<commit_before>// Copyright 2020 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Note: All the tests in this file will be executed sequentially.

package oomtest

import (
	"os"
	"strings"
	"sync"
	"testing"

	"github.com/pingcap/log"
	"github.com/pingcap/tidb/config"
	"github.com/pingcap/tidb/domain"
	"github.com/pingcap/tidb/testkit"
	"github.com/pingcap/tidb/util/testbridge"
	"github.com/stretchr/testify/require"
	"go.uber.org/goleak"
	"go.uber.org/zap"
	"go.uber.org/zap/zapcore"
)

func TestMain(m *testing.M) {
	testbridge.WorkaroundGoCheckFlags()
	registerHook()
	domain.RunAutoAnalyze = false
	config.UpdateGlobal(func(conf *config.Config) {
		conf.OOMAction = config.OOMActionLog
	})
	opts := []goleak.Option{
		goleak.IgnoreTopFunction("go.etcd.io/etcd/pkg/logutil.(*MergeLogger).outputLoop"),
		goleak.IgnoreTopFunction("go.opencensus.io/stats/view.(*worker).start"),
	}
	goleak.VerifyTestMain(m, opts...)
}

func TestMemTracker4UpdateExec(t *testing.T) {
	store, clean := testkit.CreateMockStore(t)
	defer clean()

	tk := testkit.NewTestKit(t, store)
	tk.MustExec("use test")
	tk.MustExec("create table t_MemTracker4UpdateExec (id int, a int, b int, index idx_a(`a`))")

	log.SetLevel(zap.InfoLevel)
	oom.tracker = ""
	tk.MustExec("insert into t_MemTracker4UpdateExec values (1,1,1), (2,2,2), (3,3,3)")
	require.Equal(t, "schemaLeaseChecker is not set for this transaction", oom.tracker)

	tk.Session().GetSessionVars().MemQuotaQuery = 244
	tk.MustExec("update t_MemTracker4UpdateExec set a = 4")
	require.Equal(t, "expensive_query during bootstrap phase", oom.tracker)
}

func TestMemTracker4InsertAndReplaceExec(t *testing.T) {
	store, clean := testkit.CreateMockStore(t)
	defer clean()

	tk := testkit.NewTestKit(t, store)
	tk.MustExec("use test")
	tk.MustExec("create table t (id int, a int, b int, index idx_a(`a`))")
	tk.MustExec("insert into t values (1,1,1), (2,2,2), (3,3,3)")

	tk.MustExec("create table t_MemTracker4InsertAndReplaceExec (id int, a int, b int, index idx_a(`a`))")

	log.SetLevel(zap.InfoLevel)
	oom.tracker = ""
	tk.MustExec("insert into t_MemTracker4InsertAndReplaceExec values (1,1,1), (2,2,2), (3,3,3)")
	require.Equal(t, "schemaLeaseChecker is not set for this transaction", oom.tracker)
	tk.Session().GetSessionVars().MemQuotaQuery = 1
	tk.MustExec("insert into t_MemTracker4InsertAndReplaceExec values (1,1,1), (2,2,2), (3,3,3)")
	require.Equal(t, "expensive_query during bootstrap phase", oom.tracker)
	tk.Session().GetSessionVars().MemQuotaQuery = -1

	oom.tracker = ""
	tk.MustExec("replace into t_MemTracker4InsertAndReplaceExec values (1,1,1), (2,2,2), (3,3,3)")
	require.Equal(t, "", oom.tracker)
	tk.Session().GetSessionVars().MemQuotaQuery = 1
	tk.MustExec("replace into t_MemTracker4InsertAndReplaceExec values (1,1,1), (2,2,2), (3,3,3)")
	require.Equal(t, "expensive_query during bootstrap phase", oom.tracker)
	tk.Session().GetSessionVars().MemQuotaQuery = -1

	oom.tracker = ""
	tk.MustExec("insert into t_MemTracker4InsertAndReplaceExec select * from t")
	require.Equal(t, "", oom.tracker)
	tk.Session().GetSessionVars().MemQuotaQuery = 1
	tk.MustExec("insert into t_MemTracker4InsertAndReplaceExec select * from t")
	require.Equal(t, "expensive_query during bootstrap phase", oom.tracker)
	tk.Session().GetSessionVars().MemQuotaQuery = -1

	oom.tracker = ""
	tk.MustExec("replace into t_MemTracker4InsertAndReplaceExec select * from t")
	require.Equal(t, "", oom.tracker)
	tk.Session().GetSessionVars().MemQuotaQuery = 1
	tk.MustExec("replace into t_MemTracker4InsertAndReplaceExec select * from t")
	require.Equal(t, "expensive_query during bootstrap phase", oom.tracker)
	tk.Session().GetSessionVars().MemQuotaQuery = -1

	tk.Session().GetSessionVars().DMLBatchSize = 1
	tk.Session().GetSessionVars().BatchInsert = true
	oom.tracker = ""
	tk.MustExec("insert into t_MemTracker4InsertAndReplaceExec values (1,1,1), (2,2,2), (3,3,3)")
	require.Equal(t, "", oom.tracker)
	tk.Session().GetSessionVars().MemQuotaQuery = 1
	tk.MustExec("insert into t_MemTracker4InsertAndReplaceExec values (1,1,1), (2,2,2), (3,3,3)")
	require.Equal(t, "expensive_query during bootstrap phase", oom.tracker)
	tk.Session().GetSessionVars().MemQuotaQuery = -1

	oom.tracker = ""
	tk.MustExec("replace into t_MemTracker4InsertAndReplaceExec values (1,1,1), (2,2,2), (3,3,3)")
	require.Equal(t, "", oom.tracker)
	tk.Session().GetSessionVars().MemQuotaQuery = 1
	tk.MustExec("replace into t_MemTracker4InsertAndReplaceExec values (1,1,1), (2,2,2), (3,3,3)")
	require.Equal(t, "expensive_query during bootstrap phase", oom.tracker)
	tk.Session().GetSessionVars().MemQuotaQuery = -1
}

func TestMemTracker4DeleteExec(t *testing.T) {
	store, clean := testkit.CreateMockStore(t)
	defer clean()

	tk := testkit.NewTestKit(t, store)
	tk.MustExec("use test")
	tk.MustExec("create table MemTracker4DeleteExec1 (id int, a int, b int, index idx_a(`a`))")
	tk.MustExec("create table MemTracker4DeleteExec2 (id int, a int, b int, index idx_a(`a`))")

	// delete from single table
	log.SetLevel(zap.InfoLevel)
	tk.MustExec("insert into MemTracker4DeleteExec1 values(1,1,1), (2,2,2), (3,3,3), (4,4,4), (5,5,5)")
	oom.tracker = ""
	tk.MustExec("delete from MemTracker4DeleteExec1")
	require.Equal(t, "", oom.tracker)
	tk.MustExec("insert into MemTracker4DeleteExec1 values (1,1,1), (2,2,2), (3,3,3)")
	tk.Session().GetSessionVars().MemQuotaQuery = 1
	tk.MustExec("delete from MemTracker4DeleteExec1")
	require.Equal(t, "expensive_query during bootstrap phase", oom.tracker)

	// delete from multiple table
	tk.Session().GetSessionVars().MemQuotaQuery = 100000
	tk.MustExec("insert into MemTracker4DeleteExec1 values(1,1,1)")
	tk.MustExec("insert into MemTracker4DeleteExec2 values(1,1,1)")
	oom.tracker = ""
	tk.MustExec("delete MemTracker4DeleteExec1, MemTracker4DeleteExec2 from MemTracker4DeleteExec1 join MemTracker4DeleteExec2 on MemTracker4DeleteExec1.a=MemTracker4DeleteExec2.a")
	require.Equal(t, "", oom.tracker)
	tk.MustExec("insert into MemTracker4DeleteExec1 values(1,1,1)")
	tk.MustExec("insert into MemTracker4DeleteExec2 values(1,1,1)")
	oom.tracker = ""
	tk.Session().GetSessionVars().MemQuotaQuery = 10000
	tk.MustExec("delete MemTracker4DeleteExec1, MemTracker4DeleteExec2 from MemTracker4DeleteExec1 join MemTracker4DeleteExec2 on MemTracker4DeleteExec1.a=MemTracker4DeleteExec2.a")
	require.Equal(t, "memory exceeds quota, rateLimitAction delegate to fallback action", oom.tracker)
}

var oom *oomCapture

func registerHook() {
	conf := &log.Config{Level: os.Getenv("log_level"), File: log.FileLogConfig{}}
	_, r, _ := log.InitLogger(conf)
	oom = &oomCapture{r.Core, "", sync.Mutex{}}
	lg := zap.New(oom)
	log.ReplaceGlobals(lg, r)
}

type oomCapture struct {
	zapcore.Core
	tracker string
	mu      sync.Mutex
}

func (h *oomCapture) Write(entry zapcore.Entry, fields []zapcore.Field) error {
	if entry.Message == "memory exceeds quota" {
		err, _ := fields[0].Interface.(error)
		str := err.Error()
		begin := strings.Index(str, "8001]")
		if begin == -1 {
			panic("begin not found")
		}
		end := strings.Index(str, " holds")
		if end == -1 {
			panic("end not found")
		}
		h.tracker = str[begin+len("8001]") : end]
		return nil
	}

	h.mu.Lock()
	h.tracker = entry.Message
	h.mu.Unlock()
	return nil
}

func (h *oomCapture) Check(e zapcore.Entry, ce *zapcore.CheckedEntry) *zapcore.CheckedEntry {
	if h.Enabled(e.Level) {
		return ce.AddCore(e, h)
	}
	return ce
}
<commit_msg>executor: fix data race in oomtest (#30339)<commit_after>// Copyright 2020 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Note: All the tests in this file will be executed sequentially.

package oomtest

import (
	"os"
	"strings"
	"sync"
	"testing"

	"github.com/pingcap/log"
	"github.com/pingcap/tidb/config"
	"github.com/pingcap/tidb/domain"
	"github.com/pingcap/tidb/testkit"
	"github.com/pingcap/tidb/util/testbridge"
	"github.com/stretchr/testify/require"
	"go.uber.org/goleak"
	"go.uber.org/zap"
	"go.uber.org/zap/zapcore"
)

func TestMain(m *testing.M) {
	testbridge.WorkaroundGoCheckFlags()
	registerHook()
	domain.RunAutoAnalyze = false
	config.UpdateGlobal(func(conf *config.Config) {
		conf.OOMAction = config.OOMActionLog
	})
	opts := []goleak.Option{
		goleak.IgnoreTopFunction("go.etcd.io/etcd/pkg/logutil.(*MergeLogger).outputLoop"),
		goleak.IgnoreTopFunction("go.opencensus.io/stats/view.(*worker).start"),
	}
	goleak.VerifyTestMain(m, opts...)
}

func TestMemTracker4UpdateExec(t *testing.T) {
	store, clean := testkit.CreateMockStore(t)
	defer clean()

	tk := testkit.NewTestKit(t, store)
	tk.MustExec("use test")
	tk.MustExec("create table t_MemTracker4UpdateExec (id int, a int, b int, index idx_a(`a`))")

	log.SetLevel(zap.InfoLevel)

	oom.SetTracker("")

	tk.MustExec("insert into t_MemTracker4UpdateExec values (1,1,1), (2,2,2), (3,3,3)")
	require.Equal(t, "schemaLeaseChecker is not set for this transaction", oom.GetTracker())

	tk.Session().GetSessionVars().MemQuotaQuery = 244
	tk.MustExec("update t_MemTracker4UpdateExec set a = 4")
	require.Equal(t, "expensive_query during bootstrap phase", oom.GetTracker())
}

func TestMemTracker4InsertAndReplaceExec(t *testing.T) {
	store, clean := testkit.CreateMockStore(t)
	defer clean()

	tk := testkit.NewTestKit(t, store)
	tk.MustExec("use test")
	tk.MustExec("create table t (id int, a int, b int, index idx_a(`a`))")
	tk.MustExec("insert into t values (1,1,1), (2,2,2), (3,3,3)")

	tk.MustExec("create table t_MemTracker4InsertAndReplaceExec (id int, a int, b int, index idx_a(`a`))")

	log.SetLevel(zap.InfoLevel)

	oom.SetTracker("")

	tk.MustExec("insert into t_MemTracker4InsertAndReplaceExec values (1,1,1), (2,2,2), (3,3,3)")
	require.Equal(t, "schemaLeaseChecker is not set for this transaction", oom.GetTracker())
	tk.Session().GetSessionVars().MemQuotaQuery = 1
	tk.MustExec("insert into t_MemTracker4InsertAndReplaceExec values (1,1,1), (2,2,2), (3,3,3)")
	require.Equal(t, "expensive_query during bootstrap phase", oom.GetTracker())
	tk.Session().GetSessionVars().MemQuotaQuery = -1

	oom.SetTracker("")

	tk.MustExec("replace into t_MemTracker4InsertAndReplaceExec values (1,1,1), (2,2,2), (3,3,3)")
	require.Equal(t, "", oom.GetTracker())
	tk.Session().GetSessionVars().MemQuotaQuery = 1
	tk.MustExec("replace into t_MemTracker4InsertAndReplaceExec values (1,1,1), (2,2,2), (3,3,3)")
	require.Equal(t, "expensive_query during bootstrap phase", oom.GetTracker())
	tk.Session().GetSessionVars().MemQuotaQuery = -1

	oom.SetTracker("")

	tk.MustExec("insert into t_MemTracker4InsertAndReplaceExec select * from t")
	require.Equal(t, "", oom.GetTracker())
	tk.Session().GetSessionVars().MemQuotaQuery = 1
	tk.MustExec("insert into t_MemTracker4InsertAndReplaceExec select * from t")
	require.Equal(t, "expensive_query during bootstrap phase", oom.GetTracker())
	tk.Session().GetSessionVars().MemQuotaQuery = -1

	oom.SetTracker("")

	tk.MustExec("replace into t_MemTracker4InsertAndReplaceExec select * from t")
	require.Equal(t, "", oom.GetTracker())
	tk.Session().GetSessionVars().MemQuotaQuery = 1
	tk.MustExec("replace into t_MemTracker4InsertAndReplaceExec select * from t")
	require.Equal(t, "expensive_query during bootstrap phase", oom.GetTracker())
	tk.Session().GetSessionVars().MemQuotaQuery = -1

	tk.Session().GetSessionVars().DMLBatchSize = 1
	tk.Session().GetSessionVars().BatchInsert = true

	oom.SetTracker("")

	tk.MustExec("insert into t_MemTracker4InsertAndReplaceExec values (1,1,1), (2,2,2), (3,3,3)")
	require.Equal(t, "", oom.GetTracker())
	tk.Session().GetSessionVars().MemQuotaQuery = 1
	tk.MustExec("insert into t_MemTracker4InsertAndReplaceExec values (1,1,1), (2,2,2), (3,3,3)")
	require.Equal(t, "expensive_query during bootstrap phase", oom.GetTracker())
	tk.Session().GetSessionVars().MemQuotaQuery = -1

	oom.SetTracker("")

	tk.MustExec("replace into t_MemTracker4InsertAndReplaceExec values (1,1,1), (2,2,2), (3,3,3)")
	require.Equal(t, "", oom.GetTracker())
	tk.Session().GetSessionVars().MemQuotaQuery = 1
	tk.MustExec("replace into t_MemTracker4InsertAndReplaceExec values (1,1,1), (2,2,2), (3,3,3)")
	require.Equal(t, "expensive_query during bootstrap phase", oom.GetTracker())
	tk.Session().GetSessionVars().MemQuotaQuery = -1
}

func TestMemTracker4DeleteExec(t *testing.T) {
	store, clean := testkit.CreateMockStore(t)
	defer clean()

	tk := testkit.NewTestKit(t, store)
	tk.MustExec("use test")
	tk.MustExec("create table MemTracker4DeleteExec1 (id int, a int, b int, index idx_a(`a`))")
	tk.MustExec("create table MemTracker4DeleteExec2 (id int, a int, b int, index idx_a(`a`))")

	// delete from single table
	log.SetLevel(zap.InfoLevel)
	tk.MustExec("insert into MemTracker4DeleteExec1 values(1,1,1), (2,2,2), (3,3,3), (4,4,4), (5,5,5)")

	oom.SetTracker("")

	tk.MustExec("delete from MemTracker4DeleteExec1")
	require.Equal(t, "", oom.GetTracker())
	tk.MustExec("insert into MemTracker4DeleteExec1 values (1,1,1), (2,2,2), (3,3,3)")
	tk.Session().GetSessionVars().MemQuotaQuery = 1
	tk.MustExec("delete from MemTracker4DeleteExec1")
	require.Equal(t, "expensive_query during bootstrap phase", oom.GetTracker())

	// delete from multiple table
	tk.Session().GetSessionVars().MemQuotaQuery = 100000
	tk.MustExec("insert into MemTracker4DeleteExec1 values(1,1,1)")
	tk.MustExec("insert into MemTracker4DeleteExec2 values(1,1,1)")

	oom.SetTracker("")

	tk.MustExec("delete MemTracker4DeleteExec1, MemTracker4DeleteExec2 from MemTracker4DeleteExec1 join MemTracker4DeleteExec2 on MemTracker4DeleteExec1.a=MemTracker4DeleteExec2.a")
	require.Equal(t, "", oom.GetTracker())
	tk.MustExec("insert into MemTracker4DeleteExec1 values(1,1,1)")
	tk.MustExec("insert into MemTracker4DeleteExec2 values(1,1,1)")

	oom.SetTracker("")

	tk.Session().GetSessionVars().MemQuotaQuery = 10000
	tk.MustExec("delete MemTracker4DeleteExec1, MemTracker4DeleteExec2 from MemTracker4DeleteExec1 join MemTracker4DeleteExec2 on MemTracker4DeleteExec1.a=MemTracker4DeleteExec2.a")
	require.Equal(t, "memory exceeds quota, rateLimitAction delegate to fallback action", oom.GetTracker())
}

var oom *oomCapture

func registerHook() {
	conf := &log.Config{Level: os.Getenv("log_level"), File: log.FileLogConfig{}}
	_, r, _ := log.InitLogger(conf)
	oom = &oomCapture{r.Core, "", sync.Mutex{}}
	lg := zap.New(oom)
	log.ReplaceGlobals(lg, r)
}

type oomCapture struct {
	zapcore.Core
	tracker string
	mu      sync.Mutex
}

func (h *oomCapture) SetTracker(tracker string) {
	h.mu.Lock()
	defer h.mu.Unlock()
	h.tracker = tracker
}

func (h *oomCapture) GetTracker() string {
	h.mu.Lock()
	defer h.mu.Unlock()
	return h.tracker
}

func (h *oomCapture) Write(entry zapcore.Entry, fields []zapcore.Field) error {
	if entry.Message == "memory exceeds quota" {
		err, _ := fields[0].Interface.(error)
		str := err.Error()
		begin := strings.Index(str, "8001]")
		if begin == -1 {
			panic("begin not found")
		}
		end := strings.Index(str, " holds")
		if end == -1 {
			panic("end not found")
		}
		h.tracker = str[begin+len("8001]") : end]
		return nil
	}

	h.mu.Lock()
	h.tracker = entry.Message
	h.mu.Unlock()
	return nil
}

func (h *oomCapture) Check(e zapcore.Entry, ce *zapcore.CheckedEntry) *zapcore.CheckedEntry {
	if h.Enabled(e.Level) {
		return ce.AddCore(e, h)
	}
	return ce
}
<|endoftext|>


<commit_before>package manifestlist

import (
	"encoding/json"
	"errors"
	"fmt"

	"github.com/docker/distribution"
	"github.com/docker/distribution/manifest"
	"github.com/opencontainers/go-digest"
	"github.com/opencontainers/image-spec/specs-go/v1"
)

const (
	// MediaTypeManifestList specifies the mediaType for manifest lists.
	MediaTypeManifestList = "application/vnd.docker.distribution.manifest.list.v2+json"
)

// SchemaVersion provides a pre-initialized version structure for this
// packages version of the manifest.
var SchemaVersion = manifest.Versioned{
	SchemaVersion: 2,
	MediaType:     MediaTypeManifestList,
}

// OCISchemaVersion provides a pre-initialized version structure for this
// packages OCIschema version of the manifest.
var OCISchemaVersion = manifest.Versioned{
	SchemaVersion: 2,
	MediaType:     v1.MediaTypeImageIndex,
}

func init() {
	manifestListFunc := func(b []byte) (distribution.Manifest, distribution.Descriptor, error) {
		m := new(DeserializedManifestList)
		err := m.UnmarshalJSON(b)
		if err != nil {
			return nil, distribution.Descriptor{}, err
		}

		dgst := digest.FromBytes(b)
		return m, distribution.Descriptor{Digest: dgst, Size: int64(len(b)), MediaType: MediaTypeManifestList}, err
	}
	err := distribution.RegisterManifestSchema(MediaTypeManifestList, manifestListFunc)
	if err != nil {
		panic(fmt.Sprintf("Unable to register manifest: %s", err))
	}
}

// PlatformSpec specifies a platform where a particular image manifest is
// applicable.
type PlatformSpec struct {
	// Architecture field specifies the CPU architecture, for example
	// `amd64` or `ppc64`.
	Architecture string `json:"architecture"`

	// OS specifies the operating system, for example `linux` or `windows`.
	OS string `json:"os"`

	// OSVersion is an optional field specifying the operating system
	// version, for example `10.0.10586`.
	OSVersion string `json:"os.version,omitempty"`

	// OSFeatures is an optional field specifying an array of strings,
	// each listing a required OS feature (for example on Windows `win32k`).
	OSFeatures []string `json:"os.features,omitempty"`

	// Variant is an optional field specifying a variant of the CPU, for
	// example `ppc64le` to specify a little-endian version of a PowerPC CPU.
	Variant string `json:"variant,omitempty"`

	// Features is an optional field specifying an array of strings, each
	// listing a required CPU feature (for example `sse4` or `aes`).
	Features []string `json:"features,omitempty"`
}

// A ManifestDescriptor references a platform-specific manifest.
type ManifestDescriptor struct {
	distribution.Descriptor

	// Platform specifies which platform the manifest pointed to by the
	// descriptor runs on.
	Platform PlatformSpec `json:"platform"`
}

// ManifestList references manifests for various platforms.
type ManifestList struct {
	manifest.Versioned

	// Config references the image configuration as a blob.
	Manifests []ManifestDescriptor `json:"manifests"`
}

// References returns the distribution descriptors for the referenced image
// manifests.
func (m ManifestList) References() []distribution.Descriptor {
	dependencies := make([]distribution.Descriptor, len(m.Manifests))
	for i := range m.Manifests {
		dependencies[i] = m.Manifests[i].Descriptor
	}

	return dependencies
}

// DeserializedManifestList wraps ManifestList with a copy of the original
// JSON.
type DeserializedManifestList struct {
	ManifestList

	// canonical is the canonical byte representation of the Manifest.
	canonical []byte
}

// FromDescriptors takes a slice of descriptors, and returns a
// DeserializedManifestList which contains the resulting manifest list
// and its JSON representation.
func FromDescriptors(descriptors []ManifestDescriptor) (*DeserializedManifestList, error) {
	var m ManifestList
	if len(descriptors) > 0 && descriptors[0].Descriptor.MediaType == v1.MediaTypeImageManifest {
		m = ManifestList{
			Versioned: OCISchemaVersion,
		}
	} else {
		m = ManifestList{
			Versioned: SchemaVersion,
		}
	}

	m.Manifests = make([]ManifestDescriptor, len(descriptors), len(descriptors))
	copy(m.Manifests, descriptors)

	deserialized := DeserializedManifestList{
		ManifestList: m,
	}

	var err error
	deserialized.canonical, err = json.MarshalIndent(&m, "", "   ")
	return &deserialized, err
}

// UnmarshalJSON populates a new ManifestList struct from JSON data.
func (m *DeserializedManifestList) UnmarshalJSON(b []byte) error {
	m.canonical = make([]byte, len(b), len(b))
	// store manifest list in canonical
	copy(m.canonical, b)

	// Unmarshal canonical JSON into ManifestList object
	var manifestList ManifestList
	if err := json.Unmarshal(m.canonical, &manifestList); err != nil {
		return err
	}

	m.ManifestList = manifestList

	return nil
}

// MarshalJSON returns the contents of canonical. If canonical is empty,
// marshals the inner contents.
func (m *DeserializedManifestList) MarshalJSON() ([]byte, error) {
	if len(m.canonical) > 0 {
		return m.canonical, nil
	}

	return nil, errors.New("JSON representation not initialized in DeserializedManifestList")
}

// Payload returns the raw content of the manifest list. The contents can be
// used to calculate the content identifier.
func (m DeserializedManifestList) Payload() (string, []byte, error) {
	return m.MediaType, m.canonical, nil
}
<commit_msg>register oci image index<commit_after>package manifestlist

import (
	"encoding/json"
	"errors"
	"fmt"

	"github.com/docker/distribution"
	"github.com/docker/distribution/manifest"
	"github.com/opencontainers/go-digest"
	"github.com/opencontainers/image-spec/specs-go/v1"
)

const (
	// MediaTypeManifestList specifies the mediaType for manifest lists.
	MediaTypeManifestList = "application/vnd.docker.distribution.manifest.list.v2+json"
)

// SchemaVersion provides a pre-initialized version structure for this
// packages version of the manifest.
var SchemaVersion = manifest.Versioned{
	SchemaVersion: 2,
	MediaType:     MediaTypeManifestList,
}

// OCISchemaVersion provides a pre-initialized version structure for this
// packages OCIschema version of the manifest.
var OCISchemaVersion = manifest.Versioned{
	SchemaVersion: 2,
	MediaType:     v1.MediaTypeImageIndex,
}

func init() {
	manifestListFunc := func(b []byte) (distribution.Manifest, distribution.Descriptor, error) {
		m := new(DeserializedManifestList)
		err := m.UnmarshalJSON(b)
		if err != nil {
			return nil, distribution.Descriptor{}, err
		}

		dgst := digest.FromBytes(b)
		return m, distribution.Descriptor{Digest: dgst, Size: int64(len(b)), MediaType: MediaTypeManifestList}, err
	}
	err := distribution.RegisterManifestSchema(MediaTypeManifestList, manifestListFunc)
	if err != nil {
		panic(fmt.Sprintf("Unable to register manifest: %s", err))
	}

	imageIndexFunc := func(b []byte) (distribution.Manifest, distribution.Descriptor, error) {
		m := new(DeserializedManifestList)
		err := m.UnmarshalJSON(b)
		if err != nil {
			return nil, distribution.Descriptor{}, err
		}

		dgst := digest.FromBytes(b)
		return m, distribution.Descriptor{Digest: dgst, Size: int64(len(b)), MediaType: v1.MediaTypeImageIndex}, err
	}
	err = distribution.RegisterManifestSchema(v1.MediaTypeImageIndex, imageIndexFunc)
	if err != nil {
		panic(fmt.Sprintf("Unable to register OCI Image Index: %s", err))
	}
}

// PlatformSpec specifies a platform where a particular image manifest is
// applicable.
type PlatformSpec struct {
	// Architecture field specifies the CPU architecture, for example
	// `amd64` or `ppc64`.
	Architecture string `json:"architecture"`

	// OS specifies the operating system, for example `linux` or `windows`.
	OS string `json:"os"`

	// OSVersion is an optional field specifying the operating system
	// version, for example `10.0.10586`.
	OSVersion string `json:"os.version,omitempty"`

	// OSFeatures is an optional field specifying an array of strings,
	// each listing a required OS feature (for example on Windows `win32k`).
	OSFeatures []string `json:"os.features,omitempty"`

	// Variant is an optional field specifying a variant of the CPU, for
	// example `ppc64le` to specify a little-endian version of a PowerPC CPU.
	Variant string `json:"variant,omitempty"`

	// Features is an optional field specifying an array of strings, each
	// listing a required CPU feature (for example `sse4` or `aes`).
	Features []string `json:"features,omitempty"`
}

// A ManifestDescriptor references a platform-specific manifest.
type ManifestDescriptor struct {
	distribution.Descriptor

	// Platform specifies which platform the manifest pointed to by the
	// descriptor runs on.
	Platform PlatformSpec `json:"platform"`
}

// ManifestList references manifests for various platforms.
type ManifestList struct {
	manifest.Versioned

	// Config references the image configuration as a blob.
	Manifests []ManifestDescriptor `json:"manifests"`
}

// References returns the distribution descriptors for the referenced image
// manifests.
func (m ManifestList) References() []distribution.Descriptor {
	dependencies := make([]distribution.Descriptor, len(m.Manifests))
	for i := range m.Manifests {
		dependencies[i] = m.Manifests[i].Descriptor
	}

	return dependencies
}

// DeserializedManifestList wraps ManifestList with a copy of the original
// JSON.
type DeserializedManifestList struct {
	ManifestList

	// canonical is the canonical byte representation of the Manifest.
	canonical []byte
}

// FromDescriptors takes a slice of descriptors, and returns a
// DeserializedManifestList which contains the resulting manifest list
// and its JSON representation.
func FromDescriptors(descriptors []ManifestDescriptor) (*DeserializedManifestList, error) {
	var m ManifestList
	if len(descriptors) > 0 && descriptors[0].Descriptor.MediaType == v1.MediaTypeImageManifest {
		m = ManifestList{
			Versioned: OCISchemaVersion,
		}
	} else {
		m = ManifestList{
			Versioned: SchemaVersion,
		}
	}

	m.Manifests = make([]ManifestDescriptor, len(descriptors), len(descriptors))
	copy(m.Manifests, descriptors)

	deserialized := DeserializedManifestList{
		ManifestList: m,
	}

	var err error
	deserialized.canonical, err = json.MarshalIndent(&m, "", "   ")
	return &deserialized, err
}

// UnmarshalJSON populates a new ManifestList struct from JSON data.
func (m *DeserializedManifestList) UnmarshalJSON(b []byte) error {
	m.canonical = make([]byte, len(b), len(b))
	// store manifest list in canonical
	copy(m.canonical, b)

	// Unmarshal canonical JSON into ManifestList object
	var manifestList ManifestList
	if err := json.Unmarshal(m.canonical, &manifestList); err != nil {
		return err
	}

	m.ManifestList = manifestList

	return nil
}

// MarshalJSON returns the contents of canonical. If canonical is empty,
// marshals the inner contents.
func (m *DeserializedManifestList) MarshalJSON() ([]byte, error) {
	if len(m.canonical) > 0 {
		return m.canonical, nil
	}

	return nil, errors.New("JSON representation not initialized in DeserializedManifestList")
}

// Payload returns the raw content of the manifest list. The contents can be
// used to calculate the content identifier.
func (m DeserializedManifestList) Payload() (string, []byte, error) {
	return m.MediaType, m.canonical, nil
}
<|endoftext|>


<commit_before>package url

import (
	"encoding/json"
	"fmt"
	"github.com/0x263b/Porygon2"
	"github.com/PuerkitoBio/goquery"
	"github.com/dustin/go-humanize"
	"github.com/kennygrant/sanitize"
	"html"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"net/http/cookiejar"
	"net/url"
	"regexp"
	"strconv"
	"strings"
	"time"
)

// 4chan API thread structure
// https://github.com/4chan/4chan-API
type ChanPost struct {
	Posts []struct {
		No  int    `json:"no"`
		Com string `json:"com"`
	} `json:"posts"`
}

// Reddit API thread structure
// https://www.reddit.com/dev/api
type RedditComment []struct {
	Data struct {
		Children []struct {
			Kind string `json:"kind"`
			Data struct {
				Body string `json:"body"`
			} `json:"data"`
		} `json:"children"`
	} `json:"data"`
}

// Used to parse youtube's ISO 8601 durations
// https://en.wikipedia.org/wiki/ISO_8601#Durations
func ParseDuration(str string) time.Duration {
	durationRegex := regexp.MustCompile(`P(?P<years>\d+Y)?(?P<months>\d+M)?(?P<days>\d+D)?T?(?P<hours>\d+H)?(?P<minutes>\d+M)?(?P<seconds>\d+S)?`)
	matches := durationRegex.FindStringSubmatch(str)

	years := ParseInt64(matches[1])
	months := ParseInt64(matches[2])
	days := ParseInt64(matches[3])
	hours := ParseInt64(matches[4])
	minutes := ParseInt64(matches[5])
	seconds := ParseInt64(matches[6])

	hour := int64(time.Hour)
	minute := int64(time.Minute)
	second := int64(time.Second)
	return time.Duration(years*24*365*hour + months*30*24*hour + days*24*hour + hours*hour + minutes*minute + seconds*second)
}

func ParseInt64(value string) int64 {
	if len(value) == 0 {
		return 0
	}
	parsed, err := strconv.Atoi(value[:len(value)-1])
	if err != nil {
		return 0
	}
	return int64(parsed)
}

var timeout = time.Duration(3) * time.Second

func dialTimeout(network, addr string) (net.Conn, error) {
	conn, err := net.DialTimeout(network, addr, timeout)
	return conn, err
}

func extractURL(text string) string {
	extractedURL := ""
	for _, value := range strings.Split(text, " ") {
		parsedURL, err := url.Parse(value)
		if err != nil {
			continue
		}
		if strings.HasPrefix(parsedURL.Scheme, "http") {
			extractedURL = parsedURL.String()
			break
		}
	}
	return extractedURL
}

func openGraphTitle(command *bot.PassiveCmd) (string, error) {
	if !bot.GetChannelKey(command.Channel, "urls") {
		return "", nil
	}

	URL := extractURL(command.Raw)

	if URL == "" {
		return "", nil
	}

	transport := &http.Transport{
		Dial: dialTimeout,
	}

	cookieJar, _ := cookiejar.New(nil)

	client := &http.Client{
		Jar:       cookieJar, // Some sites require cookies to show you anything (nytimes)
		Transport: transport, // Time out if connection hangs
	}

	response, err := client.Get(URL)
	if err != nil {
		return "", err
	}

	finalURL := response.Request.URL.Host

	var bytes int64 = 40960
	// Youtube's html buries the duration at the bottom
	// so we have to read the first 100kB
	if finalURL == "www.youtube.com" {
		bytes = 102400
	}

	if response.Header.Get("Content-Type") == "" {
		// Some servers don't give us anything to work with
		return fmt.Sprintf("Title | (no title) | %s", finalURL), nil
	} else if !strings.Contains(response.Header.Get("Content-Type"), "text/html") {
		if !bot.GetChannelKey(command.Channel, "files") {
			return "", nil
		}
		contentType := response.Header.Get("Content-Type")
		contentLength := response.Header.Get("Content-Length")
		size, _ := strconv.ParseUint(contentLength, 10, 64)
		return fmt.Sprintf("File | %s %s | %s", contentType, humanize.Bytes(size), finalURL), nil
	}

	body := response.Body
	chunk := io.LimitReader(body, bytes) // Download/Read first 20kB

	doc, err := goquery.NewDocumentFromReader(chunk)
	if err != nil {
		fmt.Println(err)
		return "", err
	}

	title := doc.Find("title").Text()

	// Generally <meta> tags have more useful titles
	doc.Find("meta[property='og:title']").Each(func(i int, s *goquery.Selection) {
		title = s.AttrOr("content", title)
	})

	// Get tweet content from <meta>
	if finalURL == "twitter.com" {
		doc.Find("meta[property='og:description']").Each(func(i int, s *goquery.Selection) {
			title = fmt.Sprintf("%s: %s", title, s.AttrOr("content", title))
		})
	}

	// Get video duration from <meta>
	if finalURL == "www.youtube.com" {
		doc.Find("meta[itemprop='duration']").Each(func(i int, s *goquery.Selection) {
			title = fmt.Sprintf("%s | %s", title, ParseDuration(s.AttrOr("content", title)))
		})
	}

	// Get 4chan post
	if finalURL == "boards.4chan.org" {
		if strings.Contains(response.Request.URL.Path, "/thread/") {
			path := strings.Split(response.Request.URL.Path, "/")
			postId := response.Request.URL.Fragment

			response, _ := client.Get(fmt.Sprintf("https://a.4cdn.org/%s/thread/%s.json", path[1], path[3]))

			if response.StatusCode != 200 {
				title = "404 Not Found"
			} else {

				defer response.Body.Close()
				body, _ := ioutil.ReadAll(response.Body)

				var posts ChanPost
				json.Unmarshal(body, &posts)

				title = posts.Posts[0].Com

				if postId != "" {
					for _, element := range posts.Posts {
						if strings.Contains(postId, strconv.Itoa(element.No)) {
							title = element.Com
							break
						}
					}
				}

				title = sanitize.HTML(title) // Remove any unwanted html
			}

			if len(title) < 1 {
				title = "(blank post)"
			}
		}
	}

	if finalURL == "www.reddit.com" {
		thread_title := title
		if strings.Contains(response.Request.URL.Path, "/comments/") {

			response, _ := client.Get(fmt.Sprintf("%s.json", URL))

			if response.StatusCode != 200 {
				fmt.Println(response.StatusCode)
				title = "404 Not Found"
			} else {

				defer response.Body.Close()

				body, _ := ioutil.ReadAll(response.Body)
				var comments RedditComment
				json.Unmarshal(body, &comments)

				children := comments[len(comments)-1].Data.Children
				title = children[len(children)-1].Data.Body

				if len(title) < 1 {
					title = thread_title
				}
			}
		}
	}

	reg := regexp.MustCompile("\\s+")
	title = reg.ReplaceAllString(title, " ") // Strip tabs and newlines
	title = strings.TrimSpace(title)         // then trim excessive spaces

	if len(title) > 200 {
		title = fmt.Sprintf("%s …", title[0:200])
	} else if len(title) < 1 {
		title = "(no title)"
	}

	return fmt.Sprintf("Title | %s | %s", html.UnescapeString(title), finalURL), nil
}

func toggleURLs(command *bot.Cmd, matches []string) (msg string, err error) {
	if !bot.IsAdmin(command.Nick) || !bot.IsPrivateMsg(command.Channel, command.Nick) {
		return "", nil
	}

	onOff := matches[1]
	channelToToggle := matches[2]

	if onOff == "on" {
		bot.SetChannelKey(channelToToggle, "urls", true)
		return fmt.Sprintf("Reacting to URLs in %s", channelToToggle), nil
	} else if onOff == "off" {
		bot.SetChannelKey(channelToToggle, "urls", false)
		return fmt.Sprintf("No longer displaying page titles in %s", channelToToggle), nil
	}
	return "", nil
}

func toggleFiles(command *bot.Cmd, matches []string) (msg string, err error) {
	if !bot.IsAdmin(command.Nick) || !bot.IsPrivateMsg(command.Channel, command.Nick) {
		return "", nil
	}

	onOff := matches[1]
	channelToToggle := matches[2]

	if onOff == "on" {
		bot.SetChannelKey(channelToToggle, "files", true)
		return fmt.Sprintf("Reacting to file URLs in %s", channelToToggle), nil
	} else if onOff == "off" {
		bot.SetChannelKey(channelToToggle, "files", false)
		return fmt.Sprintf("No longer displaying file info in %s", channelToToggle), nil
	}
	return "", nil
}

func init() {
	bot.RegisterPassiveCommand(
		"opengraph",
		openGraphTitle)

	bot.RegisterCommand(
		"^set urls (\\S+) (\\S+)$",
		toggleURLs)

	bot.RegisterCommand(
		"^set files (\\S+) (\\S+)$",
		toggleFiles)
}
<commit_msg>defer close<commit_after>package url

import (
	"encoding/json"
	"fmt"
	"github.com/0x263b/Porygon2"
	"github.com/PuerkitoBio/goquery"
	"github.com/dustin/go-humanize"
	"github.com/kennygrant/sanitize"
	"html"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"net/http/cookiejar"
	"net/url"
	"regexp"
	"strconv"
	"strings"
	"time"
)

// 4chan API thread structure
// https://github.com/4chan/4chan-API
type ChanPost struct {
	Posts []struct {
		No  int    `json:"no"`
		Com string `json:"com"`
	} `json:"posts"`
}

// Reddit API thread structure
// https://www.reddit.com/dev/api
type RedditComment []struct {
	Data struct {
		Children []struct {
			Kind string `json:"kind"`
			Data struct {
				Body string `json:"body"`
			} `json:"data"`
		} `json:"children"`
	} `json:"data"`
}

// Used to parse youtube's ISO 8601 durations
// https://en.wikipedia.org/wiki/ISO_8601#Durations
func ParseDuration(str string) time.Duration {
	durationRegex := regexp.MustCompile(`P(?P<years>\d+Y)?(?P<months>\d+M)?(?P<days>\d+D)?T?(?P<hours>\d+H)?(?P<minutes>\d+M)?(?P<seconds>\d+S)?`)
	matches := durationRegex.FindStringSubmatch(str)

	years := ParseInt64(matches[1])
	months := ParseInt64(matches[2])
	days := ParseInt64(matches[3])
	hours := ParseInt64(matches[4])
	minutes := ParseInt64(matches[5])
	seconds := ParseInt64(matches[6])

	hour := int64(time.Hour)
	minute := int64(time.Minute)
	second := int64(time.Second)
	return time.Duration(years*24*365*hour + months*30*24*hour + days*24*hour + hours*hour + minutes*minute + seconds*second)
}

func ParseInt64(value string) int64 {
	if len(value) == 0 {
		return 0
	}
	parsed, err := strconv.Atoi(value[:len(value)-1])
	if err != nil {
		return 0
	}
	return int64(parsed)
}

var timeout = time.Duration(3) * time.Second

func dialTimeout(network, addr string) (net.Conn, error) {
	conn, err := net.DialTimeout(network, addr, timeout)
	return conn, err
}

func extractURL(text string) string {
	extractedURL := ""
	for _, value := range strings.Split(text, " ") {
		parsedURL, err := url.Parse(value)
		if err != nil {
			continue
		}
		if strings.HasPrefix(parsedURL.Scheme, "http") {
			extractedURL = parsedURL.String()
			break
		}
	}
	return extractedURL
}

func openGraphTitle(command *bot.PassiveCmd) (string, error) {
	if !bot.GetChannelKey(command.Channel, "urls") {
		return "", nil
	}

	URL := extractURL(command.Raw)

	if URL == "" {
		return "", nil
	}

	transport := &http.Transport{
		Dial: dialTimeout,
	}

	cookieJar, _ := cookiejar.New(nil)

	client := &http.Client{
		Jar:       cookieJar, // Some sites require cookies to show you anything (nytimes)
		Transport: transport, // Time out if connection hangs
	}

	response, err := client.Get(URL)
	if err != nil {
		return "", err
	}

	finalURL := response.Request.URL.Host

	var bytes int64 = 40960
	// Youtube's html buries the duration at the bottom
	// so we have to read the first 100kB
	if finalURL == "www.youtube.com" {
		bytes = 102400
	}

	if response.Header.Get("Content-Type") == "" {
		// Some servers don't give us anything to work with
		return fmt.Sprintf("Title | (no title) | %s", finalURL), nil
	} else if !strings.Contains(response.Header.Get("Content-Type"), "text/html") {
		if !bot.GetChannelKey(command.Channel, "files") {
			return "", nil
		}
		contentType := response.Header.Get("Content-Type")
		contentLength := response.Header.Get("Content-Length")
		size, _ := strconv.ParseUint(contentLength, 10, 64)
		return fmt.Sprintf("File | %s %s | %s", contentType, humanize.Bytes(size), finalURL), nil
	}

	defer response.Body.Close()
	body := response.Body
	chunk := io.LimitReader(body, bytes) // Download/Read first 20kB

	doc, err := goquery.NewDocumentFromReader(chunk)
	if err != nil {
		fmt.Println(err)
		return "", err
	}

	title := doc.Find("title").Text()

	// Generally <meta> tags have more useful titles
	doc.Find("meta[property='og:title']").Each(func(i int, s *goquery.Selection) {
		title = s.AttrOr("content", title)
	})

	// Get tweet content from <meta>
	if finalURL == "twitter.com" {
		doc.Find("meta[property='og:description']").Each(func(i int, s *goquery.Selection) {
			title = fmt.Sprintf("%s: %s", title, s.AttrOr("content", title))
		})
	}

	// Get video duration from <meta>
	if finalURL == "www.youtube.com" {
		doc.Find("meta[itemprop='duration']").Each(func(i int, s *goquery.Selection) {
			title = fmt.Sprintf("%s | %s", title, ParseDuration(s.AttrOr("content", title)))
		})
	}

	// Get 4chan post
	if finalURL == "boards.4chan.org" {
		if strings.Contains(response.Request.URL.Path, "/thread/") {
			path := strings.Split(response.Request.URL.Path, "/")
			postId := response.Request.URL.Fragment

			response, _ := client.Get(fmt.Sprintf("https://a.4cdn.org/%s/thread/%s.json", path[1], path[3]))

			if response.StatusCode != 200 {
				title = "404 Not Found"
			} else {

				defer response.Body.Close()
				body, _ := ioutil.ReadAll(response.Body)

				var posts ChanPost
				json.Unmarshal(body, &posts)

				title = posts.Posts[0].Com

				if postId != "" {
					for _, element := range posts.Posts {
						if strings.Contains(postId, strconv.Itoa(element.No)) {
							title = element.Com
							break
						}
					}
				}

				title = sanitize.HTML(title) // Remove any unwanted html
			}

			if len(title) < 1 {
				title = "(blank post)"
			}
		}
	}

	if finalURL == "www.reddit.com" {
		thread_title := title
		if strings.Contains(response.Request.URL.Path, "/comments/") {

			response, _ := client.Get(fmt.Sprintf("%s.json", URL))

			if response.StatusCode != 200 {
				fmt.Println(response.StatusCode)
				title = "404 Not Found"
			} else {

				defer response.Body.Close()

				body, _ := ioutil.ReadAll(response.Body)
				var comments RedditComment
				json.Unmarshal(body, &comments)

				children := comments[len(comments)-1].Data.Children
				title = children[len(children)-1].Data.Body

				if len(title) < 1 {
					title = thread_title
				}
			}
		}
	}

	reg := regexp.MustCompile("\\s+")
	title = reg.ReplaceAllString(title, " ") // Strip tabs and newlines
	title = strings.TrimSpace(title)         // then trim excessive spaces

	if len(title) > 200 {
		title = fmt.Sprintf("%s …", title[0:200])
	} else if len(title) < 1 {
		title = "(no title)"
	}

	return fmt.Sprintf("Title | %s | %s", html.UnescapeString(title), finalURL), nil
}

func toggleURLs(command *bot.Cmd, matches []string) (msg string, err error) {
	if !bot.IsAdmin(command.Nick) || !bot.IsPrivateMsg(command.Channel, command.Nick) {
		return "", nil
	}

	onOff := matches[1]
	channelToToggle := matches[2]

	if onOff == "on" {
		bot.SetChannelKey(channelToToggle, "urls", true)
		return fmt.Sprintf("Reacting to URLs in %s", channelToToggle), nil
	} else if onOff == "off" {
		bot.SetChannelKey(channelToToggle, "urls", false)
		return fmt.Sprintf("No longer displaying page titles in %s", channelToToggle), nil
	}
	return "", nil
}

func toggleFiles(command *bot.Cmd, matches []string) (msg string, err error) {
	if !bot.IsAdmin(command.Nick) || !bot.IsPrivateMsg(command.Channel, command.Nick) {
		return "", nil
	}

	onOff := matches[1]
	channelToToggle := matches[2]

	if onOff == "on" {
		bot.SetChannelKey(channelToToggle, "files", true)
		return fmt.Sprintf("Reacting to file URLs in %s", channelToToggle), nil
	} else if onOff == "off" {
		bot.SetChannelKey(channelToToggle, "files", false)
		return fmt.Sprintf("No longer displaying file info in %s", channelToToggle), nil
	}
	return "", nil
}

func init() {
	bot.RegisterPassiveCommand(
		"opengraph",
		openGraphTitle)

	bot.RegisterCommand(
		"^set urls (\\S+) (\\S+)$",
		toggleURLs)

	bot.RegisterCommand(
		"^set files (\\S+) (\\S+)$",
		toggleFiles)
}
<|endoftext|>


<commit_before>package misspell

import (
	"bytes"
	"io/ioutil"
	"testing"
)

var (
	sampleClean string
	sampleDirty string
	tmpCount    int
	tmp         string
	rep         *Replacer
)

func init() {

	buf := bytes.Buffer{}
	for i := 0; i < len(DictMain); i += 2 {
		buf.WriteString(DictMain[i+1] + "\n")
	}
	sampleClean = buf.String()
	sampleDirty = sampleClean + DictMain[0] + "\n"
	rep = New()

}

// BenchmarkCleanString takes a clean string (one with no errors)
func BenchmarkCleanString(b *testing.B) {
	b.ResetTimer()
	b.ReportAllocs()
	var updated string
	var diffs []Diff
	var count int
	for n := 0; n < b.N; n++ {
		updated, diffs = rep.Replace(sampleClean)
		count += len(diffs)
	}

	// prevent compilier optimizations
	tmpCount = count
	tmp = updated
}

// BenchmarkCleanStream takes a clean reader (no misspells) and outputs to a buffer
func BenchmarkCleanStream(b *testing.B) {
	b.ResetTimer()
	b.ReportAllocs()
	count := 0
	buf := bytes.NewBufferString(sampleClean)
	out := bytes.NewBuffer(make([]byte, 0, len(sampleClean)+100))
	for n := 0; n < b.N; n++ {
		buf.Reset()
		buf.WriteString(sampleClean)
		out.Reset()
		diffs := rep.ReplaceReader(buf, out)
		count += len(diffs)
	}
	tmpCount = count
}

// BenchmarkCleanStreamDiscard takes a clean reader and discards output
func BenchmarkCleanStreamDiscard(b *testing.B) {
	b.ResetTimer()
	b.ReportAllocs()

	buf := bytes.NewBufferString(sampleClean)
	count := 0
	for n := 0; n < b.N; n++ {
		buf.Reset()
		buf.WriteString(sampleClean)
		diffs := rep.ReplaceReader(buf, ioutil.Discard)
		count += len(diffs)
	}

	tmpCount = count
}

// BenchmarkCleanString takes a clean string (one with no errors)
func BenchmarkDirtyString(b *testing.B) {
	b.ResetTimer()
	b.ReportAllocs()
	var updated string
	var diffs []Diff
	var count int
	for n := 0; n < b.N; n++ {
		updated, diffs = rep.Replace(sampleDirty)
		count += len(diffs)
	}

	// prevent compilier optimizations
	tmpCount = count
	tmp = updated
}

// BenchmarkCleanStreamDiscard takes a clean reader and discards output
func BenchmarkDirtyStreamDiscard(b *testing.B) {
	b.ResetTimer()
	b.ReportAllocs()

	buf := bytes.NewBufferString(sampleDirty)
	count := 0
	for n := 0; n < b.N; n++ {
		buf.Reset()
		buf.WriteString(sampleDirty)
		diffs := rep.ReplaceReader(buf, ioutil.Discard)
		count += len(diffs)
	}

	tmpCount = count
}
<commit_msg>cleanup benchmark<commit_after>package misspell

import (
	"bytes"
	"io/ioutil"
	"testing"
)

var (
	sampleClean string
	sampleDirty string
	tmpCount    int
	tmp         string
	rep         *Replacer
)

func init() {

	buf := bytes.Buffer{}
	for i := 0; i < len(DictMain); i += 2 {
		buf.WriteString(DictMain[i+1] + " ")
		if i%5 == 0 {
			buf.WriteString("\n")
		}
	}
	sampleClean = buf.String()
	sampleDirty = sampleClean + DictMain[0] + "\n"
	rep = New()
}

// BenchmarkCleanString takes a clean string (one with no errors)
func BenchmarkCleanString(b *testing.B) {
	b.ResetTimer()
	b.ReportAllocs()
	var updated string
	var diffs []Diff
	var count int
	for n := 0; n < b.N; n++ {
		updated, diffs = rep.Replace(sampleClean)
		count += len(diffs)
	}

	// prevent compilier optimizations
	tmpCount = count
	tmp = updated
}

func discardDiff(_ Diff) {
	tmpCount++
}

// BenchmarkCleanStream takes a clean reader (no misspells) and outputs to a buffer
func BenchmarkCleanStream(b *testing.B) {
	b.ResetTimer()
	b.ReportAllocs()
	tmpCount = 0
	buf := bytes.NewBufferString(sampleClean)
	out := bytes.NewBuffer(make([]byte, 0, len(sampleClean)+100))
	for n := 0; n < b.N; n++ {
		buf.Reset()
		buf.WriteString(sampleClean)
		out.Reset()
		rep.ReplaceReader(buf, out, discardDiff)
	}
}

// BenchmarkCleanStreamDiscard takes a clean reader and discards output
func BenchmarkCleanStreamDiscard(b *testing.B) {
	b.ResetTimer()
	b.ReportAllocs()

	buf := bytes.NewBufferString(sampleClean)
	tmpCount = 0
	for n := 0; n < b.N; n++ {
		buf.Reset()
		buf.WriteString(sampleClean)
		rep.ReplaceReader(buf, ioutil.Discard, discardDiff)
	}
}

// BenchmarkCleanString takes a clean string (one with no errors)
func BenchmarkDirtyString(b *testing.B) {
	b.ResetTimer()
	b.ReportAllocs()
	var updated string
	var diffs []Diff
	var count int
	for n := 0; n < b.N; n++ {
		updated, diffs = rep.Replace(sampleDirty)
		count += len(diffs)
	}

	// prevent compilier optimizations
	tmpCount = count
	tmp = updated
}
<|endoftext|>


<commit_before>// +build integration

package main

import (
 "github.com/amarburg/go-lazycache/stress_test"
 "testing"
)

func TestRandomWalk(t *testing.T) {
	flag.Parse()

	server := StartLazycacheServer("127.0.0.1", 5000)
	defer server.Stop()

	AddMirror(OOIRawDataRootURL)

	err := stress_test.RandomWalk("127.0.0.1:5000")
	if err != nil {
		t.Error(err)
	}
}
<commit_msg>Clean up integration testing with new go-lazycache-benchmarking package.<commit_after>// +build integration

package main

import (
 stress "github.com/amarburg/go-lazycache-benchmarking"
 "testing"
)

func TestRandomWalk(t *testing.T) {

	server := StartLazycacheServer("127.0.0.1", 5000)
	defer server.Stop()

	AddMirror(OOIRawDataRootURL)

	err := stress.RandomWalk( stress.AddUrl("http://127.0.0.1:5000/org/oceanobservatories/rawdata/files/"),
 													stress.SetCount( 100 ),
													stress.SetParallelism( 5 ) )
	if err != nil {
		t.Error(err)
	}
}
<|endoftext|>


<commit_before>// +build linux

package node

import (
	"os"
	"sync"

	libpodcgroups "github.com/containers/libpod/v2/pkg/cgroups"
	libctrcgroups "github.com/opencontainers/runc/libcontainer/cgroups"
	"github.com/pkg/errors"
)

var (
	cgroupHasMemorySwapOnce sync.Once
	cgroupHasMemorySwap     bool
	cgroupHasMemorySwapErr  error

	cgroupControllerOnce sync.Once
	cgroupControllerErr  error
	cgroupHasHugetlb     bool
	cgroupHasPid         bool

	cgroupIsV2Err error
)

func CgroupIsV2() bool {
	var cgroupIsV2 bool
	cgroupIsV2, cgroupIsV2Err = libpodcgroups.IsCgroup2UnifiedMode()
	return cgroupIsV2
}

// CgroupHasMemorySwap returns whether the memory swap controller is present
func CgroupHasMemorySwap() bool {
	cgroupHasMemorySwapOnce.Do(func() {
		if CgroupIsV2() {
			cgroupHasMemorySwap = true
			return
		}

		_, err := os.Stat("/sys/fs/cgroup/memory/memory.memsw.limit_in_bytes")
		if err != nil {
			cgroupHasMemorySwapErr = errors.New("node not configured with memory swap")
			cgroupHasMemorySwap = false
			return
		}

		cgroupHasMemorySwap = true
	})
	return cgroupHasMemorySwap
}

// CgroupHasHugetlb returns whether the hugetlb controller is present
func CgroupHasHugetlb() bool {
	checkRelevantControllers()
	return cgroupHasHugetlb
}

// CgroupHasPid returns whether the pid controller is present
func CgroupHasPid() bool {
	checkRelevantControllers()
	return cgroupHasPid
}

func checkRelevantControllers() {
	cgroupControllerOnce.Do(func() {
		relevantControllers := []struct {
			name    string
			enabled *bool
		}{
			{
				name:    "pids",
				enabled: &cgroupHasPid,
			},
			{
				name:    "hugetlb",
				enabled: &cgroupHasHugetlb,
			},
		}
		ctrls, err := libctrcgroups.GetAllSubsystems()
		if err != nil {
			cgroupControllerErr = err
			return
		}
		for _, toCheck := range relevantControllers {
			for _, ctrl := range ctrls {
				if ctrl == toCheck.name {
					*toCheck.enabled = true
					break
				}
			}
		}
	})
}
<commit_msg>cgroup: implement fix for swap memcg on cgroup v2<commit_after>// +build linux

package node

import (
	"os"
	"path/filepath"
	"sync"

	libpodcgroups "github.com/containers/libpod/v2/pkg/cgroups"
	libctrcgroups "github.com/opencontainers/runc/libcontainer/cgroups"
	"github.com/pkg/errors"
)

var (
	cgroupHasMemorySwapOnce sync.Once
	cgroupHasMemorySwap     bool
	cgroupHasMemorySwapErr  error

	cgroupControllerOnce sync.Once
	cgroupControllerErr  error
	cgroupHasHugetlb     bool
	cgroupHasPid         bool

	cgroupIsV2Err error
)

func CgroupIsV2() bool {
	var cgroupIsV2 bool
	cgroupIsV2, cgroupIsV2Err = libpodcgroups.IsCgroup2UnifiedMode()
	return cgroupIsV2
}

// CgroupHasMemorySwap returns whether the memory swap controller is present
func CgroupHasMemorySwap() bool {
	cgroupHasMemorySwapOnce.Do(func() {
		if CgroupIsV2() {
			cg, err := libctrcgroups.ParseCgroupFile("/proc/self/cgroup")
			if err != nil {
				cgroupHasMemorySwapErr = err
				cgroupHasMemorySwap = false
				return
			}
			memSwap := filepath.Join("/sys/fs/cgroup", cg[""], "memory.swap.current")
			if _, err := os.Stat(memSwap); err != nil {
				cgroupHasMemorySwap = false
				return
			}
			cgroupHasMemorySwap = true
			return
		}

		_, err := os.Stat("/sys/fs/cgroup/memory/memory.memsw.limit_in_bytes")
		if err != nil {
			cgroupHasMemorySwapErr = errors.New("node not configured with memory swap")
			cgroupHasMemorySwap = false
			return
		}

		cgroupHasMemorySwap = true
	})
	return cgroupHasMemorySwap
}

// CgroupHasHugetlb returns whether the hugetlb controller is present
func CgroupHasHugetlb() bool {
	checkRelevantControllers()
	return cgroupHasHugetlb
}

// CgroupHasPid returns whether the pid controller is present
func CgroupHasPid() bool {
	checkRelevantControllers()
	return cgroupHasPid
}

func checkRelevantControllers() {
	cgroupControllerOnce.Do(func() {
		relevantControllers := []struct {
			name    string
			enabled *bool
		}{
			{
				name:    "pids",
				enabled: &cgroupHasPid,
			},
			{
				name:    "hugetlb",
				enabled: &cgroupHasHugetlb,
			},
		}
		ctrls, err := libctrcgroups.GetAllSubsystems()
		if err != nil {
			cgroupControllerErr = err
			return
		}
		for _, toCheck := range relevantControllers {
			for _, ctrl := range ctrls {
				if ctrl == toCheck.name {
					*toCheck.enabled = true
					break
				}
			}
		}
	})
}
<|endoftext|>


<commit_before>package edgectl

import (
	"bufio"
	"bytes"
	"crypto/tls"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"strconv"
	"strings"
	"time"

	"github.com/pkg/errors"

	"github.com/datawire/ambassador/pkg/k8s"
	"github.com/datawire/ambassador/pkg/supervisor"
)

var simpleTransport = &http.Transport{
	// #nosec G402
	TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
	Proxy:           nil,
	DialContext: (&net.Dialer{
		Timeout:   10 * time.Second,
		KeepAlive: 1 * time.Second,
		DualStack: true,
	}).DialContext,
	DisableKeepAlives: true,
}

var hClient = &http.Client{
	Transport: simpleTransport,
	Timeout:   15 * time.Second,
}

// Connect the daemon to a cluster
func (d *Daemon) Connect(
	p *supervisor.Process, out *Emitter, rai *RunAsInfo,
	context, namespace, managerNs string, kargs []string,
	installID string, isCI bool,
) error {
	// Sanity checks
	if d.cluster != nil {
		out.Println("Already connected")
		out.Send("connect", "Already connected")
		return nil
	}
	if d.bridge != nil {
		out.Println("Not ready: Trying to disconnect")
		out.Send("connect", "Not ready: Trying to disconnect")
		return nil
	}
	if d.network == nil {
		out.Println("Not ready: Network overrides are paused (use \"edgectl resume\")")
		out.Send("connect", "Not ready: Paused")
		return nil
	}
	if !d.network.IsOkay() {
		out.Println("Not ready: Establishing network overrides")
		out.Send("connect", "Not ready: Establishing network overrides")
		return nil
	}

	out.Printf("Connecting to traffic manager in namespace %s...\n", managerNs)
	out.Send("connect", "Connecting...")
	cluster, err := TrackKCluster(p, rai, context, namespace, kargs)
	if err != nil {
		out.Println(err.Error())
		out.Send("failed", err.Error())
		out.SendExit(1)
		return nil
	}
	d.cluster = cluster

	previewHost, err := getClusterPreviewHostname(p, cluster)
	if err != nil {
		p.Logf("get preview URL hostname: %+v", err)
		previewHost = ""
	}

	bridge, err := CheckedRetryingCommand(
		p,
		"bridge",
		[]string{GetExe(), "teleproxy", "bridge", cluster.context, cluster.namespace},
		rai,
		checkBridge,
		15*time.Second,
	)
	if err != nil {
		out.Println(err.Error())
		out.Send("failed", err.Error())
		out.SendExit(1)
		d.cluster.Close()
		d.cluster = nil
		return nil
	}
	d.bridge = bridge
	d.cluster.SetBridgeCheck(d.bridge.IsOkay)

	out.Printf(
		"Connected to context %s (%s)\n", d.cluster.Context(), d.cluster.Server(),
	)
	out.Send("cluster.context", d.cluster.Context())
	out.Send("cluster.server", d.cluster.Server())

	tmgr, err := NewTrafficManager(p, d.cluster, managerNs, installID, isCI)
	if err != nil {
		out.Println()
		out.Println("Unable to connect to the traffic manager in your cluster.")
		out.Println("The intercept feature will not be available.")
		out.Println("Error was:", err)
		// out.Println("Use <some command> to set up the traffic manager.") // FIXME
		out.Send("intercept", false)
	} else {
		tmgr.previewHost = previewHost
		d.trafficMgr = tmgr
		out.Send("intercept", true)
	}
	return nil
}

// Disconnect from the connected cluster
func (d *Daemon) Disconnect(p *supervisor.Process, out *Emitter) error {
	// Sanity checks
	if d.cluster == nil {
		out.Println("Not connected (use 'edgectl connect' to connect to your cluster)")
		out.Send("disconnect", "Not connected")
		return nil
	}

	_ = d.ClearIntercepts(p)
	if d.bridge != nil {
		d.cluster.SetBridgeCheck(nil) // Stop depending on this bridge
		_ = d.bridge.Close()
		d.bridge = nil
	}
	if d.trafficMgr != nil {
		_ = d.trafficMgr.Close()
		d.trafficMgr = nil
	}
	err := d.cluster.Close()
	d.cluster = nil

	out.Println("Disconnected")
	out.Send("disconnect", "Disconnected")
	return err
}

// getClusterPreviewHostname returns the hostname of the first Host resource it
// finds that has Preview URLs enabled with a supported URL type.
func getClusterPreviewHostname(p *supervisor.Process, cluster *KCluster) (hostname string, err error) {
	p.Log("Looking for a Host with Preview URLs enabled")

	// kubectl get hosts, in all namespaces or in this namespace
	var outBytes []byte
	outBytes, err = func() ([]byte, error) {
		clusterCmd := cluster.GetKubectlCmdNoNamespace(p, "get", "host", "-o", "yaml", "--all-namespaces")
		if outBytes, err := clusterCmd.CombinedOutput(); err == nil {
			return outBytes, nil
		}

		nsCmd := cluster.GetKubectlCmd(p, "get", "host", "-o", "yaml")
		if outBytes, err := nsCmd.CombinedOutput(); err == nil {
			return outBytes, nil
		} else {
			return nil, err
		}
	}()
	if err != nil {
		return
	}

	// Parse the output
	hostLists, kerr := k8s.ParseResources("get hosts", string(outBytes))
	if kerr != nil {
		err = kerr
		return
	}
	if len(hostLists) != 1 {
		err = errors.Errorf("weird result with length %d", len(hostLists))
		return
	}

	// Grab the "items" slice, as the result should be a list of Host resources
	hostItems := k8s.Map(hostLists[0]).GetMaps("items")
	p.Logf("Found %d Host resources", len(hostItems))

	// Loop over Hosts looking for a Preview URL hostname
	for _, hostItem := range hostItems {
		host := k8s.Resource(hostItem)
		logEntry := fmt.Sprintf("- Host %s / %s: %%s", host.Namespace(), host.Name())

		previewUrlSpec := host.Spec().GetMap("previewUrl")
		if len(previewUrlSpec) == 0 {
			p.Logf(logEntry, "no preview URL config")
			continue
		}

		if enabled, ok := previewUrlSpec["enabled"].(bool); !ok || !enabled {
			p.Logf(logEntry, "preview URL not enabled")
			continue
		}

		// missing type, default is "Path" --> success
		// type is present, set to "Path" --> success
		// otherwise --> failure
		if pType, ok := previewUrlSpec["type"].(string); ok && pType != "Path" {
			p.Logf(logEntry+": %#v", "unsupported preview URL type", previewUrlSpec["type"])
			continue
		}

		if hostname = host.Spec().GetString("hostname"); hostname == "" {
			p.Logf(logEntry, "empty hostname???")
			continue
		}

		p.Logf(logEntry+": %q", "SUCCESS! Hostname is", hostname)
		return
	}

	p.Logf("No appropriate Host resource found.")
	return
}

// checkBridge checks the status of teleproxy bridge by doing the equivalent of
//  curl http://traffic-proxy.svc.cluster.local:8022.
// Note there is no namespace specified, as we are checking for bridge status in the
// current namespace.
func checkBridge(p *supervisor.Process) error {
	address := "traffic-proxy.svc.cluster.local:8022"
	conn, err := net.DialTimeout("tcp", address, 15*time.Second)
	if err != nil {
		return errors.Wrap(err, "tcp connect")
	}
	if conn != nil {
		defer conn.Close()
		msg, _, err := bufio.NewReader(conn).ReadLine()
		if err != nil {
			return errors.Wrap(err, "tcp read")
		}
		if !strings.Contains(string(msg), "SSH") {
			return fmt.Errorf("expected SSH prompt, got: %v", string(msg))
		}
	} else {
		return fmt.Errorf("fail to establish tcp connection to %v", address)
	}
	return nil
}

// TrafficManager is a handle to access the Traffic Manager in a
// cluster.
type TrafficManager struct {
	crc            Resource
	apiPort        int
	sshPort        int
	namespace      string
	interceptables []string
	totalClusCepts int
	snapshotSent   bool
	installID      string // edgectl's install ID
	connectCI      bool   // whether --ci was passed to connect
	apiErr         error  // holds the latest traffic-manager API error
	licenseInfo    string // license information from traffic-manager
	previewHost    string // hostname to use for preview URLs, if enabled
}

// NewTrafficManager returns a TrafficManager resource for the given
// cluster if it has a Traffic Manager service.
func NewTrafficManager(p *supervisor.Process, cluster *KCluster, managerNs string, installID string, isCI bool) (*TrafficManager, error) {
	cmd := cluster.GetKubectlCmd(p, "get", "-n", managerNs, "svc/telepresence-proxy", "deploy/telepresence-proxy")
	err := cmd.Run()
	if err != nil {
		return nil, errors.Wrap(err, "kubectl get svc/deploy telepresency-proxy")
	}

	apiPort, err := GetFreePort()
	if err != nil {
		return nil, errors.Wrap(err, "get free port for API")
	}
	sshPort, err := GetFreePort()
	if err != nil {
		return nil, errors.Wrap(err, "get free port for ssh")
	}
	kpfArgStr := fmt.Sprintf("port-forward -n %s svc/telepresence-proxy %d:8022 %d:8081", managerNs, sshPort, apiPort)
	kpfArgs := cluster.GetKubectlArgs(strings.Fields(kpfArgStr)...)
	tm := &TrafficManager{
		apiPort:   apiPort,
		sshPort:   sshPort,
		namespace: managerNs,
		installID: installID,
		connectCI: isCI,
	}

	pf, err := CheckedRetryingCommand(p, "traffic-kpf", kpfArgs, cluster.RAI(), tm.check, 15*time.Second)
	if err != nil {
		return nil, err
	}
	tm.crc = pf
	return tm, nil
}

func (tm *TrafficManager) check(p *supervisor.Process) error {
	body, code, err := tm.request("GET", "state", []byte{})
	if err != nil {
		return err
	}

	if code != http.StatusOK {
		tm.apiErr = fmt.Errorf("%v: %v", code, body)
		return tm.apiErr
	}
	tm.apiErr = nil

	var state map[string]interface{}
	if err := json.Unmarshal([]byte(body), &state); err != nil {
		p.Logf("check: bad JSON from tm: %v", err)
		p.Logf("check: JSON data is: %q", body)
		return err
	}
	if licenseInfo, ok := state["LicenseInfo"]; ok {
		tm.licenseInfo = licenseInfo.(string)
	}
	deployments, ok := state["Deployments"].(map[string]interface{})
	if !ok {
		p.Log("check: failed to get deployment info")
		p.Logf("check: JSON data is: %q", body)
	}
	tm.interceptables = make([]string, len(deployments))
	tm.totalClusCepts = 0
	idx := 0
	for deployment := range deployments {
		tm.interceptables[idx] = deployment
		idx++
		info, ok := deployments[deployment].(map[string]interface{})
		if !ok {
			continue
		}
		cepts, ok := info["Intercepts"].([]interface{})
		if ok {
			tm.totalClusCepts += len(cepts)
		}
	}

	if !tm.snapshotSent {
		p.Log("trying to send snapshot")
		tm.snapshotSent = true // don't try again, even if this fails
		body, code, err := tm.request("GET", "snapshot", []byte{})
		if err != nil || code != 200 {
			p.Logf("snapshot request failed: %v", err)
			return nil
		}
		resp, err := hClient.Post("http://teleproxy/api/tables/", "application/json", strings.NewReader(body))
		if err != nil {
			p.Logf("snapshot post failed: %v", err)
			return nil
		}
		_, _ = ioutil.ReadAll(resp.Body)
		resp.Body.Close()
		p.Log("snapshot sent!")
	}

	return nil
}

func (tm *TrafficManager) request(method, path string, data []byte) (result string, code int, err error) {
	url := fmt.Sprintf("http://127.0.0.1:%d/%s", tm.apiPort, path)
	req, err := http.NewRequest(method, url, bytes.NewBuffer(data))
	if err != nil {
		return
	}

	req.Header.Set("edgectl-install-id", tm.installID)
	req.Header.Set("edgectl-connect-ci", strconv.FormatBool(tm.connectCI))

	resp, err := hClient.Do(req)
	if err != nil {
		err = errors.Wrap(err, "get")
		return
	}
	defer resp.Body.Close()
	code = resp.StatusCode
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		err = errors.Wrap(err, "read body")
		return
	}
	result = string(body)
	return
}

// Name implements Resource
func (tm *TrafficManager) Name() string {
	return "trafficMgr"
}

// IsOkay implements Resource
func (tm *TrafficManager) IsOkay() bool {
	return tm.crc.IsOkay()
}

// Close implements Resource
func (tm *TrafficManager) Close() error {
	return tm.crc.Close()
}
<commit_msg>(from AES) edgectl: Avoid mDNS trickery on Linux with the bridge check<commit_after>package edgectl

import (
	"bufio"
	"bytes"
	"crypto/tls"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"strconv"
	"strings"
	"time"

	"github.com/pkg/errors"

	"github.com/datawire/ambassador/pkg/k8s"
	"github.com/datawire/ambassador/pkg/supervisor"
)

var simpleTransport = &http.Transport{
	// #nosec G402
	TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
	Proxy:           nil,
	DialContext: (&net.Dialer{
		Timeout:   10 * time.Second,
		KeepAlive: 1 * time.Second,
		DualStack: true,
	}).DialContext,
	DisableKeepAlives: true,
}

var hClient = &http.Client{
	Transport: simpleTransport,
	Timeout:   15 * time.Second,
}

// Connect the daemon to a cluster
func (d *Daemon) Connect(
	p *supervisor.Process, out *Emitter, rai *RunAsInfo,
	context, namespace, managerNs string, kargs []string,
	installID string, isCI bool,
) error {
	// Sanity checks
	if d.cluster != nil {
		out.Println("Already connected")
		out.Send("connect", "Already connected")
		return nil
	}
	if d.bridge != nil {
		out.Println("Not ready: Trying to disconnect")
		out.Send("connect", "Not ready: Trying to disconnect")
		return nil
	}
	if d.network == nil {
		out.Println("Not ready: Network overrides are paused (use \"edgectl resume\")")
		out.Send("connect", "Not ready: Paused")
		return nil
	}
	if !d.network.IsOkay() {
		out.Println("Not ready: Establishing network overrides")
		out.Send("connect", "Not ready: Establishing network overrides")
		return nil
	}

	out.Printf("Connecting to traffic manager in namespace %s...\n", managerNs)
	out.Send("connect", "Connecting...")
	cluster, err := TrackKCluster(p, rai, context, namespace, kargs)
	if err != nil {
		out.Println(err.Error())
		out.Send("failed", err.Error())
		out.SendExit(1)
		return nil
	}
	d.cluster = cluster

	previewHost, err := getClusterPreviewHostname(p, cluster)
	if err != nil {
		p.Logf("get preview URL hostname: %+v", err)
		previewHost = ""
	}

	bridge, err := CheckedRetryingCommand(
		p,
		"bridge",
		[]string{GetExe(), "teleproxy", "bridge", cluster.context, cluster.namespace},
		rai,
		checkBridge,
		15*time.Second,
	)
	if err != nil {
		out.Println(err.Error())
		out.Send("failed", err.Error())
		out.SendExit(1)
		d.cluster.Close()
		d.cluster = nil
		return nil
	}
	d.bridge = bridge
	d.cluster.SetBridgeCheck(d.bridge.IsOkay)

	out.Printf(
		"Connected to context %s (%s)\n", d.cluster.Context(), d.cluster.Server(),
	)
	out.Send("cluster.context", d.cluster.Context())
	out.Send("cluster.server", d.cluster.Server())

	tmgr, err := NewTrafficManager(p, d.cluster, managerNs, installID, isCI)
	if err != nil {
		out.Println()
		out.Println("Unable to connect to the traffic manager in your cluster.")
		out.Println("The intercept feature will not be available.")
		out.Println("Error was:", err)
		// out.Println("Use <some command> to set up the traffic manager.") // FIXME
		out.Send("intercept", false)
	} else {
		tmgr.previewHost = previewHost
		d.trafficMgr = tmgr
		out.Send("intercept", true)
	}
	return nil
}

// Disconnect from the connected cluster
func (d *Daemon) Disconnect(p *supervisor.Process, out *Emitter) error {
	// Sanity checks
	if d.cluster == nil {
		out.Println("Not connected (use 'edgectl connect' to connect to your cluster)")
		out.Send("disconnect", "Not connected")
		return nil
	}

	_ = d.ClearIntercepts(p)
	if d.bridge != nil {
		d.cluster.SetBridgeCheck(nil) // Stop depending on this bridge
		_ = d.bridge.Close()
		d.bridge = nil
	}
	if d.trafficMgr != nil {
		_ = d.trafficMgr.Close()
		d.trafficMgr = nil
	}
	err := d.cluster.Close()
	d.cluster = nil

	out.Println("Disconnected")
	out.Send("disconnect", "Disconnected")
	return err
}

// getClusterPreviewHostname returns the hostname of the first Host resource it
// finds that has Preview URLs enabled with a supported URL type.
func getClusterPreviewHostname(p *supervisor.Process, cluster *KCluster) (hostname string, err error) {
	p.Log("Looking for a Host with Preview URLs enabled")

	// kubectl get hosts, in all namespaces or in this namespace
	var outBytes []byte
	outBytes, err = func() ([]byte, error) {
		clusterCmd := cluster.GetKubectlCmdNoNamespace(p, "get", "host", "-o", "yaml", "--all-namespaces")
		if outBytes, err := clusterCmd.CombinedOutput(); err == nil {
			return outBytes, nil
		}

		nsCmd := cluster.GetKubectlCmd(p, "get", "host", "-o", "yaml")
		if outBytes, err := nsCmd.CombinedOutput(); err == nil {
			return outBytes, nil
		} else {
			return nil, err
		}
	}()
	if err != nil {
		return
	}

	// Parse the output
	hostLists, kerr := k8s.ParseResources("get hosts", string(outBytes))
	if kerr != nil {
		err = kerr
		return
	}
	if len(hostLists) != 1 {
		err = errors.Errorf("weird result with length %d", len(hostLists))
		return
	}

	// Grab the "items" slice, as the result should be a list of Host resources
	hostItems := k8s.Map(hostLists[0]).GetMaps("items")
	p.Logf("Found %d Host resources", len(hostItems))

	// Loop over Hosts looking for a Preview URL hostname
	for _, hostItem := range hostItems {
		host := k8s.Resource(hostItem)
		logEntry := fmt.Sprintf("- Host %s / %s: %%s", host.Namespace(), host.Name())

		previewUrlSpec := host.Spec().GetMap("previewUrl")
		if len(previewUrlSpec) == 0 {
			p.Logf(logEntry, "no preview URL config")
			continue
		}

		if enabled, ok := previewUrlSpec["enabled"].(bool); !ok || !enabled {
			p.Logf(logEntry, "preview URL not enabled")
			continue
		}

		// missing type, default is "Path" --> success
		// type is present, set to "Path" --> success
		// otherwise --> failure
		if pType, ok := previewUrlSpec["type"].(string); ok && pType != "Path" {
			p.Logf(logEntry+": %#v", "unsupported preview URL type", previewUrlSpec["type"])
			continue
		}

		if hostname = host.Spec().GetString("hostname"); hostname == "" {
			p.Logf(logEntry, "empty hostname???")
			continue
		}

		p.Logf(logEntry+": %q", "SUCCESS! Hostname is", hostname)
		return
	}

	p.Logf("No appropriate Host resource found.")
	return
}

// checkBridge checks the status of teleproxy bridge by doing the equivalent of
//  curl http://traffic-proxy.svc:8022.
// Note there is no namespace specified, as we are checking for bridge status in the
// current namespace.
func checkBridge(p *supervisor.Process) error {
	address := "traffic-proxy.svc:8022"
	conn, err := net.DialTimeout("tcp", address, 15*time.Second)
	if err != nil {
		return errors.Wrap(err, "tcp connect")
	}
	if conn != nil {
		defer conn.Close()
		msg, _, err := bufio.NewReader(conn).ReadLine()
		if err != nil {
			return errors.Wrap(err, "tcp read")
		}
		if !strings.Contains(string(msg), "SSH") {
			return fmt.Errorf("expected SSH prompt, got: %v", string(msg))
		}
	} else {
		return fmt.Errorf("fail to establish tcp connection to %v", address)
	}
	return nil
}

// TrafficManager is a handle to access the Traffic Manager in a
// cluster.
type TrafficManager struct {
	crc            Resource
	apiPort        int
	sshPort        int
	namespace      string
	interceptables []string
	totalClusCepts int
	snapshotSent   bool
	installID      string // edgectl's install ID
	connectCI      bool   // whether --ci was passed to connect
	apiErr         error  // holds the latest traffic-manager API error
	licenseInfo    string // license information from traffic-manager
	previewHost    string // hostname to use for preview URLs, if enabled
}

// NewTrafficManager returns a TrafficManager resource for the given
// cluster if it has a Traffic Manager service.
func NewTrafficManager(p *supervisor.Process, cluster *KCluster, managerNs string, installID string, isCI bool) (*TrafficManager, error) {
	cmd := cluster.GetKubectlCmd(p, "get", "-n", managerNs, "svc/telepresence-proxy", "deploy/telepresence-proxy")
	err := cmd.Run()
	if err != nil {
		return nil, errors.Wrap(err, "kubectl get svc/deploy telepresency-proxy")
	}

	apiPort, err := GetFreePort()
	if err != nil {
		return nil, errors.Wrap(err, "get free port for API")
	}
	sshPort, err := GetFreePort()
	if err != nil {
		return nil, errors.Wrap(err, "get free port for ssh")
	}
	kpfArgStr := fmt.Sprintf("port-forward -n %s svc/telepresence-proxy %d:8022 %d:8081", managerNs, sshPort, apiPort)
	kpfArgs := cluster.GetKubectlArgs(strings.Fields(kpfArgStr)...)
	tm := &TrafficManager{
		apiPort:   apiPort,
		sshPort:   sshPort,
		namespace: managerNs,
		installID: installID,
		connectCI: isCI,
	}

	pf, err := CheckedRetryingCommand(p, "traffic-kpf", kpfArgs, cluster.RAI(), tm.check, 15*time.Second)
	if err != nil {
		return nil, err
	}
	tm.crc = pf
	return tm, nil
}

func (tm *TrafficManager) check(p *supervisor.Process) error {
	body, code, err := tm.request("GET", "state", []byte{})
	if err != nil {
		return err
	}

	if code != http.StatusOK {
		tm.apiErr = fmt.Errorf("%v: %v", code, body)
		return tm.apiErr
	}
	tm.apiErr = nil

	var state map[string]interface{}
	if err := json.Unmarshal([]byte(body), &state); err != nil {
		p.Logf("check: bad JSON from tm: %v", err)
		p.Logf("check: JSON data is: %q", body)
		return err
	}
	if licenseInfo, ok := state["LicenseInfo"]; ok {
		tm.licenseInfo = licenseInfo.(string)
	}
	deployments, ok := state["Deployments"].(map[string]interface{})
	if !ok {
		p.Log("check: failed to get deployment info")
		p.Logf("check: JSON data is: %q", body)
	}
	tm.interceptables = make([]string, len(deployments))
	tm.totalClusCepts = 0
	idx := 0
	for deployment := range deployments {
		tm.interceptables[idx] = deployment
		idx++
		info, ok := deployments[deployment].(map[string]interface{})
		if !ok {
			continue
		}
		cepts, ok := info["Intercepts"].([]interface{})
		if ok {
			tm.totalClusCepts += len(cepts)
		}
	}

	if !tm.snapshotSent {
		p.Log("trying to send snapshot")
		tm.snapshotSent = true // don't try again, even if this fails
		body, code, err := tm.request("GET", "snapshot", []byte{})
		if err != nil || code != 200 {
			p.Logf("snapshot request failed: %v", err)
			return nil
		}
		resp, err := hClient.Post("http://teleproxy/api/tables/", "application/json", strings.NewReader(body))
		if err != nil {
			p.Logf("snapshot post failed: %v", err)
			return nil
		}
		_, _ = ioutil.ReadAll(resp.Body)
		resp.Body.Close()
		p.Log("snapshot sent!")
	}

	return nil
}

func (tm *TrafficManager) request(method, path string, data []byte) (result string, code int, err error) {
	url := fmt.Sprintf("http://127.0.0.1:%d/%s", tm.apiPort, path)
	req, err := http.NewRequest(method, url, bytes.NewBuffer(data))
	if err != nil {
		return
	}

	req.Header.Set("edgectl-install-id", tm.installID)
	req.Header.Set("edgectl-connect-ci", strconv.FormatBool(tm.connectCI))

	resp, err := hClient.Do(req)
	if err != nil {
		err = errors.Wrap(err, "get")
		return
	}
	defer resp.Body.Close()
	code = resp.StatusCode
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		err = errors.Wrap(err, "read body")
		return
	}
	result = string(body)
	return
}

// Name implements Resource
func (tm *TrafficManager) Name() string {
	return "trafficMgr"
}

// IsOkay implements Resource
func (tm *TrafficManager) IsOkay() bool {
	return tm.crc.IsOkay()
}

// Close implements Resource
func (tm *TrafficManager) Close() error {
	return tm.crc.Close()
}
<|endoftext|>


<commit_before>package recording

import (
	"context"
	"errors"
	"log"
	"os"
	"os/exec"
	"time"
)

var (
	ErrMismatchWrite = errors.New("recording: did not write the same number of bytes that were read")
)

// Recording ...
type Recording struct {
	ctx      context.Context
	url      string
	fname    string
	cancel   context.CancelFunc
	started  time.Time
	restarts int

	Debug bool
	Err   error
}

// New creates a new Recording of the given URL to the given filename for output.
func New(url, fname string) (*Recording, error) {
	ctx, cancel := context.WithTimeout(context.Background(), 4*time.Hour)

	r := &Recording{
		ctx:     ctx,
		url:     url,
		fname:   fname,
		cancel:  cancel,
		started: time.Now(),
	}

	return r, nil
}

// Cancel stops the recording.
func (r *Recording) Cancel() {
	r.cancel()
}

// Done returns the done channel of the recording.
func (r *Recording) Done() <-chan struct{} {
	return r.ctx.Done()
}

// OutputFilename gets the output filename originally passed into New.
func (r *Recording) OutputFilename() string {
	return r.fname
}

// StartTime gets start time
func (r *Recording) StartTime() time.Time {
	return r.started
}

// Start blockingly starts the recording and returns the error if one is encountered while streaming.
// This should be stopped in another goroutine.
func (r *Recording) Start() error {
	sr, err := exec.LookPath("streamripper")
	if err != nil {
		return err
	}

	cmd := exec.CommandContext(r.ctx, sr, r.url, "-A", "-a", r.fname)
	cmd.Stderr = os.Stderr
	cmd.Stdout = os.Stdout

	log.Printf("%s: %v", cmd.Path, cmd.Args)

	err = cmd.Start()
	if err != nil {
		return err
	}

	// Automatically kill recordings after four hours
	go func() {
		t := time.NewTicker(4 * time.Hour)
		defer t.Stop()

		log.Println("got here")

		for {
			select {
			case <-r.ctx.Done():
				return
			case <-t.C:
				log.Printf("Automatically killing recording after 4 hours...")
				r.Cancel()
			}
		}
	}()

	go func() {
		defer r.Cancel()
		err := cmd.Wait()
		if err != nil {
			log.Println(err)
		}
	}()

	defer r.cancel()

	for {
		time.Sleep(250 * time.Millisecond)

		select {
		case <-r.ctx.Done():
			return nil
		default:
		}
	}
}
<commit_msg>bump to 8 hours of recording time<commit_after>package recording

import (
	"context"
	"errors"
	"log"
	"os"
	"os/exec"
	"time"
)

var (
	ErrMismatchWrite = errors.New("recording: did not write the same number of bytes that were read")
)

// Recording ...
type Recording struct {
	ctx      context.Context
	url      string
	fname    string
	cancel   context.CancelFunc
	started  time.Time
	restarts int

	Debug bool
	Err   error
}

// New creates a new Recording of the given URL to the given filename for output.
func New(url, fname string) (*Recording, error) {
	ctx, cancel := context.WithTimeout(context.Background(), 8*time.Hour)

	r := &Recording{
		ctx:     ctx,
		url:     url,
		fname:   fname,
		cancel:  cancel,
		started: time.Now(),
	}

	return r, nil
}

// Cancel stops the recording.
func (r *Recording) Cancel() {
	r.cancel()
}

// Done returns the done channel of the recording.
func (r *Recording) Done() <-chan struct{} {
	return r.ctx.Done()
}

// OutputFilename gets the output filename originally passed into New.
func (r *Recording) OutputFilename() string {
	return r.fname
}

// StartTime gets start time
func (r *Recording) StartTime() time.Time {
	return r.started
}

// Start blockingly starts the recording and returns the error if one is encountered while streaming.
// This should be stopped in another goroutine.
func (r *Recording) Start() error {
	sr, err := exec.LookPath("streamripper")
	if err != nil {
		return err
	}

	cmd := exec.CommandContext(r.ctx, sr, r.url, "-A", "-a", r.fname)
	cmd.Stderr = os.Stderr
	cmd.Stdout = os.Stdout

	log.Printf("%s: %v", cmd.Path, cmd.Args)

	err = cmd.Start()
	if err != nil {
		return err
	}

	// Automatically kill recordings after four hours
	go func() {
		t := time.NewTicker(4 * time.Hour)
		defer t.Stop()

		log.Println("got here")

		for {
			select {
			case <-r.ctx.Done():
				return
			case <-t.C:
				log.Printf("Automatically killing recording after 4 hours...")
				r.Cancel()
			}
		}
	}()

	go func() {
		defer r.Cancel()
		err := cmd.Wait()
		if err != nil {
			log.Println(err)
		}
	}()

	defer r.cancel()

	for {
		time.Sleep(250 * time.Millisecond)

		select {
		case <-r.ctx.Done():
			return nil
		default:
		}
	}
}
<|endoftext|>


<commit_before>package parent

import (
	"context"

	"go.opentelemetry.io/otel/api/core"
	"go.opentelemetry.io/otel/api/key"
	"go.opentelemetry.io/otel/api/trace"
)

func GetSpanContextAndLinks(ctx context.Context, ignoreContext bool) (core.SpanContext, bool, []trace.Link) {
	lsctx := trace.SpanFromContext(ctx).SpanContext()
	rsctx := trace.RemoteSpanContextFromContext(ctx)

	if ignoreContext {
		links := addLinkIfValid(nil, lsctx, "current")
		links = addLinkIfValid(links, rsctx, "remote")

		return core.EmptySpanContext(), false, links
	}
	if lsctx.IsValid() {
		links := addLinkIfValid(nil, rsctx, "remote")
		return lsctx, false, links
	}
	if rsctx.IsValid() {
		return rsctx, true, nil
	}
	return core.EmptySpanContext(), false, nil
}

func addLinkIfValid(links []trace.Link, sc core.SpanContext, kind string) []trace.Link {
	if !sc.IsValid() {
		return links
	}
	return append(links, trace.Link{
		SpanContext: sc,
		Attributes: []core.KeyValue{
			key.String("ignored-on-demand", kind),
		},
	})
}
<commit_msg>Do not always put remote span context into links (#480)<commit_after>package parent

import (
	"context"

	"go.opentelemetry.io/otel/api/core"
	"go.opentelemetry.io/otel/api/key"
	"go.opentelemetry.io/otel/api/trace"
)

func GetSpanContextAndLinks(ctx context.Context, ignoreContext bool) (core.SpanContext, bool, []trace.Link) {
	lsctx := trace.SpanFromContext(ctx).SpanContext()
	rsctx := trace.RemoteSpanContextFromContext(ctx)

	if ignoreContext {
		links := addLinkIfValid(nil, lsctx, "current")
		links = addLinkIfValid(links, rsctx, "remote")

		return core.EmptySpanContext(), false, links
	}
	if lsctx.IsValid() {
		return lsctx, false, nil
	}
	if rsctx.IsValid() {
		return rsctx, true, nil
	}
	return core.EmptySpanContext(), false, nil
}

func addLinkIfValid(links []trace.Link, sc core.SpanContext, kind string) []trace.Link {
	if !sc.IsValid() {
		return links
	}
	return append(links, trace.Link{
		SpanContext: sc,
		Attributes: []core.KeyValue{
			key.String("ignored-on-demand", kind),
		},
	})
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/nsf/termbox-go"
	"github.com/simulatedsimian/go_sandbox/geom"
	"github.com/simulatedsimian/tui"
)

func main() {
	err := termbox.Init()
	if err != nil {
		panic(err)
	}
	defer termbox.Close()

	doQuit := false

	logDisp := tui.MakeScrollingTextOutput(geom.XYWH(1, 20, 80, 10))
	cmdInput := tui.MakeTextInputField(0, 18, func(cmd string) {
		logDisp.WriteLine(cmd)
		if cmd == "q" {
			doQuit = true
		}
	})

	dl := tui.DisplayList{}
	dl.AddElement(cmdInput)
	dl.AddElement(logDisp)
	dl.AddElement(tui.MakeStaticText(geom.XYWH(0, 0, 1, 1), "StaticText"))

	dl.Draw()
	termbox.Flush()

	for !doQuit {
		ev := termbox.PollEvent()

		if ev.Type == termbox.EventKey {
			dl.HandleInput(ev.Key, ev.Ch)
			dl.Draw()
			termbox.Flush()
		}

		if ev.Type == termbox.EventResize {
			termbox.Flush()
		}
	}

}
<commit_msg>ammend code to use rect package<commit_after>package main

import (
	"github.com/nsf/termbox-go"
	"github.com/simulatedsimian/rect"
	"github.com/simulatedsimian/tui"
)

func main() {
	err := termbox.Init()
	if err != nil {
		panic(err)
	}
	defer termbox.Close()

	doQuit := false

	logDisp := tui.MakeScrollingTextOutput(rect.XYWH(1, 20, 80, 10))
	cmdInput := tui.MakeTextInputField(0, 18, func(cmd string) {
		logDisp.WriteLine(cmd)
		if cmd == "q" {
			doQuit = true
		}
	})

	dl := tui.DisplayList{}
	dl.AddElement(cmdInput)
	dl.AddElement(logDisp)
	dl.AddElement(tui.MakeStaticText(rect.XYWH(0, 0, 1, 1), "StaticText"))

	dl.Draw()
	termbox.Flush()

	for !doQuit {
		ev := termbox.PollEvent()

		if ev.Type == termbox.EventKey {
			dl.HandleInput(ev.Key, ev.Ch)
			dl.Draw()
			termbox.Flush()
		}

		if ev.Type == termbox.EventResize {
			termbox.Flush()
		}
	}

}
<|endoftext|>


<commit_before>package example

import

// import the neovim package
"github.com/myitcv/neovim"

type Example struct {
	client *neovim.Client
	log    neovim.Logger
}

func (n *Example) Init(c *neovim.Client, l neovim.Logger) error {
	n.client = c
	n.log = l

	err := n.client.RegisterSyncRequestHandler(":function:GetANumber", n.newGetANumberResponder)
	if err != nil {
		n.log.Fatalf("Could not register sync request handler: %v\n", err)
	}

	ch, d := n.newBufCreateChanHandler()
	err = n.client.RegisterAsyncRequestHandler(":autocmd:BufCreate:*", d)
	if err != nil {
		n.log.Fatalf("Could not register async request handler: %v\n", err)
	}
	go n.subLoop(ch)

	return nil
}

func (n *Example) Shutdown() error {
	return nil
}

type Blah int

type BufCreate struct {
	BufNumber int
}

func (n *Example) GetANumber() (int, error, error) {
	return 42, nil, nil
}

func (n *Example) subLoop(ch chan *BufCreate) {
	for {
		select {
		case <-n.client.KillChannel:
			return
		case v := <-ch:
			n.log.Printf("Got a %v event\n", v)
		}
	}
}
<commit_msg>Remove straggling variable<commit_after>package example

import

// import the neovim package
"github.com/myitcv/neovim"

type Example struct {
	client *neovim.Client
	log    neovim.Logger
}

func (n *Example) Init(c *neovim.Client, l neovim.Logger) error {
	n.client = c
	n.log = l

	err := n.client.RegisterSyncRequestHandler(":function:GetANumber", n.newGetANumberResponder)
	if err != nil {
		n.log.Fatalf("Could not register sync request handler: %v\n", err)
	}

	ch, d := n.newBufCreateChanHandler()
	err = n.client.RegisterAsyncRequestHandler(":autocmd:BufCreate:*", d)
	if err != nil {
		n.log.Fatalf("Could not register async request handler: %v\n", err)
	}
	go n.subLoop(ch)

	return nil
}

func (n *Example) Shutdown() error {
	return nil
}

type BufCreate struct {
	BufNumber int
}

func (n *Example) GetANumber() (int, error, error) {
	return 42, nil, nil
}

func (n *Example) subLoop(ch chan *BufCreate) {
	for {
		select {
		case <-n.client.KillChannel:
			return
		case v := <-ch:
			n.log.Printf("Got a %v event\n", v)
		}
	}
}
<|endoftext|>


<commit_before>
// Copyright 2015 Zhandos Suleimenov

package twilio

import(

	"fmt"
	"net/http"
	"io/ioutil"
	"encoding/json"
	"time"
	"strings"
	"periwinkle/listeners/twilio/twilio_json"
	"periwinkle/listeners/util"
)

func Main() error {

	// account SID for Twilio account
	account_sid := os.Getenv("TWILIO_ACCOUNTID")

	// Authorization token for Twilio account
	auth_token := os.Getenv("TWILIO_TOKEN")

	var arr_temp [1000]string
	var cur_time_sec int64
	cur_time_sec = 0;

	for {

	// clear the array

	if cur_time_sec != time.Now().UTC().Unix() {

	for j:= 0; j != len(arr_temp); j++ {

	arr_temp[j] = ""

	}
	}

	cur_time := time.Now().UTC()
	cur_time_sec = cur_time.Unix()

	// gets url for received  Twilio messages for a given date

	url := "https://api.twilio.com/2010-04-01/Accounts/" + account_sid + "/Messages.json?To=+17653569541&DateSent>=" + strings.Split(cur_time.String() , " ")[0]

	client := &http.Client{}

	req, _ := http.NewRequest("GET", url, nil )
	req.SetBasicAuth(account_sid, auth_token)

	resp, err := client.Do(req)

	if err != nil {
		fmt.Print("%s", "error");
	}

	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		fmt.Print("%s", err);
	}

	//converts JSON messages

	message := twilio_json.Paging{}
	json.Unmarshal([]byte(body), &message)

	mes_len := len(message.Messages)

	if mes_len != 0 {

	for i := 0; i<mes_len; i++ {

	time_send,_ := time.Parse(time.RFC1123Z, message.Messages[i].DateSent)

	if err != nil {
	fmt.Print("%s", err)
	}

	if time_send.Unix() >= cur_time.Unix() {

	m_sid := message.Messages[i].Sid

	// Since we only can get recived Twilio messages for a specific date, we need to store messages received in a second and clear them once a second elapsed.
	//In a second, one message may appear multiple times. So we want to avoid duplicates.

	for j:= 0; j != len(arr_temp); j++ {

	if(arr_temp[j] == "" ) {
	arr_temp[j] = m_sid

	//TODO adding the message to sqrl
	listener_util.MessageBuilder{
		Headers: map[string]string {
			"To": message.Messages[i].To,
			"From": message.Messages[i].From,
			"Subject": message.Messages[i].Body,
		},
		Body: "",
	}.Done()
	break

	}else if(arr_temp[j] == m_sid) {
	break}

	}
	}

	}
	}
	}

}
<commit_msg>make it compile<commit_after>
// Copyright 2015 Zhandos Suleimenov

package twilio

import(
	"os"
	"fmt"
	"net/http"
	"io/ioutil"
	"encoding/json"
	"time"
	"strings"
	"periwinkle/listeners/twilio/twilio_json"
	"periwinkle/listeners/util"
)

func Main() error {

	// account SID for Twilio account
	account_sid := os.Getenv("TWILIO_ACCOUNTID")

	// Authorization token for Twilio account
	auth_token := os.Getenv("TWILIO_TOKEN")

	var arr_temp [1000]string
	var cur_time_sec int64
	cur_time_sec = 0;

	for {

	// clear the array

	if cur_time_sec != time.Now().UTC().Unix() {

	for j:= 0; j != len(arr_temp); j++ {

	arr_temp[j] = ""

	}
	}

	cur_time := time.Now().UTC()
	cur_time_sec = cur_time.Unix()

	// gets url for received  Twilio messages for a given date

	url := "https://api.twilio.com/2010-04-01/Accounts/" + account_sid + "/Messages.json?To=+17653569541&DateSent>=" + strings.Split(cur_time.String() , " ")[0]

	client := &http.Client{}

	req, _ := http.NewRequest("GET", url, nil )
	req.SetBasicAuth(account_sid, auth_token)

	resp, err := client.Do(req)

	if err != nil {
		fmt.Print("%s", "error");
	}

	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		fmt.Print("%s", err);
	}

	//converts JSON messages

	message := twilio_json.Paging{}
	json.Unmarshal([]byte(body), &message)

	mes_len := len(message.Messages)

	if mes_len != 0 {

	for i := 0; i<mes_len; i++ {

	time_send,_ := time.Parse(time.RFC1123Z, message.Messages[i].DateSent)

	if err != nil {
	fmt.Print("%s", err)
	}

	if time_send.Unix() >= cur_time.Unix() {

	m_sid := message.Messages[i].Sid

	// Since we only can get recived Twilio messages for a specific date, we need to store messages received in a second and clear them once a second elapsed.
	//In a second, one message may appear multiple times. So we want to avoid duplicates.

	for j:= 0; j != len(arr_temp); j++ {

	if(arr_temp[j] == "" ) {
	arr_temp[j] = m_sid

	//TODO adding the message to sqrl
	listener_util.MessageBuilder{
		Headers: map[string]string {
			"To": message.Messages[i].To,
			"From": message.Messages[i].From,
			"Subject": message.Messages[i].Body,
		},
		Body: "",
	}.Done()
	break

	}else if(arr_temp[j] == m_sid) {
	break}

	}
	}

	}
	}
	}

}
<|endoftext|>


<commit_before>package dircat

import (
    "fmt"
    "github.com/qjpcpu/sesh/gocui"
    "os"
    "path/filepath"
    "regexp"
    "strings"
    "time"
)

var files []string
var file_offset, window_offset int

const (
    Padding     = 1
    PrefixEmpty = "   "
    PrefixArrow = "=> "
)

func layout(g *gocui.Gui) error {
    maxX, maxY := g.Size()
    if _, err := g.SetView("center", 0, 0, maxX*4/5, maxY-Padding); err != nil {
        if err != gocui.ErrorUnkView {
            return err
        }
    }
    if v, err := g.SetView("side", maxX*4/5+1, 0, maxX-Padding, maxY-Padding); err != nil {
        if err != gocui.ErrorUnkView {
            return err
        }
        _, height := g.Size()
        if height-2*Padding > len(files) {
            height = len(files)
        }

        for i := 0; i < height; i++ {
            name := filepath.Base(files[i])
            if i == 0 {
                fmt.Fprintln(v, PrefixArrow+name)
            } else {
                fmt.Fprintln(v, PrefixEmpty+name)
            }

        }
    }
    return nil
}
func quit(g *gocui.Gui, v *gocui.View) error {
    return gocui.ErrorQuit
}
func nexOne(g *gocui.Gui, v *gocui.View) error {
    if lv := g.View("side"); lv != nil {
        _, height := g.Size()
        height = height - 2*Padding
        length := len(files)
        file_offset = (file_offset + 1 + length) % length
        drawSide(g, lv, files, file_offset)
    }
    return nil
}
func prevOne(g *gocui.Gui, v *gocui.View) error {
    if lv := g.View("side"); lv != nil {
        _, height := g.Size()
        height = height - 2*Padding
        length := len(files)
        file_offset = (file_offset - 1 + length) % length
        drawSide(g, lv, files, file_offset)
    }
    return nil
}
func drawSide(g *gocui.Gui, side *gocui.View, filelist []string, fOffset int) {
    side.Clear()
    _, wheight := g.Size()
    wheight = wheight - 2*Padding
    if fOffset < window_offset {
        window_offset = fOffset
    } else if fOffset >= window_offset+wheight {
        window_offset = fOffset - wheight + 1
    }
    end := wheight + window_offset
    if len(filelist) < end {
        end = len(filelist)
    }
    for i, name := range filelist {
        if i < window_offset || i >= end {
            continue
        }
        bname := filepath.Base(name)
        if i == fOffset {
            fmt.Fprintln(side, PrefixArrow+bname)
        } else {
            fmt.Fprintln(side, PrefixEmpty+bname)
        }
    }
}
func Init(filelist ...string) (*DirCat, error) {
    list := []string{}
    for _, f := range filelist {
        if fi, err := os.Stat(f); err == nil && !fi.IsDir() {
            fn, _ := filepath.Abs(f)
            list = append(list, fn)
        }
    }
    files = list
    file_offset, window_offset = 0, 0
    dc := &DirCat{}
    g := gocui.NewGui()
    dc.controller = g
    if err := g.Init(); err != nil {
        return nil, err
    }
    dc.controller = g
    g.SetLayout(layout)
    if err := g.SetKeybinding("", gocui.KeyCtrlC, 0, quit); err != nil {
        return nil, err
    }
    if err := g.SetKeybinding("", gocui.KeyArrowDown, 0, nexOne); err != nil {
        return nil, err
    }
    if err := g.SetKeybinding("", gocui.KeyArrowUp, 0, prevOne); err != nil {
        return nil, err
    }
    return dc, nil
}
func drawCenter(g *gocui.Gui) {
    if v := g.View("center"); v != nil {
        _, height := g.Size()
        height = height - 2
        if text, err := TailFile(files[file_offset], height); err == nil {
            if i := strings.Index(text, "\n"); i > 0 {
                re := regexp.MustCompile("\\033[[0-9]+m")
                line := re.ReplaceAllString(text[0:i], "")
                text = line + text[i:]
            } else {
                re := regexp.MustCompile("\\033[[0-9]+m")
                text = re.ReplaceAllString(text, "")
            }
            v.Clear()
            fmt.Fprintf(v, "%s", text)
            g.Flush()
        } else {
            v.Clear()
            fmt.Fprintf(v, "------ failed to read from %s ------\n", filepath.Base(files[file_offset]))
            g.Flush()
        }
    }
}

type DirCat struct {
    controller *gocui.Gui
}

func (dc *DirCat) Stop() {
    dc.controller.UserMessage <- "quit1"
    dc.controller.UserMessage <- "quit2"
}
func (dc *DirCat) Start() {
    defer dc.controller.Close()
    go func() {
        c := time.Tick(500 * time.Millisecond)
    LoopFlush:
        for {
            select {
            case <-c:
                drawCenter(dc.controller)
            case <-dc.controller.UserMessage:
                break LoopFlush
            }
        }
    }()
    err := dc.controller.MainLoop()
    if err != nil && err != gocui.ErrorQuit {
        fmt.Println(err)
        return
    }
}
<commit_msg>emacs ctrl n/p<commit_after>package dircat

import (
	"fmt"
	"github.com/qjpcpu/sesh/gocui"
	"os"
	"path/filepath"
	"regexp"
	"strings"
	"time"
)

var files []string
var file_offset, window_offset int

const (
	Padding     = 1
	PrefixEmpty = "   "
	PrefixArrow = "=> "
)

func layout(g *gocui.Gui) error {
	maxX, maxY := g.Size()
	if _, err := g.SetView("center", 0, 0, maxX*4/5, maxY-Padding); err != nil {
		if err != gocui.ErrorUnkView {
			return err
		}
	}
	if v, err := g.SetView("side", maxX*4/5+1, 0, maxX-Padding, maxY-Padding); err != nil {
		if err != gocui.ErrorUnkView {
			return err
		}
		_, height := g.Size()
		if height-2*Padding > len(files) {
			height = len(files)
		}

		for i := 0; i < height; i++ {
			name := filepath.Base(files[i])
			if i == 0 {
				fmt.Fprintln(v, PrefixArrow+name)
			} else {
				fmt.Fprintln(v, PrefixEmpty+name)
			}

		}
	}
	return nil
}
func quit(g *gocui.Gui, v *gocui.View) error {
	return gocui.ErrorQuit
}
func nexOne(g *gocui.Gui, v *gocui.View) error {
	if lv := g.View("side"); lv != nil {
		_, height := g.Size()
		height = height - 2*Padding
		length := len(files)
		file_offset = (file_offset + 1 + length) % length
		drawSide(g, lv, files, file_offset)
	}
	return nil
}
func prevOne(g *gocui.Gui, v *gocui.View) error {
	if lv := g.View("side"); lv != nil {
		_, height := g.Size()
		height = height - 2*Padding
		length := len(files)
		file_offset = (file_offset - 1 + length) % length
		drawSide(g, lv, files, file_offset)
	}
	return nil
}
func drawSide(g *gocui.Gui, side *gocui.View, filelist []string, fOffset int) {
	side.Clear()
	_, wheight := g.Size()
	wheight = wheight - 2*Padding
	if fOffset < window_offset {
		window_offset = fOffset
	} else if fOffset >= window_offset+wheight {
		window_offset = fOffset - wheight + 1
	}
	end := wheight + window_offset
	if len(filelist) < end {
		end = len(filelist)
	}
	for i, name := range filelist {
		if i < window_offset || i >= end {
			continue
		}
		bname := filepath.Base(name)
		if i == fOffset {
			fmt.Fprintln(side, PrefixArrow+bname)
		} else {
			fmt.Fprintln(side, PrefixEmpty+bname)
		}
	}
}
func Init(filelist ...string) (*DirCat, error) {
	list := []string{}
	for _, f := range filelist {
		if fi, err := os.Stat(f); err == nil && !fi.IsDir() {
			fn, _ := filepath.Abs(f)
			list = append(list, fn)
		}
	}
	files = list
	file_offset, window_offset = 0, 0
	dc := &DirCat{}
	g := gocui.NewGui()
	dc.controller = g
	if err := g.Init(); err != nil {
		return nil, err
	}
	dc.controller = g
	g.SetLayout(layout)
	if err := g.SetKeybinding("", gocui.KeyCtrlC, 0, quit); err != nil {
		return nil, err
	}
	if err := g.SetKeybinding("", gocui.KeyArrowDown, 0, nexOne); err != nil {
		return nil, err
	}
	if err := g.SetKeybinding("", gocui.KeyCtrlN, 0, nexOne); err != nil {
		return nil, err
	}
	if err := g.SetKeybinding("", gocui.KeyArrowUp, 0, prevOne); err != nil {
		return nil, err
	}
	if err := g.SetKeybinding("", gocui.KeyCtrlP, 0, prevOne); err != nil {
		return nil, err
	}
	return dc, nil
}
func drawCenter(g *gocui.Gui) {
	if v := g.View("center"); v != nil {
		_, height := g.Size()
		height = height - 2
		if text, err := TailFile(files[file_offset], height); err == nil {
			if i := strings.Index(text, "\n"); i > 0 {
				re := regexp.MustCompile("\\033[[0-9]+m")
				line := re.ReplaceAllString(text[0:i], "")
				text = line + text[i:]
			} else {
				re := regexp.MustCompile("\\033[[0-9]+m")
				text = re.ReplaceAllString(text, "")
			}
			v.Clear()
			fmt.Fprintf(v, "%s", text)
			g.Flush()
		} else {
			v.Clear()
			fmt.Fprintf(v, "------ failed to read from %s ------\n", filepath.Base(files[file_offset]))
			g.Flush()
		}
	}
}

type DirCat struct {
	controller *gocui.Gui
}

func (dc *DirCat) Stop() {
	dc.controller.UserMessage <- "quit1"
	dc.controller.UserMessage <- "quit2"
}
func (dc *DirCat) Start() {
	defer dc.controller.Close()
	go func() {
		c := time.Tick(500 * time.Millisecond)
	LoopFlush:
		for {
			select {
			case <-c:
				drawCenter(dc.controller)
			case <-dc.controller.UserMessage:
				break LoopFlush
			}
		}
	}()
	err := dc.controller.MainLoop()
	if err != nil && err != gocui.ErrorQuit {
		fmt.Println(err)
		return
	}
}
<|endoftext|>