<commit_before>// Copyright 2016 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License.
//
// Author: Tamir Duberstein (tamird@gmail.com)

// This utility detects new tests added in a given pull request, and runs them
// under stress in our CI infrastructure.
//
// Note that this program will directly exec `make`, so there is no need to
// process its output. See build/teamcity- test{,race}.sh for usage examples.
//
// Note that our CI infrastructure has no notion of "pull requests", forcing
// the approach taken here be quite brute-force with respect to its use of the
// GitHub API.
package main

import (
	"bufio"
	"bytes"
	"context"
	"fmt"
	"go/build"
	"io"
	"log"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"
	"time"

	"golang.org/x/oauth2"

	"github.com/google/go-github/github"
)

const githubAPITokenEnv = "GITHUB_API_TOKEN"
const teamcityVCSNumberEnv = "BUILD_VCS_NUMBER"
const makeTargetEnv = "TARGET"

// https://github.com/golang/go/blob/go1.7.3/src/cmd/go/test.go#L1260:L1262
//
// It is a Test (say) if there is a character after Test that is not a lower-case letter.
// We don't want TesticularCancer.
var newGoTestRE = regexp.MustCompile(`^\+\s*func (Test[^a-z]\w*)\(.*\*testing\.TB?\) {$`)
var newGoBenchmarkRE = regexp.MustCompile(`^\+\s*func (Benchmark[^a-z]\w*)\(.*\*testing\.T?B\) {$`)

type pkg struct {
	tests, benchmarks []string
}

// pkgsFromDiff parses a git-style diff and returns a mapping from directories
// to tests and benchmarks added in those directories in the given diff.
func pkgsFromDiff(r io.Reader) (map[string]pkg, error) {
	const newFilePrefix = "+++ b/"
	const replacement = "$1"

	pkgs := make(map[string]pkg)

	var curPkgName string
	var inPrefix bool
	for reader := bufio.NewReader(r); ; {
		line, isPrefix, err := reader.ReadLine()
		switch err {
		case nil:
		case io.EOF:
			return pkgs, nil
		default:
			return nil, err
		}
		// Ignore generated files a la embedded.go.
		if isPrefix {
			inPrefix = true
			continue
		} else if inPrefix {
			inPrefix = false
			continue
		}

		switch {
		case bytes.HasPrefix(line, []byte(newFilePrefix)):
			curPkgName = filepath.Dir(string(bytes.TrimPrefix(line, []byte(newFilePrefix))))
		case newGoTestRE.Match(line):
			curPkg := pkgs[curPkgName]
			curPkg.tests = append(curPkg.tests, string(newGoTestRE.ReplaceAll(line, []byte(replacement))))
			pkgs[curPkgName] = curPkg
		case newGoBenchmarkRE.Match(line):
			curPkg := pkgs[curPkgName]
			curPkg.benchmarks = append(curPkg.benchmarks, string(newGoBenchmarkRE.ReplaceAll(line, []byte(replacement))))
			pkgs[curPkgName] = curPkg
		}
	}
}

func main() {
	sha, ok := os.LookupEnv(teamcityVCSNumberEnv)
	if !ok {
		log.Fatalf("VCS number environment variable %s is not set", teamcityVCSNumberEnv)
	}

	target, ok := os.LookupEnv(makeTargetEnv)
	if !ok {
		log.Fatalf("make target variable %s is not set", makeTargetEnv)
	}

	const org = "cockroachdb"
	const repo = "cockroach"

	crdb, err := build.Import(fmt.Sprintf("github.com/%s/%s", org, repo), "", build.FindOnly)
	if err != nil {
		log.Fatal(err)
	}

	var httpClient *http.Client
	if token, ok := os.LookupEnv(githubAPITokenEnv); ok {
		httpClient = oauth2.NewClient(context.Background(), oauth2.StaticTokenSource(
			&oauth2.Token{AccessToken: token},
		))
	} else {
		log.Printf("GitHub API token environment variable %s is not set", githubAPITokenEnv)
	}
	client := github.NewClient(httpClient)

	pulls, _, err := client.PullRequests.List(org, repo, nil)
	if err != nil {
		log.Fatal(err)
	}
	var currentPull *github.PullRequest
	for _, pull := range pulls {
		if *pull.Head.SHA == sha {
			currentPull = pull
			break
		}
	}
	if currentPull == nil {
		log.Printf("SHA %s not found in open pull requests, skipping stress", sha)
		return
	}

	diff, _, err := client.PullRequests.GetRaw(
		org,
		repo,
		*currentPull.Number,
		github.RawOptions{Type: github.Patch},
	)
	if err != nil {
		log.Fatal(err)
	}

	pkgs, err := pkgsFromDiff(strings.NewReader(diff))
	if err != nil {
		log.Fatal(err)
	}
	if len(pkgs) > 0 {
		// 5 minutes total seems OK.
		duration := (5 * time.Minute) / time.Duration(len(pkgs))
		for name, pkg := range pkgs {
			tests := "-"
			if len(pkg.tests) > 0 {
				tests = "(" + strings.Join(pkg.tests, "|") + ")"
			}
			benchmarks := "-"
			if len(pkg.benchmarks) > 0 {
				benchmarks = "(" + strings.Join(pkg.benchmarks, "|") + ")"
			}

			cmd := exec.Command(
				"make",
				target,
				fmt.Sprintf("PKG=./%s", name),
				fmt.Sprintf("TESTS=%s", tests),
				fmt.Sprintf("BENCHES=%s", benchmarks),
				fmt.Sprintf("STRESSFLAGS=-stderr -maxfails 1 -maxtime %s", duration),
			)
			cmd.Dir = crdb.Dir
			cmd.Stdout = os.Stdout
			cmd.Stderr = os.Stderr
			log.Println(cmd.Args)
			if err := cmd.Run(); err != nil {
				log.Fatal(err)
			}
		}
	}
}
<commit_msg>cmd/github-pull-request-make: avoid stressing benchmarks<commit_after>// Copyright 2016 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License.
//
// Author: Tamir Duberstein (tamird@gmail.com)

// This utility detects new tests added in a given pull request, and runs them
// under stress in our CI infrastructure.
//
// Note that this program will directly exec `make`, so there is no need to
// process its output. See build/teamcity- test{,race}.sh for usage examples.
//
// Note that our CI infrastructure has no notion of "pull requests", forcing
// the approach taken here be quite brute-force with respect to its use of the
// GitHub API.
package main

import (
	"bufio"
	"bytes"
	"context"
	"fmt"
	"go/build"
	"io"
	"log"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"
	"time"

	"golang.org/x/oauth2"

	"github.com/google/go-github/github"
)

const githubAPITokenEnv = "GITHUB_API_TOKEN"
const teamcityVCSNumberEnv = "BUILD_VCS_NUMBER"
const makeTargetEnv = "TARGET"

// https://github.com/golang/go/blob/go1.7.3/src/cmd/go/test.go#L1260:L1262
//
// It is a Test (say) if there is a character after Test that is not a lower-case letter.
// We don't want TesticularCancer.
var newGoTestRE = regexp.MustCompile(`^\+\s*func (Test[^a-z]\w*)\(.*\*testing\.TB?\) {$`)
var newGoBenchmarkRE = regexp.MustCompile(`^\+\s*func (Benchmark[^a-z]\w*)\(.*\*testing\.T?B\) {$`)

type pkg struct {
	tests, benchmarks []string
}

// pkgsFromDiff parses a git-style diff and returns a mapping from directories
// to tests and benchmarks added in those directories in the given diff.
func pkgsFromDiff(r io.Reader) (map[string]pkg, error) {
	const newFilePrefix = "+++ b/"
	const replacement = "$1"

	pkgs := make(map[string]pkg)

	var curPkgName string
	var inPrefix bool
	for reader := bufio.NewReader(r); ; {
		line, isPrefix, err := reader.ReadLine()
		switch err {
		case nil:
		case io.EOF:
			return pkgs, nil
		default:
			return nil, err
		}
		// Ignore generated files a la embedded.go.
		if isPrefix {
			inPrefix = true
			continue
		} else if inPrefix {
			inPrefix = false
			continue
		}

		switch {
		case bytes.HasPrefix(line, []byte(newFilePrefix)):
			curPkgName = filepath.Dir(string(bytes.TrimPrefix(line, []byte(newFilePrefix))))
		case newGoTestRE.Match(line):
			curPkg := pkgs[curPkgName]
			curPkg.tests = append(curPkg.tests, string(newGoTestRE.ReplaceAll(line, []byte(replacement))))
			pkgs[curPkgName] = curPkg
		case newGoBenchmarkRE.Match(line):
			curPkg := pkgs[curPkgName]
			curPkg.benchmarks = append(curPkg.benchmarks, string(newGoBenchmarkRE.ReplaceAll(line, []byte(replacement))))
			pkgs[curPkgName] = curPkg
		}
	}
}

func main() {
	sha, ok := os.LookupEnv(teamcityVCSNumberEnv)
	if !ok {
		log.Fatalf("VCS number environment variable %s is not set", teamcityVCSNumberEnv)
	}

	target, ok := os.LookupEnv(makeTargetEnv)
	if !ok {
		log.Fatalf("make target variable %s is not set", makeTargetEnv)
	}

	const org = "cockroachdb"
	const repo = "cockroach"

	crdb, err := build.Import(fmt.Sprintf("github.com/%s/%s", org, repo), "", build.FindOnly)
	if err != nil {
		log.Fatal(err)
	}

	var httpClient *http.Client
	if token, ok := os.LookupEnv(githubAPITokenEnv); ok {
		httpClient = oauth2.NewClient(context.Background(), oauth2.StaticTokenSource(
			&oauth2.Token{AccessToken: token},
		))
	} else {
		log.Printf("GitHub API token environment variable %s is not set", githubAPITokenEnv)
	}
	client := github.NewClient(httpClient)

	pulls, _, err := client.PullRequests.List(org, repo, nil)
	if err != nil {
		log.Fatal(err)
	}
	var currentPull *github.PullRequest
	for _, pull := range pulls {
		if *pull.Head.SHA == sha {
			currentPull = pull
			break
		}
	}
	if currentPull == nil {
		log.Printf("SHA %s not found in open pull requests, skipping stress", sha)
		return
	}

	diff, _, err := client.PullRequests.GetRaw(
		org,
		repo,
		*currentPull.Number,
		github.RawOptions{Type: github.Patch},
	)
	if err != nil {
		log.Fatal(err)
	}

	pkgs, err := pkgsFromDiff(strings.NewReader(diff))
	if err != nil {
		log.Fatal(err)
	}
	if len(pkgs) > 0 {
		// 5 minutes total seems OK.
		duration := (5 * time.Minute) / time.Duration(len(pkgs))
		for name, pkg := range pkgs {
			tests := "-"
			if len(pkg.tests) > 0 {
				tests = "(" + strings.Join(pkg.tests, "|") + ")"
			}

			cmd := exec.Command(
				"make",
				target,
				fmt.Sprintf("PKG=./%s", name),
				fmt.Sprintf("TESTS=%s", tests),
				fmt.Sprintf("STRESSFLAGS=-stderr -maxfails 1 -maxtime %s", duration),
			)
			cmd.Dir = crdb.Dir
			cmd.Stdout = os.Stdout
			cmd.Stderr = os.Stderr
			log.Println(cmd.Args)
			if err := cmd.Run(); err != nil {
				log.Fatal(err)
			}
		}
	}
}
<|endoftext|>


<commit_before>// +build windows

/*
Copyright 2015 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cadvisor

import (
	"github.com/google/cadvisor/events"
	cadvisorapi "github.com/google/cadvisor/info/v1"
	cadvisorapiv2 "github.com/google/cadvisor/info/v2"
	"k8s.io/kubernetes/pkg/kubelet/winstats"
)

type cadvisorClient struct {
	winStatsClient winstats.Client
}

var _ Interface = new(cadvisorClient)

// New creates a cAdvisor and exports its API on the specified port if port > 0.
func New(address string, port uint, imageFsInfoProvider ImageFsInfoProvider, rootPath string, usingLegacyStats bool) (Interface, error) {
	client, err := winstats.NewPerfCounterClient()
	return &cadvisorClient{winStatsClient: client}, err
}

func (cu *cadvisorClient) Start() error {
	return nil
}

func (cu *cadvisorClient) DockerContainer(name string, req *cadvisorapi.ContainerInfoRequest) (cadvisorapi.ContainerInfo, error) {
	return cadvisorapi.ContainerInfo{}, nil
}

func (cu *cadvisorClient) ContainerInfo(name string, req *cadvisorapi.ContainerInfoRequest) (*cadvisorapi.ContainerInfo, error) {
	return &cadvisorapi.ContainerInfo{}, nil
}

func (cu *cadvisorClient) ContainerInfoV2(name string, options cadvisorapiv2.RequestOptions) (map[string]cadvisorapiv2.ContainerInfo, error) {
	return cu.winStatsClient.WinContainerInfos()
}

func (cu *cadvisorClient) SubcontainerInfo(name string, req *cadvisorapi.ContainerInfoRequest) (map[string]*cadvisorapi.ContainerInfo, error) {
	return nil, nil
}

func (cu *cadvisorClient) MachineInfo() (*cadvisorapi.MachineInfo, error) {
	return cu.winStatsClient.WinMachineInfo()
}

func (cu *cadvisorClient) VersionInfo() (*cadvisorapi.VersionInfo, error) {
	return cu.winStatsClient.WinVersionInfo()
}

func (cu *cadvisorClient) ImagesFsInfo() (cadvisorapiv2.FsInfo, error) {
	return cadvisorapiv2.FsInfo{}, nil
}

func (cu *cadvisorClient) RootFsInfo() (cadvisorapiv2.FsInfo, error) {
	return cadvisorapiv2.FsInfo{}, nil
}

func (cu *cadvisorClient) WatchEvents(request *events.Request) (*events.EventChannel, error) {
	return &events.EventChannel{}, nil
}

func (cu *cadvisorClient) GetDirFsInfo(path string) (cadvisorapiv2.FsInfo, error) {
	return cadvisorapiv2.FsInfo{}, nil
}
<commit_msg>Get dirFsInfo from docker image filesystem<commit_after>// +build windows

/*
Copyright 2015 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cadvisor

import (
	"github.com/google/cadvisor/events"
	cadvisorapi "github.com/google/cadvisor/info/v1"
	cadvisorapiv2 "github.com/google/cadvisor/info/v2"
	"k8s.io/kubernetes/pkg/kubelet/winstats"
)

type cadvisorClient struct {
	winStatsClient winstats.Client
}

var _ Interface = new(cadvisorClient)

// New creates a cAdvisor and exports its API on the specified port if port > 0.
func New(address string, port uint, imageFsInfoProvider ImageFsInfoProvider, rootPath string, usingLegacyStats bool) (Interface, error) {
	client, err := winstats.NewPerfCounterClient()
	return &cadvisorClient{winStatsClient: client}, err
}

func (cu *cadvisorClient) Start() error {
	return nil
}

func (cu *cadvisorClient) DockerContainer(name string, req *cadvisorapi.ContainerInfoRequest) (cadvisorapi.ContainerInfo, error) {
	return cadvisorapi.ContainerInfo{}, nil
}

func (cu *cadvisorClient) ContainerInfo(name string, req *cadvisorapi.ContainerInfoRequest) (*cadvisorapi.ContainerInfo, error) {
	return &cadvisorapi.ContainerInfo{}, nil
}

func (cu *cadvisorClient) ContainerInfoV2(name string, options cadvisorapiv2.RequestOptions) (map[string]cadvisorapiv2.ContainerInfo, error) {
	return cu.winStatsClient.WinContainerInfos()
}

func (cu *cadvisorClient) SubcontainerInfo(name string, req *cadvisorapi.ContainerInfoRequest) (map[string]*cadvisorapi.ContainerInfo, error) {
	return nil, nil
}

func (cu *cadvisorClient) MachineInfo() (*cadvisorapi.MachineInfo, error) {
	return cu.winStatsClient.WinMachineInfo()
}

func (cu *cadvisorClient) VersionInfo() (*cadvisorapi.VersionInfo, error) {
	return cu.winStatsClient.WinVersionInfo()
}

func (cu *cadvisorClient) ImagesFsInfo() (cadvisorapiv2.FsInfo, error) {
	return cadvisorapiv2.FsInfo{}, nil
}

func (cu *cadvisorClient) RootFsInfo() (cadvisorapiv2.FsInfo, error) {
	return cadvisorapiv2.FsInfo{}, nil
}

func (cu *cadvisorClient) WatchEvents(request *events.Request) (*events.EventChannel, error) {
	return &events.EventChannel{}, nil
}

func (cu *cadvisorClient) GetDirFsInfo(path string) (cadvisorapiv2.FsInfo, error) {
	return cu.winStatsClient.GetDirFsInfo(path)
}
<|endoftext|>


<commit_before>package queryrange

import (
	"context"
	"fmt"
	math "math"
	strings "strings"
	"time"

	"github.com/dustin/go-humanize"
	"github.com/go-kit/log"
	"github.com/go-kit/log/level"
	"github.com/grafana/dskit/concurrency"
	"github.com/prometheus/common/model"

	"github.com/grafana/loki/pkg/logproto"
	"github.com/grafana/loki/pkg/logql"
	"github.com/grafana/loki/pkg/logql/syntax"
	"github.com/grafana/loki/pkg/querier/queryrange/queryrangebase"
	"github.com/grafana/loki/pkg/storage/config"
	"github.com/grafana/loki/pkg/storage/stores/index/stats"
	"github.com/grafana/loki/pkg/util/spanlogger"
)

func shardResolverForConf(
	ctx context.Context,
	conf config.PeriodConfig,
	defaultLookback time.Duration,
	logger log.Logger,
	maxParallelism int,
	r queryrangebase.Request,
	handler queryrangebase.Handler,
) (logql.ShardResolver, bool) {
	if conf.IndexType == config.TSDBType {
		return &dynamicShardResolver{
			ctx:             ctx,
			logger:          logger,
			handler:         handler,
			from:            model.Time(r.GetStart()),
			through:         model.Time(r.GetEnd()),
			maxParallelism:  maxParallelism,
			defaultLookback: defaultLookback,
		}, true
	}
	if conf.RowShards < 2 {
		return nil, false
	}
	return logql.ConstantShards(conf.RowShards), true
}

type dynamicShardResolver struct {
	ctx     context.Context
	handler queryrangebase.Handler
	logger  log.Logger

	from, through   model.Time
	maxParallelism  int
	defaultLookback time.Duration
}

func (r *dynamicShardResolver) Shards(e syntax.Expr) (int, error) {
	sp, _ := spanlogger.NewWithLogger(r.ctx, r.logger, "dynamicShardResolver.Shards")
	defer sp.Finish()
	// We try to shard subtrees in the AST independently if possible, although
	// nested binary expressions can make this difficult. In this case,
	// we query the index stats for all matcher groups then sum the results.
	grps := syntax.MatcherGroups(e)

	// If there are zero matchers groups, we'll inject one to query everything
	if len(grps) == 0 {
		grps = append(grps, syntax.MatcherRange{})
	}

	results := make([]*stats.Stats, 0, len(grps))

	start := time.Now()
	if err := concurrency.ForEachJob(r.ctx, len(grps), r.maxParallelism, func(ctx context.Context, i int) error {
		matchers := syntax.MatchersString(grps[i].Matchers)
		diff := grps[i].Interval + grps[i].Offset
		adjustedFrom := r.from.Add(-diff)
		if grps[i].Interval == 0 {
			adjustedFrom = adjustedFrom.Add(-r.defaultLookback)
		}

		adjustedThrough := r.through.Add(-grps[i].Offset)

		start := time.Now()
		resp, err := r.handler.Do(r.ctx, &logproto.IndexStatsRequest{
			From:     adjustedFrom,
			Through:  adjustedThrough,
			Matchers: matchers,
		})
		if err != nil {
			return err
		}

		casted, ok := resp.(*IndexStatsResponse)
		if !ok {
			return fmt.Errorf("expected *IndexStatsResponse while querying index, got %T", resp)
		}

		results = append(results, casted.Response)
		level.Debug(sp).Log(
			"msg", "queried index",
			"type", "single",
			"matchers", matchers,
			"bytes", strings.Replace(humanize.Bytes(casted.Response.Bytes), " ", "", 1),
			"chunks", casted.Response.Chunks,
			"streams", casted.Response.Streams,
			"entries", casted.Response.Entries,
			"duration", time.Since(start),
			"from", adjustedFrom.Time(),
			"through", adjustedThrough.Time(),
			"length", adjustedThrough.Sub(adjustedFrom),
		)
		return nil
	}); err != nil {
		return 0, err
	}

	combined := stats.MergeStats(results...)
	factor := guessShardFactor(combined)
	var bytesPerShard = combined.Bytes
	if factor > 0 {
		bytesPerShard = combined.Bytes / uint64(factor)
	}
	level.Debug(sp).Log(
		"msg", "queried index",
		"type", "combined",
		"len", len(results),
		"bytes", strings.Replace(humanize.Bytes(combined.Bytes), " ", "", 1),
		"chunks", combined.Chunks,
		"streams", combined.Streams,
		"entries", combined.Entries,
		"max_parallelism", r.maxParallelism,
		"duration", time.Since(start),
		"factor", factor,
		"bytes_per_shard", strings.Replace(humanize.Bytes(bytesPerShard), " ", "", 1),
	)
	return factor, nil
}

const (
	// Just some observed values to get us started on better query planning.
	p90BytesPerSecond = 300 << 20 // 300MB/s/core
)

func guessShardFactor(stats stats.Stats) int {
	expectedSeconds := float64(stats.Bytes) / float64(p90BytesPerSecond)
	power := math.Ceil(math.Log2(expectedSeconds)) // round up to nearest power of 2

	// Parallelize down to 1s queries
	// Since x^0 == 1 and we only support factors of 2
	// reset this edge case manually
	factor := int(math.Pow(2, power))
	if factor == 1 {
		factor = 0
	}
	return factor
}
<commit_msg>fix context propagation for child spans in shard resolver (#6853)<commit_after>package queryrange

import (
	"context"
	"fmt"
	math "math"
	strings "strings"
	"time"

	"github.com/dustin/go-humanize"
	"github.com/go-kit/log"
	"github.com/go-kit/log/level"
	"github.com/grafana/dskit/concurrency"
	"github.com/prometheus/common/model"

	"github.com/grafana/loki/pkg/logproto"
	"github.com/grafana/loki/pkg/logql"
	"github.com/grafana/loki/pkg/logql/syntax"
	"github.com/grafana/loki/pkg/querier/queryrange/queryrangebase"
	"github.com/grafana/loki/pkg/storage/config"
	"github.com/grafana/loki/pkg/storage/stores/index/stats"
	"github.com/grafana/loki/pkg/util/spanlogger"
)

func shardResolverForConf(
	ctx context.Context,
	conf config.PeriodConfig,
	defaultLookback time.Duration,
	logger log.Logger,
	maxParallelism int,
	r queryrangebase.Request,
	handler queryrangebase.Handler,
) (logql.ShardResolver, bool) {
	if conf.IndexType == config.TSDBType {
		return &dynamicShardResolver{
			ctx:             ctx,
			logger:          logger,
			handler:         handler,
			from:            model.Time(r.GetStart()),
			through:         model.Time(r.GetEnd()),
			maxParallelism:  maxParallelism,
			defaultLookback: defaultLookback,
		}, true
	}
	if conf.RowShards < 2 {
		return nil, false
	}
	return logql.ConstantShards(conf.RowShards), true
}

type dynamicShardResolver struct {
	ctx     context.Context
	handler queryrangebase.Handler
	logger  log.Logger

	from, through   model.Time
	maxParallelism  int
	defaultLookback time.Duration
}

func (r *dynamicShardResolver) Shards(e syntax.Expr) (int, error) {
	sp, ctx := spanlogger.NewWithLogger(r.ctx, r.logger, "dynamicShardResolver.Shards")
	defer sp.Finish()
	// We try to shard subtrees in the AST independently if possible, although
	// nested binary expressions can make this difficult. In this case,
	// we query the index stats for all matcher groups then sum the results.
	grps := syntax.MatcherGroups(e)

	// If there are zero matchers groups, we'll inject one to query everything
	if len(grps) == 0 {
		grps = append(grps, syntax.MatcherRange{})
	}

	results := make([]*stats.Stats, 0, len(grps))

	start := time.Now()
	if err := concurrency.ForEachJob(ctx, len(grps), r.maxParallelism, func(ctx context.Context, i int) error {
		matchers := syntax.MatchersString(grps[i].Matchers)
		diff := grps[i].Interval + grps[i].Offset
		adjustedFrom := r.from.Add(-diff)
		if grps[i].Interval == 0 {
			adjustedFrom = adjustedFrom.Add(-r.defaultLookback)
		}

		adjustedThrough := r.through.Add(-grps[i].Offset)

		start := time.Now()
		resp, err := r.handler.Do(r.ctx, &logproto.IndexStatsRequest{
			From:     adjustedFrom,
			Through:  adjustedThrough,
			Matchers: matchers,
		})
		if err != nil {
			return err
		}

		casted, ok := resp.(*IndexStatsResponse)
		if !ok {
			return fmt.Errorf("expected *IndexStatsResponse while querying index, got %T", resp)
		}

		results = append(results, casted.Response)
		level.Debug(sp).Log(
			"msg", "queried index",
			"type", "single",
			"matchers", matchers,
			"bytes", strings.Replace(humanize.Bytes(casted.Response.Bytes), " ", "", 1),
			"chunks", casted.Response.Chunks,
			"streams", casted.Response.Streams,
			"entries", casted.Response.Entries,
			"duration", time.Since(start),
			"from", adjustedFrom.Time(),
			"through", adjustedThrough.Time(),
			"length", adjustedThrough.Sub(adjustedFrom),
		)
		return nil
	}); err != nil {
		return 0, err
	}

	combined := stats.MergeStats(results...)
	factor := guessShardFactor(combined)
	var bytesPerShard = combined.Bytes
	if factor > 0 {
		bytesPerShard = combined.Bytes / uint64(factor)
	}
	level.Debug(sp).Log(
		"msg", "queried index",
		"type", "combined",
		"len", len(results),
		"bytes", strings.Replace(humanize.Bytes(combined.Bytes), " ", "", 1),
		"chunks", combined.Chunks,
		"streams", combined.Streams,
		"entries", combined.Entries,
		"max_parallelism", r.maxParallelism,
		"duration", time.Since(start),
		"factor", factor,
		"bytes_per_shard", strings.Replace(humanize.Bytes(bytesPerShard), " ", "", 1),
	)
	return factor, nil
}

const (
	// Just some observed values to get us started on better query planning.
	p90BytesPerSecond = 300 << 20 // 300MB/s/core
)

func guessShardFactor(stats stats.Stats) int {
	expectedSeconds := float64(stats.Bytes) / float64(p90BytesPerSecond)
	power := math.Ceil(math.Log2(expectedSeconds)) // round up to nearest power of 2

	// Parallelize down to 1s queries
	// Since x^0 == 1 and we only support factors of 2
	// reset this edge case manually
	factor := int(math.Pow(2, power))
	if factor == 1 {
		factor = 0
	}
	return factor
}
<|endoftext|>


<commit_before>package models

import (
	"fmt"
	"strconv"
	"strings"
)

const (
	TYPE_ACCOUNT         = "account"
	TYPE_CHANNEL_MESSAGE = "channelmessage"
	TYPE_CHANNEL         = "channel"
	STATUS_ADD           = "add"
	STATUS_DELETE        = "delete"
	STATUS_UPDATE        = "update"
)

type SitemapItem struct {
	Id           int64
	TypeConstant string
	Slug         string
	Status       string
}

func NewSitemapItem() *SitemapItem {
	return &SitemapItem{}
}

func (s *SitemapItem) PrepareSetValue() string {
	return fmt.Sprintf("%d:%s:%s:%s", s.Id, s.TypeConstant, s.Slug, s.Status)
}

// Populate converts value retrieved from cache to sitemapitem
// Value must be in format as: id:type:slug:status
func (s *SitemapItem) Populate(value string) error {
	r := strings.Split(value, ":")
	if len(r) != 4 {
		return fmt.Errorf("wrong value length")
	}

	id, err := strconv.ParseInt(r[0], 0, 64)
	if err != nil {
		return fmt.Errorf("id cannot be cast", err)
	}

	s.Id = id
	s.TypeConstant = r[1]
	s.Slug = r[2]
	s.Status = r[3]

	return nil
}

func (s *SitemapItem) Definition(rootURL string) *ItemDefinition {
	d := &ItemDefinition{}
	d.Location = s.composeLocation(rootURL)

	return d
}

func (s *SitemapItem) composeLocation(rootURL string) string {
	switch s.TypeConstant {
	case TYPE_ACCOUNT:
		return fmt.Sprintf("%s/%s", rootURL, s.Slug)
	case TYPE_CHANNEL_MESSAGE:
		return fmt.Sprintf("%s/%s/%s/%s", rootURL, "Activity", "Post", s.Slug)
	case TYPE_CHANNEL:
		return fmt.Sprintf("%s/%s/%s/%s", rootURL, "Activity", "Topic", s.Slug)
	}

	return ""
}
<commit_msg>Sitemap: Add comment line to sitemapitem<commit_after>package models

import (
	"fmt"
	"strconv"
	"strings"
)

const (
	TYPE_ACCOUNT         = "account"
	TYPE_CHANNEL_MESSAGE = "channelmessage"
	TYPE_CHANNEL         = "channel"
	STATUS_ADD           = "add"
	STATUS_DELETE        = "delete"
	STATUS_UPDATE        = "update"
)

// Used as a auxiliary for forming xml data
type SitemapItem struct {
	Id           int64
	TypeConstant string
	Slug         string
	Status       string
}

func NewSitemapItem() *SitemapItem {
	return &SitemapItem{}
}

func (s *SitemapItem) PrepareSetValue() string {
	return fmt.Sprintf("%d:%s:%s:%s", s.Id, s.TypeConstant, s.Slug, s.Status)
}

// Populate converts value retrieved from cache to sitemapitem
// Value must be in format as: id:type:slug:status
func (s *SitemapItem) Populate(value string) error {
	r := strings.Split(value, ":")
	if len(r) != 4 {
		return fmt.Errorf("wrong value length")
	}

	id, err := strconv.ParseInt(r[0], 0, 64)
	if err != nil {
		return fmt.Errorf("id cannot be cast", err)
	}

	s.Id = id
	s.TypeConstant = r[1]
	s.Slug = r[2]
	s.Status = r[3]

	return nil
}

func (s *SitemapItem) Definition(rootURL string) *ItemDefinition {
	d := &ItemDefinition{}
	d.Location = s.composeLocation(rootURL)

	return d
}

func (s *SitemapItem) composeLocation(rootURL string) string {
	switch s.TypeConstant {
	case TYPE_ACCOUNT:
		return fmt.Sprintf("%s/%s", rootURL, s.Slug)
	case TYPE_CHANNEL_MESSAGE:
		return fmt.Sprintf("%s/%s/%s/%s", rootURL, "Activity", "Post", s.Slug)
	case TYPE_CHANNEL:
		return fmt.Sprintf("%s/%s/%s/%s", rootURL, "Activity", "Topic", s.Slug)
	}

	return ""
}
<|endoftext|>


<commit_before>// Copyright 2015 The Vanadium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bufio"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"os/exec"
	"strings"
	"sync"

	"v.io/v23"
	"v.io/v23/context"
	"v.io/v23/rpc"
	"v.io/v23/security"
	"v.io/x/lib/cmdline"
	"v.io/x/lib/vlog"
	"v.io/x/media_sharing"
	"v.io/x/ref/lib/signals"
	"v.io/x/ref/lib/v23cmd"
	_ "v.io/x/ref/runtime/factories/static"
)

func main() {
	// When we run under the device manager, no environment is set up.
	// We want to have some kind of path.
	if os.Getenv("PATH") == "" {
		os.Setenv("PATH", "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin")
		os.Setenv("HOME", "/home/testuser1")
	}

	cmdline.Main(root)
}

var root = &cmdline.Command{
	Name:     os.Args[0],
	ArgsName: "<name>",
	ArgsLong: "<name> Is the name to mount the server under.",
	Runner:   v23cmd.RunnerFunc(serve),
	Short:    "Serve a display to remote clients.",
}

func serve(ctx *context.T, env *cmdline.Env, args []string) error {
	name := ""
	if len(args) > 0 {
		name = args[0]
		fmt.Printf("mounting under: %s\n", name)
	}

	server, err := v23.NewServer(ctx)
	if err != nil {
		return err
	}
	eps, err := server.Listen(v23.GetListenSpec(ctx))
	if err != nil {
		return err
	}
	if err := server.Serve(name, defaultMediaServer(), security.AllowEveryone()); err != nil {
		return err
	}
	fmt.Printf("Listening at: %s", eps[0].Name())

	<-signals.ShutdownOnSignals(ctx)
	return nil
}

// handler defines the interface for a media handler.
type handler interface {
	// Matches returns true if this handler supports mimetype.
	Matches(mimetype string) bool

	// Display starts displaying the content in r.
	// This method should not be blocking and shuould return quickly.
	// The returned function should be called to completely stop and clean up
	// the media playback before starting a new display.
	Display(ctx *context.T, mimetype string, r io.ReadCloser) (func(), error)
}

type eogHandler struct{}

func (eogHandler) Matches(mimetype string) bool {
	return strings.HasPrefix(mimetype, "image/")
}

func (eogHandler) Display(ctx *context.T, mimetype string, r io.ReadCloser) (func(), error) {
	// eog cannot read from a pipe, so we have to write the file to
	// the filesystem before displaying it.
	defer r.Close()
	tmp, err := ioutil.TempFile("", "")
	if err != nil {
		return nil, err
	}
	if _, err := io.Copy(tmp, r); err != nil {
		os.Remove(tmp.Name())
		return nil, err
	}
	tmp.Close()

	cmd := exec.Command("eog", "--display", ":0", "-f", tmp.Name())
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	stop := func() {
		if err := cmd.Process.Kill(); err != nil {
			vlog.Errorf("Could not kill eog: %v", err)
		}
		cmd.Wait()
		os.Remove(tmp.Name())
	}
	if err := cmd.Start(); err != nil {
		return stop, err
	}
	return stop, nil
}

type omxHandler struct{}

func (omxHandler) Matches(mimetype string) bool {
	return strings.HasPrefix(mimetype, "video")
}

func (omxHandler) Display(ctx *context.T, mimetype string, r io.ReadCloser) (func(), error) {
	defer r.Close()
	tmp, err := ioutil.TempFile("", "")
	if err != nil {
		return nil, err
	}
	if _, err := io.Copy(tmp, r); err != nil {
		os.Remove(tmp.Name())
		return nil, err
	}
	tmp.Close()

	args := []string{
		"-b",
		tmp.Name(),
	}

	vlog.Infof("Running: omxplayer %s", strings.Join(args, " "))

	cmd := exec.Command("omxplayer", args...)
	cmd.Stdin = r
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	if err := cmd.Start(); err != nil {
		return nil, err
	}
	return func() {
		if err := cmd.Process.Kill(); err != nil {
			vlog.Errorf("Could not kill omx: %v", err)
		}
		cmd.Wait()
		os.Remove(tmp.Name())
	}, nil
}

type vlcHandler struct{}

func (vlcHandler) Matches(mimetype string) bool {
	return strings.HasPrefix(mimetype, "audio/") || strings.HasPrefix(mimetype, "video/")
}

func (vlcHandler) Display(ctx *context.T, mimetype string, r io.ReadCloser) (func(), error) {
	defer r.Close()
	tmp, err := ioutil.TempFile("", "")
	if err != nil {
		return nil, err
	}
	if _, err := io.Copy(tmp, r); err != nil {
		os.Remove(tmp.Name())
		return nil, err
	}
	tmp.Close()

	args := []string{
		"--no-video-title-show",
		"--fullscreen",
		"--x11-display", ":0",
		tmp.Name(),
		"vlc://quit",
	}

	if strings.HasPrefix(mimetype, "audio/") {
		args = append([]string{"--audio-visual=visual"}, args...)
	}

	vlog.Infof("Running: vlc %s", strings.Join(args, " "))

	cmd := exec.Command("vlc", args...)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	if err := cmd.Start(); err != nil {
		return nil, err
	}
	return func() {
		if err := cmd.Process.Kill(); err != nil {
			vlog.Errorf("Could not kill vlc: %v", err)
		}
		cmd.Wait()
		os.Remove(tmp.Name())
	}, nil
}

type media struct {
	handlers []handler

	stopMu sync.Mutex
	stop   func()
}

func defaultMediaServer() media_sharing.MediaSharingServerStub {
	m := &media{
		handlers: []handler{
			&eogHandler{},
			&omxHandler{},
			&vlcHandler{},
		},
	}
	return media_sharing.MediaSharingServer(m)
}

func detectMimeType(r *bufio.Reader) string {
	// If the server didn't tell us the content type, we will have to guess.
	// Note that I am purposefully ignoring the error from Peek.  The guesser looks
	// at up to 512 bytes.  We just get as many as we can and make our best
	// guess with that.
	data, _ := r.Peek(512)
	return http.DetectContentType(data)
}

type bufferedReadCloser struct {
	*bufio.Reader
	close func() error
}

func (bc *bufferedReadCloser) Close() error {
	return bc.close()
}

func (m *media) findHandler(mimetype string) (handler, error) {
	for _, h := range m.handlers {
		if h.Matches(mimetype) {
			return h, nil
		}
	}
	return nil, fmt.Errorf("Unsupported content type: %s", mimetype)
}

func (m *media) display(ctx *context.T, mimetype string, r io.ReadCloser) error {
	rc := &bufferedReadCloser{
		Reader: bufio.NewReader(r),
		close:  r.Close,
	}

	if mimetype == "" {
		mimetype = detectMimeType(rc.Reader)
	}

	// Find a suitable handler.
	h, err := m.findHandler(mimetype)
	if err != nil {
		return err
	}

	m.stopMu.Lock()
	if m.stop != nil {
		m.stop()
	}
	m.stop, err = h.Display(ctx, mimetype, r)
	m.stopMu.Unlock()
	return err
}

// DisplayURL will cause the server to display whatever media is at
// the given URL.  The server will rely on the ContentType response
// header it gets when fetching the url to decide how to display
// the media.
func (m *media) DisplayUrl(ctx *context.T, call rpc.ServerCall, url string) error {
	resp, err := http.Get(url)
	if err != nil {
		return err
	}
	return m.display(ctx, resp.Header.Get("Content-Type"), resp.Body)
}

// The streamReader adapts a vanadium steam of byte slices to an io.ReadCloser.
type streamReader struct {
	last   []byte
	stream interface {
		Advance() bool
		Value() []byte
		Err() error
	}
}

func (s *streamReader) Read(p []byte) (int, error) {
	if len(s.last) == 0 {
		if !s.stream.Advance() {
			err := s.stream.Err()
			if err == nil {
				err = io.EOF
			}
			return 0, err
		}
		s.last = s.stream.Value()
	}
	if len(s.last) == 0 {
		return 0, nil
	}
	n := copy(p, s.last)
	s.last = s.last[n:]
	return n, nil
}

func (s *streamReader) Close() error {
	// Do nothing.
	return nil
}

// DisplayBytes will cause the server to display whatever media is
// sent in the stream.  In the case of audio or movie media, the
// media should be played while the data is streaming.  The mediaType
// can be used by the server to decide how to display the media.
func (m *media) DisplayBytes(ctx *context.T, call media_sharing.MediaSharingDisplayBytesServerCall, mediaType string) error {
	r := &streamReader{stream: call.RecvStream()}
	return m.display(ctx, mediaType, r)
}
<commit_msg>media-sharing: Change many uses of v23.NewServer to use xrpc.New*Server instead.<commit_after>// Copyright 2015 The Vanadium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bufio"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"os/exec"
	"strings"
	"sync"

	"v.io/v23/context"
	"v.io/v23/rpc"
	"v.io/v23/security"
	"v.io/x/lib/cmdline"
	"v.io/x/lib/vlog"
	"v.io/x/media_sharing"
	"v.io/x/ref/lib/signals"
	"v.io/x/ref/lib/v23cmd"
	"v.io/x/ref/lib/xrpc"
	_ "v.io/x/ref/runtime/factories/static"
)

func main() {
	// When we run under the device manager, no environment is set up.
	// We want to have some kind of path.
	if os.Getenv("PATH") == "" {
		os.Setenv("PATH", "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin")
		os.Setenv("HOME", "/home/testuser1")
	}

	cmdline.Main(root)
}

var root = &cmdline.Command{
	Name:     os.Args[0],
	ArgsName: "<name>",
	ArgsLong: "<name> Is the name to mount the server under.",
	Runner:   v23cmd.RunnerFunc(serve),
	Short:    "Serve a display to remote clients.",
}

func serve(ctx *context.T, env *cmdline.Env, args []string) error {
	name := ""
	if len(args) > 0 {
		name = args[0]
		fmt.Printf("mounting under: %s\n", name)
	}

	server, err := xrpc.NewServer(ctx, name, defaultMediaServer(), security.AllowEveryone())
	if err != nil {
		return err
	}
	fmt.Printf("Listening at: %s", server.Status().Endpoints[0].Name())

	<-signals.ShutdownOnSignals(ctx)
	return nil
}

// handler defines the interface for a media handler.
type handler interface {
	// Matches returns true if this handler supports mimetype.
	Matches(mimetype string) bool

	// Display starts displaying the content in r.
	// This method should not be blocking and shuould return quickly.
	// The returned function should be called to completely stop and clean up
	// the media playback before starting a new display.
	Display(ctx *context.T, mimetype string, r io.ReadCloser) (func(), error)
}

type eogHandler struct{}

func (eogHandler) Matches(mimetype string) bool {
	return strings.HasPrefix(mimetype, "image/")
}

func (eogHandler) Display(ctx *context.T, mimetype string, r io.ReadCloser) (func(), error) {
	// eog cannot read from a pipe, so we have to write the file to
	// the filesystem before displaying it.
	defer r.Close()
	tmp, err := ioutil.TempFile("", "")
	if err != nil {
		return nil, err
	}
	if _, err := io.Copy(tmp, r); err != nil {
		os.Remove(tmp.Name())
		return nil, err
	}
	tmp.Close()

	cmd := exec.Command("eog", "--display", ":0", "-f", tmp.Name())
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	stop := func() {
		if err := cmd.Process.Kill(); err != nil {
			vlog.Errorf("Could not kill eog: %v", err)
		}
		cmd.Wait()
		os.Remove(tmp.Name())
	}
	if err := cmd.Start(); err != nil {
		return stop, err
	}
	return stop, nil
}

type omxHandler struct{}

func (omxHandler) Matches(mimetype string) bool {
	return strings.HasPrefix(mimetype, "video")
}

func (omxHandler) Display(ctx *context.T, mimetype string, r io.ReadCloser) (func(), error) {
	defer r.Close()
	tmp, err := ioutil.TempFile("", "")
	if err != nil {
		return nil, err
	}
	if _, err := io.Copy(tmp, r); err != nil {
		os.Remove(tmp.Name())
		return nil, err
	}
	tmp.Close()

	args := []string{
		"-b",
		tmp.Name(),
	}

	vlog.Infof("Running: omxplayer %s", strings.Join(args, " "))

	cmd := exec.Command("omxplayer", args...)
	cmd.Stdin = r
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	if err := cmd.Start(); err != nil {
		return nil, err
	}
	return func() {
		if err := cmd.Process.Kill(); err != nil {
			vlog.Errorf("Could not kill omx: %v", err)
		}
		cmd.Wait()
		os.Remove(tmp.Name())
	}, nil
}

type vlcHandler struct{}

func (vlcHandler) Matches(mimetype string) bool {
	return strings.HasPrefix(mimetype, "audio/") || strings.HasPrefix(mimetype, "video/")
}

func (vlcHandler) Display(ctx *context.T, mimetype string, r io.ReadCloser) (func(), error) {
	defer r.Close()
	tmp, err := ioutil.TempFile("", "")
	if err != nil {
		return nil, err
	}
	if _, err := io.Copy(tmp, r); err != nil {
		os.Remove(tmp.Name())
		return nil, err
	}
	tmp.Close()

	args := []string{
		"--no-video-title-show",
		"--fullscreen",
		"--x11-display", ":0",
		tmp.Name(),
		"vlc://quit",
	}

	if strings.HasPrefix(mimetype, "audio/") {
		args = append([]string{"--audio-visual=visual"}, args...)
	}

	vlog.Infof("Running: vlc %s", strings.Join(args, " "))

	cmd := exec.Command("vlc", args...)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	if err := cmd.Start(); err != nil {
		return nil, err
	}
	return func() {
		if err := cmd.Process.Kill(); err != nil {
			vlog.Errorf("Could not kill vlc: %v", err)
		}
		cmd.Wait()
		os.Remove(tmp.Name())
	}, nil
}

type media struct {
	handlers []handler

	stopMu sync.Mutex
	stop   func()
}

func defaultMediaServer() media_sharing.MediaSharingServerStub {
	m := &media{
		handlers: []handler{
			&eogHandler{},
			&omxHandler{},
			&vlcHandler{},
		},
	}
	return media_sharing.MediaSharingServer(m)
}

func detectMimeType(r *bufio.Reader) string {
	// If the server didn't tell us the content type, we will have to guess.
	// Note that I am purposefully ignoring the error from Peek.  The guesser looks
	// at up to 512 bytes.  We just get as many as we can and make our best
	// guess with that.
	data, _ := r.Peek(512)
	return http.DetectContentType(data)
}

type bufferedReadCloser struct {
	*bufio.Reader
	close func() error
}

func (bc *bufferedReadCloser) Close() error {
	return bc.close()
}

func (m *media) findHandler(mimetype string) (handler, error) {
	for _, h := range m.handlers {
		if h.Matches(mimetype) {
			return h, nil
		}
	}
	return nil, fmt.Errorf("Unsupported content type: %s", mimetype)
}

func (m *media) display(ctx *context.T, mimetype string, r io.ReadCloser) error {
	rc := &bufferedReadCloser{
		Reader: bufio.NewReader(r),
		close:  r.Close,
	}

	if mimetype == "" {
		mimetype = detectMimeType(rc.Reader)
	}

	// Find a suitable handler.
	h, err := m.findHandler(mimetype)
	if err != nil {
		return err
	}

	m.stopMu.Lock()
	if m.stop != nil {
		m.stop()
	}
	m.stop, err = h.Display(ctx, mimetype, r)
	m.stopMu.Unlock()
	return err
}

// DisplayURL will cause the server to display whatever media is at
// the given URL.  The server will rely on the ContentType response
// header it gets when fetching the url to decide how to display
// the media.
func (m *media) DisplayUrl(ctx *context.T, call rpc.ServerCall, url string) error {
	resp, err := http.Get(url)
	if err != nil {
		return err
	}
	return m.display(ctx, resp.Header.Get("Content-Type"), resp.Body)
}

// The streamReader adapts a vanadium steam of byte slices to an io.ReadCloser.
type streamReader struct {
	last   []byte
	stream interface {
		Advance() bool
		Value() []byte
		Err() error
	}
}

func (s *streamReader) Read(p []byte) (int, error) {
	if len(s.last) == 0 {
		if !s.stream.Advance() {
			err := s.stream.Err()
			if err == nil {
				err = io.EOF
			}
			return 0, err
		}
		s.last = s.stream.Value()
	}
	if len(s.last) == 0 {
		return 0, nil
	}
	n := copy(p, s.last)
	s.last = s.last[n:]
	return n, nil
}

func (s *streamReader) Close() error {
	// Do nothing.
	return nil
}

// DisplayBytes will cause the server to display whatever media is
// sent in the stream.  In the case of audio or movie media, the
// media should be played while the data is streaming.  The mediaType
// can be used by the server to decide how to display the media.
func (m *media) DisplayBytes(ctx *context.T, call media_sharing.MediaSharingDisplayBytesServerCall, mediaType string) error {
	r := &streamReader{stream: call.RecvStream()}
	return m.display(ctx, mediaType, r)
}
<|endoftext|>


<commit_before>package schemas

// ResourceChanged resource changed
type ResourceChanged struct {
	ResourceKind string `json:"resource_kind" avro:"resource_kind"`
	ChangeType   string `json:"change_type" avro:"change_type"`
	PublishedAt  string `json:"published_at" avro:"published_at"`
	ResourceURI  string `json:"resource_uri" avro:"resource_uri"`
}
<commit_msg>Update resource-change schema for event<commit_after>package schemas

// EventRecord - describes the event the resource was changed for
type EventRecord struct {
	PublishedAt   string   `json:"published_at" avro:"published_at"`
	Type          string   `json:"type" avro:"type"`
	FieldsChanged []string `json:"fields_changed" avro:"fields_changed"`
}

// ResourceChanged resource changed (json is for output in this utility, nothing to do with input to kafka api)
type ResourceChanged struct {
	ResourceKind string      `json:"resource_kind" avro:"resource_kind"`
	ResourceURI  string      `json:"resource_uri" avro:"resource_uri"`
	ContextID    string      `json:"context_id" avro:"context_id"`
	DeletedData  string      `json:"deleted_data" avro:"deleted_data"`
	Event        EventRecord `json:"event" avro:"event"`
}
<|endoftext|>


<commit_before>package environs_test

import (
	. "launchpad.net/gocheck"
	"launchpad.net/juju/go/environs"
	_ "launchpad.net/juju/go/environs/dummy"
)

type OpenSuite struct{}

var _ = Suite(&OpenSuite{})

func (OpenSuite) TestNewDummyEnviron(c *C) {
	// matches *ConfigNode.Map()
	config := map[string]interface{}{
		"name":      "foo",
		"type":      "dummy",
		"zookeeper": false,
	}
	env, err := environs.NewEnviron(config)
	c.Assert(err, IsNil)
	c.Assert(env.Bootstrap(false), IsNil)
}

func (OpenSuite) TestNewUnknownEnviron(c *C) {
	env, err := environs.NewEnviron(map[string]interface{}{"type": "wondercloud"})
	c.Assert(err, ErrorMatches, "error validating environment: no registered provider for.*")
	c.Assert(env, IsNil)
}
<commit_msg>Added test for NewConfig<commit_after>package environs_test

import (
	. "launchpad.net/gocheck"
	"launchpad.net/juju/go/environs"
	_ "launchpad.net/juju/go/environs/dummy"
)

type OpenSuite struct{}

var _ = Suite(&OpenSuite{})

func (OpenSuite) TestNewDummyEnviron(c *C) {
	// matches *ConfigNode.Map()
	config := map[string]interface{}{
		"name":      "foo",
		"type":      "dummy",
		"zookeeper": false,
	}
	env, err := environs.NewEnviron(config)
	c.Assert(err, IsNil)
	c.Assert(env.Bootstrap(false), IsNil)
}

func (OpenSuite) TestNewUnknownEnviron(c *C) {
	env, err := environs.NewEnviron(map[string]interface{}{"type": "wondercloud"})
	c.Assert(err, ErrorMatches, "error validating environment: no registered provider for.*")
	c.Assert(env, IsNil)
}

func (OpenSuite) TestValidNewConfig(c *C) {
	cfg, err := environs.NewConfig(map[string]interface{}{
		"name":      "test",
		"type":      "dummy",
		"zookeeper": false,
	})
	c.Assert(err, IsNil)
	c.Assert(cfg, NotNil)

	env, err := cfg.Open()
	c.Assert(err, IsNil)
	c.Assert(env, NotNil)
}

func (OpenSuite) TestInvalidNewConfig(c *C) {
	cfg, err := environs.NewConfig(map[string]interface{}{
		"name": "test",
		"type": "dummy",
		// zookeeper is missing
	})
	c.Assert(err, ErrorMatches, "zookeeper: expected false, got nothing")
	c.Assert(cfg, IsNil)
}
<|endoftext|>


<commit_before>package errortypes

import "fmt"

// ErrFilePos extends the error interface to add details on the file position where the error occurred.
type ErrFilePos interface {
	error
	File() string
	Line() int
	Col() int
}

// NewErrFilePosf creates an error conforming to the ErrFilePos interface.
func NewErrFilePosf(file string, line, col int, format string, args ...interface{}) error {
	return &errFilePos{
		error: fmt.Errorf(format, args),
		file:  file,
		line:  line,
		col:   col,
	}
}

// IsErrFilePos identifies whethere or not the root cause of the provided error is of the ErrFilePos type.
// Wrapped errors are unwrapped via the Cause() function.
func IsErrFilePos(err error) bool {
	if err == nil {
		return false
	}
	_, isErrFilePos := err.(ErrFilePos)
	return isErrFilePos
}

// ToErrFilePos converts the input error to an ErrFilePos if possible, or nil if not.
// If IsErrFilePos returns true, this will not return nil.
func ToErrFilePos(err error) ErrFilePos {
	if err == nil {
		return nil
	}
	if out, isErrFilePos := err.(ErrFilePos); isErrFilePos {
		return out
	}
	return nil
}

var _ ErrFilePos = &errFilePos{}

type errFilePos struct {
	error
	file string
	line int
	col  int
}

func (e *errFilePos) File() string {
	return e.file
}

func (e *errFilePos) Line() int {
	return e.line
}

func (e *errFilePos) Col() int {
	return e.col
}
<commit_msg>Fix formatting of error with file position<commit_after>package errortypes

import "fmt"

// ErrFilePos extends the error interface to add details on the file position where the error occurred.
type ErrFilePos interface {
	error
	File() string
	Line() int
	Col() int
}

// NewErrFilePosf creates an error conforming to the ErrFilePos interface.
func NewErrFilePosf(file string, line, col int, format string, args ...interface{}) error {
	return &errFilePos{
		error: fmt.Errorf(format, args...),
		file:  file,
		line:  line,
		col:   col,
	}
}

// IsErrFilePos identifies whethere or not the root cause of the provided error is of the ErrFilePos type.
// Wrapped errors are unwrapped via the Cause() function.
func IsErrFilePos(err error) bool {
	if err == nil {
		return false
	}
	_, isErrFilePos := err.(ErrFilePos)
	return isErrFilePos
}

// ToErrFilePos converts the input error to an ErrFilePos if possible, or nil if not.
// If IsErrFilePos returns true, this will not return nil.
func ToErrFilePos(err error) ErrFilePos {
	if err == nil {
		return nil
	}
	if out, isErrFilePos := err.(ErrFilePos); isErrFilePos {
		return out
	}
	return nil
}

var _ ErrFilePos = &errFilePos{}

type errFilePos struct {
	error
	file string
	line int
	col  int
}

func (e *errFilePos) File() string {
	return e.file
}

func (e *errFilePos) Line() int {
	return e.line
}

func (e *errFilePos) Col() int {
	return e.col
}
<|endoftext|>


<commit_before>package ticker_test

import (
	"testing"

	"github.com/bitfinexcom/bitfinex-api-go/pkg/models/ticker"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

func TestNewTickerFromRestRaw(t *testing.T) {
	t.Run("invalid arguments", func(t *testing.T) {
		payload := []interface{}{"tBTCUSD"}

		got, err := ticker.FromRestRaw(payload)
		require.NotNil(t, err)
		require.Nil(t, got)
	})

	t.Run("valid arguments", func(t *testing.T) {
		payload := []interface{}{
			"tBTCUSD",
			10654,
			53.62425959,
			10655,
			76.68743116,
			745.1,
			0.0752,
			10655,
			14420.34271146,
			10766,
			9889.1449809,
		}

		got, err := ticker.FromRestRaw(payload)
		require.Nil(t, err)

		expected := &ticker.Ticker{
			Symbol:          "tBTCUSD",
			Frr:             0,
			Bid:             10654,
			BidPeriod:       0,
			BidSize:         53.62425959,
			Ask:             10655,
			AskPeriod:       0,
			AskSize:         76.68743116,
			DailyChange:     745.1,
			DailyChangePerc: 0.0752,
			LastPrice:       10655,
			Volume:          14420.34271146,
			High:            10766,
			Low:             9889.1449809,
		}

		assert.Equal(t, expected, got)
	})
}

func TestFromRaw(t *testing.T) {
	t.Run("invalid arguments", func(t *testing.T) {
		payload := []interface{}{7616.5, 31.89055171}

		got, err := ticker.FromRaw("tBTCUSD", payload)
		require.NotNil(t, err)
		require.Nil(t, got)
	})

	t.Run("trading pair update / snapshot ticker", func(t *testing.T) {
		payload := []interface{}{
			7616.5,
			31.89055171,
			7617.5,
			43.358118629999986,
			-550.8,
			-0.0674,
			7617.1,
			8314.71200815,
			8257.8,
			7500,
		}

		got, err := ticker.FromRaw("tBTCUSD", payload)
		require.Nil(t, err)

		expected := &ticker.Ticker{
			Symbol:          "tBTCUSD",
			Bid:             7616.5,
			BidSize:         31.89055171,
			Ask:             7617.5,
			AskSize:         43.358118629999986,
			DailyChange:     -550.8,
			DailyChangePerc: -0.0674,
			LastPrice:       7617.1,
			Volume:          8314.71200815,
			High:            8257.8,
			Low:             7500,
		}

		assert.Equal(t, expected, got)
	})

	t.Run("funding pair spanshot ticker", func(t *testing.T) {
		payload := []interface{}{
			0.0003447013698630137,
			0.000316,
			30,
			1682003.0922634401,
			0.00031783,
			4,
			23336.958053110004,
			0.00000707,
			0.0209,
			0.0003446,
			156123478.78447533,
			0.000347,
			0.00009,
			nil,
			nil,
			146247919.52883354,
		}

		got, err := ticker.FromRaw("fUSD", payload)
		require.Nil(t, err)

		expected := &ticker.Ticker{
			Symbol:             "fUSD",
			Frr:                0.0003447013698630137,
			Bid:                0.000316,
			BidPeriod:          30,
			BidSize:            1682003.0922634401,
			Ask:                0.00031783,
			AskPeriod:          4,
			AskSize:            23336.958053110004,
			DailyChange:        0.00000707,
			DailyChangePerc:    0.0209,
			LastPrice:          0.0003446,
			Volume:             156123478.78447533,
			High:               0.000347,
			Low:                0.00009,
			FrrAmountAvailable: 146247919.52883354,
		}

		assert.Equal(t, expected, got)
	})

	t.Run("funding pair update ticker", func(t *testing.T) {
		payload := []interface{}{
			0.0003447013698630137,
			0.000316,
			30,
			1682003.0922634401,
			0.00031783,
			4,
			23336.958053110004,
			0.00000707,
			0.0209,
			0.0003446,
			156123478.78447533,
			0.000347,
			0.00009,
		}

		got, err := ticker.FromRaw("fUSD", payload)
		require.Nil(t, err)

		expected := &ticker.Ticker{
			Symbol:          "fUSD",
			Frr:             0.0003447013698630137,
			Bid:             0.000316,
			BidPeriod:       30,
			BidSize:         1682003.0922634401,
			Ask:             0.00031783,
			AskPeriod:       4,
			AskSize:         23336.958053110004,
			DailyChange:     0.00000707,
			DailyChangePerc: 0.0209,
			LastPrice:       0.0003446,
			Volume:          156123478.78447533,
			High:            0.000347,
			Low:             0.00009,
		}

		assert.Equal(t, expected, got)
	})
}

func TestSnapshotFromRaw(t *testing.T) {
	t.Run("invalid arguments", func(t *testing.T) {
		payload := [][]interface{}{}
		got, err := ticker.SnapshotFromRaw("tBTCUSD", payload)
		require.NotNil(t, err)
		require.Nil(t, got)
	})

	t.Run("partially valid arguments", func(t *testing.T) {
		payload := [][]interface{}{
			{
				9206.8,
				21.038892079999997,
				9205.9,
				31.41755015,
				38.97852525,
				0.0043,
				9205.87852525,
				1815.68824558,
				9299,
				9111.8,
			},
			{
				9206.8,
			},
		}
		got, err := ticker.SnapshotFromRaw("tBTCUSD", payload)
		require.NotNil(t, err)
		require.Nil(t, got)
	})

	t.Run("trading pair snapshot", func(t *testing.T) {
		payload := [][]interface{}{
			{
				9206.8,
				21.038892079999997,
				9205.9,
				31.41755015,
				38.97852525,
				0.0043,
				9205.87852525,
				1815.68824558,
				9299,
				9111.8,
			},
		}

		got, err := ticker.SnapshotFromRaw("tBTCUSD", payload)
		require.Nil(t, err)

		expected := &ticker.Snapshot{
			Snapshot: []*ticker.Ticker{
				{
					Symbol:          "tBTCUSD",
					Bid:             9206.8,
					BidSize:         21.038892079999997,
					Ask:             9205.9,
					AskSize:         31.41755015,
					DailyChange:     38.97852525,
					DailyChangePerc: 0.0043,
					LastPrice:       9205.87852525,
					Volume:          1815.68824558,
					High:            9299,
					Low:             9111.8,
				},
			},
		}

		assert.Equal(t, expected, got)
	})
}

func TestFromWSRaw(t *testing.T) {
	t.Run("invalid arguments", func(t *testing.T) {
		payload := []interface{}{7616.5, 31.89055171}
		got, err := ticker.FromWSRaw("tBTCUSD", payload)
		require.NotNil(t, err)
		require.Nil(t, got)
	})

	t.Run("missing arguments", func(t *testing.T) {
		payload := []interface{}{}
		got, err := ticker.FromWSRaw("tBTCUSD", payload)
		require.NotNil(t, err)
		require.Nil(t, got)
	})

	t.Run("trading pair ticker", func(t *testing.T) {
		payload := []interface{}{
			7616.5,
			31.89055171,
			7617.5,
			43.358118629999986,
			-550.8,
			-0.0674,
			7617.1,
			8314.71200815,
			8257.8,
			7500,
		}

		got, err := ticker.FromWSRaw("tBTCUSD", payload)
		require.Nil(t, err)

		expected := &ticker.Ticker{
			Symbol:          "tBTCUSD",
			Bid:             7616.5,
			BidSize:         31.89055171,
			Ask:             7617.5,
			AskSize:         43.358118629999986,
			DailyChange:     -550.8,
			DailyChangePerc: -0.0674,
			LastPrice:       7617.1,
			Volume:          8314.71200815,
			High:            8257.8,
			Low:             7500,
		}

		assert.Equal(t, expected, got)
	})

	t.Run("valid trading pair ticker snapshot", func(t *testing.T) {
		payload := []interface{}{
			[]interface{}{
				9206.8,
				21.038892079999997,
				9205.9,
				31.41755015,
				38.97852525,
				0.0043,
				9205.87852525,
				1815.68824558,
				9299,
				9111.8,
			},
		}

		got, err := ticker.FromWSRaw("tBTCUSD", payload)
		require.Nil(t, err)

		expected := &ticker.Snapshot{
			Snapshot: []*ticker.Ticker{
				{
					Symbol:          "tBTCUSD",
					Bid:             9206.8,
					BidSize:         21.038892079999997,
					Ask:             9205.9,
					AskSize:         31.41755015,
					DailyChange:     38.97852525,
					DailyChangePerc: 0.0043,
					LastPrice:       9205.87852525,
					Volume:          1815.68824558,
					High:            9299,
					Low:             9111.8,
				},
			},
		}

		assert.Equal(t, expected, got)
	})
}
<commit_msg>typo fix<commit_after>package ticker_test

import (
	"testing"

	"github.com/bitfinexcom/bitfinex-api-go/pkg/models/ticker"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

func TestNewTickerFromRestRaw(t *testing.T) {
	t.Run("invalid arguments", func(t *testing.T) {
		payload := []interface{}{"tBTCUSD"}

		got, err := ticker.FromRestRaw(payload)
		require.NotNil(t, err)
		require.Nil(t, got)
	})

	t.Run("valid arguments", func(t *testing.T) {
		payload := []interface{}{
			"tBTCUSD",
			10654,
			53.62425959,
			10655,
			76.68743116,
			745.1,
			0.0752,
			10655,
			14420.34271146,
			10766,
			9889.1449809,
		}

		got, err := ticker.FromRestRaw(payload)
		require.Nil(t, err)

		expected := &ticker.Ticker{
			Symbol:          "tBTCUSD",
			Frr:             0,
			Bid:             10654,
			BidPeriod:       0,
			BidSize:         53.62425959,
			Ask:             10655,
			AskPeriod:       0,
			AskSize:         76.68743116,
			DailyChange:     745.1,
			DailyChangePerc: 0.0752,
			LastPrice:       10655,
			Volume:          14420.34271146,
			High:            10766,
			Low:             9889.1449809,
		}

		assert.Equal(t, expected, got)
	})
}

func TestFromRaw(t *testing.T) {
	t.Run("invalid arguments", func(t *testing.T) {
		payload := []interface{}{7616.5, 31.89055171}

		got, err := ticker.FromRaw("tBTCUSD", payload)
		require.NotNil(t, err)
		require.Nil(t, got)
	})

	t.Run("trading pair update / snapshot ticker", func(t *testing.T) {
		payload := []interface{}{
			7616.5,
			31.89055171,
			7617.5,
			43.358118629999986,
			-550.8,
			-0.0674,
			7617.1,
			8314.71200815,
			8257.8,
			7500,
		}

		got, err := ticker.FromRaw("tBTCUSD", payload)
		require.Nil(t, err)

		expected := &ticker.Ticker{
			Symbol:          "tBTCUSD",
			Bid:             7616.5,
			BidSize:         31.89055171,
			Ask:             7617.5,
			AskSize:         43.358118629999986,
			DailyChange:     -550.8,
			DailyChangePerc: -0.0674,
			LastPrice:       7617.1,
			Volume:          8314.71200815,
			High:            8257.8,
			Low:             7500,
		}

		assert.Equal(t, expected, got)
	})

	t.Run("funding pair snapshot ticker", func(t *testing.T) {
		payload := []interface{}{
			0.0003447013698630137,
			0.000316,
			30,
			1682003.0922634401,
			0.00031783,
			4,
			23336.958053110004,
			0.00000707,
			0.0209,
			0.0003446,
			156123478.78447533,
			0.000347,
			0.00009,
			nil,
			nil,
			146247919.52883354,
		}

		got, err := ticker.FromRaw("fUSD", payload)
		require.Nil(t, err)

		expected := &ticker.Ticker{
			Symbol:             "fUSD",
			Frr:                0.0003447013698630137,
			Bid:                0.000316,
			BidPeriod:          30,
			BidSize:            1682003.0922634401,
			Ask:                0.00031783,
			AskPeriod:          4,
			AskSize:            23336.958053110004,
			DailyChange:        0.00000707,
			DailyChangePerc:    0.0209,
			LastPrice:          0.0003446,
			Volume:             156123478.78447533,
			High:               0.000347,
			Low:                0.00009,
			FrrAmountAvailable: 146247919.52883354,
		}

		assert.Equal(t, expected, got)
	})

	t.Run("funding pair update ticker", func(t *testing.T) {
		payload := []interface{}{
			0.0003447013698630137,
			0.000316,
			30,
			1682003.0922634401,
			0.00031783,
			4,
			23336.958053110004,
			0.00000707,
			0.0209,
			0.0003446,
			156123478.78447533,
			0.000347,
			0.00009,
		}

		got, err := ticker.FromRaw("fUSD", payload)
		require.Nil(t, err)

		expected := &ticker.Ticker{
			Symbol:          "fUSD",
			Frr:             0.0003447013698630137,
			Bid:             0.000316,
			BidPeriod:       30,
			BidSize:         1682003.0922634401,
			Ask:             0.00031783,
			AskPeriod:       4,
			AskSize:         23336.958053110004,
			DailyChange:     0.00000707,
			DailyChangePerc: 0.0209,
			LastPrice:       0.0003446,
			Volume:          156123478.78447533,
			High:            0.000347,
			Low:             0.00009,
		}

		assert.Equal(t, expected, got)
	})
}

func TestSnapshotFromRaw(t *testing.T) {
	t.Run("invalid arguments", func(t *testing.T) {
		payload := [][]interface{}{}
		got, err := ticker.SnapshotFromRaw("tBTCUSD", payload)
		require.NotNil(t, err)
		require.Nil(t, got)
	})

	t.Run("partially valid arguments", func(t *testing.T) {
		payload := [][]interface{}{
			{
				9206.8,
				21.038892079999997,
				9205.9,
				31.41755015,
				38.97852525,
				0.0043,
				9205.87852525,
				1815.68824558,
				9299,
				9111.8,
			},
			{
				9206.8,
			},
		}
		got, err := ticker.SnapshotFromRaw("tBTCUSD", payload)
		require.NotNil(t, err)
		require.Nil(t, got)
	})

	t.Run("trading pair snapshot", func(t *testing.T) {
		payload := [][]interface{}{
			{
				9206.8,
				21.038892079999997,
				9205.9,
				31.41755015,
				38.97852525,
				0.0043,
				9205.87852525,
				1815.68824558,
				9299,
				9111.8,
			},
		}

		got, err := ticker.SnapshotFromRaw("tBTCUSD", payload)
		require.Nil(t, err)

		expected := &ticker.Snapshot{
			Snapshot: []*ticker.Ticker{
				{
					Symbol:          "tBTCUSD",
					Bid:             9206.8,
					BidSize:         21.038892079999997,
					Ask:             9205.9,
					AskSize:         31.41755015,
					DailyChange:     38.97852525,
					DailyChangePerc: 0.0043,
					LastPrice:       9205.87852525,
					Volume:          1815.68824558,
					High:            9299,
					Low:             9111.8,
				},
			},
		}

		assert.Equal(t, expected, got)
	})
}

func TestFromWSRaw(t *testing.T) {
	t.Run("invalid arguments", func(t *testing.T) {
		payload := []interface{}{7616.5, 31.89055171}
		got, err := ticker.FromWSRaw("tBTCUSD", payload)
		require.NotNil(t, err)
		require.Nil(t, got)
	})

	t.Run("missing arguments", func(t *testing.T) {
		payload := []interface{}{}
		got, err := ticker.FromWSRaw("tBTCUSD", payload)
		require.NotNil(t, err)
		require.Nil(t, got)
	})

	t.Run("trading pair ticker", func(t *testing.T) {
		payload := []interface{}{
			7616.5,
			31.89055171,
			7617.5,
			43.358118629999986,
			-550.8,
			-0.0674,
			7617.1,
			8314.71200815,
			8257.8,
			7500,
		}

		got, err := ticker.FromWSRaw("tBTCUSD", payload)
		require.Nil(t, err)

		expected := &ticker.Ticker{
			Symbol:          "tBTCUSD",
			Bid:             7616.5,
			BidSize:         31.89055171,
			Ask:             7617.5,
			AskSize:         43.358118629999986,
			DailyChange:     -550.8,
			DailyChangePerc: -0.0674,
			LastPrice:       7617.1,
			Volume:          8314.71200815,
			High:            8257.8,
			Low:             7500,
		}

		assert.Equal(t, expected, got)
	})

	t.Run("valid trading pair ticker snapshot", func(t *testing.T) {
		payload := []interface{}{
			[]interface{}{
				9206.8,
				21.038892079999997,
				9205.9,
				31.41755015,
				38.97852525,
				0.0043,
				9205.87852525,
				1815.68824558,
				9299,
				9111.8,
			},
		}

		got, err := ticker.FromWSRaw("tBTCUSD", payload)
		require.Nil(t, err)

		expected := &ticker.Snapshot{
			Snapshot: []*ticker.Ticker{
				{
					Symbol:          "tBTCUSD",
					Bid:             9206.8,
					BidSize:         21.038892079999997,
					Ask:             9205.9,
					AskSize:         31.41755015,
					DailyChange:     38.97852525,
					DailyChangePerc: 0.0043,
					LastPrice:       9205.87852525,
					Volume:          1815.68824558,
					High:            9299,
					Low:             9111.8,
				},
			},
		}

		assert.Equal(t, expected, got)
	})
}
<|endoftext|>


<commit_before>package dynamodb

import (
	"bytes"
	"context"
	"encoding/binary"
	"fmt"
	"log"
	"math"
	"net/http"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/request"
	"github.com/aws/aws-sdk-go/aws/session"
	godynamodb "github.com/aws/aws-sdk-go/service/dynamodb"
	godynamodbiface "github.com/aws/aws-sdk-go/service/dynamodb/dynamodbiface"
	"github.com/pkg/errors"

	"github.com/yuuki/diamondb/pkg/config"
	"github.com/yuuki/diamondb/pkg/model"
	"github.com/yuuki/diamondb/pkg/storage/util"
	"github.com/yuuki/diamondb/pkg/timeparser"
)

//go:generate mockgen -source ../../../vendor/github.com/aws/aws-sdk-go/service/dynamodb/dynamodbiface/interface.go -destination dynamodb_mock.go -package dynamodb

// ReadWriter defines the interface for DynamoDB reader and writer.
type ReadWriter interface {
	Ping() error
	Client() godynamodbiface.DynamoDBAPI
	CreateTable(*CreateTableParam) error
	Fetch(string, time.Time, time.Time) (model.SeriesMap, error)
	batchGet(q *query) (model.SeriesMap, error)
	Put(string, string, string, int64, map[int64]float64) error
}

// DynamoDB provides a dynamodb client.
type DynamoDB struct {
	svc godynamodbiface.DynamoDBAPI
}

type timeSlot struct {
	itemEpoch int64
	step      int
}

type query struct {
	names []string
	start time.Time
	end   time.Time
	slot  *timeSlot
	// context
}

const (
	pingTimeout     = time.Duration(5) * time.Second
	batchGetTimeout = time.Duration(5) * time.Second
	updateTimeout   = time.Duration(5) * time.Second

	oneYear time.Duration = time.Duration(24*360) * time.Hour
	oneWeek time.Duration = time.Duration(24*7) * time.Hour
	oneDay  time.Duration = time.Duration(24*1) * time.Hour
)

var (
	dynamodbBatchLimit = 100

	oneYearSeconds = int(oneYear.Seconds())
	oneWeekSeconds = int(oneWeek.Seconds())
	oneDaySeconds  = int(oneDay.Seconds())
)

var _ ReadWriter = &DynamoDB{}

// New creates a new DynamoDB.
func New() (*DynamoDB, error) {
	awsConf := aws.NewConfig().WithRegion(config.Config.DynamoDBRegion)
	if config.Config.DynamoDBEndpoint != "" {
		// For dynamodb-local configuration
		awsConf.WithEndpoint(config.Config.DynamoDBEndpoint)
		awsConf.WithCredentials(credentials.NewStaticCredentials("dummy", "dummy", "dummy"))
	}
	awsConf.WithHTTPClient(&http.Client{
		Timeout:   10 * time.Second,
		Transport: http.DefaultTransport,
	})
	sess, err := session.NewSession(awsConf)
	if err != nil {
		return nil, errors.Wrapf(err,
			"failed to create session for dynamodb (%s,%s)",
			config.Config.DynamoDBRegion,
			config.Config.DynamoDBEndpoint,
		)
	}
	return &DynamoDB{
		svc: godynamodb.New(sess),
	}, nil
}

// Client returns the DynamoDB client.
func (d *DynamoDB) Client() godynamodbiface.DynamoDBAPI {
	return d.svc
}

// Ping pings DynamoDB endpoint.
func (d *DynamoDB) Ping() error {
	ctx, cancel := context.WithTimeout(context.TODO(), pingTimeout)
	defer cancel()
	var opt request.Option = func(r *request.Request) {}
	_, err := d.svc.DescribeTableWithContext(ctx, &godynamodb.DescribeTableInput{
		TableName: aws.String(config.Config.DynamoDBTableName),
	}, opt)
	if err != nil {
		if aerr, ok := err.(awserr.Error); ok && aerr.Code() == request.CanceledErrorCode {
			// If the SDK can determine the request or retry delay was canceled
			// by a context the CanceledErrorCode error code will be returned.
			return errors.Wrap(err, "failed to ping dynamodb due to timeout")
		}
		return errors.Wrapf(err, "failed to ping dynamodb")
	}
	return nil
}

// CreateTableParam is parameter set of CreateTable.
type CreateTableParam struct {
	Name string
	RCU  int64 // ReadCapacityUnits
	WCU  int64 // WriteCapacityUnits
}

// CreateTable creates a dynamodb table to store time series data.
// Skip creating table if the table already exists.
func (d *DynamoDB) CreateTable(param *CreateTableParam) error {
	_, err := d.svc.CreateTable(&godynamodb.CreateTableInput{
		TableName: aws.String(param.Name),
		AttributeDefinitions: []*godynamodb.AttributeDefinition{
			{
				AttributeName: aws.String("Name"),
				AttributeType: aws.String(godynamodb.ScalarAttributeTypeS),
			},
			{
				AttributeName: aws.String("Timestamp"),
				AttributeType: aws.String(godynamodb.ScalarAttributeTypeS),
			},
		},
		KeySchema: []*godynamodb.KeySchemaElement{
			{
				AttributeName: aws.String("Name"),
				KeyType:       aws.String("HASH"),
			},
			{
				AttributeName: aws.String("Timestamp"),
				KeyType:       aws.String("RANGE"),
			},
		},
		ProvisionedThroughput: &godynamodb.ProvisionedThroughput{
			ReadCapacityUnits:  aws.Int64(param.RCU),
			WriteCapacityUnits: aws.Int64(param.WCU),
		},
		// TODO StreamSpecification to export to s3
	})
	if err != nil {
		if awsErr, ok := err.(awserr.Error); ok {
			if awsErr.Code() == "ResourceInUseException" {
				// Skip if the table already exists
				log.Printf("Skip creating DynamoDB table because %s already exists\n", param.Name)
				return nil
			}
		}
		return errors.Wrapf(err, "failed to create dynamodb table (%s,%d,%d)",
			param.Name, param.RCU, param.WCU)
	}

	log.Printf("Creating DynamoDB table (name:%s, rcu:%d, wcu:%d) ...\n",
		param.Name, param.RCU, param.WCU)

	err = d.svc.WaitUntilTableExists(&godynamodb.DescribeTableInput{
		TableName: aws.String(param.Name),
	})
	if err != nil {
		return errors.Wrapf(err, "failed to wait until table exists (%s,%d,%d)",
			param.Name, param.RCU, param.WCU)
	}

	if config.Config.DynamoDBTTL {
		_, err = d.svc.UpdateTimeToLive(&godynamodb.UpdateTimeToLiveInput{
			TableName: aws.String(param.Name),
			TimeToLiveSpecification: &godynamodb.TimeToLiveSpecification{
				AttributeName: aws.String("TTL"),
				Enabled:       aws.Bool(true),
			},
		})
		if err != nil {
			return errors.Wrapf(err, "failed to set TTL to (%s,%d,%d)",
				param.Name, param.RCU, param.WCU)
		}
	}
	return nil
}

// Fetch fetches datapoints by name from start until end.
func (d *DynamoDB) Fetch(name string, start, end time.Time) (model.SeriesMap, error) {
	slots := selectTimeSlots(start, end)
	nameGroups := util.GroupNames(util.SplitName(name), dynamodbBatchLimit)
	numQueries := len(slots) * len(nameGroups)

	type result struct {
		value model.SeriesMap
		err   error
	}
	c := make(chan *result, numQueries)
	for _, slot := range slots {
		for _, names := range nameGroups {
			q := &query{
				names: names,
				start: start,
				end:   end,
				slot:  slot,
			}
			go func(q *query) {
				sm, err := d.batchGet(q)
				c <- &result{value: sm, err: err}
			}(q)
		}
	}
	sm := make(model.SeriesMap, len(nameGroups))
	for i := 0; i < numQueries; i++ {
		ret := <-c
		if ret.err != nil {
			return nil, ret.err
		}
		sm.MergePointsToMap(ret.value)
	}
	return sm, nil
}

func batchGetResultToMap(resp *godynamodb.BatchGetItemOutput, q *query) model.SeriesMap {
	sm := make(model.SeriesMap, len(resp.Responses))
	for _, xs := range resp.Responses {
		for _, x := range xs {
			name := (*x["Name"].S)
			points := make(model.DataPoints, 0, len(x["Values"].BS))
			for _, y := range x["Values"].BS {
				t := int64(binary.BigEndian.Uint64(y[0:8]))
				v := math.Float64frombits(binary.BigEndian.Uint64(y[8:]))
				// Trim datapoints out of [start, end]
				if t < q.start.Unix() || q.end.Unix() < t {
					continue
				}
				points = append(points, model.NewDataPoint(t, v))
			}
			sm[name] = model.NewSeriesPoint(name, points, q.slot.step)
		}
	}
	return sm
}

func (d *DynamoDB) batchGet(q *query) (model.SeriesMap, error) {
	var keys []map[string]*godynamodb.AttributeValue
	for _, name := range q.names {
		keys = append(keys, map[string]*godynamodb.AttributeValue{
			"Name":      {S: aws.String(name)},
			"Timestamp": {S: aws.String(fmt.Sprintf("%d:%d", q.slot.itemEpoch, q.slot.step))},
		})
	}
	items := make(map[string]*godynamodb.KeysAndAttributes)
	items[config.Config.DynamoDBTableName] = &godynamodb.KeysAndAttributes{Keys: keys}
	params := &godynamodb.BatchGetItemInput{
		RequestItems:           items,
		ReturnConsumedCapacity: aws.String("NONE"),
	}
	ctx, cancel := context.WithTimeout(context.TODO(), batchGetTimeout)
	defer cancel()
	var opt request.Option = func(r *request.Request) {}
	resp, err := d.svc.BatchGetItemWithContext(ctx, params, opt)
	if err != nil {
		if aerr, ok := err.(awserr.Error); ok && aerr.Code() == request.CanceledErrorCode {
			// If the SDK can determine the request or retry delay was canceled
			// by a context the CanceledErrorCode error code will be returned.
			return nil, errors.Wrap(err, "failed to batchGet dynamodb due to timeout")
		}
		if awsErr, ok := err.(awserr.Error); ok {
			if awsErr.Code() == "ResourceNotFoundException" {
				// Don't handle ResourceNotFoundException as error
				// bacause diamondb web return length 0 series as 200.
				return model.SeriesMap{}, nil
			}
		}
		return nil, errors.Wrapf(err,
			"failed to call dynamodb API batchGetItem (%s,%d,%d)",
			config.Config.DynamoDBTableName, q.slot.itemEpoch, q.slot.step,
		)
	}
	return batchGetResultToMap(resp, q), nil
}

// Put writes the datapoints into DynamoDB. It creates item
// if item doesn't exist and updates item if it exists.
func (d *DynamoDB) Put(name, slot, history string, itemEpoch int64, tv map[int64]float64) error {
	stepDuration, err := timeparser.ParseTimeOffset(slot)
	if err != nil {
		return err
	}
	historyDuration, err := timeparser.ParseTimeOffset(history)
	if err != nil {
		return err
	}
	ttl := itemEpoch + int64(historyDuration.Seconds())

	vals := make([][]byte, 0, len(tv))
	for timestamp, value := range tv {
		buf := new(bytes.Buffer)
		binary.Write(buf, binary.BigEndian, timestamp)
		binary.Write(buf, binary.BigEndian, math.Float64bits(value))
		vals = append(vals, buf.Bytes())
	}

	params := &godynamodb.UpdateItemInput{
		TableName: aws.String(config.Config.DynamoDBTableName),
		Key: map[string]*godynamodb.AttributeValue{
			"Name":      {S: aws.String(name)},
			"Timestamp": {S: aws.String(fmt.Sprintf("%d:%d", itemEpoch, int64(stepDuration.Seconds())))},
		},
		UpdateExpression: aws.String(`
			SET #ttl = :new_ttl
			ADD #values_set :new_values
		`),
		ExpressionAttributeNames: map[string]*string{
			"#ttl":        aws.String("TTL"),
			"#values_set": aws.String("Values"),
		},
		ExpressionAttributeValues: map[string]*godynamodb.AttributeValue{
			":new_ttl":    {N: aws.String(fmt.Sprintf("%d", ttl))},
			":new_values": {BS: vals},
		},
		ReturnValues: aws.String("NONE"),
	}

	ctx, cancel := context.WithTimeout(context.TODO(), updateTimeout)
	defer cancel()
	var opt request.Option = func(r *request.Request) {}
	if _, err := d.svc.UpdateItemWithContext(ctx, params, opt); err != nil {
		if aerr, ok := err.(awserr.Error); ok && aerr.Code() == request.CanceledErrorCode {
			// If the SDK can determine the request or retry delay was canceled
			// by a context the CanceledErrorCode error code will be returned.
			return errors.Wrap(err, "failed to updateItem dynamodb due to timeout")
		}
		return errors.Wrapf(err, "failed to call dynamodb API putItem (%s,%s,%d)",
			config.Config.DynamoDBTableName, name, itemEpoch)
	}
	return nil
}

func selectTimeSlots(startTime, endTime time.Time) []*timeSlot {
	var (
		step          int
		itemEpochStep int
	)
	diff := endTime.Sub(startTime)
	switch {
	case oneYear <= diff:
		itemEpochStep = oneYearSeconds
		step = 60 * 60 * 24
	case oneWeek <= diff:
		itemEpochStep = oneWeekSeconds
		step = 60 * 60
	case oneDay <= diff:
		itemEpochStep = oneDaySeconds
		step = 5 * 60
	default:
		itemEpochStep = 60 * 60
		step = 60
	}

	var slots []*timeSlot
	startItemEpoch := startTime.Unix() - startTime.Unix()%int64(itemEpochStep)
	endItemEpoch := endTime.Unix()
	for epoch := startItemEpoch; epoch < endItemEpoch; epoch += int64(itemEpochStep) {
		slots = append(slots, &timeSlot{itemEpoch: epoch, step: step})
	}

	return slots
}
<commit_msg>Make HTTPTimeout constant value<commit_after>package dynamodb

import (
	"bytes"
	"context"
	"encoding/binary"
	"fmt"
	"log"
	"math"
	"net/http"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/request"
	"github.com/aws/aws-sdk-go/aws/session"
	godynamodb "github.com/aws/aws-sdk-go/service/dynamodb"
	godynamodbiface "github.com/aws/aws-sdk-go/service/dynamodb/dynamodbiface"
	"github.com/pkg/errors"

	"github.com/yuuki/diamondb/pkg/config"
	"github.com/yuuki/diamondb/pkg/model"
	"github.com/yuuki/diamondb/pkg/storage/util"
	"github.com/yuuki/diamondb/pkg/timeparser"
)

//go:generate mockgen -source ../../../vendor/github.com/aws/aws-sdk-go/service/dynamodb/dynamodbiface/interface.go -destination dynamodb_mock.go -package dynamodb

// ReadWriter defines the interface for DynamoDB reader and writer.
type ReadWriter interface {
	Ping() error
	Client() godynamodbiface.DynamoDBAPI
	CreateTable(*CreateTableParam) error
	Fetch(string, time.Time, time.Time) (model.SeriesMap, error)
	batchGet(q *query) (model.SeriesMap, error)
	Put(string, string, string, int64, map[int64]float64) error
}

// DynamoDB provides a dynamodb client.
type DynamoDB struct {
	svc godynamodbiface.DynamoDBAPI
}

type timeSlot struct {
	itemEpoch int64
	step      int
}

type query struct {
	names []string
	start time.Time
	end   time.Time
	slot  *timeSlot
	// context
}

const (
	HTTPTimeout     = time.Duration(10) * time.Second
	pingTimeout     = time.Duration(5) * time.Second
	batchGetTimeout = time.Duration(5) * time.Second
	updateTimeout   = time.Duration(5) * time.Second

	oneYear time.Duration = time.Duration(24*360) * time.Hour
	oneWeek time.Duration = time.Duration(24*7) * time.Hour
	oneDay  time.Duration = time.Duration(24*1) * time.Hour
)

var (
	dynamodbBatchLimit = 100

	oneYearSeconds = int(oneYear.Seconds())
	oneWeekSeconds = int(oneWeek.Seconds())
	oneDaySeconds  = int(oneDay.Seconds())
)

var _ ReadWriter = &DynamoDB{}

// New creates a new DynamoDB.
func New() (*DynamoDB, error) {
	awsConf := aws.NewConfig().WithRegion(config.Config.DynamoDBRegion)
	if config.Config.DynamoDBEndpoint != "" {
		// For dynamodb-local configuration
		awsConf.WithEndpoint(config.Config.DynamoDBEndpoint)
		awsConf.WithCredentials(credentials.NewStaticCredentials("dummy", "dummy", "dummy"))
	}
	awsConf.WithHTTPClient(&http.Client{
		Timeout:   HTTPTimeout,
		Transport: http.DefaultTransport,
	})
	sess, err := session.NewSession(awsConf)
	if err != nil {
		return nil, errors.Wrapf(err,
			"failed to create session for dynamodb (%s,%s)",
			config.Config.DynamoDBRegion,
			config.Config.DynamoDBEndpoint,
		)
	}
	return &DynamoDB{
		svc: godynamodb.New(sess),
	}, nil
}

// Client returns the DynamoDB client.
func (d *DynamoDB) Client() godynamodbiface.DynamoDBAPI {
	return d.svc
}

// Ping pings DynamoDB endpoint.
func (d *DynamoDB) Ping() error {
	ctx, cancel := context.WithTimeout(context.TODO(), pingTimeout)
	defer cancel()
	var opt request.Option = func(r *request.Request) {}
	_, err := d.svc.DescribeTableWithContext(ctx, &godynamodb.DescribeTableInput{
		TableName: aws.String(config.Config.DynamoDBTableName),
	}, opt)
	if err != nil {
		if aerr, ok := err.(awserr.Error); ok && aerr.Code() == request.CanceledErrorCode {
			// If the SDK can determine the request or retry delay was canceled
			// by a context the CanceledErrorCode error code will be returned.
			return errors.Wrap(err, "failed to ping dynamodb due to timeout")
		}
		return errors.Wrapf(err, "failed to ping dynamodb")
	}
	return nil
}

// CreateTableParam is parameter set of CreateTable.
type CreateTableParam struct {
	Name string
	RCU  int64 // ReadCapacityUnits
	WCU  int64 // WriteCapacityUnits
}

// CreateTable creates a dynamodb table to store time series data.
// Skip creating table if the table already exists.
func (d *DynamoDB) CreateTable(param *CreateTableParam) error {
	_, err := d.svc.CreateTable(&godynamodb.CreateTableInput{
		TableName: aws.String(param.Name),
		AttributeDefinitions: []*godynamodb.AttributeDefinition{
			{
				AttributeName: aws.String("Name"),
				AttributeType: aws.String(godynamodb.ScalarAttributeTypeS),
			},
			{
				AttributeName: aws.String("Timestamp"),
				AttributeType: aws.String(godynamodb.ScalarAttributeTypeS),
			},
		},
		KeySchema: []*godynamodb.KeySchemaElement{
			{
				AttributeName: aws.String("Name"),
				KeyType:       aws.String("HASH"),
			},
			{
				AttributeName: aws.String("Timestamp"),
				KeyType:       aws.String("RANGE"),
			},
		},
		ProvisionedThroughput: &godynamodb.ProvisionedThroughput{
			ReadCapacityUnits:  aws.Int64(param.RCU),
			WriteCapacityUnits: aws.Int64(param.WCU),
		},
		// TODO StreamSpecification to export to s3
	})
	if err != nil {
		if awsErr, ok := err.(awserr.Error); ok {
			if awsErr.Code() == "ResourceInUseException" {
				// Skip if the table already exists
				log.Printf("Skip creating DynamoDB table because %s already exists\n", param.Name)
				return nil
			}
		}
		return errors.Wrapf(err, "failed to create dynamodb table (%s,%d,%d)",
			param.Name, param.RCU, param.WCU)
	}

	log.Printf("Creating DynamoDB table (name:%s, rcu:%d, wcu:%d) ...\n",
		param.Name, param.RCU, param.WCU)

	err = d.svc.WaitUntilTableExists(&godynamodb.DescribeTableInput{
		TableName: aws.String(param.Name),
	})
	if err != nil {
		return errors.Wrapf(err, "failed to wait until table exists (%s,%d,%d)",
			param.Name, param.RCU, param.WCU)
	}

	if config.Config.DynamoDBTTL {
		_, err = d.svc.UpdateTimeToLive(&godynamodb.UpdateTimeToLiveInput{
			TableName: aws.String(param.Name),
			TimeToLiveSpecification: &godynamodb.TimeToLiveSpecification{
				AttributeName: aws.String("TTL"),
				Enabled:       aws.Bool(true),
			},
		})
		if err != nil {
			return errors.Wrapf(err, "failed to set TTL to (%s,%d,%d)",
				param.Name, param.RCU, param.WCU)
		}
	}
	return nil
}

// Fetch fetches datapoints by name from start until end.
func (d *DynamoDB) Fetch(name string, start, end time.Time) (model.SeriesMap, error) {
	slots := selectTimeSlots(start, end)
	nameGroups := util.GroupNames(util.SplitName(name), dynamodbBatchLimit)
	numQueries := len(slots) * len(nameGroups)

	type result struct {
		value model.SeriesMap
		err   error
	}
	c := make(chan *result, numQueries)
	for _, slot := range slots {
		for _, names := range nameGroups {
			q := &query{
				names: names,
				start: start,
				end:   end,
				slot:  slot,
			}
			go func(q *query) {
				sm, err := d.batchGet(q)
				c <- &result{value: sm, err: err}
			}(q)
		}
	}
	sm := make(model.SeriesMap, len(nameGroups))
	for i := 0; i < numQueries; i++ {
		ret := <-c
		if ret.err != nil {
			return nil, ret.err
		}
		sm.MergePointsToMap(ret.value)
	}
	return sm, nil
}

func batchGetResultToMap(resp *godynamodb.BatchGetItemOutput, q *query) model.SeriesMap {
	sm := make(model.SeriesMap, len(resp.Responses))
	for _, xs := range resp.Responses {
		for _, x := range xs {
			name := (*x["Name"].S)
			points := make(model.DataPoints, 0, len(x["Values"].BS))
			for _, y := range x["Values"].BS {
				t := int64(binary.BigEndian.Uint64(y[0:8]))
				v := math.Float64frombits(binary.BigEndian.Uint64(y[8:]))
				// Trim datapoints out of [start, end]
				if t < q.start.Unix() || q.end.Unix() < t {
					continue
				}
				points = append(points, model.NewDataPoint(t, v))
			}
			sm[name] = model.NewSeriesPoint(name, points, q.slot.step)
		}
	}
	return sm
}

func (d *DynamoDB) batchGet(q *query) (model.SeriesMap, error) {
	var keys []map[string]*godynamodb.AttributeValue
	for _, name := range q.names {
		keys = append(keys, map[string]*godynamodb.AttributeValue{
			"Name":      {S: aws.String(name)},
			"Timestamp": {S: aws.String(fmt.Sprintf("%d:%d", q.slot.itemEpoch, q.slot.step))},
		})
	}
	items := make(map[string]*godynamodb.KeysAndAttributes)
	items[config.Config.DynamoDBTableName] = &godynamodb.KeysAndAttributes{Keys: keys}
	params := &godynamodb.BatchGetItemInput{
		RequestItems:           items,
		ReturnConsumedCapacity: aws.String("NONE"),
	}
	ctx, cancel := context.WithTimeout(context.TODO(), batchGetTimeout)
	defer cancel()
	var opt request.Option = func(r *request.Request) {}
	resp, err := d.svc.BatchGetItemWithContext(ctx, params, opt)
	if err != nil {
		if aerr, ok := err.(awserr.Error); ok && aerr.Code() == request.CanceledErrorCode {
			// If the SDK can determine the request or retry delay was canceled
			// by a context the CanceledErrorCode error code will be returned.
			return nil, errors.Wrap(err, "failed to batchGet dynamodb due to timeout")
		}
		if awsErr, ok := err.(awserr.Error); ok {
			if awsErr.Code() == "ResourceNotFoundException" {
				// Don't handle ResourceNotFoundException as error
				// bacause diamondb web return length 0 series as 200.
				return model.SeriesMap{}, nil
			}
		}
		return nil, errors.Wrapf(err,
			"failed to call dynamodb API batchGetItem (%s,%d,%d)",
			config.Config.DynamoDBTableName, q.slot.itemEpoch, q.slot.step,
		)
	}
	return batchGetResultToMap(resp, q), nil
}

// Put writes the datapoints into DynamoDB. It creates item
// if item doesn't exist and updates item if it exists.
func (d *DynamoDB) Put(name, slot, history string, itemEpoch int64, tv map[int64]float64) error {
	stepDuration, err := timeparser.ParseTimeOffset(slot)
	if err != nil {
		return err
	}
	historyDuration, err := timeparser.ParseTimeOffset(history)
	if err != nil {
		return err
	}
	ttl := itemEpoch + int64(historyDuration.Seconds())

	vals := make([][]byte, 0, len(tv))
	for timestamp, value := range tv {
		buf := new(bytes.Buffer)
		binary.Write(buf, binary.BigEndian, timestamp)
		binary.Write(buf, binary.BigEndian, math.Float64bits(value))
		vals = append(vals, buf.Bytes())
	}

	params := &godynamodb.UpdateItemInput{
		TableName: aws.String(config.Config.DynamoDBTableName),
		Key: map[string]*godynamodb.AttributeValue{
			"Name":      {S: aws.String(name)},
			"Timestamp": {S: aws.String(fmt.Sprintf("%d:%d", itemEpoch, int64(stepDuration.Seconds())))},
		},
		UpdateExpression: aws.String(`
			SET #ttl = :new_ttl
			ADD #values_set :new_values
		`),
		ExpressionAttributeNames: map[string]*string{
			"#ttl":        aws.String("TTL"),
			"#values_set": aws.String("Values"),
		},
		ExpressionAttributeValues: map[string]*godynamodb.AttributeValue{
			":new_ttl":    {N: aws.String(fmt.Sprintf("%d", ttl))},
			":new_values": {BS: vals},
		},
		ReturnValues: aws.String("NONE"),
	}

	ctx, cancel := context.WithTimeout(context.TODO(), updateTimeout)
	defer cancel()
	var opt request.Option = func(r *request.Request) {}
	if _, err := d.svc.UpdateItemWithContext(ctx, params, opt); err != nil {
		if aerr, ok := err.(awserr.Error); ok && aerr.Code() == request.CanceledErrorCode {
			// If the SDK can determine the request or retry delay was canceled
			// by a context the CanceledErrorCode error code will be returned.
			return errors.Wrap(err, "failed to updateItem dynamodb due to timeout")
		}
		return errors.Wrapf(err, "failed to call dynamodb API putItem (%s,%s,%d)",
			config.Config.DynamoDBTableName, name, itemEpoch)
	}
	return nil
}

func selectTimeSlots(startTime, endTime time.Time) []*timeSlot {
	var (
		step          int
		itemEpochStep int
	)
	diff := endTime.Sub(startTime)
	switch {
	case oneYear <= diff:
		itemEpochStep = oneYearSeconds
		step = 60 * 60 * 24
	case oneWeek <= diff:
		itemEpochStep = oneWeekSeconds
		step = 60 * 60
	case oneDay <= diff:
		itemEpochStep = oneDaySeconds
		step = 5 * 60
	default:
		itemEpochStep = 60 * 60
		step = 60
	}

	var slots []*timeSlot
	startItemEpoch := startTime.Unix() - startTime.Unix()%int64(itemEpochStep)
	endItemEpoch := endTime.Unix()
	for epoch := startItemEpoch; epoch < endItemEpoch; epoch += int64(itemEpochStep) {
		slots = append(slots, &timeSlot{itemEpoch: epoch, step: step})
	}

	return slots
}
<|endoftext|>


<commit_before>package webircgateway

import (
	"context"
	"strings"
	"sync"

	"golang.org/x/crypto/acme/autocert"
)

type LEManager struct {
	// ensure only one instance of the manager and handler is running
	// while allowing multiple listeners to use it
	Mutex   sync.Mutex
	Manager *autocert.Manager
	gateway *Gateway
}

func NewLetsEncryptManager(gateway *Gateway) *LEManager {
	return &LEManager{gateway: gateway}
}

func (le *LEManager) Get(certCacheDir string) *autocert.Manager {
	le.Mutex.Lock()
	defer le.Mutex.Unlock()

	// Create it if it doesn't already exist
	if le.Manager == nil {
		le.Manager = &autocert.Manager{
			Prompt: autocert.AcceptTOS,
			Cache:  autocert.DirCache(strings.TrimRight(certCacheDir, "/")),
			HostPolicy: func(ctx context.Context, host string) error {
				le.gateway.Log(2, "Automatically requesting a HTTPS certificate for %s", host)
				return nil
			},
		}
		le.gateway.HttpRouter.Handle("/.well-known/", le.Manager.HTTPHandler(le.gateway.HttpRouter))
	}

	return le.Manager
}
<commit_msg>Set autocert fallback to nil to prevent getting stuck in a loop<commit_after>package webircgateway

import (
	"context"
	"strings"
	"sync"

	"golang.org/x/crypto/acme/autocert"
)

type LEManager struct {
	// ensure only one instance of the manager and handler is running
	// while allowing multiple listeners to use it
	Mutex   sync.Mutex
	Manager *autocert.Manager
	gateway *Gateway
}

func NewLetsEncryptManager(gateway *Gateway) *LEManager {
	return &LEManager{gateway: gateway}
}

func (le *LEManager) Get(certCacheDir string) *autocert.Manager {
	le.Mutex.Lock()
	defer le.Mutex.Unlock()

	// Create it if it doesn't already exist
	if le.Manager == nil {
		le.Manager = &autocert.Manager{
			Prompt: autocert.AcceptTOS,
			Cache:  autocert.DirCache(strings.TrimRight(certCacheDir, "/")),
			HostPolicy: func(ctx context.Context, host string) error {
				le.gateway.Log(2, "Automatically requesting a HTTPS certificate for %s", host)
				return nil
			},
		}
		le.gateway.HttpRouter.Handle("/.well-known/", le.Manager.HTTPHandler(nil))
	}

	return le.Manager
}
<|endoftext|>


<commit_before>package soaap

import (
	"encoding/json"
	"errors"
	"fmt"
	"os"
	"strconv"
	"strings"
)

//
// Parse a JSON file emitted by SOAAP.
//
// The `progress` callback will be notified when major events occur
// (top-level JSON parsing begins/ends, traces are parsed, etc.).
//
func ParseJSON(f *os.File, progress func(string)) (Results, error) {
	decoder := json.NewDecoder(f)
	var top map[string]map[string]json.RawMessage

	go progress(fmt.Sprintf("Loading %s", f.Name()))
	err := decoder.Decode(&top)
	if err != nil {
		panic(err)
	}
	raw := top["soaap"]

	var soaap Results

	maxTraceSize := len(raw) - 5
	soaap.Traces = make([]CallTrace, maxTraceSize)

	parsed := 0

	// Once SOAAP issue #28 is resolved, we should be able to replace this
	// loop (as well as the `parseTrace` function) with a single call to
	// decoder.Decode().
	for k, v := range raw {
		switch k {
		case "access_origin_warning":
			// TODO

		case "classified_warning":
			// TODO

		case "private_access":
			json.Unmarshal(v, &soaap.PrivateAccess)
			for i, vuln := range soaap.PrivateAccess {
				num, err := traceNumber(vuln.TraceName)
				if err != nil {
					panic(err)
				}

				soaap.PrivateAccess[i].Trace = num
			}

		case "private_leak":
			// TODO

		case "vulnerability_warning":
			json.Unmarshal(v, &soaap.Vulnerabilities)
			for i, vuln := range soaap.Vulnerabilities {
				num, err := traceNumber(vuln.TraceName)
				if err != nil {
					panic(err)
				}

				soaap.Vulnerabilities[i].Trace = num
			}

		default:
			index, err := traceNumber(k)
			if err != nil {
				return soaap, errors.New(k + " is not a trace")
			}

			err = parseTrace(v, soaap.Traces, index)
			if err != nil {
				return soaap, err
			}
		}

		parsed += 1
		if parsed%10000 == 0 {
			go progress(fmt.Sprintf("Parsed %d traces", parsed))
		}
	}

	go progress("Done")

	return soaap, nil
}

//
// Unwrap the SOAAP encoding of a call trace.
//
// This should be simplified once SOAAP issue #28 is addressed: we won't
// need this function any more.
//
func parseTrace(j json.RawMessage, traces []CallTrace, index int) error {
	var x map[string]json.RawMessage
	err := json.Unmarshal(j, &x)
	if err != nil {
		return err
	}

	var rawCallSites []json.RawMessage
	err = json.Unmarshal(x["trace"], &rawCallSites)
	if err != nil {
		return err
	}

	// The elements in a SOAAP call trace include at least n-1 functions:
	// the final element may be a reference to another trace.
	count := len(rawCallSites)

	t := &traces[index]
	t.CallSites = make([]CallSite, 0, count)
	t.Next = -1

	for i, v := range rawCallSites {
		var tmp CallSite
		err = json.Unmarshal(v, &tmp)
		if err != nil {
			return err
		}

		// If we managed to parse a real function name, this is
		// a CallSite and not a reference to a sub-trace.
		if tmp.Function != "" {
			t.CallSites = append(t.CallSites, tmp)
		} else {
			if i != count-1 {
				panic("unable to convert function")
			}

			var tmp map[string]string
			err = json.Unmarshal(v, &tmp)
			if err != nil {
				return err
			}

			ref, err := traceNumber(tmp["trace_ref"])
			if err != nil {
				return err
			}

			if ref > len(traces) {
				return errors.New(
					fmt.Sprintf("trace %d references %d "+
						" but we only have %d traces",
						index, ref, len(traces)))
			}
			t.Next = ref
		}
	}

	return err
}

// Extract a trace number from a string like '!trace42'.
func traceNumber(name string) (int, error) {
	if !strings.HasPrefix(name, "!trace") {
		return -1, errors.New("'" + name + "' is not a trace name")
	}

	i, err := strconv.ParseInt(name[6:], 10, 32)
	return int(i), err
}
<commit_msg>Report parsing completion synchronously.<commit_after>package soaap

import (
	"encoding/json"
	"errors"
	"fmt"
	"os"
	"strconv"
	"strings"
)

//
// Parse a JSON file emitted by SOAAP.
//
// The `progress` callback will be notified when major events occur
// (top-level JSON parsing begins/ends, traces are parsed, etc.).
//
func ParseJSON(f *os.File, progress func(string)) (Results, error) {
	decoder := json.NewDecoder(f)
	var top map[string]map[string]json.RawMessage

	go progress(fmt.Sprintf("Loading %s", f.Name()))
	err := decoder.Decode(&top)
	if err != nil {
		panic(err)
	}
	raw := top["soaap"]

	var soaap Results

	maxTraceSize := len(raw) - 5
	soaap.Traces = make([]CallTrace, maxTraceSize)

	parsed := 0

	// Once SOAAP issue #28 is resolved, we should be able to replace this
	// loop (as well as the `parseTrace` function) with a single call to
	// decoder.Decode().
	for k, v := range raw {
		switch k {
		case "access_origin_warning":
			// TODO

		case "classified_warning":
			// TODO

		case "private_access":
			json.Unmarshal(v, &soaap.PrivateAccess)
			for i, vuln := range soaap.PrivateAccess {
				num, err := traceNumber(vuln.TraceName)
				if err != nil {
					panic(err)
				}

				soaap.PrivateAccess[i].Trace = num
			}

		case "private_leak":
			// TODO

		case "vulnerability_warning":
			json.Unmarshal(v, &soaap.Vulnerabilities)
			for i, vuln := range soaap.Vulnerabilities {
				num, err := traceNumber(vuln.TraceName)
				if err != nil {
					panic(err)
				}

				soaap.Vulnerabilities[i].Trace = num
			}

		default:
			index, err := traceNumber(k)
			if err != nil {
				return soaap, errors.New(k + " is not a trace")
			}

			err = parseTrace(v, soaap.Traces, index)
			if err != nil {
				return soaap, err
			}
		}

		parsed += 1
		if parsed%10000 == 0 {
			go progress(fmt.Sprintf("Parsed %d traces", parsed))
		}
	}

	progress(fmt.Sprintf("Finished parsing %s.", f.Name()))

	return soaap, nil
}

//
// Unwrap the SOAAP encoding of a call trace.
//
// This should be simplified once SOAAP issue #28 is addressed: we won't
// need this function any more.
//
func parseTrace(j json.RawMessage, traces []CallTrace, index int) error {
	var x map[string]json.RawMessage
	err := json.Unmarshal(j, &x)
	if err != nil {
		return err
	}

	var rawCallSites []json.RawMessage
	err = json.Unmarshal(x["trace"], &rawCallSites)
	if err != nil {
		return err
	}

	// The elements in a SOAAP call trace include at least n-1 functions:
	// the final element may be a reference to another trace.
	count := len(rawCallSites)

	t := &traces[index]
	t.CallSites = make([]CallSite, 0, count)
	t.Next = -1

	for i, v := range rawCallSites {
		var tmp CallSite
		err = json.Unmarshal(v, &tmp)
		if err != nil {
			return err
		}

		// If we managed to parse a real function name, this is
		// a CallSite and not a reference to a sub-trace.
		if tmp.Function != "" {
			t.CallSites = append(t.CallSites, tmp)
		} else {
			if i != count-1 {
				panic("unable to convert function")
			}

			var tmp map[string]string
			err = json.Unmarshal(v, &tmp)
			if err != nil {
				return err
			}

			ref, err := traceNumber(tmp["trace_ref"])
			if err != nil {
				return err
			}

			if ref > len(traces) {
				return errors.New(
					fmt.Sprintf("trace %d references %d "+
						" but we only have %d traces",
						index, ref, len(traces)))
			}
			t.Next = ref
		}
	}

	return err
}

// Extract a trace number from a string like '!trace42'.
func traceNumber(name string) (int, error) {
	if !strings.HasPrefix(name, "!trace") {
		return -1, errors.New("'" + name + "' is not a trace name")
	}

	i, err := strconv.ParseInt(name[6:], 10, 32)
	return int(i), err
}
<|endoftext|>