<commit_before>package funcs

import (
	"testing"
	"unicode/utf8"

	"github.com/stretchr/testify/assert"
)

func TestASCII(t *testing.T) {
	f := &RandomFuncs{}
	s, err := f.ASCII(0)
	assert.NoError(t, err)
	assert.Empty(t, s)

	s, err = f.ASCII(100)
	assert.NoError(t, err)
	assert.Len(t, s, 100)
	assert.Regexp(t, "^[[:print:]]*$", s)
}

func TestAlpha(t *testing.T) {
	f := &RandomFuncs{}
	s, err := f.Alpha(0)
	assert.NoError(t, err)
	assert.Empty(t, s)

	s, err = f.Alpha(100)
	assert.NoError(t, err)
	assert.Len(t, s, 100)
	assert.Regexp(t, "^[[:alpha:]]*$", s)
}

func TestAlphaNum(t *testing.T) {
	f := &RandomFuncs{}
	s, err := f.AlphaNum(0)
	assert.NoError(t, err)
	assert.Empty(t, s)

	s, err = f.AlphaNum(100)
	assert.NoError(t, err)
	assert.Len(t, s, 100)
	assert.Regexp(t, "^[[:alnum:]]*$", s)
}

func TestToCodePoints(t *testing.T) {
	l, u, err := toCodePoints("a", "b")
	assert.NoError(t, err)
	assert.Equal(t, 'a', l)
	assert.Equal(t, 'b', u)

	_, _, err = toCodePoints("foo", "bar")
	assert.Error(t, err)

	_, _, err = toCodePoints("0755", "bar")
	assert.Error(t, err)

	l, u, err = toCodePoints("0xD700", "0x0001FFFF")
	assert.NoError(t, err)
	assert.Equal(t, '\ud700', l)
	assert.Equal(t, '\U0001ffff', u)

	l, u, err = toCodePoints("0011", "0777")
	assert.NoError(t, err)
	assert.Equal(t, rune(0011), l)
	assert.Equal(t, rune(0777), u)

	l, u, err = toCodePoints("♬", "♟")
	assert.NoError(t, err)
	assert.Equal(t, rune(0x266C), l)
	assert.Equal(t, '♟', u)
}

func TestString(t *testing.T) {
	f := &RandomFuncs{}
	out, err := f.String(1)
	assert.NoError(t, err)
	assert.Len(t, out, 1)

	out, err = f.String(42)
	assert.NoError(t, err)
	assert.Len(t, out, 42)

	_, err = f.String(0)
	assert.Error(t, err)

	out, err = f.String(8, "[a-z]")
	assert.NoError(t, err)
	assert.Regexp(t, "^[a-z]{8}$", out)

	out, err = f.String(10, 0x23, 0x26)
	assert.NoError(t, err)
	assert.Regexp(t, "^[#$%&]{10}$", out)

	out, err = f.String(8, '\U0001f062', '\U0001f093')
	assert.NoError(t, err)
	assert.Regexp(t, "^[🁢-🂓]{8}$", out)

	out, err = f.String(8, '\U0001f062', '\U0001f093')
	assert.NoError(t, err)
	assert.Regexp(t, "^[🁢-🂓]{8}$", out)

	out, err = f.String(8, "♚", "♟")
	assert.NoError(t, err)
	assert.Regexp(t, "^[♚-♟]{8}$", out)

	out, err = f.String(100, "♠", "♣")
	assert.NoError(t, err)
	assert.Equal(t, 100, utf8.RuneCountInString(out))
	assert.Regexp(t, "^[♠-♣]{100}$", out)
}

func TestItem(t *testing.T) {
	f := &RandomFuncs{}
	_, err := f.Item(nil)
	assert.Error(t, err)

	_, err = f.Item("foo")
	assert.Error(t, err)

	i, err := f.Item([]string{"foo"})
	assert.NoError(t, err)
	assert.Equal(t, "foo", i)

	in := []string{"foo", "bar"}
	got := ""
	for j := 0; j < 10; j++ {
		i, err = f.Item(in)
		assert.NoError(t, err)
		got += i.(string)
	}
	assert.NotEqual(t, "foofoofoofoofoofoofoofoofoofoo", got)
	assert.NotEqual(t, "barbarbarbarbarbarbarbarbarbar", got)
}

func TestNumber(t *testing.T) {
	f := &RandomFuncs{}
	n, err := f.Number()
	assert.NoError(t, err)
	assert.True(t, 0 < n && n < 100)

	_, err = f.Number(-1)
	assert.Error(t, err)

	n, err = f.Number(0)
	assert.NoError(t, err)
	assert.Equal(t, int64(0), n)

	n, err = f.Number(9, 9)
	assert.NoError(t, err)
	assert.Equal(t, int64(9), n)

	n, err = f.Number(-10, -10)
	assert.NoError(t, err)
	assert.Equal(t, int64(-10), n)
}

func TestFloat(t *testing.T) {
	f := &RandomFuncs{}
	n, err := f.Float()
	assert.NoError(t, err)
	assert.InDelta(t, 0.5, n, 0.5)

	n, err = f.Float(0.5)
	assert.NoError(t, err)
	assert.InDelta(t, 0.25, n, 0.25)

	n, err = f.Float(490, 500)
	assert.NoError(t, err)
	assert.InDelta(t, 495, n, 5)

	n, err = f.Float(-500, 500)
	assert.NoError(t, err)
	assert.InDelta(t, 0, n, 500)
}
<commit_msg>Fixing random test that fails rarely<commit_after>package funcs

import (
	"testing"
	"unicode/utf8"

	"github.com/stretchr/testify/assert"
)

func TestASCII(t *testing.T) {
	f := &RandomFuncs{}
	s, err := f.ASCII(0)
	assert.NoError(t, err)
	assert.Empty(t, s)

	s, err = f.ASCII(100)
	assert.NoError(t, err)
	assert.Len(t, s, 100)
	assert.Regexp(t, "^[[:print:]]*$", s)
}

func TestAlpha(t *testing.T) {
	f := &RandomFuncs{}
	s, err := f.Alpha(0)
	assert.NoError(t, err)
	assert.Empty(t, s)

	s, err = f.Alpha(100)
	assert.NoError(t, err)
	assert.Len(t, s, 100)
	assert.Regexp(t, "^[[:alpha:]]*$", s)
}

func TestAlphaNum(t *testing.T) {
	f := &RandomFuncs{}
	s, err := f.AlphaNum(0)
	assert.NoError(t, err)
	assert.Empty(t, s)

	s, err = f.AlphaNum(100)
	assert.NoError(t, err)
	assert.Len(t, s, 100)
	assert.Regexp(t, "^[[:alnum:]]*$", s)
}

func TestToCodePoints(t *testing.T) {
	l, u, err := toCodePoints("a", "b")
	assert.NoError(t, err)
	assert.Equal(t, 'a', l)
	assert.Equal(t, 'b', u)

	_, _, err = toCodePoints("foo", "bar")
	assert.Error(t, err)

	_, _, err = toCodePoints("0755", "bar")
	assert.Error(t, err)

	l, u, err = toCodePoints("0xD700", "0x0001FFFF")
	assert.NoError(t, err)
	assert.Equal(t, '\ud700', l)
	assert.Equal(t, '\U0001ffff', u)

	l, u, err = toCodePoints("0011", "0777")
	assert.NoError(t, err)
	assert.Equal(t, rune(0011), l)
	assert.Equal(t, rune(0777), u)

	l, u, err = toCodePoints("♬", "♟")
	assert.NoError(t, err)
	assert.Equal(t, rune(0x266C), l)
	assert.Equal(t, '♟', u)
}

func TestString(t *testing.T) {
	f := &RandomFuncs{}
	out, err := f.String(1)
	assert.NoError(t, err)
	assert.Len(t, out, 1)

	out, err = f.String(42)
	assert.NoError(t, err)
	assert.Len(t, out, 42)

	_, err = f.String(0)
	assert.Error(t, err)

	out, err = f.String(8, "[a-z]")
	assert.NoError(t, err)
	assert.Regexp(t, "^[a-z]{8}$", out)

	out, err = f.String(10, 0x23, 0x26)
	assert.NoError(t, err)
	assert.Regexp(t, "^[#$%&]{10}$", out)

	out, err = f.String(8, '\U0001f062', '\U0001f093')
	assert.NoError(t, err)
	assert.Regexp(t, "^[🁢-🂓]{8}$", out)

	out, err = f.String(8, '\U0001f062', '\U0001f093')
	assert.NoError(t, err)
	assert.Regexp(t, "^[🁢-🂓]{8}$", out)

	out, err = f.String(8, "♚", "♟")
	assert.NoError(t, err)
	assert.Regexp(t, "^[♚-♟]{8}$", out)

	out, err = f.String(100, "♠", "♣")
	assert.NoError(t, err)
	assert.Equal(t, 100, utf8.RuneCountInString(out))
	assert.Regexp(t, "^[♠-♣]{100}$", out)
}

func TestItem(t *testing.T) {
	f := &RandomFuncs{}
	_, err := f.Item(nil)
	assert.Error(t, err)

	_, err = f.Item("foo")
	assert.Error(t, err)

	i, err := f.Item([]string{"foo"})
	assert.NoError(t, err)
	assert.Equal(t, "foo", i)

	in := []string{"foo", "bar"}
	got := ""
	for j := 0; j < 10; j++ {
		i, err = f.Item(in)
		assert.NoError(t, err)
		got += i.(string)
	}
	assert.NotEqual(t, "foofoofoofoofoofoofoofoofoofoo", got)
	assert.NotEqual(t, "barbarbarbarbarbarbarbarbarbar", got)
}

func TestNumber(t *testing.T) {
	f := &RandomFuncs{}
	n, err := f.Number()
	assert.NoError(t, err)
	assert.True(t, 0 < n && n <= 100, n)

	_, err = f.Number(-1)
	assert.Error(t, err)

	n, err = f.Number(0)
	assert.NoError(t, err)
	assert.Equal(t, int64(0), n)

	n, err = f.Number(9, 9)
	assert.NoError(t, err)
	assert.Equal(t, int64(9), n)

	n, err = f.Number(-10, -10)
	assert.NoError(t, err)
	assert.Equal(t, int64(-10), n)
}

func TestFloat(t *testing.T) {
	f := &RandomFuncs{}
	n, err := f.Float()
	assert.NoError(t, err)
	assert.InDelta(t, 0.5, n, 0.5)

	n, err = f.Float(0.5)
	assert.NoError(t, err)
	assert.InDelta(t, 0.25, n, 0.25)

	n, err = f.Float(490, 500)
	assert.NoError(t, err)
	assert.InDelta(t, 495, n, 5)

	n, err = f.Float(-500, 500)
	assert.NoError(t, err)
	assert.InDelta(t, 0, n, 500)
}
<|endoftext|>


<commit_before>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"time"

	"github.com/golang/glog"
	"gopkg.in/yaml.v2"
	"k8s.io/kubernetes/test/e2e/perftype"
)

const (
	// S_TO_MS is a second to millisecond ratio.
	S_TO_MS = float64((time.Second) / time.Millisecond)
)

// BenchmarkResult is a dns benchmark results structure.
type BenchmarkResult struct {
	Code   int             `yaml:"code"`
	Data   BenchmarkData   `yaml:"data"`
	Params BenchmarkParams `yaml:"params"`
}

// BenchmarkData represents dns benchmark data.
type BenchmarkData struct {
	Latency50Percentile float64 `yaml:"latency_50_percentile"`
	Latency95Percentile float64 `yaml:"latency_95_percentile"`
	Latency99Percentile float64 `yaml:"latency_99_percentile"`
	AvgLatency          float64 `yaml:"avg_latency"`
	MaxLatency          float64 `yaml:"max_latency"`
	MinLatency          float64 `yaml:"min_latency"`
	Qps                 float64 `yaml:"qps"`
	QueriesCompleted    float64 `yaml:"queries_completed"`
	QueriesLost         float64 `yaml:"queries_lost"`
	QueriesSent         float64 `yaml:"queries_sent"`
}

// BenchmarkParams represents dns benchmark params.
type BenchmarkParams struct {
	RunLengthSeconds float64  `yaml:"run_length_seconds"`
	QueryFile        string   `yaml:"query_file"`
	KubednsCpu       *float64 `yaml:"kubedns_cpu"`
	DnsmasqCpu       *float64 `yaml:"dnsmasq_cpu"`
	DnsmasqCache     *float64 `yaml:"dnsmasq_cache"`
	MaxQps           *float64 `yaml:"max_qps"`
}

func main() {
	defer glog.Flush()
	err := run()
	if err != nil {
		panic(err)
	}
}

func run() error {
	var benchmarkDirPath, jsonDirPath, benchmarkName string
	flag.CommandLine = flag.NewFlagSet(os.Args[0], flag.ExitOnError)
	flag.StringVar(&benchmarkDirPath, "benchmarkDirPath", ".", "benchmark results directory path")
	flag.StringVar(&jsonDirPath, "jsonDirPath", ".", "json results directory path")
	flag.StringVar(&benchmarkName, "benchmarkName", ".", "benchmark name")

	if err := flag.CommandLine.Parse(os.Args[1:]); err != nil {
		return fmt.Errorf("flag parse failed: %v", err)
	}

	glog.Infof("benchmarkDirPath: %v\n", benchmarkDirPath)
	glog.Infof("jsonDirPath: %v\n", jsonDirPath)
	glog.Infof("benchmarkName: %v\n", benchmarkName)

	latency := perftype.PerfData{Version: "v1"}
	latencyPerc := perftype.PerfData{Version: "v1"}
	queries := perftype.PerfData{Version: "v1"}
	qps := perftype.PerfData{Version: "v1"}

	fileList, err := getFileList(benchmarkDirPath)
	if err != nil {
		return fmt.Errorf("listing files error: %v", err)
	}

	for _, file := range fileList {
		glog.Infof("processing %s\n", file)
		result, err := readBenchmarkResult(filepath.Join(benchmarkDirPath, file))
		if err != nil {
			return err
		}
		labels := createLabels(&result.Params)
		latency.DataItems = appendLatency(latency.DataItems, labels, result)
		latencyPerc.DataItems = appendLatencyPerc(latencyPerc.DataItems, labels, result)
		queries.DataItems = appendQueries(queries.DataItems, labels, result)
		qps.DataItems = appendQps(qps.DataItems, labels, result)
	}

	timeString := time.Now().Format(time.RFC3339)
	if err = saveMetric(&latency, filepath.Join(jsonDirPath, "Latency_"+benchmarkName+"_"+timeString+".json")); err != nil {
		return err
	}
	if err = saveMetric(&latencyPerc, filepath.Join(jsonDirPath, "LatencyPerc_"+benchmarkName+"_"+timeString+".json")); err != nil {
		return err
	}
	if err = saveMetric(&queries, filepath.Join(jsonDirPath, "Queries_"+benchmarkName+"_"+timeString+".json")); err != nil {
		return err
	}
	if err = saveMetric(&qps, filepath.Join(jsonDirPath, "Qps_"+benchmarkName+"_"+timeString+".json")); err != nil {
		return err
	}

	return nil
}

// getFileList returns a list of all files with extension .out.
func getFileList(dir string) ([]string, error) {
	var fileNames []string
	files, err := ioutil.ReadDir(dir)
	if err != nil {
		return fileNames, err
	}

	for _, file := range files {
		if !file.IsDir() && filepath.Ext(file.Name()) == ".out" {
			fileNames = append(fileNames, file.Name())
		}
	}
	return fileNames, nil
}

func readBenchmarkResult(path string) (*BenchmarkResult, error) {
	var result BenchmarkResult
	bin, err := ioutil.ReadFile(path)
	if err != nil {
		return nil, fmt.Errorf("reading error: %v", err)
	}

	if err := yaml.Unmarshal(bin, &result); err != nil {
		return nil, fmt.Errorf("decoding failed: %v", err)
	}
	return &result, nil
}

func toString(v *float64) string {
	if v == nil {
		return ""
	}
	return fmt.Sprintf("%v", *v)
}

func createLabels(params *BenchmarkParams) map[string]string {
	labels := make(map[string]string)
	labels["run_length_seconds"] = fmt.Sprintf("%v", params.RunLengthSeconds)
	labels["query_file"] = params.QueryFile
	labels["kubedns_cpu"] = toString(params.KubednsCpu)
	labels["dnsmasq_cpu"] = toString(params.DnsmasqCpu)
	labels["dnsmasq_cache"] = toString(params.DnsmasqCache)
	labels["max_qps"] = toString(params.MaxQps)
	return labels

}

func appendLatency(items []perftype.DataItem, labels map[string]string, result *BenchmarkResult) []perftype.DataItem {
	return append(items, perftype.DataItem{
		Unit:   "ms",
		Labels: labels,
		Data: map[string]float64{
			"max_latency": result.Data.MaxLatency * S_TO_MS,
			"avg_latency": result.Data.AvgLatency * S_TO_MS,
			"min_latency": result.Data.MinLatency * S_TO_MS,
		},
	})
}

func appendLatencyPerc(items []perftype.DataItem, labels map[string]string, result *BenchmarkResult) []perftype.DataItem {
	return append(items, perftype.DataItem{
		Unit:   "ms",
		Labels: labels,
		Data: map[string]float64{
			"perc50": result.Data.Latency50Percentile,
			"perc90": result.Data.Latency95Percentile,
			"perc99": result.Data.Latency99Percentile,
		},
	})
}

func appendQueries(items []perftype.DataItem, labels map[string]string, result *BenchmarkResult) []perftype.DataItem {
	return append(items, perftype.DataItem{
		Unit:   "ms",
		Labels: labels,
		Data: map[string]float64{
			"queries_completed": result.Data.QueriesCompleted,
			"queries_lost":      result.Data.QueriesLost,
			"queries_sent":      result.Data.QueriesSent,
		},
	})
}

func appendQps(items []perftype.DataItem, labels map[string]string, result *BenchmarkResult) []perftype.DataItem {
	return append(items, perftype.DataItem{
		Unit:   "ms",
		Labels: labels,
		Data: map[string]float64{
			"qps": result.Data.Qps,
		},
	})
}

func saveMetric(metric *perftype.PerfData, path string) error {
	output := &bytes.Buffer{}
	if err := json.NewEncoder(output).Encode(metric); err != nil {
		return err
	}
	formatted := &bytes.Buffer{}
	if err := json.Indent(formatted, output.Bytes(), "", "  "); err != nil {
		return err
	}
	return ioutil.WriteFile(path, formatted.Bytes(), 0664)
}
<commit_msg>fixing dns metrics units<commit_after>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"time"

	"github.com/golang/glog"
	"gopkg.in/yaml.v2"
	"k8s.io/kubernetes/test/e2e/perftype"
)

const (
	// S_TO_MS is a second to millisecond ratio.
	S_TO_MS = float64((time.Second) / time.Millisecond)
)

// BenchmarkResult is a dns benchmark results structure.
type BenchmarkResult struct {
	Code   int             `yaml:"code"`
	Data   BenchmarkData   `yaml:"data"`
	Params BenchmarkParams `yaml:"params"`
}

// BenchmarkData represents dns benchmark data.
type BenchmarkData struct {
	Latency50Percentile float64 `yaml:"latency_50_percentile"`
	Latency95Percentile float64 `yaml:"latency_95_percentile"`
	Latency99Percentile float64 `yaml:"latency_99_percentile"`
	AvgLatency          float64 `yaml:"avg_latency"`
	MaxLatency          float64 `yaml:"max_latency"`
	MinLatency          float64 `yaml:"min_latency"`
	Qps                 float64 `yaml:"qps"`
	QueriesCompleted    float64 `yaml:"queries_completed"`
	QueriesLost         float64 `yaml:"queries_lost"`
	QueriesSent         float64 `yaml:"queries_sent"`
}

// BenchmarkParams represents dns benchmark params.
type BenchmarkParams struct {
	RunLengthSeconds float64  `yaml:"run_length_seconds"`
	QueryFile        string   `yaml:"query_file"`
	KubednsCpu       *float64 `yaml:"kubedns_cpu"`
	DnsmasqCpu       *float64 `yaml:"dnsmasq_cpu"`
	DnsmasqCache     *float64 `yaml:"dnsmasq_cache"`
	MaxQps           *float64 `yaml:"max_qps"`
}

func main() {
	defer glog.Flush()
	err := run()
	if err != nil {
		panic(err)
	}
}

func run() error {
	var benchmarkDirPath, jsonDirPath, benchmarkName string
	flag.CommandLine = flag.NewFlagSet(os.Args[0], flag.ExitOnError)
	flag.StringVar(&benchmarkDirPath, "benchmarkDirPath", ".", "benchmark results directory path")
	flag.StringVar(&jsonDirPath, "jsonDirPath", ".", "json results directory path")
	flag.StringVar(&benchmarkName, "benchmarkName", ".", "benchmark name")

	if err := flag.CommandLine.Parse(os.Args[1:]); err != nil {
		return fmt.Errorf("flag parse failed: %v", err)
	}

	glog.Infof("benchmarkDirPath: %v\n", benchmarkDirPath)
	glog.Infof("jsonDirPath: %v\n", jsonDirPath)
	glog.Infof("benchmarkName: %v\n", benchmarkName)

	latency := perftype.PerfData{Version: "v1"}
	latencyPerc := perftype.PerfData{Version: "v1"}
	queries := perftype.PerfData{Version: "v1"}
	qps := perftype.PerfData{Version: "v1"}

	fileList, err := getFileList(benchmarkDirPath)
	if err != nil {
		return fmt.Errorf("listing files error: %v", err)
	}

	for _, file := range fileList {
		glog.Infof("processing %s\n", file)
		result, err := readBenchmarkResult(filepath.Join(benchmarkDirPath, file))
		if err != nil {
			return err
		}
		labels := createLabels(&result.Params)
		latency.DataItems = appendLatency(latency.DataItems, labels, result)
		latencyPerc.DataItems = appendLatencyPerc(latencyPerc.DataItems, labels, result)
		queries.DataItems = appendQueries(queries.DataItems, labels, result)
		qps.DataItems = appendQps(qps.DataItems, labels, result)
	}

	timeString := time.Now().Format(time.RFC3339)
	if err = saveMetric(&latency, filepath.Join(jsonDirPath, "Latency_"+benchmarkName+"_"+timeString+".json")); err != nil {
		return err
	}
	if err = saveMetric(&latencyPerc, filepath.Join(jsonDirPath, "LatencyPerc_"+benchmarkName+"_"+timeString+".json")); err != nil {
		return err
	}
	if err = saveMetric(&queries, filepath.Join(jsonDirPath, "Queries_"+benchmarkName+"_"+timeString+".json")); err != nil {
		return err
	}
	if err = saveMetric(&qps, filepath.Join(jsonDirPath, "Qps_"+benchmarkName+"_"+timeString+".json")); err != nil {
		return err
	}

	return nil
}

// getFileList returns a list of all files with extension .out.
func getFileList(dir string) ([]string, error) {
	var fileNames []string
	files, err := ioutil.ReadDir(dir)
	if err != nil {
		return fileNames, err
	}

	for _, file := range files {
		if !file.IsDir() && filepath.Ext(file.Name()) == ".out" {
			fileNames = append(fileNames, file.Name())
		}
	}
	return fileNames, nil
}

func readBenchmarkResult(path string) (*BenchmarkResult, error) {
	var result BenchmarkResult
	bin, err := ioutil.ReadFile(path)
	if err != nil {
		return nil, fmt.Errorf("reading error: %v", err)
	}

	if err := yaml.Unmarshal(bin, &result); err != nil {
		return nil, fmt.Errorf("decoding failed: %v", err)
	}
	return &result, nil
}

func toString(v *float64) string {
	if v == nil {
		return ""
	}
	return fmt.Sprintf("%v", *v)
}

func createLabels(params *BenchmarkParams) map[string]string {
	labels := make(map[string]string)
	labels["run_length_seconds"] = fmt.Sprintf("%v", params.RunLengthSeconds)
	labels["query_file"] = params.QueryFile
	labels["kubedns_cpu"] = toString(params.KubednsCpu)
	labels["dnsmasq_cpu"] = toString(params.DnsmasqCpu)
	labels["dnsmasq_cache"] = toString(params.DnsmasqCache)
	labels["max_qps"] = toString(params.MaxQps)
	return labels

}

func appendLatency(items []perftype.DataItem, labels map[string]string, result *BenchmarkResult) []perftype.DataItem {
	return append(items, perftype.DataItem{
		Unit:   "ms",
		Labels: labels,
		Data: map[string]float64{
			"max_latency": result.Data.MaxLatency * S_TO_MS,
			"avg_latency": result.Data.AvgLatency * S_TO_MS,
			"min_latency": result.Data.MinLatency * S_TO_MS,
		},
	})
}

func appendLatencyPerc(items []perftype.DataItem, labels map[string]string, result *BenchmarkResult) []perftype.DataItem {
	return append(items, perftype.DataItem{
		Unit:   "ms",
		Labels: labels,
		Data: map[string]float64{
			"perc50": result.Data.Latency50Percentile,
			"perc90": result.Data.Latency95Percentile,
			"perc99": result.Data.Latency99Percentile,
		},
	})
}

func appendQueries(items []perftype.DataItem, labels map[string]string, result *BenchmarkResult) []perftype.DataItem {
	return append(items, perftype.DataItem{
		Unit:   "",
		Labels: labels,
		Data: map[string]float64{
			"queries_completed": result.Data.QueriesCompleted,
			"queries_lost":      result.Data.QueriesLost,
			"queries_sent":      result.Data.QueriesSent,
		},
	})
}

func appendQps(items []perftype.DataItem, labels map[string]string, result *BenchmarkResult) []perftype.DataItem {
	return append(items, perftype.DataItem{
		Unit:   "1/s",
		Labels: labels,
		Data: map[string]float64{
			"qps": result.Data.Qps,
		},
	})
}

func saveMetric(metric *perftype.PerfData, path string) error {
	output := &bytes.Buffer{}
	if err := json.NewEncoder(output).Encode(metric); err != nil {
		return err
	}
	formatted := &bytes.Buffer{}
	if err := json.Indent(formatted, output.Bytes(), "", "  "); err != nil {
		return err
	}
	return ioutil.WriteFile(path, formatted.Bytes(), 0664)
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package math

import "math"

// x^y: exponentation
export func Pow(x, y float64) float64 {
	// TODO: x or y NaN, ±Inf, maybe ±0.
	switch {
	case y == 0:
		return 1;
	case y == 1:
		return x;
	case x == 0 && y > 0:
		return 0;
	case x == 0 && y < 0:
		return sys.Inf(1);
	case y == 0.5:
		return Sqrt(x);
	case y == -0.5:
		return 1 / Sqrt(x);
	}

	absy := y;
	flip := false;
	if absy < 0 {
		absy = -absy;
		flip = true;
	}
	yi, yf := sys.modf(absy);
	if yf != 0 && x < 0 {
		return sys.NaN();
	}
	if yi >= 1<<63 {
		return Exp(y * Log(x));
	}

	ans := float64(1);

	// ans *= x^yf
	if yf != 0 {
		if yf > 0.5 {
			yf--;
			yi++;
		}
		ans = Exp(yf * Log(x));
	}

	// ans *= x^yi
	// by multiplying in successive squarings
	// of x according to bits of yi.
	// accumulate powers of two into exp.
	// will still have to do ans *= 2^exp later.
	x1, xe := sys.frexp(x);
	exp := 0;
	if i := int64(yi); i != 0 {
		for {
			if i&1 == 1 {
				ans *= x1;
				exp += xe;
			}
			i >>= 1;
			if i == 0 {
				break;
			}
			x1 *= x1;
			xe <<= 1;
			if x1 < .5 {
				x1 += x1;
				xe--;
			}
		}
	}

	// ans *= 2^exp
	// if flip { ans = 1 / ans }
	// but in the opposite order
	if flip {
		ans = 1 / ans;
		exp = -exp;
	}
	return sys.ldexp(ans, exp);
}

<commit_msg>slightly simpler math.Pow per gri's suggestion<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package math

import "math"

// x^y: exponentiation
export func Pow(x, y float64) float64 {
	// TODO: x or y NaN, ±Inf, maybe ±0.
	switch {
	case y == 0:
		return 1;
	case y == 1:
		return x;
	case x == 0 && y > 0:
		return 0;
	case x == 0 && y < 0:
		return sys.Inf(1);
	case y == 0.5:
		return Sqrt(x);
	case y == -0.5:
		return 1 / Sqrt(x);
	}

	absy := y;
	flip := false;
	if absy < 0 {
		absy = -absy;
		flip = true;
	}
	yi, yf := sys.modf(absy);
	if yf != 0 && x < 0 {
		return sys.NaN();
	}
	if yi >= 1<<63 {
		return Exp(y * Log(x));
	}

	// ans = a1 * 2^ae (= 1 for now).
	a1 := float64(1);
	ae := 0;

	// ans *= x^yf
	if yf != 0 {
		if yf > 0.5 {
			yf--;
			yi++;
		}
		a1 = Exp(yf * Log(x));
	}

	// ans *= x^yi
	// by multiplying in successive squarings
	// of x according to bits of yi.
	// accumulate powers of two into exp.
	x1, xe := sys.frexp(x);
	for i := int64(yi); i != 0; i >>= 1 {
		if i&1 == 1 {
			a1 *= x1;
			ae += xe;
		}
		x1 *= x1;
		xe <<= 1;
		if x1 < .5 {
			x1 += x1;
			xe--;
		}
	}

	// ans = a1*2^ae
	// if flip { ans = 1 / ans }
	// but in the opposite order
	if flip {
		a1 = 1 / a1;
		ae = -ae;
	}
	return sys.ldexp(a1, ae);
}
<|endoftext|>


<commit_before>package libfuse

import (
	"os"
	"strings"
	"sync"

	"bazil.org/fuse"
	"bazil.org/fuse/fs"
	keybase1 "github.com/keybase/client/go/protocol"
	"github.com/keybase/kbfs/libkbfs"
	"golang.org/x/net/context"
)

// FolderList is a node that can list all of the logged-in user's
// favorite top-level folders, on either a public or private basis.
type FolderList struct {
	fs *FS
	// only accept public folders
	public bool

	mu      sync.Mutex
	folders map[string]*TLF
}

var _ fs.Node = (*FolderList)(nil)

// Attr implements the fs.Node interface.
func (*FolderList) Attr(ctx context.Context, a *fuse.Attr) error {
	a.Mode = os.ModeDir | 0755
	return nil
}

var _ fs.NodeRequestLookuper = (*FolderList)(nil)

// Lookup implements the fs.NodeRequestLookuper interface.
func (fl *FolderList) Lookup(ctx context.Context, req *fuse.LookupRequest, resp *fuse.LookupResponse) (node fs.Node, err error) {
	fl.fs.log.CDebugf(ctx, "FL Lookup %s", req.Name)
	defer func() { fl.fs.reportErr(ctx, err) }()
	fl.mu.Lock()
	defer fl.mu.Unlock()

	specialNode := handleSpecialFile(req.Name, fl.fs, resp)
	if specialNode != nil {
		return specialNode, nil
	}

	if child, ok := fl.folders[req.Name]; ok {
		return child, nil
	}

	// Shortcut for dreaded extraneous OSX finder lookups and weird
	// Linux stuff like "_mtn".
	if strings.HasPrefix(req.Name, "._") || strings.HasPrefix(req.Name, "_") {
		return nil, fuse.ENOENT
	}

	_, err = libkbfs.ParseTlfHandle(
		ctx, fl.fs.config.KBPKI(), req.Name, fl.public)
	switch err := err.(type) {
	case nil:
		// No error.
		break

	case libkbfs.TlfNameNotCanonical:
		// Non-canonical name.
		n := &Alias{
			canon: err.NameToTry,
		}
		return n, nil

	case libkbfs.NoSuchNameError:
		// Invalid public TLF.
		return nil, fuse.ENOENT

	default:
		// Some other error.
		return nil, err
	}

	child := newTLF(fl, req.Name)
	fl.folders[req.Name] = child
	return child, nil
}

func (fl *FolderList) forgetFolder(folderName string) {
	fl.mu.Lock()
	defer fl.mu.Unlock()
	delete(fl.folders, folderName)
}

var _ fs.Handle = (*FolderList)(nil)

var _ fs.HandleReadDirAller = (*FolderList)(nil)

// ReadDirAll implements the ReadDirAll interface.
func (fl *FolderList) ReadDirAll(ctx context.Context) (res []fuse.Dirent, err error) {
	fl.fs.log.CDebugf(ctx, "FL ReadDirAll")
	defer func() {
		fl.fs.reportErr(ctx, err)
	}()
	_, _, err = fl.fs.config.KBPKI().GetCurrentUserInfo(ctx)
	isLoggedIn := err == nil

	var favs []*libkbfs.Favorite
	if isLoggedIn {
		favs, err = fl.fs.config.KBFSOps().GetFavorites(ctx)
		if err != nil {
			return nil, err
		}
	}

	res = make([]fuse.Dirent, 0, len(favs))
	for _, fav := range favs {
		if fav.Public != fl.public {
			continue
		}
		res = append(res, fuse.Dirent{
			Type: fuse.DT_Dir,
			Name: fav.Name,
		})
	}
	return res, nil
}

var _ fs.NodeRemover = (*FolderList)(nil)

// Remove implements the fs.NodeRemover interface for FolderList.
func (fl *FolderList) Remove(ctx context.Context, req *fuse.RemoveRequest) (err error) {
	fl.fs.log.CDebugf(ctx, "FolderList Remove %s", req.Name)
	defer func() { fl.fs.reportErr(ctx, err) }()

	// TODO trying to delete non-canonical folder handles
	// could be skipped.
	//
	// TODO how to handle closing down the folderbranchops
	// object? Open files may still exist long after removing
	// the favorite.
	fld := keybase1.Folder{Name: req.Name, Private: !fl.public}
	err = fl.fs.config.KeybaseDaemon().FavoriteDelete(ctx, fld)
	return err
}

func isTlfNameNotCanonical(err error) bool {
	_, ok := err.(libkbfs.TlfNameNotCanonical)
	return ok
}
<commit_msg>Revert "folderlist: ignore "_"-prefixed names"<commit_after>package libfuse

import (
	"os"
	"strings"
	"sync"

	"bazil.org/fuse"
	"bazil.org/fuse/fs"
	keybase1 "github.com/keybase/client/go/protocol"
	"github.com/keybase/kbfs/libkbfs"
	"golang.org/x/net/context"
)

// FolderList is a node that can list all of the logged-in user's
// favorite top-level folders, on either a public or private basis.
type FolderList struct {
	fs *FS
	// only accept public folders
	public bool

	mu      sync.Mutex
	folders map[string]*TLF
}

var _ fs.Node = (*FolderList)(nil)

// Attr implements the fs.Node interface.
func (*FolderList) Attr(ctx context.Context, a *fuse.Attr) error {
	a.Mode = os.ModeDir | 0755
	return nil
}

var _ fs.NodeRequestLookuper = (*FolderList)(nil)

// Lookup implements the fs.NodeRequestLookuper interface.
func (fl *FolderList) Lookup(ctx context.Context, req *fuse.LookupRequest, resp *fuse.LookupResponse) (node fs.Node, err error) {
	fl.fs.log.CDebugf(ctx, "FL Lookup %s", req.Name)
	defer func() { fl.fs.reportErr(ctx, err) }()
	fl.mu.Lock()
	defer fl.mu.Unlock()

	specialNode := handleSpecialFile(req.Name, fl.fs, resp)
	if specialNode != nil {
		return specialNode, nil
	}

	if child, ok := fl.folders[req.Name]; ok {
		return child, nil
	}

	// Shortcut for dreaded extraneous OSX finder lookups
	if strings.HasPrefix(req.Name, "._") {
		return nil, fuse.ENOENT
	}

	_, err = libkbfs.ParseTlfHandle(
		ctx, fl.fs.config.KBPKI(), req.Name, fl.public)
	switch err := err.(type) {
	case nil:
		// No error.
		break

	case libkbfs.TlfNameNotCanonical:
		// Non-canonical name.
		n := &Alias{
			canon: err.NameToTry,
		}
		return n, nil

	case libkbfs.NoSuchNameError:
		// Invalid public TLF.
		return nil, fuse.ENOENT

	default:
		// Some other error.
		return nil, err
	}

	child := newTLF(fl, req.Name)
	fl.folders[req.Name] = child
	return child, nil
}

func (fl *FolderList) forgetFolder(folderName string) {
	fl.mu.Lock()
	defer fl.mu.Unlock()
	delete(fl.folders, folderName)
}

var _ fs.Handle = (*FolderList)(nil)

var _ fs.HandleReadDirAller = (*FolderList)(nil)

// ReadDirAll implements the ReadDirAll interface.
func (fl *FolderList) ReadDirAll(ctx context.Context) (res []fuse.Dirent, err error) {
	fl.fs.log.CDebugf(ctx, "FL ReadDirAll")
	defer func() {
		fl.fs.reportErr(ctx, err)
	}()
	_, _, err = fl.fs.config.KBPKI().GetCurrentUserInfo(ctx)
	isLoggedIn := err == nil

	var favs []*libkbfs.Favorite
	if isLoggedIn {
		favs, err = fl.fs.config.KBFSOps().GetFavorites(ctx)
		if err != nil {
			return nil, err
		}
	}

	res = make([]fuse.Dirent, 0, len(favs))
	for _, fav := range favs {
		if fav.Public != fl.public {
			continue
		}
		res = append(res, fuse.Dirent{
			Type: fuse.DT_Dir,
			Name: fav.Name,
		})
	}
	return res, nil
}

var _ fs.NodeRemover = (*FolderList)(nil)

// Remove implements the fs.NodeRemover interface for FolderList.
func (fl *FolderList) Remove(ctx context.Context, req *fuse.RemoveRequest) (err error) {
	fl.fs.log.CDebugf(ctx, "FolderList Remove %s", req.Name)
	defer func() { fl.fs.reportErr(ctx, err) }()

	// TODO trying to delete non-canonical folder handles
	// could be skipped.
	//
	// TODO how to handle closing down the folderbranchops
	// object? Open files may still exist long after removing
	// the favorite.
	fld := keybase1.Folder{Name: req.Name, Private: !fl.public}
	err = fl.fs.config.KeybaseDaemon().FavoriteDelete(ctx, fld)
	return err
}

func isTlfNameNotCanonical(err error) bool {
	_, ok := err.(libkbfs.TlfNameNotCanonical)
	return ok
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

type Request struct {
	a, b	int;
	replyc	chan int;
}

type BinOp (a, b int) int;

func Run(op *BinOp, request *Request) {
	result := op(request.a, request.b);
	request.replyc <- result;
}

func Server(op *BinOp, service chan *Request) {
	for {
		request := <-service;
		go Run(op, request);  // don't wait for it
	}
}

func StartServer(op *BinOp) chan *Request {
	req := new(chan *Request);
	go Server(op, req);
	return req;
}

func main() {
	adder := StartServer(func(a, b int) int { return a + b });
	const N = 100;
	var reqs [N]Request;
	for i := 0; i < N; i++ {
		req := &reqs[i];
		req.a = i;
		req.b = i + N;
		req.replyc = new(chan int);
		adder <- req;
	}
	for i := N-1; i >= 0; i-- {   // doesn't matter what order
		if <-reqs[i].replyc != N + 2*i {
			print("fail at ", i, "\n");
		}
	}
	print("done\n");
}
<commit_msg>Use make instead of new to allocate a channel.<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

type Request struct {
	a, b	int;
	replyc	chan int;
}

type BinOp (a, b int) int;

func Run(op *BinOp, request *Request) {
	result := op(request.a, request.b);
	request.replyc <- result;
}

func Server(op *BinOp, service chan *Request) {
	for {
		request := <-service;
		go Run(op, request);  // don't wait for it
	}
}

func StartServer(op *BinOp) chan *Request {
	req := make(chan *Request);
	go Server(op, req);
	return req;
}

func main() {
	adder := StartServer(func(a, b int) int { return a + b });
	const N = 100;
	var reqs [N]Request;
	for i := 0; i < N; i++ {
		req := &reqs[i];
		req.a = i;
		req.b = i + N;
		req.replyc = make(chan int);
		adder <- req;
	}
	for i := N-1; i >= 0; i-- {   // doesn't matter what order
		if <-reqs[i].replyc != N + 2*i {
			print("fail at ", i, "\n");
		}
	}
	print("done\n");
}
<|endoftext|>


<commit_before>package commands

import (
	"bytes"
	"fmt"
	"strings"

	"rollcage/core"

	"github.com/cactus/cobra"
)

func listCmdRun(cmd *cobra.Command, args []string) {
	const propertyList = "org.freebsd.iocage:host_hostuuid," +
		"org.freebsd.iocage:tag,org.freebsd.iocage:boot"
	outputHeaders := []string{"jid", "uuid", "tag", "boot", "state"}

	out := core.ZFSMust("list", "-H", "-o", propertyList, "-d", "1", core.GetJailsPath())

	lines := strings.Split(strings.TrimSpace(string(out)), "\n")
	wf := core.NewOutputWriter(outputHeaders, MachineOutput)
	for _, line := range lines {
		if strings.HasPrefix(line, "-") {
			continue
		}
		hostUUID := strings.SplitN(line, "\t", 2)[0]
		jid := core.JlsMust("-j", fmt.Sprintf("ioc-%s", hostUUID), "jid")
		state := "up"
		if string(jid) == "" {
			jid = []byte("-")
			state = "down"
		}
		fmt.Fprintf(wf, "%s\t%s\t%s\n", bytes.TrimSpace(jid), line, state)
	}
	wf.Flush()
}

func init() {
	RootCmd.AddCommand(&cobra.Command{
		Use:   "list",
		Short: "List all jails",
		Run:   listCmdRun,
	})
}
<commit_msg>speed up<commit_after>package commands

import (
	"fmt"
	"strings"

	"rollcage/core"

	"github.com/cactus/cobra"
)

func listCmdRun(cmd *cobra.Command, args []string) {
	const propertyList = "org.freebsd.iocage:host_hostuuid," +
		"org.freebsd.iocage:tag,org.freebsd.iocage:boot"
	outputHeaders := []string{"jid", "uuid", "tag", "boot", "state"}

	running := strings.TrimSpace(string(core.JlsMust("jid", "name")))
	jails := make(map[string]string, 0)
	for _, jinfo := range strings.Split(running, "\n") {
		jail := strings.Split(jinfo, " ")
		jid := strings.TrimSpace(jail[0])
		jname := strings.TrimSpace(jail[1])
		jails[jname] = jid
	}

	out := core.ZFSMust("list", "-H", "-o", propertyList, "-d", "1", core.GetJailsPath())

	lines := strings.Split(strings.TrimSpace(string(out)), "\n")
	wf := core.NewOutputWriter(outputHeaders, MachineOutput)
	for _, line := range lines {
		if strings.HasPrefix(line, "-") {
			continue
		}

		iocid := fmt.Sprintf("ioc-%s", strings.SplitN(line, "\t", 2)[0])
		state := "up"
		jid, ok := jails[iocid]
		if !ok {
			jid = "-"
			state = "down"
		}
		fmt.Fprintf(wf, "%s\t%s\t%s\n", jid, line, state)
	}
	wf.Flush()
}

func init() {
	RootCmd.AddCommand(&cobra.Command{
		Use:   "list",
		Short: "List all jails",
		Run:   listCmdRun,
	})
}
<|endoftext|>


<commit_before>package sharding

import storagemetadata "github.com/jacksontj/dataman/src/storage_node/metadata"

// Interface that all shard methods must adhere to
type ShardFunc func(storagemetadata.FieldType, interface{}, int) (int, error)

type ShardMethod string

const (
	Jump ShardMethod = "jump"
	Mod              = "mod"
)

func (s ShardMethod) Get() ShardFunc {
	switch s {
	case Jump:
		return JumpSelect
	case Mod:
		return ModSelect
	default:
		return nil
	}
}
<commit_msg>TODO addition<commit_after>package sharding

import storagemetadata "github.com/jacksontj/dataman/src/storage_node/metadata"

// TODO: we need to also allow for compound shard-keys, the current plan is to
// have a single unified mechanism to "concat" fields together to then pass to
// the shard functions Interface that all shard methods must adhere to
type ShardFunc func(storagemetadata.FieldType, interface{}, int) (int, error)

type ShardMethod string

const (
	Jump ShardMethod = "jump"
	Mod              = "mod"
)

func (s ShardMethod) Get() ShardFunc {
	switch s {
	case Jump:
		return JumpSelect
	case Mod:
		return ModSelect
	default:
		return nil
	}
}
<|endoftext|>


<commit_before>// continuous-deploy monitors pubsub events for the GCP Container Builder and
// pushes updated images when successful images are built.

package main

import (
	"context"
	"encoding/json"
	"flag"
	"fmt"
	"os"
	"regexp"
	"strings"

	"cloud.google.com/go/pubsub"
	"go.skia.org/infra/go/auth"
	"go.skia.org/infra/go/common"
	"go.skia.org/infra/go/exec"
	"go.skia.org/infra/go/gitauth"
	"go.skia.org/infra/go/metrics2"
	"go.skia.org/infra/go/sklog"
	cloudbuild "google.golang.org/api/cloudbuild/v1"
	"google.golang.org/api/option"
)

// flags
var (
	clusterConfig = flag.String("cluster_config", "", "Absolute filename of the config.json file.")
	local         = flag.Bool("local", false, "Running locally if true. As opposed to in production.")
	project       = flag.String("project", "skia-public", "The GCE project name.")
	promPort      = flag.String("prom_port", ":20000", "Metrics service address (e.g., ':10110')")
)

var (
	parseImageName *regexp.Regexp
)

func Init() {
	parseImageName = regexp.MustCompile("^gcr.io/" + *project + "/([^:]+).*$")
}

// baseImageName returns "fiddler" from "gcr.io/skia-public/fiddler:foo".
//
// If the image name doesn't start with "gcr.io" and the project name then "" is returned.
func baseImageName(s string) string {
	matches := parseImageName.FindStringSubmatch(s)
	if len(matches) != 2 {
		return ""
	} else {
		return matches[1]
	}
}

// imagesFromInfo parses the incoming PubSub Data 'b' as JSON and then returns
// the full image names of all the images that match 'shortImageNames'.
func imagesFromInfo(shortImageNames []string, buildInfo cloudbuild.Build) []string {
	imageNames := []string{}
	for _, im := range buildInfo.Results.Images {
		sklog.Infof("ImageName: %s", im.Name)
		// Is this one of the images we are pushing?
		for _, name := range shortImageNames {
			if baseImageName(im.Name) == name {
				imageNames = append(imageNames, im.Name)
				break
			}
		}
	}
	return imageNames
}

func main() {
	common.InitWithMust(
		"continuous-deploy",
		common.PrometheusOpt(promPort),
	)
	if len(flag.Args()) == 0 {
		sklog.Fatal("continuous-deploy must be passed in at least one package name to push.")
	}
	Init()
	sklog.Infof("Pushing to: %v", flag.Args())
	ctx := context.Background()
	ts, err := auth.NewDefaultTokenSource(*local, pubsub.ScopePubSub, "https://www.googleapis.com/auth/gerritcodereview")
	if err != nil {
		sklog.Fatal(err)
	}
	if !*local {
		_, err := gitauth.New(ts, "/tmp/git-cookie", true, "skia-continuous-deploy@skia-public.iam.gserviceaccount.com")
		if err != nil {
			sklog.Fatal(err)
		}
	}
	client, err := pubsub.NewClient(ctx, *project, option.WithTokenSource(ts))
	if err != nil {
		sklog.Fatal(err)
	}
	topic := client.Topic("cloud-builds")
	hostname, err := os.Hostname()
	if err != nil {
		sklog.Fatal(err)
	}
	subName := fmt.Sprintf("continuous-deploy-%s", hostname)
	sub := client.Subscription(subName)
	ok, err := sub.Exists(ctx)
	if err != nil {
		sklog.Fatalf("Failed checking subscription existence: %s", err)
	}
	if !ok {
		sub, err = client.CreateSubscription(ctx, subName, pubsub.SubscriptionConfig{
			Topic: topic,
		})
		if err != nil {
			sklog.Fatalf("Failed creating subscription: %s", err)
		}
	}
	pushk := "/usr/local/bin/pushk"
	if *local {
		pushk = "pushk"
	}
	shortImageNames := flag.Args()

	pubSubReceive := metrics2.NewLiveness("ci_pubsub_receive", nil)
	for {
		err := sub.Receive(ctx, func(ctx context.Context, msg *pubsub.Message) {
			msg.Ack()
			sklog.Infof("Status: %s", msg.Attributes["status"])

			var buildInfo cloudbuild.Build
			if err := json.Unmarshal(msg.Data, &buildInfo); err != nil {
				sklog.Errorf("Failed to decode: %s: %q", err, string(msg.Data))
				return
			}
			repoName := "--unknown--"
			if buildInfo.Source != nil && buildInfo.Source.RepoSource != nil {
				repoName = buildInfo.Source.RepoSource.RepoName
			}

			// Record build failures so we can alert on them.
			counter := metrics2.GetCounter("ci_build_failure", map[string]string{"trigger": repoName})
			if msg.Attributes["status"] == "FAILURE" {
				counter.Inc(1)
			} else if msg.Attributes["status"] == "SUCCESS" {
				counter.Reset()
			}

			if msg.Attributes["status"] != "SUCCESS" {
				return
			}
			imageNames := imagesFromInfo(shortImageNames, buildInfo)
			if err != nil {
				sklog.Error(err)
				return
			}
			if len(imageNames) == 0 {
				sklog.Infof("No images to push.")
				return
			}
			cmd := fmt.Sprintf("%s --logtostderr %s", pushk, strings.Join(imageNames, " "))
			if *clusterConfig != "" {
				cmd += fmt.Sprintf(" --config-file=%s", *clusterConfig)
			}
			sklog.Infof("About to execute: %q", cmd)
			output, err := exec.RunSimple(ctx, cmd)
			pushFailure := metrics2.GetCounter("ci_push_failure", map[string]string{"trigger": repoName})
			if err != nil {
				sklog.Errorf("Failed to run pushk: %s: %s", output, err)
				pushFailure.Inc(1)
				return
			} else {
				sklog.Info(output)
			}
			pushFailure.Reset()
			pubSubReceive.Reset()
			sklog.Info("Finished push")
		})
		if err != nil {
			sklog.Errorf("Failed receiving pubsub message: %s", err)
		}
	}
}
<commit_msg>[sk8s] Fix order of flags passed to pushk from continuous-deploy.<commit_after>// continuous-deploy monitors pubsub events for the GCP Container Builder and
// pushes updated images when successful images are built.

package main

import (
	"context"
	"encoding/json"
	"flag"
	"fmt"
	"os"
	"regexp"
	"strings"

	"cloud.google.com/go/pubsub"
	"go.skia.org/infra/go/auth"
	"go.skia.org/infra/go/common"
	"go.skia.org/infra/go/exec"
	"go.skia.org/infra/go/gitauth"
	"go.skia.org/infra/go/metrics2"
	"go.skia.org/infra/go/sklog"
	cloudbuild "google.golang.org/api/cloudbuild/v1"
	"google.golang.org/api/option"
)

// flags
var (
	clusterConfig = flag.String("cluster_config", "", "Absolute filename of the config.json file.")
	local         = flag.Bool("local", false, "Running locally if true. As opposed to in production.")
	project       = flag.String("project", "skia-public", "The GCE project name.")
	promPort      = flag.String("prom_port", ":20000", "Metrics service address (e.g., ':10110')")
)

var (
	parseImageName *regexp.Regexp
)

func Init() {
	parseImageName = regexp.MustCompile("^gcr.io/" + *project + "/([^:]+).*$")
}

// baseImageName returns "fiddler" from "gcr.io/skia-public/fiddler:foo".
//
// If the image name doesn't start with "gcr.io" and the project name then "" is returned.
func baseImageName(s string) string {
	matches := parseImageName.FindStringSubmatch(s)
	if len(matches) != 2 {
		return ""
	} else {
		return matches[1]
	}
}

// imagesFromInfo parses the incoming PubSub Data 'b' as JSON and then returns
// the full image names of all the images that match 'shortImageNames'.
func imagesFromInfo(shortImageNames []string, buildInfo cloudbuild.Build) []string {
	imageNames := []string{}
	for _, im := range buildInfo.Results.Images {
		sklog.Infof("ImageName: %s", im.Name)
		// Is this one of the images we are pushing?
		for _, name := range shortImageNames {
			if baseImageName(im.Name) == name {
				imageNames = append(imageNames, im.Name)
				break
			}
		}
	}
	return imageNames
}

func main() {
	common.InitWithMust(
		"continuous-deploy",
		common.PrometheusOpt(promPort),
	)
	if len(flag.Args()) == 0 {
		sklog.Fatal("continuous-deploy must be passed in at least one package name to push.")
	}
	Init()
	sklog.Infof("Pushing to: %v", flag.Args())
	ctx := context.Background()
	ts, err := auth.NewDefaultTokenSource(*local, pubsub.ScopePubSub, "https://www.googleapis.com/auth/gerritcodereview")
	if err != nil {
		sklog.Fatal(err)
	}
	if !*local {
		_, err := gitauth.New(ts, "/tmp/git-cookie", true, "skia-continuous-deploy@skia-public.iam.gserviceaccount.com")
		if err != nil {
			sklog.Fatal(err)
		}
	}
	client, err := pubsub.NewClient(ctx, *project, option.WithTokenSource(ts))
	if err != nil {
		sklog.Fatal(err)
	}
	topic := client.Topic("cloud-builds")
	hostname, err := os.Hostname()
	if err != nil {
		sklog.Fatal(err)
	}
	subName := fmt.Sprintf("continuous-deploy-%s", hostname)
	sub := client.Subscription(subName)
	ok, err := sub.Exists(ctx)
	if err != nil {
		sklog.Fatalf("Failed checking subscription existence: %s", err)
	}
	if !ok {
		sub, err = client.CreateSubscription(ctx, subName, pubsub.SubscriptionConfig{
			Topic: topic,
		})
		if err != nil {
			sklog.Fatalf("Failed creating subscription: %s", err)
		}
	}
	pushk := "/usr/local/bin/pushk"
	if *local {
		pushk = "pushk"
	}
	shortImageNames := flag.Args()

	pubSubReceive := metrics2.NewLiveness("ci_pubsub_receive", nil)
	for {
		err := sub.Receive(ctx, func(ctx context.Context, msg *pubsub.Message) {
			msg.Ack()
			sklog.Infof("Status: %s", msg.Attributes["status"])

			var buildInfo cloudbuild.Build
			if err := json.Unmarshal(msg.Data, &buildInfo); err != nil {
				sklog.Errorf("Failed to decode: %s: %q", err, string(msg.Data))
				return
			}
			repoName := "--unknown--"
			if buildInfo.Source != nil && buildInfo.Source.RepoSource != nil {
				repoName = buildInfo.Source.RepoSource.RepoName
			}

			// Record build failures so we can alert on them.
			counter := metrics2.GetCounter("ci_build_failure", map[string]string{"trigger": repoName})
			if msg.Attributes["status"] == "FAILURE" {
				counter.Inc(1)
			} else if msg.Attributes["status"] == "SUCCESS" {
				counter.Reset()
			}

			if msg.Attributes["status"] != "SUCCESS" {
				return
			}
			imageNames := imagesFromInfo(shortImageNames, buildInfo)
			if err != nil {
				sklog.Error(err)
				return
			}
			if len(imageNames) == 0 {
				sklog.Infof("No images to push.")
				return
			}
			cfgFile := ""
			if *clusterConfig != "" {
				cfgFile = fmt.Sprintf(" --config-file=%s ", *clusterConfig)
			}
			cmd := fmt.Sprintf("%s --logtostderr %s %s", pushk, cfgFile, strings.Join(imageNames, " "))
			sklog.Infof("About to execute: %q", cmd)
			output, err := exec.RunSimple(ctx, cmd)
			pushFailure := metrics2.GetCounter("ci_push_failure", map[string]string{"trigger": repoName})
			if err != nil {
				sklog.Errorf("Failed to run pushk: %s: %s", output, err)
				pushFailure.Inc(1)
				return
			} else {
				sklog.Info(output)
			}
			pushFailure.Reset()
			pubSubReceive.Reset()
			sklog.Info("Finished push")
		})
		if err != nil {
			sklog.Errorf("Failed receiving pubsub message: %s", err)
		}
	}
}
<|endoftext|>


<commit_before>package lru

import lru "github.com/golang/groupcache/lru"

// GroupcacheLRU is google groupcache imprementation.
type GroupcacheLRU struct {
	cache *lru.Cache
}

// Get wraps its function, it's compatible to transparent.BackendCache
func (d *GroupcacheLRU) Get(k interface{}) (interface{}, bool) {
	return d.cache.Get(k)
}

// Add wraps its function, it's compatible to transparent.BackendCache
func (d *GroupcacheLRU) Add(k interface{}, v interface{}) {
	d.cache.Add(k, v)
}

func GroupcacheLRUNew(maxEntries int) *GroupcacheLRU {
	return &GroupcacheLRU{
		cache: lru.New(maxEntries),
	}
}
<commit_msg>Remove unused file<commit_after><|endoftext|>


<commit_before>// Copyright 2015 Keybase, Inc. All rights reserved. Use of
// this source code is governed by the included BSD license.

package libkb

import (
	"bytes"
	"compress/gzip"
	"io"
	"io/ioutil"
	"mime/multipart"
	"os"
	"path/filepath"
	"sort"
	"strings"
	"time"

	"github.com/keybase/client/go/logger"
)

// Logs is the struct to specify the path of log files
type Logs struct {
	Desktop string
	Kbfs    string
	Service string
	Updater string
	Start   string
	Install string
	System  string
	Git     string
}

// LogSendContext for LogSend
type LogSendContext struct {
	Contextified
	Logs Logs
}

func addFile(mpart *multipart.Writer, param, filename, data string) error {
	if len(data) == 0 {
		return nil
	}

	part, err := mpart.CreateFormFile(param, filename)
	if err != nil {
		return err
	}
	gz := gzip.NewWriter(part)
	if _, err := gz.Write([]byte(data)); err != nil {
		return err
	}
	return gz.Close()
}

func (l *LogSendContext) post(status, feedback, kbfsLog, svcLog, desktopLog, updaterLog, startLog, installLog, systemLog, gitLog string) (string, error) {
	l.G().Log.Debug("sending status + logs to keybase")

	var body bytes.Buffer
	mpart := multipart.NewWriter(&body)

	if feedback != "" {
		mpart.WriteField("feedback", feedback)
	}

	if err := addFile(mpart, "status_gz", "status.gz", status); err != nil {
		return "", err
	}
	if err := addFile(mpart, "kbfs_log_gz", "kbfs_log.gz", kbfsLog); err != nil {
		return "", err
	}
	if err := addFile(mpart, "keybase_log_gz", "keybase_log.gz", svcLog); err != nil {
		return "", err
	}
	if err := addFile(mpart, "updater_log_gz", "updater_log.gz", updaterLog); err != nil {
		return "", err
	}
	if err := addFile(mpart, "gui_log_gz", "gui_log.gz", desktopLog); err != nil {
		return "", err
	}
	if err := addFile(mpart, "start_log_gz", "start_log.gz", startLog); err != nil {
		return "", err
	}
	if err := addFile(mpart, "install_log_gz", "install_log.gz", installLog); err != nil {
		return "", err
	}
	if err := addFile(mpart, "system_log_gz", "system_log.gz", systemLog); err != nil {
		return "", err
	}
	if err := addFile(mpart, "git_log_gz", "git_log.gz", gitLog); err != nil {
		return "", err
	}

	if err := mpart.Close(); err != nil {
		return "", err
	}

	l.G().Log.Debug("body size: %d\n", body.Len())

	arg := APIArg{
		Endpoint:    "logdump/send",
		SessionType: APISessionTypeOPTIONAL,
	}

	// Get the login session, if any
	l.G().LoginState().LoggedInLoad()

	resp, err := l.G().API.PostRaw(arg, mpart.FormDataContentType(), &body)
	if err != nil {
		l.G().Log.Debug("post error: %s", err)
		return "", err
	}

	id, err := resp.Body.AtKey("logdump_id").GetString()
	if err != nil {
		return "", err
	}

	return id, nil
}

// tail the logs that start with the stem `stem`, which are of type `which`.
// Get the most recent `numBytes` from the concatenation of the files.
func tail(log logger.Logger, which string, stem string, numBytes int) (ret string) {

	numFiles := 0

	log.Debug("+ tailing %s file with stem %q", which, stem)
	defer func() {
		log.Debug("- collected %d bytes from %d files", len(ret), numFiles)
	}()

	if len(stem) == 0 {
		log.Debug("| skipping %s logs (since no stem given)", which)
		return
	}

	lognames := listLogFiles(log, stem)
	var parts []string
	remaining := numBytes

	// Keep reading logs in reverse chronological order until we've read nothing
	// more, or we've filled up numBytes worth of buffer, or we didn't have to read
	// the whole file.
	for _, logname := range lognames {
		data, seeked := tailFile(log, which, logname, remaining)
		if len(data) == 0 {
			break
		}
		parts = append(parts, data)
		numFiles++
		remaining -= len(data)
		if remaining <= 0 || seeked {
			break
		}
	}

	// Reverse the array; took this one-line from StackOverflow answer
	for i, j := 0, len(parts)-1; i < j; i, j = i+1, j-1 {
		parts[i], parts[j] = parts[j], parts[i]
	}

	return strings.Join(parts, "")
}

type nameAndMTime struct {
	name  string
	mtime time.Time
}

type nameAndMTimes []nameAndMTime

func (a nameAndMTimes) Len() int           { return len(a) }
func (a nameAndMTimes) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a nameAndMTimes) Less(i, j int) bool { return a[i].mtime.After(a[j].mtime) }

// List logfiles that match the glob filename*, and return then in reverse chronological order.
// We'll need to read the first, and maybe the second
func listLogFiles(log logger.Logger, stem string) (ret []string) {
	stem = filepath.Clean(stem)
	dir := filepath.Dir(stem)
	base := filepath.Base(stem)
	files, err := ioutil.ReadDir(dir)

	defer func() {
		log.Debug("listLogFiles(%q) -> %v", stem, ret)
	}()

	// In the worst case, just return the stem in an array
	defret := []string{stem}

	if err != nil {
		log.Debug("failed to list directory %q: %s", dir, err)
		return defret
	}

	var tmp []nameAndMTime
	for _, d := range files {
		fullname := filepath.Clean(filepath.Join(dir, d.Name()))
		// Use the stat on the file (and not from the directory read)
		// since the latter doesn't work reliably on Windows
		finfo, err := os.Stat(fullname)
		if err != nil {
			log.Debug("Cannot stat %q: %s", fullname, err)
			continue
		}
		if strings.HasPrefix(d.Name(), base) {
			tmp = append(tmp, nameAndMTime{fullname, finfo.ModTime()})
		}
	}
	if len(tmp) == 0 {
		log.Debug("no files found matching \"%s*\"; falling back to default glob", stem)
		return defret
	}

	// Sort the files in reverse chronological mtime order. We should get the raw stem first.
	sort.Sort(nameAndMTimes(tmp))
	log.Debug("Sorted file list: %+v", tmp)

	for _, f := range tmp {
		ret = append(ret, f.name)
	}

	// If we didn't get the raw stem first, then we have a problem, so just use only the
	// raw stem and ignore the rest of our work.
	if stem != ret[0] {
		log.Debug("Expected to get %q first, but got %q instead! Falling back to one log only", stem, ret[0])
		return defret
	}
	return ret
}

// findFirstNewline first the first newline in the given byte array, and then returns the
// rest of the byte array. Should be safe to use on utf-8 strings.
func findFirstNewline(b []byte) []byte {
	index := bytes.IndexByte(b, '\n')
	if index < 0 || index == len(b)-1 {
		return nil
	}
	return b[(index + 1):]
}

// tailFile takes the last n bytes, but advances to the first newline. Returns the log (as a string)
// and a bool, indicating if we read the full log, or we had to advance into the log to find the newline.
func tailFile(log logger.Logger, which string, filename string, numBytes int) (ret string, seeked bool) {

	log.Debug("+ tailing %s log %q (%d bytes)", which, filename, numBytes)
	defer func() {
		log.Debug("- scanned %d bytes", len(ret))
	}()

	seeked = false
	if filename == "" {
		return ret, seeked
	}
	finfo, err := os.Stat(filename)
	if os.IsNotExist(err) {
		log.Debug("log %q doesn't exist", filename)
		return ret, seeked
	}
	f, err := os.Open(filename)
	if err != nil {
		log.Errorf("error opening log %q: %s", filename, err)
		return ret, seeked
	}
	defer f.Close()
	if finfo.Size() > int64(numBytes) {
		seeked = true
		_, err = f.Seek(int64(-numBytes), io.SeekEnd)
		if err != nil {
			log.Errorf("Can't seek log %q: %s", filename, err)
			return ret, seeked
		}
	}
	buf, err := ioutil.ReadAll(f)
	if err != nil {
		log.Errorf("Failure in reading file %q: %s", filename, err)
		return ret, seeked
	}
	if seeked {
		buf = findFirstNewline(buf)
	}
	return string(buf), seeked
}

// LogSend sends the the tails of log files to kb
func (l *LogSendContext) LogSend(statusJSON, feedback string, sendLogs bool, numBytes int) (string, error) {
	logs := l.Logs
	var kbfsLog string
	var svcLog string
	var desktopLog string
	var updaterLog string
	var startLog string
	var installLog string
	var systemLog string
	var gitLog string

	if sendLogs {
		kbfsLog = tail(l.G().Log, "kbfs", logs.Kbfs, numBytes)
		svcLog = tail(l.G().Log, "service", logs.Service, numBytes)
		desktopLog = tail(l.G().Log, "desktop", logs.Desktop, numBytes)
		updaterLog = tail(l.G().Log, "updater", logs.Updater, numBytes)
		startLog = tail(l.G().Log, "start", logs.Start, numBytes)
		installLog = tail(l.G().Log, "install", logs.Install, numBytes)
		systemLog = tail(l.G().Log, "system", logs.System, numBytes)
		gitLog = tail(l.G().Log, "git", logs.Git, numBytes)
	} else {
		kbfsLog = ""
		svcLog = ""
		desktopLog = ""
		updaterLog = ""
		startLog = ""
		installLog = ""
		systemLog = ""
		gitLog = ""
	}

	return l.post(statusJSON, feedback, kbfsLog, svcLog, desktopLog, updaterLog, startLog, installLog, systemLog, gitLog)
}
<commit_msg>make `keybase log send` read from journalctl when running under systemd<commit_after>// Copyright 2015 Keybase, Inc. All rights reserved. Use of
// this source code is governed by the included BSD license.

package libkb

import (
	"bytes"
	"compress/gzip"
	"fmt"
	"io"
	"io/ioutil"
	"mime/multipart"
	"os"
	"os/exec"
	"path/filepath"
	"sort"
	"strconv"
	"strings"
	"time"

	"github.com/keybase/client/go/logger"
)

// Logs is the struct to specify the path of log files
type Logs struct {
	Desktop string
	Kbfs    string
	Service string
	Updater string
	Start   string
	Install string
	System  string
	Git     string
}

// LogSendContext for LogSend
type LogSendContext struct {
	Contextified
	Logs Logs
}

func addFile(mpart *multipart.Writer, param, filename, data string) error {
	if len(data) == 0 {
		return nil
	}

	part, err := mpart.CreateFormFile(param, filename)
	if err != nil {
		return err
	}
	gz := gzip.NewWriter(part)
	if _, err := gz.Write([]byte(data)); err != nil {
		return err
	}
	return gz.Close()
}

func (l *LogSendContext) post(status, feedback, kbfsLog, svcLog, desktopLog, updaterLog, startLog, installLog, systemLog, gitLog string) (string, error) {
	l.G().Log.Debug("sending status + logs to keybase")

	var body bytes.Buffer
	mpart := multipart.NewWriter(&body)

	if feedback != "" {
		mpart.WriteField("feedback", feedback)
	}

	if err := addFile(mpart, "status_gz", "status.gz", status); err != nil {
		return "", err
	}
	if err := addFile(mpart, "kbfs_log_gz", "kbfs_log.gz", kbfsLog); err != nil {
		return "", err
	}
	if err := addFile(mpart, "keybase_log_gz", "keybase_log.gz", svcLog); err != nil {
		return "", err
	}
	if err := addFile(mpart, "updater_log_gz", "updater_log.gz", updaterLog); err != nil {
		return "", err
	}
	if err := addFile(mpart, "gui_log_gz", "gui_log.gz", desktopLog); err != nil {
		return "", err
	}
	if err := addFile(mpart, "start_log_gz", "start_log.gz", startLog); err != nil {
		return "", err
	}
	if err := addFile(mpart, "install_log_gz", "install_log.gz", installLog); err != nil {
		return "", err
	}
	if err := addFile(mpart, "system_log_gz", "system_log.gz", systemLog); err != nil {
		return "", err
	}
	if err := addFile(mpart, "git_log_gz", "git_log.gz", gitLog); err != nil {
		return "", err
	}

	if err := mpart.Close(); err != nil {
		return "", err
	}

	l.G().Log.Debug("body size: %d\n", body.Len())

	arg := APIArg{
		Endpoint:    "logdump/send",
		SessionType: APISessionTypeOPTIONAL,
	}

	// Get the login session, if any
	l.G().LoginState().LoggedInLoad()

	resp, err := l.G().API.PostRaw(arg, mpart.FormDataContentType(), &body)
	if err != nil {
		l.G().Log.Debug("post error: %s", err)
		return "", err
	}

	id, err := resp.Body.AtKey("logdump_id").GetString()
	if err != nil {
		return "", err
	}

	return id, nil
}

// tail the logs that start with the stem `stem`, which are of type `which`.
// Get the most recent `numBytes` from the concatenation of the files.
func tail(log logger.Logger, which string, stem string, numBytes int) (ret string) {

	numFiles := 0

	log.Debug("+ tailing %s file with stem %q", which, stem)
	defer func() {
		log.Debug("- collected %d bytes from %d files", len(ret), numFiles)
	}()

	if len(stem) == 0 {
		log.Debug("| skipping %s logs (since no stem given)", which)
		return
	}

	lognames := listLogFiles(log, stem)
	var parts []string
	remaining := numBytes

	// Keep reading logs in reverse chronological order until we've read nothing
	// more, or we've filled up numBytes worth of buffer, or we didn't have to read
	// the whole file.
	for _, logname := range lognames {
		data, seeked := tailFile(log, which, logname, remaining)
		if len(data) == 0 {
			break
		}
		parts = append(parts, data)
		numFiles++
		remaining -= len(data)
		if remaining <= 0 || seeked {
			break
		}
	}

	// Reverse the array; took this one-line from StackOverflow answer
	for i, j := 0, len(parts)-1; i < j; i, j = i+1, j-1 {
		parts[i], parts[j] = parts[j], parts[i]
	}

	return strings.Join(parts, "")
}

type nameAndMTime struct {
	name  string
	mtime time.Time
}

type nameAndMTimes []nameAndMTime

func (a nameAndMTimes) Len() int           { return len(a) }
func (a nameAndMTimes) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a nameAndMTimes) Less(i, j int) bool { return a[i].mtime.After(a[j].mtime) }

// List logfiles that match the glob filename*, and return then in reverse chronological order.
// We'll need to read the first, and maybe the second
func listLogFiles(log logger.Logger, stem string) (ret []string) {
	stem = filepath.Clean(stem)
	dir := filepath.Dir(stem)
	base := filepath.Base(stem)
	files, err := ioutil.ReadDir(dir)

	defer func() {
		log.Debug("listLogFiles(%q) -> %v", stem, ret)
	}()

	// In the worst case, just return the stem in an array
	defret := []string{stem}

	if err != nil {
		log.Debug("failed to list directory %q: %s", dir, err)
		return defret
	}

	var tmp []nameAndMTime
	for _, d := range files {
		fullname := filepath.Clean(filepath.Join(dir, d.Name()))
		// Use the stat on the file (and not from the directory read)
		// since the latter doesn't work reliably on Windows
		finfo, err := os.Stat(fullname)
		if err != nil {
			log.Debug("Cannot stat %q: %s", fullname, err)
			continue
		}
		if strings.HasPrefix(d.Name(), base) {
			tmp = append(tmp, nameAndMTime{fullname, finfo.ModTime()})
		}
	}
	if len(tmp) == 0 {
		log.Debug("no files found matching \"%s*\"; falling back to default glob", stem)
		return defret
	}

	// Sort the files in reverse chronological mtime order. We should get the raw stem first.
	sort.Sort(nameAndMTimes(tmp))
	log.Debug("Sorted file list: %+v", tmp)

	for _, f := range tmp {
		ret = append(ret, f.name)
	}

	// If we didn't get the raw stem first, then we have a problem, so just use only the
	// raw stem and ignore the rest of our work.
	if stem != ret[0] {
		log.Debug("Expected to get %q first, but got %q instead! Falling back to one log only", stem, ret[0])
		return defret
	}
	return ret
}

// findFirstNewline first the first newline in the given byte array, and then returns the
// rest of the byte array. Should be safe to use on utf-8 strings.
func findFirstNewline(b []byte) []byte {
	index := bytes.IndexByte(b, '\n')
	if index < 0 || index == len(b)-1 {
		return nil
	}
	return b[(index + 1):]
}

func appendError(collected []byte, err error) []byte {
	return append(collected, []byte(fmt.Sprintf("\nERROR READING LOGS:\n%#v\n", err))...)
}

func readBytesFromPipe(log logger.Logger, pipe io.ReadCloser, numBytes int) ([]byte, error) {
	buf := [4096]byte{}
	collected := []byte{}
	for {
		n, err := pipe.Read(buf[:])
		collected = append(collected, buf[:n]...)
		if err == io.EOF {
			_ = pipe.Close()
			return collected, nil
		}
		if err != nil {
			// For non-EOF errors, append them to the logs and return what we have.
			return appendError(collected, err), err
		}
		// If the output we've collected grows bigger than it's supposed to be,
		// just drop everything we've seen so far, add a big warning at the
		// top, and keep going. This avoids consuming all the memory on the
		// system if the logs are pathoLOGical, but still lets us see the tail
		// end of the logs, which is probably what we care about.
		if len(collected) > numBytes {
			msg := "WARNING: Lines in the journal are longer than expected. Truncating log."
			log.Warning(msg)
			collected = []byte("#####\n" + msg + "\n#####\n")
		}
	}
}

// If systemd is running, we look for logs in the systemd journal for our 3
// main services.
func tailSystemdJournal(log logger.Logger, which string, numBytes int) (ret string) {
	log.Debug("+ tailing journalctl for %s (%d bytes)", which, numBytes)
	defer func() {
		log.Debug("- scanned %d bytes", len(ret))
	}()

	// Journalctl doesn't provide a "last N bytes" flag directly, so instead we
	// use "last N lines". My own logs for all three services average out to
	// between 120 and 220 bytes per line. We'll use a slightly conservative
	// assumption of 300 bytes per line.
	numLines := numBytes / 300

	journalCmd := exec.Command("journalctl", "--user", "--unit", which, "--lines", strconv.Itoa(numLines))
	journalCmd.Stderr = os.Stderr
	stdout, err := journalCmd.StdoutPipe()
	if err != nil {
		msg := fmt.Sprintf("Failed to open a pipe for journalctl: %s", err)
		log.Errorf(msg)
		return msg
	}
	err = journalCmd.Start()
	if err != nil {
		msg := fmt.Sprintf("Failed to run journalctl: %s", err)
		log.Errorf(msg)
		return msg
	}

	// Once we start reading output, don't short-circuit on errors. Just log
	// them, and return whatever we got.
	output, err := readBytesFromPipe(log, stdout, numBytes)
	if err != nil {
		log.Errorf("Error reading from journalctl pipe: %s", err)
		output = appendError(output, err)
	}
	err = journalCmd.Wait()
	if err != nil {
		log.Errorf("Journalctl exited with an error: %s", err)
		output = appendError(output, err)
	}
	return string(output)
}

// tailFile takes the last n bytes, but advances to the first newline. Returns the log (as a string)
// and a bool, indicating if we read the full log, or we had to advance into the log to find the newline.
func tailFile(log logger.Logger, which string, filename string, numBytes int) (ret string, seeked bool) {

	log.Debug("+ tailing %s log %q (%d bytes)", which, filename, numBytes)
	defer func() {
		log.Debug("- scanned %d bytes", len(ret))
	}()

	seeked = false
	if filename == "" {
		return ret, seeked
	}
	finfo, err := os.Stat(filename)
	if os.IsNotExist(err) {
		log.Debug("log %q doesn't exist", filename)
		return ret, seeked
	}
	f, err := os.Open(filename)
	if err != nil {
		log.Errorf("error opening log %q: %s", filename, err)
		return ret, seeked
	}
	defer f.Close()
	if finfo.Size() > int64(numBytes) {
		seeked = true
		_, err = f.Seek(int64(-numBytes), io.SeekEnd)
		if err != nil {
			log.Errorf("Can't seek log %q: %s", filename, err)
			return ret, seeked
		}
	}
	buf, err := ioutil.ReadAll(f)
	if err != nil {
		log.Errorf("Failure in reading file %q: %s", filename, err)
		return ret, seeked
	}
	if seeked {
		buf = findFirstNewline(buf)
	}
	return string(buf), seeked
}

// LogSend sends the the tails of log files to kb
func (l *LogSendContext) LogSend(statusJSON, feedback string, sendLogs bool, numBytes int) (string, error) {
	logs := l.Logs
	var kbfsLog string
	var svcLog string
	var desktopLog string
	var updaterLog string
	var startLog string
	var installLog string
	var systemLog string
	var gitLog string

	if sendLogs {
		if l.G().Env.WantsSystemd() {
			svcLog = tailSystemdJournal(l.G().Log, "keybase.service", numBytes)
			kbfsLog = tailSystemdJournal(l.G().Log, "kbfs.service", numBytes)
			desktopLog = tailSystemdJournal(l.G().Log, "keybase.gui.service", numBytes)
		} else {
			svcLog = tail(l.G().Log, "service", logs.Service, numBytes)
			kbfsLog = tail(l.G().Log, "kbfs", logs.Kbfs, numBytes)
			desktopLog = tail(l.G().Log, "desktop", logs.Desktop, numBytes)
		}
		updaterLog = tail(l.G().Log, "updater", logs.Updater, numBytes)
		startLog = tail(l.G().Log, "start", logs.Start, numBytes)
		installLog = tail(l.G().Log, "install", logs.Install, numBytes)
		systemLog = tail(l.G().Log, "system", logs.System, numBytes)
		gitLog = tail(l.G().Log, "git", logs.Git, numBytes)
	} else {
		kbfsLog = ""
		svcLog = ""
		desktopLog = ""
		updaterLog = ""
		startLog = ""
		installLog = ""
		systemLog = ""
		gitLog = ""
	}

	return l.post(statusJSON, feedback, kbfsLog, svcLog, desktopLog, updaterLog, startLog, installLog, systemLog, gitLog)
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"flag"
	"fmt"
	"log"
	"os"
	"os/signal"
	"syscall"

	"github.com/Shopify/sarama"
)

var (
	fConfig = flag.String("config", "", "xml file configuring processing streams")

	logger = log.New(os.Stderr, "processor", log.LstdFlags)
)

func flagbad(f string, i ...interface{}) {
	fmt.Fprintf(os.Stderr, f, i...)
	flag.PrintDefaults()
	os.Exit(1)
}

func main() {
	flag.Parse()
	if *fConfig == "" {
		flagbad("-config is empty\n")
	}

	cfg, err := ParseConfig(*fConfig)
	if err != nil {
		logger.Panicf("Failed to parse config: %v", err)
	}

	config := sarama.NewConfig()
	config.Producer.RequiredAcks = sarama.WaitForAll
	config.ClientID = "kafkaproc.processor"

	client, err := sarama.NewClient(cfg.Kafka.Peer, config)
	if err != nil {
		logger.Panicf("Creating sarama client: %v", err)
	}
	defer client.Close()

	consumer, err := sarama.NewConsumerFromClient(client)
	if err != nil {
		logger.Panicf("Creating sarama consumer: %v", err)
	}
	defer consumer.Close()

	producer, err := sarama.NewSyncProducerFromClient(client)
	if err != nil {
		logger.Panicf("Creating sarama syncproducer: %v", err)
	}
	defer producer.Close()

	cl := NewConsumerList()

	for _, stream := range cfg.Stream {
		for _, partition := range stream.Partition {
			partitionconsumer, err := consumer.ConsumePartition(stream.TopicSrc, partition, sarama.OffsetNewest)
			if err != nil {
				logger.Panicf("Creating partition consumer: %v", err)
			}
			//defer partitionconsumer.Close()
			pc := &ProcConsumer{
				consumer:  partitionconsumer,
				producer:  producer,
				src:       stream.TopicSrc,
				dst:       stream.TopicDst,
				partition: partition,
				function:  stream.Function,
				killchan:  make(chan struct{}),
			}
			cl.Add(pc)
		}
	}

	sigchan := make(chan os.Signal)
	signal.Notify(sigchan, syscall.SIGINT)
	go func() {
		<-sigchan
		fmt.Printf("SIGINT\n")
		cl.Close()
	}()
	fmt.Printf("wait\n")
	<-cl.killchan
	fmt.Printf("waited\n")
}

type ProcConsumer struct {
	consumer  sarama.PartitionConsumer
	producer  sarama.SyncProducer
	src, dst  string
	partition int32
	function  []string
	killchan  chan struct{}
}

func (pc *ProcConsumer) Close() {
	pc.consumer.Close()
	<-pc.killchan
}

func (pc *ProcConsumer) Run() {
	defer close(pc.killchan)
	defer pc.consumer.Close()
	for msg := range pc.consumer.Messages() {
		//TODO: more general function
		newval := bytes.ToUpper(msg.Value)
		message := &sarama.ProducerMessage{
			Topic:     pc.dst,
			Partition: pc.partition,
			Value:     sarama.ByteEncoder(newval),
		}
		_, _, err := pc.producer.SendMessage(message)
		if err != nil {
			logger.Panicf("Failed to send message: %v", err)
		}
	}
}

type ConsumerList struct {
	pc       []*ProcConsumer
	killchan chan struct{}
}

func NewConsumerList() *ConsumerList {
	cl := new(ConsumerList)
	cl.killchan = make(chan struct{})
	return cl
}

func (cl *ConsumerList) Close() {
	defer close(cl.killchan)
	for i := range cl.pc {
		cl.pc[i].Close()
	}
}

func (cl *ConsumerList) Add(pc *ProcConsumer) {
	go pc.Run()
	cl.pc = append(cl.pc, pc)
}
<commit_msg>Safely close<commit_after>package main

import (
	"bytes"
	"flag"
	"fmt"
	"log"
	"os"
	"os/signal"
	"sync/atomic"
	"syscall"

	"github.com/Shopify/sarama"
)

var (
	fConfig = flag.String("config", "", "xml file configuring processing streams")

	logger = log.New(os.Stderr, "processor", log.LstdFlags)
)

func flagbad(f string, i ...interface{}) {
	fmt.Fprintf(os.Stderr, f, i...)
	flag.PrintDefaults()
	os.Exit(1)
}

func main() {
	flag.Parse()
	if *fConfig == "" {
		flagbad("-config is empty\n")
	}

	cfg, err := ParseConfig(*fConfig)
	if err != nil {
		logger.Panicf("Failed to parse config: %v", err)
	}

	config := sarama.NewConfig()
	config.Producer.RequiredAcks = sarama.WaitForAll
	config.ClientID = "kafkaproc.processor"

	client, err := sarama.NewClient(cfg.Kafka.Peer, config)
	if err != nil {
		logger.Panicf("Creating sarama client: %v", err)
	}
	defer client.Close()

	consumer, err := sarama.NewConsumerFromClient(client)
	if err != nil {
		logger.Panicf("Creating sarama consumer: %v", err)
	}
	defer consumer.Close()

	producer, err := sarama.NewSyncProducerFromClient(client)
	if err != nil {
		logger.Panicf("Creating sarama syncproducer: %v", err)
	}
	defer producer.Close()

	cl := NewConsumerList()

	for _, stream := range cfg.Stream {
		for _, partition := range stream.Partition {
			partitionconsumer, err := consumer.ConsumePartition(stream.TopicSrc, partition, sarama.OffsetNewest)
			if err != nil {
				logger.Panicf("Creating partition consumer: %v", err)
			}
			//defer partitionconsumer.Close()
			pc := &ProcConsumer{
				consumer:  partitionconsumer,
				producer:  producer,
				src:       stream.TopicSrc,
				dst:       stream.TopicDst,
				partition: partition,
				function:  stream.Function,
				killchan:  make(chan struct{}),
			}
			cl.Add(pc)
		}
	}

	sigchan := make(chan os.Signal)
	signal.Notify(sigchan, syscall.SIGINT)
	go func() {
		<-sigchan
		fmt.Printf("SIGINT\n")
		cl.Close()
	}()
	fmt.Printf("wait\n")
	<-cl.killchan
	fmt.Printf("waited\n")
}

type ProcConsumer struct {
	consumer  sarama.PartitionConsumer
	producer  sarama.SyncProducer
	src, dst  string
	partition int32
	function  []string
	killchan  chan struct{}
	isclosed  int32
}

func (pc *ProcConsumer) Close() {
	if atomic.CompareAndSwapInt32(&pc.isclosed, 0, 1) {
		pc.consumer.Close()
		<-pc.killchan
	}
}

func (pc *ProcConsumer) Run() {
	defer pc.Close()
	defer close(pc.killchan)
	for msg := range pc.consumer.Messages() {
		//TODO: more general function
		newval := bytes.ToUpper(msg.Value)
		message := &sarama.ProducerMessage{
			Topic:     pc.dst,
			Partition: pc.partition,
			Value:     sarama.ByteEncoder(newval),
		}
		_, _, err := pc.producer.SendMessage(message)
		if err != nil {
			logger.Panicf("Failed to send message: %v", err)
		}
	}
}

type ConsumerList struct {
	pc       []*ProcConsumer
	killchan chan struct{}
}

func NewConsumerList() *ConsumerList {
	cl := new(ConsumerList)
	cl.killchan = make(chan struct{})
	return cl
}

func (cl *ConsumerList) Close() {
	defer close(cl.killchan)
	for i := range cl.pc {
		cl.pc[i].Close()
	}
}

func (cl *ConsumerList) Add(pc *ProcConsumer) {
	go pc.Run()
	cl.pc = append(cl.pc, pc)
}
<|endoftext|>


<commit_before>package network

import (
	"strings"
	"time"

	"github.com/scrapli/scrapligo/driver/base"
)

// SendConfig send configuration string to the device.
func (d *Driver) SendConfig(c string, o ...base.SendOption) (*base.Response, error) {
	sc := strings.Split(c, "\n")
	m, err := d.SendConfigs(sc, o...)

	r := base.NewResponse(
		d.Host,
		d.Transport.BaseTransportArgs.Port,
		c,
		m.Responses[0].FailedWhenContains,
	)

	if err != nil {
		return r, err
	}

	individualResponses := make([]string, len(sc))
	for _, response := range m.Responses {
		individualResponses = append(individualResponses, response.Result)
	}

	r.StartTime = m.StartTime
	r.EndTime = time.Now()
	r.ElapsedTime = r.EndTime.Sub(r.StartTime).Seconds()
	r.Result = strings.Join(individualResponses, "\n")

	return r, nil
}
<commit_msg>dont build response until after checking sendconfigs err<commit_after>package network

import (
	"strings"
	"time"

	"github.com/scrapli/scrapligo/driver/base"
)

// SendConfig send configuration string to the device.
func (d *Driver) SendConfig(c string, o ...base.SendOption) (*base.Response, error) {
	sc := strings.Split(c, "\n")
	m, err := d.SendConfigs(sc, o...)

	if err != nil {
		return nil, err
	}

	r := base.NewResponse(
		d.Host,
		d.Transport.BaseTransportArgs.Port,
		c,
		m.Responses[0].FailedWhenContains,
	)

	individualResponses := make([]string, len(sc))
	for _, response := range m.Responses {
		individualResponses = append(individualResponses, response.Result)
	}

	r.StartTime = m.StartTime
	r.EndTime = time.Now()
	r.ElapsedTime = r.EndTime.Sub(r.StartTime).Seconds()
	r.Result = strings.Join(individualResponses, "\n")

	return r, nil
}
<|endoftext|>


<commit_before>// Copyright 2015 Keybase, Inc. All rights reserved. Use of
// this source code is governed by the included BSD license.

package service

import (
	"encoding/json"
	"fmt"
	"os"
	"path/filepath"
	"regexp"
	"runtime"
	"strings"

	"golang.org/x/net/context"

	"github.com/keybase/client/go/engine"
	"github.com/keybase/client/go/libkb"
	keybase1 "github.com/keybase/client/go/protocol/keybase1"
	"github.com/keybase/go-framed-msgpack-rpc/rpc"
	jsonw "github.com/keybase/go-jsonw"
)

type ConfigHandler struct {
	libkb.Contextified
	xp     rpc.Transporter
	svc    *Service
	connID libkb.ConnectionID
}

var _ keybase1.ConfigInterface = (*ConfigHandler)(nil)

func NewConfigHandler(xp rpc.Transporter, i libkb.ConnectionID, g *libkb.GlobalContext, svc *Service) *ConfigHandler {
	return &ConfigHandler{
		Contextified: libkb.NewContextified(g),
		xp:           xp,
		svc:          svc,
		connID:       i,
	}
}

func (h ConfigHandler) GetCurrentStatus(ctx context.Context, sessionID int) (res keybase1.GetCurrentStatusRes, err error) {
	var cs libkb.CurrentStatus
	if cs, err = libkb.GetCurrentStatus(ctx, h.G()); err == nil {
		res = cs.Export()
	}
	return
}

func getPlatformInfo() keybase1.PlatformInfo {
	return keybase1.PlatformInfo{
		Os:        runtime.GOOS,
		Arch:      runtime.GOARCH,
		GoVersion: runtime.Version(),
	}
}

func (h ConfigHandler) GetValue(_ context.Context, path string) (ret keybase1.ConfigValue, err error) {
	var i interface{}
	i, err = h.G().Env.GetConfig().GetInterfaceAtPath(path)
	if err != nil {
		return ret, err
	}
	if i == nil {
		ret.IsNull = true
	} else {
		switch v := i.(type) {
		case int:
			ret.I = &v
		case string:
			ret.S = &v
		case bool:
			ret.B = &v
		case float64:
			tmp := int(v)
			ret.I = &tmp
		default:
			var b []byte
			b, err = json.Marshal(v)
			if err == nil {
				tmp := string(b)
				ret.O = &tmp
			}
		}
	}
	return ret, err
}

func (h ConfigHandler) SetValue(_ context.Context, arg keybase1.SetValueArg) (err error) {
	w := h.G().Env.GetConfigWriter()
	if arg.Path == "users" {
		err = fmt.Errorf("The field 'users' cannot be edited for fear of config corruption")
		return err
	}
	switch {
	case arg.Value.IsNull:
		err = w.SetNullAtPath(arg.Path)
	case arg.Value.S != nil:
		err = w.SetStringAtPath(arg.Path, *arg.Value.S)
	case arg.Value.I != nil:
		err = w.SetIntAtPath(arg.Path, *arg.Value.I)
	case arg.Value.B != nil:
		err = w.SetBoolAtPath(arg.Path, *arg.Value.B)
	case arg.Value.O != nil:
		var jw *jsonw.Wrapper
		jw, err = jsonw.Unmarshal([]byte(*arg.Value.O))
		if err == nil {
			err = w.SetWrapperAtPath(arg.Path, jw)
		}
	default:
		err = fmt.Errorf("Bad type for setting a value")
	}
	if err == nil {
		h.G().ConfigReload()
	}
	return err
}

func (h ConfigHandler) ClearValue(_ context.Context, path string) error {
	h.G().Env.GetConfigWriter().DeleteAtPath(path)
	h.G().ConfigReload()
	return nil
}

func (h ConfigHandler) GetExtendedStatus(ctx context.Context, sessionID int) (res keybase1.ExtendedStatus, err error) {
	defer h.G().Trace("ConfigHandler::GetExtendedStatus", func() error { return err })()

	res.Standalone = h.G().Env.GetStandalone()
	res.LogDir = h.G().Env.GetLogDir()

	// Should work in standalone mode too
	if h.G().ConnectionManager != nil {
		res.Clients = h.G().ConnectionManager.ListAllLabeledConnections()
	}

	err = h.G().GetFullSelfer().WithSelf(ctx, func(me *libkb.User) error {
		device, err := me.GetComputedKeyFamily().GetCurrentDevice(h.G())
		if err != nil {
			h.G().Log.Debug("| GetCurrentDevice failed: %s", err)
			res.DeviceErr = &keybase1.LoadDeviceErr{Where: "ckf.GetCurrentDevice", Desc: err.Error()}
		} else {
			res.Device = device.ProtExport()
		}

		ss := h.G().SecretStore()
		if me != nil && ss != nil {
			s, err := ss.RetrieveSecret(me.GetNormalizedName())
			if err == nil && !s.IsNil() {
				res.StoredSecret = true
			}
		}
		return nil
	})
	if err != nil {
		h.G().Log.Debug("| could not load me user: %s", err)
		res.DeviceErr = &keybase1.LoadDeviceErr{Where: "libkb.LoadMe", Desc: err.Error()}
	}

	// cached device key status
	_, _, _, sk, ek := h.G().ActiveDevice.AllFields()
	res.DeviceSigKeyCached = sk != nil
	res.DeviceEncKeyCached = ek != nil

	h.G().LoginState().Account(func(a *libkb.Account) {
		res.PassphraseStreamCached = a.PassphraseStreamCache().ValidPassphraseStream()
		res.TsecCached = a.PassphraseStreamCache().ValidTsec()

		// cached paper key status
		if a.GetUnlockedPaperSigKey() != nil {
			res.PaperSigKeyCached = true
		}
		if a.GetUnlockedPaperEncKey() != nil {
			res.PaperEncKeyCached = true
		}

		res.SecretPromptSkip = a.SkipSecretPrompt()

		if a.LoginSession() != nil {
			res.Session = a.LoginSession().Status()
		}
	}, "ConfigHandler::GetExtendedStatus")

	current, all, err := h.G().GetAllUserNames()
	if err != nil {
		h.G().Log.Debug("| died in GetAllUseranmes()")
		return res, err
	}
	res.DefaultUsername = current.String()
	p := make([]string, len(all))
	for i, u := range all {
		p[i] = u.String()
	}
	res.ProvisionedUsernames = p
	res.PlatformInfo = getPlatformInfo()
	res.DefaultDeviceID = h.G().Env.GetDeviceID()
	res.RememberPassphrase = h.G().Env.RememberPassphrase()

	return res, nil
}

func (h ConfigHandler) GetConfig(_ context.Context, sessionID int) (keybase1.Config, error) {
	var c keybase1.Config

	c.ServerURI = h.G().Env.GetServerURI()
	c.RunMode = string(h.G().Env.GetRunMode())
	var err error
	c.SocketFile, err = h.G().Env.GetSocketBindFile()
	if err != nil {
		return c, err
	}

	gpg := h.G().GetGpgClient()
	canExec, err := gpg.CanExec()
	if err == nil {
		c.GpgExists = canExec
		c.GpgPath = gpg.Path()
	}

	c.Version = libkb.VersionString()
	c.VersionShort = libkb.Version

	var v []string
	libkb.VersionMessage(func(s string) {
		v = append(v, s)
	})
	c.VersionFull = strings.Join(v, "\n")

	dir, err := filepath.Abs(filepath.Dir(os.Args[0]))
	if err == nil {
		c.Path = dir
	} else {
		h.G().Log.Warning("Failed to get service path: %s", err)
	}

	realpath, err := libkb.CurrentBinaryRealpath()
	if err == nil {
		c.BinaryRealpath = realpath
	} else {
		h.G().Log.Warning("Failed to get service realpath: %s", err)
	}

	c.ConfigPath = h.G().Env.GetConfigFilename()
	c.Label = h.G().Env.GetLabel()
	if h.svc != nil {
		if h.svc.ForkType == keybase1.ForkType_AUTO {
			c.IsAutoForked = true
		}
		c.ForkType = h.svc.ForkType
	}

	return c, nil
}

func (h ConfigHandler) SetUserConfig(_ context.Context, arg keybase1.SetUserConfigArg) (err error) {
	eng := engine.NewUserConfigEngine(&engine.UserConfigEngineArg{
		Key:   arg.Key,
		Value: arg.Value,
	}, h.G())

	ctx := &engine.Context{}
	err = engine.RunEngine(eng, ctx)
	if err != nil {
		return err
	}
	return nil
}

func (h ConfigHandler) SetPath(_ context.Context, arg keybase1.SetPathArg) error {
	h.G().Log.Debug("SetPath calling mergeIntoPath(%s)", arg.Path)
	return mergeIntoPath(h.G(), arg.Path)
}

func mergeIntoPath(g *libkb.GlobalContext, p2 string) error {

	svcPath := os.Getenv("PATH")
	g.Log.Debug("mergeIntoPath: service path = %s", svcPath)
	g.Log.Debug("mergeIntoPath: merge path   = %s", p2)

	pathenv := filepath.SplitList(svcPath)
	pathset := make(map[string]bool)
	for _, p := range pathenv {
		pathset[p] = true
	}

	var clientAdditions []string
	for _, dir := range filepath.SplitList(p2) {
		if _, ok := pathset[dir]; ok {
			continue
		}
		clientAdditions = append(clientAdditions, dir)
	}

	pathenv = append(pathenv, clientAdditions...)
	combined := strings.Join(pathenv, string(os.PathListSeparator))

	if combined == svcPath {
		g.Log.Debug("No path changes needed")
		return nil
	}

	g.Log.Debug("mergeIntoPath: merged path = %s", combined)
	os.Setenv("PATH", combined)
	return nil
}

func (h ConfigHandler) HelloIAm(_ context.Context, arg keybase1.ClientDetails) error {
	tmp := fmt.Sprintf("%v", arg.Argv)
	re := regexp.MustCompile(`\b(chat|encrypt|git|accept-invite)\b`)
	if mtch := re.FindString(tmp); len(mtch) > 0 {
		arg.Argv = []string{arg.Argv[0], mtch, "(redacted)"}
	}
	h.G().Log.Debug("HelloIAm: %d - %v", h.connID, arg)
	return h.G().ConnectionManager.Label(h.connID, arg)
}

func (h ConfigHandler) CheckAPIServerOutOfDateWarning(_ context.Context) (keybase1.OutOfDateInfo, error) {
	return h.G().GetOutOfDateInfo(), nil
}

func (h ConfigHandler) WaitForClient(_ context.Context, arg keybase1.WaitForClientArg) (bool, error) {
	return h.G().ConnectionManager.WaitForClientType(arg.ClientType, arg.Timeout.Duration()), nil
}

func (h ConfigHandler) GetBootstrapStatus(ctx context.Context, sessionID int) (keybase1.BootstrapStatus, error) {
	eng := engine.NewBootstrap(h.G())
	ectx := &engine.Context{NetContext: ctx}
	if err := engine.RunEngine(eng, ectx); err != nil {
		return keybase1.BootstrapStatus{}, err
	}

	return eng.Status(), nil
}

func (h ConfigHandler) GetRememberPassphrase(ctx context.Context, sessionID int) (bool, error) {
	return h.G().Env.RememberPassphrase(), nil
}

func (h ConfigHandler) SetRememberPassphrase(ctx context.Context, arg keybase1.SetRememberPassphraseArg) error {
	remember, err := h.GetRememberPassphrase(ctx, arg.SessionID)
	if err != nil {
		return err
	}
	if remember == arg.Remember {
		h.G().Log.Debug("SetRememberPassphrase: no change necessary (remember = %v)", remember)
		return nil
	}

	// set the config variable
	w := h.G().Env.GetConfigWriter()
	if err := w.SetRememberPassphrase(arg.Remember); err != nil {
		return err
	}
	h.G().ConfigReload()

	// replace the secret store
	if err := h.G().ReplaceSecretStore(); err != nil {
		h.G().Log.Debug("error replacing secret store for SetRememberPassphrase(%v): %s", arg.Remember, err)
		return err
	}

	h.G().Log.Debug("SetRememberPassphrase(%v) success", arg.Remember)

	return nil
}
<commit_msg>redact wallet send and import (#11495)<commit_after>// Copyright 2015 Keybase, Inc. All rights reserved. Use of
// this source code is governed by the included BSD license.

package service

import (
	"encoding/json"
	"fmt"
	"os"
	"path/filepath"
	"regexp"
	"runtime"
	"strings"

	"golang.org/x/net/context"

	"github.com/keybase/client/go/engine"
	"github.com/keybase/client/go/libkb"
	keybase1 "github.com/keybase/client/go/protocol/keybase1"
	"github.com/keybase/go-framed-msgpack-rpc/rpc"
	jsonw "github.com/keybase/go-jsonw"
)

type ConfigHandler struct {
	libkb.Contextified
	xp     rpc.Transporter
	svc    *Service
	connID libkb.ConnectionID
}

var _ keybase1.ConfigInterface = (*ConfigHandler)(nil)

func NewConfigHandler(xp rpc.Transporter, i libkb.ConnectionID, g *libkb.GlobalContext, svc *Service) *ConfigHandler {
	return &ConfigHandler{
		Contextified: libkb.NewContextified(g),
		xp:           xp,
		svc:          svc,
		connID:       i,
	}
}

func (h ConfigHandler) GetCurrentStatus(ctx context.Context, sessionID int) (res keybase1.GetCurrentStatusRes, err error) {
	var cs libkb.CurrentStatus
	if cs, err = libkb.GetCurrentStatus(ctx, h.G()); err == nil {
		res = cs.Export()
	}
	return
}

func getPlatformInfo() keybase1.PlatformInfo {
	return keybase1.PlatformInfo{
		Os:        runtime.GOOS,
		Arch:      runtime.GOARCH,
		GoVersion: runtime.Version(),
	}
}

func (h ConfigHandler) GetValue(_ context.Context, path string) (ret keybase1.ConfigValue, err error) {
	var i interface{}
	i, err = h.G().Env.GetConfig().GetInterfaceAtPath(path)
	if err != nil {
		return ret, err
	}
	if i == nil {
		ret.IsNull = true
	} else {
		switch v := i.(type) {
		case int:
			ret.I = &v
		case string:
			ret.S = &v
		case bool:
			ret.B = &v
		case float64:
			tmp := int(v)
			ret.I = &tmp
		default:
			var b []byte
			b, err = json.Marshal(v)
			if err == nil {
				tmp := string(b)
				ret.O = &tmp
			}
		}
	}
	return ret, err
}

func (h ConfigHandler) SetValue(_ context.Context, arg keybase1.SetValueArg) (err error) {
	w := h.G().Env.GetConfigWriter()
	if arg.Path == "users" {
		err = fmt.Errorf("The field 'users' cannot be edited for fear of config corruption")
		return err
	}
	switch {
	case arg.Value.IsNull:
		err = w.SetNullAtPath(arg.Path)
	case arg.Value.S != nil:
		err = w.SetStringAtPath(arg.Path, *arg.Value.S)
	case arg.Value.I != nil:
		err = w.SetIntAtPath(arg.Path, *arg.Value.I)
	case arg.Value.B != nil:
		err = w.SetBoolAtPath(arg.Path, *arg.Value.B)
	case arg.Value.O != nil:
		var jw *jsonw.Wrapper
		jw, err = jsonw.Unmarshal([]byte(*arg.Value.O))
		if err == nil {
			err = w.SetWrapperAtPath(arg.Path, jw)
		}
	default:
		err = fmt.Errorf("Bad type for setting a value")
	}
	if err == nil {
		h.G().ConfigReload()
	}
	return err
}

func (h ConfigHandler) ClearValue(_ context.Context, path string) error {
	h.G().Env.GetConfigWriter().DeleteAtPath(path)
	h.G().ConfigReload()
	return nil
}

func (h ConfigHandler) GetExtendedStatus(ctx context.Context, sessionID int) (res keybase1.ExtendedStatus, err error) {
	defer h.G().Trace("ConfigHandler::GetExtendedStatus", func() error { return err })()

	res.Standalone = h.G().Env.GetStandalone()
	res.LogDir = h.G().Env.GetLogDir()

	// Should work in standalone mode too
	if h.G().ConnectionManager != nil {
		res.Clients = h.G().ConnectionManager.ListAllLabeledConnections()
	}

	err = h.G().GetFullSelfer().WithSelf(ctx, func(me *libkb.User) error {
		device, err := me.GetComputedKeyFamily().GetCurrentDevice(h.G())
		if err != nil {
			h.G().Log.Debug("| GetCurrentDevice failed: %s", err)
			res.DeviceErr = &keybase1.LoadDeviceErr{Where: "ckf.GetCurrentDevice", Desc: err.Error()}
		} else {
			res.Device = device.ProtExport()
		}

		ss := h.G().SecretStore()
		if me != nil && ss != nil {
			s, err := ss.RetrieveSecret(me.GetNormalizedName())
			if err == nil && !s.IsNil() {
				res.StoredSecret = true
			}
		}
		return nil
	})
	if err != nil {
		h.G().Log.Debug("| could not load me user: %s", err)
		res.DeviceErr = &keybase1.LoadDeviceErr{Where: "libkb.LoadMe", Desc: err.Error()}
	}

	// cached device key status
	_, _, _, sk, ek := h.G().ActiveDevice.AllFields()
	res.DeviceSigKeyCached = sk != nil
	res.DeviceEncKeyCached = ek != nil

	h.G().LoginState().Account(func(a *libkb.Account) {
		res.PassphraseStreamCached = a.PassphraseStreamCache().ValidPassphraseStream()
		res.TsecCached = a.PassphraseStreamCache().ValidTsec()

		// cached paper key status
		if a.GetUnlockedPaperSigKey() != nil {
			res.PaperSigKeyCached = true
		}
		if a.GetUnlockedPaperEncKey() != nil {
			res.PaperEncKeyCached = true
		}

		res.SecretPromptSkip = a.SkipSecretPrompt()

		if a.LoginSession() != nil {
			res.Session = a.LoginSession().Status()
		}
	}, "ConfigHandler::GetExtendedStatus")

	current, all, err := h.G().GetAllUserNames()
	if err != nil {
		h.G().Log.Debug("| died in GetAllUseranmes()")
		return res, err
	}
	res.DefaultUsername = current.String()
	p := make([]string, len(all))
	for i, u := range all {
		p[i] = u.String()
	}
	res.ProvisionedUsernames = p
	res.PlatformInfo = getPlatformInfo()
	res.DefaultDeviceID = h.G().Env.GetDeviceID()
	res.RememberPassphrase = h.G().Env.RememberPassphrase()

	return res, nil
}

func (h ConfigHandler) GetConfig(_ context.Context, sessionID int) (keybase1.Config, error) {
	var c keybase1.Config

	c.ServerURI = h.G().Env.GetServerURI()
	c.RunMode = string(h.G().Env.GetRunMode())
	var err error
	c.SocketFile, err = h.G().Env.GetSocketBindFile()
	if err != nil {
		return c, err
	}

	gpg := h.G().GetGpgClient()
	canExec, err := gpg.CanExec()
	if err == nil {
		c.GpgExists = canExec
		c.GpgPath = gpg.Path()
	}

	c.Version = libkb.VersionString()
	c.VersionShort = libkb.Version

	var v []string
	libkb.VersionMessage(func(s string) {
		v = append(v, s)
	})
	c.VersionFull = strings.Join(v, "\n")

	dir, err := filepath.Abs(filepath.Dir(os.Args[0]))
	if err == nil {
		c.Path = dir
	} else {
		h.G().Log.Warning("Failed to get service path: %s", err)
	}

	realpath, err := libkb.CurrentBinaryRealpath()
	if err == nil {
		c.BinaryRealpath = realpath
	} else {
		h.G().Log.Warning("Failed to get service realpath: %s", err)
	}

	c.ConfigPath = h.G().Env.GetConfigFilename()
	c.Label = h.G().Env.GetLabel()
	if h.svc != nil {
		if h.svc.ForkType == keybase1.ForkType_AUTO {
			c.IsAutoForked = true
		}
		c.ForkType = h.svc.ForkType
	}

	return c, nil
}

func (h ConfigHandler) SetUserConfig(_ context.Context, arg keybase1.SetUserConfigArg) (err error) {
	eng := engine.NewUserConfigEngine(&engine.UserConfigEngineArg{
		Key:   arg.Key,
		Value: arg.Value,
	}, h.G())

	ctx := &engine.Context{}
	err = engine.RunEngine(eng, ctx)
	if err != nil {
		return err
	}
	return nil
}

func (h ConfigHandler) SetPath(_ context.Context, arg keybase1.SetPathArg) error {
	h.G().Log.Debug("SetPath calling mergeIntoPath(%s)", arg.Path)
	return mergeIntoPath(h.G(), arg.Path)
}

func mergeIntoPath(g *libkb.GlobalContext, p2 string) error {

	svcPath := os.Getenv("PATH")
	g.Log.Debug("mergeIntoPath: service path = %s", svcPath)
	g.Log.Debug("mergeIntoPath: merge path   = %s", p2)

	pathenv := filepath.SplitList(svcPath)
	pathset := make(map[string]bool)
	for _, p := range pathenv {
		pathset[p] = true
	}

	var clientAdditions []string
	for _, dir := range filepath.SplitList(p2) {
		if _, ok := pathset[dir]; ok {
			continue
		}
		clientAdditions = append(clientAdditions, dir)
	}

	pathenv = append(pathenv, clientAdditions...)
	combined := strings.Join(pathenv, string(os.PathListSeparator))

	if combined == svcPath {
		g.Log.Debug("No path changes needed")
		return nil
	}

	g.Log.Debug("mergeIntoPath: merged path = %s", combined)
	os.Setenv("PATH", combined)
	return nil
}

func (h ConfigHandler) HelloIAm(_ context.Context, arg keybase1.ClientDetails) error {
	tmp := fmt.Sprintf("%v", arg.Argv)
	re := regexp.MustCompile(`\b(chat|encrypt|git|accept-invite|wallet\s+send|wallet\s+import)\b`)
	if mtch := re.FindString(tmp); len(mtch) > 0 {
		arg.Argv = []string{arg.Argv[0], mtch, "(redacted)"}
	}
	h.G().Log.Debug("HelloIAm: %d - %v", h.connID, arg)
	return h.G().ConnectionManager.Label(h.connID, arg)
}

func (h ConfigHandler) CheckAPIServerOutOfDateWarning(_ context.Context) (keybase1.OutOfDateInfo, error) {
	return h.G().GetOutOfDateInfo(), nil
}

func (h ConfigHandler) WaitForClient(_ context.Context, arg keybase1.WaitForClientArg) (bool, error) {
	return h.G().ConnectionManager.WaitForClientType(arg.ClientType, arg.Timeout.Duration()), nil
}

func (h ConfigHandler) GetBootstrapStatus(ctx context.Context, sessionID int) (keybase1.BootstrapStatus, error) {
	eng := engine.NewBootstrap(h.G())
	ectx := &engine.Context{NetContext: ctx}
	if err := engine.RunEngine(eng, ectx); err != nil {
		return keybase1.BootstrapStatus{}, err
	}

	return eng.Status(), nil
}

func (h ConfigHandler) GetRememberPassphrase(ctx context.Context, sessionID int) (bool, error) {
	return h.G().Env.RememberPassphrase(), nil
}

func (h ConfigHandler) SetRememberPassphrase(ctx context.Context, arg keybase1.SetRememberPassphraseArg) error {
	remember, err := h.GetRememberPassphrase(ctx, arg.SessionID)
	if err != nil {
		return err
	}
	if remember == arg.Remember {
		h.G().Log.Debug("SetRememberPassphrase: no change necessary (remember = %v)", remember)
		return nil
	}

	// set the config variable
	w := h.G().Env.GetConfigWriter()
	if err := w.SetRememberPassphrase(arg.Remember); err != nil {
		return err
	}
	h.G().ConfigReload()

	// replace the secret store
	if err := h.G().ReplaceSecretStore(); err != nil {
		h.G().Log.Debug("error replacing secret store for SetRememberPassphrase(%v): %s", arg.Remember, err)
		return err
	}

	h.G().Log.Debug("SetRememberPassphrase(%v) success", arg.Remember)

	return nil
}
<|endoftext|>