<commit_before>package oplog

import (
	"encoding/base64"
	"expvar"
	"fmt"
	"io"
	"net/http"
	"strings"
	"time"

	log "github.com/Sirupsen/logrus"
)

// SSEDaemon listens for events and send them to the oplog MongoDB capped collection
type SSEDaemon struct {
	s  *http.Server
	ol *OpLog
	// Password is the shared secret to connect to a password protected oplog.
	Password string
}

func NewSSEDaemon(addr string, ol *OpLog) *SSEDaemon {
	daemon := &SSEDaemon{nil, ol, ""}
	daemon.s = &http.Server{
		Addr:           addr,
		Handler:        daemon,
		MaxHeaderBytes: 1 << 20,
	}

	return daemon
}

// authenticate checks for HTTP basic authentication if an admin password is set.
func (daemon *SSEDaemon) authenticate(r *http.Request) bool {
	if daemon.Password == "" {
		return true
	}

	s := strings.SplitN(r.Header.Get("Authorization"), " ", 2)
	if len(s) != 2 || s[0] != "Basic" {
		return false
	}

	b, err := base64.StdEncoding.DecodeString(s[1])
	if err != nil {
		return false
	}
	pair := strings.SplitN(string(b), ":", 2)
	if len(pair) != 2 {
		return false
	}

	return daemon.Password == pair[1]
}

func (daemon *SSEDaemon) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	if r.Method != "GET" {
		w.WriteHeader(405)
		return
	}
	switch r.URL.Path {
	case "/status":
		daemon.Status(w, r)
	case "/ops", "/":
		daemon.Ops(w, r)
	default:
		w.WriteHeader(404)
	}
}

func (daemon *SSEDaemon) Status(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "application/json")
	fmt.Fprintf(w, "{\"status\":\"OK\"")
	expvar.Do(func(kv expvar.KeyValue) {
		fmt.Fprintf(w, ",%q:%s", kv.Key, kv.Value)
	})
	fmt.Fprintf(w, "}")
}

func (daemon *SSEDaemon) Ops(w http.ResponseWriter, r *http.Request) {
	log.Info("SSE connection started")

	if r.Header.Get("Accept") != "text/event-stream" {
		// Not an event stream request, return a 406 Not Acceptable HTTP error
		w.WriteHeader(406)
		return
	}

	if !daemon.authenticate(r) {
		w.WriteHeader(401)
		return
	}

	h := w.Header()
	h.Set("Server", fmt.Sprintf("oplog/%s", VERSION))
	h.Set("Content-Type", "text/event-stream; charset=utf-8")
	h.Set("Cache-Control", "no-cache, no-store, must-revalidate")
	h.Set("Connection", "close")
	h.Set("Access-Control-Allow-Origin", "*")

	var lastId LastId
	var err error
	if r.Header.Get("Last-Event-ID") == "" {
		// No last id provided, use the very last id of the events collection
		lastId, err = daemon.ol.LastId()
		if err != nil {
			log.Warnf("SSE can't get last id: %s", err)
			w.WriteHeader(503)
			return
		}
	} else {
		if lastId, err = NewLastId(r.Header.Get("Last-Event-ID")); err != nil {
			log.Warnf("SSE invalid last id: %s", err)
			w.WriteHeader(400)
			return
		}
		found, err := daemon.ol.HasId(lastId)
		if err != nil {
			log.Warnf("SSE can't check last id: %s", err)
			w.WriteHeader(503)
			return
		}
		if !found {
			log.Debug("SSE last id not found, falling back to replication id: ", lastId.String())
			// If the requested event id is not found, fallback to a replication id
			olid := lastId.(*OperationLastId)
			lastId = olid.Fallback()
		}
		// Backward compat, remove when all oplogc will be updated
		h.Set("Last-Event-ID", r.Header.Get("Last-Event-ID"))
	}

	if lastId != nil {
		log.Debug("SSE using last id: ", lastId.String())
	}

	types := []string{}
	if r.URL.Query().Get("types") != "" {
		types = strings.Split(r.URL.Query().Get("types"), ",")
	}
	parents := []string{}
	if r.URL.Query().Get("parents") != "" {
		parents = strings.Split(r.URL.Query().Get("parents"), ",")
	}
	filter := OpLogFilter{
		Types:   types,
		Parents: parents,
	}

	flusher := w.(http.Flusher)
	notifier := w.(http.CloseNotifier)
	ops := make(chan io.WriterTo)
	stop := make(chan bool)
	flusher.Flush()

	go daemon.ol.Tail(lastId, filter, ops, stop)
	daemon.ol.Stats.Clients.Add(1)
	daemon.ol.Stats.Connections.Add(1)
	defer daemon.ol.Stats.Clients.Add(-1)

	for {
		select {
		case <-notifier.CloseNotify():
			log.Info("SSE connection closed")
			stop <- true
			return
		case op := <-ops:
			log.Debug("SSE sending event")
			daemon.ol.Stats.EventsSent.Add(1)
			_, err := op.WriteTo(w)
			if err != nil {
				log.Warn("SSE write error: ", err)
				continue
			}
			flusher.Flush()
		case <-time.After(25 * time.Second):
			// Send "ping" data to prevent proxy/browsers from closing the connection
			// for inactivity
			log.Debug("SSE sending a keep alive ping")
			w.Write([]byte{':', '\n'})
			flusher.Flush()
		}
	}
}

func (daemon *SSEDaemon) Run() error {
	return daemon.s.ListenAndServe()
}
<commit_msg>Handle SSE write errors as fatal error and close the connection<commit_after>package oplog

import (
	"encoding/base64"
	"expvar"
	"fmt"
	"io"
	"net/http"
	"strings"
	"time"

	log "github.com/Sirupsen/logrus"
)

// SSEDaemon listens for events and send them to the oplog MongoDB capped collection
type SSEDaemon struct {
	s  *http.Server
	ol *OpLog
	// Password is the shared secret to connect to a password protected oplog.
	Password string
}

func NewSSEDaemon(addr string, ol *OpLog) *SSEDaemon {
	daemon := &SSEDaemon{nil, ol, ""}
	daemon.s = &http.Server{
		Addr:           addr,
		Handler:        daemon,
		MaxHeaderBytes: 1 << 20,
	}

	return daemon
}

// authenticate checks for HTTP basic authentication if an admin password is set.
func (daemon *SSEDaemon) authenticate(r *http.Request) bool {
	if daemon.Password == "" {
		return true
	}

	s := strings.SplitN(r.Header.Get("Authorization"), " ", 2)
	if len(s) != 2 || s[0] != "Basic" {
		return false
	}

	b, err := base64.StdEncoding.DecodeString(s[1])
	if err != nil {
		return false
	}
	pair := strings.SplitN(string(b), ":", 2)
	if len(pair) != 2 {
		return false
	}

	return daemon.Password == pair[1]
}

func (daemon *SSEDaemon) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	if r.Method != "GET" {
		w.WriteHeader(405)
		return
	}
	switch r.URL.Path {
	case "/status":
		daemon.Status(w, r)
	case "/ops", "/":
		daemon.Ops(w, r)
	default:
		w.WriteHeader(404)
	}
}

func (daemon *SSEDaemon) Status(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "application/json")
	fmt.Fprintf(w, "{\"status\":\"OK\"")
	expvar.Do(func(kv expvar.KeyValue) {
		fmt.Fprintf(w, ",%q:%s", kv.Key, kv.Value)
	})
	fmt.Fprintf(w, "}")
}

func (daemon *SSEDaemon) Ops(w http.ResponseWriter, r *http.Request) {
	log.Info("SSE connection started")

	if r.Header.Get("Accept") != "text/event-stream" {
		// Not an event stream request, return a 406 Not Acceptable HTTP error
		w.WriteHeader(406)
		return
	}

	if !daemon.authenticate(r) {
		w.WriteHeader(401)
		return
	}

	h := w.Header()
	h.Set("Server", fmt.Sprintf("oplog/%s", VERSION))
	h.Set("Content-Type", "text/event-stream; charset=utf-8")
	h.Set("Cache-Control", "no-cache, no-store, must-revalidate")
	h.Set("Connection", "close")
	h.Set("Access-Control-Allow-Origin", "*")

	var lastId LastId
	var err error
	if r.Header.Get("Last-Event-ID") == "" {
		// No last id provided, use the very last id of the events collection
		lastId, err = daemon.ol.LastId()
		if err != nil {
			log.Warnf("SSE can't get last id: %s", err)
			w.WriteHeader(503)
			return
		}
	} else {
		if lastId, err = NewLastId(r.Header.Get("Last-Event-ID")); err != nil {
			log.Warnf("SSE invalid last id: %s", err)
			w.WriteHeader(400)
			return
		}
		found, err := daemon.ol.HasId(lastId)
		if err != nil {
			log.Warnf("SSE can't check last id: %s", err)
			w.WriteHeader(503)
			return
		}
		if !found {
			log.Debug("SSE last id not found, falling back to replication id: ", lastId.String())
			// If the requested event id is not found, fallback to a replication id
			olid := lastId.(*OperationLastId)
			lastId = olid.Fallback()
		}
		// Backward compat, remove when all oplogc will be updated
		h.Set("Last-Event-ID", r.Header.Get("Last-Event-ID"))
	}

	if lastId != nil {
		log.Debug("SSE using last id: ", lastId.String())
	}

	types := []string{}
	if r.URL.Query().Get("types") != "" {
		types = strings.Split(r.URL.Query().Get("types"), ",")
	}
	parents := []string{}
	if r.URL.Query().Get("parents") != "" {
		parents = strings.Split(r.URL.Query().Get("parents"), ",")
	}
	filter := OpLogFilter{
		Types:   types,
		Parents: parents,
	}

	flusher := w.(http.Flusher)
	notifier := w.(http.CloseNotifier)
	ops := make(chan io.WriterTo)
	stop := make(chan bool)
	flusher.Flush()

	go daemon.ol.Tail(lastId, filter, ops, stop)
	defer func() {
		// Stop the oplog tailer
		stop <- true
	}()

	daemon.ol.Stats.Clients.Add(1)
	daemon.ol.Stats.Connections.Add(1)
	defer daemon.ol.Stats.Clients.Add(-1)

	for {
		select {
		case <-notifier.CloseNotify():
			log.Info("SSE connection closed")
			return
		case op := <-ops:
			log.Debug("SSE sending event")
			daemon.ol.Stats.EventsSent.Add(1)
			if _, err := op.WriteTo(w); err != nil {
				log.Warn("SSE write error: ", err)
				return
			}
			flusher.Flush()
		case <-time.After(25 * time.Second):
			// Send "ping" data to prevent proxy/browsers from closing the connection
			// for inactivity
			log.Debug("SSE sending a keep alive ping")
			if _, err := w.Write([]byte{':', '\n'}); err != nil {
				log.Warn("SSE write error: ", err)
				return
			}
			flusher.Flush()
		}
	}
}

func (daemon *SSEDaemon) Run() error {
	return daemon.s.ListenAndServe()
}
<|endoftext|>


<commit_before>// Package tap provides support for automated Test Anything Protocol ("TAP")
// tests in Go.  For example:
//
// 		package main
//
// 		import "github.com/mndrix/tap-go"
//
// 		func main() {
// 			t := tap.New()
// 			t.Header(2)
// 			t.Ok(true, "first test")
// 			t.Ok(true, "second test")
// 		}
//
// generates the following output
//
// 		TAP version 13
// 		1..2
// 		ok 1 - first test
// 		ok 2 - second test
package tap // import "github.com/mndrix/tap-go"

import "fmt"
import "testing/quick"

// T is a type to encapsulate test state.  Methods on this type generate TAP
// output.
type T struct {
	nextTestNumber int
}

// New creates a new Tap value
func New() *T {
	return &T{
		nextTestNumber: 1,
	}
}

// Header displays a TAP header including version number and expected
// number of tests to run.  For an unknown number of tests, set
// testCount to zero (in which case the plan is not written); this is
// useful with AutoPlan.
func (t *T) Header(testCount int) {
	fmt.Printf("TAP version 13\n")
	if testCount > 0 {
		fmt.Printf("1..%d\n", testCount)
	}
}

// Ok generates TAP output indicating whether a test passed or failed.
func (t *T) Ok(test bool, description string) {
	// did the test pass or not?
	ok := "ok"
	if !test {
		ok = "not ok"
	}

	fmt.Printf("%s %d - %s\n", ok, t.nextTestNumber, description)
	t.nextTestNumber++
}

// Check runs randomized tests against a function just as "testing/quick.Check"
// does.  Success or failure generate appropriate TAP output.
func (t *T) Check(function interface{}, description string) {
	err := quick.Check(function, nil)
	if err == nil {
		t.Ok(true, description)
		return
	}

	fmt.Printf("# %s\n", err)
	t.Ok(false, description)
}

// Count returns the number of tests completed so far.
func (t *T) Count() int {
	return t.nextTestNumber - 1
}

// AutoPlan generates a test plan based on the number of tests that were run.
func (t *T) AutoPlan() {
	fmt.Printf("1..%d\n", t.Count())
}
<commit_msg>Use our own printf method for output<commit_after>// Package tap provides support for automated Test Anything Protocol ("TAP")
// tests in Go.  For example:
//
// 		package main
//
// 		import "github.com/mndrix/tap-go"
//
// 		func main() {
// 			t := tap.New()
// 			t.Header(2)
// 			t.Ok(true, "first test")
// 			t.Ok(true, "second test")
// 		}
//
// generates the following output
//
// 		TAP version 13
// 		1..2
// 		ok 1 - first test
// 		ok 2 - second test
package tap // import "github.com/mndrix/tap-go"

import (
	"fmt"
	"os"
)
import "testing/quick"

// T is a type to encapsulate test state.  Methods on this type generate TAP
// output.
type T struct {
	nextTestNumber int
}

// New creates a new Tap value
func New() *T {
	return &T{
		nextTestNumber: 1,
	}
}

func (t *T) printf(format string, a ...interface{}) {
	fmt.Printf(format, a...)
}

// Header displays a TAP header including version number and expected
// number of tests to run.  For an unknown number of tests, set
// testCount to zero (in which case the plan is not written); this is
// useful with AutoPlan.
func (t *T) Header(testCount int) {
	t.printf("TAP version 13\n")
	if testCount > 0 {
		t.printf("1..%d\n", testCount)
	}
}

// Ok generates TAP output indicating whether a test passed or failed.
func (t *T) Ok(test bool, description string) {
	// did the test pass or not?
	ok := "ok"
	if !test {
		ok = "not ok"
	}

	t.printf("%s %d - %s\n", ok, t.nextTestNumber, description)
	t.nextTestNumber++
}

// Check runs randomized tests against a function just as "testing/quick.Check"
// does.  Success or failure generate appropriate TAP output.
func (t *T) Check(function interface{}, description string) {
	err := quick.Check(function, nil)
	if err == nil {
		t.Ok(true, description)
		return
	}

	t.printf("# %s\n", err)
	t.Ok(false, description)
}

// Count returns the number of tests completed so far.
func (t *T) Count() int {
	return t.nextTestNumber - 1
}

// AutoPlan generates a test plan based on the number of tests that were run.
func (t *T) AutoPlan() {
	t.printf("1..%d\n", t.Count())
}
<|endoftext|>


<commit_before>package archiver

import (
	"archive/tar"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"strings"
)

// Tar is for Tar format
var Tar tarFormat

func init() {
	RegisterFormat("Tar", Tar)
}

type tarFormat struct{}

func (tarFormat) Match(filename string) bool {
	// TODO: read file header to identify the format
	return strings.HasSuffix(strings.ToLower(filename), ".tar")
}

// Make creates a .tar file at tarPath containing the
// contents of files listed in filePaths. File paths can
// be those of regular files or directories. Regular
// files are stored at the 'root' of the archive, and
// directories are recursively added.
func (tarFormat) Make(tarPath string, filePaths []string) error {
	out, err := os.Create(tarPath)
	if err != nil {
		return fmt.Errorf("error creating %s: %v", tarPath, err)
	}
	defer out.Close()

	tarWriter := tar.NewWriter(out)
	defer tarWriter.Close()

	return tarball(filePaths, tarWriter, tarPath)
}

// tarball writes all files listed in filePaths into tarWriter, which is
// writing into a file located at dest.
func tarball(filePaths []string, tarWriter *tar.Writer, dest string) error {
	for _, fpath := range filePaths {
		err := tarFile(tarWriter, fpath, dest)
		if err != nil {
			return err
		}
	}
	return nil
}

// tarFile writes the file at source into tarWriter. It does so
// recursively for directories.
func tarFile(tarWriter *tar.Writer, source, dest string) error {
	sourceInfo, err := os.Stat(source)
	if err != nil {
		return fmt.Errorf("%s: stat: %v", source, err)
	}

	var baseDir string
	if sourceInfo.IsDir() {
		baseDir = filepath.Base(source)
	}

	return filepath.Walk(source, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return fmt.Errorf("error walking to %s: %v", path, err)
		}

		header, err := tar.FileInfoHeader(info, path)
		if err != nil {
			return fmt.Errorf("%s: making header: %v", path, err)
		}

		if baseDir != "" {
			header.Name = filepath.Join(baseDir, strings.TrimPrefix(path, source))
		}

		if header.Name == dest {
			// our new tar file is inside the directory being archived; skip it
			return nil
		}

		if info.IsDir() {
			header.Name += "/"
		}

		err = tarWriter.WriteHeader(header)
		if err != nil {
			return fmt.Errorf("%s: writing header: %v", path, err)
		}

		if info.IsDir() {
			return nil
		}

		if header.Typeflag == tar.TypeReg {
			file, err := os.Open(path)
			if err != nil {
				return fmt.Errorf("%s: open: %v", path, err)
			}
			defer file.Close()

			_, err = io.CopyN(tarWriter, file, info.Size())
			if err != nil && err != io.EOF {
				return fmt.Errorf("%s: copying contents: %v", path, err)
			}
		}
		return nil
	})
}

// Open untars source and puts the contents into destination.
func (tarFormat) Open(source, destination string) error {
	f, err := os.Open(source)
	if err != nil {
		return fmt.Errorf("%s: failed to open archive: %v", source, err)
	}
	defer f.Close()

	return untar(tar.NewReader(f), destination)
}

// untar un-tarballs the contents of tr into destination.
func untar(tr *tar.Reader, destination string) error {
	for {
		header, err := tr.Next()
		if err == io.EOF {
			break
		} else if err != nil {
			return err
		}

		if err := untarFile(tr, header, destination); err != nil {
			return err
		}
	}
	return nil
}

// untarFile untars a single file from tr with header header into destination.
func untarFile(tr *tar.Reader, header *tar.Header, destination string) error {
	switch header.Typeflag {
	case tar.TypeDir:
		return mkdir(filepath.Join(destination, header.Name))
	case tar.TypeReg, tar.TypeRegA:
		return writeNewFile(filepath.Join(destination, header.Name), tr, header.FileInfo().Mode())
	case tar.TypeSymlink:
		return writeNewSymbolicLink(filepath.Join(destination, header.Name), header.Linkname)
	case tar.TypeLink:
		return writeNewHardLink(filepath.Join(destination, header.Name), filepath.Join(destination, header.Linkname))
	default:
		return fmt.Errorf("%s: unknown type flag: %c", header.Name, header.Typeflag)
	}
}
<commit_msg>Identify .tar file by reading file header<commit_after>package archiver

import (
	"archive/tar"
	"bytes"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"strconv"
	"strings"
)

// Tar is for Tar format
var Tar tarFormat

func init() {
	RegisterFormat("Tar", Tar)
}

type tarFormat struct{}

func (tarFormat) Match(filename string) bool {
	return strings.HasSuffix(strings.ToLower(filename), ".tar") || isTar(filename)
}

const tarBlockSize int = 512

// isTar checks the file has the Tar format header by reading its beginning
// block.
func isTar(tarPath string) bool {
	f, err := os.Open(tarPath)
	if err != nil {
		return false
	}
	defer f.Close()

	buf := make([]byte, tarBlockSize)
	if _, err = io.ReadFull(f, buf); err != nil {
		return false
	}

	return hasTarHeader(buf)

}

// hasTarHeader checks passed bytes has a valid tar header or not. buf must
// contain at least 512 bytes and if not, it always returns false.
func hasTarHeader(buf []byte) bool {
	if len(buf) < tarBlockSize {
		return false
	}

	b := buf[148:156]
	b = bytes.Trim(b, " \x00") // clean up all spaces and null bytes
	if len(b) == 0 {
		return false // unknown format
	}
	hdrSum, err := strconv.ParseUint(string(b), 8, 64)
	if err != nil {
		return false
	}

	// According to the go official archive/tar, Sun tar uses signed byte
	// values so this calcs both signed and unsigned
	var usum uint64
	var sum int64
	for i, c := range buf {
		if 148 <= i && i < 156 {
			c = ' ' // checksum field itself is counted as branks
		}
		usum += uint64(uint8(c))
		sum += int64(int8(c))
	}

	if hdrSum != usum && int64(hdrSum) != sum {
		return false // invalid checksum
	}
	// checksum OK

	magic := buf[257:265]
	if bytes.Equal(magic, []byte("ustar  \x00")) {
		return true // GNU tar
	} else if bytes.Equal(magic[:6], []byte("ustar\x00")) {
		return true // POSIX tar
	}
	return true // Old style tar
}

// Make creates a .tar file at tarPath containing the
// contents of files listed in filePaths. File paths can
// be those of regular files or directories. Regular
// files are stored at the 'root' of the archive, and
// directories are recursively added.
func (tarFormat) Make(tarPath string, filePaths []string) error {
	out, err := os.Create(tarPath)
	if err != nil {
		return fmt.Errorf("error creating %s: %v", tarPath, err)
	}
	defer out.Close()

	tarWriter := tar.NewWriter(out)
	defer tarWriter.Close()

	return tarball(filePaths, tarWriter, tarPath)
}

// tarball writes all files listed in filePaths into tarWriter, which is
// writing into a file located at dest.
func tarball(filePaths []string, tarWriter *tar.Writer, dest string) error {
	for _, fpath := range filePaths {
		err := tarFile(tarWriter, fpath, dest)
		if err != nil {
			return err
		}
	}
	return nil
}

// tarFile writes the file at source into tarWriter. It does so
// recursively for directories.
func tarFile(tarWriter *tar.Writer, source, dest string) error {
	sourceInfo, err := os.Stat(source)
	if err != nil {
		return fmt.Errorf("%s: stat: %v", source, err)
	}

	var baseDir string
	if sourceInfo.IsDir() {
		baseDir = filepath.Base(source)
	}

	return filepath.Walk(source, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return fmt.Errorf("error walking to %s: %v", path, err)
		}

		header, err := tar.FileInfoHeader(info, path)
		if err != nil {
			return fmt.Errorf("%s: making header: %v", path, err)
		}

		if baseDir != "" {
			header.Name = filepath.Join(baseDir, strings.TrimPrefix(path, source))
		}

		if header.Name == dest {
			// our new tar file is inside the directory being archived; skip it
			return nil
		}

		if info.IsDir() {
			header.Name += "/"
		}

		err = tarWriter.WriteHeader(header)
		if err != nil {
			return fmt.Errorf("%s: writing header: %v", path, err)
		}

		if info.IsDir() {
			return nil
		}

		if header.Typeflag == tar.TypeReg {
			file, err := os.Open(path)
			if err != nil {
				return fmt.Errorf("%s: open: %v", path, err)
			}
			defer file.Close()

			_, err = io.CopyN(tarWriter, file, info.Size())
			if err != nil && err != io.EOF {
				return fmt.Errorf("%s: copying contents: %v", path, err)
			}
		}
		return nil
	})
}

// Open untars source and puts the contents into destination.
func (tarFormat) Open(source, destination string) error {
	f, err := os.Open(source)
	if err != nil {
		return fmt.Errorf("%s: failed to open archive: %v", source, err)
	}
	defer f.Close()

	return untar(tar.NewReader(f), destination)
}

// untar un-tarballs the contents of tr into destination.
func untar(tr *tar.Reader, destination string) error {
	for {
		header, err := tr.Next()
		if err == io.EOF {
			break
		} else if err != nil {
			return err
		}

		if err := untarFile(tr, header, destination); err != nil {
			return err
		}
	}
	return nil
}

// untarFile untars a single file from tr with header header into destination.
func untarFile(tr *tar.Reader, header *tar.Header, destination string) error {
	switch header.Typeflag {
	case tar.TypeDir:
		return mkdir(filepath.Join(destination, header.Name))
	case tar.TypeReg, tar.TypeRegA:
		return writeNewFile(filepath.Join(destination, header.Name), tr, header.FileInfo().Mode())
	case tar.TypeSymlink:
		return writeNewSymbolicLink(filepath.Join(destination, header.Name), header.Linkname)
	case tar.TypeLink:
		return writeNewHardLink(filepath.Join(destination, header.Name), filepath.Join(destination, header.Linkname))
	default:
		return fmt.Errorf("%s: unknown type flag: %c", header.Name, header.Typeflag)
	}
}
<|endoftext|>


<commit_before>package crypto

import (
	"crypto/ed25519"
	"crypto/rand"
	"encoding/json"
	"fmt"
	"regexp"
	"strings"

	"github.com/multiformats/go-multibase"
	"github.com/multiformats/go-multicodec"
	"github.com/teserakt-io/golang-ed25519/extra25519"
	"github.com/tyler-smith/go-bip39"
	"golang.org/x/crypto/curve25519"

	"nimona.io/pkg/did"
	"nimona.io/pkg/multiheader"
)

// https://blog.filippo.io/using-ed25519-keys-for-encryption
// https://libsodium.gitbook.io/doc/advanced/ed25519-curve25519
// http://moderncrypto.org/mail-archive/curves/2014/000205.html
// https://signal.org/docs/specifications/xeddsa
// https://libsodium.gitbook.io/doc/advanced/ed25519-curve25519

// we are opting for ed to x at this point based on FiloSottile's age spec

type (
	KeyAlgorithm multicodec.Code
)

const (
	Ed25519Private KeyAlgorithm = KeyAlgorithm(multicodec.Ed25519Priv)
	Ed25519Public  KeyAlgorithm = KeyAlgorithm(multicodec.Ed25519Pub)
)

type (
	PublicKey struct {
		Algorithm KeyAlgorithm
		RawKey    ed25519.PublicKey // TODO use crypto.PublicKey
	}
	PrivateKey struct {
		Algorithm KeyAlgorithm
		RawKey    ed25519.PrivateKey // TODO use crypto.PrivateKey
	}
)

var (
	EmptyPublicKey  = PublicKey{}
	EmptyPrivateKey = PrivateKey{}
)

func (k PublicKey) String() string {
	if k.IsEmpty() {
		return ""
	}
	b := multiheader.Encode(multicodec.Code(k.Algorithm), k.RawKey)
	// nolint: errcheck // cannot error
	s, _ := multibase.Encode(multibase.Base58BTC, b)
	return s
}

func (k PublicKey) DID() did.DID {
	return did.DID{
		Method:   did.MethodKey,
		Identity: k.String(),
	}
}

func (k PublicKey) IsEmpty() bool {
	return k.RawKey == nil
}

func (k PublicKey) MarshalString() (string, error) {
	return k.String(), nil
}

func (k PublicKey) MarshalJSON() ([]byte, error) {
	return json.Marshal(k.String())
}

// UnmarshalText implements encoding.TextUnmarshaler mainly for use
// by envconfig.
func (k *PublicKey) UnmarshalText(b []byte) error {
	return k.UnmarshalString(string(b))
}

func (k *PublicKey) UnmarshalString(s string) error {
	_, b, err := multibase.Decode(s)
	if err != nil {
		return fmt.Errorf("unable to decode multibase, %w", err)
	}

	c, r, err := multiheader.Decode(b)
	if err != nil {
		return fmt.Errorf("unable to decode multiheader, %w", err)
	}

	if c != multicodec.Ed25519Pub {
		return ErrUnsupportedKeyAlgorithm
	}

	k.Algorithm = Ed25519Public
	k.RawKey = ed25519.PublicKey(r)

	return nil
}

func (k *PublicKey) UnmarshalJSON(s []byte) error {
	v := ""
	if err := json.Unmarshal(s, &v); err != nil {
		return err
	}
	return k.UnmarshalString(v)
}

func (k PrivateKey) IsEmpty() bool {
	return k.RawKey == nil
}

func (k PrivateKey) String() string {
	b := multiheader.Encode(multicodec.Code(k.Algorithm), k.RawKey)
	// nolint: errcheck // cannot error
	s, _ := multibase.Encode(multibase.Base58BTC, b)
	return s
}

func (k PrivateKey) Seed() []byte {
	return k.RawKey.Seed()
}

func (k PrivateKey) BIP39() string {
	m, _ := bip39.NewMnemonic(k.Seed())
	return m
}

func (k PrivateKey) MarshalString() (string, error) {
	return k.String(), nil
}

func (k PrivateKey) MarshalJSON() ([]byte, error) {
	return json.Marshal(k.String())
}

// UnmarshalText implements encoding.TextUnmarshaler mainly for use
// by envconfig.
func (k *PrivateKey) UnmarshalText(b []byte) error {
	return k.UnmarshalString(string(b))
}

func (k *PrivateKey) UnmarshalString(s string) error {
	_, b, err := multibase.Decode(s)
	if err != nil {
		return fmt.Errorf("unable to decode multibase, %w", err)
	}

	c, r, err := multiheader.Decode(b)
	if err != nil {
		return fmt.Errorf("unable to decode multiheader, %w", err)
	}

	if c != multicodec.Ed25519Priv {
		return ErrUnsupportedKeyAlgorithm
	}

	k.Algorithm = Ed25519Private
	k.RawKey = ed25519.PrivateKey(r)

	return nil
}

func (k *PrivateKey) UnmarshalJSON(s []byte) error {
	v := ""
	if err := json.Unmarshal(s, &v); err != nil {
		return err
	}
	return k.UnmarshalString(v)
}

func (k PrivateKey) PublicKey() PublicKey {
	return PublicKey{
		Algorithm: Ed25519Public,
		RawKey:    k.RawKey.Public().(ed25519.PublicKey),
	}
}

func NewEd25519PrivateKey() (PrivateKey, error) {
	_, k, err := ed25519.GenerateKey(rand.Reader)
	if err != nil {
		return EmptyPrivateKey, err
	}
	return PrivateKey{
		Algorithm: Ed25519Private,
		RawKey:    k,
	}, nil
}

// TODO check validity and return error
func NewEd25519PrivateKeyFromSeed(seed []byte) PrivateKey {
	b := ed25519.NewKeyFromSeed(seed)
	return PrivateKey{
		Algorithm: Ed25519Private,
		RawKey:    b,
	}
}

var (
	charRegex  = regexp.MustCompile(`[^a-zA-Z ]+`)
	spaceRegex = regexp.MustCompile(`\s+`)
)

// TODO check validity and return error
func NewEd25519PrivateKeyFromBIP39(mnemonic string) (PrivateKey, error) {
	mnemonicClean := mnemonic
	mnemonicClean = charRegex.ReplaceAllString(mnemonicClean, " ")
	mnemonicClean = spaceRegex.ReplaceAllString(mnemonicClean, " ")
	mnemonicClean = strings.TrimSpace(mnemonicClean)
	seed, err := bip39.EntropyFromMnemonic(mnemonicClean)
	if err != nil {
		return EmptyPrivateKey, fmt.Errorf("error parsing mnemonic, %w", err)
	}
	return NewEd25519PrivateKeyFromSeed(seed), nil
}

// TODO check validity and return error
func NewEd25519PublicKeyFromRaw(raw ed25519.PublicKey) PublicKey {
	return PublicKey{
		Algorithm: Ed25519Public,
		RawKey:    raw,
	}
}

func publicEd25519KeyToCurve25519(pub ed25519.PublicKey) []byte {
	var edPk [ed25519.PublicKeySize]byte
	var curveKey [32]byte
	copy(edPk[:], pub)
	if !extra25519.PublicKeyToCurve25519(&curveKey, &edPk) {
		panic("could not convert ed25519 public key to curve25519")
	}
	return curveKey[:]
}

func privateEd25519KeyToCurve25519(priv ed25519.PrivateKey) []byte {
	var edSk [ed25519.PrivateKeySize]byte
	var curveKey [32]byte
	copy(edSk[:], priv)
	extra25519.PrivateKeyToCurve25519(&curveKey, &edSk)
	return curveKey[:]
}

// CalculateSharedKey calculates a shared secret given a private an public key
func CalculateSharedKey(
	priv PrivateKey,
	pub PublicKey,
) ([]byte, error) {
	if priv.Algorithm != Ed25519Private || pub.Algorithm != Ed25519Public {
		return nil, ErrUnsupportedKeyAlgorithm
	}
	ca := privateEd25519KeyToCurve25519(priv.RawKey)
	cB := publicEd25519KeyToCurve25519(pub.RawKey)
	ss, err := curve25519.X25519(ca, cB)
	if err != nil {
		return nil, fmt.Errorf("error getting x25519, %w", err)
	}
	return ss, nil
}

// NewSharedKey calculates a shared secret given a private and a public key,
// and returns it
func NewSharedKey(
	priv PrivateKey,
	pub PublicKey,
) (PrivateKey, []byte, error) {
	ca := privateEd25519KeyToCurve25519(priv.RawKey)
	cB := publicEd25519KeyToCurve25519(pub.RawKey)
	ss, err := curve25519.X25519(ca, cB)
	if err != nil {
		return EmptyPrivateKey, nil, err
	}
	return priv, ss, nil
}

// CalculateEphemeralSharedKey creates a new ec25519 key pair, calculates a
// shared secret given a public key, and returns the created public key and
// secret
func CalculateEphemeralSharedKey(
	pub PublicKey,
) (PrivateKey, []byte, error) {
	priv, err := NewEd25519PrivateKey()
	if err != nil {
		return EmptyPrivateKey, nil, err
	}
	return NewSharedKey(priv, pub)
}

func (k PrivateKey) Sign(message []byte) []byte {
	return ed25519.Sign(k.RawKey, message)
}

func (k PublicKey) Verify(message []byte, signature []byte) error {
	ok := ed25519.Verify(k.RawKey, message, signature)
	if !ok {
		return ErrInvalidSignature
	}
	return nil
}

func (k PublicKey) Equals(w PublicKey) bool {
	return k.Algorithm == w.Algorithm &&
		k.RawKey.Equal(w.RawKey)
}

func PublicKeyFromDID(d did.DID) (*PublicKey, error) {
	pk := &PublicKey{}
	err := pk.UnmarshalString(d.Identity)
	if err != nil {
		return nil, err
	}
	return pk, nil
}
<commit_msg>feat(crypto): add public key hash method<commit_after>package crypto

import (
	"crypto/ed25519"
	"crypto/rand"
	"encoding/json"
	"fmt"
	"regexp"
	"strings"

	"github.com/multiformats/go-multibase"
	"github.com/multiformats/go-multicodec"
	"github.com/teserakt-io/golang-ed25519/extra25519"
	"github.com/tyler-smith/go-bip39"
	"golang.org/x/crypto/curve25519"

	"nimona.io/pkg/did"
	"nimona.io/pkg/multiheader"
	"nimona.io/pkg/tilde"
)

// https://blog.filippo.io/using-ed25519-keys-for-encryption
// https://libsodium.gitbook.io/doc/advanced/ed25519-curve25519
// http://moderncrypto.org/mail-archive/curves/2014/000205.html
// https://signal.org/docs/specifications/xeddsa
// https://libsodium.gitbook.io/doc/advanced/ed25519-curve25519

// we are opting for ed to x at this point based on FiloSottile's age spec

type (
	KeyAlgorithm multicodec.Code
)

const (
	Ed25519Private KeyAlgorithm = KeyAlgorithm(multicodec.Ed25519Priv)
	Ed25519Public  KeyAlgorithm = KeyAlgorithm(multicodec.Ed25519Pub)
)

type (
	PublicKey struct {
		Algorithm KeyAlgorithm
		RawKey    ed25519.PublicKey // TODO use crypto.PublicKey
	}
	PrivateKey struct {
		Algorithm KeyAlgorithm
		RawKey    ed25519.PrivateKey // TODO use crypto.PrivateKey
	}
)

var (
	EmptyPublicKey  = PublicKey{}
	EmptyPrivateKey = PrivateKey{}
)

func (k PublicKey) String() string {
	if k.IsEmpty() {
		return ""
	}
	b := multiheader.Encode(multicodec.Code(k.Algorithm), k.RawKey)
	// nolint: errcheck // cannot error
	s, _ := multibase.Encode(multibase.Base58BTC, b)
	return s
}

func (k PublicKey) DID() did.DID {
	return did.DID{
		Method:   did.MethodKey,
		Identity: k.String(),
	}
}

func (k PublicKey) IsEmpty() bool {
	return k.RawKey == nil
}

func (k PublicKey) Hash() tilde.Digest {
	return tilde.String(k.String()).Hash()
}

func (k PublicKey) MarshalString() (string, error) {
	return k.String(), nil
}

func (k PublicKey) MarshalJSON() ([]byte, error) {
	return json.Marshal(k.String())
}

// UnmarshalText implements encoding.TextUnmarshaler mainly for use
// by envconfig.
func (k *PublicKey) UnmarshalText(b []byte) error {
	return k.UnmarshalString(string(b))
}

func (k *PublicKey) UnmarshalString(s string) error {
	_, b, err := multibase.Decode(s)
	if err != nil {
		return fmt.Errorf("unable to decode multibase, %w", err)
	}

	c, r, err := multiheader.Decode(b)
	if err != nil {
		return fmt.Errorf("unable to decode multiheader, %w", err)
	}

	if c != multicodec.Ed25519Pub {
		return ErrUnsupportedKeyAlgorithm
	}

	k.Algorithm = Ed25519Public
	k.RawKey = ed25519.PublicKey(r)

	return nil
}

func (k *PublicKey) UnmarshalJSON(s []byte) error {
	v := ""
	if err := json.Unmarshal(s, &v); err != nil {
		return err
	}
	return k.UnmarshalString(v)
}

func (k PrivateKey) IsEmpty() bool {
	return k.RawKey == nil
}

func (k PrivateKey) String() string {
	b := multiheader.Encode(multicodec.Code(k.Algorithm), k.RawKey)
	// nolint: errcheck // cannot error
	s, _ := multibase.Encode(multibase.Base58BTC, b)
	return s
}

func (k PrivateKey) Seed() []byte {
	return k.RawKey.Seed()
}

func (k PrivateKey) BIP39() string {
	m, _ := bip39.NewMnemonic(k.Seed())
	return m
}

func (k PrivateKey) MarshalString() (string, error) {
	return k.String(), nil
}

func (k PrivateKey) MarshalJSON() ([]byte, error) {
	return json.Marshal(k.String())
}

// UnmarshalText implements encoding.TextUnmarshaler mainly for use
// by envconfig.
func (k *PrivateKey) UnmarshalText(b []byte) error {
	return k.UnmarshalString(string(b))
}

func (k *PrivateKey) UnmarshalString(s string) error {
	_, b, err := multibase.Decode(s)
	if err != nil {
		return fmt.Errorf("unable to decode multibase, %w", err)
	}

	c, r, err := multiheader.Decode(b)
	if err != nil {
		return fmt.Errorf("unable to decode multiheader, %w", err)
	}

	if c != multicodec.Ed25519Priv {
		return ErrUnsupportedKeyAlgorithm
	}

	k.Algorithm = Ed25519Private
	k.RawKey = ed25519.PrivateKey(r)

	return nil
}

func (k *PrivateKey) UnmarshalJSON(s []byte) error {
	v := ""
	if err := json.Unmarshal(s, &v); err != nil {
		return err
	}
	return k.UnmarshalString(v)
}

func (k PrivateKey) PublicKey() PublicKey {
	return PublicKey{
		Algorithm: Ed25519Public,
		RawKey:    k.RawKey.Public().(ed25519.PublicKey),
	}
}

func NewEd25519PrivateKey() (PrivateKey, error) {
	_, k, err := ed25519.GenerateKey(rand.Reader)
	if err != nil {
		return EmptyPrivateKey, err
	}
	return PrivateKey{
		Algorithm: Ed25519Private,
		RawKey:    k,
	}, nil
}

// TODO check validity and return error
func NewEd25519PrivateKeyFromSeed(seed []byte) PrivateKey {
	b := ed25519.NewKeyFromSeed(seed)
	return PrivateKey{
		Algorithm: Ed25519Private,
		RawKey:    b,
	}
}

var (
	charRegex  = regexp.MustCompile(`[^a-zA-Z ]+`)
	spaceRegex = regexp.MustCompile(`\s+`)
)

// TODO check validity and return error
func NewEd25519PrivateKeyFromBIP39(mnemonic string) (PrivateKey, error) {
	mnemonicClean := mnemonic
	mnemonicClean = charRegex.ReplaceAllString(mnemonicClean, " ")
	mnemonicClean = spaceRegex.ReplaceAllString(mnemonicClean, " ")
	mnemonicClean = strings.TrimSpace(mnemonicClean)
	seed, err := bip39.EntropyFromMnemonic(mnemonicClean)
	if err != nil {
		return EmptyPrivateKey, fmt.Errorf("error parsing mnemonic, %w", err)
	}
	return NewEd25519PrivateKeyFromSeed(seed), nil
}

// TODO check validity and return error
func NewEd25519PublicKeyFromRaw(raw ed25519.PublicKey) PublicKey {
	return PublicKey{
		Algorithm: Ed25519Public,
		RawKey:    raw,
	}
}

func publicEd25519KeyToCurve25519(pub ed25519.PublicKey) []byte {
	var edPk [ed25519.PublicKeySize]byte
	var curveKey [32]byte
	copy(edPk[:], pub)
	if !extra25519.PublicKeyToCurve25519(&curveKey, &edPk) {
		panic("could not convert ed25519 public key to curve25519")
	}
	return curveKey[:]
}

func privateEd25519KeyToCurve25519(priv ed25519.PrivateKey) []byte {
	var edSk [ed25519.PrivateKeySize]byte
	var curveKey [32]byte
	copy(edSk[:], priv)
	extra25519.PrivateKeyToCurve25519(&curveKey, &edSk)
	return curveKey[:]
}

// CalculateSharedKey calculates a shared secret given a private an public key
func CalculateSharedKey(
	priv PrivateKey,
	pub PublicKey,
) ([]byte, error) {
	if priv.Algorithm != Ed25519Private || pub.Algorithm != Ed25519Public {
		return nil, ErrUnsupportedKeyAlgorithm
	}
	ca := privateEd25519KeyToCurve25519(priv.RawKey)
	cB := publicEd25519KeyToCurve25519(pub.RawKey)
	ss, err := curve25519.X25519(ca, cB)
	if err != nil {
		return nil, fmt.Errorf("error getting x25519, %w", err)
	}
	return ss, nil
}

// NewSharedKey calculates a shared secret given a private and a public key,
// and returns it
func NewSharedKey(
	priv PrivateKey,
	pub PublicKey,
) (PrivateKey, []byte, error) {
	ca := privateEd25519KeyToCurve25519(priv.RawKey)
	cB := publicEd25519KeyToCurve25519(pub.RawKey)
	ss, err := curve25519.X25519(ca, cB)
	if err != nil {
		return EmptyPrivateKey, nil, err
	}
	return priv, ss, nil
}

// CalculateEphemeralSharedKey creates a new ec25519 key pair, calculates a
// shared secret given a public key, and returns the created public key and
// secret
func CalculateEphemeralSharedKey(
	pub PublicKey,
) (PrivateKey, []byte, error) {
	priv, err := NewEd25519PrivateKey()
	if err != nil {
		return EmptyPrivateKey, nil, err
	}
	return NewSharedKey(priv, pub)
}

func (k PrivateKey) Sign(message []byte) []byte {
	return ed25519.Sign(k.RawKey, message)
}

func (k PublicKey) Verify(message []byte, signature []byte) error {
	ok := ed25519.Verify(k.RawKey, message, signature)
	if !ok {
		return ErrInvalidSignature
	}
	return nil
}

func (k PublicKey) Equals(w PublicKey) bool {
	return k.Algorithm == w.Algorithm &&
		k.RawKey.Equal(w.RawKey)
}

func PublicKeyFromDID(d did.DID) (*PublicKey, error) {
	pk := &PublicKey{}
	err := pk.UnmarshalString(d.Identity)
	if err != nil {
		return nil, err
	}
	return pk, nil
}
<|endoftext|>


<commit_before>package etwlogrus

import (
	"sort"

	"github.com/Microsoft/go-winio/pkg/etw"
	"github.com/sirupsen/logrus"
)

// Hook is a Logrus hook which logs received events to ETW.
type Hook struct {
	provider      *etw.Provider
	closeProvider bool
}

// NewHook registers a new ETW provider and returns a hook to log from it. The
// provider will be closed when the hook is closed.
func NewHook(providerName string) (*Hook, error) {
	provider, err := etw.NewProvider(providerName, nil)
	if err != nil {
		return nil, err
	}

	return &Hook{provider, true}, nil
}

// NewHookFromProvider creates a new hook based on an existing ETW provider. The
// provider will not be closed when the hook is closed.
func NewHookFromProvider(provider *etw.Provider) (*Hook, error) {
	return &Hook{provider, false}, nil
}

// Levels returns the set of levels that this hook wants to receive log entries
// for.
func (h *Hook) Levels() []logrus.Level {
	return []logrus.Level{
		logrus.TraceLevel,
		logrus.DebugLevel,
		logrus.InfoLevel,
		logrus.WarnLevel,
		logrus.ErrorLevel,
		logrus.FatalLevel,
		logrus.PanicLevel,
	}
}

var logrusToETWLevelMap = map[logrus.Level]etw.Level{
	logrus.PanicLevel: etw.LevelAlways,
	logrus.FatalLevel: etw.LevelCritical,
	logrus.ErrorLevel: etw.LevelError,
	logrus.WarnLevel:  etw.LevelWarning,
	logrus.InfoLevel:  etw.LevelInfo,
	logrus.DebugLevel: etw.LevelVerbose,
	logrus.TraceLevel: etw.LevelVerbose,
}

// Fire receives each Logrus entry as it is logged, and logs it to ETW.
func (h *Hook) Fire(e *logrus.Entry) error {
	// Logrus defines more levels than ETW typically uses, but analysis is
	// easiest when using a consistent set of levels across ETW providers, so we
	// map the Logrus levels to ETW levels.
	level := logrusToETWLevelMap[e.Level]
	if !h.provider.IsEnabledForLevel(level) {
		return nil
	}

	// Sort the fields by name so they are consistent in each instance
	// of an event. Otherwise, the fields don't line up in WPA.
	names := make([]string, 0, len(e.Data))
	hasError := false
	for k := range e.Data {
		if k == logrus.ErrorKey {
			// Always put the error last because it is optional in some events.
			hasError = true
		} else {
			names = append(names, k)
		}
	}
	sort.Strings(names)

	// Reserve extra space for the message field.
	fields := make([]etw.FieldOpt, 0, len(e.Data)+1)
	fields = append(fields, etw.StringField("Message", e.Message))
	for _, k := range names {
		fields = append(fields, etw.SmartField(k, e.Data[k]))
	}
	if hasError {
		fields = append(fields, etw.SmartField(logrus.ErrorKey, e.Data[logrus.ErrorKey]))
	}

	// Firing an ETW event is essentially best effort, as the event write can
	// fail for reasons completely out of the control of the event writer (such
	// as a session listening for the event having no available space in its
	// buffers). Therefore, we don't return the error from WriteEvent, as it is
	// just noise in many cases.
	h.provider.WriteEvent(
		"LogrusEntry",
		etw.WithEventOpts(etw.WithLevel(level)),
		fields)

	return nil
}

// Close cleans up the hook and closes the ETW provider. If the provder was
// registered by etwlogrus, it will be closed as part of `Close`. If the
// provider was passed in, it will not be closed.
func (h *Hook) Close() error {
	if h.closeProvider {
		return h.provider.Close()
	}
	return nil
}
<commit_msg>Replace ETW Logrus hook levels with AllLevels<commit_after>package etwlogrus

import (
	"sort"

	"github.com/Microsoft/go-winio/pkg/etw"
	"github.com/sirupsen/logrus"
)

// Hook is a Logrus hook which logs received events to ETW.
type Hook struct {
	provider      *etw.Provider
	closeProvider bool
}

// NewHook registers a new ETW provider and returns a hook to log from it. The
// provider will be closed when the hook is closed.
func NewHook(providerName string) (*Hook, error) {
	provider, err := etw.NewProvider(providerName, nil)
	if err != nil {
		return nil, err
	}

	return &Hook{provider, true}, nil
}

// NewHookFromProvider creates a new hook based on an existing ETW provider. The
// provider will not be closed when the hook is closed.
func NewHookFromProvider(provider *etw.Provider) (*Hook, error) {
	return &Hook{provider, false}, nil
}

// Levels returns the set of levels that this hook wants to receive log entries
// for.
func (h *Hook) Levels() []logrus.Level {
	return logrus.AllLevels
}

var logrusToETWLevelMap = map[logrus.Level]etw.Level{
	logrus.PanicLevel: etw.LevelAlways,
	logrus.FatalLevel: etw.LevelCritical,
	logrus.ErrorLevel: etw.LevelError,
	logrus.WarnLevel:  etw.LevelWarning,
	logrus.InfoLevel:  etw.LevelInfo,
	logrus.DebugLevel: etw.LevelVerbose,
	logrus.TraceLevel: etw.LevelVerbose,
}

// Fire receives each Logrus entry as it is logged, and logs it to ETW.
func (h *Hook) Fire(e *logrus.Entry) error {
	// Logrus defines more levels than ETW typically uses, but analysis is
	// easiest when using a consistent set of levels across ETW providers, so we
	// map the Logrus levels to ETW levels.
	level := logrusToETWLevelMap[e.Level]
	if !h.provider.IsEnabledForLevel(level) {
		return nil
	}

	// Sort the fields by name so they are consistent in each instance
	// of an event. Otherwise, the fields don't line up in WPA.
	names := make([]string, 0, len(e.Data))
	hasError := false
	for k := range e.Data {
		if k == logrus.ErrorKey {
			// Always put the error last because it is optional in some events.
			hasError = true
		} else {
			names = append(names, k)
		}
	}
	sort.Strings(names)

	// Reserve extra space for the message field.
	fields := make([]etw.FieldOpt, 0, len(e.Data)+1)
	fields = append(fields, etw.StringField("Message", e.Message))
	for _, k := range names {
		fields = append(fields, etw.SmartField(k, e.Data[k]))
	}
	if hasError {
		fields = append(fields, etw.SmartField(logrus.ErrorKey, e.Data[logrus.ErrorKey]))
	}

	// Firing an ETW event is essentially best effort, as the event write can
	// fail for reasons completely out of the control of the event writer (such
	// as a session listening for the event having no available space in its
	// buffers). Therefore, we don't return the error from WriteEvent, as it is
	// just noise in many cases.
	h.provider.WriteEvent(
		"LogrusEntry",
		etw.WithEventOpts(etw.WithLevel(level)),
		fields)

	return nil
}

// Close cleans up the hook and closes the ETW provider. If the provder was
// registered by etwlogrus, it will be closed as part of `Close`. If the
// provider was passed in, it will not be closed.
func (h *Hook) Close() error {
	if h.closeProvider {
		return h.provider.Close()
	}
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2016-2017 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package kvstore

import (
	"fmt"

	log "github.com/Sirupsen/logrus"
)

// Client is the instance of the key-value as configured
var Client KVClient

func initClient() error {
	switch backend {
	case Consul:
		if consulConfig == nil {
			return fmt.Errorf("mising consul server address, please specify, e.g. --kvstore-opt consul.address=127.0.0.1:8501")
		}

		c, err := newConsulClient(consulConfig)
		if err != nil {
			return err
		}

		log.Infof("Using consul as key-value store")
		Client = c

	case Etcd:
		if etcdCfgPath == "" && etcdConfig == nil {
			return fmt.Errorf("missing etcd endpoints; please specify , e.g. --kvstore-opt etcd.address=....")
		}

		c, err := newEtcdClient(etcdConfig, etcdCfgPath)
		if err != nil {
			return err
		}

		log.Infof("Using etcd as key-value store")
		Client = c

	case Local:
		log.Infof("Using local storage as key-value store")
		Client = NewLocalClient()

	default:
		panic("BUG: kvstore backend not specified")
	}

	return nil
}
<commit_msg>kvstore: erroring out with the default kvstore IPs<commit_after>// Copyright 2016-2017 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package kvstore

import (
	"fmt"

	log "github.com/Sirupsen/logrus"
)

// Client is the instance of the key-value as configured
var Client KVClient

func initClient() error {
	switch backend {
	case Consul:
		if consulConfig == nil {
			return fmt.Errorf("mising consul server address, please specify, e.g. --kvstore-opt consul.address=127.0.0.1:8500")
		}

		c, err := newConsulClient(consulConfig)
		if err != nil {
			return err
		}

		log.Infof("Using consul as key-value store")
		Client = c

	case Etcd:
		if etcdCfgPath == "" && etcdConfig == nil {
			return fmt.Errorf("missing etcd endpoints; please specify , e.g. --kvstore-opt etcd.address=127.0.0.1:2379")
		}

		c, err := newEtcdClient(etcdConfig, etcdCfgPath)
		if err != nil {
			return err
		}

		log.Infof("Using etcd as key-value store")
		Client = c

	case Local:
		log.Infof("Using local storage as key-value store")
		Client = NewLocalClient()

	default:
		panic("BUG: kvstore backend not specified")
	}

	return nil
}
<|endoftext|>


<commit_before>package spanner

import (
	"context"
	"errors"
	"fmt"
	"sort"

	"cloud.google.com/go/spanner"
	admin "cloud.google.com/go/spanner/admin/database/apiv1"
	"golang.org/x/xerrors"
	"google.golang.org/api/iterator"
	"google.golang.org/api/option"
	databasepb "google.golang.org/genproto/googleapis/spanner/admin/database/v1"
)

const (
	ddlStatementsSeparator = ";"
)

type Client struct {
	config             *Config
	spannerClient      *spanner.Client
	spannerAdminClient *admin.DatabaseAdminClient
}

func NewClient(ctx context.Context, config *Config) (*Client, error) {
	opts := make([]option.ClientOption, 0)
	if config.CredentialsFile != "" {
		opts = append(opts, option.WithCredentialsFile(config.CredentialsFile))
	}

	spannerClient, err := spanner.NewClient(ctx, config.URL(), opts...)
	if err != nil {
		return nil, &Error{
			Code: ErrorCodeCreateClient,
			err:  err,
		}
	}

	spannerAdminClient, err := admin.NewDatabaseAdminClient(ctx, opts...)
	if err != nil {
		spannerClient.Close()
		return nil, &Error{
			Code: ErrorCodeCreateClient,
			err:  err,
		}
	}

	return &Client{
		config:             config,
		spannerClient:      spannerClient,
		spannerAdminClient: spannerAdminClient,
	}, nil
}

func (c *Client) CreateDatabase(ctx context.Context, ddl []byte) error {
	statements := toStatements(ddl)

	createReq := &databasepb.CreateDatabaseRequest{
		Parent:          fmt.Sprintf("projects/%s/instances/%s", c.config.Project, c.config.Instance),
		CreateStatement: fmt.Sprintf("CREATE DATABASE `%s`", c.config.Database),
		ExtraStatements: statements,
	}

	op, err := c.spannerAdminClient.CreateDatabase(ctx, createReq)
	if err != nil {
		return &Error{
			Code: ErrorCodeCreateDatabase,
			err:  err,
		}
	}

	_, err = op.Wait(ctx)
	if err != nil {
		return &Error{
			Code: ErrorCodeWaitOperation,
			err:  err,
		}
	}

	return nil
}

func (c *Client) DropDatabase(ctx context.Context) error {
	req := &databasepb.DropDatabaseRequest{Database: c.config.URL()}

	if err := c.spannerAdminClient.DropDatabase(ctx, req); err != nil {
		return &Error{
			Code: ErrorCodeDropDatabase,
			err:  err,
		}
	}

	return nil
}

func (c *Client) LoadDDL(ctx context.Context) ([]byte, error) {
	req := &databasepb.GetDatabaseDdlRequest{Database: c.config.URL()}

	res, err := c.spannerAdminClient.GetDatabaseDdl(ctx, req)
	if err != nil {
		return nil, &Error{
			Code: ErrorCodeLoadSchema,
			err:  err,
		}
	}

	var schema []byte
	last := len(res.Statements) - 1
	for index, statement := range res.Statements {
		if index != last {
			statement += ddlStatementsSeparator + "\n\n"
		} else {
			statement += ddlStatementsSeparator + "\n"
		}

		schema = append(schema[:], []byte(statement)[:]...)
	}

	return schema, nil
}

func (c *Client) ApplyDDLFile(ctx context.Context, ddl []byte) error {
	return c.ApplyDDL(ctx, toStatements(ddl))
}

func (c *Client) ApplyDDL(ctx context.Context, statements []string) error {
	req := &databasepb.UpdateDatabaseDdlRequest{
		Database:   c.config.URL(),
		Statements: statements,
	}

	op, err := c.spannerAdminClient.UpdateDatabaseDdl(ctx, req)
	if err != nil {
		return &Error{
			Code: ErrorCodeUpdateDDL,
			err:  err,
		}
	}

	err = op.Wait(ctx)
	if err != nil {
		return &Error{
			Code: ErrorCodeWaitOperation,
			err:  err,
		}
	}

	return nil
}

func (c *Client) ApplyDMLFile(ctx context.Context, ddl []byte, partitioned bool) (int64, error) {
	statements := toStatements(ddl)

	if partitioned {
		return c.ApplyPartitionedDML(ctx, statements)
	} else {
		return c.ApplyDML(ctx, statements)
	}
}

func (c *Client) ApplyDML(ctx context.Context, statements []string) (int64, error) {
	numAffectedRows := int64(0)
	_, err := c.spannerClient.ReadWriteTransaction(ctx, func(ctx context.Context, tx *spanner.ReadWriteTransaction) error {
		for _, s := range statements {
			num, err := tx.Update(ctx, spanner.Statement{
				SQL: s,
			})
			if err != nil {
				return err
			}
			numAffectedRows += num
		}
		return nil
	})
	if err != nil {
		return 0, &Error{
			Code: ErrorCodeUpdateDML,
			err:  err,
		}
	}

	return numAffectedRows, nil
}

func (c *Client) ApplyPartitionedDML(ctx context.Context, statements []string) (int64, error) {
	numAffectedRows := int64(0)

	for _, s := range statements {
		num, err := c.spannerClient.PartitionedUpdate(ctx, spanner.Statement{
			SQL: s,
		})
		if err != nil {
			return numAffectedRows, &Error{
				Code: ErrorCodeUpdatePartitionedDML,
				err:  err,
			}
		}

		numAffectedRows += num
	}

	return numAffectedRows, nil
}

func (c *Client) ExecuteMigrations(ctx context.Context, migrations Migrations, limit int, tableName string) error {
	sort.Sort(migrations)

	version, dirty, err := c.GetSchemaMigrationVersion(ctx, tableName)
	if err != nil {
		se := &Error{}
		if !xerrors.As(err, &se) || se.Code != ErrorCodeNoMigration {
			return &Error{
				Code: ErrorCodeExecuteMigrations,
				err:  err,
			}
		}
	}

	if dirty {
		return &Error{
			Code: ErrorCodeMigrationVersionDirty,
			err:  fmt.Errorf("Database version: %d is dirty, please fix it.", version),
		}
	}

	var count int
	for _, m := range migrations {
		if limit == 0 {
			break
		}

		if m.Version <= version {
			continue
		}

		if err := c.SetSchemaMigrationVersion(ctx, m.Version, true, tableName); err != nil {
			return &Error{
				Code: ErrorCodeExecuteMigrations,
				err:  err,
			}
		}

		switch m.kind {
		case statementKindDDL:
			if err := c.ApplyDDL(ctx, m.Statements); err != nil {
				return &Error{
					Code: ErrorCodeExecuteMigrations,
					err:  err,
				}
			}
		case statementKindDML:
			if _, err := c.ApplyPartitionedDML(ctx, m.Statements); err != nil {
				return &Error{
					Code: ErrorCodeExecuteMigrations,
					err:  err,
				}
			}
		default:
			return &Error{
				Code: ErrorCodeExecuteMigrations,
				err:  fmt.Errorf("Unknown query type, version: %d", m.Version),
			}
		}

		if m.Name != "" {
			fmt.Printf("%d/up %s\n", m.Version, m.Name)
		} else {
			fmt.Printf("%d/up\n", m.Version)
		}

		if err := c.SetSchemaMigrationVersion(ctx, m.Version, false, tableName); err != nil {
			return &Error{
				Code: ErrorCodeExecuteMigrations,
				err:  err,
			}
		}

		count++
		if limit > 0 && count == limit {
			break
		}
	}

	if count == 0 {
		fmt.Println("no change")
	}

	return nil
}

func (c *Client) GetSchemaMigrationVersion(ctx context.Context, tableName string) (uint, bool, error) {
	stmt := spanner.Statement{
		SQL: `SELECT Version, Dirty FROM ` + tableName + ` LIMIT 1`,
	}
	iter := c.spannerClient.Single().Query(ctx, stmt)
	defer iter.Stop()

	for {
		row, err := iter.Next()
		if err != nil {
			if err == iterator.Done {
				break
			}
			return 0, false, &Error{
				Code: ErrorCodeGetMigrationVersion,
				err:  err,
			}
		}

		var (
			v     int64
			dirty bool
		)
		if err := row.Columns(&v, &dirty); err != nil {
			return 0, false, &Error{
				Code: ErrorCodeGetMigrationVersion,
				err:  err,
			}
		}

		return uint(v), dirty, nil
	}

	return 0, false, &Error{
		Code: ErrorCodeNoMigration,
		err:  errors.New("No migration."),
	}
}

func (c *Client) SetSchemaMigrationVersion(ctx context.Context, version uint, dirty bool, tableName string) error {
	_, err := c.spannerClient.ReadWriteTransaction(ctx, func(ctx context.Context, tx *spanner.ReadWriteTransaction) error {
		m := []*spanner.Mutation{
			spanner.Delete(tableName, spanner.AllKeys()),
			spanner.Insert(
				tableName,
				[]string{"Version", "Dirty"},
				[]interface{}{int64(version), dirty},
			)}
		return tx.BufferWrite(m)
	})
	if err != nil {
		return &Error{
			Code: ErrorCodeSetMigrationVersion,
			err:  err,
		}
	}

	return nil
}

func (c *Client) EnsureMigrationTable(ctx context.Context, tableName string) error {
	iter := c.spannerClient.Single().Read(ctx, tableName, spanner.AllKeys(), []string{"Version"})
	err := iter.Do(func(r *spanner.Row) error {
		return nil
	})
	if err == nil {
		return nil
	}

	stmt := fmt.Sprintf(`CREATE TABLE %s (
    Version INT64 NOT NULL,
    Dirty    BOOL NOT NULL
	) PRIMARY KEY(Version)`, tableName)

	return c.ApplyDDL(ctx, []string{stmt})
}

func (c *Client) Close() error {
	c.spannerClient.Close()
	if err := c.spannerAdminClient.Close(); err != nil {
		return &Error{
			err:  err,
			Code: ErrorCodeCloseClient,
		}
	}

	return nil
}
<commit_msg>remove unnecessary loop<commit_after>package spanner

import (
	"context"
	"errors"
	"fmt"
	"sort"

	"cloud.google.com/go/spanner"
	admin "cloud.google.com/go/spanner/admin/database/apiv1"
	"golang.org/x/xerrors"
	"google.golang.org/api/iterator"
	"google.golang.org/api/option"
	databasepb "google.golang.org/genproto/googleapis/spanner/admin/database/v1"
)

const (
	ddlStatementsSeparator = ";"
)

type Client struct {
	config             *Config
	spannerClient      *spanner.Client
	spannerAdminClient *admin.DatabaseAdminClient
}

func NewClient(ctx context.Context, config *Config) (*Client, error) {
	opts := make([]option.ClientOption, 0)
	if config.CredentialsFile != "" {
		opts = append(opts, option.WithCredentialsFile(config.CredentialsFile))
	}

	spannerClient, err := spanner.NewClient(ctx, config.URL(), opts...)
	if err != nil {
		return nil, &Error{
			Code: ErrorCodeCreateClient,
			err:  err,
		}
	}

	spannerAdminClient, err := admin.NewDatabaseAdminClient(ctx, opts...)
	if err != nil {
		spannerClient.Close()
		return nil, &Error{
			Code: ErrorCodeCreateClient,
			err:  err,
		}
	}

	return &Client{
		config:             config,
		spannerClient:      spannerClient,
		spannerAdminClient: spannerAdminClient,
	}, nil
}

func (c *Client) CreateDatabase(ctx context.Context, ddl []byte) error {
	statements := toStatements(ddl)

	createReq := &databasepb.CreateDatabaseRequest{
		Parent:          fmt.Sprintf("projects/%s/instances/%s", c.config.Project, c.config.Instance),
		CreateStatement: fmt.Sprintf("CREATE DATABASE `%s`", c.config.Database),
		ExtraStatements: statements,
	}

	op, err := c.spannerAdminClient.CreateDatabase(ctx, createReq)
	if err != nil {
		return &Error{
			Code: ErrorCodeCreateDatabase,
			err:  err,
		}
	}

	_, err = op.Wait(ctx)
	if err != nil {
		return &Error{
			Code: ErrorCodeWaitOperation,
			err:  err,
		}
	}

	return nil
}

func (c *Client) DropDatabase(ctx context.Context) error {
	req := &databasepb.DropDatabaseRequest{Database: c.config.URL()}

	if err := c.spannerAdminClient.DropDatabase(ctx, req); err != nil {
		return &Error{
			Code: ErrorCodeDropDatabase,
			err:  err,
		}
	}

	return nil
}

func (c *Client) LoadDDL(ctx context.Context) ([]byte, error) {
	req := &databasepb.GetDatabaseDdlRequest{Database: c.config.URL()}

	res, err := c.spannerAdminClient.GetDatabaseDdl(ctx, req)
	if err != nil {
		return nil, &Error{
			Code: ErrorCodeLoadSchema,
			err:  err,
		}
	}

	var schema []byte
	last := len(res.Statements) - 1
	for index, statement := range res.Statements {
		if index != last {
			statement += ddlStatementsSeparator + "\n\n"
		} else {
			statement += ddlStatementsSeparator + "\n"
		}

		schema = append(schema[:], []byte(statement)[:]...)
	}

	return schema, nil
}

func (c *Client) ApplyDDLFile(ctx context.Context, ddl []byte) error {
	return c.ApplyDDL(ctx, toStatements(ddl))
}

func (c *Client) ApplyDDL(ctx context.Context, statements []string) error {
	req := &databasepb.UpdateDatabaseDdlRequest{
		Database:   c.config.URL(),
		Statements: statements,
	}

	op, err := c.spannerAdminClient.UpdateDatabaseDdl(ctx, req)
	if err != nil {
		return &Error{
			Code: ErrorCodeUpdateDDL,
			err:  err,
		}
	}

	err = op.Wait(ctx)
	if err != nil {
		return &Error{
			Code: ErrorCodeWaitOperation,
			err:  err,
		}
	}

	return nil
}

func (c *Client) ApplyDMLFile(ctx context.Context, ddl []byte, partitioned bool) (int64, error) {
	statements := toStatements(ddl)

	if partitioned {
		return c.ApplyPartitionedDML(ctx, statements)
	} else {
		return c.ApplyDML(ctx, statements)
	}
}

func (c *Client) ApplyDML(ctx context.Context, statements []string) (int64, error) {
	numAffectedRows := int64(0)
	_, err := c.spannerClient.ReadWriteTransaction(ctx, func(ctx context.Context, tx *spanner.ReadWriteTransaction) error {
		for _, s := range statements {
			num, err := tx.Update(ctx, spanner.Statement{
				SQL: s,
			})
			if err != nil {
				return err
			}
			numAffectedRows += num
		}
		return nil
	})
	if err != nil {
		return 0, &Error{
			Code: ErrorCodeUpdateDML,
			err:  err,
		}
	}

	return numAffectedRows, nil
}

func (c *Client) ApplyPartitionedDML(ctx context.Context, statements []string) (int64, error) {
	numAffectedRows := int64(0)

	for _, s := range statements {
		num, err := c.spannerClient.PartitionedUpdate(ctx, spanner.Statement{
			SQL: s,
		})
		if err != nil {
			return numAffectedRows, &Error{
				Code: ErrorCodeUpdatePartitionedDML,
				err:  err,
			}
		}

		numAffectedRows += num
	}

	return numAffectedRows, nil
}

func (c *Client) ExecuteMigrations(ctx context.Context, migrations Migrations, limit int, tableName string) error {
	sort.Sort(migrations)

	version, dirty, err := c.GetSchemaMigrationVersion(ctx, tableName)
	if err != nil {
		se := &Error{}
		if !xerrors.As(err, &se) || se.Code != ErrorCodeNoMigration {
			return &Error{
				Code: ErrorCodeExecuteMigrations,
				err:  err,
			}
		}
	}

	if dirty {
		return &Error{
			Code: ErrorCodeMigrationVersionDirty,
			err:  fmt.Errorf("Database version: %d is dirty, please fix it.", version),
		}
	}

	var count int
	for _, m := range migrations {
		if limit == 0 {
			break
		}

		if m.Version <= version {
			continue
		}

		if err := c.SetSchemaMigrationVersion(ctx, m.Version, true, tableName); err != nil {
			return &Error{
				Code: ErrorCodeExecuteMigrations,
				err:  err,
			}
		}

		switch m.kind {
		case statementKindDDL:
			if err := c.ApplyDDL(ctx, m.Statements); err != nil {
				return &Error{
					Code: ErrorCodeExecuteMigrations,
					err:  err,
				}
			}
		case statementKindDML:
			if _, err := c.ApplyPartitionedDML(ctx, m.Statements); err != nil {
				return &Error{
					Code: ErrorCodeExecuteMigrations,
					err:  err,
				}
			}
		default:
			return &Error{
				Code: ErrorCodeExecuteMigrations,
				err:  fmt.Errorf("Unknown query type, version: %d", m.Version),
			}
		}

		if m.Name != "" {
			fmt.Printf("%d/up %s\n", m.Version, m.Name)
		} else {
			fmt.Printf("%d/up\n", m.Version)
		}

		if err := c.SetSchemaMigrationVersion(ctx, m.Version, false, tableName); err != nil {
			return &Error{
				Code: ErrorCodeExecuteMigrations,
				err:  err,
			}
		}

		count++
		if limit > 0 && count == limit {
			break
		}
	}

	if count == 0 {
		fmt.Println("no change")
	}

	return nil
}

func (c *Client) GetSchemaMigrationVersion(ctx context.Context, tableName string) (uint, bool, error) {
	stmt := spanner.Statement{
		SQL: `SELECT Version, Dirty FROM ` + tableName + ` LIMIT 1`,
	}
	iter := c.spannerClient.Single().Query(ctx, stmt)
	defer iter.Stop()

	row, err := iter.Next()
	if err != nil {
		if err == iterator.Done {
			return 0, false, &Error{
				Code: ErrorCodeNoMigration,
				err:  errors.New("No migration."),
			}
		}
		return 0, false, &Error{
			Code: ErrorCodeGetMigrationVersion,
			err:  err,
		}
	}

	var (
		v     int64
		dirty bool
	)
	if err := row.Columns(&v, &dirty); err != nil {
		return 0, false, &Error{
			Code: ErrorCodeGetMigrationVersion,
			err:  err,
		}
	}

	return uint(v), dirty, nil
}

func (c *Client) SetSchemaMigrationVersion(ctx context.Context, version uint, dirty bool, tableName string) error {
	_, err := c.spannerClient.ReadWriteTransaction(ctx, func(ctx context.Context, tx *spanner.ReadWriteTransaction) error {
		m := []*spanner.Mutation{
			spanner.Delete(tableName, spanner.AllKeys()),
			spanner.Insert(
				tableName,
				[]string{"Version", "Dirty"},
				[]interface{}{int64(version), dirty},
			)}
		return tx.BufferWrite(m)
	})
	if err != nil {
		return &Error{
			Code: ErrorCodeSetMigrationVersion,
			err:  err,
		}
	}

	return nil
}

func (c *Client) EnsureMigrationTable(ctx context.Context, tableName string) error {
	iter := c.spannerClient.Single().Read(ctx, tableName, spanner.AllKeys(), []string{"Version"})
	err := iter.Do(func(r *spanner.Row) error {
		return nil
	})
	if err == nil {
		return nil
	}

	stmt := fmt.Sprintf(`CREATE TABLE %s (
    Version INT64 NOT NULL,
    Dirty    BOOL NOT NULL
	) PRIMARY KEY(Version)`, tableName)

	return c.ApplyDDL(ctx, []string{stmt})
}

func (c *Client) Close() error {
	c.spannerClient.Close()
	if err := c.spannerAdminClient.Close(); err != nil {
		return &Error{
			err:  err,
			Code: ErrorCodeCloseClient,
		}
	}

	return nil
}
<|endoftext|>


<commit_before>/*
Copyright The Helm Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package strvals

import (
	"bytes"
	"fmt"
	"io"
	"strconv"
	"strings"

	"github.com/pkg/errors"
	"sigs.k8s.io/yaml"
)

// ErrNotList indicates that a non-list was treated as a list.
var ErrNotList = errors.New("not a list")

// ToYAML takes a string of arguments and converts to a YAML document.
func ToYAML(s string) (string, error) {
	m, err := Parse(s)
	if err != nil {
		return "", err
	}
	d, err := yaml.Marshal(m)
	return strings.TrimSuffix(string(d), "\n"), err
}

// Parse parses a set line.
//
// A set line is of the form name1=value1,name2=value2
func Parse(s string) (map[string]interface{}, error) {
	vals := map[string]interface{}{}
	scanner := bytes.NewBufferString(s)
	t := newParser(scanner, vals, false)
	err := t.parse()
	return vals, err
}

// ParseString parses a set line and forces a string value.
//
// A set line is of the form name1=value1,name2=value2
func ParseString(s string) (map[string]interface{}, error) {
	vals := map[string]interface{}{}
	scanner := bytes.NewBufferString(s)
	t := newParser(scanner, vals, true)
	err := t.parse()
	return vals, err
}

// ParseInto parses a strvals line and merges the result into dest.
//
// If the strval string has a key that exists in dest, it overwrites the
// dest version.
func ParseInto(s string, dest map[string]interface{}) error {
	scanner := bytes.NewBufferString(s)
	t := newParser(scanner, dest, false)
	return t.parse()
}

// ParseFile parses a set line, but its final value is loaded from the file at the path specified by the original value.
//
// A set line is of the form name1=path1,name2=path2
//
// When the files at path1 and path2 contained "val1" and "val2" respectively, the set line is consumed as
// name1=val1,name2=val2
func ParseFile(s string, reader RunesValueReader) (map[string]interface{}, error) {
	vals := map[string]interface{}{}
	scanner := bytes.NewBufferString(s)
	t := newFileParser(scanner, vals, reader)
	err := t.parse()
	return vals, err
}

// ParseIntoString parses a strvals line and merges the result into dest.
//
// This method always returns a string as the value.
func ParseIntoString(s string, dest map[string]interface{}) error {
	scanner := bytes.NewBufferString(s)
	t := newParser(scanner, dest, true)
	return t.parse()
}

// ParseIntoFile parses a filevals line and merges the result into dest.
//
// This method always returns a string as the value.
func ParseIntoFile(s string, dest map[string]interface{}, reader RunesValueReader) error {
	scanner := bytes.NewBufferString(s)
	t := newFileParser(scanner, dest, reader)
	return t.parse()
}

// RunesValueReader is a function that takes the given value (a slice of runes)
// and returns the parsed value
type RunesValueReader func([]rune) (interface{}, error)

// parser is a simple parser that takes a strvals line and parses it into a
// map representation.
//
// where sc is the source of the original data being parsed
// where data is the final parsed data from the parses with correct types
type parser struct {
	sc     *bytes.Buffer
	data   map[string]interface{}
	reader RunesValueReader
}

func newParser(sc *bytes.Buffer, data map[string]interface{}, stringBool bool) *parser {
	stringConverter := func(rs []rune) (interface{}, error) {
		return typedVal(rs, stringBool), nil
	}
	return &parser{sc: sc, data: data, reader: stringConverter}
}

func newFileParser(sc *bytes.Buffer, data map[string]interface{}, reader RunesValueReader) *parser {
	return &parser{sc: sc, data: data, reader: reader}
}

func (t *parser) parse() error {
	for {
		err := t.key(t.data)
		if err == nil {
			continue
		}
		if err == io.EOF {
			return nil
		}
		return err
	}
}

func runeSet(r []rune) map[rune]bool {
	s := make(map[rune]bool, len(r))
	for _, rr := range r {
		s[rr] = true
	}
	return s
}

func (t *parser) key(data map[string]interface{}) (reterr error) {
	defer func() {
		if r := recover(); r != nil {
			reterr = fmt.Errorf("unable to parse key: %s", r)
		}
	}()
	stop := runeSet([]rune{'=', '[', ',', '.'})
	for {
		switch k, last, err := runesUntil(t.sc, stop); {
		case err != nil:
			if len(k) == 0 {
				return err
			}
			return errors.Errorf("key %q has no value", string(k))
			//set(data, string(k), "")
			//return err
		case last == '[':
			// We are in a list index context, so we need to set an index.
			i, err := t.keyIndex()
			if err != nil {
				return errors.Wrap(err, "error parsing index")
			}
			kk := string(k)
			// Find or create target list
			list := []interface{}{}
			if _, ok := data[kk]; ok {
				list = data[kk].([]interface{})
			}

			// Now we need to get the value after the ].
			list, err = t.listItem(list, i)
			set(data, kk, list)
			return err
		case last == '=':
			//End of key. Consume =, Get value.
			// FIXME: Get value list first
			vl, e := t.valList()
			switch e {
			case nil:
				set(data, string(k), vl)
				return nil
			case io.EOF:
				set(data, string(k), "")
				return e
			case ErrNotList:
				rs, e := t.val()
				if e != nil && e != io.EOF {
					return e
				}
				v, e := t.reader(rs)
				set(data, string(k), v)
				return e
			default:
				return e
			}

		case last == ',':
			// No value given. Set the value to empty string. Return error.
			set(data, string(k), "")
			return errors.Errorf("key %q has no value (cannot end with ,)", string(k))
		case last == '.':
			// First, create or find the target map.
			inner := map[string]interface{}{}
			if _, ok := data[string(k)]; ok {
				inner = data[string(k)].(map[string]interface{})
			}

			// Recurse
			e := t.key(inner)
			if len(inner) == 0 {
				return errors.Errorf("key map %q has no value", string(k))
			}
			set(data, string(k), inner)
			return e
		}
	}
}

func set(data map[string]interface{}, key string, val interface{}) {
	// If key is empty, don't set it.
	if len(key) == 0 {
		return
	}
	data[key] = val
}

func setIndex(list []interface{}, index int, val interface{}) (l2 []interface{}, err error) {
	// There are possible index values that are out of range on a target system
	// causing a panic. This will catch the panic and return an error instead.
	// The value of the index that causes a panic varies from system to system.
	defer func() {
		if r := recover(); r != nil {
			err = fmt.Errorf("error processing index %d: %s", index, r)
		}
	}()

	if index < 0 {
		return list, fmt.Errorf("negative %d index not allowed", index)
	}
	if len(list) <= index {
		newlist := make([]interface{}, index+1)
		copy(newlist, list)
		list = newlist
	}
	list[index] = val
	return list, nil
}

func (t *parser) keyIndex() (int, error) {
	// First, get the key.
	stop := runeSet([]rune{']'})
	v, _, err := runesUntil(t.sc, stop)
	if err != nil {
		return 0, err
	}
	// v should be the index
	return strconv.Atoi(string(v))

}
func (t *parser) listItem(list []interface{}, i int) ([]interface{}, error) {
	if i < 0 {
		return list, fmt.Errorf("negative %d index not allowed", i)
	}
	stop := runeSet([]rune{'[', '.', '='})
	switch k, last, err := runesUntil(t.sc, stop); {
	case len(k) > 0:
		return list, errors.Errorf("unexpected data at end of array index: %q", k)
	case err != nil:
		return list, err
	case last == '=':
		vl, e := t.valList()
		switch e {
		case nil:
			return setIndex(list, i, vl)
		case io.EOF:
			return setIndex(list, i, "")
		case ErrNotList:
			rs, e := t.val()
			if e != nil && e != io.EOF {
				return list, e
			}
			v, e := t.reader(rs)
			if e != nil {
				return list, e
			}
			return setIndex(list, i, v)
		default:
			return list, e
		}
	case last == '[':
		// now we have a nested list. Read the index and handle.
		i, err := t.keyIndex()
		if err != nil {
			return list, errors.Wrap(err, "error parsing index")
		}
		// Now we need to get the value after the ].
		list2, err := t.listItem(list, i)
		if err != nil {
			return list, err
		}
		return setIndex(list, i, list2)
	case last == '.':
		// We have a nested object. Send to t.key
		inner := map[string]interface{}{}
		if len(list) > i {
			var ok bool
			inner, ok = list[i].(map[string]interface{})
			if !ok {
				// We have indices out of order. Initialize empty value.
				list[i] = map[string]interface{}{}
				inner = list[i].(map[string]interface{})
			}
		}

		// Recurse
		e := t.key(inner)
		if e != nil {
			return list, e
		}
		return setIndex(list, i, inner)
	default:
		return nil, errors.Errorf("parse error: unexpected token %v", last)
	}
}

func (t *parser) val() ([]rune, error) {
	stop := runeSet([]rune{','})
	v, _, err := runesUntil(t.sc, stop)
	return v, err
}

func (t *parser) valList() ([]interface{}, error) {
	r, _, e := t.sc.ReadRune()
	if e != nil {
		return []interface{}{}, e
	}

	if r != '{' {
		t.sc.UnreadRune()
		return []interface{}{}, ErrNotList
	}

	list := []interface{}{}
	stop := runeSet([]rune{',', '}'})
	for {
		switch rs, last, err := runesUntil(t.sc, stop); {
		case err != nil:
			if err == io.EOF {
				err = errors.New("list must terminate with '}'")
			}
			return list, err
		case last == '}':
			// If this is followed by ',', consume it.
			if r, _, e := t.sc.ReadRune(); e == nil && r != ',' {
				t.sc.UnreadRune()
			}
			v, e := t.reader(rs)
			list = append(list, v)
			return list, e
		case last == ',':
			v, e := t.reader(rs)
			if e != nil {
				return list, e
			}
			list = append(list, v)
		}
	}
}

func runesUntil(in io.RuneReader, stop map[rune]bool) ([]rune, rune, error) {
	v := []rune{}
	for {
		switch r, _, e := in.ReadRune(); {
		case e != nil:
			return v, r, e
		case inMap(r, stop):
			return v, r, nil
		case r == '\\':
			next, _, e := in.ReadRune()
			if e != nil {
				return v, next, e
			}
			v = append(v, next)
		default:
			v = append(v, r)
		}
	}
}

func inMap(k rune, m map[rune]bool) bool {
	_, ok := m[k]
	return ok
}

func typedVal(v []rune, st bool) interface{} {
	val := string(v)

	if st {
		return val
	}

	if strings.EqualFold(val, "true") {
		return true
	}

	if strings.EqualFold(val, "false") {
		return false
	}

	if strings.EqualFold(val, "null") {
		return nil
	}

	if strings.EqualFold(val, "0") {
		return int64(0)
	}

	// If this value does not start with zero, try parsing it to an int
	if len(val) != 0 && val[0] != '0' {
		if iv, err := strconv.ParseInt(val, 10, 64); err == nil {
			return iv
		}
	}

	return val
}
<commit_msg>fix #6116<commit_after>/*
Copyright The Helm Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package strvals

import (
	"bytes"
	"fmt"
	"io"
	"strconv"
	"strings"

	"github.com/pkg/errors"
	"sigs.k8s.io/yaml"
)

// ErrNotList indicates that a non-list was treated as a list.
var ErrNotList = errors.New("not a list")

// ToYAML takes a string of arguments and converts to a YAML document.
func ToYAML(s string) (string, error) {
	m, err := Parse(s)
	if err != nil {
		return "", err
	}
	d, err := yaml.Marshal(m)
	return strings.TrimSuffix(string(d), "\n"), err
}

// Parse parses a set line.
//
// A set line is of the form name1=value1,name2=value2
func Parse(s string) (map[string]interface{}, error) {
	vals := map[string]interface{}{}
	scanner := bytes.NewBufferString(s)
	t := newParser(scanner, vals, false)
	err := t.parse()
	return vals, err
}

// ParseString parses a set line and forces a string value.
//
// A set line is of the form name1=value1,name2=value2
func ParseString(s string) (map[string]interface{}, error) {
	vals := map[string]interface{}{}
	scanner := bytes.NewBufferString(s)
	t := newParser(scanner, vals, true)
	err := t.parse()
	return vals, err
}

// ParseInto parses a strvals line and merges the result into dest.
//
// If the strval string has a key that exists in dest, it overwrites the
// dest version.
func ParseInto(s string, dest map[string]interface{}) error {
	scanner := bytes.NewBufferString(s)
	t := newParser(scanner, dest, false)
	return t.parse()
}

// ParseFile parses a set line, but its final value is loaded from the file at the path specified by the original value.
//
// A set line is of the form name1=path1,name2=path2
//
// When the files at path1 and path2 contained "val1" and "val2" respectively, the set line is consumed as
// name1=val1,name2=val2
func ParseFile(s string, reader RunesValueReader) (map[string]interface{}, error) {
	vals := map[string]interface{}{}
	scanner := bytes.NewBufferString(s)
	t := newFileParser(scanner, vals, reader)
	err := t.parse()
	return vals, err
}

// ParseIntoString parses a strvals line and merges the result into dest.
//
// This method always returns a string as the value.
func ParseIntoString(s string, dest map[string]interface{}) error {
	scanner := bytes.NewBufferString(s)
	t := newParser(scanner, dest, true)
	return t.parse()
}

// ParseIntoFile parses a filevals line and merges the result into dest.
//
// This method always returns a string as the value.
func ParseIntoFile(s string, dest map[string]interface{}, reader RunesValueReader) error {
	scanner := bytes.NewBufferString(s)
	t := newFileParser(scanner, dest, reader)
	return t.parse()
}

// RunesValueReader is a function that takes the given value (a slice of runes)
// and returns the parsed value
type RunesValueReader func([]rune) (interface{}, error)

// parser is a simple parser that takes a strvals line and parses it into a
// map representation.
//
// where sc is the source of the original data being parsed
// where data is the final parsed data from the parses with correct types
type parser struct {
	sc     *bytes.Buffer
	data   map[string]interface{}
	reader RunesValueReader
}

func newParser(sc *bytes.Buffer, data map[string]interface{}, stringBool bool) *parser {
	stringConverter := func(rs []rune) (interface{}, error) {
		return typedVal(rs, stringBool), nil
	}
	return &parser{sc: sc, data: data, reader: stringConverter}
}

func newFileParser(sc *bytes.Buffer, data map[string]interface{}, reader RunesValueReader) *parser {
	return &parser{sc: sc, data: data, reader: reader}
}

func (t *parser) parse() error {
	for {
		err := t.key(t.data)
		if err == nil {
			continue
		}
		if err == io.EOF {
			return nil
		}
		return err
	}
}

func runeSet(r []rune) map[rune]bool {
	s := make(map[rune]bool, len(r))
	for _, rr := range r {
		s[rr] = true
	}
	return s
}

func (t *parser) key(data map[string]interface{}) (reterr error) {
	defer func() {
		if r := recover(); r != nil {
			reterr = fmt.Errorf("unable to parse key: %s", r)
		}
	}()
	stop := runeSet([]rune{'=', '[', ',', '.'})
	for {
		switch k, last, err := runesUntil(t.sc, stop); {
		case err != nil:
			if len(k) == 0 {
				return err
			}
			return errors.Errorf("key %q has no value", string(k))
			//set(data, string(k), "")
			//return err
		case last == '[':
			// We are in a list index context, so we need to set an index.
			i, err := t.keyIndex()
			if err != nil {
				return errors.Wrap(err, "error parsing index")
			}
			kk := string(k)
			// Find or create target list
			list := []interface{}{}
			if _, ok := data[kk]; ok {
				list = data[kk].([]interface{})
			}

			// Now we need to get the value after the ].
			list, err = t.listItem(list, i)
			set(data, kk, list)
			return err
		case last == '=':
			//End of key. Consume =, Get value.
			// FIXME: Get value list first
			vl, e := t.valList()
			switch e {
			case nil:
				set(data, string(k), vl)
				return nil
			case io.EOF:
				set(data, string(k), "")
				return e
			case ErrNotList:
				rs, e := t.val()
				if e != nil && e != io.EOF {
					return e
				}
				v, e := t.reader(rs)
				set(data, string(k), v)
				return e
			default:
				return e
			}

		case last == ',':
			// No value given. Set the value to empty string. Return error.
			set(data, string(k), "")
			return errors.Errorf("key %q has no value (cannot end with ,)", string(k))
		case last == '.':
			// First, create or find the target map.
			inner := map[string]interface{}{}
			if _, ok := data[string(k)]; ok {
				inner = data[string(k)].(map[string]interface{})
			}

			// Recurse
			e := t.key(inner)
			if len(inner) == 0 {
				return errors.Errorf("key map %q has no value", string(k))
			}
			set(data, string(k), inner)
			return e
		}
	}
}

func set(data map[string]interface{}, key string, val interface{}) {
	// If key is empty, don't set it.
	if len(key) == 0 {
		return
	}
	data[key] = val
}

func setIndex(list []interface{}, index int, val interface{}) (l2 []interface{}, err error) {
	// There are possible index values that are out of range on a target system
	// causing a panic. This will catch the panic and return an error instead.
	// The value of the index that causes a panic varies from system to system.
	defer func() {
		if r := recover(); r != nil {
			err = fmt.Errorf("error processing index %d: %s", index, r)
		}
	}()

	if index < 0 {
		return list, fmt.Errorf("negative %d index not allowed", index)
	}
	if len(list) <= index {
		newlist := make([]interface{}, index+1)
		copy(newlist, list)
		list = newlist
	}
	list[index] = val
	return list, nil
}

func (t *parser) keyIndex() (int, error) {
	// First, get the key.
	stop := runeSet([]rune{']'})
	v, _, err := runesUntil(t.sc, stop)
	if err != nil {
		return 0, err
	}
	// v should be the index
	return strconv.Atoi(string(v))

}
func (t *parser) listItem(list []interface{}, i int) ([]interface{}, error) {
	if i < 0 {
		return list, fmt.Errorf("negative %d index not allowed", i)
	}
	stop := runeSet([]rune{'[', '.', '='})
	switch k, last, err := runesUntil(t.sc, stop); {
	case len(k) > 0:
		return list, errors.Errorf("unexpected data at end of array index: %q", k)
	case err != nil:
		return list, err
	case last == '=':
		vl, e := t.valList()
		switch e {
		case nil:
			return setIndex(list, i, vl)
		case io.EOF:
			return setIndex(list, i, "")
		case ErrNotList:
			rs, e := t.val()
			if e != nil && e != io.EOF {
				return list, e
			}
			v, e := t.reader(rs)
			if e != nil {
				return list, e
			}
			return setIndex(list, i, v)
		default:
			return list, e
		}
	case last == '[':
		// now we have a nested list. Read the index and handle.
		i, err := t.keyIndex()
		if err != nil {
			return list, errors.Wrap(err, "error parsing index")
		}
		var crtList []interface{}
		if len(list) > i {
			// If nested list already exists, take the value of list to next cycle.
			crtList = list[i].([]interface{})
		} else {
			crtList = list
		}
		// Now we need to get the value after the ].
		list2, err := t.listItem(crtList, i)
		if err != nil {
			return list, err
		}
		return setIndex(list, i, list2)
	case last == '.':
		// We have a nested object. Send to t.key
		inner := map[string]interface{}{}
		if len(list) > i {
			var ok bool
			inner, ok = list[i].(map[string]interface{})
			if !ok {
				// We have indices out of order. Initialize empty value.
				list[i] = map[string]interface{}{}
				inner = list[i].(map[string]interface{})
			}
		}

		// Recurse
		e := t.key(inner)
		if e != nil {
			return list, e
		}
		return setIndex(list, i, inner)
	default:
		return nil, errors.Errorf("parse error: unexpected token %v", last)
	}
}

func (t *parser) val() ([]rune, error) {
	stop := runeSet([]rune{','})
	v, _, err := runesUntil(t.sc, stop)
	return v, err
}

func (t *parser) valList() ([]interface{}, error) {
	r, _, e := t.sc.ReadRune()
	if e != nil {
		return []interface{}{}, e
	}

	if r != '{' {
		t.sc.UnreadRune()
		return []interface{}{}, ErrNotList
	}

	list := []interface{}{}
	stop := runeSet([]rune{',', '}'})
	for {
		switch rs, last, err := runesUntil(t.sc, stop); {
		case err != nil:
			if err == io.EOF {
				err = errors.New("list must terminate with '}'")
			}
			return list, err
		case last == '}':
			// If this is followed by ',', consume it.
			if r, _, e := t.sc.ReadRune(); e == nil && r != ',' {
				t.sc.UnreadRune()
			}
			v, e := t.reader(rs)
			list = append(list, v)
			return list, e
		case last == ',':
			v, e := t.reader(rs)
			if e != nil {
				return list, e
			}
			list = append(list, v)
		}
	}
}

func runesUntil(in io.RuneReader, stop map[rune]bool) ([]rune, rune, error) {
	v := []rune{}
	for {
		switch r, _, e := in.ReadRune(); {
		case e != nil:
			return v, r, e
		case inMap(r, stop):
			return v, r, nil
		case r == '\\':
			next, _, e := in.ReadRune()
			if e != nil {
				return v, next, e
			}
			v = append(v, next)
		default:
			v = append(v, r)
		}
	}
}

func inMap(k rune, m map[rune]bool) bool {
	_, ok := m[k]
	return ok
}

func typedVal(v []rune, st bool) interface{} {
	val := string(v)

	if st {
		return val
	}

	if strings.EqualFold(val, "true") {
		return true
	}

	if strings.EqualFold(val, "false") {
		return false
	}

	if strings.EqualFold(val, "null") {
		return nil
	}

	if strings.EqualFold(val, "0") {
		return int64(0)
	}

	// If this value does not start with zero, try parsing it to an int
	if len(val) != 0 && val[0] != '0' {
		if iv, err := strconv.ParseInt(val, 10, 64); err == nil {
			return iv
		}
	}

	return val
}
<|endoftext|>


<commit_before>package tanka

import (
	"fmt"

	"k8s.io/apimachinery/pkg/labels"

	"github.com/grafana/tanka/pkg/spec/v1alpha1"
)

const defaultParallelism = 8

type parallelOpts struct {
	JsonnetOpts JsonnetOpts
	Selector    labels.Selector
	Parallelism int
}

// parallelLoadEnvironments evaluates multiple environments in parallel
func parallelLoadEnvironments(paths []string, opts parallelOpts) ([]*v1alpha1.Environment, error) {
	jobsCh := make(chan parallelJob)
	outCh := make(chan parallelOut)

	if opts.Parallelism <= 0 {
		opts.Parallelism = defaultParallelism
	}

	for i := 0; i < opts.Parallelism; i++ {
		go parallelWorker(jobsCh, outCh)
	}

	for _, path := range paths {
		jobsCh <- parallelJob{
			path: path,
			opts: Opts{JsonnetOpts: opts.JsonnetOpts},
		}
	}
	close(jobsCh)

	var envs []*v1alpha1.Environment
	var errors []error
	for i := 0; i < len(paths); i++ {
		out := <-outCh
		if out.err != nil {
			errors = append(errors, out.err)
			continue
		}
		if opts.Selector == nil || opts.Selector.Empty() || opts.Selector.Matches(out.env.Metadata) {
			envs = append(envs, out.env)
		}
	}

	if len(errors) != 0 {
		return envs, ErrParallel{errors: errors}
	}

	return envs, nil
}

type parallelJob struct {
	path string
	opts Opts
}

type parallelOut struct {
	env *v1alpha1.Environment
	err error
}

func parallelWorker(jobsCh <-chan parallelJob, outCh chan parallelOut) {
	for job := range jobsCh {
		env, err := LoadEnvironment(job.path, job.opts)
		if err != nil {
			err = fmt.Errorf("%s:\n %w", job.path, err)
		}
		outCh <- parallelOut{env: env, err: err}
	}
}
<commit_msg>fix: recursive export (#475)<commit_after>package tanka

import (
	"fmt"

	"k8s.io/apimachinery/pkg/labels"

	"github.com/grafana/tanka/pkg/spec/v1alpha1"
)

const defaultParallelism = 8

type parallelOpts struct {
	JsonnetOpts JsonnetOpts
	Selector    labels.Selector
	Parallelism int
}

// parallelLoadEnvironments evaluates multiple environments in parallel
func parallelLoadEnvironments(paths []string, opts parallelOpts) ([]*v1alpha1.Environment, error) {
	jobsCh := make(chan parallelJob)
	outCh := make(chan parallelOut, len(paths))

	if opts.Parallelism <= 0 {
		opts.Parallelism = defaultParallelism
	}

	for i := 0; i < opts.Parallelism; i++ {
		go parallelWorker(jobsCh, outCh)
	}

	for _, path := range paths {
		jobsCh <- parallelJob{
			path: path,
			opts: Opts{JsonnetOpts: opts.JsonnetOpts},
		}
	}
	close(jobsCh)

	var envs []*v1alpha1.Environment
	var errors []error
	for i := 0; i < len(paths); i++ {
		out := <-outCh
		if out.err != nil {
			errors = append(errors, out.err)
			continue
		}
		if opts.Selector == nil || opts.Selector.Empty() || opts.Selector.Matches(out.env.Metadata) {
			envs = append(envs, out.env)
		}
	}

	if len(errors) != 0 {
		return envs, ErrParallel{errors: errors}
	}

	return envs, nil
}

type parallelJob struct {
	path string
	opts Opts
}

type parallelOut struct {
	env *v1alpha1.Environment
	err error
}

func parallelWorker(jobsCh <-chan parallelJob, outCh chan parallelOut) {
	for job := range jobsCh {
		env, err := LoadEnvironment(job.path, job.opts)
		if err != nil {
			err = fmt.Errorf("%s:\n %w", job.path, err)
		}
		outCh <- parallelOut{env: env, err: err}
	}
}
<|endoftext|>


<commit_before>package platform

import (
	"image"
	"image/color"
	"sync"

	"golang.org/x/exp/shiny/driver"
	"golang.org/x/exp/shiny/screen"
	"golang.org/x/image/math/f64"
	"golang.org/x/mobile/event/lifecycle"
	"golang.org/x/mobile/event/key"
	"golang.org/x/mobile/event/paint"
	"golang.org/x/mobile/event/size"
)

// WindowState contains what the window loop and program proper both need to touch
type WindowState struct {
	// the Width of the framebuffer
	Width int
	// the Height of the framebuffer
	Height int

	// a Mutex that must be held when reading or writing in WindowState
	Mutex sync.Mutex

	// Pix is the raw RGBA bytes of the framebuffer
	Pix []byte

	keycodeMap dumbMap
	runeMap dumbMap

	eventQueue screen.EventDeque
	drawRequested bool
}

type dumbMap struct {
	slots [256]dumbKeyVal
}
type dumbKeyVal struct {
	next *dumbKeyVal
	key int32
	val bool
}
func (m *dumbMap) Set(k int32, v bool) {
	hash := k & 0xff
	slot := &m.slots[hash]
	if slot.key == k {
		slot.val = v
		return
	}
	if slot.key == 0 {
		slot.key = k
		slot.val = v
		return
	}
	for slot.next != nil {
		slot = slot.next
		if slot.key == k {
			slot.val = v
			return
		}
	}
	slot.next = &dumbKeyVal{key: k, val: v}
}
func (m *dumbMap) Get(k int32) bool {
	hash := k & 0xff
	slotCopy := m.slots[hash]
	if slotCopy.key == k {
		return slotCopy.val
	}
	slot := slotCopy.next
	for slot != nil {
		if slot.key == k {
			return slot.val
		}
		slot = slot.next
	}
	return false
}

// CharIsDown returns the key state for that char
func (s *WindowState) CharIsDown(c rune) bool {
	return s.runeMap.Get(int32(c))
}
// CodeIsDown returns the key state for that keyCode
func (s *WindowState) CodeIsDown(c key.Code) bool {
	return s.keycodeMap.Get(int32(c))
}

func (s *WindowState) updateKeyboardState(e key.Event) {
	if e.Direction == key.DirRelease {
		s.keycodeMap.Set(int32(e.Code), false)
		s.runeMap.Set(int32(e.Rune), false)
	} else if e.Direction == key.DirPress {
		s.keycodeMap.Set(int32(e.Code), true)
		s.runeMap.Set(int32(e.Rune), true)
	}
}

// RequestDraw puts a draw request on the window loop queue
// It is assumed the mutex is already held when this function is called.
func (s *WindowState) RequestDraw() {
	if !s.drawRequested {
		s.eventQueue.Send(drawRequest{})
		s.drawRequested = true
	}
}

type drawRequest struct {}

// InitDisplayLoop creates a window and starts event loop
func InitDisplayLoop(windowWidth, windowHeight, frameWidth, frameHeight int, updateLoop func(*WindowState)) {
	driver.Main(func (s screen.Screen) {

		w, err := s.NewWindow(&screen.NewWindowOptions{windowWidth, windowHeight})
		if err != nil {
			panic(err)
		}
		defer w.Release()

		buf, err := s.NewBuffer(image.Point{frameWidth, frameHeight})
		if err != nil {
			panic(err)
		}
		tex, err := s.NewTexture(image.Point{frameWidth, frameHeight})
		if err != nil {
			panic(err)
		}

		windowState := WindowState{
			Width: frameWidth,
			Height: frameHeight,
			Pix: make([]byte, 4*frameWidth*frameHeight),
			eventQueue: w,
			keycodeMap: dumbMap{},
			runeMap: dumbMap{},
		}

		go updateLoop(&windowState)

		szRect := buf.Bounds()
		needFullRepaint := true

		for {
			publish := false

			switch e := w.NextEvent().(type) {
			case lifecycle.Event:
				if e.To == lifecycle.StageDead {
					return
				}

			case key.Event:
				windowState.Mutex.Lock()
				windowState.updateKeyboardState(e)
				windowState.Mutex.Unlock()

			case drawRequest:
				windowState.Mutex.Lock()
				copy(buf.RGBA().Pix, windowState.Pix)
				tex.Upload(image.Point{0, 0}, buf, buf.Bounds())
				windowState.drawRequested = false
				windowState.Mutex.Unlock()
				publish = true

			case size.Event:
				szRect = e.Bounds()

			case paint.Event:
				needFullRepaint = true
				publish = true
			}

			if publish {
				scaleFactX := float64(szRect.Max.X) / float64(tex.Bounds().Max.X)
				scaleFactY := float64(szRect.Max.Y) / float64(tex.Bounds().Max.Y)
				scaleFact := scaleFactX
				if scaleFactY < scaleFact {
					scaleFact = scaleFactY
				}
				// NOTE: flicker happens when scale is not an integer
				scaleFact = float64(int(scaleFact))
				newWidth := int(scaleFact * float64(tex.Bounds().Max.X))
				centerX := float64(szRect.Max.X/2 - newWidth/2)
				src2dst := f64.Aff3 {
					float64(int(scaleFact)), 0, centerX,
					0, float64(int(scaleFact)), 0,
				}
				identTrans := f64.Aff3 {
					1, 0, 0,
					0, 1, 0,
				}
				// get flicker when we do two draws all the time, so
				// only do it when we resize or get moved on/offscreen
				if needFullRepaint {
					w.DrawUniform(identTrans, color.Black, szRect, screen.Src, nil)
					needFullRepaint = false
				}
				w.Draw(src2dst, tex, tex.Bounds(), screen.Src, nil)
				w.Publish()
			}
		}
	})
}
<commit_msg>Switch to arr/map approach for keydown checks<commit_after>package platform

import (
	"image"
	"image/color"
	"sync"

	"golang.org/x/exp/shiny/driver"
	"golang.org/x/exp/shiny/screen"
	"golang.org/x/image/math/f64"
	"golang.org/x/mobile/event/lifecycle"
	"golang.org/x/mobile/event/key"
	"golang.org/x/mobile/event/paint"
	"golang.org/x/mobile/event/size"
)

// WindowState contains what the window loop and program proper both need to touch
type WindowState struct {
	// the Width of the framebuffer
	Width int
	// the Height of the framebuffer
	Height int

	// a Mutex that must be held when reading or writing in WindowState
	Mutex sync.Mutex

	// Pix is the raw RGBA bytes of the framebuffer
	Pix []byte

	keyCodeArray [256]bool
	keyCodeMap map[key.Code]bool
	keyCharArray [256]bool
	keyCharMap map[rune]bool

	eventQueue screen.EventDeque
	drawRequested bool
}

// CharIsDown returns the key state for that char
func (s *WindowState) CharIsDown(c rune) bool {
	if c < 256 {
		return s.keyCharArray[byte(c)]
	}
	return s.keyCharMap[c]
}
// CodeIsDown returns the key state for that keyCode
func (s *WindowState) CodeIsDown(c key.Code) bool {
	if c < 256 {
		return s.keyCodeArray[byte(c)]
	}
	return s.keyCodeMap[c]
}

func (s *WindowState) updateKeyboardState(e key.Event) {
	setVal := e.Direction == key.DirPress
	if setVal || e.Direction == key.DirRelease {
		if e.Code < 256 {
			s.keyCodeArray[byte(e.Code)] = setVal
		}
		s.keyCodeMap[e.Code] = setVal
		if e.Rune < 256 {
			s.keyCharArray[byte(e.Rune)] = setVal
		}
		s.keyCharMap[e.Rune] = setVal
	}
}

// RequestDraw puts a draw request on the window loop queue
// It is assumed the mutex is already held when this function is called.
func (s *WindowState) RequestDraw() {
	if !s.drawRequested {
		s.eventQueue.Send(drawRequest{})
		s.drawRequested = true
	}
}

type drawRequest struct {}

// InitDisplayLoop creates a window and starts event loop
func InitDisplayLoop(windowWidth, windowHeight, frameWidth, frameHeight int, updateLoop func(*WindowState)) {
	driver.Main(func (s screen.Screen) {

		w, err := s.NewWindow(&screen.NewWindowOptions{windowWidth, windowHeight})
		if err != nil {
			panic(err)
		}
		defer w.Release()

		buf, err := s.NewBuffer(image.Point{frameWidth, frameHeight})
		if err != nil {
			panic(err)
		}
		tex, err := s.NewTexture(image.Point{frameWidth, frameHeight})
		if err != nil {
			panic(err)
		}

		windowState := WindowState{
			Width: frameWidth,
			Height: frameHeight,
			Pix: make([]byte, 4*frameWidth*frameHeight),
			eventQueue: w,
			keyCodeMap: map[key.Code]bool{},
			keyCharMap: map[rune]bool{},
		}

		go updateLoop(&windowState)

		szRect := buf.Bounds()
		needFullRepaint := true

		for {
			publish := false

			switch e := w.NextEvent().(type) {
			case lifecycle.Event:
				if e.To == lifecycle.StageDead {
					return
				}

			case key.Event:
				windowState.Mutex.Lock()
				windowState.updateKeyboardState(e)
				windowState.Mutex.Unlock()

			case drawRequest:
				windowState.Mutex.Lock()
				copy(buf.RGBA().Pix, windowState.Pix)
				tex.Upload(image.Point{0, 0}, buf, buf.Bounds())
				windowState.drawRequested = false
				windowState.Mutex.Unlock()
				publish = true

			case size.Event:
				szRect = e.Bounds()

			case paint.Event:
				needFullRepaint = true
				publish = true
			}

			if publish {
				scaleFactX := float64(szRect.Max.X) / float64(tex.Bounds().Max.X)
				scaleFactY := float64(szRect.Max.Y) / float64(tex.Bounds().Max.Y)
				scaleFact := scaleFactX
				if scaleFactY < scaleFact {
					scaleFact = scaleFactY
				}
				// NOTE: flicker happens when scale is not an integer
				scaleFact = float64(int(scaleFact))
				newWidth := int(scaleFact * float64(tex.Bounds().Max.X))
				centerX := float64(szRect.Max.X/2 - newWidth/2)
				src2dst := f64.Aff3 {
					float64(int(scaleFact)), 0, centerX,
					0, float64(int(scaleFact)), 0,
				}
				identTrans := f64.Aff3 {
					1, 0, 0,
					0, 1, 0,
				}
				// get flicker when we do two draws all the time, so
				// only do it when we resize or get moved on/offscreen
				if needFullRepaint {
					w.DrawUniform(identTrans, color.Black, szRect, screen.Src, nil)
					needFullRepaint = false
				}
				w.Draw(src2dst, tex, tex.Bounds(), screen.Src, nil)
				w.Publish()
			}
		}
	})
}
<|endoftext|>