<commit_before>// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build ignore

// The vet/all command runs go vet on the standard library and commands.
// It compares the output against a set of whitelists
// maintained in the whitelist directory.
//
// This program attempts to build packages from golang.org/x/tools,
// which must be in your GOPATH.
package main

import (
	"bufio"
	"bytes"
	"flag"
	"fmt"
	"go/build"
	"go/types"
	"internal/testenv"
	"io"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strings"
	"sync/atomic"
)

var (
	flagPlatforms = flag.String("p", "", "platform(s) to use e.g. linux/amd64,darwin/386")
	flagAll       = flag.Bool("all", false, "run all platforms")
	flagNoLines   = flag.Bool("n", false, "don't print line numbers")
)

var cmdGoPath string
var failed uint32 // updated atomically

func main() {
	log.SetPrefix("vet/all: ")
	log.SetFlags(log.Lshortfile)

	var err error
	cmdGoPath, err = testenv.GoTool()
	if err != nil {
		log.Print("could not find cmd/go; skipping")
		// We're on a platform that can't run cmd/go.
		// We want this script to be able to run as part of all.bash,
		// so return cleanly rather than with exit code 1.
		return
	}

	flag.Parse()
	switch {
	case *flagAll && *flagPlatforms != "":
		log.Print("-all and -p flags are incompatible")
		flag.Usage()
		os.Exit(2)
	case *flagPlatforms != "":
		vetPlatforms(parseFlagPlatforms())
	case *flagAll:
		vetPlatforms(allPlatforms())
	default:
		hostPlatform.vet()
	}
	if atomic.LoadUint32(&failed) != 0 {
		os.Exit(1)
	}
}

var hostPlatform = platform{os: build.Default.GOOS, arch: build.Default.GOARCH}

func allPlatforms() []platform {
	var pp []platform
	cmd := exec.Command(cmdGoPath, "tool", "dist", "list")
	cmd.Stderr = new(strings.Builder)
	out, err := cmd.Output()
	if err != nil {
		log.Fatalf("%s: %v\n%s", strings.Join(cmd.Args, " "), err, cmd.Stderr)
	}
	lines := bytes.Split(out, []byte{'\n'})
	for _, line := range lines {
		if len(line) == 0 {
			continue
		}
		pp = append(pp, parsePlatform(string(line)))
	}
	return pp
}

func parseFlagPlatforms() []platform {
	var pp []platform
	components := strings.Split(*flagPlatforms, ",")
	for _, c := range components {
		pp = append(pp, parsePlatform(c))
	}
	return pp
}

func parsePlatform(s string) platform {
	vv := strings.Split(s, "/")
	if len(vv) != 2 {
		log.Fatalf("could not parse platform %s, must be of form goos/goarch", s)
	}
	return platform{os: vv[0], arch: vv[1]}
}

type whitelist map[string]int

// load adds entries from the whitelist file, if present, for os/arch to w.
func (w whitelist) load(goos string, goarch string) {
	sz := types.SizesFor("gc", goarch)
	if sz == nil {
		log.Fatalf("unknown type sizes for arch %q", goarch)
	}
	archbits := 8 * sz.Sizeof(types.Typ[types.UnsafePointer])

	// Look up whether goarch has a shared arch suffix,
	// such as mips64x for mips64 and mips64le.
	archsuff := goarch
	if x, ok := archAsmX[goarch]; ok {
		archsuff = x
	}

	// Load whitelists.
	filenames := []string{
		"all.txt",
		goos + ".txt",
		goarch + ".txt",
		goos + "_" + goarch + ".txt",
		fmt.Sprintf("%dbit.txt", archbits),
	}
	if goarch != archsuff {
		filenames = append(filenames,
			archsuff+".txt",
			goos+"_"+archsuff+".txt",
		)
	}

	// We allow error message templates using GOOS and GOARCH.
	if goos == "android" {
		goos = "linux" // so many special cases :(
	}

	// Read whitelists and do template substitution.
	replace := strings.NewReplacer("GOOS", goos, "GOARCH", goarch, "ARCHSUFF", archsuff)

	for _, filename := range filenames {
		path := filepath.Join("whitelist", filename)
		f, err := os.Open(path)
		if err != nil {
			// Allow not-exist errors; not all combinations have whitelists.
			if os.IsNotExist(err) {
				continue
			}
			log.Fatal(err)
		}
		scan := bufio.NewScanner(f)
		for scan.Scan() {
			line := scan.Text()
			if len(line) == 0 || strings.HasPrefix(line, "//") {
				continue
			}
			w[replace.Replace(line)]++
		}
		if err := scan.Err(); err != nil {
			log.Fatal(err)
		}
	}
}

type platform struct {
	os   string
	arch string
}

func (p platform) String() string {
	return p.os + "/" + p.arch
}

// ignorePathPrefixes are file path prefixes that should be ignored wholesale.
var ignorePathPrefixes = [...]string{
	// These testdata dirs have lots of intentionally broken/bad code for tests.
	"cmd/go/testdata/",
	"cmd/vet/testdata/",
	"go/printer/testdata/",
}

func vetPlatforms(pp []platform) {
	for _, p := range pp {
		p.vet()
	}
}

func (p platform) vet() {
	if p.os == "linux" && (p.arch == "riscv64" || p.arch == "sparc64") {
		// TODO(tklauser): enable as soon as these ports have fully landed
		fmt.Printf("skipping %s/%s\n", p.os, p.arch)
		return
	}

	if p.os == "windows" && p.arch == "arm" {
		// TODO(jordanrh1): enable as soon as the windows/arm port has fully landed
		fmt.Println("skipping windows/arm")
		return
	}

	if p.os == "aix" && p.arch == "ppc64" {
		// TODO(aix): enable as soon as the aix/ppc64 port has fully landed
		fmt.Println("skipping aix/ppc64")
		return
	}

	var buf bytes.Buffer
	fmt.Fprintf(&buf, "go run main.go -p %s\n", p)

	// Load whitelist(s).
	w := make(whitelist)
	w.load(p.os, p.arch)

	var vetCmd []string

	if os.Getenv("GO_BUILDER_NAME") == "" {
		vetCmd = []string{cmdGoPath, "vet"}
	} else {
		// Build the go/packages-based vet command from the x/tools
		// repo. It is considerably faster than "go vet", which rebuilds
		// the standard library.
		tmpdir, err := ioutil.TempDir("", "cmd-vet-all")
		if err != nil {
			log.Fatal(err)
		}
		defer os.RemoveAll(tmpdir)

		vetTool := filepath.Join(tmpdir, "vet")
		vetCmd = []string{
			vetTool,
			"-nilness=0", // expensive, uses SSA
		}

		cmd := exec.Command(cmdGoPath, "build", "-o", vetTool, "golang.org/x/tools/go/analysis/cmd/vet")
		cmd.Env = os.Environ()

		// golang.org/x/tools does not have a vendor directory, so don't try to use
		// one in module mode.
		for i, v := range cmd.Env {
			if strings.HasPrefix(v, "GOFLAGS=") {
				var goflags []string
				for _, f := range strings.Fields(strings.TrimPrefix(v, "GOFLAGS=")) {
					if f != "-mod=vendor" && f != "--mod=vendor" {
						goflags = append(goflags, f)
					}
				}
				cmd.Env[i] = strings.Join(goflags, " ")
			}
		}

		// The coordinator places a copy of golang.org/x/tools in GOPATH.
		// If we can find it there, use that specific version.
		for _, gp := range filepath.SplitList(os.Getenv("GOPATH")) {
			gopathDir := filepath.Join(gp, "src", "golang.org", "x", "tools", "go", "analysis", "cmd", "vet")
			if _, err := os.Stat(gopathDir); err == nil {
				cmd.Dir = gopathDir
			}
		}
		if cmd.Dir == "" {
			// Otherwise, move to tmpdir and let the module loader resolve the latest version.
			cmd.Dir = tmpdir
		}

		cmd.Stderr = os.Stderr
		cmd.Stdout = os.Stderr
		if err := cmd.Run(); err != nil {
			log.Fatalf("%s: %v", strings.Join(cmd.Args, " "), err)
		}
	}

	// TODO: The unsafeptr checks are disabled for now,
	// because there are so many false positives,
	// and no clear way to improve vet to eliminate large chunks of them.
	// And having them in the whitelists will just cause annoyance
	// and churn when working on the runtime.
	cmd := exec.Command(vetCmd[0],
		append(vetCmd[1:],
			"-unsafeptr=0",
			"std",
			"cmd/...",
			"cmd/compile/internal/gc/testdata",
		)...)
	cmd.Dir = filepath.Join(runtime.GOROOT(), "src")
	cmd.Env = append(os.Environ(), "GOOS="+p.os, "GOARCH="+p.arch, "CGO_ENABLED=0")
	stderr, err := cmd.StderrPipe()
	if err != nil {
		log.Fatal(err)
	}
	if err := cmd.Start(); err != nil {
		log.Fatal(err)
	}

	// Process vet output.
	scan := bufio.NewScanner(stderr)
	var parseFailed bool
NextLine:
	for scan.Scan() {
		line := scan.Text()
		if strings.HasPrefix(line, "vet: ") {
			// Typecheck failure: Malformed syntax or multiple packages or the like.
			// This will yield nicer error messages elsewhere, so ignore them here.

			// This includes warnings from asmdecl of the form:
			//   "vet: foo.s:16: [amd64] cannot check cross-package assembly function"
			continue
		}

		if strings.HasPrefix(line, "panic: ") {
			// Panic in vet. Don't filter anything, we want the complete output.
			parseFailed = true
			fmt.Fprintf(os.Stderr, "panic in vet (to reproduce: go run main.go -p %s):\n", p)
			fmt.Fprintln(os.Stderr, line)
			io.Copy(os.Stderr, stderr)
			break
		}
		if strings.HasPrefix(line, "# ") {
			// 'go vet' prefixes the output of each vet invocation by a comment:
			//    # [package]
			continue
		}

		// Parse line.
		// Assume the part before the first ": "
		// is the "file:line:col: " information.
		// TODO(adonovan): parse vet -json output.
		var file, lineno, msg string
		if i := strings.Index(line, ": "); i >= 0 {
			msg = line[i+len(": "):]

			words := strings.Split(line[:i], ":")
			switch len(words) {
			case 3:
				_ = words[2] // ignore column
				fallthrough
			case 2:
				lineno = words[1]
				fallthrough
			case 1:
				file = words[0]

				// Make the file name relative to GOROOT/src.
				if rel, err := filepath.Rel(cmd.Dir, file); err == nil {
					file = rel
				}
			default:
				// error: too many columns
			}
		}
		if file == "" {
			if !parseFailed {
				parseFailed = true
				fmt.Fprintf(os.Stderr, "failed to parse %s output:\n# %s\n", p, strings.Join(cmd.Args, " "))
			}
			fmt.Fprintln(os.Stderr, line)
			continue
		}

		msg = strings.TrimSpace(msg)

		for _, ignore := range ignorePathPrefixes {
			if strings.HasPrefix(file, filepath.FromSlash(ignore)) {
				continue NextLine
			}
		}

		key := file + ": " + msg
		if w[key] == 0 {
			// Vet error with no match in the whitelist. Print it.
			if *flagNoLines {
				fmt.Fprintf(&buf, "%s: %s\n", file, msg)
			} else {
				fmt.Fprintf(&buf, "%s:%s: %s\n", file, lineno, msg)
			}
			atomic.StoreUint32(&failed, 1)
			continue
		}
		w[key]--
	}
	if parseFailed {
		atomic.StoreUint32(&failed, 1)
		return
	}
	if scan.Err() != nil {
		log.Fatalf("failed to scan vet output: %v", scan.Err())
	}
	err = cmd.Wait()
	// We expect vet to fail.
	// Make sure it has failed appropriately, though (for example, not a PathError).
	if _, ok := err.(*exec.ExitError); !ok {
		log.Fatalf("unexpected go vet execution failure: %v", err)
	}
	printedHeader := false
	if len(w) > 0 {
		for k, v := range w {
			if v != 0 {
				if !printedHeader {
					fmt.Fprintln(&buf, "unmatched whitelist entries:")
					printedHeader = true
				}
				for i := 0; i < v; i++ {
					fmt.Fprintln(&buf, k)
				}
				atomic.StoreUint32(&failed, 1)
			}
		}
	}

	os.Stdout.Write(buf.Bytes())
}

// archAsmX maps architectures to the suffix usually used for their assembly files,
// if different than the arch name itself.
var archAsmX = map[string]string{
	"android":  "linux",
	"mips64":   "mips64x",
	"mips64le": "mips64x",
	"mips":     "mipsx",
	"mipsle":   "mipsx",
	"ppc64":    "ppc64x",
	"ppc64le":  "ppc64x",
}
<commit_msg>cmd/vet/all: don't use the x/tools vet in GOPATH<commit_after>// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build ignore

// The vet/all command runs go vet on the standard library and commands.
// It compares the output against a set of whitelists
// maintained in the whitelist directory.
//
// This program attempts to build packages from golang.org/x/tools,
// which must be in your GOPATH.
package main

import (
	"bufio"
	"bytes"
	"flag"
	"fmt"
	"go/build"
	"go/types"
	"internal/testenv"
	"io"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strings"
	"sync/atomic"
)

var (
	flagPlatforms = flag.String("p", "", "platform(s) to use e.g. linux/amd64,darwin/386")
	flagAll       = flag.Bool("all", false, "run all platforms")
	flagNoLines   = flag.Bool("n", false, "don't print line numbers")
)

var cmdGoPath string
var failed uint32 // updated atomically

func main() {
	log.SetPrefix("vet/all: ")
	log.SetFlags(log.Lshortfile)

	var err error
	cmdGoPath, err = testenv.GoTool()
	if err != nil {
		log.Print("could not find cmd/go; skipping")
		// We're on a platform that can't run cmd/go.
		// We want this script to be able to run as part of all.bash,
		// so return cleanly rather than with exit code 1.
		return
	}

	flag.Parse()
	switch {
	case *flagAll && *flagPlatforms != "":
		log.Print("-all and -p flags are incompatible")
		flag.Usage()
		os.Exit(2)
	case *flagPlatforms != "":
		vetPlatforms(parseFlagPlatforms())
	case *flagAll:
		vetPlatforms(allPlatforms())
	default:
		hostPlatform.vet()
	}
	if atomic.LoadUint32(&failed) != 0 {
		os.Exit(1)
	}
}

var hostPlatform = platform{os: build.Default.GOOS, arch: build.Default.GOARCH}

func allPlatforms() []platform {
	var pp []platform
	cmd := exec.Command(cmdGoPath, "tool", "dist", "list")
	cmd.Stderr = new(strings.Builder)
	out, err := cmd.Output()
	if err != nil {
		log.Fatalf("%s: %v\n%s", strings.Join(cmd.Args, " "), err, cmd.Stderr)
	}
	lines := bytes.Split(out, []byte{'\n'})
	for _, line := range lines {
		if len(line) == 0 {
			continue
		}
		pp = append(pp, parsePlatform(string(line)))
	}
	return pp
}

func parseFlagPlatforms() []platform {
	var pp []platform
	components := strings.Split(*flagPlatforms, ",")
	for _, c := range components {
		pp = append(pp, parsePlatform(c))
	}
	return pp
}

func parsePlatform(s string) platform {
	vv := strings.Split(s, "/")
	if len(vv) != 2 {
		log.Fatalf("could not parse platform %s, must be of form goos/goarch", s)
	}
	return platform{os: vv[0], arch: vv[1]}
}

type whitelist map[string]int

// load adds entries from the whitelist file, if present, for os/arch to w.
func (w whitelist) load(goos string, goarch string) {
	sz := types.SizesFor("gc", goarch)
	if sz == nil {
		log.Fatalf("unknown type sizes for arch %q", goarch)
	}
	archbits := 8 * sz.Sizeof(types.Typ[types.UnsafePointer])

	// Look up whether goarch has a shared arch suffix,
	// such as mips64x for mips64 and mips64le.
	archsuff := goarch
	if x, ok := archAsmX[goarch]; ok {
		archsuff = x
	}

	// Load whitelists.
	filenames := []string{
		"all.txt",
		goos + ".txt",
		goarch + ".txt",
		goos + "_" + goarch + ".txt",
		fmt.Sprintf("%dbit.txt", archbits),
	}
	if goarch != archsuff {
		filenames = append(filenames,
			archsuff+".txt",
			goos+"_"+archsuff+".txt",
		)
	}

	// We allow error message templates using GOOS and GOARCH.
	if goos == "android" {
		goos = "linux" // so many special cases :(
	}

	// Read whitelists and do template substitution.
	replace := strings.NewReplacer("GOOS", goos, "GOARCH", goarch, "ARCHSUFF", archsuff)

	for _, filename := range filenames {
		path := filepath.Join("whitelist", filename)
		f, err := os.Open(path)
		if err != nil {
			// Allow not-exist errors; not all combinations have whitelists.
			if os.IsNotExist(err) {
				continue
			}
			log.Fatal(err)
		}
		scan := bufio.NewScanner(f)
		for scan.Scan() {
			line := scan.Text()
			if len(line) == 0 || strings.HasPrefix(line, "//") {
				continue
			}
			w[replace.Replace(line)]++
		}
		if err := scan.Err(); err != nil {
			log.Fatal(err)
		}
	}
}

type platform struct {
	os   string
	arch string
}

func (p platform) String() string {
	return p.os + "/" + p.arch
}

// ignorePathPrefixes are file path prefixes that should be ignored wholesale.
var ignorePathPrefixes = [...]string{
	// These testdata dirs have lots of intentionally broken/bad code for tests.
	"cmd/go/testdata/",
	"cmd/vet/testdata/",
	"go/printer/testdata/",
}

func vetPlatforms(pp []platform) {
	for _, p := range pp {
		p.vet()
	}
}

func (p platform) vet() {
	if p.os == "linux" && (p.arch == "riscv64" || p.arch == "sparc64") {
		// TODO(tklauser): enable as soon as these ports have fully landed
		fmt.Printf("skipping %s/%s\n", p.os, p.arch)
		return
	}

	if p.os == "windows" && p.arch == "arm" {
		// TODO(jordanrh1): enable as soon as the windows/arm port has fully landed
		fmt.Println("skipping windows/arm")
		return
	}

	if p.os == "aix" && p.arch == "ppc64" {
		// TODO(aix): enable as soon as the aix/ppc64 port has fully landed
		fmt.Println("skipping aix/ppc64")
		return
	}

	var buf bytes.Buffer
	fmt.Fprintf(&buf, "go run main.go -p %s\n", p)

	// Load whitelist(s).
	w := make(whitelist)
	w.load(p.os, p.arch)

	var vetCmd []string

	if os.Getenv("GO_BUILDER_NAME") == "" {
		vetCmd = []string{cmdGoPath, "vet"}
	} else {
		// Build the go/packages-based vet command from the x/tools
		// repo. It is considerably faster than "go vet", which rebuilds
		// the standard library.
		tmpdir, err := ioutil.TempDir("", "cmd-vet-all")
		if err != nil {
			log.Fatal(err)
		}
		defer os.RemoveAll(tmpdir)

		vetTool := filepath.Join(tmpdir, "vet")
		vetCmd = []string{
			vetTool,
			"-nilness=0", // expensive, uses SSA
		}

		cmd := exec.Command(cmdGoPath, "build", "-o", vetTool, "golang.org/x/tools/go/analysis/cmd/vet")
		cmd.Env = append(os.Environ(),
			// Setting GO111MODULE to on is redundant in master
			// (Go 1.13), but not if we backport this to Go 1.11/1.12
			// release branches (for our own builder usage) or if
			// master ends up reverting its GO111MODULE default. If
			// that happens, we want to force it on here anyway, as
			// we're now depending on it.
			"GO111MODULE=on",
		)
		// Use the module that cmd/vet/all is a part of:
		cmd.Dir = filepath.Join(runtime.GOROOT(), "src", "cmd", "vet", "all")

		// golang.org/x/tools does not have a vendor directory, so don't try to use
		// one in module mode.
		for i, v := range cmd.Env {
			if strings.HasPrefix(v, "GOFLAGS=") {
				var goflags []string
				for _, f := range strings.Fields(strings.TrimPrefix(v, "GOFLAGS=")) {
					if f != "-mod=vendor" && f != "--mod=vendor" {
						goflags = append(goflags, f)
					}
				}
				cmd.Env[i] = strings.Join(goflags, " ")
			}
		}

		cmd.Stderr = os.Stderr
		cmd.Stdout = os.Stderr
		if err := cmd.Run(); err != nil {
			log.Fatalf("%s: %v", strings.Join(cmd.Args, " "), err)
		}
	}

	// TODO: The unsafeptr checks are disabled for now,
	// because there are so many false positives,
	// and no clear way to improve vet to eliminate large chunks of them.
	// And having them in the whitelists will just cause annoyance
	// and churn when working on the runtime.
	cmd := exec.Command(vetCmd[0],
		append(vetCmd[1:],
			"-unsafeptr=0",
			"std",
			"cmd/...",
			"cmd/compile/internal/gc/testdata",
		)...)
	cmd.Dir = filepath.Join(runtime.GOROOT(), "src")
	cmd.Env = append(os.Environ(), "GOOS="+p.os, "GOARCH="+p.arch, "CGO_ENABLED=0")
	stderr, err := cmd.StderrPipe()
	if err != nil {
		log.Fatal(err)
	}
	if err := cmd.Start(); err != nil {
		log.Fatal(err)
	}

	// Process vet output.
	scan := bufio.NewScanner(stderr)
	var parseFailed bool
NextLine:
	for scan.Scan() {
		line := scan.Text()
		if strings.HasPrefix(line, "vet: ") {
			// Typecheck failure: Malformed syntax or multiple packages or the like.
			// This will yield nicer error messages elsewhere, so ignore them here.

			// This includes warnings from asmdecl of the form:
			//   "vet: foo.s:16: [amd64] cannot check cross-package assembly function"
			continue
		}

		if strings.HasPrefix(line, "panic: ") {
			// Panic in vet. Don't filter anything, we want the complete output.
			parseFailed = true
			fmt.Fprintf(os.Stderr, "panic in vet (to reproduce: go run main.go -p %s):\n", p)
			fmt.Fprintln(os.Stderr, line)
			io.Copy(os.Stderr, stderr)
			break
		}
		if strings.HasPrefix(line, "# ") {
			// 'go vet' prefixes the output of each vet invocation by a comment:
			//    # [package]
			continue
		}

		// Parse line.
		// Assume the part before the first ": "
		// is the "file:line:col: " information.
		// TODO(adonovan): parse vet -json output.
		var file, lineno, msg string
		if i := strings.Index(line, ": "); i >= 0 {
			msg = line[i+len(": "):]

			words := strings.Split(line[:i], ":")
			switch len(words) {
			case 3:
				_ = words[2] // ignore column
				fallthrough
			case 2:
				lineno = words[1]
				fallthrough
			case 1:
				file = words[0]

				// Make the file name relative to GOROOT/src.
				if rel, err := filepath.Rel(cmd.Dir, file); err == nil {
					file = rel
				}
			default:
				// error: too many columns
			}
		}
		if file == "" {
			if !parseFailed {
				parseFailed = true
				fmt.Fprintf(os.Stderr, "failed to parse %s output:\n# %s\n", p, strings.Join(cmd.Args, " "))
			}
			fmt.Fprintln(os.Stderr, line)
			continue
		}

		msg = strings.TrimSpace(msg)

		for _, ignore := range ignorePathPrefixes {
			if strings.HasPrefix(file, filepath.FromSlash(ignore)) {
				continue NextLine
			}
		}

		key := file + ": " + msg
		if w[key] == 0 {
			// Vet error with no match in the whitelist. Print it.
			if *flagNoLines {
				fmt.Fprintf(&buf, "%s: %s\n", file, msg)
			} else {
				fmt.Fprintf(&buf, "%s:%s: %s\n", file, lineno, msg)
			}
			atomic.StoreUint32(&failed, 1)
			continue
		}
		w[key]--
	}
	if parseFailed {
		atomic.StoreUint32(&failed, 1)
		return
	}
	if scan.Err() != nil {
		log.Fatalf("failed to scan vet output: %v", scan.Err())
	}
	err = cmd.Wait()
	// We expect vet to fail.
	// Make sure it has failed appropriately, though (for example, not a PathError).
	if _, ok := err.(*exec.ExitError); !ok {
		log.Fatalf("unexpected go vet execution failure: %v", err)
	}
	printedHeader := false
	if len(w) > 0 {
		for k, v := range w {
			if v != 0 {
				if !printedHeader {
					fmt.Fprintln(&buf, "unmatched whitelist entries:")
					printedHeader = true
				}
				for i := 0; i < v; i++ {
					fmt.Fprintln(&buf, k)
				}
				atomic.StoreUint32(&failed, 1)
			}
		}
	}

	os.Stdout.Write(buf.Bytes())
}

// archAsmX maps architectures to the suffix usually used for their assembly files,
// if different than the arch name itself.
var archAsmX = map[string]string{
	"android":  "linux",
	"mips64":   "mips64x",
	"mips64le": "mips64x",
	"mips":     "mipsx",
	"mipsle":   "mipsx",
	"ppc64":    "ppc64x",
	"ppc64le":  "ppc64x",
}
<|endoftext|>


<commit_before>// Copyright 2014 Manu Martinez-Almeida.  All rights reserved.
// Use of this source code is governed by a MIT style
// license that can be found in the LICENSE file.

package binding

import (
	"errors"
	"fmt"
	"reflect"
	"strconv"
	"strings"
	"time"

	"github.com/gin-gonic/gin/internal/bytesconv"
	"github.com/gin-gonic/gin/internal/json"
)

var errUnknownType = errors.New("unknown type")

func mapUri(ptr interface{}, m map[string][]string) error {
	return mapFormByTag(ptr, m, "uri")
}

func mapForm(ptr interface{}, form map[string][]string) error {
	return mapFormByTag(ptr, form, "form")
}

var emptyField = reflect.StructField{}

func mapFormByTag(ptr interface{}, form map[string][]string, tag string) error {
	return mappingByPtr(ptr, formSource(form), tag)
}

// setter tries to set value on a walking by fields of a struct
type setter interface {
	TrySet(value reflect.Value, field reflect.StructField, key string, opt setOptions) (isSetted bool, err error)
}

type formSource map[string][]string

var _ setter = formSource(nil)

// TrySet tries to set a value by request's form source (like map[string][]string)
func (form formSource) TrySet(value reflect.Value, field reflect.StructField, tagValue string, opt setOptions) (isSetted bool, err error) {
	return setByForm(value, field, form, tagValue, opt)
}

func mappingByPtr(ptr interface{}, setter setter, tag string) error {
	_, err := mapping(reflect.ValueOf(ptr), emptyField, setter, tag)
	return err
}

func mapping(value reflect.Value, field reflect.StructField, setter setter, tag string) (bool, error) {
	if field.Tag.Get(tag) == "-" { // just ignoring this field
		return false, nil
	}

	var vKind = value.Kind()

	if vKind == reflect.Ptr {
		var isNew bool
		vPtr := value
		if value.IsNil() {
			isNew = true
			vPtr = reflect.New(value.Type().Elem())
		}
		isSetted, err := mapping(vPtr.Elem(), field, setter, tag)
		if err != nil {
			return false, err
		}
		if isNew && isSetted {
			value.Set(vPtr)
		}
		return isSetted, nil
	}

	if vKind != reflect.Struct || !field.Anonymous {
		ok, err := tryToSetValue(value, field, setter, tag)
		if err != nil {
			return false, err
		}
		if ok {
			return true, nil
		}
	}

	if vKind == reflect.Struct {
		tValue := value.Type()

		var isSetted bool
		for i := 0; i < value.NumField(); i++ {
			sf := tValue.Field(i)
			if sf.PkgPath != "" && !sf.Anonymous { // unexported
				continue
			}
			ok, err := mapping(value.Field(i), tValue.Field(i), setter, tag)
			if err != nil {
				return false, err
			}
			isSetted = isSetted || ok
		}
		return isSetted, nil
	}
	return false, nil
}

type setOptions struct {
	isDefaultExists bool
	defaultValue    string
}

func tryToSetValue(value reflect.Value, field reflect.StructField, setter setter, tag string) (bool, error) {
	var tagValue string
	var setOpt setOptions

	tagValue = field.Tag.Get(tag)
	tagValue, opts := head(tagValue, ",")

	if tagValue == "" { // default value is FieldName
		tagValue = field.Name
	}
	if tagValue == "" { // when field is "emptyField" variable
		return false, nil
	}

	var opt string
	for len(opts) > 0 {
		opt, opts = head(opts, ",")

		if k, v := head(opt, "="); k == "default" {
			setOpt.isDefaultExists = true
			setOpt.defaultValue = v
		}
	}

	return setter.TrySet(value, field, tagValue, setOpt)
}

func setByForm(value reflect.Value, field reflect.StructField, form map[string][]string, tagValue string, opt setOptions) (isSetted bool, err error) {
	vs, ok := form[tagValue]
	if !ok && !opt.isDefaultExists {
		return false, nil
	}

	switch value.Kind() {
	case reflect.Slice:
		if !ok {
			vs = []string{opt.defaultValue}
		}
		return true, setSlice(vs, value, field)
	case reflect.Array:
		if !ok {
			vs = []string{opt.defaultValue}
		}
		if len(vs) != value.Len() {
			return false, fmt.Errorf("%q is not valid value for %s", vs, value.Type().String())
		}
		return true, setArray(vs, value, field)
	default:
		var val string
		if !ok {
			val = opt.defaultValue
		}

		if len(vs) > 0 {
			val = vs[0]
		}
		return true, setWithProperType(val, value, field)
	}
}

func setWithProperType(val string, value reflect.Value, field reflect.StructField) error {
	switch value.Kind() {
	case reflect.Int:
		return setIntField(val, 0, value)
	case reflect.Int8:
		return setIntField(val, 8, value)
	case reflect.Int16:
		return setIntField(val, 16, value)
	case reflect.Int32:
		return setIntField(val, 32, value)
	case reflect.Int64:
		switch value.Interface().(type) {
		case time.Duration:
			return setTimeDuration(val, value, field)
		}
		return setIntField(val, 64, value)
	case reflect.Uint:
		return setUintField(val, 0, value)
	case reflect.Uint8:
		return setUintField(val, 8, value)
	case reflect.Uint16:
		return setUintField(val, 16, value)
	case reflect.Uint32:
		return setUintField(val, 32, value)
	case reflect.Uint64:
		return setUintField(val, 64, value)
	case reflect.Bool:
		return setBoolField(val, value)
	case reflect.Float32:
		return setFloatField(val, 32, value)
	case reflect.Float64:
		return setFloatField(val, 64, value)
	case reflect.String:
		value.SetString(val)
	case reflect.Struct:
		switch value.Interface().(type) {
		case time.Time:
			return setTimeField(val, field, value)
		}
		return json.Unmarshal(bytesconv.StringToBytes(val), value.Addr().Interface())
	case reflect.Map:
		return json.Unmarshal(bytesconv.StringToBytes(val), value.Addr().Interface())
	default:
		return errUnknownType
	}
	return nil
}

func setIntField(val string, bitSize int, field reflect.Value) error {
	if val == "" {
		val = "0"
	}
	intVal, err := strconv.ParseInt(val, 10, bitSize)
	if err == nil {
		field.SetInt(intVal)
	}
	return err
}

func setUintField(val string, bitSize int, field reflect.Value) error {
	if val == "" {
		val = "0"
	}
	uintVal, err := strconv.ParseUint(val, 10, bitSize)
	if err == nil {
		field.SetUint(uintVal)
	}
	return err
}

func setBoolField(val string, field reflect.Value) error {
	if val == "" {
		val = "false"
	}
	boolVal, err := strconv.ParseBool(val)
	if err == nil {
		field.SetBool(boolVal)
	}
	return err
}

func setFloatField(val string, bitSize int, field reflect.Value) error {
	if val == "" {
		val = "0.0"
	}
	floatVal, err := strconv.ParseFloat(val, bitSize)
	if err == nil {
		field.SetFloat(floatVal)
	}
	return err
}

func setTimeField(val string, structField reflect.StructField, value reflect.Value) error {
	timeFormat := structField.Tag.Get("time_format")
	if timeFormat == "" {
		timeFormat = time.RFC3339
	}

	switch tf := strings.ToLower(timeFormat); tf {
	case "unix", "unixnano":
		tv, err := strconv.ParseInt(val, 10, 0)
		if err != nil {
			return err
		}

		d := time.Duration(1)
		if tf == "unixnano" {
			d = time.Second
		}

		t := time.Unix(tv/int64(d), tv%int64(d))
		value.Set(reflect.ValueOf(t))
		return nil

	}

	if val == "" {
		value.Set(reflect.ValueOf(time.Time{}))
		return nil
	}

	l := time.Local
	if isUTC, _ := strconv.ParseBool(structField.Tag.Get("time_utc")); isUTC {
		l = time.UTC
	}

	if locTag := structField.Tag.Get("time_location"); locTag != "" {
		loc, err := time.LoadLocation(locTag)
		if err != nil {
			return err
		}
		l = loc
	}

	t, err := time.ParseInLocation(timeFormat, val, l)
	if err != nil {
		return err
	}

	value.Set(reflect.ValueOf(t))
	return nil
}

func setArray(vals []string, value reflect.Value, field reflect.StructField) error {
	for i, s := range vals {
		err := setWithProperType(s, value.Index(i), field)
		if err != nil {
			return err
		}
	}
	return nil
}

func setSlice(vals []string, value reflect.Value, field reflect.StructField) error {
	slice := reflect.MakeSlice(value.Type(), len(vals), len(vals))
	err := setArray(vals, slice, field)
	if err != nil {
		return err
	}
	value.Set(slice)
	return nil
}

func setTimeDuration(val string, value reflect.Value, field reflect.StructField) error {
	d, err := time.ParseDuration(val)
	if err != nil {
		return err
	}
	value.Set(reflect.ValueOf(d))
	return nil
}

func head(str, sep string) (head string, tail string) {
	idx := strings.Index(str, sep)
	if idx < 0 {
		return str, ""
	}
	return str[:idx], str[idx+len(sep):]
}
<commit_msg>binding: avoid 2038 problem on 32-bit architectures (#2450)<commit_after>// Copyright 2014 Manu Martinez-Almeida.  All rights reserved.
// Use of this source code is governed by a MIT style
// license that can be found in the LICENSE file.

package binding

import (
	"errors"
	"fmt"
	"reflect"
	"strconv"
	"strings"
	"time"

	"github.com/gin-gonic/gin/internal/bytesconv"
	"github.com/gin-gonic/gin/internal/json"
)

var errUnknownType = errors.New("unknown type")

func mapUri(ptr interface{}, m map[string][]string) error {
	return mapFormByTag(ptr, m, "uri")
}

func mapForm(ptr interface{}, form map[string][]string) error {
	return mapFormByTag(ptr, form, "form")
}

var emptyField = reflect.StructField{}

func mapFormByTag(ptr interface{}, form map[string][]string, tag string) error {
	return mappingByPtr(ptr, formSource(form), tag)
}

// setter tries to set value on a walking by fields of a struct
type setter interface {
	TrySet(value reflect.Value, field reflect.StructField, key string, opt setOptions) (isSetted bool, err error)
}

type formSource map[string][]string

var _ setter = formSource(nil)

// TrySet tries to set a value by request's form source (like map[string][]string)
func (form formSource) TrySet(value reflect.Value, field reflect.StructField, tagValue string, opt setOptions) (isSetted bool, err error) {
	return setByForm(value, field, form, tagValue, opt)
}

func mappingByPtr(ptr interface{}, setter setter, tag string) error {
	_, err := mapping(reflect.ValueOf(ptr), emptyField, setter, tag)
	return err
}

func mapping(value reflect.Value, field reflect.StructField, setter setter, tag string) (bool, error) {
	if field.Tag.Get(tag) == "-" { // just ignoring this field
		return false, nil
	}

	var vKind = value.Kind()

	if vKind == reflect.Ptr {
		var isNew bool
		vPtr := value
		if value.IsNil() {
			isNew = true
			vPtr = reflect.New(value.Type().Elem())
		}
		isSetted, err := mapping(vPtr.Elem(), field, setter, tag)
		if err != nil {
			return false, err
		}
		if isNew && isSetted {
			value.Set(vPtr)
		}
		return isSetted, nil
	}

	if vKind != reflect.Struct || !field.Anonymous {
		ok, err := tryToSetValue(value, field, setter, tag)
		if err != nil {
			return false, err
		}
		if ok {
			return true, nil
		}
	}

	if vKind == reflect.Struct {
		tValue := value.Type()

		var isSetted bool
		for i := 0; i < value.NumField(); i++ {
			sf := tValue.Field(i)
			if sf.PkgPath != "" && !sf.Anonymous { // unexported
				continue
			}
			ok, err := mapping(value.Field(i), tValue.Field(i), setter, tag)
			if err != nil {
				return false, err
			}
			isSetted = isSetted || ok
		}
		return isSetted, nil
	}
	return false, nil
}

type setOptions struct {
	isDefaultExists bool
	defaultValue    string
}

func tryToSetValue(value reflect.Value, field reflect.StructField, setter setter, tag string) (bool, error) {
	var tagValue string
	var setOpt setOptions

	tagValue = field.Tag.Get(tag)
	tagValue, opts := head(tagValue, ",")

	if tagValue == "" { // default value is FieldName
		tagValue = field.Name
	}
	if tagValue == "" { // when field is "emptyField" variable
		return false, nil
	}

	var opt string
	for len(opts) > 0 {
		opt, opts = head(opts, ",")

		if k, v := head(opt, "="); k == "default" {
			setOpt.isDefaultExists = true
			setOpt.defaultValue = v
		}
	}

	return setter.TrySet(value, field, tagValue, setOpt)
}

func setByForm(value reflect.Value, field reflect.StructField, form map[string][]string, tagValue string, opt setOptions) (isSetted bool, err error) {
	vs, ok := form[tagValue]
	if !ok && !opt.isDefaultExists {
		return false, nil
	}

	switch value.Kind() {
	case reflect.Slice:
		if !ok {
			vs = []string{opt.defaultValue}
		}
		return true, setSlice(vs, value, field)
	case reflect.Array:
		if !ok {
			vs = []string{opt.defaultValue}
		}
		if len(vs) != value.Len() {
			return false, fmt.Errorf("%q is not valid value for %s", vs, value.Type().String())
		}
		return true, setArray(vs, value, field)
	default:
		var val string
		if !ok {
			val = opt.defaultValue
		}

		if len(vs) > 0 {
			val = vs[0]
		}
		return true, setWithProperType(val, value, field)
	}
}

func setWithProperType(val string, value reflect.Value, field reflect.StructField) error {
	switch value.Kind() {
	case reflect.Int:
		return setIntField(val, 0, value)
	case reflect.Int8:
		return setIntField(val, 8, value)
	case reflect.Int16:
		return setIntField(val, 16, value)
	case reflect.Int32:
		return setIntField(val, 32, value)
	case reflect.Int64:
		switch value.Interface().(type) {
		case time.Duration:
			return setTimeDuration(val, value, field)
		}
		return setIntField(val, 64, value)
	case reflect.Uint:
		return setUintField(val, 0, value)
	case reflect.Uint8:
		return setUintField(val, 8, value)
	case reflect.Uint16:
		return setUintField(val, 16, value)
	case reflect.Uint32:
		return setUintField(val, 32, value)
	case reflect.Uint64:
		return setUintField(val, 64, value)
	case reflect.Bool:
		return setBoolField(val, value)
	case reflect.Float32:
		return setFloatField(val, 32, value)
	case reflect.Float64:
		return setFloatField(val, 64, value)
	case reflect.String:
		value.SetString(val)
	case reflect.Struct:
		switch value.Interface().(type) {
		case time.Time:
			return setTimeField(val, field, value)
		}
		return json.Unmarshal(bytesconv.StringToBytes(val), value.Addr().Interface())
	case reflect.Map:
		return json.Unmarshal(bytesconv.StringToBytes(val), value.Addr().Interface())
	default:
		return errUnknownType
	}
	return nil
}

func setIntField(val string, bitSize int, field reflect.Value) error {
	if val == "" {
		val = "0"
	}
	intVal, err := strconv.ParseInt(val, 10, bitSize)
	if err == nil {
		field.SetInt(intVal)
	}
	return err
}

func setUintField(val string, bitSize int, field reflect.Value) error {
	if val == "" {
		val = "0"
	}
	uintVal, err := strconv.ParseUint(val, 10, bitSize)
	if err == nil {
		field.SetUint(uintVal)
	}
	return err
}

func setBoolField(val string, field reflect.Value) error {
	if val == "" {
		val = "false"
	}
	boolVal, err := strconv.ParseBool(val)
	if err == nil {
		field.SetBool(boolVal)
	}
	return err
}

func setFloatField(val string, bitSize int, field reflect.Value) error {
	if val == "" {
		val = "0.0"
	}
	floatVal, err := strconv.ParseFloat(val, bitSize)
	if err == nil {
		field.SetFloat(floatVal)
	}
	return err
}

func setTimeField(val string, structField reflect.StructField, value reflect.Value) error {
	timeFormat := structField.Tag.Get("time_format")
	if timeFormat == "" {
		timeFormat = time.RFC3339
	}

	switch tf := strings.ToLower(timeFormat); tf {
	case "unix", "unixnano":
		tv, err := strconv.ParseInt(val, 10, 64)
		if err != nil {
			return err
		}

		d := time.Duration(1)
		if tf == "unixnano" {
			d = time.Second
		}

		t := time.Unix(tv/int64(d), tv%int64(d))
		value.Set(reflect.ValueOf(t))
		return nil

	}

	if val == "" {
		value.Set(reflect.ValueOf(time.Time{}))
		return nil
	}

	l := time.Local
	if isUTC, _ := strconv.ParseBool(structField.Tag.Get("time_utc")); isUTC {
		l = time.UTC
	}

	if locTag := structField.Tag.Get("time_location"); locTag != "" {
		loc, err := time.LoadLocation(locTag)
		if err != nil {
			return err
		}
		l = loc
	}

	t, err := time.ParseInLocation(timeFormat, val, l)
	if err != nil {
		return err
	}

	value.Set(reflect.ValueOf(t))
	return nil
}

func setArray(vals []string, value reflect.Value, field reflect.StructField) error {
	for i, s := range vals {
		err := setWithProperType(s, value.Index(i), field)
		if err != nil {
			return err
		}
	}
	return nil
}

func setSlice(vals []string, value reflect.Value, field reflect.StructField) error {
	slice := reflect.MakeSlice(value.Type(), len(vals), len(vals))
	err := setArray(vals, slice, field)
	if err != nil {
		return err
	}
	value.Set(slice)
	return nil
}

func setTimeDuration(val string, value reflect.Value, field reflect.StructField) error {
	d, err := time.ParseDuration(val)
	if err != nil {
		return err
	}
	value.Set(reflect.ValueOf(d))
	return nil
}

func head(str, sep string) (head string, tail string) {
	idx := strings.Index(str, sep)
	if idx < 0 {
		return str, ""
	}
	return str[:idx], str[idx+len(sep):]
}
<|endoftext|>


<commit_before>// Copyright © 2016 Pennock Tech, LLC.
// All rights reserved, except as granted under license.
// Licensed per file LICENSE.txt

package main

import (
	"bufio"
	"io"
	"os"
	"path/filepath"
	"strings"
	"sync"

	"github.com/sirupsen/logrus"
	"gopkg.in/fsnotify.v1"
)

// opts.aliasfile defaults to /etc/finger.conf
// The format is man-page specified on BSD; "#" for comments, nothing about blank lines;
// Unix tradition, expect a final newline
// all real lines are:
//    alias:(user|alias)
// but aliases are only permitted to be _forward_ references (which avoids loops, but is
// inverted from the usual sense).  Since we're implementing this file for compatibility,
// stick to that constraint.
// Also aliases can be fully-qualified filenames (start with a `/`) to point elsewhere.

var aliases struct {
	sync.RWMutex
	to map[string]string
}

func init() {
	aliases.to = make(map[string]string)
}

func currentAliases() map[string]string {
	aliases.RLock()
	defer aliases.RUnlock()
	return aliases.to
}

func loadMappingData(log logrus.FieldLogger) {
	log = log.WithField("file", opts.aliasfile)
	var err error
	fh, err := os.Open(opts.aliasfile)
	if err != nil {
		log.WithError(err).Info("unable to load aliases")
		return
	}
	defer fh.Close()

	concrete := make(map[string]string)

	unresolved := make([][2]string, 0, 100)

	// No size limit on the alias file, we "trust" it
	r := bufio.NewReader(fh)
	err = nil
	var line string
	for lineNum := 0; err != io.EOF; {
		line, err = r.ReadString('\n')
		if err != nil && err != io.EOF {
			log.WithError(err).Warn("problem reading config, aborting")
			return
		}
		lineNum++
		line = strings.TrimSpace(line)
		if len(line) == 0 || line[0] == '#' {
			continue
		}
		fields := strings.SplitN(line, ":", 2)
		if len(fields) != 2 || len(fields[0]) == 0 || len(fields[1]) == 0 || strings.ContainsRune(fields[0], '/') {
			log.WithField("line", lineNum).Warn("malformed line, skipping")
			continue
		}

		unresolved = append(unresolved, [2]string{fields[0], fields[1]})
	}

	for i := len(unresolved) - 1; i >= 0; i-- {
		from := strings.ToLower(unresolved[i][0])
		to := unresolved[i][1]
		if to[0] != '/' {
			to = strings.ToLower(to)
		}
		if _, ok := concrete[from]; ok {
			log.Warn("alias %q defined more than once, last one wins", from)
			continue
		}
		if chain, ok := concrete[to]; ok {
			concrete[from] = chain
		} else {
			concrete[from] = to
		}
	}

	aliases.Lock()
	aliases.to = concrete
	aliases.Unlock()
	log.WithField("alias-count", len(concrete)).Info("parsed aliases")
	return
}

// as long as the _directory_ exists, we'll detect a late file creation and handle it fine.
func scheduleAutoMappingDataReload(log logrus.FieldLogger) {
	log = log.WithField("subsystem", "fs-watcher")
	// originally mostly ripped straight from fsnotify.v1's NewWatcher example in the docs
	watcher, err := fsnotify.NewWatcher()
	if err != nil {
		log.WithError(err).Error("unable to start FS watcher, will not detect changes")
		// We continue on without aborting
		return
	}
	logrus.RegisterExitHandler(func() { _ = watcher.Close() })

	basename := filepath.Base(opts.aliasfile)
	dirname := filepath.Dir(opts.aliasfile)

	go func() {
		for {
			select {
			case event, ok := <-watcher.Events:
				if !ok {
					log.Warn("terminating config-watcher event dispatcher")
					return
				}
				l := log.WithField("event", event)
				switch filepath.Base(event.Name) {
				case basename:
					if event.Op&fsnotify.Write == fsnotify.Write {
						l.Info("modification detected")
						// The actual work!  (also in some other edge-cases just below)
						loadMappingData(log)
					} else if event.Op&fsnotify.Create == fsnotify.Create {
						// better late than never
						l.Info("creation detected (adding watch)")
						loadMappingData(log)
						watcher.Add(opts.aliasfile)
					} else if event.Op&fsnotify.Chmod == fsnotify.Chmod {
						// assume file created with 0 permissions then chmod'd more open, so our initial read might
						// have failed.  Should be harmless to re-read the file.  If it was chmod'd unreadable, we'll
						// error out cleanly.
						l.Info("chmod detected")
						loadMappingData(log)
					} else if event.Op&fsnotify.Remove == fsnotify.Remove || event.Op&fsnotify.Rename == fsnotify.Rename {
						l.Info("gone, removing watch")
						watcher.Remove(opts.aliasfile)
						// TODO: should we also remove all aliases?
						// We currently continue running with the last aliases seen, which lets us
						// steady-state on the assumption that the file is being replaced.  Perhaps
						// we should start a timer and after 5 seconds, nuke the loaded aliases?
					}
					// no other scenarios known
				case dirname:
					// usually ...
					// nothing to do; file creation will create an event named for the file, which we detect above for the file
					// which we care about; chmod ... we care less about.
					if event.Op&fsnotify.Remove == fsnotify.Remove || event.Op&fsnotify.Rename == fsnotify.Rename {
						l.Warn("directory of config file gone, nuking watcher, no notifications anymore")
						// duplicate close on shutdown is safe
						_ = watcher.Close()
						return
					}
				}
			case err, ok := <-watcher.Errors:
				if !ok {
					log.Warn("terminating config-watcher event dispatcher")
					return
				}
				log.WithError(err).Warn("error")
			}
		}
	}()

	count := 0
	for _, p := range []string{opts.aliasfile, dirname} {
		err = watcher.Add(p)
		if err != nil {
			log.WithError(err).WithField("file", p).Info("unable to start watching")
			// do not error out
		} else {
			count++
		}
	}

	if count == 0 {
		log.Warn("unable to set up any watches, terminating FS watcher, auto-reloading gone")
		_ = watcher.Close()
	}

	return
}
<commit_msg>Double-check FS notify on alias file<commit_after>// Copyright © 2016 Pennock Tech, LLC.
// All rights reserved, except as granted under license.
// Licensed per file LICENSE.txt

package main

import (
	"bufio"
	"io"
	"os"
	"path/filepath"
	"strings"
	"sync"

	"github.com/sirupsen/logrus"
	"gopkg.in/fsnotify.v1"
)

// opts.aliasfile defaults to /etc/finger.conf
// The format is man-page specified on BSD; "#" for comments, nothing about blank lines;
// Unix tradition, expect a final newline
// all real lines are:
//    alias:(user|alias)
// but aliases are only permitted to be _forward_ references (which avoids loops, but is
// inverted from the usual sense).  Since we're implementing this file for compatibility,
// stick to that constraint.
// Also aliases can be fully-qualified filenames (start with a `/`) to point elsewhere.

var aliases struct {
	sync.RWMutex
	to map[string]string
}

func init() {
	aliases.to = make(map[string]string)
}

func currentAliases() map[string]string {
	aliases.RLock()
	defer aliases.RUnlock()
	return aliases.to
}

func loadMappingData(log logrus.FieldLogger) {
	log = log.WithField("file", opts.aliasfile)
	var err error
	fh, err := os.Open(opts.aliasfile)
	if err != nil {
		log.WithError(err).Info("unable to load aliases")
		return
	}
	defer fh.Close()

	concrete := make(map[string]string)

	unresolved := make([][2]string, 0, 100)

	// No size limit on the alias file, we "trust" it
	r := bufio.NewReader(fh)
	err = nil
	var line string
	for lineNum := 0; err != io.EOF; {
		line, err = r.ReadString('\n')
		if err != nil && err != io.EOF {
			log.WithError(err).Warn("problem reading config, aborting")
			return
		}
		lineNum++
		line = strings.TrimSpace(line)
		if len(line) == 0 || line[0] == '#' {
			continue
		}
		fields := strings.SplitN(line, ":", 2)
		if len(fields) != 2 || len(fields[0]) == 0 || len(fields[1]) == 0 || strings.ContainsRune(fields[0], '/') {
			log.WithField("line", lineNum).Warn("malformed line, skipping")
			continue
		}

		unresolved = append(unresolved, [2]string{fields[0], fields[1]})
	}

	for i := len(unresolved) - 1; i >= 0; i-- {
		from := strings.ToLower(unresolved[i][0])
		to := unresolved[i][1]
		if to[0] != '/' {
			to = strings.ToLower(to)
		}
		if _, ok := concrete[from]; ok {
			log.Warn("alias %q defined more than once, last one wins", from)
			continue
		}
		if chain, ok := concrete[to]; ok {
			concrete[from] = chain
		} else {
			concrete[from] = to
		}
	}

	aliases.Lock()
	aliases.to = concrete
	aliases.Unlock()
	log.WithField("alias-count", len(concrete)).Info("parsed aliases")
	return
}

// as long as the _directory_ exists, we'll detect a late file creation and handle it fine.
func scheduleAutoMappingDataReload(log logrus.FieldLogger) {
	log = log.WithField("subsystem", "fs-watcher")
	// originally mostly ripped straight from fsnotify.v1's NewWatcher example in the docs
	watcher, err := fsnotify.NewWatcher()
	if err != nil {
		log.WithError(err).Error("unable to start FS watcher, will not detect changes")
		// We continue on without aborting
		return
	}
	logrus.RegisterExitHandler(func() { _ = watcher.Close() })

	basename := filepath.Base(opts.aliasfile)
	dirname := filepath.Dir(opts.aliasfile)

	go func() {
		for {
			select {
			case event, ok := <-watcher.Events:
				if !ok {
					log.Warn("terminating config-watcher event dispatcher")
					return
				}
				l := log.WithField("event", event)
				switch filepath.Base(event.Name) {
				case basename:
					if event.Op&fsnotify.Write == fsnotify.Write {
						l.Info("modification detected")
						// The actual work!  (also in some other edge-cases just below)
						loadMappingData(log)
					} else if event.Op&fsnotify.Create == fsnotify.Create {
						// better late than never
						l.Info("creation detected (adding watch)")
						loadMappingData(log)
						watcher.Add(opts.aliasfile)
					} else if event.Op&fsnotify.Chmod == fsnotify.Chmod {
						// assume file created with 0 permissions then chmod'd more open, so our initial read might
						// have failed.  Should be harmless to re-read the file.  If it was chmod'd unreadable, we'll
						// error out cleanly.
						l.Info("chmod detected")
						loadMappingData(log)
					} else if event.Op&fsnotify.Remove == fsnotify.Remove || event.Op&fsnotify.Rename == fsnotify.Rename {
						// I have seen this fire when the watched config file
						// is an entry in a K8S configmap and the entry was
						// modified, not deleted.
						_, err := os.Stat(event.Name)
						if err != nil && os.IsNotExist(err) {
							l.Info("file gone, confirmed, WATCH GONE")
						} else {
							l.Info("file gone, false positive, file still exists, re-watching")
							// The kernel will have removed the watch and
							// fsnotify will have removed its copy, to match.
							// We can't just ignore this, we have to add the
							// watch back.
							watcher.Add(opts.aliasfile)
						}
						// TODO: should we also remove all aliases?
						// We currently continue running with the last aliases seen, which lets us
						// steady-state on the assumption that the file is being replaced.  Perhaps
						// we should start a timer and after 5 seconds, nuke the loaded aliases?
					}
					// no other scenarios known
				case dirname:
					// usually ...
					// nothing to do; file creation will create an event named for the file, which we detect above for the file
					// which we care about; chmod ... we care less about.
					if event.Op&fsnotify.Remove == fsnotify.Remove || event.Op&fsnotify.Rename == fsnotify.Rename {
						l.Warn("directory of config file gone, nuking watcher, no notifications anymore")
						// duplicate close on shutdown is safe
						_ = watcher.Close()
						return
					}
				}
			case err, ok := <-watcher.Errors:
				if !ok {
					log.Warn("terminating config-watcher event dispatcher")
					return
				}
				log.WithError(err).Warn("error")
			}
		}
	}()

	count := 0
	for _, p := range []string{opts.aliasfile, dirname} {
		err = watcher.Add(p)
		if err != nil {
			log.WithError(err).WithField("file", p).Info("unable to start watching")
			// do not error out
		} else {
			count++
		}
	}

	if count == 0 {
		log.Warn("unable to set up any watches, terminating FS watcher, auto-reloading gone")
		_ = watcher.Close()
	}

	return
}
<|endoftext|>


<commit_before>package docker

import (
	"bufio"
	"fmt"
	"io"
	"strings"
)

type Builder struct {
	runtime *Runtime
}

func NewBuilder(runtime *Runtime) *Builder {
	return &Builder{
		runtime: runtime,
	}
}

func (builder *Builder) run(image *Image, cmd string) (*Container, error) {
	// FIXME: pass a NopWriter instead of nil
	config, err := ParseRun([]string{"-d", image.Id, "/bin/sh", "-c", cmd}, nil, builder.runtime.capabilities)
	if config.Image == "" {
		return nil, fmt.Errorf("Image not specified")
	}
	if len(config.Cmd) == 0 {
		return nil, fmt.Errorf("Command not specified")
	}
	if config.Tty {
		return nil, fmt.Errorf("The tty mode is not supported within the builder")
	}

	// Create new container
	container, err := builder.runtime.Create(config)
	if err != nil {
		return nil, err
	}
	if err := container.Start(); err != nil {
		return nil, err
	}
	return container, nil
}

func (builder *Builder) clearTmp(containers, images map[string]struct{}) {
	for c := range containers {
		tmp := builder.runtime.Get(c)
		builder.runtime.Destroy(tmp)
		Debugf("Removing container %s", c)
	}
	for i := range images {
		builder.runtime.graph.Delete(i)
		Debugf("Removing image %s", i)
	}
}

func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) error {
	var (
		image, base   *Image
		tmpContainers map[string]struct{} = make(map[string]struct{})
		tmpImages     map[string]struct{} = make(map[string]struct{})
	)
	defer builder.clearTmp(tmpContainers, tmpImages)

	file := bufio.NewReader(dockerfile)
	for {
		line, err := file.ReadString('\n')
		if err != nil {
			if err == io.EOF {
				break
			}
			return err
		}
		line = strings.TrimSpace(line)
		// Skip comments and empty line
		if len(line) == 0 || line[0] == '#' {
			continue
		}
		tmp := strings.SplitN(line, "	", 2)
		if len(tmp) != 2 {
			return fmt.Errorf("Invalid Dockerfile format")
		}
		switch tmp[0] {
		case "from":
			fmt.Fprintf(stdout, "FROM %s\n", tmp[1])
			image, err = builder.runtime.repositories.LookupImage(tmp[1])
			if err != nil {
				return err
			}
			break
		case "run":
			fmt.Fprintf(stdout, "RUN %s\n", tmp[1])
			if image == nil {
				return fmt.Errorf("Please provide a source image with `from` prior to run")
			}

			// Create the container and start it
			c, err := builder.run(image, tmp[1])
			if err != nil {
				return err
			}
			tmpContainers[c.Id] = struct{}{}

			// Wait for it to finish
			if result := c.Wait(); result != 0 {
				return fmt.Errorf("!!! '%s' return non-zero exit code '%d'. Aborting.", tmp[1], result)
			}

			// Commit the container
			base, err := builder.runtime.Commit(c.Id, "", "", "", "")
			if err != nil {
				return err
			}
			tmpImages[base.Id] = struct{}{}

			fmt.Fprintf(stdout, "===> %s\n", base.Id)
			break
		case "copy":
			return fmt.Errorf("The copy operator has not yet been implemented")
		default:
			fmt.Fprintf(stdout, "Skipping unknown op %s\n", tmp[0])
		}
	}
	if base != nil {
		// The build is successful, keep the temporary containers and images
		for i := range tmpImages {
			delete(tmpImages, i)
		}
		for i := range tmpContainers {
			delete(tmpContainers, i)
		}
		fmt.Fprintf(stdout, "Build finished. image id: %s\n", base.Id)
	} else {
		fmt.Fprintf(stdout, "An error occured during the build\n")
	}
	return nil
}
<commit_msg>make builder.Run public it now runs only given arguments without sh -c<commit_after>package docker

import (
	"bufio"
	"fmt"
	"io"
	"strings"
)

type Builder struct {
	runtime *Runtime
}

func NewBuilder(runtime *Runtime) *Builder {
	return &Builder{
		runtime: runtime,
	}
}

func (builder *Builder) Run(image *Image, cmd ...string) (*Container, error) {
	// FIXME: pass a NopWriter instead of nil
	config, err := ParseRun(append([]string{"-d", image.Id}, cmd...), nil, builder.runtime.capabilities)
	if config.Image == "" {
		return nil, fmt.Errorf("Image not specified")
	}
	if len(config.Cmd) == 0 {
		return nil, fmt.Errorf("Command not specified")
	}
	if config.Tty {
		return nil, fmt.Errorf("The tty mode is not supported within the builder")
	}

	// Create new container
	container, err := builder.runtime.Create(config)
	if err != nil {
		return nil, err
	}
	if err := container.Start(); err != nil {
		return nil, err
	}
	return container, nil
}

func (builder *Builder) clearTmp(containers, images map[string]struct{}) {
	for c := range containers {
		tmp := builder.runtime.Get(c)
		builder.runtime.Destroy(tmp)
		Debugf("Removing container %s", c)
	}
	for i := range images {
		builder.runtime.graph.Delete(i)
		Debugf("Removing image %s", i)
	}
}

func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) error {
	var (
		image, base   *Image
		tmpContainers map[string]struct{} = make(map[string]struct{})
		tmpImages     map[string]struct{} = make(map[string]struct{})
	)
	defer builder.clearTmp(tmpContainers, tmpImages)

	file := bufio.NewReader(dockerfile)
	for {
		line, err := file.ReadString('\n')
		if err != nil {
			if err == io.EOF {
				break
			}
			return err
		}
		line = strings.TrimSpace(line)
		// Skip comments and empty line
		if len(line) == 0 || line[0] == '#' {
			continue
		}
		tmp := strings.SplitN(line, "	", 2)
		if len(tmp) != 2 {
			return fmt.Errorf("Invalid Dockerfile format")
		}
		switch tmp[0] {
		case "from":
			fmt.Fprintf(stdout, "FROM %s\n", tmp[1])
			image, err = builder.runtime.repositories.LookupImage(tmp[1])
			if err != nil {
				return err
			}
			break
		case "run":
			fmt.Fprintf(stdout, "RUN %s\n", tmp[1])
			if image == nil {
				return fmt.Errorf("Please provide a source image with `from` prior to run")
			}

			// Create the container and start it
			c, err := builder.Run(image, "/bin/sh", "-c", tmp[1])
			if err != nil {
				return err
			}
			tmpContainers[c.Id] = struct{}{}

			// Wait for it to finish
			if result := c.Wait(); result != 0 {
				return fmt.Errorf("!!! '%s' return non-zero exit code '%d'. Aborting.", tmp[1], result)
			}

			// Commit the container
			base, err := builder.runtime.Commit(c.Id, "", "", "", "")
			if err != nil {
				return err
			}
			tmpImages[base.Id] = struct{}{}

			fmt.Fprintf(stdout, "===> %s\n", base.Id)
			break
		case "copy":
			return fmt.Errorf("The copy operator has not yet been implemented")
		default:
			fmt.Fprintf(stdout, "Skipping unknown op %s\n", tmp[0])
		}
	}
	if base != nil {
		// The build is successful, keep the temporary containers and images
		for i := range tmpImages {
			delete(tmpImages, i)
		}
		for i := range tmpContainers {
			delete(tmpContainers, i)
		}
		fmt.Fprintf(stdout, "Build finished. image id: %s\n", base.Id)
	} else {
		fmt.Fprintf(stdout, "An error occured during the build\n")
	}
	return nil
}
<|endoftext|>


<commit_before>//+build linux

package vsock_test

import (
	"fmt"
	"net"
	"os"
	"strings"
	"sync"
	"testing"
	"time"

	"github.com/mdlayher/vsock"
	"github.com/mdlayher/vsock/internal/vsutil"
	"golang.org/x/net/nettest"
)

func TestIntegrationListenerUnblockAcceptAfterClose(t *testing.T) {
	l, done := newListener(t)
	defer done()

	var wg sync.WaitGroup
	wg.Add(1)

	go func() {
		defer wg.Done()

		t.Log("start accept")
		_, err := vsutil.Accept(l, 10*time.Second)
		t.Log("after accept")

		nerr, ok := err.(*net.OpError)
		if !ok {
			t.Errorf("expected a net.OpError, but got: %#v", err)
		}

		if nerr.Temporary() {
			t.Errorf("expected permanent error, but got temporary one: %v", err)
		}

		// We mimic what net.TCPConn does and return an error with the same
		// string as internal/poll, so string matching is the best we can do
		// for now.
		if !strings.Contains(nerr.Err.Error(), "use of closed") {
			t.Errorf("expected close network connection error, but got: %v", nerr.Err)
		}
	}()

	time.Sleep(100 * time.Millisecond)

	if err := l.Close(); err != nil {
		t.Fatalf("failed to close listener: %v", err)
	}

	doneC := make(chan struct{})
	go func() {
		wg.Wait()
		close(doneC)
	}()

	select {
	case <-doneC:
	case <-time.After(5 * time.Second):
		t.Fatal("timeout waiting accept to unblock")
	}
}

func TestIntegrationContextIDGuest(t *testing.T) {
	if vsutil.IsHypervisor(t) {
		t.Skip("skipping, machine is not a guest")
	}

	cid, err := vsock.ContextID()
	if err != nil {
		t.Fatalf("failed to retrieve guest's context ID: %v", err)
	}

	t.Logf("guest context ID: %d", cid)

	// Guests should always have a context ID of 3 or more, since
	// 0-2 are invalid or reserved.
	if cid < 3 {
		t.Fatalf("unexpected guest context ID: %d", cid)
	}
}

func TestIntegrationContextIDHost(t *testing.T) {
	if !vsutil.IsHypervisor(t) {
		t.Skip("skipping, machine is not a hypervisor")
	}

	cid, err := vsock.ContextID()
	if err != nil {
		t.Fatalf("failed to retrieve host's context ID: %v", err)
	}

	t.Logf("host context ID: %d", cid)

	if want, got := uint32(vsock.Host), cid; want != got {
		t.Fatalf("unexpected host context ID:\n- want: %d\n-  got: %d",
			want, got)
	}
}

func TestIntegrationNettest(t *testing.T) {
	if vsutil.IsHypervisor(t) {
		t.Skip("skipping, x/net/nettest vsock integration tests must be run in a guest")
	}

	nettest.TestConn(t, makeLocalPipe(
		func() (net.Listener, error) { return vsock.Listen(0) },
		func(addr net.Addr) (net.Conn, error) {
			a := addr.(*vsock.Addr)
			return vsock.Dial(a.ContextID, a.Port)
		},
	))
}

func newListener(t *testing.T) (*vsock.Listener, func()) {
	t.Helper()

	timer := time.AfterFunc(10*time.Second, func() {
		panic("test took too long")
	})

	l, err := vsock.Listen(0)
	if err != nil {
		vsutil.SkipDeviceError(t, err)

		t.Fatalf("failed to create vsock listener: %v", err)
	}

	return l, func() {
		// Clean up the timer and this listener.
		timer.Stop()
		_ = l.Close()
	}
}

// makeLocalPipe produces a nettest.MakePipe function using the input functions
// to configure a net.Listener and point a net.Conn at the listener.
//
// This function is proposed for inclusion in x/net/nettest, and should be
// removed from here if the proposal is accepted. See:
// https://github.com/golang/go/issues/30984.
func makeLocalPipe(
	listen func() (net.Listener, error),
	dial func(addr net.Addr) (net.Conn, error),
) nettest.MakePipe {
	if listen == nil {
		panic("nil listen function passed to makeLocalPipe")
	}

	if dial == nil {
		dial = func(addr net.Addr) (net.Conn, error) {
			return net.Dial(addr.Network(), addr.String())
		}
	}

	// The majority of this code is taken from golang.org/x/net/nettest:
	// https://go.googlesource.com/net/+/9e4ed9723b84cb6661bb04e4104f7bfb3ff5d016/nettest/conntest_test.go.
	//
	// Copyright 2016 The Go Authors. All rights reserved.

	return func() (c1, c2 net.Conn, stop func(), err error) {
		ln, err := listen()
		if err != nil {
			return nil, nil, nil, fmt.Errorf("failed to create local listener: %v", err)
		}

		// Start a connection between two endpoints.
		var err1, err2 error
		done := make(chan bool)
		go func() {
			c2, err2 = ln.Accept()
			close(done)
		}()
		c1, err1 = dial(ln.Addr())
		<-done

		stop = func() {
			if err1 == nil {
				c1.Close()
			}
			if err2 == nil {
				c2.Close()
			}
			ln.Close()
			switch ln.Addr().Network() {
			case "unix", "unixpacket":
				os.Remove(ln.Addr().String())
			}
		}

		switch {
		case err1 != nil:
			stop()
			return nil, nil, nil, err1
		case err2 != nil:
			stop()
			return nil, nil, nil, err2
		default:
			return c1, c2, stop, nil
		}
	}
}
<commit_msg>vsock: add scaffolding for x/net/nettest TestListener usage<commit_after>//+build linux

package vsock_test

import (
	"fmt"
	"net"
	"os"
	"regexp"
	"strconv"
	"strings"
	"sync"
	"testing"
	"time"

	"github.com/mdlayher/vsock"
	"github.com/mdlayher/vsock/internal/vsutil"
	"golang.org/x/net/nettest"
)

func TestIntegrationListenerUnblockAcceptAfterClose(t *testing.T) {
	l, done := newListener(t)
	defer done()

	var wg sync.WaitGroup
	wg.Add(1)

	go func() {
		defer wg.Done()

		t.Log("start accept")
		_, err := vsutil.Accept(l, 10*time.Second)
		t.Log("after accept")

		nerr, ok := err.(*net.OpError)
		if !ok {
			t.Errorf("expected a net.OpError, but got: %#v", err)
		}

		if nerr.Temporary() {
			t.Errorf("expected permanent error, but got temporary one: %v", err)
		}

		// We mimic what net.TCPConn does and return an error with the same
		// string as internal/poll, so string matching is the best we can do
		// for now.
		if !strings.Contains(nerr.Err.Error(), "use of closed") {
			t.Errorf("expected close network connection error, but got: %v", nerr.Err)
		}
	}()

	time.Sleep(100 * time.Millisecond)

	if err := l.Close(); err != nil {
		t.Fatalf("failed to close listener: %v", err)
	}

	doneC := make(chan struct{})
	go func() {
		wg.Wait()
		close(doneC)
	}()

	select {
	case <-doneC:
	case <-time.After(5 * time.Second):
		t.Fatal("timeout waiting accept to unblock")
	}
}

func TestIntegrationContextIDGuest(t *testing.T) {
	if vsutil.IsHypervisor(t) {
		t.Skip("skipping, machine is not a guest")
	}

	cid, err := vsock.ContextID()
	if err != nil {
		t.Fatalf("failed to retrieve guest's context ID: %v", err)
	}

	t.Logf("guest context ID: %d", cid)

	// Guests should always have a context ID of 3 or more, since
	// 0-2 are invalid or reserved.
	if cid < 3 {
		t.Fatalf("unexpected guest context ID: %d", cid)
	}
}

func TestIntegrationContextIDHost(t *testing.T) {
	if !vsutil.IsHypervisor(t) {
		t.Skip("skipping, machine is not a hypervisor")
	}

	cid, err := vsock.ContextID()
	if err != nil {
		t.Fatalf("failed to retrieve host's context ID: %v", err)
	}

	t.Logf("host context ID: %d", cid)

	if want, got := uint32(vsock.Host), cid; want != got {
		t.Fatalf("unexpected host context ID:\n- want: %d\n-  got: %d",
			want, got)
	}
}

func TestIntegrationNettestTestConn(t *testing.T) {
	if vsutil.IsHypervisor(t) {
		t.Skip("skipping, x/net/nettest vsock integration tests must be run in a guest")
	}

	nettest.TestConn(t, makeLocalPipe(
		func() (net.Listener, error) { return vsock.Listen(0) },
		func(addr net.Addr) (net.Conn, error) {
			a := addr.(*vsock.Addr)
			return vsock.Dial(a.ContextID, a.Port)
		},
	))
}

var cidRe = regexp.MustCompile(`\S+\((\d+)\)`)

func TestIntegrationNettestTestListener(t *testing.T) {
	if vsutil.IsHypervisor(t) {
		t.Skip("skipping, x/net/nettest vsock integration tests must be run in a guest")
	}

	// This test uses the nettest.TestListener API which is being built in:
	// https://go-review.googlesource.com/c/net/+/123056.
	//
	// TODO(mdlayher): stop skipping this test once that CL lands.

	mos := func() (ln net.Listener, dial func(string, string) (net.Conn, error), stop func(), err error) {
		l, err := vsock.Listen(0)
		if err != nil {
			return nil, nil, nil, err
		}

		stop = func() {
			// TODO(mdlayher): cancel context if we use vsock.DialContext.
			_ = l.Close()
		}

		dial = func(_, addr string) (net.Conn, error) {
			host, sport, err := net.SplitHostPort(addr)
			if err != nil {
				return nil, err
			}

			// Extract the CID value from the surrounding text.
			scid := cidRe.FindStringSubmatch(host)
			cid, err := strconv.Atoi(scid[1])
			if err != nil {
				return nil, err
			}

			port, err := strconv.Atoi(sport)
			if err != nil {
				return nil, err
			}

			return vsock.Dial(uint32(cid), uint32(port))
		}

		return l, dial, stop, nil
	}

	_ = mos
	t.Skip("skipping, enable once https://go-review.googlesource.com/c/net/+/123056 is merged")
	// nettest.TestListener(t, mos)
}

func newListener(t *testing.T) (*vsock.Listener, func()) {
	t.Helper()

	timer := time.AfterFunc(10*time.Second, func() {
		panic("test took too long")
	})

	l, err := vsock.Listen(0)
	if err != nil {
		vsutil.SkipDeviceError(t, err)

		t.Fatalf("failed to create vsock listener: %v", err)
	}

	return l, func() {
		// Clean up the timer and this listener.
		timer.Stop()
		_ = l.Close()
	}
}

// makeLocalPipe produces a nettest.MakePipe function using the input functions
// to configure a net.Listener and point a net.Conn at the listener.
//
// This function is proposed for inclusion in x/net/nettest, and should be
// removed from here if the proposal is accepted. See:
// https://github.com/golang/go/issues/30984.
func makeLocalPipe(
	listen func() (net.Listener, error),
	dial func(addr net.Addr) (net.Conn, error),
) nettest.MakePipe {
	if listen == nil {
		panic("nil listen function passed to makeLocalPipe")
	}

	if dial == nil {
		dial = func(addr net.Addr) (net.Conn, error) {
			return net.Dial(addr.Network(), addr.String())
		}
	}

	// The majority of this code is taken from golang.org/x/net/nettest:
	// https://go.googlesource.com/net/+/9e4ed9723b84cb6661bb04e4104f7bfb3ff5d016/nettest/conntest_test.go.
	//
	// Copyright 2016 The Go Authors. All rights reserved.

	return func() (c1, c2 net.Conn, stop func(), err error) {
		ln, err := listen()
		if err != nil {
			return nil, nil, nil, fmt.Errorf("failed to create local listener: %v", err)
		}

		// Start a connection between two endpoints.
		var err1, err2 error
		done := make(chan bool)
		go func() {
			c2, err2 = ln.Accept()
			close(done)
		}()
		c1, err1 = dial(ln.Addr())
		<-done

		stop = func() {
			if err1 == nil {
				c1.Close()
			}
			if err2 == nil {
				c2.Close()
			}
			ln.Close()
			switch ln.Addr().Network() {
			case "unix", "unixpacket":
				os.Remove(ln.Addr().String())
			}
		}

		switch {
		case err1 != nil:
			stop()
			return nil, nil, nil, err1
		case err2 != nil:
			stop()
			return nil, nil, nil, err2
		default:
			return c1, c2, stop, nil
		}
	}
}
<|endoftext|>


<commit_before>package bild

import (
	"image"
	"math"
)

// ConvolutionMatrix interface for use as an image Kernel
type ConvolutionMatrix interface {
	At(x, y int) float64
	Sum() float64
	Normalized() ConvolutionMatrix
	Diameter() int
}

// NewKernel returns a kernel of the provided size
func NewKernel(diameter int) *Kernel {
	matrix := make([][]float64, diameter)
	for i := 0; i < diameter; i++ {
		matrix[i] = make([]float64, diameter)
	}
	return &Kernel{matrix}
}

// Kernel is used as a convolution matrix
type Kernel struct {
	Matrix [][]float64
}

// Sum returns the cumulative value of the matrix
func (k *Kernel) Sum() float64 {
	var sum float64
	diameter := k.Diameter()
	for x := 0; x < diameter; x++ {
		for y := 0; y < diameter; y++ {
			sum += k.Matrix[x][y]
		}
	}
	return sum
}

// Normalized returns a new Kernel with normalized values
func (k *Kernel) Normalized() ConvolutionMatrix {
	sum := k.Sum()
	diameter := k.Diameter()
	nk := NewKernel(diameter)

	for x := 0; x < diameter; x++ {
		for y := 0; y < diameter; y++ {
			nk.Matrix[x][y] = k.Matrix[x][y] / sum
		}
	}

	return nk
}

// Diameter returns the row/column length for the kernel
func (k *Kernel) Diameter() int {
	return len(k.Matrix)
}

// At returns the matrix value at position x, y
func (k *Kernel) At(x, y int) float64 {
	return k.Matrix[x][y]
}

// convolute applies a convolution matrix (kernel) to an image
func convolute(img image.Image, k ConvolutionMatrix) *image.RGBA {
	bounds := img.Bounds()
	src := CloneAsRGBA(img)
	dst := image.NewRGBA(bounds)

	w, h := bounds.Max.X, bounds.Max.Y
	diameter := k.Diameter()
	normKernel := k.Normalized()

	parallelize(h, func(start, end int) {
		for x := 0; x < w; x++ {
			for y := start; y < end; y++ {

				var r, g, b, a float64
				for kx := 0; kx < diameter; kx++ {
					for ky := 0; ky < diameter; ky++ {
						ix := x - diameter/2 + kx
						iy := y - diameter/2 + ky

						if ix < 0 || ix >= w || iy < 0 || iy >= h {
							continue
						}

						ipos := iy*dst.Stride + ix*4
						kvalue := normKernel.At(kx, ky)
						r += float64(src.Pix[ipos+0]) * kvalue
						g += float64(src.Pix[ipos+1]) * kvalue
						b += float64(src.Pix[ipos+2]) * kvalue
						a += float64(src.Pix[ipos+3]) * kvalue
					}
				}

				pos := y*dst.Stride + x*4

				dst.Pix[pos+0] = uint8(math.Max(math.Min(r, 255), 0))
				dst.Pix[pos+1] = uint8(math.Max(math.Min(g, 255), 0))
				dst.Pix[pos+2] = uint8(math.Max(math.Min(b, 255), 0))
				dst.Pix[pos+3] = uint8(math.Max(math.Min(a, 255), 0))
			}
		}
	})

	return dst
}
<commit_msg>switched to unnormalized kernel and added bias<commit_after>package bild

import (
	"image"
	"math"
)

// ConvolutionMatrix interface for use as an image Kernel
type ConvolutionMatrix interface {
	At(x, y int) float64
	Sum() float64
	Normalized() ConvolutionMatrix
	Diameter() int
}

// NewKernel returns a kernel of the provided size
func NewKernel(diameter int) *Kernel {
	matrix := make([][]float64, diameter)
	for i := 0; i < diameter; i++ {
		matrix[i] = make([]float64, diameter)
	}
	return &Kernel{matrix}
}

// Kernel is used as a convolution matrix
type Kernel struct {
	Matrix [][]float64
}

// Sum returns the cumulative value of the matrix
func (k *Kernel) Sum() float64 {
	var sum float64
	diameter := k.Diameter()
	for x := 0; x < diameter; x++ {
		for y := 0; y < diameter; y++ {
			sum += k.Matrix[x][y]
		}
	}
	return sum
}

// Normalized returns a new Kernel with normalized values
func (k *Kernel) Normalized() ConvolutionMatrix {
	sum := k.Sum()
	diameter := k.Diameter()
	nk := NewKernel(diameter)

	for x := 0; x < diameter; x++ {
		for y := 0; y < diameter; y++ {
			nk.Matrix[x][y] = k.Matrix[x][y] / sum
		}
	}

	return nk
}

// Diameter returns the row/column length for the kernel
func (k *Kernel) Diameter() int {
	return len(k.Matrix)
}

// At returns the matrix value at position x, y
func (k *Kernel) At(x, y int) float64 {
	return k.Matrix[x][y]
}

// convolute applies a convolution matrix (kernel) to an image
func convolute(img image.Image, k ConvolutionMatrix, bias float64) *image.RGBA {
	bounds := img.Bounds()
	src := CloneAsRGBA(img)
	dst := image.NewRGBA(bounds)

	w, h := bounds.Max.X, bounds.Max.Y
	diameter := k.Diameter()

	parallelize(h, func(start, end int) {
		for x := 0; x < w; x++ {
			for y := start; y < end; y++ {

				var r, g, b, a float64
				for kx := 0; kx < diameter; kx++ {
					for ky := 0; ky < diameter; ky++ {
						ix := x - diameter/2 + kx
						iy := y - diameter/2 + ky

						if ix < 0 || ix >= w || iy < 0 || iy >= h {
							continue
						}

						ipos := iy*dst.Stride + ix*4
						kvalue := k.At(kx, ky)
						r += float64(src.Pix[ipos+0]) * kvalue
						g += float64(src.Pix[ipos+1]) * kvalue
						b += float64(src.Pix[ipos+2]) * kvalue
						a += float64(src.Pix[ipos+3]) * kvalue
					}
				}

				pos := y*dst.Stride + x*4

				dst.Pix[pos+0] = uint8(math.Max(math.Min(r+bias, 255), 0))
				dst.Pix[pos+1] = uint8(math.Max(math.Min(g+bias, 255), 0))
				dst.Pix[pos+2] = uint8(math.Max(math.Min(b+bias, 255), 0))
				dst.Pix[pos+3] = uint8(math.Max(math.Min(a+bias, 255), 0))
			}
		}
	})

	return dst
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"net/http"
)

// Fetch the url given in parameter and unmarshall the JSON to the given out struct
func fetchURL(url string, out interface{}) (err error) {
	client := new(http.Client)
	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return
	}
	req.Header.Add("Accept", "application/vnd.github.v3+json")
	resp, err := client.Do(req)
	defer resp.Body.Close()
	dec := json.NewDecoder(resp.Body)
	err = dec.Decode(out)
	return
}
<commit_msg>Hm. Stuff ?<commit_after>package main

import (
	"encoding/json"
	"net/http"
)

// Fetch the url given in parameter and unmarshall the JSON to the given out struct
func fetchURL(url string, out interface{}) (err error) {
	c := new(http.Client)
	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return
	}
	// Add the API version in Headers
	req.Header.Add("Accept", "application/vnd.github.v3+json")
	resp, err := c.Do(req)
	defer resp.Body.Close()
	dec := json.NewDecoder(resp.Body)
	err = dec.Decode(out)
	return
}
<|endoftext|>


<commit_before>package eventbus

import (
	"fmt"
	"sync"
	"sync/atomic"
	"testing"
	"time"

	"github.com/libp2p/go-libp2p-testing/race"
)

type EventA struct{}
type EventB int

func getN() int {
	n := 50000
	if race.WithRace() {
		n = 1000
	}
	return n
}

func (EventA) String() string {
	return "Oh, Hello"
}

func TestDefaultSubIsBuffered(t *testing.T) {
	bus := NewBus()
	s, err := bus.Subscribe(new(EventA))
	if err != nil {
		t.Fatal(err)
	}
	if cap(s.(*sub).ch) == 0 {
		t.Fatalf("without any options subscribe should be buffered. was %d", cap(s.(*sub).ch))
	}
}

func TestEmit(t *testing.T) {
	bus := NewBus()
	sub, err := bus.Subscribe(new(EventA))
	if err != nil {
		t.Fatal(err)
	}

	go func() {
		defer sub.Close()
		<-sub.Out()
	}()

	em, err := bus.Emitter(new(EventA))
	if err != nil {
		t.Fatal(err)
	}
	defer em.Close()

	em.Emit(EventA{})
}

func TestSub(t *testing.T) {
	bus := NewBus()
	sub, err := bus.Subscribe(new(EventB))
	if err != nil {
		t.Fatal(err)
	}

	var event EventB

	var wait sync.WaitGroup
	wait.Add(1)

	go func() {
		defer sub.Close()
		event = (<-sub.Out()).(EventB)
		wait.Done()
	}()

	em, err := bus.Emitter(new(EventB))
	if err != nil {
		t.Fatal(err)
	}
	defer em.Close()

	em.Emit(EventB(7))
	wait.Wait()

	if event != 7 {
		t.Error("got wrong event")
	}
}

func TestEmitNoSubNoBlock(t *testing.T) {
	bus := NewBus()

	em, err := bus.Emitter(new(EventA))
	if err != nil {
		t.Fatal(err)
	}
	defer em.Close()

	em.Emit(EventA{})
}

func TestEmitOnClosed(t *testing.T) {
	bus := NewBus()

	em, err := bus.Emitter(new(EventA))
	if err != nil {
		t.Fatal(err)
	}
	em.Close()
	err = em.Emit(EventA{})
	if err == nil {
		t.Errorf("expected panic")
	}
	if err.Error() != "emitter is closed" {
		t.Error("unexpected message")
	}
}

func TestClosingRaces(t *testing.T) {
	subs := getN()
	emits := getN()

	var wg sync.WaitGroup
	var lk sync.RWMutex
	lk.Lock()

	wg.Add(subs + emits)

	b := NewBus()

	for i := 0; i < subs; i++ {
		go func() {
			lk.RLock()
			defer lk.RUnlock()

			sub, _ := b.Subscribe(new(EventA))
			time.Sleep(10 * time.Millisecond)
			sub.Close()

			wg.Done()
		}()
	}
	for i := 0; i < emits; i++ {
		go func() {
			lk.RLock()
			defer lk.RUnlock()

			emit, _ := b.Emitter(new(EventA))
			time.Sleep(10 * time.Millisecond)
			emit.Close()

			wg.Done()
		}()
	}

	time.Sleep(10 * time.Millisecond)
	lk.Unlock() // start everything

	wg.Wait()

	if len(b.(*basicBus).nodes) != 0 {
		t.Error("expected no nodes")
	}
}

func TestSubMany(t *testing.T) {
	bus := NewBus()

	var r int32

	n := getN()
	var wait sync.WaitGroup
	var ready sync.WaitGroup
	wait.Add(n)
	ready.Add(n)

	for i := 0; i < n; i++ {
		go func() {
			sub, err := bus.Subscribe(new(EventB))
			if err != nil {
				panic(err)
			}
			defer sub.Close()

			ready.Done()
			atomic.AddInt32(&r, int32((<-sub.Out()).(EventB)))
			wait.Done()
		}()
	}

	em, err := bus.Emitter(new(EventB))
	if err != nil {
		t.Fatal(err)
	}
	defer em.Close()

	ready.Wait()

	em.Emit(EventB(7))
	wait.Wait()

	if int(r) != 7*n {
		t.Error("got wrong result")
	}
}

func TestSubType(t *testing.T) {
	bus := NewBus()
	sub, err := bus.Subscribe([]interface{}{new(EventA), new(EventB)})
	if err != nil {
		t.Fatal(err)
	}

	var event fmt.Stringer

	var wait sync.WaitGroup
	wait.Add(1)

	go func() {
		defer sub.Close()
		event = (<-sub.Out()).(EventA)
		wait.Done()
	}()

	em, err := bus.Emitter(new(EventA))
	if err != nil {
		t.Fatal(err)
	}
	defer em.Close()

	em.Emit(EventA{})
	wait.Wait()

	if event.String() != "Oh, Hello" {
		t.Error("didn't get the correct message")
	}
}

func TestNonStateful(t *testing.T) {
	bus := NewBus()
	em, err := bus.Emitter(new(EventB))
	if err != nil {
		t.Fatal(err)
	}
	defer em.Close()

	sub1, err := bus.Subscribe(new(EventB), BufSize(1))
	if err != nil {
		t.Fatal(err)
	}
	defer sub1.Close()

	select {
	case <-sub1.Out():
		t.Fatal("didn't expect to get an event")
	default:
	}

	em.Emit(EventB(1))

	select {
	case e := <-sub1.Out():
		if e.(EventB) != 1 {
			t.Fatal("got wrong event")
		}
	default:
		t.Fatal("expected to get an event")
	}

	sub2, err := bus.Subscribe(new(EventB), BufSize(1))
	if err != nil {
		t.Fatal(err)
	}
	defer sub2.Close()

	select {
	case <-sub2.Out():
		t.Fatal("didn't expect to get an event")
	default:
	}
}

func TestStateful(t *testing.T) {
	bus := NewBus()
	em, err := bus.Emitter(new(EventB), Stateful)
	if err != nil {
		t.Fatal(err)
	}
	defer em.Close()

	em.Emit(EventB(2))

	sub, err := bus.Subscribe(new(EventB), BufSize(1))
	if err != nil {
		t.Fatal(err)
	}
	defer sub.Close()

	if (<-sub.Out()).(EventB) != 2 {
		t.Fatal("got wrong event")
	}
}

func TestCloseBlocking(t *testing.T) {
	bus := NewBus()
	em, err := bus.Emitter(new(EventB))
	if err != nil {
		t.Fatal(err)
	}

	sub, err := bus.Subscribe(new(EventB))
	if err != nil {
		t.Fatal(err)
	}

	go func() {
		em.Emit(EventB(159))
	}()

	time.Sleep(10 * time.Millisecond) // make sure that emit is blocked

	sub.Close() // cancel sub
}

func panicOnTimeout(d time.Duration) {
	<-time.After(d)
	panic("timeout reached")
}

func TestSubFailFully(t *testing.T) {
	bus := NewBus()
	em, err := bus.Emitter(new(EventB))
	if err != nil {
		t.Fatal(err)
	}

	_, err = bus.Subscribe([]interface{}{new(EventB), 5})
	if err == nil || err.Error() != "subscribe called with non-pointer type" {
		t.Fatal(err)
	}

	go panicOnTimeout(5 * time.Second)

	em.Emit(EventB(159)) // will hang if sub doesn't fail properly
}

func testMany(t testing.TB, subs, emits, msgs int, stateful bool) {
	if race.WithRace() && subs+emits > 5000 {
		t.SkipNow()
	}

	bus := NewBus()

	var r int64

	var wait sync.WaitGroup
	var ready sync.WaitGroup
	wait.Add(subs + emits)
	ready.Add(subs)

	for i := 0; i < subs; i++ {
		go func() {
			sub, err := bus.Subscribe(new(EventB))
			if err != nil {
				panic(err)
			}
			defer sub.Close()

			ready.Done()
			for i := 0; i < emits*msgs; i++ {
				e, ok := <-sub.Out()
				if !ok {
					panic("wat")
				}
				atomic.AddInt64(&r, int64(e.(EventB)))
			}
			wait.Done()
		}()
	}

	for i := 0; i < emits; i++ {
		go func() {
			em, err := bus.Emitter(new(EventB), func(settings interface{}) error {
				settings.(*emitterSettings).makeStateful = stateful
				return nil
			})
			if err != nil {
				panic(err)
			}
			defer em.Close()

			ready.Wait()

			for i := 0; i < msgs; i++ {
				em.Emit(EventB(97))
			}

			wait.Done()
		}()
	}

	wait.Wait()

	if int(r) != 97*subs*emits*msgs {
		t.Fatal("got wrong result")
	}
}

func TestBothMany(t *testing.T) {
	testMany(t, 10000, 100, 10, false)
}

type benchCase struct {
	subs     int
	emits    int
	stateful bool
}

func (bc benchCase) name() string {
	return fmt.Sprintf("subs-%03d/emits-%03d/stateful-%t", bc.subs, bc.emits, bc.stateful)
}

func genTestCases() []benchCase {
	ret := make([]benchCase, 0, 200)
	for stateful := 0; stateful < 2; stateful++ {
		for subs := uint(0); subs <= 8; subs = subs + 4 {
			for emits := uint(0); emits <= 8; emits = emits + 4 {
				ret = append(ret, benchCase{1 << subs, 1 << emits, stateful == 1})
			}
		}
	}
	return ret
}

func BenchmarkEvents(b *testing.B) {
	for _, bc := range genTestCases() {
		b.Run(bc.name(), benchMany(bc))
	}
}

func benchMany(bc benchCase) func(*testing.B) {
	return func(b *testing.B) {
		b.ReportAllocs()
		subs := bc.subs
		emits := bc.emits
		stateful := bc.stateful
		bus := NewBus()
		var wait sync.WaitGroup
		var ready sync.WaitGroup
		wait.Add(subs + emits)
		ready.Add(subs + emits)

		for i := 0; i < subs; i++ {
			go func() {
				sub, err := bus.Subscribe(new(EventB))
				if err != nil {
					panic(err)
				}
				defer sub.Close()

				ready.Done()
				ready.Wait()
				for i := 0; i < (b.N/emits)*emits; i++ {
					_, ok := <-sub.Out()
					if !ok {
						panic("wat")
					}
				}
				wait.Done()
			}()
		}

		for i := 0; i < emits; i++ {
			go func() {
				em, err := bus.Emitter(new(EventB), func(settings interface{}) error {
					settings.(*emitterSettings).makeStateful = stateful
					return nil
				})
				if err != nil {
					panic(err)
				}
				defer em.Close()

				ready.Done()
				ready.Wait()

				for i := 0; i < b.N/emits; i++ {
					em.Emit(EventB(97))
				}

				wait.Done()
			}()
		}
		ready.Wait()
		b.ResetTimer()
		wait.Wait()
	}
}

var div = 100

func BenchmarkSubscribe(b *testing.B) {
	b.ReportAllocs()
	for i := 0; i < b.N/div; i++ {
		bus := NewBus()
		for j := 0; j < div; j++ {
			bus.Subscribe(new(EventA))
		}
	}
}

func BenchmarkEmitter(b *testing.B) {
	b.ReportAllocs()
	for i := 0; i < b.N/div; i++ {
		bus := NewBus()
		for j := 0; j < div; j++ {
			bus.Emitter(new(EventA))
		}
	}
}

func BenchmarkSubscribeAndEmitter(b *testing.B) {
	b.ReportAllocs()
	for i := 0; i < b.N/div; i++ {
		bus := NewBus()
		for j := 0; j < div; j++ {
			bus.Subscribe(new(EventA))
			bus.Emitter(new(EventA))
		}
	}
}
<commit_msg>Update test error message<commit_after>package eventbus

import (
	"fmt"
	"sync"
	"sync/atomic"
	"testing"
	"time"

	"github.com/libp2p/go-libp2p-testing/race"
)

type EventA struct{}
type EventB int

func getN() int {
	n := 50000
	if race.WithRace() {
		n = 1000
	}
	return n
}

func (EventA) String() string {
	return "Oh, Hello"
}

func TestDefaultSubIsBuffered(t *testing.T) {
	bus := NewBus()
	s, err := bus.Subscribe(new(EventA))
	if err != nil {
		t.Fatal(err)
	}
	if cap(s.(*sub).ch) == 0 {
		t.Fatalf("without any options subscribe should be buffered. was %d", cap(s.(*sub).ch))
	}
}

func TestEmit(t *testing.T) {
	bus := NewBus()
	sub, err := bus.Subscribe(new(EventA))
	if err != nil {
		t.Fatal(err)
	}

	go func() {
		defer sub.Close()
		<-sub.Out()
	}()

	em, err := bus.Emitter(new(EventA))
	if err != nil {
		t.Fatal(err)
	}
	defer em.Close()

	em.Emit(EventA{})
}

func TestSub(t *testing.T) {
	bus := NewBus()
	sub, err := bus.Subscribe(new(EventB))
	if err != nil {
		t.Fatal(err)
	}

	var event EventB

	var wait sync.WaitGroup
	wait.Add(1)

	go func() {
		defer sub.Close()
		event = (<-sub.Out()).(EventB)
		wait.Done()
	}()

	em, err := bus.Emitter(new(EventB))
	if err != nil {
		t.Fatal(err)
	}
	defer em.Close()

	em.Emit(EventB(7))
	wait.Wait()

	if event != 7 {
		t.Error("got wrong event")
	}
}

func TestEmitNoSubNoBlock(t *testing.T) {
	bus := NewBus()

	em, err := bus.Emitter(new(EventA))
	if err != nil {
		t.Fatal(err)
	}
	defer em.Close()

	em.Emit(EventA{})
}

func TestEmitOnClosed(t *testing.T) {
	bus := NewBus()

	em, err := bus.Emitter(new(EventA))
	if err != nil {
		t.Fatal(err)
	}
	em.Close()
	err = em.Emit(EventA{})
	if err == nil {
		t.Errorf("expected error")
	}
	if err.Error() != "emitter is closed" {
		t.Error("unexpected message")
	}
}

func TestClosingRaces(t *testing.T) {
	subs := getN()
	emits := getN()

	var wg sync.WaitGroup
	var lk sync.RWMutex
	lk.Lock()

	wg.Add(subs + emits)

	b := NewBus()

	for i := 0; i < subs; i++ {
		go func() {
			lk.RLock()
			defer lk.RUnlock()

			sub, _ := b.Subscribe(new(EventA))
			time.Sleep(10 * time.Millisecond)
			sub.Close()

			wg.Done()
		}()
	}
	for i := 0; i < emits; i++ {
		go func() {
			lk.RLock()
			defer lk.RUnlock()

			emit, _ := b.Emitter(new(EventA))
			time.Sleep(10 * time.Millisecond)
			emit.Close()

			wg.Done()
		}()
	}

	time.Sleep(10 * time.Millisecond)
	lk.Unlock() // start everything

	wg.Wait()

	if len(b.(*basicBus).nodes) != 0 {
		t.Error("expected no nodes")
	}
}

func TestSubMany(t *testing.T) {
	bus := NewBus()

	var r int32

	n := getN()
	var wait sync.WaitGroup
	var ready sync.WaitGroup
	wait.Add(n)
	ready.Add(n)

	for i := 0; i < n; i++ {
		go func() {
			sub, err := bus.Subscribe(new(EventB))
			if err != nil {
				panic(err)
			}
			defer sub.Close()

			ready.Done()
			atomic.AddInt32(&r, int32((<-sub.Out()).(EventB)))
			wait.Done()
		}()
	}

	em, err := bus.Emitter(new(EventB))
	if err != nil {
		t.Fatal(err)
	}
	defer em.Close()

	ready.Wait()

	em.Emit(EventB(7))
	wait.Wait()

	if int(r) != 7*n {
		t.Error("got wrong result")
	}
}

func TestSubType(t *testing.T) {
	bus := NewBus()
	sub, err := bus.Subscribe([]interface{}{new(EventA), new(EventB)})
	if err != nil {
		t.Fatal(err)
	}

	var event fmt.Stringer

	var wait sync.WaitGroup
	wait.Add(1)

	go func() {
		defer sub.Close()
		event = (<-sub.Out()).(EventA)
		wait.Done()
	}()

	em, err := bus.Emitter(new(EventA))
	if err != nil {
		t.Fatal(err)
	}
	defer em.Close()

	em.Emit(EventA{})
	wait.Wait()

	if event.String() != "Oh, Hello" {
		t.Error("didn't get the correct message")
	}
}

func TestNonStateful(t *testing.T) {
	bus := NewBus()
	em, err := bus.Emitter(new(EventB))
	if err != nil {
		t.Fatal(err)
	}
	defer em.Close()

	sub1, err := bus.Subscribe(new(EventB), BufSize(1))
	if err != nil {
		t.Fatal(err)
	}
	defer sub1.Close()

	select {
	case <-sub1.Out():
		t.Fatal("didn't expect to get an event")
	default:
	}

	em.Emit(EventB(1))

	select {
	case e := <-sub1.Out():
		if e.(EventB) != 1 {
			t.Fatal("got wrong event")
		}
	default:
		t.Fatal("expected to get an event")
	}

	sub2, err := bus.Subscribe(new(EventB), BufSize(1))
	if err != nil {
		t.Fatal(err)
	}
	defer sub2.Close()

	select {
	case <-sub2.Out():
		t.Fatal("didn't expect to get an event")
	default:
	}
}

func TestStateful(t *testing.T) {
	bus := NewBus()
	em, err := bus.Emitter(new(EventB), Stateful)
	if err != nil {
		t.Fatal(err)
	}
	defer em.Close()

	em.Emit(EventB(2))

	sub, err := bus.Subscribe(new(EventB), BufSize(1))
	if err != nil {
		t.Fatal(err)
	}
	defer sub.Close()

	if (<-sub.Out()).(EventB) != 2 {
		t.Fatal("got wrong event")
	}
}

func TestCloseBlocking(t *testing.T) {
	bus := NewBus()
	em, err := bus.Emitter(new(EventB))
	if err != nil {
		t.Fatal(err)
	}

	sub, err := bus.Subscribe(new(EventB))
	if err != nil {
		t.Fatal(err)
	}

	go func() {
		em.Emit(EventB(159))
	}()

	time.Sleep(10 * time.Millisecond) // make sure that emit is blocked

	sub.Close() // cancel sub
}

func panicOnTimeout(d time.Duration) {
	<-time.After(d)
	panic("timeout reached")
}

func TestSubFailFully(t *testing.T) {
	bus := NewBus()
	em, err := bus.Emitter(new(EventB))
	if err != nil {
		t.Fatal(err)
	}

	_, err = bus.Subscribe([]interface{}{new(EventB), 5})
	if err == nil || err.Error() != "subscribe called with non-pointer type" {
		t.Fatal(err)
	}

	go panicOnTimeout(5 * time.Second)

	em.Emit(EventB(159)) // will hang if sub doesn't fail properly
}

func testMany(t testing.TB, subs, emits, msgs int, stateful bool) {
	if race.WithRace() && subs+emits > 5000 {
		t.SkipNow()
	}

	bus := NewBus()

	var r int64

	var wait sync.WaitGroup
	var ready sync.WaitGroup
	wait.Add(subs + emits)
	ready.Add(subs)

	for i := 0; i < subs; i++ {
		go func() {
			sub, err := bus.Subscribe(new(EventB))
			if err != nil {
				panic(err)
			}
			defer sub.Close()

			ready.Done()
			for i := 0; i < emits*msgs; i++ {
				e, ok := <-sub.Out()
				if !ok {
					panic("wat")
				}
				atomic.AddInt64(&r, int64(e.(EventB)))
			}
			wait.Done()
		}()
	}

	for i := 0; i < emits; i++ {
		go func() {
			em, err := bus.Emitter(new(EventB), func(settings interface{}) error {
				settings.(*emitterSettings).makeStateful = stateful
				return nil
			})
			if err != nil {
				panic(err)
			}
			defer em.Close()

			ready.Wait()

			for i := 0; i < msgs; i++ {
				em.Emit(EventB(97))
			}

			wait.Done()
		}()
	}

	wait.Wait()

	if int(r) != 97*subs*emits*msgs {
		t.Fatal("got wrong result")
	}
}

func TestBothMany(t *testing.T) {
	testMany(t, 10000, 100, 10, false)
}

type benchCase struct {
	subs     int
	emits    int
	stateful bool
}

func (bc benchCase) name() string {
	return fmt.Sprintf("subs-%03d/emits-%03d/stateful-%t", bc.subs, bc.emits, bc.stateful)
}

func genTestCases() []benchCase {
	ret := make([]benchCase, 0, 200)
	for stateful := 0; stateful < 2; stateful++ {
		for subs := uint(0); subs <= 8; subs = subs + 4 {
			for emits := uint(0); emits <= 8; emits = emits + 4 {
				ret = append(ret, benchCase{1 << subs, 1 << emits, stateful == 1})
			}
		}
	}
	return ret
}

func BenchmarkEvents(b *testing.B) {
	for _, bc := range genTestCases() {
		b.Run(bc.name(), benchMany(bc))
	}
}

func benchMany(bc benchCase) func(*testing.B) {
	return func(b *testing.B) {
		b.ReportAllocs()
		subs := bc.subs
		emits := bc.emits
		stateful := bc.stateful
		bus := NewBus()
		var wait sync.WaitGroup
		var ready sync.WaitGroup
		wait.Add(subs + emits)
		ready.Add(subs + emits)

		for i := 0; i < subs; i++ {
			go func() {
				sub, err := bus.Subscribe(new(EventB))
				if err != nil {
					panic(err)
				}
				defer sub.Close()

				ready.Done()
				ready.Wait()
				for i := 0; i < (b.N/emits)*emits; i++ {
					_, ok := <-sub.Out()
					if !ok {
						panic("wat")
					}
				}
				wait.Done()
			}()
		}

		for i := 0; i < emits; i++ {
			go func() {
				em, err := bus.Emitter(new(EventB), func(settings interface{}) error {
					settings.(*emitterSettings).makeStateful = stateful
					return nil
				})
				if err != nil {
					panic(err)
				}
				defer em.Close()

				ready.Done()
				ready.Wait()

				for i := 0; i < b.N/emits; i++ {
					em.Emit(EventB(97))
				}

				wait.Done()
			}()
		}
		ready.Wait()
		b.ResetTimer()
		wait.Wait()
	}
}

var div = 100

func BenchmarkSubscribe(b *testing.B) {
	b.ReportAllocs()
	for i := 0; i < b.N/div; i++ {
		bus := NewBus()
		for j := 0; j < div; j++ {
			bus.Subscribe(new(EventA))
		}
	}
}

func BenchmarkEmitter(b *testing.B) {
	b.ReportAllocs()
	for i := 0; i < b.N/div; i++ {
		bus := NewBus()
		for j := 0; j < div; j++ {
			bus.Emitter(new(EventA))
		}
	}
}

func BenchmarkSubscribeAndEmitter(b *testing.B) {
	b.ReportAllocs()
	for i := 0; i < b.N/div; i++ {
		bus := NewBus()
		for j := 0; j < div; j++ {
			bus.Subscribe(new(EventA))
			bus.Emitter(new(EventA))
		}
	}
}
<|endoftext|>


<commit_before>package docker_helpers

import (
	"bufio"
	"bytes"
	"errors"
	"fmt"
	"io"
	"os"
	"os/exec"
	"strings"

	"github.com/Sirupsen/logrus"
)

type machineCommand struct {
}

func (m *machineCommand) Create(driver, name string, opts ...string) error {
	args := []string{
		"create",
		"--driver", driver,
	}
	for _, opt := range opts {
		args = append(args, "--"+opt)
	}
	args = append(args, name)

	cmd := exec.Command("docker-machine", args...)
	cmd.Env = os.Environ()
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	logrus.Debugln("Executing", cmd.Path, cmd.Args)
	return cmd.Run()
}

func (m *machineCommand) Provision(name string) error {
	cmd := exec.Command("docker-machine", "provision", name)
	cmd.Env = os.Environ()
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	return cmd.Run()
}

func (m *machineCommand) Remove(name string) error {
	cmd := exec.Command("docker-machine", "rm", "-y", name)
	cmd.Env = os.Environ()
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	return cmd.Run()
}

func (m *machineCommand) List(nodeFilter string) (machines []string, err error) {
	cmd := exec.Command("docker-machine", "ls", "-q")
	cmd.Env = os.Environ()
	data, err := cmd.Output()
	if err != nil {
		return
	}

	reader := bufio.NewReader(bytes.NewReader(data))
	for {
		var line string

		line, err = reader.ReadString('\n')
		if err != nil {
			if err == io.EOF {
				err = nil
			}
			return
		}

		line = strings.TrimSpace(line)
		if line == "" {
			continue
		}

		var query string
		if n, _ := fmt.Sscanf(line, nodeFilter, &query); n != 1 {
			continue
		}

		machines = append(machines, line)
	}
}

func (m *machineCommand) get(args ...string) (out string, err error) {
	// Execute docker-machine to fetch IP
	cmd := exec.Command("docker-machine", args...)
	cmd.Env = os.Environ()
	data, err := cmd.Output()
	if err != nil {
		return
	}

	// Save the IP
	out = strings.TrimSpace(string(data))
	if out == "" {
		err = fmt.Errorf("failed to get %v", args)
	}
	return
}

func (m *machineCommand) IP(name string) (string, error) {
	return m.get("ip", name)
}

func (m *machineCommand) URL(name string) (string, error) {
	return m.get("url", name)
}

func (m *machineCommand) CertPath(name string) (string, error) {
	return m.get("inspect", name, "-f", "{{.HostOptions.AuthOptions.StorePath}}")
}

func (m *machineCommand) Status(name string) (string, error) {
	return m.get("status", name)
}

func (m *machineCommand) Exist(name string) bool {
	cmd := exec.Command("docker-machine", "inspect", name)
	cmd.Env = os.Environ()
	cmd.Stderr = os.Stderr
	return cmd.Run() == nil
}

func (m *machineCommand) CanConnect(name string) bool {
	// Execute docker-machine config which actively ask the machine if it is up and online
	cmd := exec.Command("docker-machine", "config", name)
	cmd.Env = os.Environ()
	err := cmd.Run()
	if err == nil {
		return true
	}
	return false
}

func (m *machineCommand) Credentials(name string) (dc DockerCredentials, err error) {
	if !m.CanConnect(name) {
		err = errors.New("Can't connect")
		return
	}

	dc.TLSVerify = true
	dc.Host, err = m.URL(name)
	if err == nil {
		dc.CertPath, err = m.CertPath(name)
	}
	return
}

func NewMachineCommand() Machine {
	return &machineCommand{}
}
<commit_msg>Execute `docker-machine ls` only once by all asking processes<commit_after>package docker_helpers

import (
	"bufio"
	"bytes"
	"errors"
	"fmt"
	"io"
	"os"
	"os/exec"
	"strings"

	"github.com/Sirupsen/logrus"
	"sync"
)

type machineCommand struct {
	lsCmd   *exec.Cmd
	lsLock  sync.Mutex
	lsCond  *sync.Cond
	lsData  []byte
	lsError error
}

func (m *machineCommand) ls() (data []byte, err error) {
	m.lsLock.Lock()
	defer m.lsLock.Unlock()

	if m.lsCond == nil {
		m.lsCond = sync.NewCond(&m.lsLock)
	}

	if m.lsCmd == nil {
		m.lsCmd := exec.Command("docker-machine", "ls", "-q")
		m.lsCmd.Env = os.Environ()
		go func() {
			m.lsData, m.lsError = m.lsCmd.Output()
			m.lsCmd = nil
			m.lsCond.Broadcast()
		}()
	}

	m.lsCond.Wait()

	return m.lsData, m.lsError
}

func (m *machineCommand) Create(driver, name string, opts ...string) error {
	args := []string{
		"create",
		"--driver", driver,
	}
	for _, opt := range opts {
		args = append(args, "--"+opt)
	}
	args = append(args, name)

	cmd := exec.Command("docker-machine", args...)
	cmd.Env = os.Environ()
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	logrus.Debugln("Executing", cmd.Path, cmd.Args)
	return cmd.Run()
}

func (m *machineCommand) Provision(name string) error {
	cmd := exec.Command("docker-machine", "provision", name)
	cmd.Env = os.Environ()
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	return cmd.Run()
}

func (m *machineCommand) Remove(name string) error {
	cmd := exec.Command("docker-machine", "rm", "-y", name)
	cmd.Env = os.Environ()
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	return cmd.Run()
}

func (m *machineCommand) List(nodeFilter string) (machines []string, err error) {
	data, err := m.ls()
	if err != nil {
		return
	}

	reader := bufio.NewReader(bytes.NewReader(data))
	for {
		var line string

		line, err = reader.ReadString('\n')
		if err != nil {
			if err == io.EOF {
				err = nil
			}
			return
		}

		line = strings.TrimSpace(line)
		if line == "" {
			continue
		}

		var query string
		if n, _ := fmt.Sscanf(line, nodeFilter, &query); n != 1 {
			continue
		}

		machines = append(machines, line)
	}
}

func (m *machineCommand) get(args ...string) (out string, err error) {
	// Execute docker-machine to fetch IP
	cmd := exec.Command("docker-machine", args...)
	cmd.Env = os.Environ()
	data, err := cmd.Output()
	if err != nil {
		return
	}

	// Save the IP
	out = strings.TrimSpace(string(data))
	if out == "" {
		err = fmt.Errorf("failed to get %v", args)
	}
	return
}

func (m *machineCommand) IP(name string) (string, error) {
	return m.get("ip", name)
}

func (m *machineCommand) URL(name string) (string, error) {
	return m.get("url", name)
}

func (m *machineCommand) CertPath(name string) (string, error) {
	return m.get("inspect", name, "-f", "{{.HostOptions.AuthOptions.StorePath}}")
}

func (m *machineCommand) Status(name string) (string, error) {
	return m.get("status", name)
}

func (m *machineCommand) Exist(name string) bool {
	cmd := exec.Command("docker-machine", "inspect", name)
	cmd.Env = os.Environ()
	cmd.Stderr = os.Stderr
	return cmd.Run() == nil
}

func (m *machineCommand) CanConnect(name string) bool {
	// Execute docker-machine config which actively ask the machine if it is up and online
	cmd := exec.Command("docker-machine", "config", name)
	cmd.Env = os.Environ()
	err := cmd.Run()
	if err == nil {
		return true
	}
	return false
}

func (m *machineCommand) Credentials(name string) (dc DockerCredentials, err error) {
	if !m.CanConnect(name) {
		err = errors.New("Can't connect")
		return
	}

	dc.TLSVerify = true
	dc.Host, err = m.URL(name)
	if err == nil {
		dc.CertPath, err = m.CertPath(name)
	}
	return
}

func NewMachineCommand() Machine {
	return &machineCommand{}
}
<|endoftext|>