<commit_before>// Copyright 2014 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"crypto/md5"
	"flag"
	"fmt"
	"go/build"
	"internal/testenv"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strings"
	"testing"
)

var tmp, exe string // populated by buildObjdump

func TestMain(m *testing.M) {
	if !testenv.HasGoBuild() {
		return
	}

	var exitcode int
	if err := buildObjdump(); err == nil {
		exitcode = m.Run()
	} else {
		fmt.Println(err)
		exitcode = 1
	}
	os.RemoveAll(tmp)
	os.Exit(exitcode)
}

func buildObjdump() error {
	var err error
	tmp, err = ioutil.TempDir("", "TestObjDump")
	if err != nil {
		return fmt.Errorf("TempDir failed: %v", err)
	}

	exe = filepath.Join(tmp, "testobjdump.exe")
	gotool, err := testenv.GoTool()
	if err != nil {
		return err
	}
	out, err := exec.Command(gotool, "build", "-o", exe, "cmd/objdump").CombinedOutput()
	if err != nil {
		os.RemoveAll(tmp)
		return fmt.Errorf("go build -o %v cmd/objdump: %v\n%s", exe, err, string(out))
	}

	return nil
}

var x86Need = []string{ // for both 386 and AMD64
	"JMP main.main(SB)",
	"CALL main.Println(SB)",
	"RET",
}

var amd64GnuNeed = []string{
	"movq",
	"callq",
	"cmpb",
}

var i386GnuNeed = []string{
	"mov",
	"call",
	"cmp",
}

var armNeed = []string{
	"B main.main(SB)",
	"BL main.Println(SB)",
	"RET",
}

var arm64Need = []string{
	"JMP main.main(SB)",
	"CALL main.Println(SB)",
	"RET",
}

var armGnuNeed = []string{ // for both ARM and AMR64
	"ldr",
	"bl",
	"cmp",
}

var ppcNeed = []string{
	"BR main.main(SB)",
	"CALL main.Println(SB)",
	"RET",
}

var ppcGnuNeed = []string{
	"mflr",
	"lbz",
	"cmpw",
}

var target = flag.String("target", "", "test disassembly of `goos/goarch` binary")

// objdump is fully cross platform: it can handle binaries
// from any known operating system and architecture.
// We could in principle add binaries to testdata and check
// all the supported systems during this test. However, the
// binaries would be about 1 MB each, and we don't want to
// add that much junk to the hg repository. Instead, build a
// binary for the current system (only) and test that objdump
// can handle that one.

func testDisasm(t *testing.T, srcfname string, printCode bool, printGnuAsm bool, flags ...string) {
	goarch := runtime.GOARCH
	if *target != "" {
		f := strings.Split(*target, "/")
		if len(f) != 2 {
			t.Fatalf("-target argument must be goos/goarch")
		}
		defer os.Setenv("GOOS", os.Getenv("GOOS"))
		defer os.Setenv("GOARCH", os.Getenv("GOARCH"))
		os.Setenv("GOOS", f[0])
		os.Setenv("GOARCH", f[1])
		goarch = f[1]
	}

	hash := md5.Sum([]byte(fmt.Sprintf("%v-%v-%v-%v", srcfname, flags, printCode, printGnuAsm)))
	hello := filepath.Join(tmp, fmt.Sprintf("hello-%x.exe", hash))
	args := []string{"build", "-o", hello}
	args = append(args, flags...)
	args = append(args, srcfname)
	cmd := exec.Command(testenv.GoToolPath(t), args...)
	// "Bad line" bug #36683 is sensitive to being run in the source directory.
	cmd.Dir = "testdata"
	// Ensure that the source file location embedded in the binary matches our
	// actual current GOROOT, instead of GOROOT_FINAL if set.
	cmd.Env = append(os.Environ(), "GOROOT_FINAL=")
	t.Logf("Running %v", cmd.Args)
	out, err := cmd.CombinedOutput()
	if err != nil {
		t.Fatalf("go build %s: %v\n%s", srcfname, err, out)
	}
	need := []string{
		"TEXT main.main(SB)",
	}

	if printCode {
		need = append(need, `	Println("hello, world")`)
	} else {
		need = append(need, srcfname+":6")
	}

	switch goarch {
	case "amd64", "386":
		need = append(need, x86Need...)
	case "arm":
		need = append(need, armNeed...)
	case "arm64":
		need = append(need, arm64Need...)
	case "ppc64", "ppc64le":
		need = append(need, ppcNeed...)
	}

	if printGnuAsm {
		switch goarch {
		case "amd64":
			need = append(need, amd64GnuNeed...)
		case "386":
			need = append(need, i386GnuNeed...)
		case "arm", "arm64":
			need = append(need, armGnuNeed...)
		case "ppc64", "ppc64le":
			need = append(need, ppcGnuNeed...)
		}
	}
	args = []string{
		"-s", "main.main",
		hello,
	}

	if printCode {
		args = append([]string{"-S"}, args...)
	}

	if printGnuAsm {
		args = append([]string{"-gnu"}, args...)
	}
	cmd = exec.Command(exe, args...)
	cmd.Dir = "testdata" // "Bad line" bug #36683 is sensitive to being run in the source directory
	out, err = cmd.CombinedOutput()
	t.Logf("Running %v", cmd.Args)

	if err != nil {
		exename := srcfname[:len(srcfname)-len(filepath.Ext(srcfname))] + ".exe"
		t.Fatalf("objdump %q: %v\n%s", exename, err, out)
	}

	text := string(out)
	ok := true
	for _, s := range need {
		if !strings.Contains(text, s) {
			t.Errorf("disassembly missing '%s'", s)
			ok = false
		}
	}
	if goarch == "386" {
		if strings.Contains(text, "(IP)") {
			t.Errorf("disassembly contains PC-Relative addressing on 386")
			ok = false
		}
	}

	if !ok {
		t.Logf("full disassembly:\n%s", text)
	}
}

func testGoAndCgoDisasm(t *testing.T, printCode bool, printGnuAsm bool) {
	t.Parallel()
	testDisasm(t, "fmthello.go", printCode, printGnuAsm)
	if build.Default.CgoEnabled {
		if runtime.GOOS == "aix" {
			t.Skipf("skipping on %s, issue 40972", runtime.GOOS)
		}
		testDisasm(t, "fmthellocgo.go", printCode, printGnuAsm)
	}
}

func TestDisasm(t *testing.T) {
	switch runtime.GOARCH {
	case "mips", "mipsle", "mips64", "mips64le":
		t.Skipf("skipping on %s, issue 12559", runtime.GOARCH)
	case "riscv64":
		t.Skipf("skipping on %s, issue 36738", runtime.GOARCH)
	case "s390x":
		t.Skipf("skipping on %s, issue 15255", runtime.GOARCH)
	}
	testGoAndCgoDisasm(t, false, false)
}

func TestDisasmCode(t *testing.T) {
	switch runtime.GOARCH {
	case "mips", "mipsle", "mips64", "mips64le", "riscv64", "s390x":
		t.Skipf("skipping on %s, issue 19160", runtime.GOARCH)
	}
	testGoAndCgoDisasm(t, true, false)
}

func TestDisasmGnuAsm(t *testing.T) {
	switch runtime.GOARCH {
	case "mips", "mipsle", "mips64", "mips64le", "riscv64", "s390x":
		t.Skipf("skipping on %s, issue 19160", runtime.GOARCH)
	}
	testGoAndCgoDisasm(t, false, true)
}

func TestDisasmExtld(t *testing.T) {
	switch runtime.GOOS {
	case "plan9", "windows":
		t.Skipf("skipping on %s", runtime.GOOS)
	}
	switch runtime.GOARCH {
	case "ppc64":
		t.Skipf("skipping on %s, no support for external linking, issue 9038", runtime.GOARCH)
	case "mips64", "mips64le", "mips", "mipsle":
		t.Skipf("skipping on %s, issue 12559 and 12560", runtime.GOARCH)
	case "riscv64":
		t.Skipf("skipping on %s, no support for external linking, issue 36739", runtime.GOARCH)
	case "s390x":
		t.Skipf("skipping on %s, issue 15255", runtime.GOARCH)
	}
	if !build.Default.CgoEnabled {
		t.Skip("skipping because cgo is not enabled")
	}
	t.Parallel()
	testDisasm(t, "fmthello.go", false, false, "-ldflags=-linkmode=external")
}

func TestDisasmGoobj(t *testing.T) {
	switch runtime.GOARCH {
	case "mips", "mipsle", "mips64", "mips64le":
		t.Skipf("skipping on %s, issue 12559", runtime.GOARCH)
	case "riscv64":
		t.Skipf("skipping on %s, issue 36738", runtime.GOARCH)
	case "s390x":
		t.Skipf("skipping on %s, issue 15255", runtime.GOARCH)
	}

	hello := filepath.Join(tmp, "hello.o")
	args := []string{"tool", "compile", "-o", hello}
	args = append(args, "testdata/fmthello.go")
	out, err := exec.Command(testenv.GoToolPath(t), args...).CombinedOutput()
	if err != nil {
		t.Fatalf("go tool compile fmthello.go: %v\n%s", err, out)
	}
	need := []string{
		"main(SB)",
		"fmthello.go:6",
	}

	args = []string{
		"-s", "main",
		hello,
	}

	out, err = exec.Command(exe, args...).CombinedOutput()
	if err != nil {
		t.Fatalf("objdump fmthello.o: %v\n%s", err, out)
	}

	text := string(out)
	ok := true
	for _, s := range need {
		if !strings.Contains(text, s) {
			t.Errorf("disassembly missing '%s'", s)
			ok = false
		}
	}
	if runtime.GOARCH == "386" {
		if strings.Contains(text, "(IP)") {
			t.Errorf("disassembly contains PC-Relative addressing on 386")
			ok = false
		}
	}
	if !ok {
		t.Logf("full disassembly:\n%s", text)
	}
}

func TestGoobjFileNumber(t *testing.T) {
	// Test that file table in Go object file is parsed correctly.
	testenv.MustHaveGoBuild(t)

	t.Parallel()

	tmpdir, err := ioutil.TempDir("", "TestGoobjFileNumber")
	if err != nil {
		t.Fatal(err)
	}
	defer os.RemoveAll(tmpdir)

	obj := filepath.Join(tmpdir, "p.a")
	cmd := exec.Command(testenv.GoToolPath(t), "build", "-o", obj)
	cmd.Dir = filepath.Join("testdata/testfilenum")
	out, err := cmd.CombinedOutput()
	if err != nil {
		t.Fatalf("build failed: %v\n%s", err, out)
	}

	cmd = exec.Command(exe, obj)
	out, err = cmd.CombinedOutput()
	if err != nil {
		t.Fatalf("objdump failed: %v\n%s", err, out)
	}

	text := string(out)
	for _, s := range []string{"a.go", "b.go", "c.go"} {
		if !strings.Contains(text, s) {
			t.Errorf("output missing '%s'", s)
		}
	}

	if t.Failed() {
		t.Logf("output:\n%s", text)
	}
}
<commit_msg>cmd/objdump: skip tests on unsupported platforms<commit_after>// Copyright 2014 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"crypto/md5"
	"flag"
	"fmt"
	"go/build"
	"internal/testenv"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strings"
	"testing"
)

var tmp, exe string // populated by buildObjdump

func TestMain(m *testing.M) {
	if !testenv.HasGoBuild() {
		return
	}

	var exitcode int
	if err := buildObjdump(); err == nil {
		exitcode = m.Run()
	} else {
		fmt.Println(err)
		exitcode = 1
	}
	os.RemoveAll(tmp)
	os.Exit(exitcode)
}

func buildObjdump() error {
	var err error
	tmp, err = ioutil.TempDir("", "TestObjDump")
	if err != nil {
		return fmt.Errorf("TempDir failed: %v", err)
	}

	exe = filepath.Join(tmp, "testobjdump.exe")
	gotool, err := testenv.GoTool()
	if err != nil {
		return err
	}
	out, err := exec.Command(gotool, "build", "-o", exe, "cmd/objdump").CombinedOutput()
	if err != nil {
		os.RemoveAll(tmp)
		return fmt.Errorf("go build -o %v cmd/objdump: %v\n%s", exe, err, string(out))
	}

	return nil
}

var x86Need = []string{ // for both 386 and AMD64
	"JMP main.main(SB)",
	"CALL main.Println(SB)",
	"RET",
}

var amd64GnuNeed = []string{
	"movq",
	"callq",
	"cmpb",
}

var i386GnuNeed = []string{
	"mov",
	"call",
	"cmp",
}

var armNeed = []string{
	"B main.main(SB)",
	"BL main.Println(SB)",
	"RET",
}

var arm64Need = []string{
	"JMP main.main(SB)",
	"CALL main.Println(SB)",
	"RET",
}

var armGnuNeed = []string{ // for both ARM and AMR64
	"ldr",
	"bl",
	"cmp",
}

var ppcNeed = []string{
	"BR main.main(SB)",
	"CALL main.Println(SB)",
	"RET",
}

var ppcGnuNeed = []string{
	"mflr",
	"lbz",
	"cmpw",
}

func mustHaveDisasm(t *testing.T) {
	switch runtime.GOARCH {
	case "mips", "mipsle", "mips64", "mips64le":
		t.Skipf("skipping on %s, issue 12559", runtime.GOARCH)
	case "riscv64":
		t.Skipf("skipping on %s, issue 36738", runtime.GOARCH)
	case "s390x":
		t.Skipf("skipping on %s, issue 15255", runtime.GOARCH)
	}
}

var target = flag.String("target", "", "test disassembly of `goos/goarch` binary")

// objdump is fully cross platform: it can handle binaries
// from any known operating system and architecture.
// We could in principle add binaries to testdata and check
// all the supported systems during this test. However, the
// binaries would be about 1 MB each, and we don't want to
// add that much junk to the hg repository. Instead, build a
// binary for the current system (only) and test that objdump
// can handle that one.

func testDisasm(t *testing.T, srcfname string, printCode bool, printGnuAsm bool, flags ...string) {
	mustHaveDisasm(t)
	goarch := runtime.GOARCH
	if *target != "" {
		f := strings.Split(*target, "/")
		if len(f) != 2 {
			t.Fatalf("-target argument must be goos/goarch")
		}
		defer os.Setenv("GOOS", os.Getenv("GOOS"))
		defer os.Setenv("GOARCH", os.Getenv("GOARCH"))
		os.Setenv("GOOS", f[0])
		os.Setenv("GOARCH", f[1])
		goarch = f[1]
	}

	hash := md5.Sum([]byte(fmt.Sprintf("%v-%v-%v-%v", srcfname, flags, printCode, printGnuAsm)))
	hello := filepath.Join(tmp, fmt.Sprintf("hello-%x.exe", hash))
	args := []string{"build", "-o", hello}
	args = append(args, flags...)
	args = append(args, srcfname)
	cmd := exec.Command(testenv.GoToolPath(t), args...)
	// "Bad line" bug #36683 is sensitive to being run in the source directory.
	cmd.Dir = "testdata"
	// Ensure that the source file location embedded in the binary matches our
	// actual current GOROOT, instead of GOROOT_FINAL if set.
	cmd.Env = append(os.Environ(), "GOROOT_FINAL=")
	t.Logf("Running %v", cmd.Args)
	out, err := cmd.CombinedOutput()
	if err != nil {
		t.Fatalf("go build %s: %v\n%s", srcfname, err, out)
	}
	need := []string{
		"TEXT main.main(SB)",
	}

	if printCode {
		need = append(need, `	Println("hello, world")`)
	} else {
		need = append(need, srcfname+":6")
	}

	switch goarch {
	case "amd64", "386":
		need = append(need, x86Need...)
	case "arm":
		need = append(need, armNeed...)
	case "arm64":
		need = append(need, arm64Need...)
	case "ppc64", "ppc64le":
		need = append(need, ppcNeed...)
	}

	if printGnuAsm {
		switch goarch {
		case "amd64":
			need = append(need, amd64GnuNeed...)
		case "386":
			need = append(need, i386GnuNeed...)
		case "arm", "arm64":
			need = append(need, armGnuNeed...)
		case "ppc64", "ppc64le":
			need = append(need, ppcGnuNeed...)
		}
	}
	args = []string{
		"-s", "main.main",
		hello,
	}

	if printCode {
		args = append([]string{"-S"}, args...)
	}

	if printGnuAsm {
		args = append([]string{"-gnu"}, args...)
	}
	cmd = exec.Command(exe, args...)
	cmd.Dir = "testdata" // "Bad line" bug #36683 is sensitive to being run in the source directory
	out, err = cmd.CombinedOutput()
	t.Logf("Running %v", cmd.Args)

	if err != nil {
		exename := srcfname[:len(srcfname)-len(filepath.Ext(srcfname))] + ".exe"
		t.Fatalf("objdump %q: %v\n%s", exename, err, out)
	}

	text := string(out)
	ok := true
	for _, s := range need {
		if !strings.Contains(text, s) {
			t.Errorf("disassembly missing '%s'", s)
			ok = false
		}
	}
	if goarch == "386" {
		if strings.Contains(text, "(IP)") {
			t.Errorf("disassembly contains PC-Relative addressing on 386")
			ok = false
		}
	}

	if !ok {
		t.Logf("full disassembly:\n%s", text)
	}
}

func testGoAndCgoDisasm(t *testing.T, printCode bool, printGnuAsm bool) {
	t.Parallel()
	testDisasm(t, "fmthello.go", printCode, printGnuAsm)
	if build.Default.CgoEnabled {
		if runtime.GOOS == "aix" {
			return // issue 40972
		}
		testDisasm(t, "fmthellocgo.go", printCode, printGnuAsm)
	}
}

func TestDisasm(t *testing.T) {
	testGoAndCgoDisasm(t, false, false)
}

func TestDisasmCode(t *testing.T) {
	testGoAndCgoDisasm(t, true, false)
}

func TestDisasmGnuAsm(t *testing.T) {
	testGoAndCgoDisasm(t, false, true)
}

func TestDisasmExtld(t *testing.T) {
	testenv.MustHaveCGO(t)
	switch runtime.GOOS {
	case "plan9", "windows":
		t.Skipf("skipping on %s", runtime.GOOS)
	}
	t.Parallel()
	testDisasm(t, "fmthello.go", false, false, "-ldflags=-linkmode=external")
}

func TestDisasmGoobj(t *testing.T) {
	mustHaveDisasm(t)

	hello := filepath.Join(tmp, "hello.o")
	args := []string{"tool", "compile", "-o", hello}
	args = append(args, "testdata/fmthello.go")
	out, err := exec.Command(testenv.GoToolPath(t), args...).CombinedOutput()
	if err != nil {
		t.Fatalf("go tool compile fmthello.go: %v\n%s", err, out)
	}
	need := []string{
		"main(SB)",
		"fmthello.go:6",
	}

	args = []string{
		"-s", "main",
		hello,
	}

	out, err = exec.Command(exe, args...).CombinedOutput()
	if err != nil {
		t.Fatalf("objdump fmthello.o: %v\n%s", err, out)
	}

	text := string(out)
	ok := true
	for _, s := range need {
		if !strings.Contains(text, s) {
			t.Errorf("disassembly missing '%s'", s)
			ok = false
		}
	}
	if runtime.GOARCH == "386" {
		if strings.Contains(text, "(IP)") {
			t.Errorf("disassembly contains PC-Relative addressing on 386")
			ok = false
		}
	}
	if !ok {
		t.Logf("full disassembly:\n%s", text)
	}
}

func TestGoobjFileNumber(t *testing.T) {
	// Test that file table in Go object file is parsed correctly.
	testenv.MustHaveGoBuild(t)
	mustHaveDisasm(t)

	t.Parallel()

	tmpdir, err := ioutil.TempDir("", "TestGoobjFileNumber")
	if err != nil {
		t.Fatal(err)
	}
	defer os.RemoveAll(tmpdir)

	obj := filepath.Join(tmpdir, "p.a")
	cmd := exec.Command(testenv.GoToolPath(t), "build", "-o", obj)
	cmd.Dir = filepath.Join("testdata/testfilenum")
	out, err := cmd.CombinedOutput()
	if err != nil {
		t.Fatalf("build failed: %v\n%s", err, out)
	}

	cmd = exec.Command(exe, obj)
	out, err = cmd.CombinedOutput()
	if err != nil {
		t.Fatalf("objdump failed: %v\n%s", err, out)
	}

	text := string(out)
	for _, s := range []string{"a.go", "b.go", "c.go"} {
		if !strings.Contains(text, s) {
			t.Errorf("output missing '%s'", s)
		}
	}

	if t.Failed() {
		t.Logf("output:\n%s", text)
	}
}
<|endoftext|>


<commit_before>// Copyright 2014 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"crypto/md5"
	"flag"
	"fmt"
	"go/build"
	"internal/testenv"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strings"
	"testing"
)

var tmp, exe string // populated by buildObjdump

func TestMain(m *testing.M) {
	if !testenv.HasGoBuild() {
		return
	}

	var exitcode int
	if err := buildObjdump(); err == nil {
		exitcode = m.Run()
	} else {
		fmt.Println(err)
		exitcode = 1
	}
	os.RemoveAll(tmp)
	os.Exit(exitcode)
}

func buildObjdump() error {
	var err error
	tmp, err = os.MkdirTemp("", "TestObjDump")
	if err != nil {
		return fmt.Errorf("TempDir failed: %v", err)
	}

	exe = filepath.Join(tmp, "testobjdump.exe")
	gotool, err := testenv.GoTool()
	if err != nil {
		return err
	}
	out, err := exec.Command(gotool, "build", "-o", exe, "cmd/objdump").CombinedOutput()
	if err != nil {
		os.RemoveAll(tmp)
		return fmt.Errorf("go build -o %v cmd/objdump: %v\n%s", exe, err, string(out))
	}

	return nil
}

var x86Need = []string{ // for both 386 and AMD64
	"JMP main.main(SB)",
	"CALL main.Println(SB)",
	"RET",
}

var amd64GnuNeed = []string{
	"movq",
	"callq",
	"cmpb",
}

var i386GnuNeed = []string{
	"mov",
	"call",
	"cmp",
}

var armNeed = []string{
	"B main.main(SB)",
	"BL main.Println(SB)",
	"RET",
}

var arm64Need = []string{
	"JMP main.main(SB)",
	"CALL main.Println(SB)",
	"RET",
}

var armGnuNeed = []string{ // for both ARM and AMR64
	"ldr",
	"bl",
	"cmp",
}

var ppcNeed = []string{
	"BR main.main(SB)",
	"CALL main.Println(SB)",
	"RET",
}

var ppcGnuNeed = []string{
	"mflr",
	"lbz",
	"cmpw",
}

func mustHaveDisasm(t *testing.T) {
	switch runtime.GOARCH {
	case "mips", "mipsle", "mips64", "mips64le":
		t.Skipf("skipping on %s, issue 12559", runtime.GOARCH)
	case "riscv64":
		t.Skipf("skipping on %s, issue 36738", runtime.GOARCH)
	case "s390x":
		t.Skipf("skipping on %s, issue 15255", runtime.GOARCH)
	}
}

var target = flag.String("target", "", "test disassembly of `goos/goarch` binary")

// objdump is fully cross platform: it can handle binaries
// from any known operating system and architecture.
// We could in principle add binaries to testdata and check
// all the supported systems during this test. However, the
// binaries would be about 1 MB each, and we don't want to
// add that much junk to the hg repository. Instead, build a
// binary for the current system (only) and test that objdump
// can handle that one.

func testDisasm(t *testing.T, srcfname string, printCode bool, printGnuAsm bool, flags ...string) {
	mustHaveDisasm(t)
	goarch := runtime.GOARCH
	if *target != "" {
		f := strings.Split(*target, "/")
		if len(f) != 2 {
			t.Fatalf("-target argument must be goos/goarch")
		}
		defer os.Setenv("GOOS", os.Getenv("GOOS"))
		defer os.Setenv("GOARCH", os.Getenv("GOARCH"))
		os.Setenv("GOOS", f[0])
		os.Setenv("GOARCH", f[1])
		goarch = f[1]
	}

	hash := md5.Sum([]byte(fmt.Sprintf("%v-%v-%v-%v", srcfname, flags, printCode, printGnuAsm)))
	hello := filepath.Join(tmp, fmt.Sprintf("hello-%x.exe", hash))
	args := []string{"build", "-o", hello}
	args = append(args, flags...)
	args = append(args, srcfname)
	cmd := exec.Command(testenv.GoToolPath(t), args...)
	// "Bad line" bug #36683 is sensitive to being run in the source directory.
	cmd.Dir = "testdata"
	// Ensure that the source file location embedded in the binary matches our
	// actual current GOROOT, instead of GOROOT_FINAL if set.
	cmd.Env = append(os.Environ(), "GOROOT_FINAL=")
	t.Logf("Running %v", cmd.Args)
	out, err := cmd.CombinedOutput()
	if err != nil {
		t.Fatalf("go build %s: %v\n%s", srcfname, err, out)
	}
	need := []string{
		"TEXT main.main(SB)",
	}

	if printCode {
		need = append(need, `	Println("hello, world")`)
	} else {
		need = append(need, srcfname+":6")
	}

	switch goarch {
	case "amd64", "386":
		need = append(need, x86Need...)
	case "arm":
		need = append(need, armNeed...)
	case "arm64":
		need = append(need, arm64Need...)
	case "ppc64", "ppc64le":
		need = append(need, ppcNeed...)
	}

	if printGnuAsm {
		switch goarch {
		case "amd64":
			need = append(need, amd64GnuNeed...)
		case "386":
			need = append(need, i386GnuNeed...)
		case "arm", "arm64":
			need = append(need, armGnuNeed...)
		case "ppc64", "ppc64le":
			need = append(need, ppcGnuNeed...)
		}
	}
	args = []string{
		"-s", "main.main",
		hello,
	}

	if printCode {
		args = append([]string{"-S"}, args...)
	}

	if printGnuAsm {
		args = append([]string{"-gnu"}, args...)
	}
	cmd = exec.Command(exe, args...)
	cmd.Dir = "testdata" // "Bad line" bug #36683 is sensitive to being run in the source directory
	out, err = cmd.CombinedOutput()
	t.Logf("Running %v", cmd.Args)

	if err != nil {
		exename := srcfname[:len(srcfname)-len(filepath.Ext(srcfname))] + ".exe"
		t.Fatalf("objdump %q: %v\n%s", exename, err, out)
	}

	text := string(out)
	ok := true
	for _, s := range need {
		if !strings.Contains(text, s) {
			t.Errorf("disassembly missing '%s'", s)
			ok = false
		}
	}
	if goarch == "386" {
		if strings.Contains(text, "(IP)") {
			t.Errorf("disassembly contains PC-Relative addressing on 386")
			ok = false
		}
	}

	if !ok {
		t.Logf("full disassembly:\n%s", text)
	}
}

func testGoAndCgoDisasm(t *testing.T, printCode bool, printGnuAsm bool) {
	t.Parallel()
	testDisasm(t, "fmthello.go", printCode, printGnuAsm)
	if build.Default.CgoEnabled {
		testDisasm(t, "fmthellocgo.go", printCode, printGnuAsm)
	}
}

func TestDisasm(t *testing.T) {
	testGoAndCgoDisasm(t, false, false)
}

func TestDisasmCode(t *testing.T) {
	testGoAndCgoDisasm(t, true, false)
}

func TestDisasmGnuAsm(t *testing.T) {
	testGoAndCgoDisasm(t, false, true)
}

func TestDisasmExtld(t *testing.T) {
	testenv.MustHaveCGO(t)
	switch runtime.GOOS {
	case "plan9", "windows":
		t.Skipf("skipping on %s", runtime.GOOS)
	}
	t.Parallel()
	testDisasm(t, "fmthello.go", false, false, "-ldflags=-linkmode=external")
}

func TestDisasmGoobj(t *testing.T) {
	mustHaveDisasm(t)

	hello := filepath.Join(tmp, "hello.o")
	args := []string{"tool", "compile", "-o", hello}
	args = append(args, "testdata/fmthello.go")
	out, err := exec.Command(testenv.GoToolPath(t), args...).CombinedOutput()
	if err != nil {
		t.Fatalf("go tool compile fmthello.go: %v\n%s", err, out)
	}
	need := []string{
		"main(SB)",
		"fmthello.go:6",
	}

	args = []string{
		"-s", "main",
		hello,
	}

	out, err = exec.Command(exe, args...).CombinedOutput()
	if err != nil {
		t.Fatalf("objdump fmthello.o: %v\n%s", err, out)
	}

	text := string(out)
	ok := true
	for _, s := range need {
		if !strings.Contains(text, s) {
			t.Errorf("disassembly missing '%s'", s)
			ok = false
		}
	}
	if runtime.GOARCH == "386" {
		if strings.Contains(text, "(IP)") {
			t.Errorf("disassembly contains PC-Relative addressing on 386")
			ok = false
		}
	}
	if !ok {
		t.Logf("full disassembly:\n%s", text)
	}
}

func TestGoobjFileNumber(t *testing.T) {
	// Test that file table in Go object file is parsed correctly.
	testenv.MustHaveGoBuild(t)
	mustHaveDisasm(t)

	t.Parallel()

	tmpdir, err := os.MkdirTemp("", "TestGoobjFileNumber")
	if err != nil {
		t.Fatal(err)
	}
	defer os.RemoveAll(tmpdir)

	obj := filepath.Join(tmpdir, "p.a")
	cmd := exec.Command(testenv.GoToolPath(t), "build", "-o", obj)
	cmd.Dir = filepath.Join("testdata/testfilenum")
	out, err := cmd.CombinedOutput()
	if err != nil {
		t.Fatalf("build failed: %v\n%s", err, out)
	}

	cmd = exec.Command(exe, obj)
	out, err = cmd.CombinedOutput()
	if err != nil {
		t.Fatalf("objdump failed: %v\n%s", err, out)
	}

	text := string(out)
	for _, s := range []string{"a.go", "b.go", "c.go"} {
		if !strings.Contains(text, s) {
			t.Errorf("output missing '%s'", s)
		}
	}

	if t.Failed() {
		t.Logf("output:\n%s", text)
	}
}
<commit_msg>cmd/objdump: update test with register ABI<commit_after>// Copyright 2014 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"crypto/md5"
	"flag"
	"fmt"
	"go/build"
	"internal/testenv"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strings"
	"testing"
)

var tmp, exe string // populated by buildObjdump

func TestMain(m *testing.M) {
	if !testenv.HasGoBuild() {
		return
	}

	var exitcode int
	if err := buildObjdump(); err == nil {
		exitcode = m.Run()
	} else {
		fmt.Println(err)
		exitcode = 1
	}
	os.RemoveAll(tmp)
	os.Exit(exitcode)
}

func buildObjdump() error {
	var err error
	tmp, err = os.MkdirTemp("", "TestObjDump")
	if err != nil {
		return fmt.Errorf("TempDir failed: %v", err)
	}

	exe = filepath.Join(tmp, "testobjdump.exe")
	gotool, err := testenv.GoTool()
	if err != nil {
		return err
	}
	out, err := exec.Command(gotool, "build", "-o", exe, "cmd/objdump").CombinedOutput()
	if err != nil {
		os.RemoveAll(tmp)
		return fmt.Errorf("go build -o %v cmd/objdump: %v\n%s", exe, err, string(out))
	}

	return nil
}

var x86Need = []string{ // for both 386 and AMD64
	"JMP main.main(SB)",
	"CALL main.Println(SB)",
	"RET",
}

var amd64GnuNeed = []string{
	"jmp",
	"callq",
	"cmpb",
}

var i386GnuNeed = []string{
	"jmp",
	"call",
	"cmp",
}

var armNeed = []string{
	"B main.main(SB)",
	"BL main.Println(SB)",
	"RET",
}

var arm64Need = []string{
	"JMP main.main(SB)",
	"CALL main.Println(SB)",
	"RET",
}

var armGnuNeed = []string{ // for both ARM and AMR64
	"ldr",
	"bl",
	"cmp",
}

var ppcNeed = []string{
	"BR main.main(SB)",
	"CALL main.Println(SB)",
	"RET",
}

var ppcGnuNeed = []string{
	"mflr",
	"lbz",
	"cmpw",
}

func mustHaveDisasm(t *testing.T) {
	switch runtime.GOARCH {
	case "mips", "mipsle", "mips64", "mips64le":
		t.Skipf("skipping on %s, issue 12559", runtime.GOARCH)
	case "riscv64":
		t.Skipf("skipping on %s, issue 36738", runtime.GOARCH)
	case "s390x":
		t.Skipf("skipping on %s, issue 15255", runtime.GOARCH)
	}
}

var target = flag.String("target", "", "test disassembly of `goos/goarch` binary")

// objdump is fully cross platform: it can handle binaries
// from any known operating system and architecture.
// We could in principle add binaries to testdata and check
// all the supported systems during this test. However, the
// binaries would be about 1 MB each, and we don't want to
// add that much junk to the hg repository. Instead, build a
// binary for the current system (only) and test that objdump
// can handle that one.

func testDisasm(t *testing.T, srcfname string, printCode bool, printGnuAsm bool, flags ...string) {
	mustHaveDisasm(t)
	goarch := runtime.GOARCH
	if *target != "" {
		f := strings.Split(*target, "/")
		if len(f) != 2 {
			t.Fatalf("-target argument must be goos/goarch")
		}
		defer os.Setenv("GOOS", os.Getenv("GOOS"))
		defer os.Setenv("GOARCH", os.Getenv("GOARCH"))
		os.Setenv("GOOS", f[0])
		os.Setenv("GOARCH", f[1])
		goarch = f[1]
	}

	hash := md5.Sum([]byte(fmt.Sprintf("%v-%v-%v-%v", srcfname, flags, printCode, printGnuAsm)))
	hello := filepath.Join(tmp, fmt.Sprintf("hello-%x.exe", hash))
	args := []string{"build", "-o", hello}
	args = append(args, flags...)
	args = append(args, srcfname)
	cmd := exec.Command(testenv.GoToolPath(t), args...)
	// "Bad line" bug #36683 is sensitive to being run in the source directory.
	cmd.Dir = "testdata"
	// Ensure that the source file location embedded in the binary matches our
	// actual current GOROOT, instead of GOROOT_FINAL if set.
	cmd.Env = append(os.Environ(), "GOROOT_FINAL=")
	t.Logf("Running %v", cmd.Args)
	out, err := cmd.CombinedOutput()
	if err != nil {
		t.Fatalf("go build %s: %v\n%s", srcfname, err, out)
	}
	need := []string{
		"TEXT main.main(SB)",
	}

	if printCode {
		need = append(need, `	Println("hello, world")`)
	} else {
		need = append(need, srcfname+":6")
	}

	switch goarch {
	case "amd64", "386":
		need = append(need, x86Need...)
	case "arm":
		need = append(need, armNeed...)
	case "arm64":
		need = append(need, arm64Need...)
	case "ppc64", "ppc64le":
		need = append(need, ppcNeed...)
	}

	if printGnuAsm {
		switch goarch {
		case "amd64":
			need = append(need, amd64GnuNeed...)
		case "386":
			need = append(need, i386GnuNeed...)
		case "arm", "arm64":
			need = append(need, armGnuNeed...)
		case "ppc64", "ppc64le":
			need = append(need, ppcGnuNeed...)
		}
	}
	args = []string{
		"-s", "main.main",
		hello,
	}

	if printCode {
		args = append([]string{"-S"}, args...)
	}

	if printGnuAsm {
		args = append([]string{"-gnu"}, args...)
	}
	cmd = exec.Command(exe, args...)
	cmd.Dir = "testdata" // "Bad line" bug #36683 is sensitive to being run in the source directory
	out, err = cmd.CombinedOutput()
	t.Logf("Running %v", cmd.Args)

	if err != nil {
		exename := srcfname[:len(srcfname)-len(filepath.Ext(srcfname))] + ".exe"
		t.Fatalf("objdump %q: %v\n%s", exename, err, out)
	}

	text := string(out)
	ok := true
	for _, s := range need {
		if !strings.Contains(text, s) {
			t.Errorf("disassembly missing '%s'", s)
			ok = false
		}
	}
	if goarch == "386" {
		if strings.Contains(text, "(IP)") {
			t.Errorf("disassembly contains PC-Relative addressing on 386")
			ok = false
		}
	}

	if !ok {
		t.Logf("full disassembly:\n%s", text)
	}
}

func testGoAndCgoDisasm(t *testing.T, printCode bool, printGnuAsm bool) {
	t.Parallel()
	testDisasm(t, "fmthello.go", printCode, printGnuAsm)
	if build.Default.CgoEnabled {
		testDisasm(t, "fmthellocgo.go", printCode, printGnuAsm)
	}
}

func TestDisasm(t *testing.T) {
	testGoAndCgoDisasm(t, false, false)
}

func TestDisasmCode(t *testing.T) {
	testGoAndCgoDisasm(t, true, false)
}

func TestDisasmGnuAsm(t *testing.T) {
	testGoAndCgoDisasm(t, false, true)
}

func TestDisasmExtld(t *testing.T) {
	testenv.MustHaveCGO(t)
	switch runtime.GOOS {
	case "plan9", "windows":
		t.Skipf("skipping on %s", runtime.GOOS)
	}
	t.Parallel()
	testDisasm(t, "fmthello.go", false, false, "-ldflags=-linkmode=external")
}

func TestDisasmGoobj(t *testing.T) {
	mustHaveDisasm(t)

	hello := filepath.Join(tmp, "hello.o")
	args := []string{"tool", "compile", "-o", hello}
	args = append(args, "testdata/fmthello.go")
	out, err := exec.Command(testenv.GoToolPath(t), args...).CombinedOutput()
	if err != nil {
		t.Fatalf("go tool compile fmthello.go: %v\n%s", err, out)
	}
	need := []string{
		"main(SB)",
		"fmthello.go:6",
	}

	args = []string{
		"-s", "main",
		hello,
	}

	out, err = exec.Command(exe, args...).CombinedOutput()
	if err != nil {
		t.Fatalf("objdump fmthello.o: %v\n%s", err, out)
	}

	text := string(out)
	ok := true
	for _, s := range need {
		if !strings.Contains(text, s) {
			t.Errorf("disassembly missing '%s'", s)
			ok = false
		}
	}
	if runtime.GOARCH == "386" {
		if strings.Contains(text, "(IP)") {
			t.Errorf("disassembly contains PC-Relative addressing on 386")
			ok = false
		}
	}
	if !ok {
		t.Logf("full disassembly:\n%s", text)
	}
}

func TestGoobjFileNumber(t *testing.T) {
	// Test that file table in Go object file is parsed correctly.
	testenv.MustHaveGoBuild(t)
	mustHaveDisasm(t)

	t.Parallel()

	tmpdir, err := os.MkdirTemp("", "TestGoobjFileNumber")
	if err != nil {
		t.Fatal(err)
	}
	defer os.RemoveAll(tmpdir)

	obj := filepath.Join(tmpdir, "p.a")
	cmd := exec.Command(testenv.GoToolPath(t), "build", "-o", obj)
	cmd.Dir = filepath.Join("testdata/testfilenum")
	out, err := cmd.CombinedOutput()
	if err != nil {
		t.Fatalf("build failed: %v\n%s", err, out)
	}

	cmd = exec.Command(exe, obj)
	out, err = cmd.CombinedOutput()
	if err != nil {
		t.Fatalf("objdump failed: %v\n%s", err, out)
	}

	text := string(out)
	for _, s := range []string{"a.go", "b.go", "c.go"} {
		if !strings.Contains(text, s) {
			t.Errorf("output missing '%s'", s)
		}
	}

	if t.Failed() {
		t.Logf("output:\n%s", text)
	}
}
<|endoftext|>


<commit_before>package algorithm

import "sort"

type Version struct {
	id     int
	order  int
	passed map[int]BuildSet
}

func NewVersion(candidate VersionCandidate) Version {
	v := Version{
		id:     candidate.VersionID,
		order:  candidate.CheckOrder,
		passed: map[int]BuildSet{},
	}

	if candidate.JobID != 0 {
		v.passed[candidate.JobID] = BuildSet{candidate.BuildID: struct{}{}}
	}

	return v
}

func (v Version) PassedAny(jobID int, builds BuildSet) bool {
	bs, found := v.passed[jobID]
	if !found {
		return true
	}

	return bs.Overlaps(builds)
}

type Versions []Version

func (vs Versions) With(candidate VersionCandidate) Versions {
	i := sort.Search(len(vs), func(i int) bool {
		return vs[i].order <= candidate.CheckOrder
	})
	if i == len(vs) {
		vs = append(vs, NewVersion(candidate))
	}

	if vs[i].id != candidate.VersionID {
		vs = append(vs, Version{})
		copy(vs[i+1:], vs[i:])
		vs[i] = NewVersion(candidate)
	} else if candidate.JobID != 0 {
		builds, found := vs[i].passed[candidate.JobID]
		if !found {
			builds = BuildSet{}
			vs[i].passed[candidate.JobID] = builds
		}

		builds[candidate.BuildID] = struct{}{}
	}

	return vs
}

func (vs Versions) Merge(v Version) Versions {
	i := sort.Search(len(vs), func(i int) bool {
		return vs[i].order <= v.order
	})
	if i == len(vs) {
		vs = append(vs, v)
	}

	if vs[i].id != v.id {
		vs = append(vs, Version{})
		copy(vs[i+1:], vs[i:])
		vs[i] = v
	} else {
		for jobID, vbuilds := range v.passed {
			builds, found := vs[i].passed[jobID]
			if !found {
				vs[i].passed[jobID] = vbuilds
				continue
			}

			for vbuild := range vbuilds {
				builds[vbuild] = struct{}{}
			}
		}
	}

	return vs
}
<commit_msg>skip redundant work<commit_after>package algorithm

import "sort"

type Version struct {
	id     int
	order  int
	passed map[int]BuildSet
}

func NewVersion(candidate VersionCandidate) Version {
	v := Version{
		id:     candidate.VersionID,
		order:  candidate.CheckOrder,
		passed: map[int]BuildSet{},
	}

	if candidate.JobID != 0 {
		v.passed[candidate.JobID] = BuildSet{candidate.BuildID: struct{}{}}
	}

	return v
}

func (v Version) PassedAny(jobID int, builds BuildSet) bool {
	bs, found := v.passed[jobID]
	if !found {
		return true
	}

	return bs.Overlaps(builds)
}

type Versions []Version

func (vs Versions) With(candidate VersionCandidate) Versions {
	i := sort.Search(len(vs), func(i int) bool {
		return vs[i].order <= candidate.CheckOrder
	})
	if i == len(vs) {
		return append(vs, NewVersion(candidate))
	}

	if vs[i].id != candidate.VersionID {
		vs = append(vs, Version{})
		copy(vs[i+1:], vs[i:])
		vs[i] = NewVersion(candidate)
	} else if candidate.JobID != 0 {
		builds, found := vs[i].passed[candidate.JobID]
		if !found {
			builds = BuildSet{}
			vs[i].passed[candidate.JobID] = builds
		}

		builds[candidate.BuildID] = struct{}{}
	}

	return vs
}

func (vs Versions) Merge(v Version) Versions {
	i := sort.Search(len(vs), func(i int) bool {
		return vs[i].order <= v.order
	})
	if i == len(vs) {
		return append(vs, v)
	}

	if vs[i].id != v.id {
		vs = append(vs, Version{})
		copy(vs[i+1:], vs[i:])
		vs[i] = v
	} else {
		for jobID, vbuilds := range v.passed {
			builds, found := vs[i].passed[jobID]
			if !found {
				vs[i].passed[jobID] = vbuilds
				continue
			}

			for vbuild := range vbuilds {
				builds[vbuild] = struct{}{}
			}
		}
	}

	return vs
}
<|endoftext|>


<commit_before>package rpcd

import (
	"bufio"
	"errors"
	"fmt"
	"github.com/Symantec/Dominator/lib/format"
	"github.com/Symantec/Dominator/lib/hash"
	"github.com/Symantec/Dominator/lib/objectcache"
	"github.com/Symantec/Dominator/lib/objectclient"
	"github.com/Symantec/Dominator/proto/sub"
	"io"
	"os"
	"path"
	"syscall"
	"time"
)

func (t *rpcType) Fetch(request sub.FetchRequest,
	reply *sub.FetchResponse) error {
	rwLock.Lock()
	defer rwLock.Unlock()
	logger.Printf("Fetch() %d objects\n", len(request.Hashes))
	if fetchInProgress {
		logger.Println("Error: fetch already in progress")
		return errors.New("fetch already in progress")
	}
	if updateInProgress {
		logger.Println("Error: update progress")
		return errors.New("update in progress")
	}
	fetchInProgress = true
	go doFetch(request)
	return nil
}

func doFetch(request sub.FetchRequest) {
	defer clearFetchInProgress()
	objectServer := objectclient.NewObjectClient(request.ServerAddress)
	benchmark := false
	if networkReaderContext.MaximumSpeed() < 1 {
		benchmark = enoughBytesForBenchmark(objectServer, request)
		if benchmark {
			objectServer.SetExclusiveGetObjects(true)
			logger.Println("Benchmarking network speed")
		}
	}
	objectsReader, err := objectServer.GetObjects(request.Hashes)
	if err != nil {
		logger.Printf("Error getting object reader:\t%s\n", err.Error())
		return
	}
	var totalLength uint64
	timeStart := time.Now()
	for _, hash := range request.Hashes {
		length, reader, err := objectsReader.NextObject()
		if err != nil {
			logger.Println(err)
			return
		}
		err = readOne(hash, networkReaderContext.NewReader(reader))
		reader.Close()
		if err != nil {
			logger.Println(err)
			return
		}
		totalLength += length
	}
	duration := time.Since(timeStart)
	speed := uint64(float64(totalLength) / duration.Seconds())
	if benchmark {
		file, err := os.Create(netbenchFilename)
		if err == nil {
			fmt.Fprintf(file, "%d\n", speed)
			file.Close()
		}
		networkReaderContext.InitialiseMaximumSpeed(speed)
	}
	logger.Printf("Fetch() complete. Read: %s in %s (%s/s)\n",
		format.FormatBytes(totalLength), duration, format.FormatBytes(speed))
	rescanObjectCacheChannel <- true
}

func enoughBytesForBenchmark(objectServer *objectclient.ObjectClient,
	request sub.FetchRequest) bool {
	lengths, err := objectServer.CheckObjects(request.Hashes)
	if err != nil {
		return false
	}
	var totalLength uint64
	for _, length := range lengths {
		totalLength += length
	}
	if totalLength > 1024*1024*64 {
		return true
	}
	return false
}

func readOne(hash hash.Hash, reader io.Reader) error {
	filename := path.Join(objectsDir, objectcache.HashToFilename(hash))
	dirname := path.Dir(filename)
	err := os.MkdirAll(dirname, syscall.S_IRUSR|syscall.S_IWUSR)
	if err != nil {
		return err
	}
	file, err := os.Create(filename)
	if err != nil {
		return err
	}
	defer file.Close()
	writer := bufio.NewWriter(file)
	defer writer.Flush()
	_, err = io.Copy(writer, reader)
	if err != nil {
		return errors.New(fmt.Sprintf("error copying: %s", err.Error()))
	}
	return nil
}

func clearFetchInProgress() {
	rwLock.Lock()
	defer rwLock.Unlock()
	fetchInProgress = false
}
<commit_msg>Add u=x permission to /.subd/objects directory.<commit_after>package rpcd

import (
	"bufio"
	"errors"
	"fmt"
	"github.com/Symantec/Dominator/lib/format"
	"github.com/Symantec/Dominator/lib/hash"
	"github.com/Symantec/Dominator/lib/objectcache"
	"github.com/Symantec/Dominator/lib/objectclient"
	"github.com/Symantec/Dominator/proto/sub"
	"io"
	"os"
	"path"
	"syscall"
	"time"
)

func (t *rpcType) Fetch(request sub.FetchRequest,
	reply *sub.FetchResponse) error {
	rwLock.Lock()
	defer rwLock.Unlock()
	logger.Printf("Fetch() %d objects\n", len(request.Hashes))
	if fetchInProgress {
		logger.Println("Error: fetch already in progress")
		return errors.New("fetch already in progress")
	}
	if updateInProgress {
		logger.Println("Error: update progress")
		return errors.New("update in progress")
	}
	fetchInProgress = true
	go doFetch(request)
	return nil
}

func doFetch(request sub.FetchRequest) {
	defer clearFetchInProgress()
	objectServer := objectclient.NewObjectClient(request.ServerAddress)
	benchmark := false
	if networkReaderContext.MaximumSpeed() < 1 {
		benchmark = enoughBytesForBenchmark(objectServer, request)
		if benchmark {
			objectServer.SetExclusiveGetObjects(true)
			logger.Println("Benchmarking network speed")
		}
	}
	objectsReader, err := objectServer.GetObjects(request.Hashes)
	if err != nil {
		logger.Printf("Error getting object reader:\t%s\n", err.Error())
		return
	}
	var totalLength uint64
	timeStart := time.Now()
	for _, hash := range request.Hashes {
		length, reader, err := objectsReader.NextObject()
		if err != nil {
			logger.Println(err)
			return
		}
		err = readOne(hash, networkReaderContext.NewReader(reader))
		reader.Close()
		if err != nil {
			logger.Println(err)
			return
		}
		totalLength += length
	}
	duration := time.Since(timeStart)
	speed := uint64(float64(totalLength) / duration.Seconds())
	if benchmark {
		file, err := os.Create(netbenchFilename)
		if err == nil {
			fmt.Fprintf(file, "%d\n", speed)
			file.Close()
		}
		networkReaderContext.InitialiseMaximumSpeed(speed)
	}
	logger.Printf("Fetch() complete. Read: %s in %s (%s/s)\n",
		format.FormatBytes(totalLength), duration, format.FormatBytes(speed))
	rescanObjectCacheChannel <- true
}

func enoughBytesForBenchmark(objectServer *objectclient.ObjectClient,
	request sub.FetchRequest) bool {
	lengths, err := objectServer.CheckObjects(request.Hashes)
	if err != nil {
		return false
	}
	var totalLength uint64
	for _, length := range lengths {
		totalLength += length
	}
	if totalLength > 1024*1024*64 {
		return true
	}
	return false
}

func readOne(hash hash.Hash, reader io.Reader) error {
	filename := path.Join(objectsDir, objectcache.HashToFilename(hash))
	dirname := path.Dir(filename)
	err := os.MkdirAll(dirname, syscall.S_IRWXU)
	if err != nil {
		return err
	}
	file, err := os.Create(filename)
	if err != nil {
		return err
	}
	defer file.Close()
	writer := bufio.NewWriter(file)
	defer writer.Flush()
	_, err = io.Copy(writer, reader)
	if err != nil {
		return errors.New(fmt.Sprintf("error copying: %s", err.Error()))
	}
	return nil
}

func clearFetchInProgress() {
	rwLock.Lock()
	defer rwLock.Unlock()
	fetchInProgress = false
}
<|endoftext|>


<commit_before>package disk

import (
	"encoding/json"
	"os"
	"crypto/rand"
	"sort"
)

/*
A type returned by the create swarm option.
Contains information related to metadata such as filehash associated with filesize and other such things.
*/
type SwarmStorage struct {
	SwarmId    string            "swid"
	amountused uint64            "amtused"
	files      map[string]uint64 "files"
	fileordering []string "fileorder"
}

//helper function to produce the correct filename
func (r SwarmStorage) getFileName(filehash string) string {
	return r.SwarmId + string(os.PathSeparator) + filehash
}

//Opens or creates directory for swarm info, and if it exists, obtains the correct amount of space used by its
//files
func CreateSwarmSystem(swarmid string) (r *SwarmStorage, err error) {
	var i SwarmStorage
	i.SwarmId = swarmid
	i.amountused = 0
	i.files=make(map[string]uint64)
	err = os.Mkdir(swarmid, os.ModeDir|os.ModePerm)
	if err != nil {
		if os.IsExist(err) {
			meta, e := os.Open(swarmid + ".conf")
			if e != nil {
			}
			defer meta.Close()
			c := json.NewDecoder(meta)
			if e != nil {
				print(e.Error())
			}
			if err = c.Decode(&i); e != nil {

			}
		}
	}
	r = &i
	return
}

func (r SwarmStorage) CreateFile(filehash string, length uint64) (written int64, err error) {
	file, err := os.Create(r.SwarmId + string(os.PathSeparator) + filehash)

	if err != nil&&os.IsExist(err) {
		//in which case, it should be safe to ignore the error
		err = nil
		if r.files[filehash]==length{
			return
		}
	}
	defer file.Close()
	err = file.Truncate(int64(length))
	if _,ok:=r.files[filehash];!ok{
		r.fileordering=append(r.fileorder,filehash)
		sort.Sort(&r.fileordering)
	}
	r.files[filehash] = uint64(length)
	written = int64(length)
	return
}
func (r SwarmStorage) FileExists(filehash string) bool{
	_,ok:=r.files[filehash]
	return ok
}

func (r SwarmStorage) DeleteFile(filehash string) error {
	size, err := os.Stat(r.getFileName(filehash))
	if err == nil {
		r.amountused -= uint64(size.Size())
		err = os.Remove(r.SwarmId + string(os.PathSeparator) + filehash)
	}
	r.files[filehash] = uint64(0)
	return err
}

func (r SwarmStorage) WriteFile(filehash string, start uint64, data []byte) error {
	path := r.SwarmId + string(os.PathSeparator) + filehash
	file, err := os.Open(path)
	if err != nil {
		return err
	}
	size, ok := r.files[filehash]
	if uint64(start)+uint64(len(data)) >= size && ok {
		r.amountused += uint64(start) + uint64(uint64(len(data))-size)
		r.files[filehash] = uint64(start) + uint64(len(data))
	}
	file.WriteAt(data, int64(start))
	file.Close()
	return nil

}
func (r SwarmStorage) ReadFile(filehash string, start uint64, data []byte) (err error) {
	file, err := os.Open(r.getFileName(filehash))
	file.ReadAt(data, int64(start))
	return
}
func (r SwarmStorage) SaveSwarm() {
	s,err:=os.Create(r.SwarmId+".conf")
	if err!=nil&&os.IsExist(err){
		s, err = os.Open(r.SwarmId + ".conf")
	}
	defer s.Close()
	js := json.NewEncoder(s)
	if err = js.Encode(&s); err != nil {
		print("From SaveSwarm")
		print(err.Error())
	}

}
func (r SwarmStorage) GetRandomByte(index uint64) byte{
	var u uint64
	c:=uint64(0)
	v:=""
	for i :=range r.fileordering{
		if u+r.files[i]>=index{
			c=index-u+index
			v=i
			break
		}
		u+=r.files[i]
	}
	b:=[]byte{0}
	r.ReadFile(v,c,b)
	return b[0]
}
<commit_msg>Fixed it so that the unit tests works. Also cleaned up a little bit.<commit_after>package disk

import (
	"encoding/json"
	"os"
	"sort"
)

/*
A type returned by the create swarm option.
Contains information related to metadata such as filehash associated with filesize and other such things.
*/
type SwarmStorage struct {
	SwarmId    string            "swid"
	amountused uint64            "amtused"
	files      map[string]uint64 "files"
	fileordering []string "fileorder"
}

//helper function to produce the correct filename
func (r SwarmStorage) getFileName(filehash string) string {
	return r.SwarmId + string(os.PathSeparator) + filehash
}

//Opens or creates directory for swarm info, and if it exists, obtains the correct amount of space used by its
//files
func CreateSwarmSystem(swarmid string) (r *SwarmStorage, err error) {
	var i SwarmStorage
	i.SwarmId = swarmid
	i.amountused = 0
	i.files=make(map[string]uint64)
	err = os.Mkdir(swarmid, os.ModeDir|os.ModePerm)
	if err != nil {
		if os.IsExist(err) {
			meta, e := os.Open(swarmid + ".conf")
			if e != nil {
			}
			defer meta.Close()
			c := json.NewDecoder(meta)
			if e != nil {
				print(e.Error())
			}
			if err = c.Decode(&i); e != nil {

			}
		}
	}
	r = &i
	return
}

func (r SwarmStorage) CreateFile(filehash string, length uint64) (written int64, err error) {
	file, err := os.Create(r.SwarmId + string(os.PathSeparator) + filehash)

	if err != nil&&os.IsExist(err) {
		//in which case, it should be safe to ignore the error
		err = nil
		if r.files[filehash]==length{
			return
		}
	}
	defer file.Close()
	err = file.Truncate(int64(length))
	if _,ok:=r.files[filehash];!ok{
		r.fileordering=append(r.fileordering,filehash)
		sort.Strings(r.fileordering)
	}
	r.files[filehash] = uint64(length)
	written = int64(length)
	return
}
func (r SwarmStorage) FileExists(filehash string) bool{
	_,ok:=r.files[filehash]
	return ok
}

func (r SwarmStorage) DeleteFile(filehash string) error {
	size, err := os.Stat(r.getFileName(filehash))
	if err == nil {
		r.amountused -= uint64(size.Size())
		err = os.Remove(r.SwarmId + string(os.PathSeparator) + filehash)
	}
	r.files[filehash] = uint64(0)
	return err
}

func (r SwarmStorage) WriteFile(filehash string, start uint64, data []byte) error {
	path := r.SwarmId + string(os.PathSeparator) + filehash
	file, err := os.Open(path)
	if err != nil {
		return err
	}
	size, ok := r.files[filehash]
	if uint64(start)+uint64(len(data)) >= size && ok {
		r.amountused += uint64(start) + uint64(uint64(len(data))-size)
		r.files[filehash] = uint64(start) + uint64(len(data))
	}
	file.WriteAt(data, int64(start))
	file.Close()
	return nil

}
func (r SwarmStorage) ReadFile(filehash string, start uint64, data []byte) (err error) {
	file, err := os.Open(r.getFileName(filehash))
	file.ReadAt(data, int64(start))
	return
}
func (r SwarmStorage) SaveSwarm() {
	s,err:=os.Create(r.SwarmId+".conf")
	if err!=nil&&os.IsExist(err){
		s, err = os.Open(r.SwarmId + ".conf")
	}
	defer s.Close()
	js := json.NewEncoder(s)
	if err = js.Encode(&r); err != nil {
		print("From SaveSwarm")
		print(err.Error())
	}

}
func (r SwarmStorage) GetRandomByte(index uint64) byte{
	var u uint64
	c:=uint64(0)
	v:=""
	for i :=range r.fileordering{
		var d=r.fileordering[i]
		if u+r.files[d]>=index{
			c=index-u+index
			v=d
			break
		}
		u+=r.files[d]
	}
	if v==""{
		return 0
	}
	b:=[]byte{0}
	r.ReadFile(v,c,b)
	return b[0]
}
<|endoftext|>


<commit_before>/*

*/

package main

import (
	"flag"
	"path/filepath"
	"os"

	"golang.org/x/exp/inotify"
	"github.com/hosom/gomandrake/plugin"
)

const (
	// VERSION is the version of the plugin
	VERSION = "0.0.1"
	// NAME is the name of the plugin
	NAME = "inotify"
)

type config struct {
	MonitorPath		string
}

func main() {
	config_path := flag.String("config",
		filepath.FromSlash("/etc/mandrake/inotify.conf"),
		"configuration file")
	version := flag.Bool("version",
		false,
		"Output version and exit.")
	flag.Parse()

	if *version {
		fmt.Println(VERSION)
		os.Exit(0)
	}

	var c config
	configuration, err := plugin.ReadConfig(config_path, c)
	if err != nil {
		log.Fatal(err)
	}

	i := plugin.NewInput(NAME)

	watcher, err := inotify.NewWatcher()
	if err != nil {
		log.Fatal(err)
	}

	err = watcher.AddWatch(config.MonitorPath, inotify.IN_CLOSE_WRITE)

	for {
		select {
		case ev := <-watcher.Event:
			i.Analyze(ev.Name)
		case ev := <-watcher.Error:
			log.Printf("Error: %s", err)
		}
	}
}<commit_msg>update inotify plugin<commit_after>/*

*/

package main

import (
	"flag"
	"path/filepath"
	"os"
	"fmt"
	"log"

	"golang.org/x/exp/inotify"
	"github.com/hosom/gomandrake/plugin"
)

const (
	// VERSION is the version of the plugin
	VERSION = "0.0.1"
	// NAME is the name of the plugin
	NAME = "inotify"
)

type config struct {
	MonitorPath		string
}

func main() {
	config_path := flag.String("config",
		filepath.FromSlash("/etc/mandrake/inotify.conf"),
		"configuration file")
	version := flag.Bool("version",
		false,
		"Output version and exit.")
	flag.Parse()

	if *version {
		fmt.Println(VERSION)
		os.Exit(0)
	}

	var c *config
	configuration, err := plugin.ReadConfig(*config_path, c)
	if err != nil {
		log.Fatal(err)
	}

	c = configuration.(*config)

	i := plugin.NewInput(NAME)

	watcher, err := inotify.NewWatcher()
	if err != nil {
		log.Fatal(err)
	}

	err = watcher.AddWatch(c.MonitorPath, inotify.IN_CLOSE_WRITE)

	for {
		select {
		case ev := <-watcher.Event:
			i.Analyze(ev.Name)
		case err := <-watcher.Error:
			log.Printf("Error: %s", err)
		}
	}
}<|endoftext|>


<commit_before>package Kinsokujiko

import (
	"bufio"
	"errors"
	"fmt"
	"os"
	"strings"
)

// Item is dictionary Item
type Item struct {
	Surf  string
	Slice string
	Kana  string
	Pos   string
}

// Dictionary is Slice any Item
type Dictionary []Item

var dicPath = "_dic/dic.txt"

func Update(dic Dictionary) (string, error) {
	_, err := _add_item(dic)
	if err != nil {
		fmt.Println(err)
	}
	for index, item := range dic {
		if !_check_item(item) {
			return "Error", errors.New(fmt.Sprintf("%d 行目に問題があるようです", index))
		}
	}
	return _add_item(dic)
}

func _check_item(it Item) bool {
	var res = false
	if strings.Count(it.Surf, "") > 1 {
		res = true
	}
	if strings.Count(it.Slice, "") > 1 {
		res = true
	}
	if strings.Count(it.Kana, "") > 1 {
		res = true
	}
	if strings.Count(it.Pos, "") > 1 {
		res = true
	}
	return res
}

func _add_item(dic Dictionary) (string, error) {
	dir, _ := os.Getwd()
	dicPath := fmt.Sprintf("%s/%s", dir, dicPath)
	file, err := os.OpenFile(dicPath, os.O_RDWR|os.O_CREATE, 0755)
	defer file.Close()
	scan := bufio.NewScanner(file)
	scan.Scan()
	res := fmt.Sprintf("ファイルの詳細：%+v", scan.Text())
	return res, err
}

func Show() (Dictionary, error) {
	return Dictionary{}, nil
}

func _get_dic() {}
<commit_msg>debug: tring bufio part 1.<commit_after>package Kinsokujiko

import (
	"bufio"
	"errors"
	"fmt"
	"os"
	"strings"
)

// Item is dictionary Item
type Item struct {
	Surf  string
	Slice string
	Kana  string
	Pos   string
}

// Dictionary is Slice any Item
type Dictionary []Item

var dicPath = "_dic/dic.txt"

func Update(dic Dictionary) (string, error) {
	_, err := _add_item(dic)
	if err != nil {
		fmt.Println(err)
	}
	for index, item := range dic {
		if !_check_item(item) {
			return "Error", errors.New(fmt.Sprintf("%d 行目に問題があるようです", index))
		}
	}
	return _add_item(dic)
}

func _check_item(it Item) bool {
	var res = false
	if strings.Count(it.Surf, "") > 1 {
		res = true
	}
	if strings.Count(it.Slice, "") > 1 {
		res = true
	}
	if strings.Count(it.Kana, "") > 1 {
		res = true
	}
	if strings.Count(it.Pos, "") > 1 {
		res = true
	}
	return res
}

func _add_item(dic Dictionary) (string, error) {
	dir, _ := os.Getwd()
	dicPath := fmt.Sprintf("%s/%s", dir, dicPath)
	file, err := os.OpenFile(dicPath, os.O_RDWR|os.O_CREATE, 0755)
	defer file.Close()
	read := bufio.NewReader(file)
	res := fmt.Sprintf("ファイルの詳細：%+v", read)
	return res, err
}

func Show() (Dictionary, error) {
	return Dictionary{}, nil
}

func _get_dic() {}
<|endoftext|>


<commit_before>// +build !windows
// Copyright 2014-2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"). You may
// not use this file except in compliance with the License. A copy of the
// License is located at
//
//	http://aws.amazon.com/apache2.0/
//
// or in the "license" file accompanying this file. This file is distributed
// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
// express or implied. See the License for the specific language governing
// permissions and limitations under the License.

package config

import (
	"fmt"
	"time"

	"github.com/aws/amazon-ecs-agent/agent/dockerclient"
)

const (
	// AgentCredentialsAddress is used to serve the credentials for tasks.
	AgentCredentialsAddress = "" // this is left blank right now for net=bridge
	// defaultAuditLogFile specifies the default audit log filename
	defaultCredentialsAuditLogFile = "/log/audit.log"
	// Default cgroup prefix for ECS tasks
	DefaultTaskCgroupPrefix = "/ecs"

	// Default cgroup memory system root path, this is the default used if the
	// path has not been configured through ECS_CGROUP_PATH
	defaultCgroupPath = "/sys/fs/cgroup"
	// defaultContainerStartTimeout specifies the value for container start timeout duration
	defaultContainerStartTimeout = 3 * time.Minute
	// minimumContainerStartTimeout specifies the minimum value for starting a container
	minimumContainerStartTimeout = 45 * time.Second
	// default docker inactivity time is extra time needed on container extraction
	defaultImagePullInactivityTimeout = 1 * time.Minute
)

// DefaultConfig returns the default configuration for Linux
func DefaultConfig() Config {
	return Config{
		DockerEndpoint:              "unix:///var/run/docker.sock",
		ReservedPorts:               []uint16{SSHPort, DockerReservedPort, DockerReservedSSLPort, AgentIntrospectionPort, AgentCredentialsPort},
		ReservedPortsUDP:            []uint16{},
		DataDir:                     "/data/",
		DataDirOnHost:               "/var/lib/ecs",
		DisableMetrics:              false,
		ReservedMemory:              0,
		AvailableLoggingDrivers:     []dockerclient.LoggingDriver{dockerclient.JSONFileDriver, dockerclient.NoneDriver},
		TaskCleanupWaitDuration:     DefaultTaskCleanupWaitDuration,
		DockerStopTimeout:           defaultDockerStopTimeout,
		ContainerStartTimeout:       defaultContainerStartTimeout,
		CredentialsAuditLogFile:     defaultCredentialsAuditLogFile,
		CredentialsAuditLogDisabled: false,
		ImageCleanupDisabled:        false,
		MinimumImageDeletionAge:     DefaultImageDeletionAge,
		ImageCleanupInterval:        DefaultImageCleanupTimeInterval,
		NumImagesToDeletePerCycle:   DefaultNumImagesToDeletePerCycle,
		CNIPluginsPath:              defaultCNIPluginsPath,
		PauseContainerTarballPath:   pauseContainerTarballPath,
		PauseContainerImageName:     DefaultPauseContainerImageName,
		PauseContainerTag:           DefaultPauseContainerTag,
		AWSVPCBlockInstanceMetdata:  false,
		ContainerMetadataEnabled:    false,
		TaskCPUMemLimit:             DefaultEnabled,
		CgroupPath:                  defaultCgroupPath,
		TaskMetadataSteadyStateRate: DefaultTaskMetadataSteadyStateRate,
		TaskMetadataBurstRate:       DefaultTaskMetadataBurstRate,
		SharedVolumeMatchFullConfig: false, // only requiring shared volumes to match on name, which is default docker behavior
	}
}

func (cfg *Config) platformOverrides() {}

// ShouldLoadPauseContainerTarball determines whether the pause container
// tarball should be loaded into Docker or not.  This function will return
// false is the default image name/tag have been overridden, because we do not
// expect the tarball to match the overridden name/tag.
func (cfg *Config) ShouldLoadPauseContainerTarball() bool {
	return cfg.PauseContainerImageName == DefaultPauseContainerImageName &&
		cfg.PauseContainerTag == DefaultPauseContainerTag
}

// platformString returns platform-specific config data that can be serialized
// to string for debugging
func (cfg *Config) platformString() string {
	if cfg.ShouldLoadPauseContainerTarball() {
		return fmt.Sprintf(", PauseContainerImageName: %s, PauseContainerTag: %s",
			cfg.PauseContainerImageName, cfg.PauseContainerTag)
	}
	return ""
}
<commit_msg>config: Add missing default on linux<commit_after>// +build !windows
// Copyright 2014-2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"). You may
// not use this file except in compliance with the License. A copy of the
// License is located at
//
//	http://aws.amazon.com/apache2.0/
//
// or in the "license" file accompanying this file. This file is distributed
// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
// express or implied. See the License for the specific language governing
// permissions and limitations under the License.

package config

import (
	"fmt"
	"time"

	"github.com/aws/amazon-ecs-agent/agent/dockerclient"
)

const (
	// AgentCredentialsAddress is used to serve the credentials for tasks.
	AgentCredentialsAddress = "" // this is left blank right now for net=bridge
	// defaultAuditLogFile specifies the default audit log filename
	defaultCredentialsAuditLogFile = "/log/audit.log"
	// Default cgroup prefix for ECS tasks
	DefaultTaskCgroupPrefix = "/ecs"

	// Default cgroup memory system root path, this is the default used if the
	// path has not been configured through ECS_CGROUP_PATH
	defaultCgroupPath = "/sys/fs/cgroup"
	// defaultContainerStartTimeout specifies the value for container start timeout duration
	defaultContainerStartTimeout = 3 * time.Minute
	// minimumContainerStartTimeout specifies the minimum value for starting a container
	minimumContainerStartTimeout = 45 * time.Second
	// default docker inactivity time is extra time needed on container extraction
	defaultImagePullInactivityTimeout = 1 * time.Minute
)

// DefaultConfig returns the default configuration for Linux
func DefaultConfig() Config {
	return Config{
		DockerEndpoint:              "unix:///var/run/docker.sock",
		ReservedPorts:               []uint16{SSHPort, DockerReservedPort, DockerReservedSSLPort, AgentIntrospectionPort, AgentCredentialsPort},
		ReservedPortsUDP:            []uint16{},
		DataDir:                     "/data/",
		DataDirOnHost:               "/var/lib/ecs",
		DisableMetrics:              false,
		ReservedMemory:              0,
		AvailableLoggingDrivers:     []dockerclient.LoggingDriver{dockerclient.JSONFileDriver, dockerclient.NoneDriver},
		TaskCleanupWaitDuration:     DefaultTaskCleanupWaitDuration,
		DockerStopTimeout:           defaultDockerStopTimeout,
		ContainerStartTimeout:       defaultContainerStartTimeout,
		CredentialsAuditLogFile:     defaultCredentialsAuditLogFile,
		CredentialsAuditLogDisabled: false,
		ImageCleanupDisabled:        false,
		MinimumImageDeletionAge:     DefaultImageDeletionAge,
		ImageCleanupInterval:        DefaultImageCleanupTimeInterval,
		ImagePullInactivityTimeout:  defaultImagePullInactivityTimeout,
		NumImagesToDeletePerCycle:   DefaultNumImagesToDeletePerCycle,
		CNIPluginsPath:              defaultCNIPluginsPath,
		PauseContainerTarballPath:   pauseContainerTarballPath,
		PauseContainerImageName:     DefaultPauseContainerImageName,
		PauseContainerTag:           DefaultPauseContainerTag,
		AWSVPCBlockInstanceMetdata:  false,
		ContainerMetadataEnabled:    false,
		TaskCPUMemLimit:             DefaultEnabled,
		CgroupPath:                  defaultCgroupPath,
		TaskMetadataSteadyStateRate: DefaultTaskMetadataSteadyStateRate,
		TaskMetadataBurstRate:       DefaultTaskMetadataBurstRate,
		SharedVolumeMatchFullConfig: false, // only requiring shared volumes to match on name, which is default docker behavior
	}
}

func (cfg *Config) platformOverrides() {}

// ShouldLoadPauseContainerTarball determines whether the pause container
// tarball should be loaded into Docker or not.  This function will return
// false is the default image name/tag have been overridden, because we do not
// expect the tarball to match the overridden name/tag.
func (cfg *Config) ShouldLoadPauseContainerTarball() bool {
	return cfg.PauseContainerImageName == DefaultPauseContainerImageName &&
		cfg.PauseContainerTag == DefaultPauseContainerTag
}

// platformString returns platform-specific config data that can be serialized
// to string for debugging
func (cfg *Config) platformString() string {
	if cfg.ShouldLoadPauseContainerTarball() {
		return fmt.Sprintf(", PauseContainerImageName: %s, PauseContainerTag: %s",
			cfg.PauseContainerImageName, cfg.PauseContainerTag)
	}
	return ""
}
<|endoftext|>


<commit_before>// Copyright 2014-2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"). You may
// not use this file except in compliance with the License. A copy of the
// License is located at
//
//	http://aws.amazon.com/apache2.0/
//
// or in the "license" file accompanying this file. This file is distributed
// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
// express or implied. See the License for the specific language governing
// permissions and limitations under the License.

// Package engine contains the core logic for managing tasks
package engine

import (
	"context"
	"fmt"
	"reflect"
	"sync"
	"testing"
	"time"

	"github.com/aws/amazon-ecs-agent/agent/api"
	"github.com/aws/amazon-ecs-agent/agent/credentials"
	"github.com/aws/amazon-ecs-agent/agent/engine/dockerclient"
	"github.com/aws/amazon-ecs-agent/agent/statechange"
	"github.com/aws/amazon-ecs-agent/agent/utils/ttime/mocks"
	docker "github.com/fsouza/go-dockerclient"
	"github.com/golang/mock/gomock"
	"github.com/stretchr/testify/assert"
)

const (
	containerID = "containerID"
)

var (
	defaultDockerClientAPIVersion = dockerclient.Version_1_17
)

func discardEvents(from interface{}) func() {
	done := make(chan bool)

	go func() {
		for {
			ndx, _, _ := reflect.Select([]reflect.SelectCase{
				{
					Dir:  reflect.SelectRecv,
					Chan: reflect.ValueOf(from),
				},
				{
					Dir:  reflect.SelectRecv,
					Chan: reflect.ValueOf(done),
				},
			})
			if ndx == 1 {
				break
			}
		}
	}()
	return func() {
		done <- true
	}
}

// TODO: Move integ tests away from relying on the statechange channel for
// determining if a task is running/stopped or not
func verifyTaskIsRunning(stateChangeEvents <-chan statechange.Event, task *api.Task) error {
	for {
		event := <-stateChangeEvents
		if event.GetEventType() != statechange.TaskEvent {
			continue
		}

		taskEvent := event.(api.TaskStateChange)
		if taskEvent.TaskARN != task.Arn {
			continue
		}
		if taskEvent.Status == api.TaskRunning {
			return nil
		}
		if taskEvent.Status > api.TaskRunning {
			return fmt.Errorf("Task went straight to %s without running, task: %s", taskEvent.Status.String(), task.Arn)
		}
	}
}

func verifyTaskIsStopped(stateChangeEvents <-chan statechange.Event, task *api.Task) {
	for {
		event := <-stateChangeEvents
		if event.GetEventType() != statechange.TaskEvent {
			continue
		}
		taskEvent := event.(api.TaskStateChange)
		if taskEvent.TaskARN == task.Arn && taskEvent.Status >= api.TaskStopped {
			return
		}
	}
}

// waitForTaskStoppedByCheckStatus verify the task is in stopped status by checking the KnownStatusUnsafe field of the task
func waitForTaskStoppedByCheckStatus(task *api.Task) {
	for {
		if task.GetKnownStatus() == api.TaskStopped {
			return
		}
		time.Sleep(50 * time.Millisecond)
	}
}

// validateContainerRunWorkflow validates the container create and start workflow
// for a test task without any resources (such as ENIs).
//
// The createdContainerName channel is used to emit the container name from the
// create operation. It can be used to validate that the name of the container
// removed matches with the generated container name during cleanup operation in the
// test.
func validateContainerRunWorkflow(t *testing.T,
	container *api.Container,
	task *api.Task,
	imageManager *MockImageManager,
	client *MockDockerClient,
	roleCredentials *credentials.TaskIAMRoleCredentials,
	containerEventsWG sync.WaitGroup,
	eventStream chan DockerContainerChangeEvent,
	createdContainerName chan<- string,
	assertions func(),
) {
	imageManager.EXPECT().AddAllImageStates(gomock.Any()).AnyTimes()
	client.EXPECT().PullImage(container.Image, nil).Return(DockerContainerMetadata{})
	imageManager.EXPECT().RecordContainerReference(container).Return(nil)
	imageManager.EXPECT().GetImageStateFromImageName(gomock.Any()).Return(nil)
	client.EXPECT().APIVersion().Return(defaultDockerClientAPIVersion, nil)
	dockerConfig, err := task.DockerConfig(container, defaultDockerClientAPIVersion)
	if err != nil {
		t.Fatal(err)
	}
	if roleCredentials != nil {
		// Container config should get updated with this during PostUnmarshalTask
		credentialsEndpointEnvValue := roleCredentials.IAMRoleCredentials.GenerateCredentialsEndpointRelativeURI()
		dockerConfig.Env = append(dockerConfig.Env, "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI="+credentialsEndpointEnvValue)
	}
	// Container config should get updated with this during CreateContainer
	dockerConfig.Labels["com.amazonaws.ecs.task-arn"] = task.Arn
	dockerConfig.Labels["com.amazonaws.ecs.container-name"] = container.Name
	dockerConfig.Labels["com.amazonaws.ecs.task-definition-family"] = task.Family
	dockerConfig.Labels["com.amazonaws.ecs.task-definition-version"] = task.Version
	dockerConfig.Labels["com.amazonaws.ecs.cluster"] = ""
	client.EXPECT().CreateContainer(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Do(
		func(config *docker.Config, y interface{}, containerName string, z time.Duration) {
			assert.True(t, reflect.DeepEqual(dockerConfig, config),
				"Mismatch in container config; expected: %v, got: %v", dockerConfig, config)
			// sleep5 task contains only one container. Just assign
			// the containerName to createdContainerName
			createdContainerName <- containerName
			containerEventsWG.Add(1)
			go func() {
				eventStream <- createDockerEvent(api.ContainerCreated)
				containerEventsWG.Done()
			}()
		}).Return(DockerContainerMetadata{DockerID: containerID})

	client.EXPECT().StartContainer(containerID, startContainerTimeout).Do(
		func(id string, timeout time.Duration) {
			containerEventsWG.Add(1)
			go func() {
				eventStream <- createDockerEvent(api.ContainerRunning)
				containerEventsWG.Done()
			}()
		}).Return(DockerContainerMetadata{DockerID: containerID})
	assertions()
}

// addTaskToEngine adds a test task to the engine. It waits for a task to reach the
// steady state before returning. Hence, this should not be used for tests, which
// expect container stops to be invoked before a task reaches its steady state
func addTaskToEngine(t *testing.T,
	ctx context.Context,
	taskEngine TaskEngine,
	sleepTask *api.Task,
	mockTime *mock_ttime.MockTime,
	createStartEventsReported sync.WaitGroup) {
	// steadyStateCheckWait is used to force the test to wait until the steady-state check
	// has been invoked at least once
	steadyStateVerify := make(chan time.Time, 1)
	mockTime.EXPECT().Now().Return(time.Now()).AnyTimes()
	gomock.InOrder(
		mockTime.EXPECT().After(steadyStateTaskVerifyInterval).Return(steadyStateVerify).AnyTimes(),
	)

	err := taskEngine.Init(ctx)
	assert.NoError(t, err)

	taskEngine.AddTask(sleepTask)
	waitForRunningEvents(t, taskEngine.StateChangeEvents())

	// Wait for all events to be consumed prior to moving it towards stopped; we
	// don't want to race the below with these or we'll end up with the "going
	// backwards in state" stop and we haven't 'expect'd for that

	// Wait for container create and start events to be processed
	createStartEventsReported.Wait()
}

func createDockerEvent(status api.ContainerStatus) DockerContainerChangeEvent {
	meta := DockerContainerMetadata{
		DockerID: containerID,
	}
	return DockerContainerChangeEvent{Status: status, DockerContainerMetadata: meta}
}

// waitForRunningEvents waits for a task to emit 'RUNNING' events for a container
// and the task
func waitForRunningEvents(t *testing.T, stateChangeEvents <-chan statechange.Event) {
	event := <-stateChangeEvents
	assert.Equal(t, event.(api.ContainerStateChange).Status, api.ContainerRunning,
		"Expected container to be RUNNING")

	event = <-stateChangeEvents
	assert.Equal(t, event.(api.TaskStateChange).Status, api.TaskRunning,
		"Expected task to be RUNNING")

	select {
	case <-stateChangeEvents:
		t.Fatal("Should be out of events")
	default:
	}
}

// waitForStopEvents waits for a task to emit 'STOPPED' events for a container
// and the task
func waitForStopEvents(t *testing.T, stateChangeEvents <-chan statechange.Event, verifyExitCode bool) {
	event := <-stateChangeEvents
	if cont := event.(api.ContainerStateChange); cont.Status != api.ContainerStopped {
		t.Fatal("Expected container to stop first")
		if verifyExitCode {
			assert.Equal(t, *cont.ExitCode, 1, "Exit code should be present")
		}
	}
	event = <-stateChangeEvents
	assert.Equal(t, event.(api.TaskStateChange).Status, api.TaskStopped, "Expected task to be STOPPED")

	select {
	case <-stateChangeEvents:
		t.Fatal("Should be out of events")
	default:
	}
}
<commit_msg>engine: remove mock validations for time.After()<commit_after>// Copyright 2014-2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"). You may
// not use this file except in compliance with the License. A copy of the
// License is located at
//
//	http://aws.amazon.com/apache2.0/
//
// or in the "license" file accompanying this file. This file is distributed
// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
// express or implied. See the License for the specific language governing
// permissions and limitations under the License.

// Package engine contains the core logic for managing tasks
package engine

import (
	"context"
	"fmt"
	"reflect"
	"sync"
	"testing"
	"time"

	"github.com/aws/amazon-ecs-agent/agent/api"
	"github.com/aws/amazon-ecs-agent/agent/credentials"
	"github.com/aws/amazon-ecs-agent/agent/engine/dockerclient"
	"github.com/aws/amazon-ecs-agent/agent/statechange"
	"github.com/aws/amazon-ecs-agent/agent/utils/ttime/mocks"
	docker "github.com/fsouza/go-dockerclient"
	"github.com/golang/mock/gomock"
	"github.com/stretchr/testify/assert"
)

const (
	containerID = "containerID"
)

var (
	defaultDockerClientAPIVersion = dockerclient.Version_1_17
)

func discardEvents(from interface{}) func() {
	done := make(chan bool)

	go func() {
		for {
			ndx, _, _ := reflect.Select([]reflect.SelectCase{
				{
					Dir:  reflect.SelectRecv,
					Chan: reflect.ValueOf(from),
				},
				{
					Dir:  reflect.SelectRecv,
					Chan: reflect.ValueOf(done),
				},
			})
			if ndx == 1 {
				break
			}
		}
	}()
	return func() {
		done <- true
	}
}

// TODO: Move integ tests away from relying on the statechange channel for
// determining if a task is running/stopped or not
func verifyTaskIsRunning(stateChangeEvents <-chan statechange.Event, task *api.Task) error {
	for {
		event := <-stateChangeEvents
		if event.GetEventType() != statechange.TaskEvent {
			continue
		}

		taskEvent := event.(api.TaskStateChange)
		if taskEvent.TaskARN != task.Arn {
			continue
		}
		if taskEvent.Status == api.TaskRunning {
			return nil
		}
		if taskEvent.Status > api.TaskRunning {
			return fmt.Errorf("Task went straight to %s without running, task: %s", taskEvent.Status.String(), task.Arn)
		}
	}
}

func verifyTaskIsStopped(stateChangeEvents <-chan statechange.Event, task *api.Task) {
	for {
		event := <-stateChangeEvents
		if event.GetEventType() != statechange.TaskEvent {
			continue
		}
		taskEvent := event.(api.TaskStateChange)
		if taskEvent.TaskARN == task.Arn && taskEvent.Status >= api.TaskStopped {
			return
		}
	}
}

// waitForTaskStoppedByCheckStatus verify the task is in stopped status by checking the KnownStatusUnsafe field of the task
func waitForTaskStoppedByCheckStatus(task *api.Task) {
	for {
		if task.GetKnownStatus() == api.TaskStopped {
			return
		}
		time.Sleep(50 * time.Millisecond)
	}
}

// validateContainerRunWorkflow validates the container create and start workflow
// for a test task without any resources (such as ENIs).
//
// The createdContainerName channel is used to emit the container name from the
// create operation. It can be used to validate that the name of the container
// removed matches with the generated container name during cleanup operation in the
// test.
func validateContainerRunWorkflow(t *testing.T,
	container *api.Container,
	task *api.Task,
	imageManager *MockImageManager,
	client *MockDockerClient,
	roleCredentials *credentials.TaskIAMRoleCredentials,
	containerEventsWG sync.WaitGroup,
	eventStream chan DockerContainerChangeEvent,
	createdContainerName chan<- string,
	assertions func(),
) {
	imageManager.EXPECT().AddAllImageStates(gomock.Any()).AnyTimes()
	client.EXPECT().PullImage(container.Image, nil).Return(DockerContainerMetadata{})
	imageManager.EXPECT().RecordContainerReference(container).Return(nil)
	imageManager.EXPECT().GetImageStateFromImageName(gomock.Any()).Return(nil)
	client.EXPECT().APIVersion().Return(defaultDockerClientAPIVersion, nil)
	dockerConfig, err := task.DockerConfig(container, defaultDockerClientAPIVersion)
	if err != nil {
		t.Fatal(err)
	}
	if roleCredentials != nil {
		// Container config should get updated with this during PostUnmarshalTask
		credentialsEndpointEnvValue := roleCredentials.IAMRoleCredentials.GenerateCredentialsEndpointRelativeURI()
		dockerConfig.Env = append(dockerConfig.Env, "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI="+credentialsEndpointEnvValue)
	}
	// Container config should get updated with this during CreateContainer
	dockerConfig.Labels["com.amazonaws.ecs.task-arn"] = task.Arn
	dockerConfig.Labels["com.amazonaws.ecs.container-name"] = container.Name
	dockerConfig.Labels["com.amazonaws.ecs.task-definition-family"] = task.Family
	dockerConfig.Labels["com.amazonaws.ecs.task-definition-version"] = task.Version
	dockerConfig.Labels["com.amazonaws.ecs.cluster"] = ""
	client.EXPECT().CreateContainer(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Do(
		func(config *docker.Config, y interface{}, containerName string, z time.Duration) {
			assert.True(t, reflect.DeepEqual(dockerConfig, config),
				"Mismatch in container config; expected: %v, got: %v", dockerConfig, config)
			// sleep5 task contains only one container. Just assign
			// the containerName to createdContainerName
			createdContainerName <- containerName
			containerEventsWG.Add(1)
			go func() {
				eventStream <- createDockerEvent(api.ContainerCreated)
				containerEventsWG.Done()
			}()
		}).Return(DockerContainerMetadata{DockerID: containerID})

	client.EXPECT().StartContainer(containerID, startContainerTimeout).Do(
		func(id string, timeout time.Duration) {
			containerEventsWG.Add(1)
			go func() {
				eventStream <- createDockerEvent(api.ContainerRunning)
				containerEventsWG.Done()
			}()
		}).Return(DockerContainerMetadata{DockerID: containerID})
	assertions()
}

// addTaskToEngine adds a test task to the engine. It waits for a task to reach the
// steady state before returning. Hence, this should not be used for tests, which
// expect container stops to be invoked before a task reaches its steady state
func addTaskToEngine(t *testing.T,
	ctx context.Context,
	taskEngine TaskEngine,
	sleepTask *api.Task,
	mockTime *mock_ttime.MockTime,
	createStartEventsReported sync.WaitGroup) {
	// steadyStateCheckWait is used to force the test to wait until the steady-state check
	// has been invoked at least once
	mockTime.EXPECT().Now().Return(time.Now()).AnyTimes()

	err := taskEngine.Init(ctx)
	assert.NoError(t, err)

	taskEngine.AddTask(sleepTask)
	waitForRunningEvents(t, taskEngine.StateChangeEvents())

	// Wait for all events to be consumed prior to moving it towards stopped; we
	// don't want to race the below with these or we'll end up with the "going
	// backwards in state" stop and we haven't 'expect'd for that

	// Wait for container create and start events to be processed
	createStartEventsReported.Wait()
}

func createDockerEvent(status api.ContainerStatus) DockerContainerChangeEvent {
	meta := DockerContainerMetadata{
		DockerID: containerID,
	}
	return DockerContainerChangeEvent{Status: status, DockerContainerMetadata: meta}
}

// waitForRunningEvents waits for a task to emit 'RUNNING' events for a container
// and the task
func waitForRunningEvents(t *testing.T, stateChangeEvents <-chan statechange.Event) {
	event := <-stateChangeEvents
	assert.Equal(t, event.(api.ContainerStateChange).Status, api.ContainerRunning,
		"Expected container to be RUNNING")

	event = <-stateChangeEvents
	assert.Equal(t, event.(api.TaskStateChange).Status, api.TaskRunning,
		"Expected task to be RUNNING")

	select {
	case <-stateChangeEvents:
		t.Fatal("Should be out of events")
	default:
	}
}

// waitForStopEvents waits for a task to emit 'STOPPED' events for a container
// and the task
func waitForStopEvents(t *testing.T, stateChangeEvents <-chan statechange.Event, verifyExitCode bool) {
	event := <-stateChangeEvents
	if cont := event.(api.ContainerStateChange); cont.Status != api.ContainerStopped {
		t.Fatal("Expected container to stop first")
		if verifyExitCode {
			assert.Equal(t, *cont.ExitCode, 1, "Exit code should be present")
		}
	}
	event = <-stateChangeEvents
	assert.Equal(t, event.(api.TaskStateChange).Status, api.TaskStopped, "Expected task to be STOPPED")

	select {
	case <-stateChangeEvents:
		t.Fatal("Should be out of events")
	default:
	}
}
<|endoftext|>


<commit_before>package agent

import (
	"bytes"
	"io"
	"io/ioutil"
	"net/http"
	"net/http/httptest"
	"os"
	"path/filepath"
	"testing"

	"github.com/buildkite/agent/api"
	"github.com/buildkite/agent/logger"
)

func TestFormUploading(t *testing.T) {
	server := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
		switch req.URL.Path {
		case `/buildkiteartifacts.com`:
			if req.ContentLength <= 0 {
				t.Error("Expected a Content-Length header")
				http.Error(rw, "Bad requests", http.StatusBadRequest)
				return
			}

			err := req.ParseMultipartForm(5 * 1024 * 1024)
			if err != nil {
				t.Error(err)
				http.Error(rw, err.Error(), http.StatusInternalServerError)
				return
			}

			// Check the ${artifact:path} interpolation is working
			path := req.FormValue("path")
			if path != "llamas.txt" {
				t.Errorf("Bad path content %q", path)
				http.Error(rw, "Bad path content", http.StatusInternalServerError)
				return
			}

			file, _, err := req.FormFile("file")
			if err != nil {
				t.Error(err)
				http.Error(rw, err.Error(), http.StatusInternalServerError)
				return
			}
			defer file.Close()

			b := &bytes.Buffer{}
			_, _ = io.Copy(b, file)

			// Check the file is attached correctly
			if b.String() != "llamas" {
				t.Errorf("Bad file content %q", b.String())
				http.Error(rw, "Bad file content", http.StatusInternalServerError)
				return
			}

		default:
			t.Errorf("Unknown path %s %s", req.Method, req.URL.Path)
			http.Error(rw, "Not found", http.StatusNotFound)
		}
	}))
	defer server.Close()

	temp, err := ioutil.TempDir("", "agent")
	if err != nil {
		t.Fatal(err)
	}
	defer os.Remove(temp)

	cwd, err := os.Getwd()
	if err != nil {
		t.Fatal(err)
	}

	tc := []string{temp, cwd}

	runtest := func(wd string) {
		abspath := filepath.Join(wd, "llamas.txt")
		err = ioutil.WriteFile(abspath, []byte("llamas"), 0700)
		defer os.Remove(abspath)

		uploader := NewFormUploader(logger.Discard, FormUploaderConfig{})
		artifact := &api.Artifact{
			ID:           "xxxxx-xxxx-xxxx-xxxx-xxxxxxxxxx",
			Path:         "llamas.txt",
			AbsolutePath: abspath,
			GlobPath:     "llamas.txt",
			ContentType:  "text/plain",
			UploadInstructions: &api.ArtifactUploadInstructions{
				Data: map[string]string{
					"path": "${artifact:path}",
				},
				Action: struct {
					URL       string "json:\"url,omitempty\""
					Method    string "json:\"method\""
					Path      string "json:\"path\""
					FileInput string "json:\"file_input\""
				}{
					URL:       server.URL,
					Method:    "POST",
					Path:      "buildkiteartifacts.com",
					FileInput: "file",
				}},
		}

		if err := uploader.Upload(artifact); err != nil {
			t.Fatal(err)
		}
	}

	for _, wd := range tc {
		runtest(wd)
	}
}
<commit_msg>Verify artifact upload backend path is relative<commit_after>package agent

import (
	"bytes"
	"io"
	"io/ioutil"
	"net/http"
	"net/http/httptest"
	"os"
	"path/filepath"
	"testing"

	"github.com/buildkite/agent/api"
	"github.com/buildkite/agent/logger"
)

func TestFormUploading(t *testing.T) {
	server := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
		switch req.URL.Path {
		case `/buildkiteartifacts.com`:
			if req.ContentLength <= 0 {
				t.Error("Expected a Content-Length header")
				http.Error(rw, "Bad requests", http.StatusBadRequest)
				return
			}

			err := req.ParseMultipartForm(5 * 1024 * 1024)
			if err != nil {
				t.Error(err)
				http.Error(rw, err.Error(), http.StatusInternalServerError)
				return
			}

			// Check the ${artifact:path} interpolation is working
			path := req.FormValue("path")
			if path != "llamas.txt" {
				t.Errorf("Bad path content %q", path)
				http.Error(rw, "Bad path content", http.StatusInternalServerError)
				return
			}

			file, fh, err := req.FormFile("file")
			if err != nil {
				t.Error(err)
				http.Error(rw, err.Error(), http.StatusInternalServerError)
				return
			}
			defer file.Close()

			b := &bytes.Buffer{}
			_, _ = io.Copy(b, file)

			// Check the file is attached correctly
			if b.String() != "llamas" {
				t.Errorf("Bad file content %q", b.String())
				http.Error(rw, "Bad file content", http.StatusInternalServerError)
				return
			}

			if fh.Filename != "llamas.txt" {
				t.Errorf("Bad filename content %q", fh.Filename)
				http.Error(rw, "Bad filename content", http.StatusInternalServerError)
				return
			}

		default:
			t.Errorf("Unknown path %s %s", req.Method, req.URL.Path)
			http.Error(rw, "Not found", http.StatusNotFound)
		}
	}))
	defer server.Close()

	temp, err := ioutil.TempDir("", "agent")
	if err != nil {
		t.Fatal(err)
	}
	defer os.Remove(temp)

	cwd, err := os.Getwd()
	if err != nil {
		t.Fatal(err)
	}

	tc := []string{temp, cwd}

	runtest := func(wd string) {
		abspath := filepath.Join(wd, "llamas.txt")
		err = ioutil.WriteFile(abspath, []byte("llamas"), 0700)
		defer os.Remove(abspath)

		uploader := NewFormUploader(logger.Discard, FormUploaderConfig{})
		artifact := &api.Artifact{
			ID:           "xxxxx-xxxx-xxxx-xxxx-xxxxxxxxxx",
			Path:         "llamas.txt",
			AbsolutePath: abspath,
			GlobPath:     "llamas.txt",
			ContentType:  "text/plain",
			UploadInstructions: &api.ArtifactUploadInstructions{
				Data: map[string]string{
					"path": "${artifact:path}",
				},
				Action: struct {
					URL       string "json:\"url,omitempty\""
					Method    string "json:\"method\""
					Path      string "json:\"path\""
					FileInput string "json:\"file_input\""
				}{
					URL:       server.URL,
					Method:    "POST",
					Path:      "buildkiteartifacts.com",
					FileInput: "file",
				}},
		}

		if err := uploader.Upload(artifact); err != nil {
			t.Fatal(err)
		}
	}

	for _, wd := range tc {
		runtest(wd)
	}
}
<|endoftext|>


<commit_before>package asset

import (
	"github.com/watermint/toolbox/domain/common/model/mo_path"
	"github.com/watermint/toolbox/domain/github/api/gh_conn"
	"github.com/watermint/toolbox/domain/github/service/sv_release"
	"github.com/watermint/toolbox/domain/github/service/sv_release_asset"
	"github.com/watermint/toolbox/infra/control/app_control"
	"github.com/watermint/toolbox/infra/report/rp_model"
	"github.com/watermint/toolbox/infra/ui/app_msg"
	"github.com/watermint/toolbox/infra/util/ut_download"
	"github.com/watermint/toolbox/quality/infra/qt_errors"
	"go.uber.org/zap"
	"os"
	"path/filepath"
)

type Download struct {
	Path        mo_path.FileSystemPath
	Owner       string
	Repository  string
	Release     string
	Peer        gh_conn.ConnGithubRepo
	Downloads   rp_model.TransactionReport
	Downloading app_msg.Message
}

func (z *Download) Preset() {
	z.Downloads.SetModel(
		&UploadFile{},
		nil,
		rp_model.HiddenColumns("result.id"),
	)
}

func (z *Download) Exec(c app_control.Control) error {
	l := c.Log()
	ui := c.UI()

	rel, err := sv_release.New(z.Peer.Context(), z.Owner, z.Repository).Get(z.Release)
	if err != nil {
		return err
	}

	sva := sv_release_asset.New(z.Peer.Context(), z.Owner, z.Repository, rel.Id)
	assets, err := sva.List()
	if err != nil {
		return err
	}

	err = os.MkdirAll(z.Path.Path(), 0755)
	if err != nil {
		return err
	}

	for _, asset := range assets {
		l.Debug("download", zap.Any("asset", asset))
		ui.Progress(z.Downloading.With("File", asset.Name))
		path := filepath.Join(z.Path.Path(), asset.Name)
		if err := ut_download.Download(c.Log(), asset.DownloadUrl, path); err != nil {
			return err
		}
	}
	return nil
}

func (z *Download) Test(c app_control.Control) error {
	return qt_errors.ErrorImplementMe
}
<commit_msg>#338 : services github release asset download<commit_after>package asset

import (
	"github.com/watermint/toolbox/domain/common/model/mo_path"
	"github.com/watermint/toolbox/domain/github/api/gh_conn"
	"github.com/watermint/toolbox/domain/github/service/sv_release"
	"github.com/watermint/toolbox/domain/github/service/sv_release_asset"
	"github.com/watermint/toolbox/infra/control/app_control"
	"github.com/watermint/toolbox/infra/recipe/rc_exec"
	"github.com/watermint/toolbox/infra/recipe/rc_recipe"
	"github.com/watermint/toolbox/infra/report/rp_model"
	"github.com/watermint/toolbox/infra/ui/app_msg"
	"github.com/watermint/toolbox/infra/util/ut_download"
	"github.com/watermint/toolbox/quality/infra/qt_file"
	"go.uber.org/zap"
	"os"
	"path/filepath"
)

type Download struct {
	Path        mo_path.FileSystemPath
	Owner       string
	Repository  string
	Release     string
	Peer        gh_conn.ConnGithubRepo
	Downloads   rp_model.TransactionReport
	Downloading app_msg.Message
}

func (z *Download) Preset() {
	z.Downloads.SetModel(
		&UploadFile{},
		nil,
		rp_model.HiddenColumns("result.id"),
	)
}

func (z *Download) Exec(c app_control.Control) error {
	l := c.Log()
	ui := c.UI()

	rel, err := sv_release.New(z.Peer.Context(), z.Owner, z.Repository).Get(z.Release)
	if err != nil {
		return err
	}

	sva := sv_release_asset.New(z.Peer.Context(), z.Owner, z.Repository, rel.Id)
	assets, err := sva.List()
	if err != nil {
		return err
	}

	err = os.MkdirAll(z.Path.Path(), 0755)
	if err != nil {
		return err
	}

	for _, asset := range assets {
		l.Debug("download", zap.Any("asset", asset))
		ui.Progress(z.Downloading.With("File", asset.Name))
		path := filepath.Join(z.Path.Path(), asset.Name)
		if err := ut_download.Download(c.Log(), asset.DownloadUrl, path); err != nil {
			return err
		}
	}
	return nil
}

func (z *Download) Test(c app_control.Control) error {
	f, err := qt_file.MakeTestFolder("download", false)
	if err != nil {
		return err
	}
	defer func() {
		os.RemoveAll(f)
	}()

	return rc_exec.ExecMock(c, &Download{}, func(r rc_recipe.Recipe) {
		m := r.(*Download)
		m.Owner = "watermint"
		m.Repository = "toolbox"
		m.Release = "65.4.233"
		m.Path = mo_path.NewFileSystemPath(f)
	})
}
<|endoftext|>


<commit_before>package sync

import (
	"fmt"
	"runtime"
	"sync"
	"time"
)

// RWMutex provides locking functions, and an ability to detect and remove
// deadlocks.
type RWMutex struct {
	openLocks        map[int]lockInfo
	openLocksCounter int
	openLocksMutex   sync.Mutex

	callDepth   int
	maxLockTime time.Duration

	mu sync.RWMutex
}

// lockInfo contains information about when and how a lock call was made.
type lockInfo struct {
	// When the lock was called.
	lockTime time.Time

	// Call stack of the caller.
	callingFiles []string
	callingLines []int
}

// New takes a maxLockTime and returns a lock. The lock will never stay locked
// for more than maxLockTime, instead printing an error and unlocking after
// maxLockTime has passed.
func New(maxLockTime time.Duration, callDepth int) *RWMutex {
	rwm := &RWMutex{
		openLocks:   make(map[int]lockInfo),
		maxLockTime: maxLockTime,
		callDepth:   callDepth,
	}

	go rwm.threadedDeadlockFinder()

	return rwm
}

// threadedDeadlockFinder occasionally freezes the mutexes and scans all open mutexes,
// reporting any that have exceeded their time limit.
func (rwm *RWMutex) threadedDeadlockFinder() {
	for {
		rwm.openLocksMutex.Lock()
		for id, info := range rwm.openLocks {
			// Check if the lock has been held for longer than 'maxLockTime'.
			if time.Now().Sub(info.lockTime) > rwm.maxLockTime {
				fmt.Printf("A lock was held for too long, id '%v'. Call stack:\n", id)
				for i := 0; i <= rwm.callDepth; i++ {
					fmt.Printf("\tFile: '%v:%v'\n", info.callingFiles[i], info.callingLines[i])
				}
			}
		}
		rwm.openLocksMutex.Unlock()

		time.Sleep(rwm.maxLockTime)
	}
}

// safeLock is the generic function for doing safe locking. If the read flag is
// set, then a readlock will be used, otherwise a lock will be used.
func (rwm *RWMutex) safeLock(read bool) int {
	// Get the call stack.
	var li lockInfo
	li.lockTime = time.Now()
	li.callingFiles = make([]string, rwm.callDepth+1)
	li.callingLines = make([]int, rwm.callDepth+1)
	for i := 0; i <= rwm.callDepth; i++ {
		_, li.callingFiles[i], li.callingLines[i], _ = runtime.Caller(2 + i)
	}

	// Safely register that a lock has been triggered.
	rwm.openLocksMutex.Lock()
	counter := rwm.openLocksCounter
	rwm.openLocks[counter] = li
	rwm.openLocksCounter++
	rwm.openLocksMutex.Unlock()

	// Lock the mutex.
	if read {
		rwm.mu.RLock()
	} else {
		rwm.mu.Lock()
	}

	return counter
}

// safeUnlock is the generic function for doing safe unlocking. If the lock had
// to be removed because a deadlock was detected, an error is printed.
func (rwm *RWMutex) safeUnlock(read bool, counter int) {
	// Get the call stack.
	callingFiles := make([]string, rwm.callDepth+1)
	callingLines := make([]int, rwm.callDepth+1)
	for i := 0; i <= rwm.callDepth; i++ {
		_, callingFiles[i], callingLines[i], _ = runtime.Caller(2 + i)
	}

	rwm.openLocksMutex.Lock()
	defer rwm.openLocksMutex.Unlock()

	// Check if a deadlock has been detected and fixed manually.
	_, exists := rwm.openLocks[counter]
	if !exists {
		fmt.Printf("A lock was held until deadlock, subsequent call to unlock failed. id '%v'. Call stack:\n", counter)
		for i := 0; i <= rwm.callDepth; i++ {
			fmt.Printf("\tFile: '%v:%v'\n", callingFiles[i], callingLines[i])
		}
		return
	}

	// Remove the lock.
	delete(rwm.openLocks, counter)
	if read {
		rwm.mu.RUnlock()
	} else {
		rwm.mu.Unlock()
	}
}

// RLock will read lock the RWMutex. The return value must be used as input
// when calling RUnlock.
func (rwm *RWMutex) RLock() int {
	return rwm.safeLock(true)
}

// RUnlock will read unlock the RWMutex. The return value of calling RLock must
// be used as input.
func (rwm *RWMutex) RUnlock(counter int) {
	rwm.safeUnlock(true, counter)
}

// Lock will lock the RWMutex. The return value must be used as input when
// calling RUnlock.
func (rwm *RWMutex) Lock() int {
	return rwm.safeLock(false)
}

// Unlock will unlock the RWMutex. The return value of calling Lock must be
// used as input.
func (rwm *RWMutex) Unlock(counter int) {
	rwm.safeUnlock(false, counter)
}
<commit_msg>register a lock as locked after the lock has been obtained<commit_after>package sync

import (
	"fmt"
	"runtime"
	"sync"
	"time"
)

// RWMutex provides locking functions, and an ability to detect and remove
// deadlocks.
type RWMutex struct {
	openLocks        map[int]lockInfo
	openLocksCounter int
	openLocksMutex   sync.Mutex

	callDepth   int
	maxLockTime time.Duration

	mu sync.RWMutex
}

// lockInfo contains information about when and how a lock call was made.
type lockInfo struct {
	// When the lock was called.
	lockTime time.Time

	// Call stack of the caller.
	callingFiles []string
	callingLines []int
}

// New takes a maxLockTime and returns a lock. The lock will never stay locked
// for more than maxLockTime, instead printing an error and unlocking after
// maxLockTime has passed.
func New(maxLockTime time.Duration, callDepth int) *RWMutex {
	rwm := &RWMutex{
		openLocks:   make(map[int]lockInfo),
		maxLockTime: maxLockTime,
		callDepth:   callDepth,
	}

	go rwm.threadedDeadlockFinder()

	return rwm
}

// threadedDeadlockFinder occasionally freezes the mutexes and scans all open mutexes,
// reporting any that have exceeded their time limit.
func (rwm *RWMutex) threadedDeadlockFinder() {
	for {
		rwm.openLocksMutex.Lock()
		for id, info := range rwm.openLocks {
			// Check if the lock has been held for longer than 'maxLockTime'.
			if time.Now().Sub(info.lockTime) > rwm.maxLockTime {
				fmt.Printf("A lock was held for too long, id '%v'. Call stack:\n", id)
				for i := 0; i <= rwm.callDepth; i++ {
					fmt.Printf("\tFile: '%v:%v'\n", info.callingFiles[i], info.callingLines[i])
				}
			}
		}
		rwm.openLocksMutex.Unlock()

		time.Sleep(rwm.maxLockTime)
	}
}

// safeLock is the generic function for doing safe locking. If the read flag is
// set, then a readlock will be used, otherwise a lock will be used.
func (rwm *RWMutex) safeLock(read bool) int {
	// Get the call stack.
	var li lockInfo
	li.callingFiles = make([]string, rwm.callDepth+1)
	li.callingLines = make([]int, rwm.callDepth+1)
	for i := 0; i <= rwm.callDepth; i++ {
		_, li.callingFiles[i], li.callingLines[i], _ = runtime.Caller(2 + i)
	}

	// Lock the mutex.
	if read {
		rwm.mu.RLock()
	} else {
		rwm.mu.Lock()
	}

	// Safely register that a lock has been triggered.
	rwm.openLocksMutex.Lock()
	li.lockTime = time.Now()
	counter := rwm.openLocksCounter
	rwm.openLocks[counter] = li
	rwm.openLocksCounter++
	rwm.openLocksMutex.Unlock()

	return counter
}

// safeUnlock is the generic function for doing safe unlocking. If the lock had
// to be removed because a deadlock was detected, an error is printed.
func (rwm *RWMutex) safeUnlock(read bool, counter int) {
	// Get the call stack.
	callingFiles := make([]string, rwm.callDepth+1)
	callingLines := make([]int, rwm.callDepth+1)
	for i := 0; i <= rwm.callDepth; i++ {
		_, callingFiles[i], callingLines[i], _ = runtime.Caller(2 + i)
	}

	rwm.openLocksMutex.Lock()
	defer rwm.openLocksMutex.Unlock()

	// Check if a deadlock has been detected and fixed manually.
	_, exists := rwm.openLocks[counter]
	if !exists {
		fmt.Printf("A lock was held until deadlock, subsequent call to unlock failed. id '%v'. Call stack:\n", counter)
		for i := 0; i <= rwm.callDepth; i++ {
			fmt.Printf("\tFile: '%v:%v'\n", callingFiles[i], callingLines[i])
		}
		return
	}

	// Remove the lock.
	delete(rwm.openLocks, counter)
	if read {
		rwm.mu.RUnlock()
	} else {
		rwm.mu.Unlock()
	}
}

// RLock will read lock the RWMutex. The return value must be used as input
// when calling RUnlock.
func (rwm *RWMutex) RLock() int {
	return rwm.safeLock(true)
}

// RUnlock will read unlock the RWMutex. The return value of calling RLock must
// be used as input.
func (rwm *RWMutex) RUnlock(counter int) {
	rwm.safeUnlock(true, counter)
}

// Lock will lock the RWMutex. The return value must be used as input when
// calling RUnlock.
func (rwm *RWMutex) Lock() int {
	return rwm.safeLock(false)
}

// Unlock will unlock the RWMutex. The return value of calling Lock must be
// used as input.
func (rwm *RWMutex) Unlock(counter int) {
	rwm.safeUnlock(false, counter)
}
<|endoftext|>