<commit_before>package integration

import (
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	"sync/atomic"
	"testing"

	"github.com/buildkite/bintest"
)

func TestCheckingOutLocalGitProject(t *testing.T) {
	t.Parallel()

	tester, err := NewBootstrapTester()
	if err != nil {
		t.Fatal(err)
	}
	defer tester.Close()

	env := []string{
		"BUILDKITE_GIT_CLONE_FLAGS=-v",
		"BUILDKITE_GIT_CLEAN_FLAGS=-fdq",
	}

	// Actually execute git commands, but with expectations
	git := tester.
		MustMock(t, "git").
		PassthroughToLocalCommand()

	// But assert which ones are called
	git.ExpectAll([][]interface{}{
		{"clone", "-v", "--", tester.Repo.Path, "."},
		{"clean", "-fdq"},
		{"fetch", "-v", "--prune", "origin", "master"},
		{"checkout", "-f", "FETCH_HEAD"},
		{"clean", "-fdq"},
		{"--no-pager", "show", "HEAD", "-s", "--format=fuller", "--no-color"},
		{"--no-pager", "branch", "--contains", "HEAD", "--no-color"},
	})

	// Mock out the meta-data calls to the agent after checkout
	agent := tester.MustMock(t, "buildkite-agent")
	agent.
		Expect("meta-data", "exists", "buildkite:git:commit").
		AndExitWith(1)
	agent.
		Expect("meta-data", "set", "buildkite:git:commit", bintest.MatchAny()).
		AndExitWith(0)
	agent.
		Expect("meta-data", "set", "buildkite:git:branch", bintest.MatchAny()).
		AndExitWith(0)

	tester.RunAndCheck(t, env...)
}

func TestCheckingOutLocalGitProjectWithSubmodules(t *testing.T) {
	t.Parallel()

	tester, err := NewBootstrapTester()
	if err != nil {
		t.Fatal(err)
	}
	defer tester.Close()

	submoduleRepo, err := createTestGitRespository()
	if err != nil {
		t.Fatal(err)
	}
	defer submoduleRepo.Close()

	out, err := tester.Repo.Execute("submodule", "add", submoduleRepo.Path)
	if err != nil {
		t.Fatalf("Adding submodule failed: %s", out)
	}

	out, err = tester.Repo.Execute("commit", "-am", "Add example submodule")
	if err != nil {
		t.Fatalf("Committing submodule failed: %s", out)
	}

	env := []string{
		"BUILDKITE_GIT_CLONE_FLAGS=-v",
		"BUILDKITE_GIT_CLEAN_FLAGS=-fdq",
	}

	// Actually execute git commands, but with expectations
	git := tester.
		MustMock(t, "git").
		PassthroughToLocalCommand()

	// But assert which ones are called
	git.ExpectAll([][]interface{}{
		{"clone", "-v", "--", tester.Repo.Path, "."},
		{"clean", "-fdq"},
		{"submodule", "foreach", "--recursive", "git", "clean", "-fdq"},
		{"fetch", "-v", "--prune", "origin", "master"},
		{"checkout", "-f", "FETCH_HEAD"},
		{"submodule", "sync", "--recursive"},
		{"submodule", "update", "--init", "--recursive", "--force"},
		{"submodule", "foreach", "--recursive", "git", "reset", "--hard"},
		{"clean", "-fdq"},
		{"submodule", "foreach", "--recursive", "git", "clean", "-fdq"},
		{"submodule", "foreach", "--recursive", "git", "ls-remote", "--get-url"},
		{"--no-pager", "show", "HEAD", "-s", "--format=fuller", "--no-color"},
		{"--no-pager", "branch", "--contains", "HEAD", "--no-color"},
	})

	// Mock out the meta-data calls to the agent after checkout
	agent := tester.MustMock(t, "buildkite-agent")
	agent.
		Expect("meta-data", "exists", "buildkite:git:commit").
		AndExitWith(1)
	agent.
		Expect("meta-data", "set", "buildkite:git:commit", bintest.MatchAny()).
		AndExitWith(0)
	agent.
		Expect("meta-data", "set", "buildkite:git:branch", bintest.MatchAny()).
		AndExitWith(0)

	tester.RunAndCheck(t, env...)
}

func TestCheckingOutSetsCorrectGitMetadataAndSendsItToBuildkite(t *testing.T) {
	t.Parallel()

	tester, err := NewBootstrapTester()
	if err != nil {
		t.Fatal(err)
	}
	defer tester.Close()

	agent := tester.MustMock(t, "buildkite-agent")
	agent.
		Expect("meta-data", "exists", "buildkite:git:commit").
		AndExitWith(1)

	agent.
		Expect("meta-data", "set", "buildkite:git:commit",
			bintest.MatchPattern(`^commit`)).
		AndExitWith(0)

	agent.
		Expect("meta-data", "set", "buildkite:git:branch",
			bintest.MatchPattern(`^\* \(HEAD detached at FETCH_HEAD\)`)).
		AndExitWith(0)

	tester.RunAndCheck(t)
}

func TestCheckingOutWithSSHKeyscan(t *testing.T) {
	t.Parallel()

	tester, err := NewBootstrapTester()
	if err != nil {
		t.Fatal(err)
	}
	defer tester.Close()

	tester.MustMock(t, "ssh-keyscan").
		Expect("github.com").
		AndWriteToStdout("github.com ssh-rsa xxx=").
		AndExitWith(0)

	git := tester.MustMock(t, "git")
	git.IgnoreUnexpectedInvocations()

	git.Expect("clone", "-v", "--", "git@github.com:buildkite/agent.git", ".").
		AndExitWith(0)

	env := []string{
		`BUILDKITE_REPO=git@github.com:buildkite/agent.git`,
		`BUILDKITE_SSH_KEYSCAN=true`,
	}

	tester.RunAndCheck(t, env...)
}

func TestCheckingOutWithoutSSHKeyscan(t *testing.T) {
	t.Parallel()

	tester, err := NewBootstrapTester()
	if err != nil {
		t.Fatal(err)
	}
	defer tester.Close()

	tester.MustMock(t, "ssh-keyscan").
		Expect("github.com").
		NotCalled()

	env := []string{
		`BUILDKITE_REPO=https://github.com/buildkite/bash-example.git`,
		`BUILDKITE_SSH_KEYSCAN=false`,
	}

	tester.RunAndCheck(t, env...)
}

func TestCheckingOutWithSSHKeyscanAndUnscannableRepo(t *testing.T) {
	t.Parallel()

	tester, err := NewBootstrapTester()
	if err != nil {
		t.Fatal(err)
	}
	defer tester.Close()

	tester.MustMock(t, "ssh-keyscan").
		Expect("github.com").
		NotCalled()

	git := tester.MustMock(t, "git")
	git.IgnoreUnexpectedInvocations()

	git.Expect("clone", "-v", "--", "https://github.com/buildkite/bash-example.git", ".").
		AndExitWith(0)

	env := []string{
		`BUILDKITE_REPO=https://github.com/buildkite/bash-example.git`,
		`BUILDKITE_SSH_KEYSCAN=true`,
	}

	tester.RunAndCheck(t, env...)
}

func TestCleaningAnExistingCheckout(t *testing.T) {
	t.Parallel()

	tester, err := NewBootstrapTester()
	if err != nil {
		t.Fatal(err)
	}
	defer tester.Close()

	// Create an existing checkout
	out, err := tester.Repo.Execute("clone", "-v", "--", tester.Repo.Path, tester.CheckoutDir())
	if err != nil {
		t.Fatalf("Clone failed with %s", out)
	}
	err = ioutil.WriteFile(filepath.Join(tester.CheckoutDir(), "test.txt"), []byte("llamas"), 0700)
	if err != nil {
		t.Fatalf("Write failed with %s", out)
	}

	// Mock out the meta-data calls to the agent after checkout
	agent := tester.MustMock(t, "buildkite-agent")
	agent.
		Expect("meta-data", "exists", "buildkite:git:commit").
		AndExitWith(0)

	tester.RunAndCheck(t)

	_, err = os.Stat(filepath.Join(tester.CheckoutDir(), "test.txt"))
	if os.IsExist(err) {
		t.Fatalf("test.txt still exitst")
	}
}

func TestForcingACleanCheckout(t *testing.T) {
	tester, err := NewBootstrapTester()
	if err != nil {
		t.Fatal(err)
	}
	defer tester.Close()

	// Mock out the meta-data calls to the agent after checkout
	agent := tester.MustMock(t, "buildkite-agent")
	agent.
		Expect("meta-data", "exists", "buildkite:git:commit").
		AndExitWith(0)

	tester.RunAndCheck(t, "BUILDKITE_CLEAN_CHECKOUT=true")

	if !strings.Contains(tester.Output, "Cleaning pipeline checkout") {
		t.Fatalf("Should have removed checkout dir")
	}
}

func TestCheckoutOnAnExistingRepositoryWithoutAGitFolder(t *testing.T) {
	tester, err := NewBootstrapTester()
	if err != nil {
		t.Fatal(err)
	}
	defer tester.Close()

	// Create an existing checkout
	out, err := tester.Repo.Execute("clone", "-v", "--", tester.Repo.Path, tester.CheckoutDir())
	if err != nil {
		t.Fatalf("Clone failed with %s", out)
	}

	if err = os.RemoveAll(filepath.Join(tester.CheckoutDir(), ".git", "refs")); err != nil {
		t.Fatal(err)
	}

	agent := tester.MustMock(t, "buildkite-agent")
	agent.
		Expect("meta-data", "exists", "buildkite:git:commit").
		AndExitWith(0)

	tester.RunAndCheck(t)
}

func TestCheckoutRetriesOnCleanFailure(t *testing.T) {
	tester, err := NewBootstrapTester()
	if err != nil {
		t.Fatal(err)
	}
	defer tester.Close()

	var cleanCounter int32

	// Mock out all git commands, passing them through to the real thing unless it's a checkout
	git := tester.MustMock(t, "git").PassthroughToLocalCommand().Before(func(i bintest.Invocation) error {
		if i.Args[0] == "clean" {
			c := atomic.AddInt32(&cleanCounter, 1)

			// NB: clean gets run twice per checkout
			if c == 1 {
				return errors.New("Sunspots have caused git clean to fail")
			}
		}
		return nil
	})

	git.Expect().AtLeastOnce().WithAnyArguments()
	tester.RunAndCheck(t)
}

func TestCheckoutRetriesOnCloneFailure(t *testing.T) {
	tester, err := NewBootstrapTester()
	if err != nil {
		t.Fatal(err)
	}
	defer tester.Close()

	var cloneCounter int32

	// Mock out all git commands, passing them through to the real thing unless it's a checkout
	git := tester.MustMock(t, "git").PassthroughToLocalCommand().Before(func(i bintest.Invocation) error {
		if i.Args[0] == "clone" {
			c := atomic.AddInt32(&cloneCounter, 1)
			if c == 1 {
				return errors.New("Sunspots have caused git clone to fail")
			}
		}
		return nil
	})

	git.Expect().AtLeastOnce().WithAnyArguments()
	tester.RunAndCheck(t)
}

func TestCheckoutDoesNotRetryOnHookFailure(t *testing.T) {
	tester, err := NewBootstrapTester()
	if err != nil {
		t.Fatal(err)
	}
	defer tester.Close()

	var checkoutCounter int32

	tester.ExpectGlobalHook("checkout").Once().AndCallFunc(func(c *bintest.Call) {
		counter := atomic.AddInt32(&checkoutCounter, 1)
		fmt.Fprintf(c.Stdout, "Checkout invocation %d\n", counter)
		if counter == 1 {
			fmt.Fprintf(c.Stdout, "Sunspots have caused checkout to fail\n")
			c.Exit(1)
		} else {
			c.Exit(0)
		}
	})

	if err = tester.Run(t); err == nil {
		t.Fatal("Expected the bootstrap to fail")
	}

	tester.CheckMocks(t)
}
<commit_msg>Force committing submodule to make windows happy<commit_after>package integration

import (
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	"sync/atomic"
	"testing"

	"github.com/buildkite/bintest"
)

func TestCheckingOutLocalGitProject(t *testing.T) {
	t.Parallel()

	tester, err := NewBootstrapTester()
	if err != nil {
		t.Fatal(err)
	}
	defer tester.Close()

	env := []string{
		"BUILDKITE_GIT_CLONE_FLAGS=-v",
		"BUILDKITE_GIT_CLEAN_FLAGS=-fdq",
	}

	// Actually execute git commands, but with expectations
	git := tester.
		MustMock(t, "git").
		PassthroughToLocalCommand()

	// But assert which ones are called
	git.ExpectAll([][]interface{}{
		{"clone", "-v", "--", tester.Repo.Path, "."},
		{"clean", "-fdq"},
		{"fetch", "-v", "--prune", "origin", "master"},
		{"checkout", "-f", "FETCH_HEAD"},
		{"clean", "-fdq"},
		{"--no-pager", "show", "HEAD", "-s", "--format=fuller", "--no-color"},
		{"--no-pager", "branch", "--contains", "HEAD", "--no-color"},
	})

	// Mock out the meta-data calls to the agent after checkout
	agent := tester.MustMock(t, "buildkite-agent")
	agent.
		Expect("meta-data", "exists", "buildkite:git:commit").
		AndExitWith(1)
	agent.
		Expect("meta-data", "set", "buildkite:git:commit", bintest.MatchAny()).
		AndExitWith(0)
	agent.
		Expect("meta-data", "set", "buildkite:git:branch", bintest.MatchAny()).
		AndExitWith(0)

	tester.RunAndCheck(t, env...)
}

func TestCheckingOutLocalGitProjectWithSubmodules(t *testing.T) {
	t.Parallel()

	tester, err := NewBootstrapTester()
	if err != nil {
		t.Fatal(err)
	}
	defer tester.Close()

	submoduleRepo, err := createTestGitRespository()
	if err != nil {
		t.Fatal(err)
	}
	defer submoduleRepo.Close()

	out, err := tester.Repo.Execute("submodule", "add", "-f", submoduleRepo.Path)
	if err != nil {
		t.Fatalf("Adding submodule failed: %s", out)
	}

	out, err = tester.Repo.Execute("commit", "-am", "Add example submodule")
	if err != nil {
		t.Fatalf("Committing submodule failed: %s", out)
	}

	env := []string{
		"BUILDKITE_GIT_CLONE_FLAGS=-v",
		"BUILDKITE_GIT_CLEAN_FLAGS=-fdq",
	}

	// Actually execute git commands, but with expectations
	git := tester.
		MustMock(t, "git").
		PassthroughToLocalCommand()

	// But assert which ones are called
	git.ExpectAll([][]interface{}{
		{"clone", "-v", "--", tester.Repo.Path, "."},
		{"clean", "-fdq"},
		{"submodule", "foreach", "--recursive", "git", "clean", "-fdq"},
		{"fetch", "-v", "--prune", "origin", "master"},
		{"checkout", "-f", "FETCH_HEAD"},
		{"submodule", "sync", "--recursive"},
		{"submodule", "update", "--init", "--recursive", "--force"},
		{"submodule", "foreach", "--recursive", "git", "reset", "--hard"},
		{"clean", "-fdq"},
		{"submodule", "foreach", "--recursive", "git", "clean", "-fdq"},
		{"submodule", "foreach", "--recursive", "git", "ls-remote", "--get-url"},
		{"--no-pager", "show", "HEAD", "-s", "--format=fuller", "--no-color"},
		{"--no-pager", "branch", "--contains", "HEAD", "--no-color"},
	})

	// Mock out the meta-data calls to the agent after checkout
	agent := tester.MustMock(t, "buildkite-agent")
	agent.
		Expect("meta-data", "exists", "buildkite:git:commit").
		AndExitWith(1)
	agent.
		Expect("meta-data", "set", "buildkite:git:commit", bintest.MatchAny()).
		AndExitWith(0)
	agent.
		Expect("meta-data", "set", "buildkite:git:branch", bintest.MatchAny()).
		AndExitWith(0)

	tester.RunAndCheck(t, env...)
}

func TestCheckingOutSetsCorrectGitMetadataAndSendsItToBuildkite(t *testing.T) {
	t.Parallel()

	tester, err := NewBootstrapTester()
	if err != nil {
		t.Fatal(err)
	}
	defer tester.Close()

	agent := tester.MustMock(t, "buildkite-agent")
	agent.
		Expect("meta-data", "exists", "buildkite:git:commit").
		AndExitWith(1)

	agent.
		Expect("meta-data", "set", "buildkite:git:commit",
			bintest.MatchPattern(`^commit`)).
		AndExitWith(0)

	agent.
		Expect("meta-data", "set", "buildkite:git:branch",
			bintest.MatchPattern(`^\* \(HEAD detached at FETCH_HEAD\)`)).
		AndExitWith(0)

	tester.RunAndCheck(t)
}

func TestCheckingOutWithSSHKeyscan(t *testing.T) {
	t.Parallel()

	tester, err := NewBootstrapTester()
	if err != nil {
		t.Fatal(err)
	}
	defer tester.Close()

	tester.MustMock(t, "ssh-keyscan").
		Expect("github.com").
		AndWriteToStdout("github.com ssh-rsa xxx=").
		AndExitWith(0)

	git := tester.MustMock(t, "git")
	git.IgnoreUnexpectedInvocations()

	git.Expect("clone", "-v", "--", "git@github.com:buildkite/agent.git", ".").
		AndExitWith(0)

	env := []string{
		`BUILDKITE_REPO=git@github.com:buildkite/agent.git`,
		`BUILDKITE_SSH_KEYSCAN=true`,
	}

	tester.RunAndCheck(t, env...)
}

func TestCheckingOutWithoutSSHKeyscan(t *testing.T) {
	t.Parallel()

	tester, err := NewBootstrapTester()
	if err != nil {
		t.Fatal(err)
	}
	defer tester.Close()

	tester.MustMock(t, "ssh-keyscan").
		Expect("github.com").
		NotCalled()

	env := []string{
		`BUILDKITE_REPO=https://github.com/buildkite/bash-example.git`,
		`BUILDKITE_SSH_KEYSCAN=false`,
	}

	tester.RunAndCheck(t, env...)
}

func TestCheckingOutWithSSHKeyscanAndUnscannableRepo(t *testing.T) {
	t.Parallel()

	tester, err := NewBootstrapTester()
	if err != nil {
		t.Fatal(err)
	}
	defer tester.Close()

	tester.MustMock(t, "ssh-keyscan").
		Expect("github.com").
		NotCalled()

	git := tester.MustMock(t, "git")
	git.IgnoreUnexpectedInvocations()

	git.Expect("clone", "-v", "--", "https://github.com/buildkite/bash-example.git", ".").
		AndExitWith(0)

	env := []string{
		`BUILDKITE_REPO=https://github.com/buildkite/bash-example.git`,
		`BUILDKITE_SSH_KEYSCAN=true`,
	}

	tester.RunAndCheck(t, env...)
}

func TestCleaningAnExistingCheckout(t *testing.T) {
	t.Parallel()

	tester, err := NewBootstrapTester()
	if err != nil {
		t.Fatal(err)
	}
	defer tester.Close()

	// Create an existing checkout
	out, err := tester.Repo.Execute("clone", "-v", "--", tester.Repo.Path, tester.CheckoutDir())
	if err != nil {
		t.Fatalf("Clone failed with %s", out)
	}
	err = ioutil.WriteFile(filepath.Join(tester.CheckoutDir(), "test.txt"), []byte("llamas"), 0700)
	if err != nil {
		t.Fatalf("Write failed with %s", out)
	}

	// Mock out the meta-data calls to the agent after checkout
	agent := tester.MustMock(t, "buildkite-agent")
	agent.
		Expect("meta-data", "exists", "buildkite:git:commit").
		AndExitWith(0)

	tester.RunAndCheck(t)

	_, err = os.Stat(filepath.Join(tester.CheckoutDir(), "test.txt"))
	if os.IsExist(err) {
		t.Fatalf("test.txt still exitst")
	}
}

func TestForcingACleanCheckout(t *testing.T) {
	tester, err := NewBootstrapTester()
	if err != nil {
		t.Fatal(err)
	}
	defer tester.Close()

	// Mock out the meta-data calls to the agent after checkout
	agent := tester.MustMock(t, "buildkite-agent")
	agent.
		Expect("meta-data", "exists", "buildkite:git:commit").
		AndExitWith(0)

	tester.RunAndCheck(t, "BUILDKITE_CLEAN_CHECKOUT=true")

	if !strings.Contains(tester.Output, "Cleaning pipeline checkout") {
		t.Fatalf("Should have removed checkout dir")
	}
}

func TestCheckoutOnAnExistingRepositoryWithoutAGitFolder(t *testing.T) {
	tester, err := NewBootstrapTester()
	if err != nil {
		t.Fatal(err)
	}
	defer tester.Close()

	// Create an existing checkout
	out, err := tester.Repo.Execute("clone", "-v", "--", tester.Repo.Path, tester.CheckoutDir())
	if err != nil {
		t.Fatalf("Clone failed with %s", out)
	}

	if err = os.RemoveAll(filepath.Join(tester.CheckoutDir(), ".git", "refs")); err != nil {
		t.Fatal(err)
	}

	agent := tester.MustMock(t, "buildkite-agent")
	agent.
		Expect("meta-data", "exists", "buildkite:git:commit").
		AndExitWith(0)

	tester.RunAndCheck(t)
}

func TestCheckoutRetriesOnCleanFailure(t *testing.T) {
	tester, err := NewBootstrapTester()
	if err != nil {
		t.Fatal(err)
	}
	defer tester.Close()

	var cleanCounter int32

	// Mock out all git commands, passing them through to the real thing unless it's a checkout
	git := tester.MustMock(t, "git").PassthroughToLocalCommand().Before(func(i bintest.Invocation) error {
		if i.Args[0] == "clean" {
			c := atomic.AddInt32(&cleanCounter, 1)

			// NB: clean gets run twice per checkout
			if c == 1 {
				return errors.New("Sunspots have caused git clean to fail")
			}
		}
		return nil
	})

	git.Expect().AtLeastOnce().WithAnyArguments()
	tester.RunAndCheck(t)
}

func TestCheckoutRetriesOnCloneFailure(t *testing.T) {
	tester, err := NewBootstrapTester()
	if err != nil {
		t.Fatal(err)
	}
	defer tester.Close()

	var cloneCounter int32

	// Mock out all git commands, passing them through to the real thing unless it's a checkout
	git := tester.MustMock(t, "git").PassthroughToLocalCommand().Before(func(i bintest.Invocation) error {
		if i.Args[0] == "clone" {
			c := atomic.AddInt32(&cloneCounter, 1)
			if c == 1 {
				return errors.New("Sunspots have caused git clone to fail")
			}
		}
		return nil
	})

	git.Expect().AtLeastOnce().WithAnyArguments()
	tester.RunAndCheck(t)
}

func TestCheckoutDoesNotRetryOnHookFailure(t *testing.T) {
	tester, err := NewBootstrapTester()
	if err != nil {
		t.Fatal(err)
	}
	defer tester.Close()

	var checkoutCounter int32

	tester.ExpectGlobalHook("checkout").Once().AndCallFunc(func(c *bintest.Call) {
		counter := atomic.AddInt32(&checkoutCounter, 1)
		fmt.Fprintf(c.Stdout, "Checkout invocation %d\n", counter)
		if counter == 1 {
			fmt.Fprintf(c.Stdout, "Sunspots have caused checkout to fail\n")
			c.Exit(1)
		} else {
			c.Exit(0)
		}
	})

	if err = tester.Run(t); err == nil {
		t.Fatal("Expected the bootstrap to fail")
	}

	tester.CheckMocks(t)
}
<|endoftext|>


<commit_before>package integration

import (
	"code.cloudfoundry.org/cli/integration/helpers"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	. "github.com/onsi/gomega/gbytes"
	. "github.com/onsi/gomega/gexec"
)

var _ = Describe("proxy", func() {
	var proxyURL string
	BeforeEach(func() {
		proxyURL = "127.0.0.1:9999"
	})

	Context("V2 Legacy", func() {
		It("handles a proxy", func() {
			session := helpers.CFWithEnv(map[string]string{"http_proxy": proxyURL}, "api", apiURL)
			Eventually(session).Should(Say("Error performing request: Get %s/v2/info: http: error connecting to proxy http://%s: dial tcp %s: getsockopt: connection refused", apiURL, proxyURL, proxyURL))
			Eventually(session).Should(Exit(1))
		})
	})

	Context("V3", func() {
		It("handles a proxy", func() {
			session := helpers.CFWithEnv(map[string]string{"http_proxy": proxyURL}, "run-task", "app", "echo")
			Eventually(session.Err).Should(Say("Get %s: http: error connecting to proxy http://%s: dial tcp %s: getsockopt: connection refused", apiURL, proxyURL, proxyURL))
			Eventually(session).Should(Exit(1))
		})
	})
})
<commit_msg>optional protocal cuz of windows<commit_after>package integration

import (
	"code.cloudfoundry.org/cli/integration/helpers"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	. "github.com/onsi/gomega/gbytes"
	. "github.com/onsi/gomega/gexec"
)

var _ = Describe("proxy", func() {
	var proxyURL, apiRegexp string

	BeforeEach(func() {
		proxyURL = "127.0.0.1:9999"
		apiRegexp = "(?:https:\\/\\/)*" + apiURL
	})

	Context("V2 Legacy", func() {
		It("handles a proxy", func() {
			session := helpers.CFWithEnv(map[string]string{"http_proxy": proxyURL}, "api", apiURL)
			Eventually(session).Should(Say("Error performing request: Get %s/v2/info: http: error connecting to proxy http://%s: dial tcp %s: getsockopt: connection refused", apiRegexp, proxyURL, proxyURL))
			Eventually(session).Should(Exit(1))
		})
	})

	Context("V3", func() {
		It("handles a proxy", func() {
			session := helpers.CFWithEnv(map[string]string{"http_proxy": proxyURL}, "run-task", "app", "echo")
			Eventually(session.Err).Should(Say("Get %s: http: error connecting to proxy http://%s: dial tcp %s: getsockopt: connection refused", apiRegexp, proxyURL, proxyURL))
			Eventually(session).Should(Exit(1))
		})
	})
})
<|endoftext|>


<commit_before>//go:build darwin || freebsd || linux
// +build darwin freebsd linux

package fuse

import (
	"context"
	"fmt"
	"path"
	"sort"
	"strings"
	"sync"
	"time"

	"github.com/restic/restic/internal/debug"
	"github.com/restic/restic/internal/restic"
)

type MetaDirData struct {
	// set if this is a symlink or a snapshot mount point
	linkTarget string
	snapshot   *restic.Snapshot
	// names is set if this is a pseudo directory
	names map[string]*MetaDirData
}

// SnapshotsDirStructure contains the directory structure for snapshots.
// It uses a paths and time template to generate a map of pathnames
// pointing to the actual snapshots. For templates that end with a time,
// also "latest" links are generated.
type SnapshotsDirStructure struct {
	root          *Root
	pathTemplates []string
	timeTemplate  string

	mutex sync.Mutex
	// "" is the root path, subdirectory paths are assembled as parent+"/"+childFn
	// thus all subdirectories are prefixed with a slash as the root is ""
	// that way we don't need path processing special cases when using the entries tree
	entries map[string]*MetaDirData

	snCount   int
	lastCheck time.Time
}

// NewSnapshotsDirStructure returns a new directory structure for snapshots.
func NewSnapshotsDirStructure(root *Root, pathTemplates []string, timeTemplate string) *SnapshotsDirStructure {
	return &SnapshotsDirStructure{
		root:          root,
		pathTemplates: pathTemplates,
		timeTemplate:  timeTemplate,
		snCount:       -1,
	}
}

// pathsFromSn generates the paths from pathTemplate and timeTemplate
// where the variables are replaced by the snapshot data.
// The time is given as suffix if the pathTemplate ends with "%T".
func pathsFromSn(pathTemplate string, timeTemplate string, sn *restic.Snapshot) (paths []string, timeSuffix string) {
	timeformat := sn.Time.Format(timeTemplate)

	inVerb := false
	writeTime := false
	out := make([]strings.Builder, 1)
	for _, c := range pathTemplate {
		if writeTime {
			for i := range out {
				out[i].WriteString(timeformat)
			}
			writeTime = false
		}

		if !inVerb {
			if c == '%' {
				inVerb = true
			} else {
				for i := range out {
					out[i].WriteRune(c)
				}
			}
			continue
		}

		var repl string
		inVerb = false
		switch c {
		case 'T':
			// lazy write; time might be returned as suffix
			writeTime = true
			continue

		case 't':
			if len(sn.Tags) == 0 {
				return nil, ""
			}
			if len(sn.Tags) != 1 {
				// needs special treatment: Rebuild the string builders
				newout := make([]strings.Builder, len(out)*len(sn.Tags))
				for i, tag := range sn.Tags {
					tag = filenameFromTag(tag)
					for j := range out {
						newout[i*len(out)+j].WriteString(out[j].String() + tag)
					}
				}
				out = newout
				continue
			}
			repl = sn.Tags[0]

		case 'i':
			repl = sn.ID().Str()

		case 'I':
			repl = sn.ID().String()

		case 'u':
			repl = sn.Username

		case 'h':
			repl = sn.Hostname

		default:
			repl = string(c)
		}

		// write replacement string to all string builders
		for i := range out {
			out[i].WriteString(repl)
		}
	}

	for i := range out {
		paths = append(paths, out[i].String())
	}

	if writeTime {
		timeSuffix = timeformat
	}

	return paths, timeSuffix
}

// Some tags are problematic when used as filenames:
//
//	""
//	".", ".."
//	anything containing '/'
//
// Replace all special character by underscores "_", an empty tag is also represented as a underscore.
func filenameFromTag(tag string) string {
	switch tag {
	case "", ".":
		return "_"
	case "..":
		return "__"
	}

	return strings.ReplaceAll(tag, "/", "_")
}

// determine static path prefix
func staticPrefix(pathTemplate string) (prefix string) {
	inVerb := false
	patternStart := -1
outer:
	for i, c := range pathTemplate {
		if !inVerb {
			if c == '%' {
				inVerb = true
			}
			continue
		}
		inVerb = false
		switch c {
		case 'i', 'I', 'u', 'h', 't', 'T':
			patternStart = i
			break outer
		}
	}
	if patternStart < 0 {
		// ignore patterns without template variable
		return ""
	}

	p := pathTemplate[:patternStart]
	idx := strings.LastIndex(p, "/")
	if idx < 0 {
		return ""
	}
	return p[:idx]
}

// uniqueName returns a unique name to be used for prefix+name.
// It appends -number to make the name unique.
func uniqueName(entries map[string]*MetaDirData, prefix, name string) string {
	newname := name
	for i := 1; ; i++ {
		if _, ok := entries[prefix+newname]; !ok {
			break
		}
		newname = fmt.Sprintf("%s-%d", name, i)
	}
	return newname
}

// makeDirs inserts all paths generated from pathTemplates and
// TimeTemplate for all given snapshots into d.names.
// Also adds d.latest links if "%T" is at end of a path template
func (d *SnapshotsDirStructure) makeDirs(snapshots restic.Snapshots) {
	entries := make(map[string]*MetaDirData)

	type mountData struct {
		sn         *restic.Snapshot
		linkTarget string // if linkTarget!= "", this is a symlink
		childFn    string
		child      *MetaDirData
	}

	// recursively build tree structure
	var mount func(path string, data mountData)
	mount = func(path string, data mountData) {
		e := entries[path]
		if e == nil {
			e = &MetaDirData{}
		}
		if data.sn != nil {
			e.snapshot = data.sn
			e.linkTarget = data.linkTarget
		} else {
			// intermediate directory, register as a child directory
			if e.names == nil {
				e.names = make(map[string]*MetaDirData)
			}
			if data.child != nil {
				e.names[data.childFn] = data.child
			}
		}
		entries[path] = e

		slashIdx := strings.LastIndex(path, "/")
		if slashIdx >= 0 {
			// add to parent dir, but without snapshot
			mount(path[:slashIdx], mountData{childFn: path[slashIdx+1:], child: e})
		}
	}

	// root directory
	mount("", mountData{})

	// insert pure directories; needed to get empty structure even if there
	// are no snapshots in these dirs
	for _, p := range d.pathTemplates {
		p = staticPrefix(p)
		if p != "" {
			mount(path.Clean("/"+p), mountData{})
		}
	}

	latestTime := make(map[string]time.Time)
	for _, sn := range snapshots {
		for _, templ := range d.pathTemplates {
			paths, timeSuffix := pathsFromSn(templ, d.timeTemplate, sn)
			for _, p := range paths {
				if p != "" {
					p = "/" + p
				}
				suffix := uniqueName(entries, p, timeSuffix)
				mount(path.Clean(p+suffix), mountData{sn: sn})
				if timeSuffix != "" {
					lt, ok := latestTime[p]
					if !ok || !sn.Time.Before(lt) {
						debug.Log("link (update) %v -> %v\n", p, suffix)
						// inject symlink
						mount(path.Clean(p+"/latest"), mountData{sn: sn, linkTarget: suffix})
						latestTime[p] = sn.Time
					}
				}
			}
		}
	}

	d.entries = entries
}

const minSnapshotsReloadTime = 60 * time.Second

// update snapshots if repository has changed
func (d *SnapshotsDirStructure) updateSnapshots(ctx context.Context) error {
	d.mutex.Lock()
	defer d.mutex.Unlock()
	if time.Since(d.lastCheck) < minSnapshotsReloadTime {
		return nil
	}

	var snapshots restic.Snapshots
	err := restic.FindFilteredSnapshots(ctx, d.root.repo.Backend(), d.root.repo, d.root.cfg.Hosts, d.root.cfg.Tags, d.root.cfg.Paths, nil, func(id string, sn *restic.Snapshot, err error) error {
		if sn != nil {
			snapshots = append(snapshots, sn)
		}
		return nil
	})
	if err != nil {
		return err
	}
	// sort snapshots ascending by time (default order is descending)
	sort.Sort(sort.Reverse(snapshots))

	if d.snCount == len(snapshots) {
		d.lastCheck = time.Now()
		return nil
	}

	err = d.root.repo.LoadIndex(ctx)
	if err != nil {
		return err
	}

	d.lastCheck = time.Now()
	d.snCount = len(snapshots)
	d.makeDirs(snapshots)
	return nil
}

func (d *SnapshotsDirStructure) UpdatePrefix(ctx context.Context, prefix string) (*MetaDirData, error) {
	err := d.updateSnapshots(ctx)
	if err != nil {
		return nil, err
	}

	d.mutex.Lock()
	defer d.mutex.Unlock()
	return d.entries[prefix], nil
}
<commit_msg>fuse: Better check for whether snapshots changed<commit_after>//go:build darwin || freebsd || linux
// +build darwin freebsd linux

package fuse

import (
	"bytes"
	"context"
	"fmt"
	"path"
	"sort"
	"strings"
	"sync"
	"time"

	"github.com/restic/restic/internal/debug"
	"github.com/restic/restic/internal/restic"

	"github.com/minio/sha256-simd"
)

type MetaDirData struct {
	// set if this is a symlink or a snapshot mount point
	linkTarget string
	snapshot   *restic.Snapshot
	// names is set if this is a pseudo directory
	names map[string]*MetaDirData
}

// SnapshotsDirStructure contains the directory structure for snapshots.
// It uses a paths and time template to generate a map of pathnames
// pointing to the actual snapshots. For templates that end with a time,
// also "latest" links are generated.
type SnapshotsDirStructure struct {
	root          *Root
	pathTemplates []string
	timeTemplate  string

	mutex sync.Mutex
	// "" is the root path, subdirectory paths are assembled as parent+"/"+childFn
	// thus all subdirectories are prefixed with a slash as the root is ""
	// that way we don't need path processing special cases when using the entries tree
	entries map[string]*MetaDirData

	hash      [sha256.Size]byte // Hash at last check.
	lastCheck time.Time
}

// NewSnapshotsDirStructure returns a new directory structure for snapshots.
func NewSnapshotsDirStructure(root *Root, pathTemplates []string, timeTemplate string) *SnapshotsDirStructure {
	return &SnapshotsDirStructure{
		root:          root,
		pathTemplates: pathTemplates,
		timeTemplate:  timeTemplate,
	}
}

// pathsFromSn generates the paths from pathTemplate and timeTemplate
// where the variables are replaced by the snapshot data.
// The time is given as suffix if the pathTemplate ends with "%T".
func pathsFromSn(pathTemplate string, timeTemplate string, sn *restic.Snapshot) (paths []string, timeSuffix string) {
	timeformat := sn.Time.Format(timeTemplate)

	inVerb := false
	writeTime := false
	out := make([]strings.Builder, 1)
	for _, c := range pathTemplate {
		if writeTime {
			for i := range out {
				out[i].WriteString(timeformat)
			}
			writeTime = false
		}

		if !inVerb {
			if c == '%' {
				inVerb = true
			} else {
				for i := range out {
					out[i].WriteRune(c)
				}
			}
			continue
		}

		var repl string
		inVerb = false
		switch c {
		case 'T':
			// lazy write; time might be returned as suffix
			writeTime = true
			continue

		case 't':
			if len(sn.Tags) == 0 {
				return nil, ""
			}
			if len(sn.Tags) != 1 {
				// needs special treatment: Rebuild the string builders
				newout := make([]strings.Builder, len(out)*len(sn.Tags))
				for i, tag := range sn.Tags {
					tag = filenameFromTag(tag)
					for j := range out {
						newout[i*len(out)+j].WriteString(out[j].String() + tag)
					}
				}
				out = newout
				continue
			}
			repl = sn.Tags[0]

		case 'i':
			repl = sn.ID().Str()

		case 'I':
			repl = sn.ID().String()

		case 'u':
			repl = sn.Username

		case 'h':
			repl = sn.Hostname

		default:
			repl = string(c)
		}

		// write replacement string to all string builders
		for i := range out {
			out[i].WriteString(repl)
		}
	}

	for i := range out {
		paths = append(paths, out[i].String())
	}

	if writeTime {
		timeSuffix = timeformat
	}

	return paths, timeSuffix
}

// Some tags are problematic when used as filenames:
//
//	""
//	".", ".."
//	anything containing '/'
//
// Replace all special character by underscores "_", an empty tag is also represented as a underscore.
func filenameFromTag(tag string) string {
	switch tag {
	case "", ".":
		return "_"
	case "..":
		return "__"
	}

	return strings.ReplaceAll(tag, "/", "_")
}

// determine static path prefix
func staticPrefix(pathTemplate string) (prefix string) {
	inVerb := false
	patternStart := -1
outer:
	for i, c := range pathTemplate {
		if !inVerb {
			if c == '%' {
				inVerb = true
			}
			continue
		}
		inVerb = false
		switch c {
		case 'i', 'I', 'u', 'h', 't', 'T':
			patternStart = i
			break outer
		}
	}
	if patternStart < 0 {
		// ignore patterns without template variable
		return ""
	}

	p := pathTemplate[:patternStart]
	idx := strings.LastIndex(p, "/")
	if idx < 0 {
		return ""
	}
	return p[:idx]
}

// uniqueName returns a unique name to be used for prefix+name.
// It appends -number to make the name unique.
func uniqueName(entries map[string]*MetaDirData, prefix, name string) string {
	newname := name
	for i := 1; ; i++ {
		if _, ok := entries[prefix+newname]; !ok {
			break
		}
		newname = fmt.Sprintf("%s-%d", name, i)
	}
	return newname
}

// makeDirs inserts all paths generated from pathTemplates and
// TimeTemplate for all given snapshots into d.names.
// Also adds d.latest links if "%T" is at end of a path template
func (d *SnapshotsDirStructure) makeDirs(snapshots restic.Snapshots) {
	entries := make(map[string]*MetaDirData)

	type mountData struct {
		sn         *restic.Snapshot
		linkTarget string // if linkTarget!= "", this is a symlink
		childFn    string
		child      *MetaDirData
	}

	// recursively build tree structure
	var mount func(path string, data mountData)
	mount = func(path string, data mountData) {
		e := entries[path]
		if e == nil {
			e = &MetaDirData{}
		}
		if data.sn != nil {
			e.snapshot = data.sn
			e.linkTarget = data.linkTarget
		} else {
			// intermediate directory, register as a child directory
			if e.names == nil {
				e.names = make(map[string]*MetaDirData)
			}
			if data.child != nil {
				e.names[data.childFn] = data.child
			}
		}
		entries[path] = e

		slashIdx := strings.LastIndex(path, "/")
		if slashIdx >= 0 {
			// add to parent dir, but without snapshot
			mount(path[:slashIdx], mountData{childFn: path[slashIdx+1:], child: e})
		}
	}

	// root directory
	mount("", mountData{})

	// insert pure directories; needed to get empty structure even if there
	// are no snapshots in these dirs
	for _, p := range d.pathTemplates {
		p = staticPrefix(p)
		if p != "" {
			mount(path.Clean("/"+p), mountData{})
		}
	}

	latestTime := make(map[string]time.Time)
	for _, sn := range snapshots {
		for _, templ := range d.pathTemplates {
			paths, timeSuffix := pathsFromSn(templ, d.timeTemplate, sn)
			for _, p := range paths {
				if p != "" {
					p = "/" + p
				}
				suffix := uniqueName(entries, p, timeSuffix)
				mount(path.Clean(p+suffix), mountData{sn: sn})
				if timeSuffix != "" {
					lt, ok := latestTime[p]
					if !ok || !sn.Time.Before(lt) {
						debug.Log("link (update) %v -> %v\n", p, suffix)
						// inject symlink
						mount(path.Clean(p+"/latest"), mountData{sn: sn, linkTarget: suffix})
						latestTime[p] = sn.Time
					}
				}
			}
		}
	}

	d.entries = entries
}

const minSnapshotsReloadTime = 60 * time.Second

// update snapshots if repository has changed
func (d *SnapshotsDirStructure) updateSnapshots(ctx context.Context) error {
	d.mutex.Lock()
	defer d.mutex.Unlock()
	if time.Since(d.lastCheck) < minSnapshotsReloadTime {
		return nil
	}

	var snapshots restic.Snapshots
	err := restic.FindFilteredSnapshots(ctx, d.root.repo.Backend(), d.root.repo, d.root.cfg.Hosts, d.root.cfg.Tags, d.root.cfg.Paths, nil, func(id string, sn *restic.Snapshot, err error) error {
		if sn != nil {
			snapshots = append(snapshots, sn)
		}
		return nil
	})
	if err != nil {
		return err
	}

	// Sort snapshots ascending by time, using the id to break ties.
	// This needs to be done before hashing.
	sort.Slice(snapshots, func(i, j int) bool {
		si, sj := snapshots[i], snapshots[j]
		if si.Time.Equal(sj.Time) {
			return bytes.Compare(si.ID()[:], sj.ID()[:]) < 0
		}
		return si.Time.Before(sj.Time)
	})

	// We update the snapshots when the hash of their id's changes.
	h := sha256.New()
	for _, sn := range snapshots {
		h.Write(sn.ID()[:])
	}
	var hash [sha256.Size]byte
	h.Sum(hash[:0])

	if d.hash == hash {
		d.lastCheck = time.Now()
		return nil
	}

	err = d.root.repo.LoadIndex(ctx)
	if err != nil {
		return err
	}

	d.lastCheck = time.Now()
	d.hash = hash
	d.makeDirs(snapshots)
	return nil
}

func (d *SnapshotsDirStructure) UpdatePrefix(ctx context.Context, prefix string) (*MetaDirData, error) {
	err := d.updateSnapshots(ctx)
	if err != nil {
		return nil, err
	}

	d.mutex.Lock()
	defer d.mutex.Unlock()
	return d.entries[prefix], nil
}
<|endoftext|>


<commit_before>package s2k

import (
	"fmt"

	"github.com/spiegel-im-spiegel/gpgpdump/internal/options"
	"github.com/spiegel-im-spiegel/gpgpdump/internal/parse/values"
)

//const
const EXPBIAS = uint32(6)

// S2K - information of public key algorithm
type S2K struct {
	*options.Options
	buf  []byte
	left int
}

//New returns new Pubkey
func New(opt *options.Options, buf []byte) *S2K {
	return &S2K{Options: opt, buf: buf}
}

//Left returns left bytes
func (s S2K) Left() int {
	return s.left
}

//Parse parsing S2K information
func (s *S2K) Parse(indent values.Indent) values.Content {
	content := values.NewContent()
	s2kAlg := values.S2KAlg(s.buf[0])
	s.left = len(s.buf) - 1

	content = append(content, indent.Fill(s2kAlg.String()))
	switch s2kAlg {
	case 0:
		content = append(content, (indent + 1).Fill(values.HashAlg(s.buf[1]).String()))
		s.left--
	case 1:
		content = append(content, (indent + 1).Fill(values.HashAlg(s.buf[1]).String()))
		content = append(content, (indent + 1).Fill(fmt.Sprintf("Salt - %s", values.DumpByte(s.buf[2:10]))))
		s.left -= 9
	case 3:
		content = append(content, (indent + 1).Fill(values.HashAlg(s.buf[1]).String()))
		content = append(content, (indent + 1).Fill(fmt.Sprintf("Salt - %s", values.DumpByte(s.buf[2:10]))))
		c := uint32(s.buf[10])
		count := (uint32(16) + (c & 15)) << ((c >> 4) + EXPBIAS)
		content = append(content, (indent + 1).Fill(fmt.Sprintf("Salt - %d", count)))
		s.left -= 10
	}
	return content
}
<commit_msg>add private S2K type<commit_after>package s2k

import (
	"fmt"

	"github.com/spiegel-im-spiegel/gpgpdump/internal/options"
	"github.com/spiegel-im-spiegel/gpgpdump/internal/parse/values"
)

//const
const EXPBIAS = uint32(6)

// S2K - information of public key algorithm
type S2K struct {
	*options.Options
	buf  []byte
	left int
}

//New returns new Pubkey
func New(opt *options.Options, buf []byte) *S2K {
	return &S2K{Options: opt, buf: buf}
}

//Left returns left bytes
func (s S2K) Left() int {
	return s.left
}

//Parse parsing S2K information
func (s *S2K) Parse(indent values.Indent) values.Content {
	content := values.NewContent()
	s2kAlg := values.S2KAlg(s.buf[0])
	s.left = len(s.buf) - 1

	content = append(content, indent.Fill(s2kAlg.String()))
	switch s2kAlg {
	case 0:
		content = append(content, (indent + 1).Fill(values.HashAlg(s.buf[1]).String()))
		s.left--
	case 1:
		content = append(content, (indent + 1).Fill(values.HashAlg(s.buf[1]).String()))
		content = append(content, (indent + 1).Fill(fmt.Sprintf("Salt - %s", values.DumpByte(s.buf[2:10]))))
		s.left -= 9
	case 3:
		content = append(content, (indent + 1).Fill(values.HashAlg(s.buf[1]).String()))
		content = append(content, (indent + 1).Fill(fmt.Sprintf("Salt - %s", values.DumpByte(s.buf[2:10]))))
		c := uint32(s.buf[10])
		count := (uint32(16) + (c & 15)) << ((c >> 4) + EXPBIAS)
		content = append(content, (indent + 1).Fill(fmt.Sprintf("Count - %d (coded count 0x%02x)", count, c)))
		s.left -= 10
	case 101:
		mrk := string(s.buf[1:5])
		switch mrk {
		case "GNU1":
			content = append(content, (indent + 1).Fill("GnuPG gnu-dummy (s2k 1001)"))
		case "GNU2":
			content = append(content, (indent + 1).Fill("GnuPG gnu-divert-to-card (s2k 1001)"))
			l := s.buf[5]
			content = append(content, (indent + 2).Fill(fmt.Sprintf("Serial Number: %s", values.DumpByte(s.buf[6:6+1]))))
			s.left -= 5 + int(l)
		default:
		}
	}
	return content
}
<|endoftext|>


<commit_before>// Copyright © 2017 Circonus, Inc. <support@circonus.com>
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//

// thanks to https://github.com/influxdata/telegraf/blob/release-1.2/plugins/inputs/prometheus/parser.go
// and https://github.com/prometheus/common/tree/master/expfmt

package promrecv

import (
	"fmt"
	"io"
	"math"
	"regexp"
	"sync"

	"github.com/circonus-labs/circonus-agent/internal/config"
	"github.com/circonus-labs/circonus-agent/internal/release"
	"github.com/circonus-labs/circonus-agent/internal/tags"
	cgm "github.com/circonus-labs/circonus-gometrics/v3"
	"github.com/pkg/errors"
	dto "github.com/prometheus/client_model/go"
	"github.com/prometheus/common/expfmt"
	"github.com/rs/zerolog"
	"github.com/rs/zerolog/log"
	"github.com/spf13/viper"
)

var (
	id            string
	baseTags      []string
	nameCleanerRx *regexp.Regexp
	metricsmu     sync.Mutex
	metrics       *cgm.CirconusMetrics
	// parseRx             *regexp.Regexp
	logger = log.With().Str("pkg", "promrecv").Logger()
)

// logshim is used to satisfy apiclient Logger interface (avoiding ptr receiver issue)
type logshim struct {
	logh zerolog.Logger
}

func (l logshim) Printf(fmt string, v ...interface{}) {
	l.logh.Printf(fmt, v...)
}

func initCGM() error {
	metricsmu.Lock()
	defer metricsmu.Unlock()

	if metrics != nil {
		return nil
	}

	cmc := &cgm.Config{
		Debug: viper.GetBool(config.KeyDebugCGM),
		Log:   logshim{logh: log.With().Str("pkg", "cgm.promrecv").Logger()},
	}
	// put cgm into manual mode (no interval, no api key, invalid submission url)
	cmc.Interval = "0"                            // disable automatic flush
	cmc.CheckManager.Check.SubmissionURL = "none" // disable check management (create/update)

	hm, err := cgm.NewCirconusMetrics(cmc)
	if err != nil {
		return errors.Wrap(err, "prom receiver cgm")
	}

	metrics = hm

	// inintialize any options for the receiver
	id = "prom"                                      // metric name (group) prefix to be used
	nameCleanerRx = regexp.MustCompile("[\r\n\"'`]") // used to strip unwanted characters

	baseTags = tags.GetBaseTags()
	baseTags = append(baseTags, []string{
		"source:" + release.NAME,
		"collector:promrecv",
	}...)

	return nil
}

// Flush returns current metrics
func Flush() *cgm.Metrics {
	_ = initCGM()

	return metrics.FlushMetrics()
}

// Parse handles incoming PUT/POST requests
func Parse(data io.Reader) error {
	if err := initCGM(); err != nil {
		return err
	}

	var parser expfmt.TextParser

	// formats supported from https://prometheus.io/docs/instrumenting/exposition_formats/

	metricFamilies, err := parser.TextToMetricFamilies(data)
	if err != nil {
		return err
	}

	for mn, mf := range metricFamilies {
		for _, m := range mf.Metric {
			metricName := nameCleanerRx.ReplaceAllString(mn, "")
			tags := getLabels(m)
			switch {
			case mf.GetType() == dto.MetricType_SUMMARY:
				metrics.Gauge(metricName+"_count", float64(m.GetSummary().GetSampleCount()))
				metrics.Gauge(metricName+"_sum", float64(m.GetSummary().GetSampleSum()))
				for qn, qv := range getQuantiles(m) {
					metrics.GaugeWithTags(metricName+"_"+qn, tags, qv)
				}
			case mf.GetType() == dto.MetricType_HISTOGRAM:
				metrics.Gauge(metricName+"_count", float64(m.GetHistogram().GetSampleCount()))
				metrics.Gauge(metricName+"_sum", float64(m.GetHistogram().GetSampleSum()))
				for bn, bv := range getBuckets(m) {
					metrics.GaugeWithTags(metricName+"_"+bn, tags, bv)
				}
			default:
				switch {
				case m.Gauge != nil:
					if m.GetGauge().Value != nil {
						metrics.GaugeWithTags(metricName, tags, *m.GetGauge().Value)
					}
				case m.Counter != nil:
					if m.GetCounter().Value != nil {
						metrics.GaugeWithTags(metricName, tags, *m.GetCounter().Value)
					}
				case m.Untyped != nil:
					if m.GetUntyped().Value != nil {
						if *m.GetUntyped().Value == math.Inf(+1) {
							logger.Warn().Str("metric", metricName).Str("type", mf.GetType().String()).Str("value", (*m).GetUntyped().String()).Msg("cannot coerce +Inf to uint64")
							continue
						}
						metrics.GaugeWithTags(metricName, tags, *m.GetUntyped().Value)
					}
				}
			}
		}
	}

	return nil
}

func getLabels(m *dto.Metric) tags.Tags {
	labels := make([]string, 0, len(m.Label))
	for _, label := range m.Label {
		if label.Name != nil && label.Value != nil {
			ln := nameCleanerRx.ReplaceAllString(*label.Name, "")
			lv := nameCleanerRx.ReplaceAllString(*label.Value, "")
			labels = append(labels, ln+tags.Delimiter+lv) // stream tags take form cat:val
		}
	}

	if len(labels) > 0 {
		tagList := make([]string, 0, len(baseTags)+len(labels))
		tagList = append(tagList, baseTags...)
		tagList = append(tagList, labels...)
		tags := tags.FromList(tagList)
		return tags
	}

	return tags.FromList(baseTags)
}

func getQuantiles(m *dto.Metric) map[string]float64 {
	ret := make(map[string]float64)
	for _, q := range m.GetSummary().Quantile {
		if q.Value != nil && !math.IsNaN(*q.Value) {
			ret[fmt.Sprint(*q.Quantile)] = *q.Value
		}
	}
	return ret
}

func getBuckets(m *dto.Metric) map[string]uint64 {
	ret := make(map[string]uint64)
	for _, b := range m.GetHistogram().Bucket {
		if b.CumulativeCount != nil {
			ret[fmt.Sprint(*b.UpperBound)] = *b.CumulativeCount
		}
	}
	return ret
}
<commit_msg>fix: lint errors upd: remove deprecated id, replaced with collector:promrecv streamtag<commit_after>// Copyright © 2017 Circonus, Inc. <support@circonus.com>
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//

// thanks to https://github.com/influxdata/telegraf/blob/release-1.2/plugins/inputs/prometheus/parser.go
// and https://github.com/prometheus/common/tree/master/expfmt

package promrecv

import (
	"fmt"
	"io"
	"math"
	"regexp"
	"sync"

	"github.com/circonus-labs/circonus-agent/internal/config"
	"github.com/circonus-labs/circonus-agent/internal/release"
	"github.com/circonus-labs/circonus-agent/internal/tags"
	cgm "github.com/circonus-labs/circonus-gometrics/v3"
	"github.com/pkg/errors"
	dto "github.com/prometheus/client_model/go"
	"github.com/prometheus/common/expfmt"
	"github.com/rs/zerolog"
	"github.com/rs/zerolog/log"
	"github.com/spf13/viper"
)

var (
	baseTags      []string
	nameCleanerRx *regexp.Regexp
	metricsmu     sync.Mutex
	metrics       *cgm.CirconusMetrics
	logger        = log.With().Str("pkg", "promrecv").Logger()
)

// logshim is used to satisfy apiclient Logger interface (avoiding ptr receiver issue)
type logshim struct {
	logh zerolog.Logger
}

func (l logshim) Printf(fmt string, v ...interface{}) {
	l.logh.Printf(fmt, v...)
}

func initCGM() error {
	metricsmu.Lock()
	defer metricsmu.Unlock()

	if metrics != nil {
		return nil
	}

	cmc := &cgm.Config{
		Debug: viper.GetBool(config.KeyDebugCGM),
		Log:   logshim{logh: log.With().Str("pkg", "cgm.promrecv").Logger()},
	}
	// put cgm into manual mode (no interval, no api key, invalid submission url)
	cmc.Interval = "0"                            // disable automatic flush
	cmc.CheckManager.Check.SubmissionURL = "none" // disable check management (create/update)

	hm, err := cgm.NewCirconusMetrics(cmc)
	if err != nil {
		return errors.Wrap(err, "prom receiver cgm")
	}

	metrics = hm

	// inintialize any options for the receiver
	nameCleanerRx = regexp.MustCompile("[\r\n\"'`]") // used to strip unwanted characters

	baseTags = tags.GetBaseTags()
	baseTags = append(baseTags, []string{
		"source:" + release.NAME,
		"collector:promrecv",
	}...)

	return nil
}

// Flush returns current metrics
func Flush() *cgm.Metrics {
	_ = initCGM()

	return metrics.FlushMetrics()
}

// Parse handles incoming PUT/POST requests
func Parse(data io.Reader) error {
	if err := initCGM(); err != nil {
		return err
	}

	var parser expfmt.TextParser

	// formats supported from https://prometheus.io/docs/instrumenting/exposition_formats/

	metricFamilies, err := parser.TextToMetricFamilies(data)
	if err != nil {
		return err
	}

	for mn, mf := range metricFamilies {
		for _, m := range mf.Metric {
			metricName := nameCleanerRx.ReplaceAllString(mn, "")
			tags := getLabels(m)
			switch {
			case mf.GetType() == dto.MetricType_SUMMARY:
				metrics.Gauge(metricName+"_count", float64(m.GetSummary().GetSampleCount()))
				metrics.Gauge(metricName+"_sum", float64(m.GetSummary().GetSampleSum()))
				for qn, qv := range getQuantiles(m) {
					metrics.GaugeWithTags(metricName+"_"+qn, tags, qv)
				}
			case mf.GetType() == dto.MetricType_HISTOGRAM:
				metrics.Gauge(metricName+"_count", float64(m.GetHistogram().GetSampleCount()))
				metrics.Gauge(metricName+"_sum", float64(m.GetHistogram().GetSampleSum()))
				for bn, bv := range getBuckets(m) {
					metrics.GaugeWithTags(metricName+"_"+bn, tags, bv)
				}
			default:
				switch {
				case m.Gauge != nil:
					if m.GetGauge().Value != nil {
						metrics.GaugeWithTags(metricName, tags, *m.GetGauge().Value)
					}
				case m.Counter != nil:
					if m.GetCounter().Value != nil {
						metrics.GaugeWithTags(metricName, tags, *m.GetCounter().Value)
					}
				case m.Untyped != nil:
					if m.GetUntyped().Value != nil {
						if *m.GetUntyped().Value == math.Inf(+1) {
							logger.Warn().Str("metric", metricName).Str("type", mf.GetType().String()).Str("value", (*m).GetUntyped().String()).Msg("cannot coerce +Inf to uint64")
							continue
						}
						metrics.GaugeWithTags(metricName, tags, *m.GetUntyped().Value)
					}
				}
			}
		}
	}

	return nil
}

func getLabels(m *dto.Metric) tags.Tags {
	labels := make([]string, 0, len(m.Label))
	for _, label := range m.Label {
		if label.Name != nil && label.Value != nil {
			ln := nameCleanerRx.ReplaceAllString(*label.Name, "")
			lv := nameCleanerRx.ReplaceAllString(*label.Value, "")
			labels = append(labels, ln+tags.Delimiter+lv) // stream tags take form cat:val
		}
	}

	if len(labels) > 0 {
		tagList := make([]string, 0, len(baseTags)+len(labels))
		tagList = append(tagList, baseTags...)
		tagList = append(tagList, labels...)
		tags := tags.FromList(tagList)
		return tags
	}

	return tags.FromList(baseTags)
}

func getQuantiles(m *dto.Metric) map[string]float64 {
	ret := make(map[string]float64)
	for _, q := range m.GetSummary().Quantile {
		if q.Value != nil && !math.IsNaN(*q.Value) {
			ret[fmt.Sprint(*q.Quantile)] = *q.Value
		}
	}
	return ret
}

func getBuckets(m *dto.Metric) map[string]uint64 {
	ret := make(map[string]uint64)
	for _, b := range m.GetHistogram().Bucket {
		if b.CumulativeCount != nil {
			ret[fmt.Sprint(*b.UpperBound)] = *b.CumulativeCount
		}
	}
	return ret
}
<|endoftext|>


<commit_before>// Copyright 2015 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package docker

import (
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"regexp"
	"strings"
	"time"

	"github.com/tsuru/config"
)

func clientWithTimeout(timeout time.Duration) *http.Client {
	dialTimeout := func(network, addr string) (net.Conn, error) {
		return net.DialTimeout(network, addr, timeout)
	}
	transport := http.Transport{
		Dial: dialTimeout,
	}
	return &http.Client{
		Transport: &transport,
	}
}

var timeoutHttpClient = clientWithTimeout(5 * time.Second)

func runHealthcheck(cont *container, w io.Writer) error {
	yamlData, err := getImageTsuruYamlDataWithFallback(cont.Image, cont.AppName)
	if err != nil {
		return err
	}
	path := yamlData.Healthcheck.Path
	method := yamlData.Healthcheck.Method
	match := yamlData.Healthcheck.Match
	status := yamlData.Healthcheck.Status
	allowedFailures := yamlData.Healthcheck.AllowedFailures
	if path == "" {
		return nil
	}
	path = strings.TrimSpace(strings.TrimLeft(path, "/"))
	if method == "" {
		method = "get"
	}
	method = strings.ToUpper(method)
	if status == 0 && match == "" {
		status = 200
	}
	var matchRE *regexp.Regexp
	if match != "" {
		match = "(?s)" + match
		matchRE, err = regexp.Compile(match)
		if err != nil {
			return err
		}
	}
	maxWaitTime, _ := config.GetDuration("docker:healthcheck:max-time")
	if maxWaitTime == 0 {
		maxWaitTime = 120
	}
	maxWaitTime = maxWaitTime * time.Second
	sleepTime := 3 * time.Second
	startedTime := time.Now()
	url := fmt.Sprintf("http://%s:%s/%s", cont.HostAddr, cont.HostPort, path)
	for {
		var lastError error = nil
		req, err := http.NewRequest(method, url, nil)
		if err != nil {
			return err
		}
		rsp, err := timeoutHttpClient.Do(req)
		if err != nil {
			lastError = fmt.Errorf("healthcheck fail(%s): %s", cont.shortID(), err.Error())
		} else {
			defer rsp.Body.Close()
			if status != 0 && rsp.StatusCode != status {
				lastError = fmt.Errorf("healthcheck fail(%s): wrong status code, expected %d, got: %d", cont.shortID(), status, rsp.StatusCode)
			} else if matchRE != nil {
				result, err := ioutil.ReadAll(rsp.Body)

				if err != nil {
					lastError = err
				}
				if !matchRE.Match(result) {
					lastError = fmt.Errorf("healthcheck fail(%s): unexpected result, expected %q, got: %s", cont.shortID(), match, string(result))
				}
			}
			if lastError != nil {
				if allowedFailures == 0 {
					return lastError
				}
				allowedFailures--
			}
		}
		if lastError == nil {
			fmt.Fprintf(w, " ---> healthcheck successful(%s)\n", cont.shortID())
			return nil
		}
		if time.Now().Sub(startedTime) > maxWaitTime {
			return lastError
		}
		fmt.Fprintf(w, " ---> %s. Trying again in %ds\n", lastError.Error(), sleepTime/time.Second)
		time.Sleep(sleepTime)
	}
}
<commit_msg>provision/docker: set maximum timeout for healthcheck requests<commit_after>// Copyright 2015 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package docker

import (
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"regexp"
	"strings"
	"time"

	"github.com/tsuru/config"
)

func clientWithTimeout(dialTimeout time.Duration) *http.Client {
	transport := http.Transport{
		Dial: (&net.Dialer{
			Timeout:   dialTimeout,
			KeepAlive: 30 * time.Second,
		}).Dial,
		TLSHandshakeTimeout: dialTimeout,
	}
	return &http.Client{
		Transport: &transport,
		Timeout:   1 * time.Minute,
	}
}

var timeoutHttpClient = clientWithTimeout(5 * time.Second)

func runHealthcheck(cont *container, w io.Writer) error {
	yamlData, err := getImageTsuruYamlDataWithFallback(cont.Image, cont.AppName)
	if err != nil {
		return err
	}
	path := yamlData.Healthcheck.Path
	method := yamlData.Healthcheck.Method
	match := yamlData.Healthcheck.Match
	status := yamlData.Healthcheck.Status
	allowedFailures := yamlData.Healthcheck.AllowedFailures
	if path == "" {
		return nil
	}
	path = strings.TrimSpace(strings.TrimLeft(path, "/"))
	if method == "" {
		method = "get"
	}
	method = strings.ToUpper(method)
	if status == 0 && match == "" {
		status = 200
	}
	var matchRE *regexp.Regexp
	if match != "" {
		match = "(?s)" + match
		matchRE, err = regexp.Compile(match)
		if err != nil {
			return err
		}
	}
	maxWaitTime, _ := config.GetDuration("docker:healthcheck:max-time")
	if maxWaitTime == 0 {
		maxWaitTime = 120
	}
	maxWaitTime = maxWaitTime * time.Second
	sleepTime := 3 * time.Second
	startedTime := time.Now()
	url := fmt.Sprintf("http://%s:%s/%s", cont.HostAddr, cont.HostPort, path)
	for {
		var lastError error = nil
		req, err := http.NewRequest(method, url, nil)
		if err != nil {
			return err
		}
		rsp, err := timeoutHttpClient.Do(req)
		if err != nil {
			lastError = fmt.Errorf("healthcheck fail(%s): %s", cont.shortID(), err.Error())
		} else {
			defer rsp.Body.Close()
			if status != 0 && rsp.StatusCode != status {
				lastError = fmt.Errorf("healthcheck fail(%s): wrong status code, expected %d, got: %d", cont.shortID(), status, rsp.StatusCode)
			} else if matchRE != nil {
				result, err := ioutil.ReadAll(rsp.Body)

				if err != nil {
					lastError = err
				}
				if !matchRE.Match(result) {
					lastError = fmt.Errorf("healthcheck fail(%s): unexpected result, expected %q, got: %s", cont.shortID(), match, string(result))
				}
			}
			if lastError != nil {
				if allowedFailures == 0 {
					return lastError
				}
				allowedFailures--
			}
		}
		if lastError == nil {
			fmt.Fprintf(w, " ---> healthcheck successful(%s)\n", cont.shortID())
			return nil
		}
		if time.Now().Sub(startedTime) > maxWaitTime {
			return lastError
		}
		fmt.Fprintf(w, " ---> %s. Trying again in %ds\n", lastError.Error(), sleepTime/time.Second)
		time.Sleep(sleepTime)
	}
}
<|endoftext|>


<commit_before>// This package implements a provisioner for Packer that executes a
// saltstack highstate within the remote machine
package salt

import (
	"fmt"
	"github.com/mitchellh/iochan"
	"github.com/mitchellh/mapstructure"
	"github.com/mitchellh/packer/packer"
	"io"
	"log"
	"sort"
	"strings"
)

var Ui packer.Ui

type config struct {
	// If true, run the salt-bootstrap script
	SkipBootstrap bool   `mapstructure:"skip_bootstrap"`
	BootstrapArgs string `mapstructure:"bootstrap_args"`
}

type Provisioner struct {
	config config
}

func (p *Provisioner) Prepare(raws ...interface{}) error {
	var md mapstructure.Metadata
	decoderConfig := &mapstructure.DecoderConfig{
		Metadata: &md,
		Result:   &p.config,
	}

	decoder, err := mapstructure.NewDecoder(decoderConfig)
	if err != nil {
		return err
	}

	for _, raw := range raws {
		err := decoder.Decode(raw)
		if err != nil {
			return err
		}
	}

	// Accumulate any errors
	errs := make([]error, 0)

	// Unused keys are errors
	if len(md.Unused) > 0 {
		sort.Strings(md.Unused)
		for _, unused := range md.Unused {
			if unused != "type" && !strings.HasPrefix(unused, "packer_") {
				errs = append(
					errs, fmt.Errorf("Unknown configuration key: %s", unused))
			}
		}
	}

	if len(errs) > 0 {
		return &packer.MultiError{errs}
	}

	return nil
}

func (p *Provisioner) Provision(ui packer.Ui, comm packer.Communicator) error {
	var err error
	Ui = ui

	if !p.config.SkipBootstrap {
		cmd := fmt.Sprintf("wget -O - http://bootstrap.saltstack.org | sudo sh -s %s", p.config.BootstrapArgs)
		Ui.Say(fmt.Sprintf("Installing Salt with command %s", cmd))
		if err = executeCommand(cmd, comm); err != nil {
			return fmt.Errorf("Unable to install Salt: %d", err)
		}
	}
	return nil
}

func executeCommand(command string, comm packer.Communicator) (err error) {
	// Setup the remote command
	stdout_r, stdout_w := io.Pipe()
	stderr_r, stderr_w := io.Pipe()

	var cmd packer.RemoteCmd
	cmd.Command = command
	cmd.Stdout = stdout_w
	cmd.Stderr = stderr_w

	log.Printf("Executing command: %s", cmd.Command)
	err = comm.Start(&cmd)
	if err != nil {
		return fmt.Errorf("Failed executing command: %s", err)
	}

	exitChan := make(chan int, 1)
	stdoutChan := iochan.DelimReader(stdout_r, '\n')
	stderrChan := iochan.DelimReader(stderr_r, '\n')

	go func() {
		defer stdout_w.Close()
		defer stderr_w.Close()

		cmd.Wait()
		exitChan <- cmd.ExitStatus
	}()

OutputLoop:
	for {
		select {
		case output := <-stderrChan:
			Ui.Message(strings.TrimSpace(output))
		case output := <-stdoutChan:
			Ui.Message(strings.TrimSpace(output))
		case exitStatus := <-exitChan:
			log.Printf("Chef Solo provisioner exited with status %d", exitStatus)

			if exitStatus != 0 {
				return fmt.Errorf("Command exited with non-zero exit status: %d", exitStatus)
			}

			break OutputLoop
		}
	}

	// Make sure we finish off stdout/stderr because we may have gotten
	// a message from the exit channel first.
	for output := range stdoutChan {
		Ui.Message(output)
	}

	for output := range stderrChan {
		Ui.Message(output)
	}

	return nil
}
<commit_msg>provisioner/salt: copy local state tree to remote, run highstate<commit_after>// This package implements a provisioner for Packer that executes a
// saltstack highstate within the remote machine
package salt

import (
	"errors"
	"fmt"
	"github.com/mitchellh/iochan"
	"github.com/mitchellh/mapstructure"
	"github.com/mitchellh/packer/packer"
	"io"
	"log"
	"os"
	"path/filepath"
	"sort"
	"strings"
)

var Ui packer.Ui

type config struct {
	// If true, run the salt-bootstrap script
	SkipBootstrap bool   `mapstructure:"skip_bootstrap"`
	BootstrapArgs string `mapstructure:"bootstrap_args"`

	// Local path to the salt state tree
	LocalStateTree string `mapstructure:"local_state_tree"`
}

type Provisioner struct {
	config config
}

func (p *Provisioner) Prepare(raws ...interface{}) error {
	var md mapstructure.Metadata
	decoderConfig := &mapstructure.DecoderConfig{
		Metadata: &md,
		Result:   &p.config,
	}

	decoder, err := mapstructure.NewDecoder(decoderConfig)
	if err != nil {
		return err
	}

	for _, raw := range raws {
		err := decoder.Decode(raw)
		if err != nil {
			return err
		}
	}

	// Accumulate any errors
	errs := make([]error, 0)

	// Unused keys are errors
	if len(md.Unused) > 0 {
		sort.Strings(md.Unused)
		for _, unused := range md.Unused {
			if unused != "type" && !strings.HasPrefix(unused, "packer_") {
				errs = append(
					errs, fmt.Errorf("Unknown configuration key: %s", unused))
			}
		}
	}

	if p.config.LocalStateTree == "" {
		errs = append(errs, errors.New("Please specify a local_state_tree"))
	}

	if len(errs) > 0 {
		return &packer.MultiError{errs}
	}

	return nil
}

func (p *Provisioner) Provision(ui packer.Ui, comm packer.Communicator) error {
	var err error
	Ui = ui

	if !p.config.SkipBootstrap {
		cmd := fmt.Sprintf("wget -O - http://bootstrap.saltstack.org | sudo sh -s %s", p.config.BootstrapArgs)
		Ui.Say(fmt.Sprintf("Installing Salt with command %s", cmd))
		if err = ExecuteCommand(cmd, comm); err != nil {
			return fmt.Errorf("Unable to install Salt: %d", err)
		}
	}

	remoteTempDir := "salt"
	Ui.Say(fmt.Sprintf("Creating remote directory: %s", remoteTempDir))
	if err = ExecuteCommand(fmt.Sprintf("mkdir -p %s", remoteTempDir), comm); err != nil {
		return fmt.Errorf("Error creating remote salt state directory: %s", err)
	}

	Ui.Say(fmt.Sprintf("Uploading local state tree: %s", p.config.LocalStateTree))
	if err = UploadLocalDirectory(p.config.LocalStateTree, remoteTempDir, comm); err != nil {
		return fmt.Errorf("Error uploading local state tree to remote: %s", err)
	}

	Ui.Say(fmt.Sprintf("Moving %s to /srv/salt", remoteTempDir))
	if err = ExecuteCommand(fmt.Sprintf("sudo mv %s /srv/salt", remoteTempDir), comm); err != nil {
		return fmt.Errorf("Unable to move %s to /srv/salt: %d", remoteTempDir, err)
	}

	Ui.Say("Running highstate")
	if err = ExecuteCommand("sudo salt-call --local state.highstate -l debug", comm); err != nil {
		return fmt.Errorf("Error executing highstate: %s", err)
	}

	Ui.Say("Removing /srv/salt")
	if err = ExecuteCommand("sudo rm -r /srv/salt", comm); err != nil {
		return fmt.Errorf("Unable to remove /srv/salt: %d", err)
	}

	return nil
}

func UploadLocalDirectory(localDir string, remoteDir string, comm packer.Communicator) (err error) {
	visitPath := func(localPath string, f os.FileInfo, err error) (err2 error) {
		localRelPath := strings.Replace(localPath, localDir, "", 1)
		remotePath := fmt.Sprintf("%s/%s", remoteDir, localRelPath)
		if f.IsDir() && f.Name() == ".git" {
			return filepath.SkipDir
		}
		if f.IsDir() {
			// Make remote directory
			err = ExecuteCommand(fmt.Sprintf("mkdir -p %s", remotePath), comm)
			if err != nil {
				return err
			}
		} else {
			// Upload file to existing directory
			file, err := os.Open(localPath)
			if err != nil {
				return fmt.Errorf("Error opening file: %s", err)
			}
			defer file.Close()

			Ui.Say(fmt.Sprintf("Uploading file %s: %s", localPath, remotePath))
			err = comm.Upload(remotePath, file)
			if err != nil {
				return fmt.Errorf("Error uploading file: %s", err)
			}
		}
		return
	}

	err = filepath.Walk(localDir, visitPath)
	if err != nil {
		return fmt.Errorf("Error uploading local directory %s: %s", localDir, err)
	}

	return nil
}

func ExecuteCommand(command string, comm packer.Communicator) (err error) {
	// Setup the remote command
	stdout_r, stdout_w := io.Pipe()
	stderr_r, stderr_w := io.Pipe()

	var cmd packer.RemoteCmd
	cmd.Command = command
	cmd.Stdout = stdout_w
	cmd.Stderr = stderr_w

	log.Printf("Executing command: %s", cmd.Command)
	err = comm.Start(&cmd)
	if err != nil {
		return fmt.Errorf("Failed executing command: %s", err)
	}

	exitChan := make(chan int, 1)
	stdoutChan := iochan.DelimReader(stdout_r, '\n')
	stderrChan := iochan.DelimReader(stderr_r, '\n')

	go func() {
		defer stdout_w.Close()
		defer stderr_w.Close()

		cmd.Wait()
		exitChan <- cmd.ExitStatus
	}()

OutputLoop:
	for {
		select {
		case output := <-stderrChan:
			Ui.Message(strings.TrimSpace(output))
		case output := <-stdoutChan:
			Ui.Message(strings.TrimSpace(output))
		case exitStatus := <-exitChan:
			log.Printf("Salt provisioner exited with status %d", exitStatus)

			if exitStatus != 0 {
				return fmt.Errorf("Command exited with non-zero exit status: %d", exitStatus)
			}

			break OutputLoop
		}
	}

	// Make sure we finish off stdout/stderr because we may have gotten
	// a message from the exit channel first.
	for output := range stdoutChan {
		Ui.Message(output)
	}

	for output := range stderrChan {
		Ui.Message(output)
	}

	return nil
}
<|endoftext|>


<commit_before>package portmapper

import (
	"errors"
	"fmt"
	"net"
	"sync"

	"github.com/Sirupsen/logrus"
	"github.com/docker/libnetwork/iptables"
	"github.com/docker/libnetwork/portallocator"
)

type mapping struct {
	proto         string
	userlandProxy userlandProxy
	host          net.Addr
	container     net.Addr
}

var newProxy = newProxyCommand

var (
	// ErrUnknownBackendAddressType refers to an unknown container or unsupported address type
	ErrUnknownBackendAddressType = errors.New("unknown container address type not supported")
	// ErrPortMappedForIP refers to a port already mapped to an ip address
	ErrPortMappedForIP = errors.New("port is already mapped to ip")
	// ErrPortNotMapped refers to an unmapped port
	ErrPortNotMapped = errors.New("port is not mapped")
)

// PortMapper manages the network address translation
type PortMapper struct {
	chain      *iptables.ChainInfo
	bridgeName string

	// udp:ip:port
	currentMappings map[string]*mapping
	lock            sync.Mutex

	Allocator *portallocator.PortAllocator
}

// New returns a new instance of PortMapper
func New() *PortMapper {
	return NewWithPortAllocator(portallocator.Get())
}

// NewWithPortAllocator returns a new instance of PortMapper which will use the specified PortAllocator
func NewWithPortAllocator(allocator *portallocator.PortAllocator) *PortMapper {
	return &PortMapper{
		currentMappings: make(map[string]*mapping),
		Allocator:       allocator,
	}
}

// SetIptablesChain sets the specified chain into portmapper
func (pm *PortMapper) SetIptablesChain(c *iptables.ChainInfo, bridgeName string) {
	pm.chain = c
	pm.bridgeName = bridgeName
}

// Map maps the specified container transport address to the host's network address and transport port
func (pm *PortMapper) Map(container net.Addr, hostIP net.IP, hostPort int, useProxy bool) (host net.Addr, err error) {
	return pm.MapRange(container, hostIP, hostPort, hostPort, useProxy)
}

// MapRange maps the specified container transport address to the host's network address and transport port range
func (pm *PortMapper) MapRange(container net.Addr, hostIP net.IP, hostPortStart, hostPortEnd int, useProxy bool) (host net.Addr, err error) {
	pm.lock.Lock()
	defer pm.lock.Unlock()

	var (
		m                 *mapping
		proto             string
		allocatedHostPort int
	)

	switch container.(type) {
	case *net.TCPAddr:
		proto = "tcp"
		if allocatedHostPort, err = pm.Allocator.RequestPortInRange(hostIP, proto, hostPortStart, hostPortEnd); err != nil {
			return nil, err
		}

		m = &mapping{
			proto:     proto,
			host:      &net.TCPAddr{IP: hostIP, Port: allocatedHostPort},
			container: container,
		}

		if useProxy {
			m.userlandProxy, err = newProxy(proto, hostIP, allocatedHostPort, container.(*net.TCPAddr).IP, container.(*net.TCPAddr).Port)
			if err != nil {
				return nil, err
			}
		} else {
			m.userlandProxy = newDummyProxy(proto, hostIP, allocatedHostPort)
		}
	case *net.UDPAddr:
		proto = "udp"
		if allocatedHostPort, err = pm.Allocator.RequestPortInRange(hostIP, proto, hostPortStart, hostPortEnd); err != nil {
			return nil, err
		}

		m = &mapping{
			proto:     proto,
			host:      &net.UDPAddr{IP: hostIP, Port: allocatedHostPort},
			container: container,
		}

		if useProxy {
			m.userlandProxy, err = newProxy(proto, hostIP, allocatedHostPort, container.(*net.UDPAddr).IP, container.(*net.UDPAddr).Port)
			if err != nil {
				return nil, err
			}
		} else {
			m.userlandProxy = newDummyProxy(proto, hostIP, allocatedHostPort)
		}
	default:
		return nil, ErrUnknownBackendAddressType
	}

	// release the allocated port on any further error during return.
	defer func() {
		if err != nil {
			pm.Allocator.ReleasePort(hostIP, proto, allocatedHostPort)
		}
	}()

	key := getKey(m.host)
	if _, exists := pm.currentMappings[key]; exists {
		return nil, ErrPortMappedForIP
	}

	containerIP, containerPort := getIPAndPort(m.container)
	if err := pm.forward(iptables.Append, m.proto, hostIP, allocatedHostPort, containerIP.String(), containerPort); err != nil {
		return nil, err
	}

	cleanup := func() error {
		// need to undo the iptables rules before we return
		m.userlandProxy.Stop()
		pm.forward(iptables.Delete, m.proto, hostIP, allocatedHostPort, containerIP.String(), containerPort)
		if err := pm.Allocator.ReleasePort(hostIP, m.proto, allocatedHostPort); err != nil {
			return err
		}

		return nil
	}

	if err := m.userlandProxy.Start(); err != nil {
		if err := cleanup(); err != nil {
			return nil, fmt.Errorf("Error during port allocation cleanup: %v", err)
		}
		return nil, err
	}

	pm.currentMappings[key] = m
	return m.host, nil
}

// Unmap removes stored mapping for the specified host transport address
func (pm *PortMapper) Unmap(host net.Addr) error {
	pm.lock.Lock()
	defer pm.lock.Unlock()

	key := getKey(host)
	data, exists := pm.currentMappings[key]
	if !exists {
		return ErrPortNotMapped
	}

	if data.userlandProxy != nil {
		data.userlandProxy.Stop()
	}

	delete(pm.currentMappings, key)

	containerIP, containerPort := getIPAndPort(data.container)
	hostIP, hostPort := getIPAndPort(data.host)
	if err := pm.forward(iptables.Delete, data.proto, hostIP, hostPort, containerIP.String(), containerPort); err != nil {
		logrus.Errorf("Error on iptables delete: %s", err)
	}

	switch a := host.(type) {
	case *net.TCPAddr:
		return pm.Allocator.ReleasePort(a.IP, "tcp", a.Port)
	case *net.UDPAddr:
		return pm.Allocator.ReleasePort(a.IP, "udp", a.Port)
	}
	return nil
}

//ReMapAll will re-apply all port mappings
func (pm *PortMapper) ReMapAll() {
	pm.lock.Lock()
	defer pm.lock.Unlock()
	logrus.Debugln("Re-applying all port mappings.")
	for _, data := range pm.currentMappings {
		containerIP, containerPort := getIPAndPort(data.container)
		hostIP, hostPort := getIPAndPort(data.host)
		if err := pm.forward(iptables.Append, data.proto, hostIP, hostPort, containerIP.String(), containerPort); err != nil {
			logrus.Errorf("Error on iptables add: %s", err)
		}
	}
}

func getKey(a net.Addr) string {
	switch t := a.(type) {
	case *net.TCPAddr:
		return fmt.Sprintf("%s:%d/%s", t.IP.String(), t.Port, "tcp")
	case *net.UDPAddr:
		return fmt.Sprintf("%s:%d/%s", t.IP.String(), t.Port, "udp")
	}
	return ""
}

func getIPAndPort(a net.Addr) (net.IP, int) {
	switch t := a.(type) {
	case *net.TCPAddr:
		return t.IP, t.Port
	case *net.UDPAddr:
		return t.IP, t.Port
	}
	return nil, 0
}

func (pm *PortMapper) forward(action iptables.Action, proto string, sourceIP net.IP, sourcePort int, containerIP string, containerPort int) error {
	if pm.chain == nil {
		return nil
	}
	return pm.chain.Forward(action, sourceIP, sourcePort, proto, containerIP, containerPort, pm.bridgeName)
}
<commit_msg>portmapper: touch iptables only for IPv4<commit_after>package portmapper

import (
	"errors"
	"fmt"
	"net"
	"sync"

	"github.com/Sirupsen/logrus"
	"github.com/docker/libnetwork/iptables"
	"github.com/docker/libnetwork/portallocator"
)

type mapping struct {
	proto         string
	userlandProxy userlandProxy
	host          net.Addr
	container     net.Addr
}

var newProxy = newProxyCommand

var (
	// ErrUnknownBackendAddressType refers to an unknown container or unsupported address type
	ErrUnknownBackendAddressType = errors.New("unknown container address type not supported")
	// ErrPortMappedForIP refers to a port already mapped to an ip address
	ErrPortMappedForIP = errors.New("port is already mapped to ip")
	// ErrPortNotMapped refers to an unmapped port
	ErrPortNotMapped = errors.New("port is not mapped")
)

// PortMapper manages the network address translation
type PortMapper struct {
	chain      *iptables.ChainInfo
	bridgeName string

	// udp:ip:port
	currentMappings map[string]*mapping
	lock            sync.Mutex

	Allocator *portallocator.PortAllocator
}

// New returns a new instance of PortMapper
func New() *PortMapper {
	return NewWithPortAllocator(portallocator.Get())
}

// NewWithPortAllocator returns a new instance of PortMapper which will use the specified PortAllocator
func NewWithPortAllocator(allocator *portallocator.PortAllocator) *PortMapper {
	return &PortMapper{
		currentMappings: make(map[string]*mapping),
		Allocator:       allocator,
	}
}

// SetIptablesChain sets the specified chain into portmapper
func (pm *PortMapper) SetIptablesChain(c *iptables.ChainInfo, bridgeName string) {
	pm.chain = c
	pm.bridgeName = bridgeName
}

// Map maps the specified container transport address to the host's network address and transport port
func (pm *PortMapper) Map(container net.Addr, hostIP net.IP, hostPort int, useProxy bool) (host net.Addr, err error) {
	return pm.MapRange(container, hostIP, hostPort, hostPort, useProxy)
}

// MapRange maps the specified container transport address to the host's network address and transport port range
func (pm *PortMapper) MapRange(container net.Addr, hostIP net.IP, hostPortStart, hostPortEnd int, useProxy bool) (host net.Addr, err error) {
	pm.lock.Lock()
	defer pm.lock.Unlock()

	var (
		m                 *mapping
		proto             string
		allocatedHostPort int
	)

	switch container.(type) {
	case *net.TCPAddr:
		proto = "tcp"
		if allocatedHostPort, err = pm.Allocator.RequestPortInRange(hostIP, proto, hostPortStart, hostPortEnd); err != nil {
			return nil, err
		}

		m = &mapping{
			proto:     proto,
			host:      &net.TCPAddr{IP: hostIP, Port: allocatedHostPort},
			container: container,
		}

		if useProxy {
			m.userlandProxy, err = newProxy(proto, hostIP, allocatedHostPort, container.(*net.TCPAddr).IP, container.(*net.TCPAddr).Port)
			if err != nil {
				return nil, err
			}
		} else {
			m.userlandProxy = newDummyProxy(proto, hostIP, allocatedHostPort)
		}
	case *net.UDPAddr:
		proto = "udp"
		if allocatedHostPort, err = pm.Allocator.RequestPortInRange(hostIP, proto, hostPortStart, hostPortEnd); err != nil {
			return nil, err
		}

		m = &mapping{
			proto:     proto,
			host:      &net.UDPAddr{IP: hostIP, Port: allocatedHostPort},
			container: container,
		}

		if useProxy {
			m.userlandProxy, err = newProxy(proto, hostIP, allocatedHostPort, container.(*net.UDPAddr).IP, container.(*net.UDPAddr).Port)
			if err != nil {
				return nil, err
			}
		} else {
			m.userlandProxy = newDummyProxy(proto, hostIP, allocatedHostPort)
		}
	default:
		return nil, ErrUnknownBackendAddressType
	}

	// release the allocated port on any further error during return.
	defer func() {
		if err != nil {
			pm.Allocator.ReleasePort(hostIP, proto, allocatedHostPort)
		}
	}()

	key := getKey(m.host)
	if _, exists := pm.currentMappings[key]; exists {
		return nil, ErrPortMappedForIP
	}

	containerIP, containerPort := getIPAndPort(m.container)
	if hostIP.To4() != nil {
		if err := pm.forward(iptables.Append, m.proto, hostIP, allocatedHostPort, containerIP.String(), containerPort); err != nil {
			return nil, err
		}
	}

	cleanup := func() error {
		// need to undo the iptables rules before we return
		m.userlandProxy.Stop()
		if hostIP.To4() != nil {
			pm.forward(iptables.Delete, m.proto, hostIP, allocatedHostPort, containerIP.String(), containerPort)
			if err := pm.Allocator.ReleasePort(hostIP, m.proto, allocatedHostPort); err != nil {
				return err
			}
		}

		return nil
	}

	if err := m.userlandProxy.Start(); err != nil {
		if err := cleanup(); err != nil {
			return nil, fmt.Errorf("Error during port allocation cleanup: %v", err)
		}
		return nil, err
	}

	pm.currentMappings[key] = m
	return m.host, nil
}

// Unmap removes stored mapping for the specified host transport address
func (pm *PortMapper) Unmap(host net.Addr) error {
	pm.lock.Lock()
	defer pm.lock.Unlock()

	key := getKey(host)
	data, exists := pm.currentMappings[key]
	if !exists {
		return ErrPortNotMapped
	}

	if data.userlandProxy != nil {
		data.userlandProxy.Stop()
	}

	delete(pm.currentMappings, key)

	containerIP, containerPort := getIPAndPort(data.container)
	hostIP, hostPort := getIPAndPort(data.host)
	if err := pm.forward(iptables.Delete, data.proto, hostIP, hostPort, containerIP.String(), containerPort); err != nil {
		logrus.Errorf("Error on iptables delete: %s", err)
	}

	switch a := host.(type) {
	case *net.TCPAddr:
		return pm.Allocator.ReleasePort(a.IP, "tcp", a.Port)
	case *net.UDPAddr:
		return pm.Allocator.ReleasePort(a.IP, "udp", a.Port)
	}
	return nil
}

//ReMapAll will re-apply all port mappings
func (pm *PortMapper) ReMapAll() {
	pm.lock.Lock()
	defer pm.lock.Unlock()
	logrus.Debugln("Re-applying all port mappings.")
	for _, data := range pm.currentMappings {
		containerIP, containerPort := getIPAndPort(data.container)
		hostIP, hostPort := getIPAndPort(data.host)
		if err := pm.forward(iptables.Append, data.proto, hostIP, hostPort, containerIP.String(), containerPort); err != nil {
			logrus.Errorf("Error on iptables add: %s", err)
		}
	}
}

func getKey(a net.Addr) string {
	switch t := a.(type) {
	case *net.TCPAddr:
		return fmt.Sprintf("%s:%d/%s", t.IP.String(), t.Port, "tcp")
	case *net.UDPAddr:
		return fmt.Sprintf("%s:%d/%s", t.IP.String(), t.Port, "udp")
	}
	return ""
}

func getIPAndPort(a net.Addr) (net.IP, int) {
	switch t := a.(type) {
	case *net.TCPAddr:
		return t.IP, t.Port
	case *net.UDPAddr:
		return t.IP, t.Port
	}
	return nil, 0
}

func (pm *PortMapper) forward(action iptables.Action, proto string, sourceIP net.IP, sourcePort int, containerIP string, containerPort int) error {
	if pm.chain == nil {
		return nil
	}
	return pm.chain.Forward(action, sourceIP, sourcePort, proto, containerIP, containerPort, pm.bridgeName)
}
<|endoftext|>


<commit_before><commit_msg>veyron/lib/textutil: Fix minor typo.<commit_after><|endoftext|>


<commit_before>package outgoing_test

import (
	"os"
	"strconv"
	"testing"
	"time"

	"github.com/lestrrat/roccaforte/client"
	"github.com/lestrrat/roccaforte/event"
	"github.com/lestrrat/roccaforte/incoming"
	"github.com/lestrrat/roccaforte/outgoing"
	"github.com/stretchr/testify/assert"
	"golang.org/x/net/context"
)

var projectID string

func init() {
	projectID = os.Getenv("DATASTORE_PROJECT_ID")
}

func TestOutgoing(t *testing.T) {
	if projectID == "" {
		t.Skip("missing project ID. please set DATASTORE_PROJECT_ID")
		return
	}

	// Enqueue
	ctx, cancel := context.WithCancel(context.Background())

	h := incoming.NewHTTPSource()
	s := incoming.NewGDatastoreStorage(projectID)
	e := incoming.New()
	e.Storage = s
	e.AddSource(h)

	go func() {
		assert.NoError(t, e.Run(ctx), "incoming engine should exit w/o errors")
	}()

	go h.Loop(ctx)
	for i := 0; i < 10; i++ {
		e.SetRule("test.notify" + strconv.Itoa(i), &incoming.Rule{})
	}

	for i := 0; i < 10; i++ {
		eventName := "test.notify" + strconv.Itoa(i)
		go func() {
			cl := client.NewHTTP("http://localhost:8080/enqueue")
			events := make([]event.Event, 100)
			for j := 0; j < 100; j++ {
				events[j] = event.NewCoreAttrs(eventName)
			}
			if !assert.NoError(t, cl.Enqueue(events...), "enqueue should succeed") {
				return
			}
		}()
	}

	o := outgoing.New(projectID)
	o.Clock = clock.NewMockClock(time.Now().Add(time.Hour))
	go o.Run(ctx)

	time.AfterFunc(10*time.Second, func() {
		t.Logf("Killing server via cancel")
		cancel()
	})
	<-ctx.Done()
}
<commit_msg>include clock<commit_after>package outgoing_test

import (
	"os"
	"strconv"
	"testing"
	"time"

	"github.com/lestrrat/roccaforte/client"
	"github.com/lestrrat/roccaforte/event"
	"github.com/lestrrat/roccaforte/incoming"
	"github.com/lestrrat/roccaforte/outgoing"
	"github.com/stretchr/testify/assert"
	"github.com/WatchBeam/clock"
	"golang.org/x/net/context"
)

var projectID string

func init() {
	projectID = os.Getenv("DATASTORE_PROJECT_ID")
}

func TestOutgoing(t *testing.T) {
	if projectID == "" {
		t.Skip("missing project ID. please set DATASTORE_PROJECT_ID")
		return
	}

	// Enqueue
	ctx, cancel := context.WithCancel(context.Background())

	h := incoming.NewHTTPSource()
	s := incoming.NewGDatastoreStorage(projectID)
	e := incoming.New()
	e.Storage = s
	e.AddSource(h)

	go func() {
		assert.NoError(t, e.Run(ctx), "incoming engine should exit w/o errors")
	}()

	go h.Loop(ctx)
	for i := 0; i < 10; i++ {
		e.SetRule("test.notify" + strconv.Itoa(i), &incoming.Rule{})
	}

	for i := 0; i < 10; i++ {
		eventName := "test.notify" + strconv.Itoa(i)
		go func() {
			cl := client.NewHTTP("http://localhost:8080/enqueue")
			events := make([]event.Event, 100)
			for j := 0; j < 100; j++ {
				events[j] = event.NewCoreAttrs(eventName)
			}
			if !assert.NoError(t, cl.Enqueue(events...), "enqueue should succeed") {
				return
			}
		}()
	}

	o := outgoing.New(projectID)
	o.Clock = clock.NewMockClock(time.Now().Add(time.Hour))
	go o.Run(ctx)

	time.AfterFunc(10*time.Second, func() {
		t.Logf("Killing server via cancel")
		cancel()
	})
	<-ctx.Done()
}
<|endoftext|>


<commit_before>package docomo

import (
	"net/http"
	"os"
	"io"
	"mime/multipart"
	"bytes"
)

const DOMAIN_URL = "https://api.apigw.smt.docomo.ne.jp"

// docomoURL
const DIALOGUE_URL = "/dialogue/v1/dialogue"
const IMAGE_URL = "/characterRecognition/v1/document"

type DocomoClient struct {
	client  *http.Client
	domain  string
	apiKey  string
	context string
}

func New() *DocomoClient {
	// TODO: Commandの引数にする
	apiKey := os.Getenv("DOCOMO_APIKEY")

	return &DocomoClient{
		client:  http.DefaultClient,
		domain:  DOMAIN_URL,
		apiKey:  apiKey,
		context: "",
	}
}

func (d *DocomoClient) PostJSON(docomoURL string, body io.Reader) (resp *http.Response, err error) {
	return d.client.Post(d.domain + docomoURL + "?APIKEY=" + d.apiKey, "application/json", body)
}

func (d *DocomoClient) PostImage(file string) (resp *http.Response, err error) {

	var b bytes.Buffer
	w := multipart.NewWriter(&b)
	// Add your image file
	f, err := os.Open(file)
	if err != nil {
		return
	}
	fw, err := w.CreateFormFile("image", file)
	if err != nil {
		return
	}
	if _, err = io.Copy(fw, f); err != nil {
		return
	}
//	// Add the other fields
//	if fw, err = w.CreateFormField("lang"); err != nil {
//		return
//	}
//	if _, err = fw.Write([]byte("png")); err != nil {
//		return
//	}
	// Don't forget to close the multipart writer.
	// If you don't close it, your request will be missing the terminating boundary.
	w.Close()

	// Now that you have a form, you can submit it to your handler.
	req, err := http.NewRequest("POST", d.domain + IMAGE_URL + "?APIKEY=" + d.apiKey, &b)
	if err != nil {
		return
	}
	// Don't forget to set the content type, this will contain the boundary.
	req.Header.Set("Content-Type", w.FormDataContentType())

	// Submit the request
	client := &http.Client{}
	res, err := client.Do(req)
	if err != nil {
		return
	}

	return res, nil
}
<commit_msg>なんかうまくいかないので一旦辞める<commit_after>package docomo

import (
	"net/http"
	"os"
	"io"
	"mime/multipart"
	"bytes"
	"fmt"
)

const DOMAIN_URL = "https://api.apigw.smt.docomo.ne.jp"

// docomoURL
const DIALOGUE_URL = "/dialogue/v1/dialogue"
const IMAGE_URL = "/characterRecognition/v1/document"

type DocomoClient struct {
	client  *http.Client
	domain  string
	apiKey  string
	context string
}

func New() *DocomoClient {
	// TODO: Commandの引数にする
	apiKey := os.Getenv("DOCOMO_APIKEY")

	return &DocomoClient{
		client:  http.DefaultClient,
		domain:  DOMAIN_URL,
		apiKey:  apiKey,
		context: "",
	}
}

func (d *DocomoClient) PostJSON(docomoURL string, body io.Reader) (resp *http.Response, err error) {
	return d.client.Post(d.domain + docomoURL + "?APIKEY=" + d.apiKey, "application/json", body)
}

func (d *DocomoClient) PostImage(file string) (resp *http.Response, err error) {

	var b bytes.Buffer
	w := multipart.NewWriter(&b)
	// Add your image file
	f, err := os.Open(file)
	if err != nil {
		return
	}
	fw, err := w.CreateFormFile("image", file)
	if err != nil {
		return
	}
	if _, err = io.Copy(fw, f); err != nil {
		return
	}

	// Add the other fields
	if fw, err = w.CreateFormField("lang"); err != nil {
		return
	}
	if _, err = fw.Write([]byte("eng")); err != nil {
		return
	}
	// Don't forget to close the multipart writer.
	// If you don't close it, your request will be missing the terminating boundary.
	w.Close()

	// Now that you have a form, you can submit it to your handler.
	req, err := http.NewRequest("POST", d.domain + IMAGE_URL + "?APIKEY=" + d.apiKey, &b)
	if err != nil {
		return
	}

	fmt.Println("ContentLength = ", req.ContentLength)

	// Don't forget to set the content type, this will contain the boundary.
	req.Header.Set("Content-Type", w.FormDataContentType())

	// Submit the request
	client := &http.Client{}
	res, err := client.Do(req)
	if err != nil {
		return
	}

	return res, nil
}
<|endoftext|>


<commit_before>// +build ignore

package main

import (
	"bufio"
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"sort"
	"strings"

	"github.com/spf13/pflag"
)

var opts = struct {
	Version string

	IgnoreBranchName         bool
	IgnoreUncommittedChanges bool
	IgnoreChangelogVersion   bool

	tarFilename string
	buildDir    string
}{}

var versionRegex = regexp.MustCompile(`^\d+\.\d+\.\d+$`)

func init() {
	pflag.BoolVar(&opts.IgnoreBranchName, "ignore-branch-name", false, "allow releasing from other branches as 'master'")
	pflag.BoolVar(&opts.IgnoreUncommittedChanges, "ignore-uncommitted-changes", false, "allow uncommitted changes")
	pflag.BoolVar(&opts.IgnoreChangelogVersion, "ignore-changelgo-version", false, "ignore missing entry in CHANGELOG.md")
	pflag.Parse()
}

func die(f string, args ...interface{}) {
	if !strings.HasSuffix(f, "\n") {
		f += "\n"
	}
	f = "\x1b[31m" + f + "\x1b[0m"
	fmt.Fprintf(os.Stderr, f, args...)
	os.Exit(1)
}

func msg(f string, args ...interface{}) {
	if !strings.HasSuffix(f, "\n") {
		f += "\n"
	}
	f = "\x1b[32m" + f + "\x1b[0m"
	fmt.Printf(f, args...)
}

func run(cmd string, args ...string) {
	c := exec.Command(cmd, args...)
	c.Stdout = os.Stdout
	c.Stderr = os.Stderr
	err := c.Run()
	if err != nil {
		die("error running %s %s: %v", cmd, args, err)
	}
}

func rm(file string) {
	err := os.Remove(file)
	if err != nil {
		die("error removing %v: %v", file, err)
	}
}

func rmdir(dir string) {
	err := os.RemoveAll(dir)
	if err != nil {
		die("error removing %v: %v", dir, err)
	}
}

func mkdir(dir string) {
	err := os.Mkdir(dir, 0755)
	if err != nil {
		die("mkdir %v: %v", dir, err)
	}
}

func getwd() string {
	pwd, err := os.Getwd()
	if err != nil {
		die("Getwd(): %v", err)
	}
	return pwd
}

func uncommittedChanges(dirs ...string) string {
	args := []string{"status", "--porcelain", "--untracked-files=no"}
	if len(dirs) > 0 {
		args = append(args, dirs...)
	}

	changes, err := exec.Command("git", args...).Output()
	if err != nil {
		die("unable to run command: %v", err)
	}

	return string(changes)
}

func preCheckBranchMaster() {
	if opts.IgnoreBranchName {
		return
	}

	branch, err := exec.Command("git", "rev-parse", "--abbrev-ref", "HEAD").Output()
	if err != nil {
		die("error running 'git': %v", err)
	}

	if strings.TrimSpace(string(branch)) != "master" {
		die("wrong branch: %s", branch)
	}
}

func preCheckUncommittedChanges() {
	if opts.IgnoreUncommittedChanges {
		return
	}

	changes := uncommittedChanges()
	if len(changes) > 0 {
		die("uncommited changes found:\n%s\n", changes)
	}
}

func preCheckVersionExists() {
	buf, err := exec.Command("git", "tag", "-l").Output()
	if err != nil {
		die("error running 'git tag -l': %v", err)
	}

	sc := bufio.NewScanner(bytes.NewReader(buf))
	for sc.Scan() {
		if sc.Err() != nil {
			die("error scanning version tags: %v", sc.Err())
		}

		if strings.TrimSpace(sc.Text()) == "v"+opts.Version {
			die("tag v%v already exists", opts.Version)
		}
	}
}

func preCheckChangelog() {
	if opts.IgnoreChangelogVersion {
		return
	}

	f, err := os.Open("CHANGELOG.md")
	if err != nil {
		die("unable to open CHANGELOG.md: %v", err)
	}
	defer f.Close()

	sc := bufio.NewScanner(f)
	for sc.Scan() {
		if sc.Err() != nil {
			die("error scanning: %v", sc.Err())
		}

		if strings.TrimSpace(sc.Text()) == fmt.Sprintf("Important Changes in %v", opts.Version) {
			return
		}
	}

	die("CHANGELOG.md does not contain version %v", opts.Version)
}

func generateFiles() {
	msg("generate files")
	run("go", "run", "build.go", "-o", "restic-generate.temp")

	mandir := filepath.Join("doc", "man")
	rmdir(mandir)
	mkdir(mandir)
	run("./restic-generate.temp", "generate",
		"--man", "doc/man",
		"--zsh-completion", "doc/zsh-completion.zsh",
		"--bash-completion", "doc/bash-completion.sh")
	rm("restic-generate.temp")

	changes := uncommittedChanges("doc")
	if len(changes) > 0 {
		msg("comitting manpages and auto-completion")
		run("git", "commit", "-m", "Update manpages and auto-completion", "doc")
	}
}

func updateVersion() {
	err := ioutil.WriteFile("VERSION", []byte(opts.Version), 0644)
	if err != nil {
		die("unable to write version to file: %v", err)
	}

	if len(uncommittedChanges("VERSION")) > 0 {
		msg("comitting file VERSION")
		run("git", "commit", "-m", fmt.Sprintf("Add VERSION for %v", opts.Version), "VERSION")
	}
}

func addTag() {
	tagname := "v" + opts.Version
	msg("add tag %v", tagname)
	run("git", "tag", "-a", "-s", "-m", tagname, tagname)
}

func exportTar() {
	cmd := fmt.Sprintf("git archive --format=tar --prefix=restic-%s/ v%s | gzip -n > %s",
		opts.Version, opts.Version, opts.tarFilename)
	run("sh", "-c", cmd)
	msg("build restic-%s.tar.gz", opts.Version)
}

func runBuild() {
	msg("building binaries...")
	run("docker", "pull", "restic/builder")
	run("docker", "run", "--volume", getwd()+":/home/build", "restic/builder", opts.tarFilename)
}

func findBuildDir() string {
	nameRegex := regexp.MustCompile(`restic-` + opts.Version + `-\d{8}-\d{6}`)

	f, err := os.Open(".")
	if err != nil {
		die("Open(.): %v", err)
	}

	entries, err := f.Readdirnames(-1)
	if err != nil {
		die("Readdirnames(): %v", err)
	}

	err = f.Close()
	if err != nil {
		die("Close(): %v", err)
	}

	sort.Slice(entries, func(i, j int) bool {
		return entries[j] < entries[i]
	})

	for _, entry := range entries {
		if nameRegex.MatchString(entry) {
			msg("found restic build dir: %v", entry)
			return entry
		}
	}

	die("restic build dir not found")
	return ""
}

func signFiles() {
	run("gpg", "--armor", "--detach-sign", filepath.Join(opts.buildDir, "SHA256SUMS"))
	run("gpg", "--armor", "--detach-sign", filepath.Join(opts.buildDir, opts.tarFilename))
}

func updateDocker() {
	cmd := fmt.Sprintf("bzcat %s/restic_%s_linux_amd64.bz2 > restic", opts.buildDir, opts.Version)
	run("sh", "-c", cmd)
	run("chmod", "+x", "restic")
	run("docker", "build", "--rm", "--tag", "restic/restic:latest", "-f", "docker/Dockerfile", ".")
}

func main() {
	if len(pflag.Args()) == 0 {
		die("USAGE: release-version [OPTIONS] VERSION")
	}

	opts.Version = pflag.Args()[0]
	if !versionRegex.MatchString(opts.Version) {
		die("invalid new version")
	}

	opts.tarFilename = fmt.Sprintf("restic-%s.tar.gz", opts.Version)

	preCheckBranchMaster()
	preCheckUncommittedChanges()
	preCheckVersionExists()
	preCheckChangelog()

	generateFiles()
	updateVersion()
	addTag()

	exportTar()
	runBuild()
	opts.buildDir = findBuildDir()
	signFiles()

	updateDocker()

	msg("done, build dir is %v", opts.buildDir)

	msg("now run:\n\ngit push --tags origin master\ndocker push restic/restic\n")
}
<commit_msg>Correct typos<commit_after>// +build ignore

package main

import (
	"bufio"
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"sort"
	"strings"

	"github.com/spf13/pflag"
)

var opts = struct {
	Version string

	IgnoreBranchName         bool
	IgnoreUncommittedChanges bool
	IgnoreChangelogVersion   bool

	tarFilename string
	buildDir    string
}{}

var versionRegex = regexp.MustCompile(`^\d+\.\d+\.\d+$`)

func init() {
	pflag.BoolVar(&opts.IgnoreBranchName, "ignore-branch-name", false, "allow releasing from other branches as 'master'")
	pflag.BoolVar(&opts.IgnoreUncommittedChanges, "ignore-uncommitted-changes", false, "allow uncommitted changes")
	pflag.BoolVar(&opts.IgnoreChangelogVersion, "ignore-changelgo-version", false, "ignore missing entry in CHANGELOG.md")
	pflag.Parse()
}

func die(f string, args ...interface{}) {
	if !strings.HasSuffix(f, "\n") {
		f += "\n"
	}
	f = "\x1b[31m" + f + "\x1b[0m"
	fmt.Fprintf(os.Stderr, f, args...)
	os.Exit(1)
}

func msg(f string, args ...interface{}) {
	if !strings.HasSuffix(f, "\n") {
		f += "\n"
	}
	f = "\x1b[32m" + f + "\x1b[0m"
	fmt.Printf(f, args...)
}

func run(cmd string, args ...string) {
	c := exec.Command(cmd, args...)
	c.Stdout = os.Stdout
	c.Stderr = os.Stderr
	err := c.Run()
	if err != nil {
		die("error running %s %s: %v", cmd, args, err)
	}
}

func rm(file string) {
	err := os.Remove(file)
	if err != nil {
		die("error removing %v: %v", file, err)
	}
}

func rmdir(dir string) {
	err := os.RemoveAll(dir)
	if err != nil {
		die("error removing %v: %v", dir, err)
	}
}

func mkdir(dir string) {
	err := os.Mkdir(dir, 0755)
	if err != nil {
		die("mkdir %v: %v", dir, err)
	}
}

func getwd() string {
	pwd, err := os.Getwd()
	if err != nil {
		die("Getwd(): %v", err)
	}
	return pwd
}

func uncommittedChanges(dirs ...string) string {
	args := []string{"status", "--porcelain", "--untracked-files=no"}
	if len(dirs) > 0 {
		args = append(args, dirs...)
	}

	changes, err := exec.Command("git", args...).Output()
	if err != nil {
		die("unable to run command: %v", err)
	}

	return string(changes)
}

func preCheckBranchMaster() {
	if opts.IgnoreBranchName {
		return
	}

	branch, err := exec.Command("git", "rev-parse", "--abbrev-ref", "HEAD").Output()
	if err != nil {
		die("error running 'git': %v", err)
	}

	if strings.TrimSpace(string(branch)) != "master" {
		die("wrong branch: %s", branch)
	}
}

func preCheckUncommittedChanges() {
	if opts.IgnoreUncommittedChanges {
		return
	}

	changes := uncommittedChanges()
	if len(changes) > 0 {
		die("uncommitted changes found:\n%s\n", changes)
	}
}

func preCheckVersionExists() {
	buf, err := exec.Command("git", "tag", "-l").Output()
	if err != nil {
		die("error running 'git tag -l': %v", err)
	}

	sc := bufio.NewScanner(bytes.NewReader(buf))
	for sc.Scan() {
		if sc.Err() != nil {
			die("error scanning version tags: %v", sc.Err())
		}

		if strings.TrimSpace(sc.Text()) == "v"+opts.Version {
			die("tag v%v already exists", opts.Version)
		}
	}
}

func preCheckChangelog() {
	if opts.IgnoreChangelogVersion {
		return
	}

	f, err := os.Open("CHANGELOG.md")
	if err != nil {
		die("unable to open CHANGELOG.md: %v", err)
	}
	defer f.Close()

	sc := bufio.NewScanner(f)
	for sc.Scan() {
		if sc.Err() != nil {
			die("error scanning: %v", sc.Err())
		}

		if strings.TrimSpace(sc.Text()) == fmt.Sprintf("Important Changes in %v", opts.Version) {
			return
		}
	}

	die("CHANGELOG.md does not contain version %v", opts.Version)
}

func generateFiles() {
	msg("generate files")
	run("go", "run", "build.go", "-o", "restic-generate.temp")

	mandir := filepath.Join("doc", "man")
	rmdir(mandir)
	mkdir(mandir)
	run("./restic-generate.temp", "generate",
		"--man", "doc/man",
		"--zsh-completion", "doc/zsh-completion.zsh",
		"--bash-completion", "doc/bash-completion.sh")
	rm("restic-generate.temp")

	changes := uncommittedChanges("doc")
	if len(changes) > 0 {
		msg("committing manpages and auto-completion")
		run("git", "commit", "-m", "Update manpages and auto-completion", "doc")
	}
}

func updateVersion() {
	err := ioutil.WriteFile("VERSION", []byte(opts.Version), 0644)
	if err != nil {
		die("unable to write version to file: %v", err)
	}

	if len(uncommittedChanges("VERSION")) > 0 {
		msg("committing file VERSION")
		run("git", "commit", "-m", fmt.Sprintf("Add VERSION for %v", opts.Version), "VERSION")
	}
}

func addTag() {
	tagname := "v" + opts.Version
	msg("add tag %v", tagname)
	run("git", "tag", "-a", "-s", "-m", tagname, tagname)
}

func exportTar() {
	cmd := fmt.Sprintf("git archive --format=tar --prefix=restic-%s/ v%s | gzip -n > %s",
		opts.Version, opts.Version, opts.tarFilename)
	run("sh", "-c", cmd)
	msg("build restic-%s.tar.gz", opts.Version)
}

func runBuild() {
	msg("building binaries...")
	run("docker", "pull", "restic/builder")
	run("docker", "run", "--volume", getwd()+":/home/build", "restic/builder", opts.tarFilename)
}

func findBuildDir() string {
	nameRegex := regexp.MustCompile(`restic-` + opts.Version + `-\d{8}-\d{6}`)

	f, err := os.Open(".")
	if err != nil {
		die("Open(.): %v", err)
	}

	entries, err := f.Readdirnames(-1)
	if err != nil {
		die("Readdirnames(): %v", err)
	}

	err = f.Close()
	if err != nil {
		die("Close(): %v", err)
	}

	sort.Slice(entries, func(i, j int) bool {
		return entries[j] < entries[i]
	})

	for _, entry := range entries {
		if nameRegex.MatchString(entry) {
			msg("found restic build dir: %v", entry)
			return entry
		}
	}

	die("restic build dir not found")
	return ""
}

func signFiles() {
	run("gpg", "--armor", "--detach-sign", filepath.Join(opts.buildDir, "SHA256SUMS"))
	run("gpg", "--armor", "--detach-sign", filepath.Join(opts.buildDir, opts.tarFilename))
}

func updateDocker() {
	cmd := fmt.Sprintf("bzcat %s/restic_%s_linux_amd64.bz2 > restic", opts.buildDir, opts.Version)
	run("sh", "-c", cmd)
	run("chmod", "+x", "restic")
	run("docker", "build", "--rm", "--tag", "restic/restic:latest", "-f", "docker/Dockerfile", ".")
}

func main() {
	if len(pflag.Args()) == 0 {
		die("USAGE: release-version [OPTIONS] VERSION")
	}

	opts.Version = pflag.Args()[0]
	if !versionRegex.MatchString(opts.Version) {
		die("invalid new version")
	}

	opts.tarFilename = fmt.Sprintf("restic-%s.tar.gz", opts.Version)

	preCheckBranchMaster()
	preCheckUncommittedChanges()
	preCheckVersionExists()
	preCheckChangelog()

	generateFiles()
	updateVersion()
	addTag()

	exportTar()
	runBuild()
	opts.buildDir = findBuildDir()
	signFiles()

	updateDocker()

	msg("done, build dir is %v", opts.buildDir)

	msg("now run:\n\ngit push --tags origin master\ndocker push restic/restic\n")
}
<|endoftext|>