<commit_before>package main

import (
	"exec"
	"flag"
	"fmt"
	"github.com/hanwen/termite/termite"
	"log"
	"os"
	"path/filepath"
	"rpc"
	"strings"
)

// TODO - this file is a mess. Clean it up.
const _TIMEOUT = 10e9

var socketRpc *rpc.Client
var topDir string

func Rpc() *rpc.Client {
	if socketRpc == nil {
		socket := termite.FindSocket()
		if socket == "" {
			log.Fatal("Could not find .termite-socket")
		}
		topDir, _ = filepath.Split(socket)
		topDir = filepath.Clean(topDir)
		conn := termite.OpenSocketConnection(socket, termite.RPC_CHANNEL, _TIMEOUT)
		socketRpc = rpc.NewClient(conn)
	}
	return socketRpc
}

func TryRunDirect(req *termite.WorkRequest) {
	if req.Argv[0] == "echo" {
		fmt.Println(strings.Join(req.Argv[1:], " "))
		os.Exit(0)
	}
	if req.Argv[0] == "true" {
		os.Exit(0)
	}
	if req.Argv[0] == "false" {
		os.Exit(1)
	}
}

var bashInternals = []string{
	"alias", "bg", "bind", "break", "builtin", "caller", "case", "cd",
	"command", "compgen", "complete", "compopt", "continue", "coproc",
	"declare", "dirs", "disown", /* echo, */ "enable", "eval", "exec", "exit",
	"export", "false", "fc", "fg", "for", "for", "function", "getopts",
	"hash", "help", "history", "if", "jobs", "kill", "let", "local",
	"logout", "mapfile", "popd", "printf", "pushd", "pwd", "read",
	"readarray", "readonly", "return", "select", "set", "shift", "shopt",
	"source", "suspend", "test", "time", "times", "trap", "true", "type",
	"typeset", "ulimit", "umask", "unalias", "unset", "until",
	"variables", "wait", "while",
}

func NewWorkRequest(cmd string, dir string, topdir string) *termite.WorkRequest {
	req := &termite.WorkRequest{
		Binary: Shell(),
		Argv:   []string{Shell(), "-c", cmd},
		Env:    cleanEnv(os.Environ()),
		Dir:    dir,
	}

	parsed := termite.ParseCommand(cmd)
	if len(parsed) > 0 {
		// Is this really necessary?
		for _, c := range bashInternals {
			if parsed[0] == c {
				return req
			}
		}

		// A no-frills command invocation: do it directly.
		var err os.Error
		req.Binary, err = exec.LookPath(parsed[0])
		if err != nil {
			log.Fatal("LookPath", err)
		}
		req.Argv = parsed
	}

	return req
}

func PrepareRun(cmd string, dir string, topdir string) (*termite.WorkRequest, *termite.LocalRule) {
	if cmd == ":" || strings.TrimRight(cmd, " ") == "" {
		os.Exit(0)
	}

	req := NewWorkRequest(cmd, dir, topdir)
	TryRunDirect(req)

	decider := termite.NewLocalDecider(topdir)
	rule := decider.ShouldRunLocally(cmd)
	if rule != nil {
		req.Debug = rule.Debug
		return req, rule
	}

	return req, nil
}

func Refresh() {
	req := 1
	rep := 1
	err := Rpc().Call("LocalMaster.RefreshAttributeCache", &req, &rep)
	if err != nil {
		log.Fatal("LocalMaster.RefreshAttributeCache: ", err)
	}
}

func cleanEnv(input []string) []string {
	env := []string{}
	for _, v := range input {
		comps := strings.SplitN(v, "=", 2)
		if comps[1] == "termite-make" {
			// TODO - more generic.
			v = fmt.Sprintf("%s=%s", comps[0], "make")
		} else if comps[0] == "MAKE_SHELL" {
			continue
		}
		env = append(env, v)
	}
	return env
}

func Inspect(files []string) {
	wd, _ := os.Getwd()
	for _, p := range files {
		if p[0] != '/' {
			p = filepath.Join(wd, p)
		}
		p = p[1:]
		req := termite.AttrRequest{Name: p}
		rep := termite.AttrResponse{}
		err := Rpc().Call("LocalMaster.InspectFile", &req, &rep)
		if err != nil {
			log.Fatal("LocalMaster.InspectFile: ", err)
		}

		for _, a := range rep.Attrs {
			log.Printf("%v", *a)
		}
	}
}

func Shell() string {
	shell := os.Getenv("SHELL")
	if shell == "" {
		shell = "/bin/sh"
	}
	return shell
}

func RunLocally(req *termite.WorkRequest, rule *termite.LocalRule) *os.Waitmsg {
	env := os.Environ()
	if !rule.Recurse {
		env = cleanEnv(env)
	}

	proc, err := os.StartProcess(req.Binary, req.Argv, &os.ProcAttr{
		Env:   env,
		Files: []*os.File{os.Stdin, os.Stdout, os.Stderr},
	})
	if err != nil {
		log.Fatalf("os.StartProcess() for %v: %v", req, err)
	}
	msg, err := proc.Wait(0)
	if err != nil {
		log.Fatalf("proc.Wait() for %v: %v", req, err)
	}
	return msg
}

func main() {
	command := flag.String("c", "", "command to run.")
	refresh := flag.Bool("refresh", false, "refresh master file cache.")
	shutdown := flag.Bool("shutdown", false, "shutdown master.")
	inspect := flag.Bool("inspect", false, "inspect files on master.")
	exec := flag.Bool("exec", false, "run command args without shell.")
	directory := flag.String("dir", "", "directory from where to run (default: cwd).")

	debug := flag.Bool("dbg", false, "set on debugging in request.")
	flag.Parse()
	log.SetPrefix("S")

	if *shutdown {
		req := 1
		rep := 1
		err := Rpc().Call("LocalMaster.Shutdown", &req, &rep)
		if err != nil {
			log.Fatal(err)
		}
		return
	}
	if *refresh {
		Refresh()
	}

	if *inspect {
		Inspect(flag.Args())
	}

	if *directory == "" {
		wd, err := os.Getwd()
		if err != nil {
			log.Fatal("Getwd", err)
		}

		directory = &wd
	}

	var req *termite.WorkRequest
	var rule *termite.LocalRule
	if *exec {
		req = &termite.WorkRequest{
			Binary: flag.Args()[0],
			Argv:   flag.Args(),
			Dir:    *directory,
			Env:    os.Environ(),
		}
	} else {
		req, rule = PrepareRun(*command, *directory, topDir)
	}
	var waitMsg *os.Waitmsg
	if rule != nil && rule.Local {
		waitMsg = RunLocally(req, rule)
		if !rule.SkipRefresh {
			Refresh()
		}
	} else {
		req.Debug = req.Debug || os.Getenv("TERMITE_DEBUG") != "" || *debug
		rep := termite.WorkResponse{}
		err := Rpc().Call("LocalMaster.Run", &req, &rep)
		if err != nil {
			log.Fatal("LocalMaster.Run: ", err)
		}

		os.Stdout.Write([]byte(rep.Stdout))
		os.Stderr.Write([]byte(rep.Stderr))

		waitMsg = &rep.Exit
	}

	if waitMsg.ExitStatus() != 0 {
		log.Printf("Failed: '%s'", *command)
	}

	// TODO - is this necessary?
	Rpc().Close()
	os.Exit(waitMsg.ExitStatus())
}
<commit_msg>Print directory contents too.<commit_after>package main

import (
	"exec"
	"flag"
	"fmt"
	"github.com/hanwen/termite/termite"
	"log"
	"os"
	"path/filepath"
	"rpc"
	"strings"
)

// TODO - this file is a mess. Clean it up.
const _TIMEOUT = 10e9

var socketRpc *rpc.Client
var topDir string

func Rpc() *rpc.Client {
	if socketRpc == nil {
		socket := termite.FindSocket()
		if socket == "" {
			log.Fatal("Could not find .termite-socket")
		}
		topDir, _ = filepath.Split(socket)
		topDir = filepath.Clean(topDir)
		conn := termite.OpenSocketConnection(socket, termite.RPC_CHANNEL, _TIMEOUT)
		socketRpc = rpc.NewClient(conn)
	}
	return socketRpc
}

func TryRunDirect(req *termite.WorkRequest) {
	if req.Argv[0] == "echo" {
		fmt.Println(strings.Join(req.Argv[1:], " "))
		os.Exit(0)
	}
	if req.Argv[0] == "true" {
		os.Exit(0)
	}
	if req.Argv[0] == "false" {
		os.Exit(1)
	}
}

var bashInternals = []string{
	"alias", "bg", "bind", "break", "builtin", "caller", "case", "cd",
	"command", "compgen", "complete", "compopt", "continue", "coproc",
	"declare", "dirs", "disown", /* echo, */ "enable", "eval", "exec", "exit",
	"export", "false", "fc", "fg", "for", "for", "function", "getopts",
	"hash", "help", "history", "if", "jobs", "kill", "let", "local",
	"logout", "mapfile", "popd", "printf", "pushd", "pwd", "read",
	"readarray", "readonly", "return", "select", "set", "shift", "shopt",
	"source", "suspend", "test", "time", "times", "trap", "true", "type",
	"typeset", "ulimit", "umask", "unalias", "unset", "until",
	"variables", "wait", "while",
}

func NewWorkRequest(cmd string, dir string, topdir string) *termite.WorkRequest {
	req := &termite.WorkRequest{
		Binary: Shell(),
		Argv:   []string{Shell(), "-c", cmd},
		Env:    cleanEnv(os.Environ()),
		Dir:    dir,
	}

	parsed := termite.ParseCommand(cmd)
	if len(parsed) > 0 {
		// Is this really necessary?
		for _, c := range bashInternals {
			if parsed[0] == c {
				return req
			}
		}

		// A no-frills command invocation: do it directly.
		var err os.Error
		req.Binary, err = exec.LookPath(parsed[0])
		if err != nil {
			log.Fatal("LookPath", err)
		}
		req.Argv = parsed
	}

	return req
}

func PrepareRun(cmd string, dir string, topdir string) (*termite.WorkRequest, *termite.LocalRule) {
	if cmd == ":" || strings.TrimRight(cmd, " ") == "" {
		os.Exit(0)
	}

	req := NewWorkRequest(cmd, dir, topdir)
	TryRunDirect(req)

	decider := termite.NewLocalDecider(topdir)
	rule := decider.ShouldRunLocally(cmd)
	if rule != nil {
		req.Debug = rule.Debug
		return req, rule
	}

	return req, nil
}

func Refresh() {
	req := 1
	rep := 1
	err := Rpc().Call("LocalMaster.RefreshAttributeCache", &req, &rep)
	if err != nil {
		log.Fatal("LocalMaster.RefreshAttributeCache: ", err)
	}
}

func cleanEnv(input []string) []string {
	env := []string{}
	for _, v := range input {
		comps := strings.SplitN(v, "=", 2)
		if comps[1] == "termite-make" {
			// TODO - more generic.
			v = fmt.Sprintf("%s=%s", comps[0], "make")
		} else if comps[0] == "MAKE_SHELL" {
			continue
		}
		env = append(env, v)
	}
	return env
}

func Inspect(files []string) {
	wd, _ := os.Getwd()
	for _, p := range files {
		if p[0] != '/' {
			p = filepath.Join(wd, p)
		}
		p = p[1:]
		req := termite.AttrRequest{Name: p}
		rep := termite.AttrResponse{}
		err := Rpc().Call("LocalMaster.InspectFile", &req, &rep)
		if err != nil {
			log.Fatal("LocalMaster.InspectFile: ", err)
		}

		for _, a := range rep.Attrs {
			log.Printf("%v", *a)
			for n, m := range a.NameModeMap {
				log.Printf(" %q: %o", n, m)
			}
		}
	}
}

func Shell() string {
	shell := os.Getenv("SHELL")
	if shell == "" {
		shell = "/bin/sh"
	}
	return shell
}

func RunLocally(req *termite.WorkRequest, rule *termite.LocalRule) *os.Waitmsg {
	env := os.Environ()
	if !rule.Recurse {
		env = cleanEnv(env)
	}

	proc, err := os.StartProcess(req.Binary, req.Argv, &os.ProcAttr{
		Env:   env,
		Files: []*os.File{os.Stdin, os.Stdout, os.Stderr},
	})
	if err != nil {
		log.Fatalf("os.StartProcess() for %v: %v", req, err)
	}
	msg, err := proc.Wait(0)
	if err != nil {
		log.Fatalf("proc.Wait() for %v: %v", req, err)
	}
	return msg
}

func main() {
	command := flag.String("c", "", "command to run.")
	refresh := flag.Bool("refresh", false, "refresh master file cache.")
	shutdown := flag.Bool("shutdown", false, "shutdown master.")
	inspect := flag.Bool("inspect", false, "inspect files on master.")
	exec := flag.Bool("exec", false, "run command args without shell.")
	directory := flag.String("dir", "", "directory from where to run (default: cwd).")

	debug := flag.Bool("dbg", false, "set on debugging in request.")
	flag.Parse()
	log.SetPrefix("S")

	if *shutdown {
		req := 1
		rep := 1
		err := Rpc().Call("LocalMaster.Shutdown", &req, &rep)
		if err != nil {
			log.Fatal(err)
		}
		return
	}
	if *refresh {
		Refresh()
	}

	if *inspect {
		Inspect(flag.Args())
	}

	if *directory == "" {
		wd, err := os.Getwd()
		if err != nil {
			log.Fatal("Getwd", err)
		}

		directory = &wd
	}

	var req *termite.WorkRequest
	var rule *termite.LocalRule
	if *exec {
		req = &termite.WorkRequest{
			Binary: flag.Args()[0],
			Argv:   flag.Args(),
			Dir:    *directory,
			Env:    os.Environ(),
		}
	} else {
		req, rule = PrepareRun(*command, *directory, topDir)
	}
	var waitMsg *os.Waitmsg
	if rule != nil && rule.Local {
		waitMsg = RunLocally(req, rule)
		if !rule.SkipRefresh {
			Refresh()
		}
	} else {
		req.Debug = req.Debug || os.Getenv("TERMITE_DEBUG") != "" || *debug
		rep := termite.WorkResponse{}
		err := Rpc().Call("LocalMaster.Run", &req, &rep)
		if err != nil {
			log.Fatal("LocalMaster.Run: ", err)
		}

		os.Stdout.Write([]byte(rep.Stdout))
		os.Stderr.Write([]byte(rep.Stderr))

		waitMsg = &rep.Exit
	}

	if waitMsg.ExitStatus() != 0 {
		log.Printf("Failed: '%s'", *command)
	}

	// TODO - is this necessary?
	Rpc().Close()
	os.Exit(waitMsg.ExitStatus())
}
<|endoftext|>


<commit_before>package api

import (
	"fmt"
	"io"
	"net/http"
	"time"

	"github.com/exercism/cli/debug"
)

var (
	// UserAgent lets the API know where the call is being made from.
	// It's overridden from the root command so that we can set the version.
	UserAgent = "github.com/exercism/cli"

	// DefaultHTTPClient configures a timeout to use by default.
	DefaultHTTPClient = &http.Client{Timeout: 10 * time.Second}
)

// Client is an http client that is configured for Exercism.
type Client struct {
	*http.Client
	ContentType string
	Token       string
	BaseURL     string
}

// NewClient returns an Exercism API client.
func NewClient(token, baseURL string) (*Client, error) {
	return &Client{
		Client:  DefaultHTTPClient,
		Token:   token,
		BaseURL: baseURL,
	}, nil
}

// NewRequest returns an http.Request with information for the Exercism API.
func (c *Client) NewRequest(method, url string, body io.Reader) (*http.Request, error) {
	if c.Client == nil {
		c.Client = DefaultHTTPClient
	}

	req, err := http.NewRequest(method, url, body)
	if err != nil {
		return nil, err
	}

	req.Header.Set("User-Agent", UserAgent)
	if c.ContentType == "" {
		req.Header.Set("Content-Type", "application/json")
	} else {
		req.Header.Set("Content-Type", c.ContentType)
	}
	if c.Token != "" {
		req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", c.Token))
	}

	return req, nil
}

// Do performs an http.Request and optionally parses the response body into the given interface.
func (c *Client) Do(req *http.Request) (*http.Response, error) {
	debug.DumpRequest(req)

	res, err := c.Client.Do(req)
	if err != nil {
		return nil, err
	}
	debug.DumpResponse(res)
	return res, nil
}

// ValidateToken calls the API to determine whether the token is valid.
func (c *Client) ValidateToken() error {
	url := fmt.Sprintf("%s/validate_token", c.BaseURL)
	req, err := c.NewRequest("GET", url, nil)
	if err != nil {
		return err
	}
	_, err = c.Do(req)

	return err
}
<commit_msg>Remove optional unmarshal in client.Do()<commit_after>package api

import (
	"fmt"
	"io"
	"net/http"
	"time"

	"github.com/exercism/cli/debug"
)

var (
	// UserAgent lets the API know where the call is being made from.
	// It's overridden from the root command so that we can set the version.
	UserAgent = "github.com/exercism/cli"

	// DefaultHTTPClient configures a timeout to use by default.
	DefaultHTTPClient = &http.Client{Timeout: 10 * time.Second}
)

// Client is an http client that is configured for Exercism.
type Client struct {
	*http.Client
	ContentType string
	Token       string
	BaseURL     string
}

// NewClient returns an Exercism API client.
func NewClient(token, baseURL string) (*Client, error) {
	return &Client{
		Client:  DefaultHTTPClient,
		Token:   token,
		BaseURL: baseURL,
	}, nil
}

// NewRequest returns an http.Request with information for the Exercism API.
func (c *Client) NewRequest(method, url string, body io.Reader) (*http.Request, error) {
	if c.Client == nil {
		c.Client = DefaultHTTPClient
	}

	req, err := http.NewRequest(method, url, body)
	if err != nil {
		return nil, err
	}

	req.Header.Set("User-Agent", UserAgent)
	if c.ContentType == "" {
		req.Header.Set("Content-Type", "application/json")
	} else {
		req.Header.Set("Content-Type", c.ContentType)
	}
	if c.Token != "" {
		req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", c.Token))
	}

	return req, nil
}

// Do performs an http.Request and optionally parses the response body into the given interface.
func (c *Client) Do(req *http.Request) (*http.Response, error) {
	debug.DumpRequest(req)

	res, err := c.Client.Do(req)
	if err != nil {
		return nil, err
	}

	debug.DumpResponse(res)
	return res, nil
}

// ValidateToken calls the API to determine whether the token is valid.
func (c *Client) ValidateToken() error {
	url := fmt.Sprintf("%s/validate_token", c.BaseURL)
	req, err := c.NewRequest("GET", url, nil)
	if err != nil {
		return err
	}
	_, err = c.Do(req)

	return err
}
<|endoftext|>


<commit_before>// Copyright 2012 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build cgo

package runtime_test

import (
	"bytes"
	"fmt"
	"internal/testenv"
	"os"
	"os/exec"
	"runtime"
	"strings"
	"testing"
	"time"
)

func TestCgoCrashHandler(t *testing.T) {
	testCrashHandler(t, true)
}

func TestCgoSignalDeadlock(t *testing.T) {
	if testing.Short() && runtime.GOOS == "windows" {
		t.Skip("Skipping in short mode") // takes up to 64 seconds
	}
	got := runTestProg(t, "testprogcgo", "CgoSignalDeadlock")
	want := "OK\n"
	if got != want {
		t.Fatalf("expected %q, but got:\n%s", want, got)
	}
}

func TestCgoTraceback(t *testing.T) {
	got := runTestProg(t, "testprogcgo", "CgoTraceback")
	want := "OK\n"
	if got != want {
		t.Fatalf("expected %q, but got:\n%s", want, got)
	}
}

func TestCgoCallbackGC(t *testing.T) {
	if runtime.GOOS == "plan9" || runtime.GOOS == "windows" {
		t.Skipf("no pthreads on %s", runtime.GOOS)
	}
	if testing.Short() {
		switch {
		case runtime.GOOS == "dragonfly":
			t.Skip("see golang.org/issue/11990")
		case runtime.GOOS == "linux" && runtime.GOARCH == "arm":
			t.Skip("too slow for arm builders")
		case runtime.GOOS == "linux" && (runtime.GOARCH == "mips64" || runtime.GOARCH == "mips64le"):
			t.Skip("too slow for mips64x builders")
		}
	}
	got := runTestProg(t, "testprogcgo", "CgoCallbackGC")
	want := "OK\n"
	if got != want {
		t.Fatalf("expected %q, but got:\n%s", want, got)
	}
}

func TestCgoExternalThreadPanic(t *testing.T) {
	if runtime.GOOS == "plan9" {
		t.Skipf("no pthreads on %s", runtime.GOOS)
	}
	got := runTestProg(t, "testprogcgo", "CgoExternalThreadPanic")
	want := "panic: BOOM"
	if !strings.Contains(got, want) {
		t.Fatalf("want failure containing %q. output:\n%s\n", want, got)
	}
}

func TestCgoExternalThreadSIGPROF(t *testing.T) {
	// issue 9456.
	switch runtime.GOOS {
	case "plan9", "windows":
		t.Skipf("no pthreads on %s", runtime.GOOS)
	case "darwin":
		if runtime.GOARCH != "arm" && runtime.GOARCH != "arm64" {
			// static constructor needs external linking, but we don't support
			// external linking on OS X 10.6.
			out, err := exec.Command("uname", "-r").Output()
			if err != nil {
				t.Fatalf("uname -r failed: %v", err)
			}
			// OS X 10.6 == Darwin 10.x
			if strings.HasPrefix(string(out), "10.") {
				t.Skipf("no external linking on OS X 10.6")
			}
		}
	}
	if runtime.GOARCH == "ppc64" {
		// TODO(austin) External linking not implemented on
		// ppc64 (issue #8912)
		t.Skipf("no external linking on ppc64")
	}
	got := runTestProg(t, "testprogcgo", "CgoExternalThreadSIGPROF")
	want := "OK\n"
	if got != want {
		t.Fatalf("expected %q, but got:\n%s", want, got)
	}
}

func TestCgoExternalThreadSignal(t *testing.T) {
	// issue 10139
	switch runtime.GOOS {
	case "plan9", "windows":
		t.Skipf("no pthreads on %s", runtime.GOOS)
	}
	got := runTestProg(t, "testprogcgo", "CgoExternalThreadSignal")
	want := "OK\n"
	if got != want {
		t.Fatalf("expected %q, but got:\n%s", want, got)
	}
}

func TestCgoDLLImports(t *testing.T) {
	// test issue 9356
	if runtime.GOOS != "windows" {
		t.Skip("skipping windows specific test")
	}
	got := runTestProg(t, "testprogcgo", "CgoDLLImportsMain")
	want := "OK\n"
	if got != want {
		t.Fatalf("expected %q, but got %v", want, got)
	}
}

func TestCgoExecSignalMask(t *testing.T) {
	// Test issue 13164.
	switch runtime.GOOS {
	case "windows", "plan9":
		t.Skipf("skipping signal mask test on %s", runtime.GOOS)
	}
	got := runTestProg(t, "testprogcgo", "CgoExecSignalMask")
	want := "OK\n"
	if got != want {
		t.Errorf("expected %q, got %v", want, got)
	}
}

func TestEnsureDropM(t *testing.T) {
	// Test for issue 13881.
	switch runtime.GOOS {
	case "windows", "plan9":
		t.Skipf("skipping dropm test on %s", runtime.GOOS)
	}
	got := runTestProg(t, "testprogcgo", "EnsureDropM")
	want := "OK\n"
	if got != want {
		t.Errorf("expected %q, got %v", want, got)
	}
}

// Test for issue 14387.
// Test that the program that doesn't need any cgo pointer checking
// takes about the same amount of time with it as without it.
func TestCgoCheckBytes(t *testing.T) {
	// Make sure we don't count the build time as part of the run time.
	testenv.MustHaveGoBuild(t)
	exe, err := buildTestProg(t, "testprogcgo")
	if err != nil {
		t.Fatal(err)
	}

	// Try it 10 times to avoid flakiness.
	const tries = 10
	var tot1, tot2 time.Duration
	for i := 0; i < tries; i++ {
		cmd := testEnv(exec.Command(exe, "CgoCheckBytes"))
		cmd.Env = append(cmd.Env, "GODEBUG=cgocheck=0", fmt.Sprintf("GO_CGOCHECKBYTES_TRY=%d", i))

		start := time.Now()
		cmd.Run()
		d1 := time.Since(start)

		cmd = testEnv(exec.Command(exe, "CgoCheckBytes"))
		cmd.Env = append(cmd.Env, fmt.Sprintf("GO_CGOCHECKBYTES_TRY=%d", i))

		start = time.Now()
		cmd.Run()
		d2 := time.Since(start)

		if d1*20 > d2 {
			// The slow version (d2) was less than 20 times
			// slower than the fast version (d1), so OK.
			return
		}

		tot1 += d1
		tot2 += d2
	}

	t.Errorf("cgo check too slow: got %v, expected at most %v", tot2/tries, (tot1/tries)*20)
}

func TestCgoPanicDeadlock(t *testing.T) {
	// test issue 14432
	got := runTestProg(t, "testprogcgo", "CgoPanicDeadlock")
	want := "panic: cgo error\n\n"
	if !strings.HasPrefix(got, want) {
		t.Fatalf("output does not start with %q:\n%s", want, got)
	}
}

func TestCgoCCodeSIGPROF(t *testing.T) {
	got := runTestProg(t, "testprogcgo", "CgoCCodeSIGPROF")
	want := "OK\n"
	if got != want {
		t.Errorf("expected %q got %v", want, got)
	}
}

func TestCgoCrashTraceback(t *testing.T) {
	if runtime.GOOS != "linux" || runtime.GOARCH != "amd64" {
		t.Skipf("not yet supported on %s/%s", runtime.GOOS, runtime.GOARCH)
	}
	got := runTestProg(t, "testprogcgo", "CrashTraceback")
	for i := 1; i <= 3; i++ {
		if !strings.Contains(got, fmt.Sprintf("cgo symbolizer:%d", i)) {
			t.Errorf("missing cgo symbolizer:%d", i)
		}
	}
}

func TestCgoTracebackContext(t *testing.T) {
	got := runTestProg(t, "testprogcgo", "TracebackContext")
	want := "OK\n"
	if got != want {
		t.Errorf("expected %q got %v", want, got)
	}
}

func testCgoPprof(t *testing.T, buildArg, runArg string) {
	if runtime.GOOS != "linux" || runtime.GOARCH != "amd64" {
		t.Skipf("not yet supported on %s/%s", runtime.GOOS, runtime.GOARCH)
	}
	testenv.MustHaveGoRun(t)

	exe, err := buildTestProg(t, "testprogcgo", buildArg)
	if err != nil {
		t.Fatal(err)
	}

	got, err := testEnv(exec.Command(exe, runArg)).CombinedOutput()
	if err != nil {
		t.Fatal(err)
	}
	fn := strings.TrimSpace(string(got))
	defer os.Remove(fn)

	top, err := exec.Command("go", "tool", "pprof", "-top", "-nodecount=1", exe, fn).CombinedOutput()
	t.Logf("%s", top)
	if err != nil {
		t.Fatal(err)
	}

	if !bytes.Contains(top, []byte("cpuHog")) {
		t.Error("missing cpuHog in pprof output")
	}
}

func TestCgoPprof(t *testing.T) {
	testCgoPprof(t, "", "CgoPprof")
}

func TestCgoPprofPIE(t *testing.T) {
	testCgoPprof(t, "-ldflags=-extldflags=-pie", "CgoPprof")
}

func TestCgoPprofThread(t *testing.T) {
	testCgoPprof(t, "", "CgoPprofThread")
}
<commit_msg>runtime: set PPROF_TMPDIR before running pprof<commit_after>// Copyright 2012 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build cgo

package runtime_test

import (
	"bytes"
	"fmt"
	"internal/testenv"
	"os"
	"os/exec"
	"runtime"
	"strings"
	"testing"
	"time"
)

func TestCgoCrashHandler(t *testing.T) {
	testCrashHandler(t, true)
}

func TestCgoSignalDeadlock(t *testing.T) {
	if testing.Short() && runtime.GOOS == "windows" {
		t.Skip("Skipping in short mode") // takes up to 64 seconds
	}
	got := runTestProg(t, "testprogcgo", "CgoSignalDeadlock")
	want := "OK\n"
	if got != want {
		t.Fatalf("expected %q, but got:\n%s", want, got)
	}
}

func TestCgoTraceback(t *testing.T) {
	got := runTestProg(t, "testprogcgo", "CgoTraceback")
	want := "OK\n"
	if got != want {
		t.Fatalf("expected %q, but got:\n%s", want, got)
	}
}

func TestCgoCallbackGC(t *testing.T) {
	if runtime.GOOS == "plan9" || runtime.GOOS == "windows" {
		t.Skipf("no pthreads on %s", runtime.GOOS)
	}
	if testing.Short() {
		switch {
		case runtime.GOOS == "dragonfly":
			t.Skip("see golang.org/issue/11990")
		case runtime.GOOS == "linux" && runtime.GOARCH == "arm":
			t.Skip("too slow for arm builders")
		case runtime.GOOS == "linux" && (runtime.GOARCH == "mips64" || runtime.GOARCH == "mips64le"):
			t.Skip("too slow for mips64x builders")
		}
	}
	got := runTestProg(t, "testprogcgo", "CgoCallbackGC")
	want := "OK\n"
	if got != want {
		t.Fatalf("expected %q, but got:\n%s", want, got)
	}
}

func TestCgoExternalThreadPanic(t *testing.T) {
	if runtime.GOOS == "plan9" {
		t.Skipf("no pthreads on %s", runtime.GOOS)
	}
	got := runTestProg(t, "testprogcgo", "CgoExternalThreadPanic")
	want := "panic: BOOM"
	if !strings.Contains(got, want) {
		t.Fatalf("want failure containing %q. output:\n%s\n", want, got)
	}
}

func TestCgoExternalThreadSIGPROF(t *testing.T) {
	// issue 9456.
	switch runtime.GOOS {
	case "plan9", "windows":
		t.Skipf("no pthreads on %s", runtime.GOOS)
	case "darwin":
		if runtime.GOARCH != "arm" && runtime.GOARCH != "arm64" {
			// static constructor needs external linking, but we don't support
			// external linking on OS X 10.6.
			out, err := exec.Command("uname", "-r").Output()
			if err != nil {
				t.Fatalf("uname -r failed: %v", err)
			}
			// OS X 10.6 == Darwin 10.x
			if strings.HasPrefix(string(out), "10.") {
				t.Skipf("no external linking on OS X 10.6")
			}
		}
	}
	if runtime.GOARCH == "ppc64" {
		// TODO(austin) External linking not implemented on
		// ppc64 (issue #8912)
		t.Skipf("no external linking on ppc64")
	}
	got := runTestProg(t, "testprogcgo", "CgoExternalThreadSIGPROF")
	want := "OK\n"
	if got != want {
		t.Fatalf("expected %q, but got:\n%s", want, got)
	}
}

func TestCgoExternalThreadSignal(t *testing.T) {
	// issue 10139
	switch runtime.GOOS {
	case "plan9", "windows":
		t.Skipf("no pthreads on %s", runtime.GOOS)
	}
	got := runTestProg(t, "testprogcgo", "CgoExternalThreadSignal")
	want := "OK\n"
	if got != want {
		t.Fatalf("expected %q, but got:\n%s", want, got)
	}
}

func TestCgoDLLImports(t *testing.T) {
	// test issue 9356
	if runtime.GOOS != "windows" {
		t.Skip("skipping windows specific test")
	}
	got := runTestProg(t, "testprogcgo", "CgoDLLImportsMain")
	want := "OK\n"
	if got != want {
		t.Fatalf("expected %q, but got %v", want, got)
	}
}

func TestCgoExecSignalMask(t *testing.T) {
	// Test issue 13164.
	switch runtime.GOOS {
	case "windows", "plan9":
		t.Skipf("skipping signal mask test on %s", runtime.GOOS)
	}
	got := runTestProg(t, "testprogcgo", "CgoExecSignalMask")
	want := "OK\n"
	if got != want {
		t.Errorf("expected %q, got %v", want, got)
	}
}

func TestEnsureDropM(t *testing.T) {
	// Test for issue 13881.
	switch runtime.GOOS {
	case "windows", "plan9":
		t.Skipf("skipping dropm test on %s", runtime.GOOS)
	}
	got := runTestProg(t, "testprogcgo", "EnsureDropM")
	want := "OK\n"
	if got != want {
		t.Errorf("expected %q, got %v", want, got)
	}
}

// Test for issue 14387.
// Test that the program that doesn't need any cgo pointer checking
// takes about the same amount of time with it as without it.
func TestCgoCheckBytes(t *testing.T) {
	// Make sure we don't count the build time as part of the run time.
	testenv.MustHaveGoBuild(t)
	exe, err := buildTestProg(t, "testprogcgo")
	if err != nil {
		t.Fatal(err)
	}

	// Try it 10 times to avoid flakiness.
	const tries = 10
	var tot1, tot2 time.Duration
	for i := 0; i < tries; i++ {
		cmd := testEnv(exec.Command(exe, "CgoCheckBytes"))
		cmd.Env = append(cmd.Env, "GODEBUG=cgocheck=0", fmt.Sprintf("GO_CGOCHECKBYTES_TRY=%d", i))

		start := time.Now()
		cmd.Run()
		d1 := time.Since(start)

		cmd = testEnv(exec.Command(exe, "CgoCheckBytes"))
		cmd.Env = append(cmd.Env, fmt.Sprintf("GO_CGOCHECKBYTES_TRY=%d", i))

		start = time.Now()
		cmd.Run()
		d2 := time.Since(start)

		if d1*20 > d2 {
			// The slow version (d2) was less than 20 times
			// slower than the fast version (d1), so OK.
			return
		}

		tot1 += d1
		tot2 += d2
	}

	t.Errorf("cgo check too slow: got %v, expected at most %v", tot2/tries, (tot1/tries)*20)
}

func TestCgoPanicDeadlock(t *testing.T) {
	// test issue 14432
	got := runTestProg(t, "testprogcgo", "CgoPanicDeadlock")
	want := "panic: cgo error\n\n"
	if !strings.HasPrefix(got, want) {
		t.Fatalf("output does not start with %q:\n%s", want, got)
	}
}

func TestCgoCCodeSIGPROF(t *testing.T) {
	got := runTestProg(t, "testprogcgo", "CgoCCodeSIGPROF")
	want := "OK\n"
	if got != want {
		t.Errorf("expected %q got %v", want, got)
	}
}

func TestCgoCrashTraceback(t *testing.T) {
	if runtime.GOOS != "linux" || runtime.GOARCH != "amd64" {
		t.Skipf("not yet supported on %s/%s", runtime.GOOS, runtime.GOARCH)
	}
	got := runTestProg(t, "testprogcgo", "CrashTraceback")
	for i := 1; i <= 3; i++ {
		if !strings.Contains(got, fmt.Sprintf("cgo symbolizer:%d", i)) {
			t.Errorf("missing cgo symbolizer:%d", i)
		}
	}
}

func TestCgoTracebackContext(t *testing.T) {
	got := runTestProg(t, "testprogcgo", "TracebackContext")
	want := "OK\n"
	if got != want {
		t.Errorf("expected %q got %v", want, got)
	}
}

func testCgoPprof(t *testing.T, buildArg, runArg string) {
	if runtime.GOOS != "linux" || runtime.GOARCH != "amd64" {
		t.Skipf("not yet supported on %s/%s", runtime.GOOS, runtime.GOARCH)
	}
	testenv.MustHaveGoRun(t)

	exe, err := buildTestProg(t, "testprogcgo", buildArg)
	if err != nil {
		t.Fatal(err)
	}

	got, err := testEnv(exec.Command(exe, runArg)).CombinedOutput()
	if err != nil {
		t.Fatal(err)
	}
	fn := strings.TrimSpace(string(got))
	defer os.Remove(fn)

	cmd := testEnv(exec.Command("go", "tool", "pprof", "-top", "-nodecount=1", exe, fn))

	found := false
	for i, e := range cmd.Env {
		if strings.HasPrefix(e, "PPROF_TMPDIR=") {
			cmd.Env[i] = "PPROF_TMPDIR=" + os.TempDir()
			found = true
			break
		}
	}
	if !found {
		cmd.Env = append(cmd.Env, "PPROF_TMPDIR="+os.TempDir())
	}

	top, err := cmd.CombinedOutput()
	t.Logf("%s", top)
	if err != nil {
		t.Fatal(err)
	}

	if !bytes.Contains(top, []byte("cpuHog")) {
		t.Error("missing cpuHog in pprof output")
	}
}

func TestCgoPprof(t *testing.T) {
	testCgoPprof(t, "", "CgoPprof")
}

func TestCgoPprofPIE(t *testing.T) {
	testCgoPprof(t, "-ldflags=-extldflags=-pie", "CgoPprof")
}

func TestCgoPprofThread(t *testing.T) {
	testCgoPprof(t, "", "CgoPprofThread")
}
<|endoftext|>


<commit_before>package api

import (
	"fmt"
	"sync"

	"github.com/CenturyLinkLabs/docker-reg-client/registry"
	"github.com/pmylund/go-cache"
)

type remoteConnection struct {
	client     *registry.Client
	layerCache *cache.Cache
}

func NewRemoteRegistry() *registryApi {
	conn := &remoteConnection{
		client:     registry.NewClient(),
		layerCache: cache.New(cacheDuration, cacheCleanupInterval),
	}
	reg := newRegistryApi(conn)

	return reg
}

func (rc *remoteConnection) GetTags(name string) (registry.TagMap, error) {
	auth, err := rc.client.Hub.GetReadToken(name)
	if err != nil {
		return nil, err
	}

	return rc.client.Repository.ListTags(name, auth)
}

func (rc *remoteConnection) Search(name string) (*registry.SearchResults, error) {
	return rc.client.Search.Query(name, 1, 10)
}

func (rc *remoteConnection) Status() (Status, error) {
	var s Status

	s.Message = "Connected to Registry"
	s.Service = "Registry Image Manager"
	return s, nil
}

func (rc *remoteConnection) GetImageLayers(name, tag string) ([]*registry.ImageMetadata, error) {
	var wg sync.WaitGroup

	auth, err := rc.client.Hub.GetReadToken(name)
	if err != nil {
		return nil, err
	}

	id, err := rc.client.Repository.GetImageID(name, tag, auth)
	if err != nil {
		return nil, err
	}

	layerIDs, err := rc.client.Image.GetAncestry(id, auth)
	if err != nil {
		return nil, err
	}

	list := make([]*registry.ImageMetadata, len(layerIDs))

	for i, layerID := range layerIDs {
		wg.Add(1)
		go func(idx int, lid string) {
			defer wg.Done()
			var err error
			var m *registry.ImageMetadata

			val, found := rc.layerCache.Get(lid)
			if found {
				m = val.(*registry.ImageMetadata)
			} else {
				m, err = rc.client.Image.GetMetadata(lid, auth)
				if err == nil && m != nil {
					rc.layerCache.Set(lid, m, cache.DefaultExpiration)
				}
			}

			list[idx] = m
		}(i, layerID)
	}
	wg.Wait()

	// Return an error if any of the layers are missing
	for i, layer := range list {
		if layer == nil {
			return nil, fmt.Errorf("Error retrieving layer: %s", layerIDs[i])
		}
	}

	return list, nil
}
<commit_msg>Add caching for tag lookup<commit_after>package api

import (
	"fmt"
	"sync"

	"github.com/CenturyLinkLabs/docker-reg-client/registry"
	"github.com/pmylund/go-cache"
)

type remoteConnection struct {
	client     *registry.Client
	layerCache *cache.Cache
	tagCache   *cache.Cache
}

func NewRemoteRegistry() *registryApi {
	conn := &remoteConnection{
		client:     registry.NewClient(),
		layerCache: cache.New(cacheDuration, cacheCleanupInterval),
		tagCache:   cache.New(cacheDuration, cacheCleanupInterval),
	}
	reg := newRegistryApi(conn)

	return reg
}

func (rc *remoteConnection) GetTags(name string) (registry.TagMap, error) {
	var tags registry.TagMap

	val, found := rc.tagCache.Get(name)
	if found {
		tags = val.(registry.TagMap)
	} else {
		auth, err := rc.client.Hub.GetReadToken(name)
		if err != nil {
			return nil, err
		}

		tags, err = rc.client.Repository.ListTags(name, auth)
		if err != nil {
			return nil, err
		}
		rc.tagCache.Set(name, tags, cache.DefaultExpiration)
	}

	return tags, nil
}

func (rc *remoteConnection) Search(name string) (*registry.SearchResults, error) {
	return rc.client.Search.Query(name, 1, 10)
}

func (rc *remoteConnection) Status() (Status, error) {
	var s Status

	s.Message = "Connected to Registry"
	s.Service = "Registry Image Manager"
	return s, nil
}

func (rc *remoteConnection) GetImageLayers(name, tag string) ([]*registry.ImageMetadata, error) {
	var wg sync.WaitGroup

	auth, err := rc.client.Hub.GetReadToken(name)
	if err != nil {
		return nil, err
	}

	id, err := rc.client.Repository.GetImageID(name, tag, auth)
	if err != nil {
		return nil, err
	}

	layerIDs, err := rc.client.Image.GetAncestry(id, auth)
	if err != nil {
		return nil, err
	}

	list := make([]*registry.ImageMetadata, len(layerIDs))

	for i, layerID := range layerIDs {
		wg.Add(1)
		go func(idx int, lid string) {
			defer wg.Done()
			var err error
			var m *registry.ImageMetadata

			val, found := rc.layerCache.Get(lid)
			if found {
				m = val.(*registry.ImageMetadata)
			} else {
				m, err = rc.client.Image.GetMetadata(lid, auth)
				if err == nil && m != nil {
					rc.layerCache.Set(lid, m, cache.DefaultExpiration)
				}
			}

			list[idx] = m
		}(i, layerID)
	}
	wg.Wait()

	// Return an error if any of the layers are missing
	for i, layer := range list {
		if layer == nil {
			return nil, fmt.Errorf("Error retrieving layer: %s", layerIDs[i])
		}
	}

	return list, nil
}
<|endoftext|>


<commit_before>package clicommand

import (
	"github.com/andrew-d/go-termutil"
	"github.com/buildkite/agent/agent"
	"github.com/buildkite/agent/api"
	"github.com/buildkite/agent/cliconfig"
	"github.com/buildkite/agent/logger"
	"github.com/buildkite/agent/retry"
	"github.com/codegangsta/cli"
	"io/ioutil"
	"os"
	"path/filepath"
	"time"
)

var PipelineUploadHelpDescription = `Usage:

   buildkite-agent pipeline upload <file> [arguments...]

Description:

   Allows you to change the pipeline of a running build by uploading a
   JSON configuration file. By default, we try to upload
   .buildkite/steps.json, but you can customize which file to use.

Example:

   $ buildkite-agent pipeline upload
   $ buildkite-agent pipeline upload my-custom-steps.json
   $ cat .buildkite/steps.yml | buildkite-agent pipeline upload`

type PipelineUploadConfig struct {
	FilePath         string `cli:"arg:0" label:"upload paths"`
	Job              string `cli:"job" validate:"required"`
	AgentAccessToken string `cli:"agent-access-token" validate:"required"`
	Endpoint         string `cli:"endpoint" validate:"required"`
	NoColor          bool   `cli:"no-color"`
	Debug            bool   `cli:"debug"`
}

var PipelineUploadCommand = cli.Command{
	Name:        "upload",
	Usage:       "Upload a description of the build pipleine and change the running build",
	Description: PipelineUploadHelpDescription,
	Flags: []cli.Flag{
		cli.StringFlag{
			Name:   "job",
			Value:  "",
			Usage:  "The job that is making the changes to it's build",
			EnvVar: "BUILDKITE_JOB_ID",
		},
		AgentAccessTokenFlag,
		EndpointFlag,
		DebugFlag,
		NoColorFlag,
	},
	Action: func(c *cli.Context) {
		// The configuration will be loaded into this struct
		cfg := PipelineUploadConfig{}

		// Load the configuration
		loader := cliconfig.Loader{CLI: c, Config: &cfg}
		if err := loader.Load(); err != nil {
			logger.Fatal("%s", err)
		}

		// Setup the any global configuration options
		HandleGlobalFlags(cfg)

		// Find the pipeline file either from STDIN or the first
		// argument
		var input []byte
		var err error
		var filename string

		if cfg.FilePath != "" {
			filename = filepath.Base(cfg.FilePath)
			input, err = ioutil.ReadFile(cfg.FilePath)
			if err != nil {
				logger.Fatal("Failed to read file: %s", err)
			}
		} else if !termutil.Isatty(os.Stdin.Fd()) {
			input, err = ioutil.ReadAll(os.Stdin)
			if err != nil {
				logger.Fatal("Failed to read from STDIN: %s", err)
			}
		} else {
			// Default to opening .buildkite/steps.json
			filename = "steps.json"
			path, _ := filepath.Abs(".buildkite/" + filename)
			input, err = ioutil.ReadFile(path)
			if err != nil {
				logger.Fatal("Failed to read file: %s", err)
			}
		}

		// Create the API client
		client := agent.APIClient{
			Endpoint: cfg.Endpoint,
			Token:    cfg.AgentAccessToken,
		}.Create()

		// Retry the pipeline upload a few times before giving up
		err = retry.Do(func(s *retry.Stats) error {
			_, err = client.Pipelines.Upload(cfg.Job, &api.Pipeline{Data: input, FileName: filename})
			if err != nil {
				logger.Warn("%s (%s)", err, s)
			}

			return err
		}, &retry.Config{Maximum: 5, Interval: 1 * time.Second})
		if err != nil {
			logger.Fatal("Failed to upload and process pipeline: %s", err)
		}

		logger.Info("Successfully uploaded and parsed pipeline config")
	},
}
<commit_msg>Tweaked pipeline upload docs.<commit_after>package clicommand

import (
	"github.com/andrew-d/go-termutil"
	"github.com/buildkite/agent/agent"
	"github.com/buildkite/agent/api"
	"github.com/buildkite/agent/cliconfig"
	"github.com/buildkite/agent/logger"
	"github.com/buildkite/agent/retry"
	"github.com/codegangsta/cli"
	"io/ioutil"
	"os"
	"path/filepath"
	"time"
)

var PipelineUploadHelpDescription = `Usage:

   buildkite-agent pipeline upload <file> [arguments...]

Description:

   Allows you to change the pipeline of a running build by uploading a
   JSON configuration file. By default, we try to upload
   .buildkite/steps.json, but you can customize which file to use.

Example:

   $ buildkite-agent pipeline upload
   $ buildkite-agent pipeline upload my-custom-steps.json
   $ ./script/dynamic_step_generator | buildkite-agent pipeline upload`

type PipelineUploadConfig struct {
	FilePath         string `cli:"arg:0" label:"upload paths"`
	Job              string `cli:"job" validate:"required"`
	AgentAccessToken string `cli:"agent-access-token" validate:"required"`
	Endpoint         string `cli:"endpoint" validate:"required"`
	NoColor          bool   `cli:"no-color"`
	Debug            bool   `cli:"debug"`
}

var PipelineUploadCommand = cli.Command{
	Name:        "upload",
	Usage:       "Upload a description of the build pipleine and change the running build",
	Description: PipelineUploadHelpDescription,
	Flags: []cli.Flag{
		cli.StringFlag{
			Name:   "job",
			Value:  "",
			Usage:  "The job that is making the changes to it's build",
			EnvVar: "BUILDKITE_JOB_ID",
		},
		AgentAccessTokenFlag,
		EndpointFlag,
		DebugFlag,
		NoColorFlag,
	},
	Action: func(c *cli.Context) {
		// The configuration will be loaded into this struct
		cfg := PipelineUploadConfig{}

		// Load the configuration
		loader := cliconfig.Loader{CLI: c, Config: &cfg}
		if err := loader.Load(); err != nil {
			logger.Fatal("%s", err)
		}

		// Setup the any global configuration options
		HandleGlobalFlags(cfg)

		// Find the pipeline file either from STDIN or the first
		// argument
		var input []byte
		var err error
		var filename string

		if cfg.FilePath != "" {
			filename = filepath.Base(cfg.FilePath)
			input, err = ioutil.ReadFile(cfg.FilePath)
			if err != nil {
				logger.Fatal("Failed to read file: %s", err)
			}
		} else if !termutil.Isatty(os.Stdin.Fd()) {
			input, err = ioutil.ReadAll(os.Stdin)
			if err != nil {
				logger.Fatal("Failed to read from STDIN: %s", err)
			}
		} else {
			// Default to opening .buildkite/steps.json
			filename = "steps.json"
			path, _ := filepath.Abs(".buildkite/" + filename)
			input, err = ioutil.ReadFile(path)
			if err != nil {
				logger.Fatal("Failed to read file: %s", err)
			}
		}

		// Create the API client
		client := agent.APIClient{
			Endpoint: cfg.Endpoint,
			Token:    cfg.AgentAccessToken,
		}.Create()

		// Retry the pipeline upload a few times before giving up
		err = retry.Do(func(s *retry.Stats) error {
			_, err = client.Pipelines.Upload(cfg.Job, &api.Pipeline{Data: input, FileName: filename})
			if err != nil {
				logger.Warn("%s (%s)", err, s)
			}

			return err
		}, &retry.Config{Maximum: 5, Interval: 1 * time.Second})
		if err != nil {
			logger.Fatal("Failed to upload and process pipeline: %s", err)
		}

		logger.Info("Successfully uploaded and parsed pipeline config")
	},
}
<|endoftext|>


<commit_before>/*
 *
 * Author     : Valentin Kuznetsov <vkuznet AT gmail dot com>
 * Description: URL fetch proxy server concurrently fetches data from
 *              provided URL list. It provides a POST HTTP interface
 *              "/fetch" which accepts urls as newline separated encoded
 *              string
 * Created    : Wed Mar 20 13:29:48 EDT 2013
 * License    : MIT
 *
 */
package urlfetch

import (
    "os"
    "log"
    "strings"
    "net/http"
    "io/ioutil"
    "crypto/tls"
    "x509proxy"
)

/*
 * Return array of certificates
 */
func Certs() (tls_certs []tls.Certificate) {
    uproxy := os.Getenv("X509_USER_PROXY")
    uckey  := os.Getenv("X509_USER_KEY")
    ucert  := os.Getenv("X509_USER_CERT")
    log.Println("X509_USER_PROXY", uproxy)
    log.Println("X509_USER_KEY", uckey)
    log.Println("X509_USER_CERT", ucert)
    if  len(uproxy) > 0 {
//        x509cert, err := tls.LoadX509KeyPair(uproxy, uproxy)
        // use local implementation of LoadX409KeyPair, see x509proxy.go
        x509cert, err := x509proxy.LoadX509Proxy(uproxy)
        if  err != nil {
            log.Println("Fail to parser proxy X509 certificate", err)
            return
        }
        tls_certs = []tls.Certificate{x509cert}
    } else if len(uckey) > 0 {
        x509cert, err := tls.LoadX509KeyPair(ucert, uckey)
        if  err != nil {
            log.Println("Fail to parser user X509 certificate", err)
            return
        }
        tls_certs = []tls.Certificate{x509cert}
    } else {
        return
    }
    return
}

/*
 * HTTP client for urlfetch server
 */
func HttpClient() (client *http.Client) {
    // create HTTP client
    certs := Certs()
    log.Println("Number of certificates", len(certs), certs)
    if  len(certs) == 0 {
        client = &http.Client{}
        return
    }
    tr := &http.Transport{
        TLSClientConfig: &tls.Config{tls.Certificates: certs},
    }
    log.Println("Create TLSClientConfig")
    client = &http.Client{Transport: tr}
    return
}

// create global HTTP client and re-use it through the code
var client = HttpClient()

/*
 * Fetch(url string, ch chan<- []byte)
 * Fetch data for provided URL and redirect results to given channel
 */
func Fetch(url string, ch chan<- []byte) {
    msg := ""
    resp, err := client.Get(url)
    if  err != nil {
        msg = "Fail to contact " + url
        log.Println(msg, err)
        ch <- []byte(msg)
        return
    }
    defer resp.Body.Close()
    body, err := ioutil.ReadAll(resp.Body)
    if  err != nil {
        msg = "Fail to parse reponse body"
        log.Println(msg, err)
        ch <- []byte(msg)
        return
    }
    ch <- body
}

/*
 * RequestHandler is used by web server to handle incoming requests
 */
func RequestHandler(w http.ResponseWriter, r *http.Request) {
    // we only accept POST request with urls (this is by design)
    if  r.Method != "POST" {
        w.WriteHeader(http.StatusBadRequest)
        return
    }

    // parse input request parameter, in this case we should pass urls
    r.ParseForm()
    urls := []string{}
    for k, v := range r.Form {
        if k == "urls" {
            urls = strings.Split(v[0], "\n")
        }
    }
    num := len(urls)
    if  num > 0 {
        first := urls[0]
        last := urls[num-1]
        log.Println("Fetch", num, "URLs:", first, "...", last)
    } else {
        w.Write([]byte("No URLs provided\n"))
        return
    }

    // loop concurently over url list and store results into channel
    ch := make(chan []byte)
    for _, url := range urls {
        go Fetch(url, ch)
    }
    // once channels are ready fill out results to response writer
    for i:=0; i<len(urls); i++ {
        w.Write(<-ch)
        w.Write([]byte("\n"))
    }
}

// proxy server. It defines /fetch public interface
func Server(port string) {
    log.Printf("Start server localhost:%s/fetch", port)
    http.HandleFunc("/fetch", RequestHandler)
    err := http.ListenAndServe(":" + port, nil)
    // NOTE: later this can be replaced with secure connection
    // replace ListenAndServe(addr string, handler Handler)
    // with TLS function
    // ListenAndServeTLS(addr string, certFile string, keyFile string, handler
    // Handler)
    if  err != nil {
        log.Fatal("ListenAndServe: ", err)
    }
}

<commit_msg>Add InsecureSkipVerify option for debugging<commit_after>/*
 *
 * Author     : Valentin Kuznetsov <vkuznet AT gmail dot com>
 * Description: URL fetch proxy server concurrently fetches data from
 *              provided URL list. It provides a POST HTTP interface
 *              "/fetch" which accepts urls as newline separated encoded
 *              string
 * Created    : Wed Mar 20 13:29:48 EDT 2013
 * License    : MIT
 *
 */
package urlfetch

import (
    "os"
    "log"
    "strings"
    "net/http"
    "io/ioutil"
    "crypto/tls"
    "x509proxy"
)

/*
 * Return array of certificates
 */
func Certs() (tls_certs []tls.Certificate) {
    uproxy := os.Getenv("X509_USER_PROXY")
    uckey  := os.Getenv("X509_USER_KEY")
    ucert  := os.Getenv("X509_USER_CERT")
    log.Println("X509_USER_PROXY", uproxy)
    log.Println("X509_USER_KEY", uckey)
    log.Println("X509_USER_CERT", ucert)
    if  len(uproxy) > 0 {
//        x509cert, err := tls.LoadX509KeyPair(uproxy, uproxy)
        // use local implementation of LoadX409KeyPair, see x509proxy.go
        x509cert, err := x509proxy.LoadX509Proxy(uproxy)
        if  err != nil {
            log.Println("Fail to parser proxy X509 certificate", err)
            return
        }
        tls_certs = []tls.Certificate{x509cert}
    } else if len(uckey) > 0 {
        x509cert, err := tls.LoadX509KeyPair(ucert, uckey)
        if  err != nil {
            log.Println("Fail to parser user X509 certificate", err)
            return
        }
        tls_certs = []tls.Certificate{x509cert}
    } else {
        return
    }
    return
}

/*
 * HTTP client for urlfetch server
 */
func HttpClient() (client *http.Client) {
    // create HTTP client
    certs := Certs()
    log.Println("Number of certificates", len(certs))
    if  len(certs) == 0 {
        client = &http.Client{}
        return
    }
    tr := &http.Transport{
        TLSClientConfig: &tls.Config{tls.Certificates: certs,
                                     tls.InsecureSkipVerify: true},
    }
    log.Println("Create TLSClientConfig")
    client = &http.Client{Transport: tr}
    return
}

// create global HTTP client and re-use it through the code
var client = HttpClient()

/*
 * Fetch(url string, ch chan<- []byte)
 * Fetch data for provided URL and redirect results to given channel
 */
func Fetch(url string, ch chan<- []byte) {
    msg := ""
    resp, err := client.Get(url)
    if  err != nil {
        msg = "Fail to contact " + url
        log.Println(msg, err)
        ch <- []byte(msg)
        return
    }
    defer resp.Body.Close()
    body, err := ioutil.ReadAll(resp.Body)
    if  err != nil {
        msg = "Fail to parse reponse body"
        log.Println(msg, err)
        ch <- []byte(msg)
        return
    }
    ch <- body
}

/*
 * RequestHandler is used by web server to handle incoming requests
 */
func RequestHandler(w http.ResponseWriter, r *http.Request) {
    // we only accept POST request with urls (this is by design)
    if  r.Method != "POST" {
        w.WriteHeader(http.StatusBadRequest)
        return
    }

    // parse input request parameter, in this case we should pass urls
    r.ParseForm()
    urls := []string{}
    for k, v := range r.Form {
        if k == "urls" {
            urls = strings.Split(v[0], "\n")
        }
    }
    num := len(urls)
    if  num > 0 {
        first := urls[0]
        last := urls[num-1]
        log.Println("Fetch", num, "URLs:", first, "...", last)
    } else {
        w.Write([]byte("No URLs provided\n"))
        return
    }

    // loop concurently over url list and store results into channel
    ch := make(chan []byte)
    for _, url := range urls {
        go Fetch(url, ch)
    }
    // once channels are ready fill out results to response writer
    for i:=0; i<len(urls); i++ {
        w.Write(<-ch)
        w.Write([]byte("\n"))
    }
}

// proxy server. It defines /fetch public interface
func Server(port string) {
    log.Printf("Start server localhost:%s/fetch", port)
    http.HandleFunc("/fetch", RequestHandler)
    err := http.ListenAndServe(":" + port, nil)
    // NOTE: later this can be replaced with secure connection
    // replace ListenAndServe(addr string, handler Handler)
    // with TLS function
    // ListenAndServeTLS(addr string, certFile string, keyFile string, handler
    // Handler)
    if  err != nil {
        log.Fatal("ListenAndServe: ", err)
    }
}

<|endoftext|>


<commit_before>// cloud-config-server starts an HTTP server, which can be accessed
// via URLs in the form of
//
//   http://<addr:port>?mac=aa:bb:cc:dd:ee:ff
//
// and returns the cloud-config YAML file specificially tailored for
// the node whose primary NIC's MAC address matches that specified in
// above URL.
package main

import (
	"errors"
	"flag"
	"fmt"
	"log"
	"net"
	"net/http"
	"os"
	"strings"

	"github.com/golang/glog"

	"github.com/gorilla/mux"
	"github.com/k8sp/sextant/cloud-config-server/certgen"
	cctemplate "github.com/k8sp/sextant/cloud-config-server/template"
	"github.com/topicai/candy"
)

func main() {
	clusterDesc := flag.String("cluster-desc", "./cluster-desc.yml", "Configurations for a k8s cluster.")
	ccTemplateDir := flag.String("cloud-config-dir", "./cloud-config.template", "cloud-config file template.")
	caCrt := flag.String("ca-crt", "", "CA certificate file, in PEM format")
	caKey := flag.String("ca-key", "", "CA private key file, in PEM format")
	addr := flag.String("addr", ":8080", "Listening address")
	staticDir := flag.String("dir", "./static/", "The directory to serve files from. Default is ./static/")
	flag.Parse()

	if len(*caCrt) == 0 || len(*caKey) == 0 {
		glog.Info("No ca.pem or ca-key.pem provided, generating now...")
		*caKey, *caCrt = certgen.GenerateRootCA("./")
	}
	// valid caKey and caCrt file is ready
	if err := fileExist(*caCrt); err != nil {
		glog.Error("No cert of ca.pem has been generated!")
		log.Panic(err)
	}
	if err := fileExist(*caKey); err != nil {
		glog.Error("No cert of ca-key.pem has been generated!")
		log.Panic(err)
	}

	glog.Info("Cloud-config server start Listenning...")
	l, e := net.Listen("tcp", *addr)
	candy.Must(e)

	// start and run the HTTP server
	router := mux.NewRouter().StrictSlash(true)
	router.HandleFunc("/cloud-config/{mac}", makeCloudConfigHandler(*clusterDesc, *ccTemplateDir, *caKey, *caCrt))
	router.PathPrefix("/static/").Handler(http.StripPrefix("/static/", http.FileServer(http.Dir(*staticDir))))

	glog.Fatal(http.Serve(l, router))
}

// makeCloudConfigHandler generate a HTTP server handler to serve cloud-config
// fetching requests
func makeCloudConfigHandler(clusterDescFile string, ccTemplateDir string, caKey, caCrt string) http.HandlerFunc {
	return makeSafeHandler(func(w http.ResponseWriter, r *http.Request) {
		hwAddr, err := net.ParseMAC(mux.Vars(r)["mac"])
		candy.Must(err)
		candy.Must(cctemplate.Execute(w, hwAddr.String(), "cc-template", ccTemplateDir, clusterDescFile, caKey, caCrt))
	})
}

func makeSafeHandler(h http.HandlerFunc) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		defer func() {
			if err := recover(); err != nil {
				http.Error(w, fmt.Sprint(err), http.StatusInternalServerError)
			}
		}()
		h(w, r)
	}
}

func fileExist(fn string) error {
	_, err := os.Stat(fn)
	if err != nil || os.IsNotExist(err) {
		return errors.New("file " + fn + " is not ready.")
	}
	return nil
}
<commit_msg>Remove unused strings<commit_after>// cloud-config-server starts an HTTP server, which can be accessed
// via URLs in the form of
//
//   http://<addr:port>?mac=aa:bb:cc:dd:ee:ff
//
// and returns the cloud-config YAML file specificially tailored for
// the node whose primary NIC's MAC address matches that specified in
// above URL.
package main

import (
	"errors"
	"flag"
	"fmt"
	"log"
	"net"
	"net/http"
	"os"

	"github.com/golang/glog"

	"github.com/gorilla/mux"
	"github.com/k8sp/sextant/cloud-config-server/certgen"
	cctemplate "github.com/k8sp/sextant/cloud-config-server/template"
	"github.com/topicai/candy"
)

func main() {
	clusterDesc := flag.String("cluster-desc", "./cluster-desc.yml", "Configurations for a k8s cluster.")
	ccTemplateDir := flag.String("cloud-config-dir", "./cloud-config.template", "cloud-config file template.")
	caCrt := flag.String("ca-crt", "", "CA certificate file, in PEM format")
	caKey := flag.String("ca-key", "", "CA private key file, in PEM format")
	addr := flag.String("addr", ":8080", "Listening address")
	staticDir := flag.String("dir", "./static/", "The directory to serve files from. Default is ./static/")
	flag.Parse()

	if len(*caCrt) == 0 || len(*caKey) == 0 {
		glog.Info("No ca.pem or ca-key.pem provided, generating now...")
		*caKey, *caCrt = certgen.GenerateRootCA("./")
	}
	// valid caKey and caCrt file is ready
	if err := fileExist(*caCrt); err != nil {
		glog.Error("No cert of ca.pem has been generated!")
		log.Panic(err)
	}
	if err := fileExist(*caKey); err != nil {
		glog.Error("No cert of ca-key.pem has been generated!")
		log.Panic(err)
	}

	glog.Info("Cloud-config server start Listenning...")
	l, e := net.Listen("tcp", *addr)
	candy.Must(e)

	// start and run the HTTP server
	router := mux.NewRouter().StrictSlash(true)
	router.HandleFunc("/cloud-config/{mac}", makeCloudConfigHandler(*clusterDesc, *ccTemplateDir, *caKey, *caCrt))
	router.PathPrefix("/static/").Handler(http.StripPrefix("/static/", http.FileServer(http.Dir(*staticDir))))

	glog.Fatal(http.Serve(l, router))
}

// makeCloudConfigHandler generate a HTTP server handler to serve cloud-config
// fetching requests
func makeCloudConfigHandler(clusterDescFile string, ccTemplateDir string, caKey, caCrt string) http.HandlerFunc {
	return makeSafeHandler(func(w http.ResponseWriter, r *http.Request) {
		hwAddr, err := net.ParseMAC(mux.Vars(r)["mac"])
		candy.Must(err)
		candy.Must(cctemplate.Execute(w, hwAddr.String(), "cc-template", ccTemplateDir, clusterDescFile, caKey, caCrt))
	})
}

func makeSafeHandler(h http.HandlerFunc) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		defer func() {
			if err := recover(); err != nil {
				http.Error(w, fmt.Sprint(err), http.StatusInternalServerError)
			}
		}()
		h(w, r)
	}
}

func fileExist(fn string) error {
	_, err := os.Stat(fn)
	if err != nil || os.IsNotExist(err) {
		return errors.New("file " + fn + " is not ready.")
	}
	return nil
}
<|endoftext|>


<commit_before>// cloud-config-server starts an HTTP server, which can be accessed
// via URLs in the form of
//
//   http://<addr:port>?mac=aa:bb:cc:dd:ee:ff
//
// and returns the cloud-config YAML file specificially tailored for
// the node whose primary NIC's MAC address matches that specified in
// above URL.
package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"path"
	"strings"
	"text/template"

	"github.com/gorilla/mux"
	"github.com/k8sp/auto-install/cloud-config-server/cache"
	"github.com/k8sp/auto-install/cloud-config-server/certgen"
	cctemplate "github.com/k8sp/auto-install/cloud-config-server/template"
	"github.com/k8sp/auto-install/config"
	"github.com/topicai/candy"
	"gopkg.in/yaml.v2"
)

func main() {
	clusterDescURL := flag.String("cluster-desc-url",
		"https://raw.githubusercontent.com/k8sp/auto-install/master/cloud-config-server/template/unisound-ailab/build_config.yml",
		"URL to remote cluster description YAML file.")
	clusterDescFile := flag.String("cluster-desc-file", "./cluster-desc.yml", "Local copy of cluster description YAML file.")

	ccTemplateURL := flag.String("cc-template-url",
		"https://raw.githubusercontent.com/k8sp/auto-install/master/cloud-config-server/template/cloud-config.template",
		"URL to cloud-config file template.")
	ccTemplateFile := flag.String("cc-template-file", "./cloud-config.template", "Local copy of cloud-config file template.")

	caCrt := flag.String("ca-crt", "", "CA certificate file, in PEM format")
	caKey := flag.String("ca-key", "", "CA private key file, in PEM format")
	addr := flag.String("addr", ":8080", "Listening address")

	dir := flag.String("dir", "./static/", "The directory to serve files from. Default is ./static/")

	flag.Parse()

	if len(*caCrt) == 0 || len(*caKey) == 0 {
		*caKey, *caCrt = certgen.GenerateRootCA("./")
	}
	c := makeCacheGetter(*clusterDescURL, *clusterDescFile)
	t := makeCacheGetter(*ccTemplateURL, *ccTemplateFile)

	l, e := net.Listen("tcp", *addr)
	candy.Must(e)
	run(c, t, l, *caKey, *caCrt, *dir)
}

// By making the first two parameters closures, we get the flexibility
// to create closures reading from the cache for production serving,
// and from constant values for testing.  Please refer to func main()
// for the former case, and server_test.go for the latter case.
func run(clusterDesc func() []byte, ccTemplate func() []byte, ln net.Listener, caKey, caCrt, dir string) {
	router := mux.NewRouter().StrictSlash(true)
	router.HandleFunc("/cloud-config/{mac}",
		makeSafeHandler(func(w http.ResponseWriter, r *http.Request) {
			mac := strings.ToLower(mux.Vars(r)["mac"])
			tmpl := template.Must(template.New("template").Parse(string(ccTemplate())))
			c := &config.Cluster{}
			candy.Must(yaml.Unmarshal(clusterDesc(), c))
			candy.Must(cctemplate.Execute(tmpl, c, mac, caKey, caCrt, w))
		}))
	router.PathPrefix("/static/").Handler(http.StripPrefix("/static/", http.FileServer(http.Dir(dir))))

	log.Printf("%v", http.Serve(ln, router))
}

func makeSafeHandler(h http.HandlerFunc) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		defer func() {
			if err := recover(); err != nil {
				http.Error(w, fmt.Sprint(err), http.StatusInternalServerError)
			}
		}()
		h(w, r)
	}
}

func makeCacheGetter(url, fn string) func() []byte {
	if len(fn) == 0 {
		dir, e := ioutil.TempDir("", "")
		candy.Must(e)
		fn = path.Join(dir, "localfile")
	}
	c := cache.New(url, fn)
	return func() []byte { return c.Get() }
}
<commit_msg>if ca-crt or ca-key is not ready, server exit.<commit_after>// cloud-config-server starts an HTTP server, which can be accessed
// via URLs in the form of
//
//   http://<addr:port>?mac=aa:bb:cc:dd:ee:ff
//
// and returns the cloud-config YAML file specificially tailored for
// the node whose primary NIC's MAC address matches that specified in
// above URL.
package main

import (
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"os"
	"path"
	"strings"
	"text/template"

	"github.com/gorilla/mux"
	"github.com/k8sp/auto-install/cloud-config-server/cache"
	"github.com/k8sp/auto-install/cloud-config-server/certgen"
	cctemplate "github.com/k8sp/auto-install/cloud-config-server/template"
	"github.com/k8sp/auto-install/config"
	"github.com/topicai/candy"
	"gopkg.in/yaml.v2"
)

func main() {
	clusterDescURL := flag.String("cluster-desc-url",
		"https://raw.githubusercontent.com/k8sp/auto-install/master/cloud-config-server/template/unisound-ailab/build_config.yml",
		"URL to remote cluster description YAML file.")
	clusterDescFile := flag.String("cluster-desc-file", "./cluster-desc.yml", "Local copy of cluster description YAML file.")

	ccTemplateURL := flag.String("cc-template-url",
		"https://raw.githubusercontent.com/k8sp/auto-install/master/cloud-config-server/template/cloud-config.template",
		"URL to cloud-config file template.")
	ccTemplateFile := flag.String("cc-template-file", "./cloud-config.template", "Local copy of cloud-config file template.")

	caCrt := flag.String("ca-crt", "", "CA certificate file, in PEM format")
	caKey := flag.String("ca-key", "", "CA private key file, in PEM format")
	addr := flag.String("addr", ":8080", "Listening address")

	dir := flag.String("dir", "./static/", "The directory to serve files from. Default is ./static/")

	flag.Parse()

	if len(*caCrt) == 0 || len(*caKey) == 0 {
		*caKey, *caCrt = certgen.GenerateRootCA("./")
	}
	// valid caKey and caCrt file is ready
	candy.Must(fileExist(*caCrt))
	candy.Must(fileExist(*caKey))

	c := makeCacheGetter(*clusterDescURL, *clusterDescFile)
	t := makeCacheGetter(*ccTemplateURL, *ccTemplateFile)

	l, e := net.Listen("tcp", *addr)
	candy.Must(e)
	run(c, t, l, *caKey, *caCrt, *dir)
}

// By making the first two parameters closures, we get the flexibility
// to create closures reading from the cache for production serving,
// and from constant values for testing.  Please refer to func main()
// for the former case, and server_test.go for the latter case.
func run(clusterDesc func() []byte, ccTemplate func() []byte, ln net.Listener, caKey, caCrt, dir string) {
	router := mux.NewRouter().StrictSlash(true)
	router.HandleFunc("/cloud-config/{mac}",
		makeSafeHandler(func(w http.ResponseWriter, r *http.Request) {
			mac := strings.ToLower(mux.Vars(r)["mac"])
			tmpl := template.Must(template.New("template").Parse(string(ccTemplate())))
			c := &config.Cluster{}
			candy.Must(yaml.Unmarshal(clusterDesc(), c))
			candy.Must(cctemplate.Execute(tmpl, c, mac, caKey, caCrt, w))
		}))
	router.PathPrefix("/static/").Handler(http.StripPrefix("/static/", http.FileServer(http.Dir(dir))))

	log.Printf("%v", http.Serve(ln, router))
}

func makeSafeHandler(h http.HandlerFunc) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		defer func() {
			if err := recover(); err != nil {
				http.Error(w, fmt.Sprint(err), http.StatusInternalServerError)
			}
		}()
		h(w, r)
	}
}

func makeCacheGetter(url, fn string) func() []byte {
	if len(fn) == 0 {
		dir, e := ioutil.TempDir("", "")
		candy.Must(e)
		fn = path.Join(dir, "localfile")
	}
	c := cache.New(url, fn)
	return func() []byte { return c.Get() }
}

func fileExist(fn string) error {
	_, err := os.Stat(fn)
	if err != nil || os.IsNotExist(err) {
		return errors.New("file " + fn + " is not ready.")
	}
	return nil
}
<|endoftext|>


<commit_before>/*
Copyright 2013 Google Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"errors"
	"flag"
	"fmt"
	"log"
	"os"

	"camlistore.org/pkg/blobserver/diskpacked"
	"camlistore.org/pkg/cmdmain"
	"camlistore.org/pkg/jsonconfig"
	"camlistore.org/pkg/osutil"
	"camlistore.org/pkg/serverinit"
)

type reindexdpCmd struct {
	overwrite, verbose bool
}

func init() {
	cmdmain.RegisterCommand("reindex-diskpacked",
		func(flags *flag.FlagSet) cmdmain.CommandRunner {
			cmd := new(reindexdpCmd)
			flags.BoolVar(&cmd.overwrite, "overwrite", false,
				"Whether to overwrite the existing index. If false, only check.")
			return cmd
		})
}

func (c *reindexdpCmd) Describe() string {
	return "Rebuild the index of the diskpacked blob store"
}

func (c *reindexdpCmd) Usage() {
	fmt.Fprintln(os.Stderr, "Usage: camtool [globalopts] reindex-diskpacked [reindex-opts]")
	fmt.Fprintln(os.Stderr, "       camtool reindex-diskpacked [--overwrite] # dir from server config")
	fmt.Fprintln(os.Stderr, "       camtool reindex-diskpacked [--overwrite] /path/to/directory")
}

func (c *reindexdpCmd) RunCommand(args []string) error {
	var path string
	var indexConf jsonconfig.Obj

	switch len(args) {
	case 0:
	case 1:
		path = args[0]
	default:
		return errors.New("More than 1 argument not allowed")
	}
	cfg, err := serverinit.LoadFile(osutil.UserServerConfigPath())
	if err != nil {
		return err
	}
	prefixes, ok := cfg.Obj["prefixes"].(map[string]interface{})
	if !ok {
		return fmt.Errorf("No 'prefixes' object in low-level (or converted) config file %s", osutil.UserServerConfigPath())
	}
	paths := []string{}
	for prefix, vei := range prefixes {
		pmap, ok := vei.(map[string]interface{})
		if !ok {
			log.Printf("prefix %q value is a %T, not an object", prefix, vei)
			continue
		}
		pconf := jsonconfig.Obj(pmap)
		handlerType := pconf.RequiredString("handler")
		handlerArgs := pconf.OptionalObject("handlerArgs")
		// no pconf.Validate, as this is a recover tool
		if handlerType != "storage-diskpacked" {
			continue
		}
		log.Printf("handlerArgs of %q: %v", prefix, handlerArgs)
		if handlerArgs == nil {
			log.Printf("no handlerArgs for %q", prefix)
			continue
		}
		aconf := jsonconfig.Obj(handlerArgs)
		apath := aconf.RequiredString("path")
		// no aconv.Validate, as this is a recover tool
		if apath == "" {
			log.Printf("path is missing for %q", prefix)
			continue
		}
		if path != "" && path != apath {
			continue
		}
		paths = append(paths, apath)
		indexConf = jsonconfig.Obj(aconf["metaIndex"].(map[string]interface{}))
		log.Printf("indexConf: %v", indexConf)
	}
	if len(paths) == 0 {
		return fmt.Errorf("Server config file %s doesn't specify a disk-packed storage handler.",
			osutil.UserServerConfigPath())
	}
	if len(paths) > 1 {
		return fmt.Errorf("Ambiguity. Server config file %s d specify more than 1 disk-packed storage handler. Please specify one of: %v", osutil.UserServerConfigPath(), paths)
	}
	path = paths[0]
	if path == "" {
		return errors.New("no path is given/found")
	}

	return diskpacked.Reindex(path, c.overwrite, indexConf)
}
<commit_msg>cmd/camtool reindex-diskpacked: don't panic on high-level conf<commit_after>/*
Copyright 2013 Google Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"errors"
	"flag"
	"fmt"
	"log"
	"os"

	"camlistore.org/pkg/blobserver/diskpacked"
	"camlistore.org/pkg/cmdmain"
	"camlistore.org/pkg/jsonconfig"
	"camlistore.org/pkg/osutil"
	"camlistore.org/pkg/serverinit"
)

type reindexdpCmd struct {
	overwrite, verbose bool
}

func init() {
	cmdmain.RegisterCommand("reindex-diskpacked",
		func(flags *flag.FlagSet) cmdmain.CommandRunner {
			cmd := new(reindexdpCmd)
			flags.BoolVar(&cmd.overwrite, "overwrite", false,
				"Whether to overwrite the existing index. If false, only check.")
			return cmd
		})
}

func (c *reindexdpCmd) Describe() string {
	return "Rebuild the index of the diskpacked blob store"
}

func (c *reindexdpCmd) Usage() {
	fmt.Fprintln(os.Stderr, "Usage: camtool [globalopts] reindex-diskpacked [reindex-opts]")
	fmt.Fprintln(os.Stderr, "       camtool reindex-diskpacked [--overwrite] # dir from server config")
	fmt.Fprintln(os.Stderr, "       camtool reindex-diskpacked [--overwrite] /path/to/directory")
}

func (c *reindexdpCmd) RunCommand(args []string) error {
	var path string
	var indexConf jsonconfig.Obj

	switch len(args) {
	case 0:
	case 1:
		path = args[0]
	default:
		return errors.New("More than 1 argument not allowed")
	}
	cfg, err := serverinit.LoadFile(osutil.UserServerConfigPath())
	if err != nil {
		return err
	}
	prefixes, ok := cfg.Obj["prefixes"].(map[string]interface{})
	if !ok {
		return fmt.Errorf("No 'prefixes' object in low-level (or converted) config file %s", osutil.UserServerConfigPath())
	}
	paths, confs := []string{}, []jsonconfig.Obj{}
	for prefix, vei := range prefixes {
		pmap, ok := vei.(map[string]interface{})
		if !ok {
			log.Printf("prefix %q value is a %T, not an object", prefix, vei)
			continue
		}
		pconf := jsonconfig.Obj(pmap)
		handlerType := pconf.RequiredString("handler")
		handlerArgs := pconf.OptionalObject("handlerArgs")
		// no pconf.Validate, as this is a recover tool
		if handlerType != "storage-diskpacked" {
			continue
		}
		log.Printf("handlerArgs of %q: %v", prefix, handlerArgs)
		if handlerArgs == nil {
			log.Printf("no handlerArgs for %q", prefix)
			continue
		}
		aconf := jsonconfig.Obj(handlerArgs)
		apath := aconf.RequiredString("path")
		// no aconv.Validate, as this is a recover tool
		if apath == "" {
			log.Printf("path is missing for %q", prefix)
			continue
		}
		if path != "" && path != apath {
			continue
		}
		paths = append(paths, apath)
		confs = append(confs, aconf)
	}
	if len(paths) == 0 {
		return fmt.Errorf("Server config file %s doesn't specify a disk-packed storage handler.",
			osutil.UserServerConfigPath())
	}
	if len(paths) > 1 {
		return fmt.Errorf("Ambiguity. Server config file %s d specify more than 1 disk-packed storage handler. Please specify one of: %v", osutil.UserServerConfigPath(), paths)
	}
	path = paths[0]
	if path == "" {
		return errors.New("no path is given/found")
	}
	// If no index is specified, the default will be used (as on the regular path).
	if mi := confs[0]["metaIndex"]; mi != nil {
		if mi, ok := mi.(map[string]interface{}); ok {
			indexConf = jsonconfig.Obj(mi)
		}
	}
	log.Printf("indexConf: %v", indexConf)

	return diskpacked.Reindex(path, c.overwrite, indexConf)
}
<|endoftext|>


<commit_before>package command

import (
	"flag"
	"fmt"
	"strings"
	"time"

	"github.com/funkygao/gafka/ctx"
	"github.com/funkygao/gafka/zk"
	"github.com/funkygao/gocli"
	"github.com/funkygao/golib/color"
	"github.com/funkygao/golib/gofmt"
)

type Controllers struct {
	Ui  cli.Ui
	Cmd string
}

func (this *Controllers) Run(args []string) (exitCode int) {
	var (
		cluster string
		zone    string
	)
	cmdFlags := flag.NewFlagSet("controllers", flag.ContinueOnError)
	cmdFlags.Usage = func() { this.Ui.Output(this.Help()) }
	cmdFlags.StringVar(&zone, "z", "", "")
	cmdFlags.StringVar(&cluster, "c", "", "") // TODO not used
	if err := cmdFlags.Parse(args); err != nil {
		return 1
	}

	if zone == "" {
		forSortedZones(func(zkzone *zk.ZkZone) {
			this.printControllers(zkzone)
			printSwallowedErrors(this.Ui, zkzone)
		})

		return
	}

	zkzone := zk.NewZkZone(zk.DefaultConfig(zone, ctx.ZoneZkAddrs(zone)))
	this.printControllers(zkzone)
	printSwallowedErrors(this.Ui, zkzone)

	return
}

// Print all controllers of all clusters within a zone.
func (this *Controllers) printControllers(zkzone *zk.ZkZone) {
	this.Ui.Output(zkzone.Name())
	zkzone.ForSortedControllers(func(cluster string, controller *zk.ControllerMeta) {
		this.Ui.Output(strings.Repeat(" ", 4) + cluster)
		if controller == nil {
			this.Ui.Output(fmt.Sprintf("\t%s", color.Red("empty")))
		} else {
			epochSince := time.Since(controller.Mtime.Time())
			epochSinceStr := gofmt.PrettySince(controller.Mtime.Time())
			if epochSince < time.Hour*2*24 {
				epochSinceStr = color.Red(epochSinceStr)
			}
			this.Ui.Output(fmt.Sprintf("\t%-2s %21s epoch:%2s/%-20s uptime:%s",
				controller.Broker.Id, controller.Broker.Addr(),
				controller.Epoch,
				epochSinceStr,
				gofmt.PrettySince(controller.Broker.Uptime())))
		}
	})

}

func (*Controllers) Synopsis() string {
	return "Print active controllers in kafka clusters"
}

func (this *Controllers) Help() string {
	help := fmt.Sprintf(`
Usage: %s controllers [options]

    Print active controllers in kafka clusters

Options:

    -z zone
      Only print kafka controllers within this zone.

    -c cluster

`, this.Cmd)
	return strings.TrimSpace(help)
}
<commit_msg>'gk controllers' can limit by cluster name<commit_after>package command

import (
	"flag"
	"fmt"
	"strings"
	"time"

	"github.com/funkygao/gafka/ctx"
	"github.com/funkygao/gafka/zk"
	"github.com/funkygao/gocli"
	"github.com/funkygao/golib/color"
	"github.com/funkygao/golib/gofmt"
)

type Controllers struct {
	Ui  cli.Ui
	Cmd string

	cluster string
}

func (this *Controllers) Run(args []string) (exitCode int) {
	var (
		zone string
	)
	cmdFlags := flag.NewFlagSet("controllers", flag.ContinueOnError)
	cmdFlags.Usage = func() { this.Ui.Output(this.Help()) }
	cmdFlags.StringVar(&zone, "z", "", "")
	cmdFlags.StringVar(&this.cluster, "c", "", "") // TODO not used
	if err := cmdFlags.Parse(args); err != nil {
		return 1
	}

	if zone == "" {
		forSortedZones(func(zkzone *zk.ZkZone) {
			this.printControllers(zkzone)
			printSwallowedErrors(this.Ui, zkzone)
		})

		return
	}

	zkzone := zk.NewZkZone(zk.DefaultConfig(zone, ctx.ZoneZkAddrs(zone)))
	this.printControllers(zkzone)
	printSwallowedErrors(this.Ui, zkzone)

	return
}

// Print all controllers of all clusters within a zone.
func (this *Controllers) printControllers(zkzone *zk.ZkZone) {
	this.Ui.Output(zkzone.Name())
	zkzone.ForSortedControllers(func(cluster string, controller *zk.ControllerMeta) {
		if !patternMatched(cluster, this.cluster) {
			return
		}

		this.Ui.Output(strings.Repeat(" ", 4) + cluster)
		if controller == nil {
			this.Ui.Output(fmt.Sprintf("\t%s", color.Red("empty")))
		} else {
			epochSince := time.Since(controller.Mtime.Time())
			epochSinceStr := gofmt.PrettySince(controller.Mtime.Time())
			if epochSince < time.Hour*2*24 {
				epochSinceStr = color.Red(epochSinceStr)
			}
			this.Ui.Output(fmt.Sprintf("\t%-2s %21s epoch:%2s/%-20s uptime:%s",
				controller.Broker.Id, controller.Broker.Addr(),
				controller.Epoch,
				epochSinceStr,
				gofmt.PrettySince(controller.Broker.Uptime())))
		}
	})

}

func (*Controllers) Synopsis() string {
	return "Print active controllers in kafka clusters"
}

func (this *Controllers) Help() string {
	help := fmt.Sprintf(`
Usage: %s controllers [options]

    Print active controllers in kafka clusters

Options:

    -z zone
      Only print kafka controllers within this zone.

    -c cluster

`, this.Cmd)
	return strings.TrimSpace(help)
}
<|endoftext|>


<commit_before>package main

import (
	"archive/tar"
	"compress/gzip"
	"errors"
	"fmt"
	"github.com/Symantec/Dominator/imageserver/client"
	"github.com/Symantec/Dominator/lib/filesystem"
	"github.com/Symantec/Dominator/lib/filesystem/scanner"
	"github.com/Symantec/Dominator/lib/filesystem/untar"
	"github.com/Symantec/Dominator/lib/filter"
	"github.com/Symantec/Dominator/lib/hash"
	"github.com/Symantec/Dominator/lib/image"
	"github.com/Symantec/Dominator/lib/mbr"
	objectclient "github.com/Symantec/Dominator/lib/objectserver/client"
	"github.com/Symantec/Dominator/lib/srpc"
	"github.com/Symantec/Dominator/lib/srpc/setupclient"
	"github.com/Symantec/Dominator/lib/wsyscall"
	"io"
	"os"
	"os/exec"
	"strings"
	"syscall"
	"time"
)

func addImagefileSubcommand(args []string) {
	imageSClient, objectClient := getClients()
	err := addImagefile(imageSClient, objectClient, args[0], args[1], args[2],
		args[3])
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error adding image: \"%s\": %s\n", args[0], err)
		os.Exit(1)
	}
	os.Exit(0)
}

func addImagefile(imageSClient *srpc.Client,
	objectClient *objectclient.ObjectClient,
	name, imageFilename, filterFilename, triggersFilename string) error {
	imageExists, err := client.CheckImage(imageSClient, name)
	if err != nil {
		return errors.New("error checking for image existance: " + err.Error())
	}
	if imageExists {
		return errors.New("image exists")
	}
	newImage := new(image.Image)
	if err := loadImageFiles(newImage, objectClient, filterFilename,
		triggersFilename); err != nil {
		return err
	}
	newImage.FileSystem, err = buildImage(imageSClient, newImage.Filter,
		imageFilename)
	if err != nil {
		return errors.New("error building image: " + err.Error())
	}
	if err := spliceComputedFiles(newImage.FileSystem); err != nil {
		return err
	}
	return addImage(imageSClient, name, newImage)
}

func addImage(imageSClient *srpc.Client, name string, img *image.Image) error {
	if *expiresIn > 0 {
		img.ExpiresAt = time.Now().Add(*expiresIn)
	}
	if err := img.Verify(); err != nil {
		return err
	}
	if err := img.VerifyRequiredPaths(requiredPaths); err != nil {
		return err
	}
	if err := client.AddImage(imageSClient, name, img); err != nil {
		return errors.New("remote error: " + err.Error())
	}
	return nil
}

type hasher struct {
	objQ *objectclient.ObjectAdderQueue
}

func (h *hasher) Hash(reader io.Reader, length uint64) (
	hash.Hash, error) {
	hash, err := h.objQ.Add(reader, length)
	if err != nil {
		return hash, errors.New("error sending image data: " + err.Error())
	}
	return hash, nil
}

func buildImage(imageSClient *srpc.Client, filter *filter.Filter,
	imageFilename string) (*filesystem.FileSystem, error) {
	var h hasher
	var err error
	h.objQ, err = objectclient.NewObjectAdderQueue(imageSClient)
	if err != nil {
		return nil, err
	}
	fs, err := buildImageWithHasher(imageSClient, filter, imageFilename, &h)
	if err != nil {
		h.objQ.Close()
		return nil, err
	}
	err = h.objQ.Close()
	if err != nil {
		return nil, err
	}
	return fs, nil
}

func buildImageWithHasher(imageSClient *srpc.Client, filter *filter.Filter,
	imageFilename string, h *hasher) (*filesystem.FileSystem, error) {
	fi, err := os.Lstat(imageFilename)
	if err != nil {
		return nil, err
	}
	if fi.IsDir() {
		sfs, err := scanner.ScanFileSystem(imageFilename, nil, filter, nil, h,
			nil)
		if err != nil {
			return nil, err
		}
		return &sfs.FileSystem, nil
	}
	imageFile, err := os.Open(imageFilename)
	if err != nil {
		return nil, errors.New("error opening image file: " + err.Error())
	}
	defer imageFile.Close()
	if partitionTable, err := mbr.Decode(imageFile); err != nil {
		return nil, err
	} else if partitionTable != nil {
		return buildImageFromRaw(imageSClient, filter, imageFile,
			partitionTable, h)
	}
	var imageReader io.Reader
	if strings.HasSuffix(imageFilename, ".tar") {
		imageReader = imageFile
	} else if strings.HasSuffix(imageFilename, ".tar.gz") ||
		strings.HasSuffix(imageFilename, ".tgz") {
		gzipReader, err := gzip.NewReader(imageFile)
		if err != nil {
			return nil, errors.New(
				"error creating gzip reader: " + err.Error())
		}
		defer gzipReader.Close()
		imageReader = gzipReader
	} else {
		return nil, errors.New("unrecognised image type")
	}
	tarReader := tar.NewReader(imageReader)
	fs, err := untar.Decode(tarReader, h, filter)
	if err != nil {
		return nil, errors.New("error building image: " + err.Error())
	}
	return fs, nil
}

func buildImageFromRaw(imageSClient *srpc.Client, filter *filter.Filter,
	imageFile *os.File, partitionTable *mbr.Mbr,
	h *hasher) (*filesystem.FileSystem, error) {
	var index uint
	var offsetOfLargest, sizeOfLargest uint64
	numPartitions := partitionTable.GetNumPartitions()
	for index = 0; index < numPartitions; index++ {
		offset := partitionTable.GetPartitionOffset(index)
		size := partitionTable.GetPartitionSize(index)
		if size > sizeOfLargest {
			offsetOfLargest = offset
			sizeOfLargest = size
		}
	}
	if sizeOfLargest < 1 {
		return nil, errors.New("unable to find largest partition")
	}
	if err := wsyscall.UnshareMountNamespace(); err != nil {
		if os.IsPermission(err) {
			// Try again with sudo(8).
			args := make([]string, 0, len(os.Args)+1)
			if sudoPath, err := exec.LookPath("sudo"); err != nil {
				return nil, err
			} else {
				args = append(args, sudoPath)
			}
			args = append(args, os.Args[0])
			args = append(args, fmt.Sprintf("-certDirectory=%s",
				setupclient.GetCertDirectory()))
			args = append(args, os.Args[1:]...)
			if err := syscall.Exec(args[0], args, os.Environ()); err != nil {
				return nil, errors.New("unable to Exec: " + err.Error())
			}
		}
		return nil, errors.New(
			"error unsharing mount namespace: " + err.Error())
	}
	cmd := exec.Command("mount", "-o",
		fmt.Sprintf("loop,offset=%d", offsetOfLargest), imageFile.Name(),
		"/mnt")
	cmd.Stdin = os.Stdin
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	if err := cmd.Run(); err != nil {
		return nil, err
	}
	return buildImageWithHasher(imageSClient, filter, "/mnt", h)
}
<commit_msg>Lookup imagetool path when re-execing via sudo.<commit_after>package main

import (
	"archive/tar"
	"compress/gzip"
	"errors"
	"fmt"
	"github.com/Symantec/Dominator/imageserver/client"
	"github.com/Symantec/Dominator/lib/filesystem"
	"github.com/Symantec/Dominator/lib/filesystem/scanner"
	"github.com/Symantec/Dominator/lib/filesystem/untar"
	"github.com/Symantec/Dominator/lib/filter"
	"github.com/Symantec/Dominator/lib/hash"
	"github.com/Symantec/Dominator/lib/image"
	"github.com/Symantec/Dominator/lib/mbr"
	objectclient "github.com/Symantec/Dominator/lib/objectserver/client"
	"github.com/Symantec/Dominator/lib/srpc"
	"github.com/Symantec/Dominator/lib/srpc/setupclient"
	"github.com/Symantec/Dominator/lib/wsyscall"
	"io"
	"os"
	"os/exec"
	"strings"
	"syscall"
	"time"
)

func addImagefileSubcommand(args []string) {
	imageSClient, objectClient := getClients()
	err := addImagefile(imageSClient, objectClient, args[0], args[1], args[2],
		args[3])
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error adding image: \"%s\": %s\n", args[0], err)
		os.Exit(1)
	}
	os.Exit(0)
}

func addImagefile(imageSClient *srpc.Client,
	objectClient *objectclient.ObjectClient,
	name, imageFilename, filterFilename, triggersFilename string) error {
	imageExists, err := client.CheckImage(imageSClient, name)
	if err != nil {
		return errors.New("error checking for image existance: " + err.Error())
	}
	if imageExists {
		return errors.New("image exists")
	}
	newImage := new(image.Image)
	if err := loadImageFiles(newImage, objectClient, filterFilename,
		triggersFilename); err != nil {
		return err
	}
	newImage.FileSystem, err = buildImage(imageSClient, newImage.Filter,
		imageFilename)
	if err != nil {
		return errors.New("error building image: " + err.Error())
	}
	if err := spliceComputedFiles(newImage.FileSystem); err != nil {
		return err
	}
	return addImage(imageSClient, name, newImage)
}

func addImage(imageSClient *srpc.Client, name string, img *image.Image) error {
	if *expiresIn > 0 {
		img.ExpiresAt = time.Now().Add(*expiresIn)
	}
	if err := img.Verify(); err != nil {
		return err
	}
	if err := img.VerifyRequiredPaths(requiredPaths); err != nil {
		return err
	}
	if err := client.AddImage(imageSClient, name, img); err != nil {
		return errors.New("remote error: " + err.Error())
	}
	return nil
}

type hasher struct {
	objQ *objectclient.ObjectAdderQueue
}

func (h *hasher) Hash(reader io.Reader, length uint64) (
	hash.Hash, error) {
	hash, err := h.objQ.Add(reader, length)
	if err != nil {
		return hash, errors.New("error sending image data: " + err.Error())
	}
	return hash, nil
}

func buildImage(imageSClient *srpc.Client, filter *filter.Filter,
	imageFilename string) (*filesystem.FileSystem, error) {
	var h hasher
	var err error
	h.objQ, err = objectclient.NewObjectAdderQueue(imageSClient)
	if err != nil {
		return nil, err
	}
	fs, err := buildImageWithHasher(imageSClient, filter, imageFilename, &h)
	if err != nil {
		h.objQ.Close()
		return nil, err
	}
	err = h.objQ.Close()
	if err != nil {
		return nil, err
	}
	return fs, nil
}

func buildImageWithHasher(imageSClient *srpc.Client, filter *filter.Filter,
	imageFilename string, h *hasher) (*filesystem.FileSystem, error) {
	fi, err := os.Lstat(imageFilename)
	if err != nil {
		return nil, err
	}
	if fi.IsDir() {
		sfs, err := scanner.ScanFileSystem(imageFilename, nil, filter, nil, h,
			nil)
		if err != nil {
			return nil, err
		}
		return &sfs.FileSystem, nil
	}
	imageFile, err := os.Open(imageFilename)
	if err != nil {
		return nil, errors.New("error opening image file: " + err.Error())
	}
	defer imageFile.Close()
	if partitionTable, err := mbr.Decode(imageFile); err != nil {
		return nil, err
	} else if partitionTable != nil {
		return buildImageFromRaw(imageSClient, filter, imageFile,
			partitionTable, h)
	}
	var imageReader io.Reader
	if strings.HasSuffix(imageFilename, ".tar") {
		imageReader = imageFile
	} else if strings.HasSuffix(imageFilename, ".tar.gz") ||
		strings.HasSuffix(imageFilename, ".tgz") {
		gzipReader, err := gzip.NewReader(imageFile)
		if err != nil {
			return nil, errors.New(
				"error creating gzip reader: " + err.Error())
		}
		defer gzipReader.Close()
		imageReader = gzipReader
	} else {
		return nil, errors.New("unrecognised image type")
	}
	tarReader := tar.NewReader(imageReader)
	fs, err := untar.Decode(tarReader, h, filter)
	if err != nil {
		return nil, errors.New("error building image: " + err.Error())
	}
	return fs, nil
}

func buildImageFromRaw(imageSClient *srpc.Client, filter *filter.Filter,
	imageFile *os.File, partitionTable *mbr.Mbr,
	h *hasher) (*filesystem.FileSystem, error) {
	var index uint
	var offsetOfLargest, sizeOfLargest uint64
	numPartitions := partitionTable.GetNumPartitions()
	for index = 0; index < numPartitions; index++ {
		offset := partitionTable.GetPartitionOffset(index)
		size := partitionTable.GetPartitionSize(index)
		if size > sizeOfLargest {
			offsetOfLargest = offset
			sizeOfLargest = size
		}
	}
	if sizeOfLargest < 1 {
		return nil, errors.New("unable to find largest partition")
	}
	if err := wsyscall.UnshareMountNamespace(); err != nil {
		if os.IsPermission(err) {
			// Try again with sudo(8).
			args := make([]string, 0, len(os.Args)+1)
			if sudoPath, err := exec.LookPath("sudo"); err != nil {
				return nil, err
			} else {
				args = append(args, sudoPath)
			}
			if myPath, err := exec.LookPath(os.Args[0]); err != nil {
				return nil, err
			} else {
				args = append(args, myPath)
			}
			args = append(args, fmt.Sprintf("-certDirectory=%s",
				setupclient.GetCertDirectory()))
			args = append(args, os.Args[1:]...)
			if err := syscall.Exec(args[0], args, os.Environ()); err != nil {
				return nil, errors.New("unable to Exec: " + err.Error())
			}
		}
		return nil, errors.New(
			"error unsharing mount namespace: " + err.Error())
	}
	cmd := exec.Command("mount", "-o",
		fmt.Sprintf("loop,offset=%d", offsetOfLargest), imageFile.Name(),
		"/mnt")
	cmd.Stdin = os.Stdin
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	if err := cmd.Run(); err != nil {
		return nil, err
	}
	return buildImageWithHasher(imageSClient, filter, "/mnt", h)
}
<|endoftext|>


<commit_before>// Copyright 2016-2017 the u-root Authors. All rights reserved
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// This program validates a file by verifying a checksum and a signature file.
//
// Synopsis:
//     validate [OPTIONS...] FILE PUBLIC_KEY_FILE
//
// Description:
//     Return code: 0-OK, 1-Any error, 2-Bad signature, 3-Bad checksum
//
// Options:
//     -a:        signature is ASCII armored
//     -i FILE:   checksum file
//     -alg FILE: algorithms to check
//     -v:        verbose
package main

import (
	"crypto"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"strings"

	_ "crypto/md5"
	_ "crypto/sha1"
	_ "crypto/sha256"
	_ "crypto/sha512"

	_ "golang.org/x/crypto/md4"
	_ "golang.org/x/crypto/openpgp"
	_ "golang.org/x/crypto/ripemd160"
	_ "golang.org/x/crypto/sha3"
)

var (
	algs = map[string]crypto.Hash{
		"MD4":       crypto.MD4,
		"MD5":       crypto.MD5,
		"SHA1":      crypto.SHA1,
		"SHA224":    crypto.SHA224,
		"SHA256":    crypto.SHA256,
		"SHA384":    crypto.SHA384,
		"SHA512":    crypto.SHA512,
		"RIPEMD160": crypto.RIPEMD160,
		"SHA3_224":  crypto.SHA3_224,
		"SHA3_256":  crypto.SHA3_256,
		"SHA3_384":  crypto.SHA3_384,
		"SHA3_512":  crypto.SHA3_512,
	}

	armored    = flag.Bool("a", false, "signature is ASCII armored")
	sumfile    = flag.String("i", "", "checksum file")
	alg        = flag.String("alg", "", "algorithms to check")
	verbose    = flag.Bool("v", false, "verbose")
	debug      = func(string, ...interface{}) {}
	try, tried []string
)

func init() {
	for v := range algs {
		try = append(try, v)
	}
}

func one(n string, b []byte, sig string) bool {
	if !algs[n].Available() {
		log.Printf("Hash %v did not get linked in", n)
		return false
	}
	debug("Check alg %v", n)
	checker := algs[n].New()
	checker.Write(b)
	r := checker.Sum([]byte{})
	debug("sum is %d bytes %v\n", len(r), r)
	tried = append(tried, n)

	// There has to be a better way.
	sumText := ""
	for _, v := range r {
		sumText += fmt.Sprintf("%02x", v)
	}
	debug("Compare to %v", sumText)
	if sumText == sig {
		return true
	}
	return false
}

func sign(n string, k crypto.PrivateKey, b []byte, sig string) bool {
	if !algs[n].Available() {
		log.Printf("Hash %v did not get linked in", n)
		return false
	}
	debug("Check alg %v", n)
	b, err := k.(crypto.Signer).Sign(nil, b, algs[n])
	log.Printf("For %v we get %v %v", n, b, err)
	return false
}

func main() {
	flag.Parse()
	if flag.NArg() < 2 {
		log.Fatalf("Need at least a file to be validated and one public key")
	}

	if *verbose {
		debug = log.Printf
	}

	v, f := flag.Args()[0], flag.Args()[1]

	sigData, err := ioutil.ReadFile(v)
	if err != nil {
		log.Fatalf("%v", err)
	}

	if *alg != "" {
		try = []string{}
		for _, v := range strings.Split(*alg, ",") {
			try = append(try, v)
		}
	}

	log.Printf("Try %v", try)

	sig := strings.Split(string(sigData), " ")

	debug("Signature is %v len %v", sig[0], len(sig[0]))

	b, err := ioutil.ReadFile(f)
	if err != nil {
		log.Fatalf("%s: %v", f, err)
	}

	if *sumfile != "" {
		sum, err := ioutil.ReadFile(*sumfile)
		if err != nil {
			log.Fatalf("Can't read sumfile: %v", *sumfile)
		}
		for i := range try {
			if sign(try[i], crypto.PrivateKey(sum), b, sig[0]) {
				fmt.Printf("%v\n", try[i])
				os.Exit(0)
			}
			debug("not ok")
		}

	} else {
		for i := range try {
			debug("Check %v", try[i])
			if one(try[i], b, sig[0]) {
				fmt.Printf("%v\n", try[i])
				os.Exit(0)
			}
			// Sometimes it's not a file in the standard format, but some binary thing.
			// Check that too.
			if one(try[i], b, string(sigData)) {
				fmt.Printf("%v\n", try[i])
				os.Exit(0)
			}
			debug("not ok")
		}
	}
	log.Fatalf("No matches found for %v", tried)
}
<commit_msg>replace for loop<commit_after>// Copyright 2016-2017 the u-root Authors. All rights reserved
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// This program validates a file by verifying a checksum and a signature file.
//
// Synopsis:
//     validate [OPTIONS...] FILE PUBLIC_KEY_FILE
//
// Description:
//     Return code: 0-OK, 1-Any error, 2-Bad signature, 3-Bad checksum
//
// Options:
//     -a:        signature is ASCII armored
//     -i FILE:   checksum file
//     -alg FILE: algorithms to check
//     -v:        verbose
package main

import (
	"crypto"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"strings"

	_ "crypto/md5"
	_ "crypto/sha1"
	_ "crypto/sha256"
	_ "crypto/sha512"

	_ "golang.org/x/crypto/md4"
	_ "golang.org/x/crypto/openpgp"
	_ "golang.org/x/crypto/ripemd160"
	_ "golang.org/x/crypto/sha3"
)

var (
	algs = map[string]crypto.Hash{
		"MD4":       crypto.MD4,
		"MD5":       crypto.MD5,
		"SHA1":      crypto.SHA1,
		"SHA224":    crypto.SHA224,
		"SHA256":    crypto.SHA256,
		"SHA384":    crypto.SHA384,
		"SHA512":    crypto.SHA512,
		"RIPEMD160": crypto.RIPEMD160,
		"SHA3_224":  crypto.SHA3_224,
		"SHA3_256":  crypto.SHA3_256,
		"SHA3_384":  crypto.SHA3_384,
		"SHA3_512":  crypto.SHA3_512,
	}

	armored    = flag.Bool("a", false, "signature is ASCII armored")
	sumfile    = flag.String("i", "", "checksum file")
	alg        = flag.String("alg", "", "algorithms to check")
	verbose    = flag.Bool("v", false, "verbose")
	debug      = func(string, ...interface{}) {}
	try, tried []string
)

func init() {
	for v := range algs {
		try = append(try, v)
	}
}

func one(n string, b []byte, sig string) bool {
	if !algs[n].Available() {
		log.Printf("Hash %v did not get linked in", n)
		return false
	}
	debug("Check alg %v", n)
	checker := algs[n].New()
	checker.Write(b)
	r := checker.Sum([]byte{})
	debug("sum is %d bytes %v\n", len(r), r)
	tried = append(tried, n)

	// There has to be a better way.
	sumText := ""
	for _, v := range r {
		sumText += fmt.Sprintf("%02x", v)
	}
	debug("Compare to %v", sumText)
	if sumText == sig {
		return true
	}
	return false
}

func sign(n string, k crypto.PrivateKey, b []byte, sig string) bool {
	if !algs[n].Available() {
		log.Printf("Hash %v did not get linked in", n)
		return false
	}
	debug("Check alg %v", n)
	b, err := k.(crypto.Signer).Sign(nil, b, algs[n])
	log.Printf("For %v we get %v %v", n, b, err)
	return false
}

func main() {
	flag.Parse()
	if flag.NArg() < 2 {
		log.Fatalf("Need at least a file to be validated and one public key")
	}

	if *verbose {
		debug = log.Printf
	}

	v, f := flag.Args()[0], flag.Args()[1]

	sigData, err := ioutil.ReadFile(v)
	if err != nil {
		log.Fatalf("%v", err)
	}

	if *alg != "" {
		try = append(try, strings.Split(*alg, ",")...)
	}

	log.Printf("Try %v", try)

	sig := strings.Split(string(sigData), " ")

	debug("Signature is %v len %v", sig[0], len(sig[0]))

	b, err := ioutil.ReadFile(f)
	if err != nil {
		log.Fatalf("%s: %v", f, err)
	}

	if *sumfile != "" {
		sum, err := ioutil.ReadFile(*sumfile)
		if err != nil {
			log.Fatalf("Can't read sumfile: %v", *sumfile)
		}
		for i := range try {
			if sign(try[i], crypto.PrivateKey(sum), b, sig[0]) {
				fmt.Printf("%v\n", try[i])
				os.Exit(0)
			}
			debug("not ok")
		}

	} else {
		for i := range try {
			debug("Check %v", try[i])
			if one(try[i], b, sig[0]) {
				fmt.Printf("%v\n", try[i])
				os.Exit(0)
			}
			// Sometimes it's not a file in the standard format, but some binary thing.
			// Check that too.
			if one(try[i], b, string(sigData)) {
				fmt.Printf("%v\n", try[i])
				os.Exit(0)
			}
			debug("not ok")
		}
	}
	log.Fatalf("No matches found for %v", tried)
}
<|endoftext|>