<commit_before>package main

import (
	"context"
	"flag"
	"fmt"
	"io"
	stdlog "log"
	"net/http"
	"os"
	"os/signal"
	"runtime"
	"syscall"
	"time"

	_ "expvar"

	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/cloudwatchlogs"
	"github.com/bepress/camo/logging"
	"github.com/bepress/camo/proxy"
	"github.com/reedobrien/cowl"
	"github.com/rs/zerolog"
)

const (
	app       = "camo"
	drainTime = 10 * time.Second
)

var (
	// BuildDate is populated on build for version info.
	BuildDate string

	// GitBranch is populated on build for version info.
	GitBranch string

	// GitHash is populated on build for version info.
	GitHash string

	// BuildVersion is populated on build for version info.
	BuildVersion string
)

func main() {
	var (
		addr    = flag.String("addr", ":443", "The address and port to listen on")
		maxsize = flag.Int64("maxsize", 5, "Maximum size to proxy in whole MB (no decimal)")
		secret  = flag.String("secret", "", "The 'shared secret' hmac key")
		tlscert = flag.String("cert", "cert.pem", "The TLS certificate to use")
		tlskey  = flag.String("key", "key.pem", "The TLS key to use")
		verbose = flag.Bool("verbose", false, "If verbose logging should take place (No-op at this time as there's no debug log statements)")
		version = flag.Bool("version", false, "Display version and build info, then exit")

		// TODO(ro) 2017-10-10 Add flags for other proxy set-ables.
		logger zerolog.Logger
	)
	flag.Parse()

	if *version {
		versionInfo()
	}

	// Create a context so we can cancel goroutines.
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel() // Make sure they go away when main exits.
	hostname, err := os.Hostname()
	if err != nil {
		panic("failed to get hostname")
	}
	// Start an aws session, and make the cloudwatchlogs service.
	session := session.Must(session.NewSession())
	svc := cloudwatchlogs.New(session)

	// Create the cloudwatchlog writer.
	cwl := cowl.MustNewWriterWithContext(ctx, svc, app, hostname+"-app")
	// Write to both stdout and cwl.
	w := io.MultiWriter(cwl, os.Stdout)
	// Set up the logger to use it.
	logger = logging.NewLogger(app, *verbose, w).With().Str(
		"handler", "proxy").Logger()
	if BuildVersion != "" {
		logger = logger.With().
			Str("app_version", BuildVersion+"-"+GitHash).
			Logger()
	}

	stdlog.SetFlags(0)
	stdlog.SetOutput(logger)

	// Stop channel to block until we get a signal. This is used for graceful
	// shutdown.
	stop := make(chan os.Signal, 1)
	signal.Notify(stop, syscall.SIGINT, syscall.SIGTERM)

	// Set up options.
	// TODO(ro) 2017-10-11 Add more options here and as flags as necessary.
	options := []func(*proxy.Proxy){}
	if *maxsize > 0 {
		options = append(options, func(p *proxy.Proxy) { p.MaxSize = *maxsize * 1024 * 1024 })
	}

	// Create proxy handler.
	p := proxy.MustNew([]byte(*secret), logger, options...)
	// Wrap proxy handler with logger.
	proxyHandler := logging.NewAccessLogger(p, logger)
	s := http.Server{Addr: *addr, Handler: proxyHandler}

	// Start TLS server.
	go func() {
		if err := s.ListenAndServeTLS(*tlscert, *tlskey); err != http.ErrServerClosed {
			logger.Fatal().Err(err).Msg("failed to start server")
		}
	}()
	// Start a server on localhost:9000 for expvar.
	go func() {
		if err := http.ListenAndServe("127.0.0.1:9000", nil); err != nil {
			logger.Fatal().Err(err).Msg("failed to start expvar server")
		}
	}()

	<-stop

	// Create a context with timeout to limit shutdown time in the event of
	// slow clients.
	ctx, cancel = context.WithTimeout(ctx, drainTime)
	defer cancel()

	if err := s.Shutdown(ctx); err != nil {
		logger.Error().Err(err).Msg("caught on proxy.Shutdown")
	}
}

func versionInfo() {
	fmt.Printf(`
build date: %s
branch    : %s
hash      : %s
version   : %s
built with: %s
`[1:], BuildDate, GitBranch, GitHash, BuildVersion, runtime.Version())
	os.Exit(0)
}
<commit_msg>make app the log stream prefix so we can aggregate<commit_after>package main

import (
	"context"
	"flag"
	"fmt"
	"io"
	stdlog "log"
	"net/http"
	"os"
	"os/signal"
	"runtime"
	"syscall"
	"time"

	_ "expvar"

	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/cloudwatchlogs"
	"github.com/bepress/camo/logging"
	"github.com/bepress/camo/proxy"
	"github.com/reedobrien/cowl"
	"github.com/rs/zerolog"
)

const (
	app       = "camo"
	drainTime = 10 * time.Second
)

var (
	// BuildDate is populated on build for version info.
	BuildDate string

	// GitBranch is populated on build for version info.
	GitBranch string

	// GitHash is populated on build for version info.
	GitHash string

	// BuildVersion is populated on build for version info.
	BuildVersion string
)

func main() {
	var (
		addr    = flag.String("addr", ":443", "The address and port to listen on")
		maxsize = flag.Int64("maxsize", 5, "Maximum size to proxy in whole MB (no decimal)")
		secret  = flag.String("secret", "", "The 'shared secret' hmac key")
		tlscert = flag.String("cert", "cert.pem", "The TLS certificate to use")
		tlskey  = flag.String("key", "key.pem", "The TLS key to use")
		verbose = flag.Bool("verbose", false, "If verbose logging should take place (No-op at this time as there's no debug log statements)")
		version = flag.Bool("version", false, "Display version and build info, then exit")

		// TODO(ro) 2017-10-10 Add flags for other proxy set-ables.
		logger zerolog.Logger
	)
	flag.Parse()

	if *version {
		versionInfo()
	}

	// Create a context so we can cancel goroutines.
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel() // Make sure they go away when main exits.
	hostname, err := os.Hostname()
	if err != nil {
		panic("failed to get hostname")
	}
	// Start an aws session, and make the cloudwatchlogs service.
	session := session.Must(session.NewSession())
	svc := cloudwatchlogs.New(session)

	// Create the cloudwatchlog writer.
	cwl := cowl.MustNewWriterWithContext(ctx, svc, app, "app-"+hostname)
	// Write to both stdout and cwl.
	w := io.MultiWriter(cwl, os.Stdout)
	// Set up the logger to use it.
	logger = logging.NewLogger(app, *verbose, w).With().Str(
		"handler", "proxy").Logger()
	if BuildVersion != "" {
		logger = logger.With().
			Str("app_version", BuildVersion+"-"+GitHash).
			Logger()
	}

	stdlog.SetFlags(0)
	stdlog.SetOutput(logger)

	// Stop channel to block until we get a signal. This is used for graceful
	// shutdown.
	stop := make(chan os.Signal, 1)
	signal.Notify(stop, syscall.SIGINT, syscall.SIGTERM)

	// Set up options.
	// TODO(ro) 2017-10-11 Add more options here and as flags as necessary.
	options := []func(*proxy.Proxy){}
	if *maxsize > 0 {
		options = append(options, func(p *proxy.Proxy) { p.MaxSize = *maxsize * 1024 * 1024 })
	}

	// Create proxy handler.
	p := proxy.MustNew([]byte(*secret), logger, options...)
	// Wrap proxy handler with logger.
	proxyHandler := logging.NewAccessLogger(p, logger)
	s := http.Server{Addr: *addr, Handler: proxyHandler}

	// Start TLS server.
	go func() {
		if err := s.ListenAndServeTLS(*tlscert, *tlskey); err != http.ErrServerClosed {
			logger.Fatal().Err(err).Msg("failed to start server")
		}
	}()
	// Start a server on localhost:9000 for expvar.
	go func() {
		if err := http.ListenAndServe("127.0.0.1:9000", nil); err != nil {
			logger.Fatal().Err(err).Msg("failed to start expvar server")
		}
	}()

	<-stop

	// Create a context with timeout to limit shutdown time in the event of
	// slow clients.
	ctx, cancel = context.WithTimeout(ctx, drainTime)
	defer cancel()

	if err := s.Shutdown(ctx); err != nil {
		logger.Error().Err(err).Msg("caught on proxy.Shutdown")
	}
}

func versionInfo() {
	fmt.Printf(`
build date: %s
branch    : %s
hash      : %s
version   : %s
built with: %s
`[1:], BuildDate, GitBranch, GitHash, BuildVersion, runtime.Version())
	os.Exit(0)
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"

	"github.com/docker/docker/api/types"
	"github.com/docker/docker/api/types/container"
	"github.com/docker/docker/client"
	"gopkg.in/yaml.v2"
)

/*
version: '3'
services:
  redis:
    image: 'redis:3.0-alpine'

  busybox:
    image: busybox
*/
type serviceConfig struct {
	Build string `yaml:"build,omitempty"`
	//Command        yaml.Command         `yaml:"command,flow,omitempty"`
	Dockerfile string `yaml:"dockerfile,omitempty"`
	//Environment    yaml.MaporEqualSlice `yaml:"environment,omitempty"`
	Image string `yaml:"image,omitempty"`
	//Links          yaml.MaporColonSlice `yaml:"links,omitempty"`
	Name        string   `yaml:"name,omitempty"`
	Ports       []string `yaml:"ports,omitempty"`
	Restart     string   `yaml:"restart,omitempty"`
	Volumes     []string `yaml:"volumes,omitempty"`
	VolumesFrom []string `yaml:"volumes_from,omitempty"`
	Expose      []string `yaml:"expose,omitempty"`
}

type dockerComposeConfig struct {
	Version  string                   `yaml:"version,omitempty"`
	Services map[string]serviceConfig `yaml:"services"`
	//networks map[string]     `yaml:"networks,omitempty"`
	//volumes map[string]                  `yaml:"volumes,omitempty"`
}

func (dcy *dockerComposeConfig) Parse(data []byte) error {
	return yaml.Unmarshal(data, dcy)
}

func main() {
	data, err := ioutil.ReadFile("docker-compose.yml")

	if err != nil {
		log.Fatal(err)
	}

	var dockerCyaml dockerComposeConfig
	if err := dockerCyaml.Parse(data); err != nil {
		log.Fatal(err)
	}

	for _, v := range dockerCyaml.Services {
		fmt.Println()
		fmt.Println(v.Image)
		fmt.Println()
		pullImage(v.Image)
	}

}

func pullImage(imagename string) {
	ctx := context.Background()
	cli, err := client.NewEnvClient()
	if err != nil {
		panic(err)
	}

	imagePullResp, err := cli.ImagePull(ctx, imagename, types.ImagePullOptions{})
	if err != nil {
		panic(err)
	}
	defer imagePullResp.Close()
	_, err = io.Copy(os.Stdout, imagePullResp)
	if err != nil {
		log.Println(err)
	}

	containerCreateResp, err := cli.ContainerCreate(ctx, &container.Config{Image: imagename}, &container.HostConfig{PublishAllPorts: true}, nil, "")
	if err != nil {
		log.Fatal(err)
	}

	err = cli.ContainerStart(ctx, containerCreateResp.ID, types.ContainerStartOptions{})
	if err != nil {
		panic(err)
	}

	containerLogResp, err := cli.ContainerLogs(ctx, containerCreateResp.ID, types.ContainerLogsOptions{ShowStdout: true, ShowStderr: true, Timestamps: true})
	if err != nil {
		panic(err)
	}
	defer containerLogResp.Close()

	_, err = io.Copy(os.Stdout, containerLogResp)
	if err != nil {
		log.Println(err)
	}
}
<commit_msg>add network<commit_after>package main

import (
	"context"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"

	"github.com/docker/docker/api/types"
	"github.com/docker/docker/api/types/container"
	"github.com/docker/docker/api/types/network"
	"github.com/docker/docker/client"
	"gopkg.in/yaml.v2"
)

/*
version: '3'
services:
  redis:
    image: 'redis:3.0-alpine'

  busybox:
    image: busybox
*/
type serviceConfig struct {
	Build string `yaml:"build,omitempty"`
	//Command        yaml.Command         `yaml:"command,flow,omitempty"`
	Dockerfile string `yaml:"dockerfile,omitempty"`
	//Environment    yaml.MaporEqualSlice `yaml:"environment,omitempty"`
	Image string `yaml:"image,omitempty"`
	//Links          yaml.MaporColonSlice `yaml:"links,omitempty"`
	Name        string   `yaml:"name,omitempty"`
	Ports       []string `yaml:"ports,omitempty"`
	Restart     string   `yaml:"restart,omitempty"`
	Volumes     []string `yaml:"volumes,omitempty"`
	VolumesFrom []string `yaml:"volumes_from,omitempty"`
	Expose      []string `yaml:"expose,omitempty"`
}

type dockerComposeConfig struct {
	Version  string                   `yaml:"version,omitempty"`
	Services map[string]serviceConfig `yaml:"services"`
	//networks map[string]     `yaml:"networks,omitempty"`
	//volumes map[string]                  `yaml:"volumes,omitempty"`
}

func (dcy *dockerComposeConfig) Parse(data []byte) error {
	return yaml.Unmarshal(data, dcy)
}

func getNetwork() (string, error) {
	// create/get newtwork
	// TODO: move this into it's own file
	ctx := context.Background()
	cli, err := client.NewEnvClient()
	if err != nil {
		return "", err
	}
	networkName := "melinetworkname"
	var typeNetworkCreate = types.NetworkCreate{
		CheckDuplicate: true,
		Driver:         "bridge",
		EnableIPv6:     false,
		IPAM:           &network.IPAM{Driver: "default"},
		Internal:       false,
		Attachable:     true,
	}
	networkCreateResponse, err := cli.NetworkCreate(
		ctx,
		networkName,
		typeNetworkCreate)
	if err != nil {
		return "", err
	}
	return networkCreateResponse.ID, nil

}

func main() {
	data, err := ioutil.ReadFile("docker-compose.yml")
	if err != nil {
		log.Fatal(err)
	}
	networkID, err := getNetwork()
	if err != nil {
		log.Fatal(err)
	}

	var dockerCyaml dockerComposeConfig
	if err := dockerCyaml.Parse(data); err != nil {
		log.Fatal(err)
	}

	for _, v := range dockerCyaml.Services {
		fmt.Println()
		fmt.Println("image & networkID: ", v.Image, networkID)
		fmt.Println()
		pullImage(v.Image, networkID)
	}

}

func pullImage(imagename, networkID string) {
	ctx := context.Background()
	cli, err := client.NewEnvClient()
	if err != nil {
		panic(err)
	}

	imagePullResp, err := cli.ImagePull(
		ctx,
		imagename,
		types.ImagePullOptions{})
	if err != nil {
		panic(err)
	}
	defer imagePullResp.Close()
	_, err = io.Copy(os.Stdout, imagePullResp)
	if err != nil {
		log.Println(err)
	}

	containerCreateResp, err := cli.ContainerCreate(
		ctx,
		&container.Config{Image: imagename},
		&container.HostConfig{PublishAllPorts: true},
		nil,
		"")
	if err != nil {
		log.Fatal(err)
	}

	//
	//Connect container to network
	err = cli.NetworkConnect(
		ctx,
		networkID,
		containerCreateResp.ID,
		&network.EndpointSettings{})
	if err != nil {
		log.Fatal(err)
	}
	//

	err = cli.ContainerStart(
		ctx,
		containerCreateResp.ID,
		types.ContainerStartOptions{})
	if err != nil {
		panic(err)
	}

	containerLogResp, err := cli.ContainerLogs(
		ctx,
		containerCreateResp.ID,
		types.ContainerLogsOptions{
			ShowStdout: true,
			ShowStderr: true,
			Timestamps: true})
	if err != nil {
		panic(err)
	}
	defer containerLogResp.Close()

	_, err = io.Copy(os.Stdout, containerLogResp)
	if err != nil {
		log.Println(err)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"os/exec"
	"sync"
	"text/template"
	"time"

	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/client"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/proxy/config"

	"github.com/golang/glog"
	flag "github.com/spf13/pflag"
)

var (
	clientConfig = &client.Config{}
	templatePath = flag.String("template_path", "/etc/k8s-haproxy/haproxy.cfg.gotemplate", "location of the haproxy template")
)

func init() {
	client.BindClientConfigFlags(flag.CommandLine, clientConfig)
	flag.Set("logtostderr", "true")
}

func main() {
	flag.Parse()

	kubeClient, err := client.New(clientConfig)
	if err != nil {
		glog.Fatalf("Invalid API configuration: %v", err)
	}

	ManageConfig(*templatePath)
	glog.Info("managing config")

	ManageHaproxy()
	glog.Info("managing haproxy")

	ManageWatch(kubeClient)
	glog.Info("managing watch")

	select {}
}

//haproxy stuff
var (
	endpointsUpdater *endpointUpdateHandler
	serviceUpdater   = &serviceUpdateHandler{}

	t    *template.Template
	lock sync.Mutex

	endpoints = []api.Endpoints{}
	services  = []api.Service{}
)

const ConfigPath = "/etc/haproxy/haproxy.cfg"

func ManageHaproxy() {
	cmd := exec.Command("haproxy", "-f", ConfigPath, "-p", "/var/run/haproxy.pid")
	err := cmd.Run()
	if err != nil {
		if o, err := cmd.CombinedOutput(); err != nil {
			glog.Error(string(o))
		}
		glog.Errorf("haproxy process died, : %v", err)
	}
}

func ManageConfig(templatePath string) {
	var err error
	t, err = template.ParseFiles(templatePath)
	if err != nil {
		glog.Fatalf("error parsing template: %v", err)
	}
}

type endpointUpdateHandler struct{}

func (e *endpointUpdateHandler) OnUpdate(newEndpoints []api.Endpoints) {
	lock.Lock()
	endpoints = newEndpoints
	lock.Unlock()
	Commit()
}

type serviceUpdateHandler struct{}

func (e *serviceUpdateHandler) OnUpdate(newServices []api.Service) {
	lock.Lock()
	services = newServices
	lock.Unlock()
	err := Commit()
	if err != nil {
		glog.Errorf("error commiting haproxy config: %v", err)
	}
}

func Commit() error {
	lock.Lock()
	defer lock.Unlock()
	f, err := os.Create(ConfigPath)
	if err != nil {
		return err
	}
	states := Convert(endpoints, services)
	err = validate(states)
	if err != nil {
		return err
	}
	err = t.Execute(f, states)
	if err != nil {
		return err
	}
	cmd := exec.Command("/reload-haproxy.sh", ConfigPath)
	b, err := cmd.CombinedOutput()
	if err != nil {
		return fmt.Errorf("error reloading haproxy: %v: %v", err, string(b))
	}
	glog.Info("updated haproxy")
	return nil
}

type ServiceState struct {
	Service   api.Service
	Endpoints api.Endpoints
}

func validate(s map[string]ServiceState) error {
	return nil
}

func Convert(es []api.Endpoints, ss []api.Service) map[string]ServiceState {
	sm := make(map[string]api.Service)
	em := make(map[string]api.Endpoints)
	for _, s := range ss {
		sm[s.Name] = s
	}
	for _, e := range es {
		em[e.Name] = e
	}
	states := make(map[string]ServiceState)
	for k, s := range sm {
		if e, found := em[k]; found {
			states[k] = ServiceState{s, e}
			continue
		}
		glog.Infof("endpoint not found for service: %+v", s)
		// what should we do here?
	}
	return states
}

//watch stuff
var (
	serviceConfig   = config.NewServiceConfig()
	endpointsConfig = config.NewEndpointsConfig()
	sourceAPI       *config.SourceAPI
)

func ManageWatch(c *client.Client) {

	serviceConfig.RegisterHandler(serviceUpdater)
	endpointsConfig.RegisterHandler(endpointsUpdater)

	sourceAPI = config.NewSourceAPI(
		c.Services(api.NamespaceAll),
		c.Endpoints(api.NamespaceAll),
		30*time.Second,
		serviceConfig.Channel("api"),
		endpointsConfig.Channel("api"),
	)
}
<commit_msg>added logging statement to be consistent<commit_after>package main

import (
	"fmt"
	"os"
	"os/exec"
	"sync"
	"text/template"
	"time"

	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/client"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/proxy/config"

	"github.com/golang/glog"
	flag "github.com/spf13/pflag"
)

var (
	clientConfig = &client.Config{}
	templatePath = flag.String("template_path", "/etc/k8s-haproxy/haproxy.cfg.gotemplate", "location of the haproxy template")
)

func init() {
	client.BindClientConfigFlags(flag.CommandLine, clientConfig)
	flag.Set("logtostderr", "true")
}

func main() {
	flag.Parse()

	kubeClient, err := client.New(clientConfig)
	if err != nil {
		glog.Fatalf("Invalid API configuration: %v", err)
	}

	ManageConfig(*templatePath)
	glog.Info("managing config")

	ManageHaproxy()
	glog.Info("managing haproxy")

	ManageWatch(kubeClient)
	glog.Info("managing watch")

	select {}
}

//haproxy stuff
var (
	endpointsUpdater *endpointUpdateHandler
	serviceUpdater   = &serviceUpdateHandler{}

	t    *template.Template
	lock sync.Mutex

	endpoints = []api.Endpoints{}
	services  = []api.Service{}
)

const ConfigPath = "/etc/haproxy/haproxy.cfg"

func ManageHaproxy() {
	cmd := exec.Command("haproxy", "-f", ConfigPath, "-p", "/var/run/haproxy.pid")
	err := cmd.Run()
	if err != nil {
		if o, err := cmd.CombinedOutput(); err != nil {
			glog.Error(string(o))
		}
		glog.Errorf("haproxy process died, : %v", err)
	}
}

func ManageConfig(templatePath string) {
	var err error
	t, err = template.ParseFiles(templatePath)
	if err != nil {
		glog.Fatalf("error parsing template: %v", err)
	}
}

type endpointUpdateHandler struct{}

func (e *endpointUpdateHandler) OnUpdate(newEndpoints []api.Endpoints) {
	lock.Lock()
	endpoints = newEndpoints
	lock.Unlock()
	err := Commit()
	if err != nil {
		glog.Errorf("error commiting haproxy config: %v", err)
	}
}

type serviceUpdateHandler struct{}

func (e *serviceUpdateHandler) OnUpdate(newServices []api.Service) {
	lock.Lock()
	services = newServices
	lock.Unlock()
	err := Commit()
	if err != nil {
		glog.Errorf("error commiting haproxy config: %v", err)
	}
}

func Commit() error {
	lock.Lock()
	defer lock.Unlock()
	f, err := os.Create(ConfigPath)
	if err != nil {
		return err
	}
	states := Convert(endpoints, services)
	err = validate(states)
	if err != nil {
		return err
	}
	err = t.Execute(f, states)
	if err != nil {
		return err
	}
	cmd := exec.Command("/reload-haproxy.sh", ConfigPath)
	b, err := cmd.CombinedOutput()
	if err != nil {
		return fmt.Errorf("error reloading haproxy: %v: %v", err, string(b))
	}
	glog.Info("updated haproxy")
	return nil
}

type ServiceState struct {
	Service   api.Service
	Endpoints api.Endpoints
}

func validate(s map[string]ServiceState) error {
	return nil
}

func Convert(es []api.Endpoints, ss []api.Service) map[string]ServiceState {
	sm := make(map[string]api.Service)
	em := make(map[string]api.Endpoints)
	for _, s := range ss {
		sm[s.Name] = s
	}
	for _, e := range es {
		em[e.Name] = e
	}
	states := make(map[string]ServiceState)
	for k, s := range sm {
		if e, found := em[k]; found {
			states[k] = ServiceState{s, e}
			continue
		}
		glog.Infof("endpoint not found for service: %+v", s)
		// what should we do here?
	}
	return states
}

//watch stuff
var (
	serviceConfig   = config.NewServiceConfig()
	endpointsConfig = config.NewEndpointsConfig()
	sourceAPI       *config.SourceAPI
)

func ManageWatch(c *client.Client) {

	serviceConfig.RegisterHandler(serviceUpdater)
	endpointsConfig.RegisterHandler(endpointsUpdater)

	sourceAPI = config.NewSourceAPI(
		c.Services(api.NamespaceAll),
		c.Endpoints(api.NamespaceAll),
		30*time.Second,
		serviceConfig.Channel("api"),
		endpointsConfig.Channel("api"),
	)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"net/http"
	"os"
	"os/signal"

	"github.com/douglasmakey/ursho/config"
	"github.com/douglasmakey/ursho/handlers"
	"github.com/douglasmakey/ursho/storages"
)

func main() {
	// Set use storage, select [Postgres, Filesystem, Redis ...]
	storage := &storages.Postgres{}

	// Read config
	config, err := config.ReadConfig()
	if err != nil {
		log.Fatal(err)
	}
	// Init storage
	if err = storage.Init(config); err != nil {
		log.Fatal(err)
	}

	// Defers
	defer storage.Close()

	// Handlers
	http.Handle("/encode/", handlers.EncodeHandler(storage))
	http.Handle("/", handlers.RedirectHandler(storage))
	http.Handle("/info/", handlers.DecodeHandler(storage))

	// Graceful shutdown
	sigquit := make(chan os.Signal, 1)
	signal.Notify(sigquit, os.Interrupt, os.Kill)

	// Wait signal
	close := make(chan bool, 1)

	// Create a server
	server := &http.Server{Addr: fmt.Sprintf("%s:%s", config.Server.Host, config.Server.Port)}

	// Start server
	go func() {
		log.Printf("Starting HTTP Server. Listening at %q", server.Addr)
		if err := server.ListenAndServe(); err != nil {
			if err := server.ListenAndServe(); err != nil {
				if err != http.ErrServerClosed {
					log.Println(err.Error())
				} else {
					log.Println("Server closed!")
				}
				close <- true
			}
		}

	}()

	// Check for a closing signal
	go func() {
		sig := <-sigquit
		log.Printf("caught sig: %+v", sig)
		log.Printf("Gracefully shutting down server...")

		if err := server.Shutdown(nil); err != nil {
			log.Println("Unable to shut down server: " + err.Error())
			close <- true
		} else {
			close <- true
			log.Println("Server stopped")
		}
	}()

	<-close
}
<commit_msg>Fix typo<commit_after>package main

import (
	"fmt"
	"log"
	"net/http"
	"os"
	"os/signal"

	"github.com/douglasmakey/ursho/config"
	"github.com/douglasmakey/ursho/handlers"
	"github.com/douglasmakey/ursho/storages"
)

func main() {
	// Set use storage, select [Postgres, Filesystem, Redis ...]
	storage := &storages.Postgres{}

	// Read config
	config, err := config.ReadConfig()
	if err != nil {
		log.Fatal(err)
	}
	// Init storage
	if err = storage.Init(config); err != nil {
		log.Fatal(err)
	}

	// Defers
	defer storage.Close()

	// Handlers
	http.Handle("/encode/", handlers.EncodeHandler(storage))
	http.Handle("/", handlers.RedirectHandler(storage))
	http.Handle("/info/", handlers.DecodeHandler(storage))

	// Graceful shutdown
	sigquit := make(chan os.Signal, 1)
	signal.Notify(sigquit, os.Interrupt, os.Kill)

	// Wait signal
	close := make(chan bool, 1)

	// Create a server
	server := &http.Server{Addr: fmt.Sprintf("%s:%s", config.Server.Host, config.Server.Port)}

	// Start server
	go func() {
		log.Printf("Starting HTTP Server. Listening at %q", server.Addr)
		if err := server.ListenAndServe(); err != nil {
			if err := server.ListenAndServe(); err != nil {
				if err != http.ErrServerClosed {
					log.Println(err.Error())
				} else {
					log.Println("Server closed!")
				}
				close <- true
			}
		}

	}()

	// Check for a closing signal
	go func() {
		sig := <-sigquit
		log.Printf("caught sig: %+v", sig)
		log.Printf("Gracefully shutting down server...")

		if err := server.Shutdown(nil); err != nil {
			log.Println("Unable to shut down server: " + err.Error())
			close <- true
		} else {
			log.Println("Server stopped")
			close <- true
		}
	}()

	<-close
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"github.com/siddontang/redis-failover/failover"
	"os"
	"os/signal"
	"strings"
	"syscall"
)

var configFile = flag.String("config", "", "failover config file")
var addr = flag.String("addr", ":11000", "failover http listen addr")
var checkInterval = flag.Int("check_interval", 1000, "check master alive every n millisecond")
var maxDownTime = flag.Int("max_down_time", 3, "max down time for a master, after that, we will do failover")

var masters = flag.String("masters", "", "redis master need to be monitored, seperated by comma")
var mastersState = flag.String("masters_state", "", "new or existing for raft, if new, we will depracted old saved masters")

var broker = flag.String("broker", "", "broker for cluster, now is raft or zk")

var raftDataDir = flag.String("raft_data_dir", "./var/store", "raft data store path")
var raftLogDir = flag.String("raft_log_dir", "./var/log", "raft log store path")
var raftAddr = flag.String("raft_addr", "", "raft listen addr, if empty, we will disable raft")
var raftCluster = flag.String("raft_cluster", "", "raft cluster,vseperated by comma")
var raftClusterState = flag.String("raft_cluster_state", "", "new or existing, if new, we will deprecate old saved cluster and use new")

var zkAddr = flag.String("zk_addr", "", "zookeeper address, seperated by comma")
var zkPath = flag.String("zk_path", "/zk/redis/failover", "base directory in zk, prefix must be /zk")

func main() {
	flag.Parse()

	var c *failover.Config
	var err error
	if len(*configFile) > 0 {
		c, err = failover.NewConfigWithFile(*configFile)
		if err != nil {
			fmt.Printf("load failover config %s err %v", *configFile, err)
			return
		}
	} else {
		fmt.Printf("no config file, use default config")
		c = new(failover.Config)
		c.Addr = ":11000"
	}

	if *checkInterval > 0 {
		c.CheckInterval = *checkInterval
	}

	if *maxDownTime > 0 {
		c.MaxDownTime = *maxDownTime
	}

	if len(*raftAddr) > 0 {
		c.Raft.Addr = *raftAddr
	}

	if len(*raftDataDir) > 0 {
		c.Raft.DataDir = *raftDataDir
	}

	if len(*raftLogDir) > 0 {
		c.Raft.LogDir = *raftLogDir
	}

	seps := strings.Split(*raftCluster, ",")
	if len(seps) > 0 && len(seps[0]) > 0 {
		c.Raft.Cluster = seps
	}

	if len(*raftClusterState) > 0 {
		c.Raft.ClusterState = *raftClusterState
	}

	seps = strings.Split(*zkAddr, ",")
	if len(seps) > 0 && len(seps[0]) > 0 {
		c.Zk.Addr = seps
	}

	if len(*zkPath) > 0 {
		c.Zk.BaseDir = *zkPath
	}

	if len(*broker) > 0 {
		c.Broker = *broker
	} else if len(c.Raft.Addr) > 0 {
		// use raft
		c.Broker = "raft"
	} else if len(c.Zk.Addr) > 0 {
		c.Broker = "zk"
	}

	seps = strings.Split(*masters, ",")
	if len(seps) > 0 && len(seps[0]) > 0 {
		c.Masters = seps
	}

	if len(*mastersState) > 0 {
		c.MastersState = *mastersState
	}

	app, err := failover.NewApp(c)
	if err != nil {
		fmt.Printf("new failover app error %v", err)
		return
	}

	sc := make(chan os.Signal, 1)
	signal.Notify(sc,
		os.Kill,
		os.Interrupt,
		syscall.SIGHUP,
		syscall.SIGINT,
		syscall.SIGTERM,
		syscall.SIGQUIT)

	go func() {
		<-sc
		app.Close()
	}()

	app.Run()
}
<commit_msg>fix flag overwrite config file<commit_after>package main

import (
	"flag"
	"fmt"
	"github.com/siddontang/redis-failover/failover"
	"os"
	"os/signal"
	"strings"
	"syscall"
)

var configFile = flag.String("config", "", "failover config file")
var addr = flag.String("addr", "", "failover http listen addr")
var checkInterval = flag.Int("check_interval", 0, "check master alive every n millisecond")
var maxDownTime = flag.Int("max_down_time", 0, "max down time for a master, after that, we will do failover")

var masters = flag.String("masters", "", "redis master need to be monitored, seperated by comma")
var mastersState = flag.String("masters_state", "", "new or existing for raft, if new, we will depracted old saved masters")

var broker = flag.String("broker", "", "broker for cluster, now is raft or zk")

var raftDataDir = flag.String("raft_data_dir", "", "raft data store path")
var raftLogDir = flag.String("raft_log_dir", "", "raft log store path")
var raftAddr = flag.String("raft_addr", "", "raft listen addr, if empty, we will disable raft")
var raftCluster = flag.String("raft_cluster", "", "raft cluster,vseperated by comma")
var raftClusterState = flag.String("raft_cluster_state", "", "new or existing, if new, we will deprecate old saved cluster and use new")

var zkAddr = flag.String("zk_addr", "", "zookeeper address, seperated by comma")
var zkPath = flag.String("zk_path", "", "base directory in zk, prefix must be /zk")

func main() {
	flag.Parse()

	var c *failover.Config
	var err error
	if len(*configFile) > 0 {
		c, err = failover.NewConfigWithFile(*configFile)
		if err != nil {
			fmt.Printf("load failover config %s err %v", *configFile, err)
			return
		}
	} else {
		fmt.Printf("no config file, use default config")
		c = new(failover.Config)
		c.Addr = ":11000"
		c.CheckInterval = 1000
		c.MaxDownTime = 3
	}

	if len(*addr) > 0 {
		c.Addr = *addr
	}

	if *checkInterval > 0 {
		c.CheckInterval = *checkInterval
	}

	if *maxDownTime > 0 {
		c.MaxDownTime = *maxDownTime
	}

	if len(*raftAddr) > 0 {
		c.Raft.Addr = *raftAddr
	}

	if len(*raftDataDir) > 0 {
		c.Raft.DataDir = *raftDataDir
	}

	if len(*raftLogDir) > 0 {
		c.Raft.LogDir = *raftLogDir
	}

	seps := strings.Split(*raftCluster, ",")
	if len(seps) > 0 && len(seps[0]) > 0 {
		c.Raft.Cluster = seps
	}

	if len(*raftClusterState) > 0 {
		c.Raft.ClusterState = *raftClusterState
	}

	seps = strings.Split(*zkAddr, ",")
	if len(seps) > 0 && len(seps[0]) > 0 {
		c.Zk.Addr = seps
	}

	if len(*zkPath) > 0 {
		c.Zk.BaseDir = *zkPath
	}

	if len(*broker) > 0 {
		c.Broker = *broker
	}

	seps = strings.Split(*masters, ",")
	if len(seps) > 0 && len(seps[0]) > 0 {
		c.Masters = seps
	}

	if len(*mastersState) > 0 {
		c.MastersState = *mastersState
	}

	app, err := failover.NewApp(c)
	if err != nil {
		fmt.Printf("new failover app error %v", err)
		return
	}

	sc := make(chan os.Signal, 1)
	signal.Notify(sc,
		os.Kill,
		os.Interrupt,
		syscall.SIGHUP,
		syscall.SIGINT,
		syscall.SIGTERM,
		syscall.SIGQUIT)

	go func() {
		<-sc
		app.Close()
	}()

	app.Run()
}
<|endoftext|>


<commit_before>// Copyright (c) 2013, Ben Morgan. All rights reserved.
// Use of this source code is governed by an MIT license
// that can be found in the LICENSE file.

// Fairsplit is a program to split expenses among a group of people.
package main

import (
	"bufio"
	"fmt"
	"io"
	"os"
	"strconv"
	"strings"
)

func main() {
	fmt.Println("Fairsplit – no one's cheating no one here.\n")
	g := buildGraphInteractively()
	simplifyGraph(g)
	printTransactions(g)
}

type graph map[string]map[string]float64

// buildGraphInteractively prints an introduction message and then proceeds to
// take input from the standard input until EOF is encountered (which can be
// inserted with Ctrl-D or Ctrl-Z, depending on OS.)
//
// If a line is erroneous, an error message should be printed, but processing
// should continue. Input from the user should be indicated by a greater than
// mark >. For example, the input might look like this:
//
//	Please enter your finance graph here.
//
//		<Person> <Sum> <Person> [Person...]
//
//	Lines have the above format, where those left of the sum
//	spend money on behalf of those to the right of the sum.
//
//	> Ben 45.67 Reed Tami
//	> Reed 60.75 Ben Tami
//	> Tami 33.20 Ben Reed
//	> Ben 20.99 Tami
//  > quit
//	I do not understand 'quit'; ignoring.
//	> CTRL-D
//
// And then it should return the map[string]map[string]float64 that it created
// in this process. It is not the job of buildGraphInteractively however, to
// simplify the "graph". Let that be anothers job.
func buildGraphInteractively() graph {
	fmt.Println(`Please enter your finance graph here.

	<Person> <Sum> <Person> [Person...]

Lines have the above format, where those left of the sum
spend money on behalf of those to the right of the sum.
`)

	g := make(graph)

	// Implementation tip: use fmt.*Scan* functions.
	r := bufio.NewReader(os.Stdin)
	for {
		// Get a line from the standard input
		fmt.Print("> ")
		line, err := r.ReadString('\n')
		if err != nil && err != io.EOF {
			fmt.Printf("error: %v\n", err)
			break
		}

		// Check that we have enough information
		fields := strings.Fields(line)
		if len(fields) < 3 {
			if len(fields) > 0 {
				fmt.Println("error: invalid format, ignoring line")
			} else if err == io.EOF {
				break
			}
			continue
		}

		// Assign the information to the correct entities
		subj := fields[0]
		obj := fields[2:]
		amount, err := strconv.ParseFloat(fields[1], 64)
		if err != nil {
			fmt.Printf("error: cannot read number %q, ignoring line\n", fields[1])
			continue
		}
		amount /= float64(len(obj) + 1)

		// Put the information in the graph
		for _, v := range obj {
			if g[v] == nil {
				g[v] = make(map[string]float64)
			}
			g[v][subj] += amount
		}
	}

	fmt.Println()
	return g
}

// simplifyGraph simplifies the "graph" g so that the number of transactions
// that need to take place is minimized.
//
// For example, if Ben owes Reed 12.00€, and Reed owes Ben 5.00€, then there
// should only be one transaction, namely that of Ben paying Reed 7.00€.
// Further simplifications should also be made, if possible; for example,
// if Ben owes Reed 5.00€, and Sonia owes Ben 5.00, then Sonia should pay
// Reed, because there is one less edge in the graph then.
//
// Edges in the graph that have a weight of 0.0 should be deleted from the
// map, so that when we iterate over the map, we have the minimal number
// of transactions right there.
func simplifyGraph(g graph) {
	// Make sure there is only ever one edge between two nodes
	for subj, others := range g {
		for recv, sum := range others {
			if sum < g[recv][subj] {
				g[recv][subj] -= sum
				delete(g[subj], recv)
			}
		}
	}

	// TODO: Try to remove edges if we can reroute the flow
}

// printTransactions iterates through the simplified graph and prints all
// the edges contained in it as transactions from one person to another.
func printTransactions(g graph) {
	fmt.Println("\nOUTSTANDING TRANSACTIONS:")
	for subj, others := range g {
		if len(others) == 0 {
			continue
		}
		fmt.Printf("%v must pay:\n", subj)
		for recv, sum := range others {
			fmt.Printf("    %.2f to %v\n", sum, recv)
		}
		fmt.Println()
	}
}
<commit_msg>Refactoring -> efficient minimization of transactions<commit_after>// Copyright (c) 2013, Ben Morgan. All rights reserved.
// Use of this source code is governed by an MIT license
// that can be found in the LICENSE file.

// Fairsplit is a program to split expenses among a group of people.
package main

import (
	"bufio"
	"fmt"
	"io"
	"os"
	"strconv"
	"strings"
)

func main() {
	fmt.Println("Fairsplit – no one's cheating no one here.\n")
	g := buildGraphInteractively()
	printTransactions(g)
}

type graph map[string]map[string]float64

// buildGraphInteractively prints an introduction message and then proceeds to
// take input from the standard input until EOF is encountered (which can be
// inserted with Ctrl-D or Ctrl-Z, depending on OS.)
//
// If a line is erroneous, an error message should be printed, but processing
// should continue. Input from the user should be indicated by a greater than
// mark >. For example, the input might look like this:
//
//	Please enter your finance graph here.
//
//		<Person> <Sum> <Person> [Person...]
//
//	Lines have the above format, where those left of the sum
//	spend money on behalf of those to the right of the sum.
//
//	> Ben 45.67 Reed Sonia
//	> Reed 60.75 Ben Sonia
//	> Sonia 33.20 Ben Reed
//	> Ben 20.99 Sonia
//	> CTRL-D
//
func buildGraphInteractively() graph {
	fmt.Println(`Please enter your finance graph here.

	<Person> <Sum> <Person> [Person...]

Lines have the above format, where those left of the sum
spend money on behalf of those to the right of the sum.
`)

	amends := make(map[string]float64)

	// Implementation tip: use fmt.*Scan* functions.
	r := bufio.NewReader(os.Stdin)
	for {
		// Get a line from the standard input
		fmt.Print("> ")
		line, err := r.ReadString('\n')
		if err != nil && err != io.EOF {
			fmt.Printf("error: %v\n", err)
			break
		}

		// Check that we have enough information
		fields := strings.Fields(line)
		if len(fields) < 3 {
			if len(fields) > 0 {
				fmt.Println("error: invalid format, ignoring line")
			} else if err == io.EOF {
				break
			}
			continue
		}

		// Assign the information to the correct entities
		subj := fields[0]
		others := fields[2:]
		amount, err := strconv.ParseFloat(fields[1], 64)
		if err != nil {
			fmt.Printf("error: cannot read number %q, ignoring line\n", fields[1])
			continue
		}

		// Put the information in the graph
		part := amount / float64(len(others)+1)
		amends[subj] -= amount - part
		for _, o := range others {
			amends[o] += part
		}
	}
	fmt.Println()

	// Convert the amends data-structure to a graph
	type obj struct {
		k string  // key = name
		v float64 // value = sum
	}
	gets := make([]obj, 0, len(amends))
	puts := make([]obj, 0, len(amends))
	for k, v := range amends {
		if v < 0 {
			gets = append(gets, obj{k, -v})
		} else if v > 0 {
			puts = append(puts, obj{k, v})
		}
	}

	g := make(graph)
	for i, j := 0, 0; i < len(puts); i++ {
		put := &puts[i]
		for put.v >= 0.01 {
			if g[put.k] == nil {
				g[put.k] = make(map[string]float64)
			}

			get := &gets[j]
			if get.v > put.v {
				g[put.k][get.k] = put.v
				get.v -= put.v
				put.v = 0
			} else if get.v <= put.v {
				g[put.k][get.k] = get.v
				put.v -= get.v
				get.v = 0
				j++
			}
		}
	}

	return g
}

// printTransactions iterates through the simplified graph and prints all
// the edges contained in it as transactions from one person to another.
func printTransactions(g graph) {
	fmt.Println("\nOUTSTANDING TRANSACTIONS:")

	for subj, others := range g {
		if len(others) == 0 {
			continue
		}
		fmt.Printf("%v must pay:\n", subj)
		for recv, sum := range others {
			fmt.Printf("    %.2f to %v\n", sum, recv)
		}
		fmt.Println()
	}
}
<|endoftext|>


<commit_before>package main

import (
	"os"

	log "github.com/sirupsen/logrus"
	"gopkg.in/urfave/cli.v2"
)

func init() {
	log.SetOutput(os.Stdout)
	//log.SetLevel(log.DebugLevel)
}

func main() {

	app := &cli.App{
		EnableShellCompletion: true,
		Name:  "go-dnsimple",
		Usage: "manipulate DNSimple domains!",
		Flags: []cli.Flag{
			&cli.BoolFlag{
				Name:    "debug",
				Aliases: []string{"D"},
				Usage:   "Enable debug output",
			},
			&cli.BoolFlag{
				Name:    "jsonOutput",
				Aliases: []string{"J"},
				Usage:   "Enable json output",
			},
			&cli.BoolFlag{
				Name:    "dynamicIP",
				Aliases: []string{"d"},
				Usage:   "Lookup current IP dynamically",
			},
			&cli.StringFlag{
				Name:    "token",
				Aliases: []string{"t"},
				Usage:   "DNSimple API OAuth Token",
				EnvVars: []string{"DNSIMPLE_AUTH_TOKEN"},
			},
			&cli.BoolFlag{
				Name:        "ipv4",
				Aliases:     []string{"4"},
				Usage:       "Dynamic lookup with IP V4 address",
				Value:       true,
				DefaultText: "true",
			},
			&cli.BoolFlag{
				Name:        "ipv6",
				Aliases:     []string{"6"},
				Usage:       "Dynamic lookup with IP V6 address",
				Value:       false,
				DefaultText: "false",
			},
		},
		Before: func(c *cli.Context) error {
			if c.Bool("debug") {
				log.SetLevel(log.DebugLevel)
			}

			if c.Bool("jsonOutput") {
				log.SetFormatter(&log.JSONFormatter{})
			}
			return nil
		},
		Commands: []*cli.Command{
			{
				Name:  "record",
				Usage: "domain record operations",
				Flags: []cli.Flag{
					&cli.StringFlag{
						Name:    "domain",
						Aliases: []string{"d"},
						Usage:   "domain name to operate on",
					},
					&cli.BoolFlag{
						Name:    "dynamicLookup",
						Usage:   "lookup current Internet IP dynamically",
						Aliases: []string{"dynamic", "dyn"},
					},
				},
				Subcommands: []*cli.Command{
					{
						Name:    "list",
						Aliases: []string{"ls"},
					},
					{
						Name:    "create",
						Aliases: []string{"add", "cr"},
						Usage:   "create a new record",
						Flags: []cli.Flag{
							&cli.StringFlag{
								Name:        "type",
								Aliases:     []string{"t"},
								DefaultText: "A",
								Usage:       "Record type to create: A, AAAA",
							},
							&cli.StringFlag{
								Name:  "data",
								Usage: "Record data",
							},
						},
						Action: func(context *cli.Context) error {
							dns, err := NewDNSimpleAPI(context.String("token"))
							if err != nil {
								log.WithError(err).Fatal("Cannot communicate with DNSimple API")
								return cli.Exit("Cannot communicate with DNSimple API", 1)
							}

							domainName := context.String("domain")
							recordName := context.Args().First()
							recordType := context.String("type")
							recordData := context.String("data")

							crudOp, err := NewOperationData(dns, recordData, recordName, domainName, recordType, false)
							if err != nil {
								log.WithError(err).Fatal("Could not create command operation data.")
								return cli.Exit("Cannot create record", 1)
							}

							err = crudOp.Create()
							if err != nil {
								log.WithError(err).Fatal("Could not create record")
								return cli.Exit("Cannot create record", 1)
							}
							return nil
						},
					},
					{
						Name:        "delete",
						Aliases:     []string{"del"},
						Description: "delete record data",
						Flags: []cli.Flag{
							&cli.StringFlag{
								Name:        "type",
								Aliases:     []string{"t"},
								DefaultText: "A",
								Usage:       "Record type to delete: A, AAAA",
							},
							&cli.StringFlag{
								Name:  "data",
								Usage: "Record data",
							},
						},
						Action: func(context *cli.Context) error {
							dns, err := NewDNSimpleAPI(context.String("token"))
							if err != nil {
								return cli.Exit("Cannot communicate with DNSimple API", 1)
							}

							domainName := context.String("domain")
							recordName := context.Args().First()
							recordType := context.String("type")
							recordData := context.String("data")

							crudOp, err := NewOperationData(dns, recordData, recordName, domainName, recordType, false)
							if err != nil {
								log.WithError(err).Fatal("Could not create command operation data.")
								return cli.Exit("Cannot delete record", 1)
							}

							err = crudOp.Delete()
							if err != nil {
								return cli.Exit("Cannot delete record", 1)
							}
							return nil
						},
					},
					{
						Name:        "upsert",
						Aliases:     []string{"up", "update"},
						Description: "upsert record data",
						Flags: []cli.Flag{
							&cli.StringFlag{
								Name:        "type",
								Aliases:     []string{"t"},
								DefaultText: "A",
								Usage:       "Record type to upsert: A, AAAA",
							},
							&cli.StringFlag{
								Name:  "data",
								Usage: "Record data",
							},
						},
						Action: func(context *cli.Context) error {
							dns, err := NewDNSimpleAPI(context.String("token"))
							if err != nil {
								return cli.Exit("Cannot communicate with DNSimple API", 1)
							}

							domainName := context.String("domain")
							recordName := context.Args().First()
							recordType := context.String("type")
							recordData := context.String("data")

							crudOp, err := NewOperationData(dns, recordData, recordName, domainName, recordType, false)
							if err != nil {
								log.WithError(err).Fatal("Could not create command operation data.")
								return cli.Exit("Cannot upsert record", 1)
							}

							err = crudOp.Upsert()
							if err != nil {
								return cli.Exit("Cannot upsert record", 1)
							}
							return nil
						},
					},
				},
			},
		},
	}

	app.Run(os.Args)
}
<commit_msg>support ipv6 properly<commit_after>package main

import (
	"os"

	log "github.com/sirupsen/logrus"
	"gopkg.in/urfave/cli.v2"
)

func init() {
	log.SetOutput(os.Stdout)
	//log.SetLevel(log.DebugLevel)
}

func main() {

	app := &cli.App{
		EnableShellCompletion: true,
		Name:  "go-dnsimple",
		Usage: "manipulate DNSimple domains!",
		Flags: []cli.Flag{
			&cli.BoolFlag{
				Name:    "debug",
				Aliases: []string{"D"},
				Usage:   "Enable debug output",
			},
			&cli.BoolFlag{
				Name:    "jsonOutput",
				Aliases: []string{"J"},
				Usage:   "Enable json output",
			},
			&cli.BoolFlag{
				Name:    "dynamicIP",
				Aliases: []string{"d"},
				Usage:   "Lookup current IP dynamically",
			},
			&cli.StringFlag{
				Name:    "token",
				Aliases: []string{"t"},
				Usage:   "DNSimple API OAuth Token",
				EnvVars: []string{"DNSIMPLE_AUTH_TOKEN"},
			},
			&cli.BoolFlag{
				Name:        "ipv4",
				Aliases:     []string{"4"},
				Usage:       "Dynamic lookup with IP V4 address",
				Value:       true,
				DefaultText: "true",
			},
			&cli.BoolFlag{
				Name:        "ipv6",
				Aliases:     []string{"6"},
				Usage:       "Dynamic lookup with IP V6 address",
				Value:       false,
				DefaultText: "false",
			},
		},
		Before: func(c *cli.Context) error {
			if c.Bool("debug") {
				log.SetLevel(log.DebugLevel)
			}

			if c.Bool("jsonOutput") {
				log.SetFormatter(&log.JSONFormatter{})
			}
			return nil
		},
		Commands: []*cli.Command{
			{
				Name:  "record",
				Usage: "domain record operations",
				Flags: []cli.Flag{
					&cli.StringFlag{
						Name:    "domain",
						Aliases: []string{"d"},
						Usage:   "domain name to operate on",
					},
					&cli.BoolFlag{
						Name:    "dynamicLookup",
						Usage:   "lookup current Internet IP dynamically",
						Aliases: []string{"dynamic", "dyn"},
					},
				},
				Subcommands: []*cli.Command{
					{
						Name:    "list",
						Aliases: []string{"ls"},
					},
					{
						Name:    "create",
						Aliases: []string{"add", "cr"},
						Usage:   "create a new record",
						Flags: []cli.Flag{
							&cli.StringFlag{
								Name:        "type",
								Aliases:     []string{"t"},
								DefaultText: "A",
								Usage:       "Record type to create: A, AAAA",
							},
							&cli.StringFlag{
								Name:  "data",
								Usage: "Record data",
							},
						},
						Action: func(context *cli.Context) error {
							dns, err := NewDNSimpleAPI(context.String("token"))
							if err != nil {
								log.WithError(err).Fatal("Cannot communicate with DNSimple API")
								return cli.Exit("Cannot communicate with DNSimple API", 1)
							}

							domainName := context.String("domain")
							recordName := context.Args().First()
							recordType := context.String("type")
							recordData := context.String("data")

							crudOp, err := NewOperationData(dns, recordData, recordName, domainName, recordType, context.Bool("ipv6"))
							if err != nil {
								log.WithError(err).Fatal("Could not create command operation data.")
								return cli.Exit("Cannot create record", 1)
							}

							err = crudOp.Create()
							if err != nil {
								log.WithError(err).Fatal("Could not create record")
								return cli.Exit("Cannot create record", 1)
							}
							return nil
						},
					},
					{
						Name:        "delete",
						Aliases:     []string{"del"},
						Description: "delete record data",
						Flags: []cli.Flag{
							&cli.StringFlag{
								Name:        "type",
								Aliases:     []string{"t"},
								DefaultText: "A",
								Usage:       "Record type to delete: A, AAAA",
							},
							&cli.StringFlag{
								Name:  "data",
								Usage: "Record data",
							},
						},
						Action: func(context *cli.Context) error {
							dns, err := NewDNSimpleAPI(context.String("token"))
							if err != nil {
								return cli.Exit("Cannot communicate with DNSimple API", 1)
							}

							domainName := context.String("domain")
							recordName := context.Args().First()
							recordType := context.String("type")
							recordData := context.String("data")

							crudOp, err := NewOperationData(dns, recordData, recordName, domainName, recordType, context.Bool("ipv6"))
							if err != nil {
								log.WithError(err).Fatal("Could not create command operation data.")
								return cli.Exit("Cannot delete record", 1)
							}

							err = crudOp.Delete()
							if err != nil {
								return cli.Exit("Cannot delete record", 1)
							}
							return nil
						},
					},
					{
						Name:        "upsert",
						Aliases:     []string{"up", "update"},
						Description: "upsert record data",
						Flags: []cli.Flag{
							&cli.StringFlag{
								Name:        "type",
								Aliases:     []string{"t"},
								DefaultText: "A",
								Usage:       "Record type to upsert: A, AAAA",
							},
							&cli.StringFlag{
								Name:  "data",
								Usage: "Record data",
							},
						},
						Action: func(context *cli.Context) error {
							dns, err := NewDNSimpleAPI(context.String("token"))
							if err != nil {
								return cli.Exit("Cannot communicate with DNSimple API", 1)
							}

							domainName := context.String("domain")
							recordName := context.Args().First()
							recordType := context.String("type")
							recordData := context.String("data")

							crudOp, err := NewOperationData(dns, recordData, recordName, domainName, recordType, context.Bool("ipv6"))
							if err != nil {
								log.WithError(err).Fatal("Could not create command operation data.")
								return cli.Exit("Cannot upsert record", 1)
							}

							err = crudOp.Upsert()
							if err != nil {
								return cli.Exit("Cannot upsert record", 1)
							}
							return nil
						},
					},
				},
			},
		},
	}

	app.Run(os.Args)
}
<|endoftext|>


<commit_before>// Copyright 2015 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Gt is a wrapper for ``go test'' that caches test results.
//
// The usage of ``gt'' is identical to that of ``go test.''
//
//	gt [-f] [-l] [arguments for "go test"]
//
// The difference between ``gt'' and ``go test'' is that when testing
// a list of packages, if a package and its dependencies have not changed
// since the last run, ``gt'' reuses the previous result.
//
// The -f flag causes gt to treat all test results as uncached, as does the
// use of any ``go test'' flag other than -short and -v.
//
// The -l flag causes gt to list the uncached tests it would run.
//
// Cached test results are saved in $HOME/Library/Caches/go-test-cache on OS X
// and $HOME/.cache/go-test-cache on other systems.
// It is always safe to delete these directories if they become too large.
//
// Gt is an experiment in what it would mean and how well it would work
// to cache test results. If the experiment proves successful, the functionality
// may move into the standard go command.
//
// Examples
//
// Run (and cache) the strings test:
//
//	$ gt strings
//	ok  	strings	0.436s
//	$
//
// List tests in str... without cached results:
//
//	$ gt -l str...
//	strconv
//	$
//
// Run str... tests:
//
//	$ gt str...
//	ok  	strconv	1.548s
//	ok  	strings	0.436s (cached)
//	$
//
// Force rerun of both:
//
//	$ gt -f str...
//	ok  	strconv	1.795s
//	ok  	strings	0.629s
//	$
//
package main

import (
	"bufio"
	"bytes"
	"crypto/sha1"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"runtime"
	"strings"
	"time"
)

func usage() {
	fmt.Fprint(os.Stderr, "usage: gt [arguments for \"go test\"]\n")
	os.Exit(2)
}

var (
	flagV      bool
	flagShort  bool
	flagList   bool
	flagForce  bool
	flagTiming bool
	failed     bool
	cacheDir   string
	start      = time.Now()
)

func main() {
	log.SetFlags(0)
	log.SetPrefix("gt: ")

	opts, pkgs := parseFlags()
	if len(pkgs) == 0 {
		if flagList {
			log.Fatal("cannot use -l without package list or with testing flags other than -v and -short")
		}
		cmd := exec.Command("go", append([]string{"test"}, os.Args[1:]...)...)
		cmd.Stdin = os.Stdin
		cmd.Stdout = os.Stdout
		cmd.Stderr = os.Stderr
		err := cmd.Run()
		if err != nil {
			log.Fatalf("go test: %v", err)
		}
		return
	}

	if flagTiming {
		log.Printf("%.2fs go list", time.Since(start).Seconds())
	}

	// Expand pkg list.
	out, err := exec.Command("go", append([]string{"list"}, pkgs...)...).CombinedOutput()
	if err != nil {
		log.Fatalf("go list: %v", err)
	}
	pkgs = strings.Fields(string(out))

	if flagTiming {
		log.Printf("%.2fs go list -json", time.Since(start).Seconds())
	}

	// Build list of all dependencies.
	readPkgs(pkgs)

	first := true
	next := pkgs
	for {
		var deps []string
		for _, path := range next {
			p := pkgInfo[path]
			if p.Incomplete {
				log.Fatalf("go list: errors loading packages")
			}
			for _, dep := range p.Deps {
				if _, ok := pkgInfo[dep]; !ok {
					pkgInfo[dep] = nil
					deps = append(deps, dep)
				}
			}
			if first {
				for _, dep := range p.TestImports {
					if _, ok := pkgInfo[dep]; !ok {
						pkgInfo[dep] = nil
						deps = append(deps, dep)
					}
				}
				for _, dep := range p.XTestImports {
					if _, ok := pkgInfo[dep]; !ok {
						pkgInfo[dep] = nil
						deps = append(deps, dep)
					}
				}
			}
		}
		if len(deps) == 0 {
			break
		}

		if flagTiming {
			log.Printf("%.2fs go list -json", time.Since(start).Seconds())
		}
		readPkgs(deps)
		next = deps
		first = false
	}

	if runtime.GOOS == "darwin" {
		cacheDir = fmt.Sprintf("%s/Library/Caches/go-test-cache", os.Getenv("HOME"))
	} else {
		cacheDir = fmt.Sprintf("%s/.cache/go-test-cache", os.Getenv("HOME"))
	}

	if flagTiming {
		log.Printf("%.2fs compute hashes", time.Since(start).Seconds())
	}

	computeStale(pkgs)

	var toRun []string
	for _, pkg := range pkgs {
		if !haveTestResult(pkg) {
			toRun = append(toRun, pkg)
		}
	}

	if flagTiming {
		log.Printf("%.2fs ready to run", time.Since(start).Seconds())
	}

	if flagList {
		for _, pkg := range toRun {
			fmt.Printf("%s\n", pkg)
		}
		return
	}

	var cmd *exec.Cmd
	pr, pw := io.Pipe()
	r := bufio.NewReader(pr)
	if len(toRun) > 0 {
		if err := os.MkdirAll(cacheDir, 0700); err != nil {
			log.Fatal(err)
		}

		args := []string{"test"}
		args = append(args, opts...)
		args = append(args, toRun...)
		cmd = exec.Command("go", args...)
		cmd.Stdout = pw
		cmd.Stderr = pw
		if err := cmd.Start(); err != nil {
			log.Fatalf("go test: %v", err)
		}
	}

	var cmdErr error
	done := make(chan bool)
	go func() {
		if cmd != nil {
			cmdErr = cmd.Wait()
		}
		pw.Close()
		done <- true
	}()

	for _, pkg := range pkgs {
		if len(toRun) > 0 && toRun[0] == pkg {
			readTestResult(r, pkg)
			toRun = toRun[1:]
		} else {
			showTestResult(pkg)
		}
	}

	io.Copy(os.Stdout, r)

	<-done
	if cmdErr != nil && !failed {
		log.Fatalf("go test: %v", cmdErr)
	}

	if flagTiming {
		log.Printf("%.2fs done", time.Since(start).Seconds())
	}

	if failed {
		os.Exit(1)
	}
}

var (
	pkgInfo   = map[string]*Package{}
	outOfSync bool
)

type Package struct {
	Dir          string
	ImportPath   string
	Standard     bool
	Goroot       bool
	Stale        bool
	GoFiles      []string
	CgoFiles     []string
	CFiles       []string
	CXXFiles     []string
	MFiles       []string
	HFiles       []string
	SFiles       []string
	SwigFiles    []string
	SwigCXXFiles []string
	SysoFiles    []string
	Imports      []string
	Deps         []string
	Incomplete   bool
	TestGoFiles  []string
	TestImports  []string
	XTestGoFiles []string
	XTestImports []string

	testHash string
	pkgHash  string
}

func readPkgs(pkgs []string) {
	out, err := exec.Command("go", append([]string{"list", "-json"}, pkgs...)...).CombinedOutput()
	if err != nil {
		log.Fatalf("go list: %v\n%s", err, out)
	}

	dec := json.NewDecoder(bytes.NewReader(out))
	for {
		var p Package
		if err := dec.Decode(&p); err != nil {
			if err == io.EOF {
				break
			}
			log.Fatalf("reading go list output: %v", err)
		}
		pkgInfo[p.ImportPath] = &p
	}
}

func computeStale(pkgs []string) {
	for _, pkg := range pkgs {
		computeTestHash(pkgInfo[pkg])
	}
}

func computeTestHash(p *Package) {
	if p.testHash != "" {
		return
	}
	p.testHash = "cycle"
	computePkgHash(p)
	h := sha1.New()
	fmt.Fprintf(h, "test\n")
	if flagShort {
		fmt.Fprintf(h, "-short\n")
	}
	if flagV {
		fmt.Fprintf(h, "-v\n")
	}
	fmt.Fprintf(h, "pkg %s\n", p.pkgHash)
	for _, imp := range p.TestImports {
		p1 := pkgInfo[imp]
		computePkgHash(p1)
		fmt.Fprintf(h, "testimport %s\n", p1.pkgHash)
	}
	for _, imp := range p.XTestImports {
		p1 := pkgInfo[imp]
		computePkgHash(p1)
		fmt.Fprintf(h, "xtestimport %s\n", p1.pkgHash)
	}
	hashFiles(h, p.Dir, p.TestGoFiles)
	hashFiles(h, p.Dir, p.XTestGoFiles)
	p.testHash = fmt.Sprintf("%x", h.Sum(nil))
}

func computePkgHash(p *Package) {
	if p.pkgHash != "" {
		return
	}
	p.pkgHash = "cycle"
	h := sha1.New()
	fmt.Fprintf(h, "pkg\n")
	for _, imp := range p.Deps {
		p1 := pkgInfo[imp]
		if p1 == nil {
			log.Fatalf("lost package: %v for %v", imp, p.ImportPath)
		}
		computePkgHash(p1)
		fmt.Fprintf(h, "import %s\n", p1.pkgHash)
	}
	hashFiles(h, p.Dir, p.GoFiles)
	hashFiles(h, p.Dir, p.CgoFiles)
	hashFiles(h, p.Dir, p.CFiles)
	hashFiles(h, p.Dir, p.CXXFiles)
	hashFiles(h, p.Dir, p.MFiles)
	hashFiles(h, p.Dir, p.HFiles)
	hashFiles(h, p.Dir, p.SFiles)
	hashFiles(h, p.Dir, p.SwigFiles)
	hashFiles(h, p.Dir, p.SwigCXXFiles)
	hashFiles(h, p.Dir, p.SysoFiles)

	p.pkgHash = fmt.Sprintf("%x", h.Sum(nil))
}

func hashFiles(h io.Writer, dir string, files []string) {
	for _, file := range files {
		f, err := os.Open(filepath.Join(dir, file))
		if err != nil {
			fmt.Fprintf(h, "%s error\n", file)
			continue
		}
		fmt.Fprintf(h, "file %s\n", file)
		n, _ := io.Copy(h, f)
		fmt.Fprintf(h, "%d bytes\n", n)
		f.Close()
	}
}

func cacheFile(p *Package) string {
	return filepath.Join(cacheDir, fmt.Sprintf("%s/%s.test", p.testHash[:3], p.testHash[3:]))
}

func haveTestResult(path string) bool {
	if flagForce {
		return false
	}
	p := pkgInfo[path]
	if p.testHash == "cycle" {
		return false
	}
	fi, err := os.Stat(cacheFile(pkgInfo[path]))
	return err == nil && fi.Mode().IsRegular()
}

var fail = []byte("FAIL")

func showTestResult(path string) {
	p := pkgInfo[path]
	if p.testHash == "cycle" {
		return
	}
	data, err := ioutil.ReadFile(cacheFile(pkgInfo[path]))
	if err != nil {
		fmt.Printf("%v\n", err)
		fmt.Printf("FAIL\t%s\t(cached)\n", path)
		return
	}
	os.Stdout.Write(data)
	data = bytes.TrimSpace(data)
	i := bytes.LastIndex(data, []byte{'\n'})
	line := data[i+1:]
	if bytes.HasPrefix(line, fail) {
		failed = true
	}
}

var endRE = regexp.MustCompile(`\A(\?|ok|FAIL) ? ? ?\t([^ \t]+)\t([0-9.]+s|\[.*\])\n\z`)

func readTestResult(r *bufio.Reader, path string) {
	var buf bytes.Buffer
	for {
		line, err := r.ReadString('\n')
		os.Stdout.WriteString(line)
		if err != nil {
			log.Fatalf("reading test output for %s: %v", path, err)
		}
		if outOfSync {
			continue
		}
		m := endRE.FindStringSubmatch(line)
		if m == nil {
			buf.WriteString(line)
			continue
		}

		if m[1] == "FAIL" {
			failed = true
		}

		fmt.Fprintf(&buf, "%s (cached)\n", strings.TrimSuffix(line, "\n"))
		file := cacheFile(pkgInfo[path])
		if err := os.MkdirAll(filepath.Dir(file), 0700); err != nil {
			log.Print(err)
		} else if err := ioutil.WriteFile(file, buf.Bytes(), 0600); err != nil {
			log.Print(err)
		}

		break
	}
}

func parseFlags() (opts, pkgs []string) {
	donePkgs := false
	for i := 1; i < len(os.Args); i++ {
		arg := os.Args[i]
		if !strings.HasPrefix(arg, "-") {
			if donePkgs {
				// additional arguments after pkg list ended
				return nil, nil
			}
			pkgs = append(pkgs, arg)
			continue
		}
		donePkgs = len(pkgs) > 0
		if strings.HasPrefix(arg, "--") && arg != "--" && !strings.HasPrefix(arg, "---") {
			arg = arg[1:]
		}
		if arg == "-gt.timing" {
			flagTiming = true
			continue
		}
		if arg == "-v" {
			flagV = true
			opts = append(opts, arg)
			donePkgs = len(pkgs) > 0
			continue
		}
		if arg == "-short" {
			flagShort = true
			opts = append(opts, arg)
			continue
		}
		if arg == "-f" {
			flagForce = true
			continue
		}
		if arg == "-l" {
			flagList = true
			continue
		}
		// unrecognized flag
		return nil, nil
	}
	return opts, pkgs
}
<commit_msg>gt: fix usage<commit_after>// Copyright 2015 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Gt is a wrapper for ``go test'' that caches test results.
//
// The usage of ``gt'' is nearly identical to that of ``go test.''
//
//	gt [-f] [-l] [arguments for "go test"]
//
// The difference between ``gt'' and ``go test'' is that when testing
// a list of packages, if a package and its dependencies have not changed
// since the last run, ``gt'' reuses the previous result.
//
// The -f flag causes gt to treat all test results as uncached, as does the
// use of any ``go test'' flag other than -short and -v.
//
// The -l flag causes gt to list the uncached tests it would run.
//
// Cached test results are saved in $HOME/Library/Caches/go-test-cache on OS X
// and $HOME/.cache/go-test-cache on other systems.
// It is always safe to delete these directories if they become too large.
//
// Gt is an experiment in what it would mean and how well it would work
// to cache test results. If the experiment proves successful, the functionality
// may move into the standard go command.
//
// Examples
//
// Run (and cache) the strings test:
//
//	$ gt strings
//	ok  	strings	0.436s
//	$
//
// List tests in str... without cached results:
//
//	$ gt -l str...
//	strconv
//	$
//
// Run str... tests:
//
//	$ gt str...
//	ok  	strconv	1.548s
//	ok  	strings	0.436s (cached)
//	$
//
// Force rerun of both:
//
//	$ gt -f str...
//	ok  	strconv	1.795s
//	ok  	strings	0.629s
//	$
//
package main

import (
	"bufio"
	"bytes"
	"crypto/sha1"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"runtime"
	"strings"
	"time"
)

func usage() {
	fmt.Fprint(os.Stderr, "usage: gt [arguments for \"go test\"]\n")
	os.Exit(2)
}

var (
	flagV      bool
	flagShort  bool
	flagList   bool
	flagForce  bool
	flagTiming bool
	failed     bool
	cacheDir   string
	start      = time.Now()
)

func main() {
	log.SetFlags(0)
	log.SetPrefix("gt: ")

	opts, pkgs := parseFlags()
	if len(pkgs) == 0 {
		if flagList {
			log.Fatal("cannot use -l without package list or with testing flags other than -v and -short")
		}
		cmd := exec.Command("go", append([]string{"test"}, os.Args[1:]...)...)
		cmd.Stdin = os.Stdin
		cmd.Stdout = os.Stdout
		cmd.Stderr = os.Stderr
		err := cmd.Run()
		if err != nil {
			log.Fatalf("go test: %v", err)
		}
		return
	}

	if flagTiming {
		log.Printf("%.2fs go list", time.Since(start).Seconds())
	}

	// Expand pkg list.
	out, err := exec.Command("go", append([]string{"list"}, pkgs...)...).CombinedOutput()
	if err != nil {
		log.Fatalf("go list: %v", err)
	}
	pkgs = strings.Fields(string(out))

	if flagTiming {
		log.Printf("%.2fs go list -json", time.Since(start).Seconds())
	}

	// Build list of all dependencies.
	readPkgs(pkgs)

	first := true
	next := pkgs
	for {
		var deps []string
		for _, path := range next {
			p := pkgInfo[path]
			if p.Incomplete {
				log.Fatalf("go list: errors loading packages")
			}
			for _, dep := range p.Deps {
				if _, ok := pkgInfo[dep]; !ok {
					pkgInfo[dep] = nil
					deps = append(deps, dep)
				}
			}
			if first {
				for _, dep := range p.TestImports {
					if _, ok := pkgInfo[dep]; !ok {
						pkgInfo[dep] = nil
						deps = append(deps, dep)
					}
				}
				for _, dep := range p.XTestImports {
					if _, ok := pkgInfo[dep]; !ok {
						pkgInfo[dep] = nil
						deps = append(deps, dep)
					}
				}
			}
		}
		if len(deps) == 0 {
			break
		}

		if flagTiming {
			log.Printf("%.2fs go list -json", time.Since(start).Seconds())
		}
		readPkgs(deps)
		next = deps
		first = false
	}

	if runtime.GOOS == "darwin" {
		cacheDir = fmt.Sprintf("%s/Library/Caches/go-test-cache", os.Getenv("HOME"))
	} else {
		cacheDir = fmt.Sprintf("%s/.cache/go-test-cache", os.Getenv("HOME"))
	}

	if flagTiming {
		log.Printf("%.2fs compute hashes", time.Since(start).Seconds())
	}

	computeStale(pkgs)

	var toRun []string
	for _, pkg := range pkgs {
		if !haveTestResult(pkg) {
			toRun = append(toRun, pkg)
		}
	}

	if flagTiming {
		log.Printf("%.2fs ready to run", time.Since(start).Seconds())
	}

	if flagList {
		for _, pkg := range toRun {
			fmt.Printf("%s\n", pkg)
		}
		return
	}

	var cmd *exec.Cmd
	pr, pw := io.Pipe()
	r := bufio.NewReader(pr)
	if len(toRun) > 0 {
		if err := os.MkdirAll(cacheDir, 0700); err != nil {
			log.Fatal(err)
		}

		args := []string{"test"}
		args = append(args, opts...)
		args = append(args, toRun...)
		cmd = exec.Command("go", args...)
		cmd.Stdout = pw
		cmd.Stderr = pw
		if err := cmd.Start(); err != nil {
			log.Fatalf("go test: %v", err)
		}
	}

	var cmdErr error
	done := make(chan bool)
	go func() {
		if cmd != nil {
			cmdErr = cmd.Wait()
		}
		pw.Close()
		done <- true
	}()

	for _, pkg := range pkgs {
		if len(toRun) > 0 && toRun[0] == pkg {
			readTestResult(r, pkg)
			toRun = toRun[1:]
		} else {
			showTestResult(pkg)
		}
	}

	io.Copy(os.Stdout, r)

	<-done
	if cmdErr != nil && !failed {
		log.Fatalf("go test: %v", cmdErr)
	}

	if flagTiming {
		log.Printf("%.2fs done", time.Since(start).Seconds())
	}

	if failed {
		os.Exit(1)
	}
}

var (
	pkgInfo   = map[string]*Package{}
	outOfSync bool
)

type Package struct {
	Dir          string
	ImportPath   string
	Standard     bool
	Goroot       bool
	Stale        bool
	GoFiles      []string
	CgoFiles     []string
	CFiles       []string
	CXXFiles     []string
	MFiles       []string
	HFiles       []string
	SFiles       []string
	SwigFiles    []string
	SwigCXXFiles []string
	SysoFiles    []string
	Imports      []string
	Deps         []string
	Incomplete   bool
	TestGoFiles  []string
	TestImports  []string
	XTestGoFiles []string
	XTestImports []string

	testHash string
	pkgHash  string
}

func readPkgs(pkgs []string) {
	out, err := exec.Command("go", append([]string{"list", "-json"}, pkgs...)...).CombinedOutput()
	if err != nil {
		log.Fatalf("go list: %v\n%s", err, out)
	}

	dec := json.NewDecoder(bytes.NewReader(out))
	for {
		var p Package
		if err := dec.Decode(&p); err != nil {
			if err == io.EOF {
				break
			}
			log.Fatalf("reading go list output: %v", err)
		}
		pkgInfo[p.ImportPath] = &p
	}
}

func computeStale(pkgs []string) {
	for _, pkg := range pkgs {
		computeTestHash(pkgInfo[pkg])
	}
}

func computeTestHash(p *Package) {
	if p.testHash != "" {
		return
	}
	p.testHash = "cycle"
	computePkgHash(p)
	h := sha1.New()
	fmt.Fprintf(h, "test\n")
	if flagShort {
		fmt.Fprintf(h, "-short\n")
	}
	if flagV {
		fmt.Fprintf(h, "-v\n")
	}
	fmt.Fprintf(h, "pkg %s\n", p.pkgHash)
	for _, imp := range p.TestImports {
		p1 := pkgInfo[imp]
		computePkgHash(p1)
		fmt.Fprintf(h, "testimport %s\n", p1.pkgHash)
	}
	for _, imp := range p.XTestImports {
		p1 := pkgInfo[imp]
		computePkgHash(p1)
		fmt.Fprintf(h, "xtestimport %s\n", p1.pkgHash)
	}
	hashFiles(h, p.Dir, p.TestGoFiles)
	hashFiles(h, p.Dir, p.XTestGoFiles)
	p.testHash = fmt.Sprintf("%x", h.Sum(nil))
}

func computePkgHash(p *Package) {
	if p.pkgHash != "" {
		return
	}
	p.pkgHash = "cycle"
	h := sha1.New()
	fmt.Fprintf(h, "pkg\n")
	for _, imp := range p.Deps {
		p1 := pkgInfo[imp]
		if p1 == nil {
			log.Fatalf("lost package: %v for %v", imp, p.ImportPath)
		}
		computePkgHash(p1)
		fmt.Fprintf(h, "import %s\n", p1.pkgHash)
	}
	hashFiles(h, p.Dir, p.GoFiles)
	hashFiles(h, p.Dir, p.CgoFiles)
	hashFiles(h, p.Dir, p.CFiles)
	hashFiles(h, p.Dir, p.CXXFiles)
	hashFiles(h, p.Dir, p.MFiles)
	hashFiles(h, p.Dir, p.HFiles)
	hashFiles(h, p.Dir, p.SFiles)
	hashFiles(h, p.Dir, p.SwigFiles)
	hashFiles(h, p.Dir, p.SwigCXXFiles)
	hashFiles(h, p.Dir, p.SysoFiles)

	p.pkgHash = fmt.Sprintf("%x", h.Sum(nil))
}

func hashFiles(h io.Writer, dir string, files []string) {
	for _, file := range files {
		f, err := os.Open(filepath.Join(dir, file))
		if err != nil {
			fmt.Fprintf(h, "%s error\n", file)
			continue
		}
		fmt.Fprintf(h, "file %s\n", file)
		n, _ := io.Copy(h, f)
		fmt.Fprintf(h, "%d bytes\n", n)
		f.Close()
	}
}

func cacheFile(p *Package) string {
	return filepath.Join(cacheDir, fmt.Sprintf("%s/%s.test", p.testHash[:3], p.testHash[3:]))
}

func haveTestResult(path string) bool {
	if flagForce {
		return false
	}
	p := pkgInfo[path]
	if p.testHash == "cycle" {
		return false
	}
	fi, err := os.Stat(cacheFile(pkgInfo[path]))
	return err == nil && fi.Mode().IsRegular()
}

var fail = []byte("FAIL")

func showTestResult(path string) {
	p := pkgInfo[path]
	if p.testHash == "cycle" {
		return
	}
	data, err := ioutil.ReadFile(cacheFile(pkgInfo[path]))
	if err != nil {
		fmt.Printf("%v\n", err)
		fmt.Printf("FAIL\t%s\t(cached)\n", path)
		return
	}
	os.Stdout.Write(data)
	data = bytes.TrimSpace(data)
	i := bytes.LastIndex(data, []byte{'\n'})
	line := data[i+1:]
	if bytes.HasPrefix(line, fail) {
		failed = true
	}
}

var endRE = regexp.MustCompile(`\A(\?|ok|FAIL) ? ? ?\t([^ \t]+)\t([0-9.]+s|\[.*\])\n\z`)

func readTestResult(r *bufio.Reader, path string) {
	var buf bytes.Buffer
	for {
		line, err := r.ReadString('\n')
		os.Stdout.WriteString(line)
		if err != nil {
			log.Fatalf("reading test output for %s: %v", path, err)
		}
		if outOfSync {
			continue
		}
		m := endRE.FindStringSubmatch(line)
		if m == nil {
			buf.WriteString(line)
			continue
		}

		if m[1] == "FAIL" {
			failed = true
		}

		fmt.Fprintf(&buf, "%s (cached)\n", strings.TrimSuffix(line, "\n"))
		file := cacheFile(pkgInfo[path])
		if err := os.MkdirAll(filepath.Dir(file), 0700); err != nil {
			log.Print(err)
		} else if err := ioutil.WriteFile(file, buf.Bytes(), 0600); err != nil {
			log.Print(err)
		}

		break
	}
}

func parseFlags() (opts, pkgs []string) {
	donePkgs := false
	for i := 1; i < len(os.Args); i++ {
		arg := os.Args[i]
		if !strings.HasPrefix(arg, "-") {
			if donePkgs {
				// additional arguments after pkg list ended
				return nil, nil
			}
			pkgs = append(pkgs, arg)
			continue
		}
		donePkgs = len(pkgs) > 0
		if strings.HasPrefix(arg, "--") && arg != "--" && !strings.HasPrefix(arg, "---") {
			arg = arg[1:]
		}
		if arg == "-gt.timing" {
			flagTiming = true
			continue
		}
		if arg == "-v" {
			flagV = true
			opts = append(opts, arg)
			donePkgs = len(pkgs) > 0
			continue
		}
		if arg == "-short" {
			flagShort = true
			opts = append(opts, arg)
			continue
		}
		if arg == "-f" {
			flagForce = true
			continue
		}
		if arg == "-l" {
			flagList = true
			continue
		}
		// unrecognized flag
		return nil, nil
	}
	return opts, pkgs
}
<|endoftext|>


<commit_before>// Copyright 2013 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// tsuru-admin is under development.
package main

import (
	"github.com/globocom/tsuru/cmd"
	"github.com/globocom/tsuru/cmd/tsuru-base"
	"os"
)

const (
	version = "0.1.8"
	header  = "Supported-Tsuru-Admin"
)

func buildManager(name string) *cmd.Manager {
	m := cmd.BuildBaseManager(name, version, header)
	m.Register(tsuru.AppList{})
	m.Register(&tsuru.SetCName{})
	m.Register(&tsuru.UnsetCName{})
	m.Register(&tokenGen{})
	return m
}

func main() {
	name := cmd.ExtractProgramName(os.Args[0])
	manager := buildManager(name)
	args := os.Args[1:]
	manager.Run(args)
}
<commit_msg>cmd/tsuru-admin: version 0.1.9<commit_after>// Copyright 2013 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// tsuru-admin is under development.
package main

import (
	"github.com/globocom/tsuru/cmd"
	"github.com/globocom/tsuru/cmd/tsuru-base"
	"os"
)

const (
	version = "0.1.9"
	header  = "Supported-Tsuru-Admin"
)

func buildManager(name string) *cmd.Manager {
	m := cmd.BuildBaseManager(name, version, header)
	m.Register(tsuru.AppList{})
	m.Register(&tsuru.SetCName{})
	m.Register(&tsuru.UnsetCName{})
	m.Register(&tokenGen{})
	return m
}

func main() {
	name := cmd.ExtractProgramName(os.Args[0])
	manager := buildManager(name)
	args := os.Args[1:]
	manager.Run(args)
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"log"
	_ "net/http/pprof"
	"os"
	"path/filepath"
	"runtime"
	"time"

	"github.com/StackExchange/bosun/_third_party/github.com/StackExchange/scollector/collect"
	"github.com/StackExchange/bosun/_third_party/github.com/howeyc/fsnotify"
	"github.com/StackExchange/bosun/conf"
	"github.com/StackExchange/bosun/sched"
	"github.com/StackExchange/bosun/search"
	"github.com/StackExchange/bosun/web"
)

var (
	confFile = flag.String("c", "dev.conf", "config file location")
	test     = flag.Bool("t", false, "Only validate config then exit")
	watch    = flag.Bool("w", false, "watch current directory and exit on changes; for use with an autorestarter")
)

func main() {
	flag.Parse()
	log.Println("GOMAXPROCS:", runtime.GOMAXPROCS(0))
	c, err := conf.ParseFile(*confFile)
	if err != nil {
		log.Fatal(err)
	}
	if *test {
		log.Println("Valid Config")
		os.Exit(0)
	}
	if err := collect.Init(c.RelayListen, "bosun"); err != nil {
		log.Fatal(err)
	}
	sched.Load(c)
	go func() { log.Fatal(search.RelayHTTP(c.RelayListen, c.TsdbHost)) }()
	go func() { log.Fatal(web.Listen(c.HttpListen, c.WebDir, c.TsdbHost)) }()
	go func() { log.Fatal(sched.Run()) }()
	go watcher()
	select {}
}

func watcher() {
	if !*watch {
		return
	}
	time.Sleep(time.Second)
	watcher, err := fsnotify.NewWatcher()
	if err != nil {
		log.Print(err)
		return
	}
	go func() {
		for {
			select {
			case ev := <-watcher.Event:
				log.Print("file changed, exiting:", ev)
				os.Exit(0)
			case err := <-watcher.Error:
				log.Print(err)
			}
		}
	}()
	filepath.Walk(".", func(path string, info os.FileInfo, err error) error {
		if !info.IsDir() || (len(path) > 1 && path[0] == '.') {
			return nil
		}
		watcher.Watch(path)
		return nil
	})
}
<commit_msg>Set maxprocs to numcpu<commit_after>package main

import (
	"flag"
	"log"
	_ "net/http/pprof"
	"os"
	"path/filepath"
	"runtime"
	"time"

	"github.com/StackExchange/bosun/_third_party/github.com/StackExchange/scollector/collect"
	"github.com/StackExchange/bosun/_third_party/github.com/howeyc/fsnotify"
	"github.com/StackExchange/bosun/conf"
	"github.com/StackExchange/bosun/sched"
	"github.com/StackExchange/bosun/search"
	"github.com/StackExchange/bosun/web"
)

var (
	confFile = flag.String("c", "dev.conf", "config file location")
	test     = flag.Bool("t", false, "Only validate config then exit")
	watch    = flag.Bool("w", false, "watch current directory and exit on changes; for use with an autorestarter")
)

func main() {
	flag.Parse()
	runtime.GOMAXPROCS(runtime.NumCPU())
	c, err := conf.ParseFile(*confFile)
	if err != nil {
		log.Fatal(err)
	}
	if *test {
		log.Println("Valid Config")
		os.Exit(0)
	}
	if err := collect.Init(c.RelayListen, "bosun"); err != nil {
		log.Fatal(err)
	}
	sched.Load(c)
	go func() { log.Fatal(search.RelayHTTP(c.RelayListen, c.TsdbHost)) }()
	go func() { log.Fatal(web.Listen(c.HttpListen, c.WebDir, c.TsdbHost)) }()
	go func() { log.Fatal(sched.Run()) }()
	go watcher()
	select {}
}

func watcher() {
	if !*watch {
		return
	}
	time.Sleep(time.Second)
	watcher, err := fsnotify.NewWatcher()
	if err != nil {
		log.Print(err)
		return
	}
	go func() {
		for {
			select {
			case ev := <-watcher.Event:
				log.Print("file changed, exiting:", ev)
				os.Exit(0)
			case err := <-watcher.Error:
				log.Print(err)
			}
		}
	}()
	filepath.Walk(".", func(path string, info os.FileInfo, err error) error {
		if !info.IsDir() || (len(path) > 1 && path[0] == '.') {
			return nil
		}
		watcher.Watch(path)
		return nil
	})
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"errors"
	"expvar"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"math"
	"net/http"
	_ "net/http/pprof"
	"net/url"
	"os"
	"runtime"
	"strconv"
	"strings"
	"time"

	"code.google.com/p/gogoprotobuf/proto"
	pb "github.com/dgryski/carbonzipper/carbonzipperpb"

	"github.com/bradfitz/gomemcache/memcache"
	"github.com/peterbourgon/g2g"
)

type zipper string

var Zipper zipper

var Metrics = struct {
	Requests         *expvar.Int
	RequestCacheHits *expvar.Int

	FindRequests  *expvar.Int
	FindCacheHits *expvar.Int

	RenderRequests *expvar.Int
}{
	Requests:         expvar.NewInt("requests"),
	RequestCacheHits: expvar.NewInt("request_cache_hits"),

	FindRequests:  expvar.NewInt("find_requests"),
	FindCacheHits: expvar.NewInt("find_cache_hits"),

	RenderRequests: expvar.NewInt("render_requests"),
}

var queryCache bytesCache
var findCache bytesCache

var timeFormats = []string{"15:04 20060102", "20060102", "01/02/06"}

// dateParamToEpoch turns a passed string parameter into an epoch we can send to the Zipper
func dateParamToEpoch(s string, d int64) int32 {

	if s == "" {
		// return the default if nothing was passed
		return int32(d)
	}

	// relative timestamp
	if s[0] == '-' {

		offset, err := intervalString(s[1:])
		if err != nil {
			return int32(d)
		}

		return int32(timeNow().Add(-time.Duration(offset) * time.Second).Unix())
	}

	if s == "now" {
		return int32(timeNow().Unix())
	}

	sint, err := strconv.Atoi(s)
	if err == nil && len(s) > 8 {
		return int32(sint) // We got a timestamp so returning it
	}

	if strings.Contains(s, "_") {
		s = strings.Replace(s, "_", " ", 1) // Go can't parse _ in date strings
	}

	for _, format := range timeFormats {
		t, err := time.Parse(format, s)
		if err == nil {
			return int32(t.Unix())
		}
	}
	return int32(d)
}

func intervalString(s string) (int32, error) {

	var j int

	for j < len(s) && s[j] >= '0' && s[j] <= '9' {
		j++
	}
	offsetStr, unitStr := s[:j], s[j:]

	var units int
	switch unitStr {
	case "s", "sec", "secs", "second", "seconds":
		units = 1
	case "min", "minute", "minutes":
		units = 60
	case "h", "hour", "hours":
		units = 60 * 60
	case "d", "day", "days":
		units = 24 * 60 * 60
	case "w", "week", "weeks":
		units = 7 * 24 * 60 * 60
	case "mon", "month", "months":
		units = 30 * 24 * 60 * 60
	case "y", "year", "years":
		units = 365 * 24 * 60 * 60
	default:
		return 0, errors.New("unknown time units")
	}

	offset, err := strconv.Atoi(offsetStr)
	if err != nil {
		return 0, err
	}

	return int32(offset * units), nil
}

// FIXME(dgryski): extract the http.Get + unproto code into its own function

func (z zipper) Find(metric string) (pb.GlobResponse, error) {

	u, _ := url.Parse(string(z) + "/metrics/find/")

	u.RawQuery = url.Values{
		"query":  []string{metric},
		"format": []string{"protobuf"},
	}.Encode()

	resp, err := http.Get(u.String())
	if err != nil {
		log.Printf("Find: http.Get: %+v\n", err)
		return pb.GlobResponse{}, err
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Printf("Find: ioutil.ReadAll: %+v\n", err)
		return pb.GlobResponse{}, err
	}

	var pbresp pb.GlobResponse

	err = proto.Unmarshal(body, &pbresp)
	if err != nil {
		log.Printf("Find: proto.Unmarshal: %+v\n", err)
		return pb.GlobResponse{}, err
	}

	return pbresp, nil
}

func (z zipper) Render(metric string, from, until int32) (pb.FetchResponse, error) {

	u, _ := url.Parse(string(z) + "/render/")

	u.RawQuery = url.Values{
		"target": []string{metric},
		"format": []string{"protobuf"},
		"from":   []string{strconv.Itoa(int(from))},
		"until":  []string{strconv.Itoa(int(until))},
	}.Encode()

	resp, err := http.Get(u.String())
	if err != nil {
		log.Printf("Render: http.Get: %s: %+v\n", metric, err)
		return pb.FetchResponse{}, err
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Printf("Render: ioutil.ReadAll: %s: %+v\n", metric, err)
		return pb.FetchResponse{}, err
	}

	var pbresp pb.FetchResponse

	err = proto.Unmarshal(body, &pbresp)
	if err != nil {
		log.Printf("Render: proto.Unmarshal: %s: %+v\n", metric, err)
		return pb.FetchResponse{}, err
	}

	return pbresp, nil
}

type limiter chan struct{}

func (l limiter) enter() { l <- struct{}{} }
func (l limiter) leave() { <-l }

var Limiter limiter

// for testing
var timeNow = time.Now

type graphitePoint struct {
	value float64
	t     int32
}

func (g graphitePoint) MarshalJSON() ([]byte, error) {
	// TODO(dgryski): fmt.Sprintf() is slow, use strconv.Append{Float,Int}
	if math.IsNaN(g.value) {
		return []byte(fmt.Sprintf("[null,%d]", g.t)), nil
	}
	return []byte(fmt.Sprintf("[%g,%d]", g.value, g.t)), nil
}

type jsonResponse struct {
	Target     string          `json:"target"`
	Datapoints []graphitePoint `json:"datapoints"`
}

func renderHandler(w http.ResponseWriter, r *http.Request) {

	Metrics.Requests.Add(1)

	err := r.ParseForm()
	if err != nil {
			http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest)
			return
	}

	targets := r.Form["target"]
	from := r.FormValue("from")
	until := r.FormValue("until")
	useCache := r.FormValue("noCache") == ""

	// make sure the cache key doesn't say noCache, because it will never hit
	r.Form.Del("noCache")

	cacheKey := r.Form.Encode()

	if response, ok := queryCache.get(cacheKey); useCache && ok {
		Metrics.RequestCacheHits.Add(1)
		w.Header().Set("Content-Type", "application/json")
		w.Write(response)
		return
	}

	// normalize from and until values
	// BUG(dgryski): doesn't handle timezones the same as graphite-web
	from32 := dateParamToEpoch(from, timeNow().Add(-24*time.Hour).Unix())
	until32 := dateParamToEpoch(until, timeNow().Unix())

	var results []*pb.FetchResponse
	metricMap := make(map[metricRequest][]*pb.FetchResponse)

	for _, target := range targets {

		exp, e, err := parseExpr(target)
		if err != nil || e != "" {
			http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest)
			return
		}

		for _, m := range exp.metrics() {

			mfetch := m
			mfetch.from += from32
			mfetch.until += until32

			if _, ok := metricMap[mfetch]; ok {
				// already fetched this metric for this request
				continue
			}

			var glob pb.GlobResponse
			var haveCacheData bool

			if response, ok := findCache.get(m.metric); useCache && ok {
				Metrics.FindCacheHits.Add(1)
				err := proto.Unmarshal(response, &glob)
				haveCacheData = err == nil
			}

			if !haveCacheData {
				var err error
				Metrics.FindRequests.Add(1)
				glob, err = Zipper.Find(m.metric)
				if err != nil {
					continue
				}
				b, err := proto.Marshal(&glob)
				if err == nil {
					findCache.set(m.metric, b)
				}
			}

			// For each metric returned in the Find response, query Render
			// This is a conscious decision to *not* cache render data
			rch := make(chan *pb.FetchResponse, len(glob.GetMatches()))
			leaves := 0
			for _, m := range glob.GetMatches() {
				if !m.GetIsLeaf() {
					continue
				}
				Metrics.RenderRequests.Add(1)
				leaves++
				Limiter.enter()
				go func(m *pb.GlobMatch, from, until int32) {
					var rptr *pb.FetchResponse
					r, err := Zipper.Render(m.GetPath(), from, until)
					if err == nil {
						rptr = &r
					}
					rch <- rptr
					Limiter.leave()
				}(m, mfetch.from, mfetch.until)
			}

			for i := 0; i < leaves; i++ {
				r := <-rch
				if r != nil {
					metricMap[mfetch] = append(metricMap[mfetch], r)
				}
			}
		}

		exprs := evalExpr(exp, from32, until32, metricMap)
		results = append(results, exprs...)
	}

	var jresults []jsonResponse

	for _, r := range results {
		if r == nil {
			log.Println("skipping nil result")
		}
		datapoints := make([]graphitePoint, 0, len(r.Values))
		t := *r.StartTime
		for i, v := range r.Values {
			if r.IsAbsent[i] {
				v = math.NaN()
			}
			datapoints = append(datapoints, graphitePoint{value: v, t: t})
			t += *r.StepTime
		}
		jresults = append(jresults, jsonResponse{Target: r.GetName(), Datapoints: datapoints})
	}

	jout, err := json.Marshal(jresults)
	if err != nil {
		http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
		return
	}

	queryCache.set(cacheKey, jout)

	w.Header().Set("Content-Type", "application/json")
	w.Write(jout)
}

func lbcheckHandler(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintf(w, "Ok\n")
}

func main() {

	z := flag.String("z", "", "zipper")
	port := flag.Int("p", 8080, "port")
	l := flag.Int("l", 20, "concurrency limit")
	mc := flag.String("mc", "", "comma separated memcached server list")
	cpus := flag.Int("cpus", 0, "number of CPUs to use")

	flag.Parse()

	if p := os.Getenv("PORT"); p != "" {
		*port, _ = strconv.Atoi(p)
	}

	Limiter = make(chan struct{}, *l)

	if *z == "" {
		log.Fatal("no zipper provided")
	}

	if _, err := url.Parse(*z); err != nil {
		log.Fatal("unable to parze zipper:", err)
	}

	log.Println("using zipper", *z)

	if *mc != "" {
		servers := strings.Split(*mc, ",")
		log.Println("using memcache servers:", servers)
		queryCache = &memcachedCache{client: memcache.New(servers...)}
		findCache = &memcachedCache{client: memcache.New(servers...)}
	} else {
		queryCache = &expireCache{cache: make(map[string]cacheElement)}
		go queryCache.(*expireCache).cleaner()

		findCache = &expireCache{cache: make(map[string]cacheElement)}
		go findCache.(*expireCache).cleaner()
	}

	Zipper = zipper(*z)

	if *cpus != 0 {
		log.Println("using GOMAXPROCS", *cpus)
		runtime.GOMAXPROCS(*cpus)
	}

	if host := os.Getenv("GRAPHITEHOST") + ":" + os.Getenv("GRAPHITEPORT"); host != ":" {

		log.Println("Using graphite host", host)

		// register our metrics with graphite
		graphite, err := g2g.NewGraphite(host, 60*time.Second, 10*time.Second)
		if err != nil {
			log.Fatal("unable to connect to to graphite: ", host, ":", err)
		}

		hostname, _ := os.Hostname()
		hostname = strings.Replace(hostname, ".", "_", -1)

		graphite.Register(fmt.Sprintf("carbon.api.%s.requests", hostname), Metrics.Requests)
		graphite.Register(fmt.Sprintf("carbon.api.%s.request_cache_hits", hostname), Metrics.RequestCacheHits)

		graphite.Register(fmt.Sprintf("carbon.api.%s.find_requests", hostname), Metrics.FindRequests)
		graphite.Register(fmt.Sprintf("carbon.api.%s.find_cache_hits", hostname), Metrics.FindCacheHits)

		graphite.Register(fmt.Sprintf("carbon.api.%s.render_requests", hostname), Metrics.RenderRequests)
	}

	http.HandleFunc("/render/", renderHandler)
	http.HandleFunc("/lbcheck", lbcheckHandler)

	log.Println("listening on port", *port)
	log.Fatalln(http.ListenAndServe(":"+strconv.Itoa(*port), nil))
}
<commit_msg>todos++<commit_after>package main

import (
	"encoding/json"
	"errors"
	"expvar"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"math"
	"net/http"
	_ "net/http/pprof"
	"net/url"
	"os"
	"runtime"
	"strconv"
	"strings"
	"time"

	"code.google.com/p/gogoprotobuf/proto"
	pb "github.com/dgryski/carbonzipper/carbonzipperpb"

	"github.com/bradfitz/gomemcache/memcache"
	"github.com/peterbourgon/g2g"
)

type zipper string

var Zipper zipper

var Metrics = struct {
	Requests         *expvar.Int
	RequestCacheHits *expvar.Int

	FindRequests  *expvar.Int
	FindCacheHits *expvar.Int

	RenderRequests *expvar.Int
}{
	Requests:         expvar.NewInt("requests"),
	RequestCacheHits: expvar.NewInt("request_cache_hits"),

	FindRequests:  expvar.NewInt("find_requests"),
	FindCacheHits: expvar.NewInt("find_cache_hits"),

	RenderRequests: expvar.NewInt("render_requests"),
}

var queryCache bytesCache
var findCache bytesCache

var timeFormats = []string{"15:04 20060102", "20060102", "01/02/06"}

// dateParamToEpoch turns a passed string parameter into an epoch we can send to the Zipper
func dateParamToEpoch(s string, d int64) int32 {

	if s == "" {
		// return the default if nothing was passed
		return int32(d)
	}

	// relative timestamp
	if s[0] == '-' {

		offset, err := intervalString(s[1:])
		if err != nil {
			return int32(d)
		}

		return int32(timeNow().Add(-time.Duration(offset) * time.Second).Unix())
	}

	if s == "now" {
		return int32(timeNow().Unix())
	}

	sint, err := strconv.Atoi(s)
	if err == nil && len(s) > 8 {
		return int32(sint) // We got a timestamp so returning it
	}

	if strings.Contains(s, "_") {
		s = strings.Replace(s, "_", " ", 1) // Go can't parse _ in date strings
	}

	for _, format := range timeFormats {
		t, err := time.Parse(format, s)
		if err == nil {
			return int32(t.Unix())
		}
	}
	return int32(d)
}

func intervalString(s string) (int32, error) {

	// TODO(dgryski): add defaultSign param for use if there is no +/- sign provided by string

	var j int

	for j < len(s) && s[j] >= '0' && s[j] <= '9' {
		j++
	}
	offsetStr, unitStr := s[:j], s[j:]

	var units int
	switch unitStr {
	case "s", "sec", "secs", "second", "seconds":
		units = 1
	case "min", "minute", "minutes":
		units = 60
	case "h", "hour", "hours":
		units = 60 * 60
	case "d", "day", "days":
		units = 24 * 60 * 60
	case "w", "week", "weeks":
		units = 7 * 24 * 60 * 60
	case "mon", "month", "months":
		units = 30 * 24 * 60 * 60
	case "y", "year", "years":
		units = 365 * 24 * 60 * 60
	default:
		return 0, errors.New("unknown time units")
	}

	offset, err := strconv.Atoi(offsetStr)
	if err != nil {
		return 0, err
	}

	return int32(offset * units), nil
}

// FIXME(dgryski): extract the http.Get + unproto code into its own function

func (z zipper) Find(metric string) (pb.GlobResponse, error) {

	u, _ := url.Parse(string(z) + "/metrics/find/")

	u.RawQuery = url.Values{
		"query":  []string{metric},
		"format": []string{"protobuf"},
	}.Encode()

	resp, err := http.Get(u.String())
	if err != nil {
		log.Printf("Find: http.Get: %+v\n", err)
		return pb.GlobResponse{}, err
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Printf("Find: ioutil.ReadAll: %+v\n", err)
		return pb.GlobResponse{}, err
	}

	var pbresp pb.GlobResponse

	err = proto.Unmarshal(body, &pbresp)
	if err != nil {
		log.Printf("Find: proto.Unmarshal: %+v\n", err)
		return pb.GlobResponse{}, err
	}

	return pbresp, nil
}

func (z zipper) Render(metric string, from, until int32) (pb.FetchResponse, error) {

	u, _ := url.Parse(string(z) + "/render/")

	u.RawQuery = url.Values{
		"target": []string{metric},
		"format": []string{"protobuf"},
		"from":   []string{strconv.Itoa(int(from))},
		"until":  []string{strconv.Itoa(int(until))},
	}.Encode()

	resp, err := http.Get(u.String())
	if err != nil {
		log.Printf("Render: http.Get: %s: %+v\n", metric, err)
		return pb.FetchResponse{}, err
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Printf("Render: ioutil.ReadAll: %s: %+v\n", metric, err)
		return pb.FetchResponse{}, err
	}

	var pbresp pb.FetchResponse

	err = proto.Unmarshal(body, &pbresp)
	if err != nil {
		log.Printf("Render: proto.Unmarshal: %s: %+v\n", metric, err)
		return pb.FetchResponse{}, err
	}

	return pbresp, nil
}

type limiter chan struct{}

func (l limiter) enter() { l <- struct{}{} }
func (l limiter) leave() { <-l }

var Limiter limiter

// for testing
var timeNow = time.Now

type graphitePoint struct {
	value float64
	t     int32
}

func (g graphitePoint) MarshalJSON() ([]byte, error) {
	// TODO(dgryski): fmt.Sprintf() is slow, use strconv.Append{Float,Int}
	// TODO(dgryski): MarshalJSON call should be on jsonResponse to reduce overhead
	if math.IsNaN(g.value) {
		return []byte(fmt.Sprintf("[null,%d]", g.t)), nil
	}
	return []byte(fmt.Sprintf("[%g,%d]", g.value, g.t)), nil
}

type jsonResponse struct {
	Target     string          `json:"target"`
	Datapoints []graphitePoint `json:"datapoints"`
}

func renderHandler(w http.ResponseWriter, r *http.Request) {

	Metrics.Requests.Add(1)

	err := r.ParseForm()
	if err != nil {
			http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest)
			return
	}

	targets := r.Form["target"]
	from := r.FormValue("from")
	until := r.FormValue("until")
	useCache := r.FormValue("noCache") == ""

	// make sure the cache key doesn't say noCache, because it will never hit
	r.Form.Del("noCache")

	cacheKey := r.Form.Encode()

	if response, ok := queryCache.get(cacheKey); useCache && ok {
		Metrics.RequestCacheHits.Add(1)
		w.Header().Set("Content-Type", "application/json")
		w.Write(response)
		return
	}

	// normalize from and until values
	// BUG(dgryski): doesn't handle timezones the same as graphite-web
	from32 := dateParamToEpoch(from, timeNow().Add(-24*time.Hour).Unix())
	until32 := dateParamToEpoch(until, timeNow().Unix())

	var results []*pb.FetchResponse
	metricMap := make(map[metricRequest][]*pb.FetchResponse)

	for _, target := range targets {

		exp, e, err := parseExpr(target)
		if err != nil || e != "" {
			http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest)
			return
		}

		for _, m := range exp.metrics() {

			mfetch := m
			mfetch.from += from32
			mfetch.until += until32

			if _, ok := metricMap[mfetch]; ok {
				// already fetched this metric for this request
				continue
			}

			var glob pb.GlobResponse
			var haveCacheData bool

			if response, ok := findCache.get(m.metric); useCache && ok {
				Metrics.FindCacheHits.Add(1)
				err := proto.Unmarshal(response, &glob)
				haveCacheData = err == nil
			}

			if !haveCacheData {
				var err error
				Metrics.FindRequests.Add(1)
				glob, err = Zipper.Find(m.metric)
				if err != nil {
					continue
				}
				b, err := proto.Marshal(&glob)
				if err == nil {
					findCache.set(m.metric, b)
				}
			}

			// For each metric returned in the Find response, query Render
			// This is a conscious decision to *not* cache render data
			rch := make(chan *pb.FetchResponse, len(glob.GetMatches()))
			leaves := 0
			for _, m := range glob.GetMatches() {
				if !m.GetIsLeaf() {
					continue
				}
				Metrics.RenderRequests.Add(1)
				leaves++
				Limiter.enter()
				go func(m *pb.GlobMatch, from, until int32) {
					var rptr *pb.FetchResponse
					r, err := Zipper.Render(m.GetPath(), from, until)
					if err == nil {
						rptr = &r
					}
					rch <- rptr
					Limiter.leave()
				}(m, mfetch.from, mfetch.until)
			}

			for i := 0; i < leaves; i++ {
				r := <-rch
				if r != nil {
					metricMap[mfetch] = append(metricMap[mfetch], r)
				}
			}
		}

		exprs := evalExpr(exp, from32, until32, metricMap)
		results = append(results, exprs...)
	}

	var jresults []jsonResponse

	for _, r := range results {
		if r == nil {
			log.Println("skipping nil result")
		}
		datapoints := make([]graphitePoint, 0, len(r.Values))
		t := *r.StartTime
		for i, v := range r.Values {
			if r.IsAbsent[i] {
				v = math.NaN()
			}
			datapoints = append(datapoints, graphitePoint{value: v, t: t})
			t += *r.StepTime
		}
		jresults = append(jresults, jsonResponse{Target: r.GetName(), Datapoints: datapoints})
	}

	jout, err := json.Marshal(jresults)
	if err != nil {
		http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
		return
	}

	queryCache.set(cacheKey, jout)

	w.Header().Set("Content-Type", "application/json")
	w.Write(jout)
}

func lbcheckHandler(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintf(w, "Ok\n")
}

func main() {

	z := flag.String("z", "", "zipper")
	port := flag.Int("p", 8080, "port")
	l := flag.Int("l", 20, "concurrency limit")
	mc := flag.String("mc", "", "comma separated memcached server list")
	cpus := flag.Int("cpus", 0, "number of CPUs to use")

	flag.Parse()

	if p := os.Getenv("PORT"); p != "" {
		*port, _ = strconv.Atoi(p)
	}

	Limiter = make(chan struct{}, *l)

	if *z == "" {
		log.Fatal("no zipper provided")
	}

	if _, err := url.Parse(*z); err != nil {
		log.Fatal("unable to parze zipper:", err)
	}

	log.Println("using zipper", *z)

	if *mc != "" {
		servers := strings.Split(*mc, ",")
		log.Println("using memcache servers:", servers)
		queryCache = &memcachedCache{client: memcache.New(servers...)}
		findCache = &memcachedCache{client: memcache.New(servers...)}
	} else {
		queryCache = &expireCache{cache: make(map[string]cacheElement)}
		go queryCache.(*expireCache).cleaner()

		findCache = &expireCache{cache: make(map[string]cacheElement)}
		go findCache.(*expireCache).cleaner()
	}

	Zipper = zipper(*z)

	if *cpus != 0 {
		log.Println("using GOMAXPROCS", *cpus)
		runtime.GOMAXPROCS(*cpus)
	}

	if host := os.Getenv("GRAPHITEHOST") + ":" + os.Getenv("GRAPHITEPORT"); host != ":" {

		log.Println("Using graphite host", host)

		// register our metrics with graphite
		graphite, err := g2g.NewGraphite(host, 60*time.Second, 10*time.Second)
		if err != nil {
			log.Fatal("unable to connect to to graphite: ", host, ":", err)
		}

		hostname, _ := os.Hostname()
		hostname = strings.Replace(hostname, ".", "_", -1)

		graphite.Register(fmt.Sprintf("carbon.api.%s.requests", hostname), Metrics.Requests)
		graphite.Register(fmt.Sprintf("carbon.api.%s.request_cache_hits", hostname), Metrics.RequestCacheHits)

		graphite.Register(fmt.Sprintf("carbon.api.%s.find_requests", hostname), Metrics.FindRequests)
		graphite.Register(fmt.Sprintf("carbon.api.%s.find_cache_hits", hostname), Metrics.FindCacheHits)

		graphite.Register(fmt.Sprintf("carbon.api.%s.render_requests", hostname), Metrics.RenderRequests)
	}

	http.HandleFunc("/render/", renderHandler)
	http.HandleFunc("/lbcheck", lbcheckHandler)

	log.Println("listening on port", *port)
	log.Fatalln(http.ListenAndServe(":"+strconv.Itoa(*port), nil))
}
<|endoftext|>