<commit_before>package events

import log "github.com/mozilla-services/reaper/reaperlog"

// DatadogStatistics implements EventReporter encapsulates Datadog, sends statistics to Datadog
// uses godspeed, requires dd-agent running
type DatadogStatistics struct {
	Datadog
}

// NewDatadogStatistics returns a new instance of Datadog
func NewDatadogStatistics(c *DatadogConfig) *DatadogStatistics {
	c.Name = "DatadogStatistics"
	return &DatadogStatistics{Datadog{Config: c}}
}

// NewStatistic is a method of EventReporter
// NewStatistic reports a gauge to Datadog
func (e *DatadogStatistics) NewStatistic(name string, value float64, tags []string) error {
	if e.Config.DryRun {
		if log.Extras() {
			log.Info("DryRun: Not reporting %s", name)
		}
		return nil
	}

	log.Info("Reporting statistic %s: %d, tags: %v", name, value, tags)

	g, err := e.godspeed()
	if err != nil {
		return err
	}
	err = g.Gauge(name, value, tags)
	return err
}

// NewCountStatistic is a method of EventReporter
// NewCountStatistic reports an Incr to Datadog
func (e *DatadogStatistics) NewCountStatistic(name string, tags []string) error {
	if e.Config.DryRun {
		if log.Extras() {
			log.Info("DryRun: Not reporting %s", name)
		}
		return nil
	}

	log.Info("Reporting count statistic %s", name)

	g, err := e.godspeed()
	if err != nil {
		return err
	}
	err = g.Incr(name, tags)
	return err
}

// GetConfig is a method of EventReporter
func (e *DatadogStatistics) GetConfig() EventReporterConfig {
	return *e.Config.EventReporterConfig
}

// NewReapableEvent is a method of EventReporter
func (e *DatadogStatistics) NewReapableEvent(r Reapable, tags []string) error {
	return nil
}

// NewBatchReapableEvent is a method of EventReporter
func (e *DatadogStatistics) NewBatchReapableEvent(rs []Reapable, tags []string) error {
	return nil
}

// NewEvent is a method of EventReporter
func (e *DatadogStatistics) NewEvent(string, string, map[string]string, []string) error {
	return nil
}
<commit_msg>put extra stat logging behind Extras log option<commit_after>package events

import log "github.com/mozilla-services/reaper/reaperlog"

// DatadogStatistics implements EventReporter encapsulates Datadog, sends statistics to Datadog
// uses godspeed, requires dd-agent running
type DatadogStatistics struct {
	Datadog
}

// NewDatadogStatistics returns a new instance of Datadog
func NewDatadogStatistics(c *DatadogConfig) *DatadogStatistics {
	c.Name = "DatadogStatistics"
	return &DatadogStatistics{Datadog{Config: c}}
}

// NewStatistic is a method of EventReporter
// NewStatistic reports a gauge to Datadog
func (e *DatadogStatistics) NewStatistic(name string, value float64, tags []string) error {
	if e.Config.DryRun {
		if log.Extras() {
			log.Info("DryRun: Not reporting %s", name)
		}
		return nil
	}
	if log.Extras() {
		log.Info("DatadogStatistics: reporting %s: %f, tags: %v", name, value, tags)
	}
	g, err := e.godspeed()
	if err != nil {
		return err
	}
	err = g.Gauge(name, value, tags)
	return err
}

// NewCountStatistic is a method of EventReporter
// NewCountStatistic reports an Incr to Datadog
func (e *DatadogStatistics) NewCountStatistic(name string, tags []string) error {
	if e.Config.DryRun {
		if log.Extras() {
			log.Info("DryRun: Not reporting %s", name)
		}
		return nil
	}

	log.Info("Reporting count statistic %s", name)

	g, err := e.godspeed()
	if err != nil {
		return err
	}
	err = g.Incr(name, tags)
	return err
}

// GetConfig is a method of EventReporter
func (e *DatadogStatistics) GetConfig() EventReporterConfig {
	return *e.Config.EventReporterConfig
}

// NewReapableEvent is a method of EventReporter
func (e *DatadogStatistics) NewReapableEvent(r Reapable, tags []string) error {
	return nil
}

// NewBatchReapableEvent is a method of EventReporter
func (e *DatadogStatistics) NewBatchReapableEvent(rs []Reapable, tags []string) error {
	return nil
}

// NewEvent is a method of EventReporter
func (e *DatadogStatistics) NewEvent(string, string, map[string]string, []string) error {
	return nil
}
<|endoftext|>


<commit_before>package enaml

type Deployment interface {
	GetDeployment() DeploymentManifest
}

type CloudConfig interface {
	GetManifest() CloudConfigManifest
}

type DeploymentManifest struct {
	Name           string              `yaml:"name"`
	DirectorUUID   string              `yaml:"director_uuid,omitempty"`
	Releases       []Release           `yaml:"releases,omitempty"`
	Stemcells      []Stemcell          `yaml:"stemcells,omitempty"`
	InstanceGroups []*InstanceGroup    `yaml:"instance_groups,omitempty"`
	Networks       []DeploymentNetwork `yaml:"networks,omitempty"`
	ResourcePools  []ResourcePool      `yaml:"resource_pools,omitempty"`
	DiskPools      []DiskPool          `yaml:"disk_pools,omitempty"`
	Compilation    *Compilation        `yaml:"compilation,omitempty"`
	Update         Update              `yaml:"update,omitempty"`
	Jobs           []Job               `yaml:"jobs,omitempty"`
	Properties     Properties          `yaml:"properties,omitempty"`
	CloudProvider  *CloudProvider      `yaml:"cloud_provider,omitempty"`
}

type InstanceGroup struct {
	Name               string        `yaml:"name"`
	ResourcePool       string        `yaml:"resource_pool,omitempty"`
	PersistentDisk     int           `yaml:"persistent_disk,omitempty"`
	PersistentDiskType string        `yaml:"persistent_disk_type,omitempty"`
	Instances          int           `yaml:"instances"`
	VMType             string        `yaml:"vm_type,omitempty"`
	Stemcell           string        `yaml:"stemcell,omitempty"`
	AZs                []string      `yaml:"azs,flow,omitempty"`
	Networks           []Network     `yaml:"networks,flow"`
	Jobs               []InstanceJob `yaml:"jobs"`
}

type InstanceJob struct {
	Name       string      `yaml:"name"`
	Release    string      `yaml:"release"`
	Properties interface{} `yaml:"properties"`
}

type DeploymentNetwork interface{}

type Release struct {
	Name    string `yaml:"name"`
	Version string `yaml:"version,omitempty"`
	URL     string `yaml:"url,omitempty"`
	SHA1    string `yaml:"sha1,omitempty"`
}

func NewVIPNetwork(name string) VIPNetwork {
	return VIPNetwork{
		Name: name,
		Type: "vip",
	}
}

type VIPNetwork struct {
	Name            string          `yaml:"name"`
	Type            string          `yaml:"type"`
	CloudProperties CloudProperties `yaml:"cloud_properties"`
}

type DynamicNetwork struct {
	Name            string          `yaml:"name"`
	Type            string          `yaml:"type"`
	DNS             []string        `yaml:"dns"`
	CloudProperties CloudProperties `yaml:"cloud_properties"`
}

func NewManualNetwork(name string) ManualNetwork {
	return ManualNetwork{
		Name: name,
		Type: "manual",
	}
}

type ManualNetwork struct {
	Name    string   `yaml:"name"`
	Type    string   `yaml:"type"`
	Subnets []Subnet `yaml:"subnets"`
}

func (s *ManualNetwork) AddSubnet(subnet Subnet) (err error) {
	s.Subnets = append(s.Subnets, subnet)
	return
}

func NewSubnet(octet string, azName string) Subnet {
	return Subnet{
		Range:   octet + ".0/24",
		Gateway: octet + ".1",
		AZ:      azName,
	}
}

type Subnet struct {
	Range           string          `yaml:"range,omitempty"`
	Gateway         string          `yaml:"gateway,omitempty"`
	DNS             []string        `yaml:"dns,omitempty"`
	Reserved        []string        `yaml:"reserved,omitempty"`
	Static          []string        `yaml:"static,omitempty"`
	AZ              string          `yaml:"az,omitempty"`
	AZs             []string        `yaml:"azs,omitempty"`
	CloudProperties CloudProperties `yaml:"cloud_properties"`
}

func (s *Subnet) AddDNS(dns string) (err error) {
	s.DNS = append(s.DNS, dns)
	return
}

func (s *Subnet) AddReserved(rsv string) (err error) {
	s.Reserved = append(s.Reserved, rsv)
	return
}

type ResourcePool struct {
	Name            string                 `yaml:"name"`
	Network         string                 `yaml:"network"`
	Size            int                    `yaml:"size,omitempty"`
	Stemcell        Stemcell               `yaml:"stemcell"`
	CloudProperties CloudProperties        `yaml:"cloud_properties"`
	Env             map[string]interface{} `yaml:"env,omitempty"`
}

type Stemcell struct {
	Alias   string `yaml:"alias,omitempty"`
	OS      string `yaml:"os,omitempty"`
	Name    string `yaml:"name,omitempty"`
	Version string `yaml:"version,omitempty"`
	URL     string `yaml:"url,omitempty"`
	SHA1    string `yaml:"sha1,omitempty"`
}

type DiskType DiskPool
type DiskPool struct {
	Name            string          `yaml:"name"`
	DiskSize        int             `yaml:"disk_size"`
	CloudProperties CloudProperties `yaml:"cloud_properties"`
}

type Compilation struct {
	Workers             int             `yaml:"workers"`
	Network             string          `yaml:"network"`
	ReuseCompilationVMs bool            `yaml:"reuse_compilation_vms"`
	CloudProperties     CloudProperties `yaml:"cloud_properties,omitempty"`
	VMType              string          `yaml:"vm_type,omitempty"`
	AZ                  string          `yaml:"az,omitempty"`
}

type Update struct {
	Canaries        int    `yaml:"canaries"`
	MaxInFlight     int    `yaml:"max_in_flight"`
	CanaryWatchTime string `yaml:"canary_watch_time"`
	UpdateWatchTime string `yaml:"update_watch_time"`
	Serial          bool   `yaml:"serial,omitempty"`
}

type Job struct {
	Name           string                 `yaml:"name"`
	Templates      []Template             `yaml:"templates,flow"`
	Lifecycle      string                 `yaml:"lifeycle,omitempty"`
	PersistentDisk string                 `yaml:"persistent_disk,omitempty"`
	Properties     Properties             `yaml:"properties,omitempty"`
	ResourcePool   string                 `yaml:"resource_pool"`
	Update         map[string]interface{} `yaml:"update,omitempty"`
	Instances      int                    `yaml:"instances"`
	Networks       []Network              `yaml:"networks"`
}

type Network struct {
	Name      string        `yaml:"name"`
	StaticIPs []string      `yaml:"static_ips,omitempty"`
	Default   []interface{} `yaml:"default,omitempty"`
}

type Template struct {
	Name    string `yaml:"name"`
	Release string `yaml:"release"`
}

type CloudProvider struct {
	Template   Template                `yaml:"template,flow"`
	MBus       string                  `yaml:"mbus"`
	Properties CloudProviderProperties `yaml:"properties"`
}
type CloudProviderProperties interface{}
type Properties map[string]interface{}
type CloudProperties interface{}
<commit_msg>[#118197197] adding omitempty for a optional cloudconfig on vips<commit_after>package enaml

type Deployment interface {
	GetDeployment() DeploymentManifest
}

type CloudConfig interface {
	GetManifest() CloudConfigManifest
}

type DeploymentManifest struct {
	Name           string              `yaml:"name"`
	DirectorUUID   string              `yaml:"director_uuid,omitempty"`
	Releases       []Release           `yaml:"releases,omitempty"`
	Stemcells      []Stemcell          `yaml:"stemcells,omitempty"`
	InstanceGroups []*InstanceGroup    `yaml:"instance_groups,omitempty"`
	Networks       []DeploymentNetwork `yaml:"networks,omitempty"`
	ResourcePools  []ResourcePool      `yaml:"resource_pools,omitempty"`
	DiskPools      []DiskPool          `yaml:"disk_pools,omitempty"`
	Compilation    *Compilation        `yaml:"compilation,omitempty"`
	Update         Update              `yaml:"update,omitempty"`
	Jobs           []Job               `yaml:"jobs,omitempty"`
	Properties     Properties          `yaml:"properties,omitempty"`
	CloudProvider  *CloudProvider      `yaml:"cloud_provider,omitempty"`
}

type InstanceGroup struct {
	Name               string        `yaml:"name"`
	ResourcePool       string        `yaml:"resource_pool,omitempty"`
	PersistentDisk     int           `yaml:"persistent_disk,omitempty"`
	PersistentDiskType string        `yaml:"persistent_disk_type,omitempty"`
	Instances          int           `yaml:"instances"`
	VMType             string        `yaml:"vm_type,omitempty"`
	Stemcell           string        `yaml:"stemcell,omitempty"`
	AZs                []string      `yaml:"azs,flow,omitempty"`
	Networks           []Network     `yaml:"networks,flow"`
	Jobs               []InstanceJob `yaml:"jobs"`
}

type InstanceJob struct {
	Name       string      `yaml:"name"`
	Release    string      `yaml:"release"`
	Properties interface{} `yaml:"properties"`
}

type DeploymentNetwork interface{}

type Release struct {
	Name    string `yaml:"name"`
	Version string `yaml:"version,omitempty"`
	URL     string `yaml:"url,omitempty"`
	SHA1    string `yaml:"sha1,omitempty"`
}

func NewVIPNetwork(name string) VIPNetwork {
	return VIPNetwork{
		Name: name,
		Type: "vip",
	}
}

type VIPNetwork struct {
	Name            string          `yaml:"name"`
	Type            string          `yaml:"type"`
	CloudProperties CloudProperties `yaml:"cloud_properties,omitempty"`
}

type DynamicNetwork struct {
	Name            string          `yaml:"name"`
	Type            string          `yaml:"type"`
	DNS             []string        `yaml:"dns"`
	CloudProperties CloudProperties `yaml:"cloud_properties"`
}

func NewManualNetwork(name string) ManualNetwork {
	return ManualNetwork{
		Name: name,
		Type: "manual",
	}
}

type ManualNetwork struct {
	Name    string   `yaml:"name"`
	Type    string   `yaml:"type"`
	Subnets []Subnet `yaml:"subnets"`
}

func (s *ManualNetwork) AddSubnet(subnet Subnet) (err error) {
	s.Subnets = append(s.Subnets, subnet)
	return
}

func NewSubnet(octet string, azName string) Subnet {
	return Subnet{
		Range:   octet + ".0/24",
		Gateway: octet + ".1",
		AZ:      azName,
	}
}

type Subnet struct {
	Range           string          `yaml:"range,omitempty"`
	Gateway         string          `yaml:"gateway,omitempty"`
	DNS             []string        `yaml:"dns,omitempty"`
	Reserved        []string        `yaml:"reserved,omitempty"`
	Static          []string        `yaml:"static,omitempty"`
	AZ              string          `yaml:"az,omitempty"`
	AZs             []string        `yaml:"azs,omitempty"`
	CloudProperties CloudProperties `yaml:"cloud_properties"`
}

func (s *Subnet) AddDNS(dns string) (err error) {
	s.DNS = append(s.DNS, dns)
	return
}

func (s *Subnet) AddReserved(rsv string) (err error) {
	s.Reserved = append(s.Reserved, rsv)
	return
}

type ResourcePool struct {
	Name            string                 `yaml:"name"`
	Network         string                 `yaml:"network"`
	Size            int                    `yaml:"size,omitempty"`
	Stemcell        Stemcell               `yaml:"stemcell"`
	CloudProperties CloudProperties        `yaml:"cloud_properties"`
	Env             map[string]interface{} `yaml:"env,omitempty"`
}

type Stemcell struct {
	Alias   string `yaml:"alias,omitempty"`
	OS      string `yaml:"os,omitempty"`
	Name    string `yaml:"name,omitempty"`
	Version string `yaml:"version,omitempty"`
	URL     string `yaml:"url,omitempty"`
	SHA1    string `yaml:"sha1,omitempty"`
}

type DiskType DiskPool
type DiskPool struct {
	Name            string          `yaml:"name"`
	DiskSize        int             `yaml:"disk_size"`
	CloudProperties CloudProperties `yaml:"cloud_properties"`
}

type Compilation struct {
	Workers             int             `yaml:"workers"`
	Network             string          `yaml:"network"`
	ReuseCompilationVMs bool            `yaml:"reuse_compilation_vms"`
	CloudProperties     CloudProperties `yaml:"cloud_properties,omitempty"`
	VMType              string          `yaml:"vm_type,omitempty"`
	AZ                  string          `yaml:"az,omitempty"`
}

type Update struct {
	Canaries        int    `yaml:"canaries"`
	MaxInFlight     int    `yaml:"max_in_flight"`
	CanaryWatchTime string `yaml:"canary_watch_time"`
	UpdateWatchTime string `yaml:"update_watch_time"`
	Serial          bool   `yaml:"serial,omitempty"`
}

type Job struct {
	Name           string                 `yaml:"name"`
	Templates      []Template             `yaml:"templates,flow"`
	Lifecycle      string                 `yaml:"lifeycle,omitempty"`
	PersistentDisk string                 `yaml:"persistent_disk,omitempty"`
	Properties     Properties             `yaml:"properties,omitempty"`
	ResourcePool   string                 `yaml:"resource_pool"`
	Update         map[string]interface{} `yaml:"update,omitempty"`
	Instances      int                    `yaml:"instances"`
	Networks       []Network              `yaml:"networks"`
}

type Network struct {
	Name      string        `yaml:"name"`
	StaticIPs []string      `yaml:"static_ips,omitempty"`
	Default   []interface{} `yaml:"default,omitempty"`
}

type Template struct {
	Name    string `yaml:"name"`
	Release string `yaml:"release"`
}

type CloudProvider struct {
	Template   Template                `yaml:"template,flow"`
	MBus       string                  `yaml:"mbus"`
	Properties CloudProviderProperties `yaml:"properties"`
}
type CloudProviderProperties interface{}
type Properties map[string]interface{}
type CloudProperties interface{}
<|endoftext|>


<commit_before>package eval

import (
	"context"
	"fmt"
	"io"
	"os"
	"os/exec"
	"os/signal"
	"strings"
	"syscall"

	"github.com/pkg/errors"

	"github.com/elpinal/coco3/ast"
	"github.com/elpinal/coco3/token"
)

var ErrInterrupted = errors.New("signal caught: interrupt")

func New(in io.Reader, out, err io.Writer) *Evaluator {
	return &Evaluator{
		in:     in,
		out:    out,
		err:    err,
		ExitCh: make(chan int, 1),
	}
}

func (e *Evaluator) Eval(stmts []ast.Stmt) error {
	for _, stmt := range stmts {
		err := e.eval(stmt)
		if err != nil {
			return err
		}
	}
	return nil
}

type Evaluator struct {
	in  io.Reader
	out io.Writer
	err io.Writer

	closeAfterStart []io.Closer

	ExitCh chan int
}

func (e *Evaluator) eval(stmt ast.Stmt) error {
	switch x := stmt.(type) {
	case *ast.PipeStmt:
		commands := make([][]string, 0, len(x.Args))
		for _, c := range x.Args {
			args := make([]string, 0, len(c.Args))
			for _, arg := range c.Args {
				s, err := e.evalExpr(arg)
				if err != nil {
					return err
				}
				args = append(args, s...)
			}
			if len(args) == 0 {
				return errors.New("no command to execute")
			}
			commands = append(commands, args)
		}
		return e.execPipe(commands)
	case *ast.ExecStmt:
		args := make([]string, 0, len(x.Args))
		for _, arg := range x.Args {
			s, err := e.evalExpr(arg)
			if err != nil {
				return err
			}
			args = append(args, s...)
		}
		if len(args) == 0 {
			return nil
		}
		return e.execCmd(args[0], args[1:])
	}
	return fmt.Errorf("unexpected type: %T", stmt)
}

func (e *Evaluator) evalExpr(expr ast.Expr) ([]string, error) {
	switch x := expr.(type) {
	case *ast.Ident:
		return []string{strings.Replace(x.Name, "~", os.Getenv("HOME"), -1)}, nil
	case *ast.BasicLit:
		s := strings.TrimPrefix(x.Value, "'")
		s = strings.TrimSuffix(s, "'")
		s = strings.Replace(s, "''", "'", -1)
		return []string{s}, nil
	case *ast.ParenExpr:
		var list []string
		for _, expr := range x.Exprs {
			s, err := e.evalExpr(expr)
			if err != nil {
				return nil, err
			}
			list = append(list, s...)
		}
		return list, nil
	case *ast.UnaryExpr:
		s, err := e.evalExpr(x.X)
		if err != nil {
			return nil, err
		}
		if len(s) == 0 {
			return nil, fmt.Errorf("cannot redirect")
		}
		if len(s) > 1 {
			return nil, fmt.Errorf("cannot redirect to multi-word filename")
		}
		switch x.Op {
		case token.REDIRIN:
			f, err := os.Open(s[0])
			if err != nil {
				return nil, err
			}
			e.in = f
			e.closeAfterStart = append(e.closeAfterStart, f)
		case token.REDIROUT:
			f, err := os.Create(s[0])
			if err != nil {
				return nil, err
			}
			e.out = f
			e.closeAfterStart = append(e.closeAfterStart, f)
		}
		return nil, nil
	case nil:
		return nil, nil
	}
	return nil, fmt.Errorf("unexpected type: %T", expr)
}

func (e *Evaluator) execCmd(name string, args []string) error {
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()
	cmd := e.CommandContext(ctx, name, args...)
	cmd.SetStdin(e.in)
	cmd.SetStdout(e.out)
	cmd.SetStderr(e.err)
	return e.run(cmd)
}

func wait(fn func() error) <-chan error {
	c := make(chan error)
	go func() {
		c <- fn()
	}()
	return c
}

func (e *Evaluator) execPipe(commands [][]string) error {
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()
	cmds, err := e.makePipe(ctx, commands)
	if err != nil {
		return errors.Wrap(err, "constructing pipe")
	}
	return e.run(cmds)
}

type runner interface {
	Run() error
}

func (e *Evaluator) run(cmd runner) error {
	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt)
	defer func() {
		for _, closer := range e.closeAfterStart {
			closer.Close()
		}
	}()
	select {
	case err := <-wait(cmd.Run):
		return err
	}
}

func (e *Evaluator) makePipe(ctx context.Context, commands [][]string) (pipeCmd, error) {
	cmds := make([]Cmd, len(commands))
	for i, c := range commands {
		name := c[0]
		args := c[1:]
		cmds[i] = e.CommandContext(ctx, name, args...)
		if i > 0 {
			pipe, err := cmds[i-1].StdoutPipe()
			if err != nil {
				return pipeCmd{}, err
			}
			cmds[i].SetStdin(pipe)
		}
		cmds[i].SetStderr(e.err)
	}
	cmds[0].SetStdin(e.in)
	cmds[len(cmds)-1].SetStdout(e.out)
	return pipeCmd{cmds: cmds, errStream: e.err}, nil
}

type pipeCmd struct {
	cmds      []Cmd
	errStream io.Writer
}

func (p pipeCmd) errorf(format string, err error) {
	fmt.Fprintf(p.errStream, format, err)
}

func (p pipeCmd) start() error {
	for _, cmd := range p {
		if err := cmd.Start(); err != nil {
			return err
		}
	}
	return nil
}

func isInterrupt(err error) bool {
	ee, ok := err.(*exec.ExitError)
	if !ok {
		return false
	}
	status := ee.ProcessState.Sys().(syscall.WaitStatus)
	return status.Signal() == syscall.SIGINT
}

func (p pipeCmd) wait() error {
	for _, cmd := range p {
		if err := cmd.Wait(); err != nil {
			return err
		}
	}
	return nil
}

func (p pipeCmd) Run() error {
	err := p.start()
	if err != nil {
		return err
	}
	return p.wait()
}
<commit_msg>Don't return until all child processes is done even if errors occur<commit_after>package eval

import (
	"context"
	"fmt"
	"io"
	"os"
	"os/exec"
	"os/signal"
	"strings"
	"syscall"

	"github.com/pkg/errors"

	"github.com/elpinal/coco3/ast"
	"github.com/elpinal/coco3/token"
)

var ErrInterrupted = errors.New("signal caught: interrupt")

func New(in io.Reader, out, err io.Writer) *Evaluator {
	return &Evaluator{
		in:     in,
		out:    out,
		err:    err,
		ExitCh: make(chan int, 1),
	}
}

func (e *Evaluator) Eval(stmts []ast.Stmt) error {
	for _, stmt := range stmts {
		err := e.eval(stmt)
		if err != nil {
			return err
		}
	}
	return nil
}

type Evaluator struct {
	in  io.Reader
	out io.Writer
	err io.Writer

	closeAfterStart []io.Closer

	ExitCh chan int
}

func (e *Evaluator) eval(stmt ast.Stmt) error {
	switch x := stmt.(type) {
	case *ast.PipeStmt:
		commands := make([][]string, 0, len(x.Args))
		for _, c := range x.Args {
			args := make([]string, 0, len(c.Args))
			for _, arg := range c.Args {
				s, err := e.evalExpr(arg)
				if err != nil {
					return err
				}
				args = append(args, s...)
			}
			if len(args) == 0 {
				return errors.New("no command to execute")
			}
			commands = append(commands, args)
		}
		return e.execPipe(commands)
	case *ast.ExecStmt:
		args := make([]string, 0, len(x.Args))
		for _, arg := range x.Args {
			s, err := e.evalExpr(arg)
			if err != nil {
				return err
			}
			args = append(args, s...)
		}
		if len(args) == 0 {
			return nil
		}
		return e.execCmd(args[0], args[1:])
	}
	return fmt.Errorf("unexpected type: %T", stmt)
}

func (e *Evaluator) evalExpr(expr ast.Expr) ([]string, error) {
	switch x := expr.(type) {
	case *ast.Ident:
		return []string{strings.Replace(x.Name, "~", os.Getenv("HOME"), -1)}, nil
	case *ast.BasicLit:
		s := strings.TrimPrefix(x.Value, "'")
		s = strings.TrimSuffix(s, "'")
		s = strings.Replace(s, "''", "'", -1)
		return []string{s}, nil
	case *ast.ParenExpr:
		var list []string
		for _, expr := range x.Exprs {
			s, err := e.evalExpr(expr)
			if err != nil {
				return nil, err
			}
			list = append(list, s...)
		}
		return list, nil
	case *ast.UnaryExpr:
		s, err := e.evalExpr(x.X)
		if err != nil {
			return nil, err
		}
		if len(s) == 0 {
			return nil, fmt.Errorf("cannot redirect")
		}
		if len(s) > 1 {
			return nil, fmt.Errorf("cannot redirect to multi-word filename")
		}
		switch x.Op {
		case token.REDIRIN:
			f, err := os.Open(s[0])
			if err != nil {
				return nil, err
			}
			e.in = f
			e.closeAfterStart = append(e.closeAfterStart, f)
		case token.REDIROUT:
			f, err := os.Create(s[0])
			if err != nil {
				return nil, err
			}
			e.out = f
			e.closeAfterStart = append(e.closeAfterStart, f)
		}
		return nil, nil
	case nil:
		return nil, nil
	}
	return nil, fmt.Errorf("unexpected type: %T", expr)
}

func (e *Evaluator) execCmd(name string, args []string) error {
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()
	cmd := e.CommandContext(ctx, name, args...)
	cmd.SetStdin(e.in)
	cmd.SetStdout(e.out)
	cmd.SetStderr(e.err)
	return e.run(cmd)
}

func wait(fn func() error) <-chan error {
	c := make(chan error)
	go func() {
		c <- fn()
	}()
	return c
}

func (e *Evaluator) execPipe(commands [][]string) error {
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()
	cmds, err := e.makePipe(ctx, commands)
	if err != nil {
		return errors.Wrap(err, "constructing pipe")
	}
	return e.run(cmds)
}

type runner interface {
	Run() error
}

func (e *Evaluator) run(cmd runner) error {
	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt)
	defer func() {
		for _, closer := range e.closeAfterStart {
			closer.Close()
		}
	}()
	select {
	case err := <-wait(cmd.Run):
		return err
	}
}

func (e *Evaluator) makePipe(ctx context.Context, commands [][]string) (pipeCmd, error) {
	cmds := make([]Cmd, len(commands))
	for i, c := range commands {
		name := c[0]
		args := c[1:]
		cmds[i] = e.CommandContext(ctx, name, args...)
		if i > 0 {
			pipe, err := cmds[i-1].StdoutPipe()
			if err != nil {
				return pipeCmd{}, err
			}
			cmds[i].SetStdin(pipe)
		}
		cmds[i].SetStderr(e.err)
	}
	cmds[0].SetStdin(e.in)
	cmds[len(cmds)-1].SetStdout(e.out)
	return pipeCmd{cmds: cmds, errStream: e.err}, nil
}

type pipeCmd struct {
	cmds      []Cmd
	errStream io.Writer
}

func (p pipeCmd) errorf(format string, err error) {
	fmt.Fprintf(p.errStream, format, err)
}

func (p pipeCmd) start() error {
	for _, cmd := range p.cmds {
		if err := cmd.Start(); err != nil {
			p.errorf("%v\n", err)
		}
	}
	return nil
}

func isInterrupt(err error) bool {
	ee, ok := err.(*exec.ExitError)
	if !ok {
		return false
	}
	status := ee.ProcessState.Sys().(syscall.WaitStatus)
	return status.Signal() == syscall.SIGINT
}

func (p pipeCmd) wait() error {
	for _, cmd := range p.cmds {
		err := cmd.Wait()
		if err == nil || isInterrupt(err) {
			continue
		}
		p.errorf("%v\n", err)
	}
	return nil
}

func (p pipeCmd) Run() error {
	err := p.start()
	if err != nil {
		return err
	}
	return p.wait()
}
<|endoftext|>


<commit_before>// Copyright © 2015 The Things Network
// Use of this source code is governed by the MIT license that can be found in the LICENSE file.

package components

import (
	"bytes"
	"encoding/binary"
	"fmt"

	"github.com/brocaar/lorawan"
)

type entryReadWriter struct {
	err  error
	data *bytes.Buffer
}

func NewEntryReadWriter(buf []byte) *entryReadWriter {
	return &entryReadWriter{
		err:  nil,
		data: bytes.NewBuffer(buf),
	}
}

func (w *entryReadWriter) Write(data interface{}) {
	var raw []byte
	switch data.(type) {
	case []byte:
		raw = data.([]byte)
	case lorawan.AES128Key:
		data := data.(lorawan.AES128Key)
		raw = data[:]
	case lorawan.EUI64:
		data := data.(lorawan.EUI64)
		raw = data[:]
	case lorawan.DevAddr:
		data := data.(lorawan.DevAddr)
		raw = data[:]
	case string:
		raw = []byte(data.(string))
	default:
		panic(fmt.Errorf("Unreckognized data type: %v", data))
	}
	w.DirectWrite(uint16(len(raw)))
	w.DirectWrite(raw)
}

func (w *entryReadWriter) DirectWrite(data interface{}) {
	if w.err != nil {
		return
	}
	w.err = binary.Write(w.data, binary.BigEndian, data)
}

func (w *entryReadWriter) Read(to func(data []byte)) {
	if w.err != nil {
		return
	}
	lenTo := new(uint16)
	if w.err = binary.Read(w.data, binary.BigEndian, lenTo); w.err != nil {
		return
	}
	to(w.data.Next(int(*lenTo)))
}

func (w entryReadWriter) Bytes() ([]byte, error) {
	if w.err != nil {
		return nil, w.err
	}
	return w.data.Bytes(), nil
}

func (w entryReadWriter) Err() error {
	return w.err
}
<commit_msg>[in memory storage] Fix entry read/writer initialisation buffer<commit_after>// Copyright © 2015 The Things Network
// Use of this source code is governed by the MIT license that can be found in the LICENSE file.

package components

import (
	"bytes"
	"encoding/binary"
	"fmt"

	"github.com/brocaar/lorawan"
)

type entryReadWriter struct {
	err  error
	data *bytes.Buffer
}

func NewEntryReadWriter(buf []byte) *entryReadWriter {
	return &entryReadWriter{
		err:  nil,
		data: new(bytes.Buffer),
	}
}

func (w *entryReadWriter) Write(data interface{}) {
	var raw []byte
	switch data.(type) {
	case []byte:
		raw = data.([]byte)
	case lorawan.AES128Key:
		data := data.(lorawan.AES128Key)
		raw = data[:]
	case lorawan.EUI64:
		data := data.(lorawan.EUI64)
		raw = data[:]
	case lorawan.DevAddr:
		data := data.(lorawan.DevAddr)
		raw = data[:]
	case string:
		raw = []byte(data.(string))
	default:
		panic(fmt.Errorf("Unreckognized data type: %v", data))
	}
	w.DirectWrite(uint16(len(raw)))
	w.DirectWrite(raw)
}

func (w *entryReadWriter) DirectWrite(data interface{}) {
	if w.err != nil {
		return
	}
	in := w.data.Next(w.data.Len())
	binary.Write(w.data, binary.BigEndian, in)
	w.err = binary.Write(w.data, binary.BigEndian, data)
	fmt.Println(w.data.Bytes())
}

func (w *entryReadWriter) Read(to func(data []byte)) {
	if w.err != nil {
		return
	}

	lenTo := new(uint16)
	if w.err = binary.Read(w.data, binary.BigEndian, lenTo); w.err != nil {
		return
	}
	to(w.data.Next(int(*lenTo)))
}

func (w entryReadWriter) Bytes() ([]byte, error) {
	if w.err != nil {
		return nil, w.err
	}
	return w.data.Bytes(), nil
}

func (w entryReadWriter) Err() error {
	return w.err
}
<|endoftext|>


<commit_before>package cli

import (
	"io/ioutil"
	"os"
	"time"

	"github.com/jinzhu/now"
)

// Time wrap time.Time
type Time time.Time

func init() {
	now.TimeFormats = append(now.TimeFormats, time.ANSIC)
	now.TimeFormats = append(now.TimeFormats, time.UnixDate)
	now.TimeFormats = append(now.TimeFormats, time.RubyDate)
	now.TimeFormats = append(now.TimeFormats, time.RFC822)
	now.TimeFormats = append(now.TimeFormats, time.RFC822Z)
	now.TimeFormats = append(now.TimeFormats, time.RFC850)
	now.TimeFormats = append(now.TimeFormats, time.RFC1123)
	now.TimeFormats = append(now.TimeFormats, time.RFC1123Z)
	now.TimeFormats = append(now.TimeFormats, time.RFC3339)
	now.TimeFormats = append(now.TimeFormats, time.RFC3339Nano)
	now.TimeFormats = append(now.TimeFormats, time.Kitchen)
	now.TimeFormats = append(now.TimeFormats, time.Stamp)
	now.TimeFormats = append(now.TimeFormats, time.StampMilli)
	now.TimeFormats = append(now.TimeFormats, time.StampMicro)
	now.TimeFormats = append(now.TimeFormats, time.StampNano)
}

func (t *Time) Decode(s string) error {
	now.TimeFormats
	v, err := now.Parse(s)
	if err != nil {
		return err
	}
	*t = Time(v)
	return nil
}

func (t Time) Encode() string {
	return time.Time(t).Format(time.RFC3339Nano)
}

// Duration wrap time.Duration
type Duration time.Duration

func (d *Duration) Decode(s string) error {
	v, err := time.ParseDuration(s)
	if err != nil {
		return err
	}
	*d = Duration(v)
}

func (d Duration) Encode() string {
	return time.Duration(d).String()
}

// File reads data from file or stdin(if filename is empty)
type File struct {
	filename string
	data     []byte
}

func (rf File) Data() []byte {
	return rf.data
}

func (rf File) String() string {
	return string(rf.data)
}

func (rf *File) Decode(s string) error {
	var (
		data []byte
		err  error
	)
	if s == "" {
		data, err = ioutil.ReadAll(os.Stdin)
	} else {
		data, err = ioutil.ReadFile(s)
	}
	if err != nil {
		return err
	}
	rf.data = data
	rf.filename = s
	return nil
}

func (rf File) Encode() string {
	return rf.filename
}
<commit_msg>fix syntax error and add PidFile<commit_after>package cli

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/jinzhu/now"
)

// Time wrap time.Time
type Time time.Time

func init() {
	now.TimeFormats = append(now.TimeFormats, time.ANSIC)
	now.TimeFormats = append(now.TimeFormats, time.UnixDate)
	now.TimeFormats = append(now.TimeFormats, time.RubyDate)
	now.TimeFormats = append(now.TimeFormats, time.RFC822)
	now.TimeFormats = append(now.TimeFormats, time.RFC822Z)
	now.TimeFormats = append(now.TimeFormats, time.RFC850)
	now.TimeFormats = append(now.TimeFormats, time.RFC1123)
	now.TimeFormats = append(now.TimeFormats, time.RFC1123Z)
	now.TimeFormats = append(now.TimeFormats, time.RFC3339)
	now.TimeFormats = append(now.TimeFormats, time.RFC3339Nano)
	now.TimeFormats = append(now.TimeFormats, time.Kitchen)
	now.TimeFormats = append(now.TimeFormats, time.Stamp)
	now.TimeFormats = append(now.TimeFormats, time.StampMilli)
	now.TimeFormats = append(now.TimeFormats, time.StampMicro)
	now.TimeFormats = append(now.TimeFormats, time.StampNano)
}

func (t *Time) Decode(s string) error {
	v, err := now.Parse(s)
	if err != nil {
		return err
	}
	*t = Time(v)
	return nil
}

func (t Time) Encode() string {
	return time.Time(t).Format(time.RFC3339Nano)
}

// Duration wrap time.Duration
type Duration time.Duration

func (d *Duration) Decode(s string) error {
	v, err := time.ParseDuration(s)
	if err != nil {
		return err
	}
	*d = Duration(v)
	return nil
}

func (d Duration) Encode() string {
	return time.Duration(d).String()
}

// File reads data from file or stdin(if filename is empty)
type File struct {
	filename string
	data     []byte
}

func (rf File) Data() []byte {
	return rf.data
}

func (rf File) String() string {
	return string(rf.data)
}

func (rf *File) Decode(s string) error {
	var (
		data []byte
		err  error
	)
	if s == "" {
		data, err = ioutil.ReadAll(os.Stdin)
	} else {
		data, err = ioutil.ReadFile(s)
	}
	if err != nil {
		return err
	}
	rf.data = data
	rf.filename = s
	return nil
}

func (rf File) Encode() string {
	return rf.filename
}

// PidFile
type PidFile struct {
	filename string
}

func (pid PidFile) String() string {
	return pid.filename
}

func (pid *PidFile) Decode(s string) error {
	pid.filename = s
	dir, _ := filepath.Split(pid.filename)
	if err := os.MkdirAll(dir, 0755); err != nil {
		return err
	}
	if content, err := ioutil.ReadFile(pid.filename); err == nil {
		pidStr := strings.TrimSpace(string(content))
		if pidStr == "" {
			return nil
		}
		if _, err := os.Stat(filepath.Join("/proc", pidStr)); err == nil {
			return fmt.Errorf("pid file found, ensoure %s is not running", os.Args[0])
		}
	}
	if err := ioutil.WriteFile(pid.filename, []byte(fmt.Sprintf("%d", os.Getpid())), 0644); err != nil {
		return err
	}
	return nil
}

func (pid PidFile) Remove() error {
	if pid.filename != "" {
		return os.Remove(pid.filename)
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	. "github.com/smartystreets/goconvey/convey"
	"testing"
)

func TestFileSHA1(t *testing.T) {
	filepath := "test/gopher.jpg"
	Convey("SHA1 of file", t, func() {
		s, err := getFileSHA1(filepath)
		So(err, ShouldBeNil)
		So(s, ShouldEqual, "070b45ae488fb1967aaf618561a7d6ba4d28a1c9")
	})
}

func TestFileID(t *testing.T) {
	f := File{}
	f.hash = "070b45ae488fb1967aaf618561a7d6ba4d28a1c9"
	f.size = 12345
	f.height = 1080
	f.width = 1920
	f.filetype = "png"
	Convey("File ID", t, func() {
		Convey("Generation", func() {
			gotFileID := f.String()
			expectedID := "070b45ae488fb1967aaf618561a7d6ba4d28a1c9-12345-1920-1080-png"
			So(gotFileID, ShouldEqual, expectedID)
		})
		Convey("Parsing", func() {
			fid := "070b45ae488fb1967aaf618561a7d6ba4d28a1c9-12345-1920-1080-png"
			parsed, err := FileFromID(fid)
			So(err, ShouldBeNil)
			So(parsed.hash, ShouldEqual, f.hash)
			So(parsed.size, ShouldEqual, f.size)
			So(parsed.width, ShouldEqual, f.width)
			So(parsed.height, ShouldEqual, f.height)
			So(parsed.filetype, ShouldEqual, f.filetype)
			Convey("Error handling", func() {
				examples := []string{
					"070b45ae488fb1967aaf618561a7d6ba4d28a1c9-?-1920-1080-png",
					"kek-12345-1920-1080a-png",
					"070b45ae488fb1967aaf618561a7d6ba4d28a1c9-12345-pek-1080-png",
					"070b45ae488fb1967aaf618561a7d6ba4d28a1c9-12345-1223-2f-png",
				}
				for _, example := range examples {
					_, err := FileFromID(example)
					So(err, ShouldNotBeNil)
				}
			})
		})
	})
}
<commit_msg>added test for Path<commit_after>package main

import (
	"testing"

	. "github.com/smartystreets/goconvey/convey"
)

func TestFileSHA1(t *testing.T) {
	filepath := "test/gopher.jpg"
	Convey("SHA1 of file", t, func() {
		s, err := getFileSHA1(filepath)
		So(err, ShouldBeNil)
		So(s, ShouldEqual, "070b45ae488fb1967aaf618561a7d6ba4d28a1c9")
	})
}

func TestFileID(t *testing.T) {
	f := File{}
	f.hash = "070b45ae488fb1967aaf618561a7d6ba4d28a1c9"
	f.size = 12345
	f.height = 1080
	f.width = 1920
	f.filetype = "png"
	Convey("File ID", t, func() {
		Convey("Generation", func() {
			gotFileID := f.String()
			expectedID := "070b45ae488fb1967aaf618561a7d6ba4d28a1c9-12345-1920-1080-png"
			So(gotFileID, ShouldEqual, expectedID)
		})
		Convey("Path", func() {
			expected := "07/070b45ae488fb1967aaf618561a7d6ba4d28a1c9-12345-1920-1080-png"
			actual := f.Path()
			So(expected, ShouldEqual, actual)
		})
		Convey("Parsing", func() {
			fid := "070b45ae488fb1967aaf618561a7d6ba4d28a1c9-12345-1920-1080-png"
			parsed, err := FileFromID(fid)
			So(err, ShouldBeNil)
			So(parsed.hash, ShouldEqual, f.hash)
			So(parsed.size, ShouldEqual, f.size)
			So(parsed.width, ShouldEqual, f.width)
			So(parsed.height, ShouldEqual, f.height)
			So(parsed.filetype, ShouldEqual, f.filetype)
			Convey("Error handling", func() {
				examples := []string{
					"070b45ae488fb1967aaf618561a7d6ba4d28a1c9-?-1920-1080-png",
					"kek-12345-1920-1080a-png",
					"070b45ae488fb1967aaf618561a7d6ba4d28a1c9-12345-pek-1080-png",
					"070b45ae488fb1967aaf618561a7d6ba4d28a1c9-12345-1223-2f-png",
				}
				for _, example := range examples {
					_, err := FileFromID(example)
					So(err, ShouldNotBeNil)
				}
			})
		})
	})
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"crypto/tls"
	"flag"
	"net"
	"net/http"
	"os"
	"os/signal"
	"strings"
	"syscall"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/coreos/go-semver/semver"
	"github.com/fnproject/fn/fnlb/lb"
)

const VERSION = "0.0.25"

func main() {
	// XXX (reed): normalize
	fnodes := flag.String("nodes", "", "comma separated list of functions nodes")
	minAPIVersion := flag.String("min-api-version", "0.0.7", "minimal node API to accept")

	var conf lb.Config
	flag.StringVar(&conf.DBurl, "db", "sqlite3://:memory:", "backend to store nodes, default to in memory")
	flag.StringVar(&conf.Listen, "listen", ":8081", "port to run on")
	flag.IntVar(&conf.HealthcheckInterval, "hc-interval", 3, "how often to check f(x) nodes, in seconds")
	flag.StringVar(&conf.HealthcheckEndpoint, "hc-path", "/version", "endpoint to determine node health")
	flag.IntVar(&conf.HealthcheckUnhealthy, "hc-unhealthy", 2, "threshold of failed checks to declare node unhealthy")
	flag.IntVar(&conf.HealthcheckTimeout, "hc-timeout", 5, "timeout of healthcheck endpoint, in seconds")
	flag.StringVar(&conf.ZipkinURL, "zipkin", "", "zipkin endpoint to send traces")
	flag.Parse()

	conf.MinAPIVersion = semver.New(*minAPIVersion)

	if len(*fnodes) > 0 {
		// starting w/o nodes is fine too
		conf.Nodes = strings.Split(*fnodes, ",")
	}

	conf.Transport = &http.Transport{
		Proxy: http.ProxyFromEnvironment,
		Dial: (&net.Dialer{
			Timeout:   10 * time.Second,
			KeepAlive: 120 * time.Second,
		}).Dial,
		MaxIdleConnsPerHost: 512,
		TLSHandshakeTimeout: 10 * time.Second,
		TLSClientConfig: &tls.Config{
			ClientSessionCache: tls.NewLRUClientSessionCache(4096),
		},
	}

	g, err := lb.NewAllGrouper(conf)
	if err != nil {
		logrus.WithError(err).Fatal("error setting up grouper")
	}

	r := lb.NewConsistentRouter(conf)
	k := func(r *http.Request) (string, error) {
		return r.URL.Path, nil
	}

	h := lb.NewProxy(k, g, r, conf)
	h = g.Wrap(h) // add/del/list endpoints
	h = r.Wrap(h) // stats / dash endpoint

	err = serve(conf.Listen, h)
	if err != nil {
		logrus.WithError(err).Fatal("server error")
	}
}

func serve(addr string, handler http.Handler) error {
	server := &http.Server{Addr: addr, Handler: handler}

	ch := make(chan os.Signal, 1)
	signal.Notify(ch, syscall.SIGQUIT, syscall.SIGINT)
	go func() {
		for sig := range ch {
			logrus.WithFields(logrus.Fields{"signal": sig}).Info("received signal")
			server.Shutdown(context.Background()) // safe shutdown
			return
		}
	}()
	return server.ListenAndServe()
}
<commit_msg>fnlb: 0.0.26 release [skip ci]<commit_after>package main

import (
	"context"
	"crypto/tls"
	"flag"
	"net"
	"net/http"
	"os"
	"os/signal"
	"strings"
	"syscall"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/coreos/go-semver/semver"
	"github.com/fnproject/fn/fnlb/lb"
)

const VERSION = "0.0.26"

func main() {
	// XXX (reed): normalize
	fnodes := flag.String("nodes", "", "comma separated list of functions nodes")
	minAPIVersion := flag.String("min-api-version", "0.0.7", "minimal node API to accept")

	var conf lb.Config
	flag.StringVar(&conf.DBurl, "db", "sqlite3://:memory:", "backend to store nodes, default to in memory")
	flag.StringVar(&conf.Listen, "listen", ":8081", "port to run on")
	flag.IntVar(&conf.HealthcheckInterval, "hc-interval", 3, "how often to check f(x) nodes, in seconds")
	flag.StringVar(&conf.HealthcheckEndpoint, "hc-path", "/version", "endpoint to determine node health")
	flag.IntVar(&conf.HealthcheckUnhealthy, "hc-unhealthy", 2, "threshold of failed checks to declare node unhealthy")
	flag.IntVar(&conf.HealthcheckTimeout, "hc-timeout", 5, "timeout of healthcheck endpoint, in seconds")
	flag.StringVar(&conf.ZipkinURL, "zipkin", "", "zipkin endpoint to send traces")
	flag.Parse()

	conf.MinAPIVersion = semver.New(*minAPIVersion)

	if len(*fnodes) > 0 {
		// starting w/o nodes is fine too
		conf.Nodes = strings.Split(*fnodes, ",")
	}

	conf.Transport = &http.Transport{
		Proxy: http.ProxyFromEnvironment,
		Dial: (&net.Dialer{
			Timeout:   10 * time.Second,
			KeepAlive: 120 * time.Second,
		}).Dial,
		MaxIdleConnsPerHost: 512,
		TLSHandshakeTimeout: 10 * time.Second,
		TLSClientConfig: &tls.Config{
			ClientSessionCache: tls.NewLRUClientSessionCache(4096),
		},
	}

	g, err := lb.NewAllGrouper(conf)
	if err != nil {
		logrus.WithError(err).Fatal("error setting up grouper")
	}

	r := lb.NewConsistentRouter(conf)
	k := func(r *http.Request) (string, error) {
		return r.URL.Path, nil
	}

	h := lb.NewProxy(k, g, r, conf)
	h = g.Wrap(h) // add/del/list endpoints
	h = r.Wrap(h) // stats / dash endpoint

	err = serve(conf.Listen, h)
	if err != nil {
		logrus.WithError(err).Fatal("server error")
	}
}

func serve(addr string, handler http.Handler) error {
	server := &http.Server{Addr: addr, Handler: handler}

	ch := make(chan os.Signal, 1)
	signal.Notify(ch, syscall.SIGQUIT, syscall.SIGINT)
	go func() {
		for sig := range ch {
			logrus.WithFields(logrus.Fields{"signal": sig}).Info("received signal")
			server.Shutdown(context.Background()) // safe shutdown
			return
		}
	}()
	return server.ListenAndServe()
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"crypto/tls"
	"flag"
	"net"
	"net/http"
	"os"
	"os/signal"
	"strings"
	"syscall"
	"time"

	"github.com/coreos/go-semver/semver"
	"github.com/fnproject/fn/fnlb/lb"
	"github.com/sirupsen/logrus"
)

const VERSION = "0.0.264"

func main() {
	// XXX (reed): normalize
	level, err := logrus.ParseLevel(os.Getenv("LOG_LEVEL"))
	if err != nil {
		level = logrus.InfoLevel
	}
	logrus.SetLevel(level)

	fnodes := flag.String("nodes", "", "comma separated list of functions nodes")
	minAPIVersion := flag.String("min-api-version", "0.0.231", "minimal node API to accept")

	var conf lb.Config
	flag.StringVar(&conf.DBurl, "db", "sqlite3://:memory:", "backend to store nodes, default to in memory; use k8s for kuberneted")
	flag.StringVar(&conf.Listen, "listen", ":8081", "port to run on")
	flag.StringVar(&conf.MgmtListen, "mgmt-listen", ":8081", "management port to run on")
	flag.IntVar(&conf.ShutdownTimeout, "shutdown-timeout", 0, "graceful shutdown timeout")
	flag.IntVar(&conf.HealthcheckInterval, "hc-interval", 3, "how often to check f(x) nodes, in seconds")
	flag.StringVar(&conf.HealthcheckEndpoint, "hc-path", "/version", "endpoint to determine node health")
	flag.IntVar(&conf.HealthcheckUnhealthy, "hc-unhealthy", 2, "threshold of failed checks to declare node unhealthy")
	flag.IntVar(&conf.HealthcheckHealthy, "hc-healthy", 1, "threshold of success checks to declare node healthy")
	flag.IntVar(&conf.HealthcheckTimeout, "hc-timeout", 5, "timeout of healthcheck endpoint, in seconds")
	flag.StringVar(&conf.ZipkinURL, "zipkin", "", "zipkin endpoint to send traces")
	flag.StringVar(&conf.Namespace, "namespace", "", "kubernetes namespace to monitor")
	flag.StringVar(&conf.LabelSelector, "label-selector", "", "kubernetes label selector to monitor")
	flag.IntVar(&conf.TargetPort, "target-port", 8080, "kubernetes port to target on selected pods")

	flag.Parse()

	conf.MinAPIVersion = semver.New(*minAPIVersion)

	if len(*fnodes) > 0 {
		// starting w/o nodes is fine too
		conf.Nodes = strings.Split(*fnodes, ",")
	}

	conf.Transport = &http.Transport{
		Proxy: http.ProxyFromEnvironment,
		Dial: (&net.Dialer{
			Timeout:   10 * time.Second,
			KeepAlive: 120 * time.Second,
		}).Dial,
		MaxIdleConnsPerHost: 512,
		TLSHandshakeTimeout: 10 * time.Second,
		TLSClientConfig: &tls.Config{
			ClientSessionCache: tls.NewLRUClientSessionCache(4096),
		},
	}

	db, err := lb.NewDB(conf) // Handles case where DBurl == "k8s"
	if err != nil {
		logrus.WithError(err).Fatal("error setting up database")
	}
	defer db.Close()

	g, err := lb.NewAllGrouper(conf, db)
	if err != nil {
		logrus.WithError(err).Fatal("error setting up grouper")
	}

	r := lb.NewConsistentRouter(conf)
	k := func(r *http.Request) (string, error) {
		return r.URL.Path, nil
	}

	servers := make([]*http.Server, 0, 1)
	handler := lb.NewProxy(k, g, r, conf)

	// a separate mgmt listener is requested? then let's create a LB traffic only server
	if conf.Listen != conf.MgmtListen {
		servers = append(servers, &http.Server{Addr: conf.Listen, Handler: handler})
		handler = lb.NullHandler()
	}

	// add mgmt endpoints to the handler
	handler = g.Wrap(handler) // add/del/list endpoints
	handler = r.Wrap(handler) // stats / dash endpoint

	servers = append(servers, &http.Server{Addr: conf.MgmtListen, Handler: handler})
	serve(servers, &conf)
}

func serve(servers []*http.Server, conf *lb.Config) {

	ch := make(chan os.Signal, 1)
	signal.Notify(ch, syscall.SIGQUIT, syscall.SIGINT)

	for i := 0; i < len(servers); i++ {
		go func(idx int) {
			err := servers[idx].ListenAndServe()
			if err != nil && err != http.ErrServerClosed {
				logrus.WithFields(logrus.Fields{"server_id": idx}).WithError(err).Fatal("server error")
			} else {
				logrus.WithFields(logrus.Fields{"server_id": idx}).Info("server stopped")
			}
		}(i)
	}

	sig := <-ch
	logrus.WithFields(logrus.Fields{"signal": sig}).Info("received signal")

	for i := 0; i < len(servers); i++ {

		ctx := context.Background()

		if conf.ShutdownTimeout > 0 {
			tmpCtx, cancel := context.WithTimeout(context.Background(), time.Duration(conf.ShutdownTimeout)*time.Second)
			ctx = tmpCtx
			defer cancel()
		}

		err := servers[i].Shutdown(ctx) // safe shutdown
		if err != nil {
			logrus.WithFields(logrus.Fields{"server_id": i}).WithError(err).Fatal("server shutdown error")
		} else {
			logrus.WithFields(logrus.Fields{"server_id": i}).Info("server shutdown")
		}
	}
}
<commit_msg>fnlb: 0.0.265 release [skip ci]<commit_after>package main

import (
	"context"
	"crypto/tls"
	"flag"
	"net"
	"net/http"
	"os"
	"os/signal"
	"strings"
	"syscall"
	"time"

	"github.com/coreos/go-semver/semver"
	"github.com/fnproject/fn/fnlb/lb"
	"github.com/sirupsen/logrus"
)

const VERSION = "0.0.265"

func main() {
	// XXX (reed): normalize
	level, err := logrus.ParseLevel(os.Getenv("LOG_LEVEL"))
	if err != nil {
		level = logrus.InfoLevel
	}
	logrus.SetLevel(level)

	fnodes := flag.String("nodes", "", "comma separated list of functions nodes")
	minAPIVersion := flag.String("min-api-version", "0.0.232", "minimal node API to accept")

	var conf lb.Config
	flag.StringVar(&conf.DBurl, "db", "sqlite3://:memory:", "backend to store nodes, default to in memory; use k8s for kuberneted")
	flag.StringVar(&conf.Listen, "listen", ":8081", "port to run on")
	flag.StringVar(&conf.MgmtListen, "mgmt-listen", ":8081", "management port to run on")
	flag.IntVar(&conf.ShutdownTimeout, "shutdown-timeout", 0, "graceful shutdown timeout")
	flag.IntVar(&conf.HealthcheckInterval, "hc-interval", 3, "how often to check f(x) nodes, in seconds")
	flag.StringVar(&conf.HealthcheckEndpoint, "hc-path", "/version", "endpoint to determine node health")
	flag.IntVar(&conf.HealthcheckUnhealthy, "hc-unhealthy", 2, "threshold of failed checks to declare node unhealthy")
	flag.IntVar(&conf.HealthcheckHealthy, "hc-healthy", 1, "threshold of success checks to declare node healthy")
	flag.IntVar(&conf.HealthcheckTimeout, "hc-timeout", 5, "timeout of healthcheck endpoint, in seconds")
	flag.StringVar(&conf.ZipkinURL, "zipkin", "", "zipkin endpoint to send traces")
	flag.StringVar(&conf.Namespace, "namespace", "", "kubernetes namespace to monitor")
	flag.StringVar(&conf.LabelSelector, "label-selector", "", "kubernetes label selector to monitor")
	flag.IntVar(&conf.TargetPort, "target-port", 8080, "kubernetes port to target on selected pods")

	flag.Parse()

	conf.MinAPIVersion = semver.New(*minAPIVersion)

	if len(*fnodes) > 0 {
		// starting w/o nodes is fine too
		conf.Nodes = strings.Split(*fnodes, ",")
	}

	conf.Transport = &http.Transport{
		Proxy: http.ProxyFromEnvironment,
		Dial: (&net.Dialer{
			Timeout:   10 * time.Second,
			KeepAlive: 120 * time.Second,
		}).Dial,
		MaxIdleConnsPerHost: 512,
		TLSHandshakeTimeout: 10 * time.Second,
		TLSClientConfig: &tls.Config{
			ClientSessionCache: tls.NewLRUClientSessionCache(4096),
		},
	}

	db, err := lb.NewDB(conf) // Handles case where DBurl == "k8s"
	if err != nil {
		logrus.WithError(err).Fatal("error setting up database")
	}
	defer db.Close()

	g, err := lb.NewAllGrouper(conf, db)
	if err != nil {
		logrus.WithError(err).Fatal("error setting up grouper")
	}

	r := lb.NewConsistentRouter(conf)
	k := func(r *http.Request) (string, error) {
		return r.URL.Path, nil
	}

	servers := make([]*http.Server, 0, 1)
	handler := lb.NewProxy(k, g, r, conf)

	// a separate mgmt listener is requested? then let's create a LB traffic only server
	if conf.Listen != conf.MgmtListen {
		servers = append(servers, &http.Server{Addr: conf.Listen, Handler: handler})
		handler = lb.NullHandler()
	}

	// add mgmt endpoints to the handler
	handler = g.Wrap(handler) // add/del/list endpoints
	handler = r.Wrap(handler) // stats / dash endpoint

	servers = append(servers, &http.Server{Addr: conf.MgmtListen, Handler: handler})
	serve(servers, &conf)
}

func serve(servers []*http.Server, conf *lb.Config) {

	ch := make(chan os.Signal, 1)
	signal.Notify(ch, syscall.SIGQUIT, syscall.SIGINT)

	for i := 0; i < len(servers); i++ {
		go func(idx int) {
			err := servers[idx].ListenAndServe()
			if err != nil && err != http.ErrServerClosed {
				logrus.WithFields(logrus.Fields{"server_id": idx}).WithError(err).Fatal("server error")
			} else {
				logrus.WithFields(logrus.Fields{"server_id": idx}).Info("server stopped")
			}
		}(i)
	}

	sig := <-ch
	logrus.WithFields(logrus.Fields{"signal": sig}).Info("received signal")

	for i := 0; i < len(servers); i++ {

		ctx := context.Background()

		if conf.ShutdownTimeout > 0 {
			tmpCtx, cancel := context.WithTimeout(context.Background(), time.Duration(conf.ShutdownTimeout)*time.Second)
			ctx = tmpCtx
			defer cancel()
		}

		err := servers[i].Shutdown(ctx) // safe shutdown
		if err != nil {
			logrus.WithFields(logrus.Fields{"server_id": i}).WithError(err).Fatal("server shutdown error")
		} else {
			logrus.WithFields(logrus.Fields{"server_id": i}).Info("server shutdown")
		}
	}
}
<|endoftext|>


<commit_before>package aws

import (
	"fmt"
	"testing"

	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/terraform"
	"github.com/mitchellh/goamz/autoscaling"
)

func TestAccAWSLaunchConfiguration(t *testing.T) {
	var conf autoscaling.LaunchConfiguration

	resource.Test(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckAWSLaunchConfigurationDestroy,
		Steps: []resource.TestStep{
			resource.TestStep{
				Config: testAccAWSLaunchConfigurationConfig,
				Check: resource.ComposeTestCheckFunc(
					testAccCheckAWSLaunchConfigurationExists("aws_launch_configuration.bar", &conf),
					testAccCheckAWSLaunchConfigurationAttributes(&conf),
					resource.TestCheckResourceAttr(
						"aws_launch_configuration.bar", "image_id", "ami-21f78e11"),
					resource.TestCheckResourceAttr(
						"aws_launch_configuration.bar", "name", "foobar-terraform-test"),
					resource.TestCheckResourceAttr(
						"aws_launch_configuration.bar", "instance_type", "t1.micro"),
					resource.TestCheckResourceAttr(
						"aws_launch_configuration.bar", "associate_public_ip_address", "true"),
				),
			},
		},
	})
}

func testAccCheckAWSLaunchConfigurationDestroy(s *terraform.State) error {
	conn := testAccProvider.Meta().(*AWSClient).autoscalingconn

	for _, rs := range s.RootModule().Resources {
		if rs.Type != "aws_launch_configuration" {
			continue
		}

		describe, err := conn.DescribeLaunchConfigurations(
			&autoscaling.DescribeLaunchConfigurations{
				Names: []string{rs.Primary.ID},
			})

		if err == nil {
			if len(describe.LaunchConfigurations) != 0 &&
				describe.LaunchConfigurations[0].Name == rs.Primary.ID {
				return fmt.Errorf("Launch Configuration still exists")
			}
		}

		// Verify the error
		providerErr, ok := err.(*autoscaling.Error)
		if !ok {
			return err
		}
		if providerErr.Code != "InvalidLaunchConfiguration.NotFound" {
			return err
		}
	}

	return nil
}

func testAccCheckAWSLaunchConfigurationAttributes(conf *autoscaling.LaunchConfiguration) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		if conf.ImageId != "ami-21f78e11" {
			return fmt.Errorf("Bad image_id: %s", conf.ImageId)
		}

		if conf.Name != "foobar-terraform-test" {
			return fmt.Errorf("Bad name: %s", conf.Name)
		}

		if conf.InstanceType != "t1.micro" {
			return fmt.Errorf("Bad instance_type: %s", conf.InstanceType)
		}

		return nil
	}
}

func testAccCheckAWSLaunchConfigurationExists(n string, res *autoscaling.LaunchConfiguration) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		rs, ok := s.RootModule().Resources[n]
		if !ok {
			return fmt.Errorf("Not found: %s", n)
		}

		if rs.Primary.ID == "" {
			return fmt.Errorf("No Launch Configuration ID is set")
		}

		conn := testAccProvider.Meta().(*AWSClient).autoscalingconn

		describeOpts := autoscaling.DescribeLaunchConfigurations{
			Names: []string{rs.Primary.ID},
		}
		describe, err := conn.DescribeLaunchConfigurations(&describeOpts)

		if err != nil {
			return err
		}

		if len(describe.LaunchConfigurations) != 1 ||
			describe.LaunchConfigurations[0].Name != rs.Primary.ID {
			return fmt.Errorf("Launch Configuration Group not found")
		}

		*res = describe.LaunchConfigurations[0]

		return nil
	}
}

const testAccAWSLaunchConfigurationConfig = `
resource "aws_launch_configuration" "bar" {
  name = "foobar-terraform-test"
  image_id = "ami-21f78e11"
  instance_type = "t1.micro"
  user_data = "foobar-user-data"
  associate_public_ip_address = true
}
`
<commit_msg>Add test for launch configuration with spot price<commit_after>package aws

import (
	"fmt"
	"testing"

	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/terraform"
	"github.com/mitchellh/goamz/autoscaling"
)

func TestAccAWSLaunchConfiguration(t *testing.T) {
	var conf autoscaling.LaunchConfiguration

	resource.Test(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckAWSLaunchConfigurationDestroy,
		Steps: []resource.TestStep{
			resource.TestStep{
				Config: testAccAWSLaunchConfigurationConfig,
				Check: resource.ComposeTestCheckFunc(
					testAccCheckAWSLaunchConfigurationExists("aws_launch_configuration.bar", &conf),
					testAccCheckAWSLaunchConfigurationAttributes(&conf),
					resource.TestCheckResourceAttr(
						"aws_launch_configuration.bar", "image_id", "ami-21f78e11"),
					resource.TestCheckResourceAttr(
						"aws_launch_configuration.bar", "name", "foobar-terraform-test"),
					resource.TestCheckResourceAttr(
						"aws_launch_configuration.bar", "instance_type", "t1.micro"),
					resource.TestCheckResourceAttr(
						"aws_launch_configuration.bar", "associate_public_ip_address", "true"),
					resource.TestCheckResourceAttr(
						"aws_launch_configuration.bar", "spot_price", ""),
				),
			},

			resource.TestStep{
				Config: TestAccAWSLaunchConfigurationWithSpotPriceConfig,
				Check: resource.ComposeTestCheckFunc(
					testAccCheckAWSLaunchConfigurationExists("aws_launch_configuration.bar", &conf),
					testAccCheckAWSLaunchConfigurationAttributes(&conf),
					resource.TestCheckResourceAttr(
						"aws_launch_configuration.bar", "spot_price", "0.01"),
				),
			},
		},
	})
}

func testAccCheckAWSLaunchConfigurationDestroy(s *terraform.State) error {
	conn := testAccProvider.Meta().(*AWSClient).autoscalingconn

	for _, rs := range s.RootModule().Resources {
		if rs.Type != "aws_launch_configuration" {
			continue
		}

		describe, err := conn.DescribeLaunchConfigurations(
			&autoscaling.DescribeLaunchConfigurations{
				Names: []string{rs.Primary.ID},
			})

		if err == nil {
			if len(describe.LaunchConfigurations) != 0 &&
				describe.LaunchConfigurations[0].Name == rs.Primary.ID {
				return fmt.Errorf("Launch Configuration still exists")
			}
		}

		// Verify the error
		providerErr, ok := err.(*autoscaling.Error)
		if !ok {
			return err
		}
		if providerErr.Code != "InvalidLaunchConfiguration.NotFound" {
			return err
		}
	}

	return nil
}

func testAccCheckAWSLaunchConfigurationAttributes(conf *autoscaling.LaunchConfiguration) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		if conf.ImageId != "ami-21f78e11" {
			return fmt.Errorf("Bad image_id: %s", conf.ImageId)
		}

		if conf.Name != "foobar-terraform-test" {
			return fmt.Errorf("Bad name: %s", conf.Name)
		}

		if conf.InstanceType != "t1.micro" {
			return fmt.Errorf("Bad instance_type: %s", conf.InstanceType)
		}

		return nil
	}
}

func testAccCheckAWSLaunchConfigurationExists(n string, res *autoscaling.LaunchConfiguration) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		rs, ok := s.RootModule().Resources[n]
		if !ok {
			return fmt.Errorf("Not found: %s", n)
		}

		if rs.Primary.ID == "" {
			return fmt.Errorf("No Launch Configuration ID is set")
		}

		conn := testAccProvider.Meta().(*AWSClient).autoscalingconn

		describeOpts := autoscaling.DescribeLaunchConfigurations{
			Names: []string{rs.Primary.ID},
		}
		describe, err := conn.DescribeLaunchConfigurations(&describeOpts)

		if err != nil {
			return err
		}

		if len(describe.LaunchConfigurations) != 1 ||
			describe.LaunchConfigurations[0].Name != rs.Primary.ID {
			return fmt.Errorf("Launch Configuration Group not found")
		}

		*res = describe.LaunchConfigurations[0]

		return nil
	}
}

const testAccAWSLaunchConfigurationConfig = `
resource "aws_launch_configuration" "bar" {
  name = "foobar-terraform-test"
  image_id = "ami-21f78e11"
  instance_type = "t1.micro"
  user_data = "foobar-user-data"
  associate_public_ip_address = true
}
`

const TestAccAWSLaunchConfigurationWithSpotPriceConfig = `
resource "aws_launch_configuration" "bar" {
  name = "foobar-terraform-test"
  image_id = "ami-21f78e11"
  instance_type = "t1.micro"
  user_data = "foobar-user-data"
  associate_public_ip_address = true
  spot_price = "0.01"
}
`
<|endoftext|>


<commit_before>// (c) 2019-2020, Ava Labs, Inc. All rights reserved.
// See the file LICENSE for licensing terms.

package queue

import (
	"fmt"

	"github.com/ava-labs/avalanchego/cache"
	"github.com/ava-labs/avalanchego/cache/metercacher"
	"github.com/ava-labs/avalanchego/database"
	"github.com/ava-labs/avalanchego/database/linkeddb"
	"github.com/ava-labs/avalanchego/database/prefixdb"
	"github.com/ava-labs/avalanchego/ids"
	"github.com/prometheus/client_golang/prometheus"
)

const (
	dependentsCacheSize = 1024
	jobsCacheSize       = 2048
)

var (
	runnableJobIDsKey = []byte("runnable")
	jobsKey           = []byte("jobs")
	dependenciesKey   = []byte("dependencies")
	missingJobIDsKey  = []byte("missing job IDs")
)

type state struct {
	parser         Parser
	runnableJobIDs linkeddb.LinkedDB
	jobsCache      cache.Cacher
	jobs           database.Database
	// Should be prefixed with the jobID that we are attempting to find the
	// dependencies of. This prefixdb.Database should then be wrapped in a
	// linkeddb.LinkedDB to read the dependencies.
	dependencies database.Database
	// This is a cache that tracks LinkedDB iterators that have recently been
	// made.
	dependentsCache cache.Cacher
	missingJobIDs   linkeddb.LinkedDB
}

func newState(
	db database.Database,
	metricsNamespace string,
	metricsRegisterer prometheus.Registerer,
) (*state, error) {
	jobsCacheMetricsNamespace := fmt.Sprintf("%s_jobs_cache", metricsNamespace)
	jobsCache, err := metercacher.New(jobsCacheMetricsNamespace, metricsRegisterer, &cache.LRU{Size: jobsCacheSize})
	if err != nil {
		return nil, fmt.Errorf("couldn't create metered cache: %s", err)
	}
	return &state{
		runnableJobIDs:  linkeddb.NewDefault(prefixdb.New(runnableJobIDsKey, db)),
		jobsCache:       jobsCache,
		jobs:            prefixdb.New(jobsKey, db),
		dependencies:    prefixdb.New(dependenciesKey, db),
		dependentsCache: &cache.LRU{Size: dependentsCacheSize},
		missingJobIDs:   linkeddb.NewDefault(prefixdb.New(missingJobIDsKey, db)),
	}, nil
}

// AddRunnableJob adds [jobID] to the runnable queue
func (ps *state) AddRunnableJob(jobID ids.ID) error {
	return ps.runnableJobIDs.Put(jobID[:], nil)
}

// HasRunnableJob returns if there is a job that can be run on the queue
func (ps *state) HasRunnableJob() (bool, error) {
	isEmpty, err := ps.runnableJobIDs.IsEmpty()
	return !isEmpty, err
}

// RemoveRunnableJob fetches and deletes the next job from the runnable queue
func (ps *state) RemoveRunnableJob() (Job, error) {
	jobIDBytes, err := ps.runnableJobIDs.HeadKey()
	if err != nil {
		return nil, err
	}
	if err := ps.runnableJobIDs.Delete(jobIDBytes); err != nil {
		return nil, err
	}

	jobID, err := ids.ToID(jobIDBytes)
	if err != nil {
		return nil, fmt.Errorf("couldn't convert job ID bytes to job ID: %s", err)
	}
	jobIntf, exists := ps.jobsCache.Get(jobID)
	if exists {
		return jobIntf.(Job), ps.jobs.Delete(jobIDBytes)
	}

	jobBytes, err := ps.jobs.Get(jobIDBytes)
	if err != nil {
		return nil, err
	}
	job, err := ps.parser.Parse(jobBytes)
	if err != nil {
		return nil, err
	}
	return job, ps.jobs.Delete(jobIDBytes)
}

// PutJob adds the job to the queue
func (ps *state) PutJob(job Job) error {
	id := job.ID()
	ps.jobsCache.Put(id, job)
	return ps.jobs.Put(id[:], job.Bytes())
}

// HasJob returns true if the job [id] is in the queue
func (ps *state) HasJob(id ids.ID) (bool, error) {
	if _, exists := ps.jobsCache.Get(id); exists {
		return true, nil
	}
	return ps.jobs.Has(id[:])
}

// GetJob returns the job [id]
func (ps *state) GetJob(id ids.ID) (Job, error) {
	if job, exists := ps.jobsCache.Get(id); exists {
		return job.(Job), nil
	}
	jobBytes, err := ps.jobs.Get(id[:])
	if err != nil {
		return nil, err
	}
	return ps.parser.Parse(jobBytes)
}

// AddBlocking adds [dependent] as blocking on [dependency] being completed
func (ps *state) AddDependency(dependency, dependent ids.ID) error {
	dependentsDB := ps.getDependents(dependency)
	return dependentsDB.Put(dependent[:], nil)
}

// Blocking returns the set of IDs that are blocking on the completion of
// [dependency] and removes them from the database.
func (ps *state) RemoveDependencies(dependency ids.ID) ([]ids.ID, error) {
	dependentsDB := ps.getDependents(dependency)
	iterator := dependentsDB.NewIterator()
	defer iterator.Release()

	dependents := []ids.ID(nil)
	for iterator.Next() {
		dependentKey := iterator.Key()
		if err := dependentsDB.Delete(dependentKey); err != nil {
			return nil, err
		}
		dependent, err := ids.ToID(dependentKey)
		if err != nil {
			return nil, err
		}
		dependents = append(dependents, dependent)
	}
	return dependents, iterator.Error()
}

func (ps *state) AddMissingJobIDs(missingIDs ids.Set) error {
	for missingID := range missingIDs {
		missingID := missingID
		if err := ps.missingJobIDs.Put(missingID[:], nil); err != nil {
			return err
		}
	}
	return nil
}

func (ps *state) RemoveMissingJobIDs(missingIDs ids.Set) error {
	for missingID := range missingIDs {
		missingID := missingID
		if err := ps.missingJobIDs.Delete(missingID[:]); err != nil {
			return err
		}
	}
	return nil
}

func (ps *state) MissingJobIDs() ([]ids.ID, error) {
	iterator := ps.missingJobIDs.NewIterator()
	defer iterator.Release()

	missingIDs := []ids.ID(nil)
	for iterator.Next() {
		missingID, err := ids.ToID(iterator.Key())
		if err != nil {
			return nil, err
		}
		missingIDs = append(missingIDs, missingID)
	}
	return missingIDs, nil
}

func (ps *state) getDependents(dependency ids.ID) linkeddb.LinkedDB {
	if dependentsDBIntf, ok := ps.dependentsCache.Get(dependency); ok {
		return dependentsDBIntf.(linkeddb.LinkedDB)
	}
	dependencyDB := prefixdb.New(dependency[:], ps.dependencies)
	dependentsDB := linkeddb.NewDefault(dependencyDB)
	ps.dependentsCache.Put(dependency, dependentsDB)
	return dependentsDB
}
<commit_msg>evict from cache on delete from jobs list<commit_after>// (c) 2019-2020, Ava Labs, Inc. All rights reserved.
// See the file LICENSE for licensing terms.

package queue

import (
	"fmt"

	"github.com/ava-labs/avalanchego/cache"
	"github.com/ava-labs/avalanchego/cache/metercacher"
	"github.com/ava-labs/avalanchego/database"
	"github.com/ava-labs/avalanchego/database/linkeddb"
	"github.com/ava-labs/avalanchego/database/prefixdb"
	"github.com/ava-labs/avalanchego/ids"
	"github.com/prometheus/client_golang/prometheus"
)

const (
	dependentsCacheSize = 1024
	jobsCacheSize       = 2048
)

var (
	runnableJobIDsKey = []byte("runnable")
	jobsKey           = []byte("jobs")
	dependenciesKey   = []byte("dependencies")
	missingJobIDsKey  = []byte("missing job IDs")
)

type state struct {
	parser         Parser
	runnableJobIDs linkeddb.LinkedDB
	jobsCache      cache.Cacher
	jobs           database.Database
	// Should be prefixed with the jobID that we are attempting to find the
	// dependencies of. This prefixdb.Database should then be wrapped in a
	// linkeddb.LinkedDB to read the dependencies.
	dependencies database.Database
	// This is a cache that tracks LinkedDB iterators that have recently been
	// made.
	dependentsCache cache.Cacher
	missingJobIDs   linkeddb.LinkedDB
}

func newState(
	db database.Database,
	metricsNamespace string,
	metricsRegisterer prometheus.Registerer,
) (*state, error) {
	jobsCacheMetricsNamespace := fmt.Sprintf("%s_jobs_cache", metricsNamespace)
	jobsCache, err := metercacher.New(jobsCacheMetricsNamespace, metricsRegisterer, &cache.LRU{Size: jobsCacheSize})
	if err != nil {
		return nil, fmt.Errorf("couldn't create metered cache: %s", err)
	}
	return &state{
		runnableJobIDs:  linkeddb.NewDefault(prefixdb.New(runnableJobIDsKey, db)),
		jobsCache:       jobsCache,
		jobs:            prefixdb.New(jobsKey, db),
		dependencies:    prefixdb.New(dependenciesKey, db),
		dependentsCache: &cache.LRU{Size: dependentsCacheSize},
		missingJobIDs:   linkeddb.NewDefault(prefixdb.New(missingJobIDsKey, db)),
	}, nil
}

// AddRunnableJob adds [jobID] to the runnable queue
func (ps *state) AddRunnableJob(jobID ids.ID) error {
	return ps.runnableJobIDs.Put(jobID[:], nil)
}

// HasRunnableJob returns if there is a job that can be run on the queue
func (ps *state) HasRunnableJob() (bool, error) {
	isEmpty, err := ps.runnableJobIDs.IsEmpty()
	return !isEmpty, err
}

// RemoveRunnableJob fetches and deletes the next job from the runnable queue
func (ps *state) RemoveRunnableJob() (Job, error) {
	jobIDBytes, err := ps.runnableJobIDs.HeadKey()
	if err != nil {
		return nil, err
	}
	if err := ps.runnableJobIDs.Delete(jobIDBytes); err != nil {
		return nil, err
	}

	jobID, err := ids.ToID(jobIDBytes)
	if err != nil {
		return nil, fmt.Errorf("couldn't convert job ID bytes to job ID: %s", err)
	}
	jobIntf, exists := ps.jobsCache.Get(jobID)
	if exists {
		ps.jobsCache.Evict(jobID)
		return jobIntf.(Job), ps.jobs.Delete(jobIDBytes)
	}

	jobBytes, err := ps.jobs.Get(jobIDBytes)
	if err != nil {
		return nil, err
	}
	job, err := ps.parser.Parse(jobBytes)
	if err != nil {
		return nil, err
	}
	ps.jobsCache.Evict(jobID)
	return job, ps.jobs.Delete(jobIDBytes)
}

// PutJob adds the job to the queue
func (ps *state) PutJob(job Job) error {
	id := job.ID()
	ps.jobsCache.Put(id, job)
	return ps.jobs.Put(id[:], job.Bytes())
}

// HasJob returns true if the job [id] is in the queue
func (ps *state) HasJob(id ids.ID) (bool, error) {
	if _, exists := ps.jobsCache.Get(id); exists {
		return true, nil
	}
	return ps.jobs.Has(id[:])
}

// GetJob returns the job [id]
func (ps *state) GetJob(id ids.ID) (Job, error) {
	if job, exists := ps.jobsCache.Get(id); exists {
		return job.(Job), nil
	}
	jobBytes, err := ps.jobs.Get(id[:])
	if err != nil {
		return nil, err
	}
	return ps.parser.Parse(jobBytes)
}

// AddBlocking adds [dependent] as blocking on [dependency] being completed
func (ps *state) AddDependency(dependency, dependent ids.ID) error {
	dependentsDB := ps.getDependents(dependency)
	return dependentsDB.Put(dependent[:], nil)
}

// Blocking returns the set of IDs that are blocking on the completion of
// [dependency] and removes them from the database.
func (ps *state) RemoveDependencies(dependency ids.ID) ([]ids.ID, error) {
	dependentsDB := ps.getDependents(dependency)
	iterator := dependentsDB.NewIterator()
	defer iterator.Release()

	dependents := []ids.ID(nil)
	for iterator.Next() {
		dependentKey := iterator.Key()
		if err := dependentsDB.Delete(dependentKey); err != nil {
			return nil, err
		}
		dependent, err := ids.ToID(dependentKey)
		if err != nil {
			return nil, err
		}
		dependents = append(dependents, dependent)
	}
	return dependents, iterator.Error()
}

func (ps *state) AddMissingJobIDs(missingIDs ids.Set) error {
	for missingID := range missingIDs {
		missingID := missingID
		if err := ps.missingJobIDs.Put(missingID[:], nil); err != nil {
			return err
		}
	}
	return nil
}

func (ps *state) RemoveMissingJobIDs(missingIDs ids.Set) error {
	for missingID := range missingIDs {
		missingID := missingID
		if err := ps.missingJobIDs.Delete(missingID[:]); err != nil {
			return err
		}
	}
	return nil
}

func (ps *state) MissingJobIDs() ([]ids.ID, error) {
	iterator := ps.missingJobIDs.NewIterator()
	defer iterator.Release()

	missingIDs := []ids.ID(nil)
	for iterator.Next() {
		missingID, err := ids.ToID(iterator.Key())
		if err != nil {
			return nil, err
		}
		missingIDs = append(missingIDs, missingID)
	}
	return missingIDs, nil
}

func (ps *state) getDependents(dependency ids.ID) linkeddb.LinkedDB {
	if dependentsDBIntf, ok := ps.dependentsCache.Get(dependency); ok {
		return dependentsDBIntf.(linkeddb.LinkedDB)
	}
	dependencyDB := prefixdb.New(dependency[:], ps.dependencies)
	dependentsDB := linkeddb.NewDefault(dependencyDB)
	ps.dependentsCache.Put(dependency, dependentsDB)
	return dependentsDB
}
<|endoftext|>


<commit_before>// Package fs provides a tall backend based on filesystem
package fs

import (
	"bytes"
	"github.com/jamesruan/tall"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"sync"
)

// implements tall.Backend, tall.BackendScavenger
type FSBackend struct {
	entry  string
	sm     *SuperMeta
	logger *log.Logger
	debug  bool
	lock   *sync.Mutex
}

func New(entry string) (*FSBackend, error) {
	var supermetafile *os.File
	var err error
	path := filepath.Join(entry, SUPERMATPATH)
	supermetafile, err = os.OpenFile(path, os.O_RDWR, DefaultMode)
	if err != nil {
		return nil, err
	}
	sm := new(SuperMeta)
	if err = sm.Load(supermetafile); err != nil {
		supermetafile.Close()
		return nil, err
	}
	be := &FSBackend{
		entry: entry,
		sm:    sm,
		lock:  new(sync.Mutex),
	}
	return be, nil
}

// Store save file to a tempfile, calculate score for the data
// and move to the file with the score as its name.
// The path for the file is scattered according to SuperMeta.Level
func (b *FSBackend) Store(data []byte) (score tall.HexBytes, err error) {
	var agent *FSWriteAgent
	r := bytes.NewBuffer(data)
	if agent, err = b.NewWriteAgent(); err != nil {
		return
	}

	if _, err = io.Copy(agent, r); err != nil {
		return
	}
	if err = agent.Close(); err != nil {
		return
	}
	score = agent.Score()

	return score, nil
}

func (b *FSBackend) NewWriteAgent() (*FSWriteAgent, error) {
	var tempfile *os.File
	var err error

	// TODO: use journal
	if tempfile, err = ioutil.TempFile(filepath.Join(b.entry, TEMPPATH), "data"); err != nil {
		return nil, err
	}

	pr, pw := io.Pipe()
	w := io.MultiWriter(tempfile, pw)

	return &FSWriteAgent{
		b:        b,
		sch:      ScoreFrom(pr),
		tempfile: tempfile,
		w:        w,
		pw:       pw,
		pr:       pr,
	}, nil
}

func (b *FSBackend) Load(score tall.HexBytes) (data []byte, err error) {
	buf := new(bytes.Buffer)
	var r io.ReadCloser
	r, err = b.Open(score)
	defer r.Close()
	if _, err = io.Copy(buf, r); err != nil {
		return nil, err
	}
	return buf.Bytes(), nil
}

func (b *FSBackend) Open(score tall.HexBytes) (*os.File, error) {
	var file *os.File
	var err error
	path := filepath.Join(b.entry, scoreToPath(score, b.sm.Level))
	if file, err = os.Open(path); err != nil {
		return nil, err
	}
	return file, nil
}

func (b *FSBackend) Has(score tall.HexBytes) (ok bool, err error) {
	path := filepath.Join(b.entry, scoreToPath(score, b.sm.Level))
	if _, err = os.Stat(path); err != nil {
		return false, err
	} else {
		return true, nil
	}
}

func (b *FSBackend) SetLogger(logger *log.Logger, debug bool) {
	b.lock.Lock()
	b.logger = logger
	b.debug = debug
	b.lock.Unlock()
}

func scoreToPath(score tall.HexBytes, level int) string {
	block := 2
	scoreleft := string(score)
	index := 0
	arr := []string{}
	for index < level {
		var v string
		v, scoreleft = scoreleft[:block*index], scoreleft[block*index:]
		arr = append(arr, v)
		index += 1
	}
	arr = append(arr, string(score))
	return filepath.Join(arr...)
}

func getScoreAndWrite(f io.WriteCloser, r io.Reader) (tall.HexBytes, error) {
	var err error
	pr, pw := io.Pipe()
	defer pr.Close()
	defer pw.Close()
	writer := io.MultiWriter(f, pw)

	score := ScoreFrom(pr)

	if _, err = io.Copy(writer, r); err != nil {
		return "", err
	}
	return <-score, err
}
<commit_msg>backend/fs: hide some method<commit_after>// Package fs provides a tall backend based on filesystem
package fs

import (
	"bytes"
	"github.com/jamesruan/tall"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"sync"
)

// implements tall.Backend, tall.BackendScavenger
type FSBackend struct {
	entry  string
	sm     *SuperMeta
	logger *log.Logger
	debug  bool
	lock   *sync.Mutex
}

func New(entry string) (*FSBackend, error) {
	var supermetafile *os.File
	var err error
	path := filepath.Join(entry, SUPERMATPATH)
	supermetafile, err = os.OpenFile(path, os.O_RDWR, DefaultMode)
	if err != nil {
		return nil, err
	}
	sm := new(SuperMeta)
	if err = sm.Load(supermetafile); err != nil {
		supermetafile.Close()
		return nil, err
	}
	be := &FSBackend{
		entry: entry,
		sm:    sm,
		lock:  new(sync.Mutex),
	}
	return be, nil
}

func (b *FSBackend) Score(d []byte) tall.HexBytes {
	return Score(d)
}

// Store save file to a tempfile, calculate score for the data
// and move to the file with the score as its name.
// The path for the file is scattered according to SuperMeta.Level
func (b *FSBackend) Store(data []byte) (score tall.HexBytes, err error) {
	var agent *FSWriteAgent
	r := bytes.NewBuffer(data)
	if agent, err = b.newWriteAgent(); err != nil {
		return
	}

	if _, err = io.Copy(agent, r); err != nil {
		return
	}
	if err = agent.Close(); err != nil {
		return
	}
	score = agent.Score()

	return score, nil
}

func (b *FSBackend) newWriteAgent() (*FSWriteAgent, error) {
	var tempfile *os.File
	var err error

	// TODO: use journal
	if tempfile, err = ioutil.TempFile(filepath.Join(b.entry, TEMPPATH), "data"); err != nil {
		return nil, err
	}

	pr, pw := io.Pipe()
	w := io.MultiWriter(tempfile, pw)

	return &FSWriteAgent{
		b:        b,
		sch:      ScoreFrom(pr),
		tempfile: tempfile,
		w:        w,
		pw:       pw,
		pr:       pr,
	}, nil
}

func (b *FSBackend) Load(score tall.HexBytes) (data []byte, err error) {
	buf := new(bytes.Buffer)
	var r io.ReadCloser
	r, err = b.open(score)
	defer r.Close()
	if _, err = io.Copy(buf, r); err != nil {
		return nil, err
	}
	return buf.Bytes(), nil
}

func (b *FSBackend) open(score tall.HexBytes) (*os.File, error) {
	var file *os.File
	var err error
	path := filepath.Join(b.entry, scoreToPath(score, b.sm.Level))
	if file, err = os.Open(path); err != nil {
		return nil, err
	}
	return file, nil
}

func (b *FSBackend) Has(score tall.HexBytes) (ok bool, err error) {
	path := filepath.Join(b.entry, scoreToPath(score, b.sm.Level))
	if _, err = os.Stat(path); err != nil {
		return false, err
	} else {
		return true, nil
	}
}

func (b *FSBackend) SetLogger(logger *log.Logger, debug bool) {
	b.lock.Lock()
	b.logger = logger
	b.debug = debug
	b.lock.Unlock()
}

func scoreToPath(score tall.HexBytes, level int) string {
	block := 2
	scoreleft := string(score)
	index := 0
	arr := []string{}
	for index < level {
		var v string
		v, scoreleft = scoreleft[:block*index], scoreleft[block*index:]
		arr = append(arr, v)
		index += 1
	}
	arr = append(arr, string(score))
	return filepath.Join(arr...)
}

func getScoreAndWrite(f io.WriteCloser, r io.Reader) (tall.HexBytes, error) {
	var err error
	pr, pw := io.Pipe()
	defer pr.Close()
	defer pw.Close()
	writer := io.MultiWriter(f, pw)

	score := ScoreFrom(pr)

	if _, err = io.Copy(writer, r); err != nil {
		return "", err
	}
	return <-score, err
}
<|endoftext|>