<commit_before>package common

import (
	"context"
	"fmt"
	"log"
	"os"
	"reflect"
	"strings"
	"time"

	"github.com/hashicorp/packer/helper/multistep"
	"github.com/hashicorp/packer/packer"
)

func newRunner(steps []multistep.Step, config PackerConfig, ui packer.Ui) (multistep.Runner, multistep.DebugPauseFn) {
	switch config.PackerOnError {
	case "", "cleanup":
	case "abort":
		for i, step := range steps {
			steps[i] = abortStep{step, ui}
		}
	case "ask":
		for i, step := range steps {
			steps[i] = askStep{step, ui}
		}
	}

	if config.PackerDebug {
		pauseFn := MultistepDebugFn(ui)
		return &multistep.DebugRunner{Steps: steps, PauseFn: pauseFn}, pauseFn
	} else {
		return &multistep.BasicRunner{Steps: steps}, nil
	}
}

// NewRunner returns a multistep.Runner that runs steps augmented with support
// for -debug and -on-error command line arguments.
func NewRunner(steps []multistep.Step, config PackerConfig, ui packer.Ui) multistep.Runner {
	runner, _ := newRunner(steps, config, ui)
	return runner
}

// NewRunnerWithPauseFn returns a multistep.Runner that runs steps augmented
// with support for -debug and -on-error command line arguments.  With -debug it
// puts the multistep.DebugPauseFn that will pause execution between steps into
// the state under the key "pauseFn".
func NewRunnerWithPauseFn(steps []multistep.Step, config PackerConfig, ui packer.Ui, state multistep.StateBag) multistep.Runner {
	runner, pauseFn := newRunner(steps, config, ui)
	if pauseFn != nil {
		state.Put("pauseFn", pauseFn)
	}
	return runner
}

func typeName(i interface{}) string {
	return reflect.Indirect(reflect.ValueOf(i)).Type().Name()
}

type abortStep struct {
	step multistep.Step
	ui   packer.Ui
}

func (s abortStep) InnerStepName() string {
	return typeName(s.step)
}

func (s abortStep) Run(ctx context.Context, state multistep.StateBag) multistep.StepAction {
	return s.step.Run(ctx, state)
}

func (s abortStep) Cleanup(state multistep.StateBag) {
	err, ok := state.GetOk("error")
	if ok {
		s.ui.Error(err.(error).Error())
	}
	if _, ok := state.GetOk(multistep.StateCancelled); ok {
		s.ui.Error("Interrupted, aborting...")
		os.Exit(1)
	}
	if _, ok := state.GetOk(multistep.StateHalted); ok {
		s.ui.Error(fmt.Sprintf("Step %q failed, aborting...", typeName(s.step)))
		os.Exit(1)
	}
	s.step.Cleanup(state)
}

type askStep struct {
	step multistep.Step
	ui   packer.Ui
}

func (s askStep) InnerStepName() string {
	return typeName(s.step)
}

func (s askStep) Run(ctx context.Context, state multistep.StateBag) (action multistep.StepAction) {
	for {
		action = s.step.Run(ctx, state)

		if action != multistep.ActionHalt {
			return
		}

		err, ok := state.GetOk("error")
		if ok {
			s.ui.Error(err.(error).Error())
		}

		switch ask(s.ui, typeName(s.step), state) {
		case askCleanup:
			return
		case askAbort:
			os.Exit(1)
		case askRetry:
			continue
		}
	}
}

func (s askStep) Cleanup(state multistep.StateBag) {
	s.step.Cleanup(state)
}

type askResponse int

const (
	askCleanup askResponse = iota
	askAbort
	askRetry
)

func ask(ui packer.Ui, name string, state multistep.StateBag) askResponse {
	ui.Say(fmt.Sprintf("Step %q failed", name))

	result := make(chan askResponse)
	go func() {
		result <- askPrompt(ui)
	}()

	for {
		select {
		case response := <-result:
			return response
		case <-time.After(100 * time.Millisecond):
			if _, ok := state.GetOk(multistep.StateCancelled); ok {
				return askCleanup
			}
		}
	}
}

func askPrompt(ui packer.Ui) askResponse {
	for {
		line, err := ui.Ask("[c] Clean up and exit, [a] abort without cleanup, or [r] retry step (build may fail even if retry succeeds)?")
		if err != nil {
			log.Printf("Error asking for input: %s", err)
		}

		input := strings.ToLower(line) + "c"
		switch input[0] {
		case 'c':
			return askCleanup
		case 'a':
			return askAbort
		case 'r':
			return askRetry
		}
		ui.Say(fmt.Sprintf("Incorrect input: %#v", line))
	}
}
<commit_msg>Use fmt to convert whatever's in error to a string.<commit_after>package common

import (
	"context"
	"fmt"
	"log"
	"os"
	"reflect"
	"strings"
	"time"

	"github.com/hashicorp/packer/helper/multistep"
	"github.com/hashicorp/packer/packer"
)

func newRunner(steps []multistep.Step, config PackerConfig, ui packer.Ui) (multistep.Runner, multistep.DebugPauseFn) {
	switch config.PackerOnError {
	case "", "cleanup":
	case "abort":
		for i, step := range steps {
			steps[i] = abortStep{step, ui}
		}
	case "ask":
		for i, step := range steps {
			steps[i] = askStep{step, ui}
		}
	}

	if config.PackerDebug {
		pauseFn := MultistepDebugFn(ui)
		return &multistep.DebugRunner{Steps: steps, PauseFn: pauseFn}, pauseFn
	} else {
		return &multistep.BasicRunner{Steps: steps}, nil
	}
}

// NewRunner returns a multistep.Runner that runs steps augmented with support
// for -debug and -on-error command line arguments.
func NewRunner(steps []multistep.Step, config PackerConfig, ui packer.Ui) multistep.Runner {
	runner, _ := newRunner(steps, config, ui)
	return runner
}

// NewRunnerWithPauseFn returns a multistep.Runner that runs steps augmented
// with support for -debug and -on-error command line arguments.  With -debug it
// puts the multistep.DebugPauseFn that will pause execution between steps into
// the state under the key "pauseFn".
func NewRunnerWithPauseFn(steps []multistep.Step, config PackerConfig, ui packer.Ui, state multistep.StateBag) multistep.Runner {
	runner, pauseFn := newRunner(steps, config, ui)
	if pauseFn != nil {
		state.Put("pauseFn", pauseFn)
	}
	return runner
}

func typeName(i interface{}) string {
	return reflect.Indirect(reflect.ValueOf(i)).Type().Name()
}

type abortStep struct {
	step multistep.Step
	ui   packer.Ui
}

func (s abortStep) InnerStepName() string {
	return typeName(s.step)
}

func (s abortStep) Run(ctx context.Context, state multistep.StateBag) multistep.StepAction {
	return s.step.Run(ctx, state)
}

func (s abortStep) Cleanup(state multistep.StateBag) {
	err, ok := state.GetOk("error")
	if ok {
		s.ui.Error(fmt.Sprintf("%s", err))
	}
	if _, ok := state.GetOk(multistep.StateCancelled); ok {
		s.ui.Error("Interrupted, aborting...")
		os.Exit(1)
	}
	if _, ok := state.GetOk(multistep.StateHalted); ok {
		s.ui.Error(fmt.Sprintf("Step %q failed, aborting...", typeName(s.step)))
		os.Exit(1)
	}
	s.step.Cleanup(state)
}

type askStep struct {
	step multistep.Step
	ui   packer.Ui
}

func (s askStep) InnerStepName() string {
	return typeName(s.step)
}

func (s askStep) Run(ctx context.Context, state multistep.StateBag) (action multistep.StepAction) {
	for {
		action = s.step.Run(ctx, state)

		if action != multistep.ActionHalt {
			return
		}

		err, ok := state.GetOk("error")
		if ok {
			s.ui.Error(fmt.Sprintf("%s", err))
		}

		switch ask(s.ui, typeName(s.step), state) {
		case askCleanup:
			return
		case askAbort:
			os.Exit(1)
		case askRetry:
			continue
		}
	}
}

func (s askStep) Cleanup(state multistep.StateBag) {
	s.step.Cleanup(state)
}

type askResponse int

const (
	askCleanup askResponse = iota
	askAbort
	askRetry
)

func ask(ui packer.Ui, name string, state multistep.StateBag) askResponse {
	ui.Say(fmt.Sprintf("Step %q failed", name))

	result := make(chan askResponse)
	go func() {
		result <- askPrompt(ui)
	}()

	for {
		select {
		case response := <-result:
			return response
		case <-time.After(100 * time.Millisecond):
			if _, ok := state.GetOk(multistep.StateCancelled); ok {
				return askCleanup
			}
		}
	}
}

func askPrompt(ui packer.Ui) askResponse {
	for {
		line, err := ui.Ask("[c] Clean up and exit, [a] abort without cleanup, or [r] retry step (build may fail even if retry succeeds)?")
		if err != nil {
			log.Printf("Error asking for input: %s", err)
		}

		input := strings.ToLower(line) + "c"
		switch input[0] {
		case 'c':
			return askCleanup
		case 'a':
			return askAbort
		case 'r':
			return askRetry
		}
		ui.Say(fmt.Sprintf("Incorrect input: %#v", line))
	}
}
<|endoftext|>


<commit_before>package scamp

import (
	"fmt"
	"math/rand"
	"sort"
	"time"
)

const (
	MAX_RETRIES = 20
)

// MakeJSONRequest retreives the appropriate service proxy based on the message action, and makes a
// JSON request.
func MakeJSONRequest(sector, action string, version int, msg *Message) (message *Message, err error) {
	var msgType string
	if msg.Envelope == EnvelopeJSON {
		msgType = "json"
	} else if msg.Envelope == EnvelopeJSONSTORE {
		msgType = "jsonstore"
	} else {
		err = fmt.Errorf("unsupported envelope type: `%d`", msg.Envelope)
		return
	}

	err = DefaultCache.Refresh()
	if err != nil {
		return
	}
	//TODO: add retry logic in case service proxies are nil
	var serviceProxies []*serviceProxy

	serviceProxies, err = DefaultCache.SearchByAction(sector, action, version, msgType)
	if err != nil {
		return
	}
	if len(serviceProxies) == 0 {
		err = fmt.Errorf("could not find %s:%s~%d#%s", sector, action, version, msgType)
		return
	}

	msg.SetAction(action)
	msg.SetVersion(version)

	sent := false
	var responseChan chan *Message

	var clients []*Client
	for _, serviceProxy := range serviceProxies {
		if serviceProxy != nil {
			client, err := serviceProxy.GetClient()
			if err != nil || client == nil {
				continue
			}

			clients = append(clients, client)
		}
	}

	rand.Shuffle(len(clients), func(i, j int) {
		clients[i], clients[j] = clients[j], clients[i]
	})

	// Sort based on queue depth.
	sort.Slice(clients, func(i, j int) bool {
		ilen := len(clients[i].openReplies)
		jlen := len(clients[j].openReplies)
		return ilen < jlen
	})

	for _, client := range clients {
		responseChan, err = client.Send(msg)
		if err == nil {
			sent = true
			break
		}
	}

	if !sent {
		err = fmt.Errorf("Request failed: %s.%s not found: %s", sector, action, err)
		return
	}

RetryLoop:
	for attempts := 0; attempts < MAX_RETRIES; attempts++ {
		select {
		case msg, ok := <-responseChan:
			if !ok {
				break RetryLoop
			}

			if msg == nil {
				continue RetryLoop
			}

			message = msg
			return
		case <-time.After(300 * time.Second):
			//close(responseChan)
			err = fmt.Errorf("request timed out")
			return
		}
	}

	return
}
<commit_msg>Return error on missing message<commit_after>package scamp

import (
	"fmt"
	"math/rand"
	"sort"
	"time"
)

const (
	MAX_RETRIES = 20
)

// MakeJSONRequest retreives the appropriate service proxy based on the message action, and makes a
// JSON request.
func MakeJSONRequest(sector, action string, version int, msg *Message) (message *Message, err error) {
	var msgType string
	if msg.Envelope == EnvelopeJSON {
		msgType = "json"
	} else if msg.Envelope == EnvelopeJSONSTORE {
		msgType = "jsonstore"
	} else {
		err = fmt.Errorf("unsupported envelope type: `%d`", msg.Envelope)
		return
	}

	err = DefaultCache.Refresh()
	if err != nil {
		return
	}
	//TODO: add retry logic in case service proxies are nil
	var serviceProxies []*serviceProxy

	serviceProxies, err = DefaultCache.SearchByAction(sector, action, version, msgType)
	if err != nil {
		return
	}
	if len(serviceProxies) == 0 {
		err = fmt.Errorf("could not find %s:%s~%d#%s", sector, action, version, msgType)
		return
	}

	msg.SetAction(action)
	msg.SetVersion(version)

	sent := false
	var responseChan chan *Message

	var clients []*Client
	for _, serviceProxy := range serviceProxies {
		if serviceProxy != nil {
			client, err := serviceProxy.GetClient()
			if err != nil || client == nil {
				continue
			}

			clients = append(clients, client)
		}
	}

	rand.Shuffle(len(clients), func(i, j int) {
		clients[i], clients[j] = clients[j], clients[i]
	})

	// Sort based on queue depth.
	sort.Slice(clients, func(i, j int) bool {
		ilen := len(clients[i].openReplies)
		jlen := len(clients[j].openReplies)
		return ilen < jlen
	})

	for _, client := range clients {
		responseChan, err = client.Send(msg)
		if err == nil {
			sent = true
			break
		}
	}

	if !sent {
		err = fmt.Errorf("Request failed: %s.%s not found: %s", sector, action, err)
		return
	}

RetryLoop:
	for attempts := 0; attempts < MAX_RETRIES; attempts++ {
		select {
		case msg, ok := <-responseChan:
			if !ok && msg == nil {
				break RetryLoop
			}

			if msg == nil {
				continue RetryLoop
			}

			message = msg
			return
		case <-time.After(300 * time.Second):
			//close(responseChan)
			err = fmt.Errorf("request timed out")
			return
		}
	}

	if msg == nil {
		err = fmt.Errorf("no response was found")
		return
	}

	return
}
<|endoftext|>


<commit_before>// Copyright 2016 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package resource

// TODO(ericsnow) Move this file to the charm repo?

import (
	"io"

	charmresource "gopkg.in/juju/charm.v6-unstable/resource"
)

// Content holds a reader for the content of a resource along
// with details about that content.
type Content struct {
	// Data holds the resouce content, ready to be read (once).
	Data io.Reader

	// Size is the byte count of the data.
	Size int64

	// Fingerprint holds the checksum of the data.
	Fingerprint charmresource.Fingerprint
}
<commit_msg>Add resource.GenerateContent().<commit_after>// Copyright 2016 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package resource

// TODO(ericsnow) Move this file to the charm repo?

import (
	"io"
	"os"

	"github.com/juju/errors"
	"github.com/juju/utils"
	charmresource "gopkg.in/juju/charm.v6-unstable/resource"
)

// Content holds a reader for the content of a resource along
// with details about that content.
type Content struct {
	// Data holds the resouce content, ready to be read (once).
	Data io.Reader

	// Size is the byte count of the data.
	Size int64

	// Fingerprint holds the checksum of the data.
	Fingerprint charmresource.Fingerprint
}

// GenerateContent returns a new Content for the given data stream.
func GenerateContent(reader io.ReadSeeker) (Content, error) {
	var sizer utils.SizeTracker
	sizingReader := io.TeeReader(reader, &sizer)
	fp, err := charmresource.GenerateFingerprint(sizingReader)
	if err != nil {
		return Content{}, errors.Trace(err)
	}
	if _, err := reader.Seek(0, os.SEEK_SET); err != nil {
		return Content{}, errors.Trace(err)
	}
	size := sizer.Size()

	content := Content{
		Data:        reader,
		Size:        size,
		Fingerprint: fp,
	}
	return content, nil
}
<|endoftext|>


<commit_before>package resource

import "time"

// Droplet is the result of staging an application package.
// There are two types (lifecycles) of droplets: buildpack and
// docker. In the case of buildpacks, the droplet contains the
// bits produced by the buildpack.
type Droplet struct {
	GUID              string            `json:"guid"`
	CreatedAt         time.Time         `json:"created_at"`
	UpdatedAt         time.Time         `json:"updated_at"`
	State             string            `json:"state"`
	Error             *string           `json:"error"`
	Lifecycle         Lifecycle         `json:"lifecycle"`
	Links             map[string]Link   `json:"links"`
	ExecutionMetadata string            `json:"execution_metadata"`
	ProcessTypes      map[string]string `json:"process_types"`
	Metadata          Metadata          `json:"metadata"`
	Relationships     AppRelationship   `json:"relationships"`

	// Only specified when the droplet is using the Docker lifecycle.
	Image *string `json:"image"`

	// The following fields are specified when the droplet is using
	// the buildpack lifecycle.
	Checksum struct {
		Type  string `json:"type"`
		Value string `json:"value"`
	} `json:"checksum"`
	Stack      string              `json:"stack"`
	Buildpacks []DetectedBuildpack `json:"buildpacks"`
}

type DropletCreate struct {
	Relationships AppRelationship   `json:"relationships"`
	ProcessTypes  map[string]string `json:"process_types"`
}

type DropletList struct {
	Pagination Pagination `json:"pagination,omitempty"`
	Resources  []*Droplet `json:"resources,omitempty"`
}

type DropletUpdate struct {
	Metadata Metadata `json:"metadata,omitempty"`
	Image    string   `json:"image"`
}

type DropletCurrent struct {
	Data  Relationship    `json:"data"`
	Links map[string]Link `json:"links"`
}

type DropletCopy struct {
	Relationships AppRelationship `json:"relationships"`
}

type DetectedBuildpack struct {
	Name          string `json:"name"`           // system buildpack name
	BuildpackName string `json:"buildpack_name"` // name reported by the buildpack
	DetectOutput  string `json:"detect_output"`  // output during detect process
	Version       string `json:"version"`
}

func NewDropletCreate(appGUID string) *DropletCreate {
	return &DropletCreate{
		Relationships: AppRelationship{
			App: ToOneRelationship{
				Data: &Relationship{
					GUID: appGUID,
				},
			},
		},
		ProcessTypes: make(map[string]string),
	}
}

func NewDropletCopy(appGUID string) *DropletCopy {
	return &DropletCopy{
		Relationships: AppRelationship{
			App: ToOneRelationship{
				Data: &Relationship{
					GUID: appGUID,
				},
			},
		},
	}
}
<commit_msg>Add DropletState type and constants<commit_after>package resource

import "time"

type DropletState string

// The 3 lifecycle states
const (
	DropletStateAwaitingUpload   BuildState = "AWAITING_UPLOAD"
	DropletStateProcessingUpload BuildState = "PROCESSING_UPLOAD"
	DropletStateStaged           BuildState = "STAGED"
	DropletStateCopying          BuildState = "COPYING"
	DropletStateFailed           BuildState = "FAILED"
	DropletStateExpired          BuildState = "EXPIRED"
)

// Droplet is the result of staging an application package.
// There are two types (lifecycles) of droplets: buildpack and
// docker. In the case of buildpacks, the droplet contains the
// bits produced by the buildpack.
type Droplet struct {
	GUID              string            `json:"guid"`
	CreatedAt         time.Time         `json:"created_at"`
	UpdatedAt         time.Time         `json:"updated_at"`
	State             DropletState      `json:"state"`
	Error             *string           `json:"error"`
	Lifecycle         Lifecycle         `json:"lifecycle"`
	Links             map[string]Link   `json:"links"`
	ExecutionMetadata string            `json:"execution_metadata"`
	ProcessTypes      map[string]string `json:"process_types"`
	Metadata          Metadata          `json:"metadata"`
	Relationships     AppRelationship   `json:"relationships"`

	// Only specified when the droplet is using the Docker lifecycle.
	Image *string `json:"image"`

	// The following fields are specified when the droplet is using
	// the buildpack lifecycle.
	Checksum struct {
		Type  string `json:"type"`
		Value string `json:"value"`
	} `json:"checksum"`
	Stack      string              `json:"stack"`
	Buildpacks []DetectedBuildpack `json:"buildpacks"`
}

type DropletCreate struct {
	Relationships AppRelationship   `json:"relationships"`
	ProcessTypes  map[string]string `json:"process_types"`
}

type DropletList struct {
	Pagination Pagination `json:"pagination,omitempty"`
	Resources  []*Droplet `json:"resources,omitempty"`
}

type DropletUpdate struct {
	Metadata Metadata `json:"metadata,omitempty"`
	Image    string   `json:"image"`
}

type DropletCurrent struct {
	Data  Relationship    `json:"data"`
	Links map[string]Link `json:"links"`
}

type DropletCopy struct {
	Relationships AppRelationship `json:"relationships"`
}

type DetectedBuildpack struct {
	Name          string `json:"name"`           // system buildpack name
	BuildpackName string `json:"buildpack_name"` // name reported by the buildpack
	DetectOutput  string `json:"detect_output"`  // output during detect process
	Version       string `json:"version"`
}

func NewDropletCreate(appGUID string) *DropletCreate {
	return &DropletCreate{
		Relationships: AppRelationship{
			App: ToOneRelationship{
				Data: &Relationship{
					GUID: appGUID,
				},
			},
		},
		ProcessTypes: make(map[string]string),
	}
}

func NewDropletCopy(appGUID string) *DropletCopy {
	return &DropletCopy{
		Relationships: AppRelationship{
			App: ToOneRelationship{
				Data: &Relationship{
					GUID: appGUID,
				},
			},
		},
	}
}
<|endoftext|>


<commit_before>package v2

import (
	"fmt"
	"os"

	"code.cloudfoundry.org/cli/cf/cmd"
	"code.cloudfoundry.org/cli/commands"
	"code.cloudfoundry.org/cli/commands/flags"
)

type AppCommand struct {
	RequiredArgs flags.AppName `positional-args:"yes"`
	GUID         bool          `long:"guid" description:"Retrieve and display the given app's guid.  All other health and status output for the app is suppressed."`
	usage        interface{}   `usage:"CF_NAME app APP_NAME"`
}

func (_ AppCommand) Setup(config commands.Config) error {
	return nil
}

func (_ AppCommand) Execute(args []string) error {
	fmt.Println("executing app command")
	cmd.Main(os.Getenv("CF_TRACE"), os.Args)
	return nil
}
<commit_msg>yeah, lets not keep debug output<commit_after>package v2

import (
	"os"

	"code.cloudfoundry.org/cli/cf/cmd"
	"code.cloudfoundry.org/cli/commands"
	"code.cloudfoundry.org/cli/commands/flags"
)

type AppCommand struct {
	RequiredArgs flags.AppName `positional-args:"yes"`
	GUID         bool          `long:"guid" description:"Retrieve and display the given app's guid.  All other health and status output for the app is suppressed."`
	usage        interface{}   `usage:"CF_NAME app APP_NAME"`
}

func (_ AppCommand) Setup(config commands.Config) error {
	return nil
}

func (_ AppCommand) Execute(args []string) error {
	cmd.Main(os.Getenv("CF_TRACE"), os.Args)
	return nil
}
<|endoftext|>


<commit_before>package aws

import (
	"fmt"
	"log"

	"github.com/hashicorp/terraform/helper/diff"
	"github.com/hashicorp/terraform/terraform"
	"github.com/mitchellh/goamz/ec2"
)

func resource_aws_eip_create(
	s *terraform.ResourceState,
	d *terraform.ResourceDiff,
	meta interface{}) (*terraform.ResourceState, error) {
	p := meta.(*ResourceProvider)
	ec2conn := p.ec2conn

	// Merge the diff into the state so that we have all the attributes
	// properly.
	rs := s.MergeDiff(d)

	// By default, we're not in a VPC
	vpc := false
	domainOpt := ""
	if rs.Attributes["vpc"] == "true" {
		vpc = true
		domainOpt = "vpc"
	}

	allocOpts := ec2.AllocateAddress{
		Domain: domainOpt,
	}

	log.Printf("[DEBUG] EIP create configuration: %#v", allocOpts)
	allocResp, err := ec2conn.AllocateAddress(&allocOpts)
	if err != nil {
		return nil, fmt.Errorf("Error creating EIP: %s", err)
	}

	// Assign the eips (unique) allocation id for use later
	// the EIP api has a conditional unique ID (really), so
	// if we're in a VPC we need to save the ID as such, otherwise
	// it defaults to using the public IP
	if vpc {
		rs.ID = allocResp.AllocationId
	} else {
		rs.ID = allocResp.PublicIp
	}

	log.Printf("[INFO] EIP ID: %s (vpc: %v)", rs.ID, vpc)

	// If we have an instance to register, do it
	instanceId := rs.Attributes["instance"]

	// Only register with an instance if we have one
	if instanceId != "" {
		assocOpts := ec2.AssociateAddress{
			InstanceId: instanceId,
			PublicIp:   rs.ID,
		}
		// more unique ID conditionals
		if vpc {
			assocOpts = ec2.AssociateAddress{
				InstanceId:   instanceId,
				AllocationId: rs.ID,
			}
		}

		log.Printf("[DEBUG] EIP associate configuration: %#v (vpc: %v)", assocOpts, vpc)
		_, err := ec2conn.AssociateAddress(&assocOpts)

		if err != nil {
			return rs, fmt.Errorf("Failure associating instances: %s", err)
		}
	}

	// Get the full address description for saving to state for
	// use in other resources
	assocIds := []string{}
	publicIps := []string{}
	if vpc {
		assocIds = []string{rs.ID}
	} else {
		publicIps = []string{rs.ID}
	}

	log.Printf("[DEBUG] EIP describe configuration: %#v, %#v (vpc: %v)", assocIds, publicIps, vpc)

	describeAddresses, err := ec2conn.Addresses(publicIps, assocIds, nil)

	if err != nil {
		return rs, fmt.Errorf("Error retrieving EIP: %s", err)
	}

	// Verify AWS returned our EIP
	if len(describeAddresses.Addresses) != 1 ||
		describeAddresses.Addresses[0].AllocationId != rs.ID {
		if err != nil {
			return rs, fmt.Errorf("Unable to find EIP: %#v", describeAddresses.Addresses)
		}
	}

	address := describeAddresses.Addresses[0]

	return resource_aws_eip_update_state(rs, &address)
}

func resource_aws_eip_update(
	s *terraform.ResourceState,
	d *terraform.ResourceDiff,
	meta interface{}) (*terraform.ResourceState, error) {

	rs := s.MergeDiff(d)
	log.Printf("ResourceDiff: %s", d)
	log.Printf("ResourceState: %s", s)
	log.Printf("Merged: %s", rs)

	return nil, fmt.Errorf("Did not update")
}

func resource_aws_eip_destroy(
	s *terraform.ResourceState,
	meta interface{}) error {

	return nil
}

func resource_aws_eip_refresh(
	s *terraform.ResourceState,
	meta interface{}) (*terraform.ResourceState, error) {

	address := &ec2.Address{}

	return resource_aws_eip_update_state(s, address)
}

func resource_aws_eip_diff(
	s *terraform.ResourceState,
	c *terraform.ResourceConfig,
	meta interface{}) (*terraform.ResourceDiff, error) {

	b := &diff.ResourceBuilder{
		Attrs: map[string]diff.AttrType{
			"vpc":      diff.AttrTypeCreate,
			"instance": diff.AttrTypeUpdate,
		},

		ComputedAttrs: []string{
			"public_ip",
			"private_ip",
		},
	}

	return b.Diff(s, c)
}

func resource_aws_eip_update_state(
	s *terraform.ResourceState,
	address *ec2.Address) (*terraform.ResourceState, error) {

	s.Attributes["private_ip"] = address.PrivateIpAddress
	s.Attributes["public_ip"] = address.PublicIp

	return s, nil
}
<commit_msg>providers/aws: eip destroy and drefresh<commit_after>package aws

import (
	"fmt"
	"log"

	"github.com/hashicorp/terraform/helper/diff"
	"github.com/hashicorp/terraform/terraform"
	"github.com/mitchellh/goamz/ec2"
)

func resource_aws_eip_create(
	s *terraform.ResourceState,
	d *terraform.ResourceDiff,
	meta interface{}) (*terraform.ResourceState, error) {
	p := meta.(*ResourceProvider)
	ec2conn := p.ec2conn

	// Merge the diff into the state so that we have all the attributes
	// properly.
	rs := s.MergeDiff(d)

	// By default, we're not in a VPC
	vpc := false
	domainOpt := ""
	if rs.Attributes["vpc"] == "true" {
		vpc = true
		domainOpt = "vpc"
	}

	allocOpts := ec2.AllocateAddress{
		Domain: domainOpt,
	}

	log.Printf("[DEBUG] EIP create configuration: %#v", allocOpts)
	allocResp, err := ec2conn.AllocateAddress(&allocOpts)
	if err != nil {
		return nil, fmt.Errorf("Error creating EIP: %s", err)
	}

	// Assign the eips (unique) allocation id for use later
	// the EIP api has a conditional unique ID (really), so
	// if we're in a VPC we need to save the ID as such, otherwise
	// it defaults to using the public IP
	if vpc {
		rs.ID = allocResp.AllocationId
	} else {
		rs.ID = allocResp.PublicIp
	}

	log.Printf("[INFO] EIP ID: %s (vpc: %v)", rs.ID, vpc)

	// If we have an instance to register, do it
	instanceId := rs.Attributes["instance"]

	// Only register with an instance if we have one
	if instanceId != "" {
		assocOpts := ec2.AssociateAddress{
			InstanceId: instanceId,
			PublicIp:   rs.ID,
		}
		// more unique ID conditionals
		if vpc {
			assocOpts = ec2.AssociateAddress{
				InstanceId:   instanceId,
				AllocationId: rs.ID,
			}
		}

		log.Printf("[DEBUG] EIP associate configuration: %#v (vpc: %v)", assocOpts, vpc)
		_, err := ec2conn.AssociateAddress(&assocOpts)

		if err != nil {
			return rs, fmt.Errorf("Failure associating instances: %s", err)
		}
	}

	// Get the full address description for saving to state for
	// use in other resources
	assocIds := []string{}
	publicIps := []string{}
	if vpc {
		assocIds = []string{rs.ID}
	} else {
		publicIps = []string{rs.ID}
	}

	log.Printf("[DEBUG] EIP describe configuration: %#v, %#v (vpc: %v)", assocIds, publicIps, vpc)

	describeAddresses, err := ec2conn.Addresses(publicIps, assocIds, nil)

	if err != nil {
		return rs, fmt.Errorf("Error retrieving EIP: %s", err)
	}

	// Verify AWS returned our EIP
	if len(describeAddresses.Addresses) != 1 ||
		describeAddresses.Addresses[0].AllocationId != rs.ID {
		if err != nil {
			return rs, fmt.Errorf("Unable to find EIP: %#v", describeAddresses.Addresses)
		}
	}

	address := describeAddresses.Addresses[0]

	return resource_aws_eip_update_state(rs, &address)
}

func resource_aws_eip_update(
	s *terraform.ResourceState,
	d *terraform.ResourceDiff,
	meta interface{}) (*terraform.ResourceState, error) {

	rs := s.MergeDiff(d)
	log.Printf("ResourceDiff: %s", d)
	log.Printf("ResourceState: %s", s)
	log.Printf("Merged: %s", rs)

	return nil, fmt.Errorf("Did not update")
}

func resource_aws_eip_destroy(
	s *terraform.ResourceState,
	meta interface{}) error {
	p := meta.(*ResourceProvider)
	ec2conn := p.ec2conn

	log.Printf("[DEBUG] EIP release (destroy) address: %v", s.ID)

	_, err := ec2conn.ReleaseAddress(s.ID)

	return err
}

func resource_aws_eip_refresh(
	s *terraform.ResourceState,
	meta interface{}) (*terraform.ResourceState, error) {
	p := meta.(*ResourceProvider)
	ec2conn := p.ec2conn

	vpc := false
	if s.Attributes["vpc"] == "true" {
		vpc = true
	}

	// Get the full address description for saving to state for
	// use in other resources
	assocIds := []string{}
	publicIps := []string{}
	if vpc {
		assocIds = []string{s.ID}
	} else {
		publicIps = []string{s.ID}
	}

	log.Printf("[DEBUG] EIP describe configuration: %#v, %#v (vpc: %v)", assocIds, publicIps, vpc)

	describeAddresses, err := ec2conn.Addresses(publicIps, assocIds, nil)

	if err != nil {
		return s, fmt.Errorf("Error retrieving EIP: %s", err)
	}

	// Verify AWS returned our EIP
	if len(describeAddresses.Addresses) != 1 ||
		describeAddresses.Addresses[0].AllocationId != s.ID {
		if err != nil {
			return s, fmt.Errorf("Unable to find EIP: %#v", describeAddresses.Addresses)
		}
	}

	address := describeAddresses.Addresses[0]

	return resource_aws_eip_update_state(s, &address)
}

func resource_aws_eip_diff(
	s *terraform.ResourceState,
	c *terraform.ResourceConfig,
	meta interface{}) (*terraform.ResourceDiff, error) {

	b := &diff.ResourceBuilder{
		Attrs: map[string]diff.AttrType{
			"vpc":      diff.AttrTypeCreate,
			"instance": diff.AttrTypeUpdate,
		},

		ComputedAttrs: []string{
			"public_ip",
			"private_ip",
		},
	}

	return b.Diff(s, c)
}

func resource_aws_eip_update_state(
	s *terraform.ResourceState,
	address *ec2.Address) (*terraform.ResourceState, error) {

	s.Attributes["private_ip"] = address.PrivateIpAddress
	s.Attributes["public_ip"] = address.PublicIp

	return s, nil
}
<|endoftext|>


<commit_before>package bitsgo

import (
	"bytes"
	"crypto/sha1"
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"strings"
	"time"

	"github.com/petergtz/bitsgo/logger"
)

type MetricsService interface {
	SendTimingMetric(name string, duration time.Duration)
}

type StateForbiddenError struct {
	error
}

func NewStateForbiddenError() *StateForbiddenError {
	return &StateForbiddenError{fmt.Errorf("StateForbiddenError")}
}

type Updater interface {
	NotifyProcessingUpload(guid string) error
	NotifyUploadSucceeded(guid string, sha1 string, sha2 string) error
	NotifyUploadFailed(guid string, e error) error
}

type NullUpdater struct{}

func (u *NullUpdater) NotifyProcessingUpload(guid string) error                          { return nil }
func (u *NullUpdater) NotifyUploadSucceeded(guid string, sha1 string, sha2 string) error { return nil }
func (u *NullUpdater) NotifyUploadFailed(guid string, e error) error                     { return nil }

type ResourceHandler struct {
	blobstore        Blobstore
	resourceType     string
	metricsService   MetricsService
	maxBodySizeLimit uint64
	updater          Updater
}

type responseBody struct {
	Guid      string    `json:"guid"`
	State     string    `json:"state"`
	Type      string    `json:"type"`
	CreatedAt time.Time `json:"created_at"`
}

func NewResourceHandler(blobstore Blobstore, resourceType string, metricsService MetricsService, maxBodySizeLimit uint64) *ResourceHandler {
	return &ResourceHandler{
		blobstore:        blobstore,
		resourceType:     resourceType,
		metricsService:   metricsService,
		maxBodySizeLimit: maxBodySizeLimit,
		updater:          &NullUpdater{},
	}
}

func NewResourceHandlerWithUpdater(blobstore Blobstore, updater Updater, resourceType string, metricsService MetricsService, maxBodySizeLimit uint64) *ResourceHandler {
	return &ResourceHandler{
		blobstore:        blobstore,
		resourceType:     resourceType,
		metricsService:   metricsService,
		maxBodySizeLimit: maxBodySizeLimit,
		updater:          updater,
	}
}

// TODO: instead of params, we could use `identifier string` to make the interface more type-safe.
//       Here and in the other methods.
func (handler *ResourceHandler) AddOrReplaceWithDigestInHeader(responseWriter http.ResponseWriter, request *http.Request, params map[string]string) {
	if !HandleBodySizeLimits(responseWriter, request, handler.maxBodySizeLimit) {
		return
	}
	logger.From(request).Debugw("Octet-Stream")

	digest := request.Header.Get("Digest")
	if digest == "" {
		badRequest(responseWriter, "No Digest header")
		return
	}
	parts := strings.Split(digest, "=")
	if len(parts) != 2 {
		badRequest(responseWriter, "Digest must have format sha256=value, but is "+digest)
		return
	}
	alg, value := strings.ToLower(parts[0]), parts[1]
	if alg != "sha256" {
		badRequest(responseWriter, "Digest must have format sha256=value, but is "+digest)
		return
	}
	if value == "" {
		badRequest(responseWriter, "Digest must have format sha256=value. Value cannot be empty")
		return
	}

	// TODO this can cause an out of memory panic. Should be smart about writing big files to disk instead.
	content, e := ioutil.ReadAll(request.Body)
	if e != nil {
		internalServerError(responseWriter, e)
		return
	}
	e = handler.blobstore.Put(params["identifier"]+"/"+value, bytes.NewReader(content))
	// TODO use Clock instead:
	writeResponseBasedOn("", e, responseWriter, http.StatusCreated, nil, &responseBody{Guid: params["identifier"], State: "READY", Type: "bits", CreatedAt: time.Now()}, "")
}

// TODO: instead of params, we could use `identifier string` to make the interface more type-safe.
//       Here and in the other methods.
func (handler *ResourceHandler) AddOrReplace(responseWriter http.ResponseWriter, request *http.Request, params map[string]string) {
	if !HandleBodySizeLimits(responseWriter, request, handler.maxBodySizeLimit) {
		return
	}
	file, _, e := request.FormFile(handler.resourceType)
	if e == http.ErrMissingFile {
		file, _, e = request.FormFile("bits")
	}
	if e == http.ErrMissingFile {
		badRequest(responseWriter, "Could not retrieve form parameter '%s' or 'bits", handler.resourceType)
		return
	}
	if e != nil {
		internalServerError(responseWriter, e)
		return
	}
	defer file.Close()

	e = handler.updater.NotifyProcessingUpload(params["identifier"])
	if handleNotificationError(e, responseWriter) {
		return
	}
	buffer, e := ioutil.ReadAll(file)
	if e != nil {
		internalServerError(responseWriter, e)
		return
	}

	startTime := time.Now()
	e = handler.blobstore.Put(params["identifier"], bytes.NewReader(buffer))
	handler.metricsService.SendTimingMetric(handler.resourceType+"-cp_to_blobstore-time", time.Since(startTime))

	if e != nil {
		logger.From(request).Infow("Failed to upload to blobstore.", "error", e)
		notifyErr := handler.updater.NotifyUploadFailed(params["identifier"], e)
		if notifyErr != nil {
			logger.From(request).Errorw("Failed to notifying CC about failed upload.", "error", notifyErr)
		}
		if _, noSpaceLeft := e.(*NoSpaceLeftError); noSpaceLeft {
			http.Error(responseWriter, descriptionAndCodeAsJSON("500000", "Request Entity Too Large"), http.StatusInsufficientStorage)
			return
		}
		internalServerError(responseWriter, e)
		return
	}
	sha1, sha256 := sha1.Sum(buffer), sha256.Sum256(buffer)
	e = handler.updater.NotifyUploadSucceeded(params["identifier"], hex.EncodeToString(sha1[:]), hex.EncodeToString(sha256[:]))
	if e != nil {
		internalServerError(responseWriter, e)
		return
	}

	writeResponseBasedOn("", nil, responseWriter, http.StatusCreated, nil, &responseBody{Guid: params["identifier"], State: "READY", Type: "bits", CreatedAt: time.Now()}, "")
}

func handleNotificationError(e error, responseWriter http.ResponseWriter) (wasError bool) {
	switch e.(type) {
	case *StateForbiddenError:
		responseWriter.WriteHeader(http.StatusBadRequest)
		fprintDescriptionAndCodeAsJSON(responseWriter, "290008", "Cannot update an existing package.")
		return true
	case *NotFoundError:
		responseWriter.WriteHeader(http.StatusNotFound)
		fprintDescriptionAndCodeAsJSON(responseWriter, "10010", e.Error())
		return true
	case error:
		internalServerError(responseWriter, e)
		return true
	}
	return false
}

func (handler *ResourceHandler) CopySourceGuid(responseWriter http.ResponseWriter, request *http.Request, params map[string]string) {
	if !HandleBodySizeLimits(responseWriter, request, handler.maxBodySizeLimit) {
		return
	}
	sourceGuid := sourceGuidFrom(request.Body, responseWriter)
	if sourceGuid == "" {
		return // response is already handled in sourceGuidFrom
	}
	e := handler.blobstore.Copy(sourceGuid, params["identifier"])
	// TODO use Clock instead:
	writeResponseBasedOn("", e, responseWriter, http.StatusCreated, nil, &responseBody{Guid: params["identifier"], State: "READY", Type: "bits", CreatedAt: time.Now()}, "")
}

func sourceGuidFrom(body io.ReadCloser, responseWriter http.ResponseWriter) string {
	content, e := ioutil.ReadAll(body)
	if e != nil {
		internalServerError(responseWriter, e)
		return ""
	}
	var payload struct {
		SourceGuid string `json:"source_guid"`
	}
	e = json.Unmarshal(content, &payload)
	if e != nil {
		badRequest(responseWriter, "Body must be valid JSON when request is not multipart/form-data. %+v", e)
		return ""
	}
	return payload.SourceGuid
}

func (handler *ResourceHandler) HeadOrRedirectAsGet(responseWriter http.ResponseWriter, request *http.Request, params map[string]string) {
	redirectLocation, e := handler.blobstore.HeadOrRedirectAsGet(params["identifier"])
	writeResponseBasedOn(redirectLocation, e, responseWriter, http.StatusOK, nil, nil, "")
}

func (handler *ResourceHandler) Get(responseWriter http.ResponseWriter, request *http.Request, params map[string]string) {
	body, redirectLocation, e := handler.blobstore.GetOrRedirect(params["identifier"])
	writeResponseBasedOn(redirectLocation, e, responseWriter, http.StatusOK, body, nil, request.Header.Get("If-None-Modify"))
}

func (handler *ResourceHandler) Delete(responseWriter http.ResponseWriter, request *http.Request, params map[string]string) {
	// TODO nothing should be S3 specific here
	// this check is needed, because S3 does not return a NotFound on a Delete request:
	exists, e := handler.blobstore.Exists(params["identifier"])
	if e != nil {
		internalServerError(responseWriter, e)
		return
	}
	if !exists {
		responseWriter.WriteHeader(http.StatusNotFound)
		return
	}
	e = handler.blobstore.Delete(params["identifier"])
	writeResponseBasedOn("", e, responseWriter, http.StatusNoContent, nil, nil, "")
}

func (handler *ResourceHandler) DeleteDir(responseWriter http.ResponseWriter, request *http.Request, params map[string]string) {
	e := handler.blobstore.DeleteDir(params["identifier"])
	switch e.(type) {
	case *NotFoundError:
		responseWriter.WriteHeader(http.StatusNoContent)
		return
	}
	writeResponseBasedOn("", e, responseWriter, http.StatusNoContent, nil, nil, "")
}

var emptyReader = ioutil.NopCloser(bytes.NewReader(nil))

func writeResponseBasedOn(redirectLocation string, e error, responseWriter http.ResponseWriter, statusCode int, body io.ReadCloser, jsonBody *responseBody, ifNoneModify string) {
	switch e.(type) {
	case *NotFoundError:
		responseWriter.WriteHeader(http.StatusNotFound)
		return
	case *NoSpaceLeftError:
		http.Error(responseWriter, descriptionAndCodeAsJSON("500000", "Request Entity Too Large"), http.StatusInsufficientStorage)
		return
	case error:
		internalServerError(responseWriter, e)
		return
	}
	if redirectLocation != "" {
		redirect(responseWriter, redirectLocation)
		return
	}
	if body != nil {
		defer body.Close()
		var buffer bytes.Buffer
		sha := sha1.New()
		_, e := io.Copy(io.MultiWriter(&buffer, sha), body)
		if e != nil {
			internalServerError(responseWriter, e)
			return
		}
		eTag := hex.EncodeToString(sha.Sum(nil))
		logger.Log.Debugw("Cache check", "if-none-modify", ifNoneModify, "etag", eTag)
		responseWriter.Header().Set("ETag", eTag)
		if ifNoneModify == eTag {
			responseWriter.WriteHeader(http.StatusNotModified)
			return
		}
		responseWriter.WriteHeader(statusCode)
		io.Copy(responseWriter, &buffer)
		return
	}
	if jsonBody != nil {
		respBody, marshallingErr := json.Marshal(jsonBody)
		if marshallingErr != nil {
			internalServerError(responseWriter, marshallingErr)
			return
		}
		responseWriter.WriteHeader(statusCode)
		responseWriter.Write(respBody)
		return
	}
	responseWriter.WriteHeader(statusCode)
}

func redirect(responseWriter http.ResponseWriter, redirectLocation string) {
	responseWriter.Header().Set("Location", redirectLocation)
	responseWriter.WriteHeader(http.StatusFound)
}

func internalServerError(responseWriter http.ResponseWriter, e error) {
	logger.Log.Errorw("Internal Server Error.", "error", fmt.Sprintf("%+v", e))
	responseWriter.WriteHeader(http.StatusInternalServerError)
}

func badRequest(responseWriter http.ResponseWriter, message string, args ...interface{}) {
	responseBody := fmt.Sprintf(message, args...)
	logger.Log.Infow("Bad request", "body", responseBody)
	responseWriter.WriteHeader(http.StatusBadRequest)
	fprintDescriptionAndCodeAsJSON(responseWriter, "290003", message, args...)
}

func fprintDescriptionAndCodeAsJSON(responseWriter http.ResponseWriter, code string, description string, a ...interface{}) {
	fmt.Fprintf(responseWriter, descriptionAndCodeAsJSON(code, description, a...))
}

func descriptionAndCodeAsJSON(code string, description string, a ...interface{}) string {
	return fmt.Sprintf(`{"description":"%v","code":%v}`, fmt.Sprintf(description, a...), code)
}
<commit_msg>Write uploaded resources to temp file for further handling<commit_after>package bitsgo

import (
	"bytes"
	"crypto/sha1"
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"strings"
	"time"

	"github.com/petergtz/bitsgo/logger"
	"github.com/pkg/errors"
)

type MetricsService interface {
	SendTimingMetric(name string, duration time.Duration)
}

type StateForbiddenError struct {
	error
}

func NewStateForbiddenError() *StateForbiddenError {
	return &StateForbiddenError{fmt.Errorf("StateForbiddenError")}
}

type Updater interface {
	NotifyProcessingUpload(guid string) error
	NotifyUploadSucceeded(guid string, sha1 string, sha2 string) error
	NotifyUploadFailed(guid string, e error) error
}

type NullUpdater struct{}

func (u *NullUpdater) NotifyProcessingUpload(guid string) error                          { return nil }
func (u *NullUpdater) NotifyUploadSucceeded(guid string, sha1 string, sha2 string) error { return nil }
func (u *NullUpdater) NotifyUploadFailed(guid string, e error) error                     { return nil }

type ResourceHandler struct {
	blobstore        Blobstore
	resourceType     string
	metricsService   MetricsService
	maxBodySizeLimit uint64
	updater          Updater
}

type responseBody struct {
	Guid      string    `json:"guid"`
	State     string    `json:"state"`
	Type      string    `json:"type"`
	CreatedAt time.Time `json:"created_at"`
}

func NewResourceHandler(blobstore Blobstore, resourceType string, metricsService MetricsService, maxBodySizeLimit uint64) *ResourceHandler {
	return &ResourceHandler{
		blobstore:        blobstore,
		resourceType:     resourceType,
		metricsService:   metricsService,
		maxBodySizeLimit: maxBodySizeLimit,
		updater:          &NullUpdater{},
	}
}

func NewResourceHandlerWithUpdater(blobstore Blobstore, updater Updater, resourceType string, metricsService MetricsService, maxBodySizeLimit uint64) *ResourceHandler {
	return &ResourceHandler{
		blobstore:        blobstore,
		resourceType:     resourceType,
		metricsService:   metricsService,
		maxBodySizeLimit: maxBodySizeLimit,
		updater:          updater,
	}
}

// TODO: instead of params, we could use `identifier string` to make the interface more type-safe.
//       Here and in the other methods.
func (handler *ResourceHandler) AddOrReplaceWithDigestInHeader(responseWriter http.ResponseWriter, request *http.Request, params map[string]string) {
	if !HandleBodySizeLimits(responseWriter, request, handler.maxBodySizeLimit) {
		return
	}
	logger.From(request).Debugw("Octet-Stream")

	digest := request.Header.Get("Digest")
	if digest == "" {
		badRequest(responseWriter, "No Digest header")
		return
	}
	parts := strings.Split(digest, "=")
	if len(parts) != 2 {
		badRequest(responseWriter, "Digest must have format sha256=value, but is "+digest)
		return
	}
	alg, value := strings.ToLower(parts[0]), parts[1]
	if alg != "sha256" {
		badRequest(responseWriter, "Digest must have format sha256=value, but is "+digest)
		return
	}
	if value == "" {
		badRequest(responseWriter, "Digest must have format sha256=value. Value cannot be empty")
		return
	}

	// TODO this can cause an out of memory panic. Should be smart about writing big files to disk instead.
	content, e := ioutil.ReadAll(request.Body)
	if e != nil {
		internalServerError(responseWriter, e)
		return
	}
	e = handler.blobstore.Put(params["identifier"]+"/"+value, bytes.NewReader(content))
	// TODO use Clock instead:
	writeResponseBasedOn("", e, responseWriter, http.StatusCreated, nil, &responseBody{Guid: params["identifier"], State: "READY", Type: "bits", CreatedAt: time.Now()}, "")
}

// TODO: instead of params, we could use `identifier string` to make the interface more type-safe.
//       Here and in the other methods.
func (handler *ResourceHandler) AddOrReplace(responseWriter http.ResponseWriter, request *http.Request, params map[string]string) {
	if !HandleBodySizeLimits(responseWriter, request, handler.maxBodySizeLimit) {
		return
	}
	file, _, e := request.FormFile(handler.resourceType)
	if e == http.ErrMissingFile {
		file, _, e = request.FormFile("bits")
	}
	if e == http.ErrMissingFile {
		badRequest(responseWriter, "Could not retrieve form parameter '%s' or 'bits", handler.resourceType)
		return
	}
	if e != nil {
		internalServerError(responseWriter, e)
		return
	}
	defer file.Close()

	e = handler.updater.NotifyProcessingUpload(params["identifier"])
	if handleNotificationError(e, responseWriter) {
		return
	}

	tempFilename, e := CreateTempFileWithContent(file)
	if e != nil {
		internalServerError(responseWriter, e)
		return
	}

	// Should go into (Go-)Routine
	defer os.Remove(tempFilename)

	tempFile, e := os.Open(tempFilename)
	if e != nil {
		internalServerError(responseWriter, e)
		return
	}
	defer tempFile.Close()

	startTime := time.Now()
	e = handler.blobstore.Put(params["identifier"], tempFile)
	handler.metricsService.SendTimingMetric(handler.resourceType+"-cp_to_blobstore-time", time.Since(startTime))

	if e != nil {
		logger.From(request).Infow("Failed to upload to blobstore.", "error", e)
		notifyErr := handler.updater.NotifyUploadFailed(params["identifier"], e)
		if notifyErr != nil {
			logger.From(request).Errorw("Failed to notifying CC about failed upload.", "error", notifyErr)
		}
		if _, noSpaceLeft := e.(*NoSpaceLeftError); noSpaceLeft {
			http.Error(responseWriter, descriptionAndCodeAsJSON("500000", "Request Entity Too Large"), http.StatusInsufficientStorage)
			return
		}
		internalServerError(responseWriter, e)
		return
	}
	sha1, sha256, e := ShaSums(tempFilename)
	if e != nil {
		notifyErr := handler.updater.NotifyUploadFailed(params["identifier"], e)
		if notifyErr != nil {
			logger.From(request).Errorw("Failed to notifying CC about failed upload.", "error", notifyErr)
		}
		internalServerError(responseWriter, e)
		return
	}
	e = handler.updater.NotifyUploadSucceeded(params["identifier"], hex.EncodeToString(sha1), hex.EncodeToString(sha256))
	if e != nil {
		internalServerError(responseWriter, e)
		return
	}

	writeResponseBasedOn("", nil, responseWriter, http.StatusCreated, nil, &responseBody{Guid: params["identifier"], State: "READY", Type: "bits", CreatedAt: time.Now()}, "")
}

func CreateTempFileWithContent(reader io.Reader) (string, error) {
	uploadedFile, e := ioutil.TempFile("", "bits")
	if e != nil {
		return "", errors.WithStack(e)
	}
	_, e = io.Copy(uploadedFile, reader)
	if e != nil {
		uploadedFile.Close()
		return "", errors.WithStack(e)
	}
	uploadedFile.Close()

	return uploadedFile.Name(), nil
}

func ShaSums(filename string) (sha1Sum []byte, sha256Sum []byte, e error) {
	file, e := os.Open(filename)
	if e != nil {
		return nil, nil, errors.WithStack(e)
	}
	defer file.Close()
	sha1Hash := sha1.New()
	sha256Hash := sha256.New()
	_, e = io.Copy(io.MultiWriter(sha1Hash, sha256Hash), file)
	if e != nil {
		return nil, nil, errors.WithStack(e)
	}
	return sha1Hash.Sum(nil), sha256Hash.Sum(nil), nil
}

func handleNotificationError(e error, responseWriter http.ResponseWriter) (wasError bool) {
	switch e.(type) {
	case *StateForbiddenError:
		responseWriter.WriteHeader(http.StatusBadRequest)
		fprintDescriptionAndCodeAsJSON(responseWriter, "290008", "Cannot update an existing package.")
		return true
	case *NotFoundError:
		responseWriter.WriteHeader(http.StatusNotFound)
		fprintDescriptionAndCodeAsJSON(responseWriter, "10010", e.Error())
		return true
	case error:
		internalServerError(responseWriter, e)
		return true
	}
	return false
}

func (handler *ResourceHandler) CopySourceGuid(responseWriter http.ResponseWriter, request *http.Request, params map[string]string) {
	if !HandleBodySizeLimits(responseWriter, request, handler.maxBodySizeLimit) {
		return
	}
	sourceGuid := sourceGuidFrom(request.Body, responseWriter)
	if sourceGuid == "" {
		return // response is already handled in sourceGuidFrom
	}
	e := handler.blobstore.Copy(sourceGuid, params["identifier"])
	// TODO use Clock instead:
	writeResponseBasedOn("", e, responseWriter, http.StatusCreated, nil, &responseBody{Guid: params["identifier"], State: "READY", Type: "bits", CreatedAt: time.Now()}, "")
}

func sourceGuidFrom(body io.ReadCloser, responseWriter http.ResponseWriter) string {
	content, e := ioutil.ReadAll(body)
	if e != nil {
		internalServerError(responseWriter, e)
		return ""
	}
	var payload struct {
		SourceGuid string `json:"source_guid"`
	}
	e = json.Unmarshal(content, &payload)
	if e != nil {
		badRequest(responseWriter, "Body must be valid JSON when request is not multipart/form-data. %+v", e)
		return ""
	}
	return payload.SourceGuid
}

func (handler *ResourceHandler) HeadOrRedirectAsGet(responseWriter http.ResponseWriter, request *http.Request, params map[string]string) {
	redirectLocation, e := handler.blobstore.HeadOrRedirectAsGet(params["identifier"])
	writeResponseBasedOn(redirectLocation, e, responseWriter, http.StatusOK, nil, nil, "")
}

func (handler *ResourceHandler) Get(responseWriter http.ResponseWriter, request *http.Request, params map[string]string) {
	body, redirectLocation, e := handler.blobstore.GetOrRedirect(params["identifier"])
	writeResponseBasedOn(redirectLocation, e, responseWriter, http.StatusOK, body, nil, request.Header.Get("If-None-Modify"))
}

func (handler *ResourceHandler) Delete(responseWriter http.ResponseWriter, request *http.Request, params map[string]string) {
	// TODO nothing should be S3 specific here
	// this check is needed, because S3 does not return a NotFound on a Delete request:
	exists, e := handler.blobstore.Exists(params["identifier"])
	if e != nil {
		internalServerError(responseWriter, e)
		return
	}
	if !exists {
		responseWriter.WriteHeader(http.StatusNotFound)
		return
	}
	e = handler.blobstore.Delete(params["identifier"])
	writeResponseBasedOn("", e, responseWriter, http.StatusNoContent, nil, nil, "")
}

func (handler *ResourceHandler) DeleteDir(responseWriter http.ResponseWriter, request *http.Request, params map[string]string) {
	e := handler.blobstore.DeleteDir(params["identifier"])
	switch e.(type) {
	case *NotFoundError:
		responseWriter.WriteHeader(http.StatusNoContent)
		return
	}
	writeResponseBasedOn("", e, responseWriter, http.StatusNoContent, nil, nil, "")
}

var emptyReader = ioutil.NopCloser(bytes.NewReader(nil))

func writeResponseBasedOn(redirectLocation string, e error, responseWriter http.ResponseWriter, statusCode int, body io.ReadCloser, jsonBody *responseBody, ifNoneModify string) {
	switch e.(type) {
	case *NotFoundError:
		responseWriter.WriteHeader(http.StatusNotFound)
		return
	case *NoSpaceLeftError:
		http.Error(responseWriter, descriptionAndCodeAsJSON("500000", "Request Entity Too Large"), http.StatusInsufficientStorage)
		return
	case error:
		internalServerError(responseWriter, e)
		return
	}
	if redirectLocation != "" {
		redirect(responseWriter, redirectLocation)
		return
	}
	if body != nil {
		defer body.Close()
		var buffer bytes.Buffer
		sha := sha1.New()
		_, e := io.Copy(io.MultiWriter(&buffer, sha), body)
		if e != nil {
			internalServerError(responseWriter, e)
			return
		}
		eTag := hex.EncodeToString(sha.Sum(nil))
		logger.Log.Debugw("Cache check", "if-none-modify", ifNoneModify, "etag", eTag)
		responseWriter.Header().Set("ETag", eTag)
		if ifNoneModify == eTag {
			responseWriter.WriteHeader(http.StatusNotModified)
			return
		}
		responseWriter.WriteHeader(statusCode)
		io.Copy(responseWriter, &buffer)
		return
	}
	if jsonBody != nil {
		respBody, marshallingErr := json.Marshal(jsonBody)
		if marshallingErr != nil {
			internalServerError(responseWriter, marshallingErr)
			return
		}
		responseWriter.WriteHeader(statusCode)
		responseWriter.Write(respBody)
		return
	}
	responseWriter.WriteHeader(statusCode)
}

func redirect(responseWriter http.ResponseWriter, redirectLocation string) {
	responseWriter.Header().Set("Location", redirectLocation)
	responseWriter.WriteHeader(http.StatusFound)
}

func internalServerError(responseWriter http.ResponseWriter, e error) {
	logger.Log.Errorw("Internal Server Error.", "error", fmt.Sprintf("%+v", e))
	responseWriter.WriteHeader(http.StatusInternalServerError)
}

func badRequest(responseWriter http.ResponseWriter, message string, args ...interface{}) {
	responseBody := fmt.Sprintf(message, args...)
	logger.Log.Infow("Bad request", "body", responseBody)
	responseWriter.WriteHeader(http.StatusBadRequest)
	fprintDescriptionAndCodeAsJSON(responseWriter, "290003", message, args...)
}

func fprintDescriptionAndCodeAsJSON(responseWriter http.ResponseWriter, code string, description string, a ...interface{}) {
	fmt.Fprintf(responseWriter, descriptionAndCodeAsJSON(code, description, a...))
}

func descriptionAndCodeAsJSON(code string, description string, a ...interface{}) string {
	return fmt.Sprintf(`{"description":"%v","code":%v}`, fmt.Sprintf(description, a...), code)
}
<|endoftext|>


<commit_before>// Copyright (C) 2015 Eiichiro Watanabe
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"fmt"
	"net"
	"reflect"
	"strconv"
	"strings"
	"sync"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/armon/go-radix"
	set "github.com/deckarep/golang-set"
	"github.com/grafov/bcast"
	"github.com/osrg/gobgp/packet"
)

const (
	UNKNOWN = iota
	REQ_LOAD
	REQ_RELOAD
	REQ_CURRENT_SERIAL
	REQ_CURRENT_LIST
	REQ_LIST_TO_BE_DELETED
	REQ_LIST_TO_BE_ADDED
	REQ_IF_SERIAL_EXISTS
	REQ_BEGIN_TRANSACTION
	REQ_END_TRANSACTION
)

type RequestType int

type Request struct {
	RequestType RequestType
	Key         interface{}
	Value       string
	Response    chan *Response
	transaction chan Request
}

type reqKeys struct {
	RF bgp.RouteFamily
	SN uint32
}

type FakeROA struct {
	Prefix    net.IP
	PrefixLen uint8
	MaxLen    uint8
	AS        uint32
}

type Response struct {
	Error error
	Data  interface{}
}

type ResourceManager struct {
	ch           chan Request
	serialNotify *bcast.Group
	init         sync.Once
}

func (r *ResourceManager) Load(files []string) error {
	r.init.Do(func() {
		go r.serialNotify.Broadcasting(0)
		r.ch = make(chan Request)
		go r.run()
	})
	result := make(chan *Response)
	r.ch <- Request{RequestType: REQ_LOAD, Key: files, Response: result}
	res := <-result
	return res.Error
}

func (r *ResourceManager) Reload() error {
	result := make(chan *Response)
	r.ch <- Request{RequestType: REQ_RELOAD, Response: result}
	res := <-result
	return res.Error
}

func (r *ResourceManager) CurrentSerial() uint32 {
	result := make(chan *Response)
	r.ch <- Request{RequestType: REQ_CURRENT_SERIAL, Response: result}
	res := <-result
	return res.Data.(uint32)
}

func (r *ResourceManager) CurrentList(rf bgp.RouteFamily) []*FakeROA {
	result := make(chan *Response)
	r.ch <- Request{RequestType: REQ_CURRENT_LIST, Key: reqKeys{RF: rf}, Response: result}
	res := <-result
	return res.Data.([]*FakeROA)
}

func (r *ResourceManager) ToBeDeleted(rf bgp.RouteFamily, sn uint32) []*FakeROA {
	result := make(chan *Response)
	r.ch <- Request{RequestType: REQ_LIST_TO_BE_DELETED, Key: reqKeys{RF: rf, SN: sn}, Response: result}
	res := <-result
	return res.Data.([]*FakeROA)
}

func (r *ResourceManager) ToBeAdded(rf bgp.RouteFamily, sn uint32) []*FakeROA {
	result := make(chan *Response)
	r.ch <- Request{RequestType: REQ_LIST_TO_BE_ADDED, Key: reqKeys{RF: rf, SN: sn}, Response: result}
	res := <-result
	return res.Data.([]*FakeROA)
}

func (r *ResourceManager) HasKey(sn uint32) bool {
	result := make(chan *Response)
	r.ch <- Request{RequestType: REQ_IF_SERIAL_EXISTS, Key: sn, Response: result}
	res := <-result
	return res.Data.(bool)
}

func (r *ResourceManager) BeginTransaction() *ResourceManager {
	result := make(chan *Response)
	t := make(chan Request)
	r.ch <- Request{RequestType: REQ_BEGIN_TRANSACTION, Response: result, transaction: t}
	return &ResourceManager{
		ch:           t,
		serialNotify: r.serialNotify,
		init:         r.init,
	}
}

func (r *ResourceManager) EndTransaction() {
	r.ch <- Request{RequestType: REQ_END_TRANSACTION}
}

func handleRequests(r *ResourceManager, rsrc *resource) {
	var err error
	for {
		req := <-r.ch
		switch req.RequestType {
		case REQ_LOAD:
			rsrc, err = newResource(req.Key.([]string))
			log.Infof("Resource has been loaded. (SN: %v)", rsrc.currentSN)
			req.Response <- &Response{Error: err}
		case REQ_CURRENT_SERIAL:
			req.Response <- &Response{Data: rsrc.currentSN}
		case REQ_RELOAD:
			serialNotify := false
			nextSN := uint32(time.Now().Unix())
			rsrc, err = rsrc.loadAs(nextSN)
			if err != nil {
				req.Response <- &Response{Data: nil, Error: err}
				break
			}

			if eql := reflect.DeepEqual(rsrc.table[rsrc.currentSN], rsrc.table[nextSN]); !eql {
				log.Infof("Resource has been updated. (SN: %v -> %v)", rsrc.currentSN, nextSN)
				rsrc.currentSN = nextSN
				serialNotify = true
			} else {
				delete(rsrc.table, nextSN)
			}

			for k, _ := range rsrc.table {
				if rsrc.currentSN != k {
					t := time.Now()
					if int64(k) < t.Add(-24*time.Hour).Unix() {
						delete(rsrc.table, k)
						log.Infof("Resource as of %v was expired. (SN: %v)", time.Unix(int64(k), 0).Format("2006/01/02 15:04:05"), k)
					}
				}
			}
			if serialNotify {
				r.serialNotify.Send(true)
			}
			req.Response <- &Response{Error: nil}
		case REQ_CURRENT_LIST:
			keys := req.Key.(reqKeys)
			req.Response <- &Response{Data: fakeROALists(rsrc, treeToSet(rsrc.table[rsrc.currentSN][keys.RF]))}
		case REQ_LIST_TO_BE_DELETED:
			keys := req.Key.(reqKeys)
			req.Response <- &Response{Data: fakeROALists(rsrc, treeToSet(rsrc.table[keys.SN][keys.RF]).Difference(treeToSet(rsrc.table[rsrc.currentSN][keys.RF])))}
		case REQ_LIST_TO_BE_ADDED:
			keys := req.Key.(reqKeys)
			req.Response <- &Response{Data: fakeROALists(rsrc, treeToSet(rsrc.table[rsrc.currentSN][keys.RF]).Difference(treeToSet(rsrc.table[keys.SN][keys.RF])))}
		case REQ_IF_SERIAL_EXISTS:
			_, ok := rsrc.table[req.Key.(uint32)]
			req.Response <- &Response{Data: ok}
		case REQ_BEGIN_TRANSACTION:
			transaction := &ResourceManager{ch: req.transaction}
			handleRequests(transaction, rsrc)
		case REQ_END_TRANSACTION:
			return
		}
	}
}

func fakeROALists(rsrc *resource, list set.Set) []*FakeROA {
	fakeROAs := make([]*FakeROA, 0)
	for _, item := range list.ToSlice() {
		addr, plen, mlen, asn := stringToValues(item.(string))
		fakeROAs = append(fakeROAs, &FakeROA{
			Prefix:    addr,
			PrefixLen: plen,
			MaxLen:    mlen,
			AS:        asn,
		})
	}
	return fakeROAs
}

func treeToSet(table *radix.Tree) set.Set {
	i := 0
	tableMap := make([]*prefixResource, table.Len())
	table.Walk(func(s string, v interface{}) bool {
		ct, _ := v.(*prefixResource)
		tableMap[i] = ct
		i++
		return false
	})
	result := set.NewSet()
	for _, x := range tableMap {
		for _, y := range x.values {
			for _, z := range y.asns {
				result.Add(fmt.Sprintf("%v/%v-%v-%d", x.prefix, x.prefixLen, y.maxLen, z))
			}
		}
	}
	return result
}

func stringToValues(str string) (net.IP, uint8, uint8, uint32) {
	arr := strings.Split(str, "-")
	_, n, _, _ := parseCIDR(arr[0])
	addr := n.IP
	m, _ := n.Mask.Size()
	maxLen, _ := strconv.ParseUint(arr[1], 10, 8)
	asn, _ := strconv.ParseUint(arr[2], 10, 32)
	return addr, uint8(m), uint8(maxLen), uint32(asn)
}

func (r *ResourceManager) run() {
	var rsrc *resource
	handleRequests(r, rsrc)
}
<commit_msg>Fix params<commit_after>// Copyright (C) 2015 Eiichiro Watanabe
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"fmt"
	"net"
	"reflect"
	"strconv"
	"strings"
	"sync"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/armon/go-radix"
	set "github.com/deckarep/golang-set"
	"github.com/grafov/bcast"
	"github.com/osrg/gobgp/packet"
)

const (
	UNKNOWN = iota
	REQ_LOAD
	REQ_RELOAD
	REQ_CURRENT_SERIAL
	REQ_CURRENT_LIST
	REQ_LIST_TO_BE_DELETED
	REQ_LIST_TO_BE_ADDED
	REQ_IF_SERIAL_EXISTS
	REQ_BEGIN_TRANSACTION
	REQ_END_TRANSACTION
)

type RequestType int

type Request struct {
	RequestType RequestType
	Key         interface{}
	Value       string
	Response    chan *Response
	transaction chan Request
}

type reqKeys struct {
	RF bgp.RouteFamily
	SN uint32
}

type FakeROA struct {
	Prefix    net.IP
	PrefixLen uint8
	MaxLen    uint8
	AS        uint32
}

type Response struct {
	Error error
	Data  interface{}
}

type ResourceManager struct {
	ch           chan Request
	serialNotify *bcast.Group
	init         sync.Once
}

func (r *ResourceManager) Load(files []string) error {
	r.init.Do(func() {
		go r.serialNotify.Broadcasting(0)
		r.ch = make(chan Request)
		go r.run()
	})
	result := make(chan *Response)
	r.ch <- Request{RequestType: REQ_LOAD, Key: files, Response: result}
	res := <-result
	return res.Error
}

func (r *ResourceManager) Reload() error {
	result := make(chan *Response)
	r.ch <- Request{RequestType: REQ_RELOAD, Response: result}
	res := <-result
	return res.Error
}

func (r *ResourceManager) CurrentSerial() uint32 {
	result := make(chan *Response)
	r.ch <- Request{RequestType: REQ_CURRENT_SERIAL, Response: result}
	res := <-result
	return res.Data.(uint32)
}

func (r *ResourceManager) CurrentList(rf bgp.RouteFamily) []*FakeROA {
	result := make(chan *Response)
	r.ch <- Request{RequestType: REQ_CURRENT_LIST, Key: reqKeys{RF: rf}, Response: result}
	res := <-result
	return res.Data.([]*FakeROA)
}

func (r *ResourceManager) ToBeDeleted(rf bgp.RouteFamily, sn uint32) []*FakeROA {
	result := make(chan *Response)
	r.ch <- Request{RequestType: REQ_LIST_TO_BE_DELETED, Key: reqKeys{RF: rf, SN: sn}, Response: result}
	res := <-result
	return res.Data.([]*FakeROA)
}

func (r *ResourceManager) ToBeAdded(rf bgp.RouteFamily, sn uint32) []*FakeROA {
	result := make(chan *Response)
	r.ch <- Request{RequestType: REQ_LIST_TO_BE_ADDED, Key: reqKeys{RF: rf, SN: sn}, Response: result}
	res := <-result
	return res.Data.([]*FakeROA)
}

func (r *ResourceManager) HasKey(sn uint32) bool {
	result := make(chan *Response)
	r.ch <- Request{RequestType: REQ_IF_SERIAL_EXISTS, Key: sn, Response: result}
	res := <-result
	return res.Data.(bool)
}

func (r *ResourceManager) BeginTransaction() *ResourceManager {
	result := make(chan *Response)
	t := make(chan Request)
	r.ch <- Request{RequestType: REQ_BEGIN_TRANSACTION, Response: result, transaction: t}
	return &ResourceManager{
		ch:           t,
		serialNotify: r.serialNotify,
		init:         r.init,
	}
}

func (r *ResourceManager) EndTransaction() {
	r.ch <- Request{RequestType: REQ_END_TRANSACTION}
}

func handleRequests(r *ResourceManager, rsrc *resource) {
	var err error
	for {
		req := <-r.ch
		switch req.RequestType {
		case REQ_LOAD:
			rsrc, err = newResource(req.Key.([]string))
			log.Infof("Resource has been loaded. (SN: %v)", rsrc.currentSN)
			req.Response <- &Response{Error: err}
		case REQ_CURRENT_SERIAL:
			req.Response <- &Response{Data: rsrc.currentSN}
		case REQ_RELOAD:
			serialNotify := false
			nextSN := uint32(time.Now().Unix())
			rsrc, err = rsrc.loadAs(nextSN)
			if err != nil {
				req.Response <- &Response{Error: err}
				break
			}

			if eql := reflect.DeepEqual(rsrc.table[rsrc.currentSN], rsrc.table[nextSN]); !eql {
				log.Infof("Resource has been updated. (SN: %v -> %v)", rsrc.currentSN, nextSN)
				rsrc.currentSN = nextSN
				serialNotify = true
			} else {
				delete(rsrc.table, nextSN)
			}

			for k, _ := range rsrc.table {
				if rsrc.currentSN != k {
					t := time.Now()
					if int64(k) < t.Add(-24*time.Hour).Unix() {
						delete(rsrc.table, k)
						log.Infof("Resource as of %v was expired. (SN: %v)", time.Unix(int64(k), 0).Format("2006/01/02 15:04:05"), k)
					}
				}
			}
			if serialNotify {
				r.serialNotify.Send(true)
			}
			req.Response <- &Response{Error: nil}
		case REQ_CURRENT_LIST:
			keys := req.Key.(reqKeys)
			req.Response <- &Response{Data: fakeROALists(rsrc, treeToSet(rsrc.table[rsrc.currentSN][keys.RF]))}
		case REQ_LIST_TO_BE_DELETED:
			keys := req.Key.(reqKeys)
			req.Response <- &Response{Data: fakeROALists(rsrc, treeToSet(rsrc.table[keys.SN][keys.RF]).Difference(treeToSet(rsrc.table[rsrc.currentSN][keys.RF])))}
		case REQ_LIST_TO_BE_ADDED:
			keys := req.Key.(reqKeys)
			req.Response <- &Response{Data: fakeROALists(rsrc, treeToSet(rsrc.table[rsrc.currentSN][keys.RF]).Difference(treeToSet(rsrc.table[keys.SN][keys.RF])))}
		case REQ_IF_SERIAL_EXISTS:
			_, ok := rsrc.table[req.Key.(uint32)]
			req.Response <- &Response{Data: ok}
		case REQ_BEGIN_TRANSACTION:
			transaction := &ResourceManager{ch: req.transaction}
			handleRequests(transaction, rsrc)
		case REQ_END_TRANSACTION:
			return
		}
	}
}

func fakeROALists(rsrc *resource, list set.Set) []*FakeROA {
	fakeROAs := make([]*FakeROA, 0)
	for _, item := range list.ToSlice() {
		addr, plen, mlen, asn := stringToValues(item.(string))
		fakeROAs = append(fakeROAs, &FakeROA{
			Prefix:    addr,
			PrefixLen: plen,
			MaxLen:    mlen,
			AS:        asn,
		})
	}
	return fakeROAs
}

func treeToSet(table *radix.Tree) set.Set {
	i := 0
	tableMap := make([]*prefixResource, table.Len())
	table.Walk(func(s string, v interface{}) bool {
		ct, _ := v.(*prefixResource)
		tableMap[i] = ct
		i++
		return false
	})
	result := set.NewSet()
	for _, x := range tableMap {
		for _, y := range x.values {
			for _, z := range y.asns {
				result.Add(fmt.Sprintf("%v/%v-%v-%d", x.prefix, x.prefixLen, y.maxLen, z))
			}
		}
	}
	return result
}

func stringToValues(str string) (net.IP, uint8, uint8, uint32) {
	arr := strings.Split(str, "-")
	_, n, _, _ := parseCIDR(arr[0])
	addr := n.IP
	m, _ := n.Mask.Size()
	maxLen, _ := strconv.ParseUint(arr[1], 10, 8)
	asn, _ := strconv.ParseUint(arr[2], 10, 32)
	return addr, uint8(m), uint8(maxLen), uint32(asn)
}

func (r *ResourceManager) run() {
	var rsrc *resource
	handleRequests(r, rsrc)
}
<|endoftext|>


<commit_before>package protocol

import (
	v2net "github.com/v2ray/v2ray-core/common/net"
	"github.com/v2ray/v2ray-core/common/uuid"
)

type RequestCommand byte

const (
	RequestCommandTCP = RequestCommand(0x01)
	RequestCommandUDP = RequestCommand(0x02)
)

const (
	RequestOptionChunkStream     = RequestOption(0x01)
	RequestOptionConnectionReuse = RequestOption(0x02)
)

type RequestOption byte

func (this RequestOption) Has(option RequestOption) bool {
	return (this & option) == option
}

func (this *RequestOption) Set(option RequestOption) {
	*this = (*this | option)
}

func (this *RequestOption) Clear(option RequestOption) {
	*this = (*this & (^option))
}

type RequestHeader struct {
	Version byte
	User    *User
	Command RequestCommand
	Option  RequestOption
	Address v2net.Address
	Port    v2net.Port
}

func (this *RequestHeader) Destination() v2net.Destination {
	if this.Command == RequestCommandUDP {
		return v2net.UDPDestination(this.Address, this.Port)
	}
	return v2net.TCPDestination(this.Address, this.Port)
}

type ResponseOption byte

var (
	ResponseOptionConnectionReuse = ResponseOption(1)
)

func (this *ResponseOption) Set(option ResponseOption) {
	*this = (*this | option)
}

func (this ResponseOption) Has(option ResponseOption) bool {
	return (this | option) == option
}

func (this *ResponseOption) Clear(option ResponseOption) {
	*this = (*this & (^option))
}

type ResponseCommand interface{}

type ResponseHeader struct {
	Option  ResponseOption
	Command ResponseCommand
}

type CommandSwitchAccount struct {
	Host     v2net.Address
	Port     v2net.Port
	ID       *uuid.UUID
	AlterIds uint16
	Level    UserLevel
	ValidMin byte
}
<commit_msg>var -> const<commit_after>package protocol

import (
	v2net "github.com/v2ray/v2ray-core/common/net"
	"github.com/v2ray/v2ray-core/common/uuid"
)

type RequestCommand byte

const (
	RequestCommandTCP = RequestCommand(0x01)
	RequestCommandUDP = RequestCommand(0x02)
)

type RequestOption byte

const (
	RequestOptionChunkStream     = RequestOption(0x01)
	RequestOptionConnectionReuse = RequestOption(0x02)
)

func (this RequestOption) Has(option RequestOption) bool {
	return (this & option) == option
}

func (this *RequestOption) Set(option RequestOption) {
	*this = (*this | option)
}

func (this *RequestOption) Clear(option RequestOption) {
	*this = (*this & (^option))
}

type RequestHeader struct {
	Version byte
	User    *User
	Command RequestCommand
	Option  RequestOption
	Address v2net.Address
	Port    v2net.Port
}

func (this *RequestHeader) Destination() v2net.Destination {
	if this.Command == RequestCommandUDP {
		return v2net.UDPDestination(this.Address, this.Port)
	}
	return v2net.TCPDestination(this.Address, this.Port)
}

type ResponseOption byte

const (
	ResponseOptionConnectionReuse = ResponseOption(1)
)

func (this *ResponseOption) Set(option ResponseOption) {
	*this = (*this | option)
}

func (this ResponseOption) Has(option ResponseOption) bool {
	return (this | option) == option
}

func (this *ResponseOption) Clear(option ResponseOption) {
	*this = (*this & (^option))
}

type ResponseCommand interface{}

type ResponseHeader struct {
	Option  ResponseOption
	Command ResponseCommand
}

type CommandSwitchAccount struct {
	Host     v2net.Address
	Port     v2net.Port
	ID       *uuid.UUID
	AlterIds uint16
	Level    UserLevel
	ValidMin byte
}
<|endoftext|>


<commit_before>// Copyright © 2016 Jip J. Dekker <jip@dekker.li>
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package compiler

import (
	"os"
	"path/filepath"

	log "github.com/Sirupsen/logrus"
	"github.com/jjdekker/ponder/helpers"
	"github.com/jjdekker/ponder/settings"
)

// compilePath calls the given function on all files that aren't in any
// of the ignored directories
func compilePath(root string, opts *settings.Settings,
	f func(string, os.FileInfo) error) filepath.WalkFunc {
	return func(path string, info os.FileInfo, walkError error) error {
		// Handle walking error
		if walkError != nil {
			log.WithFields(log.Fields{
				"error": walkError,
				"path":  path,
			}).Warning("error occurred transversing project path")
			return nil
		}

		if info.IsDir() {
			// Skip directories that are ignored
			relPath, err := filepath.Rel(root, path)
			helpers.Check(err, "Unable to create relative Path")
			for _, dir := range append(append(opts.IgnoreDirs, opts.LilypondIncludes...), opts.OutputDir) {
				if relPath == dir || (filepath.IsAbs(dir) && path == dir) {
					log.WithFields(log.Fields{"path": path}).Debug("Ignoring directory")
					return filepath.SkipDir
				}
			}
		} else {
			// Call function on non-directory
			err := f(path, info)
			if err != nil {
				log.WithFields(log.Fields{
					"error": err,
					"path":  path,
				}).Error("error occured processing files")
			}
		}
		return nil
	}
}
<commit_msg>Adds a function to get the output path for files<commit_after>// Copyright © 2016 Jip J. Dekker <jip@dekker.li>
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package compiler

import (
	"os"
	"path/filepath"
	"strings"

	log "github.com/Sirupsen/logrus"
	"github.com/jjdekker/ponder/helpers"
	"github.com/jjdekker/ponder/settings"
)

// compilePath calls the given function on all files that aren't in any
// of the ignored directories
func compilePath(root string, opts *settings.Settings,
	f func(string, os.FileInfo) error) filepath.WalkFunc {
	return func(path string, info os.FileInfo, walkError error) error {
		// Handle walking error
		if walkError != nil {
			log.WithFields(log.Fields{
				"error": walkError,
				"path":  path,
			}).Warning("error occurred transversing project path")
			return nil
		}

		if info.IsDir() {
			// Skip directories that are ignored
			relPath, err := filepath.Rel(root, path)
			helpers.Check(err, "Unable to create relative Path")
			for _, dir := range append(append(opts.IgnoreDirs, opts.LilypondIncludes...), opts.OutputDir) {
				if relPath == dir || (filepath.IsAbs(dir) && path == dir) {
					log.WithFields(log.Fields{"path": path}).Debug("Ignoring directory")
					return filepath.SkipDir
				}
			}
		} else {
			// Call function on non-directory
			err := f(path, info)
			if err != nil {
				log.WithFields(log.Fields{
					"error": err,
					"path":  path,
				}).Error("error occured processing files")
			}
		}
		return nil
	}
}

// outputPath returns the path that the compiled file will take
func outputPath(source string, opts *settings.Settings) string {
	file := filepath.Base(source)
	dot := strings.LastIndex(file, ".")
	if dot == -1 {
		log.WithFields(log.Fields{"path": source}).Error("Unable to compute output path")
	}
	file = file[:dot] + "pdf"
	return filepath.Join(opts.OutputDir, file)
}
<|endoftext|>


<commit_before>package river

import (
	"flag"
	"fmt"
	"os"
	"testing"
	"time"

	"github.com/siddontang/go-mysql-elasticsearch/elastic"
	"github.com/siddontang/go-mysql/client"
	. "gopkg.in/check.v1"
)

var my_addr = flag.String("my_addr", "127.0.0.1:3306", "MySQL addr")
var es_addr = flag.String("es_addr", "127.0.0.1:9200", "Elasticsearch addr")

func Test(t *testing.T) {
	TestingT(t)
}

type riverTestSuite struct {
	c *client.Conn
	r *River
}

var _ = Suite(&riverTestSuite{})

func (s *riverTestSuite) SetUpSuite(c *C) {
	var err error
	s.c, err = client.Connect(*my_addr, "root", "", "test")
	c.Assert(err, IsNil)

	s.testExecute(c, "SET SESSION binlog_format = 'ROW'")

	schema := `
        CREATE TABLE IF NOT EXISTS %s (
            id INT,
            title VARCHAR(256),
            content VARCHAR(256),
            mylist VARCHAR(256),
            tenum ENUM("e1", "e2", "e3"),
            tset SET("a", "b", "c"),
            tbit BIT(1) default 1,
            PRIMARY KEY(id)) ENGINE=INNODB;
    `

	s.testExecute(c, "DROP TABLE IF EXISTS test_river")
	s.testExecute(c, fmt.Sprintf(schema, "test_river"))
	s.testExecute(c, fmt.Sprintf(schema, "test_for_id"))

	for i := 0; i < 10; i++ {
		table := fmt.Sprintf("test_river_%04d", i)
		s.testExecute(c, fmt.Sprintf("DROP TABLE IF EXISTS %s", table))
		s.testExecute(c, fmt.Sprintf(schema, table))
	}

	cfg := new(Config)
	cfg.MyAddr = *my_addr
	cfg.MyUser = "root"
	cfg.MyPassword = ""
	cfg.MyCharset = "utf8"
	cfg.ESAddr = *es_addr

	cfg.ServerID = 1001
	cfg.Flavor = "mysql"

	cfg.DataDir = "/tmp/test_river"
	cfg.DumpExec = "mysqldump"

	cfg.StatAddr = "127.0.0.1:12800"
	cfg.BulkSize = 1
	cfg.FlushBulkTime = TomlDuration{3 * time.Millisecond}

	os.RemoveAll(cfg.DataDir)

	cfg.Sources = []SourceConfig{SourceConfig{Schema: "test", Tables: []string{"test_river", "test_river_[0-9]{4}", "test_for_id"}}}

	cfg.Rules = []*Rule{
		&Rule{Schema: "test",
			Table:        "test_river",
			Index:        "river",
			Type:         "river",
			FieldMapping: map[string]string{"title": "es_title", "mylist": "es_mylist,list"},
		},

		&Rule{Schema: "test",
			Table:        "test_for_id",
			Index:        "river",
			Type:         "river",
			ID:           []string{"id", "title"},
			FieldMapping: map[string]string{"title": "es_title", "mylist": "es_mylist,list"},
		},

		&Rule{Schema: "test",
			Table:        "test_river_[0-9]{4}",
			Index:        "river",
			Type:         "river",
			FieldMapping: map[string]string{"title": "es_title", "mylist": "es_mylist,list"},
		},
	}

	s.r, err = NewRiver(cfg)
	c.Assert(err, IsNil)

	//err = s.r.es.DeleteIndex("river")
	//c.Assert(err, IsNil)
}

func (s *riverTestSuite) TearDownSuite(c *C) {
	if s.c != nil {
		s.c.Close()
	}

	if s.r != nil {
		s.r.Close()
	}
}

func (s *riverTestSuite) TestConfig(c *C) {
	str := `
my_addr = "127.0.0.1:3306"
my_user = "root"
my_pass = ""
my_charset = "utf8"
es_addr = "127.0.0.1:9200"
es_user = ""
es_pass = ""
data_dir = "./var"

[[source]]
schema = "test"

tables = ["test_river", "test_river_[0-9]{4}", "test_for_id"]

[[rule]]
schema = "test"
table = "test_river"
index = "river"
type = "river"
parent = "pid"

    [rule.field]
    title = "es_title"
    mylist = "es_mylist,list"

[[rule]]
schema = "test"
table = "test_for_id"
index = "river"
type = "river"
parent = "pid"
id = ["id", "title"]
    [rule.field]
    title = "es_title"
    mylist = "es_mylist,list"

[[rule]]
schema = "test"
table = "test_river_[0-9]{4}"
index = "river"
type = "river"

    [rule.field]
    title = "es_title"
    mylist = "es_mylist,list"

`

	cfg, err := NewConfig(str)
	c.Assert(err, IsNil)
	c.Assert(cfg.Sources, HasLen, 1)
	c.Assert(cfg.Sources[0].Tables, HasLen, 3)
	c.Assert(cfg.Rules, HasLen, 3)
}

func (s *riverTestSuite) testExecute(c *C, query string, args ...interface{}) {
	_, err := s.c.Execute(query, args...)
	c.Assert(err, IsNil)
}

func (s *riverTestSuite) testPrepareData(c *C) {
	s.testExecute(c, "INSERT INTO test_river (id, title, content, tenum, tset) VALUES (?, ?, ?, ?, ?)", 1, "first", "hello go 1", "e1", "a,b")
	s.testExecute(c, "INSERT INTO test_river (id, title, content, tenum, tset) VALUES (?, ?, ?, ?, ?)", 2, "second", "hello mysql 2", "e2", "b,c")
	s.testExecute(c, "INSERT INTO test_river (id, title, content, tenum, tset) VALUES (?, ?, ?, ?, ?)", 3, "third", "hello elaticsearch 3", "e3", "c")
	s.testExecute(c, "INSERT INTO test_river (id, title, content, tenum, tset, tbit) VALUES (?, ?, ?, ?, ?, ?)", 4, "fouth", "hello go-mysql-elasticserach 4", "e1", "a,b,c", 0)
	s.testExecute(c, "INSERT INTO test_for_id (id, title, content, tenum, tset) VALUES (?, ?, ?, ?, ?)", 1, "first", "hello go 1", "e1", "a,b")

	for i := 0; i < 10; i++ {
		table := fmt.Sprintf("test_river_%04d", i)
		s.testExecute(c, fmt.Sprintf("INSERT INTO %s (id, title, content, tenum, tset) VALUES (?, ?, ?, ?, ?)", table), 5+i, "abc", "hello", "e1", "a,b,c")
	}
}

func (s *riverTestSuite) testElasticGet(c *C, id string) *elastic.Response {
	index := "river"
	docType := "river"

	r, err := s.r.es.Get(index, docType, id)
	if err != nil {
		r := new(Response)
		r.Found = false
		return r
	}

	return r
}

func testWaitSyncDone(c *C, r *River) {
	<-r.canal.WaitDumpDone()

	err := r.canal.CatchMasterPos(10 * time.Second)
	c.Assert(err, IsNil)

	for i := 0; i < 1000; i++ {
		if len(r.syncCh) == 0 {
			return
		}

		time.Sleep(10 * time.Millisecond)
	}

	c.Fatalf("wait 1s but still have %d items to be synced", len(r.syncCh))
}

func (s *riverTestSuite) TestRiver(c *C) {
	s.testPrepareData(c)

	s.r.Start()

	testWaitSyncDone(c, s.r)

	var r *elastic.Response
	r = s.testElasticGet(c, "1")
	c.Assert(r.Found, Equals, true)
	c.Assert(r.Source["tenum"], Equals, "e1")
	c.Assert(r.Source["tset"], Equals, "a,b")

	r = s.testElasticGet(c, "1:first")
	c.Assert(r.Found, Equals, true)

	r = s.testElasticGet(c, "100")
	c.Assert(r.Found, Equals, false)

	for i := 0; i < 10; i++ {
		r = s.testElasticGet(c, fmt.Sprintf("%d", 5+i))
		c.Assert(r.Found, Equals, true)
		c.Assert(r.Source["es_title"], Equals, "abc")
	}

	s.testExecute(c, "UPDATE test_river SET title = ?, tenum = ?, tset = ?, mylist = ? WHERE id = ?", "second 2", "e3", "a,b,c", "a,b,c", 2)
	s.testExecute(c, "DELETE FROM test_river WHERE id = ?", 1)
	s.testExecute(c, "UPDATE test_river SET title = ?, id = ? WHERE id = ?", "second 30", 30, 3)

	// so we can insert invalid data
	s.testExecute(c, `SET SESSION sql_mode="NO_ENGINE_SUBSTITUTION";`)

	// bad insert
	s.testExecute(c, "UPDATE test_river SET title = ?, tenum = ?, tset = ? WHERE id = ?", "second 2", "e5", "a,b,c,d", 4)

	for i := 0; i < 10; i++ {
		table := fmt.Sprintf("test_river_%04d", i)
		s.testExecute(c, fmt.Sprintf("UPDATE %s SET title = ? WHERE id = ?", table), "hello", 5+i)
	}

	testWaitSyncDone(c, s.r)

	r = s.testElasticGet(c, "1")
	c.Assert(r.Found, Equals, false)

	r = s.testElasticGet(c, "2")
	c.Assert(r.Found, Equals, true)
	c.Assert(r.Source["es_title"], Equals, "second 2")
	c.Assert(r.Source["tenum"], Equals, "e3")
	c.Assert(r.Source["tset"], Equals, "a,b,c")
	c.Assert(r.Source["es_mylist"], DeepEquals, []interface{}{"a", "b", "c"})
	c.Assert(r.Source["tbit"], Equals, float64(1))

	r = s.testElasticGet(c, "4")
	c.Assert(r.Found, Equals, true)
	c.Assert(r.Source["tenum"], Equals, "")
	c.Assert(r.Source["tset"], Equals, "a,b,c")
	c.Assert(r.Source["tbit"], Equals, float64(0))

	r = s.testElasticGet(c, "3")
	c.Assert(r.Found, Equals, false)

	r = s.testElasticGet(c, "30")
	c.Assert(r.Found, Equals, true)
	c.Assert(r.Source["es_title"], Equals, "second 30")

	for i := 0; i < 10; i++ {
		r = s.testElasticGet(c, fmt.Sprintf("%d", 5+i))
		c.Assert(r.Found, Equals, true)
		c.Assert(r.Source["es_title"], Equals, "hello")
	}
}
<commit_msg>Update river_test.go<commit_after>package river

import (
	"flag"
	"fmt"
	"os"
	"testing"
	"time"

	"github.com/siddontang/go-mysql-elasticsearch/elastic"
	"github.com/siddontang/go-mysql/client"
	. "gopkg.in/check.v1"
)

var my_addr = flag.String("my_addr", "127.0.0.1:3306", "MySQL addr")
var es_addr = flag.String("es_addr", "127.0.0.1:9200", "Elasticsearch addr")

func Test(t *testing.T) {
	TestingT(t)
}

type riverTestSuite struct {
	c *client.Conn
	r *River
}

var _ = Suite(&riverTestSuite{})

func (s *riverTestSuite) SetUpSuite(c *C) {
	var err error
	s.c, err = client.Connect(*my_addr, "root", "", "test")
	c.Assert(err, IsNil)

	s.testExecute(c, "SET SESSION binlog_format = 'ROW'")

	schema := `
        CREATE TABLE IF NOT EXISTS %s (
            id INT,
            title VARCHAR(256),
            content VARCHAR(256),
            mylist VARCHAR(256),
            tenum ENUM("e1", "e2", "e3"),
            tset SET("a", "b", "c"),
            tbit BIT(1) default 1,
            PRIMARY KEY(id)) ENGINE=INNODB;
    `

	s.testExecute(c, "DROP TABLE IF EXISTS test_river")
	s.testExecute(c, fmt.Sprintf(schema, "test_river"))
	s.testExecute(c, fmt.Sprintf(schema, "test_for_id"))

	for i := 0; i < 10; i++ {
		table := fmt.Sprintf("test_river_%04d", i)
		s.testExecute(c, fmt.Sprintf("DROP TABLE IF EXISTS %s", table))
		s.testExecute(c, fmt.Sprintf(schema, table))
	}

	cfg := new(Config)
	cfg.MyAddr = *my_addr
	cfg.MyUser = "root"
	cfg.MyPassword = ""
	cfg.MyCharset = "utf8"
	cfg.ESAddr = *es_addr

	cfg.ServerID = 1001
	cfg.Flavor = "mysql"

	cfg.DataDir = "/tmp/test_river"
	cfg.DumpExec = "mysqldump"

	cfg.StatAddr = "127.0.0.1:12800"
	cfg.BulkSize = 1
	cfg.FlushBulkTime = TomlDuration{3 * time.Millisecond}

	os.RemoveAll(cfg.DataDir)

	cfg.Sources = []SourceConfig{SourceConfig{Schema: "test", Tables: []string{"test_river", "test_river_[0-9]{4}", "test_for_id"}}}

	cfg.Rules = []*Rule{
		&Rule{Schema: "test",
			Table:        "test_river",
			Index:        "river",
			Type:         "river",
			FieldMapping: map[string]string{"title": "es_title", "mylist": "es_mylist,list"},
		},

		&Rule{Schema: "test",
			Table:        "test_for_id",
			Index:        "river",
			Type:         "river",
			ID:           []string{"id", "title"},
			FieldMapping: map[string]string{"title": "es_title", "mylist": "es_mylist,list"},
		},

		&Rule{Schema: "test",
			Table:        "test_river_[0-9]{4}",
			Index:        "river",
			Type:         "river",
			FieldMapping: map[string]string{"title": "es_title", "mylist": "es_mylist,list"},
		},
	}

	s.r, err = NewRiver(cfg)
	c.Assert(err, IsNil)

	//err = s.r.es.DeleteIndex("river")
	//c.Assert(err, IsNil)
}

func (s *riverTestSuite) TearDownSuite(c *C) {
	if s.c != nil {
		s.c.Close()
	}

	if s.r != nil {
		s.r.Close()
	}
}

func (s *riverTestSuite) TestConfig(c *C) {
	str := `
my_addr = "127.0.0.1:3306"
my_user = "root"
my_pass = ""
my_charset = "utf8"
es_addr = "127.0.0.1:9200"
es_user = ""
es_pass = ""
data_dir = "./var"

[[source]]
schema = "test"

tables = ["test_river", "test_river_[0-9]{4}", "test_for_id"]

[[rule]]
schema = "test"
table = "test_river"
index = "river"
type = "river"
parent = "pid"

    [rule.field]
    title = "es_title"
    mylist = "es_mylist,list"

[[rule]]
schema = "test"
table = "test_for_id"
index = "river"
type = "river"
parent = "pid"
id = ["id", "title"]
    [rule.field]
    title = "es_title"
    mylist = "es_mylist,list"

[[rule]]
schema = "test"
table = "test_river_[0-9]{4}"
index = "river"
type = "river"

    [rule.field]
    title = "es_title"
    mylist = "es_mylist,list"

`

	cfg, err := NewConfig(str)
	c.Assert(err, IsNil)
	c.Assert(cfg.Sources, HasLen, 1)
	c.Assert(cfg.Sources[0].Tables, HasLen, 3)
	c.Assert(cfg.Rules, HasLen, 3)
}

func (s *riverTestSuite) testExecute(c *C, query string, args ...interface{}) {
	_, err := s.c.Execute(query, args...)
	c.Assert(err, IsNil)
}

func (s *riverTestSuite) testPrepareData(c *C) {
	s.testExecute(c, "INSERT INTO test_river (id, title, content, tenum, tset) VALUES (?, ?, ?, ?, ?)", 1, "first", "hello go 1", "e1", "a,b")
	s.testExecute(c, "INSERT INTO test_river (id, title, content, tenum, tset) VALUES (?, ?, ?, ?, ?)", 2, "second", "hello mysql 2", "e2", "b,c")
	s.testExecute(c, "INSERT INTO test_river (id, title, content, tenum, tset) VALUES (?, ?, ?, ?, ?)", 3, "third", "hello elaticsearch 3", "e3", "c")
	s.testExecute(c, "INSERT INTO test_river (id, title, content, tenum, tset, tbit) VALUES (?, ?, ?, ?, ?, ?)", 4, "fouth", "hello go-mysql-elasticserach 4", "e1", "a,b,c", 0)
	s.testExecute(c, "INSERT INTO test_for_id (id, title, content, tenum, tset) VALUES (?, ?, ?, ?, ?)", 1, "first", "hello go 1", "e1", "a,b")

	for i := 0; i < 10; i++ {
		table := fmt.Sprintf("test_river_%04d", i)
		s.testExecute(c, fmt.Sprintf("INSERT INTO %s (id, title, content, tenum, tset) VALUES (?, ?, ?, ?, ?)", table), 5+i, "abc", "hello", "e1", "a,b,c")
	}
}

func (s *riverTestSuite) testElasticGet(c *C, id string) *elastic.Response {
	index := "river"
	docType := "river"

	r, err := s.r.es.Get(index, docType, id)
	if err != nil {
		r := new(elastic.Response)
		r.Found = false
		return r
	}

	return r
}

func testWaitSyncDone(c *C, r *River) {
	<-r.canal.WaitDumpDone()

	err := r.canal.CatchMasterPos(10 * time.Second)
	c.Assert(err, IsNil)

	for i := 0; i < 1000; i++ {
		if len(r.syncCh) == 0 {
			return
		}

		time.Sleep(10 * time.Millisecond)
	}

	c.Fatalf("wait 1s but still have %d items to be synced", len(r.syncCh))
}

func (s *riverTestSuite) TestRiver(c *C) {
	s.testPrepareData(c)

	s.r.Start()

	testWaitSyncDone(c, s.r)

	var r *elastic.Response
	r = s.testElasticGet(c, "1")
	c.Assert(r.Found, Equals, true)
	c.Assert(r.Source["tenum"], Equals, "e1")
	c.Assert(r.Source["tset"], Equals, "a,b")

	r = s.testElasticGet(c, "1:first")
	c.Assert(r.Found, Equals, true)

	r = s.testElasticGet(c, "100")
	c.Assert(r.Found, Equals, false)

	for i := 0; i < 10; i++ {
		r = s.testElasticGet(c, fmt.Sprintf("%d", 5+i))
		c.Assert(r.Found, Equals, true)
		c.Assert(r.Source["es_title"], Equals, "abc")
	}

	s.testExecute(c, "UPDATE test_river SET title = ?, tenum = ?, tset = ?, mylist = ? WHERE id = ?", "second 2", "e3", "a,b,c", "a,b,c", 2)
	s.testExecute(c, "DELETE FROM test_river WHERE id = ?", 1)
	s.testExecute(c, "UPDATE test_river SET title = ?, id = ? WHERE id = ?", "second 30", 30, 3)

	// so we can insert invalid data
	s.testExecute(c, `SET SESSION sql_mode="NO_ENGINE_SUBSTITUTION";`)

	// bad insert
	s.testExecute(c, "UPDATE test_river SET title = ?, tenum = ?, tset = ? WHERE id = ?", "second 2", "e5", "a,b,c,d", 4)

	for i := 0; i < 10; i++ {
		table := fmt.Sprintf("test_river_%04d", i)
		s.testExecute(c, fmt.Sprintf("UPDATE %s SET title = ? WHERE id = ?", table), "hello", 5+i)
	}

	testWaitSyncDone(c, s.r)

	r = s.testElasticGet(c, "1")
	c.Assert(r.Found, Equals, false)

	r = s.testElasticGet(c, "2")
	c.Assert(r.Found, Equals, true)
	c.Assert(r.Source["es_title"], Equals, "second 2")
	c.Assert(r.Source["tenum"], Equals, "e3")
	c.Assert(r.Source["tset"], Equals, "a,b,c")
	c.Assert(r.Source["es_mylist"], DeepEquals, []interface{}{"a", "b", "c"})
	c.Assert(r.Source["tbit"], Equals, float64(1))

	r = s.testElasticGet(c, "4")
	c.Assert(r.Found, Equals, true)
	c.Assert(r.Source["tenum"], Equals, "")
	c.Assert(r.Source["tset"], Equals, "a,b,c")
	c.Assert(r.Source["tbit"], Equals, float64(0))

	r = s.testElasticGet(c, "3")
	c.Assert(r.Found, Equals, false)

	r = s.testElasticGet(c, "30")
	c.Assert(r.Found, Equals, true)
	c.Assert(r.Source["es_title"], Equals, "second 30")

	for i := 0; i < 10; i++ {
		r = s.testElasticGet(c, fmt.Sprintf("%d", 5+i))
		c.Assert(r.Found, Equals, true)
		c.Assert(r.Source["es_title"], Equals, "hello")
	}
}
<|endoftext|>


<commit_before>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"path/filepath"
	"runtime"
	"testing"
)

func runTest(t *testing.T, path, pkg string) {
	exitCode = 0
	*pkgName = pkg
	*recursive = false

	if pkg == "" {
		processFiles([]string{path}, true)
	} else {
		processDirectory(path)
	}

	if exitCode != 0 {
		t.Errorf("processing %s failed: exitCode = %d", path, exitCode)
	}
}

var tests = []struct {
	path string
	pkg  string
}{
	// individual files
	{"testdata/test1.go", ""},

	// directories
	{filepath.Join(runtime.GOROOT(), "src/pkg/go/ast"), "ast"},
	{filepath.Join(runtime.GOROOT(), "src/pkg/go/doc"), "doc"},
	{filepath.Join(runtime.GOROOT(), "src/pkg/go/token"), "scanner"},
	{filepath.Join(runtime.GOROOT(), "src/pkg/go/scanner"), "scanner"},
	{filepath.Join(runtime.GOROOT(), "src/pkg/go/parser"), "parser"},
	{filepath.Join(runtime.GOROOT(), "src/pkg/go/types"), "types"},
}

func Test(t *testing.T) {
	for _, test := range tests {
		runTest(t, test.path, test.pkg)
	}
}
<commit_msg>gotype: fix build (update test)<commit_after>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"path/filepath"
	"runtime"
	"testing"
)

func runTest(t *testing.T, path, pkg string) {
	exitCode = 0
	*pkgName = pkg
	*recursive = false

	if pkg == "" {
		processFiles([]string{path}, true)
	} else {
		processDirectory(path)
	}

	if exitCode != 0 {
		t.Errorf("processing %s failed: exitCode = %d", path, exitCode)
	}
}

var tests = []struct {
	path string
	pkg  string
}{
	// individual files
	{"testdata/test1.go", ""},

	// directories
	{filepath.Join(runtime.GOROOT(), "src/pkg/go/ast"), "ast"},
	{filepath.Join(runtime.GOROOT(), "src/pkg/go/doc"), "doc"},
	{filepath.Join(runtime.GOROOT(), "src/pkg/go/token"), "scanner"},
	{filepath.Join(runtime.GOROOT(), "src/pkg/go/scanner"), "scanner"},
	{filepath.Join(runtime.GOROOT(), "src/pkg/go/parser"), "parser"},
	{filepath.Join(runtime.GOROOT(), "src/pkg/exp/types"), "types"},
}

func Test(t *testing.T) {
	for _, test := range tests {
		runTest(t, test.path, test.pkg)
	}
}
<|endoftext|>