<commit_before>package openzwave

//
// Provides a facade for the C++ API that exposes just enough of the underlying C++
// API to be useful to implementing the Ninja Zwave driver
//
// The functions in this module are responsible for marshalling to and from the C functions
// declared in api.hpp and api.cpp.
//

//
// The following #cgo directives assumed that 'go' is a symbolic link that references the gopath that contains the current directory, e.g. ../../../..
//
// All 'go' packages that have this package as a dependency should include such a go link and will then inherit the library built in this package.
//

// #cgo LDFLAGS: -lopenzwave -Lgo/src/github.com/ninjasphere/go-openzwave/openzwave
// #cgo CPPFLAGS: -Iopenzwave/cpp/src/platform -Iopenzwave/cpp/src -Iopenzwave/cpp/src/value_classes
//
// #include "api.h"
import "C"

import "fmt"
import "os"
import "os/signal"
import "unsafe"

import "github.com/ninjasphere/go-openzwave/NT"
import "github.com/ninjasphere/go-openzwave/VT"
import "github.com/ninjasphere/go-openzwave/CODE"

type api struct {
	options       C.Options // an opaque reference to C++ Options object
	manager       C.Manager // an opaque reference to C++ Manager opject
	notifications chan Notification
}

//
// The Phase0 -> Phase5 interface represent 6 different states in the evolution of the api from
// creation, through configuration, through use.
//
// Each phase includes at least one method that allows transition to the next phase.
//
// Use of strong typing like this helps guide the consumer of the api package
// to construct a valid build sequence.
//

type Phase0 interface {
	StartOptions(configPath string, logPath string) Phase1
}

type Phase1 interface {
	AddIntOption(option string, value int) Phase1
	AddBoolOption(option string, value bool) Phase1
	EndOptions() Phase2
}

type Phase2 interface {
	CreateManager() Phase3
}

type Phase3 interface {
	SetNotificationChannel(channel chan Notification) Phase4
}

type Phase4 interface {
	StartDriver(device string) Phase5
}

type Phase5 interface {
	Run() int
}

type Notification struct {
	notification *C.Notification
}

func (self Notification) String() string {
	return fmt.Sprintf(
		"Notification[\n"+
			"notificationType=%s,\n"+
			"notificationCode=%s,\n"+
			"homeId=0x%08x,\n"+
			"nodeId=0x%02x,\n"+
			"valueType=%s,\n"+
			"valueId=0x%08x]\n",
		NT.ToEnum(int(self.notification.notificationType)),
		CODE.ToEnum(int(self.notification.notificationCode)),
		self.notification.nodeId.homeId,
		self.notification.nodeId.nodeId,
		VT.ToEnum(int(self.notification.valueId.valueType)),
		self.notification.valueId.valueId)
}

// allocate the control block used to track the state of the API
func API() Phase0 {
	return api{nil, nil, nil}
}

// create and stash the C++ Options object
func (self api) StartOptions(configPath string, logPath string) Phase1 {
	var cConfigPath *C.char = C.CString(configPath)
	var cLogPath *C.char = C.CString(logPath)
	defer C.free(unsafe.Pointer(cConfigPath))
	defer C.free(unsafe.Pointer(cLogPath))
	self.options = C.startOptions(cConfigPath, cLogPath)
	return self
}

// configure the C++ Options object with an integer value
func (self api) AddIntOption(option string, value int) Phase1 {
	var cOption *C.char = C.CString(option)
	defer C.free(unsafe.Pointer(cOption))

	C.addIntOption(self.options, cOption, C.int(value))
	return self
}

// configure the C++ Options object with a boolean value
func (self api) AddBoolOption(option string, value bool) Phase1 {
	var cOption *C.char = C.CString(option)
	var cBool C.int

	defer C.free(unsafe.Pointer(cOption))
	if value {
		cBool = C.TRUE
	} else {
		cBool = C.FALSE
	}
	C.addBoolOption(self.options, cOption, cBool)
	return self
}

// lock the options object
func (self api) EndOptions() Phase2 {
	C.endOptions(self.options)
	return self
}

// create the manager object
func (self api) CreateManager() Phase3 {
	self.manager = C.createManager()
	return self
}

// add a watcher
func (self api) SetNotificationChannel(channel chan Notification) Phase4 {
	self.notifications = channel
	C.setNotificationWatcher(self.manager, unsafe.Pointer(&self))
	return self
}

// add a driver.
func (self api) StartDriver(device string) Phase5 {
	if device == "" {
		device = defaultDriverName
	}
	var cDevice *C.char = C.CString(device)
	defer C.free(unsafe.Pointer(cDevice))

	C.addDriver(self.manager, cDevice)
	return self
}

func (self api) Run() int {
	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt, os.Kill)

	// Block until a signal is received.
	s := <-c
	fmt.Printf("received signal: %v", s)
	return 1
}

//export OnNotificationWrapper
func OnNotificationWrapper(notification *C.Notification, context unsafe.Pointer) {
	(*api)(context).notifications <- Notification{notification}
}
<commit_msg>Remove 3 phases from the builder pattern, add an EventLoop type.<commit_after>package openzwave

//
// Provides a facade for the C++ API that exposes just enough of the underlying C++
// API to be useful to implementing the Ninja Zwave driver
//
// The functions in this module are responsible for marshalling to and from the C functions
// declared in api.hpp and api.cpp.
//

//
// The following #cgo directives assumed that 'go' is a symbolic link that references the gopath that contains the current directory, e.g. ../../../..
//
// All 'go' packages that have this package as a dependency should include such a go link and will then inherit the library built in this package.
//

// #cgo LDFLAGS: -lopenzwave -Lgo/src/github.com/ninjasphere/go-openzwave/openzwave
// #cgo CPPFLAGS: -Iopenzwave/cpp/src/platform -Iopenzwave/cpp/src -Iopenzwave/cpp/src/value_classes
//
// #include "api.h"
import "C"

import "fmt"
import "os"
import "os/signal"
import "unsafe"

import "github.com/ninjasphere/go-openzwave/NT"
import "github.com/ninjasphere/go-openzwave/VT"
import "github.com/ninjasphere/go-openzwave/CODE"

type api struct {
	options       C.Options // an opaque reference to C++ Options object
	manager       C.Manager // an opaque reference to C++ Manager opject
	notifications chan Notification
}

//
// The Phase0 -> Phase2 interface represent 3 different states in the evolution of the api from
// creation, through configuration, through use.
//
// Each phase includes at least one method that allows transition to the next phase.
//
// Use of strong typing like this helps guide the consumer of the api package
// to construct a valid build sequence.
//

type Phase0 interface {
	StartOptions(configPath string, logPath string) Phase1
}

type Phase1 interface {
	AddIntOption(option string, value int) Phase1
	AddBoolOption(option string, value bool) Phase1
	StartDriver(device string) Phase2
}

type EventLoop func (chan Notification);

type Phase2 interface {
	Run(loop EventLoop) int
}

type Notification struct {
	notification *C.Notification
}

func (self Notification) String() string {
	return fmt.Sprintf(
		"Notification[\n"+
			"notificationType=%s,\n"+
			"notificationCode=%s,\n"+
			"homeId=0x%08x,\n"+
			"nodeId=0x%02x,\n"+
			"valueType=%s,\n"+
			"valueId=0x%08x]\n",
		NT.ToEnum(int(self.notification.notificationType)),
		CODE.ToEnum(int(self.notification.notificationCode)),
		self.notification.nodeId.homeId,
		self.notification.nodeId.nodeId,
		VT.ToEnum(int(self.notification.valueId.valueType)),
		self.notification.valueId.valueId)
}

// allocate the control block used to track the state of the API
func API() Phase0 {
	return api{nil, nil, nil}
}

// create and stash the C++ Options object
func (self api) StartOptions(configPath string, logPath string) Phase1 {
	var cConfigPath *C.char = C.CString(configPath)
	var cLogPath *C.char = C.CString(logPath)
	defer C.free(unsafe.Pointer(cConfigPath))
	defer C.free(unsafe.Pointer(cLogPath))
	self.options = C.startOptions(cConfigPath, cLogPath)
	return self
}

// configure the C++ Options object with an integer value
func (self api) AddIntOption(option string, value int) Phase1 {
	var cOption *C.char = C.CString(option)
	defer C.free(unsafe.Pointer(cOption))

	C.addIntOption(self.options, cOption, C.int(value))
	return self
}

// configure the C++ Options object with a boolean value
func (self api) AddBoolOption(option string, value bool) Phase1 {
	var cOption *C.char = C.CString(option)
	var cBool C.int

	defer C.free(unsafe.Pointer(cOption))
	if value {
		cBool = C.TRUE
	} else {
		cBool = C.FALSE
	}
	C.addBoolOption(self.options, cOption, cBool)
	return self
}

// add a driver.
func (self api) StartDriver(device string) Phase2 {
	C.endOptions(self.options)

	self.manager = C.createManager()
	self.notifications = make(chan Notification)
	C.setNotificationWatcher(self.manager, unsafe.Pointer(&self))

	if device == "" {
		device = defaultDriverName
	}
	var cDevice *C.char = C.CString(device)
	defer C.free(unsafe.Pointer(cDevice))

	C.addDriver(self.manager, cDevice)
	return self
}

func (self api) Run(loop EventLoop) int {

	signals := make(chan os.Signal, 1)
	signal.Notify(signals, os.Interrupt, os.Kill)

	go loop(self.notifications);

	// Block until a signal is received.
	signal := <-signals
	fmt.Printf("received signal: %v", signal)
	return 1
}

//export OnNotificationWrapper
func OnNotificationWrapper(notification *C.Notification, context unsafe.Pointer) {
	(*api)(context).notifications <- Notification{notification}
}
<|endoftext|>


<commit_before>// Copyright 2015 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.

package trie

import (
	"errors"
	"fmt"

	"github.com/ethereum/go-ethereum/common"
	"github.com/ethereum/go-ethereum/ethdb"
	"gopkg.in/karalabe/cookiejar.v2/collections/prque"
)

// ErrNotRequested is returned by the trie sync when it's requested to process a
// node it did not request.
var ErrNotRequested = errors.New("not requested")

// request represents a scheduled or already in-flight state retrieval request.
type request struct {
	hash   common.Hash // Hash of the node data content to retrieve
	data   []byte      // Data content of the node, cached until all subtrees complete
	object *node       // Target node to populate with retrieved data (hashnode originally)

	parents []*request // Parent state nodes referencing this entry (notify all upon completion)
	depth   int        // Depth level within the trie the node is located to prioritise DFS
	deps    int        // Number of dependencies before allowed to commit this node

	callback TrieSyncLeafCallback // Callback to invoke if a leaf node it reached on this branch
}

// SyncResult is a simple list to return missing nodes along with their request
// hashes.
type SyncResult struct {
	Hash common.Hash // Hash of the originally unknown trie node
	Data []byte      // Data content of the retrieved node
}

// TrieSyncLeafCallback is a callback type invoked when a trie sync reaches a
// leaf node. It's used by state syncing to check if the leaf node requires some
// further data syncing.
type TrieSyncLeafCallback func(leaf []byte, parent common.Hash) error

// TrieSync is the main state trie synchronisation scheduler, which provides yet
// unknown trie hashes to retrieve, accepts node data associated with said hashes
// and reconstructs the trie step by step until all is done.
type TrieSync struct {
	database ethdb.Database           // State database for storing all the assembled node data
	requests map[common.Hash]*request // Pending requests pertaining to a key hash
	queue    *prque.Prque             // Priority queue with the pending requests
}

// NewTrieSync creates a new trie data download scheduler.
func NewTrieSync(root common.Hash, database ethdb.Database, callback TrieSyncLeafCallback) *TrieSync {
	ts := &TrieSync{
		database: database,
		requests: make(map[common.Hash]*request),
		queue:    prque.New(),
	}
	ts.AddSubTrie(root, 0, common.Hash{}, callback)
	return ts
}

// AddSubTrie registers a new trie to the sync code, rooted at the designated parent.
func (s *TrieSync) AddSubTrie(root common.Hash, depth int, parent common.Hash, callback TrieSyncLeafCallback) {
	// Short circuit if the trie is empty or already known
	if root == emptyRoot {
		return
	}
	key := root.Bytes()
	blob, _ := s.database.Get(key)
	if local, err := decodeNode(key, blob, 0); local != nil && err == nil {
		return
	}
	// Assemble the new sub-trie sync request
	node := node(hashNode(root.Bytes()))
	req := &request{
		object:   &node,
		hash:     root,
		depth:    depth,
		callback: callback,
	}
	// If this sub-trie has a designated parent, link them together
	if parent != (common.Hash{}) {
		ancestor := s.requests[parent]
		if ancestor == nil {
			panic(fmt.Sprintf("sub-trie ancestor not found: %x", parent))
		}
		ancestor.deps++
		req.parents = append(req.parents, ancestor)
	}
	s.schedule(req)
}

// AddRawEntry schedules the direct retrieval of a state entry that should not be
// interpreted as a trie node, but rather accepted and stored into the database
// as is. This method's goal is to support misc state metadata retrievals (e.g.
// contract code).
func (s *TrieSync) AddRawEntry(hash common.Hash, depth int, parent common.Hash) {
	// Short circuit if the entry is empty or already known
	if hash == emptyState {
		return
	}
	if blob, _ := s.database.Get(hash.Bytes()); blob != nil {
		return
	}
	// Assemble the new sub-trie sync request
	req := &request{
		hash:  hash,
		depth: depth,
	}
	// If this sub-trie has a designated parent, link them together
	if parent != (common.Hash{}) {
		ancestor := s.requests[parent]
		if ancestor == nil {
			panic(fmt.Sprintf("raw-entry ancestor not found: %x", parent))
		}
		ancestor.deps++
		req.parents = append(req.parents, ancestor)
	}
	s.schedule(req)
}

// Missing retrieves the known missing nodes from the trie for retrieval.
func (s *TrieSync) Missing(max int) []common.Hash {
	requests := []common.Hash{}
	for !s.queue.Empty() && (max == 0 || len(requests) < max) {
		requests = append(requests, s.queue.PopItem().(common.Hash))
	}
	return requests
}

// Process injects a batch of retrieved trie nodes data.
func (s *TrieSync) Process(results []SyncResult) (int, error) {
	for i, item := range results {
		// If the item was not requested, bail out
		request := s.requests[item.Hash]
		if request == nil {
			return i, ErrNotRequested
		}
		// If the item is a raw entry request, commit directly
		if request.object == nil {
			request.data = item.Data
			s.commit(request, nil)
			continue
		}
		// Decode the node data content and update the request
		node, err := decodeNode(item.Hash[:], item.Data, 0)
		if err != nil {
			return i, err
		}
		*request.object = node
		request.data = item.Data

		// Create and schedule a request for all the children nodes
		requests, err := s.children(request)
		if err != nil {
			return i, err
		}
		if len(requests) == 0 && request.deps == 0 {
			s.commit(request, nil)
			continue
		}
		request.deps += len(requests)
		for _, child := range requests {
			s.schedule(child)
		}
	}
	return 0, nil
}

// Pending returns the number of state entries currently pending for download.
func (s *TrieSync) Pending() int {
	return len(s.requests)
}

// schedule inserts a new state retrieval request into the fetch queue. If there
// is already a pending request for this node, the new request will be discarded
// and only a parent reference added to the old one.
func (s *TrieSync) schedule(req *request) {
	// If we're already requesting this node, add a new reference and stop
	if old, ok := s.requests[req.hash]; ok {
		old.parents = append(old.parents, req.parents...)
		return
	}
	// Schedule the request for future retrieval
	s.queue.Push(req.hash, float32(req.depth))
	s.requests[req.hash] = req
}

// children retrieves all the missing children of a state trie entry for future
// retrieval scheduling.
func (s *TrieSync) children(req *request) ([]*request, error) {
	// Gather all the children of the node, irrelevant whether known or not
	type child struct {
		node  *node
		depth int
	}
	children := []child{}

	switch node := (*req.object).(type) {
	case *shortNode:
		node = node.copy() // Prevents linking all downloaded nodes together.
		children = []child{{
			node:  &node.Val,
			depth: req.depth + len(node.Key),
		}}
	case *fullNode:
		node = node.copy()
		for i := 0; i < 17; i++ {
			if node.Children[i] != nil {
				children = append(children, child{
					node:  &node.Children[i],
					depth: req.depth + 1,
				})
			}
		}
	default:
		panic(fmt.Sprintf("unknown node: %+v", node))
	}
	// Iterate over the children, and request all unknown ones
	requests := make([]*request, 0, len(children))
	for _, child := range children {
		// Notify any external watcher of a new key/value node
		if req.callback != nil {
			if node, ok := (*child.node).(valueNode); ok {
				if err := req.callback(node, req.hash); err != nil {
					return nil, err
				}
			}
		}
		// If the child references another node, resolve or schedule
		if node, ok := (*child.node).(hashNode); ok {
			// Try to resolve the node from the local database
			blob, _ := s.database.Get(node)
			if local, err := decodeNode(node[:], blob, 0); local != nil && err == nil {
				*child.node = local
				continue
			}
			// Locally unknown node, schedule for retrieval
			requests = append(requests, &request{
				object:   child.node,
				hash:     common.BytesToHash(node),
				parents:  []*request{req},
				depth:    child.depth,
				callback: req.callback,
			})
		}
	}
	return requests, nil
}

// commit finalizes a retrieval request and stores it into the database. If any
// of the referencing parent requests complete due to this commit, they are also
// committed themselves.
func (s *TrieSync) commit(req *request, batch ethdb.Batch) (err error) {
	// Create a new batch if none was specified
	if batch == nil {
		batch = s.database.NewBatch()
		defer func() {
			err = batch.Write()
		}()
	}
	// Write the node content to disk
	if err := batch.Put(req.hash[:], req.data); err != nil {
		return err
	}
	delete(s.requests, req.hash)

	// Check all parents for completion
	for _, parent := range req.parents {
		parent.deps--
		if parent.deps == 0 {
			if err := s.commit(parent, batch); err != nil {
				return err
			}
		}
	}
	return nil
}
<commit_msg>trie: while fast syncing, don't keep trie nodes in memory (#3186)<commit_after>// Copyright 2015 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.

package trie

import (
	"errors"
	"fmt"

	"github.com/ethereum/go-ethereum/common"
	"github.com/ethereum/go-ethereum/ethdb"
	"gopkg.in/karalabe/cookiejar.v2/collections/prque"
)

// ErrNotRequested is returned by the trie sync when it's requested to process a
// node it did not request.
var ErrNotRequested = errors.New("not requested")

// request represents a scheduled or already in-flight state retrieval request.
type request struct {
	hash common.Hash // Hash of the node data content to retrieve
	data []byte      // Data content of the node, cached until all subtrees complete
	raw  bool        // Whether this is a raw entry (code) or a trie node

	parents []*request // Parent state nodes referencing this entry (notify all upon completion)
	depth   int        // Depth level within the trie the node is located to prioritise DFS
	deps    int        // Number of dependencies before allowed to commit this node

	callback TrieSyncLeafCallback // Callback to invoke if a leaf node it reached on this branch
}

// SyncResult is a simple list to return missing nodes along with their request
// hashes.
type SyncResult struct {
	Hash common.Hash // Hash of the originally unknown trie node
	Data []byte      // Data content of the retrieved node
}

// TrieSyncLeafCallback is a callback type invoked when a trie sync reaches a
// leaf node. It's used by state syncing to check if the leaf node requires some
// further data syncing.
type TrieSyncLeafCallback func(leaf []byte, parent common.Hash) error

// TrieSync is the main state trie synchronisation scheduler, which provides yet
// unknown trie hashes to retrieve, accepts node data associated with said hashes
// and reconstructs the trie step by step until all is done.
type TrieSync struct {
	database ethdb.Database           // State database for storing all the assembled node data
	requests map[common.Hash]*request // Pending requests pertaining to a key hash
	queue    *prque.Prque             // Priority queue with the pending requests
}

// NewTrieSync creates a new trie data download scheduler.
func NewTrieSync(root common.Hash, database ethdb.Database, callback TrieSyncLeafCallback) *TrieSync {
	ts := &TrieSync{
		database: database,
		requests: make(map[common.Hash]*request),
		queue:    prque.New(),
	}
	ts.AddSubTrie(root, 0, common.Hash{}, callback)
	return ts
}

// AddSubTrie registers a new trie to the sync code, rooted at the designated parent.
func (s *TrieSync) AddSubTrie(root common.Hash, depth int, parent common.Hash, callback TrieSyncLeafCallback) {
	// Short circuit if the trie is empty or already known
	if root == emptyRoot {
		return
	}
	key := root.Bytes()
	blob, _ := s.database.Get(key)
	if local, err := decodeNode(key, blob, 0); local != nil && err == nil {
		return
	}
	// Assemble the new sub-trie sync request
	req := &request{
		hash:     root,
		depth:    depth,
		callback: callback,
	}
	// If this sub-trie has a designated parent, link them together
	if parent != (common.Hash{}) {
		ancestor := s.requests[parent]
		if ancestor == nil {
			panic(fmt.Sprintf("sub-trie ancestor not found: %x", parent))
		}
		ancestor.deps++
		req.parents = append(req.parents, ancestor)
	}
	s.schedule(req)
}

// AddRawEntry schedules the direct retrieval of a state entry that should not be
// interpreted as a trie node, but rather accepted and stored into the database
// as is. This method's goal is to support misc state metadata retrievals (e.g.
// contract code).
func (s *TrieSync) AddRawEntry(hash common.Hash, depth int, parent common.Hash) {
	// Short circuit if the entry is empty or already known
	if hash == emptyState {
		return
	}
	if blob, _ := s.database.Get(hash.Bytes()); blob != nil {
		return
	}
	// Assemble the new sub-trie sync request
	req := &request{
		hash:  hash,
		raw:   true,
		depth: depth,
	}
	// If this sub-trie has a designated parent, link them together
	if parent != (common.Hash{}) {
		ancestor := s.requests[parent]
		if ancestor == nil {
			panic(fmt.Sprintf("raw-entry ancestor not found: %x", parent))
		}
		ancestor.deps++
		req.parents = append(req.parents, ancestor)
	}
	s.schedule(req)
}

// Missing retrieves the known missing nodes from the trie for retrieval.
func (s *TrieSync) Missing(max int) []common.Hash {
	requests := []common.Hash{}
	for !s.queue.Empty() && (max == 0 || len(requests) < max) {
		requests = append(requests, s.queue.PopItem().(common.Hash))
	}
	return requests
}

// Process injects a batch of retrieved trie nodes data.
func (s *TrieSync) Process(results []SyncResult) (int, error) {
	for i, item := range results {
		// If the item was not requested, bail out
		request := s.requests[item.Hash]
		if request == nil {
			return i, ErrNotRequested
		}
		// If the item is a raw entry request, commit directly
		if request.raw {
			request.data = item.Data
			s.commit(request, nil)
			continue
		}
		// Decode the node data content and update the request
		node, err := decodeNode(item.Hash[:], item.Data, 0)
		if err != nil {
			return i, err
		}
		request.data = item.Data

		// Create and schedule a request for all the children nodes
		requests, err := s.children(request, node)
		if err != nil {
			return i, err
		}
		if len(requests) == 0 && request.deps == 0 {
			s.commit(request, nil)
			continue
		}
		request.deps += len(requests)
		for _, child := range requests {
			s.schedule(child)
		}
	}
	return 0, nil
}

// Pending returns the number of state entries currently pending for download.
func (s *TrieSync) Pending() int {
	return len(s.requests)
}

// schedule inserts a new state retrieval request into the fetch queue. If there
// is already a pending request for this node, the new request will be discarded
// and only a parent reference added to the old one.
func (s *TrieSync) schedule(req *request) {
	// If we're already requesting this node, add a new reference and stop
	if old, ok := s.requests[req.hash]; ok {
		old.parents = append(old.parents, req.parents...)
		return
	}
	// Schedule the request for future retrieval
	s.queue.Push(req.hash, float32(req.depth))
	s.requests[req.hash] = req
}

// children retrieves all the missing children of a state trie entry for future
// retrieval scheduling.
func (s *TrieSync) children(req *request, object node) ([]*request, error) {
	// Gather all the children of the node, irrelevant whether known or not
	type child struct {
		node  node
		depth int
	}
	children := []child{}

	switch node := (object).(type) {
	case *shortNode:
		children = []child{{
			node:  node.Val,
			depth: req.depth + len(node.Key),
		}}
	case *fullNode:
		for i := 0; i < 17; i++ {
			if node.Children[i] != nil {
				children = append(children, child{
					node:  node.Children[i],
					depth: req.depth + 1,
				})
			}
		}
	default:
		panic(fmt.Sprintf("unknown node: %+v", node))
	}
	// Iterate over the children, and request all unknown ones
	requests := make([]*request, 0, len(children))
	for _, child := range children {
		// Notify any external watcher of a new key/value node
		if req.callback != nil {
			if node, ok := (child.node).(valueNode); ok {
				if err := req.callback(node, req.hash); err != nil {
					return nil, err
				}
			}
		}
		// If the child references another node, resolve or schedule
		if node, ok := (child.node).(hashNode); ok {
			// Try to resolve the node from the local database
			blob, _ := s.database.Get(node)
			if local, err := decodeNode(node[:], blob, 0); local != nil && err == nil {
				continue
			}
			// Locally unknown node, schedule for retrieval
			requests = append(requests, &request{
				hash:     common.BytesToHash(node),
				parents:  []*request{req},
				depth:    child.depth,
				callback: req.callback,
			})
		}
	}
	return requests, nil
}

// commit finalizes a retrieval request and stores it into the database. If any
// of the referencing parent requests complete due to this commit, they are also
// committed themselves.
func (s *TrieSync) commit(req *request, batch ethdb.Batch) (err error) {
	// Create a new batch if none was specified
	if batch == nil {
		batch = s.database.NewBatch()
		defer func() {
			err = batch.Write()
		}()
	}
	// Write the node content to disk
	if err := batch.Put(req.hash[:], req.data); err != nil {
		return err
	}
	delete(s.requests, req.hash)

	// Check all parents for completion
	for _, parent := range req.parents {
		parent.deps--
		if parent.deps == 0 {
			if err := s.commit(parent, batch); err != nil {
				return err
			}
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package usn

import (
	"errors"
	"fmt"
	"time"
	"unsafe"

	"golang.org/x/sys/windows"
)

const (
	recordHeaderSize = 8
	recordV2Size     = 56
	recordV3Size     = 60
	recordV4Size     = 80
)

// Record represents a change journal record.
type Record struct {
	RecordLength              uint32
	MajorVersion              uint16
	MinorVersion              uint16
	FileReferenceNumber       uint64
	ParentFileReferenceNumber uint64
	USN                       USN
	TimeStamp                 time.Time
	Reason                    uint32
	SourceInfo                uint32
	FileName                  string
}

// UnmarshalBinary attempts to parse a single record from the given data.
func (r *Record) UnmarshalBinary(data []byte) error {
	bufSize := len(data)
	if bufSize < recordHeaderSize {
		return errors.New("insufficient data for USN record header")
	}

	hdr := (*RawRecordHeader)(unsafe.Pointer(&data[0]))
	r.RecordLength = hdr.RecordLength
	r.MajorVersion = hdr.MajorVersion
	r.MinorVersion = hdr.MinorVersion

	switch hdr.MajorVersion {
	case 2:
		if bufSize < recordV2Size {
			return errors.New("insufficient data for v2 USN record")
		}
		raw := (*RawRecordV2)(unsafe.Pointer(&data[0]))
		r.USN = raw.USN
		r.TimeStamp = time.Unix(0, raw.TimeStamp.Nanoseconds())
		r.Reason = raw.Reason
		r.SourceInfo = raw.SourceInfo
		start := int(raw.FileNameOffset)
		end := start + int(raw.FileNameLength)
		if end > bufSize {
			return errors.New("insufficient data for v2 USN record file name")
		}
		r.FileName = utf16BytesToString(data[start:end])
	default:
		return fmt.Errorf("unsupported USN record version: %d.%d", hdr.MajorVersion, hdr.MinorVersion)
	}
	return nil
}

// RawRecordHeader represents the raw form of the common USN journal record
// header.
type RawRecordHeader struct {
	RecordLength uint32
	MajorVersion uint16
	MinorVersion uint16
}

// RawRecordV1 represents the raw form of a version 1 change journal record.
type RawRecordV1 struct {
	RawRecordHeader
	FileReferenceNumber       uint64
	ParentFileReferenceNumber uint64
	USN                       USN
	TimeStamp                 windows.Filetime
}

// RawRecordV2 represents the raw form of a version 2 change journal record.
type RawRecordV2 struct {
	RawRecordHeader
	FileReferenceNumber       uint64
	ParentFileReferenceNumber uint64
	USN                       USN
	TimeStamp                 windows.Filetime
	Reason                    uint32
	SourceInfo                uint32
	SecurityID                uint32
	FileAttributes            uint32
	FileNameLength            uint16
	FileNameOffset            uint16
	_                         uint16
}

// RawRecordV3 represents the raw form of a version 3 change journal record.
type RawRecordV3 struct {
	RawRecordHeader
	FileReferenceNumber       FileID128
	ParentFileReferenceNumber FileID128
	USN                       USN
	TimeStamp                 windows.Filetime
	Reason                    uint32
	SourceInfo                uint32
	SecurityID                uint32
	FileAttributes            uint32
	FileNameLength            uint16
	FileNameOffset            uint16
	_                         uint16
}

// RawRecordV4 represents the raw form of a version 4 change journal record.
type RawRecordV4 struct {
	RawRecordHeader
	FileReferenceNumber       FileID128
	ParentFileReferenceNumber FileID128
	USN                       USN
	Reason                    uint32
	SourceInfo                uint32
	RemainingExtents          uint32
	NumberOfExtents           uint16
	ExtentSize                uint16
	_                         RecordExtent
}

// RecordExtent represents a change journal record extent.
type RecordExtent struct {
	Offset int64
	Length int64
}

//var RecordSet []Record

//func (rs *RecordSet) Read()
<commit_msg>usn: Improved data validation for records<commit_after>package usn

import (
	"errors"
	"fmt"
	"time"
	"unsafe"

	"golang.org/x/sys/windows"
)

const (
	recordHeaderSize = 8
	recordV2Size     = 56
	recordV3Size     = 60
	recordV4Size     = 80
)

var (
	// ErrTruncatedRecord is returned when the data buffer containing a USN
	// record appears to have been truncated.
	ErrTruncatedRecord = errors.New("USN record data was truncated")

	// ErrInvalidRecordLength is returned when the data buffer containing a USN
	// record contains has an invalid record length value.
	ErrInvalidRecordLength = errors.New("USN record length is invalid (possible data corruption)")
)

// Record represents a change journal record.
type Record struct {
	RecordLength              uint32
	MajorVersion              uint16
	MinorVersion              uint16
	FileReferenceNumber       uint64
	ParentFileReferenceNumber uint64
	USN                       USN
	TimeStamp                 time.Time
	Reason                    uint32
	SourceInfo                uint32
	FileName                  string
}

// UnmarshalBinary attempts to parse a single record from the given data.
func (r *Record) UnmarshalBinary(data []byte) error {
	bufSize := len(data)
	if bufSize < recordHeaderSize {
		return ErrTruncatedRecord
	}

	hdr := (*RawRecordHeader)(unsafe.Pointer(&data[0]))
	r.RecordLength = hdr.RecordLength
	r.MajorVersion = hdr.MajorVersion
	r.MinorVersion = hdr.MinorVersion

	if r.RecordLength == 0 {
		return ErrInvalidRecordLength
	}

	switch hdr.MajorVersion {
	case 2:
		if bufSize < recordV2Size {
			return ErrTruncatedRecord
		}
		if r.RecordLength < recordV2Size {
			return ErrInvalidRecordLength
		}
		raw := (*RawRecordV2)(unsafe.Pointer(&data[0]))
		r.USN = raw.USN
		r.TimeStamp = time.Unix(0, raw.TimeStamp.Nanoseconds())
		r.Reason = raw.Reason
		r.SourceInfo = raw.SourceInfo
		start := int(raw.FileNameOffset)
		end := start + int(raw.FileNameLength)
		if end > bufSize {
			return ErrTruncatedRecord
		}
		if end > int(r.RecordLength) {
			return ErrInvalidRecordLength
		}
		r.FileName = utf16BytesToString(data[start:end])
	default:
		return fmt.Errorf("unsupported USN record version: %d.%d", hdr.MajorVersion, hdr.MinorVersion)
	}
	return nil
}

// RawRecordHeader represents the raw form of the common USN journal record
// header.
type RawRecordHeader struct {
	RecordLength uint32
	MajorVersion uint16
	MinorVersion uint16
}

// RawRecordV1 represents the raw form of a version 1 change journal record.
type RawRecordV1 struct {
	RawRecordHeader
	FileReferenceNumber       uint64
	ParentFileReferenceNumber uint64
	USN                       USN
	TimeStamp                 windows.Filetime
}

// RawRecordV2 represents the raw form of a version 2 change journal record.
type RawRecordV2 struct {
	RawRecordHeader
	FileReferenceNumber       uint64
	ParentFileReferenceNumber uint64
	USN                       USN
	TimeStamp                 windows.Filetime
	Reason                    uint32
	SourceInfo                uint32
	SecurityID                uint32
	FileAttributes            uint32
	FileNameLength            uint16
	FileNameOffset            uint16
	_                         uint16
}

// RawRecordV3 represents the raw form of a version 3 change journal record.
type RawRecordV3 struct {
	RawRecordHeader
	FileReferenceNumber       FileID128
	ParentFileReferenceNumber FileID128
	USN                       USN
	TimeStamp                 windows.Filetime
	Reason                    uint32
	SourceInfo                uint32
	SecurityID                uint32
	FileAttributes            uint32
	FileNameLength            uint16
	FileNameOffset            uint16
	_                         uint16
}

// RawRecordV4 represents the raw form of a version 4 change journal record.
type RawRecordV4 struct {
	RawRecordHeader
	FileReferenceNumber       FileID128
	ParentFileReferenceNumber FileID128
	USN                       USN
	Reason                    uint32
	SourceInfo                uint32
	RemainingExtents          uint32
	NumberOfExtents           uint16
	ExtentSize                uint16
	_                         RecordExtent
}

// RecordExtent represents a change journal record extent.
type RecordExtent struct {
	Offset int64
	Length int64
}

//var RecordSet []Record

//func (rs *RecordSet) Read()
<|endoftext|>


<commit_before>package brokerapi

import (
	"encoding/json"
	"net/http"
	"strconv"

	"code.cloudfoundry.org/lager"
	"github.com/gorilla/mux"
	"github.com/pivotal-cf/brokerapi/auth"
)

const provisionLogKey = "provision"
const deprovisionLogKey = "deprovision"
const bindLogKey = "bind"
const unbindLogKey = "unbind"
const lastOperationLogKey = "lastOperation"

const instanceIDLogKey = "instance-id"
const instanceDetailsLogKey = "instance-details"
const bindingIDLogKey = "binding-id"

const invalidServiceDetailsErrorKey = "invalid-service-details"
const invalidBindDetailsErrorKey = "invalid-bind-details"
const invalidUnbindDetailsErrorKey = "invalid-unbind-details"
const invalidDeprovisionDetailsErrorKey = "invalid-deprovision-details"
const instanceLimitReachedErrorKey = "instance-limit-reached"
const instanceAlreadyExistsErrorKey = "instance-already-exists"
const bindingAlreadyExistsErrorKey = "binding-already-exists"
const instanceMissingErrorKey = "instance-missing"
const bindingMissingErrorKey = "binding-missing"
const asyncRequiredKey = "async-required"
const planChangeNotSupportedKey = "plan-change-not-supported"
const unknownErrorKey = "unknown-error"
const invalidRawParamsKey = "invalid-raw-params"
const appGuidNotProvidedErrorKey = "app-guid-not-provided"

const statusUnprocessableEntity = 422

type BrokerCredentials struct {
	Username string
	Password string
}

func New(serviceBroker ServiceBroker, logger lager.Logger, brokerCredentials BrokerCredentials) http.Handler {
	router := mux.NewRouter()
	AttachRoutes(router, serviceBroker, logger)
	return auth.NewWrapper(brokerCredentials.Username, brokerCredentials.Password).Wrap(router)
}

func AttachRoutes(router *mux.Router, serviceBroker ServiceBroker, logger lager.Logger) {
	handler := serviceBrokerHandler{serviceBroker: serviceBroker, logger: logger}
	router.HandleFunc("/v2/catalog", handler.catalog).Methods("GET")

	router.HandleFunc("/v2/service_instances/{instance_id}", handler.provision).Methods("PUT")
	router.HandleFunc("/v2/service_instances/{instance_id}", handler.deprovision).Methods("DELETE")
	router.HandleFunc("/v2/service_instances/{instance_id}/last_operation", handler.lastOperation).Methods("GET")
	router.HandleFunc("/v2/service_instances/{instance_id}", handler.update).Methods("PATCH")

	router.HandleFunc("/v2/service_instances/{instance_id}/service_bindings/{binding_id}", handler.bind).Methods("PUT")
	router.HandleFunc("/v2/service_instances/{instance_id}/service_bindings/{binding_id}", handler.unbind).Methods("DELETE")
}

type serviceBrokerHandler struct {
	serviceBroker ServiceBroker
	logger        lager.Logger
}

func (h serviceBrokerHandler) catalog(w http.ResponseWriter, req *http.Request) {
	catalog := CatalogResponse{
		Services: h.serviceBroker.Services(req.Context()),
	}

	h.respond(w, http.StatusOK, catalog)
}

func (h serviceBrokerHandler) provision(w http.ResponseWriter, req *http.Request) {
	vars := mux.Vars(req)
	instanceID := vars["instance_id"]

	logger := h.logger.Session(provisionLogKey, lager.Data{
		instanceIDLogKey: instanceID,
	})

	var details ProvisionDetails
	if err := json.NewDecoder(req.Body).Decode(&details); err != nil {
		logger.Error(invalidServiceDetailsErrorKey, err)
		h.respond(w, statusUnprocessableEntity, ErrorResponse{
			Description: err.Error(),
		})
		return
	}

	acceptsIncompleteFlag, _ := strconv.ParseBool(req.URL.Query().Get("accepts_incomplete"))

	logger = logger.WithData(lager.Data{
		instanceDetailsLogKey: details,
	})

	provisionResponse, err := h.serviceBroker.Provision(req.Context(), instanceID, details, acceptsIncompleteFlag)

	if err != nil {
		switch err {
		case ErrRawParamsInvalid:
			logger.Error(invalidRawParamsKey, err)
			h.respond(w, 422, ErrorResponse{
				Description: err.Error(),
			})
		case ErrInstanceAlreadyExists:
			logger.Error(instanceAlreadyExistsErrorKey, err)
			h.respond(w, http.StatusConflict, EmptyResponse{})
		case ErrInstanceLimitMet:
			logger.Error(instanceLimitReachedErrorKey, err)
			h.respond(w, http.StatusInternalServerError, ErrorResponse{
				Description: err.Error(),
			})
		case ErrAsyncRequired:
			logger.Error(asyncRequiredKey, err)
			h.respond(w, 422, ErrorResponse{
				Error:       "AsyncRequired",
				Description: err.Error(),
			})
		default:
			logger.Error(unknownErrorKey, err)
			h.respond(w, http.StatusInternalServerError, ErrorResponse{
				Description: err.Error(),
			})
		}
		return
	}

	if provisionResponse.IsAsync {
		h.respond(w, http.StatusAccepted, ProvisioningResponse{
			DashboardURL:  provisionResponse.DashboardURL,
			OperationData: provisionResponse.OperationData,
		})
	} else {
		h.respond(w, http.StatusCreated, ProvisioningResponse{
			DashboardURL: provisionResponse.DashboardURL,
		})
	}
}

func (h serviceBrokerHandler) update(w http.ResponseWriter, req *http.Request) {
	vars := mux.Vars(req)
	instanceID := vars["instance_id"]

	var details UpdateDetails
	if err := json.NewDecoder(req.Body).Decode(&details); err != nil {
		h.logger.Error(invalidServiceDetailsErrorKey, err)
		h.respond(w, statusUnprocessableEntity, ErrorResponse{
			Description: err.Error(),
		})
		return
	}

	acceptsIncompleteFlag, _ := strconv.ParseBool(req.URL.Query().Get("accepts_incomplete"))

	updateServiceSpec, err := h.serviceBroker.Update(req.Context(), instanceID, details, acceptsIncompleteFlag)
	if err != nil {
		switch err {
		case ErrAsyncRequired:
			h.logger.Error(asyncRequiredKey, err)
			h.respond(w, 422, ErrorResponse{
				Error:       "AsyncRequired",
				Description: err.Error(),
			})
			return

		case ErrPlanChangeNotSupported:
			h.logger.Error(planChangeNotSupportedKey, err)
			h.respond(w, 422, ErrorResponse{
				Error:       "PlanChangeNotSupported",
				Description: err.Error(),
			})
			return

		default:
			h.logger.Error(unknownErrorKey, err)
			h.respond(w, http.StatusInternalServerError, ErrorResponse{
				Description: err.Error(),
			})
			return
		}
	}

	statusCode := http.StatusOK
	if updateServiceSpec.IsAsync {
		statusCode = http.StatusAccepted
	}
	h.respond(w, statusCode, UpdateResponse{OperationData: updateServiceSpec.OperationData})
}

func (h serviceBrokerHandler) deprovision(w http.ResponseWriter, req *http.Request) {
	vars := mux.Vars(req)
	instanceID := vars["instance_id"]
	logger := h.logger.Session(deprovisionLogKey, lager.Data{
		instanceIDLogKey: instanceID,
	})

	details := DeprovisionDetails{
		PlanID:    req.FormValue("plan_id"),
		ServiceID: req.FormValue("service_id"),
	}
	asyncAllowed := req.FormValue("accepts_incomplete") == "true"

	deprovisionSpec, err := h.serviceBroker.Deprovision(req.Context(), instanceID, details, asyncAllowed)
	if err != nil {
		switch err {
		case ErrInstanceDoesNotExist:
			logger.Error(instanceMissingErrorKey, err)
			h.respond(w, http.StatusGone, EmptyResponse{})
		case ErrAsyncRequired:
			logger.Error(asyncRequiredKey, err)
			h.respond(w, 422, ErrorResponse{
				Error:       "AsyncRequired",
				Description: err.Error(),
			})
		default:
			logger.Error(unknownErrorKey, err)
			h.respond(w, http.StatusInternalServerError, ErrorResponse{
				Description: err.Error(),
			})
		}
		return
	}

	if deprovisionSpec.IsAsync {
		h.respond(w, http.StatusAccepted, DeprovisionResponse{OperationData: deprovisionSpec.OperationData})
	} else {
		h.respond(w, http.StatusOK, EmptyResponse{})
	}
}

func (h serviceBrokerHandler) bind(w http.ResponseWriter, req *http.Request) {
	vars := mux.Vars(req)
	instanceID := vars["instance_id"]
	bindingID := vars["binding_id"]

	logger := h.logger.Session(bindLogKey, lager.Data{
		instanceIDLogKey: instanceID,
		bindingIDLogKey:  bindingID,
	})

	var details BindDetails
	if err := json.NewDecoder(req.Body).Decode(&details); err != nil {
		logger.Error(invalidBindDetailsErrorKey, err)
		h.respond(w, statusUnprocessableEntity, ErrorResponse{
			Description: err.Error(),
		})
		return
	}

	binding, err := h.serviceBroker.Bind(req.Context(), instanceID, bindingID, details)
	if err != nil {
		switch err {
		case ErrInstanceDoesNotExist:
			logger.Error(instanceMissingErrorKey, err)
			h.respond(w, http.StatusNotFound, ErrorResponse{
				Description: err.Error(),
			})
		case ErrBindingAlreadyExists:
			logger.Error(bindingAlreadyExistsErrorKey, err)
			h.respond(w, http.StatusConflict, ErrorResponse{
				Description: err.Error(),
			})
		case ErrAppGuidNotProvided:
			logger.Error(appGuidNotProvidedErrorKey, err)
			h.respond(w, statusUnprocessableEntity, ErrorResponse{
				Description: err.Error(),
			})
		default:
			logger.Error(unknownErrorKey, err)
			h.respond(w, http.StatusInternalServerError, ErrorResponse{
				Description: err.Error(),
			})
		}
		return
	}

	brokerAPIVersion := req.Header.Get("X-Broker-Api-Version")
	if brokerAPIVersion == "2.8" || brokerAPIVersion == "2.9" {
		experimentalVols := []ExperimentalVolumeMount{}

		for _, vol := range binding.VolumeMounts {
			experimentalConfig, err := json.Marshal(vol.Device.MountConfig)
			if err != nil {
				logger.Error(unknownErrorKey, err)
				h.respond(w, http.StatusInternalServerError, ErrorResponse{Description: err.Error()})
				return
			}

			experimentalVols = append(experimentalVols, ExperimentalVolumeMount{
				ContainerPath: vol.ContainerDir,
				Mode:          vol.Mode,
				Private: ExperimentalVolumeMountPrivate{
					Driver:  vol.Driver,
					GroupID: vol.Device.VolumeId,
					Config:  string(experimentalConfig),
				},
			})
		}

		experimentalBinding := ExperimentalVolumeMountBindingResponse{
			Credentials:     binding.Credentials,
			RouteServiceURL: binding.RouteServiceURL,
			SyslogDrainURL:  binding.SyslogDrainURL,
			VolumeMounts:    experimentalVols,
		}
		h.respond(w, http.StatusCreated, experimentalBinding)
		return
	}

	h.respond(w, http.StatusCreated, binding)
}

func (h serviceBrokerHandler) unbind(w http.ResponseWriter, req *http.Request) {
	vars := mux.Vars(req)
	instanceID := vars["instance_id"]
	bindingID := vars["binding_id"]

	logger := h.logger.Session(unbindLogKey, lager.Data{
		instanceIDLogKey: instanceID,
		bindingIDLogKey:  bindingID,
	})

	details := UnbindDetails{
		PlanID:    req.FormValue("plan_id"),
		ServiceID: req.FormValue("service_id"),
	}

	if err := h.serviceBroker.Unbind(req.Context(), instanceID, bindingID, details); err != nil {
		switch err {
		case ErrInstanceDoesNotExist:
			logger.Error(instanceMissingErrorKey, err)
			h.respond(w, http.StatusGone, EmptyResponse{})
		case ErrBindingDoesNotExist:
			logger.Error(bindingMissingErrorKey, err)
			h.respond(w, http.StatusGone, EmptyResponse{})
		default:
			logger.Error(unknownErrorKey, err)
			h.respond(w, http.StatusInternalServerError, ErrorResponse{
				Description: err.Error(),
			})
		}
		return
	}

	h.respond(w, http.StatusOK, EmptyResponse{})
}

func (h serviceBrokerHandler) lastOperation(w http.ResponseWriter, req *http.Request) {
	vars := mux.Vars(req)
	instanceID := vars["instance_id"]
	operationData := req.FormValue("operation")

	logger := h.logger.Session(lastOperationLogKey, lager.Data{
		instanceIDLogKey: instanceID,
	})

	logger.Info("starting-check-for-operation")

	lastOperation, err := h.serviceBroker.LastOperation(req.Context(), instanceID, operationData)

	if err != nil {
		switch err {
		case ErrInstanceDoesNotExist:
			logger.Error(instanceMissingErrorKey, err)
			h.respond(w, http.StatusGone, ErrorResponse{
				Description: err.Error(),
			})
		default:
			logger.Error(unknownErrorKey, err)
			h.respond(w, http.StatusInternalServerError, ErrorResponse{
				Description: err.Error(),
			})
		}

		return
	}

	logger.WithData(lager.Data{"state": lastOperation.State}).Info("done-check-for-operation")

	lastOperationResponse := LastOperationResponse{
		State:       lastOperation.State,
		Description: lastOperation.Description,
	}

	h.respond(w, http.StatusOK, lastOperationResponse)
}

func (h serviceBrokerHandler) respond(w http.ResponseWriter, status int, response interface{}) {
	w.Header().Set("Content-Type", "application/json")
	w.WriteHeader(status)

	encoder := json.NewEncoder(w)
	err := encoder.Encode(response)
	if err != nil {
		h.logger.Error("encoding response", err, lager.Data{"status": status, "response": response})
	}
}
<commit_msg>Remove unused constants<commit_after>package brokerapi

import (
	"encoding/json"
	"net/http"
	"strconv"

	"code.cloudfoundry.org/lager"
	"github.com/gorilla/mux"
	"github.com/pivotal-cf/brokerapi/auth"
)

const provisionLogKey = "provision"
const deprovisionLogKey = "deprovision"
const bindLogKey = "bind"
const unbindLogKey = "unbind"
const lastOperationLogKey = "lastOperation"

const instanceIDLogKey = "instance-id"
const instanceDetailsLogKey = "instance-details"
const bindingIDLogKey = "binding-id"

const invalidServiceDetailsErrorKey = "invalid-service-details"
const invalidBindDetailsErrorKey = "invalid-bind-details"
const instanceLimitReachedErrorKey = "instance-limit-reached"
const instanceAlreadyExistsErrorKey = "instance-already-exists"
const bindingAlreadyExistsErrorKey = "binding-already-exists"
const instanceMissingErrorKey = "instance-missing"
const bindingMissingErrorKey = "binding-missing"
const asyncRequiredKey = "async-required"
const planChangeNotSupportedKey = "plan-change-not-supported"
const unknownErrorKey = "unknown-error"
const invalidRawParamsKey = "invalid-raw-params"
const appGuidNotProvidedErrorKey = "app-guid-not-provided"

const statusUnprocessableEntity = 422

type BrokerCredentials struct {
	Username string
	Password string
}

func New(serviceBroker ServiceBroker, logger lager.Logger, brokerCredentials BrokerCredentials) http.Handler {
	router := mux.NewRouter()
	AttachRoutes(router, serviceBroker, logger)
	return auth.NewWrapper(brokerCredentials.Username, brokerCredentials.Password).Wrap(router)
}

func AttachRoutes(router *mux.Router, serviceBroker ServiceBroker, logger lager.Logger) {
	handler := serviceBrokerHandler{serviceBroker: serviceBroker, logger: logger}
	router.HandleFunc("/v2/catalog", handler.catalog).Methods("GET")

	router.HandleFunc("/v2/service_instances/{instance_id}", handler.provision).Methods("PUT")
	router.HandleFunc("/v2/service_instances/{instance_id}", handler.deprovision).Methods("DELETE")
	router.HandleFunc("/v2/service_instances/{instance_id}/last_operation", handler.lastOperation).Methods("GET")
	router.HandleFunc("/v2/service_instances/{instance_id}", handler.update).Methods("PATCH")

	router.HandleFunc("/v2/service_instances/{instance_id}/service_bindings/{binding_id}", handler.bind).Methods("PUT")
	router.HandleFunc("/v2/service_instances/{instance_id}/service_bindings/{binding_id}", handler.unbind).Methods("DELETE")
}

type serviceBrokerHandler struct {
	serviceBroker ServiceBroker
	logger        lager.Logger
}

func (h serviceBrokerHandler) catalog(w http.ResponseWriter, req *http.Request) {
	catalog := CatalogResponse{
		Services: h.serviceBroker.Services(req.Context()),
	}

	h.respond(w, http.StatusOK, catalog)
}

func (h serviceBrokerHandler) provision(w http.ResponseWriter, req *http.Request) {
	vars := mux.Vars(req)
	instanceID := vars["instance_id"]

	logger := h.logger.Session(provisionLogKey, lager.Data{
		instanceIDLogKey: instanceID,
	})

	var details ProvisionDetails
	if err := json.NewDecoder(req.Body).Decode(&details); err != nil {
		logger.Error(invalidServiceDetailsErrorKey, err)
		h.respond(w, statusUnprocessableEntity, ErrorResponse{
			Description: err.Error(),
		})
		return
	}

	acceptsIncompleteFlag, _ := strconv.ParseBool(req.URL.Query().Get("accepts_incomplete"))

	logger = logger.WithData(lager.Data{
		instanceDetailsLogKey: details,
	})

	provisionResponse, err := h.serviceBroker.Provision(req.Context(), instanceID, details, acceptsIncompleteFlag)

	if err != nil {
		switch err {
		case ErrRawParamsInvalid:
			logger.Error(invalidRawParamsKey, err)
			h.respond(w, 422, ErrorResponse{
				Description: err.Error(),
			})
		case ErrInstanceAlreadyExists:
			logger.Error(instanceAlreadyExistsErrorKey, err)
			h.respond(w, http.StatusConflict, EmptyResponse{})
		case ErrInstanceLimitMet:
			logger.Error(instanceLimitReachedErrorKey, err)
			h.respond(w, http.StatusInternalServerError, ErrorResponse{
				Description: err.Error(),
			})
		case ErrAsyncRequired:
			logger.Error(asyncRequiredKey, err)
			h.respond(w, 422, ErrorResponse{
				Error:       "AsyncRequired",
				Description: err.Error(),
			})
		default:
			logger.Error(unknownErrorKey, err)
			h.respond(w, http.StatusInternalServerError, ErrorResponse{
				Description: err.Error(),
			})
		}
		return
	}

	if provisionResponse.IsAsync {
		h.respond(w, http.StatusAccepted, ProvisioningResponse{
			DashboardURL:  provisionResponse.DashboardURL,
			OperationData: provisionResponse.OperationData,
		})
	} else {
		h.respond(w, http.StatusCreated, ProvisioningResponse{
			DashboardURL: provisionResponse.DashboardURL,
		})
	}
}

func (h serviceBrokerHandler) update(w http.ResponseWriter, req *http.Request) {
	vars := mux.Vars(req)
	instanceID := vars["instance_id"]

	var details UpdateDetails
	if err := json.NewDecoder(req.Body).Decode(&details); err != nil {
		h.logger.Error(invalidServiceDetailsErrorKey, err)
		h.respond(w, statusUnprocessableEntity, ErrorResponse{
			Description: err.Error(),
		})
		return
	}

	acceptsIncompleteFlag, _ := strconv.ParseBool(req.URL.Query().Get("accepts_incomplete"))

	updateServiceSpec, err := h.serviceBroker.Update(req.Context(), instanceID, details, acceptsIncompleteFlag)
	if err != nil {
		switch err {
		case ErrAsyncRequired:
			h.logger.Error(asyncRequiredKey, err)
			h.respond(w, 422, ErrorResponse{
				Error:       "AsyncRequired",
				Description: err.Error(),
			})
			return

		case ErrPlanChangeNotSupported:
			h.logger.Error(planChangeNotSupportedKey, err)
			h.respond(w, 422, ErrorResponse{
				Error:       "PlanChangeNotSupported",
				Description: err.Error(),
			})
			return

		default:
			h.logger.Error(unknownErrorKey, err)
			h.respond(w, http.StatusInternalServerError, ErrorResponse{
				Description: err.Error(),
			})
			return
		}
	}

	statusCode := http.StatusOK
	if updateServiceSpec.IsAsync {
		statusCode = http.StatusAccepted
	}
	h.respond(w, statusCode, UpdateResponse{OperationData: updateServiceSpec.OperationData})
}

func (h serviceBrokerHandler) deprovision(w http.ResponseWriter, req *http.Request) {
	vars := mux.Vars(req)
	instanceID := vars["instance_id"]
	logger := h.logger.Session(deprovisionLogKey, lager.Data{
		instanceIDLogKey: instanceID,
	})

	details := DeprovisionDetails{
		PlanID:    req.FormValue("plan_id"),
		ServiceID: req.FormValue("service_id"),
	}
	asyncAllowed := req.FormValue("accepts_incomplete") == "true"

	deprovisionSpec, err := h.serviceBroker.Deprovision(req.Context(), instanceID, details, asyncAllowed)
	if err != nil {
		switch err {
		case ErrInstanceDoesNotExist:
			logger.Error(instanceMissingErrorKey, err)
			h.respond(w, http.StatusGone, EmptyResponse{})
		case ErrAsyncRequired:
			logger.Error(asyncRequiredKey, err)
			h.respond(w, 422, ErrorResponse{
				Error:       "AsyncRequired",
				Description: err.Error(),
			})
		default:
			logger.Error(unknownErrorKey, err)
			h.respond(w, http.StatusInternalServerError, ErrorResponse{
				Description: err.Error(),
			})
		}
		return
	}

	if deprovisionSpec.IsAsync {
		h.respond(w, http.StatusAccepted, DeprovisionResponse{OperationData: deprovisionSpec.OperationData})
	} else {
		h.respond(w, http.StatusOK, EmptyResponse{})
	}
}

func (h serviceBrokerHandler) bind(w http.ResponseWriter, req *http.Request) {
	vars := mux.Vars(req)
	instanceID := vars["instance_id"]
	bindingID := vars["binding_id"]

	logger := h.logger.Session(bindLogKey, lager.Data{
		instanceIDLogKey: instanceID,
		bindingIDLogKey:  bindingID,
	})

	var details BindDetails
	if err := json.NewDecoder(req.Body).Decode(&details); err != nil {
		logger.Error(invalidBindDetailsErrorKey, err)
		h.respond(w, statusUnprocessableEntity, ErrorResponse{
			Description: err.Error(),
		})
		return
	}

	binding, err := h.serviceBroker.Bind(req.Context(), instanceID, bindingID, details)
	if err != nil {
		switch err {
		case ErrInstanceDoesNotExist:
			logger.Error(instanceMissingErrorKey, err)
			h.respond(w, http.StatusNotFound, ErrorResponse{
				Description: err.Error(),
			})
		case ErrBindingAlreadyExists:
			logger.Error(bindingAlreadyExistsErrorKey, err)
			h.respond(w, http.StatusConflict, ErrorResponse{
				Description: err.Error(),
			})
		case ErrAppGuidNotProvided:
			logger.Error(appGuidNotProvidedErrorKey, err)
			h.respond(w, statusUnprocessableEntity, ErrorResponse{
				Description: err.Error(),
			})
		default:
			logger.Error(unknownErrorKey, err)
			h.respond(w, http.StatusInternalServerError, ErrorResponse{
				Description: err.Error(),
			})
		}
		return
	}

	brokerAPIVersion := req.Header.Get("X-Broker-Api-Version")
	if brokerAPIVersion == "2.8" || brokerAPIVersion == "2.9" {
		experimentalVols := []ExperimentalVolumeMount{}

		for _, vol := range binding.VolumeMounts {
			experimentalConfig, err := json.Marshal(vol.Device.MountConfig)
			if err != nil {
				logger.Error(unknownErrorKey, err)
				h.respond(w, http.StatusInternalServerError, ErrorResponse{Description: err.Error()})
				return
			}

			experimentalVols = append(experimentalVols, ExperimentalVolumeMount{
				ContainerPath: vol.ContainerDir,
				Mode:          vol.Mode,
				Private: ExperimentalVolumeMountPrivate{
					Driver:  vol.Driver,
					GroupID: vol.Device.VolumeId,
					Config:  string(experimentalConfig),
				},
			})
		}

		experimentalBinding := ExperimentalVolumeMountBindingResponse{
			Credentials:     binding.Credentials,
			RouteServiceURL: binding.RouteServiceURL,
			SyslogDrainURL:  binding.SyslogDrainURL,
			VolumeMounts:    experimentalVols,
		}
		h.respond(w, http.StatusCreated, experimentalBinding)
		return
	}

	h.respond(w, http.StatusCreated, binding)
}

func (h serviceBrokerHandler) unbind(w http.ResponseWriter, req *http.Request) {
	vars := mux.Vars(req)
	instanceID := vars["instance_id"]
	bindingID := vars["binding_id"]

	logger := h.logger.Session(unbindLogKey, lager.Data{
		instanceIDLogKey: instanceID,
		bindingIDLogKey:  bindingID,
	})

	details := UnbindDetails{
		PlanID:    req.FormValue("plan_id"),
		ServiceID: req.FormValue("service_id"),
	}

	if err := h.serviceBroker.Unbind(req.Context(), instanceID, bindingID, details); err != nil {
		switch err {
		case ErrInstanceDoesNotExist:
			logger.Error(instanceMissingErrorKey, err)
			h.respond(w, http.StatusGone, EmptyResponse{})
		case ErrBindingDoesNotExist:
			logger.Error(bindingMissingErrorKey, err)
			h.respond(w, http.StatusGone, EmptyResponse{})
		default:
			logger.Error(unknownErrorKey, err)
			h.respond(w, http.StatusInternalServerError, ErrorResponse{
				Description: err.Error(),
			})
		}
		return
	}

	h.respond(w, http.StatusOK, EmptyResponse{})
}

func (h serviceBrokerHandler) lastOperation(w http.ResponseWriter, req *http.Request) {
	vars := mux.Vars(req)
	instanceID := vars["instance_id"]
	operationData := req.FormValue("operation")

	logger := h.logger.Session(lastOperationLogKey, lager.Data{
		instanceIDLogKey: instanceID,
	})

	logger.Info("starting-check-for-operation")

	lastOperation, err := h.serviceBroker.LastOperation(req.Context(), instanceID, operationData)

	if err != nil {
		switch err {
		case ErrInstanceDoesNotExist:
			logger.Error(instanceMissingErrorKey, err)
			h.respond(w, http.StatusGone, ErrorResponse{
				Description: err.Error(),
			})
		default:
			logger.Error(unknownErrorKey, err)
			h.respond(w, http.StatusInternalServerError, ErrorResponse{
				Description: err.Error(),
			})
		}

		return
	}

	logger.WithData(lager.Data{"state": lastOperation.State}).Info("done-check-for-operation")

	lastOperationResponse := LastOperationResponse{
		State:       lastOperation.State,
		Description: lastOperation.Description,
	}

	h.respond(w, http.StatusOK, lastOperationResponse)
}

func (h serviceBrokerHandler) respond(w http.ResponseWriter, status int, response interface{}) {
	w.Header().Set("Content-Type", "application/json")
	w.WriteHeader(status)

	encoder := json.NewEncoder(w)
	err := encoder.Encode(response)
	if err != nil {
		h.logger.Error("encoding response", err, lager.Data{"status": status, "response": response})
	}
}
<|endoftext|>


<commit_before>package api

import (
	"bytes"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"net/http/httputil"
	"net/url"
	"strings"

	"encoding/json"
	"encoding/xml"
)

const (
	POST   = "POST"
	GET    = "GET"
	HEAD   = "HEAD"
	PUT    = "PUT"
	DELETE = "DELETE"
	PATCH  = "PATCH"
)

var types = map[string]string{
	"html":       "text/html",
	"json":       "application/json",
	"xml":        "application/xml",
	"text":       "text/plain",
	"urlencoded": "application/x-www-form-urlencoded",
	"form":       "application/x-www-form-urlencoded",
	"form-data":  "application/x-www-form-urlencoded",
}

type Agent struct {
	u       *url.URL
	t       string
	m       string
	heads   map[string]string
	query   url.Values
	cookies []*http.Cookie
	data    io.Reader
	length  int
	Error   error
	debug   bool
	conn    *http.Client
}

func URL(aurl string) *Agent {
	u, err := url.Parse(aurl)
	return &Agent{
		u:       u,
		t:       types["html"],
		m:       GET,
		heads:   make(map[string]string),
		query:   url.Values{},
		cookies: make([]*http.Cookie, 0),
		Error:   err,
		conn:    http.DefaultClient,
	}
}

func Get(aurl string) *Agent {
	return URL(aurl).Method(GET)
}

func Post(aurl string) *Agent {
	return URL(aurl).Method(POST)
}

func Patch(aurl string) *Agent {
	return URL(aurl).Method(PATCH)
}

func Put(aurl string) *Agent {
	return URL(aurl).Method(PUT)
}

func Head(aurl string) *Agent {
	return URL(aurl).Method(HEAD)
}

func HTTP(host string) *Agent {
	return &Agent{
		u: &url.URL{
			Scheme: "http",
			Host:   host,
		},
		t:       types["html"],
		m:       GET,
		heads:   make(map[string]string),
		query:   url.Values{},
		cookies: make([]*http.Cookie, 0),
	}
}

func HTTPs(host string) *Agent {
	return &Agent{
		u: &url.URL{
			Scheme: "https",
			Host:   host,
		},
		t:       types["html"],
		m:       GET,
		heads:   make(map[string]string),
		query:   url.Values{},
		cookies: make([]*http.Cookie, 0),
	}
}

func (a *Agent) Debug(flag bool) *Agent {
	a.debug = flag
	return a
}
func (a *Agent) URI(uri string) *Agent {
	a.u.Path = uri
	return a
}

func (a *Agent) QueryGet() url.Values {
	q := a.u.Query()
	for k, v := range a.query {
		for _, vv := range v {
			q.Add(k, vv)
		}
	}
	return q
}

func (a *Agent) QuerySet(key string, value string) *Agent {
	a.query.Set(key, value)
	return a
}

func (a *Agent) QueryAdd(key string, value string) *Agent {
	a.query.Add(key, value)
	return a
}

func (a *Agent) QueryDel(key string) *Agent {
	a.query.Del(key)
	return a
}

func (a *Agent) HeadSet(key string, value string) *Agent {
	a.heads[key] = value
	return a
}

func (a *Agent) HeadDel(key string) *Agent {
	delete(a.heads, key)
	return a
}

func (a *Agent) BasicAuthSet(user, password string) *Agent {
	a.u.User = url.UserPassword(user, password)
	return a
}

func (a *Agent) BasicAuthDel() *Agent {
	a.u.User = nil
	return a
}

func (a *Agent) CookiesAdd(cookies ...*http.Cookie) *Agent {
	a.cookies = append(a.cookies, cookies...)
	return a
}

func (a *Agent) Method(m string) *Agent {
	a.m = m
	return a
}

func (a *Agent) ContentType(t string) *Agent {
	a.m = types[t]
	return a
}

func (a *Agent) FormData(form map[string][]string) *Agent {
	data := url.Values(form).Encode()
	a.data = strings.NewReader(data)
	a.length = len(data)
	return a
}

func JSONMarshal(v interface{}, unescape bool) ([]byte, error) {
	b, err := json.Marshal(v)

	if unescape {
		b = bytes.Replace(b, []byte("\\u003c"), []byte("<"), -1)
		b = bytes.Replace(b, []byte("\\u003e"), []byte(">"), -1)
		b = bytes.Replace(b, []byte("\\u0026"), []byte("&"), -1)
	}
	return b, err
}

func (a *Agent) JSONData(args ...interface{}) *Agent {
	if len(args) == 1 {
		data, err := JSONMarshal(args[0], false)
		a.data = bytes.NewBuffer(data)
		a.length = len(data)
		a.Error = err
	}

	if len(args) == 2 {
		data, err := JSONMarshal(args[0], args[1])
		a.data = bytes.NewBuffer(data)
		a.length = len(data)
		a.Error = err
	}
	return a
}

func (a *Agent) XMLData(obj interface{}) *Agent {
	data, err := xml.Marshal(obj)
	a.data = bytes.NewBuffer(data)
	a.length = len(data)
	a.Error = err
	return a
}

func (a *Agent) Bytes() (*http.Response, []byte, error) {
	req, err := http.NewRequest(a.m, a.u.String(), a.data)
	if err != nil {
		a.Error = err
		return nil, nil, err
	}

	//! headers
	req.Header.Set("Content-Type", types[a.t])
	for k, v := range a.heads {
		req.Header.Set(k, v)
	}

	//! query
	q := req.URL.Query()
	for k, v := range a.query {
		for _, vv := range v {
			q.Add(k, vv)
		}
	}
	req.URL.RawQuery = q.Encode()

	//! basic auth
	if a.u.User != nil {
		if password, ok := a.u.User.Password(); ok {
			req.SetBasicAuth(a.u.User.Username(), password)
		}
	}

	//! cookies
	for _, cookie := range a.cookies {
		req.AddCookie(cookie)
	}

	//! do
	if a.debug {
		dump, _ := httputil.DumpRequest(req, true)
		log.Printf("api request\n-------------------------------\n%s\n", string(dump))
	}

	resp, err := a.conn.Do(req)
	if err != nil {
		a.Error = err
		return nil, nil, err
	}
	defer resp.Body.Close()

	if a.debug {
		dump, _ := httputil.DumpResponse(resp, true)
		log.Printf("api response\n--------------------------------\n%s\n", string(dump))
	}

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		a.Error = err
		return nil, nil, err
	}

	resp.Body = ioutil.NopCloser(bytes.NewBuffer(body))
	return resp, body, a.Error
}

func (a *Agent) Text() (*http.Response, string, error) {
	a.t = "text"

	resp, bytes, err := a.Bytes()
	return resp, string(bytes), err
}

func (a *Agent) JSON(obj interface{}) (*http.Response, []byte, error) {
	a.t = "json"

	resp, bytes, err := a.Bytes()
	if err != nil {
		return resp, bytes, err
	}

	//! decode bytes to json
	if obj != nil {
		if err := json.NewDecoder(resp.Body).Decode(&obj); err != nil {
			a.Error = err
			return resp, bytes, err
		}
	}

	return resp, bytes, a.Error
}

func (a *Agent) XML(obj interface{}) (*http.Response, []byte, error) {
	a.t = "xml"

	resp, bytes, err := a.Bytes()
	if err != nil {
		return resp, bytes, err
	}

	//! decode bytes to json
	if obj != nil {
		if err := xml.NewDecoder(resp.Body).Decode(&obj); err != nil {
			a.Error = err
			return resp, bytes, err
		}
	}

	return resp, bytes, a.Error
}
<commit_msg>fixed type assertion<commit_after>package api

import (
	"bytes"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"net/http/httputil"
	"net/url"
	"strings"

	"encoding/json"
	"encoding/xml"
)

const (
	POST   = "POST"
	GET    = "GET"
	HEAD   = "HEAD"
	PUT    = "PUT"
	DELETE = "DELETE"
	PATCH  = "PATCH"
)

var types = map[string]string{
	"html":       "text/html",
	"json":       "application/json",
	"xml":        "application/xml",
	"text":       "text/plain",
	"urlencoded": "application/x-www-form-urlencoded",
	"form":       "application/x-www-form-urlencoded",
	"form-data":  "application/x-www-form-urlencoded",
}

type Agent struct {
	u       *url.URL
	t       string
	m       string
	heads   map[string]string
	query   url.Values
	cookies []*http.Cookie
	data    io.Reader
	length  int
	Error   error
	debug   bool
	conn    *http.Client
}

func URL(aurl string) *Agent {
	u, err := url.Parse(aurl)
	return &Agent{
		u:       u,
		t:       types["html"],
		m:       GET,
		heads:   make(map[string]string),
		query:   url.Values{},
		cookies: make([]*http.Cookie, 0),
		Error:   err,
		conn:    http.DefaultClient,
	}
}

func Get(aurl string) *Agent {
	return URL(aurl).Method(GET)
}

func Post(aurl string) *Agent {
	return URL(aurl).Method(POST)
}

func Patch(aurl string) *Agent {
	return URL(aurl).Method(PATCH)
}

func Put(aurl string) *Agent {
	return URL(aurl).Method(PUT)
}

func Head(aurl string) *Agent {
	return URL(aurl).Method(HEAD)
}

func HTTP(host string) *Agent {
	return &Agent{
		u: &url.URL{
			Scheme: "http",
			Host:   host,
		},
		t:       types["html"],
		m:       GET,
		heads:   make(map[string]string),
		query:   url.Values{},
		cookies: make([]*http.Cookie, 0),
	}
}

func HTTPs(host string) *Agent {
	return &Agent{
		u: &url.URL{
			Scheme: "https",
			Host:   host,
		},
		t:       types["html"],
		m:       GET,
		heads:   make(map[string]string),
		query:   url.Values{},
		cookies: make([]*http.Cookie, 0),
	}
}

func (a *Agent) Debug(flag bool) *Agent {
	a.debug = flag
	return a
}
func (a *Agent) URI(uri string) *Agent {
	a.u.Path = uri
	return a
}

func (a *Agent) QueryGet() url.Values {
	q := a.u.Query()
	for k, v := range a.query {
		for _, vv := range v {
			q.Add(k, vv)
		}
	}
	return q
}

func (a *Agent) QuerySet(key string, value string) *Agent {
	a.query.Set(key, value)
	return a
}

func (a *Agent) QueryAdd(key string, value string) *Agent {
	a.query.Add(key, value)
	return a
}

func (a *Agent) QueryDel(key string) *Agent {
	a.query.Del(key)
	return a
}

func (a *Agent) HeadSet(key string, value string) *Agent {
	a.heads[key] = value
	return a
}

func (a *Agent) HeadDel(key string) *Agent {
	delete(a.heads, key)
	return a
}

func (a *Agent) BasicAuthSet(user, password string) *Agent {
	a.u.User = url.UserPassword(user, password)
	return a
}

func (a *Agent) BasicAuthDel() *Agent {
	a.u.User = nil
	return a
}

func (a *Agent) CookiesAdd(cookies ...*http.Cookie) *Agent {
	a.cookies = append(a.cookies, cookies...)
	return a
}

func (a *Agent) Method(m string) *Agent {
	a.m = m
	return a
}

func (a *Agent) ContentType(t string) *Agent {
	a.m = types[t]
	return a
}

func (a *Agent) FormData(form map[string][]string) *Agent {
	data := url.Values(form).Encode()
	a.data = strings.NewReader(data)
	a.length = len(data)
	return a
}

func JSONMarshal(v interface{}, unescape bool) ([]byte, error) {
	b, err := json.Marshal(v)

	if unescape {
		b = bytes.Replace(b, []byte("\\u003c"), []byte("<"), -1)
		b = bytes.Replace(b, []byte("\\u003e"), []byte(">"), -1)
		b = bytes.Replace(b, []byte("\\u0026"), []byte("&"), -1)
	}
	return b, err
}

func (a *Agent) JSONData(args ...interface{}) *Agent {
	if len(args) == 1 {
		data, err := JSONMarshal(args[0], false)
		a.data = bytes.NewBuffer(data)
		a.length = len(data)
		a.Error = err
	}

	if len(args) == 2 {
		data, err := JSONMarshal(args[0], args[1].(bool))
		a.data = bytes.NewBuffer(data)
		a.length = len(data)
		a.Error = err
	}
	return a
}

func (a *Agent) XMLData(obj interface{}) *Agent {
	data, err := xml.Marshal(obj)
	a.data = bytes.NewBuffer(data)
	a.length = len(data)
	a.Error = err
	return a
}

func (a *Agent) Bytes() (*http.Response, []byte, error) {
	req, err := http.NewRequest(a.m, a.u.String(), a.data)
	if err != nil {
		a.Error = err
		return nil, nil, err
	}

	//! headers
	req.Header.Set("Content-Type", types[a.t])
	for k, v := range a.heads {
		req.Header.Set(k, v)
	}

	//! query
	q := req.URL.Query()
	for k, v := range a.query {
		for _, vv := range v {
			q.Add(k, vv)
		}
	}
	req.URL.RawQuery = q.Encode()

	//! basic auth
	if a.u.User != nil {
		if password, ok := a.u.User.Password(); ok {
			req.SetBasicAuth(a.u.User.Username(), password)
		}
	}

	//! cookies
	for _, cookie := range a.cookies {
		req.AddCookie(cookie)
	}

	//! do
	if a.debug {
		dump, _ := httputil.DumpRequest(req, true)
		log.Printf("api request\n-------------------------------\n%s\n", string(dump))
	}

	resp, err := a.conn.Do(req)
	if err != nil {
		a.Error = err
		return nil, nil, err
	}
	defer resp.Body.Close()

	if a.debug {
		dump, _ := httputil.DumpResponse(resp, true)
		log.Printf("api response\n--------------------------------\n%s\n", string(dump))
	}

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		a.Error = err
		return nil, nil, err
	}

	resp.Body = ioutil.NopCloser(bytes.NewBuffer(body))
	return resp, body, a.Error
}

func (a *Agent) Text() (*http.Response, string, error) {
	a.t = "text"

	resp, bytes, err := a.Bytes()
	return resp, string(bytes), err
}

func (a *Agent) JSON(obj interface{}) (*http.Response, []byte, error) {
	a.t = "json"

	resp, bytes, err := a.Bytes()
	if err != nil {
		return resp, bytes, err
	}

	//! decode bytes to json
	if obj != nil {
		if err := json.NewDecoder(resp.Body).Decode(&obj); err != nil {
			a.Error = err
			return resp, bytes, err
		}
	}

	return resp, bytes, a.Error
}

func (a *Agent) XML(obj interface{}) (*http.Response, []byte, error) {
	a.t = "xml"

	resp, bytes, err := a.Bytes()
	if err != nil {
		return resp, bytes, err
	}

	//! decode bytes to json
	if obj != nil {
		if err := xml.NewDecoder(resp.Body).Decode(&obj); err != nil {
			a.Error = err
			return resp, bytes, err
		}
	}

	return resp, bytes, a.Error
}
<|endoftext|>


<commit_before>// Copyright 2015 Prometheus Team
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"encoding/json"
	"fmt"
	"net/http"
	"strconv"
	"sync"
	"time"

	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/common/log"
	"github.com/prometheus/common/model"
	"github.com/prometheus/common/route"
	"golang.org/x/net/context"

	"github.com/prometheus/alertmanager/provider"
	"github.com/prometheus/alertmanager/types"
	"github.com/prometheus/alertmanager/version"
)

var (
	numReceivedAlerts = prometheus.NewCounterVec(prometheus.CounterOpts{
		Namespace: "alertmanager",
		Name:      "alerts_received_total",
		Help:      "The total number of received alerts.",
	}, []string{"status"})

	numInvalidAlerts = prometheus.NewCounter(prometheus.CounterOpts{
		Namespace: "alertmanager",
		Name:      "alerts_invalid_total",
		Help:      "The total number of received alerts that were invalid.",
	})
)

func init() {
	prometheus.Register(numReceivedAlerts)
	prometheus.Register(numInvalidAlerts)
}

// API provides registration of handlers for API routes.
type API struct {
	alerts         provider.Alerts
	silences       provider.Silences
	config         string
	resolveTimeout time.Duration
	uptime         time.Time

	groups func() AlertOverview

	// context is an indirection for testing.
	context func(r *http.Request) context.Context
	mtx     sync.RWMutex
}

// NewAPI returns a new API.
func NewAPI(alerts provider.Alerts, silences provider.Silences, gf func() AlertOverview) *API {
	return &API{
		context:  route.Context,
		alerts:   alerts,
		silences: silences,
		groups:   gf,
		uptime:   time.Now(),
	}
}

// Register regieters the API handlers under their correct routes
// in the given router.
func (api *API) Register(r *route.Router) {
	ihf := prometheus.InstrumentHandlerFunc

	// Register legacy forwarder for alert pushing.
	r.Post("/alerts", ihf("legacy_add_alerts", api.legacyAddAlerts))

	// Register actual API.
	r = r.WithPrefix("/v1")

	r.Get("/status", ihf("status", api.status))
	r.Get("/alerts/groups", ihf("alert_groups", api.alertGroups))

	r.Get("/alerts", ihf("list_alerts", api.listAlerts))
	r.Post("/alerts", ihf("add_alerts", api.addAlerts))

	r.Get("/silences", ihf("list_silences", api.listSilences))
	r.Post("/silences", ihf("add_silence", api.addSilence))
	r.Get("/silence/:sid", ihf("get_silence", api.getSilence))
	r.Del("/silence/:sid", ihf("del_silence", api.delSilence))
}

// Update sets the configuration string to a new value.
func (api *API) Update(config string, resolveTimeout time.Duration) {
	api.mtx.Lock()
	defer api.mtx.Unlock()

	api.config = config
	api.resolveTimeout = resolveTimeout
}

type errorType string

const (
	errorNone     errorType = ""
	errorInternal           = "server_error"
	errorBadData            = "bad_data"
)

type apiError struct {
	typ errorType
	err error
}

func (e *apiError) Error() string {
	return fmt.Sprintf("%s: %s", e.typ, e.err)
}

func (api *API) status(w http.ResponseWriter, req *http.Request) {
	api.mtx.RLock()

	var status = struct {
		Config      string            `json:"config"`
		VersionInfo map[string]string `json:"versionInfo"`
		Uptime      time.Time         `json:"uptime"`
	}{
		Config:      api.config,
		VersionInfo: version.Map,
		Uptime:      api.uptime,
	}

	api.mtx.RUnlock()

	respond(w, status)
}

func (api *API) alertGroups(w http.ResponseWriter, req *http.Request) {
	respond(w, api.groups())
}

func (api *API) listAlerts(w http.ResponseWriter, r *http.Request) {
	alerts := api.alerts.GetPending()
	defer alerts.Close()

	var (
		err error
		res []*types.Alert
	)
	// TODO(fabxc): enforce a sensible timeout.
	for a := range alerts.Next() {
		if err = alerts.Err(); err != nil {
			break
		}
		res = append(res, a)
	}

	if err != nil {
		respondError(w, apiError{
			typ: errorInternal,
			err: err,
		}, nil)
		return
	}
	respond(w, types.Alerts(res...))
}

func (api *API) legacyAddAlerts(w http.ResponseWriter, r *http.Request) {
	var legacyAlerts = []struct {
		Summary     model.LabelValue `json:"summary"`
		Description model.LabelValue `json:"description"`
		Runbook     model.LabelValue `json:"runbook"`
		Labels      model.LabelSet   `json:"labels"`
		Payload     model.LabelSet   `json:"payload"`
	}{}
	if err := receive(r, &legacyAlerts); err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}

	var alerts []*types.Alert
	for _, la := range legacyAlerts {
		a := &types.Alert{
			Alert: model.Alert{
				Labels:      la.Labels,
				Annotations: la.Payload,
			},
		}
		if a.Annotations == nil {
			a.Annotations = model.LabelSet{}
		}
		a.Annotations["summary"] = la.Summary
		a.Annotations["description"] = la.Description
		a.Annotations["runbook"] = la.Runbook

		alerts = append(alerts, a)
	}

	api.insertAlerts(w, r, alerts...)
}

func (api *API) addAlerts(w http.ResponseWriter, r *http.Request) {
	var alerts []*types.Alert
	if err := receive(r, &alerts); err != nil {
		respondError(w, apiError{
			typ: errorBadData,
			err: err,
		}, nil)
		return
	}

	api.insertAlerts(w, r, alerts...)
}

func (api *API) insertAlerts(w http.ResponseWriter, r *http.Request, alerts ...*types.Alert) {
	now := time.Now()

	for _, alert := range alerts {
		alert.UpdatedAt = now

		// Ensure StartsAt is set.
		if alert.StartsAt.IsZero() {
			alert.StartsAt = now
		}
		// If no end time is defined, set a timeout after which an alert
		// is marked resolved if it is not updated.
		if alert.EndsAt.IsZero() {
			alert.Timeout = true
			alert.EndsAt = alert.StartsAt.Add(api.resolveTimeout)

			numReceivedAlerts.WithLabelValues("firing").Inc()
		} else {
			numReceivedAlerts.WithLabelValues("resolved").Inc()
		}
	}

	// Make a best effort to insert all alerts that are valid.
	var (
		validAlerts    = make([]*types.Alert, 0, len(alerts))
		validationErrs = &types.MultiError{}
	)
	for _, a := range alerts {
		if err := a.Validate(); err != nil {
			validationErrs.Add(err)
			numInvalidAlerts.Inc()
			continue
		}
		validAlerts = append(validAlerts, a)
	}

	if err := api.alerts.Put(validAlerts...); err != nil {
		respondError(w, apiError{
			typ: errorInternal,
			err: err,
		}, nil)
		return
	}

	if validationErrs.Len() > 0 {
		respondError(w, apiError{
			typ: errorBadData,
			err: validationErrs,
		}, nil)
		return
	}

	respond(w, nil)
}

func (api *API) addSilence(w http.ResponseWriter, r *http.Request) {
	var sil types.Silence
	if err := receive(r, &sil); err != nil {
		respondError(w, apiError{
			typ: errorBadData,
			err: err,
		}, nil)
		return
	}

	if sil.CreatedAt.IsZero() {
		sil.CreatedAt = time.Now()
	}

	if err := sil.Validate(); err != nil {
		respondError(w, apiError{
			typ: errorBadData,
			err: err,
		}, nil)
		return
	}

	sid, err := api.silences.Set(&sil)
	if err != nil {
		respondError(w, apiError{
			typ: errorInternal,
			err: err,
		}, nil)
		return
	}

	respond(w, struct {
		SilenceID uint64 `json:"silenceId"`
	}{
		SilenceID: sid,
	})
}

func (api *API) getSilence(w http.ResponseWriter, r *http.Request) {
	sids := route.Param(api.context(r), "sid")
	sid, err := strconv.ParseUint(sids, 10, 64)
	if err != nil {
		respondError(w, apiError{
			typ: errorBadData,
			err: err,
		}, nil)
		return
	}

	sil, err := api.silences.Get(sid)
	if err != nil {
		http.Error(w, fmt.Sprint("Error getting silence: ", err), http.StatusNotFound)
		return
	}

	respond(w, &sil)
}

func (api *API) delSilence(w http.ResponseWriter, r *http.Request) {
	sids := route.Param(api.context(r), "sid")
	sid, err := strconv.ParseUint(sids, 10, 64)
	if err != nil {
		respondError(w, apiError{
			typ: errorBadData,
			err: err,
		}, nil)
		return
	}

	if err := api.silences.Del(sid); err != nil {
		respondError(w, apiError{
			typ: errorInternal,
			err: err,
		}, nil)
		return
	}
	respond(w, nil)
}

func (api *API) listSilences(w http.ResponseWriter, r *http.Request) {
	sils, err := api.silences.All()
	if err != nil {
		respondError(w, apiError{
			typ: errorInternal,
			err: err,
		}, nil)
		return
	}
	respond(w, sils)
}

type status string

const (
	statusSuccess status = "success"
	statusError          = "error"
)

type response struct {
	Status    status      `json:"status"`
	Data      interface{} `json:"data,omitempty"`
	ErrorType errorType   `json:"errorType,omitempty"`
	Error     string      `json:"error,omitempty"`
}

func respond(w http.ResponseWriter, data interface{}) {
	w.Header().Set("Content-Type", "application/json")
	w.WriteHeader(200)

	b, err := json.Marshal(&response{
		Status: statusSuccess,
		Data:   data,
	})
	if err != nil {
		return
	}
	w.Write(b)
}

func respondError(w http.ResponseWriter, apiErr apiError, data interface{}) {
	w.Header().Set("Content-Type", "application/json")

	switch apiErr.typ {
	case errorBadData:
		w.WriteHeader(http.StatusBadRequest)
	case errorInternal:
		w.WriteHeader(http.StatusInternalServerError)
	default:
		panic(fmt.Sprintf("unknown error type", apiErr))
	}

	b, err := json.Marshal(&response{
		Status:    statusError,
		ErrorType: apiErr.typ,
		Error:     apiErr.err.Error(),
		Data:      data,
	})
	if err != nil {
		return
	}
	log.Errorf("api error: %v", apiErr)

	w.Write(b)
}

func receive(r *http.Request, v interface{}) error {
	dec := json.NewDecoder(r.Body)
	defer r.Body.Close()

	err := dec.Decode(v)
	if err != nil {
		log.Debugf("Decoding request failed: %v", err)
	}
	return err
}
<commit_msg>Fix assigned reosolve timeout<commit_after>// Copyright 2015 Prometheus Team
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"encoding/json"
	"fmt"
	"net/http"
	"strconv"
	"sync"
	"time"

	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/common/log"
	"github.com/prometheus/common/model"
	"github.com/prometheus/common/route"
	"golang.org/x/net/context"

	"github.com/prometheus/alertmanager/provider"
	"github.com/prometheus/alertmanager/types"
	"github.com/prometheus/alertmanager/version"
)

var (
	numReceivedAlerts = prometheus.NewCounterVec(prometheus.CounterOpts{
		Namespace: "alertmanager",
		Name:      "alerts_received_total",
		Help:      "The total number of received alerts.",
	}, []string{"status"})

	numInvalidAlerts = prometheus.NewCounter(prometheus.CounterOpts{
		Namespace: "alertmanager",
		Name:      "alerts_invalid_total",
		Help:      "The total number of received alerts that were invalid.",
	})
)

func init() {
	prometheus.Register(numReceivedAlerts)
	prometheus.Register(numInvalidAlerts)
}

// API provides registration of handlers for API routes.
type API struct {
	alerts         provider.Alerts
	silences       provider.Silences
	config         string
	resolveTimeout time.Duration
	uptime         time.Time

	groups func() AlertOverview

	// context is an indirection for testing.
	context func(r *http.Request) context.Context
	mtx     sync.RWMutex
}

// NewAPI returns a new API.
func NewAPI(alerts provider.Alerts, silences provider.Silences, gf func() AlertOverview) *API {
	return &API{
		context:  route.Context,
		alerts:   alerts,
		silences: silences,
		groups:   gf,
		uptime:   time.Now(),
	}
}

// Register regieters the API handlers under their correct routes
// in the given router.
func (api *API) Register(r *route.Router) {
	ihf := prometheus.InstrumentHandlerFunc

	// Register legacy forwarder for alert pushing.
	r.Post("/alerts", ihf("legacy_add_alerts", api.legacyAddAlerts))

	// Register actual API.
	r = r.WithPrefix("/v1")

	r.Get("/status", ihf("status", api.status))
	r.Get("/alerts/groups", ihf("alert_groups", api.alertGroups))

	r.Get("/alerts", ihf("list_alerts", api.listAlerts))
	r.Post("/alerts", ihf("add_alerts", api.addAlerts))

	r.Get("/silences", ihf("list_silences", api.listSilences))
	r.Post("/silences", ihf("add_silence", api.addSilence))
	r.Get("/silence/:sid", ihf("get_silence", api.getSilence))
	r.Del("/silence/:sid", ihf("del_silence", api.delSilence))
}

// Update sets the configuration string to a new value.
func (api *API) Update(config string, resolveTimeout time.Duration) {
	api.mtx.Lock()
	defer api.mtx.Unlock()

	api.config = config
	api.resolveTimeout = resolveTimeout
}

type errorType string

const (
	errorNone     errorType = ""
	errorInternal           = "server_error"
	errorBadData            = "bad_data"
)

type apiError struct {
	typ errorType
	err error
}

func (e *apiError) Error() string {
	return fmt.Sprintf("%s: %s", e.typ, e.err)
}

func (api *API) status(w http.ResponseWriter, req *http.Request) {
	api.mtx.RLock()

	var status = struct {
		Config      string            `json:"config"`
		VersionInfo map[string]string `json:"versionInfo"`
		Uptime      time.Time         `json:"uptime"`
	}{
		Config:      api.config,
		VersionInfo: version.Map,
		Uptime:      api.uptime,
	}

	api.mtx.RUnlock()

	respond(w, status)
}

func (api *API) alertGroups(w http.ResponseWriter, req *http.Request) {
	respond(w, api.groups())
}

func (api *API) listAlerts(w http.ResponseWriter, r *http.Request) {
	alerts := api.alerts.GetPending()
	defer alerts.Close()

	var (
		err error
		res []*types.Alert
	)
	// TODO(fabxc): enforce a sensible timeout.
	for a := range alerts.Next() {
		if err = alerts.Err(); err != nil {
			break
		}
		res = append(res, a)
	}

	if err != nil {
		respondError(w, apiError{
			typ: errorInternal,
			err: err,
		}, nil)
		return
	}
	respond(w, types.Alerts(res...))
}

func (api *API) legacyAddAlerts(w http.ResponseWriter, r *http.Request) {
	var legacyAlerts = []struct {
		Summary     model.LabelValue `json:"summary"`
		Description model.LabelValue `json:"description"`
		Runbook     model.LabelValue `json:"runbook"`
		Labels      model.LabelSet   `json:"labels"`
		Payload     model.LabelSet   `json:"payload"`
	}{}
	if err := receive(r, &legacyAlerts); err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}

	var alerts []*types.Alert
	for _, la := range legacyAlerts {
		a := &types.Alert{
			Alert: model.Alert{
				Labels:      la.Labels,
				Annotations: la.Payload,
			},
		}
		if a.Annotations == nil {
			a.Annotations = model.LabelSet{}
		}
		a.Annotations["summary"] = la.Summary
		a.Annotations["description"] = la.Description
		a.Annotations["runbook"] = la.Runbook

		alerts = append(alerts, a)
	}

	api.insertAlerts(w, r, alerts...)
}

func (api *API) addAlerts(w http.ResponseWriter, r *http.Request) {
	var alerts []*types.Alert
	if err := receive(r, &alerts); err != nil {
		respondError(w, apiError{
			typ: errorBadData,
			err: err,
		}, nil)
		return
	}

	api.insertAlerts(w, r, alerts...)
}

func (api *API) insertAlerts(w http.ResponseWriter, r *http.Request, alerts ...*types.Alert) {
	now := time.Now()

	for _, alert := range alerts {
		alert.UpdatedAt = now

		// Ensure StartsAt is set.
		if alert.StartsAt.IsZero() {
			alert.StartsAt = now
		}
		// If no end time is defined, set a timeout after which an alert
		// is marked resolved if it is not updated.
		if alert.EndsAt.IsZero() {
			alert.Timeout = true
			alert.EndsAt = now.Add(api.resolveTimeout)

			numReceivedAlerts.WithLabelValues("firing").Inc()
		} else {
			numReceivedAlerts.WithLabelValues("resolved").Inc()
		}
	}

	// Make a best effort to insert all alerts that are valid.
	var (
		validAlerts    = make([]*types.Alert, 0, len(alerts))
		validationErrs = &types.MultiError{}
	)
	for _, a := range alerts {
		if err := a.Validate(); err != nil {
			validationErrs.Add(err)
			numInvalidAlerts.Inc()
			continue
		}
		validAlerts = append(validAlerts, a)
	}
	if err := api.alerts.Put(validAlerts...); err != nil {
		respondError(w, apiError{
			typ: errorInternal,
			err: err,
		}, nil)
		return
	}

	if validationErrs.Len() > 0 {
		respondError(w, apiError{
			typ: errorBadData,
			err: validationErrs,
		}, nil)
		return
	}

	respond(w, nil)
}

func (api *API) addSilence(w http.ResponseWriter, r *http.Request) {
	var sil types.Silence
	if err := receive(r, &sil); err != nil {
		respondError(w, apiError{
			typ: errorBadData,
			err: err,
		}, nil)
		return
	}

	if sil.CreatedAt.IsZero() {
		sil.CreatedAt = time.Now()
	}

	if err := sil.Validate(); err != nil {
		respondError(w, apiError{
			typ: errorBadData,
			err: err,
		}, nil)
		return
	}

	sid, err := api.silences.Set(&sil)
	if err != nil {
		respondError(w, apiError{
			typ: errorInternal,
			err: err,
		}, nil)
		return
	}

	respond(w, struct {
		SilenceID uint64 `json:"silenceId"`
	}{
		SilenceID: sid,
	})
}

func (api *API) getSilence(w http.ResponseWriter, r *http.Request) {
	sids := route.Param(api.context(r), "sid")
	sid, err := strconv.ParseUint(sids, 10, 64)
	if err != nil {
		respondError(w, apiError{
			typ: errorBadData,
			err: err,
		}, nil)
		return
	}

	sil, err := api.silences.Get(sid)
	if err != nil {
		http.Error(w, fmt.Sprint("Error getting silence: ", err), http.StatusNotFound)
		return
	}

	respond(w, &sil)
}

func (api *API) delSilence(w http.ResponseWriter, r *http.Request) {
	sids := route.Param(api.context(r), "sid")
	sid, err := strconv.ParseUint(sids, 10, 64)
	if err != nil {
		respondError(w, apiError{
			typ: errorBadData,
			err: err,
		}, nil)
		return
	}

	if err := api.silences.Del(sid); err != nil {
		respondError(w, apiError{
			typ: errorInternal,
			err: err,
		}, nil)
		return
	}
	respond(w, nil)
}

func (api *API) listSilences(w http.ResponseWriter, r *http.Request) {
	sils, err := api.silences.All()
	if err != nil {
		respondError(w, apiError{
			typ: errorInternal,
			err: err,
		}, nil)
		return
	}
	respond(w, sils)
}

type status string

const (
	statusSuccess status = "success"
	statusError          = "error"
)

type response struct {
	Status    status      `json:"status"`
	Data      interface{} `json:"data,omitempty"`
	ErrorType errorType   `json:"errorType,omitempty"`
	Error     string      `json:"error,omitempty"`
}

func respond(w http.ResponseWriter, data interface{}) {
	w.Header().Set("Content-Type", "application/json")
	w.WriteHeader(200)

	b, err := json.Marshal(&response{
		Status: statusSuccess,
		Data:   data,
	})
	if err != nil {
		return
	}
	w.Write(b)
}

func respondError(w http.ResponseWriter, apiErr apiError, data interface{}) {
	w.Header().Set("Content-Type", "application/json")

	switch apiErr.typ {
	case errorBadData:
		w.WriteHeader(http.StatusBadRequest)
	case errorInternal:
		w.WriteHeader(http.StatusInternalServerError)
	default:
		panic(fmt.Sprintf("unknown error type", apiErr))
	}

	b, err := json.Marshal(&response{
		Status:    statusError,
		ErrorType: apiErr.typ,
		Error:     apiErr.err.Error(),
		Data:      data,
	})
	if err != nil {
		return
	}
	log.Errorf("api error: %v", apiErr)

	w.Write(b)
}

func receive(r *http.Request, v interface{}) error {
	dec := json.NewDecoder(r.Body)
	defer r.Body.Close()

	err := dec.Decode(v)
	if err != nil {
		log.Debugf("Decoding request failed: %v", err)
	}
	return err
}
<|endoftext|>


<commit_before>package logmetrics

import (
	"fmt"
	"log"
	"net"
	"time"
)

type keyPushStats struct {
	key_pushed    int64
	byte_pushed   int64
	last_report   time.Time
	hostname      string
	interval      int
	pusher_number int
}

func (f *keyPushStats) inc(data_written int) {
	f.key_pushed++
	f.byte_pushed += int64(data_written)
}

func (f *keyPushStats) getLine() []string {
	t := time.Now()

	f.last_report = t

	line := make([]string, 2)
	line[0] = fmt.Sprintf("logmetrics_collector.pusher.key_sent %d %d host=%s pusher_number=%d", t.Unix(), f.key_pushed, f.hostname, f.pusher_number)
	line[1] = fmt.Sprintf("logmetrics_collector.pusher.byte_sent %d %d host=%s pusher_number=%d", t.Unix(), f.byte_pushed, f.hostname, f.pusher_number)

	return line
}

func (f *keyPushStats) isTimeForStats() bool {
	return time.Now().Sub(f.last_report) > time.Duration(f.interval)*time.Second
}

func writeLine(config *Config, doNotSend bool, conn net.Conn, line string) (int, net.Conn) {
	if config.pushType == "tsd" {
		line = ("put " + line + "\n")
	} else {
		line = line + "\n"
	}

	byte_line := []byte(line)
	byte_written := len(byte_line)

	var err error
	if doNotSend {
		fmt.Print(line)
	} else {
		for {
			//Reconnect if needed
			if conn == nil {
				target := config.GetTsdTarget()
				log.Printf("Reconnecting to %s", target)

				if conn, err = net.Dial(config.pushProto, target); err != nil {
					log.Printf("Unable to reconnect: %s", err)
					time.Sleep(time.Duration(config.pushWait) * time.Second)
				}
			}

			if conn != nil {
				_, err = conn.Write(byte_line)

				if err != nil {
					log.Printf("Error writting data: %s", err)
					conn.Close()
					conn = nil
					time.Sleep(time.Duration(config.pushWait) * time.Second)
				} else {
					break
				}
			}

		}
	}

	return byte_written, conn
}

func StartTsdPushers(config *Config, tsd_pushers []chan []string, doNotSend bool) {
	if config.pushPort == 0 {
		return
	}

	hostname := getHostname()

	for i, _ := range tsd_pushers {
		channel_number := i

		log.Printf("TsdPusher[%d] started. Pushing keys to %s:%d over %s in %s format", channel_number, config.pushHost,
			config.pushPort, config.pushProto, config.pushType)

		tsd_push := tsd_pushers[channel_number]
		go func() {
			key_push_stats := keyPushStats{last_report: time.Now(), hostname: hostname, interval: config.stats_wait, pusher_number: channel_number}

			//Check if TSD has something to say
			//if config.pushType == "tsd" {
			//	go func() {
			//		response_buffer := make([]byte, 1024)
			//		for {
			//			if conn != nil {
			//				if size, read_err := conn.Read(response_buffer); read_err != nil && read_err != io.EOF {
			//					log.Printf("Unable to read response: %s %+V", read_err, read_err)
			//				} else if size > 0 {
			//					log.Print(string(response_buffer))
			//				}
			//			}
			//		}
			//	}()
			//}

			var conn net.Conn
			for keys := range tsd_push {
				for _, line := range keys {
					var bytes_written int
					bytes_written, conn = writeLine(config, doNotSend, conn, line)

					key_push_stats.inc(bytes_written)

					//Stats on key pushed, limit checks with modulo (now() is a syscall)
					if (key_push_stats.key_pushed%10000) == 0 && key_push_stats.isTimeForStats() {
						for _, local_line := range key_push_stats.getLine() {
							bytes_written, conn = writeLine(config, doNotSend, conn, local_line)
							key_push_stats.inc(bytes_written)
						}
					}
				}
			}

		}()
	}
}
<commit_msg>Check for output stats more often.<commit_after>package logmetrics

import (
	"fmt"
	"log"
	"net"
	"time"
)

type keyPushStats struct {
	key_pushed    int64
	byte_pushed   int64
	last_report   time.Time
	hostname      string
	interval      int
	pusher_number int
}

func (f *keyPushStats) inc(data_written int) {
	f.key_pushed++
	f.byte_pushed += int64(data_written)
}

func (f *keyPushStats) getLine() []string {
	t := time.Now()

	f.last_report = t

	line := make([]string, 2)
	line[0] = fmt.Sprintf("logmetrics_collector.pusher.key_sent %d %d host=%s pusher_number=%d", t.Unix(), f.key_pushed, f.hostname, f.pusher_number)
	line[1] = fmt.Sprintf("logmetrics_collector.pusher.byte_sent %d %d host=%s pusher_number=%d", t.Unix(), f.byte_pushed, f.hostname, f.pusher_number)

	return line
}

func (f *keyPushStats) isTimeForStats() bool {
	return time.Now().Sub(f.last_report) > time.Duration(f.interval)*time.Second
}

func writeLine(config *Config, doNotSend bool, conn net.Conn, line string) (int, net.Conn) {
	if config.pushType == "tsd" {
		line = ("put " + line + "\n")
	} else {
		line = line + "\n"
	}

	byte_line := []byte(line)
	byte_written := len(byte_line)

	var err error
	if doNotSend {
		fmt.Print(line)
	} else {
		for {
			//Reconnect if needed
			if conn == nil {
				target := config.GetTsdTarget()
				log.Printf("Reconnecting to %s", target)

				if conn, err = net.Dial(config.pushProto, target); err != nil {
					log.Printf("Unable to reconnect: %s", err)
					time.Sleep(time.Duration(config.pushWait) * time.Second)
				}
			}

			if conn != nil {
				_, err = conn.Write(byte_line)

				if err != nil {
					log.Printf("Error writting data: %s", err)
					conn.Close()
					conn = nil
					time.Sleep(time.Duration(config.pushWait) * time.Second)
				} else {
					break
				}
			}

		}
	}

	return byte_written, conn
}

func StartTsdPushers(config *Config, tsd_pushers []chan []string, doNotSend bool) {
	if config.pushPort == 0 {
		return
	}

	hostname := getHostname()

	for i, _ := range tsd_pushers {
		channel_number := i

		log.Printf("TsdPusher[%d] started. Pushing keys to %s:%d over %s in %s format", channel_number, config.pushHost,
			config.pushPort, config.pushProto, config.pushType)

		tsd_push := tsd_pushers[channel_number]
		go func() {
			key_push_stats := keyPushStats{last_report: time.Now(), hostname: hostname, interval: config.stats_wait, pusher_number: channel_number}

			//Check if TSD has something to say
			//if config.pushType == "tsd" {
			//	go func() {
			//		response_buffer := make([]byte, 1024)
			//		for {
			//			if conn != nil {
			//				if size, read_err := conn.Read(response_buffer); read_err != nil && read_err != io.EOF {
			//					log.Printf("Unable to read response: %s %+V", read_err, read_err)
			//				} else if size > 0 {
			//					log.Print(string(response_buffer))
			//				}
			//			}
			//		}
			//	}()
			//}

			var conn net.Conn
			for keys := range tsd_push {
				for _, line := range keys {
					var bytes_written int
					bytes_written, conn = writeLine(config, doNotSend, conn, line)

					key_push_stats.inc(bytes_written)

					//Stats on key pushed, limit checks with modulo (now() is a syscall)
					if (key_push_stats.key_pushed%1000) == 0 && key_push_stats.isTimeForStats() {
						for _, local_line := range key_push_stats.getLine() {
							bytes_written, conn = writeLine(config, doNotSend, conn, local_line)
							key_push_stats.inc(bytes_written)
						}
					}
				}
			}

		}()
	}
}
<|endoftext|>