<commit_before>package idt

import (
	"fmt"
	"log"
	"runtime"
	"sync"

	"code.google.com/p/biogo.store/kdtree"
)

var Debug = false

type IDT struct {
	coordmap map[string]int
	values   []int
	tree     kdtree.Tree
}

func NewIDT(coords [][2]float64, values []int) (*IDT, error) {
	if len(coords) != len(values) {
		return nil, fmt.Errorf("coords and values must be of the same length")
	}
	kdpts := kdtree.Points{}
	coordmap := make(map[string]int)
	for i, v := range coords {
		kdpt := kdtree.Point{v[0], v[1]}
		kdpts = append(kdpts, kdpt)
		coordmap[fmt.Sprint(kdpt)] = i
	}
	tree := kdtree.New(kdpts, false)
	return &IDT{coordmap: coordmap, values: values, tree: *tree}, nil
}

type OrdPt struct {
	index int
	pt    kdtree.Point
}

type OrdVal struct {
	index int
	val   int16
}

func (idt IDT) genOrdPts(base [][2]int, in chan OrdPt) {
	for i, v := range base {
		in <- OrdPt{index: i, pt: kdtree.Point{float64(v[0]), float64(v[1])}}
	}
	close(in)
}

func (idt IDT) Call(base [][2]int, nnear int, majority bool) (outarr []int16, err error) {
	outarr = make([]int16, len(base))

	in := make(chan OrdPt)
	out := make(chan OrdVal)

	var wg sync.WaitGroup

	if Debug {
		log.Printf("Now running IDT on %d CPUs!", runtime.NumCPU())
	}

	nCPU := runtime.NumCPU()
	runtime.GOMAXPROCS(nCPU)
	for i := 0; i < nCPU*nCPU; i++ {
		wg.Add(1)
		go func(i int) {
			if Debug {
				log.Printf("Starting reducer #%d!", i)
			}
			defer wg.Done()
			idt.Reduce(in, out, nnear, majority, i)
		}(i)
	}
	go func() { wg.Wait(); close(out) }()
	go idt.genOrdPts(base, in)

	for ordval := range out {
		outarr[ordval.index] = ordval.val
	}

	return outarr, nil
}

func (idt IDT) Reduce(in chan OrdPt, out chan OrdVal, nnear int, majority bool, i int) {
	for ordpt := range in {
		ordval := new(OrdVal)
		ordval.index = ordpt.index
		q := ordpt.pt

		nk := kdtree.NewNKeeper(nnear)
		idt.tree.NearestSet(nk, q)
		distance := []float64{}
		index := []int{}
		for _, val := range nk.Heap {
			if val.Comparable != nil {
				distance = append(distance, val.Dist)
				index = append(index, idt.coordmap[fmt.Sprint(val.Comparable)])
			}
		}
		if Debug {
			log.Print("distance: ", distance)
			log.Print("index: ", index)
		}
		var wz int
		if nnear == 1 || distance[0] < 1e-10 {
			wz = idt.values[index[0]]
		} else {
			w := make([]float64, len(distance))
			var sumw float64
			for i, d := range distance {
				w[i] = 1.0 / d
				sumw += w[i]
			}
			if Debug {
				log.Print("w: ", w)
				log.Print("sumw: ", sumw)
			}
			for i := range distance {
				w[i] /= sumw
			}
			if Debug {
				log.Print("new w: ", w)
			}
			values := make([]int, len(index))
			for i, v := range index {
				values[i] = idt.values[v]
			}
			if Debug {
				log.Print("values: ", values)
			}
			if majority {
				majordict := make(map[int]float64)
				for i, v := range values {
					majordict[v] += w[i]
				}
				if Debug {
					log.Print("majordict: ", majordict)
				}
				var max float64
				for k, v := range majordict {
					if v > max {
						wz = k
					}
				}
			} else {
				var err error
				wz, err = dot(w, values)
				if err != nil {
					panic(err)
				}
			}
		}
		if Debug {
			log.Print("wz: ", wz)
		}
		ordval.val = int16(wz)
		out <- *ordval
	}
}

func dot(x []float64, y []int) (int, error) {
	var retval float64
	if len(x) != len(y) {
		return 0, fmt.Errorf("lengths do not match")
	}
	for i := range x {
		retval += x[i] * float64(y[i])
	}
	return int(retval), nil
}
<commit_msg>Fixed bug with IDT and majority.<commit_after>package idt

import (
	"fmt"
	"log"
	"runtime"
	"sync"

	"code.google.com/p/biogo.store/kdtree"
)

var Debug = false

type IDT struct {
	coordmap map[string]int
	values   []int
	tree     kdtree.Tree
}

func NewIDT(coords [][2]float64, values []int) (*IDT, error) {
	if len(coords) != len(values) {
		return nil, fmt.Errorf("coords and values must be of the same length")
	}
	kdpts := kdtree.Points{}
	coordmap := make(map[string]int)
	for i, v := range coords {
		kdpt := kdtree.Point{v[0], v[1]}
		kdpts = append(kdpts, kdpt)
		coordmap[fmt.Sprint(kdpt)] = i
	}
	tree := kdtree.New(kdpts, false)
	return &IDT{coordmap: coordmap, values: values, tree: *tree}, nil
}

type OrdPt struct {
	index int
	pt    kdtree.Point
}

type OrdVal struct {
	index int
	val   int16
}

func (idt IDT) genOrdPts(base [][2]int, in chan OrdPt) {
	for i, v := range base {
		in <- OrdPt{index: i, pt: kdtree.Point{float64(v[0]), float64(v[1])}}
	}
	close(in)
}

func (idt IDT) Call(base [][2]int, nnear int, majority bool) (outarr []int16, err error) {
	outarr = make([]int16, len(base))

	in := make(chan OrdPt)
	out := make(chan OrdVal)

	var wg sync.WaitGroup

	nCPU := runtime.NumCPU()
	numWorkers := nCPU * nCPU
	if Debug {
		log.Printf("debug mode -- only one worker!")
		numWorkers = 1
	}

	for i := 0; i < numWorkers; i++ {
		wg.Add(1)
		go func(i int) {
			if Debug {
				log.Printf("Starting reducer #%d!", i)
			}
			defer wg.Done()
			idt.Reduce(in, out, nnear, majority, i)
		}(i)
	}
	go func() { wg.Wait(); close(out) }()
	go idt.genOrdPts(base, in)

	for ordval := range out {
		outarr[ordval.index] = ordval.val
	}

	return outarr, nil
}

func (idt IDT) Reduce(in chan OrdPt, out chan OrdVal, nnear int, majority bool, i int) {
	for ordpt := range in {
		ordval := new(OrdVal)
		ordval.index = ordpt.index
		q := ordpt.pt

		nk := kdtree.NewNKeeper(nnear)
		idt.tree.NearestSet(nk, q)
		distance := []float64{}
		index := []int{}
		for _, val := range nk.Heap {
			if val.Comparable != nil {
				distance = append(distance, val.Dist)
				index = append(index, idt.coordmap[fmt.Sprint(val.Comparable)])
			}
		}
		if Debug {
			log.Print("distance: ", distance)
			log.Print("index: ", index)
		}
		var wz int
		if nnear == 1 || distance[0] < 1e-10 {
			wz = idt.values[index[0]]
		} else {
			w := make([]float64, len(distance))
			var sumw float64
			for i, d := range distance {
				w[i] = 1.0 / d
				sumw += w[i]
			}
			if Debug {
				log.Print("w: ", w)
				log.Print("sumw: ", sumw)
			}
			for i := range distance {
				w[i] /= sumw
			}
			if Debug {
				log.Print("new w: ", w)
			}
			values := make([]int, len(index))
			for i, v := range index {
				values[i] = idt.values[v]
			}
			if Debug {
				log.Print("values: ", values)
			}
			if majority {
				majordict := make(map[int]float64)
				for i, v := range values {
					majordict[v] += w[i]
				}
				if Debug {
					log.Print("majordict: ")
					for k, v := range majordict {
						log.Printf(" %d: %f", k, v)
					}
				}
				major := float64(0)
				for k, v := range majordict {
					if v > major {
						if Debug {
							log.Printf("new max value %f with ind %d", v, k)
						}
						major = v
						wz = k
					}
				}
			} else {
				var err error
				wz, err = dot(w, values)
				if err != nil {
					panic(err)
				}
			}
		}
		if Debug {
			log.Print("wz: ", wz)
		}
		ordval.val = int16(wz)
		out <- *ordval
	}
}

func dot(x []float64, y []int) (int, error) {
	var retval float64
	if len(x) != len(y) {
		return 0, fmt.Errorf("lengths do not match")
	}
	for i := range x {
		retval += x[i] * float64(y[i])
	}
	return int(retval), nil
}
<|endoftext|>


<commit_before>package lnwire

import (
	"io"

	"github.com/btcsuite/btcd/chaincfg/chainhash"
)

// QueryChannelRange is a message sent by a node in order to query the
// receiving node of the set of open channel they know of with short channel
// ID's after the specified block height, capped at the number of blocks beyond
// that block height. This will be used by nodes upon initial connect to
// synchronize their views of the network.
type QueryChannelRange struct {
	// ChainHash denotes the target chain that we're trying to synchronize
	// channel graph state for.
	ChainHash chainhash.Hash

	// FirstBlockHeight is the first block in the query range. The
	// responder should send all new short channel IDs from this block
	// until this block plus the specified number of blocks.
	FirstBlockHeight uint32

	// NumBlocks is the number of blocks beyond the first block that short
	// channel ID's should be sent for.
	NumBlocks uint32
}

// NewQueryChannelRange creates a new empty QueryChannelRange message.
func NewQueryChannelRange() *QueryChannelRange {
	return &QueryChannelRange{}
}

// A compile time check to ensure QueryChannelRange implements the
// lnwire.Message interface.
var _ Message = (*QueryChannelRange)(nil)

// Decode deserializes a serialized QueryChannelRange message stored in the
// passed io.Reader observing the specified protocol version.
//
// This is part of the lnwire.Message interface.
func (q *QueryChannelRange) Decode(r io.Reader, pver uint32) error {
	return ReadElements(r,
		q.ChainHash[:],
		&q.FirstBlockHeight,
		&q.NumBlocks,
	)
}

// Encode serializes the target QueryChannelRange into the passed io.Writer
// observing the protocol version specified.
//
// This is part of the lnwire.Message interface.
func (q *QueryChannelRange) Encode(w io.Writer, pver uint32) error {
	return WriteElements(w,
		q.ChainHash[:],
		q.FirstBlockHeight,
		q.NumBlocks,
	)
}

// MsgType returns the integer uniquely identifying this message type on the
// wire.
//
// This is part of the lnwire.Message interface.
func (q *QueryChannelRange) MsgType() MessageType {
	return MsgQueryChannelRange
}

// MaxPayloadLength returns the maximum allowed payload size for a
// QueryChannelRange complete message observing the specified protocol version.
//
// This is part of the lnwire.Message interface.
func (q *QueryChannelRange) MaxPayloadLength(uint32) uint32 {
	// 32 + 4 + 4
	return 40
}
<commit_msg>lnwire: add LastBlockHeight method to QueryChannelRange<commit_after>package lnwire

import (
	"io"
	"math"

	"github.com/btcsuite/btcd/chaincfg/chainhash"
)

// QueryChannelRange is a message sent by a node in order to query the
// receiving node of the set of open channel they know of with short channel
// ID's after the specified block height, capped at the number of blocks beyond
// that block height. This will be used by nodes upon initial connect to
// synchronize their views of the network.
type QueryChannelRange struct {
	// ChainHash denotes the target chain that we're trying to synchronize
	// channel graph state for.
	ChainHash chainhash.Hash

	// FirstBlockHeight is the first block in the query range. The
	// responder should send all new short channel IDs from this block
	// until this block plus the specified number of blocks.
	FirstBlockHeight uint32

	// NumBlocks is the number of blocks beyond the first block that short
	// channel ID's should be sent for.
	NumBlocks uint32
}

// NewQueryChannelRange creates a new empty QueryChannelRange message.
func NewQueryChannelRange() *QueryChannelRange {
	return &QueryChannelRange{}
}

// A compile time check to ensure QueryChannelRange implements the
// lnwire.Message interface.
var _ Message = (*QueryChannelRange)(nil)

// Decode deserializes a serialized QueryChannelRange message stored in the
// passed io.Reader observing the specified protocol version.
//
// This is part of the lnwire.Message interface.
func (q *QueryChannelRange) Decode(r io.Reader, pver uint32) error {
	return ReadElements(r,
		q.ChainHash[:],
		&q.FirstBlockHeight,
		&q.NumBlocks,
	)
}

// Encode serializes the target QueryChannelRange into the passed io.Writer
// observing the protocol version specified.
//
// This is part of the lnwire.Message interface.
func (q *QueryChannelRange) Encode(w io.Writer, pver uint32) error {
	return WriteElements(w,
		q.ChainHash[:],
		q.FirstBlockHeight,
		q.NumBlocks,
	)
}

// MsgType returns the integer uniquely identifying this message type on the
// wire.
//
// This is part of the lnwire.Message interface.
func (q *QueryChannelRange) MsgType() MessageType {
	return MsgQueryChannelRange
}

// MaxPayloadLength returns the maximum allowed payload size for a
// QueryChannelRange complete message observing the specified protocol version.
//
// This is part of the lnwire.Message interface.
func (q *QueryChannelRange) MaxPayloadLength(uint32) uint32 {
	// 32 + 4 + 4
	return 40
}

// LastBlockHeight returns the last block height covered by the range of a
// QueryChannelRange message.
func (q *QueryChannelRange) LastBlockHeight() uint32 {
	// Handle overflows by casting to uint64.
	lastBlockHeight := uint64(q.FirstBlockHeight) + uint64(q.NumBlocks) - 1
	if lastBlockHeight > math.MaxUint32 {
		return math.MaxUint32
	}
	return uint32(lastBlockHeight)
}
<|endoftext|>


<commit_before>// Copyright (c) 2017 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package config

import (
	"bytes"
	"fmt"
	"reflect"
	"strconv"

	"github.com/go-validator/validator"
)

const (
	bucketPrimitive = iota
	bucketArray
	bucketObject
	bucketMap
	bucketSlice
)

type fieldInfo struct {
	FieldName    string
	DefaultValue string
	Required     bool
}

func getFieldInfo(field reflect.StructField) fieldInfo {
	return fieldInfo{
		FieldName:    field.Tag.Get("yaml"),
		DefaultValue: field.Tag.Get("default"),
	}
}

func derefType(t reflect.Type) reflect.Type {
	if t.Kind() == reflect.Ptr {
		t = t.Elem()
	}

	return t
}

func convertValueFromStruct(value interface{}, targetType reflect.Type, fieldType reflect.Type, fieldValue reflect.Value) error {
	// The fieldType is probably a custom type here. We will try and set the fieldValue by
	// the custom type
	// TODO: refactor switch cases into isType functions
	// TODO(alsam) Fix overflows/negatives for unsigned types...
	switch fieldType.Kind() {
	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
		fieldValue.SetInt(int64(value.(int)))
	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
		fieldValue.SetUint(uint64(value.(int)))
	case reflect.Float32, reflect.Float64:
		fieldValue.SetFloat(value.(float64))
	case reflect.Bool:
		fieldValue.SetBool(value.(bool))
	case reflect.String:
		fieldValue.SetString(value.(string))
	default:
		return fmt.Errorf("can't convert %v to %v", reflect.TypeOf(value).String(), targetType)
	}
	return nil
}

func getBucket(t reflect.Type) int {
	kind := t.Kind()
	if kind == reflect.Ptr {
		kind = t.Elem().Kind()
	}

	switch kind {
	case reflect.Chan:
		fallthrough
	case reflect.Interface:
		fallthrough
	case reflect.Func:
		fallthrough
	case reflect.Map:
		return bucketMap
	case reflect.Array:
		return bucketArray
	case reflect.Slice:
		return bucketSlice
	case reflect.Struct:
		return bucketObject
	}
	return bucketPrimitive
}

type decoder struct {
	*Value
	m map[interface{}]struct{}
}

func (d *decoder) getGlobalProvider() Provider {
	if d.root == nil {
		return d.provider
	}

	return d.root
}

// Sets value to a primitive type.
func (d *decoder) scalar(childKey string, value reflect.Value, def string) error {
	valueType := value.Type()
	global := d.getGlobalProvider()
	var val interface{}

	if valueType.Kind() == reflect.Ptr {
		if v1 := global.Get(childKey); v1.HasValue() {
			val = v1.Value()
			if val != nil {
				// We cannot assign reflect.ValueOf(Val) to it as is to value.
				// value is a pointer, which currently points to non address.
				// We need to set a new reflect.Value with field type same as the field we are populating
				// before assigning the value (val) parsed from yaml
				if value.IsNil() {
					value.Set(reflect.New(valueType.Elem()))
				}

				// We cannot assign reflect.ValueOf(val) to value as is, when field is a user defined type
				// We need to find the Kind of the custom type and set the value to the specific type
				// that user defined type is defined with.
				kind := valueType.Elem().Kind()
				switch kind {
				case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
					value.Elem().SetInt(int64(val.(int)))
				case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
					value.Elem().SetUint(uint64(val.(int)))
				case reflect.Float32, reflect.Float64:
					value.Elem().SetFloat(val.(float64))
				case reflect.Bool:
					value.Elem().SetBool(val.(bool))
				case reflect.String:
					value.Elem().SetString(val.(string))
				default:
					value.Elem().Set(reflect.ValueOf(val))
				}
			}
		}

		return nil
	}

	// For primitive values, just get the value and set it into the field
	if v2 := global.Get(childKey); v2.HasValue() {
		val = v2.Value()
	} else if def != "" {
		val = def
	}

	if val != nil {
		// First try to convert primitive type values, if convertValue wasn't able
		// to convert to primitive,try converting the value as a struct value
		if ret, err := convertValue(val, valueType); ret != nil {
			if err != nil {
				return err
			}

			value.Set(reflect.ValueOf(ret))
		} else {
			return convertValueFromStruct(val, value.Type(), valueType, value)
		}
	}

	return nil
}

// Set value for a sequence type
// TODO(alsam) We stop populating sequence on a first nil value. Can we do better?
func (d *decoder) sequence(childKey string, value reflect.Value) error {
	valueType := value.Type()
	global := d.getGlobalProvider()
	destSlice := reflect.MakeSlice(valueType, 0, 4)

	// start looking for child values.
	elementType := derefType(valueType).Elem()
	childKey += _separator

	for ai := 0; ; ai++ {
		arrayKey := childKey + strconv.Itoa(ai)

		// Iterate until we find first missing value.
		if v2 := global.Get(arrayKey); v2.HasValue() {
			val := reflect.New(elementType).Elem()

			// Unmarshal current element.
			if err := d.unmarshal(arrayKey, val, ""); err != nil {
				return err
			}

			// Append element to the slice
			if destSlice.Len() <= ai {
				destSlice = reflect.Append(destSlice, reflect.Zero(elementType))
			}

			destSlice.Index(ai).Set(val)
		} else {
			break
		}
	}

	if destSlice.Len() > 0 {
		value.Set(destSlice)
	}

	return nil
}

// Set value for the array type
func (d *decoder) array(childKey string, value reflect.Value) error {
	valueType := value.Type()
	global := d.getGlobalProvider()

	// start looking for child values.
	elementType := derefType(valueType).Elem()
	childKey += _separator

	for ai := 0; ai < value.Len(); ai++ {
		arrayKey := childKey + strconv.Itoa(ai)

		// Iterate until we find first missing value.
		if v2 := global.Get(arrayKey); v2.HasValue() {
			val := reflect.New(elementType).Elem()

			// Unmarshal current element.
			if err := d.unmarshal(arrayKey, val, ""); err != nil {
				return err
			}

			value.Index(ai).Set(val)
		} else if value.Index(ai).Kind() == reflect.Struct {
			if err := d.valueStruct(arrayKey, value.Index(ai).Addr().Interface()); err != nil {
				return err
			}
		}
	}

	return nil
}

// Sets value to a map type.
func (d *decoder) mapping(childKey string, value reflect.Value, def string) error {
	valueType := value.Type()
	global := d.getGlobalProvider()

	v := global.Get(childKey)
	if !v.HasValue() || v.Value() == nil {
		return nil
	}

	val := v.Value()

	// We fallthrough for interface to maps
	if valueType.Kind() == reflect.Interface {
		value.Set(reflect.ValueOf(val))
		return nil
	}

	destMap := reflect.ValueOf(reflect.MakeMap(valueType).Interface())

	// child yamlNode parsed from yaml file is of type map[interface{}]interface{}
	// type casting here makes sure that we are iterating over a parsed map.
	if v, ok := val.(map[interface{}]interface{}); ok {
		childKey += _separator
		for key := range v {
			mapKey := childKey + fmt.Sprintf("%v", key)
			itemValue := reflect.New(valueType.Elem()).Elem()

			// Try to unmarshal value and save it in the map.
			if err := d.unmarshal(mapKey, itemValue, def); err != nil {
				return err
			}

			destMap.SetMapIndex(reflect.ValueOf(key), itemValue)
		}

		value.Set(destMap)
	}

	return nil
}

// Sets value to an object type.
func (d *decoder) object(childKey string, value reflect.Value) error {
	valueType := value.Type()
	global := d.getGlobalProvider()

	v2 := global.Get(childKey)

	if !v2.HasValue() && valueType.Kind() == reflect.Ptr {
		// in this case we will keep the pointer value as not defined.
		return nil
	}

	if !value.CanSet() {
		return nil
	}

	if valueType.Kind() != reflect.Ptr {
		value = value.Addr()
	}

	if value.IsNil() {
		tmp := reflect.New(value.Type().Elem())
		value.Set(tmp)
	}

	return d.valueStruct(childKey, value.Interface())
}

// Walk through the struct and start asking the providers for values at each key.
//
// - for individual values, we terminate
// - for array values, we start asking for indexes
// - for object values, we recurse.
func (d *decoder) valueStruct(key string, target interface{}) error {
	tarGet := reflect.Indirect(reflect.ValueOf(target))
	targetType := tarGet.Type()
	for i := 0; i < targetType.NumField(); i++ {
		field := targetType.Field(i)

		// Check for the private field
		if field.PkgPath != "" && !field.Anonymous {
			continue
		}

		fieldName := field.Name
		fieldInfo := getFieldInfo(field)
		if fieldInfo.FieldName != "" {
			fieldName = fieldInfo.FieldName
		}

		if key != "" {
			fieldName = key + _separator + fieldName
		}

		fieldValue := tarGet.Field(i)
		if fieldValue.Kind() == reflect.Ptr && fieldValue.IsNil() {
			fieldValue.Set(reflect.New(fieldValue.Type()).Elem())
		}

		if err := d.unmarshal(fieldName, fieldValue, getFieldInfo(field).DefaultValue); err != nil {
			return err
		}
	}

	return validator.Validate(target)
}

// Dispatch un-marshalling functions based on the value type.
func (d *decoder) unmarshal(name string, value reflect.Value, def string) error {
	if value.Type().Comparable() {
		val := value.Interface()
		kind := value.Kind()
		if _, ok := d.m[val]; ok {
			if kind == reflect.Ptr && !value.IsNil() {
				buf := &bytes.Buffer{}
				for k := range d.m {
					fmt.Fprintf(buf, "%+v -> ", k)
				}

				fmt.Fprintf(buf, "%+v", value.Interface())
				return fmt.Errorf("cycles detected: %s", buf.String())
			}
		}

		d.m[val] = struct{}{}
	}

	switch getBucket(value.Type()) {
	case bucketPrimitive:
		return d.scalar(name, value, def)
	case bucketObject:
		return d.object(name, value)
	case bucketArray:
		return d.array(name, value)
	case bucketSlice:
		return d.sequence(name, value)
	case bucketMap:
		return d.mapping(name, value, def)
	}

	return nil
}
<commit_msg>Remove code duplication in scalar function (#390)<commit_after>// Copyright (c) 2017 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package config

import (
	"bytes"
	"fmt"
	"reflect"
	"strconv"

	"github.com/go-validator/validator"
)

const (
	bucketPrimitive = iota
	bucketArray
	bucketObject
	bucketMap
	bucketSlice
)

type fieldInfo struct {
	FieldName    string
	DefaultValue string
	Required     bool
}

func getFieldInfo(field reflect.StructField) fieldInfo {
	return fieldInfo{
		FieldName:    field.Tag.Get("yaml"),
		DefaultValue: field.Tag.Get("default"),
	}
}

func derefType(t reflect.Type) reflect.Type {
	if t.Kind() == reflect.Ptr {
		t = t.Elem()
	}

	return t
}

func convertValueFromStruct(value interface{}, targetType reflect.Type, fieldType reflect.Type, fieldValue reflect.Value) error {
	// The fieldType is probably a custom type here. We will try and set the fieldValue by
	// the custom type
	// TODO: refactor switch cases into isType functions
	// TODO(alsam) Fix overflows/negatives for unsigned types...
	switch fieldType.Kind() {
	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
		fieldValue.SetInt(int64(value.(int)))
	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
		fieldValue.SetUint(uint64(value.(int)))
	case reflect.Float32, reflect.Float64:
		fieldValue.SetFloat(value.(float64))
	case reflect.Bool:
		fieldValue.SetBool(value.(bool))
	case reflect.String:
		fieldValue.SetString(value.(string))
	default:
		return fmt.Errorf("can't convert %v to %v", reflect.TypeOf(value).String(), targetType)
	}
	return nil
}

func getBucket(t reflect.Type) int {
	kind := t.Kind()
	if kind == reflect.Ptr {
		kind = t.Elem().Kind()
	}

	switch kind {
	case reflect.Chan:
		fallthrough
	case reflect.Interface:
		fallthrough
	case reflect.Func:
		fallthrough
	case reflect.Map:
		return bucketMap
	case reflect.Array:
		return bucketArray
	case reflect.Slice:
		return bucketSlice
	case reflect.Struct:
		return bucketObject
	}
	return bucketPrimitive
}

type decoder struct {
	*Value
	m map[interface{}]struct{}
}

func (d *decoder) getGlobalProvider() Provider {
	if d.root == nil {
		return d.provider
	}

	return d.root
}

// Sets value to a primitive type.
func (d *decoder) scalar(childKey string, value reflect.Value, def string) error {
	valueType := value.Type()
	global := d.getGlobalProvider()
	var val interface{}

	if valueType.Kind() == reflect.Ptr {
		if v1 := global.Get(childKey); v1.HasValue() {
			val = v1.Value()
			if val != nil {
				// We cannot assign reflect.ValueOf(Val) to it as is to value.
				// value is a pointer, which currently points to non address.
				// We need to set a new reflect.Value with field type same as the field we are populating
				// before assigning the value (val) parsed from yaml
				if value.IsNil() {
					value.Set(reflect.New(valueType.Elem()))
				}

				return d.scalar(childKey, value.Elem(), def)
			}
		}

		return nil
	}

	// For primitive values, just get the value and set it into the field
	if v2 := global.Get(childKey); v2.HasValue() {
		val = v2.Value()
	} else if def != "" {
		val = def
	}

	if val != nil {
		// First try to convert primitive type values, if convertValue wasn't able
		// to convert to primitive,try converting the value as a struct value
		if ret, err := convertValue(val, valueType); ret != nil {
			if err != nil {
				return err
			}

			value.Set(reflect.ValueOf(ret))
		} else {
			return convertValueFromStruct(val, value.Type(), valueType, value)
		}
	}

	return nil
}

// Set value for a sequence type
// TODO(alsam) We stop populating sequence on a first nil value. Can we do better?
func (d *decoder) sequence(childKey string, value reflect.Value) error {
	valueType := value.Type()
	global := d.getGlobalProvider()
	destSlice := reflect.MakeSlice(valueType, 0, 4)

	// start looking for child values.
	elementType := derefType(valueType).Elem()
	childKey += _separator

	for ai := 0; ; ai++ {
		arrayKey := childKey + strconv.Itoa(ai)

		// Iterate until we find first missing value.
		if v2 := global.Get(arrayKey); v2.HasValue() {
			val := reflect.New(elementType).Elem()

			// Unmarshal current element.
			if err := d.unmarshal(arrayKey, val, ""); err != nil {
				return err
			}

			// Append element to the slice
			if destSlice.Len() <= ai {
				destSlice = reflect.Append(destSlice, reflect.Zero(elementType))
			}

			destSlice.Index(ai).Set(val)
		} else {
			break
		}
	}

	if destSlice.Len() > 0 {
		value.Set(destSlice)
	}

	return nil
}

// Set value for the array type
func (d *decoder) array(childKey string, value reflect.Value) error {
	valueType := value.Type()
	global := d.getGlobalProvider()

	// start looking for child values.
	elementType := derefType(valueType).Elem()
	childKey += _separator

	for ai := 0; ai < value.Len(); ai++ {
		arrayKey := childKey + strconv.Itoa(ai)

		// Iterate until we find first missing value.
		if v2 := global.Get(arrayKey); v2.HasValue() {
			val := reflect.New(elementType).Elem()

			// Unmarshal current element.
			if err := d.unmarshal(arrayKey, val, ""); err != nil {
				return err
			}

			value.Index(ai).Set(val)
		} else if value.Index(ai).Kind() == reflect.Struct {
			if err := d.valueStruct(arrayKey, value.Index(ai).Addr().Interface()); err != nil {
				return err
			}
		}
	}

	return nil
}

// Sets value to a map type.
func (d *decoder) mapping(childKey string, value reflect.Value, def string) error {
	valueType := value.Type()
	global := d.getGlobalProvider()

	v := global.Get(childKey)
	if !v.HasValue() || v.Value() == nil {
		return nil
	}

	val := v.Value()

	// We fallthrough for interface to maps
	if valueType.Kind() == reflect.Interface {
		value.Set(reflect.ValueOf(val))
		return nil
	}

	destMap := reflect.ValueOf(reflect.MakeMap(valueType).Interface())

	// child yamlNode parsed from yaml file is of type map[interface{}]interface{}
	// type casting here makes sure that we are iterating over a parsed map.
	if v, ok := val.(map[interface{}]interface{}); ok {
		childKey += _separator
		for key := range v {
			mapKey := childKey + fmt.Sprintf("%v", key)
			itemValue := reflect.New(valueType.Elem()).Elem()

			// Try to unmarshal value and save it in the map.
			if err := d.unmarshal(mapKey, itemValue, def); err != nil {
				return err
			}

			destMap.SetMapIndex(reflect.ValueOf(key), itemValue)
		}

		value.Set(destMap)
	}

	return nil
}

// Sets value to an object type.
func (d *decoder) object(childKey string, value reflect.Value) error {
	valueType := value.Type()
	global := d.getGlobalProvider()

	v2 := global.Get(childKey)

	if !v2.HasValue() && valueType.Kind() == reflect.Ptr {
		// in this case we will keep the pointer value as not defined.
		return nil
	}

	if !value.CanSet() {
		return nil
	}

	if valueType.Kind() != reflect.Ptr {
		value = value.Addr()
	}

	if value.IsNil() {
		tmp := reflect.New(value.Type().Elem())
		value.Set(tmp)
	}

	return d.valueStruct(childKey, value.Interface())
}

// Walk through the struct and start asking the providers for values at each key.
//
// - for individual values, we terminate
// - for array values, we start asking for indexes
// - for object values, we recurse.
func (d *decoder) valueStruct(key string, target interface{}) error {
	tarGet := reflect.Indirect(reflect.ValueOf(target))
	targetType := tarGet.Type()
	for i := 0; i < targetType.NumField(); i++ {
		field := targetType.Field(i)

		// Check for the private field
		if field.PkgPath != "" && !field.Anonymous {
			continue
		}

		fieldName := field.Name
		fieldInfo := getFieldInfo(field)
		if fieldInfo.FieldName != "" {
			fieldName = fieldInfo.FieldName
		}

		if key != "" {
			fieldName = key + _separator + fieldName
		}

		fieldValue := tarGet.Field(i)
		if fieldValue.Kind() == reflect.Ptr && fieldValue.IsNil() {
			fieldValue.Set(reflect.New(fieldValue.Type()).Elem())
		}

		if err := d.unmarshal(fieldName, fieldValue, getFieldInfo(field).DefaultValue); err != nil {
			return err
		}
	}

	return validator.Validate(target)
}

// Dispatch un-marshalling functions based on the value type.
func (d *decoder) unmarshal(name string, value reflect.Value, def string) error {
	if value.Type().Comparable() {
		val := value.Interface()
		kind := value.Kind()
		if _, ok := d.m[val]; ok {
			if kind == reflect.Ptr && !value.IsNil() {
				buf := &bytes.Buffer{}
				for k := range d.m {
					fmt.Fprintf(buf, "%+v -> ", k)
				}

				fmt.Fprintf(buf, "%+v", value.Interface())
				return fmt.Errorf("cycles detected: %s", buf.String())
			}
		}

		d.m[val] = struct{}{}
	}

	switch getBucket(value.Type()) {
	case bucketPrimitive:
		return d.scalar(name, value, def)
	case bucketObject:
		return d.object(name, value)
	case bucketArray:
		return d.array(name, value)
	case bucketSlice:
		return d.sequence(name, value)
	case bucketMap:
		return d.mapping(name, value, def)
	}

	return nil
}
<|endoftext|>


<commit_before>package config

func NewConfig() *Config {
	return &Config{
		Debug: DEBUG,
		State: StateConfig{
			Required: false,
			Dev:      "LABEL=RANCHER_STATE",
			FsType:   "auto",
		},
		SystemDocker: DockerConfig{
			Args: []string{
				"docker",
				"-d",
				"-s",
				"overlay",
				"-b",
				"none",
				"--restart=false",
				"-g", "/var/lib/system-docker",
				"-H", DOCKER_SYSTEM_HOST,
			},
		},
		Modules: []string{},
		UserDocker: DockerConfig{
			TLSArgs: []string{
				"--tlsverify",
				"--tlscacert=ca.pem",
				"--tlscert=server-cert.pem",
				"--tlskey=server-key.pem",
				"-H=0.0.0.0:2376",
			},
			Args: []string{
				"docker",
				"-d",
				"-s", "overlay",
				"-G", "docker",
				"-H", DOCKER_HOST,
			},
		},
		Network: NetworkConfig{
			Dns: DnsConfig{
				Nameservers: []string{"8.8.8.8", "8.8.4.4"},
			},
			Interfaces: map[string]InterfaceConfig{
				"eth*": {
					DHCP: true,
				},
				"lo": {
					Address: "127.0.0.1/8",
				},
			},
		},
		CloudInit: CloudInit{
			Datasources: []string{"configdrive:/media/config-2"},
		},
		Upgrade: UpgradeConfig{
			Url:   "https://releases.rancher.com/os/versions.yml",
			Image: "rancher/os",
		},
		BootstrapContainers: []ContainerConfig{
			{
				Id: "udev",
				Cmd: "--name=udev " +
					"--net=none " +
					"--privileged " +
					"--rm " +
					"-v=/dev:/host/dev " +
					"-v=/lib/modules:/lib/modules:ro " +
					"udev",
			},
		},
		SystemContainers: []ContainerConfig{
			{
				Id: "system-volumes",
				Cmd: "--name=system-volumes " +
					"--net=none " +
					"--read-only " +
					"-v=/etc/ssl/certs/ca-certificates.crt:/etc/ssl/certs/ca-certificates.crt " +
					"-v=/var/lib/rancher/conf:/var/lib/rancher/conf " +
					"-v=/lib/modules:/lib/modules:ro " +
					"-v=/var/run:/var/run " +
					"-v=/var/log:/var/log " +
					"state",
			},
			{
				Id: "command-volumes",
				Cmd: "--name=command-volumes " +
					"--net=none " +
					"--read-only " +
					"-v=/init:/sbin/halt:ro " +
					"-v=/init:/sbin/poweroff:ro " +
					"-v=/init:/sbin/reboot:ro " +
					"-v=/init:/sbin/shutdown:ro " +
					"-v=/init:/sbin/netconf:ro " +
					"-v=/init:/usr/bin/cloud-init:ro " +
					"-v=/init:/usr/bin/rancherctl:ro " +
					"-v=/init:/usr/bin/respawn:ro " +
					"-v=/init:/usr/bin/system-docker:ro " +
					"-v=/lib/modules:/lib/modules:ro " +
					"-v=/usr/bin/docker:/usr/bin/docker:ro " +
					"state",
			},
			{
				Id: "user-volumes",
				Cmd: "--name=user-volumes " +
					"--net=none " +
					"--read-only " +
					"-v=/home:/home " +
					"-v=/opt:/opt " +
					"state",
			},
			{
				Id: "docker-volumes",
				Cmd: "--name=docker-volumes " +
					"--net=none " +
					"--read-only " +
					"-v=/var/lib/docker:/var/lib/docker " +
					"-v=/var/lib/system-docker:/var/lib/system-docker " +
					"state",
			},
			{
				Id: "all-volumes",
				Cmd: "--name=all-volumes " +
					"--rm " +
					"--net=none " +
					"--read-only " +
					"--volumes-from=docker-volumes " +
					"--volumes-from=command-volumes " +
					"--volumes-from=user-volumes " +
					"--volumes-from=system-volumes " +
					"state",
			},
			{
				Id: "cloud-init-pre",
				Cmd: "--name=cloud-init-pre " +
					"--rm " +
					"--privileged " +
					"--net=host " +
					"-e CLOUD_INIT_NETWORK=false " +
					"--volumes-from=command-volumes " +
					"--volumes-from=system-volumes " +
					"cloudinit",
				ReloadConfig: true,
			},
			{
				Id: "network",
				Cmd: "--name=network " +
					"--rm " +
					"--cap-add=NET_ADMIN " +
					"--net=host " +
					"--volumes-from=command-volumes " +
					"--volumes-from=system-volumes " +
					"network",
			},
			{
				Id: "cloud-init",
				Cmd: "--name=cloud-init " +
					"--rm " +
					"--privileged " +
					"--net=host " +
					"--volumes-from=command-volumes " +
					"--volumes-from=system-volumes " +
					"cloudinit",
				ReloadConfig: true,
			},
			{
				Id: "ntp",
				Cmd: "--name=ntp " +
					"--rm " +
					"-d " +
					"--privileged " +
					"--net=host " +
					"ntp",
			},
			{
				Id: "syslog",
				Cmd: "--name=syslog " +
					"-d " +
					"--rm " +
					"--privileged " +
					"--net=host " +
					"--ipc=host " +
					"--pid=host " +
					"--volumes-from=system-volumes " +
					"syslog",
			},
			{
				Id: "userdocker",
				Cmd: "--name=userdocker " +
					"-d " +
					"--rm " +
					"--restart=always " +
					"--ipc=host " +
					"--pid=host " +
					"--net=host " +
					"--privileged " +
					"--volumes-from=all-volumes " +
					"userdocker",
			},
			{
				Id: "console",
				Cmd: "--name=console " +
					"-d " +
					"--rm " +
					"--privileged " +
					"--volumes-from=all-volumes " +
					"--restart=always " +
					"--ipc=host " +
					"--net=host " +
					"--pid=host " +
					"console",
			},
		},
		EnabledAddons: []string{},
		Addons: map[string]Config{
			"ubuntu-console": {
				SystemContainers: []ContainerConfig{
					{
						Id: "console",
						Cmd: "--name=ubuntu-console " +
							"-d " +
							"--rm " +
							"--privileged " +
							"--volumes-from=all-volumes " +
							"--restart=always " +
							"--ipc=host " +
							"--net=host " +
							"--pid=host " +
							"rancher/ubuntuconsole:v0.0.2",
					},
				},
			},
		},
	}
}
<commit_msg>Add udev as create only container in system-docker<commit_after>package config

func NewConfig() *Config {
	return &Config{
		Debug: DEBUG,
		State: StateConfig{
			Required: false,
			Dev:      "LABEL=RANCHER_STATE",
			FsType:   "auto",
		},
		SystemDocker: DockerConfig{
			Args: []string{
				"docker",
				"-d",
				"-s",
				"overlay",
				"-b",
				"none",
				"--restart=false",
				"-g", "/var/lib/system-docker",
				"-H", DOCKER_SYSTEM_HOST,
			},
		},
		Modules: []string{},
		UserDocker: DockerConfig{
			TLSArgs: []string{
				"--tlsverify",
				"--tlscacert=ca.pem",
				"--tlscert=server-cert.pem",
				"--tlskey=server-key.pem",
				"-H=0.0.0.0:2376",
			},
			Args: []string{
				"docker",
				"-d",
				"-s", "overlay",
				"-G", "docker",
				"-H", DOCKER_HOST,
			},
		},
		Network: NetworkConfig{
			Dns: DnsConfig{
				Nameservers: []string{"8.8.8.8", "8.8.4.4"},
			},
			Interfaces: map[string]InterfaceConfig{
				"eth*": {
					DHCP: true,
				},
				"lo": {
					Address: "127.0.0.1/8",
				},
			},
		},
		CloudInit: CloudInit{
			Datasources: []string{"configdrive:/media/config-2"},
		},
		Upgrade: UpgradeConfig{
			Url:   "https://releases.rancher.com/os/versions.yml",
			Image: "rancher/os",
		},
		BootstrapContainers: []ContainerConfig{
			{
				Id: "udev",
				Cmd: "--name=udev " +
					"--net=none " +
					"--privileged " +
					"--rm " +
					"-v=/dev:/host/dev " +
					"-v=/lib/modules:/lib/modules:ro " +
					"udev",
			},
		},
		SystemContainers: []ContainerConfig{
			{
				Id: "udev",
				Cmd: "--name=udev " +
					"--net=none " +
					"--privileged " +
					"--rm " +
					"-v=/dev:/host/dev " +
					"-v=/lib/modules:/lib/modules:ro " +
					"udev",
				CreateOnly: true,
			},
			{
				Id: "system-volumes",
				Cmd: "--name=system-volumes " +
					"--net=none " +
					"--read-only " +
					"-v=/etc/ssl/certs/ca-certificates.crt:/etc/ssl/certs/ca-certificates.crt " +
					"-v=/var/lib/rancher/conf:/var/lib/rancher/conf " +
					"-v=/lib/modules:/lib/modules:ro " +
					"-v=/var/run:/var/run " +
					"-v=/var/log:/var/log " +
					"state",
			},
			{
				Id: "command-volumes",
				Cmd: "--name=command-volumes " +
					"--net=none " +
					"--read-only " +
					"-v=/init:/sbin/halt:ro " +
					"-v=/init:/sbin/poweroff:ro " +
					"-v=/init:/sbin/reboot:ro " +
					"-v=/init:/sbin/shutdown:ro " +
					"-v=/init:/sbin/netconf:ro " +
					"-v=/init:/usr/bin/cloud-init:ro " +
					"-v=/init:/usr/bin/rancherctl:ro " +
					"-v=/init:/usr/bin/respawn:ro " +
					"-v=/init:/usr/bin/system-docker:ro " +
					"-v=/lib/modules:/lib/modules:ro " +
					"-v=/usr/bin/docker:/usr/bin/docker:ro " +
					"state",
			},
			{
				Id: "user-volumes",
				Cmd: "--name=user-volumes " +
					"--net=none " +
					"--read-only " +
					"-v=/home:/home " +
					"-v=/opt:/opt " +
					"state",
			},
			{
				Id: "docker-volumes",
				Cmd: "--name=docker-volumes " +
					"--net=none " +
					"--read-only " +
					"-v=/var/lib/docker:/var/lib/docker " +
					"-v=/var/lib/system-docker:/var/lib/system-docker " +
					"state",
			},
			{
				Id: "all-volumes",
				Cmd: "--name=all-volumes " +
					"--rm " +
					"--net=none " +
					"--read-only " +
					"--volumes-from=docker-volumes " +
					"--volumes-from=command-volumes " +
					"--volumes-from=user-volumes " +
					"--volumes-from=system-volumes " +
					"state",
			},
			{
				Id: "cloud-init-pre",
				Cmd: "--name=cloud-init-pre " +
					"--rm " +
					"--privileged " +
					"--net=host " +
					"-e CLOUD_INIT_NETWORK=false " +
					"--volumes-from=command-volumes " +
					"--volumes-from=system-volumes " +
					"cloudinit",
				ReloadConfig: true,
			},
			{
				Id: "network",
				Cmd: "--name=network " +
					"--rm " +
					"--cap-add=NET_ADMIN " +
					"--net=host " +
					"--volumes-from=command-volumes " +
					"--volumes-from=system-volumes " +
					"network",
			},
			{
				Id: "cloud-init",
				Cmd: "--name=cloud-init " +
					"--rm " +
					"--privileged " +
					"--net=host " +
					"--volumes-from=command-volumes " +
					"--volumes-from=system-volumes " +
					"cloudinit",
				ReloadConfig: true,
			},
			{
				Id: "ntp",
				Cmd: "--name=ntp " +
					"--rm " +
					"-d " +
					"--privileged " +
					"--net=host " +
					"ntp",
			},
			{
				Id: "syslog",
				Cmd: "--name=syslog " +
					"-d " +
					"--rm " +
					"--privileged " +
					"--net=host " +
					"--ipc=host " +
					"--pid=host " +
					"--volumes-from=system-volumes " +
					"syslog",
			},
			{
				Id: "userdocker",
				Cmd: "--name=userdocker " +
					"-d " +
					"--rm " +
					"--restart=always " +
					"--ipc=host " +
					"--pid=host " +
					"--net=host " +
					"--privileged " +
					"--volumes-from=all-volumes " +
					"userdocker",
			},
			{
				Id: "console",
				Cmd: "--name=console " +
					"-d " +
					"--rm " +
					"--privileged " +
					"--volumes-from=all-volumes " +
					"--restart=always " +
					"--ipc=host " +
					"--net=host " +
					"--pid=host " +
					"console",
			},
		},
		EnabledAddons: []string{},
		Addons: map[string]Config{
			"ubuntu-console": {
				SystemContainers: []ContainerConfig{
					{
						Id: "console",
						Cmd: "--name=ubuntu-console " +
							"-d " +
							"--rm " +
							"--privileged " +
							"--volumes-from=all-volumes " +
							"--restart=always " +
							"--ipc=host " +
							"--net=host " +
							"--pid=host " +
							"rancher/ubuntuconsole:v0.0.2",
					},
				},
			},
		},
	}
}
<|endoftext|>


<commit_before>package lxd

import (
	"fmt"
	"log"
	"time"

	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/helper/schema"

	"github.com/lxc/lxd"
	"github.com/lxc/lxd/shared"
)

func resourceLxdContainer() *schema.Resource {
	return &schema.Resource{
		Create: resourceLxdContainerCreate,
		Update: resourceLxdContainerUpdate,
		Delete: resourceLxdContainerDelete,
		Exists: resourceLxdContainerExists,
		Read:   resourceLxdContainerRead,

		Schema: map[string]*schema.Schema{
			"name": &schema.Schema{
				Type:     schema.TypeString,
				ForceNew: true,
				Required: true,
			},

			"image": &schema.Schema{
				Type:     schema.TypeString,
				ForceNew: true,
				Required: true,
			},

			"profiles": &schema.Schema{
				Type:     schema.TypeList,
				Optional: true,
				Elem:     &schema.Schema{Type: schema.TypeString},
				ForceNew: false,
			},

			"config": &schema.Schema{
				Type:     schema.TypeMap,
				Optional: true,
				ForceNew: true,
			},

			"ephemeral": &schema.Schema{
				Type:     schema.TypeBool,
				Optional: true,
				Default:  false,
				ForceNew: true,
			},

			"privileged": &schema.Schema{
				Type:     schema.TypeBool,
				Optional: true,
				Default:  false,
				ForceNew: false,
			},

			"ip_address": &schema.Schema{
				Type:     schema.TypeString,
				Computed: true,
				ForceNew: false,
			},

			"status": &schema.Schema{
				Type:     schema.TypeString,
				Computed: true,
			},

			"mac_address": &schema.Schema{
				Type:     schema.TypeString,
				Computed: true,
			},
		},
	}
}

func resourceLxdContainerCreate(d *schema.ResourceData, meta interface{}) error {
	var err error
	client := meta.(*LxdProvider).Client
	remote := meta.(*LxdProvider).Remote

	name := d.Get("name").(string)
	ephem := d.Get("ephemeral").(bool)
	image := d.Get("image").(string)
	config := resourceLxdContainerConfigMap(d.Get("config"))

	/*
	 * requested_empty_profiles means user requested empty
	 * !requested_empty_profiles but len(profArgs) == 0 means use profile default
	 */
	profiles := []string{}
	if v, ok := d.GetOk("profiles"); ok {
		for _, v := range v.([]interface{}) {
			profiles = append(profiles, v.(string))
		}
	}

	//client.Init = (name string, imgremote string, image string, profiles *[]string, config map[string]string, ephem bool)
	var resp *lxd.Response
	if resp, err = client.Init(name, remote, image, &profiles, config, nil, ephem); err != nil {
		return err
	}

	// Wait for the LXC container to be created
	err = client.WaitForSuccess(resp.Operation)
	if err != nil {
		return err
	}

	// Start container
	_, err = client.Action(name, shared.Start, -1, false, false)
	if err != nil {
		// Container has been created, but daemon rejected start request
		return err
	}

	// Wait until the container is in a Running state
	stateConf := &resource.StateChangeConf{
		Target:     []string{"Running"},
		Refresh:    resourceLxdContainerRefresh(client, name),
		Timeout:    3 * time.Minute,
		Delay:      10 * time.Second,
		MinTimeout: 3 * time.Second,
	}

	if _, err = stateConf.WaitForState(); err != nil {
		return fmt.Errorf("Error waiting for container (%s) to become active: %s", name, err)
	}

	d.SetId(name)
	return resourceLxdContainerRead(d, meta)
}

func resourceLxdContainerRead(d *schema.ResourceData, meta interface{}) error {
	client := meta.(*LxdProvider).Client
	name := d.Id()

	container, err := client.ContainerInfo(name)
	if err != nil {
		return err
	}
	log.Printf("[DEBUG] Retrieved container %s: %#v", name, container)

	sshIP := ""

	ct, err := client.ContainerState(name)
	if err != nil {
		return err
	}

	d.Set("status", ct.Status)

	for iface, net := range ct.Network {
		if iface != "lo" {
			for _, ip := range net.Addresses {
				if ip.Family == "inet" {
					d.Set("ip_address", ip.Address)
					sshIP = ip.Address
					d.Set("mac_address", net.Hwaddr)
				}
			}
		}
	}

	// Initialize the connection info
	d.SetConnInfo(map[string]string{
		"type": "ssh",
		"host": sshIP,
	})

	return nil
}

func resourceLxdContainerUpdate(d *schema.ResourceData, meta interface{}) error {
	client := meta.(*LxdProvider).Client
	name := d.Id()

	// st will hold the updated container information.
	var st shared.BriefContainerInfo
	var changed bool

	ct, err := client.ContainerInfo(name)
	if err != nil {
		return err
	}
	st.Devices = ct.Devices

	if d.HasChange("profiles") {
		_, newProfiles := d.GetChange("profiles")
		if v, ok := newProfiles.([]interface{}); ok {
			changed = true
			var profiles []string
			for _, p := range v {
				profiles = append(profiles, p.(string))
			}
			st.Profiles = profiles
		}
	}

	if changed {
		err := client.UpdateContainerConfig(name, st)
		if err != nil {
			return err
		}
	}

	return nil
}

func resourceLxdContainerDelete(d *schema.ResourceData, meta interface{}) (err error) {
	client := meta.(*LxdProvider).Client
	name := d.Id()

	ct, _ := client.ContainerState(name)
	if ct.Status == "Running" {
		if _, err := client.Action(name, shared.Stop, 30, true, false); err != nil {
			return err
		}

		// Wait until the container is in a Stopped state
		stateConf := &resource.StateChangeConf{
			Target:     []string{"Stopped"},
			Refresh:    resourceLxdContainerRefresh(client, name),
			Timeout:    3 * time.Minute,
			Delay:      10 * time.Second,
			MinTimeout: 3 * time.Second,
		}

		if _, err = stateConf.WaitForState(); err != nil {
			return fmt.Errorf("Error waiting for container (%s) to stop: %s", name, err)
		}
	}

	if _, err = client.Delete(name); err != nil {
		return err
	}

	return nil
}

func resourceLxdContainerExists(d *schema.ResourceData, meta interface{}) (exists bool, err error) {
	client := meta.(*LxdProvider).Client
	name := d.Id()

	exists = false

	ct, err := client.ContainerState(name)
	if err == nil && ct != nil {
		exists = true
	}

	return
}

func resourceLxdContainerConfigMap(c interface{}) map[string]string {
	config := make(map[string]string)
	if v, ok := c.(map[string]interface{}); ok {
		for key, val := range v {
			config[key] = val.(string)
		}
	}

	log.Printf("[DEBUG] LXD Container Configuration Map: %#v", config)

	return config
}

func resourceLxdContainerRefresh(client *lxd.Client, name string) resource.StateRefreshFunc {
	return func() (interface{}, string, error) {
		ct, err := client.ContainerState(name)
		if err != nil {
			return ct, "Error", err
		}

		return ct, ct.Status, nil
	}
}
<commit_msg>Add device argument<commit_after>package lxd

import (
	"fmt"
	"log"
	"time"

	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/helper/schema"

	"github.com/lxc/lxd"
	"github.com/lxc/lxd/shared"
)

func resourceLxdContainer() *schema.Resource {
	return &schema.Resource{
		Create: resourceLxdContainerCreate,
		Update: resourceLxdContainerUpdate,
		Delete: resourceLxdContainerDelete,
		Exists: resourceLxdContainerExists,
		Read:   resourceLxdContainerRead,

		Schema: map[string]*schema.Schema{
			"name": &schema.Schema{
				Type:     schema.TypeString,
				ForceNew: true,
				Required: true,
			},

			"image": &schema.Schema{
				Type:     schema.TypeString,
				ForceNew: true,
				Required: true,
			},

			"profiles": &schema.Schema{
				Type:     schema.TypeList,
				Optional: true,
				Elem:     &schema.Schema{Type: schema.TypeString},
				ForceNew: false,
			},

			"device": &schema.Schema{
				Type:     schema.TypeList,
				Optional: true,
				Elem: &schema.Schema{
					Type: schema.TypeMap,
				},
			},

			"config": &schema.Schema{
				Type:     schema.TypeMap,
				Optional: true,
				ForceNew: true,
			},

			"ephemeral": &schema.Schema{
				Type:     schema.TypeBool,
				Optional: true,
				Default:  false,
				ForceNew: true,
			},

			"privileged": &schema.Schema{
				Type:     schema.TypeBool,
				Optional: true,
				Default:  false,
				ForceNew: false,
			},

			"ip_address": &schema.Schema{
				Type:     schema.TypeString,
				Computed: true,
				ForceNew: false,
			},

			"status": &schema.Schema{
				Type:     schema.TypeString,
				Computed: true,
			},

			"mac_address": &schema.Schema{
				Type:     schema.TypeString,
				Computed: true,
			},
		},
	}
}

func resourceLxdContainerCreate(d *schema.ResourceData, meta interface{}) error {
	var err error
	client := meta.(*LxdProvider).Client
	remote := meta.(*LxdProvider).Remote

	name := d.Get("name").(string)
	ephem := d.Get("ephemeral").(bool)
	image := d.Get("image").(string)
	config := resourceLxdContainerConfigMap(d.Get("config"))
	devices := resourceLxdContainerDevices(d.Get("device"))

	/*
	 * requested_empty_profiles means user requested empty
	 * !requested_empty_profiles but len(profArgs) == 0 means use profile default
	 */
	profiles := []string{}
	if v, ok := d.GetOk("profiles"); ok {
		for _, v := range v.([]interface{}) {
			profiles = append(profiles, v.(string))
		}
	}

	// client.Init = (name string, imgremote string, image string, profiles *[]string, config map[string]string, devices shared.Devices, ephem bool)
	var resp *lxd.Response
	if resp, err = client.Init(name, remote, image, &profiles, config, devices, ephem); err != nil {
		return err
	}

	// Wait for the LXC container to be created
	err = client.WaitForSuccess(resp.Operation)
	if err != nil {
		return err
	}

	// Start container
	_, err = client.Action(name, shared.Start, -1, false, false)
	if err != nil {
		// Container has been created, but daemon rejected start request
		return err
	}

	// Wait until the container is in a Running state
	stateConf := &resource.StateChangeConf{
		Target:     []string{"Running"},
		Refresh:    resourceLxdContainerRefresh(client, name),
		Timeout:    3 * time.Minute,
		Delay:      10 * time.Second,
		MinTimeout: 3 * time.Second,
	}

	if _, err = stateConf.WaitForState(); err != nil {
		return fmt.Errorf("Error waiting for container (%s) to become active: %s", name, err)
	}

	d.SetId(name)
	return resourceLxdContainerRead(d, meta)
}

func resourceLxdContainerRead(d *schema.ResourceData, meta interface{}) error {
	client := meta.(*LxdProvider).Client
	name := d.Id()

	container, err := client.ContainerInfo(name)
	if err != nil {
		return err
	}
	log.Printf("[DEBUG] Retrieved container %s: %#v", name, container)

	ct, err := client.ContainerState(name)
	if err != nil {
		return err
	}

	d.Set("status", ct.Status)

	log.Printf("[DEBUG] omg %#v", d.Get("device"))

	sshIP := ""
	for iface, net := range ct.Network {
		if iface != "lo" {
			for _, ip := range net.Addresses {
				if ip.Family == "inet" {
					d.Set("ip_address", ip.Address)
					sshIP = ip.Address
					d.Set("mac_address", net.Hwaddr)
				}
			}
		}
	}

	// Initialize the connection info
	d.SetConnInfo(map[string]string{
		"type": "ssh",
		"host": sshIP,
	})

	return nil
}

func resourceLxdContainerUpdate(d *schema.ResourceData, meta interface{}) error {
	client := meta.(*LxdProvider).Client
	name := d.Id()

	// st will hold the updated container information.
	var st shared.BriefContainerInfo
	var changed bool

	ct, err := client.ContainerInfo(name)
	if err != nil {
		return err
	}
	st.Devices = ct.Devices

	if d.HasChange("profiles") {
		_, newProfiles := d.GetChange("profiles")
		if v, ok := newProfiles.([]interface{}); ok {
			changed = true
			var profiles []string
			for _, p := range v {
				profiles = append(profiles, p.(string))
			}
			st.Profiles = profiles
		}
	}

	if d.HasChange("device") {
		changed = true
		old, new := d.GetChange("device")
		oldDevices := resourceLxdContainerDevices(old)
		newDevices := resourceLxdContainerDevices(new)

		for n, _ := range oldDevices {
			delete(st.Devices, n)
		}

		for n, d := range newDevices {
			if n != "" {
				st.Devices[n] = d
			}
		}

		log.Printf("[DEBUG] Updated device list: %#v", st.Devices)
	}

	if changed {
		err := client.UpdateContainerConfig(name, st)
		if err != nil {
			return err
		}
	}

	return nil
}

func resourceLxdContainerDelete(d *schema.ResourceData, meta interface{}) (err error) {
	client := meta.(*LxdProvider).Client
	name := d.Id()

	ct, _ := client.ContainerState(name)
	if ct.Status == "Running" {
		if _, err := client.Action(name, shared.Stop, 30, true, false); err != nil {
			return err
		}

		// Wait until the container is in a Stopped state
		stateConf := &resource.StateChangeConf{
			Target:     []string{"Stopped"},
			Refresh:    resourceLxdContainerRefresh(client, name),
			Timeout:    3 * time.Minute,
			Delay:      10 * time.Second,
			MinTimeout: 3 * time.Second,
		}

		if _, err = stateConf.WaitForState(); err != nil {
			return fmt.Errorf("Error waiting for container (%s) to stop: %s", name, err)
		}
	}

	if _, err = client.Delete(name); err != nil {
		return err
	}

	return nil
}

func resourceLxdContainerExists(d *schema.ResourceData, meta interface{}) (exists bool, err error) {
	client := meta.(*LxdProvider).Client
	name := d.Id()

	exists = false

	ct, err := client.ContainerState(name)
	if err == nil && ct != nil {
		exists = true
	}

	return
}

func resourceLxdContainerConfigMap(c interface{}) map[string]string {
	config := make(map[string]string)
	if v, ok := c.(map[string]interface{}); ok {
		for key, val := range v {
			config[key] = val.(string)
		}
	}

	log.Printf("[DEBUG] LXD Container Configuration Map: %#v", config)

	return config
}

func resourceLxdContainerDevices(d interface{}) shared.Devices {
	devices := make(shared.Devices)
	if deviceList, ok := d.([]interface{}); ok {
		for _, v := range deviceList {
			if v, ok := v.(map[string]interface{}); ok {
				if n, ok := v["name"]; ok {
					name := n.(string)
					device := make(shared.Device)
					for key, val := range v {
						if key != "name" {
							device[key] = val.(string)
						}
					}
					devices[name] = device
				}
			}
		}
	}

	log.Printf("[DEBUG] LXD Container Devices: %#v", devices)

	return devices
}

func resourceLxdContainerRefresh(client *lxd.Client, name string) resource.StateRefreshFunc {
	return func() (interface{}, string, error) {
		ct, err := client.ContainerState(name)
		if err != nil {
			return ct, "Error", err
		}

		return ct, ct.Status, nil
	}
}
<|endoftext|>


<commit_before>package datastore

import (
	"bytes"
	"cluster"
	"configuration"
	"fmt"
	"math"
	"os"
	"path/filepath"
	"protocol"
	"sync"
	"time"

	"datastore/storage"

	log "code.google.com/p/log4go"
	"github.com/BurntSushi/toml"
)

type ShardDatastore struct {
	baseDbDir      string
	config         *configuration.Configuration
	shards         map[uint32]*Shard
	lastAccess     map[uint32]int64
	shardRefCounts map[uint32]int
	shardsToClose  map[uint32]bool
	shardsLock     sync.RWMutex
	writeBuffer    *cluster.WriteBuffer
	maxOpenShards  int
	pointBatchSize int
	writeBatchSize int
}

const (
	ONE_KILOBYTE                    = 1024
	ONE_MEGABYTE                    = 1024 * 1024
	SHARD_BLOOM_FILTER_BITS_PER_KEY = 10
	SHARD_DATABASE_DIR              = "shard_db"
)

var (

	// This datastore implements the PersistentAtomicInteger interface. All of the persistent
	// integers start with this prefix, followed by their name
	ATOMIC_INCREMENT_PREFIX = []byte{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFD}
	// NEXT_ID_KEY holds the next id. ids are used to "intern" timeseries and column names
	NEXT_ID_KEY = []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}
	// SERIES_COLUMN_INDEX_PREFIX is the prefix of the series to column names index
	SERIES_COLUMN_INDEX_PREFIX = []byte{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFE}
	// DATABASE_SERIES_INDEX_PREFIX is the prefix of the database to series names index
	DATABASE_SERIES_INDEX_PREFIX = []byte{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}
	MAX_SEQUENCE                 = []byte{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}

	// replicateWrite = protocol.Request_REPLICATION_WRITE

	TRUE = true
)

type Field struct {
	Id   []byte
	Name string
}

type rawColumnValue struct {
	time     []byte
	sequence []byte
	value    []byte
}

func NewShardDatastore(config *configuration.Configuration) (*ShardDatastore, error) {
	baseDbDir := filepath.Join(config.DataDir, SHARD_DATABASE_DIR)
	err := os.MkdirAll(baseDbDir, 0744)
	if err != nil {
		return nil, err
	}

	return &ShardDatastore{
		baseDbDir:      baseDbDir,
		config:         config,
		shards:         make(map[uint32]*Shard),
		maxOpenShards:  config.StorageMaxOpenShards,
		lastAccess:     make(map[uint32]int64),
		shardRefCounts: make(map[uint32]int),
		shardsToClose:  make(map[uint32]bool),
		pointBatchSize: config.StoragePointBatchSize,
		writeBatchSize: config.StorageWriteBatchSize,
	}, nil
}

func (self *ShardDatastore) Close() {
	self.shardsLock.Lock()
	defer self.shardsLock.Unlock()
	for _, shard := range self.shards {
		shard.close()
	}
}

func (self *ShardDatastore) GetOrCreateShard(id uint32) (cluster.LocalShardDb, error) {
	now := time.Now().Unix()
	self.shardsLock.Lock()
	defer self.shardsLock.Unlock()
	db := self.shards[id]
	self.lastAccess[id] = now

	if db != nil {
		self.incrementShardRefCountAndCloseOldestIfNeeded(id)
		return db, nil
	}

	dbDir := self.shardDir(id)

	log.Info("DATASTORE: opening or creating shard %s", dbDir)
	defaultEngine := self.config.StorageDefaultEngine
	init, err := storage.GetInitializer(defaultEngine)
	if err != nil {
		log.Error("Error opening shard: ", err)
		return nil, err
	}
	c := init.NewConfig()
	conf, ok := self.config.StorageEngineConfigs[defaultEngine]
	if err := toml.PrimitiveDecode(conf, c); ok && err != nil {
		return nil, err
	}

	// TODO: this is for backward compatability with the old
	// configuration
	if leveldbConfig, ok := c.(*storage.LevelDbConfiguration); ok {
		if leveldbConfig.LruCacheSize == 0 {
			leveldbConfig.LruCacheSize = configuration.Size(self.config.LevelDbLruCacheSize)
		}

		if leveldbConfig.MaxOpenFiles == 0 {
			leveldbConfig.MaxOpenFiles = self.config.LevelDbMaxOpenFiles
		}
	}

	se, err := init.Initialize(dbDir, c)
	db, err = NewShard(se, self.pointBatchSize, self.writeBatchSize)
	if err != nil {
		log.Error("Error creating shard: ", err)
		se.Close()
		return nil, err
	}
	self.shards[id] = db
	self.incrementShardRefCountAndCloseOldestIfNeeded(id)
	return db, nil
}

func (self *ShardDatastore) incrementShardRefCountAndCloseOldestIfNeeded(id uint32) {
	self.shardRefCounts[id] += 1
	delete(self.shardsToClose, id)
	if self.maxOpenShards > 0 && len(self.shards) > self.maxOpenShards {
		for i := len(self.shards) - self.maxOpenShards; i > 0; i-- {
			self.closeOldestShard()
		}
	}
}

func (self *ShardDatastore) ReturnShard(id uint32) {
	self.shardsLock.Lock()
	defer self.shardsLock.Unlock()
	self.shardRefCounts[id] -= 1
	if self.shardsToClose[id] && self.shardRefCounts[id] == 0 {
		self.closeShard(id)
	}
}

func (self *ShardDatastore) Write(request *protocol.Request) error {
	shardDb, err := self.GetOrCreateShard(*request.ShardId)
	if err != nil {
		return err
	}
	defer self.ReturnShard(*request.ShardId)
	return shardDb.Write(*request.Database, request.MultiSeries)
}

func (self *ShardDatastore) BufferWrite(request *protocol.Request) {
	self.writeBuffer.Write(request)
}

func (self *ShardDatastore) SetWriteBuffer(writeBuffer *cluster.WriteBuffer) {
	self.writeBuffer = writeBuffer
}

func (self *ShardDatastore) DeleteShard(shardId uint32) error {
	self.shardsLock.Lock()
	shardDb := self.shards[shardId]
	delete(self.shards, shardId)
	delete(self.lastAccess, shardId)
	self.shardsLock.Unlock()

	if shardDb != nil {
		shardDb.close()
	}

	dir := self.shardDir(shardId)
	log.Info("DATASTORE: dropping shard %s", dir)
	return os.RemoveAll(dir)
}

func (self *ShardDatastore) shardDir(id uint32) string {
	return filepath.Join(self.baseDbDir, fmt.Sprintf("%.5d", id))
}

func (self *ShardDatastore) closeOldestShard() {
	var oldestId uint32
	oldestAccess := int64(math.MaxInt64)
	for id, lastAccess := range self.lastAccess {
		if lastAccess < oldestAccess && self.shardsToClose[id] == false {
			oldestId = id
			oldestAccess = lastAccess
		}
	}
	if self.shardRefCounts[oldestId] == 0 {
		self.closeShard(oldestId)
	} else {
		self.shardsToClose[oldestId] = true
	}
}

func (self *ShardDatastore) closeShard(id uint32) {
	shard := self.shards[id]
	if shard != nil {
		shard.close()
	}
	delete(self.shardRefCounts, id)
	delete(self.shards, id)
	delete(self.lastAccess, id)
	delete(self.shardsToClose, id)
	log.Debug("DATASTORE: closing shard %s", self.shardDir(id))
}

// // returns true if the point has the correct field id and is
// // in the given time range
func isPointInRange(fieldId, startTime, endTime, point []byte) bool {
	id := point[:8]
	time := point[8:16]
	return bytes.Equal(id, fieldId) && bytes.Compare(time, startTime) > -1 && bytes.Compare(time, endTime) < 1
}

type FieldLookupError struct {
	message string
}

func (self FieldLookupError) Error() string {
	return self.message
}

// depending on the query order (whether it's ascending or not) returns
// the min (or max in case of descending query) of the current
// [timestamp,sequence] and the self's [timestamp,sequence]
//
// This is used to determine what the next point's timestamp
// and sequence number should be.
func (self *rawColumnValue) updatePointTimeAndSequence(currentTimeRaw, currentSequenceRaw []byte, isAscendingQuery bool) ([]byte, []byte) {
	if currentTimeRaw == nil {
		return self.time, self.sequence
	}

	compareValue := 1
	if isAscendingQuery {
		compareValue = -1
	}

	timeCompare := bytes.Compare(self.time, currentTimeRaw)
	if timeCompare == compareValue {
		return self.time, self.sequence
	}

	if timeCompare != 0 {
		return currentTimeRaw, currentSequenceRaw
	}

	if bytes.Compare(self.sequence, currentSequenceRaw) == compareValue {
		return currentTimeRaw, self.sequence
	}

	return currentTimeRaw, currentSequenceRaw
}
<commit_msg>keep track of the database type<commit_after>package datastore

import (
	"bytes"
	"cluster"
	"configuration"
	"fmt"
	"io/ioutil"
	"math"
	"os"
	"path"
	"path/filepath"
	"protocol"
	"sync"
	"time"

	"datastore/storage"

	log "code.google.com/p/log4go"
	"github.com/BurntSushi/toml"
)

type ShardDatastore struct {
	baseDbDir      string
	config         *configuration.Configuration
	shards         map[uint32]*Shard
	lastAccess     map[uint32]int64
	shardRefCounts map[uint32]int
	shardsToClose  map[uint32]bool
	shardsLock     sync.RWMutex
	writeBuffer    *cluster.WriteBuffer
	maxOpenShards  int
	pointBatchSize int
	writeBatchSize int
}

const (
	ONE_KILOBYTE                    = 1024
	ONE_MEGABYTE                    = 1024 * 1024
	SHARD_BLOOM_FILTER_BITS_PER_KEY = 10
	SHARD_DATABASE_DIR              = "shard_db"
)

var (

	// This datastore implements the PersistentAtomicInteger interface. All of the persistent
	// integers start with this prefix, followed by their name
	ATOMIC_INCREMENT_PREFIX = []byte{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFD}
	// NEXT_ID_KEY holds the next id. ids are used to "intern" timeseries and column names
	NEXT_ID_KEY = []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}
	// SERIES_COLUMN_INDEX_PREFIX is the prefix of the series to column names index
	SERIES_COLUMN_INDEX_PREFIX = []byte{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFE}
	// DATABASE_SERIES_INDEX_PREFIX is the prefix of the database to series names index
	DATABASE_SERIES_INDEX_PREFIX = []byte{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}
	MAX_SEQUENCE                 = []byte{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}

	// replicateWrite = protocol.Request_REPLICATION_WRITE

	TRUE = true
)

type Field struct {
	Id   []byte
	Name string
}

type rawColumnValue struct {
	time     []byte
	sequence []byte
	value    []byte
}

func NewShardDatastore(config *configuration.Configuration) (*ShardDatastore, error) {
	baseDbDir := filepath.Join(config.DataDir, SHARD_DATABASE_DIR)
	err := os.MkdirAll(baseDbDir, 0744)
	if err != nil {
		return nil, err
	}

	return &ShardDatastore{
		baseDbDir:      baseDbDir,
		config:         config,
		shards:         make(map[uint32]*Shard),
		maxOpenShards:  config.StorageMaxOpenShards,
		lastAccess:     make(map[uint32]int64),
		shardRefCounts: make(map[uint32]int),
		shardsToClose:  make(map[uint32]bool),
		pointBatchSize: config.StoragePointBatchSize,
		writeBatchSize: config.StorageWriteBatchSize,
	}, nil
}

func (self *ShardDatastore) Close() {
	self.shardsLock.Lock()
	defer self.shardsLock.Unlock()
	for _, shard := range self.shards {
		shard.close()
	}
}

// Get the engine that was used when the shard was created if it
// exists or set the type of the default engine type
func (self *ShardDatastore) getEngine(dir string) (string, error) {
	shardExist := true
	info, err := os.Stat(dir)
	if err != nil {
		if !os.IsNotExist(err) {
			return "", err
		}

		shardExist = false
		if err := os.MkdirAll(dir, 0755); err != nil {
			return "", err
		}
	} else if !info.IsDir() {
		return "", fmt.Errorf("%s isn't a directory", dir)
	}

	f, err := os.OpenFile(path.Join(dir, "type"), os.O_RDWR|os.O_CREATE, 0644)
	if err != nil {
		return "", err
	}
	defer f.Close()

	body, err := ioutil.ReadAll(f)
	if err != nil {
		return "", err
	}

	if s := string(body); s != "" {
		return s, nil
	}

	// If the shard existed already but the 'type' file didn't, assume
	// it's leveldb
	t := self.config.StorageDefaultEngine
	if shardExist {
		t = "leveldb"
	}

	if _, err := f.WriteString(t); err != nil {
		return "", err
	}

	if err := f.Sync(); err != nil {
		return "", err
	}

	return t, nil
}

func (self *ShardDatastore) GetOrCreateShard(id uint32) (cluster.LocalShardDb, error) {
	now := time.Now().Unix()
	self.shardsLock.Lock()
	defer self.shardsLock.Unlock()
	db := self.shards[id]
	self.lastAccess[id] = now

	if db != nil {
		self.incrementShardRefCountAndCloseOldestIfNeeded(id)
		return db, nil
	}

	dbDir := self.shardDir(id)

	log.Info("DATASTORE: opening or creating shard %s", dbDir)
	engine, err := self.getEngine(dbDir)
	if err != nil {
		return nil, err
	}
	init, err := storage.GetInitializer(engine)
	if err != nil {
		log.Error("Error opening shard: ", err)
		return nil, err
	}
	c := init.NewConfig()
	conf, ok := self.config.StorageEngineConfigs[engine]
	if err := toml.PrimitiveDecode(conf, c); ok && err != nil {
		return nil, err
	}

	// TODO: this is for backward compatability with the old
	// configuration
	if leveldbConfig, ok := c.(*storage.LevelDbConfiguration); ok {
		if leveldbConfig.LruCacheSize == 0 {
			leveldbConfig.LruCacheSize = configuration.Size(self.config.LevelDbLruCacheSize)
		}

		if leveldbConfig.MaxOpenFiles == 0 {
			leveldbConfig.MaxOpenFiles = self.config.LevelDbMaxOpenFiles
		}
	}

	se, err := init.Initialize(dbDir, c)
	db, err = NewShard(se, self.pointBatchSize, self.writeBatchSize)
	if err != nil {
		log.Error("Error creating shard: ", err)
		se.Close()
		return nil, err
	}
	self.shards[id] = db
	self.incrementShardRefCountAndCloseOldestIfNeeded(id)
	return db, nil
}

func (self *ShardDatastore) incrementShardRefCountAndCloseOldestIfNeeded(id uint32) {
	self.shardRefCounts[id] += 1
	delete(self.shardsToClose, id)
	if self.maxOpenShards > 0 && len(self.shards) > self.maxOpenShards {
		for i := len(self.shards) - self.maxOpenShards; i > 0; i-- {
			self.closeOldestShard()
		}
	}
}

func (self *ShardDatastore) ReturnShard(id uint32) {
	self.shardsLock.Lock()
	defer self.shardsLock.Unlock()
	self.shardRefCounts[id] -= 1
	if self.shardsToClose[id] && self.shardRefCounts[id] == 0 {
		self.closeShard(id)
	}
}

func (self *ShardDatastore) Write(request *protocol.Request) error {
	shardDb, err := self.GetOrCreateShard(*request.ShardId)
	if err != nil {
		return err
	}
	defer self.ReturnShard(*request.ShardId)
	return shardDb.Write(*request.Database, request.MultiSeries)
}

func (self *ShardDatastore) BufferWrite(request *protocol.Request) {
	self.writeBuffer.Write(request)
}

func (self *ShardDatastore) SetWriteBuffer(writeBuffer *cluster.WriteBuffer) {
	self.writeBuffer = writeBuffer
}

func (self *ShardDatastore) DeleteShard(shardId uint32) error {
	self.shardsLock.Lock()
	shardDb := self.shards[shardId]
	delete(self.shards, shardId)
	delete(self.lastAccess, shardId)
	self.shardsLock.Unlock()

	if shardDb != nil {
		shardDb.close()
	}

	dir := self.shardDir(shardId)
	log.Info("DATASTORE: dropping shard %s", dir)
	return os.RemoveAll(dir)
}

func (self *ShardDatastore) shardDir(id uint32) string {
	return filepath.Join(self.baseDbDir, fmt.Sprintf("%.5d", id))
}

func (self *ShardDatastore) closeOldestShard() {
	var oldestId uint32
	oldestAccess := int64(math.MaxInt64)
	for id, lastAccess := range self.lastAccess {
		if lastAccess < oldestAccess && self.shardsToClose[id] == false {
			oldestId = id
			oldestAccess = lastAccess
		}
	}
	if self.shardRefCounts[oldestId] == 0 {
		self.closeShard(oldestId)
	} else {
		self.shardsToClose[oldestId] = true
	}
}

func (self *ShardDatastore) closeShard(id uint32) {
	shard := self.shards[id]
	if shard != nil {
		shard.close()
	}
	delete(self.shardRefCounts, id)
	delete(self.shards, id)
	delete(self.lastAccess, id)
	delete(self.shardsToClose, id)
	log.Debug("DATASTORE: closing shard %s", self.shardDir(id))
}

// // returns true if the point has the correct field id and is
// // in the given time range
func isPointInRange(fieldId, startTime, endTime, point []byte) bool {
	id := point[:8]
	time := point[8:16]
	return bytes.Equal(id, fieldId) && bytes.Compare(time, startTime) > -1 && bytes.Compare(time, endTime) < 1
}

type FieldLookupError struct {
	message string
}

func (self FieldLookupError) Error() string {
	return self.message
}

// depending on the query order (whether it's ascending or not) returns
// the min (or max in case of descending query) of the current
// [timestamp,sequence] and the self's [timestamp,sequence]
//
// This is used to determine what the next point's timestamp
// and sequence number should be.
func (self *rawColumnValue) updatePointTimeAndSequence(currentTimeRaw, currentSequenceRaw []byte, isAscendingQuery bool) ([]byte, []byte) {
	if currentTimeRaw == nil {
		return self.time, self.sequence
	}

	compareValue := 1
	if isAscendingQuery {
		compareValue = -1
	}

	timeCompare := bytes.Compare(self.time, currentTimeRaw)
	if timeCompare == compareValue {
		return self.time, self.sequence
	}

	if timeCompare != 0 {
		return currentTimeRaw, currentSequenceRaw
	}

	if bytes.Compare(self.sequence, currentSequenceRaw) == compareValue {
		return currentTimeRaw, self.sequence
	}

	return currentTimeRaw, currentSequenceRaw
}
<|endoftext|>


<commit_before>package abelanaEndpoints

import (
	//    "fmt"

	"encoding/json"
	"log"
	"net/http"
	"time"
	// "appengine"
	//    "appengine/datastore"
	//	"github.com/cloud-abelana-go/appengine/endpoints/tokens"
	"github.com/go-martini/martini"
)

// The initial version of this will provide stubs for everything.  If there are changes, then
// be sure to get the Android app updated.  We are trying to put this together so that we can later
// refactor into several modules.

// How to generate Discovery Documentation & Client Liraries
//
// Android Apps
//  URL='https://endpoints-dot-abelana-222.appspot.com/_ah/api/discovery/v1/apis/management/v1/rest'
//  curl -s $URL > management.rest.discovery
//
// $ endpointscfg.py gen_client_lib java -bs gradle -o . management.rest.discovery
// iOS Apps
// Note the rpc suffix in the URL:
// $ URL='https://my-app-id.appspot.com/_ah/api/discovery/v1/apis/greeting/v1/rpc'
// $ curl -s $URL > greetings.rpc.discovery

// Comment holds all comments
type Comment struct {
	UserID string
	Text   string
}

// TLEntry holds timeline entries
type TLEntry struct {
	Date    time.Time
	UserID  string
	PhotoID string
	Likes   int
}

type Timeline struct {
	Status  string
	Entries []TLEntry
}

// Friend holds information about our friends
type Friend struct {
	UserID    string
	UserPhoto string
	Email     string
	Name      string
	ShareTo   bool
	ShareFrom bool
}

// func CustomMartini() *martini.ClassicMartini {
// 	r := martini.NewRouter()
// 	m := martini.New()
// 	m.Use(martini.Logger())
// 	m.Use(martini.Recovery())
// 	m.Use(martini.Static("public"))
// 	m.MapTo(r, (*martini.Routes)(nil))
// 	m.Action(r.Handle)
// 	return &martini.ClassicMartini{m, r}
// }

// func AppEngine(c appengine.Context, r *http.Request) {
// 	c.Map(appengine.NewContext(r))
// }

func init() {
	m := martini.Classic()
	// m.MapTo(appengine.NewContext(r), (*appengine.Context)(nil))

	// m.Use(func(c martini.Context, req *http.Request) {
	// 	ctx := appengine.NewContext(req)
	// 	c.MapTo(ctx, (*appengine.Context)(nil))
	// })

	m.Get("/", func() string {
		return "Hello from Martini/server!"
	})
	m.Get("/hello/:name", func(p martini.Params) string {
		return "Hi there " + p["name"]
	})
	log.Print("b4 things")

	m.Get("/login/:gittok", Login)

	// m.Use(AuthMe) // Hopefully, only do this for atok's

	// The basics of REST
	// When dealing with a Collection URI like: http://example.com/resources/
	// GET: List the members of the collection, complete with their member URIs for further navigation.
	// For example, list all the cars for sale.
	// PUT: Meaning defined as "replace the entire collection with another collection".
	// POST: Create a new entry in the collection where the ID is assigned automatically by the
	// collection. The ID created is usually included as part of the data returned by this operation.
	// DELETE: Meaning defined as "delete the entire collection".
	//
	// When dealing with a Member URI like: http://example.com/resources/7HOU57Y
	// GET: Retrieve a representation of the addressed member of the collection expressed in an
	// appropriate MIME type.
	// PUT: Update the addressed member of the collection or create it with the specified ID.
	// POST: Treats the addressed member as a collection in its own right and creates a new subordinate
	// of it.
	// DELETE: Delete the addressed member of the collection.

	m.Get("/user/:atok/refresh", Refresh)
	m.Delete("/user/:atok", Wipeout)
	m.Post("/user/:atok/facebook/:fbkey", Import)
	m.Post("/user/:atok/plus/:plkey", Import)
	m.Post("/user/:atok/yahoo/:ykey", Import)
	m.Get("/user/:atok/friends/", GetFriendsList)
	m.Get("/user/:atok/photo", GetUserPhoto)
	m.Put("/user/:atok/friends/:friendid", AddFriend)

	m.Get("/friend/:atok/:friendid", GetFriend)

	m.Put("/device/:atok/:regid", Register)
	m.Delete("/device/:atok/:regid", Unregister)

	m.Get("/timeline/:atok/:lastid", GetTimeLine)
	m.Get("/profile/:atok/:lastid", GetMyProfile)
	m.Get("/profile/:atok/:userid/:lastid", GetFriendsProfile)

	m.Post("/photo/:atok/:photoid/comment", SetPhotoComments)
	m.Put("/photo/:atok/:photoid/like", Like)
	m.Delete("/photo/:atok/:photoid/like", Unlike)
	m.Get("/photo/:atok/:photoid/comments", GetComments)

	http.Handle("/", m)
}

func Ok() string {
	return `{"Status": "Ok"}`
}

// replyJson Given an object, convert to JSON and reply with it
func replyJson(w http.ResponseWriter, v interface{}) {
	b, err := json.Marshal(v)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	h := w.Header()
	h.Add("Content-Type", "application/json")
	_, err = w.Write(b)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}
}

///////////////////////////////////////////////////////////////////////////////////////////////////
// Timeline
///////////////////////////////////////////////////////////////////////////////////////////////////

// TlfReq Timeline Friend Request
type TlfReq struct {
	ATok   string
	friend string
}

// TlResp Response timeline
type TlResp struct {
	resp   []TLEntry
	Status string
}

// GetTimeLine - get the timeline for the user (token) : TlResp
func GetTimeLine(p martini.Params, w http.ResponseWriter, req *http.Request) {
	timeline := []TLEntry{
		TLEntry{time.Now(), "00001", "0001", 1},
		TLEntry{time.Now(), "00001", "0002", 99},
		TLEntry{time.Now(), "00001", "0003", 0},
		TLEntry{time.Now(), "00001", "0004", 3},
		TLEntry{time.Now(), "00001", "0005", 21}}
	tl := &Timeline{"Ok", timeline}

	replyJson(w, tl)
}

// GetMyProfile - Get my entries only (token) : TlResp
func GetMyProfile(p martini.Params) string {
	return Ok()
}

// GetFriendsProfile - Get a specific friends entries only (TlfReq) : TlResp
func GetFriendsProfile(p martini.Params) string {
	return Ok()
}

func GetUserPhoto(p martini.Params) string {
	return Ok()
}

///////////////////////////////////////////////////////////////////////////////////////////////////
// Import
///////////////////////////////////////////////////////////////////////////////////////////////////

// Import for Facebook / G+ / ... (xcred) : StatusResp
func Import(p martini.Params) string {
	return Ok()
}

///////////////////////////////////////////////////////////////////////////////////////////////////
// Friend
///////////////////////////////////////////////////////////////////////////////////////////////////

// FlResp A list of friends
type FlResp struct {
	Friends []Friend
}

// FrReq Request sharing change
type FrReq struct {
	ATok     string
	FriendID string
	shareTo  bool
}

// GetFriendsList - A list of our friends (AToken) : FlResp
func GetFriendsList(p martini.Params) string {
	return Ok()
}

// GetFriend -- find out about someone (FReq) : Friend
func GetFriend(p martini.Params) string {
	return Ok()
}

// AddFriend - will tell us about a new possible friend (FrReq) : Status
func AddFriend(p martini.Params) string {
	return Ok()
}

///////////////////////////////////////////////////////////////////////////////////////////////////
// Photo
///////////////////////////////////////////////////////////////////////////////////////////////////

// SetPhotoComments allows the users voice to be heard (PhotoComment) : Status
func SetPhotoComments(p martini.Params) string {
	return Ok()
}

// Like let's the user tell of their joy (Photo) : Status
func Like(p martini.Params) string {
	return Ok()
}

// Unlike let's the user recind their +1 (Photo) : Status
func Unlike(p martini.Params) string {
	return Ok()
}

func GetComments(p martini.Params) string {
	return `{"Status": "Ok", "Comments":[]}`
}

///////////////////////////////////////////////////////////////////////////////////////////////////
// Management
///////////////////////////////////////////////////////////////////////////////////////////////////

// Login will validate the user and return an Access Token -- Null if invalid. (LoginReq) : ATok
func Login(p martini.Params) string {
	return `{"Status": "Ok", "Atok": "LES001"}`
}

// Refresh will refresh an Access Token (ATok)
func Refresh(p martini.Params) string {
	return `{"Status": "Ok", "Atok": "LES002"}`
}

// Wipeout will erase all data you are working on. (Atok) : Status
func Wipeout(p martini.Params) string {
	return Ok()
}

// Register will start GCM messages to your device (GCMReq) : Status
func Register(p martini.Params) string {
	return Ok()
}

// Unregister will stop GCM messages from going to your device (GCMReq) : Status
func Unregister(p martini.Params) string {
	return Ok()
}
<commit_msg>A bit more stubbing...<commit_after>package abelanaEndpoints

import (
	//    "fmt"

	"encoding/json"
	"log"
	"net/http"
	"time"
	// "appengine"
	//    "appengine/datastore"
	//	"github.com/cloud-abelana-go/appengine/endpoints/tokens"
	"github.com/go-martini/martini"
)

// The initial version of this will provide stubs for everything.  If there are changes, then
// be sure to get the Android app updated.  We are trying to put this together so that we can later
// refactor into several modules.

// How to generate Discovery Documentation & Client Liraries
//
// Android Apps
//  URL='https://endpoints-dot-abelana-222.appspot.com/_ah/api/discovery/v1/apis/management/v1/rest'
//  curl -s $URL > management.rest.discovery
//
// $ endpointscfg.py gen_client_lib java -bs gradle -o . management.rest.discovery
// iOS Apps
// Note the rpc suffix in the URL:
// $ URL='https://my-app-id.appspot.com/_ah/api/discovery/v1/apis/greeting/v1/rpc'
// $ curl -s $URL > greetings.rpc.discovery

// Comment holds all comments
type Comment struct {
	UserID string
	Text   string
}

// TLEntry holds timeline entries
type TLEntry struct {
	Date    time.Time
	UserID  string
	PhotoID string
	Likes   int
}

type Timeline struct {
	Status  string
	Entries []TLEntry
}

// Friend holds information about our friends
type Friend struct {
	UserID    string
	UserPhoto string
	Email     string
	Name      string
	ShareTo   bool
	ShareFrom bool
}

// func CustomMartini() *martini.ClassicMartini {
// 	r := martini.NewRouter()
// 	m := martini.New()
// 	m.Use(martini.Logger())
// 	m.Use(martini.Recovery())
// 	m.Use(martini.Static("public"))
// 	m.MapTo(r, (*martini.Routes)(nil))
// 	m.Action(r.Handle)
// 	return &martini.ClassicMartini{m, r}
// }

// func AppEngine(c appengine.Context, r *http.Request) {
// 	c.Map(appengine.NewContext(r))
// }

func init() {
	m := martini.Classic()
	// m.MapTo(appengine.NewContext(r), (*appengine.Context)(nil))

	// m.Use(func(c martini.Context, req *http.Request) {
	// 	ctx := appengine.NewContext(req)
	// 	c.MapTo(ctx, (*appengine.Context)(nil))
	// })

	m.Get("/", func() string {
		return "Hello from Martini/server!"
	})
	m.Get("/hello/:name", func(p martini.Params) string {
		return "Hi there " + p["name"]
	})
	log.Print("b4 things")

	m.Get("/login/:gittok", Login)

	// m.Use(AuthMe) // Hopefully, only do this for atok's

	// The basics of REST
	// When dealing with a Collection URI like: http://example.com/resources/
	// GET: List the members of the collection, complete with their member URIs for further navigation.
	// For example, list all the cars for sale.
	// PUT: Meaning defined as "replace the entire collection with another collection".
	// POST: Create a new entry in the collection where the ID is assigned automatically by the
	// collection. The ID created is usually included as part of the data returned by this operation.
	// DELETE: Meaning defined as "delete the entire collection".
	//
	// When dealing with a Member URI like: http://example.com/resources/7HOU57Y
	// GET: Retrieve a representation of the addressed member of the collection expressed in an
	// appropriate MIME type.
	// PUT: Update the addressed member of the collection or create it with the specified ID.
	// POST: Treats the addressed member as a collection in its own right and creates a new subordinate
	// of it.
	// DELETE: Delete the addressed member of the collection.

	m.Get("/user/:atok/refresh", Refresh)
	m.Delete("/user/:atok", Wipeout)
	m.Post("/user/:atok/facebook/:fbkey", Import)
	m.Post("/user/:atok/plus/:plkey", Import)
	m.Post("/user/:atok/yahoo/:ykey", Import)
	m.Get("/user/:atok/friends/", GetFriendsList)
	m.Get("/user/:atok/photo", GetUserPhoto)
	m.Put("/user/:atok/friends/:friendid", AddFriend)

	m.Get("/friend/:atok/:friendid", GetFriend)

	m.Put("/device/:atok/:regid", Register)
	m.Delete("/device/:atok/:regid", Unregister)

	m.Get("/timeline/:atok/:lastid", GetTimeLine)
	m.Get("/profile/:atok/:lastid", GetMyProfile)
	m.Get("/profile/:atok/:userid/:lastid", GetFriendsProfile)

	m.Post("/photo/:atok/:photoid/comment", SetPhotoComments)
	m.Put("/photo/:atok/:photoid/like", Like)
	m.Delete("/photo/:atok/:photoid/like", Unlike)
	m.Get("/photo/:atok/:photoid/comments", GetComments)

	http.Handle("/", m)
}

func Ok() string {
	return `{"Status": "Ok"}`
}

// replyJson Given an object, convert to JSON and reply with it
func replyJson(w http.ResponseWriter, v interface{}) {
	b, err := json.Marshal(v)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	h := w.Header()
	h.Add("Content-Type", "application/json")
	_, err = w.Write(b)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}
}

///////////////////////////////////////////////////////////////////////////////////////////////////
// Timeline
///////////////////////////////////////////////////////////////////////////////////////////////////

// TlfReq Timeline Friend Request
type TlfReq struct {
	ATok   string
	friend string
}

// TlResp Response timeline
type TlResp struct {
	resp   []TLEntry
	Status string
}

// GetTimeLine - get the timeline for the user (token) : TlResp
func GetTimeLine(p martini.Params, w http.ResponseWriter, req *http.Request) {
	timeline := []TLEntry{
		TLEntry{time.Now(), "00001", "0001", 1},
		TLEntry{time.Now(), "00001", "0002", 99},
		TLEntry{time.Now(), "00001", "0003", 0},
		TLEntry{time.Now(), "00001", "0004", 3},
		TLEntry{time.Now(), "00001", "0005", 1},
		TLEntry{time.Now(), "00001", "0006", 99},
		TLEntry{time.Now(), "00001", "0007", 0},
		TLEntry{time.Now(), "00001", "0008", 3},
		TLEntry{time.Now(), "00001", "0009", 1},
		TLEntry{time.Now(), "00001", "0010", 99},
		TLEntry{time.Now(), "00001", "0011", 0},
		TLEntry{time.Now(), "00001", "0004", 3},
		TLEntry{time.Now(), "00001", "0001", 1},
		TLEntry{time.Now(), "00001", "0002", 99},
		TLEntry{time.Now(), "00001", "0003", 0},
		TLEntry{time.Now(), "00001", "0005", 3},
		TLEntry{time.Now(), "00001", "0007", 1},
		TLEntry{time.Now(), "00001", "0006", 99},
		TLEntry{time.Now(), "00001", "0011", 0},
		TLEntry{time.Now(), "00001", "0009", 3},
		TLEntry{time.Now(), "00001", "0002", 99},
		TLEntry{time.Now(), "00001", "0003", 0},
		TLEntry{time.Now(), "00001", "0004", 3},
		TLEntry{time.Now(), "00001", "0005", 1},
		TLEntry{time.Now(), "00001", "0006", 99},
		TLEntry{time.Now(), "00001", "0007", 0},
		TLEntry{time.Now(), "00001", "0008", 3},
		TLEntry{time.Now(), "00001", "0009", 1},
		TLEntry{time.Now(), "00001", "0010", 99},
		TLEntry{time.Now(), "00001", "0011", 0},
		TLEntry{time.Now(), "00001", "0004", 3},
		TLEntry{time.Now(), "00001", "0001", 1},
		TLEntry{time.Now(), "00001", "0002", 99},
		TLEntry{time.Now(), "00001", "0003", 0},
		TLEntry{time.Now(), "00001", "0005", 3},
		TLEntry{time.Now(), "00001", "0007", 1},
		TLEntry{time.Now(), "00001", "0006", 99},
		TLEntry{time.Now(), "00001", "0011", 0},
		TLEntry{time.Now(), "00001", "0009", 3},
		TLEntry{time.Now(), "00001", "0005", 21}}
	tl := &Timeline{"Ok", timeline}

	replyJson(w, tl)
}

// GetMyProfile - Get my entries only (token) : TlResp
func GetMyProfile(p martini.Params) string {
	return Ok()
}

// GetFriendsProfile - Get a specific friends entries only (TlfReq) : TlResp
func GetFriendsProfile(p martini.Params) string {
	return Ok()
}

func GetUserPhoto(p martini.Params) string {
	return Ok()
}

///////////////////////////////////////////////////////////////////////////////////////////////////
// Import
///////////////////////////////////////////////////////////////////////////////////////////////////

// Import for Facebook / G+ / ... (xcred) : StatusResp
func Import(p martini.Params) string {
	return Ok()
}

///////////////////////////////////////////////////////////////////////////////////////////////////
// Friend
///////////////////////////////////////////////////////////////////////////////////////////////////

// FlResp A list of friends
type FlResp struct {
	Friends []Friend
}

// FrReq Request sharing change
type FrReq struct {
	ATok     string
	FriendID string
	shareTo  bool
}

// GetFriendsList - A list of our friends (AToken) : FlResp
func GetFriendsList(p martini.Params) string {
	return Ok()
}

// GetFriend -- find out about someone (FReq) : Friend
func GetFriend(p martini.Params) string {
	return Ok()
}

// AddFriend - will tell us about a new possible friend (FrReq) : Status
func AddFriend(p martini.Params) string {
	return Ok()
}

///////////////////////////////////////////////////////////////////////////////////////////////////
// Photo
///////////////////////////////////////////////////////////////////////////////////////////////////

// SetPhotoComments allows the users voice to be heard (PhotoComment) : Status
func SetPhotoComments(p martini.Params) string {
	return Ok()
}

// Like let's the user tell of their joy (Photo) : Status
func Like(p martini.Params) string {
	return Ok()
}

// Unlike let's the user recind their +1 (Photo) : Status
func Unlike(p martini.Params) string {
	return Ok()
}

func GetComments(p martini.Params) string {
	return `{"Status": "Ok", "Comments":[]}`
}

///////////////////////////////////////////////////////////////////////////////////////////////////
// Management
///////////////////////////////////////////////////////////////////////////////////////////////////

// Login will validate the user and return an Access Token -- Null if invalid. (LoginReq) : ATok
func Login(p martini.Params) string {
	return `{"Status": "Ok", "Atok": "LES001"}`
}

// Refresh will refresh an Access Token (ATok)
func Refresh(p martini.Params) string {
	return `{"Status": "Ok", "Atok": "LES002"}`
}

// Wipeout will erase all data you are working on. (Atok) : Status
func Wipeout(p martini.Params) string {
	return Ok()
}

// Register will start GCM messages to your device (GCMReq) : Status
func Register(p martini.Params) string {
	return Ok()
}

// Unregister will stop GCM messages from going to your device (GCMReq) : Status
func Unregister(p martini.Params) string {
	return Ok()
}
<|endoftext|>


<commit_before>// LTSV formatter for logrus
package logrusltsv

import (
	"bytes"
	"fmt"
	"regexp"
	"sort"
	"time"

	"github.com/Sirupsen/logrus"
)

type LogrusLTSVFormatter struct {
	conf LogrusLTSVConfig
}

type LogrusLTSVConfig struct {
	TimestampFormat string
	FieldPrefix     string
	Filters         []Filter
}

type Filter func(string) string

var reNL = regexp.MustCompile(`\n`)

func EscapeNewLine(s string) string {
	return reNL.ReplaceAllString(s, " ")
}

// NewDefaultFormatter create LogrusLTSVFormatter with default configuration.
func NewDefaultFormatter() *LogrusLTSVFormatter {
	c := LogrusLTSVConfig{
		TimestampFormat: logrus.DefaultTimestampFormat,
		FieldPrefix:     "field.",
		Filters:         []Filter{EscapeNewLine},
	}
	return NewFormatter(c)
}

// NewFormatter create LogrusLTSVFormatter
// with given configuration.
func NewFormatter(config LogrusLTSVConfig) *LogrusLTSVFormatter {
	return &LogrusLTSVFormatter{
		conf: config,
	}
}

func (f *LogrusLTSVFormatter) Format(entry *logrus.Entry) ([]byte, error) {
	var keys []string
	for k, _ := range entry.Data {
		keys = append(keys, k)
	}
	sort.Strings(keys)

	buf := &bytes.Buffer{}

	timestampFormat := f.conf.TimestampFormat
	if timestampFormat == "" {
		timestampFormat = logrus.DefaultTimestampFormat
	}

	fmt.Fprintf(
		buf,
		"time:%s\tlevel:%s\t",
		entry.Time.Format(timestampFormat),
		entry.Level.String(),
	)

	var val string
	for _, k := range keys {
		switch v := entry.Data[k].(type) {
		case string:
			val = v
		case time.Time:
			val = v.Format(timestampFormat)
		default:
			val = fmt.Sprintf("%v", v)
		}

		for _, filter := range f.conf.Filters {
			val = filter(val)
		}

		fmt.Fprintf(buf, "%s%s:%s\t", f.conf.FieldPrefix, k, val)
	}

	fmt.Fprintf(buf, "msg:%s\n", entry.Message)

	return buf.Bytes(), nil
}
<commit_msg>Apply filters to msg<commit_after>// LTSV formatter for logrus
package logrusltsv

import (
	"bytes"
	"fmt"
	"regexp"
	"sort"
	"time"

	"github.com/Sirupsen/logrus"
)

type LogrusLTSVFormatter struct {
	conf LogrusLTSVConfig
}

type LogrusLTSVConfig struct {
	TimestampFormat string
	FieldPrefix     string
	Filters         []Filter
}

type Filter func(string) string

var reNL = regexp.MustCompile(`\n`)

func EscapeNewLine(s string) string {
	return reNL.ReplaceAllString(s, " ")
}

// NewDefaultFormatter create LogrusLTSVFormatter with default configuration.
func NewDefaultFormatter() *LogrusLTSVFormatter {
	c := LogrusLTSVConfig{
		TimestampFormat: logrus.DefaultTimestampFormat,
		FieldPrefix:     "field.",
		Filters:         []Filter{EscapeNewLine},
	}
	return NewFormatter(c)
}

// NewFormatter create LogrusLTSVFormatter
// with given configuration.
func NewFormatter(config LogrusLTSVConfig) *LogrusLTSVFormatter {
	return &LogrusLTSVFormatter{
		conf: config,
	}
}

func (f *LogrusLTSVFormatter) filtering(val string) string {
	s := val
	for _, filter := range f.conf.Filters {
		s = filter(s)
	}
	return s
}

func (f *LogrusLTSVFormatter) Format(entry *logrus.Entry) ([]byte, error) {
	var keys []string
	for k, _ := range entry.Data {
		keys = append(keys, k)
	}
	sort.Strings(keys)

	buf := &bytes.Buffer{}

	timestampFormat := f.conf.TimestampFormat
	if timestampFormat == "" {
		timestampFormat = logrus.DefaultTimestampFormat
	}

	fmt.Fprintf(
		buf,
		"time:%s\tlevel:%s\t",
		entry.Time.Format(timestampFormat),
		entry.Level.String(),
	)

	var val string
	for _, k := range keys {
		switch v := entry.Data[k].(type) {
		case string:
			val = v
		case time.Time:
			val = v.Format(timestampFormat)
		default:
			val = fmt.Sprintf("%v", v)
		}

		val = f.filtering(val)

		fmt.Fprintf(buf, "%s%s:%s\t", f.conf.FieldPrefix, k, val)
	}

	fmt.Fprintf(buf, "msg:%s\n", f.filtering(entry.Message))

	return buf.Bytes(), nil
}
<|endoftext|>