<commit_before>package ignoreCaseSorted

import (
	"sort"
	"strings"
)

type KeyValue[T any] struct {
	Key   string
	Value T
}

type Dictionary[T any] struct {
	maps  map[string]KeyValue[T]
	order []string
}

func (d *Dictionary[T]) Len() int {
	return len(d.maps)
}

// Store is same as Set that is a compatible method with "sync".Map
func (d *Dictionary[T]) Store(key string, val T) {
	d.Set(key, val)
}

func (d *Dictionary[T]) Set(key string, val T) {
	lowerKey := strings.ToLower(key)
	if d.maps == nil {
		d.maps = make(map[string]KeyValue[T])
	}
	d.maps[lowerKey] = KeyValue[T]{Key: key, Value: val}
	if d.order != nil && len(d.order) > 0 {
		d.order = d.order[:0]
	}
}

func (d *Dictionary[T]) Delete(key string) {
	if d.maps == nil {
		return
	}
	lowerKey := strings.ToLower(key)
	delete(d.maps, lowerKey)
	if d.order != nil && len(d.order) > 0 {
		d.order = d.order[:0]
	}
}

// Load is same as Get that is a compatible method with "sync".Map
func (d *Dictionary[T]) Load(key string) (val T, ok bool) {
	return d.Get(key)
}

func (d *Dictionary[T]) Get(key string) (val T, ok bool) {
	if d.maps == nil {
		return
	}
	lowerKey := strings.ToLower(key)
	var v KeyValue[T]
	v, ok = d.maps[lowerKey]
	if ok {
		val = v.Value
	}
	return
}

func (d *Dictionary[t]) makeOrder() {
	if d.order == nil {
		d.order = make([]string, 0, len(d.maps))
	} else if len(d.order) > 0 {
		return
	}
	for key := range d.maps {
		d.order = append(d.order, key)
	}
	sort.Strings(d.order)
}

func (d *Dictionary[T]) Range(f func(string, T) bool) {
	d.makeOrder()
	for _, lowerKey := range d.order {
		p := d.maps[lowerKey]
		if !f(p.Key, p.Value) {
			break
		}
	}
}

func MapToDictionary[T any](source map[string]T) *Dictionary[T] {
	var d Dictionary[T]
	for key, val := range source {
		d.Set(key, val)
	}
	return &d
}

type Ascending[T any] struct {
	Key   string
	Value T
	maps  map[string]KeyValue[T]
	order []string
}

func (d *Dictionary[T]) Ascend() *Ascending[T] {
	d.makeOrder()
	if d.maps == nil || len(d.maps) <= 0 {
		return nil
	}
	p := d.maps[d.order[0]]
	return &Ascending[T]{
		maps:  d.maps,
		order: d.order[1:],
		Key:   p.Key,
		Value: p.Value,
	}
}

func (a *Ascending[T]) Next() *Ascending[T] {
	if a == nil || a.order == nil || len(a.order) < 1 {
		return nil
	}
	p := a.maps[a.order[0]]
	a.Key = p.Key
	a.Value = p.Value
	a.order = a.order[1:]
	return a
}
<commit_msg>internal/go-ignorecase-sorted: hide KeyValue by rename as _Pair<commit_after>package ignoreCaseSorted

import (
	"sort"
	"strings"
)

type _Pair[T any] struct {
	Key   string
	Value T
}

type Dictionary[T any] struct {
	maps  map[string]_Pair[T]
	order []string
}

func (d *Dictionary[T]) Len() int {
	return len(d.maps)
}

// Store is same as Set that is a compatible method with "sync".Map
func (d *Dictionary[T]) Store(key string, val T) {
	d.Set(key, val)
}

func (d *Dictionary[T]) Set(key string, val T) {
	lowerKey := strings.ToLower(key)
	if d.maps == nil {
		d.maps = make(map[string]_Pair[T])
	}
	d.maps[lowerKey] = _Pair[T]{Key: key, Value: val}
	if d.order != nil && len(d.order) > 0 {
		d.order = d.order[:0]
	}
}

func (d *Dictionary[T]) Delete(key string) {
	if d.maps == nil {
		return
	}
	lowerKey := strings.ToLower(key)
	delete(d.maps, lowerKey)
	if d.order != nil && len(d.order) > 0 {
		d.order = d.order[:0]
	}
}

// Load is same as Get that is a compatible method with "sync".Map
func (d *Dictionary[T]) Load(key string) (val T, ok bool) {
	return d.Get(key)
}

func (d *Dictionary[T]) Get(key string) (val T, ok bool) {
	if d.maps == nil {
		return
	}
	lowerKey := strings.ToLower(key)
	var v _Pair[T]
	v, ok = d.maps[lowerKey]
	if ok {
		val = v.Value
	}
	return
}

func (d *Dictionary[t]) makeOrder() {
	if d.order == nil {
		d.order = make([]string, 0, len(d.maps))
	} else if len(d.order) > 0 {
		return
	}
	for key := range d.maps {
		d.order = append(d.order, key)
	}
	sort.Strings(d.order)
}

func (d *Dictionary[T]) Range(f func(string, T) bool) {
	d.makeOrder()
	for _, lowerKey := range d.order {
		p := d.maps[lowerKey]
		if !f(p.Key, p.Value) {
			break
		}
	}
}

func MapToDictionary[T any](source map[string]T) *Dictionary[T] {
	var d Dictionary[T]
	for key, val := range source {
		d.Set(key, val)
	}
	return &d
}

type Ascending[T any] struct {
	Key   string
	Value T
	maps  map[string]_Pair[T]
	order []string
}

func (d *Dictionary[T]) Ascend() *Ascending[T] {
	d.makeOrder()
	if d.maps == nil || len(d.maps) <= 0 {
		return nil
	}
	p := d.maps[d.order[0]]
	return &Ascending[T]{
		maps:  d.maps,
		order: d.order[1:],
		Key:   p.Key,
		Value: p.Value,
	}
}

func (a *Ascending[T]) Next() *Ascending[T] {
	if a == nil || a.order == nil || len(a.order) < 1 {
		return nil
	}
	p := a.maps[a.order[0]]
	a.Key = p.Key
	a.Value = p.Value
	a.order = a.order[1:]
	return a
}
<|endoftext|>


<commit_before>package adhier

import (
	"fmt"
	"math"

	"github.com/ready-steady/numeric/basis/linhat"
	"github.com/ready-steady/numeric/grid/newcot"
)

// Interpolation in one dimension.
func ExampleInterpolator_step() {
	const (
		inputs    = 1
		outputs   = 1
		tolerance = 1e-4
	)

	grid, basis := newcot.NewClosed(inputs), linhat.NewClosed(inputs)
	interpolator := New(grid, basis, NewConfig())

	quantity := NewQuantity(inputs, outputs)
	quantity.ComputeHandler = func(x, y []float64) {
		if x[0] <= 0.5 {
			y[0] = 1
		} else {
			y[0] = 0
		}
	}
	quantity.RefineHandler = func(Δ []float64) bool {
		return math.Abs(Δ[0]) > tolerance
	}

	surrogate := interpolator.Compute(quantity)

	fmt.Println(surrogate)

	// Output:
	// Surrogate{inputs: 1, outputs: 1, level: 9, nodes: 18}
}

// Interpolation in two dimensions.
func ExampleInterpolator_cube() {
	const (
		inputs    = 2
		outputs   = 1
		tolerance = 1e-4
	)

	grid, basis := newcot.NewClosed(inputs), linhat.NewClosed(inputs)
	interpolator := New(grid, basis, NewConfig())

	quantity := NewQuantity(inputs, outputs)
	quantity.ComputeHandler = func(x, y []float64) {
		if math.Abs(2*x[0]-1) < 0.45 && math.Abs(2*x[1]-1) < 0.45 {
			y[0] = 1
		} else {
			y[0] = 0
		}
	}
	quantity.RefineHandler = func(Δ []float64) bool {
		return math.Abs(Δ[0]) > tolerance
	}

	surrogate := interpolator.Compute(quantity)

	fmt.Println(surrogate)

	// Output:
	// Surrogate{inputs: 2, outputs: 1, level: 9, nodes: 377}
}
<commit_msg>adhier: rename quantity into target<commit_after>package adhier

import (
	"fmt"
	"math"

	"github.com/ready-steady/numeric/basis/linhat"
	"github.com/ready-steady/numeric/grid/newcot"
)

// Interpolation in one dimension.
func ExampleInterpolator_step() {
	const (
		inputs    = 1
		outputs   = 1
		tolerance = 1e-4
	)

	grid, basis := newcot.NewClosed(inputs), linhat.NewClosed(inputs)
	interpolator := New(grid, basis, NewConfig())

	target := NewQuantity(inputs, outputs)
	target.ComputeHandler = func(x, y []float64) {
		if x[0] <= 0.5 {
			y[0] = 1
		} else {
			y[0] = 0
		}
	}
	target.RefineHandler = func(Δ []float64) bool {
		return math.Abs(Δ[0]) > tolerance
	}

	surrogate := interpolator.Compute(target)

	fmt.Println(surrogate)

	// Output:
	// Surrogate{inputs: 1, outputs: 1, level: 9, nodes: 18}
}

// Interpolation in two dimensions.
func ExampleInterpolator_cube() {
	const (
		inputs    = 2
		outputs   = 1
		tolerance = 1e-4
	)

	grid, basis := newcot.NewClosed(inputs), linhat.NewClosed(inputs)
	interpolator := New(grid, basis, NewConfig())

	target := NewQuantity(inputs, outputs)
	target.ComputeHandler = func(x, y []float64) {
		if math.Abs(2*x[0]-1) < 0.45 && math.Abs(2*x[1]-1) < 0.45 {
			y[0] = 1
		} else {
			y[0] = 0
		}
	}
	target.RefineHandler = func(Δ []float64) bool {
		return math.Abs(Δ[0]) > tolerance
	}

	surrogate := interpolator.Compute(target)

	fmt.Println(surrogate)

	// Output:
	// Surrogate{inputs: 2, outputs: 1, level: 9, nodes: 377}
}
<|endoftext|>


<commit_before>package wizutil

const maxBufLen = 128 * 1024 // 128KB buffer

// ByteView allows treating an io.ReaderAt as a byte
// array.
type ByteView struct {
	Input    *SliceReader
	LookBack int64

	buf       []byte
	bufOffset int64
	bufLen    int64
}

// Get returns the byte at index i, or -1 if we
// failed to read
func (bv *ByteView) Get(i int64) int {
	if bv.buf == nil {
		bv.buf = make([]byte, maxBufLen)
	}

	// already got it in buf?
	posInBuffer := i - bv.bufOffset
	if posInBuffer >= 0 && posInBuffer < bv.bufLen {
		return int(bv.buf[posInBuffer])
	}

	newOffset := max(0, i-bv.LookBack)
	newEnd := min(newOffset+maxBufLen-1, bv.Input.Size()-1)
	newBufLen := (newEnd - newOffset) + 1
	if newBufLen <= 0 {
		// input isn't big enough
		return -1
	}

	bv.bufOffset = newOffset
	bv.bufLen = newBufLen

	// don't got it in buf! must read.
	_, err := bv.Input.ReadAt(bv.buf[:bv.bufLen], bv.bufOffset)
	if err != nil {
		// that's pretty bad
		return -1
	}

	posInBuffer = i - bv.bufOffset
	return int(bv.buf[posInBuffer])
}

func min(a, b int64) int64 {
	if a < b {
		return a
	}
	return b
}

func max(a, b int64) int64 {
	if a > b {
		return a
	}
	return b
}
<commit_msg>ByteView: return -1 on out of range reads, closes https://github.com/itchio/itch/issues/2113<commit_after>package wizutil

const maxBufLen = 128 * 1024 // 128KB buffer

// ByteView allows treating an io.ReaderAt as a byte
// array.
type ByteView struct {
	Input    *SliceReader
	LookBack int64

	buf       []byte
	bufOffset int64
	bufLen    int64
}

// Get returns the byte at index i, or -1 if we
// failed to read
func (bv *ByteView) Get(i int64) int {
	if i < 0 {
		// yeah that's out of range, don't bother reading
		return 1
	}

	if bv.buf == nil {
		bv.buf = make([]byte, maxBufLen)
	}

	// already got it in buf?
	posInBuffer := i - bv.bufOffset
	if posInBuffer >= 0 && posInBuffer < bv.bufLen {
		return int(bv.buf[posInBuffer])
	}

	newOffset := max(0, i-bv.LookBack)
	newEnd := min(newOffset+maxBufLen-1, bv.Input.Size()-1)
	newBufLen := (newEnd - newOffset) + 1
	if newBufLen <= 0 {
		// input isn't big enough
		return -1
	}

	bv.bufOffset = newOffset
	bv.bufLen = newBufLen

	// don't got it in buf! must read.
	_, err := bv.Input.ReadAt(bv.buf[:bv.bufLen], bv.bufOffset)
	if err != nil {
		// that's pretty bad
		return -1
	}

	posInBuffer = i - bv.bufOffset
	return int(bv.buf[posInBuffer])
}

func min(a, b int64) int64 {
	if a < b {
		return a
	}
	return b
}

func max(a, b int64) int64 {
	if a > b {
		return a
	}
	return b
}
<|endoftext|>


<commit_before>package zenodb

import (
	"fmt"
	"hash"
	"reflect"
	"strings"
	"time"

	"github.com/dustin/go-humanize"
	"github.com/getlantern/bytemap"
	"github.com/getlantern/errors"
	"github.com/getlantern/wal"
	"github.com/getlantern/zenodb/encoding"
)

func (db *DB) Insert(stream string, ts time.Time, dims map[string]interface{}, vals map[string]interface{}) error {
	return db.InsertRaw(stream, ts, bytemap.New(dims), bytemap.New(vals))
}

func (db *DB) InsertRaw(stream string, ts time.Time, dims bytemap.ByteMap, vals bytemap.ByteMap) error {
	if db.opts.Follow != nil {
		return errors.New("Declining to insert data directly to follower")
	}

	stream = strings.TrimSpace(strings.ToLower(stream))
	db.tablesMutex.Lock()
	w := db.streams[stream]
	db.tablesMutex.Unlock()
	if w == nil {
		return fmt.Errorf("No wal found for stream %v", stream)
	}

	if len(db.opts.WhitelistedDimensions) > 0 {
		dims = dims.Slice(db.opts.WhitelistedDimensions...)
	}

	tsd := make([]byte, encoding.Width64bits)
	encoding.EncodeTime(tsd, ts)
	dimsLen := make([]byte, encoding.Width32bits)
	encoding.WriteInt32(dimsLen, len(dims))
	valsLen := make([]byte, encoding.Width32bits)
	encoding.WriteInt32(valsLen, len(vals))
	if db.log.IsTraceEnabled() {
		db.log.Tracef("Writing to wal with dims length %d: %v", len(dims), bytemap.ByteMap(dims).AsMap())
	}
	err := w.Write(tsd, dimsLen, dims, valsLen, vals)
	if err != nil {
		db.log.Error(err)
	}
	return err
}

type walRead struct {
	data   []byte
	offset wal.Offset
	source int
}

func (t *table) processWALInserts() {
	in := make(chan *walRead)
	t.db.Go(func(stop <-chan interface{}) {
		t.processInserts(in, stop)
	})

	for {
		data, err := t.wal.Read()
		if err != nil {
			t.db.Panic(fmt.Errorf("Unable to read from WAL: %v", err))
		}
		in <- &walRead{data, t.wal.Offset(), 0}
	}
}

func (t *table) processInserts(in chan *walRead, stop <-chan interface{}) {
	isFollower := t.db.opts.Follow != nil
	start := time.Now()
	inserted := 0
	skipped := 0
	bytesRead := 0

	h := partitionHash()
loop:
	for {
		select {
		case <-stop:
			return
		case read := <-in:
			if read.data == nil {
				// Ignore empty data
				continue loop
			}
			bytesRead += len(read.data)
			if t.insert(read.data, isFollower, h, read.offset, read.source) {
				inserted++
			} else {
				// Did not insert (probably due to WHERE clause)
				t.skip(read.offset, read.source)
				skipped++
			}
			t.db.walBuffers.Put(read.data)
			delta := time.Now().Sub(start)
			if delta > 1*time.Minute {
				t.log.Debugf("Read %v at %v per second", humanize.Bytes(uint64(bytesRead)), humanize.Bytes(uint64(float64(bytesRead)/delta.Seconds())))
				t.log.Debugf("Inserted %v points at %v per second", humanize.Comma(int64(inserted)), humanize.Commaf(float64(inserted)/delta.Seconds()))
				t.log.Debugf("Skipped %v points at %v per second", humanize.Comma(int64(skipped)), humanize.Commaf(float64(skipped)/delta.Seconds()))
				inserted = 0
				skipped = 0
				bytesRead = 0
				start = time.Now()
			}
		}
	}
}

func (t *table) insert(data []byte, isFollower bool, h hash.Hash32, offset wal.Offset, source int) bool {
	defer func() {
		p := recover()
		if p != nil {
			t.log.Errorf("Panic in inserting: %v", p)
		}
	}()

	tsd, remain := encoding.Read(data, encoding.Width64bits)
	ts := encoding.TimeFromBytes(tsd)
	if ts.Before(t.truncateBefore()) {
		// Ignore old data
		return false
	}
	dimsLen, remain := encoding.ReadInt32(remain)
	dims, remain := encoding.Read(remain, dimsLen)
	if isFollower && !t.db.inPartition(h, dims, t.PartitionBy, t.db.opts.Partition) {
		// data not relevant to follower on this table
		if t.log.IsTraceEnabled() {
			t.log.Tracef("Not in partition: %v", bytemap.ByteMap(dims).AsMap())
		}
		return false
	}

	valsLen, remain := encoding.ReadInt32(remain)
	vals, _ := encoding.Read(remain, valsLen)
	// Split the dims and vals so that holding on to one doesn't force holding on
	// to the other. Also, we need copies for both because the WAL read buffer
	// will change on next call to wal.Read().
	dimsBM := make(bytemap.ByteMap, len(dims))
	valsBM := make(bytemap.ByteMap, len(vals))
	copy(dimsBM, dims)
	copy(valsBM, vals)
	if t.log.IsTraceEnabled() {
		t.log.Tracef("Length of data: %d", len(data))
		if len(vals) > 0 {
			t.log.Tracef("Length of vals: %d", len(valsBM))
			t.log.Tracef("Vals are %v", valsBM.AsMap())
		}
		if len(dims) > 0 {
			t.log.Tracef("Length of dims: %d", len(dimsBM))
			t.log.Tracef("Dims are %v", dimsBM.AsMap())
		}
	}
	return t.doInsert(ts, dimsBM, valsBM, offset, source)
}

// Skip informs the table of a new offset so that we can store it
func (t *table) skip(offset wal.Offset, source int) {
	t.rowStore.insert(&insert{nil, nil, nil, offset, source})
}

func (t *table) doInsert(ts time.Time, dims bytemap.ByteMap, vals bytemap.ByteMap, offset wal.Offset, source int) bool {
	where := t.getWhere()

	if where != nil {
		ok := where.Eval(dims)
		if !ok.(bool) {
			if t.log.IsTraceEnabled() {
				t.log.Tracef("Filtering out inbound point at %v due to %v: %v", ts, where, dims.AsMap())
			}
			t.statsMutex.Lock()
			t.stats.FilteredPoints++
			t.statsMutex.Unlock()
			return false
		}
	}
	t.db.clock.Advance(ts)

	if t.log.IsTraceEnabled() {
		t.log.Tracef("Including inbound point at %v: %v", ts, dims.AsMap())
	}

	var key bytemap.ByteMap
	if len(t.GroupBy) == 0 {
		key = dims
	} else {
		// Reslice dimensions
		names := make([]string, 0, len(t.GroupBy))
		values := make([]interface{}, 0, len(t.GroupBy))
		for _, groupBy := range t.GroupBy {
			val := groupBy.Expr.Eval(dims)
			if val != nil {
				names = append(names, groupBy.Name)
				values = append(values, val)
			}
		}
		key = bytemap.FromSortedKeysAndValues(names, values)
	}

	// Do separate inserts rows for array values if necessary
	var additionalVals []bytemap.ByteMap
	hasMainValue := false
	mainVals := bytemap.Build(func(_include func(string, interface{})) {
		include := func(key string, val float64) {
			_include(key, val)
			hasMainValue = true
		}
		vals.IterateValues(func(key string, value interface{}) bool {
			switch v := value.(type) {
			case float64:
				include(key, v)
			case int:
				include(key, float64(v))
			case []float64:
				// include first value with main vals
				include(key, v[0])
				// do separate inserts for additional values
				for i := 1; i < len(v); i++ {
					subVals := bytemap.Build(func(subInclude func(string, interface{})) {
						subInclude(key, v[i])
					}, nil, true)
					additionalVals = append(additionalVals, subVals)
				}
			case []int:
				// include first value with main vals
				include(key, float64(v[0]))
				// do separate inserts for additional values
				for i := 1; i < len(v); i++ {
					subVals := bytemap.Build(func(subInclude func(string, interface{})) {
						subInclude(key, float64(v[i]))
					}, nil, true)
					additionalVals = append(additionalVals, subVals)
				}
			default:
				t.log.Errorf("Key %v contained value '%v' of unsupported type %v, ignoring", key, value, reflect.TypeOf(value))
			}
			return true
		})
	}, nil, true)

	t.db.capMemorySize(true)
	inserted := len(additionalVals)
	if hasMainValue {
		t.rowStore.insert(&insert{key, encoding.NewTSParams(ts, mainVals), dims, offset, source})
		inserted++
	}
	for _, subVals := range additionalVals {
		t.rowStore.insert(&insert{key, encoding.NewTSParams(ts, subVals), dims, offset, source})
	}
	t.statsMutex.Lock()
	t.stats.InsertedPoints += int64(inserted)
	t.statsMutex.Unlock()

	return true
}

func (t *table) recordQueued() {
	t.statsMutex.Lock()
	t.stats.QueuedPoints++
	t.statsMutex.Unlock()
}
<commit_msg>More trace logging<commit_after>package zenodb

import (
	"fmt"
	"hash"
	"reflect"
	"strings"
	"time"

	"github.com/dustin/go-humanize"
	"github.com/getlantern/bytemap"
	"github.com/getlantern/errors"
	"github.com/getlantern/wal"
	"github.com/getlantern/zenodb/encoding"
)

func (db *DB) Insert(stream string, ts time.Time, dims map[string]interface{}, vals map[string]interface{}) error {
	return db.InsertRaw(stream, ts, bytemap.New(dims), bytemap.New(vals))
}

func (db *DB) InsertRaw(stream string, ts time.Time, dims bytemap.ByteMap, vals bytemap.ByteMap) error {
	if db.opts.Follow != nil {
		return errors.New("Declining to insert data directly to follower")
	}

	stream = strings.TrimSpace(strings.ToLower(stream))
	db.tablesMutex.Lock()
	w := db.streams[stream]
	db.tablesMutex.Unlock()
	if w == nil {
		return fmt.Errorf("No wal found for stream %v", stream)
	}

	if len(db.opts.WhitelistedDimensions) > 0 {
		if db.log.IsTraceEnabled() {
			db.log.Tracef("Whitelist Dims Original dims: %v", dims.AsMap())
			db.log.Tracef("Whitelist Dims Slicing dims with whitelist: %v", db.opts.WhitelistedDimensions)
		}
		dims = dims.Slice(db.opts.WhitelistedDimensions...)
		if db.log.IsTraceEnabled() {
			db.log.Tracef("Whitelist Dims Sliced dims: %v", dims.AsMap())
		}
	}

	tsd := make([]byte, encoding.Width64bits)
	encoding.EncodeTime(tsd, ts)
	dimsLen := make([]byte, encoding.Width32bits)
	encoding.WriteInt32(dimsLen, len(dims))
	valsLen := make([]byte, encoding.Width32bits)
	encoding.WriteInt32(valsLen, len(vals))
	if db.log.IsTraceEnabled() {
		db.log.Tracef("Writing to wal with dims length %d: %v", len(dims), bytemap.ByteMap(dims).AsMap())
	}
	err := w.Write(tsd, dimsLen, dims, valsLen, vals)
	if err != nil {
		db.log.Error(err)
	}
	return err
}

type walRead struct {
	data   []byte
	offset wal.Offset
	source int
}

func (t *table) processWALInserts() {
	in := make(chan *walRead)
	t.db.Go(func(stop <-chan interface{}) {
		t.processInserts(in, stop)
	})

	for {
		data, err := t.wal.Read()
		if err != nil {
			t.db.Panic(fmt.Errorf("Unable to read from WAL: %v", err))
		}
		in <- &walRead{data, t.wal.Offset(), 0}
	}
}

func (t *table) processInserts(in chan *walRead, stop <-chan interface{}) {
	isFollower := t.db.opts.Follow != nil
	start := time.Now()
	inserted := 0
	skipped := 0
	bytesRead := 0

	h := partitionHash()
loop:
	for {
		select {
		case <-stop:
			return
		case read := <-in:
			if read.data == nil {
				// Ignore empty data
				continue loop
			}
			bytesRead += len(read.data)
			if t.insert(read.data, isFollower, h, read.offset, read.source) {
				inserted++
			} else {
				// Did not insert (probably due to WHERE clause)
				t.skip(read.offset, read.source)
				skipped++
			}
			t.db.walBuffers.Put(read.data)
			delta := time.Now().Sub(start)
			if delta > 1*time.Minute {
				t.log.Debugf("Read %v at %v per second", humanize.Bytes(uint64(bytesRead)), humanize.Bytes(uint64(float64(bytesRead)/delta.Seconds())))
				t.log.Debugf("Inserted %v points at %v per second", humanize.Comma(int64(inserted)), humanize.Commaf(float64(inserted)/delta.Seconds()))
				t.log.Debugf("Skipped %v points at %v per second", humanize.Comma(int64(skipped)), humanize.Commaf(float64(skipped)/delta.Seconds()))
				inserted = 0
				skipped = 0
				bytesRead = 0
				start = time.Now()
			}
		}
	}
}

func (t *table) insert(data []byte, isFollower bool, h hash.Hash32, offset wal.Offset, source int) bool {
	defer func() {
		p := recover()
		if p != nil {
			t.log.Errorf("Panic in inserting: %v", p)
		}
	}()

	tsd, remain := encoding.Read(data, encoding.Width64bits)
	ts := encoding.TimeFromBytes(tsd)
	if ts.Before(t.truncateBefore()) {
		// Ignore old data
		return false
	}
	dimsLen, remain := encoding.ReadInt32(remain)
	dims, remain := encoding.Read(remain, dimsLen)
	if isFollower && !t.db.inPartition(h, dims, t.PartitionBy, t.db.opts.Partition) {
		// data not relevant to follower on this table
		if t.log.IsTraceEnabled() {
			t.log.Tracef("Not in partition: %v", bytemap.ByteMap(dims).AsMap())
		}
		return false
	}

	valsLen, remain := encoding.ReadInt32(remain)
	vals, _ := encoding.Read(remain, valsLen)
	// Split the dims and vals so that holding on to one doesn't force holding on
	// to the other. Also, we need copies for both because the WAL read buffer
	// will change on next call to wal.Read().
	dimsBM := make(bytemap.ByteMap, len(dims))
	valsBM := make(bytemap.ByteMap, len(vals))
	copy(dimsBM, dims)
	copy(valsBM, vals)
	if t.log.IsTraceEnabled() {
		t.log.Tracef("Length of data: %d", len(data))
		if len(vals) > 0 {
			t.log.Tracef("Length of vals: %d", len(valsBM))
			t.log.Tracef("Vals are %v", valsBM.AsMap())
		}
		if len(dims) > 0 {
			t.log.Tracef("Length of dims: %d", len(dimsBM))
			t.log.Tracef("Dims are %v", dimsBM.AsMap())
		}
	}
	return t.doInsert(ts, dimsBM, valsBM, offset, source)
}

// Skip informs the table of a new offset so that we can store it
func (t *table) skip(offset wal.Offset, source int) {
	t.rowStore.insert(&insert{nil, nil, nil, offset, source})
}

func (t *table) doInsert(ts time.Time, dims bytemap.ByteMap, vals bytemap.ByteMap, offset wal.Offset, source int) bool {
	where := t.getWhere()

	if where != nil {
		ok := where.Eval(dims)
		if !ok.(bool) {
			if t.log.IsTraceEnabled() {
				t.log.Tracef("Filtering out inbound point at %v due to %v: %v", ts, where, dims.AsMap())
			}
			t.statsMutex.Lock()
			t.stats.FilteredPoints++
			t.statsMutex.Unlock()
			return false
		}
	}
	t.db.clock.Advance(ts)

	if t.log.IsTraceEnabled() {
		t.log.Tracef("Including inbound point at %v: %v", ts, dims.AsMap())
	}

	var key bytemap.ByteMap
	if len(t.GroupBy) == 0 {
		key = dims
	} else {
		// Reslice dimensions
		names := make([]string, 0, len(t.GroupBy))
		values := make([]interface{}, 0, len(t.GroupBy))
		for _, groupBy := range t.GroupBy {
			val := groupBy.Expr.Eval(dims)
			if val != nil {
				names = append(names, groupBy.Name)
				values = append(values, val)
			}
		}
		key = bytemap.FromSortedKeysAndValues(names, values)
	}

	// Do separate inserts rows for array values if necessary
	var additionalVals []bytemap.ByteMap
	hasMainValue := false
	mainVals := bytemap.Build(func(_include func(string, interface{})) {
		include := func(key string, val float64) {
			_include(key, val)
			hasMainValue = true
		}
		vals.IterateValues(func(key string, value interface{}) bool {
			switch v := value.(type) {
			case float64:
				include(key, v)
			case int:
				include(key, float64(v))
			case []float64:
				// include first value with main vals
				include(key, v[0])
				// do separate inserts for additional values
				for i := 1; i < len(v); i++ {
					subVals := bytemap.Build(func(subInclude func(string, interface{})) {
						subInclude(key, v[i])
					}, nil, true)
					additionalVals = append(additionalVals, subVals)
				}
			case []int:
				// include first value with main vals
				include(key, float64(v[0]))
				// do separate inserts for additional values
				for i := 1; i < len(v); i++ {
					subVals := bytemap.Build(func(subInclude func(string, interface{})) {
						subInclude(key, float64(v[i]))
					}, nil, true)
					additionalVals = append(additionalVals, subVals)
				}
			default:
				t.log.Errorf("Key %v contained value '%v' of unsupported type %v, ignoring", key, value, reflect.TypeOf(value))
			}
			return true
		})
	}, nil, true)

	t.db.capMemorySize(true)
	inserted := len(additionalVals)
	if hasMainValue {
		t.rowStore.insert(&insert{key, encoding.NewTSParams(ts, mainVals), dims, offset, source})
		inserted++
	}
	for _, subVals := range additionalVals {
		t.rowStore.insert(&insert{key, encoding.NewTSParams(ts, subVals), dims, offset, source})
	}
	t.statsMutex.Lock()
	t.stats.InsertedPoints += int64(inserted)
	t.statsMutex.Unlock()

	return true
}

func (t *table) recordQueued() {
	t.statsMutex.Lock()
	t.stats.QueuedPoints++
	t.statsMutex.Unlock()
}
<|endoftext|>


<commit_before>package workers

import (
	"fmt"
	"github.com/APTrust/exchange/constants"
	"github.com/APTrust/exchange/context"
	"github.com/APTrust/exchange/models"
	"github.com/APTrust/exchange/network"
	"github.com/APTrust/exchange/stats"
	"github.com/APTrust/exchange/util"
	"github.com/aws/aws-sdk-go/service/s3"
	"net/url"
	"strconv"
	"strings"
	"time"
)

// If Config.BucketReaderCacheHours isn't set to something
// sensible, cache Ingest WorkItems up to this many hours old.
const DEFAULT_CACHE_HOURS = 24

// How many S3 keys should we fetch in each batch when
// we're getting the contents of a bucket?
const MAX_KEYS = 1000

// APTBucketReader scans APTrust receiving buckets for items
// that need to be ingested. It creates a WorkItem record and
// an NSQ entry for each qualifying bag, and is responsible for
// knowing whether items in the receiving buckets actually need
// to be queued.
type APTBucketReader struct {
	Context           *context.Context
	Institutions      map[string]*models.Institution
	RecentIngestItems map[string]*models.WorkItem
	stats             *stats.APTBucketReaderStats
	statsEnabled      bool
}

// Creates a new bucket reader with the given context.
// Param enableStats is generally used for integration tests.
// Enabling stats in production can cause high memory usage,
// so keep that off unless you're trying to diagnose specific problems.
func NewAPTBucketReader(context *context.Context, enableStats bool) *APTBucketReader {
	reader := &APTBucketReader{
		Context:           context,
		Institutions:      make(map[string]*models.Institution),
		RecentIngestItems: make(map[string]*models.WorkItem),
		statsEnabled:      enableStats,
	}
	if enableStats {
		reader.stats = stats.NewAPTBucketReaderStats()
	}
	return reader
}

func (reader *APTBucketReader) Run() error {
	err := reader.cacheInstitutions()
	if err != nil {
		return err
	}
	err = reader.cacheRecentIngestItems()
	if err != nil {
		return err
	}
	reader.readAllBuckets()
	return nil
}

// Cache a list of all institutions. There are < 20.
// Exit on failure.
func (reader *APTBucketReader) cacheInstitutions() error {
	params := url.Values{}
	params.Add("page", "1")
	params.Add("per_page", "100")
	resp := reader.Context.PharosClient.InstitutionList(params)
	if resp.Error != nil {
		if reader.stats != nil {
			reader.stats.AddError(resp.Error.Error())
		}
		return resp.Error
	}
	if resp.Response.StatusCode != 200 {
		return reader.processPharosError(resp)
	}
	for _, inst := range resp.Institutions() {
		reader.Institutions[inst.Identifier] = inst
		if reader.stats != nil {
			reader.stats.AddToInstitutionsCached(inst)
		}
	}
	reader.Context.MessageLog.Info("Loaded %d institutions", len(reader.Institutions))
	return nil
}

// Cache a list of Ingest items that have been added to
// the list of WorkItems in the past X hours, so we won't
// have to do 1000 lookups. (X is usually around 24, but
// check Config.BucketReaderCacheHours for the exact value.
// This function exits on failure.
func (reader *APTBucketReader) cacheRecentIngestItems() error {
	hours := reader.Context.Config.BucketReaderCacheHours
	if hours < 1 {
		hours = DEFAULT_CACHE_HOURS
	}
	createdAfter := time.Now().Add(time.Duration(-1*hours) * time.Hour).UTC()
	params := url.Values{}
	params.Add("page", "1")
	params.Add("per_page", "100") // change to 4 to test hasMoreResults loop
	params.Add("item_action", constants.ActionIngest)
	params.Add("created_after", createdAfter.Format(time.RFC3339))
	hasMoreResults := true
	for hasMoreResults {
		resp := reader.Context.PharosClient.WorkItemList(params)
		if resp.Error != nil {
			if reader.stats != nil {
				reader.stats.AddError(resp.Error.Error())
			}
			return resp.Error
		}
		if resp.Response.StatusCode != 200 {
			return reader.processPharosError(resp)
		}
		reader.Context.MessageLog.Debug("%s", resp.Request.URL.String())
		for _, workItem := range resp.WorkItems() {
			hashKey := reader.makeHashKey(workItem.Name, workItem.ETag,
				workItem.BagDate.Format(time.RFC3339))
			reader.RecentIngestItems[hashKey] = workItem
			if reader.stats != nil {
				reader.stats.AddWorkItem("WorkItemsCached", workItem)
			}
		}
		if resp.Next != nil {
			pageNum, err := strconv.Atoi(params.Get("page"))
			if err != nil {
				msg := fmt.Sprintf("Page number '%s' doesn't look like a number!", params.Get("page"))
				if reader.stats != nil {
					reader.stats.AddError(msg)
				}
				return fmt.Errorf(msg)
			}
			params.Set("page", strconv.Itoa(pageNum+1))
		} else {
			hasMoreResults = false
		}
	}
	reader.Context.MessageLog.Info("Loaded %d recent ingest WorkItems", len(reader.RecentIngestItems))
	return nil
}

func (reader *APTBucketReader) makeHashKey(key, etag, lastModified string) string {
	return fmt.Sprintf("%s|%s|%s", key, etag, lastModified)
}

func (reader *APTBucketReader) readAllBuckets() {
	for _, bucketName := range reader.Context.Config.ReceivingBuckets {
		reader.processBucket(bucketName)
	}
}

func (reader *APTBucketReader) processBucket(bucketName string) {
	reader.Context.MessageLog.Debug("Checking bucket %s", bucketName)
	s3ObjList := network.NewS3ObjectList(reader.Context.Config.APTrustS3Region,
		bucketName, MAX_KEYS)
	keepFetching := true
	for keepFetching {
		s3ObjList.GetList("")
		if s3ObjList.ErrorMessage != "" {
			if reader.stats != nil {
				reader.stats.AddError(s3ObjList.ErrorMessage)
			}
			reader.Context.MessageLog.Error(s3ObjList.ErrorMessage)
			break
		}
		for _, s3Object := range s3ObjList.Response.Contents {
			reader.stats.AddS3Item(fmt.Sprintf("%s/%s", bucketName, *s3Object.Key))
			reader.processS3Object(s3Object, bucketName)
		}
		keepFetching = *s3ObjList.Response.IsTruncated
	}
}

func (reader *APTBucketReader) processS3Object(s3Object *s3.Object, bucketName string) {
	if reader.Context.Config.MaxFileSize > int64(0) && *s3Object.Size > reader.Context.Config.MaxFileSize {
		msg := fmt.Sprintf("Skipping %s/%s because size %d is greater than "+
			"current max file size %d", bucketName, *s3Object.Key, *s3Object.Size,
			reader.Context.Config.MaxFileSize)
		if reader.stats != nil {
			reader.stats.AddWarning(msg)
		}
		reader.Context.MessageLog.Debug(msg)
		return
	}
	workItem, err := reader.findWorkItem(bucketName, *s3Object.Key, *s3Object.LastModified)
	if err != nil {
		// Don't create a work item, because one may already exist.
		// Error will be logged and added to stats at source.
		msg := fmt.Sprintf("Not creating WorkItem for %s/%s because "+
			"we can't tell if one already exists.", bucketName, *s3Object.Key)
		reader.Context.MessageLog.Warning(msg)
		if reader.stats != nil {
			reader.stats.AddWarning(msg)
		}
		return
	}
	if workItem == nil {
		workItem = reader.createWorkItem(bucketName, s3Object)
		if workItem == nil {
			// Error logged and statted at source.
			return
		}
	}
	if workItem.QueuedAt == nil || workItem.QueuedAt.IsZero() {
		reader.addToNSQ(workItem)
		reader.markAsQueued(workItem)
	}
}

func (reader *APTBucketReader) findWorkItem(key, etag string, lastModified time.Time) (*models.WorkItem, error) {
	hashKey := reader.makeHashKey(key, etag, lastModified.Format(time.RFC3339))
	if workItem, ok := reader.RecentIngestItems[hashKey]; ok {
		reader.Context.MessageLog.Debug("Found hash key '%s' in cache", hashKey)
		return workItem, nil
	}
	reader.Context.MessageLog.Debug("Looking up hash key '%s' in Pharos", hashKey)
	params := url.Values{}
	params.Add("page", "1")
	params.Add("per_page", "1")
	params.Add("item_action", constants.ActionIngest)
	params.Add("name", key)
	params.Add("etag", etag)
	//params.Add("bag_date", lastModified.Format(time.RFC3339))
	resp := reader.Context.PharosClient.WorkItemList(params)
	if resp.Error != nil {
		errMsg := fmt.Sprintf("Error getting WorkItem for name '%s', etag '%s', time '%s': %v",
			params.Get("name"), params.Get("etag"), params.Get("bag_date"), resp.Error)
		reader.Context.MessageLog.Debug("%s", resp.Request.URL.String())
		reader.Context.MessageLog.Error(errMsg)
		if reader.stats != nil {
			reader.stats.AddError(errMsg)
		}
		return nil, fmt.Errorf(errMsg)
	}
	if resp.Response.StatusCode != 200 {
		err := reader.processPharosError(resp)
		return nil, err
	}
	workItem := resp.WorkItem()
	if workItem != nil {
		reader.Context.MessageLog.Debug("Found WorkItem for hash key '%s' in Pharos", hashKey)
		if reader.stats != nil {
			reader.stats.AddWorkItem("WorkItemsFetched", workItem)
		}
	} else {
		reader.Context.MessageLog.Debug("Did not find WorkItem for hash key '%s' in Pharos", hashKey)
	}
	return workItem, nil
}

func (reader *APTBucketReader) createWorkItem(bucket string, s3Object *s3.Object) *models.WorkItem {
	// Create a WorkItem in Pharos
	institution := reader.Institutions[util.OwnerOf(bucket)]
	if institution == nil {
		errMsg := fmt.Sprintf("Cannot find institution record for item %s/%s. "+
			"Owner computes to '%s'", bucket, *s3Object.Key, util.OwnerOf(bucket))
		reader.Context.MessageLog.Error(errMsg)
		if reader.stats != nil {
			reader.stats.AddError(errMsg)
		}
		return nil
	}
	workItem := &models.WorkItem{}
	workItem.Name = *s3Object.Key
	workItem.Bucket = bucket
	workItem.ETag = strings.Replace(*s3Object.ETag, "\"", "", -1)
	workItem.Size = *s3Object.Size
	workItem.BagDate = *s3Object.LastModified
	workItem.InstitutionId = institution.Id
	workItem.User = constants.APTrustSystemUser
	workItem.Date = time.Now().UTC()
	workItem.Note = "Bag is in receiving bucket"
	workItem.Action = constants.ActionIngest
	workItem.Stage = constants.StageReceive
	workItem.Status = constants.StatusPending
	workItem.Outcome = "Item is pending ingest"
	workItem.Retry = true
	resp := reader.Context.PharosClient.WorkItemSave(workItem)

	if resp.Error != nil {
		errMsg := fmt.Sprintf("Error creating WorkItem for name '%s', etag '%s', time '%s': %v",
			workItem.Name, workItem.ETag, workItem.BagDate, resp.Error)
		reader.Context.MessageLog.Debug("%s", resp.Request.URL.String())
		reader.Context.MessageLog.Error(errMsg)
		if reader.stats != nil {
			reader.stats.AddError(errMsg)
		}
		return nil
	}
	if resp.Response.StatusCode != 201 {
		reader.processPharosError(resp)
		return nil
	}

	savedWorkItem := resp.WorkItem()
	reader.Context.MessageLog.Debug("Created WorkItem with id %d for %s/%s in Pharos",
		savedWorkItem.Id, bucket, *s3Object.Key)
	if reader.stats != nil {
		reader.stats.AddWorkItem("WorkItemsCreated", savedWorkItem)
	}
	return savedWorkItem
}

func (reader *APTBucketReader) addToNSQ(workItem *models.WorkItem) {
	client := network.NewNSQClient(reader.Context.Config.NsqdHttpAddress)
	err := client.Enqueue(reader.Context.Config.FetchWorker.NsqTopic, workItem.Id)
	if err != nil {
		msg := fmt.Sprintf("Error sending WorkItem %d to NSQ: %v", workItem.Id, err)
		if reader.stats != nil {
			reader.stats.AddError(msg)
		}
		reader.Context.MessageLog.Error(msg)
		return
	}
	reader.Context.MessageLog.Info("Added WorkItem id %d to NSQ (%s/%s)",
		workItem.Id, workItem.Bucket, workItem.Name)
	if reader.stats != nil {
		reader.stats.AddWorkItem("WorkItemsQueued", workItem)
	}
	return
}

func (reader *APTBucketReader) markAsQueued(workItem *models.WorkItem) *models.WorkItem {
	utcNow := time.Now().UTC()
	workItem.QueuedAt = &utcNow
	resp := reader.Context.PharosClient.WorkItemSave(workItem)
	if resp.Error != nil {
		errMsg := fmt.Sprintf("Error setting QueuedAt for WorkItem with id %d: %v",
			workItem.Id, resp.Error)
		reader.Context.MessageLog.Debug("%s", resp.Request.URL.String())
		reader.Context.MessageLog.Error(errMsg)
		if reader.stats != nil {
			reader.stats.AddError(errMsg)
		}
		return nil
	}
	if resp.Response.StatusCode != 200 {
		reader.processPharosError(resp)
		return nil
	}
	if reader.stats != nil {
		reader.stats.AddWorkItem("WorkItemsMarkedAsQueued", workItem)
	}
	return resp.WorkItem()
}

func (reader *APTBucketReader) processPharosError(resp *network.PharosResponse) error {
	respBody := ""
	bytesRead, readErr := resp.RawResponseData()
	if readErr == nil {
		respBody = string(bytesRead)
	} else {
		respBody = fmt.Sprintf("[Could not read response body: %v]", readErr)
	}
	msg := fmt.Sprintf("%s %s returned status code %d. Response body: %s",
		resp.Request.Method, resp.Request.URL, resp.Response.StatusCode, respBody)
	reader.Context.MessageLog.Error(msg)
	if reader.stats != nil {
		reader.stats.AddError(msg)
	}
	return fmt.Errorf(msg)
}

func (reader *APTBucketReader) GetStats() *stats.APTBucketReaderStats {
	return reader.stats
}
<commit_msg>Set WorkItem.Id to 0. Omit datetime in cached items hash key.<commit_after>package workers

import (
	"fmt"
	"github.com/APTrust/exchange/constants"
	"github.com/APTrust/exchange/context"
	"github.com/APTrust/exchange/models"
	"github.com/APTrust/exchange/network"
	"github.com/APTrust/exchange/stats"
	"github.com/APTrust/exchange/util"
	"github.com/aws/aws-sdk-go/service/s3"
	"net/url"
	"strconv"
	"strings"
	"time"
)

// If Config.BucketReaderCacheHours isn't set to something
// sensible, cache Ingest WorkItems up to this many hours old.
const DEFAULT_CACHE_HOURS = 24

// How many S3 keys should we fetch in each batch when
// we're getting the contents of a bucket?
const MAX_KEYS = 1000

// APTBucketReader scans APTrust receiving buckets for items
// that need to be ingested. It creates a WorkItem record and
// an NSQ entry for each qualifying bag, and is responsible for
// knowing whether items in the receiving buckets actually need
// to be queued.
type APTBucketReader struct {
	Context           *context.Context
	Institutions      map[string]*models.Institution
	RecentIngestItems map[string]*models.WorkItem
	stats             *stats.APTBucketReaderStats
	statsEnabled      bool
}

// Creates a new bucket reader with the given context.
// Param enableStats is generally used for integration tests.
// Enabling stats in production can cause high memory usage,
// so keep that off unless you're trying to diagnose specific problems.
func NewAPTBucketReader(context *context.Context, enableStats bool) *APTBucketReader {
	reader := &APTBucketReader{
		Context:           context,
		Institutions:      make(map[string]*models.Institution),
		RecentIngestItems: make(map[string]*models.WorkItem),
		statsEnabled:      enableStats,
	}
	if enableStats {
		reader.stats = stats.NewAPTBucketReaderStats()
	}
	return reader
}

func (reader *APTBucketReader) Run() error {
	err := reader.cacheInstitutions()
	if err != nil {
		return err
	}
	err = reader.cacheRecentIngestItems()
	if err != nil {
		return err
	}
	reader.readAllBuckets()
	return nil
}

// Cache a list of all institutions. There are < 20.
// Exit on failure.
func (reader *APTBucketReader) cacheInstitutions() error {
	params := url.Values{}
	params.Add("page", "1")
	params.Add("per_page", "100")
	resp := reader.Context.PharosClient.InstitutionList(params)
	if resp.Error != nil {
		if reader.stats != nil {
			reader.stats.AddError(resp.Error.Error())
		}
		return resp.Error
	}
	if resp.Response.StatusCode != 200 {
		return reader.processPharosError(resp)
	}
	for _, inst := range resp.Institutions() {
		reader.Institutions[inst.Identifier] = inst
		if reader.stats != nil {
			reader.stats.AddToInstitutionsCached(inst)
		}
	}
	reader.Context.MessageLog.Info("Loaded %d institutions", len(reader.Institutions))
	return nil
}

// Cache a list of Ingest items that have been added to
// the list of WorkItems in the past X hours, so we won't
// have to do 1000 lookups. (X is usually around 24, but
// check Config.BucketReaderCacheHours for the exact value.
// This function exits on failure.
func (reader *APTBucketReader) cacheRecentIngestItems() error {
	hours := reader.Context.Config.BucketReaderCacheHours
	if hours < 1 {
		hours = DEFAULT_CACHE_HOURS
	}
	createdAfter := time.Now().Add(time.Duration(-1*hours) * time.Hour).UTC()
	params := url.Values{}
	params.Add("page", "1")
	params.Add("per_page", "100") // change to 4 to test hasMoreResults loop
	params.Add("item_action", constants.ActionIngest)
	params.Add("created_after", createdAfter.Format(time.RFC3339))
	hasMoreResults := true
	for hasMoreResults {
		resp := reader.Context.PharosClient.WorkItemList(params)
		if resp.Error != nil {
			if reader.stats != nil {
				reader.stats.AddError(resp.Error.Error())
			}
			return resp.Error
		}
		if resp.Response.StatusCode != 200 {
			return reader.processPharosError(resp)
		}
		reader.Context.MessageLog.Debug("%s", resp.Request.URL.String())
		for _, workItem := range resp.WorkItems() {
			hashKey := reader.makeHashKey(workItem.Name, workItem.ETag)
			reader.RecentIngestItems[hashKey] = workItem
			if reader.stats != nil {
				reader.stats.AddWorkItem("WorkItemsCached", workItem)
			}
		}
		if resp.Next != nil {
			pageNum, err := strconv.Atoi(params.Get("page"))
			if err != nil {
				msg := fmt.Sprintf("Page number '%s' doesn't look like a number!", params.Get("page"))
				if reader.stats != nil {
					reader.stats.AddError(msg)
				}
				return fmt.Errorf(msg)
			}
			params.Set("page", strconv.Itoa(pageNum+1))
		} else {
			hasMoreResults = false
		}
	}
	reader.Context.MessageLog.Info("Loaded %d recent ingest WorkItems", len(reader.RecentIngestItems))
	return nil
}

func (reader *APTBucketReader) makeHashKey(key, etag string) string {
	return fmt.Sprintf("%s|%s", key, etag)
}

func (reader *APTBucketReader) readAllBuckets() {
	for _, bucketName := range reader.Context.Config.ReceivingBuckets {
		reader.processBucket(bucketName)
	}
}

func (reader *APTBucketReader) processBucket(bucketName string) {
	reader.Context.MessageLog.Debug("Checking bucket %s", bucketName)
	s3ObjList := network.NewS3ObjectList(reader.Context.Config.APTrustS3Region,
		bucketName, MAX_KEYS)
	keepFetching := true
	for keepFetching {
		s3ObjList.GetList("")
		if s3ObjList.ErrorMessage != "" {
			if reader.stats != nil {
				reader.stats.AddError(s3ObjList.ErrorMessage)
			}
			reader.Context.MessageLog.Error(s3ObjList.ErrorMessage)
			break
		}
		for _, s3Object := range s3ObjList.Response.Contents {
			reader.stats.AddS3Item(fmt.Sprintf("%s/%s", bucketName, *s3Object.Key))
			reader.processS3Object(s3Object, bucketName)
		}
		keepFetching = *s3ObjList.Response.IsTruncated
	}
}

func (reader *APTBucketReader) processS3Object(s3Object *s3.Object, bucketName string) {
	if reader.Context.Config.MaxFileSize > int64(0) && *s3Object.Size > reader.Context.Config.MaxFileSize {
		msg := fmt.Sprintf("Skipping %s/%s because size %d is greater than "+
			"current max file size %d", bucketName, *s3Object.Key, *s3Object.Size,
			reader.Context.Config.MaxFileSize)
		if reader.stats != nil {
			reader.stats.AddWarning(msg)
		}
		reader.Context.MessageLog.Debug(msg)
		return
	}
	workItem, err := reader.findWorkItem(bucketName, *s3Object.Key)
	if err != nil {
		// Don't create a work item, because one may already exist.
		// Error will be logged and added to stats at source.
		msg := fmt.Sprintf("Not creating WorkItem for %s/%s because "+
			"we can't tell if one already exists.", bucketName, *s3Object.Key)
		reader.Context.MessageLog.Warning(msg)
		if reader.stats != nil {
			reader.stats.AddWarning(msg)
		}
		return
	}
	if workItem == nil {
		workItem = reader.createWorkItem(bucketName, s3Object)
		if workItem == nil {
			// Error logged and statted at source.
			return
		}
	}
	if workItem.QueuedAt == nil || workItem.QueuedAt.IsZero() {
		reader.addToNSQ(workItem)
		reader.markAsQueued(workItem)
	}
}

func (reader *APTBucketReader) findWorkItem(key, etag string) (*models.WorkItem, error) {
	hashKey := reader.makeHashKey(key, etag)
	if workItem, ok := reader.RecentIngestItems[hashKey]; ok {
		reader.Context.MessageLog.Debug("Found hash key '%s' in cache", hashKey)
		return workItem, nil
	}
	reader.Context.MessageLog.Debug("Looking up hash key '%s' in Pharos", hashKey)
	params := url.Values{}
	params.Add("page", "1")
	params.Add("per_page", "1")
	params.Add("item_action", constants.ActionIngest)
	params.Add("name", key)
	params.Add("etag", etag)
	//params.Add("bag_date", lastModified.Format(time.RFC3339))
	resp := reader.Context.PharosClient.WorkItemList(params)
	if resp.Error != nil {
		errMsg := fmt.Sprintf("Error getting WorkItem for name '%s', etag '%s', time '%s': %v",
			params.Get("name"), params.Get("etag"), params.Get("bag_date"), resp.Error)
		reader.Context.MessageLog.Debug("%s", resp.Request.URL.String())
		reader.Context.MessageLog.Error(errMsg)
		if reader.stats != nil {
			reader.stats.AddError(errMsg)
		}
		return nil, fmt.Errorf(errMsg)
	}
	if resp.Response.StatusCode != 200 {
		err := reader.processPharosError(resp)
		return nil, err
	}
	workItem := resp.WorkItem()
	if workItem != nil {
		reader.Context.MessageLog.Debug("Found WorkItem for hash key '%s' in Pharos", hashKey)
		if reader.stats != nil {
			reader.stats.AddWorkItem("WorkItemsFetched", workItem)
		}
	} else {
		reader.Context.MessageLog.Debug("Did not find WorkItem for hash key '%s' in Pharos", hashKey)
	}
	return workItem, nil
}

func (reader *APTBucketReader) createWorkItem(bucket string, s3Object *s3.Object) *models.WorkItem {
	// Create a WorkItem in Pharos
	institution := reader.Institutions[util.OwnerOf(bucket)]
	if institution == nil {
		errMsg := fmt.Sprintf("Cannot find institution record for item %s/%s. "+
			"Owner computes to '%s'", bucket, *s3Object.Key, util.OwnerOf(bucket))
		reader.Context.MessageLog.Error(errMsg)
		if reader.stats != nil {
			reader.stats.AddError(errMsg)
		}
		return nil
	}
	workItem := &models.WorkItem{}
	workItem.Id = 0
	workItem.Name = *s3Object.Key
	workItem.Bucket = bucket
	workItem.ETag = strings.Replace(*s3Object.ETag, "\"", "", -1)
	workItem.Size = *s3Object.Size
	workItem.BagDate = *s3Object.LastModified
	workItem.InstitutionId = institution.Id
	workItem.User = constants.APTrustSystemUser
	workItem.Date = time.Now().UTC()
	workItem.Note = "Bag is in receiving bucket"
	workItem.Action = constants.ActionIngest
	workItem.Stage = constants.StageReceive
	workItem.Status = constants.StatusPending
	workItem.Outcome = "Item is pending ingest"
	workItem.Retry = true
	resp := reader.Context.PharosClient.WorkItemSave(workItem)

	if resp.Error != nil {
		errMsg := fmt.Sprintf("Error creating WorkItem for name '%s', etag '%s', time '%s': %v",
			workItem.Name, workItem.ETag, workItem.BagDate, resp.Error)
		reader.Context.MessageLog.Debug("%s", resp.Request.URL.String())
		reader.Context.MessageLog.Error(errMsg)
		if reader.stats != nil {
			reader.stats.AddError(errMsg)
		}
		return nil
	}
	if resp.Response.StatusCode != 201 {
		reader.processPharosError(resp)
		return nil
	}

	savedWorkItem := resp.WorkItem()
	reader.Context.MessageLog.Debug("Created WorkItem with id %d for %s/%s in Pharos",
		savedWorkItem.Id, bucket, *s3Object.Key)
	if reader.stats != nil {
		reader.stats.AddWorkItem("WorkItemsCreated", savedWorkItem)
	}
	return savedWorkItem
}

func (reader *APTBucketReader) addToNSQ(workItem *models.WorkItem) {
	client := network.NewNSQClient(reader.Context.Config.NsqdHttpAddress)
	err := client.Enqueue(reader.Context.Config.FetchWorker.NsqTopic, workItem.Id)
	if err != nil {
		msg := fmt.Sprintf("Error sending WorkItem %d to NSQ: %v", workItem.Id, err)
		if reader.stats != nil {
			reader.stats.AddError(msg)
		}
		reader.Context.MessageLog.Error(msg)
		return
	}
	reader.Context.MessageLog.Info("Added WorkItem id %d to NSQ (%s/%s)",
		workItem.Id, workItem.Bucket, workItem.Name)
	if reader.stats != nil {
		reader.stats.AddWorkItem("WorkItemsQueued", workItem)
	}
	return
}

func (reader *APTBucketReader) markAsQueued(workItem *models.WorkItem) *models.WorkItem {
	utcNow := time.Now().UTC()
	workItem.QueuedAt = &utcNow
	resp := reader.Context.PharosClient.WorkItemSave(workItem)
	if resp.Error != nil {
		errMsg := fmt.Sprintf("Error setting QueuedAt for WorkItem with id %d: %v",
			workItem.Id, resp.Error)
		reader.Context.MessageLog.Debug("%s", resp.Request.URL.String())
		reader.Context.MessageLog.Error(errMsg)
		if reader.stats != nil {
			reader.stats.AddError(errMsg)
		}
		return nil
	}
	if resp.Response.StatusCode != 200 {
		reader.processPharosError(resp)
		return nil
	}
	if reader.stats != nil {
		reader.stats.AddWorkItem("WorkItemsMarkedAsQueued", workItem)
	}
	return resp.WorkItem()
}

func (reader *APTBucketReader) processPharosError(resp *network.PharosResponse) error {
	respBody := ""
	bytesRead, readErr := resp.RawResponseData()
	if readErr == nil {
		respBody = string(bytesRead)
	} else {
		respBody = fmt.Sprintf("[Could not read response body: %v]", readErr)
	}
	msg := fmt.Sprintf("%s %s returned status code %d. Response body: %s",
		resp.Request.Method, resp.Request.URL, resp.Response.StatusCode, respBody)
	reader.Context.MessageLog.Error(msg)
	if reader.stats != nil {
		reader.stats.AddError(msg)
	}
	return fmt.Errorf(msg)
}

func (reader *APTBucketReader) GetStats() *stats.APTBucketReaderStats {
	return reader.stats
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"os"
	"os/signal"
	"path/filepath"
	"runtime"
	"sync"
	"syscall"
	"text/tabwriter"
	"time"

	"github.com/voxelbrain/goptions"
)

const (
	VERSION = "1.2.0"
)

var (
	DefaultBlocksize = 4 * KiloByte
	options          = struct {
		Cores     int           `goptions:"-c, --cores, description='Number of cores to use'"`
		Threads   int           `goptions:"-t, --threads, description='Number of threads to use'"`
		Blocksize *Datasize     `goptions:"-b, --block-size, description='Number of bytes to write with each call'"`
		Filesize  *Datasize     `goptions:"-f, --file-size, obligatory, description='Number of zeroes to write to each file'"`
		OutputDir string        `goptions:"-o, --output-dir, description='Output directory'"`
		Sync      bool          `goptions:"-s, --sync, description='Sync after every written block'"`
		KeepFiles bool          `goptions:"-k, --keep-files, description='Dont delete files when done'"`
		Help      goptions.Help `goptions:"-h, --help, description='Show this help'"`
	}{
		Cores:     runtime.NumCPU(),
		Threads:   runtime.NumCPU(),
		Blocksize: &DefaultBlocksize,
		OutputDir: os.TempDir(),
	}
	Timestamp = fmt.Sprintf("%d", time.Now().UnixNano())
)

func init() {
	goptions.ParseAndFail(&options)
	options.OutputDir = filepath.Clean(options.OutputDir + "/iowhip_" + Timestamp)
}

type Result struct {
	Index       int
	TotalTime   time.Duration
	WritingTime time.Duration
}

func main() {
	err := os.MkdirAll(options.OutputDir, os.FileMode(0755))
	if err != nil {
		log.Fatalf("Could not create output directory %s: %s", options.OutputDir, err)
	}
	cleanup := make(chan os.Signal)
	signal.Notify(cleanup, syscall.SIGTERM, syscall.SIGKILL, syscall.SIGINT)
	go func() {
		<-cleanup
		if !options.KeepFiles {
			os.RemoveAll(options.OutputDir)
		}
	}()

	runtime.GOMAXPROCS(options.Cores)
	log.Printf("Starting %d workers on %d cores writing %s bytes (%s per call)...", options.Threads, options.Cores, options.Filesize, options.Blocksize)

	results := make(chan Result)
	wg := &sync.WaitGroup{}
	wg.Add(options.Threads)
	for i := 0; i < options.Threads; i++ {
		go writeFile(i, results, wg)
	}
	wg.Wait()

	prettyPrint(results)
	cleanup <- syscall.SIGINT
}

func prettyPrint(results <-chan Result) {
	w := tabwriter.NewWriter(os.Stdout, 3, 1, 2, ' ', tabwriter.AlignRight)
	defer w.Flush()
	fmt.Fprintf(w, "\nThread\tTotal Time\tTotal Speed\tWriting Time\tWriting Speed\t\n")
	for i := 0; i < options.Threads; i++ {
		r := <-results
		fmt.Fprintf(w, "%d\t%s\t%s/s\t%s\t%s/s\t\n", r.Index,
			r.TotalTime, Datasize(int64(float64(*options.Filesize)*float64(time.Second)/float64(r.TotalTime))),
			r.WritingTime, Datasize(int64(float64(*options.Filesize)*float64(time.Second)/float64(r.WritingTime))))
	}
}

func writeFile(idx int, c chan Result, wg *sync.WaitGroup) {
	result := Result{
		Index: idx,
	}
	defer func() { c <- result }()
	defer wg.Done()
	filename := fmt.Sprintf("%s/%d", options.OutputDir, idx)
	log.Printf("Thread %d: Using %s", idx, filename)

	amount := *options.Filesize
	data := make([]byte, int(*options.Blocksize))
	start_creating := time.Now()
	f, err := os.Create(filename)
	if err != nil {
		log.Printf("Thread %d: Could not open file %s: %s", idx, filename, err)
		return
	}
	defer f.Close()

	start_writing := time.Now()
	for amount > 0 {
		n, err := f.Write(data)
		if err != nil {
			log.Printf("Thread %d: Write to %s failed: %s", idx, filename, err)
			return
		}
		amount -= Datasize(n)
		if options.Sync {
			f.Sync()
		}
	}
	f.Sync()
	result.WritingTime = time.Since(start_writing)
	f.Close()
	result.TotalTime = time.Since(start_creating)
}
<commit_msg>Fix cleanup<commit_after>package main

import (
	"fmt"
	"log"
	"os"
	"os/signal"
	"path/filepath"
	"runtime"
	"sync"
	"syscall"
	"text/tabwriter"
	"time"

	"github.com/voxelbrain/goptions"
)

const (
	VERSION = "1.2.0"
)

var (
	DefaultBlocksize = 4 * KiloByte
	options          = struct {
		Cores     int           `goptions:"-c, --cores, description='Number of cores to use'"`
		Threads   int           `goptions:"-t, --threads, description='Number of threads to use'"`
		Blocksize *Datasize     `goptions:"-b, --block-size, description='Number of bytes to write with each call'"`
		Filesize  *Datasize     `goptions:"-f, --file-size, obligatory, description='Number of zeroes to write to each file'"`
		OutputDir string        `goptions:"-o, --output-dir, description='Output directory'"`
		Sync      bool          `goptions:"-s, --sync, description='Sync after every written block'"`
		KeepFiles bool          `goptions:"-k, --keep-files, description='Dont delete files when done'"`
		Help      goptions.Help `goptions:"-h, --help, description='Show this help'"`
	}{
		Cores:     runtime.NumCPU(),
		Threads:   runtime.NumCPU(),
		Blocksize: &DefaultBlocksize,
		OutputDir: os.TempDir(),
	}
	Timestamp = fmt.Sprintf("%d", time.Now().UnixNano())
)

func init() {
	goptions.ParseAndFail(&options)
	options.OutputDir = filepath.Clean(options.OutputDir + "/iowhip_" + Timestamp)

	cleanupc := make(chan os.Signal)
	signal.Notify(cleanupc, syscall.SIGTERM, syscall.SIGKILL, syscall.SIGINT)
	go func() {
		<-cleanupc
		cleanup()
	}()
}

type Result struct {
	Index       int
	TotalTime   time.Duration
	WritingTime time.Duration
}

func cleanup() {
	if !options.KeepFiles {
		os.RemoveAll(options.OutputDir)
	}
}

func main() {
	err := os.MkdirAll(options.OutputDir, os.FileMode(0755))
	if err != nil {
		log.Fatalf("Could not create output directory %s: %s", options.OutputDir, err)
	}

	runtime.GOMAXPROCS(options.Cores)
	log.Printf("Starting %d workers on %d cores writing %s bytes (%s per call)...", options.Threads, options.Cores, options.Filesize, options.Blocksize)

	results := make(chan Result)
	wg := &sync.WaitGroup{}
	wg.Add(options.Threads)
	for i := 0; i < options.Threads; i++ {
		go writeFile(i, results, wg)
	}
	wg.Wait()

	prettyPrint(results)
	cleanup()
}

func prettyPrint(results <-chan Result) {
	w := tabwriter.NewWriter(os.Stdout, 3, 1, 2, ' ', tabwriter.AlignRight)
	defer w.Flush()
	fmt.Fprintf(w, "\nThread\tTotal Time\tTotal Speed\tWriting Time\tWriting Speed\t\n")
	for i := 0; i < options.Threads; i++ {
		r := <-results
		fmt.Fprintf(w, "%d\t%s\t%s/s\t%s\t%s/s\t\n", r.Index,
			r.TotalTime, Datasize(int64(float64(*options.Filesize)*float64(time.Second)/float64(r.TotalTime))),
			r.WritingTime, Datasize(int64(float64(*options.Filesize)*float64(time.Second)/float64(r.WritingTime))))
	}
}

func writeFile(idx int, c chan Result, wg *sync.WaitGroup) {
	result := Result{
		Index: idx,
	}
	defer func() { c <- result }()
	defer wg.Done()
	filename := fmt.Sprintf("%s/%d", options.OutputDir, idx)
	log.Printf("Thread %d: Using %s", idx, filename)

	amount := *options.Filesize
	data := make([]byte, int(*options.Blocksize))
	start_creating := time.Now()
	f, err := os.Create(filename)
	if err != nil {
		log.Printf("Thread %d: Could not open file %s: %s", idx, filename, err)
		return
	}
	defer f.Close()

	start_writing := time.Now()
	for amount > 0 {
		n, err := f.Write(data)
		if err != nil {
			log.Printf("Thread %d: Write to %s failed: %s", idx, filename, err)
			return
		}
		amount -= Datasize(n)
		if options.Sync {
			f.Sync()
		}
	}
	f.Sync()
	result.WritingTime = time.Since(start_writing)
	f.Close()
	result.TotalTime = time.Since(start_creating)
}
<|endoftext|>


<commit_before>package server

import (
	"fmt"
	"regexp"
	"sort"
	"sync"

	"github.com/blang/semver"
	"github.com/google/go-github/github"
)

var (
	updateAssetRe = regexp.MustCompile(`^update_(darwin|windows|linux)_(arm|386|amd64)\.?.*$`)

	emptyVersion semver.Version
)

// Arch holds architecture names.
var Arch = struct {
	X64 string
	X86 string
	ARM string
}{
	"amd64",
	"386",
	"arm",
}

// OS holds operating system names.
var OS = struct {
	Windows string
	Linux   string
	Darwin  string
}{
	"windows",
	"linux",
	"darwin",
}

// Release struct represents a single github release.
type Release struct {
	id      int
	URL     string
	Version semver.Version
	Assets  []Asset
}

type releasesByID []Release

// Asset struct represents a file included as part of a Release.
type Asset struct {
	id        int
	v         semver.Version
	Name      string
	URL       string
	LocalFile string
	Checksum  string
	Signature string
	AssetInfo
}

// AssetInfo struct holds OS and Arch information of an asset.
type AssetInfo struct {
	OS   string
	Arch string
}

// ReleaseManager struct defines a repository to pull releases from.
type ReleaseManager struct {
	client          *github.Client
	owner           string
	repo            string
	updateAssetsMap map[string]map[string]map[string]*Asset
	latestAssetsMap map[string]map[string]*Asset
	mu              *sync.RWMutex
}

func (a releasesByID) Len() int {
	return len(a)
}

func (a releasesByID) Swap(i, j int) {
	a[i], a[j] = a[j], a[i]
}

func (a releasesByID) Less(i, j int) bool {
	return a[i].id < a[j].id
}

// NewReleaseManager creates a wrapper of github.Client.
func NewReleaseManager(owner string, repo string) *ReleaseManager {

	ghc := &ReleaseManager{
		client:          github.NewClient(nil),
		owner:           owner,
		repo:            repo,
		mu:              new(sync.RWMutex),
		updateAssetsMap: make(map[string]map[string]map[string]*Asset),
		latestAssetsMap: make(map[string]map[string]*Asset),
	}

	return ghc
}

// getReleases queries github for all product releases.
func (g *ReleaseManager) getReleases() ([]Release, error) {
	var releases []Release

	for page := 1; true; page++ {
		opt := &github.ListOptions{Page: page}

		rels, _, err := g.client.Repositories.ListReleases(g.owner, g.repo, opt)

		if err != nil {
			return nil, err
		}

		if len(rels) == 0 {
			break
		}

		releases = make([]Release, 0, len(rels))

		for i := range rels {
			version := *rels[i].TagName
			v, err := semver.Parse(version)
			if err != nil {
				log.Debugf("Release %q is not semantically versioned (%q). Skipping.", version, err)
				continue
			}
			rel := Release{
				id:      *rels[i].ID,
				URL:     *rels[i].ZipballURL,
				Version: v,
			}
			rel.Assets = make([]Asset, 0, len(rels[i].Assets))
			for _, asset := range rels[i].Assets {
				rel.Assets = append(rel.Assets, Asset{
					id:   *asset.ID,
					Name: *asset.Name,
					URL:  *asset.BrowserDownloadURL,
				})
			}
			log.Debugf("Release %q has %d assets...", version, len(rel.Assets))
			releases = append(releases, rel)
		}
	}

	sort.Sort(sort.Reverse(releasesByID(releases)))

	return releases, nil
}

// UpdateAssetsMap will pull published releases, scan for compatible
// update-only binaries and will add them to the updateAssetsMap.
func (g *ReleaseManager) UpdateAssetsMap() (err error) {

	var rs []Release

	log.Debugf("Getting releases...")
	if rs, err = g.getReleases(); err != nil {
		return err
	}

	log.Debugf("Getting assets...")
	for i := range rs {
		log.Debugf("Getting assets for release %q...", rs[i].Version)
		for j := range rs[i].Assets {
			log.Debugf("Found %q.", rs[i].Assets[j].Name)
			// Does this asset represent a binary update?
			if isUpdateAsset(rs[i].Assets[j].Name) {
				log.Debugf("%q is an auto-update asset.", rs[i].Assets[j].Name)
				asset := rs[i].Assets[j]
				asset.v = rs[i].Version
				info, err := getAssetInfo(asset.Name)
				if err != nil {
					return fmt.Errorf("Could not get asset info: %q", err)
				}
				if err = g.pushAsset(info.OS, info.Arch, &asset); err != nil {
					return fmt.Errorf("Could not push asset: %q", err)
				}
			} else {
				log.Debugf("%q is not an auto-update asset. Skipping.", rs[i].Assets[j].Name)
			}
		}
	}

	return nil
}

func (g *ReleaseManager) getProductUpdate(os string, arch string) (asset *Asset, err error) {
	g.mu.RLock()
	defer g.mu.RUnlock()

	if g.latestAssetsMap == nil {
		return nil, fmt.Errorf("No updates available.")
	}

	if g.latestAssetsMap[os] == nil {
		return nil, fmt.Errorf("No such OS.")
	}

	if g.latestAssetsMap[os][arch] == nil {
		return nil, fmt.Errorf("No such Arch.")
	}

	return g.latestAssetsMap[os][arch], nil
}

func (g *ReleaseManager) lookupAssetWithChecksum(os string, arch string, checksum string) (asset *Asset, err error) {
	g.mu.RLock()
	defer g.mu.RUnlock()

	if g.updateAssetsMap == nil {
		return nil, fmt.Errorf("No updates available.")
	}

	if g.updateAssetsMap[os] == nil {
		return nil, fmt.Errorf("No such OS.")
	}

	if g.updateAssetsMap[os][arch] == nil {
		return nil, fmt.Errorf("No such Arch.")
	}

	for _, a := range g.updateAssetsMap[os][arch] {
		if a.Checksum == checksum {
			return a, nil
		}
	}

	return nil, fmt.Errorf("Could not find a matching checksum in assets list.")
}

func (g *ReleaseManager) lookupAssetWithVersion(os string, arch string, version string) (asset *Asset, err error) {
	g.mu.RLock()
	defer g.mu.RUnlock()

	if g.updateAssetsMap == nil {
		return nil, fmt.Errorf("No updates available.")
	}

	if g.updateAssetsMap[os] == nil {
		return nil, fmt.Errorf("No such OS.")
	}

	if g.updateAssetsMap[os][arch] == nil {
		return nil, fmt.Errorf("No such Arch.")
	}

	for _, a := range g.updateAssetsMap[os][arch] {
		if a.v.String() == version {
			return a, nil
		}
	}

	return nil, fmt.Errorf("Could not find a matching version in assets list.")
}

func (g *ReleaseManager) pushAsset(os string, arch string, asset *Asset) (err error) {
	g.mu.Lock()
	defer g.mu.Unlock()

	version := asset.v

	asset.OS = os
	asset.Arch = arch

	if version.EQ(emptyVersion) {
		return fmt.Errorf("Missing asset version.")
	}

	var localfile string
	if localfile, err = downloadAsset(asset.URL); err != nil {
		return err
	}

	if asset.Checksum, err = checksumForFile(localfile); err != nil {
		return err
	}

	if asset.Signature, err = signatureForFile(localfile); err != nil {
		return err
	}

	// Pushing version.
	if g.updateAssetsMap[os] == nil {
		g.updateAssetsMap[os] = make(map[string]map[string]*Asset)
	}
	if g.updateAssetsMap[os][arch] == nil {
		g.updateAssetsMap[os][arch] = make(map[string]*Asset)
	}
	g.updateAssetsMap[os][arch][version.String()] = asset

	// Setting latest version.
	if g.latestAssetsMap[os] == nil {
		g.latestAssetsMap[os] = make(map[string]*Asset)
	}
	if g.latestAssetsMap[os][arch] == nil {
		g.latestAssetsMap[os][arch] = asset
	} else {
		// Compare against already set version. Manoto versions can't be on top.
		if asset.v.GT(g.latestAssetsMap[os][arch].v) && !buildStringContainsManoto(asset.v) {
			g.latestAssetsMap[os][arch] = asset
		}
	}

	return nil
}

func getAssetInfo(s string) (*AssetInfo, error) {
	matches := updateAssetRe.FindStringSubmatch(s)
	if len(matches) >= 3 {
		if matches[1] != OS.Windows && matches[1] != OS.Linux && matches[1] != OS.Darwin {
			return nil, fmt.Errorf("Unknown OS: \"%s\".", matches[1])
		}
		if matches[2] != Arch.X64 && matches[2] != Arch.X86 && matches[2] != Arch.ARM {
			return nil, fmt.Errorf("Unknown architecture \"%s\".", matches[2])
		}
		info := &AssetInfo{
			OS:   matches[1],
			Arch: matches[2],
		}
		return info, nil
	}
	return nil, fmt.Errorf("Could not find asset info.")
}

func isUpdateAsset(s string) bool {
	return updateAssetRe.MatchString(s)
}
<commit_msg>Making sure manoto build won't make it to the latestAssetsMap map.<commit_after>package server

import (
	"fmt"
	"regexp"
	"sort"
	"sync"

	"github.com/blang/semver"
	"github.com/google/go-github/github"
)

var (
	updateAssetRe = regexp.MustCompile(`^update_(darwin|windows|linux)_(arm|386|amd64)\.?.*$`)

	emptyVersion semver.Version
)

// Arch holds architecture names.
var Arch = struct {
	X64 string
	X86 string
	ARM string
}{
	"amd64",
	"386",
	"arm",
}

// OS holds operating system names.
var OS = struct {
	Windows string
	Linux   string
	Darwin  string
}{
	"windows",
	"linux",
	"darwin",
}

// Release struct represents a single github release.
type Release struct {
	id      int
	URL     string
	Version semver.Version
	Assets  []Asset
}

type releasesByID []Release

// Asset struct represents a file included as part of a Release.
type Asset struct {
	id        int
	v         semver.Version
	Name      string
	URL       string
	LocalFile string
	Checksum  string
	Signature string
	AssetInfo
}

// AssetInfo struct holds OS and Arch information of an asset.
type AssetInfo struct {
	OS   string
	Arch string
}

// ReleaseManager struct defines a repository to pull releases from.
type ReleaseManager struct {
	client          *github.Client
	owner           string
	repo            string
	updateAssetsMap map[string]map[string]map[string]*Asset
	latestAssetsMap map[string]map[string]*Asset
	mu              *sync.RWMutex
}

func (a releasesByID) Len() int {
	return len(a)
}

func (a releasesByID) Swap(i, j int) {
	a[i], a[j] = a[j], a[i]
}

func (a releasesByID) Less(i, j int) bool {
	return a[i].id < a[j].id
}

// NewReleaseManager creates a wrapper of github.Client.
func NewReleaseManager(owner string, repo string) *ReleaseManager {

	ghc := &ReleaseManager{
		client:          github.NewClient(nil),
		owner:           owner,
		repo:            repo,
		mu:              new(sync.RWMutex),
		updateAssetsMap: make(map[string]map[string]map[string]*Asset),
		latestAssetsMap: make(map[string]map[string]*Asset),
	}

	return ghc
}

// getReleases queries github for all product releases.
func (g *ReleaseManager) getReleases() ([]Release, error) {
	var releases []Release

	for page := 1; true; page++ {
		opt := &github.ListOptions{Page: page}

		rels, _, err := g.client.Repositories.ListReleases(g.owner, g.repo, opt)

		if err != nil {
			return nil, err
		}

		if len(rels) == 0 {
			break
		}

		releases = make([]Release, 0, len(rels))

		for i := range rels {
			version := *rels[i].TagName
			v, err := semver.Parse(version)
			if err != nil {
				log.Debugf("Release %q is not semantically versioned (%q). Skipping.", version, err)
				continue
			}
			rel := Release{
				id:      *rels[i].ID,
				URL:     *rels[i].ZipballURL,
				Version: v,
			}
			rel.Assets = make([]Asset, 0, len(rels[i].Assets))
			for _, asset := range rels[i].Assets {
				rel.Assets = append(rel.Assets, Asset{
					id:   *asset.ID,
					Name: *asset.Name,
					URL:  *asset.BrowserDownloadURL,
				})
			}
			log.Debugf("Release %q has %d assets...", version, len(rel.Assets))
			releases = append(releases, rel)
		}
	}

	sort.Sort(sort.Reverse(releasesByID(releases)))

	return releases, nil
}

// UpdateAssetsMap will pull published releases, scan for compatible
// update-only binaries and will add them to the updateAssetsMap.
func (g *ReleaseManager) UpdateAssetsMap() (err error) {

	var rs []Release

	log.Debugf("Getting releases...")
	if rs, err = g.getReleases(); err != nil {
		return err
	}

	log.Debugf("Getting assets...")
	for i := range rs {
		log.Debugf("Getting assets for release %q...", rs[i].Version)
		for j := range rs[i].Assets {
			log.Debugf("Found %q.", rs[i].Assets[j].Name)
			// Does this asset represent a binary update?
			if isUpdateAsset(rs[i].Assets[j].Name) {
				log.Debugf("%q is an auto-update asset.", rs[i].Assets[j].Name)
				asset := rs[i].Assets[j]
				asset.v = rs[i].Version
				info, err := getAssetInfo(asset.Name)
				if err != nil {
					return fmt.Errorf("Could not get asset info: %q", err)
				}
				if err = g.pushAsset(info.OS, info.Arch, &asset); err != nil {
					return fmt.Errorf("Could not push asset: %q", err)
				}
			} else {
				log.Debugf("%q is not an auto-update asset. Skipping.", rs[i].Assets[j].Name)
			}
		}
	}

	return nil
}

func (g *ReleaseManager) getProductUpdate(os string, arch string) (asset *Asset, err error) {
	g.mu.RLock()
	defer g.mu.RUnlock()

	if g.latestAssetsMap == nil {
		return nil, fmt.Errorf("No updates available.")
	}

	if g.latestAssetsMap[os] == nil {
		return nil, fmt.Errorf("No such OS.")
	}

	if g.latestAssetsMap[os][arch] == nil {
		return nil, fmt.Errorf("No such Arch.")
	}

	return g.latestAssetsMap[os][arch], nil
}

func (g *ReleaseManager) lookupAssetWithChecksum(os string, arch string, checksum string) (asset *Asset, err error) {
	g.mu.RLock()
	defer g.mu.RUnlock()

	if g.updateAssetsMap == nil {
		return nil, fmt.Errorf("No updates available.")
	}

	if g.updateAssetsMap[os] == nil {
		return nil, fmt.Errorf("No such OS.")
	}

	if g.updateAssetsMap[os][arch] == nil {
		return nil, fmt.Errorf("No such Arch.")
	}

	for _, a := range g.updateAssetsMap[os][arch] {
		if a.Checksum == checksum {
			return a, nil
		}
	}

	return nil, fmt.Errorf("Could not find a matching checksum in assets list.")
}

func (g *ReleaseManager) lookupAssetWithVersion(os string, arch string, version string) (asset *Asset, err error) {
	g.mu.RLock()
	defer g.mu.RUnlock()

	if g.updateAssetsMap == nil {
		return nil, fmt.Errorf("No updates available.")
	}

	if g.updateAssetsMap[os] == nil {
		return nil, fmt.Errorf("No such OS.")
	}

	if g.updateAssetsMap[os][arch] == nil {
		return nil, fmt.Errorf("No such Arch.")
	}

	for _, a := range g.updateAssetsMap[os][arch] {
		if a.v.String() == version {
			return a, nil
		}
	}

	return nil, fmt.Errorf("Could not find a matching version in assets list.")
}

func (g *ReleaseManager) pushAsset(os string, arch string, asset *Asset) (err error) {
	g.mu.Lock()
	defer g.mu.Unlock()

	version := asset.v

	asset.OS = os
	asset.Arch = arch

	if version.EQ(emptyVersion) {
		return fmt.Errorf("Missing asset version.")
	}

	var localfile string
	if localfile, err = downloadAsset(asset.URL); err != nil {
		return err
	}

	if asset.Checksum, err = checksumForFile(localfile); err != nil {
		return err
	}

	if asset.Signature, err = signatureForFile(localfile); err != nil {
		return err
	}

	// Pushing version.
	if g.updateAssetsMap[os] == nil {
		g.updateAssetsMap[os] = make(map[string]map[string]*Asset)
	}
	if g.updateAssetsMap[os][arch] == nil {
		g.updateAssetsMap[os][arch] = make(map[string]*Asset)
	}
	g.updateAssetsMap[os][arch][version.String()] = asset

	// Setting latest version.
	if g.latestAssetsMap[os] == nil {
		g.latestAssetsMap[os] = make(map[string]*Asset)
	}

	// Only considering non-manoto versions for the latestAssetsMap
	if !buildStringContainsManoto(asset.v) {
		if g.latestAssetsMap[os][arch] == nil {
			g.latestAssetsMap[os][arch] = asset
		} else {
			// Compare against already set version.
			if asset.v.GT(g.latestAssetsMap[os][arch].v) {
				g.latestAssetsMap[os][arch] = asset
			}
		}
	}

	return nil
}

func getAssetInfo(s string) (*AssetInfo, error) {
	matches := updateAssetRe.FindStringSubmatch(s)
	if len(matches) >= 3 {
		if matches[1] != OS.Windows && matches[1] != OS.Linux && matches[1] != OS.Darwin {
			return nil, fmt.Errorf("Unknown OS: \"%s\".", matches[1])
		}
		if matches[2] != Arch.X64 && matches[2] != Arch.X86 && matches[2] != Arch.ARM {
			return nil, fmt.Errorf("Unknown architecture \"%s\".", matches[2])
		}
		info := &AssetInfo{
			OS:   matches[1],
			Arch: matches[2],
		}
		return info, nil
	}
	return nil, fmt.Errorf("Could not find asset info.")
}

func isUpdateAsset(s string) bool {
	return updateAssetRe.MatchString(s)
}
<|endoftext|>


<commit_before>package ecsbackend

import (
	"testing"

	"github.com/golang/mock/gomock"
	"github.com/quintilesims/layer0/common/aws/cloudwatchlogs"
	"github.com/quintilesims/layer0/common/aws/cloudwatchlogs/mock_cloudwatchlogs"
	"github.com/quintilesims/layer0/common/config"
	"github.com/quintilesims/layer0/common/models"
	"github.com/quintilesims/layer0/common/testutils"
)

func testCreateDeploy(models.CreateDeployRequest) (*models.Deploy, error) {
	model := &models.Deploy{
		DeployID: "renderedid.2",
		Version:  "2",
	}

	return model, nil
}

func TestGetLogs(t *testing.T) {
	taskARN := "arn:aws:ecs:region:aws_account_id:task/taskARN"

	testCases := []testutils.TestCase{
		{
			Name: "Should add proper logs lines to models",
			Setup: func(reporter *testutils.Reporter, ctrl *gomock.Controller) interface{} {
				mockCW := mock_cloudwatchlogs.NewMockProvider(ctrl)

				stream := cloudwatchlogs.NewLogStream("prefix/container_name/taskARN")

				mockCW.EXPECT().
					DescribeLogStreams(config.AWSLogGroupID(), "LogStreamName").
					Return([]*cloudwatchlogs.LogStream{stream}, nil)

				event := cloudwatchlogs.NewOutputLogEvent("some_message")

				mockCW.EXPECT().
					GetLogEvents(
						config.AWSLogGroupID(),
						*stream.LogStreamName,
						int64(30),
					).Return([]*cloudwatchlogs.OutputLogEvent{event}, nil)

				return mockCW
			},
			Run: func(reporter *testutils.Reporter, target interface{}) {
				provider := target.(cloudwatchlogs.Provider)

				logs, err := GetLogs(provider, []*string{stringp(taskARN)}, 30)
				if err != nil {
					reporter.Fatal(err)
				}

				testutils.AssertEqual(t, len(logs), 1)
				testutils.AssertInSlice(t, "some_message", logs[0].Lines)
			},
		},
	}

	testutils.RunTests(t, testCases)
}
<commit_msg>Updated test TestGetLogs to order by LastEventTime<commit_after>package ecsbackend

import (
	"testing"

	"github.com/golang/mock/gomock"
	"github.com/quintilesims/layer0/common/aws/cloudwatchlogs"
	"github.com/quintilesims/layer0/common/aws/cloudwatchlogs/mock_cloudwatchlogs"
	"github.com/quintilesims/layer0/common/config"
	"github.com/quintilesims/layer0/common/models"
	"github.com/quintilesims/layer0/common/testutils"
)

func testCreateDeploy(models.CreateDeployRequest) (*models.Deploy, error) {
	model := &models.Deploy{
		DeployID: "renderedid.2",
		Version:  "2",
	}

	return model, nil
}

func TestGetLogs(t *testing.T) {
	taskARN := "arn:aws:ecs:region:aws_account_id:task/taskARN"

	testCases := []testutils.TestCase{
		{
			Name: "Should add proper logs lines to models",
			Setup: func(reporter *testutils.Reporter, ctrl *gomock.Controller) interface{} {
				mockCW := mock_cloudwatchlogs.NewMockProvider(ctrl)

				stream := cloudwatchlogs.NewLogStream("prefix/container_name/taskARN")

				mockCW.EXPECT().
					DescribeLogStreams(config.AWSLogGroupID(), "LastEventTime").
					Return([]*cloudwatchlogs.LogStream{stream}, nil)

				event := cloudwatchlogs.NewOutputLogEvent("some_message")

				mockCW.EXPECT().
					GetLogEvents(
						config.AWSLogGroupID(),
						*stream.LogStreamName,
						int64(30),
					).Return([]*cloudwatchlogs.OutputLogEvent{event}, nil)

				return mockCW
			},
			Run: func(reporter *testutils.Reporter, target interface{}) {
				provider := target.(cloudwatchlogs.Provider)

				logs, err := GetLogs(provider, []*string{stringp(taskARN)}, 30)
				if err != nil {
					reporter.Fatal(err)
				}

				testutils.AssertEqual(t, len(logs), 1)
				testutils.AssertInSlice(t, "some_message", logs[0].Lines)
			},
		},
	}

	testutils.RunTests(t, testCases)
}
<|endoftext|>


<commit_before>// Copyright 2016 The Vanadium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package benchmark_test

import (
	"bytes"
	"testing"
	"time"

	"v.io/v23/rpc"
	"v.io/v23/uniqueid"
	vtime "v.io/v23/vdlroot/time"
	"v.io/v23/vom"
	"v.io/v23/vtrace"
)

var req = rpc.Request{
	Method:        "Echo",
	NumPosArgs:    1,
	Deadline:      vtime.Deadline{Time: time.Now()},
	EndStreamArgs: false,
	TraceRequest: vtrace.Request{
		SpanId:  uniqueid.Id{0x2a, 0xa2, 0xfd, 0xce, 0x42, 0xa7, 0xbd, 0x34, 0x88, 0x8, 0x11, 0x2f, 0x71, 0xce, 0x80, 0xa2},
		TraceId: uniqueid.Id{0x2a, 0xa2, 0xfd, 0xce, 0x42, 0xa7, 0xbd, 0x34, 0x88, 0x8, 0x11, 0x2f, 0x71, 0xce, 0x80, 0xa0},
	},
	Language: "en-US.UTF-8",
}

var resp = rpc.Response{
	EndStreamResults: true,
	NumPosResults:    1,
	TraceResponse: vtrace.Response{
		Trace: vtrace.TraceRecord{
			Id: uniqueid.Id{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0},
		},
	},
}

func BenchmarkVomEncodeRequest(b *testing.B) {
	benchmarkEncode(b, req)
}

func BenchmarkVomDecodeRequest(b *testing.B) {
	benchmarkDecode(b, req, func() interface{} { return &rpc.Request{} })
}

func BenchmarkVomEncodeResponse(b *testing.B) {
	benchmarkEncode(b, resp)
}

func BenchmarkVomDecodeResponse(b *testing.B) {
	benchmarkDecode(b, resp, func() interface{} { return &rpc.Response{} })
}

func benchmarkEncode(b *testing.B, value interface{}) {
	typeBuf := &bytes.Buffer{}
	valBuf := &bytes.Buffer{}

	tenc := vom.NewTypeEncoder(typeBuf)
	enc := vom.NewEncoderWithTypeEncoder(valBuf, tenc)

	if err := enc.Encode(value); err != nil {
		b.Fatal(err)
	}
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		typeBuf.Reset()
		valBuf.Reset()
		if err := enc.Encode(value); err != nil {
			b.Fatal(err)
		}
	}
}

func benchmarkDecode(b *testing.B, value interface{}, zero func() interface{}) {
	typeBuf := &bytes.Buffer{}
	valBuf := &bytes.Buffer{}

	tenc := vom.NewTypeEncoder(typeBuf)
	enc := vom.NewEncoderWithTypeEncoder(valBuf, tenc)
	tdec := vom.NewTypeDecoder(typeBuf)
	tdec.Start()
	dec := vom.NewDecoderWithTypeDecoder(valBuf, tdec)

	for i := 0; i < b.N; i++ {
		if err := enc.Encode(value); err != nil {
			b.Fatal(err)
		}
	}
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		if err := dec.Decode(zero()); err != nil {
			b.Fatal(err)
		}
	}
}
<commit_msg>rpc/benchmark: Fix a bug where we initialized a typedecoder before writing all the data into a bytes.Buffer.  This doesn't work because bytes.Buffer returns EOF if a read happens before the data is written.<commit_after>// Copyright 2016 The Vanadium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package benchmark_test

import (
	"bytes"
	"testing"
	"time"

	"v.io/v23/rpc"
	"v.io/v23/uniqueid"
	vtime "v.io/v23/vdlroot/time"
	"v.io/v23/vom"
	"v.io/v23/vtrace"
)

var req = rpc.Request{
	Method:        "Echo",
	NumPosArgs:    1,
	Deadline:      vtime.Deadline{Time: time.Now()},
	EndStreamArgs: false,
	TraceRequest: vtrace.Request{
		SpanId:  uniqueid.Id{0x2a, 0xa2, 0xfd, 0xce, 0x42, 0xa7, 0xbd, 0x34, 0x88, 0x8, 0x11, 0x2f, 0x71, 0xce, 0x80, 0xa2},
		TraceId: uniqueid.Id{0x2a, 0xa2, 0xfd, 0xce, 0x42, 0xa7, 0xbd, 0x34, 0x88, 0x8, 0x11, 0x2f, 0x71, 0xce, 0x80, 0xa0},
	},
	Language: "en-US.UTF-8",
}

var resp = rpc.Response{
	EndStreamResults: true,
	NumPosResults:    1,
	TraceResponse: vtrace.Response{
		Trace: vtrace.TraceRecord{
			Id: uniqueid.Id{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0},
		},
	},
}

func BenchmarkVomEncodeRequest(b *testing.B) {
	benchmarkEncode(b, req)
}

func BenchmarkVomDecodeRequest(b *testing.B) {
	benchmarkDecode(b, req, func() interface{} { return &rpc.Request{} })
}

func BenchmarkVomEncodeResponse(b *testing.B) {
	benchmarkEncode(b, resp)
}

func BenchmarkVomDecodeResponse(b *testing.B) {
	benchmarkDecode(b, resp, func() interface{} { return &rpc.Response{} })
}

func benchmarkEncode(b *testing.B, value interface{}) {
	typeBuf := &bytes.Buffer{}
	valBuf := &bytes.Buffer{}

	tenc := vom.NewTypeEncoder(typeBuf)
	enc := vom.NewEncoderWithTypeEncoder(valBuf, tenc)

	if err := enc.Encode(value); err != nil {
		b.Fatal(err)
	}
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		typeBuf.Reset()
		valBuf.Reset()
		if err := enc.Encode(value); err != nil {
			b.Fatal(err)
		}
	}
}

func benchmarkDecode(b *testing.B, value interface{}, zero func() interface{}) {
	typeBuf := &bytes.Buffer{}
	valBuf := &bytes.Buffer{}

	tenc := vom.NewTypeEncoder(typeBuf)
	enc := vom.NewEncoderWithTypeEncoder(valBuf, tenc)
	for i := 0; i < b.N; i++ {
		if err := enc.Encode(value); err != nil {
			b.Fatal(err)
		}
	}

	tdec := vom.NewTypeDecoder(typeBuf)
	tdec.Start()
	defer tdec.Stop()
	dec := vom.NewDecoderWithTypeDecoder(valBuf, tdec)
	b.ResetTimer()

	for i := 0; i < b.N; i++ {
		if err := dec.Decode(zero()); err != nil {
			b.Fatal(i, err)
		}
	}
}
<|endoftext|>


<commit_before>package dnode

import (
	"fmt"
	"reflect"
	"strconv"
	"strings"
)

const functionPlaceholder = `"[Function]"`

// Callback is a function to send in arguments.
// Wrap your function to make it marshalable before sending.
type Callback func(*Partial)

func (p Callback) MarshalJSON() ([]byte, error) {
	return []byte(functionPlaceholder), nil
}

// Funciont is a callable function with arbitrary args and error return value.
// It is used to wrap the callback function on receiving side.
type Function func(...interface{}) error

// UnmarshalJSON marshals the callback as "nil".
// Value of the callback is not important in dnode protocol.
// Skips unmarshal errors when unmarshalling a callback placeholder to Callback.
func (p *Function) UnmarshalJSON(data []byte) error {
	return nil
}

// Path represents a callback function's path in the arguments structure.
// Contains mixture of string and integer values.
type Path []interface{}

// CallbackSpec is a structure encapsulating a Callback and it's Path.
type CallbackSpec struct {
	// Path represents the callback's path in the arguments structure.
	Path     Path
	Callback Function
}

func (c *CallbackSpec) Apply(value reflect.Value) error {
	l.Printf("Apply: %#v\n", value.Interface())
	i := 0
	for {
		l.Printf("Apply value: %#v\n", value)
		l.Printf("value.Kind(): %s\n", value.Kind())
		switch value.Kind() {
		case reflect.Slice:
			if i == len(c.Path) {
				return fmt.Errorf("Callback path too short: %v", c.Path)
			}

			// Path component may be a string or an integer.
			var index int
			var err error
			switch v := c.Path[i].(type) {
			case string:
				index, err = strconv.Atoi(v)
				if err != nil {
					return fmt.Errorf("Integer expected in callback path, got '%v'.", c.Path[i])
				}
			case int:
				index = v
			default:
				panic(fmt.Errorf("Unknown type: %#v", c.Path[i]))
			}

			value = value.Index(index)
			i++
		case reflect.Map:
			if i == len(c.Path) {
				return fmt.Errorf("Callback path too short: %v", c.Path)
			}
			if i == len(c.Path)-1 && value.Type().Elem().Kind() == reflect.Interface {
				value.SetMapIndex(reflect.ValueOf(c.Path[i]), reflect.ValueOf(c.Callback))
				return nil
			}
			value = value.MapIndex(reflect.ValueOf(c.Path[i]))
			i++
		case reflect.Ptr:
			value = value.Elem()
		case reflect.Interface:
			if i == len(c.Path) {
				value.Set(reflect.ValueOf(c.Callback))
				return nil
			}
			value = value.Elem()
		case reflect.Struct:
			if innerPartial, ok := value.Addr().Interface().(*Partial); ok {
				spec := CallbackSpec{c.Path[i:], c.Callback}
				innerPartial.CallbackSpecs = append(innerPartial.CallbackSpecs, spec)
				return nil
			}

			// Path component may be a string or an integer.
			name, ok := c.Path[i].(string)
			if !ok {
				return fmt.Errorf("Invalid path: %#v", c.Path[i])
			}

			value = value.FieldByName(strings.ToUpper(name[0:1]) + name[1:])
			i++
		case reflect.Func:
			value.Set(reflect.ValueOf(c.Callback))
			return nil
		case reflect.Invalid:
			// callback path does not exist, skip
			return nil
		default:
			return fmt.Errorf("Unhandled value of kind '%v' in callback path: %s", value.Kind(), value.Interface())
		}
	}
	return nil
}
<commit_msg>kite: fixed typo<commit_after>package dnode

import (
	"fmt"
	"reflect"
	"strconv"
	"strings"
)

const functionPlaceholder = `"[Function]"`

// Callback is a function to send in arguments.
// Wrap your function to make it marshalable before sending.
type Callback func(*Partial)

func (p Callback) MarshalJSON() ([]byte, error) {
	return []byte(functionPlaceholder), nil
}

// Funcion is a callable function with arbitrary args and error return value.
// It is used to wrap the callback function on receiving side.
type Function func(...interface{}) error

// UnmarshalJSON marshals the callback as "nil".
// Value of the callback is not important in dnode protocol.
// Skips unmarshal errors when unmarshalling a callback placeholder to Callback.
func (f *Function) UnmarshalJSON(data []byte) error {
	return nil
}

// Path represents a callback function's path in the arguments structure.
// Contains mixture of string and integer values.
type Path []interface{}

// CallbackSpec is a structure encapsulating a Callback and it's Path.
type CallbackSpec struct {
	// Path represents the callback's path in the arguments structure.
	Path     Path
	Callback Function
}

func (c *CallbackSpec) Apply(value reflect.Value) error {
	l.Printf("Apply: %#v\n", value.Interface())
	i := 0
	for {
		l.Printf("Apply value: %#v\n", value)
		l.Printf("value.Kind(): %s\n", value.Kind())
		switch value.Kind() {
		case reflect.Slice:
			if i == len(c.Path) {
				return fmt.Errorf("Callback path too short: %v", c.Path)
			}

			// Path component may be a string or an integer.
			var index int
			var err error
			switch v := c.Path[i].(type) {
			case string:
				index, err = strconv.Atoi(v)
				if err != nil {
					return fmt.Errorf("Integer expected in callback path, got '%v'.", c.Path[i])
				}
			case int:
				index = v
			default:
				panic(fmt.Errorf("Unknown type: %#v", c.Path[i]))
			}

			value = value.Index(index)
			i++
		case reflect.Map:
			if i == len(c.Path) {
				return fmt.Errorf("Callback path too short: %v", c.Path)
			}
			if i == len(c.Path)-1 && value.Type().Elem().Kind() == reflect.Interface {
				value.SetMapIndex(reflect.ValueOf(c.Path[i]), reflect.ValueOf(c.Callback))
				return nil
			}
			value = value.MapIndex(reflect.ValueOf(c.Path[i]))
			i++
		case reflect.Ptr:
			value = value.Elem()
		case reflect.Interface:
			if i == len(c.Path) {
				value.Set(reflect.ValueOf(c.Callback))
				return nil
			}
			value = value.Elem()
		case reflect.Struct:
			if innerPartial, ok := value.Addr().Interface().(*Partial); ok {
				spec := CallbackSpec{c.Path[i:], c.Callback}
				innerPartial.CallbackSpecs = append(innerPartial.CallbackSpecs, spec)
				return nil
			}

			// Path component may be a string or an integer.
			name, ok := c.Path[i].(string)
			if !ok {
				return fmt.Errorf("Invalid path: %#v", c.Path[i])
			}

			value = value.FieldByName(strings.ToUpper(name[0:1]) + name[1:])
			i++
		case reflect.Func:
			value.Set(reflect.ValueOf(c.Callback))
			return nil
		case reflect.Invalid:
			// callback path does not exist, skip
			return nil
		default:
			return fmt.Errorf("Unhandled value of kind '%v' in callback path: %s", value.Kind(), value.Interface())
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package models

import (
	"errors"
	"time"

	"github.com/koding/bongo"
)

type MessageReply struct {
	// unique identifier of the MessageReply
	Id int64 `json:"id"`

	// Id of the interacted message
	MessageId int64 `json:"messageId"             sql:"NOT NULL"`

	// Id of the reply
	ReplyId int64 `json:"replyId"                 sql:"NOT NULL"`

	// Creation of the MessageReply
	CreatedAt time.Time `json:"createdAt"         sql:"NOT NULL"`
}

func (m *MessageReply) GetId() int64 {
	return m.Id
}

func (m MessageReply) TableName() string {
	return "api.message_reply"
}

func NewMessageReply() *MessageReply {
	return &MessageReply{}
}

func (m *MessageReply) AfterCreate() {
	bongo.B.AfterCreate(m)
}

func (m *MessageReply) AfterUpdate() {
	bongo.B.AfterUpdate(m)
}

func (m *MessageReply) AfterDelete() {
	bongo.B.AfterDelete(m)
}

func (m *MessageReply) Fetch() error {
	return bongo.B.Fetch(m)
}

func (m *MessageReply) Create() error {
	return bongo.B.Create(m)
}

func (m *MessageReply) Delete() error {
	if err := bongo.B.DB.
		Where("message_id = ? and reply_id = ?", m.MessageId, m.ReplyId).
		Delete(m).Error; err != nil {
		return err
	}
	return nil
}

func (m *MessageReply) Some(data interface{}, q *bongo.Query) error {

	return bongo.B.Some(m, data, q)
}

func (m *MessageReply) One(q *bongo.Query) error {

	return bongo.B.One(m, m, q)
}

func (m *MessageReply) List() ([]ChannelMessage, error) {
	var replies []int64

	if m.MessageId == 0 {
		return nil, errors.New("MessageId is not set")
	}

	// TODO change this with FetchReplyIds methods
	if err := bongo.B.DB.Table(m.TableName()).
		Where("message_id = ?", m.MessageId).
		Pluck("reply_id", &replies).
		Error; err != nil {
		return nil, err
	}

	parent := NewChannelMessage()
	channelMessageReplies, err := parent.FetchByIds(replies)
	if err != nil {
		return nil, err
	}

	return channelMessageReplies, nil
}

func (m *MessageReply) FetchReplyIds(p *bongo.Pagination) ([]int64, error) {
	var replyIds []int64
	q := &bongo.Query{
		Selector: map[string]interface{}{
			"message_id": m.MessageId,
		},
		Pagination: *p,
		Pluck: "reply_id",
		Sort: map[string]string{
     		"created_at": "desc",
    	},
	}
	if err := m.Some(&replyIds, q); err != nil {
		return nil, err
	}

	return replyIds, nil
}

func (m *MessageReply) FetchByReplyId() error {
	q := &bongo.Query{
		Selector: map[string]interface{}{
			"reply_id": m.ReplyId,
		},
	}
	return m.One(q)
}
<commit_msg>SocialApi: FetchFirstAccountReply method is added for retrieval of notifiers after users first reply message<commit_after>package models

import (
	"errors"
	"time"

	"github.com/koding/bongo"
)

type MessageReply struct {
	// unique identifier of the MessageReply
	Id int64 `json:"id"`

	// Id of the interacted message
	MessageId int64 `json:"messageId"             sql:"NOT NULL"`

	// Id of the reply
	ReplyId int64 `json:"replyId"                 sql:"NOT NULL"`

	// Creation of the MessageReply
	CreatedAt time.Time `json:"createdAt"         sql:"NOT NULL"`
}

func (m *MessageReply) GetId() int64 {
	return m.Id
}

func (m MessageReply) TableName() string {
	return "api.message_reply"
}

func NewMessageReply() *MessageReply {
	return &MessageReply{}
}

func (m *MessageReply) AfterCreate() {
	bongo.B.AfterCreate(m)
}

func (m *MessageReply) AfterUpdate() {
	bongo.B.AfterUpdate(m)
}

func (m *MessageReply) AfterDelete() {
	bongo.B.AfterDelete(m)
}

func (m *MessageReply) Fetch() error {
	return bongo.B.Fetch(m)
}

func (m *MessageReply) Create() error {
	return bongo.B.Create(m)
}

func (m *MessageReply) Delete() error {
	if err := bongo.B.DB.
		Where("message_id = ? and reply_id = ?", m.MessageId, m.ReplyId).
		Delete(m).Error; err != nil {
		return err
	}
	return nil
}

func (m *MessageReply) Some(data interface{}, q *bongo.Query) error {

	return bongo.B.Some(m, data, q)
}

func (m *MessageReply) One(q *bongo.Query) error {

	return bongo.B.One(m, m, q)
}

func (m *MessageReply) List() ([]ChannelMessage, error) {
	var replies []int64

	if m.MessageId == 0 {
		return nil, errors.New("MessageId is not set")
	}

	// TODO change this with FetchReplyIds methods
	if err := bongo.B.DB.Table(m.TableName()).
		Where("message_id = ?", m.MessageId).
		Pluck("reply_id", &replies).
		Error; err != nil {
		return nil, err
	}

	parent := NewChannelMessage()
	channelMessageReplies, err := parent.FetchByIds(replies)
	if err != nil {
		return nil, err
	}

	return channelMessageReplies, nil
}

func (m *MessageReply) FetchReplyIds(p *bongo.Pagination) ([]int64, error) {
	var replyIds []int64
	q := &bongo.Query{
		Selector: map[string]interface{}{
			"message_id": m.MessageId,
		},
		Pagination: *p,
		Pluck: "reply_id",
		Sort: map[string]string{
     		"created_at": "desc",
    	},
	}
	if err := m.Some(&replyIds, q); err != nil {
		return nil, err
	}

	return replyIds, nil
}

func (m *MessageReply) FetchByReplyId() error {
	q := &bongo.Query{
		Selector: map[string]interface{}{
			"reply_id": m.ReplyId,
		},
	}
	return m.One(q)
}

func (m *MessageReply) FetchFirstAccountReply(accountId int64) error {
	cm := NewChannelMessage()
	rows, err := bongo.B.DB.Raw("SELECT mr.reply_id, mr.created_at "+
		"FROM "+m.TableName()+" mr "+
		"LEFT JOIN "+cm.TableName()+" cm ON cm.id = mr.reply_id "+
		"WHERE cm.account_id = ? AND mr.message_id = ? "+
		"ORDER BY cm.created_at ASC "+
		"LIMIT 1", accountId, m.MessageId).Rows()

	// probably gorm.ErrNotFound error must be caught
	if err != nil {
		return err
	}
	if rows.Next() {
		// i could not handle it by selecting all columns
		rows.Scan(&m.ReplyId, &m.CreatedAt)
	}

	return nil
}
<|endoftext|>