<commit_before>package metrics

import (
	"fmt"
	"math"
	"strings"
	"sync"
	"time"
)

// InmemSink provides a MetricSink that does in-memory aggregation
// without sending metrics over a network. It can be embedded within
// an application to provide profiling information.
type InmemSink struct {
	// How long is each aggregation interval
	interval time.Duration

	// Retain controls how many metrics interval we keep
	retain time.Duration

	// maxIntervals is the maximum length of intervals.
	// It is retain / interval.
	maxIntervals int

	// intervals is a slice of the retained intervals
	intervals    []*IntervalMetrics
	intervalLock sync.RWMutex
}

// IntervalMetrics stores the aggregated metrics
// for a specific interval
type IntervalMetrics struct {
	sync.RWMutex

	// The start time of the interval
	Interval time.Time

	// Gauges maps the key to the last set value
	Gauges map[string]float32

	// Points maps the string to the list of emitted values
	// from EmitKey
	Points map[string][]float32

	// Counters maps the string key to a sum of the counter
	// values
	Counters map[string]*AggregateSample

	// Samples maps the key to an AggregateSample,
	// which has the rolled up view of a sample
	Samples map[string]*AggregateSample
}

// NewIntervalMetrics creates a new IntervalMetrics for a given interval
func NewIntervalMetrics(intv time.Time) *IntervalMetrics {
	return &IntervalMetrics{
		Interval: intv,
		Gauges:   make(map[string]float32),
		Points:   make(map[string][]float32),
		Counters: make(map[string]*AggregateSample),
		Samples:  make(map[string]*AggregateSample),
	}
}

// AggregateSample is used to hold aggregate metrics
// about a sample
type AggregateSample struct {
	Count       int       // The count of emitted pairs
	Sum         float64   // The sum of values
	SumSq       float64   // The sum of squared values
	Min         float64   // Minimum value
	Max         float64   // Maximum value
	LastUpdated time.Time // When value was last updated
}

// Computes a Stddev of the values
func (a *AggregateSample) Stddev() float64 {
	num := (float64(a.Count) * a.SumSq) - math.Pow(a.Sum, 2)
	div := float64(a.Count * (a.Count - 1))
	if div == 0 {
		return 0
	}
	return math.Sqrt(num / div)
}

// Computes a mean of the values
func (a *AggregateSample) Mean() float64 {
	if a.Count == 0 {
		return 0
	}
	return a.Sum / float64(a.Count)
}

// Ingest is used to update a sample
func (a *AggregateSample) Ingest(v float64) {
	a.Count++
	a.Sum += v
	a.SumSq += (v * v)
	if v < a.Min || a.Count == 1 {
		a.Min = v
	}
	if v > a.Max || a.Count == 1 {
		a.Max = v
	}
	a.LastUpdated = time.Now()
}

func (a *AggregateSample) String() string {
	if a.Count == 0 {
		return "Count: 0"
	} else if a.Stddev() == 0 {
		return fmt.Sprintf("Count: %d Sum: %0.3f LastUpdated: %s", a.Count, a.Sum, a.LastUpdated)
	} else {
		return fmt.Sprintf("Count: %d Min: %0.3f Mean: %0.3f Max: %0.3f Stddev: %0.3f Sum: %0.3f LastUpdated: %s",
			a.Count, a.Min, a.Mean(), a.Max, a.Stddev(), a.Sum, a.LastUpdated)
	}
}

// NewInmemSink is used to construct a new in-memory sink.
// Uses an aggregation interval and maximum retention period.
func NewInmemSink(interval, retain time.Duration) *InmemSink {
	i := &InmemSink{
		interval:     interval,
		retain:       retain,
		maxIntervals: int(retain / interval),
	}
	i.intervals = make([]*IntervalMetrics, 0, i.maxIntervals)
	return i
}

func (i *InmemSink) SetGauge(key []string, val float32) {
	k := i.flattenKey(key)
	intv := i.getInterval()

	intv.Lock()
	defer intv.Unlock()
	intv.Gauges[k] = val
}

func (i *InmemSink) EmitKey(key []string, val float32) {
	k := i.flattenKey(key)
	intv := i.getInterval()

	intv.Lock()
	defer intv.Unlock()
	vals := intv.Points[k]
	intv.Points[k] = append(vals, val)
}

func (i *InmemSink) IncrCounter(key []string, val float32) {
	k := i.flattenKey(key)
	intv := i.getInterval()

	intv.Lock()
	defer intv.Unlock()

	agg := intv.Counters[k]
	if agg == nil {
		agg = &AggregateSample{}
		intv.Counters[k] = agg
	}
	agg.Ingest(float64(val))
}

func (i *InmemSink) AddSample(key []string, val float32) {
	k := i.flattenKey(key)
	intv := i.getInterval()

	intv.Lock()
	defer intv.Unlock()

	agg := intv.Samples[k]
	if agg == nil {
		agg = &AggregateSample{}
		intv.Samples[k] = agg
	}
	agg.Ingest(float64(val))
}

// Data is used to retrieve all the aggregated metrics
// Intervals may be in use, and a read lock should be acquired
func (i *InmemSink) Data() []*IntervalMetrics {
	// Get the current interval, forces creation
	i.getInterval()

	i.intervalLock.RLock()
	defer i.intervalLock.RUnlock()

	intervals := make([]*IntervalMetrics, len(i.intervals))
	copy(intervals, i.intervals)
	return intervals
}

func (i *InmemSink) getExistingInterval(intv time.Time) *IntervalMetrics {
	i.intervalLock.RLock()
	defer i.intervalLock.RUnlock()

	n := len(i.intervals)
	if n > 0 && i.intervals[n-1].Interval == intv {
		return i.intervals[n-1]
	}
	return nil
}

func (i *InmemSink) createInterval(intv time.Time) *IntervalMetrics {
	i.intervalLock.Lock()
	defer i.intervalLock.Unlock()

	// Check for an existing interval
	n := len(i.intervals)
	if n > 0 && i.intervals[n-1].Interval == intv {
		return i.intervals[n-1]
	}

	// Add the current interval
	current := NewIntervalMetrics(intv)
	i.intervals = append(i.intervals, current)
	n++

	// Truncate the intervals if they are too long
	if n >= i.maxIntervals {
		copy(i.intervals[0:], i.intervals[n-i.maxIntervals:])
		i.intervals = i.intervals[:i.maxIntervals]
	}
	return current
}

// getInterval returns the current interval to write to
func (i *InmemSink) getInterval() *IntervalMetrics {
	intv := time.Now().Truncate(i.interval)
	if m := i.getExistingInterval(intv); m != nil {
		return m
	}
	return i.createInterval(intv)
}

// Flattens the key for formatting, removes spaces
func (i *InmemSink) flattenKey(parts []string) string {
	joined := strings.Join(parts, ".")
	return strings.Replace(joined, " ", "_", -1)
}
<commit_msg>Added Throughput caclulation<commit_after>package metrics

import (
	"fmt"
	"math"
	"strings"
	"sync"
	"time"
)

// InmemSink provides a MetricSink that does in-memory aggregation
// without sending metrics over a network. It can be embedded within
// an application to provide profiling information.
type InmemSink struct {
	// How long is each aggregation interval
	interval time.Duration

	// Retain controls how many metrics interval we keep
	retain time.Duration

	// maxIntervals is the maximum length of intervals.
	// It is retain / interval.
	maxIntervals int

	// intervals is a slice of the retained intervals
	intervals    []*IntervalMetrics
	intervalLock sync.RWMutex

	throughputDenom float64
}

// IntervalMetrics stores the aggregated metrics
// for a specific interval
type IntervalMetrics struct {
	sync.RWMutex

	// The start time of the interval
	Interval time.Time

	// Gauges maps the key to the last set value
	Gauges map[string]float32

	// Points maps the string to the list of emitted values
	// from EmitKey
	Points map[string][]float32

	// Counters maps the string key to a sum of the counter
	// values
	Counters map[string]*AggregateSample

	// Samples maps the key to an AggregateSample,
	// which has the rolled up view of a sample
	Samples map[string]*AggregateSample
}

// NewIntervalMetrics creates a new IntervalMetrics for a given interval
func NewIntervalMetrics(intv time.Time) *IntervalMetrics {
	return &IntervalMetrics{
		Interval: intv,
		Gauges:   make(map[string]float32),
		Points:   make(map[string][]float32),
		Counters: make(map[string]*AggregateSample),
		Samples:  make(map[string]*AggregateSample),
	}
}

// AggregateSample is used to hold aggregate metrics
// about a sample
type AggregateSample struct {
	Count       int       // The count of emitted pairs
	Throughput  float64   // The count of emitted pairs per time unit (usually 1 second)
	Sum         float64   // The sum of values
	SumSq       float64   // The sum of squared values
	Min         float64   // Minimum value
	Max         float64   // Maximum value
	LastUpdated time.Time // When value was last updated
}

// Computes a Stddev of the values
func (a *AggregateSample) Stddev() float64 {
	num := (float64(a.Count) * a.SumSq) - math.Pow(a.Sum, 2)
	div := float64(a.Count * (a.Count - 1))
	if div == 0 {
		return 0
	}
	return math.Sqrt(num / div)
}

// Computes a mean of the values
func (a *AggregateSample) Mean() float64 {
	if a.Count == 0 {
		return 0
	}
	return a.Sum / float64(a.Count)
}

// Ingest is used to update a sample
func (a *AggregateSample) Ingest(v float64, throughputDenom float64) {
	a.Count++
	a.Sum += v
	a.SumSq += (v * v)
	if v < a.Min || a.Count == 1 {
		a.Min = v
	}
	if v > a.Max || a.Count == 1 {
		a.Max = v
	}
	a.Throughput = float64(a.Count)/throughputDenom
	a.LastUpdated = time.Now()
}

func (a *AggregateSample) String() string {
	if a.Count == 0 {
		return "Count: 0"
	} else if a.Stddev() == 0 {
		return fmt.Sprintf("Count: %d Sum: %0.3f LastUpdated: %s", a.Count, a.Sum, a.LastUpdated)
	} else {
		return fmt.Sprintf("Count: %d Min: %0.3f Mean: %0.3f Max: %0.3f Stddev: %0.3f Sum: %0.3f LastUpdated: %s",
			a.Count, a.Min, a.Mean(), a.Max, a.Stddev(), a.Sum, a.LastUpdated)
	}
}

// NewInmemSink is used to construct a new in-memory sink.
// Uses an aggregation interval and maximum retention period.
func NewInmemSink(interval, retain time.Duration) *InmemSink {
	throughputTimeUnit := time.Second
	i := &InmemSink{
		interval:     interval,
		retain:       retain,
		maxIntervals: int(retain / interval),
		throughputDenom: float64(interval.Nanoseconds()) / float64(throughputTimeUnit.Nanoseconds()),
	}
	i.intervals = make([]*IntervalMetrics, 0, i.maxIntervals)
	return i
}

func (i *InmemSink) SetGauge(key []string, val float32) {
	k := i.flattenKey(key)
	intv := i.getInterval()

	intv.Lock()
	defer intv.Unlock()
	intv.Gauges[k] = val
}

func (i *InmemSink) EmitKey(key []string, val float32) {
	k := i.flattenKey(key)
	intv := i.getInterval()

	intv.Lock()
	defer intv.Unlock()
	vals := intv.Points[k]
	intv.Points[k] = append(vals, val)
}

func (i *InmemSink) IncrCounter(key []string, val float32) {
	k := i.flattenKey(key)
	intv := i.getInterval()

	intv.Lock()
	defer intv.Unlock()

	agg := intv.Counters[k]
	if agg == nil {
		agg = &AggregateSample{}
		intv.Counters[k] = agg
	}
	agg.Ingest(float64(val), i.throughputDenom)
}

func (i *InmemSink) AddSample(key []string, val float32) {
	k := i.flattenKey(key)
	intv := i.getInterval()

	intv.Lock()
	defer intv.Unlock()

	agg := intv.Samples[k]
	if agg == nil {
		agg = &AggregateSample{}
		intv.Samples[k] = agg
	}
	agg.Ingest(float64(val), i.throughputDenom)
}

// Data is used to retrieve all the aggregated metrics
// Intervals may be in use, and a read lock should be acquired
func (i *InmemSink) Data() []*IntervalMetrics {
	// Get the current interval, forces creation
	i.getInterval()

	i.intervalLock.RLock()
	defer i.intervalLock.RUnlock()

	intervals := make([]*IntervalMetrics, len(i.intervals))
	copy(intervals, i.intervals)
	return intervals
}

func (i *InmemSink) getExistingInterval(intv time.Time) *IntervalMetrics {
	i.intervalLock.RLock()
	defer i.intervalLock.RUnlock()

	n := len(i.intervals)
	if n > 0 && i.intervals[n-1].Interval == intv {
		return i.intervals[n-1]
	}
	return nil
}

func (i *InmemSink) createInterval(intv time.Time) *IntervalMetrics {
	i.intervalLock.Lock()
	defer i.intervalLock.Unlock()

	// Check for an existing interval
	n := len(i.intervals)
	if n > 0 && i.intervals[n-1].Interval == intv {
		return i.intervals[n-1]
	}

	// Add the current interval
	current := NewIntervalMetrics(intv)
	i.intervals = append(i.intervals, current)
	n++

	// Truncate the intervals if they are too long
	if n >= i.maxIntervals {
		copy(i.intervals[0:], i.intervals[n-i.maxIntervals:])
		i.intervals = i.intervals[:i.maxIntervals]
	}
	return current
}

// getInterval returns the current interval to write to
func (i *InmemSink) getInterval() *IntervalMetrics {
	intv := time.Now().Truncate(i.interval)
	if m := i.getExistingInterval(intv); m != nil {
		return m
	}
	return i.createInterval(intv)
}

// Flattens the key for formatting, removes spaces
func (i *InmemSink) flattenKey(parts []string) string {
	joined := strings.Join(parts, ".")
	return strings.Replace(joined, " ", "_", -1)
}
<|endoftext|>


<commit_before><commit_msg>useless ring<commit_after><|endoftext|>


<commit_before>package arn

import (
	"sort"
	"strings"

	"github.com/aerogo/flow"
)

// MinimumStringSimilarity is the minimum JaroWinkler distance we accept for search results.
const MinimumStringSimilarity = 0.9

// SearchIndex ...
type SearchIndex struct {
	TextToID map[string]string `json:"textToId"`
}

// NewSearchIndex ...
func NewSearchIndex() *SearchIndex {
	return &SearchIndex{
		TextToID: make(map[string]string),
	}
}

// GetSearchIndex ...
func GetSearchIndex(id string) (*SearchIndex, error) {
	obj, err := DB.Get("SearchIndex", id)
	return obj.(*SearchIndex), err
}

// Search is a fuzzy search.
func Search(term string, maxUsers, maxAnime int) ([]*User, []*Anime) {
	term = strings.ToLower(term)

	if term == "" {
		return nil, nil
	}

	var userResults []*User
	var animeResults []*Anime

	type SearchItem struct {
		text       string
		similarity float64
	}

	// Search everything in parallel
	flow.Parallel(func() {
		// Search userResults
		var user *User

		userSearchIndex, err := GetSearchIndex("User")

		if err != nil {
			return
		}

		textToID := userSearchIndex.TextToID

		// Search items
		items := make([]*SearchItem, 0)

		for name := range textToID {
			s := StringSimilarity(term, name)

			if strings.Index(name, term) != -1 {
				s += 0.5
			}

			if s < MinimumStringSimilarity {
				continue
			}

			items = append(items, &SearchItem{
				text:       name,
				similarity: s,
			})
		}

		// Sort
		sort.Slice(items, func(i, j int) bool {
			return items[i].similarity > items[j].similarity
		})

		// Limit
		if len(items) >= maxUsers {
			items = items[:maxUsers]
		}

		// Fetch data
		for _, item := range items {
			user, err = GetUser(textToID[item.text])

			if err != nil {
				continue
			}

			userResults = append(userResults, user)
		}
	}, func() {
		// Search anime
		var anime *Anime

		animeSearchIndex, err := GetSearchIndex("Anime")

		if err != nil {
			return
		}

		textToID := animeSearchIndex.TextToID

		// Search items
		items := make([]*SearchItem, 0)
		animeIDAdded := map[string]bool{}

		for name, id := range textToID {
			_, found := animeIDAdded[id]

			// Skip existing anime IDs
			if found {
				continue
			}

			s := StringSimilarity(term, name)

			if strings.Index(name, term) != -1 {
				s += 0.5
			}

			if s < MinimumStringSimilarity {
				continue
			}

			items = append(items, &SearchItem{
				text:       name,
				similarity: s,
			})

			animeIDAdded[id] = true
		}

		// Sort
		sort.Slice(items, func(i, j int) bool {
			return items[i].similarity > items[j].similarity
		})

		// Limit
		if len(items) >= maxAnime {
			items = items[:maxAnime]
		}

		// Fetch data
		for _, item := range items {
			anime, err = GetAnime(textToID[item.text])

			if err != nil {
				continue
			}

			animeResults = append(animeResults, anime)
		}
	})

	return userResults, animeResults
}
<commit_msg>Improved anime search<commit_after>package arn

import (
	"sort"
	"strings"

	"github.com/aerogo/flow"
)

// MinimumStringSimilarity is the minimum JaroWinkler distance we accept for search results.
const MinimumStringSimilarity = 0.9

// SearchIndex ...
type SearchIndex struct {
	TextToID map[string]string `json:"textToId"`
}

// NewSearchIndex ...
func NewSearchIndex() *SearchIndex {
	return &SearchIndex{
		TextToID: make(map[string]string),
	}
}

// GetSearchIndex ...
func GetSearchIndex(id string) (*SearchIndex, error) {
	obj, err := DB.Get("SearchIndex", id)
	return obj.(*SearchIndex), err
}

// Search is a fuzzy search.
func Search(term string, maxUsers, maxAnime int) ([]*User, []*Anime) {
	term = strings.ToLower(term)

	if term == "" {
		return nil, nil
	}

	var userResults []*User
	var animeResults []*Anime

	type SearchItem struct {
		text       string
		similarity float64
	}

	// Search everything in parallel
	flow.Parallel(func() {
		// Search userResults
		var user *User

		userSearchIndex, err := GetSearchIndex("User")

		if err != nil {
			return
		}

		textToID := userSearchIndex.TextToID

		// Search items
		items := make([]*SearchItem, 0)

		for name := range textToID {
			s := StringSimilarity(term, name)

			if strings.Index(name, term) != -1 {
				s += 0.5
			}

			if s < MinimumStringSimilarity {
				continue
			}

			items = append(items, &SearchItem{
				text:       name,
				similarity: s,
			})
		}

		// Sort
		sort.Slice(items, func(i, j int) bool {
			return items[i].similarity > items[j].similarity
		})

		// Limit
		if len(items) >= maxUsers {
			items = items[:maxUsers]
		}

		// Fetch data
		for _, item := range items {
			user, err = GetUser(textToID[item.text])

			if err != nil {
				continue
			}

			userResults = append(userResults, user)
		}
	}, func() {
		// Search anime
		var anime *Anime

		animeSearchIndex, err := GetSearchIndex("Anime")

		if err != nil {
			return
		}

		textToID := animeSearchIndex.TextToID

		// Search items
		items := make([]*SearchItem, 0)
		animeIDAdded := map[string]*SearchItem{}

		for name, id := range textToID {
			s := StringSimilarity(term, name)

			if strings.Index(name, term) != -1 {
				s += 0.5
			}

			if s < MinimumStringSimilarity {
				continue
			}

			addedEntry, found := animeIDAdded[id]

			// Skip existing anime IDs
			if found {
				// But update existing entry with new similarity if it's higher
				if s > addedEntry.similarity {
					addedEntry.similarity = s
				}

				continue
			}

			item := &SearchItem{
				text:       name,
				similarity: s,
			}
			items = append(items, item)

			animeIDAdded[id] = item
		}

		// Sort
		sort.Slice(items, func(i, j int) bool {
			return items[i].similarity > items[j].similarity
		})

		// Limit
		if len(items) >= maxAnime {
			items = items[:maxAnime]
		}

		// Fetch data
		for _, item := range items {
			anime, err = GetAnime(textToID[item.text])

			if err != nil {
				continue
			}

			animeResults = append(animeResults, anime)
		}
	})

	return userResults, animeResults
}
<|endoftext|>


<commit_before>// Copyright (c) 2014-present Stuart Herbert
// All rights reserved
package modlog

import (
	"fmt"
	"log"
)

// LogFormatter is the signature that all output formatters much satisfy
type LogFormatter func(*Logger, *LogEntry) string

// FormattableLog is the interface for anything that supports formatting the
// metadata for a log message
type FormattableLog interface {
	AddFormatter(string, LogFormatter)
	RemoveFormatter(string)
}

// a list of the standard slots to insert formatters into
const (
	FormatTimestamp = "timestamp"
	FormatLogLevel  = "loglevel"
	FormatModule    = "module"
)

// DateTimeFormatter converts the 'When' field to the date/time format
// specified by the SetFlags() call
func StdlibDateTimeFormatter(logger *Logger, entry *LogEntry) string {
	output := ""

	// do we need to work out the date?
	if logger.StdlibFlags&log.Ldate != 0 {
		year, month, day := entry.When.Date()
		output = fmt.Sprintf("%04d/%02d/%02d", year, month, day)
	}

	// what about the time?
	if logger.StdlibFlags&log.Ltime != 0 {
		if len(output) > 0 {
			output = output + " "
		}

		hour, mins, secs := entry.When.Clock()
		output = output + fmt.Sprintf("%02d:%02d:%02d", hour, mins, secs)
	}

	// nanoseconds, anyone?
	if logger.StdlibFlags&log.Lmicroseconds != 0 {
		if len(output) > 0 {
			output = output + " "
		}

		ms := entry.When.Nanosecond() / 1e3
		output = output + fmt.Sprintf(".%06d", ms)
	}

	// all done
	return output
}

func StandardLogLevelFormatter(logger *Logger, entry *LogEntry) string {
	return fmt.Sprintf("%-9s", entry.LogLevel.String())
}

func ShortLogLevelFormatter(logger *Logger, entry *LogEntry) string {
	return entry.LogLevel.ShortString()
}

func StdlibPrefixFormatter(logger *Logger, entry *LogEntry) string {
	return logger.StdlibPrefix
}
<commit_msg>Fix: improve compatibility for timestamp formatting<commit_after>// Copyright (c) 2014-present Stuart Herbert
// All rights reserved
package modlog

import (
	"fmt"
	"log"
)

// LogFormatter is the signature that all output formatters much satisfy
type LogFormatter func(*Logger, *LogEntry) string

// FormattableLog is the interface for anything that supports formatting the
// metadata for a log message
type FormattableLog interface {
	AddFormatter(string, LogFormatter)
	RemoveFormatter(string)
}

// a list of the standard slots to insert formatters into
const (
	FormatTimestamp = "timestamp"
	FormatLogLevel  = "loglevel"
	FormatModule    = "module"
)

// DateTimeFormatter converts the 'When' field to the date/time format
// specified by the SetFlags() call
func StdlibDateTimeFormatter(logger *Logger, entry *LogEntry) string {
	output := ""

	// do we need to work out the date?
	if logger.StdlibFlags&log.Ldate != 0 {
		year, month, day := entry.When.Date()
		output = fmt.Sprintf("%04d/%02d/%02d", year, month, day)
	}

	// what about the time?
	//
	// note: Ltime is optional if Lmicroseconds is set
	//
	// Implicit behaviour :(
	if logger.StdlibFlags&log.Ltime != 0 || logger.StdlibFlags&log.Lmicroseconds != 0 {
		if len(output) > 0 {
			output = output + " "
		}

		hour, mins, secs := entry.When.Clock()
		output = output + fmt.Sprintf("%02d:%02d:%02d", hour, mins, secs)
	}

	// nanoseconds, anyone?
	if logger.StdlibFlags&log.Lmicroseconds != 0 {
		ms := entry.When.Nanosecond() / 1e3
		output = output + fmt.Sprintf(".%06d", ms)
	}

	// all done
	return output
}

func StandardLogLevelFormatter(logger *Logger, entry *LogEntry) string {
	return fmt.Sprintf("%-9s", entry.LogLevel.String())
}

func ShortLogLevelFormatter(logger *Logger, entry *LogEntry) string {
	return entry.LogLevel.ShortString()
}

func StdlibPrefixFormatter(logger *Logger, entry *LogEntry) string {
	return logger.StdlibPrefix
}
<|endoftext|>


<commit_before>package generators

import (
	"fmt"
	"os"
	"path/filepath"

	"github.com/yext/edward/services"
	"github.com/yext/errgo"
)

type Generator interface {
	Name() string
	StartWalk(basePath string)
	StopWalk()
	VisitDir(path string, f os.FileInfo, err error) error
	Found() []*services.ServiceConfig
}

type ConfigGenerator func(path string) ([]*services.ServiceConfig, []*services.ServiceGroupConfig, error)

var Generators map[string]Generator

func RegisterGenerator(g Generator) {
	if Generators == nil {
		Generators = make(map[string]Generator)
	}
	Generators[g.Name()] = g
}

func GenerateServices(path string) ([]*services.ServiceConfig, error) {
	var outServices []*services.ServiceConfig

	err := validateDir(path)
	if err != nil {
		return outServices, err
	}

	for name, generator := range Generators {
		generator.StartWalk(path)
		err := filepath.Walk(path, func(path string, f os.FileInfo, err error) error {
			if _, err := os.Stat(path); err != nil {
				return errgo.Mask(err)
			}

			if f.Mode().IsDir() {
				return errgo.Mask(generator.VisitDir(path, f, err))
			}
			return nil
		})
		generator.StopWalk()
		if err != nil {
			fmt.Println("Error in generator", name, ":", err)
		} else {
			outServices = append(outServices, generator.Found()...)
		}
	}

	return outServices, nil
}
<commit_msg>Handle SkipDir correctly.<commit_after>package generators

import (
	"fmt"
	"os"
	"path/filepath"

	"github.com/yext/edward/services"
	"github.com/yext/errgo"
)

type Generator interface {
	Name() string
	StartWalk(basePath string)
	StopWalk()
	VisitDir(path string, f os.FileInfo, err error) error
	Found() []*services.ServiceConfig
}

type ConfigGenerator func(path string) ([]*services.ServiceConfig, []*services.ServiceGroupConfig, error)

var Generators map[string]Generator

func RegisterGenerator(g Generator) {
	if Generators == nil {
		Generators = make(map[string]Generator)
	}
	Generators[g.Name()] = g
}

func GenerateServices(path string) ([]*services.ServiceConfig, error) {
	var outServices []*services.ServiceConfig

	err := validateDir(path)
	if err != nil {
		return outServices, err
	}

	for name, generator := range Generators {
		generator.StartWalk(path)
		err := filepath.Walk(path, func(path string, f os.FileInfo, err error) error {
			if _, err := os.Stat(path); err != nil {
				return errgo.Mask(err)
			}

			if f.Mode().IsDir() {
				err := generator.VisitDir(path, f, err)
				if err == filepath.SkipDir {
					return err
				}
				return errgo.Mask(err)
			}
			return nil
		})
		generator.StopWalk()
		if err != nil {
			fmt.Println("Error in generator", name, ":", err)
		} else {
			outServices = append(outServices, generator.Found()...)
		}
	}

	return outServices, nil
}
<|endoftext|>


<commit_before>package main

import (
	"fyne.io/fyne/v2"
	"fyne.io/fyne/v2/app"
	"fyne.io/fyne/v2/canvas"
	"fyne.io/fyne/v2/container"
	"image/color"
	"time"
)

type snakePart struct {
	x, y float32
}

type moveType int

const (
	moveUp moveType = iota
	moveDown
	moveLeft
	moveRight
)

var snakeParts []snakePart
var game *fyne.Container
var head *canvas.Rectangle
var move = moveUp

func keyTyped(e *fyne.KeyEvent) {
	switch e.Name {
	case fyne.KeyUp:
		move = moveUp
	case fyne.KeyDown:
		move = moveDown
	case fyne.KeyLeft:
		move = moveLeft
	case fyne.KeyRight:
		move = moveRight
	}
}

func main() {
	a := app.New()

	w := a.NewWindow("Snake")
	w.Resize(fyne.NewSize(200, 200))
	w.SetFixedSize(true)

	game = setupGame()
	w.SetContent(game)
	w.Canvas().SetOnTypedKey(keyTyped)

	go runGame()
	w.ShowAndRun()
}

func refreshGame() {
	for i, seg := range snakeParts {
		rect := game.Objects[i]
		rect.Move(fyne.NewPos(seg.x*10, seg.y*10))
	}

	game.Refresh()
}

func runGame() {
	nextPart := snakePart{snakeParts[0].x, snakeParts[0].y - 1}
	for {
		oldPos := fyne.NewPos(snakeParts[0].x*10, snakeParts[0].y*10)
		newPos := fyne.NewPos(nextPart.x*10, nextPart.y*10)
		canvas.NewPositionAnimation(oldPos, newPos, time.Millisecond*250, func(p fyne.Position) {
			head.Move(p)
			canvas.Refresh(head)
		}).Start()

		end := len(snakeParts) - 1
		canvas.NewPositionAnimation(fyne.NewPos(snakeParts[end].x*10, snakeParts[end].y*10),
			fyne.NewPos(snakeParts[end-1].x*10, snakeParts[end-1].y*10), time.Millisecond*250,
			func(p fyne.Position) {
				tail := game.Objects[end]
				tail.Move(p)
				canvas.Refresh(tail)
			}).Start()

		time.Sleep(time.Millisecond * 250)
		for i := len(snakeParts) - 1; i >= 1; i-- {
			snakeParts[i] = snakeParts[i-1]
		}
		snakeParts[0] = nextPart
		refreshGame()
		switch move {
		case moveUp:
			nextPart = snakePart{nextPart.x, nextPart.y - 1}
		case moveDown:
			nextPart = snakePart{nextPart.x, nextPart.y + 1}
		case moveLeft:
			nextPart = snakePart{nextPart.x - 1, nextPart.y}
		case moveRight:
			nextPart = snakePart{nextPart.x + 1, nextPart.y}
		}
	}
}

func setupGame() *fyne.Container {
	var segments []fyne.CanvasObject
	for i := 0; i < 10; i++ {
		seg := snakePart{9, float32(5 + i)}
		snakeParts = append(snakeParts, seg)

		r := canvas.NewRectangle(&color.RGBA{G: 0x66, A: 0xff})
		r.Resize(fyne.NewSize(10, 10))
		r.Move(fyne.NewPos(90, float32(50+i*10)))
		segments = append(segments, r)
	}

	head = canvas.NewRectangle(&color.RGBA{G: 0x66, A: 0xff})
	head.Resize(fyne.NewSize(10, 10))
	head.Move(fyne.NewPos(snakeParts[0].x*10, snakeParts[0].y*10))
	segments = append(segments, head)
	return container.NewWithoutLayout(segments...)
}
<commit_msg>05/22/2021 08:42:21 AM snake/snake.go<commit_after>package main

import (
	"fyne.io/fyne/v2"
	"fyne.io/fyne/v2/app"
	"fyne.io/fyne/v2/canvas"
	"fyne.io/fyne/v2/container"
	"image/color"
	"time"
)

type snakePart struct {
	x, y float32
}

type moveType int

const (
	moveUp moveType = iota
	moveDown
	moveLeft
	moveRight
)

var snakeParts []snakePart
var game *fyne.Container
var head *canvas.Rectangle
var move = moveUp

func keyTyped(e *fyne.KeyEvent) {
	switch e.Name {
	case fyne.KeyUp:
		move = moveUp
	case fyne.KeyDown:
		move = moveDown
	case fyne.KeyLeft:
		move = moveLeft
	case fyne.KeyRight:
		move = moveRight
	}
}

func main() {
	a := app.New()

	w := a.NewWindow("Snake")
	w.Resize(fyne.NewSize(800, 800))
	w.SetFixedSize(true)

	game = setupGame()
	w.SetContent(game)
	w.Canvas().SetOnTypedKey(keyTyped)

	go runGame()
	w.ShowAndRun()
}

func refreshGame() {
	for i, seg := range snakeParts {
		rect := game.Objects[i]
		rect.Move(fyne.NewPos(seg.x*10, seg.y*10))
	}

	game.Refresh()
}

func runGame() {
	nextPart := snakePart{snakeParts[0].x, snakeParts[0].y - 1}
	for {
		oldPos := fyne.NewPos(snakeParts[0].x*10, snakeParts[0].y*10)
		newPos := fyne.NewPos(nextPart.x*10, nextPart.y*10)
		canvas.NewPositionAnimation(oldPos, newPos, time.Millisecond*250, func(p fyne.Position) {
			head.Move(p)
			canvas.Refresh(head)
		}).Start()

		end := len(snakeParts) - 1
		canvas.NewPositionAnimation(fyne.NewPos(snakeParts[end].x*10, snakeParts[end].y*10),
			fyne.NewPos(snakeParts[end-1].x*10, snakeParts[end-1].y*10), time.Millisecond*250,
			func(p fyne.Position) {
				tail := game.Objects[end]
				tail.Move(p)
				canvas.Refresh(tail)
			}).Start()

		time.Sleep(time.Millisecond * 250)
		for i := len(snakeParts) - 1; i >= 1; i-- {
			snakeParts[i] = snakeParts[i-1]
		}
		snakeParts[0] = nextPart
		refreshGame()
		switch move {
		case moveUp:
			nextPart = snakePart{nextPart.x, nextPart.y - 1}
		case moveDown:
			nextPart = snakePart{nextPart.x, nextPart.y + 1}
		case moveLeft:
			nextPart = snakePart{nextPart.x - 1, nextPart.y}
		case moveRight:
			nextPart = snakePart{nextPart.x + 1, nextPart.y}
		}
	}
}

func setupGame() *fyne.Container {
	var segments []fyne.CanvasObject
	for i := 0; i < 10; i++ {
		seg := snakePart{9, float32(5 + i)}
		snakeParts = append(snakeParts, seg)

		r := canvas.NewRectangle(&color.RGBA{G: 0x66, A: 0xff})
		r.Resize(fyne.NewSize(10, 10))
		r.Move(fyne.NewPos(90, float32(50+i*10)))
		segments = append(segments, r)
	}

	head = canvas.NewRectangle(&color.RGBA{G: 0x66, A: 0xff})
	head.Resize(fyne.NewSize(10, 10))
	head.Move(fyne.NewPos(snakeParts[0].x*10, snakeParts[0].y*10))
	segments = append(segments, head)
	return container.NewWithoutLayout(segments...)
}
<|endoftext|>


<commit_before>// +build !windows

package fs

import (
	"os"
	"path/filepath"
	"syscall"
)

func diskUsage(roots ...string) (Usage, error) {
	type inode struct {
		// TODO(stevvooe): Can probably reduce memory usage by not tracking
		// device, but we can leave this right for now.
		dev, ino uint64
	}

	var (
		size   int64
		inodes = map[inode]struct{}{} // expensive!
	)

	for _, root := range roots {
		if err := filepath.Walk(root, func(path string, fi os.FileInfo, err error) error {
			if err != nil {
				return err
			}

			stat := fi.Sys().(*syscall.Stat_t)
			inodes[inode{dev: uint64(stat.Dev), ino: uint64(stat.Ino)}] = struct{}{}
			size += fi.Size()
			return nil
		}); err != nil {
			return Usage{}, err
		}
	}

	return Usage{
		Inodes: int64(len(inodes)),
		Size:   size,
	}, nil
}
<commit_msg>fs: fix hardlinks detection on usage calc<commit_after>// +build !windows

package fs

import (
	"os"
	"path/filepath"
	"syscall"
)

func diskUsage(roots ...string) (Usage, error) {
	type inode struct {
		// TODO(stevvooe): Can probably reduce memory usage by not tracking
		// device, but we can leave this right for now.
		dev, ino uint64
	}

	var (
		size   int64
		inodes = map[inode]struct{}{} // expensive!
	)

	for _, root := range roots {
		if err := filepath.Walk(root, func(path string, fi os.FileInfo, err error) error {
			if err != nil {
				return err
			}

			stat := fi.Sys().(*syscall.Stat_t)

			inoKey := inode{dev: uint64(stat.Dev), ino: uint64(stat.Ino)}
			if _, ok := inodes[inoKey]; !ok {
				inodes[inoKey] = struct{}{}
				size += fi.Size()
			}

			return nil
		}); err != nil {
			return Usage{}, err
		}
	}

	return Usage{
		Inodes: int64(len(inodes)),
		Size:   size,
	}, nil
}
<|endoftext|>


<commit_before>/*

This provides Run for use in creating test suites

To use this declare a TestMain

// TestMain drives the tests
func TestMain(m *testing.M) {
	fstest.TestMain(m)
}

And then make and destroy a Run in each test

func TestMkdir(t *testing.T) {
	r := fstest.NewRun(t)
	defer r.Finalise()
	// test stuff
}

This will make r.Fremote and r.Flocal for a remote remote and a local
remote.  The remote is determined by the -remote flag passed in.

*/

package fstest

import (
	"bytes"
	"flag"
	"io/ioutil"
	"log"
	"os"
	"path"
	"path/filepath"
	"sort"
	"testing"
	"time"

	"github.com/ncw/rclone/fs"
	"github.com/ncw/rclone/fs/fserrors"
	"github.com/ncw/rclone/fs/object"
	"github.com/ncw/rclone/fs/walk"
	"github.com/stretchr/testify/require"
)

// Run holds the remotes for a test run
type Run struct {
	LocalName    string
	Flocal       fs.Fs
	Fremote      fs.Fs
	FremoteName  string
	cleanRemote  func()
	mkdir        map[string]bool // whether the remote has been made yet for the fs name
	Logf, Fatalf func(text string, args ...interface{})
}

// TestMain drives the tests
func TestMain(m *testing.M) {
	flag.Parse()
	if !*Individual {
		oneRun = newRun()
	}
	rc := m.Run()
	if !*Individual {
		oneRun.Finalise()
	}
	os.Exit(rc)
}

// oneRun holds the master run data if individual is not set
var oneRun *Run

// newRun initialise the remote and local for testing and returns a
// run object.
//
// r.Flocal is an empty local Fs
// r.Fremote is an empty remote Fs
//
// Finalise() will tidy them away when done.
func newRun() *Run {
	r := &Run{
		Logf:   log.Printf,
		Fatalf: log.Fatalf,
		mkdir:  make(map[string]bool),
	}

	Initialise()

	var err error
	r.Fremote, r.FremoteName, r.cleanRemote, err = RandomRemote(*RemoteName, *SubDir)
	if err != nil {
		r.Fatalf("Failed to open remote %q: %v", *RemoteName, err)
	}

	r.LocalName, err = ioutil.TempDir("", "rclone")
	if err != nil {
		r.Fatalf("Failed to create temp dir: %v", err)
	}
	r.LocalName = filepath.ToSlash(r.LocalName)
	r.Flocal, err = fs.NewFs(r.LocalName)
	if err != nil {
		r.Fatalf("Failed to make %q: %v", r.LocalName, err)
	}
	fs.CalculateModifyWindow(r.Fremote, r.Flocal)
	return r
}

// dirsToRemove sorts by string length
type dirsToRemove []string

func (d dirsToRemove) Len() int           { return len(d) }
func (d dirsToRemove) Swap(i, j int)      { d[i], d[j] = d[j], d[i] }
func (d dirsToRemove) Less(i, j int) bool { return len(d[i]) > len(d[j]) }

// NewRun initialise the remote and local for testing and returns a
// run object.  Call this from the tests.
//
// r.Flocal is an empty local Fs
// r.Fremote is an empty remote Fs
//
// Finalise() will tidy them away when done.
func NewRun(t *testing.T) *Run {
	var r *Run
	if *Individual {
		r = newRun()
	} else {
		// If not individual, use the global one with the clean method overridden
		r = new(Run)
		*r = *oneRun
		r.cleanRemote = func() {
			var toDelete dirsToRemove
			err := walk.Walk(r.Fremote, "", true, -1, func(dirPath string, entries fs.DirEntries, err error) error {
				if err != nil {
					if err == fs.ErrorDirNotFound {
						return nil
					}
					t.Fatalf("Error listing: %v", err)
				}
				for _, entry := range entries {
					switch x := entry.(type) {
					case fs.Object:
						err = x.Remove()
						if err != nil {
							t.Errorf("Error removing file %q: %v", x.Remote(), err)
						}
					case fs.Directory:
						toDelete = append(toDelete, x.Remote())
					}
				}
				return nil
			})
			if err == fs.ErrorDirNotFound {
				return
			}
			require.NoError(t, err)
			sort.Sort(toDelete)
			for _, dir := range toDelete {
				err := r.Fremote.Rmdir(dir)
				if err != nil {
					t.Errorf("Error removing dir %q: %v", dir, err)
				}
			}
			// Check remote is empty
			CheckListingWithPrecision(t, r.Fremote, []Item{}, []string{}, r.Fremote.Precision())
		}
	}
	r.Logf = t.Logf
	r.Fatalf = t.Fatalf
	r.Logf("Remote %q, Local %q, Modify Window %q", r.Fremote, r.Flocal, fs.Config.ModifyWindow)
	return r
}

// RenameFile renames a file in local
func (r *Run) RenameFile(item Item, newpath string) Item {
	oldFilepath := path.Join(r.LocalName, item.Path)
	newFilepath := path.Join(r.LocalName, newpath)
	if err := os.Rename(oldFilepath, newFilepath); err != nil {
		r.Fatalf("Failed to rename file from %q to %q: %v", item.Path, newpath, err)
	}

	item.Path = newpath

	return item
}

// WriteFile writes a file to local
func (r *Run) WriteFile(filePath, content string, t time.Time) Item {
	item := NewItem(filePath, content, t)
	// FIXME make directories?
	filePath = path.Join(r.LocalName, filePath)
	dirPath := path.Dir(filePath)
	err := os.MkdirAll(dirPath, 0770)
	if err != nil {
		r.Fatalf("Failed to make directories %q: %v", dirPath, err)
	}
	err = ioutil.WriteFile(filePath, []byte(content), 0600)
	if err != nil {
		r.Fatalf("Failed to write file %q: %v", filePath, err)
	}
	err = os.Chtimes(filePath, t, t)
	if err != nil {
		r.Fatalf("Failed to chtimes file %q: %v", filePath, err)
	}
	return item
}

// ForceMkdir creates the remote
func (r *Run) ForceMkdir(f fs.Fs) {
	err := f.Mkdir("")
	if err != nil {
		r.Fatalf("Failed to mkdir %q: %v", f, err)
	}
	r.mkdir[f.String()] = true
}

// Mkdir creates the remote if it hasn't been created already
func (r *Run) Mkdir(f fs.Fs) {
	if !r.mkdir[f.String()] {
		r.ForceMkdir(f)
	}
}

// WriteObjectTo writes an object to the fs, remote passed in
func (r *Run) WriteObjectTo(f fs.Fs, remote, content string, modTime time.Time, useUnchecked bool) Item {
	put := f.Put
	if useUnchecked {
		put = f.Features().PutUnchecked
		if put == nil {
			r.Fatalf("Fs doesn't support PutUnchecked")
		}
	}
	r.Mkdir(f)
	const maxTries = 10
	for tries := 1; ; tries++ {
		in := bytes.NewBufferString(content)
		objinfo := object.NewStaticObjectInfo(remote, modTime, int64(len(content)), true, nil, nil)
		_, err := put(in, objinfo)
		if err == nil {
			break
		}
		// Retry if err returned a retry error
		if fserrors.IsRetryError(err) && tries < maxTries {
			r.Logf("Retry Put of %q to %v: %d/%d (%v)", remote, f, tries, maxTries, err)
			time.Sleep(2 * time.Second)
			continue
		}
		r.Fatalf("Failed to put %q to %q: %v", remote, f, err)
	}
	return NewItem(remote, content, modTime)
}

// WriteObject writes an object to the remote
func (r *Run) WriteObject(remote, content string, modTime time.Time) Item {
	return r.WriteObjectTo(r.Fremote, remote, content, modTime, false)
}

// WriteUncheckedObject writes an object to the remote not checking for duplicates
func (r *Run) WriteUncheckedObject(remote, content string, modTime time.Time) Item {
	return r.WriteObjectTo(r.Fremote, remote, content, modTime, true)
}

// WriteBoth calls WriteObject and WriteFile with the same arguments
func (r *Run) WriteBoth(remote, content string, modTime time.Time) Item {
	r.WriteFile(remote, content, modTime)
	return r.WriteObject(remote, content, modTime)
}

// CheckWithDuplicates does a test but allows duplicates
func (r *Run) CheckWithDuplicates(t *testing.T, items ...Item) {
	panic("FIXME")
	// objects, size, err := operations.Count(r.Fremote)
	// require.NoError(t, err)
	// assert.Equal(t, int64(len(items)), objects)
	// wantSize := int64(0)
	// for _, item := range items {
	// 	wantSize += item.Size
	// }
	// assert.Equal(t, wantSize, size)
}

// Clean the temporary directory
func (r *Run) cleanTempDir() {
	err := os.RemoveAll(r.LocalName)
	if err != nil {
		r.Logf("Failed to clean temporary directory %q: %v", r.LocalName, err)
	}
}

// Finalise cleans the remote and local
func (r *Run) Finalise() {
	// r.Logf("Cleaning remote %q", r.Fremote)
	r.cleanRemote()
	// r.Logf("Cleaning local %q", r.LocalName)
	r.cleanTempDir()
}
<commit_msg>fstest: Fix CheckWithDuplicates after code reshuffle to not use operations<commit_after>/*

This provides Run for use in creating test suites

To use this declare a TestMain

// TestMain drives the tests
func TestMain(m *testing.M) {
	fstest.TestMain(m)
}

And then make and destroy a Run in each test

func TestMkdir(t *testing.T) {
	r := fstest.NewRun(t)
	defer r.Finalise()
	// test stuff
}

This will make r.Fremote and r.Flocal for a remote remote and a local
remote.  The remote is determined by the -remote flag passed in.

*/

package fstest

import (
	"bytes"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path"
	"path/filepath"
	"sort"
	"testing"
	"time"

	"github.com/ncw/rclone/fs"
	"github.com/ncw/rclone/fs/fserrors"
	"github.com/ncw/rclone/fs/object"
	"github.com/ncw/rclone/fs/walk"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

// Run holds the remotes for a test run
type Run struct {
	LocalName    string
	Flocal       fs.Fs
	Fremote      fs.Fs
	FremoteName  string
	cleanRemote  func()
	mkdir        map[string]bool // whether the remote has been made yet for the fs name
	Logf, Fatalf func(text string, args ...interface{})
}

// TestMain drives the tests
func TestMain(m *testing.M) {
	flag.Parse()
	if !*Individual {
		oneRun = newRun()
	}
	rc := m.Run()
	if !*Individual {
		oneRun.Finalise()
	}
	os.Exit(rc)
}

// oneRun holds the master run data if individual is not set
var oneRun *Run

// newRun initialise the remote and local for testing and returns a
// run object.
//
// r.Flocal is an empty local Fs
// r.Fremote is an empty remote Fs
//
// Finalise() will tidy them away when done.
func newRun() *Run {
	r := &Run{
		Logf:   log.Printf,
		Fatalf: log.Fatalf,
		mkdir:  make(map[string]bool),
	}

	Initialise()

	var err error
	r.Fremote, r.FremoteName, r.cleanRemote, err = RandomRemote(*RemoteName, *SubDir)
	if err != nil {
		r.Fatalf("Failed to open remote %q: %v", *RemoteName, err)
	}

	r.LocalName, err = ioutil.TempDir("", "rclone")
	if err != nil {
		r.Fatalf("Failed to create temp dir: %v", err)
	}
	r.LocalName = filepath.ToSlash(r.LocalName)
	r.Flocal, err = fs.NewFs(r.LocalName)
	if err != nil {
		r.Fatalf("Failed to make %q: %v", r.LocalName, err)
	}
	fs.CalculateModifyWindow(r.Fremote, r.Flocal)
	return r
}

// dirsToRemove sorts by string length
type dirsToRemove []string

func (d dirsToRemove) Len() int           { return len(d) }
func (d dirsToRemove) Swap(i, j int)      { d[i], d[j] = d[j], d[i] }
func (d dirsToRemove) Less(i, j int) bool { return len(d[i]) > len(d[j]) }

// NewRun initialise the remote and local for testing and returns a
// run object.  Call this from the tests.
//
// r.Flocal is an empty local Fs
// r.Fremote is an empty remote Fs
//
// Finalise() will tidy them away when done.
func NewRun(t *testing.T) *Run {
	var r *Run
	if *Individual {
		r = newRun()
	} else {
		// If not individual, use the global one with the clean method overridden
		r = new(Run)
		*r = *oneRun
		r.cleanRemote = func() {
			var toDelete dirsToRemove
			err := walk.Walk(r.Fremote, "", true, -1, func(dirPath string, entries fs.DirEntries, err error) error {
				if err != nil {
					if err == fs.ErrorDirNotFound {
						return nil
					}
					t.Fatalf("Error listing: %v", err)
				}
				for _, entry := range entries {
					switch x := entry.(type) {
					case fs.Object:
						err = x.Remove()
						if err != nil {
							t.Errorf("Error removing file %q: %v", x.Remote(), err)
						}
					case fs.Directory:
						toDelete = append(toDelete, x.Remote())
					}
				}
				return nil
			})
			if err == fs.ErrorDirNotFound {
				return
			}
			require.NoError(t, err)
			sort.Sort(toDelete)
			for _, dir := range toDelete {
				err := r.Fremote.Rmdir(dir)
				if err != nil {
					t.Errorf("Error removing dir %q: %v", dir, err)
				}
			}
			// Check remote is empty
			CheckListingWithPrecision(t, r.Fremote, []Item{}, []string{}, r.Fremote.Precision())
		}
	}
	r.Logf = t.Logf
	r.Fatalf = t.Fatalf
	r.Logf("Remote %q, Local %q, Modify Window %q", r.Fremote, r.Flocal, fs.Config.ModifyWindow)
	return r
}

// RenameFile renames a file in local
func (r *Run) RenameFile(item Item, newpath string) Item {
	oldFilepath := path.Join(r.LocalName, item.Path)
	newFilepath := path.Join(r.LocalName, newpath)
	if err := os.Rename(oldFilepath, newFilepath); err != nil {
		r.Fatalf("Failed to rename file from %q to %q: %v", item.Path, newpath, err)
	}

	item.Path = newpath

	return item
}

// WriteFile writes a file to local
func (r *Run) WriteFile(filePath, content string, t time.Time) Item {
	item := NewItem(filePath, content, t)
	// FIXME make directories?
	filePath = path.Join(r.LocalName, filePath)
	dirPath := path.Dir(filePath)
	err := os.MkdirAll(dirPath, 0770)
	if err != nil {
		r.Fatalf("Failed to make directories %q: %v", dirPath, err)
	}
	err = ioutil.WriteFile(filePath, []byte(content), 0600)
	if err != nil {
		r.Fatalf("Failed to write file %q: %v", filePath, err)
	}
	err = os.Chtimes(filePath, t, t)
	if err != nil {
		r.Fatalf("Failed to chtimes file %q: %v", filePath, err)
	}
	return item
}

// ForceMkdir creates the remote
func (r *Run) ForceMkdir(f fs.Fs) {
	err := f.Mkdir("")
	if err != nil {
		r.Fatalf("Failed to mkdir %q: %v", f, err)
	}
	r.mkdir[f.String()] = true
}

// Mkdir creates the remote if it hasn't been created already
func (r *Run) Mkdir(f fs.Fs) {
	if !r.mkdir[f.String()] {
		r.ForceMkdir(f)
	}
}

// WriteObjectTo writes an object to the fs, remote passed in
func (r *Run) WriteObjectTo(f fs.Fs, remote, content string, modTime time.Time, useUnchecked bool) Item {
	put := f.Put
	if useUnchecked {
		put = f.Features().PutUnchecked
		if put == nil {
			r.Fatalf("Fs doesn't support PutUnchecked")
		}
	}
	r.Mkdir(f)
	const maxTries = 10
	for tries := 1; ; tries++ {
		in := bytes.NewBufferString(content)
		objinfo := object.NewStaticObjectInfo(remote, modTime, int64(len(content)), true, nil, nil)
		_, err := put(in, objinfo)
		if err == nil {
			break
		}
		// Retry if err returned a retry error
		if fserrors.IsRetryError(err) && tries < maxTries {
			r.Logf("Retry Put of %q to %v: %d/%d (%v)", remote, f, tries, maxTries, err)
			time.Sleep(2 * time.Second)
			continue
		}
		r.Fatalf("Failed to put %q to %q: %v", remote, f, err)
	}
	return NewItem(remote, content, modTime)
}

// WriteObject writes an object to the remote
func (r *Run) WriteObject(remote, content string, modTime time.Time) Item {
	return r.WriteObjectTo(r.Fremote, remote, content, modTime, false)
}

// WriteUncheckedObject writes an object to the remote not checking for duplicates
func (r *Run) WriteUncheckedObject(remote, content string, modTime time.Time) Item {
	return r.WriteObjectTo(r.Fremote, remote, content, modTime, true)
}

// WriteBoth calls WriteObject and WriteFile with the same arguments
func (r *Run) WriteBoth(remote, content string, modTime time.Time) Item {
	r.WriteFile(remote, content, modTime)
	return r.WriteObject(remote, content, modTime)
}

// CheckWithDuplicates does a test but allows duplicates
func (r *Run) CheckWithDuplicates(t *testing.T, items ...Item) {
	var want, got []string

	// construct a []string of desired items
	for _, item := range items {
		want = append(want, fmt.Sprintf("%q %d", item.Path, item.Size))
	}
	sort.Strings(want)

	// do the listing
	objs, _, err := walk.GetAll(r.Fremote, "", true, -1)
	if err != nil && err != fs.ErrorDirNotFound {
		t.Fatalf("Error listing: %v", err)
	}

	// construct a []string of actual items
	for _, o := range objs {
		got = append(got, fmt.Sprintf("%q %d", o.Remote(), o.Size()))
	}
	sort.Strings(got)

	assert.Equal(t, want, got)
}

// Clean the temporary directory
func (r *Run) cleanTempDir() {
	err := os.RemoveAll(r.LocalName)
	if err != nil {
		r.Logf("Failed to clean temporary directory %q: %v", r.LocalName, err)
	}
}

// Finalise cleans the remote and local
func (r *Run) Finalise() {
	// r.Logf("Cleaning remote %q", r.Fremote)
	r.cleanRemote()
	// r.Logf("Cleaning local %q", r.LocalName)
	r.cleanTempDir()
}
<|endoftext|>


<commit_before>// Copyright 2016 The go-github AUTHORS. All rights reserved.
//
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package github_test

import (
	"context"
	"fmt"
	"log"

	"github.com/google/go-github/github"
)

func ExampleClient_Markdown() {
	client := github.NewClient(nil)

	input := "# heading #\n\nLink to issue #1"
	opt := &github.MarkdownOptions{Mode: "gfm", Context: "google/go-github"}

	output, _, err := client.Markdown(context.Background(), input, opt)
	if err != nil {
		fmt.Println(err)
	}

	fmt.Println(output)
}

func ExampleRepositoriesService_GetReadme() {
	client := github.NewClient(nil)

	readme, _, err := client.Repositories.GetReadme(context.Background(), "google", "go-github", nil)
	if err != nil {
		fmt.Println(err)
		return
	}

	content, err := readme.GetContent()
	if err != nil {
		fmt.Println(err)
		return
	}

	fmt.Printf("google/go-github README:\n%v\n", content)
}

func ExampleRepositoriesService_List() {
	client := github.NewClient(nil)

	user := "willnorris"
	opt := &github.RepositoryListOptions{Type: "owner", Sort: "updated", Direction: "desc"}

	repos, _, err := client.Repositories.List(context.Background(), user, opt)
	if err != nil {
		fmt.Println(err)
	}

	fmt.Printf("Recently updated repositories by %q: %v", user, github.Stringify(repos))
}

func ExampleUsersService_ListAll() {
	client := github.NewClient(nil)
	opts := &github.UserListOptions{}
	for {
		users, _, err := client.Users.ListAll(context.Background(), opts)
		if err != nil {
			log.Fatalf("error listing users: %v", err)
		}
		if len(users) == 0 {
			break
		}
		opts.Since = *users[len(users)-1].ID
		// Process users...
	}
}

func ExamplePullRequestsService_Create() {
	// In this example we're creating a PR and displaying the HTML url at the end.

	// Note that authentication is needed here as you are performing a modification
	// so you will need to modify the example to provide an oauth client to
	// github.NewClient() instead of nil. See the following documentation for more
	// information on how to authenticate with the client:
	// https://godoc.org/github.com/google/go-github/github#hdr-Authentication
	client := github.NewClient(nil)

	newPR := &github.NewPullRequest{
		Title:               github.String("My awesome pull request"),
		Head:                github.String("branch_to_merge"),
		Base:                github.String("master"),
		Body:                github.String("This is the description of the PR created with the package `github.com/google/go-github/github`"),
		MaintainerCanModify: github.Bool(true),
	}

	pr, _, err := client.PullRequests.Create(context.Background(), "myOrganization", "myRepository", newPR)
	if err != nil {
		fmt.Println(err)
		return
	}

	fmt.Printf("PR created: %s\n", pr.GetHTMLURL())
}
<commit_msg>Add example for OrganizationsService.ListTeams. (#896)<commit_after>// Copyright 2016 The go-github AUTHORS. All rights reserved.
//
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package github_test

import (
	"context"
	"fmt"
	"log"

	"github.com/google/go-github/github"
)

func ExampleClient_Markdown() {
	client := github.NewClient(nil)

	input := "# heading #\n\nLink to issue #1"
	opt := &github.MarkdownOptions{Mode: "gfm", Context: "google/go-github"}

	output, _, err := client.Markdown(context.Background(), input, opt)
	if err != nil {
		fmt.Println(err)
	}

	fmt.Println(output)
}

func ExampleRepositoriesService_GetReadme() {
	client := github.NewClient(nil)

	readme, _, err := client.Repositories.GetReadme(context.Background(), "google", "go-github", nil)
	if err != nil {
		fmt.Println(err)
		return
	}

	content, err := readme.GetContent()
	if err != nil {
		fmt.Println(err)
		return
	}

	fmt.Printf("google/go-github README:\n%v\n", content)
}

func ExampleRepositoriesService_List() {
	client := github.NewClient(nil)

	user := "willnorris"
	opt := &github.RepositoryListOptions{Type: "owner", Sort: "updated", Direction: "desc"}

	repos, _, err := client.Repositories.List(context.Background(), user, opt)
	if err != nil {
		fmt.Println(err)
	}

	fmt.Printf("Recently updated repositories by %q: %v", user, github.Stringify(repos))
}

func ExampleUsersService_ListAll() {
	client := github.NewClient(nil)
	opts := &github.UserListOptions{}
	for {
		users, _, err := client.Users.ListAll(context.Background(), opts)
		if err != nil {
			log.Fatalf("error listing users: %v", err)
		}
		if len(users) == 0 {
			break
		}
		opts.Since = *users[len(users)-1].ID
		// Process users...
	}
}

func ExamplePullRequestsService_Create() {
	// In this example we're creating a PR and displaying the HTML url at the end.

	// Note that authentication is needed here as you are performing a modification
	// so you will need to modify the example to provide an oauth client to
	// github.NewClient() instead of nil. See the following documentation for more
	// information on how to authenticate with the client:
	// https://godoc.org/github.com/google/go-github/github#hdr-Authentication
	client := github.NewClient(nil)

	newPR := &github.NewPullRequest{
		Title:               github.String("My awesome pull request"),
		Head:                github.String("branch_to_merge"),
		Base:                github.String("master"),
		Body:                github.String("This is the description of the PR created with the package `github.com/google/go-github/github`"),
		MaintainerCanModify: github.Bool(true),
	}

	pr, _, err := client.PullRequests.Create(context.Background(), "myOrganization", "myRepository", newPR)
	if err != nil {
		fmt.Println(err)
		return
	}

	fmt.Printf("PR created: %s\n", pr.GetHTMLURL())
}

func ExampleOrganizationsService_ListTeams() {
	// This example shows how to get a team ID corresponding to a given team name.

	// Note that authentication is needed here as you are performing a lookup on
	// an organization's administrative configuration, so you will need to modify
	// the example to provide an oauth client to github.NewClient() instead of nil.
	// See the following documentation for more information on how to authenticate
	// with the client:
	// https://godoc.org/github.com/google/go-github/github#hdr-Authentication
	client := github.NewClient(nil)

	teamName := "Developers team"
	ctx := context.Background()
	opts := &github.ListOptions{}

	for {
		teams, resp, err := client.Organizations.ListTeams(ctx, "myOrganization", opts)
		if err != nil {
			fmt.Println(err)
			return
		}
		for _, t := range teams {
			if t.GetName() == teamName {
				fmt.Printf("Team %q has ID %d\n", teamName, t.GetID())
				return
			}
		}
		if resp.NextPage == 0 {
			break
		}
		opts.Page = resp.NextPage
	}

	fmt.Printf("Team %q was not found\n", teamName)
}
<|endoftext|>


<commit_before>// Copyright 2015 The Vanadium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package test

import (
	"path/filepath"
	"time"

	"v.io/jiri/collect"
	"v.io/jiri/project"
	"v.io/jiri/runutil"
	"v.io/jiri/tool"
	"v.io/x/devtools/internal/test"
)

const (
	defaultPlaygroundTestTimeout = 5 * time.Minute
)

// vanadiumPlaygroundTest runs integration tests for the Vanadium playground.
//
// TODO(ivanpi): Port the namespace browser test logic from shell to Go. Add more tests.
func vanadiumPlaygroundTest(ctx *tool.Context, testName string, _ ...Opt) (_ *test.Result, e error) {
	root, err := project.JiriRoot()
	if err != nil {
		return nil, err
	}

	// Initialize the test.
	// Need the new-stype base profile since many web tests will build
	// go apps that need it.
	cleanup, err := initTest(ctx, testName, []string{"base", "nodejs"})
	if err != nil {
		return nil, internalTestError{err, "Init"}
	}
	defer collect.Error(func() error { return cleanup() }, &e)

	playgroundDir := filepath.Join(root, "release", "projects", "playground")
	backendDir := filepath.Join(playgroundDir, "go", "src", "v.io", "x", "playground")
	clientDir := filepath.Join(playgroundDir, "client")

	// Clean the playground client build.
	if err := ctx.Run().Chdir(clientDir); err != nil {
		return nil, err
	}
	if err := ctx.Run().Command("make", "clean"); err != nil {
		return nil, err
	}

	// Run builder integration test.
	if testResult, err := vanadiumPlaygroundSubtest(ctx, testName, "builder integration", backendDir, "test"); testResult != nil || err != nil {
		return testResult, err
	}

	// Run client embedded example test.
	if testResult, err := vanadiumPlaygroundSubtest(ctx, testName, "client embedded example", clientDir, "test"); testResult != nil || err != nil {
		return testResult, err
	}

	return &test.Result{Status: test.Passed}, nil
}

// Runs specified make target in the specified directory as a test case.
// On success, both return values are nil.
func vanadiumPlaygroundSubtest(ctx *tool.Context, testName, caseName, casePath, caseTarget string) (tr *test.Result, err error) {
	if err = ctx.Run().Chdir(casePath); err != nil {
		return
	}
	if err := ctx.Run().TimedCommand(defaultPlaygroundTestTimeout, "make", caseTarget); err != nil {
		if err == runutil.CommandTimedOutErr {
			return &test.Result{
				Status:       test.TimedOut,
				TimeoutValue: defaultPlaygroundTestTimeout,
			}, nil
		} else {
			return nil, internalTestError{err, "Make " + caseTarget}
		}
	}
	return
}
<commit_msg>v.io/x/devtools/jiri-test: extend playground timeout to 10mins.<commit_after>// Copyright 2015 The Vanadium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package test

import (
	"path/filepath"
	"time"

	"v.io/jiri/collect"
	"v.io/jiri/project"
	"v.io/jiri/runutil"
	"v.io/jiri/tool"
	"v.io/x/devtools/internal/test"
)

const (
	defaultPlaygroundTestTimeout = 10 * time.Minute
)

// vanadiumPlaygroundTest runs integration tests for the Vanadium playground.
//
// TODO(ivanpi): Port the namespace browser test logic from shell to Go. Add more tests.
func vanadiumPlaygroundTest(ctx *tool.Context, testName string, _ ...Opt) (_ *test.Result, e error) {
	root, err := project.JiriRoot()
	if err != nil {
		return nil, err
	}

	// Initialize the test.
	// Need the new-stype base profile since many web tests will build
	// go apps that need it.
	cleanup, err := initTest(ctx, testName, []string{"base", "nodejs"})
	if err != nil {
		return nil, internalTestError{err, "Init"}
	}
	defer collect.Error(func() error { return cleanup() }, &e)

	playgroundDir := filepath.Join(root, "release", "projects", "playground")
	backendDir := filepath.Join(playgroundDir, "go", "src", "v.io", "x", "playground")
	clientDir := filepath.Join(playgroundDir, "client")

	// Clean the playground client build.
	if err := ctx.Run().Chdir(clientDir); err != nil {
		return nil, err
	}
	if err := ctx.Run().Command("make", "clean"); err != nil {
		return nil, err
	}

	// Run builder integration test.
	if testResult, err := vanadiumPlaygroundSubtest(ctx, testName, "builder integration", backendDir, "test"); testResult != nil || err != nil {
		return testResult, err
	}

	// Run client embedded example test.
	if testResult, err := vanadiumPlaygroundSubtest(ctx, testName, "client embedded example", clientDir, "test"); testResult != nil || err != nil {
		return testResult, err
	}

	return &test.Result{Status: test.Passed}, nil
}

// Runs specified make target in the specified directory as a test case.
// On success, both return values are nil.
func vanadiumPlaygroundSubtest(ctx *tool.Context, testName, caseName, casePath, caseTarget string) (tr *test.Result, err error) {
	if err = ctx.Run().Chdir(casePath); err != nil {
		return
	}
	if err := ctx.Run().TimedCommand(defaultPlaygroundTestTimeout, "make", caseTarget); err != nil {
		if err == runutil.CommandTimedOutErr {
			return &test.Result{
				Status:       test.TimedOut,
				TimeoutValue: defaultPlaygroundTestTimeout,
			}, nil
		} else {
			return nil, internalTestError{err, "Make " + caseTarget}
		}
	}
	return
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/CapillarySoftware/gostat/stat"
	"github.com/CapillarySoftware/gostat/bucketer"
	"github.com/CapillarySoftware/gostat/aggregator"
	"fmt"
	"time"
	"os"
	"os/signal"
	"math/rand"
	nano "github.com/op/go-nanomsg"
	log  "github.com/cihub/seelog"
)

func main() {
	stats              := make(chan *stat.Stat)   // stats received from producers
	bucketedStats      := make(chan []*stat.Stat) // raw bucketed (non-aggregated) stats are output here
	shutdownBucketer   := make(chan bool)         // used to signal the bucketer we are done
	shutdownListener   := make(chan bool)         // used to signal the socket listener we are done
	shutdownAggregator := make(chan bool)         // used to signal the aggregator we are done

  installCtrlCHandler(shutdownBucketer, shutdownListener, shutdownAggregator)

	// create and start a Bucketer
	b := bucketer.NewBucketer(stats, bucketedStats, shutdownBucketer)
	go b.Run(time.Second * 5) // publish stats ~ every 15 seconds

	// TODO: replace this simulation of an Aggregator with a real one
	go func(bucketed <-chan []*stat.Stat, shutdown <-chan bool) {
		for {
			select {
				case bucket := <-bucketed : log.Debugf("aggregator simulation got stats bucket with length %d", len(bucket))
																		for _, stat := range bucket {
																			log.Debugf("aggregator sim: %#v", stat)
																		}
																		a := aggregator.Aggregate(bucket)
																		log.Debugf("aggregator sim aggregate: %#v", a)
				case <-shutdown : log.Info("aggregator simulation shutting down")
				                  break
			}
	}
	}(bucketedStats, shutdownAggregator)

	// start a socket listener
	go bindSocketListener(stats, shutdownListener)

	// TODO: remove this simulation of stats when the socket listener is really processing data
	for true {
		<-time.After(time.Second * time.Duration(rand.Intn(3))) // sleep 0-3 seconds

		// create a stat randomly named "stat1 ... stat10" with a random value between 1-100
		stat := stat.Stat{Name : fmt.Sprintf("stat%v", (rand.Intn(9)+1)), Timestamp : time.Now().UTC(), Value : float64(rand.Intn(99)+1)}
		stats <- &stat // send it to the Bucketer
	}
}

// installCtrlCHandler starts a goroutine that will signal the workers when it's time
// to shut down
func installCtrlCHandler(shutdown ...chan<- bool) {
	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt)

	go func() {
	  for sig := range c {
	    log.Infof(" captured %v, stopping stats collection and exiting...\n", sig)
	    for _, s := range shutdown {
	    	s <-true
	    }
	    <-time.After(time.Second * 5) // wait for a clean shutdown, TODO: wait on signal from all routines
	    log.Info("Done")
	    os.Exit(1)
	  }
	}()
}

func bindSocketListener(stats chan<- *stat.Stat, shutdown <-chan bool) {
	var (
		msg []byte
		err error
		done = false
	)
	socket, err := nano.NewPullSocket()

	if nil != err {
		log.Error(err)
	}

	socket.SetRecvTimeout(time.Second)

	defer socket.Close()
	_, err = socket.Bind("tcp://*:2025")
	if nil != err {
		log.Error(err)
	}
	log.Info("Ready to receive data")

	for !done {
		msg, err = socket.Recv(0) //blocking
		if nil != err {
			log.Error(err)
		}
		if nil != msg {
			// TODO: new up a Stat and send it on the stats channel
			log.Debug("Received message: ", msg)
		}

		select {
			case done = <- shutdown : break
			default                 : break
		}
	}

	log.Info("Exiting socket listener")
}

<commit_msg>removed bad comment<commit_after>package main

import (
	"github.com/CapillarySoftware/gostat/stat"
	"github.com/CapillarySoftware/gostat/bucketer"
	"github.com/CapillarySoftware/gostat/aggregator"
	"fmt"
	"time"
	"os"
	"os/signal"
	"math/rand"
	nano "github.com/op/go-nanomsg"
	log  "github.com/cihub/seelog"
)

func main() {
	stats              := make(chan *stat.Stat)   // stats received from producers
	bucketedStats      := make(chan []*stat.Stat) // raw bucketed (non-aggregated) stats are output here
	shutdownBucketer   := make(chan bool)         // used to signal the bucketer we are done
	shutdownListener   := make(chan bool)         // used to signal the socket listener we are done
	shutdownAggregator := make(chan bool)         // used to signal the aggregator we are done

  installCtrlCHandler(shutdownBucketer, shutdownListener, shutdownAggregator)

	// create and start a Bucketer
	b := bucketer.NewBucketer(stats, bucketedStats, shutdownBucketer)
	go b.Run(time.Second * 5)

	// TODO: replace this simulation of an Aggregator with a real one
	go func(bucketed <-chan []*stat.Stat, shutdown <-chan bool) {
		for {
			select {
				case bucket := <-bucketed : log.Debugf("aggregator simulation got stats bucket with length %d", len(bucket))
																		for _, stat := range bucket {
																			log.Debugf("aggregator sim: %#v", stat)
																		}
																		a := aggregator.Aggregate(bucket)
																		log.Debugf("aggregator sim aggregate: %#v", a)
				case <-shutdown : log.Info("aggregator simulation shutting down")
				                  break
			}
	}
	}(bucketedStats, shutdownAggregator)

	// start a socket listener
	go bindSocketListener(stats, shutdownListener)

	// TODO: remove this simulation of stats when the socket listener is really processing data
	for true {
		<-time.After(time.Second * time.Duration(rand.Intn(3))) // sleep 0-3 seconds

		// create a stat randomly named "stat1 ... stat10" with a random value between 1-100
		stat := stat.Stat{Name : fmt.Sprintf("stat%v", (rand.Intn(9)+1)), Timestamp : time.Now().UTC(), Value : float64(rand.Intn(99)+1)}
		stats <- &stat // send it to the Bucketer
	}
}

// installCtrlCHandler starts a goroutine that will signal the workers when it's time
// to shut down
func installCtrlCHandler(shutdown ...chan<- bool) {
	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt)

	go func() {
	  for sig := range c {
	    log.Infof(" captured %v, stopping stats collection and exiting...\n", sig)
	    for _, s := range shutdown {
	    	s <-true
	    }
	    <-time.After(time.Second * 5) // wait for a clean shutdown, TODO: wait on signal from all routines
	    log.Info("Done")
	    os.Exit(1)
	  }
	}()
}

func bindSocketListener(stats chan<- *stat.Stat, shutdown <-chan bool) {
	var (
		msg []byte
		err error
		done = false
	)
	socket, err := nano.NewPullSocket()

	if nil != err {
		log.Error(err)
	}

	socket.SetRecvTimeout(time.Second)

	defer socket.Close()
	_, err = socket.Bind("tcp://*:2025")
	if nil != err {
		log.Error(err)
	}
	log.Info("Ready to receive data")

	for !done {
		msg, err = socket.Recv(0) //blocking
		if nil != err {
			log.Error(err)
		}
		if nil != msg {
			// TODO: new up a Stat and send it on the stats channel
			log.Debug("Received message: ", msg)
		}

		select {
			case done = <- shutdown : break
			default                 : break
		}
	}

	log.Info("Exiting socket listener")
}

<|endoftext|>


<commit_before>package core

// Arguments represents a structured set of arguments passed to a predicate.
// It allows destructive operations to internal properties because it is
// guaranteed by Thunks that arguments objects are never reused as a function
// call creates a Thunk.
type Arguments struct {
	positionals   []Value
	expandedList  Value
	keywords      []KeywordArgument
	expandedDicts []Value
}

// NewArguments creates a new Arguments.
func NewArguments(
	ps []PositionalArgument,
	ks []KeywordArgument,
	ds []Value) Arguments {
	vs := make([]Value, 0, len(ps))
	l := Value(nil)

	for i, p := range ps {
		if p.expanded {
			l = mergePositionalArguments(ps[i:])
			break
		}

		vs = append(vs, p.value)
	}

	return Arguments{vs, l, ks, ds}
}

// NewPositionalArguments creates an Arguments which consists of unexpanded
// positional arguments.
func NewPositionalArguments(vs ...Value) Arguments {
	return Arguments{vs, nil, nil, nil}
}

func mergePositionalArguments(ps []PositionalArgument) Value {
	v := Value(EmptyList)

	// Optimization for a common pattern of (func a b c ... ..xs).
	// Note that Merge is O(n) but Prepend is O(1).
	if last := len(ps) - 1; ps[last].expanded {
		v = ps[last].value
		ps = ps[:last]
	}

	for i := len(ps) - 1; i >= 0; i-- {
		p := ps[i]

		if p.expanded {
			v = PApp(Merge, p.value, v)
		} else {
			v = cons(p.value, v)
		}
	}

	return v
}

func (args *Arguments) nextPositional() Value {
	if len(args.positionals) != 0 {
		v := args.positionals[0]
		args.positionals = args.positionals[1:]
		return v
	}

	if args.expandedList == nil {
		return nil
	}

	l := args.expandedList
	args.expandedList = PApp(Rest, l)
	return PApp(First, l)
}

func (args *Arguments) restPositionals() Value {
	vs := args.positionals
	l := args.expandedList
	args.positionals = nil
	args.expandedList = nil

	if l == nil {
		return NewList(vs...)
	} else if len(vs) == 0 {
		return l
	}

	return StrictPrepend(vs, l)
}

func (args *Arguments) searchKeyword(s string) Value {
	for i, k := range args.keywords {
		if s == k.name {
			args.keywords = append(args.keywords[:i], args.keywords[i+1:]...)
			return k.value
		}
	}

	for i, v := range args.expandedDicts {
		d, ok := EvalPure(v).(*DictionaryType)

		if !ok {
			return NotDictionaryError(v)
		}

		k := NewString(s)

		// Using DictionaryType.{Search,Remove} methods is safe here
		// because the key is always StringType.
		if v, ok := d.Search(k); ok {
			args.expandedDicts = append([]Value{}, args.expandedDicts...)
			args.expandedDicts[i] = d.Remove(k)
			return v
		}
	}

	return nil
}

func (args *Arguments) restKeywords() Value {
	ks := args.keywords
	ds := args.expandedDicts
	args.keywords = nil
	args.expandedDicts = nil

	d := EmptyDictionary

	for _, k := range ks {
		// Using DictionaryType.Insert method is safe here
		// because the key is always StringType.
		d = d.Insert(NewString(k.name), k.value)
	}

	return PApp(Merge, append([]Value{d}, ds...)...)
}

// Merge merges 2 sets of arguments into one.
func (args Arguments) Merge(old Arguments) Arguments {
	ks := append(args.keywords, old.keywords...)
	ds := append(args.expandedDicts, old.expandedDicts...)

	if args.expandedList == nil {
		return Arguments{append(args.positionals, old.positionals...), old.expandedList, ks, ds}
	}

	l := Value(EmptyList)

	if old.expandedList != nil {
		l = old.expandedList
	}

	return Arguments{
		args.positionals,
		PApp(Merge, args.expandedList, StrictPrepend(old.positionals, l)),
		ks,
		ds,
	}
}

func (args Arguments) empty() Value {
	if len(args.positionals) > 0 {
		return argumentError("%d positional arguments are left", len(args.positionals))
	}

	// Testing args.expandedList is impossible because we cannot know its length
	// without evaluating it.

	n := 0

	for _, t := range args.expandedDicts {
		v := EvalPure(t)
		d, ok := v.(*DictionaryType)

		if !ok {
			return NotDictionaryError(v)
		}

		n += d.Size()
	}

	if n != 0 || len(args.keywords) > 0 {
		return argumentError("%d keyword arguments are left", len(args.keywords)+n)
	}

	return nil
}
<commit_msg>Refactor arguments.go<commit_after>package core

// Arguments represents a structured set of arguments passed to a predicate.
// It allows destructive operations to internal properties because it is
// guaranteed by Thunks that arguments objects are never reused as a function
// call creates a Thunk.
type Arguments struct {
	positionals   []Value
	expandedList  Value
	keywords      []KeywordArgument
	expandedDicts []Value
}

// NewArguments creates a new Arguments.
func NewArguments(
	ps []PositionalArgument,
	ks []KeywordArgument,
	ds []Value) Arguments {
	vs := make([]Value, 0, len(ps))
	l := Value(nil)

	for i, p := range ps {
		if p.expanded {
			l = mergePositionalArguments(ps[i:])
			break
		}

		vs = append(vs, p.value)
	}

	return Arguments{vs, l, ks, ds}
}

// NewPositionalArguments creates an Arguments which consists of unexpanded
// positional arguments.
func NewPositionalArguments(vs ...Value) Arguments {
	return Arguments{vs, nil, nil, nil}
}

func mergePositionalArguments(ps []PositionalArgument) Value {
	v := Value(EmptyList)

	// Optimization for a common pattern of (func a b c ... ..xs).
	// Note that Merge is O(n) but Prepend is O(1).
	if last := len(ps) - 1; ps[last].expanded {
		v = ps[last].value
		ps = ps[:last]
	}

	for i := len(ps) - 1; i >= 0; i-- {
		p := ps[i]

		if p.expanded {
			v = PApp(Merge, p.value, v)
		} else {
			v = cons(p.value, v)
		}
	}

	return v
}

func (args *Arguments) nextPositional() Value {
	if len(args.positionals) != 0 {
		v := args.positionals[0]
		args.positionals = args.positionals[1:]
		return v
	}

	if args.expandedList == nil {
		return nil
	}

	l := args.expandedList
	args.expandedList = PApp(Rest, l)
	return PApp(First, l)
}

func (args *Arguments) restPositionals() Value {
	vs := args.positionals
	l := args.expandedList
	args.positionals = nil
	args.expandedList = nil

	if l == nil {
		return NewList(vs...)
	} else if len(vs) == 0 {
		return l
	}

	return StrictPrepend(vs, l)
}

func (args *Arguments) searchKeyword(s string) Value {
	for i, k := range args.keywords {
		if s == k.name {
			args.keywords = append(args.keywords[:i], args.keywords[i+1:]...)
			return k.value
		}
	}

	for i, v := range args.expandedDicts {
		d, ok := EvalPure(v).(*DictionaryType)

		if !ok {
			return NotDictionaryError(v)
		}

		k := NewString(s)

		// Using DictionaryType.{Search,Remove} methods is safe here
		// because the key is always StringType.
		if v, ok := d.Search(k); ok {
			args.expandedDicts = append([]Value{}, args.expandedDicts...)
			args.expandedDicts[i] = d.Remove(k)
			return v
		}
	}

	return nil
}

func (args *Arguments) restKeywords() Value {
	ks := args.keywords
	ds := args.expandedDicts
	args.keywords = nil
	args.expandedDicts = nil

	d := EmptyDictionary

	for _, k := range ks {
		// Using DictionaryType.Insert method is safe here
		// because the key is always StringType.
		d = d.Insert(NewString(k.name), k.value)
	}

	return PApp(Merge, append([]Value{d}, ds...)...)
}

// Merge merges 2 sets of arguments into one.
func (args Arguments) Merge(old Arguments) Arguments {
	ks := append(args.keywords, old.keywords...)
	ds := append(args.expandedDicts, old.expandedDicts...)

	if args.expandedList == nil {
		return Arguments{append(args.positionals, old.positionals...), old.expandedList, ks, ds}
	}

	l := Value(EmptyList)

	if old.expandedList != nil {
		l = old.expandedList
	}

	return Arguments{
		args.positionals,
		PApp(Merge, args.expandedList, StrictPrepend(old.positionals, l)),
		ks,
		ds,
	}
}

func (args Arguments) empty() Value {
	if len(args.positionals) > 0 {
		return argumentError("%d positional arguments are left", len(args.positionals))
	}

	// Testing args.expandedList is impossible because we cannot know its length
	// without evaluating it.

	n := 0

	for _, v := range args.expandedDicts {
		d, err := EvalDictionary(v)

		if err != nil {
			return err
		}

		n += d.Size()
	}

	if n != 0 || len(args.keywords) > 0 {
		return argumentError("%d keyword arguments are left", len(args.keywords)+n)
	}

	return nil
}
<|endoftext|>


<commit_before>package janus

import (
	"net/http"
	"strings"

	log "github.com/Sirupsen/logrus"
	"github.com/gin-gonic/gin"
	"github.com/urfave/negroni"
	"gopkg.in/alexcesaro/statsd.v2"
)

const (
	methodAll = "ALL"
)

// ProxyRegister represents a register proxy
type ProxyRegister struct {
	Router       Router
	proxies      []Proxy
	statsdClient *statsd.Client
}

// RegisterMany registers many proxies at once
func (p *ProxyRegister) RegisterMany(proxies []Proxy, beforeHandlers []negroni.HandlerFunc, afterHandlers []negroni.HandlerFunc) {
	for _, proxy := range proxies {
		p.Register(proxy, beforeHandlers, afterHandlers)
	}
}

// Register register a new proxy
func (p *ProxyRegister) Register(proxy Proxy, beforeHandlers []negroni.HandlerFunc, afterHandlers []negroni.HandlerFunc) {
	var handlers []gin.HandlerFunc

	router := negroni.New()
	p.registerHandlers(router, beforeHandlers)
	router.UseHandlerFunc(p.ToHandler(proxy))
	p.registerHandlers(router, afterHandlers)

	if false == p.Exists(proxy) {
		log.WithFields(log.Fields{
			"listen_path": proxy.ListenPath,
		}).Info("Registering a proxy")

		for _, method := range proxy.Methods {
			if strings.ToUpper(method) == methodAll {
				p.Router.Any(proxy.ListenPath, router.ServeHTTP)
			}

			p.Router.Handle(strings.ToUpper(method), proxy.ListenPath, router.ServeHTTP)
		}

		p.proxies = append(p.proxies, proxy)
	}
}

// Exists checks if a proxy is already registered in the manager
func (p *ProxyRegister) Exists(proxy Proxy) bool {
	for _, route := range p.proxies {
		if route.ListenPath == proxy.ListenPath {
			return true
		}
	}

	return false
}

// ToHandler turns a proxy configuration into a handler
func (p *ProxyRegister) ToHandler(proxy Proxy) http.HandlerFunc {
	return func(rw http.ResponseWriter, r *http.Request) {
		transport := &transport{http.DefaultTransport, p.statsdClient}
		reverseProxy := NewSingleHostReverseProxy(proxy, transport)
		reverseProxy.ServeHTTP(rw, r)
	}
}

func (p *ProxyRegister) registerHandlers(router *negroni.Negroni, handlers []negroni.HandlerFunc) {
	for _, handler := range handlers {
		router.UseFunc(handler)
	}
}
<commit_msg>Replaced gin<commit_after>package janus

import (
	"net/http"
	"strings"

	log "github.com/Sirupsen/logrus"
	"github.com/urfave/negroni"
	"gopkg.in/alexcesaro/statsd.v2"
)

const (
	methodAll = "ALL"
)

// ProxyRegister represents a register proxy
type ProxyRegister struct {
	Router       Router
	proxies      []Proxy
	statsdClient *statsd.Client
}

// RegisterMany registers many proxies at once
func (p *ProxyRegister) RegisterMany(proxies []Proxy, beforeHandlers []negroni.HandlerFunc, afterHandlers []negroni.HandlerFunc) {
	for _, proxy := range proxies {
		p.Register(proxy, beforeHandlers, afterHandlers)
	}
}

// Register register a new proxy
func (p *ProxyRegister) Register(proxy Proxy, beforeHandlers []negroni.HandlerFunc, afterHandlers []negroni.HandlerFunc) {
	var handlers []negroni.HandlerFunc

	router := negroni.New()
	p.registerHandlers(router, beforeHandlers)
	router.UseHandlerFunc(p.ToHandler(proxy))
	p.registerHandlers(router, afterHandlers)

	if false == p.Exists(proxy) {
		log.WithFields(log.Fields{
			"listen_path": proxy.ListenPath,
		}).Info("Registering a proxy")

		for _, method := range proxy.Methods {
			if strings.ToUpper(method) == methodAll {
				p.Router.Any(proxy.ListenPath, router.ServeHTTP)
			}

			p.Router.Handle(strings.ToUpper(method), proxy.ListenPath, router.ServeHTTP)
		}

		p.proxies = append(p.proxies, proxy)
	}
}

// Exists checks if a proxy is already registered in the manager
func (p *ProxyRegister) Exists(proxy Proxy) bool {
	for _, route := range p.proxies {
		if route.ListenPath == proxy.ListenPath {
			return true
		}
	}

	return false
}

// ToHandler turns a proxy configuration into a handler
func (p *ProxyRegister) ToHandler(proxy Proxy) http.HandlerFunc {
	return func(rw http.ResponseWriter, r *http.Request) {
		transport := &transport{http.DefaultTransport, p.statsdClient}
		reverseProxy := NewSingleHostReverseProxy(proxy, transport)
		reverseProxy.ServeHTTP(rw, r)
	}
}

func (p *ProxyRegister) registerHandlers(router *negroni.Negroni, handlers []negroni.HandlerFunc) {
	for _, handler := range handlers {
		router.UseFunc(handler)
	}
}
<|endoftext|>


<commit_before><commit_msg>version 0.11.6<commit_after><|endoftext|>


<commit_before>package gval

import (
	"regexp/syntax"
	"testing"
)

func TestParsingFailure(t *testing.T) {
	testEvaluate(
		[]evaluationTest{
			{
				name:       "Invalid equality comparator",
				expression: "1 = 1",
				wantErr:    unexpected(`"="`, "operator"),
			},
			{
				name:       "Invalid equality comparator",
				expression: "1 === 1",
				wantErr:    unknownOp("==="),
			},
			{
				name:       "Too many characters for logical operator",
				expression: "true &&& false",
				wantErr:    unknownOp("&&&"),
			},
			{

				name:       "Too many characters for logical operator",
				expression: "true ||| false",
				wantErr:    unknownOp("|||"),
			},
			{

				name:       "Premature end to expression, via modifier",
				expression: "10 > 5 +",
				wantErr:    unexpected("EOF", "extensions"),
			},
			{
				name:       "Premature end to expression, via comparator",
				expression: "10 + 5 >",
				wantErr:    unexpected("EOF", "extensions"),
			},
			{
				name:       "Premature end to expression, via logical operator",
				expression: "10 > 5 &&",
				wantErr:    unexpected("EOF", "extensions"),
			},
			{

				name:       "Premature end to expression, via ternary operator",
				expression: "true ?",
				wantErr:    unexpected("EOF", "extensions"),
			},
			{
				name:       "Hanging REQ",
				expression: "`wat` =~",
				wantErr:    unexpected("EOF", "extensions"),
			},
			{

				name:       "Invalid operator change to REQ",
				expression: " / =~",
				wantErr:    unexpected(`"/"`, "extensions"),
			},
			{
				name:       "Invalid starting token, comparator",
				expression: "> 10",
				wantErr:    unexpected(`">"`, "extensions"),
			},
			{
				name:       "Invalid starting token, modifier",
				expression: "+ 5",
				wantErr:    unexpected(`"+"`, "extensions"),
			},
			{
				name:       "Invalid starting token, logical operator",
				expression: "&& 5 < 10",
				wantErr:    unexpected(`"&"`, "extensions"),
			},
			{
				name:       "Invalid NUMERIC transition",
				expression: "10 10",
				wantErr:    unexpected(`Int`, "operator"),
			},
			{
				name:       "Invalid STRING transition",
				expression: "`foo` `foo`",
				wantErr:    unexpected(`String`, "operator"),
			},
			{
				name:       "Invalid operator transition",
				expression: "10 > < 10",
				wantErr:    unexpected(`"<"`, "extensions"),
			},
			{

				name:       "Starting with unbalanced parens",
				expression: " ) ( arg2",
				wantErr:    unexpected(`")"`, "extensions"),
			},
			{

				name:       "Unclosed bracket",
				expression: "[foo bar",
				wantErr:    unexpected(`EOF`, "extensions"),
			},
			{

				name:       "Unclosed quote",
				expression: "foo == `responseTime",
				wantErr:    "could not parse string",
			},
			{

				name:       "Constant regex pattern fail to compile",
				expression: "foo =~ `[abc`",
				wantErr:    string(syntax.ErrMissingBracket),
			},
			{

				name:       "Unbalanced parentheses",
				expression: "10 > (1 + 50",
				wantErr:    unexpected(`EOF`, "parentheses"),
			},
			{

				name:       "Multiple radix",
				expression: "127.0.0.1",
				wantErr:    unexpected(`Float`, "operator"),
			},
			{

				name:       "Hanging accessor",
				expression: "foo.Bar.",
				wantErr:    unexpected(`EOF`, "field"),
			},
			{
				name:       "Incomplete Hex",
				expression: "0x",
				wantErr:    `strconv.ParseFloat: parsing "0x": invalid syntax`,
			},
			{
				name:       "Invalid Hex literal",
				expression: "0x > 0",
				wantErr:    `strconv.ParseFloat: parsing "0x": invalid syntax`,
			},
			{
				name:       "Hex float (Unsupported)",
				expression: "0x1.1",
				wantErr:    `strconv.ParseFloat: parsing "0x1": invalid syntax`,
			},
			{
				name:       "Hex invalid letter",
				expression: "0x12g1",
				wantErr:    `strconv.ParseFloat: parsing "0x12": invalid syntax`,
			},
			{
				name:       "Error after camouflage",
				expression: "0 + ,",
				wantErr:    `unexpected "," while scanning extensions`,
			},
		},
		t,
	)
}

func unknownOp(op string) string {
	return "unknown operator " + op
}

func unexpected(token, unit string) string {
	return "unexpected " + token + " while scanning " + unit
}
<commit_msg>adjust Invalid STRING transition to go 1.11<commit_after>package gval

import (
	"regexp/syntax"
	"testing"
)

func TestParsingFailure(t *testing.T) {
	testEvaluate(
		[]evaluationTest{
			{
				name:       "Invalid equality comparator",
				expression: "1 = 1",
				wantErr:    unexpected(`"="`, "operator"),
			},
			{
				name:       "Invalid equality comparator",
				expression: "1 === 1",
				wantErr:    unknownOp("==="),
			},
			{
				name:       "Too many characters for logical operator",
				expression: "true &&& false",
				wantErr:    unknownOp("&&&"),
			},
			{

				name:       "Too many characters for logical operator",
				expression: "true ||| false",
				wantErr:    unknownOp("|||"),
			},
			{

				name:       "Premature end to expression, via modifier",
				expression: "10 > 5 +",
				wantErr:    unexpected("EOF", "extensions"),
			},
			{
				name:       "Premature end to expression, via comparator",
				expression: "10 + 5 >",
				wantErr:    unexpected("EOF", "extensions"),
			},
			{
				name:       "Premature end to expression, via logical operator",
				expression: "10 > 5 &&",
				wantErr:    unexpected("EOF", "extensions"),
			},
			{

				name:       "Premature end to expression, via ternary operator",
				expression: "true ?",
				wantErr:    unexpected("EOF", "extensions"),
			},
			{
				name:       "Hanging REQ",
				expression: "`wat` =~",
				wantErr:    unexpected("EOF", "extensions"),
			},
			{

				name:       "Invalid operator change to REQ",
				expression: " / =~",
				wantErr:    unexpected(`"/"`, "extensions"),
			},
			{
				name:       "Invalid starting token, comparator",
				expression: "> 10",
				wantErr:    unexpected(`">"`, "extensions"),
			},
			{
				name:       "Invalid starting token, modifier",
				expression: "+ 5",
				wantErr:    unexpected(`"+"`, "extensions"),
			},
			{
				name:       "Invalid starting token, logical operator",
				expression: "&& 5 < 10",
				wantErr:    unexpected(`"&"`, "extensions"),
			},
			{
				name:       "Invalid NUMERIC transition",
				expression: "10 10",
				wantErr:    unexpected(`Int`, "operator"),
			},
			{
				name:       "Invalid STRING transition",
				expression: "`foo` `foo`",
				wantErr:    `String while scanning operator`, // can't use func unexpected because the token was changed from String to RawString in go 1.11
			},
			{
				name:       "Invalid operator transition",
				expression: "10 > < 10",
				wantErr:    unexpected(`"<"`, "extensions"),
			},
			{

				name:       "Starting with unbalanced parens",
				expression: " ) ( arg2",
				wantErr:    unexpected(`")"`, "extensions"),
			},
			{

				name:       "Unclosed bracket",
				expression: "[foo bar",
				wantErr:    unexpected(`EOF`, "extensions"),
			},
			{

				name:       "Unclosed quote",
				expression: "foo == `responseTime",
				wantErr:    "could not parse string",
			},
			{

				name:       "Constant regex pattern fail to compile",
				expression: "foo =~ `[abc`",
				wantErr:    string(syntax.ErrMissingBracket),
			},
			{

				name:       "Unbalanced parentheses",
				expression: "10 > (1 + 50",
				wantErr:    unexpected(`EOF`, "parentheses"),
			},
			{

				name:       "Multiple radix",
				expression: "127.0.0.1",
				wantErr:    unexpected(`Float`, "operator"),
			},
			{

				name:       "Hanging accessor",
				expression: "foo.Bar.",
				wantErr:    unexpected(`EOF`, "field"),
			},
			{
				name:       "Incomplete Hex",
				expression: "0x",
				wantErr:    `strconv.ParseFloat: parsing "0x": invalid syntax`,
			},
			{
				name:       "Invalid Hex literal",
				expression: "0x > 0",
				wantErr:    `strconv.ParseFloat: parsing "0x": invalid syntax`,
			},
			{
				name:       "Hex float (Unsupported)",
				expression: "0x1.1",
				wantErr:    `strconv.ParseFloat: parsing "0x1": invalid syntax`,
			},
			{
				name:       "Hex invalid letter",
				expression: "0x12g1",
				wantErr:    `strconv.ParseFloat: parsing "0x12": invalid syntax`,
			},
			{
				name:       "Error after camouflage",
				expression: "0 + ,",
				wantErr:    `unexpected "," while scanning extensions`,
			},
		},
		t,
	)
}

func unknownOp(op string) string {
	return "unknown operator " + op
}

func unexpected(token, unit string) string {
	return "unexpected " + token + " while scanning " + unit
}
<|endoftext|>


<commit_before>package escapingio

import (
	"bufio"
	"io"
)

// Handler is a callback for handling an escaped char.  Reader would skip
// the escape char and passed char if returns true; otherwise, it preserves them
// in output
type Handler func(c byte) bool

// NewReader returns a reader that escapes the c character (following new lines),
// in the same manner OpenSSH handling, which defaults to `~`.
//
// For illustrative purposes, we use `~` in documentation as a shorthand for escaping character.
//
// If following a new line, reader sees:
//  * `~~`, only one is emitted
//  * `~.` (or any character), the handler is invoked with the character.
//     If handler returns true, `~.` will be skipped; otherwise, it's propagated.
//  * `~` and it's the last character in stream, it's propagated
//
// Appearances of `~` when not preceded by a new line are propagated unmodified.
func NewReader(r io.Reader, c byte, h Handler) io.Reader {
	pr, pw := io.Pipe()
	reader := &reader{
		impl:       r,
		escapeChar: c,
		handler:    h,
		pr:         pr,
		pw:         pw,
	}
	go reader.pipe()
	return reader
}

// lookState represents the state of reader for what character of `\n~.` sequence
// reader is looking for
type lookState int

const (
	// sLookNewLine indicates that reader is looking for new line
	sLookNewLine lookState = iota

	// sLookEscapeChar indicates that reader is looking for ~
	sLookEscapeChar

	// sLookChar indicates that reader just read `~` is waiting for next character
	// before acting
	sLookChar
)

// to ease comments, i'll assume escape character to be `~`
type reader struct {
	impl       io.Reader
	escapeChar uint8
	handler    Handler

	// buffers
	pw *io.PipeWriter
	pr *io.PipeReader
}

func (r *reader) Read(buf []byte) (int, error) {
	return r.pr.Read(buf)
}

func (r *reader) pipe() {
	rb := make([]byte, 4096)
	bw := bufio.NewWriter(r.pw)

	state := sLookEscapeChar

	for {
		n, err := r.impl.Read(rb)

		if n > 0 {
			state = r.processBuf(bw, rb, n, state)
			bw.Flush()
			if state == sLookChar {
				// terminated with ~ - let's read one more character
				n, err = r.impl.Read(rb[:1])
				if n == 1 {
					state = sLookNewLine
					if rb[0] == r.escapeChar {
						// only emit escape character once
						bw.WriteByte(rb[0])
						bw.Flush()
					} else if r.handler(rb[0]) {
						// skip if handled
					} else {
						bw.WriteByte(r.escapeChar)
						bw.WriteByte(rb[0])
						bw.Flush()
					}
				}
			}
		}

		if err != nil {
			// write ~ if it's the last thing
			if state == sLookChar {
				bw.WriteByte(r.escapeChar)
			}
			bw.Flush()
			r.pw.CloseWithError(err)
			break
		}
	}
}

// processBuf process buffer and emits all output to writer
// if the last part of buffer is a new line followed by sequnce, it writes
// all output until the new line and returns sLookChar
func (r *reader) processBuf(bw io.Writer, buf []byte, n int, s lookState) lookState {
	i := 0

	wi := 0

START:
	if s == sLookEscapeChar && buf[i] == r.escapeChar {
		if i+1 >= n {
			// buf terminates with ~ - write all before
			bw.Write(buf[wi:i])
			return sLookChar
		}

		nc := buf[i+1]
		if nc == r.escapeChar {
			// skip one escape char
			bw.Write(buf[wi:i])
			i++
			wi = i
		} else if r.handler(nc) {
			// skip both characters
			bw.Write(buf[wi:i])
			i = i + 2
			wi = i
		} else {
			i = i + 2
			// need to write everything keep going
		}
	}

	// search until we get \n~, or buf terminates
	for {
		if i >= n {
			// got to end without new line, write and return
			bw.Write(buf[wi:n])
			return sLookNewLine
		}

		if buf[i] == '\n' || buf[i] == '\r' {
			// buf terminated at new line
			if i+1 >= n {
				bw.Write(buf[wi:n])
				return sLookEscapeChar
			}

			// peek to see escape character go back to START if so
			if buf[i+1] == r.escapeChar {
				s = sLookEscapeChar
				i++
				goto START
			}
		}

		i++
	}
}
<commit_msg>Handle when a new line follows an escaping char<commit_after>package escapingio

import (
	"bufio"
	"io"
)

// Handler is a callback for handling an escaped char.  Reader would skip
// the escape char and passed char if returns true; otherwise, it preserves them
// in output
type Handler func(c byte) bool

// NewReader returns a reader that escapes the c character (following new lines),
// in the same manner OpenSSH handling, which defaults to `~`.
//
// For illustrative purposes, we use `~` in documentation as a shorthand for escaping character.
//
// If following a new line, reader sees:
//  * `~~`, only one is emitted
//  * `~.` (or any character), the handler is invoked with the character.
//     If handler returns true, `~.` will be skipped; otherwise, it's propagated.
//  * `~` and it's the last character in stream, it's propagated
//
// Appearances of `~` when not preceded by a new line are propagated unmodified.
func NewReader(r io.Reader, c byte, h Handler) io.Reader {
	pr, pw := io.Pipe()
	reader := &reader{
		impl:       r,
		escapeChar: c,
		handler:    h,
		pr:         pr,
		pw:         pw,
	}
	go reader.pipe()
	return reader
}

// lookState represents the state of reader for what character of `\n~.` sequence
// reader is looking for
type lookState int

const (
	// sLookNewLine indicates that reader is looking for new line
	sLookNewLine lookState = iota

	// sLookEscapeChar indicates that reader is looking for ~
	sLookEscapeChar

	// sLookChar indicates that reader just read `~` is waiting for next character
	// before acting
	sLookChar
)

// to ease comments, i'll assume escape character to be `~`
type reader struct {
	impl       io.Reader
	escapeChar uint8
	handler    Handler

	// buffers
	pw *io.PipeWriter
	pr *io.PipeReader
}

func (r *reader) Read(buf []byte) (int, error) {
	return r.pr.Read(buf)
}

func (r *reader) pipe() {
	rb := make([]byte, 4096)
	bw := bufio.NewWriter(r.pw)

	state := sLookEscapeChar

	for {
		n, err := r.impl.Read(rb)

		if n > 0 {
			state = r.processBuf(bw, rb, n, state)
			bw.Flush()
			if state == sLookChar {
				// terminated with ~ - let's read one more character
				n, err = r.impl.Read(rb[:1])
				if n == 1 {
					state = sLookNewLine
					if rb[0] == r.escapeChar {
						// only emit escape character once
						bw.WriteByte(rb[0])
						bw.Flush()
					} else if r.handler(rb[0]) {
						// skip if handled
					} else {
						bw.WriteByte(r.escapeChar)
						bw.WriteByte(rb[0])
						bw.Flush()
						if rb[0] == '\n' || rb[0] == '\r' {
							state = sLookEscapeChar
						}
					}
				}
			}
		}

		if err != nil {
			// write ~ if it's the last thing
			if state == sLookChar {
				bw.WriteByte(r.escapeChar)
			}
			bw.Flush()
			r.pw.CloseWithError(err)
			break
		}
	}
}

// processBuf process buffer and emits all output to writer
// if the last part of buffer is a new line followed by sequnce, it writes
// all output until the new line and returns sLookChar
func (r *reader) processBuf(bw io.Writer, buf []byte, n int, s lookState) lookState {
	i := 0

	wi := 0

START:
	if s == sLookEscapeChar && buf[i] == r.escapeChar {
		if i+1 >= n {
			// buf terminates with ~ - write all before
			bw.Write(buf[wi:i])
			return sLookChar
		}

		nc := buf[i+1]
		if nc == r.escapeChar {
			// skip one escape char
			bw.Write(buf[wi:i])
			i++
			wi = i
		} else if r.handler(nc) {
			// skip both characters
			bw.Write(buf[wi:i])
			i = i + 2
			wi = i
		} else if nc == '\n' || nc == '\r' {
			i = i + 2
			s = sLookEscapeChar
			goto START
		} else {
			i = i + 2
			// need to write everything keep going
		}
	}

	// search until we get \n~, or buf terminates
	for {
		if i >= n {
			// got to end without new line, write and return
			bw.Write(buf[wi:n])
			return sLookNewLine
		}

		if buf[i] == '\n' || buf[i] == '\r' {
			// buf terminated at new line
			if i+1 >= n {
				bw.Write(buf[wi:n])
				return sLookEscapeChar
			}

			// peek to see escape character go back to START if so
			if buf[i+1] == r.escapeChar {
				s = sLookEscapeChar
				i++
				goto START
			}
		}

		i++
	}
}
<|endoftext|>