<commit_before>// Package taskqueue provides the functionality for receiving, handling and executing tasks.
// In this file are the routines for the taskqueue itself.
package taskqueue

import (
	"fmt"
	"github.com/jpillora/backoff"
	"github.com/mediocregopher/radix.v2/pool"
	"github.com/nevsnode/gordon/config"
	"github.com/nevsnode/gordon/output"
	"github.com/nevsnode/gordon/stats"
	"strings"
	"sync"
	"time"
)

const (
	backlog = 1
)

type failedTask struct {
	configTask config.Task
	queueTask  QueueTask
}

var (
	errorNoNewTask          = fmt.Errorf("No new task available")
	errorNoNewTasksAccepted = fmt.Errorf("No new tasks accepted")

	conf            config.Config
	shutdown        bool
	shutdownChan    chan bool
	waitGroup       sync.WaitGroup
	waitGroupFailed sync.WaitGroup
	redisPool       *pool.Pool
	errorBackoff    map[string]*backoff.Backoff
	workerChan      map[string]chan QueueTask
	failedChan      chan failedTask
)

// Start initialises several variables and creates necessary go-routines
func Start(c config.Config) {
	conf = c

	poolSize := 1
	for _, ct := range conf.Tasks {
		if ct.FailedTasksTTL > 0 {
			poolSize++
			break
		}
	}

	var err error
	redisPool, err = pool.New(conf.RedisNetwork, conf.RedisAddress, poolSize)
	if err != nil {
		output.NotifyError("redis pool.New():", err)
	}

	stats.InitTasks(conf.Tasks)

	workerChan = make(map[string]chan QueueTask)
	failedChan = make(chan failedTask)
	shutdownChan = make(chan bool, 1)

	errorBackoff = make(map[string]*backoff.Backoff)
	for _, ct := range conf.Tasks {
		if ct.BackoffEnabled {
			errorBackoff[ct.Type] = &backoff.Backoff{
				Min:    time.Duration(ct.BackoffMin) * time.Millisecond,
				Max:    time.Duration(ct.BackoffMax) * time.Millisecond,
				Factor: ct.BackoffFactor,
				Jitter: true,
			}
		}

		workerChan[ct.Type] = make(chan QueueTask, backlog)

		for i := 0; i < ct.Workers; i++ {
			waitGroup.Add(1)
			go taskWorker(ct)
		}
	}

	waitGroupFailed.Add(1)
	go failedTaskWorker()

	waitGroup.Add(1)
	go queueWorker()
}

// Stop will cause the taskqueue to stop accepting new tasks and shutdown the
// worker routines after they've finished their current tasks
func Stop() {
	if shutdown {
		return
	}

	shutdown = true
	shutdownChan <- true

	for taskType := range workerChan {
		close(workerChan[taskType])
	}
}

// Wait waits, to keep the application running as long as there are workers
func Wait() {
	waitGroup.Wait()

	close(failedChan)
	waitGroupFailed.Wait()

	redisPool.Empty()
}

func queueWorker() {
	interval := backoff.Backoff{
		Min:    time.Duration(conf.IntervalMin) * time.Millisecond,
		Max:    time.Duration(conf.IntervalMax) * time.Millisecond,
		Factor: conf.IntervalFactor,
	}

	runIntervalLoop := make(chan bool)
	var runningIntervalLoop sync.WaitGroup
	go func() {
		for {
			runningIntervalLoop.Wait()
			time.Sleep(interval.Duration())

			if shutdown {
				break
			}

			runIntervalLoop <- true
			runningIntervalLoop.Add(1)
		}
	}()

	go func() {
		for {
			select {
			case <-shutdownChan:
				runIntervalLoop <- false
			}
		}
	}()

	for <-runIntervalLoop {
		for taskType, configTask := range conf.Tasks {
			if shutdown {
				break
			}

			output.Debug("Checking for new tasks (" + taskType + ")")

			// check if there are available workers
			if !acceptsTasks(taskType) {
				continue
			}

			queueKey := conf.RedisQueueKey + ":" + taskType

			llen, err := redisPool.Cmd("LLEN", queueKey).Int()
			if err != nil {
				// Errors here are likely redis-connection errors, so we'll
				// need to notify about it
				output.NotifyError("redisPool.Cmd() Error:", err)
				break
			}

			// there are no new tasks in redis
			if llen == 0 {
				continue
			}

			// iterate over all entries in redis, until no more are available,
			// or all workers are busy, for a maximum of 2 * workers
			for i := 0; i < (configTask.Workers * 2); i++ {
				if !acceptsTasks(taskType) {
					break
				}

				value, err := redisPool.Cmd("LPOP", queueKey).Str()
				if err != nil {
					// no more tasks found
					break
				}

				output.Debug("Fetched task for type", taskType, "with payload", value)

				task, err := NewQueueTask(value)
				if err != nil {
					output.NotifyError("NewQueueTask():", err)
					continue
				}

				workerChan[taskType] <- task

				// we've actually are handling new tasks so reset the interval
				interval.Reset()
			}
		}

		runningIntervalLoop.Done()
	}

	Stop()
	waitGroup.Done()
}

func taskWorker(ct config.Task) {
	for task := range workerChan[ct.Type] {
		output.Debug("Executing task type", ct.Type, "with arguments", task.Args)
		stats.IncrTaskCount(ct.Type)

		err := task.Execute(ct.Script)
		if err != nil {
			task.ErrorMessage = fmt.Sprintf("%s", err)
			failedChan <- failedTask{
				configTask: ct,
				queueTask:  task,
			}

			msg := fmt.Sprintf("Failed executing task: %s \"%s\"\n%s", ct.Script, strings.Join(task.Args, "\" \""), err)
			output.NotifyError(msg)
		}

		if errorBackoff[ct.Type] != nil {
			if err == nil {
				errorBackoff[ct.Type].Reset()
			} else {
				time.Sleep(errorBackoff[ct.Type].Duration())
			}
		}
	}

	waitGroup.Done()
}

func failedTaskWorker() {
	for ft := range failedChan {
		ct := ft.configTask
		qt := ft.queueTask

		if ct.FailedTasksTTL == 0 {
			return
		}

		rc, err := redisPool.Get()
		if err != nil {
			output.NotifyError("redisPool.Get():", err)
			return
		}
		defer redisPool.Put(rc)

		queueKey := conf.RedisQueueKey + ":" + ct.Type + ":failed"

		jsonString, err := qt.GetJSONString()
		if err != nil {
			output.NotifyError("failedTaskWorker(), qt.GetJSONString():", err)
			return
		}

		// add to list
		reply := rc.Cmd("RPUSH", queueKey, jsonString)
		if reply.Err != nil {
			output.NotifyError("failedTaskWorker(), RPUSH:", reply.Err)
			return
		}

		// set expire
		reply = rc.Cmd("EXPIRE", queueKey, ct.FailedTasksTTL)
		if reply.Err != nil {
			output.NotifyError("failedTaskWorker(), EXPIRE:", reply.Err)
			return
		}
	}

	waitGroupFailed.Done()
}

func acceptsTasks(taskType string) bool {
	if shutdown {
		return false
	}

	return len(workerChan[taskType]) < backlog
}
<commit_msg>added debugging output<commit_after>// Package taskqueue provides the functionality for receiving, handling and executing tasks.
// In this file are the routines for the taskqueue itself.
package taskqueue

import (
	"fmt"
	"github.com/jpillora/backoff"
	"github.com/mediocregopher/radix.v2/pool"
	"github.com/nevsnode/gordon/config"
	"github.com/nevsnode/gordon/output"
	"github.com/nevsnode/gordon/stats"
	"strings"
	"sync"
	"time"
)

const (
	backlog = 1
)

type failedTask struct {
	configTask config.Task
	queueTask  QueueTask
}

var (
	errorNoNewTask          = fmt.Errorf("No new task available")
	errorNoNewTasksAccepted = fmt.Errorf("No new tasks accepted")

	conf            config.Config
	shutdown        bool
	shutdownChan    chan bool
	waitGroup       sync.WaitGroup
	waitGroupFailed sync.WaitGroup
	redisPool       *pool.Pool
	errorBackoff    map[string]*backoff.Backoff
	workerChan      map[string]chan QueueTask
	failedChan      chan failedTask
)

// Start initialises several variables and creates necessary go-routines
func Start(c config.Config) {
	conf = c

	poolSize := 1
	for _, ct := range conf.Tasks {
		if ct.FailedTasksTTL > 0 {
			poolSize++
			break
		}
	}

	var err error
	redisPool, err = pool.New(conf.RedisNetwork, conf.RedisAddress, poolSize)
	if err != nil {
		output.NotifyError("redis pool.New():", err)
	}

	stats.InitTasks(conf.Tasks)

	workerChan = make(map[string]chan QueueTask)
	failedChan = make(chan failedTask)
	shutdownChan = make(chan bool, 1)

	errorBackoff = make(map[string]*backoff.Backoff)
	for _, ct := range conf.Tasks {
		if ct.BackoffEnabled {
			errorBackoff[ct.Type] = &backoff.Backoff{
				Min:    time.Duration(ct.BackoffMin) * time.Millisecond,
				Max:    time.Duration(ct.BackoffMax) * time.Millisecond,
				Factor: ct.BackoffFactor,
				Jitter: true,
			}
		}

		workerChan[ct.Type] = make(chan QueueTask, backlog)

		for i := 0; i < ct.Workers; i++ {
			waitGroup.Add(1)
			go taskWorker(ct)
		}
	}

	waitGroupFailed.Add(1)
	go failedTaskWorker()

	waitGroup.Add(1)
	go queueWorker()
}

// Stop will cause the taskqueue to stop accepting new tasks and shutdown the
// worker routines after they've finished their current tasks
func Stop() {
	if shutdown {
		return
	}

	shutdown = true
	shutdownChan <- true

	for taskType := range workerChan {
		close(workerChan[taskType])
	}
}

// Wait waits, to keep the application running as long as there are workers
func Wait() {
	waitGroup.Wait()

	close(failedChan)
	waitGroupFailed.Wait()

	redisPool.Empty()
}

func queueWorker() {
	interval := backoff.Backoff{
		Min:    time.Duration(conf.IntervalMin) * time.Millisecond,
		Max:    time.Duration(conf.IntervalMax) * time.Millisecond,
		Factor: conf.IntervalFactor,
	}

	runIntervalLoop := make(chan bool)
	var runningIntervalLoop sync.WaitGroup
	go func() {
		for {
			runningIntervalLoop.Wait()
			time.Sleep(interval.Duration())

			if shutdown {
				break
			}

			runIntervalLoop <- true
			runningIntervalLoop.Add(1)
		}
	}()

	go func() {
		for {
			select {
			case <-shutdownChan:
				runIntervalLoop <- false
			}
		}
	}()

	for <-runIntervalLoop {
		for taskType, configTask := range conf.Tasks {
			if shutdown {
				break
			}

			output.Debug("Checking for new tasks (" + taskType + ")")

			// check if there are available workers
			if !acceptsTasks(taskType) {
				continue
			}

			queueKey := conf.RedisQueueKey + ":" + taskType

			llen, err := redisPool.Cmd("LLEN", queueKey).Int()
			if err != nil {
				// Errors here are likely redis-connection errors, so we'll
				// need to notify about it
				output.NotifyError("redisPool.Cmd() Error:", err)
				break
			}

			// there are no new tasks in redis
			if llen == 0 {
				continue
			}

			// iterate over all entries in redis, until no more are available,
			// or all workers are busy, for a maximum of 2 * workers
			for i := 0; i < (configTask.Workers * 2); i++ {
				if !acceptsTasks(taskType) {
					break
				}

				value, err := redisPool.Cmd("LPOP", queueKey).Str()
				if err != nil {
					// no more tasks found
					break
				}

				output.Debug("Fetched task for type", taskType, "with payload", value)

				task, err := NewQueueTask(value)
				if err != nil {
					output.NotifyError("NewQueueTask():", err)
					continue
				}

				workerChan[taskType] <- task

				// we've actually are handling new tasks so reset the interval
				interval.Reset()
			}
		}

		runningIntervalLoop.Done()
	}

	Stop()
	waitGroup.Done()
}

func taskWorker(ct config.Task) {
	for task := range workerChan[ct.Type] {
		output.Debug("Executing task type", ct.Type, "with arguments", task.Args)
		stats.IncrTaskCount(ct.Type)

		err := task.Execute(ct.Script)
		if err != nil {
			task.ErrorMessage = fmt.Sprintf("%s", err)
			failedChan <- failedTask{
				configTask: ct,
				queueTask:  task,
			}

			msg := fmt.Sprintf("Failed executing task: %s \"%s\"\n%s", ct.Script, strings.Join(task.Args, "\" \""), err)
			output.NotifyError(msg)
		}

		if errorBackoff[ct.Type] != nil {
			if err == nil {
				errorBackoff[ct.Type].Reset()
			} else {
				time.Sleep(errorBackoff[ct.Type].Duration())
			}
		}

		output.Debug("Finished task type", ct.Type, "with arguments", task.Args)
	}

	waitGroup.Done()
}

func failedTaskWorker() {
	for ft := range failedChan {
		ct := ft.configTask
		qt := ft.queueTask

		if ct.FailedTasksTTL == 0 {
			return
		}

		rc, err := redisPool.Get()
		if err != nil {
			output.NotifyError("redisPool.Get():", err)
			return
		}
		defer redisPool.Put(rc)

		queueKey := conf.RedisQueueKey + ":" + ct.Type + ":failed"

		jsonString, err := qt.GetJSONString()
		if err != nil {
			output.NotifyError("failedTaskWorker(), qt.GetJSONString():", err)
			return
		}

		// add to list
		reply := rc.Cmd("RPUSH", queueKey, jsonString)
		if reply.Err != nil {
			output.NotifyError("failedTaskWorker(), RPUSH:", reply.Err)
			return
		}

		// set expire
		reply = rc.Cmd("EXPIRE", queueKey, ct.FailedTasksTTL)
		if reply.Err != nil {
			output.NotifyError("failedTaskWorker(), EXPIRE:", reply.Err)
			return
		}
	}

	waitGroupFailed.Done()
}

func acceptsTasks(taskType string) bool {
	if shutdown {
		return false
	}

	return len(workerChan[taskType]) < backlog
}
<|endoftext|>


<commit_before>package gapi

import (
	"bytes"
	"encoding/json"
	"fmt"
	"regexp"
	"strconv"
)

var headerNameRegex = regexp.MustCompile(`^httpHeaderName(\d+)$`)

// DataSource represents a Grafana data source.
type DataSource struct {
	ID     int64  `json:"id,omitempty"`
	UID    string `json:"uid,omitempty"`
	Name   string `json:"name"`
	Type   string `json:"type"`
	URL    string `json:"url"`
	Access string `json:"access"`

	Database string `json:"database,omitempty"`
	User     string `json:"user,omitempty"`
	// Deprecated: Use secureJsonData.password instead.
	Password string `json:"password,omitempty"`

	OrgID     int64 `json:"orgId,omitempty"`
	IsDefault bool  `json:"isDefault"`

	BasicAuth     bool   `json:"basicAuth"`
	BasicAuthUser string `json:"basicAuthUser,omitempty"`
	// Deprecated: Use secureJsonData.basicAuthPassword instead.
	BasicAuthPassword string `json:"basicAuthPassword,omitempty"`

	// Helper to read/write http headers
	HTTPHeaders map[string]string `json:"-"`

	JSONData       JSONData       `json:"jsonData,omitempty"`
	SecureJSONData SecureJSONData `json:"secureJsonData,omitempty"`
}

// Required to avoid recursion during (un)marshal
type _DataSource DataSource

// Marshal DataSource
func (ds *DataSource) MarshalJSON() ([]byte, error) {
	dataSource := _DataSource(*ds)
	for name, value := range ds.HTTPHeaders {
		dataSource.JSONData.httpHeaderNames = append(dataSource.JSONData.httpHeaderNames, name)
		dataSource.SecureJSONData.httpHeaderValues = append(dataSource.SecureJSONData.httpHeaderValues, value)
	}
	return json.Marshal(dataSource)
}

// Unmarshal DataSource
func (ds *DataSource) UnmarshalJSON(b []byte) (err error) {
	dataSource := _DataSource(*ds)
	if err = json.Unmarshal(b, &dataSource); err == nil {
		*ds = DataSource(dataSource)
	}
	ds.HTTPHeaders = make(map[string]string)
	for _, value := range ds.JSONData.httpHeaderNames {
		ds.HTTPHeaders[value] = "true" // HTTP Headers are not returned by the API
	}
	return err
}

// JSONData is a representation of the datasource `jsonData` property
type JSONData struct {
	// Used by all datasources
	TLSAuth           bool `json:"tlsAuth,omitempty"`
	TLSAuthWithCACert bool `json:"tlsAuthWithCACert,omitempty"`
	TLSSkipVerify     bool `json:"tlsSkipVerify,omitempty"`
	httpHeaderNames   []string

	// Used by Github
	GitHubURL string `json:"githubUrl,omitempty"`

	// Used by Graphite
	GraphiteVersion string `json:"graphiteVersion,omitempty"`

	// Used by Prometheus, Elasticsearch, InfluxDB, MySQL, PostgreSQL and MSSQL
	TimeInterval string `json:"timeInterval,omitempty"`

	// Used by Elasticsearch
	// From Grafana 8.x esVersion is the semantic version of Elasticsearch.
	EsVersion                  string `json:"esVersion,omitempty"`
	TimeField                  string `json:"timeField,omitempty"`
	Interval                   string `json:"interval,omitempty"`
	LogMessageField            string `json:"logMessageField,omitempty"`
	LogLevelField              string `json:"logLevelField,omitempty"`
	MaxConcurrentShardRequests int64  `json:"maxConcurrentShardRequests,omitempty"`

	// Used by Cloudwatch
	AuthType                string `json:"authType,omitempty"`
	AssumeRoleArn           string `json:"assumeRoleArn,omitempty"`
	DefaultRegion           string `json:"defaultRegion,omitempty"`
	CustomMetricsNamespaces string `json:"customMetricsNamespaces,omitempty"`
	Profile                 string `json:"profile,omitempty"`

	// Used by OpenTSDB
	TsdbVersion    string `json:"tsdbVersion,omitempty"`
	TsdbResolution string `json:"tsdbResolution,omitempty"`

	// Used by MSSQL
	Encrypt string `json:"encrypt,omitempty"`

	// Used by PostgreSQL
	Sslmode         string `json:"sslmode,omitempty"`
	PostgresVersion int64  `json:"postgresVersion,omitempty"`
	Timescaledb     bool   `json:"timescaledb,omitempty"`

	// Used by MySQL, PostgreSQL and MSSQL
	MaxOpenConns    int64 `json:"maxOpenConns,omitempty"`
	MaxIdleConns    int64 `json:"maxIdleConns,omitempty"`
	ConnMaxLifetime int64 `json:"connMaxLifetime,omitempty"`

	// Used by Prometheus
	HTTPMethod   string `json:"httpMethod,omitempty"`
	QueryTimeout string `json:"queryTimeout,omitempty"`

	// Used by Stackdriver
	AuthenticationType string `json:"authenticationType,omitempty"`
	ClientEmail        string `json:"clientEmail,omitempty"`
	DefaultProject     string `json:"defaultProject,omitempty"`
	TokenURI           string `json:"tokenUri,omitempty"`

	// Used by Prometheus and Elasticsearch
	SigV4AssumeRoleArn string `json:"sigV4AssumeRoleArn,omitempty"`
	SigV4Auth          bool   `json:"sigV4Auth,omitempty"`
	SigV4AuthType      string `json:"sigV4AuthType,omitempty"`
	SigV4ExternalID    string `json:"sigV4ExternalID,omitempty"`
	SigV4Profile       string `json:"sigV4Profile,omitempty"`
	SigV4Region        string `json:"sigV4Region,omitempty"`

	// Used by Prometheus and Loki
	ManageAlerts    bool   `json:"manageAlerts,omitempty"`
	AlertmanagerUID string `json:"alertmanagerUid,omitempty"`

	// Used by Alertmanager
	Implementation string `json:"implementation,omitempty"`
}

// Required to avoid recursion during (un)marshal
type _JSONData JSONData

// Marshal JSONData
func (jd JSONData) MarshalJSON() ([]byte, error) {
	jsonData := _JSONData(jd)
	b, err := json.Marshal(jsonData)
	if err != nil {
		return nil, err
	}
	fields := make(map[string]interface{})
	if err = json.Unmarshal(b, &fields); err != nil {
		return nil, err
	}
	for index, name := range jd.httpHeaderNames {
		fields[fmt.Sprintf("httpHeaderName%d", index+1)] = name
	}
	return json.Marshal(fields)
}

// Unmarshal JSONData
func (jd *JSONData) UnmarshalJSON(b []byte) (err error) {
	jsonData := _JSONData(*jd)
	if err = json.Unmarshal(b, &jsonData); err == nil {
		*jd = JSONData(jsonData)
	}
	fields := make(map[string]interface{})
	if err = json.Unmarshal(b, &fields); err == nil {
		headerCount := 0
		for name := range fields {
			match := headerNameRegex.FindStringSubmatch(name)
			if len(match) > 0 {
				headerCount++
			}
		}

		jd.httpHeaderNames = make([]string, headerCount)
		for name, value := range fields {
			match := headerNameRegex.FindStringSubmatch(name)
			if len(match) == 2 {
				index, err := strconv.ParseInt(match[1], 10, 64)
				if err != nil {
					return err
				}
				jd.httpHeaderNames[index-1] = value.(string)
			}
		}
	}
	return err
}

// SecureJSONData is a representation of the datasource `secureJsonData` property
type SecureJSONData struct {
	// Used by all datasources
	TLSCACert         string `json:"tlsCACert,omitempty"`
	TLSClientCert     string `json:"tlsClientCert,omitempty"`
	TLSClientKey      string `json:"tlsClientKey,omitempty"`
	Password          string `json:"password,omitempty"`
	BasicAuthPassword string `json:"basicAuthPassword,omitempty"`
	httpHeaderValues  []string

	// Used by Cloudwatch
	AccessKey string `json:"accessKey,omitempty"`
	SecretKey string `json:"secretKey,omitempty"`

	// Used by Stackdriver
	PrivateKey string `json:"privateKey,omitempty"`

	// Used by Prometheus and Elasticsearch
	SigV4AccessKey string `json:"sigV4AccessKey,omitempty"`
	SigV4SecretKey string `json:"sigV4SecretKey,omitempty"`

	// Used by GitHub
	AccessToken string `json:"accessToken,omitempty"`
}

// Required to avoid recursion during unmarshal
type _SecureJSONData SecureJSONData

// Marshal SecureJSONData
func (sjd SecureJSONData) MarshalJSON() ([]byte, error) {
	secureJSONData := _SecureJSONData(sjd)
	b, err := json.Marshal(secureJSONData)
	if err != nil {
		return nil, err
	}
	fields := make(map[string]interface{})
	if err = json.Unmarshal(b, &fields); err != nil {
		return nil, err
	}
	for index, value := range sjd.httpHeaderValues {
		fields[fmt.Sprintf("httpHeaderValue%d", index+1)] = value
	}
	return json.Marshal(fields)
}

// NewDataSource creates a new Grafana data source.
func (c *Client) NewDataSource(s *DataSource) (int64, error) {
	data, err := json.Marshal(s)
	if err != nil {
		return 0, err
	}

	result := struct {
		ID int64 `json:"id"`
	}{}

	err = c.request("POST", "/api/datasources", nil, bytes.NewBuffer(data), &result)
	if err != nil {
		return 0, err
	}

	return result.ID, err
}

// UpdateDataSource updates a Grafana data source.
func (c *Client) UpdateDataSource(s *DataSource) error {
	path := fmt.Sprintf("/api/datasources/%d", s.ID)
	data, err := json.Marshal(s)
	if err != nil {
		return err
	}

	return c.request("PUT", path, nil, bytes.NewBuffer(data), nil)
}

// DataSource fetches and returns the Grafana data source whose ID it's passed.
func (c *Client) DataSource(id int64) (*DataSource, error) {
	path := fmt.Sprintf("/api/datasources/%d", id)
	result := &DataSource{}
	err := c.request("GET", path, nil, nil, result)
	if err != nil {
		return nil, err
	}

	return result, err
}

// DeleteDataSource deletes the Grafana data source whose ID it's passed.
func (c *Client) DeleteDataSource(id int64) error {
	path := fmt.Sprintf("/api/datasources/%d", id)

	return c.request("DELETE", path, nil, nil, nil)
}
<commit_msg>Add fields for AWS Athena datasource<commit_after>package gapi

import (
	"bytes"
	"encoding/json"
	"fmt"
	"regexp"
	"strconv"
)

var headerNameRegex = regexp.MustCompile(`^httpHeaderName(\d+)$`)

// DataSource represents a Grafana data source.
type DataSource struct {
	ID     int64  `json:"id,omitempty"`
	UID    string `json:"uid,omitempty"`
	Name   string `json:"name"`
	Type   string `json:"type"`
	URL    string `json:"url"`
	Access string `json:"access"`

	Database string `json:"database,omitempty"`
	User     string `json:"user,omitempty"`
	// Deprecated: Use secureJsonData.password instead.
	Password string `json:"password,omitempty"`

	OrgID     int64 `json:"orgId,omitempty"`
	IsDefault bool  `json:"isDefault"`

	BasicAuth     bool   `json:"basicAuth"`
	BasicAuthUser string `json:"basicAuthUser,omitempty"`
	// Deprecated: Use secureJsonData.basicAuthPassword instead.
	BasicAuthPassword string `json:"basicAuthPassword,omitempty"`

	// Helper to read/write http headers
	HTTPHeaders map[string]string `json:"-"`

	JSONData       JSONData       `json:"jsonData,omitempty"`
	SecureJSONData SecureJSONData `json:"secureJsonData,omitempty"`
}

// Required to avoid recursion during (un)marshal
type _DataSource DataSource

// Marshal DataSource
func (ds *DataSource) MarshalJSON() ([]byte, error) {
	dataSource := _DataSource(*ds)
	for name, value := range ds.HTTPHeaders {
		dataSource.JSONData.httpHeaderNames = append(dataSource.JSONData.httpHeaderNames, name)
		dataSource.SecureJSONData.httpHeaderValues = append(dataSource.SecureJSONData.httpHeaderValues, value)
	}
	return json.Marshal(dataSource)
}

// Unmarshal DataSource
func (ds *DataSource) UnmarshalJSON(b []byte) (err error) {
	dataSource := _DataSource(*ds)
	if err = json.Unmarshal(b, &dataSource); err == nil {
		*ds = DataSource(dataSource)
	}
	ds.HTTPHeaders = make(map[string]string)
	for _, value := range ds.JSONData.httpHeaderNames {
		ds.HTTPHeaders[value] = "true" // HTTP Headers are not returned by the API
	}
	return err
}

// JSONData is a representation of the datasource `jsonData` property
type JSONData struct {
	// Used by all datasources
	TLSAuth           bool `json:"tlsAuth,omitempty"`
	TLSAuthWithCACert bool `json:"tlsAuthWithCACert,omitempty"`
	TLSSkipVerify     bool `json:"tlsSkipVerify,omitempty"`
	httpHeaderNames   []string

	// Used by Athena
	Catalog        string `json:"catalog,omitempty"`
	Database       string `json:"database,omitempty"`
	OutputLocation string `json:"outputLocation,omitempty"`
	Workgroup      string `json:"workgroup,omitempty"`

	// Used by Github
	GitHubURL string `json:"githubUrl,omitempty"`

	// Used by Graphite
	GraphiteVersion string `json:"graphiteVersion,omitempty"`

	// Used by Prometheus, Elasticsearch, InfluxDB, MySQL, PostgreSQL and MSSQL
	TimeInterval string `json:"timeInterval,omitempty"`

	// Used by Elasticsearch
	// From Grafana 8.x esVersion is the semantic version of Elasticsearch.
	EsVersion                  string `json:"esVersion,omitempty"`
	TimeField                  string `json:"timeField,omitempty"`
	Interval                   string `json:"interval,omitempty"`
	LogMessageField            string `json:"logMessageField,omitempty"`
	LogLevelField              string `json:"logLevelField,omitempty"`
	MaxConcurrentShardRequests int64  `json:"maxConcurrentShardRequests,omitempty"`

	// Used by Cloudwatch
	CustomMetricsNamespaces string `json:"customMetricsNamespaces,omitempty"`

	// Used by Cloudwatch, Athena
	AuthType      string `json:"authType,omitempty"`
	AssumeRoleArn string `json:"assumeRoleArn,omitempty"`
	DefaultRegion string `json:"defaultRegion,omitempty"`
	Endpoint      string `json:"endpoint,omitempty"`
	ExternalID    string `json:"externalId,omitempty"`
	Profile       string `json:"profile,omitempty"`

	// Used by OpenTSDB
	TsdbVersion    string `json:"tsdbVersion,omitempty"`
	TsdbResolution string `json:"tsdbResolution,omitempty"`

	// Used by MSSQL
	Encrypt string `json:"encrypt,omitempty"`

	// Used by PostgreSQL
	Sslmode         string `json:"sslmode,omitempty"`
	PostgresVersion int64  `json:"postgresVersion,omitempty"`
	Timescaledb     bool   `json:"timescaledb,omitempty"`

	// Used by MySQL, PostgreSQL and MSSQL
	MaxOpenConns    int64 `json:"maxOpenConns,omitempty"`
	MaxIdleConns    int64 `json:"maxIdleConns,omitempty"`
	ConnMaxLifetime int64 `json:"connMaxLifetime,omitempty"`

	// Used by Prometheus
	HTTPMethod   string `json:"httpMethod,omitempty"`
	QueryTimeout string `json:"queryTimeout,omitempty"`

	// Used by Stackdriver
	AuthenticationType string `json:"authenticationType,omitempty"`
	ClientEmail        string `json:"clientEmail,omitempty"`
	DefaultProject     string `json:"defaultProject,omitempty"`
	TokenURI           string `json:"tokenUri,omitempty"`

	// Used by Prometheus and Elasticsearch
	SigV4AssumeRoleArn string `json:"sigV4AssumeRoleArn,omitempty"`
	SigV4Auth          bool   `json:"sigV4Auth,omitempty"`
	SigV4AuthType      string `json:"sigV4AuthType,omitempty"`
	SigV4ExternalID    string `json:"sigV4ExternalID,omitempty"`
	SigV4Profile       string `json:"sigV4Profile,omitempty"`
	SigV4Region        string `json:"sigV4Region,omitempty"`

	// Used by Prometheus and Loki
	ManageAlerts    bool   `json:"manageAlerts,omitempty"`
	AlertmanagerUID string `json:"alertmanagerUid,omitempty"`

	// Used by Alertmanager
	Implementation string `json:"implementation,omitempty"`
}

// Required to avoid recursion during (un)marshal
type _JSONData JSONData

// Marshal JSONData
func (jd JSONData) MarshalJSON() ([]byte, error) {
	jsonData := _JSONData(jd)
	b, err := json.Marshal(jsonData)
	if err != nil {
		return nil, err
	}
	fields := make(map[string]interface{})
	if err = json.Unmarshal(b, &fields); err != nil {
		return nil, err
	}
	for index, name := range jd.httpHeaderNames {
		fields[fmt.Sprintf("httpHeaderName%d", index+1)] = name
	}
	return json.Marshal(fields)
}

// Unmarshal JSONData
func (jd *JSONData) UnmarshalJSON(b []byte) (err error) {
	jsonData := _JSONData(*jd)
	if err = json.Unmarshal(b, &jsonData); err == nil {
		*jd = JSONData(jsonData)
	}
	fields := make(map[string]interface{})
	if err = json.Unmarshal(b, &fields); err == nil {
		headerCount := 0
		for name := range fields {
			match := headerNameRegex.FindStringSubmatch(name)
			if len(match) > 0 {
				headerCount++
			}
		}

		jd.httpHeaderNames = make([]string, headerCount)
		for name, value := range fields {
			match := headerNameRegex.FindStringSubmatch(name)
			if len(match) == 2 {
				index, err := strconv.ParseInt(match[1], 10, 64)
				if err != nil {
					return err
				}
				jd.httpHeaderNames[index-1] = value.(string)
			}
		}
	}
	return err
}

// SecureJSONData is a representation of the datasource `secureJsonData` property
type SecureJSONData struct {
	// Used by all datasources
	TLSCACert         string `json:"tlsCACert,omitempty"`
	TLSClientCert     string `json:"tlsClientCert,omitempty"`
	TLSClientKey      string `json:"tlsClientKey,omitempty"`
	Password          string `json:"password,omitempty"`
	BasicAuthPassword string `json:"basicAuthPassword,omitempty"`
	httpHeaderValues  []string

	// Used by Cloudwatch, Athena
	AccessKey string `json:"accessKey,omitempty"`
	SecretKey string `json:"secretKey,omitempty"`

	// Used by Stackdriver
	PrivateKey string `json:"privateKey,omitempty"`

	// Used by Prometheus and Elasticsearch
	SigV4AccessKey string `json:"sigV4AccessKey,omitempty"`
	SigV4SecretKey string `json:"sigV4SecretKey,omitempty"`

	// Used by GitHub
	AccessToken string `json:"accessToken,omitempty"`
}

// Required to avoid recursion during unmarshal
type _SecureJSONData SecureJSONData

// Marshal SecureJSONData
func (sjd SecureJSONData) MarshalJSON() ([]byte, error) {
	secureJSONData := _SecureJSONData(sjd)
	b, err := json.Marshal(secureJSONData)
	if err != nil {
		return nil, err
	}
	fields := make(map[string]interface{})
	if err = json.Unmarshal(b, &fields); err != nil {
		return nil, err
	}
	for index, value := range sjd.httpHeaderValues {
		fields[fmt.Sprintf("httpHeaderValue%d", index+1)] = value
	}
	return json.Marshal(fields)
}

// NewDataSource creates a new Grafana data source.
func (c *Client) NewDataSource(s *DataSource) (int64, error) {
	data, err := json.Marshal(s)
	if err != nil {
		return 0, err
	}

	result := struct {
		ID int64 `json:"id"`
	}{}

	err = c.request("POST", "/api/datasources", nil, bytes.NewBuffer(data), &result)
	if err != nil {
		return 0, err
	}

	return result.ID, err
}

// UpdateDataSource updates a Grafana data source.
func (c *Client) UpdateDataSource(s *DataSource) error {
	path := fmt.Sprintf("/api/datasources/%d", s.ID)
	data, err := json.Marshal(s)
	if err != nil {
		return err
	}

	return c.request("PUT", path, nil, bytes.NewBuffer(data), nil)
}

// DataSource fetches and returns the Grafana data source whose ID it's passed.
func (c *Client) DataSource(id int64) (*DataSource, error) {
	path := fmt.Sprintf("/api/datasources/%d", id)
	result := &DataSource{}
	err := c.request("GET", path, nil, nil, result)
	if err != nil {
		return nil, err
	}

	return result, err
}

// DeleteDataSource deletes the Grafana data source whose ID it's passed.
func (c *Client) DeleteDataSource(id int64) error {
	path := fmt.Sprintf("/api/datasources/%d", id)

	return c.request("DELETE", path, nil, nil, nil)
}
<|endoftext|>


<commit_before>package bundled

const epmElv = `# Verbosity configuration
debug-mode = $false

# Configuration for common domains
-default-domain-config = [
  &"github.com"= [
    &method= git
    &protocol= https
    &levels= 2
  ]
  &"bitbucket.org"= [
    &method= git
    &protocol= https
    &levels= 2
  ]
  &"gitlab.com"= [
    &method= git
    &protocol= https
    &levels= 2
  ]
]

# Internal configuration
-data-dir = ~/.elvish
-lib-dir = $-data-dir/lib

# Runtime state - records get copied from -default-domain-config
# as needed
-domain-config = [&]

# Utility functions
fn -debug [text]{
  if $debug-mode {
    print (edit:styled '=> ' blue)
    echo $text
  }
}

fn -info [text]{
  print (edit:styled '=> ' green)
  echo $text
}

fn -warn [text]{
  print (edit:styled '=> ' yellow)
  echo $text
}

fn -error [text]{
  print (edit:styled '=> ' red)
  echo $text
}

fn dest [pkg]{
  put $-lib-dir/$pkg
}

fn is-installed [pkg]{
  put ?(test -e (dest $pkg))
}

fn -package-domain [pkg]{
  splits / $pkg | take 1
}

fn -package-without-domain [pkg]{
  joins / [(splits / $pkg | drop 1)]
}

# Known method handlers. Each entry is indexed by method name (the
# value of the "method" key in the domain configs), and must contain
# two keys: install and upgrade, each one must be a closure that
# received two arguments: package name and the domain config entry
-method-handler = [
  &git= [
    &install= [pkg dom-cfg]{
      dest = (dest $pkg)
      -info "Installing "$pkg
      mkdir -p $dest
      git clone $dom-cfg[protocol]"://"$pkg $dest
    }

    &upgrade= [pkg dom-cfg]{
      dest = (dest $pkg)
      -info "Updating "$pkg
      git -C $dest pull
    }
  ]

  &rsync= [
    &install= [pkg dom-cfg]{
      dest = (dest $pkg)
      pkgd = (-package-without-domain $pkg)
      -info "Installing "$pkg
      rsync -av $dom-cfg[location]/$pkgd/ $dest
    }

    &upgrade= [pkg dom-cfg]{
      dest = (dest $pkg)
      pkgd = (-package-without-domain $pkg)
      if (not (is-installed $pkg)) {
        -error "Package "$pkg" is not installed."
        return
      }
      -info "Updating "$pkg
      rsync -av $dom-cfg[location]/$pkgd/ $dest
    }
  ]
]

fn -domain-config-file [dom]{
  put $-lib-dir/$dom/epm-domain.cfg
}

fn -package-metadata-file [pkg]{
  put (dest $pkg)/metadata.json
}

fn -read-domain-config [dom]{
  cfgfile = (-domain-config-file $dom)
  # Only read config if it hasn't been loaded already
  if (not (has-key $-domain-config $dom)) {
    if ?(test -f $cfgfile) {
      # If the config file exists, read it...
      -domain-config[$dom] = (cat $cfgfile | from-json)
      -debug "Read domain config for "$dom": "(to-string $-domain-config[$dom])
    } else {
      # ...otherwise check if we have a default config for the domain, and save it
      if (has-key $-default-domain-config $dom) {
        -domain-config[$dom] = $-default-domain-config[$dom]
        -debug "No existing config for "$dom", using the default: "(to-string $-domain-config[$dom])
        mkdir -p (dirname $cfgfile)
        put $-domain-config[$dom] | to-json > $cfgfile
      } else {
        fail "No existing config for "$dom" and no default available. Please create config file "(tilde-abbr $cfgfile)" by hand"
      }
    }
  }
}

# Invoke package operations defined in $-method-handler above
fn -package-op [pkg what]{
  dom = (-package-domain $pkg)
  -read-domain-config $dom
  if (has-key $-domain-config $dom) {
    cfg = $-domain-config[$dom]
    method = $cfg[method]
    if (has-key $-method-handler $method) {
      if (has-key $-method-handler[$method] $what) {
        $-method-handler[$method][$what] $pkg $cfg
      } else {
        fail "No handler for '"$what"' defined in method '"$method"'"
      }
    } else {
      fail "No handler defined for method '"$method"', specified in in config file "(-domain-config-file $dom)
    }
  }
}

fn metadata [pkg]{
  res = [&]
  mdata = (-package-metadata-file $pkg)
  if (and (is-installed $pkg) ?(test -f $mdata)) {
    res = (cat $mdata | from-json)
  }
  put $res
}

fn -uninstall-package [pkg]{
  if (not (is-installed $pkg)) {
    -error "Package "$pkg" is not installed."
    return
  }
  dest = (dest $pkg)
  -info "Removing package "$pkg
  rm -rf $dest
}

fn installed {
  e:ls $-lib-dir | each [dom]{
    if ?(test -f (-domain-config-file $dom)) {
      -read-domain-config $dom
      lvl = $-domain-config[$dom][levels]
      find $-lib-dir/$dom -type d -depth $lvl | each [pkg]{
        replaces $-lib-dir/ "" $pkg
      }
    }
  }
}

######################################################################

fn install [&silent-if-installed=$false @pkgs]{
  for pkg $pkgs {
    if (is-installed $pkg) {
      if (not $silent-if-installed) {
        -info "Package "$pkg" is already installed."
      }
    } else {
      -package-op $pkg install
    }
  }
}

fn upgrade [@pkgs]{
  if (eq $pkgs []) {
    pkgs = [(installed)]
    -info 'Upgrading all installed packages'
  }
  for pkg $pkgs {
    if (not (is-installed $pkg)) {
      -error "Package "$pkg" is not installed."
    } else {
      -package-op $pkg upgrade
    }
  }
}

fn uninstall [@pkgs]{
  if (eq $pkgs []) {
    fail 'Must specify at least one package.'
    return
  }
  for pkg $pkgs {
    -uninstall-package $pkg
  }
}`
<commit_msg>Initial epm:query function<commit_after>package bundled

const epmElv = `# Verbosity configuration
debug-mode = $false

# Configuration for common domains
-default-domain-config = [
  &"github.com"= [
    &method= git
    &protocol= https
    &levels= 2
  ]
  &"bitbucket.org"= [
    &method= git
    &protocol= https
    &levels= 2
  ]
  &"gitlab.com"= [
    &method= git
    &protocol= https
    &levels= 2
  ]
]

# Internal configuration
-data-dir = ~/.elvish
-lib-dir = $-data-dir/lib

# Runtime state - records get copied from -default-domain-config
# as needed
-domain-config = [&]

# Utility functions
fn -debug [text]{
  if $debug-mode {
    print (edit:styled '=> ' blue)
    echo $text
  }
}

fn -info [text]{
  print (edit:styled '=> ' green)
  echo $text
}

fn -warn [text]{
  print (edit:styled '=> ' yellow)
  echo $text
}

fn -error [text]{
  print (edit:styled '=> ' red)
  echo $text
}

fn dest [pkg]{
  put $-lib-dir/$pkg
}

fn is-installed [pkg]{
  put ?(test -e (dest $pkg))
}

fn -package-domain [pkg]{
  splits / $pkg | take 1
}

fn -package-without-domain [pkg]{
  joins / [(splits / $pkg | drop 1)]
}

# Known method handlers. Each entry is indexed by method name (the
# value of the "method" key in the domain configs), and must contain
# two keys: install and upgrade, each one must be a closure that
# received two arguments: package name and the domain config entry
-method-handler = [
  &git= [
    &install= [pkg dom-cfg]{
      dest = (dest $pkg)
      -info "Installing "$pkg
      mkdir -p $dest
      git clone $dom-cfg[protocol]"://"$pkg $dest
    }

    &upgrade= [pkg dom-cfg]{
      dest = (dest $pkg)
      -info "Updating "$pkg
      git -C $dest pull
    }
  ]

  &rsync= [
    &install= [pkg dom-cfg]{
      dest = (dest $pkg)
      pkgd = (-package-without-domain $pkg)
      -info "Installing "$pkg
      rsync -av $dom-cfg[location]/$pkgd/ $dest
    }

    &upgrade= [pkg dom-cfg]{
      dest = (dest $pkg)
      pkgd = (-package-without-domain $pkg)
      if (not (is-installed $pkg)) {
        -error "Package "$pkg" is not installed."
        return
      }
      -info "Updating "$pkg
      rsync -av $dom-cfg[location]/$pkgd/ $dest
    }
  ]
]

fn -domain-config-file [dom]{
  put $-lib-dir/$dom/epm-domain.cfg
}

fn -package-metadata-file [pkg]{
  put (dest $pkg)/metadata.json
}

fn -read-domain-config [dom]{
  cfgfile = (-domain-config-file $dom)
  # Only read config if it hasn't been loaded already
  if (not (has-key $-domain-config $dom)) {
    if ?(test -f $cfgfile) {
      # If the config file exists, read it...
      -domain-config[$dom] = (cat $cfgfile | from-json)
      -debug "Read domain config for "$dom": "(to-string $-domain-config[$dom])
    } else {
      # ...otherwise check if we have a default config for the domain, and save it
      if (has-key $-default-domain-config $dom) {
        -domain-config[$dom] = $-default-domain-config[$dom]
        -debug "No existing config for "$dom", using the default: "(to-string $-domain-config[$dom])
        mkdir -p (dirname $cfgfile)
        put $-domain-config[$dom] | to-json > $cfgfile
      } else {
        fail "No existing config for "$dom" and no default available. Please create config file "(tilde-abbr $cfgfile)" by hand"
      }
    }
  }
}

# Invoke package operations defined in $-method-handler above
fn -package-op [pkg what]{
  dom = (-package-domain $pkg)
  -read-domain-config $dom
  if (has-key $-domain-config $dom) {
    cfg = $-domain-config[$dom]
    method = $cfg[method]
    if (has-key $-method-handler $method) {
      if (has-key $-method-handler[$method] $what) {
        $-method-handler[$method][$what] $pkg $cfg
      } else {
        fail "No handler for '"$what"' defined in method '"$method"'"
      }
    } else {
      fail "No handler defined for method '"$method"', specified in in config file "(-domain-config-file $dom)
    }
  }
}

fn metadata [pkg]{
  res = [&]
  mdata = (-package-metadata-file $pkg)
  if (and (is-installed $pkg) ?(test -f $mdata)) {
    res = (cat $mdata | from-json)
  }
  put $res
}

fn query [pkg]{
  data = (metadata $pkg)
  echo (edit:styled "Package "$pkg cyan)
  if (is-installed $pkg) {
    echo (edit:styled "Installed at "(dest $pkg) green)
  } else {
    echo (edit:styled "Not installed" red)
  }
  keys $data | each [key]{
    echo (edit:styled $key":" blue) $data[$key]
  }
}

fn -uninstall-package [pkg]{
  if (not (is-installed $pkg)) {
    -error "Package "$pkg" is not installed."
    return
  }
  dest = (dest $pkg)
  -info "Removing package "$pkg
  rm -rf $dest
}

fn installed {
  e:ls $-lib-dir | each [dom]{
    if ?(test -f (-domain-config-file $dom)) {
      -read-domain-config $dom
      lvl = $-domain-config[$dom][levels]
      find $-lib-dir/$dom -type d -depth $lvl | each [pkg]{
        replaces $-lib-dir/ "" $pkg
      }
    }
  }
}

######################################################################

fn install [&silent-if-installed=$false @pkgs]{
  for pkg $pkgs {
    if (is-installed $pkg) {
      if (not $silent-if-installed) {
        -info "Package "$pkg" is already installed."
      }
    } else {
      -package-op $pkg install
    }
  }
}

fn upgrade [@pkgs]{
  if (eq $pkgs []) {
    pkgs = [(installed)]
    -info 'Upgrading all installed packages'
  }
  for pkg $pkgs {
    if (not (is-installed $pkg)) {
      -error "Package "$pkg" is not installed."
    } else {
      -package-op $pkg upgrade
    }
  }
}

fn uninstall [@pkgs]{
  if (eq $pkgs []) {
    fail 'Must specify at least one package.'
    return
  }
  for pkg $pkgs {
    -uninstall-package $pkg
  }
}`
<|endoftext|>


<commit_before>package db

import (
	"encoding/json"
	"io/ioutil"
	"os"
	"path"
	"testing"
)

/*
 * Lowest-layer database tests
 */

func Test_DbFileCreation(t *testing.T) {
	dir, err := ioutil.TempDir("", "rqlite-test-")
	defer os.RemoveAll(dir)

	db, err := Open(path.Join(dir, "test_db"))
	if err != nil {
		t.Fatalf("failed to open new database: %s", err.Error())
	}
	if db == nil {
		t.Fatal("database is nil")
	}
	err = db.Close()
	if err != nil {
		t.Fatalf("failed to close database: %s", err.Error())
	}
}

func Test_TableCreation(t *testing.T) {
	db, path := mustOpenDatabase()
	defer db.Close()
	defer os.Remove(path)

	err := db.Execute("create table foo (id integer not null primary key, name text)")
	if err != nil {
		t.Fatalf("failed to create table: %s", err.Error())
	}

	r, err := db.Query("SELECT * FROM foo")
	if err != nil {
		t.Fatalf("failed to query empty table: %s", err.Error())
	}
	if len(r) != 0 {
		t.Fatalf("expected 0 results, got %d", len(r))
	}
}

func Test_SimpleStatements(t *testing.T) {
	db, path := mustOpenDatabase()
	defer db.Close()
	defer os.Remove(path)

	if err := db.Execute("create table foo (id integer not null primary key, name text)"); err != nil {
		t.Fatalf("failed to create table: %s", err.Error())
	}

	if err := db.Execute(`INSERT INTO foo(name) VALUES("fiona")`); err != nil {
		t.Fatalf("failed to insert record: %s", err.Error())
	}
	r, err := db.Query("SELECT * FROM foo")
	if err != nil {
		t.Fatalf("failed to query empty table: %s", err.Error())
	}
	if exp, got := `[{"id":"1","name":"fiona"}]`, asJson(r); exp != got {
		t.Fatalf("unexpected results for query, expected %s, got %s", exp, got)
	}

	if err := db.Execute(`INSERT INTO foo(name) VALUES("aoife")`); err != nil {
		t.Fatalf("failed to insert record: %s", err.Error())
	}
	r, err = db.Query("SELECT * FROM foo")
	if err != nil {
		t.Fatalf("failed to query table: %s", err.Error())
	}
	if exp, got := `[{"id":"1","name":"fiona"},{"id":"2","name":"aoife"}]`, asJson(r); exp != got {
		t.Fatalf("unexpected results for query, expected %s, got %s", exp, got)
	}

	if err := db.Execute("UPDATE foo SET Name='Who knows?' WHERE Id=1"); err != nil {
		t.Fatalf("failed to update record: %s", err.Error())
	}
	r, err = db.Query("SELECT * FROM foo")
	if err != nil {
		t.Fatalf("failed to query table: %s", err.Error())
	}
	if exp, got := `[{"id":"1","name":"Who knows?"},{"id":"2","name":"aoife"}]`, asJson(r); exp != got {
		t.Fatalf("unexpected results for query, expected %s, got %s", exp, got)
	}

	if err := db.Execute("DELETE FROM foo WHERE Id=2"); err != nil {
		t.Fatalf("failed to delete record: %s", err.Error())
	}
	r, err = db.Query("SELECT * FROM foo")
	if err != nil {
		t.Fatalf("failed to query table: %s", err.Error())
	}
	if exp, got := `[{"id":"1","name":"Who knows?"}]`, asJson(r); exp != got {
		t.Fatalf("unexpected results for query, expected %s, got %s", exp, got)
	}
}

func Test_FailingSimpleStatements(t *testing.T) {
	db, path := mustOpenDatabase()
	defer db.Close()
	defer os.Remove(path)

	err := db.Execute(`INSERT INTO foo(name) VALUES("fiona")`)
	if err == nil {
		t.Fatal("inserted record into empty table OK")
	}
	if err.Error() != "no such table: foo" {
		t.Fatal("unexpected error returned")
	}

	if err = db.Execute("create table foo (id integer not null primary key, name text)"); err != nil {
		t.Fatalf("failed to create table: %s", err.Error())
	}
	if err = db.Execute("create table foo (id integer not null primary key, name text)"); err == nil {
		t.Fatal("duplicate table created OK")
	}
	if err.Error() != "table foo already exists" {
		t.Fatalf("unexpected error returned: %s", err.Error())
	}

	if err = db.Execute(`INSERT INTO foo(id, name) VALUES(11, "fiona")`); err != nil {
		t.Fatalf("failed to insert record: %s", err.Error())
	}
	if err = db.Execute(`INSERT INTO foo(id, name) VALUES(11, "fiona")`); err == nil {
		t.Fatal("duplicated record inserted OK")
	}
	if err.Error() != "UNIQUE constraint failed: foo.id" {
		t.Fatal("unexpected error returned")
	}

	if err = db.Execute("SELECT * FROM bar"); err == nil {
		t.Fatal("no error when querying non-existant table")
	}
	if err.Error() != "no such table: bar" {
		t.Fatal("unexpected error returned")
	}

	if err = db.Execute("utter nonsense"); err == nil {
		t.Fatal("no error when issuing nonsense query")
	}
	if err.Error() != `near "utter": syntax error` {
		t.Fatal("unexpected error returned")
	}
}

func Test_SimpleTransactions(t *testing.T) {
	db, path := mustOpenDatabase()
	defer db.Close()
	defer os.Remove(path)

	if err := db.Execute("create table foo (id integer not null primary key, name text)"); err != nil {
		t.Fatalf("failed to create table: %s", err.Error())
	}

	if err := db.StartTransaction(); err != nil {
		t.Fatalf("failed to start transaction: %s", err.Error())
	}
	for i := 0; i < 10; i++ {
		_ = db.Execute(`INSERT INTO foo(name) VALUES("philip")`)
	}
	if err := db.CommitTransaction(); err != nil {
		t.Fatalf("failed to commit transaction: %s", err.Error())
	}

	r, err := db.Query("SELECT name FROM foo")
	if err != nil {
		t.Fatalf("failed to query after commited transaction: %s", err.Error())
	}
	if len(r) != 10 {
		t.Fatalf("incorrect number of results returned: %d", len(r))
	}

	if err := db.StartTransaction(); err != nil {
		t.Fatalf("failed to start transaction: %s", err.Error())
	}
	for i := 0; i < 10; i++ {
		_ = db.Execute(`INSERT INTO foo(name) VALUES("philip")`)
	}
	if err := db.RollbackTransaction(); err != nil {
		t.Fatalf("failed to rollback transaction: %s", err.Error())
	}

	r, err = db.Query("SELECT name FROM foo")
	if err != nil {
		t.Fatalf("failed to query after commited transaction: %s", err.Error())
	}
	if len(r) != 10 {
		t.Fatalf("incorrect number of results returned: %d", len(r))
	}
}

// func Test_TransactionsConstraintViolation(t *testing.T) {
// 	dir, err := ioutil.TempDir("", "rqlite-test-")
// 	defer os.RemoveAll(dir)
// 	db := New(path.Join(dir, "test_db"))
// 	defer db.Close()

// 	err = db.Execute("create table foo (id integer not null primary key, name text)")
// 	c.Assert(err, IsNil)

// 	err = db.StartTransaction()
// 	c.Assert(err, IsNil)
// 	err = db.Execute("INSERT INTO foo(id, name) VALUES(1, \"fiona\")")
// 	c.Assert(err, IsNil)
// 	err = db.Execute("INSERT INTO foo(id, name) VALUES(1, \"fiona\")")
// 	c.Assert(err, NotNil)
// 	err = db.RollbackTransaction()
// 	c.Assert(err, IsNil)

// 	r, err := db.Query("SELECT * FROM foo")
// 	c.Assert(len(r), Equals, 0)
// }

// func Test_TransactionsHardFail(t *testing.T) {
// 	dir, err := ioutil.TempDir("", "rqlite-test-")
// 	defer os.RemoveAll(dir)
// 	db := New(path.Join(dir, "test_db"))

// 	err = db.Execute("create table foo (id integer not null primary key, name text)")
// 	c.Assert(err, IsNil)
// 	err = db.Execute("INSERT INTO foo(id, name) VALUES(1, \"fiona\")")
// 	c.Assert(err, IsNil)

// 	err = db.StartTransaction()
// 	c.Assert(err, IsNil)
// 	err = db.Execute("INSERT INTO foo(id, name) VALUES(2, \"dana\")")
// 	c.Assert(err, IsNil)
// 	db.Close()

// 	db = Open(path.Join(dir, "test_db"))
// 	c.Assert(db, NotNil)
// 	r, err := db.Query("SELECT * FROM foo")
// 	c.Assert(len(r), Equals, 1)
// 	c.Assert(r[0]["name"], Equals, "fiona")
// 	db.Close()
// }

func mustOpenDatabase() (*DB, string) {
	var err error
	f, err := ioutil.TempFile("", "rqlilte-test-")
	if err != nil {
		panic("failed to create temp file")
	}
	f.Close()

	db, err := Open(f.Name())
	if err != nil {
		panic("failed to open database")
	}

	return db, f.Name()
}

func asJson(v interface{}) string {
	b, err := json.Marshal(v)
	if err != nil {
		panic("failed to JSON marshal value")
	}
	return string(b)
}
<commit_msg>Test_TransactionsConstraintViolation now passing<commit_after>package db

import (
	"encoding/json"
	"io/ioutil"
	"os"
	"path"
	"testing"
)

/*
 * Lowest-layer database tests
 */

func Test_DbFileCreation(t *testing.T) {
	dir, err := ioutil.TempDir("", "rqlite-test-")
	defer os.RemoveAll(dir)

	db, err := Open(path.Join(dir, "test_db"))
	if err != nil {
		t.Fatalf("failed to open new database: %s", err.Error())
	}
	if db == nil {
		t.Fatal("database is nil")
	}
	err = db.Close()
	if err != nil {
		t.Fatalf("failed to close database: %s", err.Error())
	}
}

func Test_TableCreation(t *testing.T) {
	db, path := mustOpenDatabase()
	defer db.Close()
	defer os.Remove(path)

	err := db.Execute("create table foo (id integer not null primary key, name text)")
	if err != nil {
		t.Fatalf("failed to create table: %s", err.Error())
	}

	r, err := db.Query("SELECT * FROM foo")
	if err != nil {
		t.Fatalf("failed to query empty table: %s", err.Error())
	}
	if len(r) != 0 {
		t.Fatalf("expected 0 results, got %d", len(r))
	}
}

func Test_SimpleStatements(t *testing.T) {
	db, path := mustOpenDatabase()
	defer db.Close()
	defer os.Remove(path)

	if err := db.Execute("create table foo (id integer not null primary key, name text)"); err != nil {
		t.Fatalf("failed to create table: %s", err.Error())
	}

	if err := db.Execute(`INSERT INTO foo(name) VALUES("fiona")`); err != nil {
		t.Fatalf("failed to insert record: %s", err.Error())
	}
	r, err := db.Query("SELECT * FROM foo")
	if err != nil {
		t.Fatalf("failed to query empty table: %s", err.Error())
	}
	if exp, got := `[{"id":"1","name":"fiona"}]`, asJson(r); exp != got {
		t.Fatalf("unexpected results for query, expected %s, got %s", exp, got)
	}

	if err := db.Execute(`INSERT INTO foo(name) VALUES("aoife")`); err != nil {
		t.Fatalf("failed to insert record: %s", err.Error())
	}
	r, err = db.Query("SELECT * FROM foo")
	if err != nil {
		t.Fatalf("failed to query table: %s", err.Error())
	}
	if exp, got := `[{"id":"1","name":"fiona"},{"id":"2","name":"aoife"}]`, asJson(r); exp != got {
		t.Fatalf("unexpected results for query, expected %s, got %s", exp, got)
	}

	if err := db.Execute("UPDATE foo SET Name='Who knows?' WHERE Id=1"); err != nil {
		t.Fatalf("failed to update record: %s", err.Error())
	}
	r, err = db.Query("SELECT * FROM foo")
	if err != nil {
		t.Fatalf("failed to query table: %s", err.Error())
	}
	if exp, got := `[{"id":"1","name":"Who knows?"},{"id":"2","name":"aoife"}]`, asJson(r); exp != got {
		t.Fatalf("unexpected results for query, expected %s, got %s", exp, got)
	}

	if err := db.Execute("DELETE FROM foo WHERE Id=2"); err != nil {
		t.Fatalf("failed to delete record: %s", err.Error())
	}
	r, err = db.Query("SELECT * FROM foo")
	if err != nil {
		t.Fatalf("failed to query table: %s", err.Error())
	}
	if exp, got := `[{"id":"1","name":"Who knows?"}]`, asJson(r); exp != got {
		t.Fatalf("unexpected results for query, expected %s, got %s", exp, got)
	}
}

func Test_FailingSimpleStatements(t *testing.T) {
	db, path := mustOpenDatabase()
	defer db.Close()
	defer os.Remove(path)

	err := db.Execute(`INSERT INTO foo(name) VALUES("fiona")`)
	if err == nil {
		t.Fatal("inserted record into empty table OK")
	}
	if err.Error() != "no such table: foo" {
		t.Fatal("unexpected error returned")
	}

	if err = db.Execute("create table foo (id integer not null primary key, name text)"); err != nil {
		t.Fatalf("failed to create table: %s", err.Error())
	}
	if err = db.Execute("create table foo (id integer not null primary key, name text)"); err == nil {
		t.Fatal("duplicate table created OK")
	}
	if err.Error() != "table foo already exists" {
		t.Fatalf("unexpected error returned: %s", err.Error())
	}

	if err = db.Execute(`INSERT INTO foo(id, name) VALUES(11, "fiona")`); err != nil {
		t.Fatalf("failed to insert record: %s", err.Error())
	}
	if err = db.Execute(`INSERT INTO foo(id, name) VALUES(11, "fiona")`); err == nil {
		t.Fatal("duplicated record inserted OK")
	}
	if err.Error() != "UNIQUE constraint failed: foo.id" {
		t.Fatal("unexpected error returned")
	}

	if err = db.Execute("SELECT * FROM bar"); err == nil {
		t.Fatal("no error when querying non-existant table")
	}
	if err.Error() != "no such table: bar" {
		t.Fatal("unexpected error returned")
	}

	if err = db.Execute("utter nonsense"); err == nil {
		t.Fatal("no error when issuing nonsense query")
	}
	if err.Error() != `near "utter": syntax error` {
		t.Fatal("unexpected error returned")
	}
}

func Test_SimpleTransactions(t *testing.T) {
	db, path := mustOpenDatabase()
	defer db.Close()
	defer os.Remove(path)

	if err := db.Execute("create table foo (id integer not null primary key, name text)"); err != nil {
		t.Fatalf("failed to create table: %s", err.Error())
	}

	if err := db.StartTransaction(); err != nil {
		t.Fatalf("failed to start transaction: %s", err.Error())
	}
	for i := 0; i < 10; i++ {
		_ = db.Execute(`INSERT INTO foo(name) VALUES("philip")`)
	}
	if err := db.CommitTransaction(); err != nil {
		t.Fatalf("failed to commit transaction: %s", err.Error())
	}

	r, err := db.Query("SELECT name FROM foo")
	if err != nil {
		t.Fatalf("failed to query after commited transaction: %s", err.Error())
	}
	if len(r) != 10 {
		t.Fatalf("incorrect number of results returned: %d", len(r))
	}

	if err := db.StartTransaction(); err != nil {
		t.Fatalf("failed to start transaction: %s", err.Error())
	}
	for i := 0; i < 10; i++ {
		_ = db.Execute(`INSERT INTO foo(name) VALUES("philip")`)
	}
	if err := db.RollbackTransaction(); err != nil {
		t.Fatalf("failed to rollback transaction: %s", err.Error())
	}

	r, err = db.Query("SELECT name FROM foo")
	if err != nil {
		t.Fatalf("failed to query after commited transaction: %s", err.Error())
	}
	if len(r) != 10 {
		t.Fatalf("incorrect number of results returned: %d", len(r))
	}
}

func Test_TransactionsConstraintViolation(t *testing.T) {
	db, path := mustOpenDatabase()
	defer db.Close()
	defer os.Remove(path)
	var err error

	if err = db.Execute("create table foo (id integer not null primary key, name text)"); err != nil {
		t.Fatalf("failed to create table: %s", err.Error())
	}

	if err = db.StartTransaction(); err != nil {
		t.Fatalf("failed to start transaction: %s", err.Error())
	}
	if err = db.Execute(`INSERT INTO foo(id, name) VALUES(11, "fiona")`); err != nil {
		t.Fatalf("failed to insert record: %s", err.Error())
	}
	if err = db.Execute(`INSERT INTO foo(id, name) VALUES(11, "fiona")`); err == nil {
		t.Fatal("duplicated record inserted OK")
	}
	if err.Error() != "UNIQUE constraint failed: foo.id" {
		t.Fatal("unexpected error returned")
	}
	if err := db.RollbackTransaction(); err != nil {
		t.Fatalf("failed to rollback transaction: %s", err.Error())
	}
	r, err := db.Query("SELECT name FROM foo")
	if err != nil {
		t.Fatalf("failed to query after commited transaction: %s", err.Error())
	}
	if len(r) != 0 {
		t.Fatalf("incorrect number of results returned: %d", len(r))
	}
}

// func Test_TransactionsHardFail(t *testing.T) {
// 	dir, err := ioutil.TempDir("", "rqlite-test-")
// 	defer os.RemoveAll(dir)
// 	db := New(path.Join(dir, "test_db"))

// 	err = db.Execute("create table foo (id integer not null primary key, name text)")
// 	c.Assert(err, IsNil)
// 	err = db.Execute("INSERT INTO foo(id, name) VALUES(1, \"fiona\")")
// 	c.Assert(err, IsNil)

// 	err = db.StartTransaction()
// 	c.Assert(err, IsNil)
// 	err = db.Execute("INSERT INTO foo(id, name) VALUES(2, \"dana\")")
// 	c.Assert(err, IsNil)
// 	db.Close()

// 	db = Open(path.Join(dir, "test_db"))
// 	c.Assert(db, NotNil)
// 	r, err := db.Query("SELECT * FROM foo")
// 	c.Assert(len(r), Equals, 1)
// 	c.Assert(r[0]["name"], Equals, "fiona")
// 	db.Close()
// }

func mustOpenDatabase() (*DB, string) {
	var err error
	f, err := ioutil.TempFile("", "rqlilte-test-")
	if err != nil {
		panic("failed to create temp file")
	}
	f.Close()

	db, err := Open(f.Name())
	if err != nil {
		panic("failed to open database")
	}

	return db, f.Name()
}

func asJson(v interface{}) string {
	b, err := json.Marshal(v)
	if err != nil {
		panic("failed to JSON marshal value")
	}
	return string(b)
}
<|endoftext|>


<commit_before>package db

import (
	"errors"
	"reflect"
	"testing"

	"github.com/microservices-demo/user/users"
)

var (
	TestDB       = fake{}
	ErrFakeError = errors.New("Fake error")
	TestAddress  = users.Address{
		Street:  "street",
		Number:  "51b",
		Country: "Netherlands",
		City:    "Amsterdam",
		ID:      "000056",
	}
)

func TestInit(t *testing.T) {
	err := Init()
	if err == nil {
		t.Error("Expected no registered db error")
	}
	Register("test", TestDB)
	database = "test"
	err = Init()
	if err != ErrFakeError {
		t.Error("expected fake db error from init")
	}
}

func TestRegister(t *testing.T) {
	l := len(DBTypes)
	Register("test2", TestDB)
	if len(DBTypes) != l+1 {
		t.Errorf("Expecting %v DB types received %v", l+1, len(DBTypes))
	}
	l = len(DBTypes)
	Register("test2", TestDB)
	if len(DBTypes) != l {
		t.Errorf("Expecting %v DB types received %v duplicate names", l, len(DBTypes))
	}
}

func TestCreate(t *testing.T) {
	err := Create(&users.User{})
	if err != ErrFakeError {
		t.Error("expected fake db error from create")
	}
}

func TestGetById(t *testing.T) {
	_, err := GetUser("test")
	if err != ErrFakeError {
		t.Error("expected fake db error from get")
	}
}

func TestGetUserName(t *testing.T) {
	_, err := GetUserName("test")
	if err != ErrFakeError {
		t.Error("expected fake db error from get")
	}
}

func TestGetUserAttributes(t *testing.T) {
	u := users.New()
	GetUserAttributes(&u)
	if len(u.Addresses) != 1 {
		t.Error("expected one address added for GetUserAttributes")
	}
	if !reflect.DeepEqual(u.Addresses[0], TestAddress) {
		t.Error("expected matching addresses")
	}
}

type fake struct{}

func (f fake) Init() error {
	return ErrFakeError
}
func (f fake) GetUserByName(name string) (users.User, error) {
	return users.User{}, ErrFakeError
}
func (f fake) GetUser(name string) (users.User, error) {
	return users.User{}, ErrFakeError
}
func (f fake) GetUserAttributes(u *users.User) error {
	u.Addresses = append(u.Addresses, TestAddress)
	return nil
}
func (f fake) CreateUser(*users.User) error {
	return ErrFakeError
}
<commit_msg>added some bogus tests to pass<commit_after>package db

import (
	"errors"
	"reflect"
	"testing"

	"github.com/microservices-demo/user/users"
)

var (
	TestDB       = fake{}
	ErrFakeError = errors.New("Fake error")
	TestAddress  = users.Address{
		Street:  "street",
		Number:  "51b",
		Country: "Netherlands",
		City:    "Amsterdam",
		ID:      "000056",
	}
)

func TestInit(t *testing.T) {
	err := Init()
	if err == nil {
		t.Error("Expected no registered db error")
	}
	Register("test", TestDB)
	database = "test"
	err = Init()
	if err != ErrFakeError {
		t.Error("expected fake db error from init")
	}
}

func TestRegister(t *testing.T) {
	l := len(DBTypes)
	Register("test2", TestDB)
	if len(DBTypes) != l+1 {
		t.Errorf("Expecting %v DB types received %v", l+1, len(DBTypes))
	}
	l = len(DBTypes)
	Register("test2", TestDB)
	if len(DBTypes) != l {
		t.Errorf("Expecting %v DB types received %v duplicate names", l, len(DBTypes))
	}
}

func TestCreateUser(t *testing.T) {
	err := CreateUser(&users.User{})
	if err != ErrFakeError {
		t.Error("expected fake db error from create")
	}
}

func TestGetUser(t *testing.T) {
	_, err := GetUser("test")
	if err != ErrFakeError {
		t.Error("expected fake db error from get")
	}
}

func TestGetUserByName(t *testing.T) {
	_, err := GetUserByName("test")
	if err != ErrFakeError {
		t.Error("expected fake db error from get")
	}
}

func TestGetUserAttributes(t *testing.T) {
	u := users.New()
	GetUserAttributes(&u)
	if len(u.Addresses) != 1 {
		t.Error("expected one address added for GetUserAttributes")
	}
	if !reflect.DeepEqual(u.Addresses[0], TestAddress) {
		t.Error("expected matching addresses")
	}
}

type fake struct{}

func (f fake) Init() error {
	return ErrFakeError
}
func (f fake) GetUserByName(name string) (users.User, error) {
	return users.User{}, ErrFakeError
}
func (f fake) GetUser(id string) (users.User, error) {
	return users.User{}, ErrFakeError
}

func (f fake) GetUsers() ([]users.User, error) {
	return make([]users.User, 0), ErrFakeError
}

func (f fake) CreateUser(*users.User) error {
	return ErrFakeError
}

func (f fake) GetUserAttributes(u *users.User) error {
	u.Addresses = append(u.Addresses, TestAddress)
	return nil
}

func (f fake) GetCard(id string) (users.Card, error) {
	return users.Card{}, ErrFakeError
}

func (f fake) GetCards() ([]users.Card, error) {
	return make([]users.Card, 0), ErrFakeError
}

func (f fake) CreateCard(c *users.Card, id string) error {
	return ErrFakeError
}

func (f fake) GetAddress(id string) (users.Address, error) {
	return users.Address{}, ErrFakeError
}

func (f fake) GetAddresses() ([]users.Address, error) {
	return make([]users.Address, 0), ErrFakeError
}

func (f fake) CreateAddress(u *users.Address, id string) error {
	return ErrFakeError
}
<|endoftext|>


<commit_before>package db

import (
	"fmt"
)

type Pointer struct {
	Season  uint
	Episode uint
}

func (p Pointer) String() string {
	return fmt.Sprintf("S%02dE%02d", p.Season, p.Episode)
}
<commit_msg>Implement parsing pointer from string<commit_after>package db

import (
	"errors"
	"fmt"
	"regexp"
	"strconv"
)

type Pointer struct {
	Season  uint
	Episode uint
}

func (p Pointer) String() string {
	return fmt.Sprintf("S%02dE%02d", p.Season, p.Episode)
}

func PointerFromString(s string) (Pointer, error) {
	re := regexp.MustCompile(`^S(\d+)E(\d+)$`)
	match := re.FindStringSubmatch(s)
	if match == nil {
		return Pointer{}, errors.New("Invalid pointer")
	}

	season, _ := strconv.Atoi(match[1])
	episode, _ := strconv.Atoi(match[2])

	return Pointer{
		Season:  uint(season),
		Episode: uint(episode),
	}, nil
}
<|endoftext|>


<commit_before>package main

import (
	zmq "github.com/alecthomas/gozmq"
	"log"
)

type ZmqOutput struct {
	Addresses []interface{}
	Formatter JsonFormatter
	connected bool
	socket    *zmq.Socket
}

func (z *ZmqOutput) connect() {
	ctx, _ := zmq.NewContext()
	socket, _ := ctx.NewSocket(zmq.PUSH)
	socket.SetSndTimeout(0)
	for _, addr := range z.Addresses {
		log.Printf("[zmq] Connecting to: %s\n", addr)
		socket.Connect(addr.(string))
	}

	z.socket = socket
	z.connected = true
}

func (z *ZmqOutput) Emit(event Event) {
	if !z.connected {
		z.connect()
	}

	z.socket.Send(z.Formatter.Format(event), 0)
}
<commit_msg>remove crap<commit_after>package main

import (
	zmq "github.com/alecthomas/gozmq"
	"log"
)

type ZmqOutput struct {
	Addresses []interface{}
	Formatter JsonFormatter
	connected bool
	socket    *zmq.Socket
}

func (z *ZmqOutput) connect() {
	ctx, _ := zmq.NewContext()
	socket, _ := ctx.NewSocket(zmq.PUSH)
	for _, addr := range z.Addresses {
		log.Printf("[zmq] Connecting to: %s\n", addr)
		socket.Connect(addr.(string))
	}

	z.socket = socket
	z.connected = true
}

func (z *ZmqOutput) Emit(event Event) {
	if !z.connected {
		z.connect()
	}

	z.socket.Send(z.Formatter.Format(event), 0)
}
<|endoftext|>


<commit_before>package collision

import (
	"fmt"
	"strconv"

	"github.com/Sythe2o0/rtreego"
	"github.com/oakmound/oak/dlog"
	"github.com/oakmound/oak/event"
	"github.com/oakmound/oak/physics"
)

// ID Types constant
const (
	NONE = iota
	CID
	PID
)

// A Space is a rectangle
// with a couple of ways of identifying
// an underlying object.
type Space struct {
	Location *rtreego.Rect
	// A label can store type information.
	// Recommended to use with an enum.
	Label Label
	// A CID can be used to get the exact
	// entity which this rectangle belongs to.
	CID event.CID
	// Type represents which ID space the above ID
	// corresponds to.
	Type int
}

// Bounds satisfies the rtreego.Spatial interface.
func (s *Space) Bounds() *rtreego.Rect {
	return s.Location
}

// GetX returns a space's x position (leftmost)
func (s *Space) GetX() float64 {
	return s.Location.PointCoord(0)
}

// GetY returns a space's y position (upmost)
func (s *Space) GetY() float64 {
	return s.Location.PointCoord(1)
}

// GetW returns a space's width (rightmost x - leftmost x)
func (s *Space) GetW() float64 {
	return s.Location.LengthsCoord(0)
}

// GetH returns a space's height (upper y - lower y)
func (s *Space) GetH() float64 {
	return s.Location.LengthsCoord(1)
}

// GetCenter returns the center point of the space
func (s *Space) GetCenter() (float64, float64) {
	return s.GetX() + s.GetW()/2, s.GetY() + s.GetH()/2
}

// GetPos returns both y and x
func (s *Space) GetPos() (float64, float64) {
	return s.Location.PointCoord(1), s.Location.PointCoord(0)
}

// Above returns how much above this space another space is
// Important note: (10,10) is Above (10,20), because in oak's
// display, lower y values are higher than higher y values.
func (s *Space) Above(other *Space) float64 {
	return other.GetY() - s.GetY()
}

// Below returns how much below this space another space is,
// Equivalent to -1 * Above
func (s *Space) Below(other *Space) float64 {
	return s.GetY() - other.GetY()
}

// Contains returns whether this space contains other
func (s *Space) Contains(other *Space) bool {
	//You contain another space if it is fully inside your space
	//If you are the same size and location as the space you are checking then you both contain eachother
	if s.GetX() > other.GetX() || s.GetX()+s.GetW() < other.GetX()+other.GetW() ||
		s.GetY() > other.GetY() || s.GetY()+s.GetH() < other.GetY()+other.GetH() {
		return false
	}
	return true
}

// LeftOf returns how far to the left other is of this space
func (s *Space) LeftOf(other *Space) float64 {
	return other.GetX() - s.GetX()
}

// RightOf returns how far to the right other is of this space.
// Equivalent to -1 * LeftOf
func (s *Space) RightOf(other *Space) float64 {
	return s.GetX() - other.GetX()
}

// Overlap returns how much this space overlaps with another space
func (s *Space) Overlap(other *Space) (xOver, yOver float64) {
	if s.GetX() > other.GetX() {
		x2 := other.GetX() + other.GetW()
		if s.GetX() < x2 {
			xOver = s.GetX() - x2
		}
	} else {
		x2 := s.GetX() + s.GetW()
		if other.GetX() < x2 {
			xOver = x2 - other.GetX()
		}
	}
	if s.GetY() > other.GetY() {
		y2 := other.GetY() + other.GetH()
		if s.GetY() < y2 {
			yOver = s.GetY() - y2
		}
	} else {
		y2 := s.GetY() + s.GetW()
		if other.GetY() < y2 {
			yOver = y2 - other.GetY()
		}
	}
	return
}

// OverlapVector returns Overlap as a vector
func (s *Space) OverlapVector(other *Space) physics.Vector {
	xover, yover := s.Overlap(other)
	// Todo: why are we multiplying by -1 here, shouldn't that
	// also be happening in Overlap at least?
	return physics.NewVector(-xover, -yover)
}

// SubtractRect removes a rectangle from this rectangle and
// returns the rectangles remaining after the portion has been
// removed. The input x,y is relative to the original space:
// Example: removing 1,1 from 10,10 -> 12,12 is OK, but removing
// 11,11 from 10,10 -> 12,12 will not act as expected.
func (s *Space) SubtractRect(x2, y2, w2, h2 float64) []*Space {
	x1 := s.GetX()
	y1 := s.GetY()
	w1 := s.GetW()
	h1 := s.GetH()

	// Left, Top, Right, Bottom
	// X, Y, W, H
	rects := [4][4]float64{}

	rects[0][0] = x1
	rects[0][1] = y1
	rects[0][2] = x2
	rects[0][3] = h1

	// Todo: these spaces overlap on the corners. We could remove that.
	rects[1][0] = x1
	rects[1][1] = y1
	rects[1][2] = w1
	rects[1][3] = y2

	rects[2][0] = x1 + x2 + w2
	rects[2][1] = y1
	rects[2][2] = w1 - (x2 + w2)
	rects[2][3] = h1

	rects[3][0] = x1
	rects[3][1] = y1 + y2 + h2
	rects[3][2] = w1
	rects[3][3] = h1 - (y2 + h2)

	var spaces []*Space

	for _, r := range rects {
		if r[2] > 0 && r[3] > 0 {
			spaces = append(spaces, NewFullSpace(r[0], r[1], r[2], r[3], s.Label, s.CID))
		}
	}

	return spaces
}

func (s *Space) String() string {
	return strconv.FormatFloat(s.GetX(), 'f', 2, 32) + "," +
		strconv.FormatFloat(s.GetY(), 'f', 2, 32) + "," +
		strconv.FormatFloat(s.GetW(), 'f', 2, 32) + "," +
		strconv.FormatFloat(s.GetH(), 'f', 2, 32) + "::" +
		strconv.Itoa(int(s.CID)) + "::" + fmt.Sprintf("%p", s)
}

// NewUnassignedSpace returns a space that just has a rectangle
func NewUnassignedSpace(x, y, w, h float64) *Space {
	return NewLabeledSpace(x, y, w, h, NilLabel)
}

// NewSpace returns a space with an associated caller id
func NewSpace(x, y, w, h float64, cID event.CID) *Space {
	return NewFullSpace(x, y, w, h, NilLabel, cID)
}

// NewLabeledSpace returns a space with an associated integer label
func NewLabeledSpace(x, y, w, h float64, l Label) *Space {
	rect := NewRect(x, y, w, h)
	return &Space{
		Location: rect,
		Label:    l,
		Type:     NONE,
	}
}

// NewFullSpace returns a space with both a label and a caller id
func NewFullSpace(x, y, w, h float64, l Label, cID event.CID) *Space {
	rect := NewRect(x, y, w, h)
	return &Space{
		rect,
		l,
		cID,
		CID, // todo: This is hard to read as distinct from cID
		// todo: a way to generate non-CID typed spaces that isn't
		// package specific (see render/particle)
	}
}

// NewRect is a wrapper around rtreego.NewRect,
// casting the given x,y to an rtreego.Point.
// Used to not expose rtreego.Point to the user.
func NewRect(x, y, w, h float64) *rtreego.Rect {
	rect, err := rtreego.NewRect(rtreego.Point{x, y}, [3]float64{w, h, 1})
	if err != nil {
		dlog.Error(err)
	}
	return &rect
}
<commit_msg>Fixed Space.Overlap calculating wrong Y<commit_after>package collision

import (
	"fmt"
	"strconv"

	"github.com/Sythe2o0/rtreego"
	"github.com/oakmound/oak/dlog"
	"github.com/oakmound/oak/event"
	"github.com/oakmound/oak/physics"
)

// ID Types constant
const (
	NONE = iota
	CID
	PID
)

// A Space is a rectangle
// with a couple of ways of identifying
// an underlying object.
type Space struct {
	Location *rtreego.Rect
	// A label can store type information.
	// Recommended to use with an enum.
	Label Label
	// A CID can be used to get the exact
	// entity which this rectangle belongs to.
	CID event.CID
	// Type represents which ID space the above ID
	// corresponds to.
	Type int
}

// Bounds satisfies the rtreego.Spatial interface.
func (s *Space) Bounds() *rtreego.Rect {
	return s.Location
}

// GetX returns a space's x position (leftmost)
func (s *Space) GetX() float64 {
	return s.Location.PointCoord(0)
}

// GetY returns a space's y position (upmost)
func (s *Space) GetY() float64 {
	return s.Location.PointCoord(1)
}

// GetW returns a space's width (rightmost x - leftmost x)
func (s *Space) GetW() float64 {
	return s.Location.LengthsCoord(0)
}

// GetH returns a space's height (upper y - lower y)
func (s *Space) GetH() float64 {
	return s.Location.LengthsCoord(1)
}

// GetCenter returns the center point of the space
func (s *Space) GetCenter() (float64, float64) {
	return s.GetX() + s.GetW()/2, s.GetY() + s.GetH()/2
}

// GetPos returns both y and x
func (s *Space) GetPos() (float64, float64) {
	return s.Location.PointCoord(1), s.Location.PointCoord(0)
}

// Above returns how much above this space another space is
// Important note: (10,10) is Above (10,20), because in oak's
// display, lower y values are higher than higher y values.
func (s *Space) Above(other *Space) float64 {
	return other.GetY() - s.GetY()
}

// Below returns how much below this space another space is,
// Equivalent to -1 * Above
func (s *Space) Below(other *Space) float64 {
	return s.GetY() - other.GetY()
}

// Contains returns whether this space contains other
func (s *Space) Contains(other *Space) bool {
	//You contain another space if it is fully inside your space
	//If you are the same size and location as the space you are checking then you both contain eachother
	if s.GetX() > other.GetX() || s.GetX()+s.GetW() < other.GetX()+other.GetW() ||
		s.GetY() > other.GetY() || s.GetY()+s.GetH() < other.GetY()+other.GetH() {
		return false
	}
	return true
}

// LeftOf returns how far to the left other is of this space
func (s *Space) LeftOf(other *Space) float64 {
	return other.GetX() - s.GetX()
}

// RightOf returns how far to the right other is of this space.
// Equivalent to -1 * LeftOf
func (s *Space) RightOf(other *Space) float64 {
	return s.GetX() - other.GetX()
}

// Overlap returns how much this space overlaps with another space
func (s *Space) Overlap(other *Space) (xOver, yOver float64) {
	if s.GetX() > other.GetX() {
		x2 := other.GetX() + other.GetW()
		if s.GetX() < x2 {
			xOver = s.GetX() - x2
		}
	} else {
		x2 := s.GetX() + s.GetW()
		if other.GetX() < x2 {
			xOver = x2 - other.GetX()
		}
	}
	if s.GetY() > other.GetY() {
		y2 := other.GetY() + other.GetH()
		if s.GetY() < y2 {
			yOver = s.GetY() - y2
		}
	} else {
		y2 := s.GetY() + s.GetH()
		if other.GetY() < y2 {
			yOver = y2 - other.GetY()
		}
	}
	return
}

// OverlapVector returns Overlap as a vector
func (s *Space) OverlapVector(other *Space) physics.Vector {
	xover, yover := s.Overlap(other)
	// Todo: why are we multiplying by -1 here, shouldn't that
	// also be happening in Overlap at least?
	return physics.NewVector(-xover, -yover)
}

// SubtractRect removes a rectangle from this rectangle and
// returns the rectangles remaining after the portion has been
// removed. The input x,y is relative to the original space:
// Example: removing 1,1 from 10,10 -> 12,12 is OK, but removing
// 11,11 from 10,10 -> 12,12 will not act as expected.
func (s *Space) SubtractRect(x2, y2, w2, h2 float64) []*Space {
	x1 := s.GetX()
	y1 := s.GetY()
	w1 := s.GetW()
	h1 := s.GetH()

	// Left, Top, Right, Bottom
	// X, Y, W, H
	rects := [4][4]float64{}

	rects[0][0] = x1
	rects[0][1] = y1
	rects[0][2] = x2
	rects[0][3] = h1

	// Todo: these spaces overlap on the corners. We could remove that.
	rects[1][0] = x1
	rects[1][1] = y1
	rects[1][2] = w1
	rects[1][3] = y2

	rects[2][0] = x1 + x2 + w2
	rects[2][1] = y1
	rects[2][2] = w1 - (x2 + w2)
	rects[2][3] = h1

	rects[3][0] = x1
	rects[3][1] = y1 + y2 + h2
	rects[3][2] = w1
	rects[3][3] = h1 - (y2 + h2)

	var spaces []*Space

	for _, r := range rects {
		if r[2] > 0 && r[3] > 0 {
			spaces = append(spaces, NewFullSpace(r[0], r[1], r[2], r[3], s.Label, s.CID))
		}
	}

	return spaces
}

func (s *Space) String() string {
	return strconv.FormatFloat(s.GetX(), 'f', 2, 32) + "," +
		strconv.FormatFloat(s.GetY(), 'f', 2, 32) + "," +
		strconv.FormatFloat(s.GetW(), 'f', 2, 32) + "," +
		strconv.FormatFloat(s.GetH(), 'f', 2, 32) + "::" +
		strconv.Itoa(int(s.CID)) + "::" + fmt.Sprintf("%p", s)
}

// NewUnassignedSpace returns a space that just has a rectangle
func NewUnassignedSpace(x, y, w, h float64) *Space {
	return NewLabeledSpace(x, y, w, h, NilLabel)
}

// NewSpace returns a space with an associated caller id
func NewSpace(x, y, w, h float64, cID event.CID) *Space {
	return NewFullSpace(x, y, w, h, NilLabel, cID)
}

// NewLabeledSpace returns a space with an associated integer label
func NewLabeledSpace(x, y, w, h float64, l Label) *Space {
	rect := NewRect(x, y, w, h)
	return &Space{
		Location: rect,
		Label:    l,
		Type:     NONE,
	}
}

// NewFullSpace returns a space with both a label and a caller id
func NewFullSpace(x, y, w, h float64, l Label, cID event.CID) *Space {
	rect := NewRect(x, y, w, h)
	return &Space{
		rect,
		l,
		cID,
		CID, // todo: This is hard to read as distinct from cID
		// todo: a way to generate non-CID typed spaces that isn't
		// package specific (see render/particle)
	}
}

// NewRect is a wrapper around rtreego.NewRect,
// casting the given x,y to an rtreego.Point.
// Used to not expose rtreego.Point to the user.
func NewRect(x, y, w, h float64) *rtreego.Rect {
	rect, err := rtreego.NewRect(rtreego.Point{x, y}, [3]float64{w, h, 1})
	if err != nil {
		dlog.Error(err)
	}
	return &rect
}
<|endoftext|>


<commit_before>//
// command/archive.go
//
// Copyright (c) 2016 Junpei Kawamoto
//
// This software is released under the MIT License.
//
// http://opensource.org/licenses/mit-license.php
//

package command

import (
	"archive/tar"
	"bufio"
	"compress/gzip"
	"fmt"
	"io"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
)

// Archive makes a tar.gz file consists of files maintained a git repository.
func Archive(dir string, filename string) (err error) {

	writeFile, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE, 0600)
	if err != nil {
		return
	}
	defer writeFile.Close()

	writer := bufio.NewWriter(writeFile)
	defer writer.Flush()

	zipWriter, err := gzip.NewWriterLevel(writer, gzip.BestCompression)
	if err != nil {
		return
	}
	defer zipWriter.Close()

	tarWriter := tar.NewWriter(zipWriter)
	defer tarWriter.Close()

	// Change dir and run.
	cd, err := os.Getwd()
	if err != nil {
		return
	}
	if err = os.Chdir(dir); err != nil {
		return
	}
	defer os.Chdir(cd)

	// Listing up and write to a tarball.
	ch := make(chan string)
	doneLGS := make(chan error)
	doneLGR := make(chan error)
	doneTB := make(chan error)

	go listupGitSources(ch, doneLGS)
	go listupGitRepository(ch, doneLGR)
	go tarballing(tarWriter, ch, doneTB)

	err1 := <-doneLGS
	err2 := <-doneLGR
	close(ch)
	if err1 != nil {
		return err1
	} else if err2 != nil {
		return err2
	}

	return <-doneTB

}

// listupGitSources is a go-routine which lists up git sources and puts
// finding paths to a given ch. After listing up all sources, put nil
// to done. If an error occurs, put the error to done.
func listupGitSources(ch chan<- string, done chan<- error) {

	cmd := exec.Command("git", "ls-files")
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		done <- err
		return
	}
	defer stdout.Close()

	doneRL := make(chan error)
	defer close(doneRL)

	go readLine(stdout, ch, doneRL)
	if err = cmd.Run(); err != nil {
		<-doneRL
		done <- err
		return
	}

	err = <-doneRL
	done <- err
	return

}

// listupGitRepository is a go-routine which lists up git repository
// and puts founded paths to a given ch. After listing up all paths,
// put nil to done. If an error occurs, put the error to done.
func listupGitRepository(ch chan<- string, done chan<- error) {

	err := filepath.Walk(".git", func(path string, info os.FileInfo, err error) error {

		if err != nil {
			return err
		}

		if info.IsDir() {
			return nil
		}

		ch <- path
		return nil

	})

	done <- err

}

// tarballing is a go-routine which write a file given via ch to a tar writer.
// It puts nil to done when it ends. If an error occurs, it puts the error to
// done.
func tarballing(writer *tar.Writer, ch <-chan string, done chan<- error) {

	var info os.FileInfo
	var header *tar.Header
	var err error

	for {
		path, ok := <-ch
		if !ok {
			break
		}

		// For Windows: Replace path delimiters.
		path = filepath.ToSlash(path)
		fmt.Println(path)

		// Write a file header.
		info, err = os.Stat(path)
		if err != nil {
			fmt.Printf("Cannot find %s (%s)", path, err.Error())
			break
		}

		header, err = tar.FileInfoHeader(info, path)
		if err != nil {
			fmt.Println(err.Error())
			break
		}

		if strings.HasPrefix(path, "../") {
			header.Name = path[3:]
		} else {
			header.Name = path
		}
		writer.WriteHeader(header)

		// Write the body.
		if err = copyFile(path, writer); err != nil {
			break
		}
	}
	done <- err

}

// readLine is a go-routine which reads a line from rd and puts it to ch.
// It sends nil to done when reads all line or some error when an error
// occurs.
func readLine(rd io.Reader, ch chan<- string, done chan<- error) {

	scanner := bufio.NewScanner(rd)
	for scanner.Scan() {
		ch <- scanner.Text()
	}
	// done <- scanner.Err()
	done <- nil

}

// copyFile opens a given file and put its body to a given writer.
func copyFile(path string, writer io.Writer) (err error) {

	// Prepare to write a file body.
	fp, err := os.Open(path)
	if err != nil {
		return
	}
	defer fp.Close()

	_, err = io.Copy(writer, fp)
	return

}
<commit_msg>Fixed dead lock when tarballing raise errors.<commit_after>//
// command/archive.go
//
// Copyright (c) 2016 Junpei Kawamoto
//
// This software is released under the MIT License.
//
// http://opensource.org/licenses/mit-license.php
//

package command

import (
	"archive/tar"
	"bufio"
	"compress/gzip"
	"fmt"
	"io"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"sync"

	"golang.org/x/net/context"
)

// Archive makes a tar.gz file consists of files maintained a git repository.
func Archive(dir string, filename string) (err error) {

	writeFile, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE, 0600)
	if err != nil {
		return
	}
	defer writeFile.Close()

	writer := bufio.NewWriter(writeFile)
	defer writer.Flush()

	zipWriter, err := gzip.NewWriterLevel(writer, gzip.BestCompression)
	if err != nil {
		return
	}
	defer zipWriter.Close()

	tarWriter := tar.NewWriter(zipWriter)
	defer tarWriter.Close()

	// Change dir and run.
	cd, err := os.Getwd()
	if err != nil {
		return
	}
	if err = os.Chdir(dir); err != nil {
		return
	}
	defer os.Chdir(cd)

	// Listing up and write to a tarball.
	ctx := context.Background()
	ctx, cancel := context.WithCancel(ctx)

	ch := make(chan string)
	doneLGS := make(chan error)
	doneLGR := make(chan error)
	doneTB := make(chan error)

	var wg sync.WaitGroup
	wg.Add(1)
	go func() {
		defer wg.Done()
		doneLGS <- listupGitSources(ctx, ch)
	}()
	wg.Add(1)
	go func() {
		defer wg.Done()
		doneLGR <- listupGitRepository(ctx, ch)
	}()
	go func() {
		wg.Wait()
		close(ch)
	}()

	// Require to close channel ch to end this goroutine.
	go tarballing(tarWriter, ch, doneTB)
	err = <-doneTB
	if err != nil {
		cancel()
		return
	}

	err1 := <-doneLGS
	err2 := <-doneLGR
	if err1 != nil {
		return err1
	} else if err2 != nil {
		return err2
	}

	return
}

// tarballing is a go-routine which write a file given via ch to a tar writer.
// It puts nil to done when it ends. If an error occurs, it puts the error to
// done.
func tarballing(writer *tar.Writer, ch <-chan string, done chan<- error) {

	var info os.FileInfo
	var header *tar.Header
	var err error

	for path := range ch {

		// For Windows: Replace path delimiters.
		path = filepath.ToSlash(path)
		fmt.Println(path)

		// Write a file header.
		info, err = os.Stat(path)
		if err != nil {
			fmt.Printf("Cannot find %s (%s)", path, err.Error())
			break
		}

		header, err = tar.FileInfoHeader(info, path)
		if err != nil {
			fmt.Println(err.Error())
			break
		}

		if strings.HasPrefix(path, "../") {
			header.Name = path[3:]
		} else {
			header.Name = path
		}
		writer.WriteHeader(header)

		// Write the body.
		if err = copyFile(path, writer); err != nil {
			break
		}
	}
	done <- err

}

// listupGitRepository lists up git repository and puts founded paths to a given ch.
func listupGitRepository(ctx context.Context, ch chan<- string) error {

	return filepath.Walk(".git", func(path string, info os.FileInfo, err error) error {

		select {
		case <-ctx.Done():
			return ctx.Err()

		default:
			if err != nil {
				return err
			}

			if info.IsDir() {
				return nil
			}

			ch <- path
			return nil
		}

	})

}

// listupGitSources lists up git sources and puts finding paths to a given ch.
func listupGitSources(ctx context.Context, ch chan<- string) (err error) {

	cmd := exec.Command("git", "ls-files")
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		return
	}
	defer stdout.Close()

	doneRL := make(chan error)
	defer close(doneRL)

	go readLine(ctx, stdout, ch, doneRL)
	if err = cmd.Run(); err != nil {
		<-doneRL
		return
	}

	return <-doneRL

}

// readLine is a go-routine which reads a line from rd and puts it to ch.
// It sends nil to done when reads all line or some error when an error
// occurs.
func readLine(ctx context.Context, rd io.Reader, ch chan<- string, done chan<- error) {

	scanner := bufio.NewScanner(rd)
	for scanner.Scan() {
		select {
		case <-ctx.Done():
			done <- ctx.Err()
			return
		default:
			ch <- scanner.Text()
		}
	}
	// done <- scanner.Err()
	done <- nil

}

// copyFile opens a given file and put its body to a given writer.
func copyFile(path string, writer io.Writer) (err error) {

	// Prepare to write a file body.
	fp, err := os.Open(path)
	if err != nil {
		return
	}
	defer fp.Close()

	_, err = io.Copy(writer, fp)
	return

}
<|endoftext|>


<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"encoding/json"
	"fmt"
	"net/http"
	"strings"
	"sync"
	"time"

	"github.com/sirupsen/logrus"

	"k8s.io/test-infra/prow/config"
	"k8s.io/test-infra/prow/errorutil"
	"k8s.io/test-infra/prow/tide"
	"k8s.io/test-infra/prow/tide/history"
)

type tidePools struct {
	Queries     []string
	TideQueries []config.TideQuery
	Pools       []tide.Pool
}

type tideHistory struct {
	History map[string][]history.Record
}

type tideAgent struct {
	log          *logrus.Entry
	path         string
	updatePeriod func() time.Duration

	// Config for hiding repos
	hiddenRepos []string
	hiddenOnly  bool
	showHidden  bool

	sync.Mutex
	pools   []tide.Pool
	history map[string][]history.Record
}

func (ta *tideAgent) start() {
	startTimePool := time.Now()
	if err := ta.updatePools(); err != nil {
		ta.log.WithError(err).Error("Updating pools the first time.")
	}
	startTimeHistory := time.Now()
	if err := ta.updateHistory(); err != nil {
		ta.log.WithError(err).Error("Updating history the first time.")
	}

	go func() {
		for {
			time.Sleep(time.Until(startTimePool.Add(ta.updatePeriod())))
			startTimePool = time.Now()
			if err := ta.updatePools(); err != nil {
				ta.log.WithError(err).Error("Updating pools.")
			}
		}
	}()
	go func() {
		for {
			time.Sleep(time.Until(startTimeHistory.Add(ta.updatePeriod())))
			startTimeHistory = time.Now()
			if err := ta.updateHistory(); err != nil {
				ta.log.WithError(err).Error("Updating history.")
			}
		}
	}()
}

func fetchTideData(log *logrus.Entry, path string, data interface{}) error {
	var prevErrs []error
	var err error
	backoff := 5 * time.Second
	for i := 0; i < 4; i++ {
		var resp *http.Response
		if err != nil {
			prevErrs = append(prevErrs, err)
			time.Sleep(backoff)
			backoff *= 4
		}
		resp, err = http.Get(path)
		if err == nil {
			defer resp.Body.Close()
			if resp.StatusCode < 200 || resp.StatusCode > 299 {
				err = fmt.Errorf("response has status code %d", resp.StatusCode)
				continue
			}
			if err = json.NewDecoder(resp.Body).Decode(data); err != nil {
				break
			}
			break
		}
	}

	// Either combine previous errors with the returned error, or if we succeeded
	// warn once about any errors we saw before succeeding.
	prevErr := errorutil.NewAggregate(prevErrs...)
	if err != nil {
		return errorutil.NewAggregate(err, prevErr)
	}
	if prevErr != nil {
		log.WithError(prevErr).Warnf(
			"Failed %d retries fetching Tide data before success: %v.",
			len(prevErrs),
			prevErr,
		)
	}
	return nil
}

func (ta *tideAgent) updatePools() error {
	var pools []tide.Pool
	if err := fetchTideData(ta.log, ta.path, &pools); err != nil {
		return err
	}
	pools = ta.filterHiddenPools(pools)

	ta.Lock()
	defer ta.Unlock()
	ta.pools = pools
	return nil
}

func (ta *tideAgent) updateHistory() error {
	path := strings.TrimSuffix(ta.path, "/") + "/history"
	var history map[string][]history.Record
	if err := fetchTideData(ta.log, path, &history); err != nil {
		return err
	}
	history = ta.filterHiddenHistory(history)

	ta.Lock()
	defer ta.Unlock()
	ta.history = history
	return nil
}

func (ta *tideAgent) filterHiddenPools(pools []tide.Pool) []tide.Pool {
	if len(ta.hiddenRepos) == 0 {
		return pools
	}

	filtered := make([]tide.Pool, 0, len(pools))
	for _, pool := range pools {
		needsHide := matches(pool.Org+"/"+pool.Repo, ta.hiddenRepos)
		if needsHide && ta.showHidden {
			filtered = append(filtered, pool)
		} else if needsHide == ta.hiddenOnly {
			filtered = append(filtered, pool)
		} else {
			ta.log.Debugf("Ignoring pool for %s.", pool.Org+"/"+pool.Repo)
		}
	}
	return filtered
}

func (ta *tideAgent) filterHiddenHistory(hist map[string][]history.Record) map[string][]history.Record {
	if len(ta.hiddenRepos) == 0 {
		return hist
	}

	filtered := make(map[string][]history.Record, len(hist))
	for pool, records := range hist {
		needsHide := matches(strings.Split(pool, ":")[0], ta.hiddenRepos)
		if needsHide && ta.showHidden {
			filtered[pool] = records
		} else if needsHide == ta.hiddenOnly {
			filtered[pool] = records
		} else {
			ta.log.Debugf("Ignoring history for %s.", pool)
		}
	}
	return filtered
}

func (ta *tideAgent) filterHiddenQueries(queries []config.TideQuery) []config.TideQuery {
	if len(ta.hiddenRepos) == 0 {
		return queries
	}

	filtered := make([]config.TideQuery, 0, len(queries))
	for _, qc := range queries {
		includesHidden := false
		// This will exclude the query even if a single
		// repo in the query is included in hiddenRepos.
		for _, repo := range qc.Repos {
			if matches(repo, ta.hiddenRepos) {
				includesHidden = true
				break
			}
		}
		if includesHidden && ta.showHidden {
			filtered = append(filtered, qc)
		} else if includesHidden == ta.hiddenOnly {
			filtered = append(filtered, qc)
		} else {
			ta.log.Debugf("Ignoring query: %s", qc.Query())
		}
	}
	return filtered
}

// matches returns whether the provided repo intersects
// with repos. repo has always the "org/repo" format but
// repos can include both orgs and repos.
func matches(repo string, repos []string) bool {
	org := strings.Split(repo, "/")[0]
	for _, r := range repos {
		if r == repo || r == org {
			return true
		}
	}
	return false
}
<commit_msg>Downgrade Tide retries log from warning to info.<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"encoding/json"
	"fmt"
	"net/http"
	"strings"
	"sync"
	"time"

	"github.com/sirupsen/logrus"

	"k8s.io/test-infra/prow/config"
	"k8s.io/test-infra/prow/errorutil"
	"k8s.io/test-infra/prow/tide"
	"k8s.io/test-infra/prow/tide/history"
)

type tidePools struct {
	Queries     []string
	TideQueries []config.TideQuery
	Pools       []tide.Pool
}

type tideHistory struct {
	History map[string][]history.Record
}

type tideAgent struct {
	log          *logrus.Entry
	path         string
	updatePeriod func() time.Duration

	// Config for hiding repos
	hiddenRepos []string
	hiddenOnly  bool
	showHidden  bool

	sync.Mutex
	pools   []tide.Pool
	history map[string][]history.Record
}

func (ta *tideAgent) start() {
	startTimePool := time.Now()
	if err := ta.updatePools(); err != nil {
		ta.log.WithError(err).Error("Updating pools the first time.")
	}
	startTimeHistory := time.Now()
	if err := ta.updateHistory(); err != nil {
		ta.log.WithError(err).Error("Updating history the first time.")
	}

	go func() {
		for {
			time.Sleep(time.Until(startTimePool.Add(ta.updatePeriod())))
			startTimePool = time.Now()
			if err := ta.updatePools(); err != nil {
				ta.log.WithError(err).Error("Updating pools.")
			}
		}
	}()
	go func() {
		for {
			time.Sleep(time.Until(startTimeHistory.Add(ta.updatePeriod())))
			startTimeHistory = time.Now()
			if err := ta.updateHistory(); err != nil {
				ta.log.WithError(err).Error("Updating history.")
			}
		}
	}()
}

func fetchTideData(log *logrus.Entry, path string, data interface{}) error {
	var prevErrs []error
	var err error
	backoff := 5 * time.Second
	for i := 0; i < 4; i++ {
		var resp *http.Response
		if err != nil {
			prevErrs = append(prevErrs, err)
			time.Sleep(backoff)
			backoff *= 4
		}
		resp, err = http.Get(path)
		if err == nil {
			defer resp.Body.Close()
			if resp.StatusCode < 200 || resp.StatusCode > 299 {
				err = fmt.Errorf("response has status code %d", resp.StatusCode)
				continue
			}
			if err = json.NewDecoder(resp.Body).Decode(data); err != nil {
				break
			}
			break
		}
	}

	// Either combine previous errors with the returned error, or if we succeeded
	// log once about any errors we saw before succeeding.
	prevErr := errorutil.NewAggregate(prevErrs...)
	if err != nil {
		return errorutil.NewAggregate(err, prevErr)
	}
	if prevErr != nil {
		log.WithError(prevErr).Infof(
			"Failed %d retries fetching Tide data before success: %v.",
			len(prevErrs),
			prevErr,
		)
	}
	return nil
}

func (ta *tideAgent) updatePools() error {
	var pools []tide.Pool
	if err := fetchTideData(ta.log, ta.path, &pools); err != nil {
		return err
	}
	pools = ta.filterHiddenPools(pools)

	ta.Lock()
	defer ta.Unlock()
	ta.pools = pools
	return nil
}

func (ta *tideAgent) updateHistory() error {
	path := strings.TrimSuffix(ta.path, "/") + "/history"
	var history map[string][]history.Record
	if err := fetchTideData(ta.log, path, &history); err != nil {
		return err
	}
	history = ta.filterHiddenHistory(history)

	ta.Lock()
	defer ta.Unlock()
	ta.history = history
	return nil
}

func (ta *tideAgent) filterHiddenPools(pools []tide.Pool) []tide.Pool {
	if len(ta.hiddenRepos) == 0 {
		return pools
	}

	filtered := make([]tide.Pool, 0, len(pools))
	for _, pool := range pools {
		needsHide := matches(pool.Org+"/"+pool.Repo, ta.hiddenRepos)
		if needsHide && ta.showHidden {
			filtered = append(filtered, pool)
		} else if needsHide == ta.hiddenOnly {
			filtered = append(filtered, pool)
		} else {
			ta.log.Debugf("Ignoring pool for %s.", pool.Org+"/"+pool.Repo)
		}
	}
	return filtered
}

func (ta *tideAgent) filterHiddenHistory(hist map[string][]history.Record) map[string][]history.Record {
	if len(ta.hiddenRepos) == 0 {
		return hist
	}

	filtered := make(map[string][]history.Record, len(hist))
	for pool, records := range hist {
		needsHide := matches(strings.Split(pool, ":")[0], ta.hiddenRepos)
		if needsHide && ta.showHidden {
			filtered[pool] = records
		} else if needsHide == ta.hiddenOnly {
			filtered[pool] = records
		} else {
			ta.log.Debugf("Ignoring history for %s.", pool)
		}
	}
	return filtered
}

func (ta *tideAgent) filterHiddenQueries(queries []config.TideQuery) []config.TideQuery {
	if len(ta.hiddenRepos) == 0 {
		return queries
	}

	filtered := make([]config.TideQuery, 0, len(queries))
	for _, qc := range queries {
		includesHidden := false
		// This will exclude the query even if a single
		// repo in the query is included in hiddenRepos.
		for _, repo := range qc.Repos {
			if matches(repo, ta.hiddenRepos) {
				includesHidden = true
				break
			}
		}
		if includesHidden && ta.showHidden {
			filtered = append(filtered, qc)
		} else if includesHidden == ta.hiddenOnly {
			filtered = append(filtered, qc)
		} else {
			ta.log.Debugf("Ignoring query: %s", qc.Query())
		}
	}
	return filtered
}

// matches returns whether the provided repo intersects
// with repos. repo has always the "org/repo" format but
// repos can include both orgs and repos.
func matches(repo string, repos []string) bool {
	org := strings.Split(repo, "/")[0]
	for _, r := range repos {
		if r == repo || r == org {
			return true
		}
	}
	return false
}
<|endoftext|>