<commit_before>package datastore

import (
	"errors"
	"time"

	query "github.com/ipfs/go-datastore/query"
)

/*
Datastore represents storage for any key-value pair.

Datastores are general enough to be backed by all kinds of different storage:
in-memory caches, databases, a remote datastore, flat files on disk, etc.

The general idea is to wrap a more complicated storage facility in a simple,
uniform interface, keeping the freedom of using the right tools for the job.
In particular, a Datastore can aggregate other datastores in interesting ways,
like sharded (to distribute load) or tiered access (caches before databases).

While Datastores should be written general enough to accept all sorts of
values, some implementations will undoubtedly have to be specific (e.g. SQL
databases where fields should be decomposed into columns), particularly to
support queries efficiently. Moreover, certain datastores may enforce certain
types of values (e.g. requiring an io.Reader, a specific struct, etc) or
serialization formats (JSON, Protobufs, etc).

IMPORTANT: No Datastore should ever Panic! This is a cross-module interface,
and thus it should behave predictably and handle exceptional conditions with
proper error reporting. Thus, all Datastore calls may return errors, which
should be checked by callers.
*/
type Datastore interface {
	// Put stores the object `value` named by `key`.
	//
	// The generalized Datastore interface does not impose a value type,
	// allowing various datastore middleware implementations (which do not
	// handle the values directly) to be composed together.
	//
	// Ultimately, the lowest-level datastore will need to do some value checking
	// or risk getting incorrect values. It may also be useful to expose a more
	// type-safe interface to your application, and do the checking up-front.
	Put(key Key, value []byte) error

	// Get retrieves the object `value` named by `key`.
	// Get will return ErrNotFound if the key is not mapped to a value.
	Get(key Key) (value []byte, err error)

	// Has returns whether the `key` is mapped to a `value`.
	// In some contexts, it may be much cheaper only to check for existence of
	// a value, rather than retrieving the value itself. (e.g. HTTP HEAD).
	// The default implementation is found in `GetBackedHas`.
	Has(key Key) (exists bool, err error)

	// Delete removes the value for given `key`.
	Delete(key Key) error

	// Query searches the datastore and returns a query result. This function
	// may return before the query actually runs. To wait for the query:
	//
	//   result, _ := ds.Query(q)
	//
	//   // use the channel interface; result may come in at different times
	//   for entry := range result.Next() { ... }
	//
	//   // or wait for the query to be completely done
	//   entries, _ := result.Rest()
	//   for entry := range entries { ... }
	//
	Query(q query.Query) (query.Results, error)
}

type Batching interface {
	Datastore

	Batch() (Batch, error)
}

var ErrBatchUnsupported = errors.New("this datastore does not support batching")

// ThreadSafeDatastore is an interface that all threadsafe datastore should
// implement to leverage type safety checks.
type ThreadSafeDatastore interface {
	Datastore

	IsThreadSafe()
}

// CheckedDatastore is an interface that should be implemented by datastores
// which may need checking on-disk data integrity.
type CheckedDatastore interface {
	Datastore

	Check() error
}

// CheckedDatastore is an interface that should be implemented by datastores
// which want to provide a mechanism to check data integrity and/or
// error correction.
type ScrubbedDatastore interface {
	Datastore

	Scrub() error
}

// GCDatastore is an interface that should be implemented by datastores which
// don't free disk space by just removing data from them.
type GCDatastore interface {
	Datastore

	CollectGarbage() error
}

// PersistentDatastore is an interface that should be implemented by datastores
// which can report disk usage.
type PersistentDatastore interface {
	Datastore

	// DiskUsage returns the space used by a datastore, in bytes.
	DiskUsage() (uint64, error)
}

// DiskUsage checks if a Datastore is a
// PersistentDatastore and returns its DiskUsage(),
// otherwise returns 0.
func DiskUsage(d Datastore) (uint64, error) {
	persDs, ok := d.(PersistentDatastore)
	if !ok {
		return 0, nil
	}
	return persDs.DiskUsage()
}

// TTLDatastore is an interface that should be implemented by datastores that
// support expiring entries.
type TTLDatastore interface {
	Datastore

	PutWithTTL(key Key, value interface{}, ttl time.Duration) error
	SetTTL(key Key, ttl time.Duration) error
}

// Txn is an interface for transactions that can be committed or rolled back.
type Txn interface {
	Datastore

	Commit() error
	Rollback()
}

// TxDatastore is an interface that should be implemented by datastores that
// support transactions.
type TxDatastore interface {
	Datastore

	NewTransaction() Txn
}

// Errors

// ErrNotFound is returned by Get, Has, and Delete when a datastore does not
// map the given key to a value.
var ErrNotFound = errors.New("datastore: key not found")

// ErrInvalidType is returned by Put when a given value is incopatible with
// the type the datastore supports. This means a conversion (or serialization)
// is needed beforehand.
var ErrInvalidType = errors.New("datastore: invalid type error")

// GetBackedHas provides a default Datastore.Has implementation.
// It exists so Datastore.Has implementations can use it, like so:
//
// func (*d SomeDatastore) Has(key Key) (exists bool, err error) {
//   return GetBackedHas(d, key)
// }
func GetBackedHas(ds Datastore, key Key) (bool, error) {
	_, err := ds.Get(key)
	switch err {
	case nil:
		return true, nil
	case ErrNotFound:
		return false, nil
	default:
		return false, err
	}
}

type Batch interface {
	Put(key Key, val []byte) error

	Delete(key Key) error

	Commit() error
}
<commit_msg>Add support for read-only transactions<commit_after>package datastore

import (
	"errors"
	"time"

	query "github.com/ipfs/go-datastore/query"
)

/*
Datastore represents storage for any key-value pair.

Datastores are general enough to be backed by all kinds of different storage:
in-memory caches, databases, a remote datastore, flat files on disk, etc.

The general idea is to wrap a more complicated storage facility in a simple,
uniform interface, keeping the freedom of using the right tools for the job.
In particular, a Datastore can aggregate other datastores in interesting ways,
like sharded (to distribute load) or tiered access (caches before databases).

While Datastores should be written general enough to accept all sorts of
values, some implementations will undoubtedly have to be specific (e.g. SQL
databases where fields should be decomposed into columns), particularly to
support queries efficiently. Moreover, certain datastores may enforce certain
types of values (e.g. requiring an io.Reader, a specific struct, etc) or
serialization formats (JSON, Protobufs, etc).

IMPORTANT: No Datastore should ever Panic! This is a cross-module interface,
and thus it should behave predictably and handle exceptional conditions with
proper error reporting. Thus, all Datastore calls may return errors, which
should be checked by callers.
*/
type Datastore interface {
	// Put stores the object `value` named by `key`.
	//
	// The generalized Datastore interface does not impose a value type,
	// allowing various datastore middleware implementations (which do not
	// handle the values directly) to be composed together.
	//
	// Ultimately, the lowest-level datastore will need to do some value checking
	// or risk getting incorrect values. It may also be useful to expose a more
	// type-safe interface to your application, and do the checking up-front.
	Put(key Key, value []byte) error

	// Get retrieves the object `value` named by `key`.
	// Get will return ErrNotFound if the key is not mapped to a value.
	Get(key Key) (value []byte, err error)

	// Has returns whether the `key` is mapped to a `value`.
	// In some contexts, it may be much cheaper only to check for existence of
	// a value, rather than retrieving the value itself. (e.g. HTTP HEAD).
	// The default implementation is found in `GetBackedHas`.
	Has(key Key) (exists bool, err error)

	// Delete removes the value for given `key`.
	Delete(key Key) error

	// Query searches the datastore and returns a query result. This function
	// may return before the query actually runs. To wait for the query:
	//
	//   result, _ := ds.Query(q)
	//
	//   // use the channel interface; result may come in at different times
	//   for entry := range result.Next() { ... }
	//
	//   // or wait for the query to be completely done
	//   entries, _ := result.Rest()
	//   for entry := range entries { ... }
	//
	Query(q query.Query) (query.Results, error)
}

type Batching interface {
	Datastore

	Batch() (Batch, error)
}

var ErrBatchUnsupported = errors.New("this datastore does not support batching")

// ThreadSafeDatastore is an interface that all threadsafe datastore should
// implement to leverage type safety checks.
type ThreadSafeDatastore interface {
	Datastore

	IsThreadSafe()
}

// CheckedDatastore is an interface that should be implemented by datastores
// which may need checking on-disk data integrity.
type CheckedDatastore interface {
	Datastore

	Check() error
}

// CheckedDatastore is an interface that should be implemented by datastores
// which want to provide a mechanism to check data integrity and/or
// error correction.
type ScrubbedDatastore interface {
	Datastore

	Scrub() error
}

// GCDatastore is an interface that should be implemented by datastores which
// don't free disk space by just removing data from them.
type GCDatastore interface {
	Datastore

	CollectGarbage() error
}

// PersistentDatastore is an interface that should be implemented by datastores
// which can report disk usage.
type PersistentDatastore interface {
	Datastore

	// DiskUsage returns the space used by a datastore, in bytes.
	DiskUsage() (uint64, error)
}

// DiskUsage checks if a Datastore is a
// PersistentDatastore and returns its DiskUsage(),
// otherwise returns 0.
func DiskUsage(d Datastore) (uint64, error) {
	persDs, ok := d.(PersistentDatastore)
	if !ok {
		return 0, nil
	}
	return persDs.DiskUsage()
}

// TTLDatastore is an interface that should be implemented by datastores that
// support expiring entries.
type TTLDatastore interface {
	Datastore

	PutWithTTL(key Key, value interface{}, ttl time.Duration) error
	SetTTL(key Key, ttl time.Duration) error
}

// Txn is an interface for transactions that can be committed or rolled back.
type Txn interface {
	Datastore

	Commit() error
	Rollback()
}

// TxDatastore is an interface that should be implemented by datastores that
// support transactions.
type TxDatastore interface {
	Datastore

	NewTransaction(readOnly bool) Txn
}

// Errors

// ErrNotFound is returned by Get, Has, and Delete when a datastore does not
// map the given key to a value.
var ErrNotFound = errors.New("datastore: key not found")

// ErrInvalidType is returned by Put when a given value is incopatible with
// the type the datastore supports. This means a conversion (or serialization)
// is needed beforehand.
var ErrInvalidType = errors.New("datastore: invalid type error")

// GetBackedHas provides a default Datastore.Has implementation.
// It exists so Datastore.Has implementations can use it, like so:
//
// func (*d SomeDatastore) Has(key Key) (exists bool, err error) {
//   return GetBackedHas(d, key)
// }
func GetBackedHas(ds Datastore, key Key) (bool, error) {
	_, err := ds.Get(key)
	switch err {
	case nil:
		return true, nil
	case ErrNotFound:
		return false, nil
	default:
		return false, err
	}
}

type Batch interface {
	Put(key Key, val []byte) error

	Delete(key Key) error

	Commit() error
}
<|endoftext|>


<commit_before>package common

import (
	"errors"
	"fmt"
	"net/url"
	"os"
	"path"
	"strconv"
	"strings"

	"github.com/Sirupsen/logrus"
	"gitlab.com/gitlab-org/gitlab-ci-multi-runner/helpers"
	"time"
)

type BuildState string

type GitStrategy int

const (
	GitClone GitStrategy = iota
	GitFetch
)

const (
	Pending BuildState = "pending"
	Running            = "running"
	Failed             = "failed"
	Success            = "success"
)

type Build struct {
	GetBuildResponse `yaml:",inline"`

	Trace           BuildTrace
	SystemInterrupt chan os.Signal `json:"-" yaml:"-"`
	RootDir         string         `json:"-" yaml:"-"`
	BuildDir        string         `json:"-" yaml:"-"`
	CacheDir        string         `json:"-" yaml:"-"`
	Hostname        string         `json:"-" yaml:"-"`
	Runner          *RunnerConfig  `json:"runner"`
	ExecutorData    ExecutorData

	// Unique ID for all running builds on this runner
	RunnerID int `json:"runner_id"`

	// Unique ID for all running builds on this runner and this project
	ProjectRunnerID int `json:"project_runner_id"`
}

func (b *Build) Log() *logrus.Entry {
	return b.Runner.Log().WithField("build", b.ID).WithField("project", b.ProjectID)
}

func (b *Build) ProjectUniqueName() string {
	return fmt.Sprintf("runner-%s-project-%d-concurrent-%d",
		b.Runner.ShortDescription(), b.ProjectID, b.ProjectRunnerID)
}

func (b *Build) ProjectSlug() (string, error) {
	url, err := url.Parse(b.RepoURL)
	if err != nil {
		return "", err
	}
	if url.Host == "" {
		return "", errors.New("only URI reference supported")
	}

	slug := url.Path
	slug = strings.TrimSuffix(slug, ".git")
	slug = path.Clean(slug)
	if slug == "." {
		return "", errors.New("invalid path")
	}
	if strings.Contains(slug, "..") {
		return "", errors.New("it doesn't look like a valid path")
	}
	return slug, nil
}

func (b *Build) ProjectUniqueDir(sharedDir bool) string {
	dir, err := b.ProjectSlug()
	if err != nil {
		dir = fmt.Sprintf("project-%d", b.ProjectID)
	}

	// for shared dirs path is constructed like this:
	// <some-path>/runner-short-id/concurrent-id/group-name/project-name/
	// ex.<some-path>/01234567/0/group/repo/
	if sharedDir {
		dir = path.Join(
			fmt.Sprintf("%s", b.Runner.ShortDescription()),
			fmt.Sprintf("%d", b.ProjectRunnerID),
			dir,
		)
	}
	return dir
}

func (b *Build) FullProjectDir() string {
	return helpers.ToSlash(b.BuildDir)
}

func (b *Build) StartBuild(rootDir, cacheDir string, sharedDir bool) {
	b.RootDir = rootDir
	b.BuildDir = path.Join(rootDir, b.ProjectUniqueDir(sharedDir))
	b.CacheDir = path.Join(cacheDir, b.ProjectUniqueDir(false))
}

func (b *Build) executeShellScript(scriptType ShellScriptType, executor Executor, abort chan interface{}) error {
	shell := executor.Shell()
	if shell == nil {
		return errors.New("No shell defined")
	}

	script, err := GenerateShellScript(scriptType, *shell)
	if err != nil {
		return err
	}

	// Nothing to execute
	if script == "" {
		return nil
	}

	cmd := ExecutorCommand{
		Script: script,
		Abort:  abort,
	}

	switch scriptType {
	case ShellBuildScript, ShellAfterScript: // use custom build environment
		cmd.Predefined = false
	default: // all other stages use a predefined build environment
		cmd.Predefined = true
	}

	return executor.Run(cmd)
}

func (b *Build) executeUploadArtifacts(state error, executor Executor, abort chan interface{}) (err error) {
	when, _ := b.Options.GetString("artifacts", "when")

	if state == nil {
		// Previous stages were successful
		if when == "" || when == "on_success" || when == "always" {
			err = b.executeShellScript(ShellUploadArtifacts, executor, abort)
		}
	} else {
		// Previous stage did fail
		if when == "on_failure" || when == "always" {
			err = b.executeShellScript(ShellUploadArtifacts, executor, abort)
		}
	}

	// Use previous error if set
	if state != nil {
		err = state
	}
	return
}

func (b *Build) executeScript(executor Executor, abort chan interface{}) error {
	// Execute pre script (git clone, cache restore, artifacts download)
	err := b.executeShellScript(ShellPrepareScript, executor, abort)

	if err == nil {
		// Execute user build script (before_script + script)
		err = b.executeShellScript(ShellBuildScript, executor, abort)

		// Execute after script (after_script)
		timeoutCh := make(chan interface{}, 1)
		timeout := time.AfterFunc(time.Minute*5, func() {
			close(timeoutCh)
		})
		b.executeShellScript(ShellAfterScript, executor, timeoutCh)
		timeout.Stop()
	}

	// Execute post script (cache store, artifacts upload)
	if err == nil {
		err = b.executeShellScript(ShellArchiveCache, executor, abort)
	}
	err = b.executeUploadArtifacts(err, executor, abort)
	return err
}

func (b *Build) run(executor Executor) (err error) {
	buildTimeout := b.Timeout
	if buildTimeout <= 0 {
		buildTimeout = DefaultTimeout
	}

	buildFinish := make(chan error, 1)
	buildAbort := make(chan interface{})

	// Run build script
	go func() {
		buildFinish <- b.executeScript(executor, buildAbort)
	}()

	// Wait for signals: cancel, timeout, abort or finish
	b.Log().Debugln("Waiting for signals...")
	select {
	case <-b.Trace.Aborted():
		err = &BuildError{Inner: errors.New("canceled")}

	case <-time.After(time.Duration(buildTimeout) * time.Second):
		err = &BuildError{Inner: fmt.Errorf("execution took longer than %v seconds", buildTimeout)}

	case signal := <-b.SystemInterrupt:
		err = fmt.Errorf("aborted: %v", signal)

	case err = <-buildFinish:
		return err
	}

	b.Log().WithError(err).Debugln("Waiting for build to finish...")

	// Wait till we receive that build did finish
	for {
		select {
		case buildAbort <- true:
		case <-buildFinish:
			return err
		}
	}
}

func (b *Build) createExecutor(globalConfig *Config) (executor Executor, err error) {
	executor = NewExecutor(b.Runner.Executor)
	if executor == nil {
		err = errors.New("executor not found")
		return
	}

	err = executor.Prepare(globalConfig, b.Runner, b)
	return
}

func (b *Build) retryCreateExecutor(globalConfig *Config, logger BuildLogger) (executor Executor, err error) {
	for tries := 0; tries < PreparationRetries; tries++ {
		executor, err = b.createExecutor(globalConfig)
		if err == nil {
			break
		}
		if executor != nil {
			executor.Cleanup()
			executor = nil
		}
		logger.SoftErrorln("Preparation failed:", err)
		logger.Infoln("Will be retried in", PreparationRetryInterval, "...")
		time.Sleep(PreparationRetryInterval)
	}
	return
}

func (b *Build) Run(globalConfig *Config, trace BuildTrace) (err error) {
	var executor Executor

	logger := NewBuildLogger(trace, b.Log())
	logger.Println("Running with " + AppVersion.Line() + helpers.ANSI_RESET)

	defer func() {
		if _, ok := err.(*BuildError); ok {
			logger.SoftErrorln("Build failed:", err)
			trace.Fail(err)
		} else if err != nil {
			logger.Errorln("Build failed (system failure):", err)
			trace.Fail(err)
		} else {
			logger.Infoln("Build succeeded")
			trace.Success()
		}
		if executor != nil {
			executor.Cleanup()
		}
	}()

	b.Trace = trace

	executor, err = b.retryCreateExecutor(globalConfig, logger)
	if err == nil {
		err = b.run(executor)
	}
	if executor != nil {
		executor.Finish(err)
	}
	return err
}

func (b *Build) String() string {
	return helpers.ToYAML(b)
}

func (b *Build) GetDefaultVariables() BuildVariables {
	return BuildVariables{
		{"CI", "true", true, true, false},
		{"CI_BUILD_REF", b.Sha, true, true, false},
		{"CI_BUILD_BEFORE_SHA", b.BeforeSha, true, true, false},
		{"CI_BUILD_REF_NAME", b.RefName, true, true, false},
		{"CI_BUILD_ID", strconv.Itoa(b.ID), true, true, false},
		{"CI_BUILD_REPO", b.RepoURL, true, true, false},
		{"CI_BUILD_TOKEN", b.Token, true, true, false},
		{"CI_PROJECT_ID", strconv.Itoa(b.ProjectID), true, true, false},
		{"CI_PROJECT_DIR", b.FullProjectDir(), true, true, false},
		{"CI_SERVER", "yes", true, true, false},
		{"CI_SERVER_NAME", "GitLab CI", true, true, false},
		{"CI_SERVER_VERSION", "", true, true, false},
		{"CI_SERVER_REVISION", "", true, true, false},
		{"GITLAB_CI", "true", true, true, false},
	}
}

func (b *Build) GetAllVariables() BuildVariables {
	variables := b.Runner.GetVariables()
	variables = append(variables, b.GetDefaultVariables()...)
	variables = append(variables, b.Variables...)
	return variables.Expand()
}

func (b *Build) GetGitDepth() string {
	return b.GetAllVariables().Get("GIT_DEPTH")
}

func (b *Build) GetGitStrategy() GitStrategy {
	switch b.GetAllVariables().Get("GIT_STRATEGY") {
	case "clone":
		return GitClone

	case "fetch":
		return GitFetch

	default:
		if b.AllowGitFetch {
			return GitFetch
		}

		return GitClone
	}
}
<commit_msg>Use ExecutorProvider instead<commit_after>package common

import (
	"errors"
	"fmt"
	"net/url"
	"os"
	"path"
	"strconv"
	"strings"

	"github.com/Sirupsen/logrus"
	"gitlab.com/gitlab-org/gitlab-ci-multi-runner/helpers"
	"time"
)

type BuildState string

type GitStrategy int

const (
	GitClone GitStrategy = iota
	GitFetch
)

const (
	Pending BuildState = "pending"
	Running            = "running"
	Failed             = "failed"
	Success            = "success"
)

type Build struct {
	GetBuildResponse `yaml:",inline"`

	Trace           BuildTrace
	SystemInterrupt chan os.Signal `json:"-" yaml:"-"`
	RootDir         string         `json:"-" yaml:"-"`
	BuildDir        string         `json:"-" yaml:"-"`
	CacheDir        string         `json:"-" yaml:"-"`
	Hostname        string         `json:"-" yaml:"-"`
	Runner          *RunnerConfig  `json:"runner"`
	ExecutorData    ExecutorData

	// Unique ID for all running builds on this runner
	RunnerID int `json:"runner_id"`

	// Unique ID for all running builds on this runner and this project
	ProjectRunnerID int `json:"project_runner_id"`
}

func (b *Build) Log() *logrus.Entry {
	return b.Runner.Log().WithField("build", b.ID).WithField("project", b.ProjectID)
}

func (b *Build) ProjectUniqueName() string {
	return fmt.Sprintf("runner-%s-project-%d-concurrent-%d",
		b.Runner.ShortDescription(), b.ProjectID, b.ProjectRunnerID)
}

func (b *Build) ProjectSlug() (string, error) {
	url, err := url.Parse(b.RepoURL)
	if err != nil {
		return "", err
	}
	if url.Host == "" {
		return "", errors.New("only URI reference supported")
	}

	slug := url.Path
	slug = strings.TrimSuffix(slug, ".git")
	slug = path.Clean(slug)
	if slug == "." {
		return "", errors.New("invalid path")
	}
	if strings.Contains(slug, "..") {
		return "", errors.New("it doesn't look like a valid path")
	}
	return slug, nil
}

func (b *Build) ProjectUniqueDir(sharedDir bool) string {
	dir, err := b.ProjectSlug()
	if err != nil {
		dir = fmt.Sprintf("project-%d", b.ProjectID)
	}

	// for shared dirs path is constructed like this:
	// <some-path>/runner-short-id/concurrent-id/group-name/project-name/
	// ex.<some-path>/01234567/0/group/repo/
	if sharedDir {
		dir = path.Join(
			fmt.Sprintf("%s", b.Runner.ShortDescription()),
			fmt.Sprintf("%d", b.ProjectRunnerID),
			dir,
		)
	}
	return dir
}

func (b *Build) FullProjectDir() string {
	return helpers.ToSlash(b.BuildDir)
}

func (b *Build) StartBuild(rootDir, cacheDir string, sharedDir bool) {
	b.RootDir = rootDir
	b.BuildDir = path.Join(rootDir, b.ProjectUniqueDir(sharedDir))
	b.CacheDir = path.Join(cacheDir, b.ProjectUniqueDir(false))
}

func (b *Build) executeShellScript(scriptType ShellScriptType, executor Executor, abort chan interface{}) error {
	shell := executor.Shell()
	if shell == nil {
		return errors.New("No shell defined")
	}

	script, err := GenerateShellScript(scriptType, *shell)
	if err != nil {
		return err
	}

	// Nothing to execute
	if script == "" {
		return nil
	}

	cmd := ExecutorCommand{
		Script: script,
		Abort:  abort,
	}

	switch scriptType {
	case ShellBuildScript, ShellAfterScript: // use custom build environment
		cmd.Predefined = false
	default: // all other stages use a predefined build environment
		cmd.Predefined = true
	}

	return executor.Run(cmd)
}

func (b *Build) executeUploadArtifacts(state error, executor Executor, abort chan interface{}) (err error) {
	when, _ := b.Options.GetString("artifacts", "when")

	if state == nil {
		// Previous stages were successful
		if when == "" || when == "on_success" || when == "always" {
			err = b.executeShellScript(ShellUploadArtifacts, executor, abort)
		}
	} else {
		// Previous stage did fail
		if when == "on_failure" || when == "always" {
			err = b.executeShellScript(ShellUploadArtifacts, executor, abort)
		}
	}

	// Use previous error if set
	if state != nil {
		err = state
	}
	return
}

func (b *Build) executeScript(executor Executor, abort chan interface{}) error {
	// Execute pre script (git clone, cache restore, artifacts download)
	err := b.executeShellScript(ShellPrepareScript, executor, abort)

	if err == nil {
		// Execute user build script (before_script + script)
		err = b.executeShellScript(ShellBuildScript, executor, abort)

		// Execute after script (after_script)
		timeoutCh := make(chan interface{}, 1)
		timeout := time.AfterFunc(time.Minute*5, func() {
			close(timeoutCh)
		})
		b.executeShellScript(ShellAfterScript, executor, timeoutCh)
		timeout.Stop()
	}

	// Execute post script (cache store, artifacts upload)
	if err == nil {
		err = b.executeShellScript(ShellArchiveCache, executor, abort)
	}
	err = b.executeUploadArtifacts(err, executor, abort)
	return err
}

func (b *Build) run(executor Executor) (err error) {
	buildTimeout := b.Timeout
	if buildTimeout <= 0 {
		buildTimeout = DefaultTimeout
	}

	buildFinish := make(chan error, 1)
	buildAbort := make(chan interface{})

	// Run build script
	go func() {
		buildFinish <- b.executeScript(executor, buildAbort)
	}()

	// Wait for signals: cancel, timeout, abort or finish
	b.Log().Debugln("Waiting for signals...")
	select {
	case <-b.Trace.Aborted():
		err = &BuildError{Inner: errors.New("canceled")}

	case <-time.After(time.Duration(buildTimeout) * time.Second):
		err = &BuildError{Inner: fmt.Errorf("execution took longer than %v seconds", buildTimeout)}

	case signal := <-b.SystemInterrupt:
		err = fmt.Errorf("aborted: %v", signal)

	case err = <-buildFinish:
		return err
	}

	b.Log().WithError(err).Debugln("Waiting for build to finish...")

	// Wait till we receive that build did finish
	for {
		select {
		case buildAbort <- true:
		case <-buildFinish:
			return err
		}
	}
}

func (b *Build) retryCreateExecutor(globalConfig *Config, provider ExecutorProvider, logger BuildLogger) (executor Executor, err error) {
	for tries := 0; tries < PreparationRetries; tries++ {
		executor = provider.Create()
		if executor == nil {
			err = errors.New("failed to create executor")
			return
		}

		err = executor.Prepare(globalConfig, b.Runner, b)
		if err == nil {
			break
		}
		if executor != nil {
			executor.Cleanup()
			executor = nil
		}

		logger.SoftErrorln("Preparation failed:", err)
		logger.Infoln("Will be retried in", PreparationRetryInterval, "...")
		time.Sleep(PreparationRetryInterval)
	}
	return
}

func (b *Build) Run(globalConfig *Config, trace BuildTrace) (err error) {
	var executor Executor

	logger := NewBuildLogger(trace, b.Log())
	logger.Println("Running with " + AppVersion.Line() + helpers.ANSI_RESET)

	defer func() {
		if _, ok := err.(*BuildError); ok {
			logger.SoftErrorln("Build failed:", err)
			trace.Fail(err)
		} else if err != nil {
			logger.Errorln("Build failed (system failure):", err)
			trace.Fail(err)
		} else {
			logger.Infoln("Build succeeded")
			trace.Success()
		}
		if executor != nil {
			executor.Cleanup()
		}
	}()

	b.Trace = trace

	provider := GetExecutor(b.Runner.Executor)
	if executor == nil {
		return errors.New("executor not found")
	}

	executor, err = b.retryCreateExecutor(globalConfig, provider, logger)
	if err == nil {
		err = b.run(executor)
	}
	if executor != nil {
		executor.Finish(err)
	}
	return err
}

func (b *Build) String() string {
	return helpers.ToYAML(b)
}

func (b *Build) GetDefaultVariables() BuildVariables {
	return BuildVariables{
		{"CI", "true", true, true, false},
		{"CI_BUILD_REF", b.Sha, true, true, false},
		{"CI_BUILD_BEFORE_SHA", b.BeforeSha, true, true, false},
		{"CI_BUILD_REF_NAME", b.RefName, true, true, false},
		{"CI_BUILD_ID", strconv.Itoa(b.ID), true, true, false},
		{"CI_BUILD_REPO", b.RepoURL, true, true, false},
		{"CI_BUILD_TOKEN", b.Token, true, true, false},
		{"CI_PROJECT_ID", strconv.Itoa(b.ProjectID), true, true, false},
		{"CI_PROJECT_DIR", b.FullProjectDir(), true, true, false},
		{"CI_SERVER", "yes", true, true, false},
		{"CI_SERVER_NAME", "GitLab CI", true, true, false},
		{"CI_SERVER_VERSION", "", true, true, false},
		{"CI_SERVER_REVISION", "", true, true, false},
		{"GITLAB_CI", "true", true, true, false},
	}
}

func (b *Build) GetAllVariables() BuildVariables {
	variables := b.Runner.GetVariables()
	variables = append(variables, b.GetDefaultVariables()...)
	variables = append(variables, b.Variables...)
	return variables.Expand()
}

func (b *Build) GetGitDepth() string {
	return b.GetAllVariables().Get("GIT_DEPTH")
}

func (b *Build) GetGitStrategy() GitStrategy {
	switch b.GetAllVariables().Get("GIT_STRATEGY") {
	case "clone":
		return GitClone

	case "fetch":
		return GitFetch

	default:
		if b.AllowGitFetch {
			return GitFetch
		}

		return GitClone
	}
}
<|endoftext|>


<commit_before>package db

const BLANK = ""

const (
	DELETE_OPERATION = 1
	INSERT_OPERATION = 2
	UPDATE_OPERATION = 3

	SentCollection       = "sent_notifications"
	StatsCollection      = "last_seen_at"
	TopicCollection      = "topics"
	GullyCollection      = "gullys"
	UserLocaleCollection = "user_locales"

	SavedEmailCollection     = "saved_email"
	SavedPushCollection      = "saved_push"
	SavedWebCollection       = "saved_web"
	AvailableTopicCollection = "topics_available"

	ProcessedCollection = "processed"
)

type Processed struct {
	BaseModel    `bson:",inline"`
	User         string `bson:"user"`
	Organization string `bson:"org"`
}

type AvailableTopic struct {
	BaseModel `bson:",inline"`
	Ident     string   `json:"ident" bson:"ident" required:"true" form:"ident" binding:"required"`
	AppName   string   `json:"app_name" bson:"app_name" required:"true" form:"app_name" binding:"required"`
	Channels  []string `json:"channels" bson:"channels" required:"true" form:"channels" binding:"required"`
}

type SentItem struct {
	BaseModel      `bson:",inline"`
	CreatedBy      string                 `json:"created_by" bson:"created_by" required:"true"`
	Organization   string                 `json:"org" bson:"org" required:"true"`
	AppName        string                 `json:"app_name" bson:"app_name" required:"true"`
	Topic          string                 `json:"topic" bson:"topic" required:"true"`
	User           string                 `json:"user" bson:"user" required:"true"`
	DestinationUri string                 `json:"destination_uri" bson:"destination_uri" required:"true"`
	Text           string                 `json:"text" bson:"text" required:"true"`
	IsRead         bool                   `json:"is_read" bson:"is_read"`
	Context        map[string]interface{} `json:"context" bson:"context"`
	Entity         string                 `json:"entity" bson:"entity" required:"true"`
}

type SavedItem struct {
	BaseModel `bson:",inline"`
	Data      interface{} `bson:"data"`
	Details   PendingItem `bson:"details"`
}

type PendingItem struct {
	BaseModel      `bson:",inline"`
	CreatedBy      string                 `json:"created_by" bson:"created_by" required:"true"`
	Organization   string                 `json:"org" bson:"org" required:"true"`
	AppName        string                 `json:"app_name" bson:"app_name" required:"true"`
	Topic          string                 `json:"topic" bson:"topic" required:"true"`
	IsPending      bool                   `json:"is_pending" bson:"is_pending" required:"true"`
	User           string                 `json:"user" bson:"user" required:"true"`
	DestinationUri string                 `json:"destination_uri" bson:"destination_uri" required:"true"`
	Context        map[string]interface{} `json:"context" bson:"context" required:"true"`
	IsRead         bool                   `json:"is_read" bson:"is_read"`
	Entity         string                 `json:"entity" bson:"entity" required:"true"`
}

func (self *PendingItem) IsValid() bool {
	if len(self.Context) == 0 {
		return false
	}
	return true
}

type LastSeen struct {
	BaseModel    `bson:",inline"`
	User         string `json:"user" bson:"user" required:"true"`
	Organization string `json:"org" bson:"org"`
	AppName      string `json:"app_name" bson:"app_name"`
	Timestamp    int64  `json:"timestamp" bson:"timestamp" required:"true"`
}

type UserLocale struct {
	BaseModel `bson:",inline"`
	User      string `json:"user" bson:"user" required:"true"`
	Locale    string `json:"locale" bson:"locale" required:"true"`
	TimeZone  string `json:"timezone" bson:"timezone" required:"true"`
}

func (self *UserLocale) IsValid() bool {
	if len(self.Locale) == 0 || len(self.User) == 0 || len(self.TimeZone) == 0 {
		return false
	}
	return true
}

type Channel struct {
	Name    string `json:"name" bson:"name"`
	Enabled bool   `json:"enabled" bson:"enabled"`
	Default bool   `json:"default" bson:"default"`
	Locked  bool   `json:"locked" bson:"locked"`
}

type Topic struct {
	BaseModel    `bson:",inline"`
	User         string    `json:"user" bson:"user"`
	Organization string    `json:"org" bson:"org"`
	Channels     []Channel `json:"channels" bson:"channels" required:"true"`
	Ident        string    `json:"ident" bson:"ident" required:"true" form:"ident" binding:"required"`
}
<commit_msg>remove collection name<commit_after>package db

const BLANK = ""

const (
	DELETE_OPERATION = 1
	INSERT_OPERATION = 2
	UPDATE_OPERATION = 3

	SentCollection       = "sent_notifications"
	StatsCollection      = "last_seen_at"
	TopicCollection      = "topics"
	UserLocaleCollection = "user_locales"

	SavedEmailCollection     = "saved_email"
	SavedPushCollection      = "saved_push"
	SavedWebCollection       = "saved_web"
	AvailableTopicCollection = "topics_available"

	ProcessedCollection = "processed"
)

type Processed struct {
	BaseModel    `bson:",inline"`
	User         string `bson:"user"`
	Organization string `bson:"org"`
}

type AvailableTopic struct {
	BaseModel `bson:",inline"`
	Ident     string   `json:"ident" bson:"ident" required:"true" form:"ident" binding:"required"`
	AppName   string   `json:"app_name" bson:"app_name" required:"true" form:"app_name" binding:"required"`
	Channels  []string `json:"channels" bson:"channels" required:"true" form:"channels" binding:"required"`
}

type SentItem struct {
	BaseModel      `bson:",inline"`
	CreatedBy      string                 `json:"created_by" bson:"created_by" required:"true"`
	Organization   string                 `json:"org" bson:"org" required:"true"`
	AppName        string                 `json:"app_name" bson:"app_name" required:"true"`
	Topic          string                 `json:"topic" bson:"topic" required:"true"`
	User           string                 `json:"user" bson:"user" required:"true"`
	DestinationUri string                 `json:"destination_uri" bson:"destination_uri" required:"true"`
	Text           string                 `json:"text" bson:"text" required:"true"`
	IsRead         bool                   `json:"is_read" bson:"is_read"`
	Context        map[string]interface{} `json:"context" bson:"context"`
	Entity         string                 `json:"entity" bson:"entity" required:"true"`
}

type SavedItem struct {
	BaseModel `bson:",inline"`
	Data      interface{} `bson:"data"`
	Details   PendingItem `bson:"details"`
}

type PendingItem struct {
	BaseModel      `bson:",inline"`
	CreatedBy      string                 `json:"created_by" bson:"created_by" required:"true"`
	Organization   string                 `json:"org" bson:"org" required:"true"`
	AppName        string                 `json:"app_name" bson:"app_name" required:"true"`
	Topic          string                 `json:"topic" bson:"topic" required:"true"`
	IsPending      bool                   `json:"is_pending" bson:"is_pending" required:"true"`
	User           string                 `json:"user" bson:"user" required:"true"`
	DestinationUri string                 `json:"destination_uri" bson:"destination_uri" required:"true"`
	Context        map[string]interface{} `json:"context" bson:"context" required:"true"`
	IsRead         bool                   `json:"is_read" bson:"is_read"`
	Entity         string                 `json:"entity" bson:"entity" required:"true"`
}

func (self *PendingItem) IsValid() bool {
	if len(self.Context) == 0 {
		return false
	}
	return true
}

type LastSeen struct {
	BaseModel    `bson:",inline"`
	User         string `json:"user" bson:"user" required:"true"`
	Organization string `json:"org" bson:"org"`
	AppName      string `json:"app_name" bson:"app_name"`
	Timestamp    int64  `json:"timestamp" bson:"timestamp" required:"true"`
}

type UserLocale struct {
	BaseModel `bson:",inline"`
	User      string `json:"user" bson:"user" required:"true"`
	Locale    string `json:"locale" bson:"locale" required:"true"`
	TimeZone  string `json:"timezone" bson:"timezone" required:"true"`
}

func (self *UserLocale) IsValid() bool {
	if len(self.Locale) == 0 || len(self.User) == 0 || len(self.TimeZone) == 0 {
		return false
	}
	return true
}

type Channel struct {
	Name    string `json:"name" bson:"name"`
	Enabled bool   `json:"enabled" bson:"enabled"`
	Default bool   `json:"default" bson:"default"`
	Locked  bool   `json:"locked" bson:"locked"`
}

type Topic struct {
	BaseModel    `bson:",inline"`
	User         string    `json:"user" bson:"user"`
	Organization string    `json:"org" bson:"org"`
	Channels     []Channel `json:"channels" bson:"channels" required:"true"`
	Ident        string    `json:"ident" bson:"ident" required:"true" form:"ident" binding:"required"`
}
<|endoftext|>


<commit_before>package collector

import (
	"bufio"
	"bytes"
	"io"
	"io/ioutil"
	"strconv"
	"strings"
)

type Meminfo struct {
	Buffers   uint64
	Cached    uint64
	MemTotal  uint64
	MemFree   uint64
	SwapTotal uint64
	SwapUsed  uint64
	SwapFree  uint64
}

func MemInfo() (*Meminfo, error) {
	want := map[string]bool{
		"Buffers:":   true,
		"Cached:":    true,
		"MemTotal:":  true,
		"MemFree:":   true,
		"SwapTotal:": true,
		"SwapFree:":  true}

	contents, err := ioutil.ReadFile("/proc/meminfo")
	if err != nil {
		return nil, err
	}

	memInfo := &Meminfo{}

	reader := bufio.NewReader(bytes.NewBuffer(contents))

	for {
		line, _, err := reader.ReadLine()
		if err == io.EOF {
			break
		}

		fields := strings.Fields(string(line))
		fieldName := fields[0]

		_, ok := want[fieldName]
		if ok && len(fields) == 3 {
			val, numerr := strconv.ParseUint(fields[1], 10, 64)
			if numerr != nil {
				continue
			}
			switch fieldName {
			case "Buffers:":
				memInfo.Buffers = val
			case "Cached:":
				memInfo.Cached = val
			case "MemTotal:":
				memInfo.MemTotal = val
			case "MemFree:":
				memInfo.MemFree = val
			case "SwapTotal:":
				memInfo.SwapTotal = val
			case "SwapFree:":
				memInfo.SwapFree = val
			}
		}
	}
	memInfo.SwapUsed = memInfo.SwapTotal - memInfo.SwapFree

	return memInfo, nil
}
<commit_msg>mem val * 1024<commit_after>package collector

import (
	"bufio"
	"bytes"
	"io"
	"io/ioutil"
	"strconv"
	"strings"
)

type Meminfo struct {
	Buffers   uint64
	Cached    uint64
	MemTotal  uint64
	MemFree   uint64
	SwapTotal uint64
	SwapUsed  uint64
	SwapFree  uint64
}

var Multi uint64 = 1024

func MemInfo() (*Meminfo, error) {
	want := map[string]bool{
		"Buffers:":   true,
		"Cached:":    true,
		"MemTotal:":  true,
		"MemFree:":   true,
		"SwapTotal:": true,
		"SwapFree:":  true}

	contents, err := ioutil.ReadFile("/proc/meminfo")
	if err != nil {
		return nil, err
	}

	memInfo := &Meminfo{}

	reader := bufio.NewReader(bytes.NewBuffer(contents))

	for {
		line, _, err := reader.ReadLine()
		if err == io.EOF {
			break
		}

		fields := strings.Fields(string(line))
		fieldName := fields[0]

		_, ok := want[fieldName]
		if ok && len(fields) == 3 {
			val, numerr := strconv.ParseUint(fields[1], 10, 64)
			if numerr != nil {
				continue
			}
			switch fieldName {
			case "Buffers:":
				memInfo.Buffers = val * Multi
			case "Cached:":
				memInfo.Cached = val * Multi
			case "MemTotal:":
				memInfo.MemTotal = val * Multi
			case "MemFree:":
				memInfo.MemFree = val * Multi
			case "SwapTotal:":
				memInfo.SwapTotal = val * Multi
			case "SwapFree:":
				memInfo.SwapFree = val * Multi
			}
		}
	}
	memInfo.SwapUsed = memInfo.SwapTotal - memInfo.SwapFree

	return memInfo, nil
}
<|endoftext|>


<commit_before>package library

import (
	"bytes"
	"encoding/json"
	"errors"
	"io/ioutil"
	"net/http"

	"github.com/nytlabs/gojee"
	"github.com/nytlabs/streamtools/st/blocks" // blocks
	"github.com/nytlabs/streamtools/st/util"
)

func parseHeaders(headerRule map[string]interface{}) (map[string]string, error) {
	t := make(map[string]string)

	for k, v := range headerRule {
		switch r := v.(type) {
		case string:
			t[k] = r
		default:
			return nil, errors.New("value is not a string")
		}
	}

	return t, nil
}

// specify those channels we're going to use to communicate with streamtools
type WebRequest struct {
	blocks.Block
	queryrule chan blocks.MsgChan
	inrule    blocks.MsgChan
	inpoll    blocks.MsgChan
	in        blocks.MsgChan
	out       blocks.MsgChan
	quit      blocks.MsgChan
}

// we need to build a simple factory so that streamtools can make new blocks of this kind
func NewWebRequest() blocks.BlockInterface {
	return &WebRequest{}
}

// Setup is called once before running the block. We build up the channels and specify what kind of block this is.
func (b *WebRequest) Setup() {
	b.Kind = "Network I/O"
	b.Desc = "Makes requests to a given URL with specified HTTP method."
	b.in = b.InRoute("in")
	b.inrule = b.InRoute("rule")
	b.queryrule = b.QueryRoute("rule")
	b.out = b.Broadcast()
	b.quit = b.Quit()
}

// Run is the block's main loop. Here we listen on the different channels we set up.
func (b *WebRequest) Run() {
	var err error
	var ok bool

	url := ""
	requestUrl := ""
	var urlPath string
	var urlTree *jee.TokenTree

	bodyPath := "."
	var bodyTree *jee.TokenTree

	token, err := jee.Lexer(bodyPath)
	if err != nil {
		b.Error(err)
	}

	bodyTree, err = jee.Parser(token)
	if err != nil {
		b.Error(err)
	}

	var httpMethod string
	headerRule := map[string]interface{}{}
	headers, _ := parseHeaders(headerRule)

	transport := http.Transport{
		Dial: dialTimeout,
	}

	client := &http.Client{
		Transport: &transport,
	}

	for {
		select {
		case ruleI := <-b.inrule:
			httpMethod, err = util.ParseString(ruleI, "Method")
			if err != nil {
				b.Error(err)
				continue
			}

			url, err = util.ParseString(ruleI, "Url")
			if err != nil {
				b.Error(err)
				continue
			}

			urlPath, err = util.ParseString(ruleI, "UrlPath")
			if err != nil {
				b.Error(err)
				continue
			}

			if len(url) != 0 && len(urlPath) != 0 {
				b.Error(errors.New("Specify either a url or a path to a url"))
				continue
			}

			if len(urlPath) == 0 {
				urlTree = nil
			}

			if len(url) == 0 {
				token, err := jee.Lexer(urlPath)
				if err != nil {
					b.Error(err)
					continue
				}

				urlTree, err = jee.Parser(token)
				if err != nil {
					b.Error(err)
					continue
				}
			}

			bodyPath, err = util.ParseString(ruleI, "BodyPath")
			if err != nil {
				b.Error(err)
				continue
			}
			token, err := jee.Lexer(bodyPath)
			if err != nil {
				b.Error(err)
				continue
			}

			bodyTree, err = jee.Parser(token)
			if err != nil {
				b.Error(err)
				continue
			}

			rule := ruleI.(map[string]interface{})
			headerRuleI, ok := rule["Headers"]
			if !ok {
				continue
			}
			headerRule = headerRuleI.(map[string]interface{})
			p, err := parseHeaders(headerRule)
			if err == nil {
				headers = p
			} else {
				b.Error(err)
				continue
			}
		case <-b.quit:
			return

		case msg := <-b.in:
			var req *http.Request

			if urlTree != nil {
				urlInterface, err := jee.Eval(urlTree, msg)
				if err != nil {
					b.Error(err)
					continue
				}
				// use the url found via rule.UrlPath in the request
				requestUrl, ok = urlInterface.(string)
				if !ok {
					b.Error(errors.New("couldn't assert url to a string"))
					continue
				}
			}

			// use the rule.Url in the request
			if len(url) != 0 {
				requestUrl = url
			}

			if httpMethod == "POST" || httpMethod == "PUT" {
				bodyInterface, err := jee.Eval(bodyTree, msg)
				if err != nil {
					b.Error(err)
					continue
				}
				requestBody, err := json.Marshal(bodyInterface)
				if err != nil {
					b.Error(errors.New("couldn't marshal body"))
					continue
				}

				req, err = http.NewRequest(httpMethod, requestUrl, bytes.NewReader(requestBody))
				if err != nil {
					b.Error(err)
					break
				}

			} else {
				req, err = http.NewRequest(httpMethod, requestUrl, nil)
				if err != nil {
					b.Error(err)
					break
				}
			}

			for key, value := range headers {
				if key == "Host" {
					req.Host = value
				} else {
					req.Header.Set(key, value)
				}
			}

			resp, err := client.Do(req)
			if err != nil {
				b.Error(err)
				break
			}

			body, err := ioutil.ReadAll(resp.Body)
			if err != nil {
				b.Error(err)
				break
			}

			var responseBody interface{}
			err = json.Unmarshal(body, &responseBody)
			if err != nil {
				responseBody = string(body)
			}

			// always return body/headers/status even if it's nasty xml
			outMsg := map[string]interface{}{
				"body":    responseBody,
				"headers": resp.Header,
				"status":  resp.Status,
			}

			resp.Body.Close()

			b.out <- outMsg

		case resp := <-b.queryrule:
			resp <- map[string]interface{}{
				"Url":      url,
				"UrlPath":  urlPath,
				"BodyPath": bodyPath,
				"Method":   httpMethod,
				"Headers":  headerRule,
			}
		}
	}
}
<commit_msg>defer close in case of later error<commit_after>package library

import (
	"bytes"
	"encoding/json"
	"errors"
	"io/ioutil"
	"net/http"

	"github.com/nytlabs/gojee"
	"github.com/nytlabs/streamtools/st/blocks" // blocks
	"github.com/nytlabs/streamtools/st/util"
)

func parseHeaders(headerRule map[string]interface{}) (map[string]string, error) {
	t := make(map[string]string)

	for k, v := range headerRule {
		switch r := v.(type) {
		case string:
			t[k] = r
		default:
			return nil, errors.New("value is not a string")
		}
	}

	return t, nil
}

// specify those channels we're going to use to communicate with streamtools
type WebRequest struct {
	blocks.Block
	queryrule chan blocks.MsgChan
	inrule    blocks.MsgChan
	inpoll    blocks.MsgChan
	in        blocks.MsgChan
	out       blocks.MsgChan
	quit      blocks.MsgChan
}

// we need to build a simple factory so that streamtools can make new blocks of this kind
func NewWebRequest() blocks.BlockInterface {
	return &WebRequest{}
}

// Setup is called once before running the block. We build up the channels and specify what kind of block this is.
func (b *WebRequest) Setup() {
	b.Kind = "Network I/O"
	b.Desc = "Makes requests to a given URL with specified HTTP method."
	b.in = b.InRoute("in")
	b.inrule = b.InRoute("rule")
	b.queryrule = b.QueryRoute("rule")
	b.out = b.Broadcast()
	b.quit = b.Quit()
}

// Run is the block's main loop. Here we listen on the different channels we set up.
func (b *WebRequest) Run() {
	var err error
	var ok bool

	url := ""
	requestUrl := ""
	var urlPath string
	var urlTree *jee.TokenTree

	bodyPath := "."
	var bodyTree *jee.TokenTree

	token, err := jee.Lexer(bodyPath)
	if err != nil {
		b.Error(err)
	}

	bodyTree, err = jee.Parser(token)
	if err != nil {
		b.Error(err)
	}

	var httpMethod string
	headerRule := map[string]interface{}{}
	headers, _ := parseHeaders(headerRule)

	transport := http.Transport{
		Dial: dialTimeout,
	}

	client := &http.Client{
		Transport: &transport,
	}

	for {
		select {
		case ruleI := <-b.inrule:
			httpMethod, err = util.ParseString(ruleI, "Method")
			if err != nil {
				b.Error(err)
				continue
			}

			url, err = util.ParseString(ruleI, "Url")
			if err != nil {
				b.Error(err)
				continue
			}

			urlPath, err = util.ParseString(ruleI, "UrlPath")
			if err != nil {
				b.Error(err)
				continue
			}

			if len(url) != 0 && len(urlPath) != 0 {
				b.Error(errors.New("Specify either a url or a path to a url"))
				continue
			}

			if len(urlPath) == 0 {
				urlTree = nil
			}

			if len(url) == 0 {
				token, err := jee.Lexer(urlPath)
				if err != nil {
					b.Error(err)
					continue
				}

				urlTree, err = jee.Parser(token)
				if err != nil {
					b.Error(err)
					continue
				}
			}

			bodyPath, err = util.ParseString(ruleI, "BodyPath")
			if err != nil {
				b.Error(err)
				continue
			}
			token, err := jee.Lexer(bodyPath)
			if err != nil {
				b.Error(err)
				continue
			}

			bodyTree, err = jee.Parser(token)
			if err != nil {
				b.Error(err)
				continue
			}

			rule := ruleI.(map[string]interface{})
			headerRuleI, ok := rule["Headers"]
			if !ok {
				continue
			}
			headerRule = headerRuleI.(map[string]interface{})
			p, err := parseHeaders(headerRule)
			if err == nil {
				headers = p
			} else {
				b.Error(err)
				continue
			}
		case <-b.quit:
			return

		case msg := <-b.in:
			var req *http.Request

			if urlTree != nil {
				urlInterface, err := jee.Eval(urlTree, msg)
				if err != nil {
					b.Error(err)
					continue
				}
				// use the url found via rule.UrlPath in the request
				requestUrl, ok = urlInterface.(string)
				if !ok {
					b.Error(errors.New("couldn't assert url to a string"))
					continue
				}
			}

			// use the rule.Url in the request
			if len(url) != 0 {
				requestUrl = url
			}

			if httpMethod == "POST" || httpMethod == "PUT" {
				bodyInterface, err := jee.Eval(bodyTree, msg)
				if err != nil {
					b.Error(err)
					continue
				}
				requestBody, err := json.Marshal(bodyInterface)
				if err != nil {
					b.Error(errors.New("couldn't marshal body"))
					continue
				}

				req, err = http.NewRequest(httpMethod, requestUrl, bytes.NewReader(requestBody))
				if err != nil {
					b.Error(err)
					break
				}

			} else {
				req, err = http.NewRequest(httpMethod, requestUrl, nil)
				if err != nil {
					b.Error(err)
					break
				}
			}

			for key, value := range headers {
				if key == "Host" {
					req.Host = value
				} else {
					req.Header.Set(key, value)
				}
			}

			resp, err := client.Do(req)
			if err != nil {
				b.Error(err)
				break
			}
			defer resp.Body.Close()

			body, err := ioutil.ReadAll(resp.Body)
			if err != nil {
				b.Error(err)
				break
			}

			var responseBody interface{}
			err = json.Unmarshal(body, &responseBody)
			if err != nil {
				responseBody = string(body)
			}

			// always return body/headers/status even if it's nasty xml
			outMsg := map[string]interface{}{
				"body":    responseBody,
				"headers": resp.Header,
				"status":  resp.Status,
			}

			b.out <- outMsg

		case resp := <-b.queryrule:
			resp <- map[string]interface{}{
				"Url":      url,
				"UrlPath":  urlPath,
				"BodyPath": bodyPath,
				"Method":   httpMethod,
				"Headers":  headerRule,
			}
		}
	}
}
<|endoftext|>


<commit_before>package dcpu

import (
	"fmt"
	"unsafe"
)

type Word uint16

type ProtectionError struct {
	Address            Word
	Opcode             Word
	OperandA, OperandB Word
}

func (err *ProtectionError) Error() string {
	return fmt.Sprintf("protection violation at address %#x (instruction %#x, operands %#x, %#x)",
		err.Address, err.Opcode, err.OperandA, err.OperandB)
}

type Registers struct {
	A, B, C, X, Y, Z, I, J Word
	PC                     Word
	SP                     Word
	O                      Word
}

type Region struct {
	Start  Word
	Length Word
}

func (r Region) Contains(address Word) bool {
	return address >= r.Start && address < r.Start+r.Length
}

// End() returns the first address not contained in the region
func (r Region) End() Word {
	return r.Start + r.Length
}

func (r Region) Union(r2 Region) Region {
	var reg Region
	if r2.Start < r.Start {
		reg.Start = r2.Start
	} else {
		reg.Start = r.Start
	}
	if r2.End() > r.End() {
		reg.Length = r2.End() - reg.Start
	} else {
		reg.Length = r.End() - reg.Start
	}
	return reg
}

type State struct {
	Registers
	Ram       [0x10000]Word
	Protected []Region
}

func decodeOpcode(opcode Word) (oooo, aaaaaa, bbbbbb Word) {
	oooo = opcode & 0xF
	aaaaaa = (opcode >> 4) & 0x3F
	bbbbbb = (opcode >> 10) & 0x3F
	return
}

// wordCount counts the number of words in the instruction identified by the given opcode
func wordCount(opcode Word) Word {
	_, a, b := decodeOpcode(opcode)
	count := Word(1)
	switch {
	case a >= 16 && a <= 23:
	case a == 30:
	case a == 31:
		count++
	}
	switch {
	case b >= 16 && b <= 23:
	case b == 30:
	case b == 31:
		count++
	}
	return count
}

func (s *State) translateOperand(op Word) (val Word, assignable *Word) {
	switch op {
	// 0-7: register value - register values
	case 0:
		assignable = &s.A
	case 1:
		assignable = &s.B
	case 2:
		assignable = &s.C
	case 3:
		assignable = &s.X
	case 4:
		assignable = &s.Y
	case 5:
		assignable = &s.Z
	case 6:
		assignable = &s.I
	case 7:
		assignable = &s.J
	// 8-15: [register value] - value at address in registries
	case 8:
		assignable = &s.Ram[s.A]
	case 9:
		assignable = &s.Ram[s.B]
	case 10:
		assignable = &s.Ram[s.C]
	case 11:
		assignable = &s.Ram[s.X]
	case 12:
		assignable = &s.Ram[s.Y]
	case 13:
		assignable = &s.Ram[s.Z]
	case 14:
		assignable = &s.Ram[s.I]
	case 15:
		assignable = &s.Ram[s.J]
	// 16-23: [next word of ram + register value] - memory address offset by register value
	case 16:
		assignable = &s.Ram[s.Ram[s.PC]+s.A]
		s.PC++
	case 17:
		assignable = &s.Ram[s.Ram[s.PC]+s.B]
		s.PC++
	case 18:
		assignable = &s.Ram[s.Ram[s.PC]+s.C]
		s.PC++
	case 19:
		assignable = &s.Ram[s.Ram[s.PC]+s.X]
		s.PC++
	case 20:
		assignable = &s.Ram[s.Ram[s.PC]+s.Y]
		s.PC++
	case 21:
		assignable = &s.Ram[s.Ram[s.PC]+s.Z]
		s.PC++
	case 22:
		assignable = &s.Ram[s.Ram[s.PC]+s.I]
		s.PC++
	case 23:
		assignable = &s.Ram[s.Ram[s.PC]+s.J]
		s.PC++
	// 24: POP - value at stack address, then increases stack counter
	case 24:
		assignable = &s.Ram[s.SP]
		s.SP++
	// 25: PEEK - value at stack address
	case 25:
		assignable = &s.Ram[s.SP]
	case 26:
		// 26: PUSH - decreases stack address, then value at stack address
		s.SP--
		assignable = &s.Ram[s.SP]
	// 27: SP - current stack pointer value - current stack address
	case 27:
		assignable = &s.SP
	// 28: PC - program counter- current program counter
	case 28:
		assignable = &s.PC
	// 29: O - overflow - current value of the overflow
	case 29:
		assignable = &s.O
	// 30: [next word of ram] - memory address
	case 30:
		assignable = &s.Ram[s.Ram[s.PC]]
		s.PC++
	// 31: next word of ram - literal, does nothing on assign
	case 31:
		val = s.Ram[s.PC]
		s.PC++
	default:
		if op >= 64 {
			panic("Out of bounds operand")
		}
		val = op - 32
	}
	if assignable != nil {
		val = *assignable
	}
	return
}

func (s *State) isProtected(address Word) bool {
	for _, region := range s.Protected {
		if region.Contains(address) {
			return true
		}
		if region.Start > address {
			break
		}
	}
	return false
}

// Step iterates the CPU by one instruction.
func (s *State) Step() error {
	// fetch
	opcode := s.Ram[s.PC]
	s.PC++

	// decode
	ins, a, b := decodeOpcode(opcode)

	var assignable *Word
	a, assignable = s.translateOperand(a)
	b, _ = s.translateOperand(b)

	// execute
	var val Word
	switch ins {
	case 0:
		// marked RESERVED, lets just treat it as a NOP
	case 1:
		// SET a, b - sets value of b to a
		val = b
	case 2:
		// ADD a, b - adds b to a, sets O
		result := uint32(a) + uint32(b)
		val = Word(result & 0xFFFF)
		s.O = Word(result >> 16)
	case 3:
		// SUB a, b - subtracts b from a, sets O
		result := uint32(a) - uint32(b)
		val = Word(result & 0xFFFF)
		s.O = Word(result >> 16)
	case 4:
		// MUL a, b - multiplies a by b, sets O
		result := uint32(a) * uint32(b)
		val = Word(result & 0xFFFF)
		s.O = Word(result >> 16)
	case 5:
		// DIV a, b - divides a by b, sets O
		// NB: how can this overflow?
		// assuming for the moment that O is supposed to be the mod
		val = a / b
		s.O = a % b
	case 6:
		// MOD a, b - remainder of a over b
		val = a % b
	case 7:
		// SHL a, b - shifts a left b places, sets O
		result := uint32(a) << uint32(b)
		val = Word(result & 0xFFFF)
		s.O = Word(result >> 16)
	case 8:
		// SHR a, b - shifts a right b places, sets O
		// NB: how can this overflow?
		val = a >> b
	case 9:
		// AND a, b - binary and of a and b
		val = a & b
	case 10:
		// BOR a, b - binary or of a and b
		val = a | b
	case 11:
		// XOR a, b - binary xor of a and b
		val = a ^ b
	case 12:
		// IFE a, b - skips one instruction if a!=b
		if a != b {
			s.PC += wordCount(s.Ram[s.PC])
		}
	case 13:
		// IFN a, b - skips one instruction if a==b
		if a == b {
			s.PC += wordCount(s.Ram[s.PC])
		}
	case 14:
		// IFG a, b - skips one instruction if a<=b
		if a <= b {
			s.PC += wordCount(s.Ram[s.PC])
		}
	case 15:
		// IFB a, b - skips one instruction if (a&b)==0
		if (a & b) == 0 {
			s.PC += wordCount(s.Ram[s.PC])
		}
	default:
		panic("Out of bounds opcode")
	}

	// store
	if ins >= 1 && ins <= 11 && assignable != nil {
		// test memory protection
		// are we in our ram?
		assPtr := uintptr(unsafe.Pointer(assignable))
		ramStart := uintptr(unsafe.Pointer(&s.Ram[0]))
		ramEnd := uintptr(unsafe.Pointer(&s.Ram[len(s.Ram)-1]))
		if assPtr >= ramStart && assPtr <= ramEnd {
			index := Word((assPtr - ramStart) / unsafe.Sizeof(s.Ram[0]))
			for _, region := range s.Protected {
				if region.Contains(index) {
					// protection error
					return &ProtectionError{
						Address:  index,
						Opcode:   opcode,
						OperandA: a,
						OperandB: b,
					}
				} else if region.Start > index {
					break
				}
			}
		}
		// go ahead and store
		*assignable = val
	}

	return nil
}
<commit_msg>Update implementation as per the new spec<commit_after>package dcpu

import (
	"fmt"
	"unsafe"
)

type Word uint16

type ProtectionError struct {
	Address            Word
	Opcode             Word
	OperandA, OperandB Word
}

func (err *ProtectionError) Error() string {
	return fmt.Sprintf("protection violation at address %#x (instruction %#x, operands %#x, %#x)",
		err.Address, err.Opcode, err.OperandA, err.OperandB)
}

type Registers struct {
	A, B, C, X, Y, Z, I, J Word
	PC                     Word
	SP                     Word
	O                      Word
}

type Region struct {
	Start  Word
	Length Word
}

func (r Region) Contains(address Word) bool {
	return address >= r.Start && address < r.Start+r.Length
}

// End() returns the first address not contained in the region
func (r Region) End() Word {
	return r.Start + r.Length
}

func (r Region) Union(r2 Region) Region {
	var reg Region
	if r2.Start < r.Start {
		reg.Start = r2.Start
	} else {
		reg.Start = r.Start
	}
	if r2.End() > r.End() {
		reg.Length = r2.End() - reg.Start
	} else {
		reg.Length = r.End() - reg.Start
	}
	return reg
}

type State struct {
	Registers
	Ram       [0x10000]Word
	Protected []Region
}

func decodeOpcode(opcode Word) (oooo, aaaaaa, bbbbbb Word) {
	oooo = opcode & 0xF
	aaaaaa = (opcode >> 4) & 0x3F
	bbbbbb = (opcode >> 10) & 0x3F
	return
}

// wordCount counts the number of words in the instruction identified by the given opcode
func wordCount(opcode Word) Word {
	_, a, b := decodeOpcode(opcode)
	count := Word(1)
	switch {
	case a >= 16 && a <= 23:
	case a == 30:
	case a == 31:
		count++
	}
	switch {
	case b >= 16 && b <= 23:
	case b == 30:
	case b == 31:
		count++
	}
	return count
}

func (s *State) translateOperand(op Word) (val Word, assignable *Word) {
	switch op {
	// 0-7: register value - register values
	case 0:
		assignable = &s.A
	case 1:
		assignable = &s.B
	case 2:
		assignable = &s.C
	case 3:
		assignable = &s.X
	case 4:
		assignable = &s.Y
	case 5:
		assignable = &s.Z
	case 6:
		assignable = &s.I
	case 7:
		assignable = &s.J
	// 8-15: [register value] - value at address in registries
	case 8:
		assignable = &s.Ram[s.A]
	case 9:
		assignable = &s.Ram[s.B]
	case 10:
		assignable = &s.Ram[s.C]
	case 11:
		assignable = &s.Ram[s.X]
	case 12:
		assignable = &s.Ram[s.Y]
	case 13:
		assignable = &s.Ram[s.Z]
	case 14:
		assignable = &s.Ram[s.I]
	case 15:
		assignable = &s.Ram[s.J]
	// 16-23: [next word of ram + register value] - memory address offset by register value
	case 16:
		assignable = &s.Ram[s.Ram[s.PC]+s.A]
		s.PC++
	case 17:
		assignable = &s.Ram[s.Ram[s.PC]+s.B]
		s.PC++
	case 18:
		assignable = &s.Ram[s.Ram[s.PC]+s.C]
		s.PC++
	case 19:
		assignable = &s.Ram[s.Ram[s.PC]+s.X]
		s.PC++
	case 20:
		assignable = &s.Ram[s.Ram[s.PC]+s.Y]
		s.PC++
	case 21:
		assignable = &s.Ram[s.Ram[s.PC]+s.Z]
		s.PC++
	case 22:
		assignable = &s.Ram[s.Ram[s.PC]+s.I]
		s.PC++
	case 23:
		assignable = &s.Ram[s.Ram[s.PC]+s.J]
		s.PC++
	// 24: POP - value at stack address, then increases stack counter
	case 24:
		assignable = &s.Ram[s.SP]
		s.SP++
	// 25: PEEK - value at stack address
	case 25:
		assignable = &s.Ram[s.SP]
	case 26:
		// 26: PUSH - decreases stack address, then value at stack address
		s.SP--
		assignable = &s.Ram[s.SP]
	// 27: SP - current stack pointer value - current stack address
	case 27:
		assignable = &s.SP
	// 28: PC - program counter- current program counter
	case 28:
		assignable = &s.PC
	// 29: O - overflow - current value of the overflow
	case 29:
		assignable = &s.O
	// 30: [next word of ram] - memory address
	case 30:
		assignable = &s.Ram[s.Ram[s.PC]]
		s.PC++
	// 31: next word of ram - literal, does nothing on assign
	case 31:
		val = s.Ram[s.PC]
		s.PC++
	default:
		if op >= 64 {
			panic("Out of bounds operand")
		}
		val = op - 32
	}
	if assignable != nil {
		val = *assignable
	}
	return
}

func (s *State) isProtected(address Word) bool {
	for _, region := range s.Protected {
		if region.Contains(address) {
			return true
		}
		if region.Start > address {
			break
		}
	}
	return false
}

// Step iterates the CPU by one instruction.
func (s *State) Step() error {
	// fetch
	opcode := s.Ram[s.PC]
	s.PC++

	// decode
	ins, a, b := decodeOpcode(opcode)

	var assignable *Word
	a, assignable = s.translateOperand(a)
	b, _ = s.translateOperand(b)

	// execute
	var val Word
	switch ins {
	case 0:
		// marked RESERVED, lets just treat it as a NOP
	case 1:
		// SET a, b - sets a to b
		val = b
	case 2:
		// ADD a, b - sets a to a+b, sets O to 0x0001 if there's an overflow, 0x0 otherwise
		result := uint32(a) + uint32(b)
		val = Word(result & 0xFFFF)
		s.O = Word(result >> 16) // will always be 0x0 or 0x1
	case 3:
		// SUB a, b - sets a to a-b, sets O to 0xffff if there's an underflow, 0x0 otherwise
		result := uint32(a) - uint32(b)
		val = Word(result & 0xFFFF)
		s.O = Word(result >> 16) // will always be 0x0 or 0xffff
	case 4:
		// MUL a, b - sets a to a*b, sets O to ((a*b)>>16)&0xffff
		result := uint32(a) * uint32(b)
		val = Word(result & 0xFFFF)
		s.O = Word(result >> 16)
	case 5:
		// DIV a, b - sets a to a/b, sets O to ((a<<16)/b)&0xffff. if b==0, sets a and O to 0 instead.
		if b == 0 {
			val, s.O = 0, 0
		} else {
			val = a / b
			s.O = Word(((uint32(a) << 16) / uint32(b)))
		}
	case 6:
		// MOD a, b - sets a to a%b. if b==0, sets a to 0 instead.
		if b == 0 {
			val = 0
		} else {
			val = a % b
		}
	case 7:
		// SHL a, b - sets a to a<<b, sets O to ((a<<b)>>16)&0xffff
		result := uint32(a) << uint32(b)
		val = Word(result & 0xFFFF)
		s.O = Word(result >> 16)
	case 8:
		// SHR a, b - sets a to a>>b, sets O to ((a<<16)>>b)&0xffff
		val = a >> b
		s.O = Word((uint32(a) << 16) >> b)
	case 9:
		// AND a, b - sets a to a&b
		val = a & b
	case 10:
		// BOR a, b - sets a to a|b
		val = a | b
	case 11:
		// XOR a, b - sets a to a^b
		val = a ^ b
	case 12:
		// IFE a, b - performs next instruction only if a==b
		if a != b {
			s.PC += wordCount(s.Ram[s.PC])
		}
	case 13:
		// IFN a, b - performs next instruction only if a!=b
		if a == b {
			s.PC += wordCount(s.Ram[s.PC])
		}
	case 14:
		// IFG a, b - performs next instruction only if a>b
		if a <= b {
			s.PC += wordCount(s.Ram[s.PC])
		}
	case 15:
		// IFB a, b - performs next instruction only if (a&b)!=0
		if (a & b) == 0 {
			s.PC += wordCount(s.Ram[s.PC])
		}
	default:
		panic("Out of bounds opcode")
	}

	// store
	if ins >= 1 && ins <= 11 && assignable != nil {
		// test memory protection
		// are we in our ram?
		assPtr := uintptr(unsafe.Pointer(assignable))
		ramStart := uintptr(unsafe.Pointer(&s.Ram[0]))
		ramEnd := uintptr(unsafe.Pointer(&s.Ram[len(s.Ram)-1]))
		if assPtr >= ramStart && assPtr <= ramEnd {
			index := Word((assPtr - ramStart) / unsafe.Sizeof(s.Ram[0]))
			for _, region := range s.Protected {
				if region.Contains(index) {
					// protection error
					return &ProtectionError{
						Address:  index,
						Opcode:   opcode,
						OperandA: a,
						OperandB: b,
					}
				} else if region.Start > index {
					break
				}
			}
		}
		// go ahead and store
		*assignable = val
	}

	return nil
}
<|endoftext|>


<commit_before>// Copyright 2016 Andrew O'Neill, Nordstrom

// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at

//     http://www.apache.org/licenses/LICENSE-2.0

// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"encoding/json"
	"fmt"
	"io"
	"log"
	"net"
	"net/http"
	"net/http/pprof"
	"os"
	"os/signal"
	"syscall"
	"time"

	"github.com/Nordstrom/choices"
	"github.com/foolusion/elwinprotos/elwin"
	"github.com/pkg/errors"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promhttp"
	"github.com/spf13/viper"
	"golang.org/x/net/context"
	"k8s.io/apimachinery/pkg/labels"
	"k8s.io/apimachinery/pkg/selection"
)

var (
	requestDurations = prometheus.NewHistogram(prometheus.HistogramOpts{
		Namespace: "nordstrom",
		Subsystem: "elwin",
		Name:      "request_durations_nanoseconds",
		Help:      "request latency distributions.",
		Buckets:   prometheus.ExponentialBuckets(1, 10, 10),
	})
	updateErrors = prometheus.NewCounter(prometheus.CounterOpts{
		Namespace: "nordstrom",
		Subsystem: "elwin",
		Name:      "update_errors",
		Help:      "The number of errors while updating storage.",
	})
)

const (
	cfgStorageAddr = "storage_address"
	cfgJSONAddr    = "json_address"
	cfgUInterval   = "update_interval"
	cfgRTimeout    = "read_timeout"
	cfgWTimeout    = "write_timeout"
	cfgITimeout    = "idle_timeout"
	cfgUFTimeout   = "update_fail_timeout"
	cfgProf        = "profiler"
)

func bind(s []string) error {
	if len(s) == 0 {
		return nil
	}
	if err := viper.BindEnv(s[0]); err != nil {
		return err
	}
	return bind(s[1:])
}

func main() {
	log.Println("Starting elwin...")

	viper.SetDefault(cfgStorageAddr, "elwin-storage:80")
	viper.SetDefault(cfgJSONAddr, ":8080")
	viper.SetDefault(cfgUInterval, "10s")
	viper.SetDefault(cfgRTimeout, "5s")
	viper.SetDefault(cfgWTimeout, "5s")
	viper.SetDefault(cfgITimeout, "30s")
	viper.SetDefault(cfgUFTimeout, "15m")

	viper.SetConfigName("config")
	viper.AddConfigPath(".")
	viper.AddConfigPath("/etc/elwin")
	err := viper.ReadInConfig()
	if err != nil {
		switch err.(type) {
		case viper.ConfigFileNotFoundError:
			log.Println("no config file found")
		default:
			log.Fatalf("could not read config: %v", err)
		}
	}

	viper.SetEnvPrefix("elwin")
	if err := bind([]string{
		cfgStorageAddr,
		cfgJSONAddr,
		cfgUInterval,
		cfgRTimeout,
		cfgWTimeout,
		cfgITimeout,
		cfgUFTimeout,
		"profiler",
	}); err != nil {
		log.Fatal(err)
	}

	// create elwin config
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	var interval, failTimeout time.Duration
	if ui, err := time.ParseDuration(viper.GetString(cfgUInterval)); err != nil {
		log.Fatal(err)
	} else if muft, err := time.ParseDuration(viper.GetString(cfgUFTimeout)); err != nil {
		log.Fatal(err)
	} else {
		interval, failTimeout = ui, muft
	}

	ec, err := choices.NewChoices(
		ctx,
		choices.WithGlobalSalt("choices"),
		choices.WithStorageConfig(viper.GetString(cfgStorageAddr), interval),
		choices.WithUpdateInterval(interval),
		choices.WithMaxUpdateFailTime(failTimeout),
	)
	if err != nil {
		log.Fatal(err)
	}

	e := &elwinServer{ec}

	// register prometheus metrics
	prometheus.MustRegister(updateErrors)
	prometheus.MustRegister(requestDurations)

	ljson, err := net.Listen("tcp", viper.GetString(cfgJSONAddr))
	if err != nil {
		log.Fatalf("could not listen on %s: %v", viper.GetString(cfgJSONAddr), err)
	}
	defer ljson.Close()
	log.Printf("Listening for json on %s", viper.GetString(cfgJSONAddr))

	mux := http.NewServeMux()
	mux.HandleFunc("/healthz", healthzHandler(map[string]interface{}{"storage": ec}))
	mux.HandleFunc("/readiness", healthzHandler(map[string]interface{}{"storage": ec}))
	mux.HandleFunc("/elwin/v1/experiments", e.json)
	mux.Handle("/metrics", promhttp.Handler())
	if viper.IsSet(cfgProf) {
		mux.HandleFunc("/debug/pprof/profile", pprof.Profile)
	}
	srv := http.Server{
		Handler: mux,
	}
	if rt, err := time.ParseDuration(viper.GetString(cfgRTimeout)); err != nil {
		log.Fatal(err)
	} else if wt, err := time.ParseDuration(viper.GetString(cfgWTimeout)); err != nil {
		log.Fatal(err)
	} else if it, err := time.ParseDuration(viper.GetString(cfgITimeout)); err != nil {
		log.Fatal(err)
	} else {
		srv.ReadTimeout = rt
		srv.WriteTimeout = wt
		srv.IdleTimeout = it
	}

	go func() {
		ec.ErrChan <- srv.Serve(ljson)
	}()

	signalChan := make(chan os.Signal, 1)
	signal.Notify(signalChan, syscall.SIGINT, syscall.SIGTERM)

	for {
		select {
		case err := <-ec.ErrChan:
			switch errors.Cause(err).(type) {
			case choices.ErrUpdateStorage:
				updateErrors.Inc()
			}
			log.Println(err)
		case s := <-signalChan:
			log.Printf("Captured %v. Exitting...", s)
			cancel()
			ctx, sdcancel := context.WithTimeout(context.Background(), 5*time.Second)
			defer sdcancel()
			srv.Shutdown(ctx)
			// send StatusServiceUnavailable to new requestors
			// block server from accepting new requests
			os.Exit(0)
		}
	}
}

type elwinServer struct {
	*choices.Config
}

func (e *elwinServer) Get(ctx context.Context, r *elwin.GetRequest) (*elwin.GetReply, error) {
	start := time.Now()
	defer func() {
		requestDurations.Observe(float64(time.Since(start)))
	}()
	selector := labels.NewSelector()
	for _, requirement := range r.Requirements {
		var op selection.Operator
		switch requirement.Op {
		case elwin.EXISTS:
			op = selection.Exists
		case elwin.EQUAL:
			op = selection.Equals
		case elwin.NOT_EQUAL:
			op = selection.NotEquals
		case elwin.IN:
			op = selection.In
		case elwin.NOT_IN:
			op = selection.NotIn
		default:
			return nil, errors.New("invalid operator in requirements")
		}
		req, err := labels.NewRequirement(requirement.Key, op, requirement.Values)
		if err != nil {
			return nil, errors.Wrap(err, "could not create requirement")
		}
		selector = selector.Add(*req)
	}

	// TODO: replace this with a pool or something similar
	resp := make([]*choices.ExperimentResponse, 0, 100)
	resp, err := e.Experiments(resp, r.UserID, selector)
	if err != nil {
		return nil, fmt.Errorf("error evaluating experiments for %s, %s: %v", r.Requirements, r.UserID, err)
	}

	//DEBUG
	/* log.Print(r) */

	if r.By != "" {
		byResp := make(map[string]*elwin.ExperimentList, 10)

		//DEBUG
		/*
			log.Print("#Selector: ", selector)
			log.Print("resp: ", resp)
			log.Print("byResp: ", byResp)
		*/

		for _, v := range resp {

			if v != nil {
				//DEBUG
				/*
					log.Print("#### Value of v == ", v.Labels)
					log.Print("#### Value of r.By == ", r.By)
					log.Print("#### Value of v.Labels[r.By] == ", v.Labels[r.By])
				*/

				if group, ok := v.Labels[r.By]; !ok {
					appendToGroup(byResp, v, "None")
				} else {
					appendToGroup(byResp, v, group)
				}
				//DEBUG
				//log.Print("#### Value of byResp: ", byResp)
			}
		}
		return &elwin.GetReply{
			Group: byResp,
		}, nil
	}

	exp := &elwin.GetReply{
		Experiments: make([]*elwin.Experiment, len(resp)),
	}

	for i, v := range resp {
		exp.Experiments[i] = &elwin.Experiment{
			Name:      v.Name,
			Namespace: v.Namespace,
			Labels:    v.Labels,
			Params:    make([]*elwin.Param, len(v.Params)),
		}

		for j, p := range v.Params {
			exp.Experiments[i].Params[j] = &elwin.Param{
				Name:  p.Name,
				Value: p.Value,
			}
		}
	}
	return exp, nil
}

func appendToGroup(br map[string]*elwin.ExperimentList, e *choices.ExperimentResponse, group string) {
	if br[group] == nil {
		br[group] = &elwin.ExperimentList{}
	}
	elist := br[group].Experiments
	ee := &elwin.Experiment{
		Name:      e.Name,
		Namespace: e.Namespace,
		Labels:    e.Labels,
		Params:    make([]*elwin.Param, len(e.Params)),
	}
	for i, p := range e.Params {
		ee.Params[i] = &elwin.Param{
			Name:  p.Name,
			Value: p.Value,
		}
	}
	elist = append(elist, ee)
	br[group].Experiments = elist
}

func (e *elwinServer) json(w http.ResponseWriter, r *http.Request) {
	if r.Method != "POST" {
		http.Error(w, "Method Not Allowed", http.StatusBadRequest)
		return
	}
	defer r.Body.Close()
	dec := json.NewDecoder(r.Body)
	data := new(elwin.GetRequest)
	if err := dec.Decode(data); err != nil {
		http.Error(w, "could not parse json", http.StatusBadRequest)
		return
	}

	ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second)
	defer cancel()
	resp, err := e.Get(ctx, data)
	if err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}
	enc := json.NewEncoder(w)
	if err := enc.Encode(resp); err != nil {
		http.Error(w, "could not marshal json", http.StatusInternalServerError)
		return
	}
}

func logCloseErr(c io.Closer) {
	if err := c.Close(); err != nil {
		log.Printf("could not close response body: %s", err)
	}
}

func healthzHandler(healthChecks map[string]interface{}) http.HandlerFunc {
	type healthy interface {
		IsHealthy() error
	}

	return func(w http.ResponseWriter, r *http.Request) {
		errs := make(map[string]string, len(healthChecks))
		for key, healthChecker := range healthChecks {
			if hc, ok := healthChecker.(healthy); ok {
				err := hc.IsHealthy()
				if err != nil {
					errs[key] = err.Error()
				}
			}
		}
		if len(errs) != 0 {
			w.WriteHeader(http.StatusInternalServerError)
			w.Header().Set("Content-Type", "application/json")
			enc := json.NewEncoder(w)
			if err := enc.Encode(errs); err != nil {
				http.Error(w, err.Error(), http.StatusInternalServerError)
				return
			}
			return
		}
		w.WriteHeader(http.StatusOK)
		w.Header().Set("Content-Type", "text/plain")
		if _, err := w.Write([]byte("OK")); err != nil {
			log.Printf("could not write to healthz connection: %s", err)
		}
	}
}
<commit_msg>adding in header to response to return json<commit_after>// Copyright 2016 Andrew O'Neill, Nordstrom

// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at

//     http://www.apache.org/licenses/LICENSE-2.0

// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"encoding/json"
	"fmt"
	"io"
	"log"
	"net"
	"net/http"
	"net/http/pprof"
	"os"
	"os/signal"
	"syscall"
	"time"

	"github.com/Nordstrom/choices"
	"github.com/foolusion/elwinprotos/elwin"
	"github.com/pkg/errors"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promhttp"
	"github.com/spf13/viper"
	"golang.org/x/net/context"
	"k8s.io/apimachinery/pkg/labels"
	"k8s.io/apimachinery/pkg/selection"
)

var (
	requestDurations = prometheus.NewHistogram(prometheus.HistogramOpts{
		Namespace: "nordstrom",
		Subsystem: "elwin",
		Name:      "request_durations_nanoseconds",
		Help:      "request latency distributions.",
		Buckets:   prometheus.ExponentialBuckets(1, 10, 10),
	})
	updateErrors = prometheus.NewCounter(prometheus.CounterOpts{
		Namespace: "nordstrom",
		Subsystem: "elwin",
		Name:      "update_errors",
		Help:      "The number of errors while updating storage.",
	})
)

const (
	cfgStorageAddr = "storage_address"
	cfgJSONAddr    = "json_address"
	cfgUInterval   = "update_interval"
	cfgRTimeout    = "read_timeout"
	cfgWTimeout    = "write_timeout"
	cfgITimeout    = "idle_timeout"
	cfgUFTimeout   = "update_fail_timeout"
	cfgProf        = "profiler"
)

func bind(s []string) error {
	if len(s) == 0 {
		return nil
	}
	if err := viper.BindEnv(s[0]); err != nil {
		return err
	}
	return bind(s[1:])
}

func main() {
	log.Println("Starting elwin...")

	viper.SetDefault(cfgStorageAddr, "elwin-storage:80")
	viper.SetDefault(cfgJSONAddr, ":8080")
	viper.SetDefault(cfgUInterval, "10s")
	viper.SetDefault(cfgRTimeout, "5s")
	viper.SetDefault(cfgWTimeout, "5s")
	viper.SetDefault(cfgITimeout, "30s")
	viper.SetDefault(cfgUFTimeout, "15m")

	viper.SetConfigName("config")
	viper.AddConfigPath(".")
	viper.AddConfigPath("/etc/elwin")
	err := viper.ReadInConfig()
	if err != nil {
		switch err.(type) {
		case viper.ConfigFileNotFoundError:
			log.Println("no config file found")
		default:
			log.Fatalf("could not read config: %v", err)
		}
	}

	viper.SetEnvPrefix("elwin")
	if err := bind([]string{
		cfgStorageAddr,
		cfgJSONAddr,
		cfgUInterval,
		cfgRTimeout,
		cfgWTimeout,
		cfgITimeout,
		cfgUFTimeout,
		"profiler",
	}); err != nil {
		log.Fatal(err)
	}

	// create elwin config
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	var interval, failTimeout time.Duration
	if ui, err := time.ParseDuration(viper.GetString(cfgUInterval)); err != nil {
		log.Fatal(err)
	} else if muft, err := time.ParseDuration(viper.GetString(cfgUFTimeout)); err != nil {
		log.Fatal(err)
	} else {
		interval, failTimeout = ui, muft
	}

	ec, err := choices.NewChoices(
		ctx,
		choices.WithGlobalSalt("choices"),
		choices.WithStorageConfig(viper.GetString(cfgStorageAddr), interval),
		choices.WithUpdateInterval(interval),
		choices.WithMaxUpdateFailTime(failTimeout),
	)
	if err != nil {
		log.Fatal(err)
	}

	e := &elwinServer{ec}

	// register prometheus metrics
	prometheus.MustRegister(updateErrors)
	prometheus.MustRegister(requestDurations)

	ljson, err := net.Listen("tcp", viper.GetString(cfgJSONAddr))
	if err != nil {
		log.Fatalf("could not listen on %s: %v", viper.GetString(cfgJSONAddr), err)
	}
	defer ljson.Close()
	log.Printf("Listening for json on %s", viper.GetString(cfgJSONAddr))

	mux := http.NewServeMux()
	mux.HandleFunc("/healthz", healthzHandler(map[string]interface{}{"storage": ec}))
	mux.HandleFunc("/readiness", healthzHandler(map[string]interface{}{"storage": ec}))
	mux.HandleFunc("/elwin/v1/experiments", e.json)
	mux.Handle("/metrics", promhttp.Handler())
	if viper.IsSet(cfgProf) {
		mux.HandleFunc("/debug/pprof/profile", pprof.Profile)
	}
	srv := http.Server{
		Handler: mux,
	}
	if rt, err := time.ParseDuration(viper.GetString(cfgRTimeout)); err != nil {
		log.Fatal(err)
	} else if wt, err := time.ParseDuration(viper.GetString(cfgWTimeout)); err != nil {
		log.Fatal(err)
	} else if it, err := time.ParseDuration(viper.GetString(cfgITimeout)); err != nil {
		log.Fatal(err)
	} else {
		srv.ReadTimeout = rt
		srv.WriteTimeout = wt
		srv.IdleTimeout = it
	}

	go func() {
		ec.ErrChan <- srv.Serve(ljson)
	}()

	signalChan := make(chan os.Signal, 1)
	signal.Notify(signalChan, syscall.SIGINT, syscall.SIGTERM)

	for {
		select {
		case err := <-ec.ErrChan:
			switch errors.Cause(err).(type) {
			case choices.ErrUpdateStorage:
				updateErrors.Inc()
			}
			log.Println(err)
		case s := <-signalChan:
			log.Printf("Captured %v. Exitting...", s)
			cancel()
			ctx, sdcancel := context.WithTimeout(context.Background(), 5*time.Second)
			defer sdcancel()
			srv.Shutdown(ctx)
			// send StatusServiceUnavailable to new requestors
			// block server from accepting new requests
			os.Exit(0)
		}
	}
}

type elwinServer struct {
	*choices.Config
}

func (e *elwinServer) Get(ctx context.Context, r *elwin.GetRequest) (*elwin.GetReply, error) {
	start := time.Now()
	defer func() {
		requestDurations.Observe(float64(time.Since(start)))
	}()
	selector := labels.NewSelector()
	for _, requirement := range r.Requirements {
		var op selection.Operator
		switch requirement.Op {
		case elwin.EXISTS:
			op = selection.Exists
		case elwin.EQUAL:
			op = selection.Equals
		case elwin.NOT_EQUAL:
			op = selection.NotEquals
		case elwin.IN:
			op = selection.In
		case elwin.NOT_IN:
			op = selection.NotIn
		default:
			return nil, errors.New("invalid operator in requirements")
		}
		req, err := labels.NewRequirement(requirement.Key, op, requirement.Values)
		if err != nil {
			return nil, errors.Wrap(err, "could not create requirement")
		}
		selector = selector.Add(*req)
	}

	// TODO: replace this with a pool or something similar
	resp := make([]*choices.ExperimentResponse, 0, 100)
	resp, err := e.Experiments(resp, r.UserID, selector)
	if err != nil {
		return nil, fmt.Errorf("error evaluating experiments for %s, %s: %v", r.Requirements, r.UserID, err)
	}

	//DEBUG
	/* log.Print(r) */

	if r.By != "" {
		byResp := make(map[string]*elwin.ExperimentList, 10)

		//DEBUG
		/*
			log.Print("#Selector: ", selector)
			log.Print("resp: ", resp)
			log.Print("byResp: ", byResp)
		*/

		for _, v := range resp {

			if v != nil {
				//DEBUG
				/*
					log.Print("#### Value of v == ", v.Labels)
					log.Print("#### Value of r.By == ", r.By)
					log.Print("#### Value of v.Labels[r.By] == ", v.Labels[r.By])
				*/

				if group, ok := v.Labels[r.By]; !ok {
					appendToGroup(byResp, v, "None")
				} else {
					appendToGroup(byResp, v, group)
				}
				//DEBUG
				//log.Print("#### Value of byResp: ", byResp)
			}
		}
		return &elwin.GetReply{
			Group: byResp,
		}, nil
	}

	exp := &elwin.GetReply{
		Experiments: make([]*elwin.Experiment, len(resp)),
	}

	for i, v := range resp {
		exp.Experiments[i] = &elwin.Experiment{
			Name:      v.Name,
			Namespace: v.Namespace,
			Labels:    v.Labels,
			Params:    make([]*elwin.Param, len(v.Params)),
		}

		for j, p := range v.Params {
			exp.Experiments[i].Params[j] = &elwin.Param{
				Name:  p.Name,
				Value: p.Value,
			}
		}
	}
	return exp, nil
}

func appendToGroup(br map[string]*elwin.ExperimentList, e *choices.ExperimentResponse, group string) {
	if br[group] == nil {
		br[group] = &elwin.ExperimentList{}
	}
	elist := br[group].Experiments
	ee := &elwin.Experiment{
		Name:      e.Name,
		Namespace: e.Namespace,
		Labels:    e.Labels,
		Params:    make([]*elwin.Param, len(e.Params)),
	}
	for i, p := range e.Params {
		ee.Params[i] = &elwin.Param{
			Name:  p.Name,
			Value: p.Value,
		}
	}
	elist = append(elist, ee)
	br[group].Experiments = elist
}

func (e *elwinServer) json(w http.ResponseWriter, r *http.Request) {
	if r.Method != "POST" {
		http.Error(w, "Method Not Allowed", http.StatusBadRequest)
		return
	}
	defer r.Body.Close()
	dec := json.NewDecoder(r.Body)
	data := new(elwin.GetRequest)
	if err := dec.Decode(data); err != nil {
		http.Error(w, "could not parse json", http.StatusBadRequest)
		return
	}

	ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second)
	defer cancel()
	resp, err := e.Get(ctx, data)
	if err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}
	enc := json.NewEncoder(w)
	w.Header().Set("Content-Type", "application/json")
	if err := enc.Encode(resp); err != nil {
		http.Error(w, "could not marshal json", http.StatusInternalServerError)
		return
	}
}

func logCloseErr(c io.Closer) {
	if err := c.Close(); err != nil {
		log.Printf("could not close response body: %s", err)
	}
}

func healthzHandler(healthChecks map[string]interface{}) http.HandlerFunc {
	type healthy interface {
		IsHealthy() error
	}

	return func(w http.ResponseWriter, r *http.Request) {
		errs := make(map[string]string, len(healthChecks))
		for key, healthChecker := range healthChecks {
			if hc, ok := healthChecker.(healthy); ok {
				err := hc.IsHealthy()
				if err != nil {
					errs[key] = err.Error()
				}
			}
		}
		if len(errs) != 0 {
			w.WriteHeader(http.StatusInternalServerError)
			w.Header().Set("Content-Type", "application/json")
			enc := json.NewEncoder(w)
			if err := enc.Encode(errs); err != nil {
				http.Error(w, err.Error(), http.StatusInternalServerError)
				return
			}
			return
		}
		w.WriteHeader(http.StatusOK)
		w.Header().Set("Content-Type", "text/plain")
		if _, err := w.Write([]byte("OK")); err != nil {
			log.Printf("could not write to healthz connection: %s", err)
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"go/build"
	"io"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strconv"
	"syscall"

	"github.com/DATA-DOG/godog"
	"github.com/DATA-DOG/godog/colors"
)

var statusMatch = regexp.MustCompile("^exit status (\\d+)")
var parsedStatus int

func buildAndRun() (int, error) {
	var status int

	bin, err := filepath.Abs("godog.test")
	if err != nil {
		return 1, err
	}
	if build.Default.GOOS == "windows" {
		bin += ".exe"
	}
	if err = godog.Build(bin); err != nil {
		return 1, err
	}
	defer os.Remove(bin)

	cmd := exec.Command(bin, os.Args[1:]...)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	cmd.Env = os.Environ()

	if err = cmd.Start(); err != nil {
		return status, err
	}

	if err = cmd.Wait(); err != nil {
		if exiterr, ok := err.(*exec.ExitError); ok {
			// The program has exited with an exit code != 0
			status = 1

			// This works on both Unix and Windows. Although package
			// syscall is generally platform dependent, WaitStatus is
			// defined for both Unix and Windows and in both cases has
			// an ExitStatus() method with the same signature.
			if st, ok := exiterr.Sys().(syscall.WaitStatus); ok {
				status = st.ExitStatus()
			}
			return status, nil
		}
		return status, err
	}
	return status, nil
}

func main() {
	var vers bool
	var output string

	opt := godog.Options{Output: colors.Colored(os.Stdout)}
	flagSet := godog.FlagSet(&opt)
	flagSet.BoolVar(&vers, "version", false, "Show current version.")
	flagSet.StringVar(&output, "o", "", "Build and output test runner executable to given target path.")
	flagSet.StringVar(&output, "output", "", "Build and output test runner executable to given target path.")

	if err := flagSet.Parse(os.Args[1:]); err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}

	if len(output) > 0 {
		bin, err := filepath.Abs(output)
		if err != nil {
			fmt.Fprintln(os.Stderr, "could not locate absolute path for:", output, err)
			os.Exit(1)
		}
		if err = godog.Build(bin); err != nil {
			fmt.Fprintln(os.Stderr, "could not build binary at:", output, err)
			os.Exit(1)
		}
		os.Exit(0)
	}

	if vers {
		fmt.Fprintln(os.Stdout, "Godog version is:", godog.Version)
		os.Exit(0) // should it be 0?
	}

	status, err := buildAndRun()
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
	// it might be a case, that status might not be resolved
	// in some OSes. this is attempt to parse it from stderr
	if parsedStatus > status {
		status = parsedStatus
	}
	os.Exit(status)
}

func statusOutputFilter(w io.Writer) io.Writer {
	return writerFunc(func(b []byte) (int, error) {
		if m := statusMatch.FindStringSubmatch(string(b)); len(m) > 1 {
			parsedStatus, _ = strconv.Atoi(m[1])
			// skip status stderr output
			return len(b), nil
		}
		return w.Write(b)
	})
}

type writerFunc func([]byte) (int, error)

func (w writerFunc) Write(b []byte) (int, error) {
	return w(b)
}
<commit_msg>Fixed: assigned os.Stdin to cmd.Stdin when launching godog<commit_after>package main

import (
	"fmt"
	"go/build"
	"io"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strconv"
	"syscall"

	"github.com/DATA-DOG/godog"
	"github.com/DATA-DOG/godog/colors"
)

var statusMatch = regexp.MustCompile("^exit status (\\d+)")
var parsedStatus int

func buildAndRun() (int, error) {
	var status int

	bin, err := filepath.Abs("godog.test")
	if err != nil {
		return 1, err
	}
	if build.Default.GOOS == "windows" {
		bin += ".exe"
	}
	if err = godog.Build(bin); err != nil {
		return 1, err
	}
	defer os.Remove(bin)

	cmd := exec.Command(bin, os.Args[1:]...)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	cmd.Stdin = os.Stdin
	cmd.Env = os.Environ()

	if err = cmd.Start(); err != nil {
		return status, err
	}

	if err = cmd.Wait(); err != nil {
		if exiterr, ok := err.(*exec.ExitError); ok {
			// The program has exited with an exit code != 0
			status = 1

			// This works on both Unix and Windows. Although package
			// syscall is generally platform dependent, WaitStatus is
			// defined for both Unix and Windows and in both cases has
			// an ExitStatus() method with the same signature.
			if st, ok := exiterr.Sys().(syscall.WaitStatus); ok {
				status = st.ExitStatus()
			}
			return status, nil
		}
		return status, err
	}
	return status, nil
}

func main() {
	var vers bool
	var output string

	opt := godog.Options{Output: colors.Colored(os.Stdout)}
	flagSet := godog.FlagSet(&opt)
	flagSet.BoolVar(&vers, "version", false, "Show current version.")
	flagSet.StringVar(&output, "o", "", "Build and output test runner executable to given target path.")
	flagSet.StringVar(&output, "output", "", "Build and output test runner executable to given target path.")

	if err := flagSet.Parse(os.Args[1:]); err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}

	if len(output) > 0 {
		bin, err := filepath.Abs(output)
		if err != nil {
			fmt.Fprintln(os.Stderr, "could not locate absolute path for:", output, err)
			os.Exit(1)
		}
		if err = godog.Build(bin); err != nil {
			fmt.Fprintln(os.Stderr, "could not build binary at:", output, err)
			os.Exit(1)
		}
		os.Exit(0)
	}

	if vers {
		fmt.Fprintln(os.Stdout, "Godog version is:", godog.Version)
		os.Exit(0) // should it be 0?
	}

	status, err := buildAndRun()
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
	// it might be a case, that status might not be resolved
	// in some OSes. this is attempt to parse it from stderr
	if parsedStatus > status {
		status = parsedStatus
	}
	os.Exit(status)
}

func statusOutputFilter(w io.Writer) io.Writer {
	return writerFunc(func(b []byte) (int, error) {
		if m := statusMatch.FindStringSubmatch(string(b)); len(m) > 1 {
			parsedStatus, _ = strconv.Atoi(m[1])
			// skip status stderr output
			return len(b), nil
		}
		return w.Write(b)
	})
}

type writerFunc func([]byte) (int, error)

func (w writerFunc) Write(b []byte) (int, error) {
	return w(b)
}
<|endoftext|>


<commit_before>// Copyright 2014 Volker Dobler.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"fmt"

	"github.com/vdobler/ht/ht"
)

var cmdVersion = &Command{
	Run:         runVersion,
	Usage:       "version",
	Description: "print version information",
	Help: `
Version prints version information about ht.
	`,
}

var (
	version = "0.2beta"
)

func runVersion(cmd *Command, suites []*ht.Suite) {
	fmt.Printf("ht version %s\n", version)
}
<commit_msg>cmd/ht: fix version display<commit_after>// Copyright 2014 Volker Dobler.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"flag"
	"fmt"

	"github.com/vdobler/ht/ht"
)

var cmdVersion = &Command{
	Run:         runVersion,
	Usage:       "version",
	Description: "print version information",
	Flag:        flag.NewFlagSet("version", flag.ContinueOnError),
	Help: `
Version prints version information about ht.
	`,
}

var (
	version = "0.2beta"
)

func runVersion(cmd *Command, suites []*ht.Suite) {
	fmt.Printf("ht version %s\n", version)
}
<|endoftext|>