<commit_before>package cwl

import (
	//"errors"
	"fmt"
	"path"
	"strings"

	//"github.com/davecgh/go-spew/spew"
	"reflect"

	"github.com/davecgh/go-spew/spew"
	"github.com/mitchellh/mapstructure"
)

// http://www.commonwl.org/v1.0/CommandLineTool.html#CommandLineTool
type CommandLineTool struct {
	//Id                 string                   `yaml:"id,omitempty" bson:"id,omitempty" json:"id,omitempty"`
	//Class              string                   `yaml:"class,omitempty" bson:"class,omitempty" json:"class,omitempty"`
	CWLObjectImpl      `yaml:",inline" json:",inline" bson:",inline" mapstructure:",squash"`
	CWL_class_Impl     `yaml:",inline" json:",inline" bson:",inline" mapstructure:",squash"`
	IdentifierImpl     `yaml:",inline" json:",inline" bson:",inline" mapstructure:",squash"`
	ProcessImpl        `yaml:",inline" bson:",inline" json:",inline" mapstructure:",squash"`
	BaseCommand        []string                 `yaml:"baseCommand,omitempty" bson:"baseCommand,omitempty" json:"baseCommand,omitempty" mapstructure:"baseCommand,omitempty"`
	Inputs             []CommandInputParameter  `yaml:"inputs" bson:"inputs" json:"inputs" mapstructure:"inputs"`
	Outputs            []CommandOutputParameter `yaml:"outputs" bson:"outputs" json:"outputs" mapstructure:"outputs"`
	Hints              []Requirement            `yaml:"hints,omitempty" bson:"hints,omitempty" json:"hints,omitempty" mapstructure:"hints,omitempty"`
	Requirements       []Requirement            `yaml:"requirements,omitempty" bson:"requirements,omitempty" json:"requirements,omitempty" mapstructure:"requirements,omitempty"`
	Doc                string                   `yaml:"doc,omitempty" bson:"doc,omitempty" json:"doc,omitempty" mapstructure:"doc,omitempty"`
	Label              string                   `yaml:"label,omitempty" bson:"label,omitempty" json:"label,omitempty" mapstructure:"label,omitempty"`
	Description        string                   `yaml:"description,omitempty" bson:"description,omitempty" json:"description,omitempty" mapstructure:"description,omitempty"`
	CwlVersion         CWLVersion               `yaml:"cwlVersion,omitempty" bson:"cwlVersion,omitempty" json:"cwlVersion,omitempty" mapstructure:"cwlVersion,omitempty"`
	Arguments          []CommandLineBinding     `yaml:"arguments,omitempty" bson:"arguments,omitempty" json:"arguments,omitempty" mapstructure:"arguments,omitempty"`
	Stdin              string                   `yaml:"stdin,omitempty" bson:"stdin,omitempty" json:"stdin,omitempty" mapstructure:"stdin,omitempty"`     // TODO support Expression
	Stderr             string                   `yaml:"stderr,omitempty" bson:"stderr,omitempty" json:"stderr,omitempty" mapstructure:"stderr,omitempty"` // TODO support Expression
	Stdout             string                   `yaml:"stdout,omitempty" bson:"stdout,omitempty" json:"stdout,omitempty" mapstructure:"stdout,omitempty"` // TODO support Expression
	SuccessCodes       []int                    `yaml:"successCodes,omitempty" bson:"successCodes,omitempty" json:"successCodes,omitempty" mapstructure:"successCodes,omitempty"`
	TemporaryFailCodes []int                    `yaml:"temporaryFailCodes,omitempty" bson:"temporaryFailCodes,omitempty" json:"temporaryFailCodes,omitempty" mapstructure:"temporaryFailCodes,omitempty"`
	PermanentFailCodes []int                    `yaml:"permanentFailCodes,omitempty" bson:"permanentFailCodes,omitempty" json:"permanentFailCodes,omitempty" mapstructure:"permanentFailCodes,omitempty"`
	Namespaces         map[string]string        `yaml:"$namespaces,omitempty" bson:"_DOLLAR_namespaces,omitempty" json:"$namespaces,omitempty" mapstructure:"$namespaces,omitempty"`
}

// IsCWLMinimal _
func (c *CommandLineTool) IsCWLMinimal() {}

// IsProcess _
func (c *CommandLineTool) IsProcess() {}

// keyname will be converted into 'Id'-field

// NewCommandLineTool _
// parentIdentifier is used to convert relative id to absolute id
// objectIdentifier is used when there is no local is, in case of file or embedded tool
func NewCommandLineTool(generic interface{}, parentIdentifier string, objectIdentifier string, injectedRequirements []Requirement, context *WorkflowContext) (commandLineTool *CommandLineTool, schemata []CWLType_Type, err error) {

	//fmt.Println("NewCommandLineTool() generic:")
	//spew.Dump(generic)

	//switch type()
	object, ok := generic.(map[string]interface{})
	if !ok {
		err = fmt.Errorf("other types than map[string]interface{} not supported yet (got %s)", reflect.TypeOf(generic))
		return
	}
	//spew.Dump(generic)

	//fmt.Println("NewCommandLineTool() object:")
	//spew.Dump(object)

	commandLineTool = &CommandLineTool{}
	commandLineTool.Class = "CommandLineTool"

	//fmt.Printf("(NewCommandLineTool) requirements %d\n", len(requirements_array))
	//spew.Dump(requirements_array)
	//scs := spew.ConfigState{Indent: "\t"}
	//scs.Dump(object["requirements"])

	if objectIdentifier != "" {
		object["id"] = objectIdentifier
	} else {

		objectIDIf, hasID := object["id"]
		if hasID {

			objectID := objectIDIf.(string)

			if !strings.HasPrefix(objectID, "#") {
				if parentIdentifier == "" {
					err = fmt.Errorf("(NewCommandLineTool) parentIdentifier is needed but empty, objectID=%s", objectID)
					return
				}
				objectID = path.Join(parentIdentifier, objectID)
			}

			if !strings.HasPrefix(objectID, "#") {
				err = fmt.Errorf("(NewCommandLineTool) not absolute: objectID=%s , parentIdentifier=%s, objectIdentifier=%s", objectID, parentIdentifier, objectIdentifier)
				return
			}
			object["id"] = objectID
		}
	}

	// extract SchemaDefRequirement
	var schemaDefReq *SchemaDefRequirement
	//var schemataNew []CWLType_Type
	hasSchemaDefReq := false

	requirementsIf, hasRequirements := object["requirements"]
	if hasRequirements {
		schemaDefReq, hasSchemaDefReq, err = GetSchemaDefRequirement(requirementsIf, context)
		if err != nil {
			err = fmt.Errorf("(NewCommandLineTool) GetSchemaDefRequirement returned: %s", err.Error())
			return
		}
	}

	//if has_schema_def_req {
	//	for i, _ := range schemataNew {
	//		schemata = append(schemata, schemataNew[i])
	//	}
	//}

	inputs := []*CommandInputParameter{}

	inputsIf, ok := object["inputs"]
	if ok {
		// Convert map of inputs into array of inputs
		err, inputs = CreateCommandInputArray(inputsIf, schemata, context)
		if err != nil {
			err = fmt.Errorf("(NewCommandLineTool) error in CreateCommandInputArray: %s", err.Error())
			return
		}
	}

	object["inputs"] = inputs

	var copa []interface{}

	outputs, hasOutputs := object["outputs"]
	if hasOutputs {

		//fmt.Println("NewCommandLineTool() object/outputs:")
		//spew.Dump(outputs)

		// Convert map of outputs into array of outputs

		copa, err = NewCommandOutputParameterArray(outputs, schemata, context)
		if err != nil {
			//fmt.Println("NewCommandLineTool after error")
			//spew.Dump(object)

			err = fmt.Errorf("(NewCommandLineTool) error in NewCommandOutputParameterArray: %s", err.Error())
			return
		}
		object["outputs"] = copa
	} else {
		err = fmt.Errorf("(NewCommandLineTool) no outputs !?")
		return
		//object["outputs"] = []*CommandOutputParameter{}
	}

	baseCommand, ok := object["baseCommand"]
	if ok {
		object["baseCommand"], err = NewBaseCommandArray(baseCommand)
		if err != nil {
			err = fmt.Errorf("(NewCommandLineTool) error in NewBaseCommandArray: %s", err.Error())
			return
		}
	}

	arguments, hasArguments := object["arguments"]
	if hasArguments {
		// Convert map of outputs into array of outputs
		var argumentsObject []CommandLineBinding
		argumentsObject, err = NewCommandLineBindingArray(arguments, context)
		if err != nil {
			err = fmt.Errorf("(NewCommandLineTool) error in NewCommandLineBindingArray: %s", err.Error())
			return
		}
		//delete(object, "arguments")
		object["arguments"] = argumentsObject
	}

	if hasSchemaDefReq {
		injectedRequirements = append(injectedRequirements, schemaDefReq)
	}

	var requirementsArray []Requirement

	//fmt.Printf("(NewCommandLineTool) Injecting %d\n", len(requirementsArray))
	//spew.Dump(requirementsArray)
	requirementsArray, err = CreateRequirementArrayAndInject(requirementsIf, injectedRequirements, nil, context)
	if err != nil {
		err = fmt.Errorf("(NewCommandLineTool) error in CreateRequirementArray (requirements): %s", err.Error())
		return
	}

	//for i, _ := range schemataNew {
	//	schemata = append(schemata, schemataNew[i])
	//}

	object["requirements"] = requirementsArray

	hints, ok := object["hints"]
	if ok && (hints != nil) {
		//var schemataNew []CWLType_Type

		var hintsArray []Requirement
		hintsArray, err = CreateHintsArray(hints, injectedRequirements, inputs, context)
		if err != nil {
			err = fmt.Errorf("(NewCommandLineTool) error in CreateRequirementArray (hints): %s", err.Error())
			return
		}
		//for i, _ := range schemataNew {
		//	schemata = append(schemata, schemataNew[i])
		//}
		object["hints"] = hintsArray
	}

	err = mapstructure.Decode(object, commandLineTool)
	if err != nil {
		err = fmt.Errorf("(NewCommandLineTool) error parsing CommandLineTool class: %s", err.Error())
		return
	}

	if commandLineTool.ID == "" {
		err = fmt.Errorf("(NewCommandLineTool) id is empty!?")
		return
	}

	if !strings.HasPrefix(commandLineTool.ID, "#") {
		err = fmt.Errorf("(NewCommandLineTool) id is not absolute!?")
		return
	}

	//fmt.Println("commandLineTool:")
	//spew.Dump(commandLineTool)

	if commandLineTool.CwlVersion == "" {
		commandLineTool.CwlVersion = context.CwlVersion
	}

	if context == nil {
		err = fmt.Errorf("(NewCommandLineTool) context == nil")
		return
	}

	if context.Namespaces != nil {
		commandLineTool.Namespaces = context.Namespaces
	}

	if context != nil {

		thisID := commandLineTool.ID
		if thisID == "" {
			err = fmt.Errorf("(NewCommandLineTool) did not expect empty Id")
			return
		}

		// err = context.Add(commandLineTool.Id, commandLineTool, "NewCommandLineTool")
		// if err != nil {
		// 	err = fmt.Errorf("(NewCommandLineTool) (add commandLineTool) context.Add returned: %s", err.Error())
		// 	return
		// }

		// for i := range commandLineTool.Inputs {
		// 	inp := &commandLineTool.Inputs[i]
		// 	inpID := path.Join(thisID, inp.Id)

		// 	err = context.AddObject(inpID, inp, "NewCommandLineTool")
		// 	if err != nil {
		// 		err = fmt.Errorf("(NewCommandLineTool) X (add commandLineToolInput)  context.Add returned: %s", err.Error())
		// 		return
		// 	}
		// }
	}
	return

}

// NewBaseCommandArray _
func NewBaseCommandArray(original interface{}) (newArray []string, err error) {
	newArray = []string{}
	switch original.(type) {
	case []interface{}:
		originalArray := original.([]interface{})
		for _, v := range originalArray {

			vStr, ok := v.(string)
			if !ok {
				spew.Dump(originalArray)
				err = fmt.Errorf("(NewBaseCommandArray) []interface{} array element is not a string, it is %s", reflect.TypeOf(v))
				return
			}
			newArray = append(newArray, vStr)
		}

		return
	case string:
		orgStr, _ := original.(string)
		newArray = append(newArray, orgStr)
		return
	default:
		err = fmt.Errorf("(NewBaseCommandArray) type unknown")

	}
	return
}

// Evaluate _
func (c *CommandLineTool) Evaluate(inputs interface{}, context *WorkflowContext) (err error) {

	for i := range c.Requirements {

		r := c.Requirements[i]

		err = r.Evaluate(inputs, context)
		if err != nil {
			err = fmt.Errorf("(CommandLineTool/Evaluate) Requirements r.Evaluate returned: %s", err.Error())
			return
		}

	}

	for i := range c.Hints {

		r := c.Hints[i]

		err = r.Evaluate(inputs, context)
		if err != nil {
			err = fmt.Errorf("(CommandLineTool/Evaluate) Hints r.Evaluate returned: %s", err.Error())
			return
		}

	}

	return
}
<commit_msg>fix commandlinetool ID parsing<commit_after>package cwl

import (
	//"errors"
	"fmt"
	"path"
	"strings"

	//"github.com/davecgh/go-spew/spew"
	"reflect"

	uuid "github.com/MG-RAST/golib/go-uuid/uuid"
	"github.com/davecgh/go-spew/spew"
	"github.com/mitchellh/mapstructure"
)

// http://www.commonwl.org/v1.0/CommandLineTool.html#CommandLineTool
type CommandLineTool struct {
	//Id                 string                   `yaml:"id,omitempty" bson:"id,omitempty" json:"id,omitempty"`
	//Class              string                   `yaml:"class,omitempty" bson:"class,omitempty" json:"class,omitempty"`
	CWLObjectImpl      `yaml:",inline" json:",inline" bson:",inline" mapstructure:",squash"`
	CWL_class_Impl     `yaml:",inline" json:",inline" bson:",inline" mapstructure:",squash"`
	IdentifierImpl     `yaml:",inline" json:",inline" bson:",inline" mapstructure:",squash"`
	ProcessImpl        `yaml:",inline" bson:",inline" json:",inline" mapstructure:",squash"`
	BaseCommand        []string                 `yaml:"baseCommand,omitempty" bson:"baseCommand,omitempty" json:"baseCommand,omitempty" mapstructure:"baseCommand,omitempty"`
	Inputs             []CommandInputParameter  `yaml:"inputs" bson:"inputs" json:"inputs" mapstructure:"inputs"`
	Outputs            []CommandOutputParameter `yaml:"outputs" bson:"outputs" json:"outputs" mapstructure:"outputs"`
	Hints              []Requirement            `yaml:"hints,omitempty" bson:"hints,omitempty" json:"hints,omitempty" mapstructure:"hints,omitempty"`
	Requirements       []Requirement            `yaml:"requirements,omitempty" bson:"requirements,omitempty" json:"requirements,omitempty" mapstructure:"requirements,omitempty"`
	Doc                string                   `yaml:"doc,omitempty" bson:"doc,omitempty" json:"doc,omitempty" mapstructure:"doc,omitempty"`
	Label              string                   `yaml:"label,omitempty" bson:"label,omitempty" json:"label,omitempty" mapstructure:"label,omitempty"`
	Description        string                   `yaml:"description,omitempty" bson:"description,omitempty" json:"description,omitempty" mapstructure:"description,omitempty"`
	CwlVersion         CWLVersion               `yaml:"cwlVersion,omitempty" bson:"cwlVersion,omitempty" json:"cwlVersion,omitempty" mapstructure:"cwlVersion,omitempty"`
	Arguments          []CommandLineBinding     `yaml:"arguments,omitempty" bson:"arguments,omitempty" json:"arguments,omitempty" mapstructure:"arguments,omitempty"`
	Stdin              string                   `yaml:"stdin,omitempty" bson:"stdin,omitempty" json:"stdin,omitempty" mapstructure:"stdin,omitempty"`     // TODO support Expression
	Stderr             string                   `yaml:"stderr,omitempty" bson:"stderr,omitempty" json:"stderr,omitempty" mapstructure:"stderr,omitempty"` // TODO support Expression
	Stdout             string                   `yaml:"stdout,omitempty" bson:"stdout,omitempty" json:"stdout,omitempty" mapstructure:"stdout,omitempty"` // TODO support Expression
	SuccessCodes       []int                    `yaml:"successCodes,omitempty" bson:"successCodes,omitempty" json:"successCodes,omitempty" mapstructure:"successCodes,omitempty"`
	TemporaryFailCodes []int                    `yaml:"temporaryFailCodes,omitempty" bson:"temporaryFailCodes,omitempty" json:"temporaryFailCodes,omitempty" mapstructure:"temporaryFailCodes,omitempty"`
	PermanentFailCodes []int                    `yaml:"permanentFailCodes,omitempty" bson:"permanentFailCodes,omitempty" json:"permanentFailCodes,omitempty" mapstructure:"permanentFailCodes,omitempty"`
	Namespaces         map[string]string        `yaml:"$namespaces,omitempty" bson:"_DOLLAR_namespaces,omitempty" json:"$namespaces,omitempty" mapstructure:"$namespaces,omitempty"`
}

// IsCWLMinimal _
func (c *CommandLineTool) IsCWLMinimal() {}

// IsProcess _
func (c *CommandLineTool) IsProcess() {}

// keyname will be converted into 'Id'-field

// NewCommandLineTool _
// parentIdentifier is used to convert relative id to absolute id
// objectIdentifier is used when there is no local is, in case of file or embedded tool
func NewCommandLineTool(generic interface{}, parentIdentifier string, objectIdentifier string, injectedRequirements []Requirement, context *WorkflowContext) (commandLineTool *CommandLineTool, schemata []CWLType_Type, err error) {

	//fmt.Println("NewCommandLineTool() generic:")
	//spew.Dump(generic)

	//switch type()
	object, ok := generic.(map[string]interface{})
	if !ok {
		err = fmt.Errorf("other types than map[string]interface{} not supported yet (got %s)", reflect.TypeOf(generic))
		return
	}
	//spew.Dump(generic)

	//fmt.Println("NewCommandLineTool() object:")
	//spew.Dump(object)

	commandLineTool = &CommandLineTool{}
	commandLineTool.Class = "CommandLineTool"

	//fmt.Printf("(NewCommandLineTool) requirements %d\n", len(requirements_array))
	//spew.Dump(requirements_array)
	//scs := spew.ConfigState{Indent: "\t"}
	//scs.Dump(object["requirements"])

	if objectIdentifier != "" {
		object["id"] = objectIdentifier
	} else {

		objectIDIf, hasID := object["id"]
		if hasID {

			objectID := objectIDIf.(string)

			if !strings.HasPrefix(objectID, "#") {
				if parentIdentifier == "" {
					err = fmt.Errorf("(NewCommandLineTool) A) parentIdentifier is needed but empty, objectID=%s", objectID)
					return
				}
				objectID = path.Join(parentIdentifier, objectID)
			}

			if !strings.HasPrefix(objectID, "#") {
				err = fmt.Errorf("(NewCommandLineTool) not absolute: objectID=%s , parentIdentifier=%s, objectIdentifier=%s", objectID, parentIdentifier, objectIdentifier)
				return
			}
			object["id"] = objectID
		} else {
			if parentIdentifier == "" {
				err = fmt.Errorf("(NewCommandLineTool) B) parentIdentifier is needed but empty")
				return
			}

			objectID := path.Join(parentIdentifier, uuid.New())
			object["id"] = objectID
		}
	}

	// extract SchemaDefRequirement
	var schemaDefReq *SchemaDefRequirement
	//var schemataNew []CWLType_Type
	hasSchemaDefReq := false

	requirementsIf, hasRequirements := object["requirements"]
	if hasRequirements {
		schemaDefReq, hasSchemaDefReq, err = GetSchemaDefRequirement(requirementsIf, context)
		if err != nil {
			err = fmt.Errorf("(NewCommandLineTool) GetSchemaDefRequirement returned: %s", err.Error())
			return
		}
	}

	//if has_schema_def_req {
	//	for i, _ := range schemataNew {
	//		schemata = append(schemata, schemataNew[i])
	//	}
	//}

	inputs := []*CommandInputParameter{}

	inputsIf, ok := object["inputs"]
	if ok {
		// Convert map of inputs into array of inputs
		err, inputs = CreateCommandInputArray(inputsIf, schemata, context)
		if err != nil {
			err = fmt.Errorf("(NewCommandLineTool) error in CreateCommandInputArray: %s", err.Error())
			return
		}
	}

	object["inputs"] = inputs

	var copa []interface{}

	outputs, hasOutputs := object["outputs"]
	if hasOutputs {

		//fmt.Println("NewCommandLineTool() object/outputs:")
		//spew.Dump(outputs)

		// Convert map of outputs into array of outputs

		copa, err = NewCommandOutputParameterArray(outputs, schemata, context)
		if err != nil {
			//fmt.Println("NewCommandLineTool after error")
			//spew.Dump(object)

			err = fmt.Errorf("(NewCommandLineTool) error in NewCommandOutputParameterArray: %s", err.Error())
			return
		}
		object["outputs"] = copa
	} else {
		err = fmt.Errorf("(NewCommandLineTool) no outputs !?")
		return
		//object["outputs"] = []*CommandOutputParameter{}
	}

	baseCommand, ok := object["baseCommand"]
	if ok {
		object["baseCommand"], err = NewBaseCommandArray(baseCommand)
		if err != nil {
			err = fmt.Errorf("(NewCommandLineTool) error in NewBaseCommandArray: %s", err.Error())
			return
		}
	}

	arguments, hasArguments := object["arguments"]
	if hasArguments {
		// Convert map of outputs into array of outputs
		var argumentsObject []CommandLineBinding
		argumentsObject, err = NewCommandLineBindingArray(arguments, context)
		if err != nil {
			err = fmt.Errorf("(NewCommandLineTool) error in NewCommandLineBindingArray: %s", err.Error())
			return
		}
		//delete(object, "arguments")
		object["arguments"] = argumentsObject
	}

	if hasSchemaDefReq {
		injectedRequirements = append(injectedRequirements, schemaDefReq)
	}

	var requirementsArray []Requirement

	//fmt.Printf("(NewCommandLineTool) Injecting %d\n", len(requirementsArray))
	//spew.Dump(requirementsArray)
	requirementsArray, err = CreateRequirementArrayAndInject(requirementsIf, injectedRequirements, nil, context)
	if err != nil {
		err = fmt.Errorf("(NewCommandLineTool) error in CreateRequirementArray (requirements): %s", err.Error())
		return
	}

	//for i, _ := range schemataNew {
	//	schemata = append(schemata, schemataNew[i])
	//}

	object["requirements"] = requirementsArray

	hints, ok := object["hints"]
	if ok && (hints != nil) {
		//var schemataNew []CWLType_Type

		var hintsArray []Requirement
		hintsArray, err = CreateHintsArray(hints, injectedRequirements, inputs, context)
		if err != nil {
			err = fmt.Errorf("(NewCommandLineTool) error in CreateRequirementArray (hints): %s", err.Error())
			return
		}
		//for i, _ := range schemataNew {
		//	schemata = append(schemata, schemataNew[i])
		//}
		object["hints"] = hintsArray
	}

	err = mapstructure.Decode(object, commandLineTool)
	if err != nil {
		err = fmt.Errorf("(NewCommandLineTool) error parsing CommandLineTool class: %s", err.Error())
		return
	}

	if commandLineTool.ID == "" {
		err = fmt.Errorf("(NewCommandLineTool) id is empty!?")
		return
	}

	if !strings.HasPrefix(commandLineTool.ID, "#") {
		err = fmt.Errorf("(NewCommandLineTool) id is not absolute!?")
		return
	}

	//fmt.Println("commandLineTool:")
	//spew.Dump(commandLineTool)

	if commandLineTool.CwlVersion == "" {
		commandLineTool.CwlVersion = context.CwlVersion
	}

	if context == nil {
		err = fmt.Errorf("(NewCommandLineTool) context == nil")
		return
	}

	if context.Namespaces != nil {
		commandLineTool.Namespaces = context.Namespaces
	}

	if context != nil {

		thisID := commandLineTool.ID
		if thisID == "" {
			err = fmt.Errorf("(NewCommandLineTool) did not expect empty Id")
			return
		}

		// err = context.Add(commandLineTool.Id, commandLineTool, "NewCommandLineTool")
		// if err != nil {
		// 	err = fmt.Errorf("(NewCommandLineTool) (add commandLineTool) context.Add returned: %s", err.Error())
		// 	return
		// }

		// for i := range commandLineTool.Inputs {
		// 	inp := &commandLineTool.Inputs[i]
		// 	inpID := path.Join(thisID, inp.Id)

		// 	err = context.AddObject(inpID, inp, "NewCommandLineTool")
		// 	if err != nil {
		// 		err = fmt.Errorf("(NewCommandLineTool) X (add commandLineToolInput)  context.Add returned: %s", err.Error())
		// 		return
		// 	}
		// }
	}
	return

}

// NewBaseCommandArray _
func NewBaseCommandArray(original interface{}) (newArray []string, err error) {
	newArray = []string{}
	switch original.(type) {
	case []interface{}:
		originalArray := original.([]interface{})
		for _, v := range originalArray {

			vStr, ok := v.(string)
			if !ok {
				spew.Dump(originalArray)
				err = fmt.Errorf("(NewBaseCommandArray) []interface{} array element is not a string, it is %s", reflect.TypeOf(v))
				return
			}
			newArray = append(newArray, vStr)
		}

		return
	case string:
		orgStr, _ := original.(string)
		newArray = append(newArray, orgStr)
		return
	default:
		err = fmt.Errorf("(NewBaseCommandArray) type unknown")

	}
	return
}

// Evaluate _
func (c *CommandLineTool) Evaluate(inputs interface{}, context *WorkflowContext) (err error) {

	for i := range c.Requirements {

		r := c.Requirements[i]

		err = r.Evaluate(inputs, context)
		if err != nil {
			err = fmt.Errorf("(CommandLineTool/Evaluate) Requirements r.Evaluate returned: %s", err.Error())
			return
		}

	}

	for i := range c.Hints {

		r := c.Hints[i]

		err = r.Evaluate(inputs, context)
		if err != nil {
			err = fmt.Errorf("(CommandLineTool/Evaluate) Hints r.Evaluate returned: %s", err.Error())
			return
		}

	}

	return
}
<|endoftext|>


<commit_before>package provision

import (
	"bytes"
	"fmt"
	"text/template"

	"github.com/docker/machine/drivers"
	"github.com/docker/machine/libmachine/auth"
	"github.com/docker/machine/libmachine/engine"
	"github.com/docker/machine/libmachine/swarm"
)

type GenericProvisioner struct {
	OsReleaseId       string
	DockerOptionsDir  string
	DaemonOptionsFile string
	KubernetesManifestFile string
	Packages          []string
	OsReleaseInfo     *OsRelease
	Driver            drivers.Driver
	AuthOptions       auth.AuthOptions
	EngineOptions     engine.EngineOptions
	SwarmOptions      swarm.SwarmOptions
}

func (provisioner *GenericProvisioner) Hostname() (string, error) {
	return provisioner.SSHCommand("hostname")
}

func (provisioner *GenericProvisioner) SetHostname(hostname string) error {
	if _, err := provisioner.SSHCommand(fmt.Sprintf(
		"sudo hostname %s && echo %q | sudo tee /etc/hostname",
		hostname,
		hostname,
	)); err != nil {
		return err
	}

	// ubuntu/debian use 127.0.1.1 for non "localhost" loopback hostnames: https://www.debian.org/doc/manuals/debian-reference/ch05.en.html#_the_hostname_resolution
	if _, err := provisioner.SSHCommand(fmt.Sprintf(
		"if grep -xq 127.0.1.1.* /etc/hosts; then sudo sed -i 's/^127.0.1.1.*/127.0.1.1 %s/g' /etc/hosts; else echo '127.0.1.1 %s' | sudo tee -a /etc/hosts; fi",
		hostname,
		hostname,
	)); err != nil {
		return err
	}

	return nil
}

func (provisioner *GenericProvisioner) GetDockerOptionsDir() string {
	return provisioner.DockerOptionsDir
}

func (provisioner *GenericProvisioner) SSHCommand(args string) (string, error) {
	return drivers.RunSSHCommandFromDriver(provisioner.Driver, args)
}

func (provisioner *GenericProvisioner) CompatibleWithHost() bool {
	return provisioner.OsReleaseInfo.Id == provisioner.OsReleaseId
}

func (provisioner *GenericProvisioner) GetAuthOptions() auth.AuthOptions {
	return provisioner.AuthOptions
}

func (provisioner *GenericProvisioner) SetOsReleaseInfo(info *OsRelease) {
	provisioner.OsReleaseInfo = info
}

func (provisioner *GenericProvisioner) Generatek8sOptions() (*k8sOptions, error) {
	var (
		k8sCfg bytes.Buffer
	)

	k8sConfigTmpl := `
{
"apiVersion": "v1beta3",
"kind": "Pod",
"metadata": {"name":"kubernetes"},
"spec":{
  "hostNetwork": true,
  "containers":[
    {
      "name": "controller-manager",
      "image": "gcr.io/google_containers/hyperkube:v0.17.0",
      "command": [
              "/hyperkube",
              "controller-manager",
              "--master=127.0.0.1:8080",
              "--machines=127.0.0.1",
              "--sync_nodes=true",
              "--v=2"
        ]
    },
    {
      "name": "apiserver",
      "image": "gcr.io/google_containers/hyperkube:v0.17.0",
      "volumeMounts": [
         {"name": "token-volume",
          "mountPath": "/tmp/tokenfile.txt",
          "readOnly": true }
          ],
      "command": [
              "/hyperkube",
              "apiserver",
              "--token-auth-file=/tmp/tokenfile.txt",
              "--portal_net=10.0.0.1/24",
              "--address=0.0.0.0",
              "--etcd_servers=http://127.0.0.1:4001",
              "--cluster_name=kmachine",
              "--v=2"
        ]
    },
    {
      "name": "scheduler",
      "image": "gcr.io/google_containers/hyperkube:v0.17.0",
      "command": [
              "/hyperkube",
              "scheduler",
              "--master=127.0.0.1:8080",
              "--v=2"
        ]
    }
  ],
  "volumes":[
    { "name": "token-volume",
      "hostPath": {
        "path": "/tmp/tokenfile.txt"}
    }
 ]
 }
}
`
	t, err := template.New("k8sConfig").Parse(k8sConfigTmpl)
	if err != nil {
		return nil, err
	}

	k8sContext := EngineConfigContext{
		DockerPort:    1234,
		AuthOptions:   provisioner.AuthOptions,
		EngineOptions: provisioner.EngineOptions,
	}

	t.Execute(&k8sCfg, k8sContext)

	return &k8sOptions{
		k8sOptions:     k8sCfg.String(),
		k8sOptionsPath: provisioner.KubernetesManifestFile,
	}, nil
}

func (provisioner *GenericProvisioner) GenerateDockerOptions(dockerPort int) (*DockerOptions, error) {
	var (
		engineCfg bytes.Buffer
	)

	driverNameLabel := fmt.Sprintf("provider=%s", provisioner.Driver.DriverName())
	provisioner.EngineOptions.Labels = append(provisioner.EngineOptions.Labels, driverNameLabel)

	engineConfigTmpl := `
DOCKER_OPTS='
-H tcp://0.0.0.0:{{.DockerPort}}
-H unix:///var/run/docker.sock
--storage-driver {{.EngineOptions.StorageDriver}}
--tlsverify
--tlscacert {{.AuthOptions.CaCertRemotePath}}
--tlscert {{.AuthOptions.ServerCertRemotePath}}
--tlskey {{.AuthOptions.ServerKeyRemotePath}}
{{ range .EngineOptions.Labels }}--label {{.}}
{{ end }}{{ range .EngineOptions.InsecureRegistry }}--insecure-registry {{.}}
{{ end }}{{ range .EngineOptions.RegistryMirror }}--registry-mirror {{.}}
{{ end }}{{ range .EngineOptions.ArbitraryFlags }}--{{.}}
{{ end }}
'
`
	t, err := template.New("engineConfig").Parse(engineConfigTmpl)
	if err != nil {
		return nil, err
	}

	engineConfigContext := EngineConfigContext{
		DockerPort:    dockerPort,
		AuthOptions:   provisioner.AuthOptions,
		EngineOptions: provisioner.EngineOptions,
	}

	t.Execute(&engineCfg, engineConfigContext)

	return &DockerOptions{
		EngineOptions:     engineCfg.String(),
		EngineOptionsPath: provisioner.DaemonOptionsFile,
	}, nil
}

func (provisioner *GenericProvisioner) GetDriver() drivers.Driver {
	return provisioner.Driver
}
<commit_msg>Forgotten commit<commit_after>package provision

import (
	"bytes"
	"fmt"
	"text/template"

	"github.com/docker/machine/drivers"
	"github.com/docker/machine/libmachine/auth"
	"github.com/docker/machine/libmachine/engine"
	"github.com/docker/machine/libmachine/swarm"
)

type GenericProvisioner struct {
	OsReleaseId       string
	DockerOptionsDir  string
	DaemonOptionsFile string
	KubernetesManifestFile string
	Packages          []string
	OsReleaseInfo     *OsRelease
	Driver            drivers.Driver
	AuthOptions       auth.AuthOptions
	EngineOptions     engine.EngineOptions
	SwarmOptions      swarm.SwarmOptions
}

func (provisioner *GenericProvisioner) Hostname() (string, error) {
	return provisioner.SSHCommand("hostname")
}

func (provisioner *GenericProvisioner) SetHostname(hostname string) error {
	if _, err := provisioner.SSHCommand(fmt.Sprintf(
		"sudo hostname %s && echo %q | sudo tee /etc/hostname",
		hostname,
		hostname,
	)); err != nil {
		return err
	}

	// ubuntu/debian use 127.0.1.1 for non "localhost" loopback hostnames: https://www.debian.org/doc/manuals/debian-reference/ch05.en.html#_the_hostname_resolution
	if _, err := provisioner.SSHCommand(fmt.Sprintf(
		"if grep -xq 127.0.1.1.* /etc/hosts; then sudo sed -i 's/^127.0.1.1.*/127.0.1.1 %s/g' /etc/hosts; else echo '127.0.1.1 %s' | sudo tee -a /etc/hosts; fi",
		hostname,
		hostname,
	)); err != nil {
		return err
	}

	return nil
}

func (provisioner *GenericProvisioner) GetDockerOptionsDir() string {
	return provisioner.DockerOptionsDir
}

func (provisioner *GenericProvisioner) SSHCommand(args string) (string, error) {
	return drivers.RunSSHCommandFromDriver(provisioner.Driver, args)
}

func (provisioner *GenericProvisioner) CompatibleWithHost() bool {
	return provisioner.OsReleaseInfo.Id == provisioner.OsReleaseId
}

func (provisioner *GenericProvisioner) GetAuthOptions() auth.AuthOptions {
	return provisioner.AuthOptions
}

func (provisioner *GenericProvisioner) SetOsReleaseInfo(info *OsRelease) {
	provisioner.OsReleaseInfo = info
}

func (provisioner *GenericProvisioner) Generatek8sOptions() (*k8sOptions, error) {
	var (
		k8sCfg bytes.Buffer
	)

	k8sConfigTmpl := `
{
"apiVersion": "v1beta3",
"kind": "Pod",
"metadata": {"name":"kubernetes"},
"spec":{
  "hostNetwork": true,
  "containers":[
    {
      "name": "controller-manager",
      "image": "gcr.io/google_containers/hyperkube:v0.17.0",
      "command": [
              "/hyperkube",
              "controller-manager",
              "--master=127.0.0.1:8080",
              "--machines=127.0.0.1",
              "--sync_nodes=true",
              "--v=2"
        ]
    },
    {
      "name": "apiserver",
      "image": "gcr.io/google_containers/hyperkube:v0.17.0",
      "volumeMounts": [
         {"name": "token-volume",
          "mountPath": "/tmp/tokenfile.txt",
          "readOnly": true }
          ],
      "command": [
              "/hyperkube",
              "apiserver",
              "--token-auth-file=/tmp/tokenfile.txt",
              "--portal-net=10.0.0.1/24",
              "--address=0.0.0.0",
              "--etcd_servers=http://127.0.0.1:4001",
              "--cluster_name=kmachine",
              "--v=2"
        ]
    },
    {
      "name": "scheduler",
      "image": "gcr.io/google_containers/hyperkube:v0.17.0",
      "command": [
              "/hyperkube",
              "scheduler",
              "--master=127.0.0.1:8080",
              "--v=2"
        ]
    }
  ],
  "volumes":[
    { "name": "token-volume",
      "hostPath": {
        "path": "/tmp/tokenfile.txt"}
    }
 ]
 }
}
`
	t, err := template.New("k8sConfig").Parse(k8sConfigTmpl)
	if err != nil {
		return nil, err
	}

	k8sContext := EngineConfigContext{
		DockerPort:    1234,
		AuthOptions:   provisioner.AuthOptions,
		EngineOptions: provisioner.EngineOptions,
	}

	t.Execute(&k8sCfg, k8sContext)

	return &k8sOptions{
		k8sOptions:     k8sCfg.String(),
		k8sOptionsPath: provisioner.KubernetesManifestFile,
	}, nil
}

func (provisioner *GenericProvisioner) GenerateDockerOptions(dockerPort int) (*DockerOptions, error) {
	var (
		engineCfg bytes.Buffer
	)

	driverNameLabel := fmt.Sprintf("provider=%s", provisioner.Driver.DriverName())
	provisioner.EngineOptions.Labels = append(provisioner.EngineOptions.Labels, driverNameLabel)

	engineConfigTmpl := `
DOCKER_OPTS='
-H tcp://0.0.0.0:{{.DockerPort}}
-H unix:///var/run/docker.sock
--storage-driver {{.EngineOptions.StorageDriver}}
--tlsverify
--tlscacert {{.AuthOptions.CaCertRemotePath}}
--tlscert {{.AuthOptions.ServerCertRemotePath}}
--tlskey {{.AuthOptions.ServerKeyRemotePath}}
{{ range .EngineOptions.Labels }}--label {{.}}
{{ end }}{{ range .EngineOptions.InsecureRegistry }}--insecure-registry {{.}}
{{ end }}{{ range .EngineOptions.RegistryMirror }}--registry-mirror {{.}}
{{ end }}{{ range .EngineOptions.ArbitraryFlags }}--{{.}}
{{ end }}
'
`
	t, err := template.New("engineConfig").Parse(engineConfigTmpl)
	if err != nil {
		return nil, err
	}

	engineConfigContext := EngineConfigContext{
		DockerPort:    dockerPort,
		AuthOptions:   provisioner.AuthOptions,
		EngineOptions: provisioner.EngineOptions,
	}

	t.Execute(&engineCfg, engineConfigContext)

	return &DockerOptions{
		EngineOptions:     engineCfg.String(),
		EngineOptionsPath: provisioner.DaemonOptionsFile,
	}, nil
}

func (provisioner *GenericProvisioner) GetDriver() drivers.Driver {
	return provisioner.Driver
}
<|endoftext|>


<commit_before>package bytepool

import (
  "io"
)

type Item struct {
  pool *Pool
  length int
  bytes []byte
}

func newItem(capacity int, pool *Pool) *Item {
  return &Item {
    pool: pool,
    bytes: make([]byte, capacity),
  }
}

func (item *Item) Write(b []byte) {
  item.length += copy(item.bytes[item.length:], b)
}

func (item *Item) WriteString(s string) {
  item.length += copy(item.bytes[item.length:], s)
}

func (item *Item) ReadFrom(reader io.Reader) (int64, error) {
  var read int64
  for {
    r, err := reader.Read(item.bytes[item.length:])
    read += int64(r)
    item.length += r
    if err == io.EOF { return read, nil }
    if err != nil { return read, err }
  }
}

func (item *Item) Bytes() []byte {
  return item.bytes[0:item.length]
}

func (item *Item) String() string {
  return string(item.Bytes())
}

func (item *Item) Len() int {
  return item.length;
}

func (item *Item) Position(position int) {
  item.length = position
}

func (item *Item) Close() {
  item.length = 0
  if item.pool != nil {
    item.pool.list <- item
  }
}
<commit_msg>item properly implements io.Close<commit_after>package bytepool

import (
  "io"
)

type Item struct {
  pool *Pool
  length int
  bytes []byte
}

func newItem(capacity int, pool *Pool) *Item {
  return &Item {
    pool: pool,
    bytes: make([]byte, capacity),
  }
}

func (item *Item) Write(b []byte) {
  item.length += copy(item.bytes[item.length:], b)
}

func (item *Item) WriteString(s string) {
  item.length += copy(item.bytes[item.length:], s)
}

func (item *Item) ReadFrom(reader io.Reader) (int64, error) {
  var read int64
  for {
    r, err := reader.Read(item.bytes[item.length:])
    read += int64(r)
    item.length += r
    if err == io.EOF { return read, nil }
    if err != nil { return read, err }
  }
}

func (item *Item) Bytes() []byte {
  return item.bytes[0:item.length]
}

func (item *Item) String() string {
  return string(item.Bytes())
}

func (item *Item) Len() int {
  return item.length;
}

func (item *Item) Position(position int) {
  item.length = position
}

func (item *Item) Close() error{
  item.length = 0
  if item.pool != nil {
    item.pool.list <- item
  }
  return nil
}
<|endoftext|>


<commit_before>package gojobs

import (
	"database/sql"
	"encoding/json"
	"reflect"
	"sync"
	"sync/atomic"
	"time"

	"github.com/golang/glog"
	"github.com/pascallouisperez/goutil/errors"
	"github.com/pascallouisperez/reflext"
)

const (
	statusPending    = "pending"
	statusProcessing = "processing"
	statusFailed     = "failed"
	statusCompleted  = "completed"
)

const (
	_ int32 = iota
	queueStopped
	queueStarting
	queueRunning
	queueStopping
)

type clock interface {
	// UnixNow returns the number of seconds elapsed since January 1, 1970 UTC
	// Equivalent of time.Now().Unix()
	UnixNow() int64
}

// JobQueue is the entry point to registering, scheduling, and querying
// jobs.
type JobQueue struct {
	numFetchers int
	numWorkers  int

	db          *sql.DB
	processorId int64
	configs     map[string]jobConfig
	status      *int32
	workersWg   sync.WaitGroup
	fetcherWg   sync.WaitGroup
	maybeJobIds chan int64
	clock       clock
}

type Job interface {
	// ID returns the identifier of the job.
	ID() int64

	// Name returns the name of the job.
	Name() string

	// Params returns the specific struct associated with this job.
	Params() (interface{}, error)
}

type jobConfig struct {
	name             string
	handler          reflect.Value
	paramsType       reflect.Type
	attempts         int
	backoffInSeconds int64
}

type jobRecord struct {
	id            int64
	name          string
	params        []byte
	remaining     int
	schedulableAt int64
}

func (rec *jobRecord) attempt(conf jobConfig) int {
	return conf.attempts - rec.remaining
}

type realClock struct{}

func (realClock) UnixNow() int64 {
	return time.Now().Unix()
}

// QueueConfiguration groups optional queue configuration parameters. For
// all parameters, reasonable defaults are provided by the library.
type QueueConfiguration struct {
	// NumWorkers specifies the number of workers (i.e. goroutines processing
	// jobs) to spawn when this queue is started. Default is 5.
	NumWorkers int
}

func NewJobQueue(db *sql.DB, processorId int64, optConfs ...QueueConfiguration) *JobQueue {
	status := queueStopped
	jq := JobQueue{
		db:          db,
		processorId: processorId,
		configs:     make(map[string]jobConfig, 10),
		status:      &status,
		clock:       realClock{},
		maybeJobIds: make(chan int64),
		numFetchers: 1,
		numWorkers:  5,
	}

	// Optional configuration.
	if size := len(optConfs); size > 1 {
		panic("too many configurations provided")
	} else if size == 1 {
		conf := optConfs[0]
		if conf.NumWorkers > 0 {
			jq.numWorkers = conf.NumWorkers
		}
	}

	return &jq
}

var handlerMatcher = reflext.MustCompile("func ({*struct}) error")

// JobConfiguration groups optional job configuration parameters. For
// all parameters, reasonable defaults are provided by the library.
type JobConfiguration struct {
	// Attempts specifies the number processing attempts to try before aborting
	// a job, and marking is failed.
	Attempts int

	// Backoff specifies the base duration off which the total exponential backoff
	// is calculated when the job is retried one, twice, thrice, etc.
	// While time.Duration can be expressed in nanoseconds, only durations of seconds
	// or more are considered valid.
	Backoff time.Duration
}

func (jq *JobQueue) Register(name string, handler interface{}, optConfs ...JobConfiguration) error {
	if name == "" {
		return errors.New("job name cannot be empty")
	}

	if _, ok := jq.configs[name]; ok {
		return errors.New("job %s: already registered", name)
	}

	if st := atomic.LoadInt32(jq.status); st != queueStopped {
		// TODO(pascal): if we cared about super pedantic, we should increment
		// a wait group on entry to Register, and wait on this wg in Start to
		// allow concurrent registrations to complete. But then, we should
		// also be careful about concurrent registrations writing to the shared
		// configs map. For now, we're going to assume registration is done in a
		// simple way.
		return errors.New("job %s: unable to register once queue has started", name)
	}

	if handler == nil {
		return errors.New("job %s: missing handler", name)
	}

	var paramsType reflect.Type
	if types, ok := handlerMatcher.FindAll(handler); ok {
		paramsType = types[0]
	} else {
		return errors.New("job %s: expected %s, was %T", name, handlerMatcher, handler)
	}

	// Defaults.
	conf := jobConfig{
		name:             name,
		handler:          reflect.ValueOf(handler),
		paramsType:       paramsType,
		attempts:         3,
		backoffInSeconds: 5,
	}

	// Optional configuration.
	if l := len(optConfs); l > 1 {
		return errors.New("job %s: too many optional configurations provided", name)
	} else if l == 1 {
		optConf := optConfs[0]
		if optConf.Attempts > 0 {
			conf.attempts = optConf.Attempts
		}
		if b := optConf.Backoff.Nanoseconds() / 1000000000; b > 0 {
			conf.backoffInSeconds = b
		}
	}

	jq.configs[name] = conf
	return nil
}

func (jq *JobQueue) fetcher(fetcherIndex int) {
	jq.fetcherWg.Add(1)
	defer jq.fetcherWg.Done()

	for {
		if st := atomic.LoadInt32(jq.status); st == queueStopping {
			return
		}

		jobId, hasNext, err := jq.maybeNext()
		if err != nil {
			glog.Infof("fetcher[%d], internal error while fetching potential next job: %s", fetcherIndex, err)
		}
		if !hasNext {
			time.Sleep(1 * time.Second)
			continue
		}

		jq.maybeJobIds <- jobId
	}
}

func (jq *JobQueue) worker(workerIndex int) {
	jq.workersWg.Add(1)
	defer jq.workersWg.Done()

	for {
		if st := atomic.LoadInt32(jq.status); st == queueStopping {
			return
		}

		jobId := <-jq.maybeJobIds

		locked, err := jq.attemptLock(jobId)
		if err != nil {
			glog.Infof("worker[%d] internal error while attempting to lock job: %s", workerIndex, err)
		}
		if !locked {
			continue
		}

		rec, err := jq.getJobRecord(jobId)
		if err != nil {
			glog.Infof("worker[%d] internal error while fetching job record: %s", workerIndex, err)
			continue
		}

		jobErr := jq.safeProcess(rec, jq.configs[rec.name])
		if jobErr != nil {
			if rec.remaining > 0 {
				err = jq.reEnqueue(rec)
			} else {
				err = jq.markAs(rec.id, statusFailed)
			}
		} else {
			err = jq.markAs(rec.id, statusCompleted)
		}

		if err != nil {
			glog.Infof("worker[%d] internal error in post process of job: %s", err)
		}
	}

}

func (jq *JobQueue) Start() error {
	// Mark queue as starting.
	if !atomic.CompareAndSwapInt32(jq.status, queueStopped, queueStarting) {
		return errors.New("can only start a queue which is stopped")
	}

	// Start fetchers.
	for i := 0; i < jq.numFetchers; i++ {
		go jq.fetcher(i)
	}

	// Start workers.
	for i := 0; i < jq.numWorkers; i++ {
		go jq.worker(i)
	}

	// Mark queue as running.
	atomic.StoreInt32(jq.status, queueRunning)

	return nil
}

func (jq *JobQueue) Stop() error {
	if !atomic.CompareAndSwapInt32(jq.status, queueRunning, queueStopping) {
		return errors.New("unable to stop")
	}
	close(jq.maybeJobIds)
	jq.workersWg.Wait()
	jq.fetcherWg.Wait()
	atomic.StoreInt32(jq.status, queueStopped)
	return nil
}

func (jq *JobQueue) Enqueue(tx *sql.Tx, name string, params interface{}) (int64, error) {
	conf, ok := jq.configs[name]
	if !ok {
		return -1, errors.New("unknown job: %s", name)
	}

	var (
		paramsAsBytes []byte
		err           error
	)
	if params != nil {
		actualParamsType := reflect.TypeOf(params)
		if !actualParamsType.AssignableTo(conf.paramsType) {
			return -1, errors.New("job %s: incorrect param type, expected %s, got %s",
				conf.name, conf.paramsType, actualParamsType)
		}

		paramsAsBytes, err = json.Marshal(params)
		if err != nil {
			return -1, errors.New("job %s: unable to marshall parms %s", err)
		}
	}

	now := jq.clock.UnixNow()
	res, err := tx.Exec(`
		insert into job_queue (name, params, remaining, status, created_at, schedulable_at)
		values (?, ?, ?, ?, ?, ?)`,
		name, paramsAsBytes, conf.attempts, statusPending, now, now)
	if err != nil {
		return -1, err
	}
	jobId, err := res.LastInsertId()
	if err != nil {
		return -1, err
	}
	return jobId, nil
}

func (jq *JobQueue) Get(jobId int64) (Job, error) {
	rec, err := jq.getJobRecord(jobId)
	if err != nil {
		return nil, err
	}
	conf, ok := jq.configs[rec.name]
	if !ok {
		return nil, errors.New("job %s has not been registered", rec.name)
	}
	return &jobRecordExternal{rec, conf.paramsType}, nil
}

func (jq *JobQueue) safeProcess(rec *jobRecord, conf jobConfig) error {
	var (
		err     error
		elapsed int64
	)

	// Process, with panic handling.
	func() {
		elapsed = time.Now().UnixNano()
		defer func() {
			if r := recover(); r != nil {
				err = errors.New("job %s panicked: %s", conf.name, r)
			}
		}()

		err = func() error {
			params, err := (&jobRecordExternal{rec, conf.paramsType}).Params()
			if err != nil {
				return err
			}
			returns := conf.handler.Call([]reflect.Value{reflect.ValueOf(params)})
			if !returns[0].IsNil() {
				return returns[0].Interface().(error)
			}
			return nil
		}()
	}()

	// Logging.
	elapsed = (time.Now().UnixNano() - elapsed) / 1000000
	if err == nil {
		glog.Infof("job %s(%d) succeeded: attempt=%d, elapsed=%dms",
			rec.name, rec.id,
			rec.attempt(conf), elapsed)
	} else {
		glog.Infof("job %s(%d) failed: attempt=%d, elapsed=%dms, remaining=%d, err=%s",
			rec.name, rec.id,
			rec.attempt(conf), elapsed, rec.remaining, err)
	}

	// Done
	return err
}

func (jq *JobQueue) maybeNext() (int64, bool, error) {
	now := jq.clock.UnixNow()
	rows, err := jq.db.Query(`
		select id from job_queue
		where status = ? and remaining > 0 and schedulable_at <= ?
		order by id asc limit 1`,
		statusPending, now)
	if err != nil {
		return -1, false, err
	}
	defer rows.Close()
	if !rows.Next() {
		return -1, false, nil
	}
	var jobId int64
	err = rows.Scan(&jobId)
	if err != nil {
		return -1, false, err
	}
	return jobId, true, nil
}

func (jq *JobQueue) attemptLock(jobId int64) (bool, error) {
	res, err := jq.db.Exec(
		"update job_queue set status = ?, processor_id = ?, remaining = remaining - 1 where id = ? and status = ?",
		statusProcessing, jq.processorId, jobId, statusPending)
	if err != nil {
		return false, err
	}
	changed, err := res.RowsAffected()
	if err != nil {
		return false, err
	}
	return changed == 1, nil
}

func (jq *JobQueue) markAs(jobId int64, status string) error {
	_, err := jq.db.Exec(
		"update job_queue set status = ?, processor_id = null where id = ?",
		status, jobId)
	return err
}

func (jq *JobQueue) reEnqueue(rec *jobRecord) error {
	conf := jq.configs[rec.name]
	backoffMultiplier := 1 << uint(rec.attempt(conf)-1)
	newSchedulableAt := jq.clock.UnixNow() + conf.backoffInSeconds*int64(backoffMultiplier)
	_, err := jq.db.Exec(
		"update job_queue set status = ?, schedulable_at = ?, processor_id = null where id = ?",
		statusPending, newSchedulableAt, rec.id)
	return err
}

func (jq *JobQueue) getJobRecord(jobId int64) (*jobRecord, error) {
	var (
		name          string
		params        []byte
		remaining     int
		schedulableAt int64
	)
	err := jq.db.
		QueryRow(
			"select name, params, remaining, schedulable_at from job_queue where id = ?",
			jobId).
		Scan(&name, &params, &remaining, &schedulableAt)
	if err != nil {
		return nil, err
	}
	rec := jobRecord{
		id:            jobId,
		name:          name,
		params:        params,
		remaining:     remaining,
		schedulableAt: schedulableAt,
	}
	return &rec, nil
}

type jobRecordExternal struct {
	*jobRecord
	paramsType reflect.Type
}

// Assert jobRecord implements the Job interface.
var _ Job = &jobRecordExternal{}

func (rec *jobRecordExternal) ID() int64 {
	return rec.id
}

func (rec *jobRecordExternal) Name() string {
	return rec.name
}

func (rec *jobRecordExternal) Params() (interface{}, error) {
	params := reflect.New(rec.paramsType.Elem()).Interface()
	if rec.params != nil {
		if err := json.Unmarshal(rec.params, &params); err != nil {
			return nil, errors.New("job %s unmarshall error: %s", rec.name, err)
		}
	}
	return params, nil
}
<commit_msg>More respectful of when things should run.<commit_after>package gojobs

import (
	"database/sql"
	"encoding/json"
	"reflect"
	"sync"
	"sync/atomic"
	"time"

	"github.com/golang/glog"
	"github.com/pascallouisperez/goutil/errors"
	"github.com/pascallouisperez/reflext"
)

const (
	statusPending    = "pending"
	statusProcessing = "processing"
	statusFailed     = "failed"
	statusCompleted  = "completed"
)

const (
	_ int32 = iota
	queueStopped
	queueStarting
	queueRunning
	queueStopping
)

type clock interface {
	// UnixNow returns the number of seconds elapsed since January 1, 1970 UTC
	// Equivalent of time.Now().Unix()
	UnixNow() int64
}

// JobQueue is the entry point to registering, scheduling, and querying
// jobs.
type JobQueue struct {
	numFetchers int
	numWorkers  int

	db          *sql.DB
	processorId int64
	configs     map[string]jobConfig
	status      *int32
	workersWg   sync.WaitGroup
	fetcherWg   sync.WaitGroup
	maybeJobIds chan int64
	clock       clock
}

type Job interface {
	// ID returns the identifier of the job.
	ID() int64

	// Name returns the name of the job.
	Name() string

	// Params returns the specific struct associated with this job.
	Params() (interface{}, error)
}

type jobConfig struct {
	name             string
	handler          reflect.Value
	paramsType       reflect.Type
	attempts         int
	backoffInSeconds int64
}

type jobRecord struct {
	id            int64
	name          string
	params        []byte
	remaining     int
	schedulableAt int64
}

func (rec *jobRecord) attempt(conf jobConfig) int {
	return conf.attempts - rec.remaining
}

type realClock struct{}

func (realClock) UnixNow() int64 {
	return time.Now().Unix()
}

// QueueConfiguration groups optional queue configuration parameters. For
// all parameters, reasonable defaults are provided by the library.
type QueueConfiguration struct {
	// NumWorkers specifies the number of workers (i.e. goroutines processing
	// jobs) to spawn when this queue is started. Default is 5.
	NumWorkers int
}

func NewJobQueue(db *sql.DB, processorId int64, optConfs ...QueueConfiguration) *JobQueue {
	status := queueStopped
	jq := JobQueue{
		db:          db,
		processorId: processorId,
		configs:     make(map[string]jobConfig, 10),
		status:      &status,
		clock:       realClock{},
		maybeJobIds: make(chan int64),
		numFetchers: 1,
		numWorkers:  5,
	}

	// Optional configuration.
	if size := len(optConfs); size > 1 {
		panic("too many configurations provided")
	} else if size == 1 {
		conf := optConfs[0]
		if conf.NumWorkers > 0 {
			jq.numWorkers = conf.NumWorkers
		}
	}

	return &jq
}

var handlerMatcher = reflext.MustCompile("func ({*struct}) error")

// JobConfiguration groups optional job configuration parameters. For
// all parameters, reasonable defaults are provided by the library.
type JobConfiguration struct {
	// Attempts specifies the number processing attempts to try before aborting
	// a job, and marking is failed.
	Attempts int

	// Backoff specifies the base duration off which the total exponential backoff
	// is calculated when the job is retried one, twice, thrice, etc.
	// While time.Duration can be expressed in nanoseconds, only durations of seconds
	// or more are considered valid.
	Backoff time.Duration
}

func (jq *JobQueue) Register(name string, handler interface{}, optConfs ...JobConfiguration) error {
	if name == "" {
		return errors.New("job name cannot be empty")
	}

	if _, ok := jq.configs[name]; ok {
		return errors.New("job %s: already registered", name)
	}

	if st := atomic.LoadInt32(jq.status); st != queueStopped {
		// TODO(pascal): if we cared about super pedantic, we should increment
		// a wait group on entry to Register, and wait on this wg in Start to
		// allow concurrent registrations to complete. But then, we should
		// also be careful about concurrent registrations writing to the shared
		// configs map. For now, we're going to assume registration is done in a
		// simple way.
		return errors.New("job %s: unable to register once queue has started", name)
	}

	if handler == nil {
		return errors.New("job %s: missing handler", name)
	}

	var paramsType reflect.Type
	if types, ok := handlerMatcher.FindAll(handler); ok {
		paramsType = types[0]
	} else {
		return errors.New("job %s: expected %s, was %T", name, handlerMatcher, handler)
	}

	// Defaults.
	conf := jobConfig{
		name:             name,
		handler:          reflect.ValueOf(handler),
		paramsType:       paramsType,
		attempts:         3,
		backoffInSeconds: 5,
	}

	// Optional configuration.
	if l := len(optConfs); l > 1 {
		return errors.New("job %s: too many optional configurations provided", name)
	} else if l == 1 {
		optConf := optConfs[0]
		if optConf.Attempts > 0 {
			conf.attempts = optConf.Attempts
		}
		if b := optConf.Backoff.Nanoseconds() / 1000000000; b > 0 {
			conf.backoffInSeconds = b
		}
	}

	jq.configs[name] = conf
	return nil
}

func (jq *JobQueue) fetcher(fetcherIndex int) {
	jq.fetcherWg.Add(1)
	defer jq.fetcherWg.Done()

	for {
		if st := atomic.LoadInt32(jq.status); st == queueStopping {
			return
		}

		jobId, hasNext, err := jq.maybeNext()
		if err != nil {
			glog.Infof("fetcher[%d], internal error while fetching potential next job: %s", fetcherIndex, err)
		}
		if !hasNext {
			time.Sleep(1 * time.Second)
			continue
		}

		jq.maybeJobIds <- jobId
	}
}

func (jq *JobQueue) worker(workerIndex int) {
	jq.workersWg.Add(1)
	defer jq.workersWg.Done()

	for {
		if st := atomic.LoadInt32(jq.status); st == queueStopping {
			return
		}

		jobId := <-jq.maybeJobIds

		locked, err := jq.attemptLock(jobId)
		if err != nil {
			glog.Infof("worker[%d] internal error while attempting to lock job: %s", workerIndex, err)
		}
		if !locked {
			continue
		}

		rec, err := jq.getJobRecord(jobId)
		if err != nil {
			glog.Infof("worker[%d] internal error while fetching job record: %s", workerIndex, err)
			continue
		}

		jobErr := jq.safeProcess(rec, jq.configs[rec.name])
		if jobErr != nil {
			if rec.remaining > 0 {
				err = jq.reEnqueue(rec)
			} else {
				err = jq.markAs(rec.id, statusFailed)
			}
		} else {
			err = jq.markAs(rec.id, statusCompleted)
		}

		if err != nil {
			glog.Infof("worker[%d] internal error in post process of job: %s", err)
		}
	}

}

func (jq *JobQueue) Start() error {
	// Mark queue as starting.
	if !atomic.CompareAndSwapInt32(jq.status, queueStopped, queueStarting) {
		return errors.New("can only start a queue which is stopped")
	}

	// Start fetchers.
	for i := 0; i < jq.numFetchers; i++ {
		go jq.fetcher(i)
	}

	// Start workers.
	for i := 0; i < jq.numWorkers; i++ {
		go jq.worker(i)
	}

	// Mark queue as running.
	atomic.StoreInt32(jq.status, queueRunning)

	return nil
}

func (jq *JobQueue) Stop() error {
	if !atomic.CompareAndSwapInt32(jq.status, queueRunning, queueStopping) {
		return errors.New("unable to stop")
	}
	close(jq.maybeJobIds)
	jq.workersWg.Wait()
	jq.fetcherWg.Wait()
	atomic.StoreInt32(jq.status, queueStopped)
	return nil
}

func (jq *JobQueue) Enqueue(tx *sql.Tx, name string, params interface{}) (int64, error) {
	conf, ok := jq.configs[name]
	if !ok {
		return -1, errors.New("unknown job: %s", name)
	}

	var (
		paramsAsBytes []byte
		err           error
	)
	if params != nil {
		actualParamsType := reflect.TypeOf(params)
		if !actualParamsType.AssignableTo(conf.paramsType) {
			return -1, errors.New("job %s: incorrect param type, expected %s, got %s",
				conf.name, conf.paramsType, actualParamsType)
		}

		paramsAsBytes, err = json.Marshal(params)
		if err != nil {
			return -1, errors.New("job %s: unable to marshall parms %s", err)
		}
	}

	now := jq.clock.UnixNow()
	res, err := tx.Exec(`
		insert into job_queue (name, params, remaining, status, created_at, schedulable_at)
		values (?, ?, ?, ?, ?, ?)`,
		name, paramsAsBytes, conf.attempts, statusPending, now, now)
	if err != nil {
		return -1, err
	}
	jobId, err := res.LastInsertId()
	if err != nil {
		return -1, err
	}
	return jobId, nil
}

func (jq *JobQueue) Get(jobId int64) (Job, error) {
	rec, err := jq.getJobRecord(jobId)
	if err != nil {
		return nil, err
	}
	conf, ok := jq.configs[rec.name]
	if !ok {
		return nil, errors.New("job %s has not been registered", rec.name)
	}
	return &jobRecordExternal{rec, conf.paramsType}, nil
}

func (jq *JobQueue) safeProcess(rec *jobRecord, conf jobConfig) error {
	var (
		err     error
		elapsed int64
	)

	// Process, with panic handling.
	func() {
		elapsed = time.Now().UnixNano()
		defer func() {
			if r := recover(); r != nil {
				err = errors.New("job %s panicked: %s", conf.name, r)
			}
		}()

		err = func() error {
			params, err := (&jobRecordExternal{rec, conf.paramsType}).Params()
			if err != nil {
				return err
			}
			returns := conf.handler.Call([]reflect.Value{reflect.ValueOf(params)})
			if !returns[0].IsNil() {
				return returns[0].Interface().(error)
			}
			return nil
		}()
	}()

	// Logging.
	elapsed = (time.Now().UnixNano() - elapsed) / 1000000
	if err == nil {
		glog.Infof("job %s(%d) succeeded: attempt=%d, elapsed=%dms",
			rec.name, rec.id,
			rec.attempt(conf), elapsed)
	} else {
		glog.Infof("job %s(%d) failed: attempt=%d, elapsed=%dms, remaining=%d, err=%s",
			rec.name, rec.id,
			rec.attempt(conf), elapsed, rec.remaining, err)
	}

	// Done
	return err
}

func (jq *JobQueue) maybeNext() (int64, bool, error) {
	now := jq.clock.UnixNow()
	rows, err := jq.db.Query(`
		select id from job_queue
		where status = ? and remaining > 0 and schedulable_at <= ?
		order by schedulable_at asc limit 1`,
		statusPending, now)
	if err != nil {
		return -1, false, err
	}
	defer rows.Close()
	if !rows.Next() {
		return -1, false, nil
	}
	var jobId int64
	err = rows.Scan(&jobId)
	if err != nil {
		return -1, false, err
	}
	return jobId, true, nil
}

func (jq *JobQueue) attemptLock(jobId int64) (bool, error) {
	res, err := jq.db.Exec(
		"update job_queue set status = ?, processor_id = ?, remaining = remaining - 1 where id = ? and status = ?",
		statusProcessing, jq.processorId, jobId, statusPending)
	if err != nil {
		return false, err
	}
	changed, err := res.RowsAffected()
	if err != nil {
		return false, err
	}
	return changed == 1, nil
}

func (jq *JobQueue) markAs(jobId int64, status string) error {
	_, err := jq.db.Exec(
		"update job_queue set status = ?, processor_id = null where id = ?",
		status, jobId)
	return err
}

func (jq *JobQueue) reEnqueue(rec *jobRecord) error {
	conf := jq.configs[rec.name]
	backoffMultiplier := 1 << uint(rec.attempt(conf)-1)
	newSchedulableAt := jq.clock.UnixNow() + conf.backoffInSeconds*int64(backoffMultiplier)
	_, err := jq.db.Exec(
		"update job_queue set status = ?, schedulable_at = ?, processor_id = null where id = ?",
		statusPending, newSchedulableAt, rec.id)
	return err
}

func (jq *JobQueue) getJobRecord(jobId int64) (*jobRecord, error) {
	var (
		name          string
		params        []byte
		remaining     int
		schedulableAt int64
	)
	err := jq.db.
		QueryRow(
			"select name, params, remaining, schedulable_at from job_queue where id = ?",
			jobId).
		Scan(&name, &params, &remaining, &schedulableAt)
	if err != nil {
		return nil, err
	}
	rec := jobRecord{
		id:            jobId,
		name:          name,
		params:        params,
		remaining:     remaining,
		schedulableAt: schedulableAt,
	}
	return &rec, nil
}

type jobRecordExternal struct {
	*jobRecord
	paramsType reflect.Type
}

// Assert jobRecord implements the Job interface.
var _ Job = &jobRecordExternal{}

func (rec *jobRecordExternal) ID() int64 {
	return rec.id
}

func (rec *jobRecordExternal) Name() string {
	return rec.name
}

func (rec *jobRecordExternal) Params() (interface{}, error) {
	params := reflect.New(rec.paramsType.Elem()).Interface()
	if rec.params != nil {
		if err := json.Unmarshal(rec.params, &params); err != nil {
			return nil, errors.New("job %s unmarshall error: %s", rec.name, err)
		}
	}
	return params, nil
}
<|endoftext|>


<commit_before>package vaultutil

import (
	vaultapi "github.com/hashicorp/vault/api"
)

func GetGroups(c *vaultapi.Client) (*api.Secret, error) {
	resp, err := c.Logical().List("auth/ldap/groups")
	if err != nil {
		return resp, err
	}
	return resp, err
}
<commit_msg>Adding GetGroupPolicy call<commit_after>package vaultutil

import (
	vaultapi "github.com/hashicorp/vault/api"
)

func GetGroups(c *vaultapi.Client) (*vaultapi.Secret, error) {
	resp, err := c.Logical().List("auth/ldap/groups")
	if err != nil {
		return resp, err
	}
	return resp, err
}

func GetGroupPolicy(c *vaultapi.Client, group_name string) (*vaultapi.Secret, error) {
	resp, err := c.Logical().Read("auth/ldap/groups/" + group_name)
	if err != nil {
		return resp, err
	}
	return resp, err
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"net"
	"os"
	"path"
	"strconv"
	"text/tabwriter"

	"github.com/boltdb/bolt"
	"github.com/jfrazelle/netns/ipallocator"
	"github.com/vishvananda/netlink"
)

type network struct {
	vethPair *netlink.Veth
	ip       net.IP
	pid      int
	status   string
}

func listNetworks() error {
	// open the database
	dbpath := path.Join(stateDir, ipallocator.DBFile)
	db, err := bolt.Open(dbpath, 0666, nil)
	if err != nil {
		return fmt.Errorf("Opening database at %s failed: %v", dbpath, err)
	}
	defer db.Close()

	var networks []network
	if err := db.View(func(tx *bolt.Tx) error {
		// Retrieve the jobs bucket.
		b := tx.Bucket(ipallocator.IPBucket)

		return b.ForEach(func(k, v []byte) error {
			n := network{
				ip: net.ParseIP(string(k)),
			}

			// get the pid
			n.pid, err = strconv.Atoi(string(v))
			if err != nil {
				return fmt.Errorf("parsing pid %s as int failed: %v", v, err)
			}
			// check the process
			_, err := os.FindProcess(n.pid)
			if err != nil {
				n.status = "does not exist"
			} else {
				n.status = "running"
			}

			// get the veth pair from the pid
			n.vethPair, err = vethPair(n.pid, bridgeName)
			if err != nil {
				return fmt.Errorf("Getting vethpair failed for pid %d: %v", n.pid, err)
			}

			// try to check if the veth link exists
			// if a netns exited, it would destroy it along with the exit
			if _, err := netlink.LinkByName(n.vethPair.Attrs().Name); err.Error() == "Link not found" {
				n.status = "destroyed"
			}

			networks = append(networks, n)
			return nil
		})
	}); err != nil {
		return fmt.Errorf("Getting networks from db failed: %v", err)
	}

	w := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0)
	fmt.Fprint(w, "IP\tLOCAL VETH\tPID\tSTATUS\n")
	for _, n := range networks {
		fmt.Fprintf(w, "%s\t%s\t%d\t%s\n", n.ip.String(), n.vethPair.Attrs().Name, n.pid, n.status)
	}
	w.Flush()

	return nil
}
<commit_msg>better list<commit_after>package main

import (
	"fmt"
	"net"
	"os"
	"path"
	"strconv"
	"text/tabwriter"

	"github.com/boltdb/bolt"
	"github.com/jfrazelle/netns/ipallocator"
	"github.com/vishvananda/netlink"
	"github.com/vishvananda/netns"
)

type network struct {
	vethPair *netlink.Veth
	ip       net.IP
	pid      int
	status   string
	fd       netns.NsHandle
}

func listNetworks() error {
	// open the database
	dbpath := path.Join(stateDir, ipallocator.DBFile)
	db, err := bolt.Open(dbpath, 0666, nil)
	if err != nil {
		return fmt.Errorf("Opening database at %s failed: %v", dbpath, err)
	}
	defer db.Close()

	var networks []network
	if err := db.View(func(tx *bolt.Tx) error {
		// Retrieve the jobs bucket.
		b := tx.Bucket(ipallocator.IPBucket)

		return b.ForEach(func(k, v []byte) error {
			n := network{
				ip: net.ParseIP(string(k)),
			}

			// get the pid
			n.pid, err = strconv.Atoi(string(v))
			if err != nil {
				return fmt.Errorf("parsing pid %s as int failed: %v", v, err)
			}
			// check the process
			_, err := os.FindProcess(n.pid)
			if err != nil {
				n.status = "does not exist"
			} else {
				n.status = "running"
			}

			// get the veth pair from the pid
			n.vethPair, err = vethPair(n.pid, bridgeName)
			if err != nil {
				return fmt.Errorf("Getting vethpair failed for pid %d: %v", n.pid, err)
			}

			// try to get the namespace handle
			n.fd, _ = netns.GetFromPid(n.pid)
			if n.fd <= 0 {
				n.status = "destroyed"
			}

			networks = append(networks, n)
			return nil
		})
	}); err != nil {
		return fmt.Errorf("Getting networks from db failed: %v", err)
	}

	w := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0)
	fmt.Fprint(w, "IP\tLOCAL VETH\tPID\tSTATUS\tNS FD\n")
	for _, n := range networks {
		fmt.Fprintf(w, "%s\t%s\t%d\t%s\t%d\n", n.ip.String(), n.vethPair.Attrs().Name, n.pid, n.status, n.fd)
	}
	w.Flush()

	return nil
}
<|endoftext|>


<commit_before>// Package llog is a generic logging library used by leven labs. The log methods
// come in different severities: Debug, Info, Warn, Error, and Fatal.
//
// The log methods take in a string describing the error, and a set of key/value
// pairs giving the specific context around the error. The string is intended to
// always be the same no matter what, while the key/value pairs give information
// like which userID the error happened to, or any other relevant dynamic
// information.
//
// By default logs will be output to Stdout, without a timestamp attached to
// them, and only showing entries of level Info or above. All of these can be
// configured.
//
// All public functions in this package are thread-safe and can be called at any
// time. The public variables in this package are NOT thread-safe and should
// only be modified before any logging takes place
//
// Examples:
//
//	Info("Something important has occurred")
//	Error("Could not open file", llog.KV{"filename": filename}, llog.ErrKV(err))
//
package llog

import (
	"fmt"
	"io"
	"os"
	"sort"
	"strconv"
	"strings"
	"sync"
	"time"
)

// Out is the io.Writer all log entries will be written to. It can be changed to
// anything you like, but the change should happen before any logging occurs. If
// an error occurs while writing to Out the entry will be written to Stdout
// instead
var Out io.Writer = os.Stdout
var defaultOut = os.Stdout

// DisplayTimestamp determines whether or not a timestamp is displayed in the
// log messages. By default one is not displayed. This can be changed by it
// should only be changed before any logging occurs
var DisplayTimestamp bool

// Level describes the severity of a particular log message
type Level int

// All defined log levels
const (
	DebugLevel Level = iota
	InfoLevel
	WarnLevel
	ErrorLevel
	FatalLevel
)

func (l Level) String() string {
	switch l {
	case DebugLevel:
		return "DEBUG"
	case InfoLevel:
		return "INFO"
	case WarnLevel:
		return "WARN"
	case ErrorLevel:
		return "ERROR"
	case FatalLevel:
		return "FATAL"
	}
	return "unknown level"
}

var currLevel = InfoLevel
var currLevelLock sync.RWMutex

// GetLevel returns the current log level
func GetLevel() Level {
	currLevelLock.RLock()
	defer currLevelLock.RUnlock()
	return currLevel
}

// SetLevel sets the current minimum log level which will be written to Out
func SetLevel(l Level) {
	currLevelLock.Lock()
	defer currLevelLock.Unlock()
	currLevel = l
}

// SetLevelFromString attempts to interpret the given string as a log level and
// sets the current log level to that. If the string can't be interpreted an
// error is returned and the log level remains what it was
func SetLevelFromString(ls string) error {
	switch strings.ToUpper(ls) {
	case "DEBUG":
		SetLevel(DebugLevel)
	case "INFO":
		SetLevel(InfoLevel)
	case "WARN":
		SetLevel(WarnLevel)
	case "ERROR":
		SetLevel(ErrorLevel)
	case "FATAL":
		SetLevel(FatalLevel)
	default:
		return fmt.Errorf("unknown log level %q", ls)
	}

	return nil
}

// KV is used to provide context to a log entry in the form of a dynamic set of
// key/value pairs which can be different for every entry.
type KV map[string]interface{}

// Copy returns a copy of the KV being called on. This method will never return
// nil
func (kv KV) Copy() KV {
	nkv := make(KV, len(kv))
	for k, v := range kv {
		nkv[k] = v
	}
	return nkv
}

// Merge takes in multiple KVs and returns a single KV which is the union of all
// the passed in ones. Key/vals on the rightmost of the set take precedence over
// conflicting ones to the left. This function will never return nil
func Merge(kvs ...KV) KV {
	kv := make(KV, len(kvs))
	for i := range kvs {
		for k, v := range kvs[i] {
			kv[k] = v
		}
	}
	return kv
}

// Set returns a copy of the KV being called on with the given key/val set on
// it. The original KV is unaffected
func (kv KV) Set(k string, v interface{}) KV {
	nkv := kv.Copy()
	nkv[k] = v
	return nkv
}

// StringSlice converts the KV into a slice of [2]string entries (first index is
// the key, second is the string form of the value).
func (kv KV) StringSlice() [][2]string {
	slice := make([][2]string, 0, len(kv))
	for kstr, v := range kv {
		vstr := fmt.Sprint(v)
		// TODO this is only here because logstash is dumb and doesn't
		// properly handle escaped quotes. Once
		// https://github.com/elastic/logstash/issues/1645
		// gets figured out this Replace can be removed
		vstr = strings.Replace(vstr, `"`, `'`, -1)
		vstr = strconv.QuoteToASCII(vstr)
		slice = append(slice, [2]string{kstr, vstr})
	}
	sort.Slice(slice, func(i, j int) bool {
		return slice[i][0] < slice[j][0]
	})
	return slice
}

type entry struct {
	level   Level
	msg     string
	kvSlice [][2]string
	blockCh chan struct{} // can be nil
}

var (
	prefix         = []byte("~ ")
	separator      = []byte(" --")
	separatorSpace = append(separator, ' ')
	tsPrefix       = []byte("[")
	tsSuffix       = []byte("] ")
	space          = []byte(" ")
	equals         = []byte("=")
	newline        = []byte("\n")
)

func (e entry) printOut(w io.Writer, displayTS bool) error {
	var err error
	write := func(b []byte) {
		if err == nil {
			_, err = w.Write(b)
		}
	}

	write(prefix)
	if displayTS {
		write(tsPrefix)
		write([]byte(time.Now().String()))
		write(tsSuffix)
	}
	write([]byte(e.level.String()))
	write(separatorSpace)
	write([]byte(e.msg))
	if len(e.kvSlice) > 0 {
		write(separator)
		for _, kve := range e.kvSlice {
			write(space)
			write([]byte(kve[0]))
			write(equals)
			write([]byte(kve[1]))
		}
	}
	write(newline)

	return err
}

type syncer interface {
	Sync()
}

type flusher interface {
	Flush()
}

var entryCh = make(chan entry)
var flushCh = make(chan chan bool)

func init() {
	go func() {
		for {
			select {
			case doneCh := <-flushCh:
				flush()
				close(doneCh)
			case e := <-entryCh:
				err := e.printOut(Out, DisplayTimestamp)

				// If we couldn't write the entry to Out we write an error to that
				// effect to Stdout, then try to write the original entry as well
				if err != nil && Out != defaultOut {
					erre := entry{
						level:   ErrorLevel,
						msg:     "Could not write to error Out",
						kvSlice: ErrKV(err).StringSlice(),
					}
					erre.printOut(defaultOut, DisplayTimestamp)
					e.printOut(defaultOut, DisplayTimestamp)
				}

				// If the error level is fatal this is the last entry we should ever
				// write. We do want to attempt to flush Out though, in case it's
				// buffered, otherwise exiting now will cause the fatal message to
				// never be shown.
				if e.level == FatalLevel {
					flush()
				}

				if e.blockCh != nil {
					close(e.blockCh)
				}
			}
		}
	}()
}

// does a raw flush on Out. Shouldn't be called outside the main loop
func flush() {
	// We try to cast to either an interface with a Sync or a Flush command as a
	// form of ghetto reflection, to see if the writer has either, and use one
	// if found.
	if so, ok := Out.(syncer); ok {
		so.Sync()
	} else if fo, ok := Out.(flusher); ok {
		fo.Flush()
	}
}

func logEntry(l Level, msg string, kvs []KV, blockCh chan struct{}) {
	if l >= GetLevel() {
		entryCh <- entry{
			level:   l,
			msg:     msg,
			kvSlice: Merge(kvs...).StringSlice(),
			blockCh: blockCh,
		}
	}
}

// LogFunc is the function signature used by the different log functions (Debug,
// Info, Warn, Error, and Fatal). It's useful for writing wrapper functions
type LogFunc func(string, ...KV)

// Debug writes a Debug message to Out, with an optional set of key/value pairs
// which will be Merge'd together.
func Debug(msg string, kv ...KV) {
	logEntry(DebugLevel, msg, kv, nil)
}

// Info writes an Info message to Out, with an optional set of key/value pairs
// which will be Merge'd together.
func Info(msg string, kv ...KV) {
	logEntry(InfoLevel, msg, kv, nil)
}

// Warn writes a Warn message to Out, with an optional set of key/value pairs
// which will be Merge'd together.
func Warn(msg string, kv ...KV) {
	logEntry(WarnLevel, msg, kv, nil)
}

// Error writes an Error message to Out, with an optional set of key/value pairs
// which will be Merge'd together.
func Error(msg string, kv ...KV) {
	logEntry(ErrorLevel, msg, kv, nil)
}

// Fatal writes a Fatal message to Out, with an optional set of key/value pairs
// which will be Merge'd together. Once written the process will be exited with
// an exit code of 1
func Fatal(msg string, kv ...KV) {
	blockCh := make(chan struct{})
	logEntry(FatalLevel, msg, kv, blockCh)
	<-blockCh
	os.Exit(1)
}

// Flush will attempts to flush any buffered data in Out. Will block until the
// flushing has been completed
func Flush() {
	doneCh := make(chan bool)
	flushCh <- doneCh
	<-doneCh
}
<commit_msg>don't quote ascii in StringsSlice<commit_after>// Package llog is a generic logging library used by leven labs. The log methods
// come in different severities: Debug, Info, Warn, Error, and Fatal.
//
// The log methods take in a string describing the error, and a set of key/value
// pairs giving the specific context around the error. The string is intended to
// always be the same no matter what, while the key/value pairs give information
// like which userID the error happened to, or any other relevant dynamic
// information.
//
// By default logs will be output to Stdout, without a timestamp attached to
// them, and only showing entries of level Info or above. All of these can be
// configured.
//
// All public functions in this package are thread-safe and can be called at any
// time. The public variables in this package are NOT thread-safe and should
// only be modified before any logging takes place
//
// Examples:
//
//	Info("Something important has occurred")
//	Error("Could not open file", llog.KV{"filename": filename}, llog.ErrKV(err))
//
package llog

import (
	"fmt"
	"io"
	"os"
	"sort"
	"strconv"
	"strings"
	"sync"
	"time"
)

// Out is the io.Writer all log entries will be written to. It can be changed to
// anything you like, but the change should happen before any logging occurs. If
// an error occurs while writing to Out the entry will be written to Stdout
// instead
var Out io.Writer = os.Stdout
var defaultOut = os.Stdout

// DisplayTimestamp determines whether or not a timestamp is displayed in the
// log messages. By default one is not displayed. This can be changed by it
// should only be changed before any logging occurs
var DisplayTimestamp bool

// Level describes the severity of a particular log message
type Level int

// All defined log levels
const (
	DebugLevel Level = iota
	InfoLevel
	WarnLevel
	ErrorLevel
	FatalLevel
)

func (l Level) String() string {
	switch l {
	case DebugLevel:
		return "DEBUG"
	case InfoLevel:
		return "INFO"
	case WarnLevel:
		return "WARN"
	case ErrorLevel:
		return "ERROR"
	case FatalLevel:
		return "FATAL"
	}
	return "unknown level"
}

var currLevel = InfoLevel
var currLevelLock sync.RWMutex

// GetLevel returns the current log level
func GetLevel() Level {
	currLevelLock.RLock()
	defer currLevelLock.RUnlock()
	return currLevel
}

// SetLevel sets the current minimum log level which will be written to Out
func SetLevel(l Level) {
	currLevelLock.Lock()
	defer currLevelLock.Unlock()
	currLevel = l
}

// SetLevelFromString attempts to interpret the given string as a log level and
// sets the current log level to that. If the string can't be interpreted an
// error is returned and the log level remains what it was
func SetLevelFromString(ls string) error {
	switch strings.ToUpper(ls) {
	case "DEBUG":
		SetLevel(DebugLevel)
	case "INFO":
		SetLevel(InfoLevel)
	case "WARN":
		SetLevel(WarnLevel)
	case "ERROR":
		SetLevel(ErrorLevel)
	case "FATAL":
		SetLevel(FatalLevel)
	default:
		return fmt.Errorf("unknown log level %q", ls)
	}

	return nil
}

// KV is used to provide context to a log entry in the form of a dynamic set of
// key/value pairs which can be different for every entry.
type KV map[string]interface{}

// Copy returns a copy of the KV being called on. This method will never return
// nil
func (kv KV) Copy() KV {
	nkv := make(KV, len(kv))
	for k, v := range kv {
		nkv[k] = v
	}
	return nkv
}

// Merge takes in multiple KVs and returns a single KV which is the union of all
// the passed in ones. Key/vals on the rightmost of the set take precedence over
// conflicting ones to the left. This function will never return nil
func Merge(kvs ...KV) KV {
	kv := make(KV, len(kvs))
	for i := range kvs {
		for k, v := range kvs[i] {
			kv[k] = v
		}
	}
	return kv
}

// Set returns a copy of the KV being called on with the given key/val set on
// it. The original KV is unaffected
func (kv KV) Set(k string, v interface{}) KV {
	nkv := kv.Copy()
	nkv[k] = v
	return nkv
}

// StringSlice converts the KV into a slice of [2]string entries (first index is
// the key, second is the string form of the value).
func (kv KV) StringSlice() [][2]string {
	slice := make([][2]string, 0, len(kv))
	for kstr, v := range kv {
		vstr := fmt.Sprint(v)
		// TODO this is only here because logstash is dumb and doesn't
		// properly handle escaped quotes. Once
		// https://github.com/elastic/logstash/issues/1645
		// gets figured out this Replace can be removed
		vstr = strings.Replace(vstr, `"`, `'`, -1)
		slice = append(slice, [2]string{kstr, vstr})
	}
	sort.Slice(slice, func(i, j int) bool {
		return slice[i][0] < slice[j][0]
	})
	return slice
}

type entry struct {
	level   Level
	msg     string
	kvSlice [][2]string
	blockCh chan struct{} // can be nil
}

var (
	prefix         = []byte("~ ")
	separator      = []byte(" --")
	separatorSpace = append(separator, ' ')
	tsPrefix       = []byte("[")
	tsSuffix       = []byte("] ")
	space          = []byte(" ")
	equals         = []byte("=")
	newline        = []byte("\n")
)

func (e entry) printOut(w io.Writer, displayTS bool) error {
	var err error
	write := func(b []byte) {
		if err == nil {
			_, err = w.Write(b)
		}
	}

	write(prefix)
	if displayTS {
		write(tsPrefix)
		write([]byte(time.Now().String()))
		write(tsSuffix)
	}
	write([]byte(e.level.String()))
	write(separatorSpace)
	write([]byte(e.msg))
	if len(e.kvSlice) > 0 {
		write(separator)
		for _, kve := range e.kvSlice {
			write(space)
			write([]byte(kve[0]))
			write(equals)
			write([]byte(strconv.QuoteToASCII(kve[1])))
		}
	}
	write(newline)

	return err
}

type syncer interface {
	Sync()
}

type flusher interface {
	Flush()
}

var entryCh = make(chan entry)
var flushCh = make(chan chan bool)

func init() {
	go func() {
		for {
			select {
			case doneCh := <-flushCh:
				flush()
				close(doneCh)
			case e := <-entryCh:
				err := e.printOut(Out, DisplayTimestamp)

				// If we couldn't write the entry to Out we write an error to that
				// effect to Stdout, then try to write the original entry as well
				if err != nil && Out != defaultOut {
					erre := entry{
						level:   ErrorLevel,
						msg:     "Could not write to error Out",
						kvSlice: ErrKV(err).StringSlice(),
					}
					erre.printOut(defaultOut, DisplayTimestamp)
					e.printOut(defaultOut, DisplayTimestamp)
				}

				// If the error level is fatal this is the last entry we should ever
				// write. We do want to attempt to flush Out though, in case it's
				// buffered, otherwise exiting now will cause the fatal message to
				// never be shown.
				if e.level == FatalLevel {
					flush()
				}

				if e.blockCh != nil {
					close(e.blockCh)
				}
			}
		}
	}()
}

// does a raw flush on Out. Shouldn't be called outside the main loop
func flush() {
	// We try to cast to either an interface with a Sync or a Flush command as a
	// form of ghetto reflection, to see if the writer has either, and use one
	// if found.
	if so, ok := Out.(syncer); ok {
		so.Sync()
	} else if fo, ok := Out.(flusher); ok {
		fo.Flush()
	}
}

func logEntry(l Level, msg string, kvs []KV, blockCh chan struct{}) {
	if l >= GetLevel() {
		entryCh <- entry{
			level:   l,
			msg:     msg,
			kvSlice: Merge(kvs...).StringSlice(),
			blockCh: blockCh,
		}
	}
}

// LogFunc is the function signature used by the different log functions (Debug,
// Info, Warn, Error, and Fatal). It's useful for writing wrapper functions
type LogFunc func(string, ...KV)

// Debug writes a Debug message to Out, with an optional set of key/value pairs
// which will be Merge'd together.
func Debug(msg string, kv ...KV) {
	logEntry(DebugLevel, msg, kv, nil)
}

// Info writes an Info message to Out, with an optional set of key/value pairs
// which will be Merge'd together.
func Info(msg string, kv ...KV) {
	logEntry(InfoLevel, msg, kv, nil)
}

// Warn writes a Warn message to Out, with an optional set of key/value pairs
// which will be Merge'd together.
func Warn(msg string, kv ...KV) {
	logEntry(WarnLevel, msg, kv, nil)
}

// Error writes an Error message to Out, with an optional set of key/value pairs
// which will be Merge'd together.
func Error(msg string, kv ...KV) {
	logEntry(ErrorLevel, msg, kv, nil)
}

// Fatal writes a Fatal message to Out, with an optional set of key/value pairs
// which will be Merge'd together. Once written the process will be exited with
// an exit code of 1
func Fatal(msg string, kv ...KV) {
	blockCh := make(chan struct{})
	logEntry(FatalLevel, msg, kv, blockCh)
	<-blockCh
	os.Exit(1)
}

// Flush will attempts to flush any buffered data in Out. Will block until the
// flushing has been completed
func Flush() {
	doneCh := make(chan bool)
	flushCh <- doneCh
	<-doneCh
}
<|endoftext|>


<commit_before>package main

import (
	"strconv"
	"sync"
	"time"

	"github.com/gizak/termui"
)

var (
	mu                 sync.Mutex
	gcCounter, stwTime int
	gcPercent          *termui.Gauge
	gcRate             *termui.Par
	liveHeap, goalHeap *termui.LineChart
	wallTime, cpuTime  *termui.BarChart
)

func refreshGCSummary() {
	ticker := time.NewTicker(5 * time.Second)

	for range ticker.C {
		gcRate.Text = strconv.Itoa(gcCounter / 5)
		gcPercent.Percent = 100 * stwTime / 5e6

		termui.Render(termui.Body)

		mu.Lock()
		gcCounter = 0
		stwTime = 0
		mu.Unlock()
	}
}

func refreshGraphs(data gcInfo) {
	liveHeap.Data = append(liveHeap.Data[1:], data.size.live)
	goalHeap.Data = append(goalHeap.Data[1:], data.size.goal)

	wallTime.Data = []int{
		data.wallTime.sweepTermination,
		data.wallTime.markAndSwap,
		data.wallTime.markTermination,
	}
	cpuTime.Data = []int{
		data.cpuTime.sweepTermination,
		data.cpuTime.markAndSwap,
		data.cpuTime.markTermination,
	}

	mu.Lock()
	gcCounter++
	stwTime += data.wallTime.sweepTermination
	stwTime += data.wallTime.markTermination
	mu.Unlock()
}

func sendEvents() {
	for data := range readStdin() {
		termui.SendCustomEvt("/feed", data)
	}
}

func main() {
	if err := termui.Init(); err != nil {
		panic(err)
	}
	defer termui.Close()

	gcPercent = newGauge("Percentage of Time Spent in GC")
	gcRate = newPar("GC Events per Second")

	liveHeap = newLineChart("Live heap size, MB")
	goalHeap = newLineChart("Goal heap size, MB")

	wallTime = newBarChart("Wall-clock time, us",
		[]string{"STW Sweep Termination", "Concurrent Mark & Swap", "STW Mark Termination"})
	cpuTime = newBarChart("CPU time, us",
		[]string{"STW Sweep Termination", "Concurrent Mark & Swap", "STW Mark Termination"})

	termui.Body.AddRows(
		termui.NewRow(
			termui.NewCol(10, 0, gcPercent),
			termui.NewCol(2, 0, gcRate)),
		termui.NewRow(
			termui.NewCol(12, 0, liveHeap)),
		termui.NewRow(
			termui.NewCol(12, 0, goalHeap)),
		termui.NewRow(
			termui.NewCol(6, 0, wallTime),
			termui.NewCol(6, 0, cpuTime)))

	termui.Body.Align()

	termui.Handle("/sys/kbd/q", func(termui.Event) {
		termui.StopLoop()
	})

	termui.Handle("/feed", func(e termui.Event) {
		if data, ok := e.Data.(gcInfo); ok {
			refreshGraphs(data)

			termui.Render(termui.Body)
		}
	})

	termui.Render(termui.Body)

	go sendEvents()

	go refreshGCSummary()

	termui.Loop()
}
<commit_msg>Serialize UI rendering<commit_after>package main

import (
	"strconv"
	"sync"
	"time"

	"github.com/gizak/termui"
)

var (
	dataMutex, renderMutex sync.Mutex
	gcCounter, stwTime     int
	gcPercent              *termui.Gauge
	gcRate                 *termui.Par
	liveHeap, goalHeap     *termui.LineChart
	wallTime, cpuTime      *termui.BarChart
)

func render() {
	renderMutex.Lock()
	defer renderMutex.Unlock()
	termui.Render(termui.Body)
}

func refreshGCSummary() {
	ticker := time.NewTicker(5 * time.Second)

	for range ticker.C {
		dataMutex.Lock()

		gcRate.Text = strconv.Itoa(gcCounter / 5)
		gcPercent.Percent = 100 * stwTime / 5e6

		gcCounter = 0
		stwTime = 0

		dataMutex.Unlock()

		render()
	}
}

func refreshGraphs(data gcInfo) {
	liveHeap.Data = append(liveHeap.Data[1:], data.size.live)
	goalHeap.Data = append(goalHeap.Data[1:], data.size.goal)

	wallTime.Data = []int{
		data.wallTime.sweepTermination,
		data.wallTime.markAndSwap,
		data.wallTime.markTermination,
	}
	cpuTime.Data = []int{
		data.cpuTime.sweepTermination,
		data.cpuTime.markAndSwap,
		data.cpuTime.markTermination,
	}

	dataMutex.Lock()
	gcCounter++
	stwTime += data.wallTime.sweepTermination
	stwTime += data.wallTime.markTermination
	dataMutex.Unlock()

	render()
}

func sendEvents() {
	for data := range readStdin() {
		termui.SendCustomEvt("/feed", data)
	}
}

func main() {
	if err := termui.Init(); err != nil {
		panic(err)
	}
	defer termui.Close()

	gcPercent = newGauge("Percentage of Time Spent in GC")
	gcRate = newPar("GC Events per Second")

	liveHeap = newLineChart("Live heap size, MB")
	goalHeap = newLineChart("Goal heap size, MB")

	wallTime = newBarChart("Wall-clock time, us",
		[]string{"STW Sweep Termination", "Concurrent Mark & Swap", "STW Mark Termination"})
	cpuTime = newBarChart("CPU time, us",
		[]string{"STW Sweep Termination", "Concurrent Mark & Swap", "STW Mark Termination"})

	termui.Body.AddRows(
		termui.NewRow(
			termui.NewCol(10, 0, gcPercent),
			termui.NewCol(2, 0, gcRate)),
		termui.NewRow(
			termui.NewCol(12, 0, liveHeap)),
		termui.NewRow(
			termui.NewCol(12, 0, goalHeap)),
		termui.NewRow(
			termui.NewCol(6, 0, wallTime),
			termui.NewCol(6, 0, cpuTime)))

	termui.Body.Align()

	termui.Handle("/sys/kbd/q", func(termui.Event) {
		termui.StopLoop()
	})

	termui.Handle("/feed", func(e termui.Event) {
		if data, ok := e.Data.(gcInfo); ok {
			refreshGraphs(data)

		}
	})

	render()

	go sendEvents()

	go refreshGCSummary()

	termui.Loop()
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"os"
	"sync"
	"time"
)

const (
	// DefaultBaseURL is the API root URL
	defaultBaseURL = "https://pikacloud.com/api/"
	apiVersion     = "v1"
)

// Agent describes the agent
type Agent struct {
	Client   *Client
	ID       string `json:"aid"`
	PingURL  string `json:"ping_url"`
	TTL      int    `json:"ttl"`
	Hostname string `json:"hostname"`
}

// CreateAgentOptions represents the agent Create() options
type CreateAgentOptions struct {
	Hostname string `json:"hostname"`
}

// Create an agent
func (agent *Agent) Create(opt *CreateAgentOptions) error {
	status, err := agent.Client.Post("run/agents/", opt, &agent)
	if err != nil {
		return fmt.Errorf(err.Error())
	}
	if status != 200 {
		return fmt.Errorf("Failed to create agent http code: %d", status)
	} else {
		log.Printf("Agent %s registered with hostname %s\n", agent.ID, agent.Hostname)
	}
	return nil
}

func (agent *Agent) infinitePing() {
	for {
		err := agent.Ping()
		if err != nil {
			log.Println(err)
		}
		log.Println("Ping OK")
		time.Sleep(3 * time.Second)
	}
}

// Ping agent
func (agent *Agent) Ping() error {
	pingURI := fmt.Sprintf("run/agents/%s/ping/", agent.ID)
	// for {
	status, err := agent.Client.Post(pingURI, nil, nil)
	if err != nil {
		log.Println("Trying to register lost agent")
		newAgentOpts := CreateAgentOptions{
			Hostname: agent.Hostname,
		}
		return agent.Create(&newAgentOpts)
	}
	if status != 200 {
		return fmt.Errorf("Ping to %s returns %d\n", pingURI, status)
	}
	return nil
}

func main() {
	apiToken := os.Getenv("PIKACLOUD_API_TOKEN")
	baseURL := os.Getenv("PIKACLOUD_AGENT_URL")
	if apiToken == "" {
		log.Fatalln("PIKACLOUD_API_TOKEN is empty")
	}
	hostname, err := os.Hostname()
	if err != nil {
		log.Fatalf("Unable to retrieve agent hostname: %v", err)
	}
	if baseURL == "" {
		baseURL = defaultBaseURL
	}
	c := NewClient(apiToken)
	c.BaseURL = baseURL

	agent := Agent{
		Client:   c,
		Hostname: hostname,
	}
	newAgentOpts := CreateAgentOptions{
		Hostname: hostname,
	}
	err = agent.Create(&newAgentOpts)
	if err != nil {
		log.Fatal(err)
	}
	wg := sync.WaitGroup{}
	defer wg.Wait()
	wg.Add(1)
	go agent.infinitePing()
	wg.Add(1)
	go agent.syncDockerInfo()
	wg.Add(1)
	go agent.syncDockerContainers()

}
<commit_msg>fix grammar<commit_after>package main

import (
	"fmt"
	"log"
	"os"
	"sync"
	"time"
)

const (
	// DefaultBaseURL is the API root URL
	defaultBaseURL = "https://pikacloud.com/api/"
	apiVersion     = "v1"
)

// Agent describes the agent
type Agent struct {
	Client   *Client
	ID       string `json:"aid"`
	PingURL  string `json:"ping_url"`
	TTL      int    `json:"ttl"`
	Hostname string `json:"hostname"`
}

// CreateAgentOptions represents the agent Create() options
type CreateAgentOptions struct {
	Hostname string `json:"hostname"`
}

// Create an agent
func (agent *Agent) Create(opt *CreateAgentOptions) error {
	status, err := agent.Client.Post("run/agents/", opt, &agent)
	if err != nil {
		return fmt.Errorf(err.Error())
	}
	if status != 200 {
		return fmt.Errorf("Failed to create agent http code: %d", status)
	}
	log.Printf("Agent %s registered with hostname %s\n", agent.ID, agent.Hostname)
	return nil
}

func (agent *Agent) infinitePing() {
	for {
		err := agent.Ping()
		if err != nil {
			log.Println(err)
		}
		log.Println("Ping OK")
		time.Sleep(3 * time.Second)
	}
}

// Ping agent
func (agent *Agent) Ping() error {
	pingURI := fmt.Sprintf("run/agents/%s/ping/", agent.ID)
	// for {
	status, err := agent.Client.Post(pingURI, nil, nil)
	if err != nil {
		log.Println("Trying to register lost agent")
		newAgentOpts := CreateAgentOptions{
			Hostname: agent.Hostname,
		}
		return agent.Create(&newAgentOpts)
	}
	if status != 200 {
		return fmt.Errorf("Ping to %s returns %d\n", pingURI, status)
	}
	return nil
}

func main() {
	apiToken := os.Getenv("PIKACLOUD_API_TOKEN")
	baseURL := os.Getenv("PIKACLOUD_AGENT_URL")
	if apiToken == "" {
		log.Fatalln("PIKACLOUD_API_TOKEN is empty")
	}
	hostname, err := os.Hostname()
	if err != nil {
		log.Fatalf("Unable to retrieve agent hostname: %v", err)
	}
	if baseURL == "" {
		baseURL = defaultBaseURL
	}
	c := NewClient(apiToken)
	c.BaseURL = baseURL

	agent := Agent{
		Client:   c,
		Hostname: hostname,
	}
	newAgentOpts := CreateAgentOptions{
		Hostname: hostname,
	}
	err = agent.Create(&newAgentOpts)
	if err != nil {
		log.Fatal(err)
	}
	wg := sync.WaitGroup{}
	defer wg.Wait()
	wg.Add(1)
	go agent.infinitePing()
	wg.Add(1)
	go agent.syncDockerInfo()
	wg.Add(1)
	go agent.syncDockerContainers()

}
<|endoftext|>


<commit_before>package main

import "fmt"

func main() {
	fmt.Printf("Hello, world.\n")
}
<commit_msg>New server settings<commit_after>package main

import "net/http"

func main() {
	http.Handle("/", http.FileServer(http.Dir("src")))
	http.ListenAndServe(":8000", nil)
}
<|endoftext|>