<commit_before>package vsphere

import (
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"html/template"
	"time"

	"github.com/rockbears/log"
	"github.com/vmware/govmomi/object"
	"github.com/vmware/govmomi/vim25/types"

	"github.com/ovh/cds/sdk"
	"github.com/ovh/cds/sdk/hatchery"
)

type annotation struct {
	HatcheryName            string    `json:"hatchery_name"`
	WorkerName              string    `json:"worker_name"`
	RegisterOnly            bool      `json:"register_only"`
	WorkerModelPath         string    `json:"worker_model_path"`
	WorkerModelLastModified string    `json:"worker_model_last_modified"`
	Model                   bool      `json:"model"`
	ToDelete                bool      `json:"to_delete"`
	Created                 time.Time `json:"created"`
}

// SpawnWorker creates a new vm instance
func (h *HatcheryVSphere) SpawnWorker(ctx context.Context, spawnArgs hatchery.SpawnArguments) error {
	if spawnArgs.JobID == 0 && !spawnArgs.RegisterOnly {
		return sdk.WithStack(fmt.Errorf("no job ID and no register"))
	}

	var vm *object.VirtualMachine
	_, err := h.getModelByName(ctx, spawnArgs.Model.Name)

	if err != nil || spawnArgs.Model.NeedRegistration {
		// Generate worker model vm
		log.Info(ctx, "creating virtual machine model %q", spawnArgs.Model.Name)
		vm, err = h.createVMModel(ctx, *spawnArgs.Model, spawnArgs.WorkerName)
		if err != nil {
			log.Error(ctx, "Unable to create VM Model: %v", err)
			return err
		}
	}

	if vm == nil {
		log.Info(ctx, "creating virtual machine %q", spawnArgs.Model.Name)
		if vm, err = h.finder.VirtualMachine(ctx, spawnArgs.Model.Name); err != nil {
			return sdk.WrapError(err, "cannot find virtual machine with this model")
		}
	}

	annot := annotation{
		HatcheryName:            h.Name(),
		WorkerName:              spawnArgs.WorkerName,
		RegisterOnly:            spawnArgs.RegisterOnly,
		WorkerModelLastModified: fmt.Sprintf("%d", spawnArgs.Model.UserLastModified.Unix()),
		WorkerModelPath:         spawnArgs.ModelName(),
		Created:                 time.Now(),
	}

	cloneSpec, folder, err := h.createVMConfig(ctx, vm, annot, spawnArgs.WorkerName)
	if err != nil {
		return sdk.WrapError(err, "cannot create VM configuration")
	}

	log.Info(ctx, "Create vm to exec worker %s", spawnArgs.WorkerName)
	defer log.Info(ctx, "Terminate to create vm for worker %s", spawnArgs.WorkerName)
	task, errC := vm.Clone(ctx, folder, spawnArgs.WorkerName, *cloneSpec)
	if errC != nil {
		return sdk.WrapError(errC, "cannot clone VM")
	}

	info, err := task.WaitForResult(ctx, nil)
	if err != nil || info.State == types.TaskInfoStateError {
		return sdk.WrapError(err, "state in error")
	}

	// Wait for IP
	vmWorker := object.NewVirtualMachine(h.vclient.Client, info.Result.(types.ManagedObjectReference))
	ip, err := vmWorker.WaitForIP(ctx, true)
	if err != nil {
		return sdk.WrapError(err, "SpawnWorker> cannot get an ip")
	}
	log.Debug(ctx, "SpawnWorker>  New IP: %s", ip)

	return h.launchScriptWorker(ctx, spawnArgs.WorkerName, spawnArgs.JobID, spawnArgs.WorkerToken, *spawnArgs.Model, spawnArgs.RegisterOnly, vmWorker)
}

// createVMModel create a model for a specific worker model
func (h *HatcheryVSphere) createVMModel(ctx context.Context, model sdk.Model, workerName string) (*object.VirtualMachine, error) {
	log.Info(ctx, "Create vm model %s from %s", model.Name, model.ModelVirtualMachine.Image)
	defer func() {
		log.Info(ctx, "Create vm model %s from %s DONE", model.Name, model.ModelVirtualMachine.Image)
	}()

	vm, errV := h.finder.VirtualMachine(ctx, model.ModelVirtualMachine.Image)
	if errV != nil {
		return vm, sdk.WrapError(errV, "createVMModel> Cannot find virtual machine")
	}

	annot := annotation{
		HatcheryName:            h.Name(),
		WorkerModelLastModified: fmt.Sprintf("%d", model.UserLastModified.Unix()),
		WorkerModelPath:         model.Path(),
		Model:                   true,
		Created:                 time.Now(),
	}

	cloneSpec, folder, errCfg := h.createVMConfig(ctx, vm, annot, workerName)
	if errCfg != nil {
		return vm, sdk.WrapError(errCfg, "createVMModel> cannot create VM configuration")
	}

	log.Info(ctx, "cloning %q to %s", workerName, model.Name+"-tmp")

	task, errC := vm.Clone(ctx, folder, model.Name+"-tmp", *cloneSpec)
	if errC != nil {
		return vm, sdk.WrapError(errC, "createVMModel> cannot clone VM")
	}

	info, errWr := task.WaitForResult(ctx, nil)
	if errWr != nil || info.State == types.TaskInfoStateError {
		return vm, sdk.WrapError(errWr, "createVMModel> state in error")
	}

	vm = object.NewVirtualMachine(h.vclient.Client, info.Result.(types.ManagedObjectReference))

	ip, errW := vm.WaitForIP(ctx, true)
	if errW != nil {
		return vm, sdk.WrapError(errW, "createVMModel> cannot get an ip")
	}

	log.Info(ctx, "virtual machine %q has IP %q", vm.Name(), ip)

	if _, errS := h.launchClientOp(ctx, vm, model.ModelVirtualMachine, model.ModelVirtualMachine.PostCmd, nil); errS != nil {
		log.Warn(ctx, "createVMModel> cannot start program %s", errS)
		annot := annotation{ToDelete: true}
		if annotStr, err := json.Marshal(annot); err == nil {
			vm.Reconfigure(ctx, types.VirtualMachineConfigSpec{
				Annotation: string(annotStr),
			})
		}
	}

	ctxTo, cancel := context.WithTimeout(ctx, 10*time.Minute)
	defer cancel()
	if err := vm.WaitForPowerState(ctxTo, types.VirtualMachinePowerStatePoweredOff); err != nil {
		return nil, sdk.WrapError(err, "cannot wait for power state result")
	}

	modelFound, err := h.getModelByName(ctx, model.Name)
	if err == nil {
		if err := h.deleteServer(modelFound); err != nil {
			log.Warn(ctx, "createVMModel> Cannot delete previous model %s : %s", model.Name, err)
		}
	}

	ctxTo, cancel = context.WithTimeout(ctx, 15*time.Second)
	defer cancel()
	task, errR := vm.Rename(ctxTo, model.Name)
	if errR != nil {
		return vm, sdk.WrapError(errR, "createVMModel> Cannot rename model %s", model.Name)
	}

	ctxTo, cancel = context.WithTimeout(ctx, 15*time.Second)
	defer cancel()
	if _, err := task.WaitForResult(ctxTo, nil); err != nil {
		return vm, sdk.WrapError(err, "error on waiting result for vm renaming %s", model.Name)
	}

	if err := vm.MarkAsTemplate(ctx); err != nil {
		return vm, sdk.WrapError(err, "unable to mark vm as template")
	}

	return vm, nil
}

// launchScriptWorker launch a script on the worker
func (h *HatcheryVSphere) launchScriptWorker(ctx context.Context, name string, jobID int64, token string, model sdk.Model, registerOnly bool, vm *object.VirtualMachine) error {
	ctxTo, cancel := context.WithTimeout(ctx, 2*time.Minute)
	defer cancel()

	if _, errW := vm.WaitForIP(ctxTo); errW != nil {
		return sdk.WrapError(errW, "createVMModel> error on waiting ip")
	}

	env := []string{
		"CDS_FROM_WORKER_IMAGE=true",
	}

	env = append(env, h.getGraylogGrpcEnv(model)...)

	udata := model.ModelVirtualMachine.PreCmd + "\n" + model.ModelVirtualMachine.Cmd

	if registerOnly {
		udata += " register"
	}
	udata += "\n" + model.ModelVirtualMachine.PostCmd

	tmpl, errt := template.New("udata").Parse(udata)
	if errt != nil {
		return errt
	}
	udataParam := sdk.WorkerArgs{
		API:               h.Configuration().API.HTTP.URL,
		Name:              name,
		Token:             token,
		Model:             model.Group.Name + "/" + model.Name,
		HatcheryName:      h.Name(),
		TTL:               h.Config.WorkerTTL,
		FromWorkerImage:   true,
		GraylogHost:       h.Configuration().Provision.WorkerLogsOptions.Graylog.Host,
		GraylogPort:       h.Configuration().Provision.WorkerLogsOptions.Graylog.Port,
		GraylogExtraKey:   h.Configuration().Provision.WorkerLogsOptions.Graylog.ExtraKey,
		GraylogExtraValue: h.Configuration().Provision.WorkerLogsOptions.Graylog.ExtraValue,
		WorkflowJobID:     jobID,
	}

	var buffer bytes.Buffer
	if err := tmpl.Execute(&buffer, udataParam); err != nil {
		return err
	}

	if _, errS := h.launchClientOp(ctx, vm, model.ModelVirtualMachine, buffer.String(), env); errS != nil {
		log.Warn(ctx, "launchScript> cannot start program %s", errS)

		// tag vm to delete
		annot := annotation{ToDelete: true}
		if annotStr, err := json.Marshal(annot); err == nil {
			vm.Reconfigure(ctx, types.VirtualMachineConfigSpec{
				Annotation: string(annotStr),
			})
		}

		return errS
	}

	return nil
}
<commit_msg>feat(hatchery/vsphere): add logs in createVMModel function (#5689)<commit_after>package vsphere

import (
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"html/template"
	"time"

	"github.com/rockbears/log"
	"github.com/vmware/govmomi/object"
	"github.com/vmware/govmomi/vim25/types"

	"github.com/ovh/cds/sdk"
	"github.com/ovh/cds/sdk/hatchery"
)

type annotation struct {
	HatcheryName            string    `json:"hatchery_name"`
	WorkerName              string    `json:"worker_name"`
	RegisterOnly            bool      `json:"register_only"`
	WorkerModelPath         string    `json:"worker_model_path"`
	WorkerModelLastModified string    `json:"worker_model_last_modified"`
	Model                   bool      `json:"model"`
	ToDelete                bool      `json:"to_delete"`
	Created                 time.Time `json:"created"`
}

// SpawnWorker creates a new vm instance
func (h *HatcheryVSphere) SpawnWorker(ctx context.Context, spawnArgs hatchery.SpawnArguments) error {
	if spawnArgs.JobID == 0 && !spawnArgs.RegisterOnly {
		return sdk.WithStack(fmt.Errorf("no job ID and no register"))
	}

	var vm *object.VirtualMachine
	_, err := h.getModelByName(ctx, spawnArgs.Model.Name)

	if err != nil || spawnArgs.Model.NeedRegistration {
		// Generate worker model vm
		log.Info(ctx, "creating virtual machine model %q", spawnArgs.Model.Name)
		vm, err = h.createVMModel(ctx, *spawnArgs.Model, spawnArgs.WorkerName)
		if err != nil {
			log.Error(ctx, "Unable to create VM Model: %v", err)
			return err
		}
	}

	if vm == nil {
		log.Info(ctx, "creating virtual machine %q", spawnArgs.Model.Name)
		if vm, err = h.finder.VirtualMachine(ctx, spawnArgs.Model.Name); err != nil {
			return sdk.WrapError(err, "cannot find virtual machine with this model")
		}
	}

	annot := annotation{
		HatcheryName:            h.Name(),
		WorkerName:              spawnArgs.WorkerName,
		RegisterOnly:            spawnArgs.RegisterOnly,
		WorkerModelLastModified: fmt.Sprintf("%d", spawnArgs.Model.UserLastModified.Unix()),
		WorkerModelPath:         spawnArgs.ModelName(),
		Created:                 time.Now(),
	}

	cloneSpec, folder, err := h.createVMConfig(ctx, vm, annot, spawnArgs.WorkerName)
	if err != nil {
		return sdk.WrapError(err, "cannot create VM configuration")
	}

	log.Info(ctx, "Create vm to exec worker %s", spawnArgs.WorkerName)
	defer log.Info(ctx, "Terminate to create vm for worker %s", spawnArgs.WorkerName)
	task, errC := vm.Clone(ctx, folder, spawnArgs.WorkerName, *cloneSpec)
	if errC != nil {
		return sdk.WrapError(errC, "cannot clone VM")
	}

	info, err := task.WaitForResult(ctx, nil)
	if err != nil || info.State == types.TaskInfoStateError {
		return sdk.WrapError(err, "state in error")
	}

	// Wait for IP
	vmWorker := object.NewVirtualMachine(h.vclient.Client, info.Result.(types.ManagedObjectReference))
	ip, err := vmWorker.WaitForIP(ctx, true)
	if err != nil {
		return sdk.WrapError(err, "SpawnWorker> cannot get an ip")
	}
	log.Debug(ctx, "SpawnWorker>  New IP: %s", ip)

	return h.launchScriptWorker(ctx, spawnArgs.WorkerName, spawnArgs.JobID, spawnArgs.WorkerToken, *spawnArgs.Model, spawnArgs.RegisterOnly, vmWorker)
}

// createVMModel create a model for a specific worker model
func (h *HatcheryVSphere) createVMModel(ctx context.Context, model sdk.Model, workerName string) (vm *object.VirtualMachine, err error) {
	ctx, cancel := context.WithTimeout(ctx, 10*time.Minute)
	defer cancel()

	log.Info(ctx, "Create vm model %q from %q", model.Name, model.ModelVirtualMachine.Image)
	defer func() {
		if err != nil {
			ctx = sdk.ContextWithStacktrace(ctx, err)
			log.Error(ctx, "Create vm model %s from %s ERROR: %v", model.Name, model.ModelVirtualMachine.Image, err)
		} else {
			log.Info(ctx, "Create vm model %s from %s DONE", model.Name, model.ModelVirtualMachine.Image)
		}
	}()

	vm, err = h.finder.VirtualMachine(ctx, model.ModelVirtualMachine.Image)
	if err != nil {
		return vm, sdk.WrapError(err, "createVMModel> Cannot find virtual machine")
	}

	annot := annotation{
		HatcheryName:            h.Name(),
		WorkerModelLastModified: fmt.Sprintf("%d", model.UserLastModified.Unix()),
		WorkerModelPath:         model.Path(),
		Model:                   true,
		Created:                 time.Now(),
	}

	cloneSpec, folder, err := h.createVMConfig(ctx, vm, annot, workerName)
	if err != nil {
		return vm, sdk.WrapError(err, "createVMModel> cannot create VM configuration")
	}

	log.Info(ctx, "creating worker %q by cloning vm to %q ", workerName, model.Name+"-tmp")

	task, err := vm.Clone(ctx, folder, model.Name+"-tmp", *cloneSpec)
	if err != nil {
		return vm, sdk.WrapError(err, "createVMModel> cannot clone VM")
	}

	log.Debug(ctx, "waiting for result...")

	info, err := task.WaitForResult(ctx, nil)
	if err != nil || info.State == types.TaskInfoStateError {
		return vm, sdk.WrapError(err, "createVMModel> state in error")
	}

	log.Debug(ctx, "new virtual machine...")

	vm = object.NewVirtualMachine(h.vclient.Client, info.Result.(types.ManagedObjectReference))

	log.Debug(ctx, "waiting for IP...")

	ip, errW := vm.WaitForIP(ctx, true)
	if errW != nil {
		return vm, sdk.WrapError(errW, "createVMModel> cannot get an ip")
	}

	log.Info(ctx, "virtual machine %q has IP %q", vm.Name(), ip)

	if _, errS := h.launchClientOp(ctx, vm, model.ModelVirtualMachine, model.ModelVirtualMachine.PostCmd, nil); errS != nil {
		log.Warn(ctx, "createVMModel> cannot start program %s", errS)
		annot := annotation{ToDelete: true}
		if annotStr, err := json.Marshal(annot); err == nil {
			vm.Reconfigure(ctx, types.VirtualMachineConfigSpec{
				Annotation: string(annotStr),
			})
		}
	}

	ctxTo, cancel := context.WithTimeout(ctx, 10*time.Minute)
	defer cancel()

	log.Debug(ctx, "waiting virtual machine %q to be powered off...", vm.Name())
	if err := vm.WaitForPowerState(ctxTo, types.VirtualMachinePowerStatePoweredOff); err != nil {
		return nil, sdk.WrapError(err, "cannot wait for power state result")
	}

	modelFound, err := h.getModelByName(ctx, model.Name)
	if err == nil {
		if err := h.deleteServer(modelFound); err != nil {
			log.Warn(ctx, "createVMModel> Cannot delete previous model %s : %s", model.Name, err)
		}
	}

	ctxTo, cancel = context.WithTimeout(ctx, 15*time.Second)
	defer cancel()
	log.Debug(ctx, "renaming virtual machine %q to %q...", vm.Name(), model.Name)

	task, errR := vm.Rename(ctxTo, model.Name)
	if errR != nil {
		return vm, sdk.WrapError(errR, "createVMModel> Cannot rename model %s", model.Name)
	}

	ctxTo, cancel = context.WithTimeout(ctx, 15*time.Second)
	defer cancel()
	if _, err := task.WaitForResult(ctxTo, nil); err != nil {
		return vm, sdk.WrapError(err, "error on waiting result for vm renaming %s", model.Name)
	}

	log.Debug(ctx, "renaming virtual machine %q to %q DONE", vm.Name(), model.Name)

	log.Info(ctx, "mark virtual machine %q as template", vm.Name(), model.Name)
	if err := vm.MarkAsTemplate(ctx); err != nil {
		return vm, sdk.WrapError(err, "unable to mark vm as template")
	}

	return vm, nil
}

// launchScriptWorker launch a script on the worker
func (h *HatcheryVSphere) launchScriptWorker(ctx context.Context, name string, jobID int64, token string, model sdk.Model, registerOnly bool, vm *object.VirtualMachine) error {
	ctxTo, cancel := context.WithTimeout(ctx, 2*time.Minute)
	defer cancel()

	if _, errW := vm.WaitForIP(ctxTo); errW != nil {
		return sdk.WrapError(errW, "createVMModel> error on waiting ip")
	}

	env := []string{
		"CDS_FROM_WORKER_IMAGE=true",
	}

	env = append(env, h.getGraylogGrpcEnv(model)...)

	udata := model.ModelVirtualMachine.PreCmd + "\n" + model.ModelVirtualMachine.Cmd

	if registerOnly {
		udata += " register"
	}
	udata += "\n" + model.ModelVirtualMachine.PostCmd

	tmpl, errt := template.New("udata").Parse(udata)
	if errt != nil {
		return errt
	}
	udataParam := sdk.WorkerArgs{
		API:               h.Configuration().API.HTTP.URL,
		Name:              name,
		Token:             token,
		Model:             model.Group.Name + "/" + model.Name,
		HatcheryName:      h.Name(),
		TTL:               h.Config.WorkerTTL,
		FromWorkerImage:   true,
		GraylogHost:       h.Configuration().Provision.WorkerLogsOptions.Graylog.Host,
		GraylogPort:       h.Configuration().Provision.WorkerLogsOptions.Graylog.Port,
		GraylogExtraKey:   h.Configuration().Provision.WorkerLogsOptions.Graylog.ExtraKey,
		GraylogExtraValue: h.Configuration().Provision.WorkerLogsOptions.Graylog.ExtraValue,
		WorkflowJobID:     jobID,
	}

	var buffer bytes.Buffer
	if err := tmpl.Execute(&buffer, udataParam); err != nil {
		return err
	}

	if _, errS := h.launchClientOp(ctx, vm, model.ModelVirtualMachine, buffer.String(), env); errS != nil {
		log.Warn(ctx, "launchScript> cannot start program %s", errS)

		// tag vm to delete
		annot := annotation{ToDelete: true}
		if annotStr, err := json.Marshal(annot); err == nil {
			vm.Reconfigure(ctx, types.VirtualMachineConfigSpec{
				Annotation: string(annotStr),
			})
		}

		return errS
	}

	return nil
}
<|endoftext|>


<commit_before>package management

import (
	"fmt"
	"time"
)

// Content Field Types
type FieldType string

// Content Field Type constants
const (
	// ShortText fields do not support ordering or strict equality. 1 to 256 characters.
	ShortText FieldType = "Symbol"
	// LongText can be a maximum length of 50,000 characters
	LongText = "Text"
	Integer  = "Integer"
	Number   = "Number"
	Boolean  = "Boolean"
	// Date must be ISO8601 formatted and do not require a time portion
	Date     = "Date"
	Location = "Location"
	Object   = "Object"
	LinkType = "Link"
	Array    = "Array"
)

// Field describes a single allowed field value of an an entry.
// Each field type corresponds to a JSON type, though there are
// more field types than JSON types.
type Field struct {
	ID       string    `json:"id,omitempty"`
	Name     string    `json:"name,omitempty"`
	LinkType string    `json:"linkType,omitempty"`
	Type     FieldType `json:"type,omitempty"`

	Localized   bool              `json:"localized,omitempty"`
	Required    bool              `json:"required,omitempty"`
	Validations []FieldValidation `json:"validations,omitempty"`
	Disabled    bool              `json:"disabled,omitempty"`

	// Omitted fields will stil be present in CMA APIs but omitted from CDA and CPA APIs
	Omitted bool `json:"omitted,omitempty"`
}

// FieldValidation describes validation rules associated with a field, if any.
type FieldValidation struct {
	Size              *SizeFieldValidation      `json:"size,omitempty"`
	DateRange         *DateRangeFieldValidation `json:"dateRange,omitempty"`
	RegularExpression *RegExFieldValidation     `json:"regexp,omitempty"`

	LinkMIMETypeGroup []string      `json:"linkMimetypeGroup,omitempty"`
	LinkContentTypes  []string      `json:"linkContentType,omitempty"`
	In                []interface{} `json:"in,omitempty"`
	Message           *string       `json:"message,omitempty"`
}

// RegExFieldValidation permits validation with regular expression.
type RegExFieldValidation struct {
	Pattern string `json:"pattern"`
}

// SizeFieldValidation permits validation with size. You can specify
// either minimum size, maximum size, or both.
type SizeFieldValidation struct {
	Min *float64 `json:"min,omitempty"`
	Max *float64 `json:"max,omitempty"`
}

// DateRangeFieldValidation permits validation with date ranges. You can specify
// either minimum date, maximum date, or both.
type DateRangeFieldValidation struct {
	Min *time.Time `json:"min,omitempty"`
	Max *time.Time `json:"max,omitempty"`
}

// ContentType are schemas that define the fields of entries.
// Every entry can only contain values in the fields defined by
// its content type, and the values of those fields must match
// the data type defined in the content type. There is a limit
// of 50 fields per content type.
type ContentType struct {
	System `json:"sys"`

	Name         string  `json:"name"`
	Description  string  `json:"description,omitempty"`
	DisplayField string  `json:"displayField,omitempty"`
	Fields       []Field `json:"fields"`
}

// Validate will validate the content type. An error is returned if the content
// type is not  valid.
func (t *ContentType) Validate() error {
	if len(t.Name) == 0 {
		return fmt.Errorf("Content type name cannot be empty")
	}

	if t.Space == nil || t.Space.ID == "" {
		return fmt.Errorf("Locale must specify valid System.Space.ID!")
	}

	if len(t.ID) == 0 {
		return fmt.Errorf("Content type must specify an identifier!")
	}

	return nil
}

// FetchContentTypes returns all content types for a given space. You can filter
// this further by toggling the published flag
func (c *Client) FetchContentTypes(spaceID string, published bool, limit int, offset int) (contentTypes []*ContentType, pagination *Pagination, err error) {
	if spaceID == "" {
		return nil, nil, fmt.Errorf("FetchContentTypes failed. Space identifier is not valid!")
	}

	if limit <= 0 {
		return nil, nil, fmt.Errorf("FetchContentTypes failed. Limit must be greater than 0")
	}

	if limit > 100 {
		limit = 100
	}

	c.rl.Wait()

	type contentTypesResponse struct {
		*Pagination
		Items []*ContentType `json:"items"`
	}

	results := new(contentTypesResponse)
	contentfulError := new(ContentfulError)
	path := func() string {
		if published {
			return fmt.Sprintf("spaces/%v/public/content_types", spaceID)
		}

		return fmt.Sprintf("spaces/%v/content_types", spaceID)
	}

	req, err := c.sling.New().
		Get(path()).
		Request()

	if err != nil {
		return
	}

	// Add query parameters
	q := req.URL.Query()
	q.Set("skip", fmt.Sprintf("%v", offset))
	q.Set("limit", fmt.Sprintf("%v", limit))
	req.URL.RawQuery = q.Encode()

	_, err = c.sling.Do(req, results, contentfulError)

	return results.Items, results.Pagination, handleError(err, contentfulError)

}

// CreateContentType will create a content type. It's recommended that you
// control the ID of the created content type and associated fields.
func (c *Client) CreateContentType(contentType *ContentType) (created *ContentType, err error) {
	if contentType == nil {
		return nil, fmt.Errorf("CreateContentType failed. Type argument was nil!")
	}

	if err = contentType.Validate(); err != nil {
		return
	}

	c.rl.Wait()

	created = new(ContentType)
	contentfulError := new(ContentfulError)
	path := fmt.Sprintf("spaces/%v/content_types/%v", contentType.Space.ID, contentType.ID)
	_, err = c.sling.New().
		Put(path).
		Set("X-Contentful-Version", fmt.Sprintf("%v", contentType.Version)).
		BodyJSON(contentType).
		Receive(created, contentfulError)

	return created, handleError(err, contentfulError)
}

// FetchContentType will return a content type for the specified space and
// content type identifier.
func (c *Client) FetchContentType(spaceID string, contentTypeID string) (contentType *ContentType, err error) {
	if spaceID == "" || contentTypeID == "" {
		err = fmt.Errorf("FetchContentType failed. Invalid spaceID or contentTypeID.")
		return
	}

	c.rl.Wait()

	contentType = new(ContentType)
	contentfulError := new(ContentfulError)
	path := fmt.Sprintf("spaces/%v/content_types/%v", spaceID, contentTypeID)
	_, err = c.sling.New().Get(path).Receive(contentType, contentfulError)

	return contentType, handleError(err, contentfulError)
}

// UpdateContentType will update the content type with the specified changes.
func (c *Client) UpdateContentType(contentType *ContentType) (updated *ContentType, err error) {
	return c.CreateContentType(contentType)
}

// DeleteContentType will delete a content type. Before you can delete a content
// type you need to deactivate it.
func (c *Client) DeleteContentType(spaceID string, contentTypeID string) (err error) {
	c.rl.Wait()

	contentfulError := new(ContentfulError)
	path := fmt.Sprintf("spaces/%v/content_types/%v", spaceID, contentTypeID)
	_, err = c.sling.New().
		Delete(path).
		Receive(nil, contentfulError)

	return handleError(err, contentfulError)
}

// ActivateContentType makes the content type available for creating entries
func (c *Client) ActivateContentType(contentType *ContentType) (activated *ContentType, err error) {
	if contentType == nil {
		return nil, fmt.Errorf("CreateContentType failed. Type argument was nil!")
	}

	if err = contentType.Validate(); err != nil {
		return
	}

	c.rl.Wait()

	activated = new(ContentType)
	contentfulError := new(ContentfulError)
	path := fmt.Sprintf("spaces/%v/content_types/%v/published", contentType.Space.ID, contentType.ID)
	_, err = c.sling.New().
		Put(path).
		Set("X-Contentful-Version", fmt.Sprintf("%v", contentType.Version)).
		Receive(activated, contentfulError)

	return activated, handleError(err, contentfulError)
}

// DeactivateContentType removes the availability for creating entries
func (c *Client) DeactivateContentType(contentType *ContentType) (deactivated *ContentType, err error) {
	if contentType == nil {
		return nil, fmt.Errorf("CreateContentType failed. Type argument was nil!")
	}

	if err = contentType.Validate(); err != nil {
		return
	}

	c.rl.Wait()

	deactivated = new(ContentType)
	contentfulError := new(ContentfulError)
	path := fmt.Sprintf("spaces/%v/content_types/%v/published", contentType.Space.ID, contentType.ID)
	_, err = c.sling.New().
		Delete(path).
		Set("X-Contentful-Version", fmt.Sprintf("%v", contentType.Version)).
		Receive(deactivated, contentfulError)

	return deactivated, handleError(err, contentfulError)
}
<commit_msg>Adding field validation for assetImageDimensions<commit_after>package management

import (
	"fmt"
	"time"
)

// Content Field Types
type FieldType string

// Content Field Type constants
const (
	// ShortText fields do not support ordering or strict equality. 1 to 256 characters.
	ShortText FieldType = "Symbol"
	// LongText can be a maximum length of 50,000 characters
	LongText = "Text"
	Integer  = "Integer"
	Number   = "Number"
	Boolean  = "Boolean"
	// Date must be ISO8601 formatted and do not require a time portion
	Date     = "Date"
	Location = "Location"
	Object   = "Object"
	LinkType = "Link"
	Array    = "Array"
)

// Field describes a single allowed field value of an an entry.
// Each field type corresponds to a JSON type, though there are
// more field types than JSON types.
type Field struct {
	ID       string    `json:"id,omitempty"`
	Name     string    `json:"name,omitempty"`
	LinkType string    `json:"linkType,omitempty"`
	Type     FieldType `json:"type,omitempty"`

	Localized   bool              `json:"localized,omitempty"`
	Required    bool              `json:"required,omitempty"`
	Validations []FieldValidation `json:"validations,omitempty"`
	Disabled    bool              `json:"disabled,omitempty"`

	// Omitted fields will stil be present in CMA APIs but omitted from CDA and CPA APIs
	Omitted bool `json:"omitted,omitempty"`
}

// FieldValidation describes validation rules associated with a field, if any.
type FieldValidation struct {
	Size                 *SizeFieldValidation      `json:"size,omitempty"`
	DateRange            *DateRangeFieldValidation `json:"dateRange,omitempty"`
	RegularExpression    *RegExFieldValidation     `json:"regexp,omitempty"`
	AssetImageValidation *AssetImageValidation     `json:"assetImageDimensions,omitempty"`

	LinkMIMETypeGroup []string      `json:"linkMimetypeGroup,omitempty"`
	LinkContentTypes  []string      `json:"linkContentType,omitempty"`
	In                []interface{} `json:"in,omitempty"`
	Message           *string       `json:"message,omitempty"`
}

// RegExFieldValidation permits validation with regular expression.
type RegExFieldValidation struct {
	Pattern string `json:"pattern"`
}

// SizeFieldValidation permits validation with size. You can specify
// either minimum size, maximum size, or both.
type SizeFieldValidation struct {
	Min *float64 `json:"min,omitempty"`
	Max *float64 `json:"max,omitempty"`
}

// AssetImageValidation permits asset validation around size
type AssetImageValidation struct {
	Width  *SizeFieldValidation `json:"width,omitempty"`
	Height *SizeFieldValidation `json:"height,omitempty"`
}

// DateRangeFieldValidation permits validation with date ranges. You can specify
// either minimum date, maximum date, or both.
type DateRangeFieldValidation struct {
	Min *time.Time `json:"min,omitempty"`
	Max *time.Time `json:"max,omitempty"`
}

// ContentType are schemas that define the fields of entries.
// Every entry can only contain values in the fields defined by
// its content type, and the values of those fields must match
// the data type defined in the content type. There is a limit
// of 50 fields per content type.
type ContentType struct {
	System `json:"sys"`

	Name         string  `json:"name"`
	Description  string  `json:"description,omitempty"`
	DisplayField string  `json:"displayField,omitempty"`
	Fields       []Field `json:"fields"`
}

// Validate will validate the content type. An error is returned if the content
// type is not  valid.
func (t *ContentType) Validate() error {
	if len(t.Name) == 0 {
		return fmt.Errorf("Content type name cannot be empty")
	}

	if t.Space == nil || t.Space.ID == "" {
		return fmt.Errorf("Locale must specify valid System.Space.ID!")
	}

	if len(t.ID) == 0 {
		return fmt.Errorf("Content type must specify an identifier!")
	}

	return nil
}

// FetchContentTypes returns all content types for a given space. You can filter
// this further by toggling the published flag
func (c *Client) FetchContentTypes(spaceID string, published bool, limit int, offset int) (contentTypes []*ContentType, pagination *Pagination, err error) {
	if spaceID == "" {
		return nil, nil, fmt.Errorf("FetchContentTypes failed. Space identifier is not valid!")
	}

	if limit <= 0 {
		return nil, nil, fmt.Errorf("FetchContentTypes failed. Limit must be greater than 0")
	}

	if limit > 100 {
		limit = 100
	}

	c.rl.Wait()

	type contentTypesResponse struct {
		*Pagination
		Items []*ContentType `json:"items"`
	}

	results := new(contentTypesResponse)
	contentfulError := new(ContentfulError)
	path := func() string {
		if published {
			return fmt.Sprintf("spaces/%v/public/content_types", spaceID)
		}

		return fmt.Sprintf("spaces/%v/content_types", spaceID)
	}

	req, err := c.sling.New().
		Get(path()).
		Request()

	if err != nil {
		return
	}

	// Add query parameters
	q := req.URL.Query()
	q.Set("skip", fmt.Sprintf("%v", offset))
	q.Set("limit", fmt.Sprintf("%v", limit))
	req.URL.RawQuery = q.Encode()

	_, err = c.sling.Do(req, results, contentfulError)

	return results.Items, results.Pagination, handleError(err, contentfulError)

}

// CreateContentType will create a content type. It's recommended that you
// control the ID of the created content type and associated fields.
func (c *Client) CreateContentType(contentType *ContentType) (created *ContentType, err error) {
	if contentType == nil {
		return nil, fmt.Errorf("CreateContentType failed. Type argument was nil!")
	}

	if err = contentType.Validate(); err != nil {
		return
	}

	c.rl.Wait()

	created = new(ContentType)
	contentfulError := new(ContentfulError)
	path := fmt.Sprintf("spaces/%v/content_types/%v", contentType.Space.ID, contentType.ID)
	_, err = c.sling.New().
		Put(path).
		Set("X-Contentful-Version", fmt.Sprintf("%v", contentType.Version)).
		BodyJSON(contentType).
		Receive(created, contentfulError)

	return created, handleError(err, contentfulError)
}

// FetchContentType will return a content type for the specified space and
// content type identifier.
func (c *Client) FetchContentType(spaceID string, contentTypeID string) (contentType *ContentType, err error) {
	if spaceID == "" || contentTypeID == "" {
		err = fmt.Errorf("FetchContentType failed. Invalid spaceID or contentTypeID.")
		return
	}

	c.rl.Wait()

	contentType = new(ContentType)
	contentfulError := new(ContentfulError)
	path := fmt.Sprintf("spaces/%v/content_types/%v", spaceID, contentTypeID)
	_, err = c.sling.New().Get(path).Receive(contentType, contentfulError)

	return contentType, handleError(err, contentfulError)
}

// UpdateContentType will update the content type with the specified changes.
func (c *Client) UpdateContentType(contentType *ContentType) (updated *ContentType, err error) {
	return c.CreateContentType(contentType)
}

// DeleteContentType will delete a content type. Before you can delete a content
// type you need to deactivate it.
func (c *Client) DeleteContentType(spaceID string, contentTypeID string) (err error) {
	c.rl.Wait()

	contentfulError := new(ContentfulError)
	path := fmt.Sprintf("spaces/%v/content_types/%v", spaceID, contentTypeID)
	_, err = c.sling.New().
		Delete(path).
		Receive(nil, contentfulError)

	return handleError(err, contentfulError)
}

// ActivateContentType makes the content type available for creating entries
func (c *Client) ActivateContentType(contentType *ContentType) (activated *ContentType, err error) {
	if contentType == nil {
		return nil, fmt.Errorf("CreateContentType failed. Type argument was nil!")
	}

	if err = contentType.Validate(); err != nil {
		return
	}

	c.rl.Wait()

	activated = new(ContentType)
	contentfulError := new(ContentfulError)
	path := fmt.Sprintf("spaces/%v/content_types/%v/published", contentType.Space.ID, contentType.ID)
	_, err = c.sling.New().
		Put(path).
		Set("X-Contentful-Version", fmt.Sprintf("%v", contentType.Version)).
		Receive(activated, contentfulError)

	return activated, handleError(err, contentfulError)
}

// DeactivateContentType removes the availability for creating entries
func (c *Client) DeactivateContentType(contentType *ContentType) (deactivated *ContentType, err error) {
	if contentType == nil {
		return nil, fmt.Errorf("CreateContentType failed. Type argument was nil!")
	}

	if err = contentType.Validate(); err != nil {
		return
	}

	c.rl.Wait()

	deactivated = new(ContentType)
	contentfulError := new(ContentfulError)
	path := fmt.Sprintf("spaces/%v/content_types/%v/published", contentType.Space.ID, contentType.ID)
	_, err = c.sling.New().
		Delete(path).
		Set("X-Contentful-Version", fmt.Sprintf("%v", contentType.Version)).
		Receive(deactivated, contentfulError)

	return deactivated, handleError(err, contentfulError)
}
<|endoftext|>


<commit_before>// Package editor enables users to create edit views from their content
// structs so that admins can manage content
package editor

import (
	"bytes"
)

// Editable ensures data is editable
type Editable interface {
	SetContentID(id int)
	ContentID() int
	ContentName() string
	SetSlug(slug string)
	Editor() *Editor
	MarshalEditor() ([]byte, error)
}

// Sortable ensures data is sortable by time
type Sortable interface {
	Time() int64
	Touch() int64
	ContentID() int
}

// Editor is a view containing fields to manage content
type Editor struct {
	ViewBuf *bytes.Buffer
}

// Field is used to create the editable view for a field
// within a particular content struct
type Field struct {
	View []byte
}

// Form takes editable content and any number of Field funcs to describe the edit
// page for any content struct added by a user
func Form(post Editable, fields ...Field) ([]byte, error) {
	editor := post.Editor()

	editor.ViewBuf = &bytes.Buffer{}
	editor.ViewBuf.Write([]byte(`<table><tbody class="row"><tr class="col s8"><td>`))

	for _, f := range fields {
		addFieldToEditorView(editor, f)
	}

	editor.ViewBuf.Write([]byte(`</td></tr>`))

	// content items with Item embedded have some default fields we need to render
	editor.ViewBuf.Write([]byte(`<tr class="col s4 default-fields"><td>`))

	publishTime := `

		MM
		<select class="month __ponzu browser-default">
			<option value="1">Jan - 01</option>
			<option value="2">Feb - 02</option>
			<option value="3">Mar - 03</option>
			<option value="4">Apr - 04</option>
			<option value="5">May - 05</option>
			<option value="6">Jun - 06</option>
			<option value="7">Jul - 07</option>
			<option value="8">Aug - 08</option>
			<option value="9">Sep - 09</option>
			<option value="10">Oct - 10</option>
			<option value="11">Nov - 11</option>
			<option value="12">Dec - 12</option>
		</select>

		DD
		<input value="" class="day __ponzu" maxlength="2" type="text" placeholder="DD" />

		YYYY
		<input value="" class="year __ponzu" maxlength="4" type="text" placeholder="YYYY" />

		HH
		<input value="" class="hour __ponzu" maxlength="2" type="text" placeholder="HH" />

:

		MM
		<input value="" class="minute __ponzu" maxlength="2" type="text" placeholder="MM" />

		Period
		<select class="period __ponzu browser-default">
			<option value="AM">AM</option>
			<option value="PM">PM</option>
		</select>

	`

	editor.ViewBuf.Write([]byte(publishTime))

	addPostDefaultFieldsToEditorView(post, editor)

	submit := `

	<button class="right waves-effect waves-light btn green save-post" type="submit">Save</button>
	<button class="right waves-effect waves-light btn red delete-post" type="submit">Delete</button>

This post is pending approval. By clicking 'Approve' it will be immediately published.

	<button class="right waves-effect waves-light btn blue approve-post" type="submit">Approve</button>

`
	editor.ViewBuf.Write([]byte(submit + `</td></tr></tbody></table>`))

	return editor.ViewBuf.Bytes(), nil
}

func addFieldToEditorView(e *Editor, f Field) {
	e.ViewBuf.Write(f.View)
}

func addPostDefaultFieldsToEditorView(p Editable, e *Editor) {
	defaults := []Field{
		Field{
			View: Input("Slug", p, map[string]string{
				"label":       "URL Slug",
				"type":        "text",
				"disabled":    "true",
				"placeholder": "Will be set automatically",
			}),
		},
		Field{
			View: Timestamp("Timestamp", p, map[string]string{
				"type":  "hidden",
				"class": "timestamp __ponzu",
			}),
		},
		Field{
			View: Timestamp("Updated", p, map[string]string{
				"type":  "hidden",
				"class": "updated __ponzu",
			}),
		},
	}

	for _, f := range defaults {
		addFieldToEditorView(e, f)
	}

}
<commit_msg>updating UI<commit_after>// Package editor enables users to create edit views from their content
// structs so that admins can manage content
package editor

import (
	"bytes"
)

// Editable ensures data is editable
type Editable interface {
	SetContentID(id int)
	ContentID() int
	ContentName() string
	SetSlug(slug string)
	Editor() *Editor
	MarshalEditor() ([]byte, error)
}

// Sortable ensures data is sortable by time
type Sortable interface {
	Time() int64
	Touch() int64
	ContentID() int
}

// Editor is a view containing fields to manage content
type Editor struct {
	ViewBuf *bytes.Buffer
}

// Field is used to create the editable view for a field
// within a particular content struct
type Field struct {
	View []byte
}

// Form takes editable content and any number of Field funcs to describe the edit
// page for any content struct added by a user
func Form(post Editable, fields ...Field) ([]byte, error) {
	editor := post.Editor()

	editor.ViewBuf = &bytes.Buffer{}
	editor.ViewBuf.Write([]byte(`<table><tbody class="row"><tr class="col s8"><td>`))

	for _, f := range fields {
		addFieldToEditorView(editor, f)
	}

	editor.ViewBuf.Write([]byte(`</td></tr>`))

	// content items with Item embedded have some default fields we need to render
	editor.ViewBuf.Write([]byte(`<tr class="col s4 default-fields"><td>`))

	publishTime := `

		MM
		<select class="month __ponzu browser-default">
			<option value="1">Jan - 01</option>
			<option value="2">Feb - 02</option>
			<option value="3">Mar - 03</option>
			<option value="4">Apr - 04</option>
			<option value="5">May - 05</option>
			<option value="6">Jun - 06</option>
			<option value="7">Jul - 07</option>
			<option value="8">Aug - 08</option>
			<option value="9">Sep - 09</option>
			<option value="10">Oct - 10</option>
			<option value="11">Nov - 11</option>
			<option value="12">Dec - 12</option>
		</select>

		DD
		<input value="" class="day __ponzu" maxlength="2" type="text" placeholder="DD" />

		YYYY
		<input value="" class="year __ponzu" maxlength="4" type="text" placeholder="YYYY" />

		HH
		<input value="" class="hour __ponzu" maxlength="2" type="text" placeholder="HH" />

:

		MM
		<input value="" class="minute __ponzu" maxlength="2" type="text" placeholder="MM" />

		Period
		<select class="period __ponzu browser-default">
			<option value="AM">AM</option>
			<option value="PM">PM</option>
		</select>

	`

	editor.ViewBuf.Write([]byte(publishTime))

	addPostDefaultFieldsToEditorView(post, editor)

	submit := `

	<button class="right waves-effect waves-light btn green save-post" type="submit">Save</button>
	<button class="right waves-effect waves-light btn red delete-post" type="submit">Delete</button>

		<button class="right waves-effect waves-light btn blue approve-post" type="submit">Approve</button>

This post is pending approval. By clicking 'Approve', it will be immediately published.

`
	editor.ViewBuf.Write([]byte(submit + `</td></tr></tbody></table>`))

	return editor.ViewBuf.Bytes(), nil
}

func addFieldToEditorView(e *Editor, f Field) {
	e.ViewBuf.Write(f.View)
}

func addPostDefaultFieldsToEditorView(p Editable, e *Editor) {
	defaults := []Field{
		Field{
			View: Input("Slug", p, map[string]string{
				"label":       "URL Slug",
				"type":        "text",
				"disabled":    "true",
				"placeholder": "Will be set automatically",
			}),
		},
		Field{
			View: Timestamp("Timestamp", p, map[string]string{
				"type":  "hidden",
				"class": "timestamp __ponzu",
			}),
		},
		Field{
			View: Timestamp("Updated", p, map[string]string{
				"type":  "hidden",
				"class": "updated __ponzu",
			}),
		},
	}

	for _, f := range defaults {
		addFieldToEditorView(e, f)
	}

}
<|endoftext|>


<commit_before>package master

import (
	. "github.com/KIT-MAMID/mamid/model"
	"github.com/jinzhu/gorm"
)

type ClusterAllocator struct {
}

type persistence uint

const (
	Persistent persistence = 0
	Volatile   persistence = 1
)

type memberCountTuple map[persistence]uint

func (c *ClusterAllocator) CompileMongodLayout(tx *gorm.DB) (err error) {

	defer func() {
		r := recover()
		if r == nil {
			return
		}
		switch r {
		case r == nil:
			return
		case r == gorm.ErrInvalidTransaction:
			err = r.(error)
		default:
			panic(r)
		}
	}()

	replicaSets := c.replicaSets(tx)
	for _, r := range replicaSets {
		c.removeUnneededMembers(tx, r)
		c.addMembers(tx, r)
	}

	return err
}

func (c *ClusterAllocator) replicaSets(tx *gorm.DB) []*ReplicaSet {
	return []*ReplicaSet{}
}

func (c *ClusterAllocator) removeUnneededMembers(tx *gorm.DB, r *ReplicaSet) {
	for persistence, count := range c.effectiveMemberCount(tx, r) {
		c.removeUnneededMembersByPersistence(tx, r, persistence, count)
	}
}

func slavePersistence(s *Slave) persistence {
	switch s.PersistentStorage {
	case true:
		return Persistent
	default:
		return Volatile
	}
}

func (c *ClusterAllocator) removeUnneededMembersByPersistence(tx *gorm.DB, r *ReplicaSet, p persistence, initialCount uint) {

	var configuredMemberCount uint
	if p == Persistent {
		configuredMemberCount = r.PersistentMemberCount
	} else if p == Volatile {
		configuredMemberCount = r.VolatileMemberCount
	}

	if err := tx.Model(r).Related(&r.Mongods, "Mongods").Error; err != nil {
		panic(err)
	}

	for initialCount > configuredMemberCount {
		// Destroy any Mongod running on disabled slaves (higher priority)
		for _, m := range r.Mongods {

			if err := tx.Model(m).Related(&m.ParentSlave, "ParentSlave").Error; err != nil {
				panic(err)
			}

			if m.ParentSlave.ConfiguredState == SlaveStateDisabled &&
				slavePersistence(m.ParentSlave) == p {
				// destroy
				panic("not implemented")

				initialCount--
			}
		}
	}

	for initialCount > configuredMemberCount {
		// Destroy any Mongod (lower priority)
		for _, m := range r.Mongods {

			// Only fetch ParentSlave where not already fetched
			if m.ParentSlave == nil {
				if err := tx.Model(m).Related(&m.ParentSlave, "ParentSlave").Error; err != nil {
					panic(err)
				}
			}

			if slavePersistence(m.ParentSlave) == p {
				// destroy
				panic("not implemented")
			}

		}

	}

}

func (c *ClusterAllocator) effectiveMemberCount(tx *gorm.DB, r *ReplicaSet) memberCountTuple {

	var res memberCountTuple

	traverseReplicaSetMongods(tx, r, func(m *Mongod) {

		if m.ObservedState.ExecutionState == MongodExecutionStateRunning &&
			m.DesiredState.ExecutionState == MongodExecutionStateRunning {
			if m.ParentSlave.PersistentStorage {
				res[Persistent]++
			} else {
				res[Volatile]++
			}
		}
	})

	return res
}

func (c *ClusterAllocator) addMembers(tx *gorm.DB, r *ReplicaSet) {
	for persistence, count := range c.alreadyAddedMemberCount(tx, r) {
		c.addMembersByPersistence(tx, r, persistence, count)
	}
}

func (c *ClusterAllocator) alreadyAddedMemberCount(tx *gorm.DB, r *ReplicaSet) memberCountTuple {

	var res memberCountTuple

	traverseReplicaSetMongods(tx, r, func(m *Mongod) {

		if m.ParentSlave.ConfiguredState != SlaveStateDisabled &&
			m.DesiredState.ExecutionState != MongodExecutionStateNotRunning &&
			m.DesiredState.ExecutionState != MongodExecutionStateDestroyed {
			if m.ParentSlave.PersistentStorage {
				res[Persistent]++
			} else {
				res[Volatile]++
			}
		}

	})

	return res
}

func (c *ClusterAllocator) addMembersByPersistence(tx *gorm.DB, r *ReplicaSet, p persistence, initialCount uint) {
	replicaSets := c.pqReplicaSets(tx)
	riskGroups := c.pqRiskGroups(tx)
	_ = replicaSets
	_ = riskGroups
}

// Traverse a Replica Set's Mongods, for which the following Attributes have been fetched
// 	ParentSlave
//	ObservedState
// 	DesiredState
func traverseReplicaSetMongods(tx *gorm.DB, r *ReplicaSet, handler func(m *Mongod)) {

	if err := tx.Model(r).Related(&r.Mongods, "Mongods").Error; err != nil {
		panic(err)
	}

	for _, m := range r.Mongods {

		if err := tx.Model(m).Related(&m.ObservedState, "ObservedState").Error; err != nil {
			panic(err)
		}
		if err := tx.Model(m).Related(&m.DesiredState, "DesiredState").Error; err != nil {
			panic(err)
		}
		if err := tx.Model(m).Related(&m.ParentSlave, "ParentSlave").Error; err != nil {
			panic(err)
		}

		handler(m)

	}

}
<commit_msg>elaborate wireframe representation of add-members step.<commit_after>package master

import (
	. "github.com/KIT-MAMID/mamid/model"
	"github.com/jinzhu/gorm"
)

type ClusterAllocator struct {
}

type persistence uint

const (
	Persistent persistence = 0
	Volatile   persistence = 1
)

type memberCountTuple map[persistence]uint

func (c *ClusterAllocator) CompileMongodLayout(tx *gorm.DB) (err error) {

	defer func() {
		r := recover()
		if r == nil {
			return
		}
		switch r {
		case r == nil:
			return
		case r == gorm.ErrInvalidTransaction:
			err = r.(error)
		default:
			panic(r)
		}
	}()

	replicaSets := c.replicaSets(tx)
	for _, r := range replicaSets {
		c.removeUnneededMembers(tx, r)
	}

	addMembers(replicaSets)

	return err
}

func (c *ClusterAllocator) replicaSets(tx *gorm.DB) []*ReplicaSet {
	return []*ReplicaSet{}
}

func (c *ClusterAllocator) removeUnneededMembers(tx *gorm.DB, r *ReplicaSet) {
	for persistence, count := range c.effectiveMemberCount(tx, r) {
		c.removeUnneededMembersByPersistence(tx, r, persistence, count)
	}
}

func slavePersistence(s *Slave) persistence {
	switch s.PersistentStorage {
	case true:
		return Persistent
	default:
		return Volatile
	}
}

func (c *ClusterAllocator) removeUnneededMembersByPersistence(tx *gorm.DB, r *ReplicaSet, p persistence, initialCount uint) {

	var configuredMemberCount uint
	if p == Persistent {
		configuredMemberCount = r.PersistentMemberCount
	} else if p == Volatile {
		configuredMemberCount = r.VolatileMemberCount
	}

	if err := tx.Model(r).Related(&r.Mongods, "Mongods").Error; err != nil {
		panic(err)
	}

	for initialCount > configuredMemberCount {
		// Destroy any Mongod running on disabled slaves (higher priority)
		for _, m := range r.Mongods {

			if err := tx.Model(m).Related(&m.ParentSlave, "ParentSlave").Error; err != nil {
				panic(err)
			}

			if m.ParentSlave.ConfiguredState == SlaveStateDisabled &&
				slavePersistence(m.ParentSlave) == p {
				// destroy
				panic("not implemented")

				initialCount--
			}
		}
	}

	for initialCount > configuredMemberCount {
		// Destroy any Mongod (lower priority)
		for _, m := range r.Mongods {

			// Only fetch ParentSlave where not already fetched
			if m.ParentSlave == nil {
				if err := tx.Model(m).Related(&m.ParentSlave, "ParentSlave").Error; err != nil {
					panic(err)
				}
			}

			if slavePersistence(m.ParentSlave) == p {
				// destroy
				panic("not implemented")
			}

		}

	}

}

func (c *ClusterAllocator) effectiveMemberCount(tx *gorm.DB, r *ReplicaSet) memberCountTuple {

	var res memberCountTuple

	traverseReplicaSetMongods(tx, r, func(m *Mongod) {

		if m.ObservedState.ExecutionState == MongodExecutionStateRunning &&
			m.DesiredState.ExecutionState == MongodExecutionStateRunning {
			if m.ParentSlave.PersistentStorage {
				res[Persistent]++
			} else {
				res[Volatile]++
			}
		}
	})

	return res
}

func (c *ClusterAllocator) addMembers(tx *gorm.DB, replicaSets []*ReplicaSet) {

	for _, persistence := range []persistence{Volatile, Persistent} {

		// build prioritization datastructures
		// will only return items that match current persistence and actually need more members

		pqReplicaSets := c.pqReplicaSets(replicaSets, persistence)
		pqRiskGroups := c.pqRiskGroups(tx, persistence)

		for r := pqReplicaSets.Pop(); r != nil; {

			if s := pqRiskGroups.popSlaveinNonconflictingRiskGroup(r); g != nil {

				// spawn new Mongod m on s and add it to r.Mongods
				// compute MongodState for m and set the DesiredState variable
				panic("not implemented")

				if replicaSetNeedsMoreMembers(r, p) {
					pqReplicaSets.Push(r)
				}

				if slaveHasFreePorts(s) {
					pqRiskGroups.pushSlave(s)
				}

			} else {
				// send constraint not fulfilled notification
				panic("not implemented")
			}
		}

	}
}

func (c *ClusterAllocator) alreadyAddedMemberCount(tx *gorm.DB, r *ReplicaSet) memberCountTuple {

	var res memberCountTuple

	traverseReplicaSetMongods(tx, r, func(m *Mongod) {

		if m.ParentSlave.ConfiguredState != SlaveStateDisabled &&
			m.DesiredState.ExecutionState != MongodExecutionStateNotRunning &&
			m.DesiredState.ExecutionState != MongodExecutionStateDestroyed {
			if m.ParentSlave.PersistentStorage {
				res[Persistent]++
			} else {
				res[Volatile]++
			}
		}

	})

	return res
}

// Traverse a Replica Set's Mongods, for which the following Attributes have been fetched
// 	ParentSlave
//	ObservedState
// 	DesiredState
func traverseReplicaSetMongods(tx *gorm.DB, r *ReplicaSet, handler func(m *Mongod)) {

	if err := tx.Model(r).Related(&r.Mongods, "Mongods").Error; err != nil {
		panic(err)
	}

	for _, m := range r.Mongods {

		if err := tx.Model(m).Related(&m.ObservedState, "ObservedState").Error; err != nil {
			panic(err)
		}
		if err := tx.Model(m).Related(&m.DesiredState, "DesiredState").Error; err != nil {
			panic(err)
		}
		if err := tx.Model(m).Related(&m.ParentSlave, "ParentSlave").Error; err != nil {
			panic(err)
		}

		handler(m)

	}

}
<|endoftext|>


<commit_before>package google

import (
	"encoding/json"
	"reflect"
)

// Convert between two types by converting to/from JSON. Intended to switch
// between multiple API versions, as they are strict supersets of one another.
// item and out are pointers to structs
func Convert(item, out interface{}) error {
	bytes, err := json.Marshal(item)
	if err != nil {
		return err
	}

	err = json.Unmarshal(bytes, out)
	if err != nil {
		return err
	}

	// Converting between maps and structs only occurs when autogenerated resources convert the result
	// of an HTTP request. Those results do not contain omitted fields, so no need to set them.
	if _, ok := item.(map[string]interface{}); !ok {
		setOmittedFields(item, out)
	}

	return nil
}

func setOmittedFields(item, out interface{}) {
	// Both inputs must be pointers, see https://blog.golang.org/laws-of-reflection:
	// "To modify a reflection object, the value must be settable."
	iVal := reflect.ValueOf(item).Elem()
	oVal := reflect.ValueOf(out).Elem()

	// Loop through all the fields of the struct to look for omitted fields and nested fields
	for i := 0; i < iVal.NumField(); i++ {
		iField := iVal.Field(i)
		if isEmptyValue(iField) {
			continue
		}

		fieldInfo := iVal.Type().Field(i)
		oField := oVal.FieldByName(fieldInfo.Name)

		// Only look at fields that exist in the output struct
		if !oField.IsValid() {
			continue
		}

		// If the field contains a 'json:"="' tag, then it was omitted from the Marshal/Unmarshal
		// call and needs to be added back in.
		if fieldInfo.Tag.Get("json") == "-" {
			oField.Set(iField)
		}

		// If this field is a struct, *struct, []struct, or []*struct, recurse.
		if iField.Kind() == reflect.Struct {
			setOmittedFields(iField.Addr().Interface(), oField.Addr().Interface())
		}
		if iField.Kind() == reflect.Ptr && iField.Type().Elem().Kind() == reflect.Struct {
			setOmittedFields(iField.Interface(), oField.Interface())
		}
		if iField.Kind() == reflect.Slice && iField.Type().Elem().Kind() == reflect.Struct {
			for j := 0; j < iField.Len(); j++ {
				setOmittedFields(iField.Index(j).Addr().Interface(), oField.Index(j).Addr().Interface())
			}
		}
		if iField.Kind() == reflect.Slice && iField.Type().Elem().Kind() == reflect.Ptr &&
			iField.Type().Elem().Elem().Kind() == reflect.Struct {
			for j := 0; j < iField.Len(); j++ {
				setOmittedFields(iField.Index(j).Interface(), oField.Index(j).Interface())
			}
		}
	}
}
<commit_msg>Add a method for converting structs -> maps when bridging MM -> TF (#12)<commit_after>package google

import (
	"encoding/json"
	"reflect"
)

// Convert between two types by converting to/from JSON. Intended to switch
// between multiple API versions, as they are strict supersets of one another.
// item and out are pointers to structs
func Convert(item, out interface{}) error {
	bytes, err := json.Marshal(item)
	if err != nil {
		return err
	}

	err = json.Unmarshal(bytes, out)
	if err != nil {
		return err
	}

	// Converting between maps and structs only occurs when autogenerated resources convert the result
	// of an HTTP request. Those results do not contain omitted fields, so no need to set them.
	if _, ok := item.(map[string]interface{}); !ok {
		setOmittedFields(item, out)
	}

	return nil
}

// When converting to a map, we can't use setOmittedFields because FieldByName
// fails. Luckily, we don't use the omitted fields anymore with generated
// resources, and this function is used to bridge from handwritten -> generated.
// Since this is a known type, we can create it inline instead of needing to
// pass an object in.
func ConvertToMap(item interface{}) (map[string]interface{}, error) {
	out := make(map[string]interface{})
	bytes, err := json.Marshal(item)
	if err != nil {
		return nil, err
	}

	err = json.Unmarshal(bytes, &out)
	if err != nil {
		return nil, err
	}

	return out, nil
}

func setOmittedFields(item, out interface{}) {
	// Both inputs must be pointers, see https://blog.golang.org/laws-of-reflection:
	// "To modify a reflection object, the value must be settable."
	iVal := reflect.ValueOf(item).Elem()
	oVal := reflect.ValueOf(out).Elem()

	// Loop through all the fields of the struct to look for omitted fields and nested fields
	for i := 0; i < iVal.NumField(); i++ {
		iField := iVal.Field(i)
		if isEmptyValue(iField) {
			continue
		}

		fieldInfo := iVal.Type().Field(i)
		oField := oVal.FieldByName(fieldInfo.Name)

		// Only look at fields that exist in the output struct
		if !oField.IsValid() {
			continue
		}

		// If the field contains a 'json:"="' tag, then it was omitted from the Marshal/Unmarshal
		// call and needs to be added back in.
		if fieldInfo.Tag.Get("json") == "-" {
			oField.Set(iField)
		}

		// If this field is a struct, *struct, []struct, or []*struct, recurse.
		if iField.Kind() == reflect.Struct {
			setOmittedFields(iField.Addr().Interface(), oField.Addr().Interface())
		}
		if iField.Kind() == reflect.Ptr && iField.Type().Elem().Kind() == reflect.Struct {
			setOmittedFields(iField.Interface(), oField.Interface())
		}
		if iField.Kind() == reflect.Slice && iField.Type().Elem().Kind() == reflect.Struct {
			for j := 0; j < iField.Len(); j++ {
				setOmittedFields(iField.Index(j).Addr().Interface(), oField.Index(j).Addr().Interface())
			}
		}
		if iField.Kind() == reflect.Slice && iField.Type().Elem().Kind() == reflect.Ptr &&
			iField.Type().Elem().Elem().Kind() == reflect.Struct {
			for j := 0; j < iField.Len(); j++ {
				setOmittedFields(iField.Index(j).Interface(), oField.Index(j).Interface())
			}
		}
	}
}
<|endoftext|>


<commit_before>package goprocess

import (
	"testing"
	"time"
)

type tree struct {
	Process
	c []tree
}

func setupHierarchy(p Process) tree {
	t := func(n Process, ts ...tree) tree {
		return tree{n, ts}
	}

	a := WithParent(p)
	b1 := WithParent(a)
	b2 := WithParent(a)
	c1 := WithParent(b1)
	c2 := WithParent(b1)
	c3 := WithParent(b2)
	c4 := WithParent(b2)

	return t(a, t(b1, t(c1), t(c2)), t(b2, t(c3), t(c4)))
}

func TestClosingClosed(t *testing.T) {

	a := WithParent(Background())
	b := WithParent(a)

	Q := make(chan string, 3)

	go func() {
		<-a.Closing()
		Q <- "closing"
		b.Close()
	}()

	go func() {
		<-a.Closed()
		Q <- "closed"
	}()

	go func() {
		a.Close()
		Q <- "closed"
	}()

	if q := <-Q; q != "closing" {
		t.Error("order incorrect. closing not first")
	}
	if q := <-Q; q != "closed" {
		t.Error("order incorrect. closing not first")
	}
	if q := <-Q; q != "closed" {
		t.Error("order incorrect. closing not first")
	}
}

func TestChildFunc(t *testing.T) {
	a := WithParent(Background())

	wait1 := make(chan struct{})
	wait2 := make(chan struct{})
	wait3 := make(chan struct{})
	wait4 := make(chan struct{})
	go func() {
		a.Close()
		wait4 <- struct{}{}
	}()

	a.Go(func(process Process) {
		wait1 <- struct{}{}
		<-wait2
		wait3 <- struct{}{}
	})

	<-wait1
	select {
	case <-wait3:
		t.Error("should not be closed yet")
	case <-wait4:
		t.Error("should not be closed yet")
	case <-a.Closed():
		t.Error("should not be closed yet")
	default:
	}

	wait2 <- struct{}{}

	select {
	case <-wait3:
	case <-time.After(time.Second):
		t.Error("should be closed now")
	}

	select {
	case <-wait4:
	case <-time.After(time.Second):
		t.Error("should be closed now")
	}
}

func TestTeardownCalledOnce(t *testing.T) {
	a := setupHierarchy(Background())

	onlyOnce := func() func() error {
		count := 0
		return func() error {
			count++
			if count > 1 {
				t.Error("called", count, "times")
			}
			return nil
		}
	}

	setTeardown := func(t tree, tf TeardownFunc) {
		t.Process.(*process).teardown = tf
	}

	setTeardown(a, onlyOnce())
	setTeardown(a.c[0], onlyOnce())
	setTeardown(a.c[0].c[0], onlyOnce())
	setTeardown(a.c[0].c[1], onlyOnce())
	setTeardown(a.c[1], onlyOnce())
	setTeardown(a.c[1].c[0], onlyOnce())
	setTeardown(a.c[1].c[1], onlyOnce())

	a.c[0].c[0].Close()
	a.c[0].c[0].Close()
	a.c[0].c[0].Close()
	a.c[0].c[0].Close()
	a.c[0].Close()
	a.c[0].Close()
	a.c[0].Close()
	a.c[0].Close()
	a.Close()
	a.Close()
	a.Close()
	a.Close()
	a.c[1].Close()
	a.c[1].Close()
	a.c[1].Close()
	a.c[1].Close()
}

func TestOnClosed(t *testing.T) {

	p := WithParent(Background())
	a := setupHierarchy(p)
	Q := make(chan string, 10)

	onClosed := func(s string, p Process) {
		<-p.Closed()
		Q <- s
	}

	go onClosed("0", a.c[0])
	go onClosed("10", a.c[1].c[0])
	go onClosed("", a)
	go onClosed("00", a.c[0].c[0])
	go onClosed("1", a.c[1])
	go onClosed("01", a.c[0].c[1])
	go onClosed("11", a.c[1].c[1])

	test := func(ss ...string) {
		s1 := <-Q
		for _, s2 := range ss {
			if s1 == s2 {
				return
			}
		}
		t.Error("context not in group", s1, ss)
	}

	go p.Close()

	test("00", "01", "10", "11")
	test("00", "01", "10", "11")
	test("00", "01", "10", "11")
	test("00", "01", "10", "11")
	test("0", "1")
	test("0", "1")
	test("")
}
<commit_msg>more tests<commit_after>package goprocess

import (
	"syscall"
	"testing"
	"time"
)

type tree struct {
	Process
	c []tree
}

func setupHierarchy(p Process) tree {
	t := func(n Process, ts ...tree) tree {
		return tree{n, ts}
	}

	a := WithParent(p)
	b1 := WithParent(a)
	b2 := WithParent(a)
	c1 := WithParent(b1)
	c2 := WithParent(b1)
	c3 := WithParent(b2)
	c4 := WithParent(b2)

	return t(a, t(b1, t(c1), t(c2)), t(b2, t(c3), t(c4)))
}

func TestClosingClosed(t *testing.T) {

	a := WithParent(Background())
	b := WithParent(a)

	Q := make(chan string, 3)

	go func() {
		<-a.Closing()
		Q <- "closing"
		b.Close()
	}()

	go func() {
		<-a.Closed()
		Q <- "closed"
	}()

	go func() {
		a.Close()
		Q <- "closed"
	}()

	if q := <-Q; q != "closing" {
		t.Error("order incorrect. closing not first")
	}
	if q := <-Q; q != "closed" {
		t.Error("order incorrect. closing not first")
	}
	if q := <-Q; q != "closed" {
		t.Error("order incorrect. closing not first")
	}
}

func TestChildFunc(t *testing.T) {
	a := WithParent(Background())

	wait1 := make(chan struct{})
	wait2 := make(chan struct{})
	wait3 := make(chan struct{})
	wait4 := make(chan struct{})
	go func() {
		a.Close()
		wait4 <- struct{}{}
	}()

	a.Go(func(process Process) {
		wait1 <- struct{}{}
		<-wait2
		wait3 <- struct{}{}
	})

	<-wait1
	select {
	case <-wait3:
		t.Error("should not be closed yet")
	case <-wait4:
		t.Error("should not be closed yet")
	case <-a.Closed():
		t.Error("should not be closed yet")
	default:
	}

	wait2 <- struct{}{}

	select {
	case <-wait3:
	case <-time.After(time.Second):
		t.Error("should be closed now")
	}

	select {
	case <-wait4:
	case <-time.After(time.Second):
		t.Error("should be closed now")
	}
}

func TestTeardownCalledOnce(t *testing.T) {
	a := setupHierarchy(Background())

	onlyOnce := func() func() error {
		count := 0
		return func() error {
			count++
			if count > 1 {
				t.Error("called", count, "times")
			}
			return nil
		}
	}

	setTeardown := func(t tree, tf TeardownFunc) {
		t.Process.(*process).teardown = tf
	}

	setTeardown(a, onlyOnce())
	setTeardown(a.c[0], onlyOnce())
	setTeardown(a.c[0].c[0], onlyOnce())
	setTeardown(a.c[0].c[1], onlyOnce())
	setTeardown(a.c[1], onlyOnce())
	setTeardown(a.c[1].c[0], onlyOnce())
	setTeardown(a.c[1].c[1], onlyOnce())

	a.c[0].c[0].Close()
	a.c[0].c[0].Close()
	a.c[0].c[0].Close()
	a.c[0].c[0].Close()
	a.c[0].Close()
	a.c[0].Close()
	a.c[0].Close()
	a.c[0].Close()
	a.Close()
	a.Close()
	a.Close()
	a.Close()
	a.c[1].Close()
	a.c[1].Close()
	a.c[1].Close()
	a.c[1].Close()
}

func TestOnClosed(t *testing.T) {

	Q := make(chan string, 10)
	p := WithParent(Background())
	a := setupHierarchy(p)

	go onClosedStr(Q, "0", a.c[0])
	go onClosedStr(Q, "10", a.c[1].c[0])
	go onClosedStr(Q, "", a)
	go onClosedStr(Q, "00", a.c[0].c[0])
	go onClosedStr(Q, "1", a.c[1])
	go onClosedStr(Q, "01", a.c[0].c[1])
	go onClosedStr(Q, "11", a.c[1].c[1])

	go p.Close()

	testStrs(t, Q, "00", "01", "10", "11")
	testStrs(t, Q, "00", "01", "10", "11")
	testStrs(t, Q, "00", "01", "10", "11")
	testStrs(t, Q, "00", "01", "10", "11")
	testStrs(t, Q, "0", "1")
	testStrs(t, Q, "0", "1")
	testStrs(t, Q, "")
}

func TestWaitFor(t *testing.T) {

	Q := make(chan string, 5)
	a := WithParent(Background())
	b := WithParent(Background())
	c := WithParent(Background())
	d := WithParent(Background())
	e := WithParent(Background())

	go onClosedStr(Q, "a", a)
	go onClosedStr(Q, "b", b)
	go onClosedStr(Q, "c", c)
	go onClosedStr(Q, "d", d)
	go onClosedStr(Q, "e", e)

	testNone(t, Q)
	a.WaitFor(b)
	a.WaitFor(c)
	b.WaitFor(d)
	e.WaitFor(d)
	testNone(t, Q)

	go a.Close() // should do nothing.
	testNone(t, Q)

	go e.Close()
	testNone(t, Q)

	d.Close()
	testStrs(t, Q, "d", "e")
	testStrs(t, Q, "d", "e")

	c.Close()
	testStrs(t, Q, "c")

	b.Close()
	testStrs(t, Q, "a", "b")
	testStrs(t, Q, "a", "b")
}

func TestAddChildNoWait(t *testing.T) {

	Q := make(chan string, 5)
	a := WithParent(Background())
	b := WithParent(Background())
	c := WithParent(Background())
	d := WithParent(Background())
	e := WithParent(Background())

	go onClosedStr(Q, "a", a)
	go onClosedStr(Q, "b", b)
	go onClosedStr(Q, "c", c)
	go onClosedStr(Q, "d", d)
	go onClosedStr(Q, "e", e)

	testNone(t, Q)
	a.AddChildNoWait(b)
	a.AddChildNoWait(c)
	b.AddChildNoWait(d)
	e.AddChildNoWait(d)
	testNone(t, Q)

	b.Close()
	testStrs(t, Q, "b", "d")
	testStrs(t, Q, "b", "d")

	a.Close()
	testStrs(t, Q, "a", "c")
	testStrs(t, Q, "a", "c")

	e.Close()
	testStrs(t, Q, "e")
}

func TestAddChild(t *testing.T) {

	a := WithParent(Background())
	b := WithParent(Background())
	c := WithParent(Background())
	d := WithParent(Background())
	e := WithParent(Background())
	Q := make(chan string, 5)

	go onClosedStr(Q, "a", a)
	go onClosedStr(Q, "b", b)
	go onClosedStr(Q, "c", c)
	go onClosedStr(Q, "d", d)
	go onClosedStr(Q, "e", e)

	testNone(t, Q)
	a.AddChild(b)
	a.AddChild(c)
	b.AddChild(d)
	e.AddChild(d)
	testNone(t, Q)

	go b.Close()
	testNone(t, Q)
	d.Close()
	testStrs(t, Q, "b", "d")
	testStrs(t, Q, "b", "d")

	go a.Close()
	testNone(t, Q)
	c.Close()
	testStrs(t, Q, "a", "c")
	testStrs(t, Q, "a", "c")

	e.Close()
	testStrs(t, Q, "e")
}

func TestBackground(t *testing.T) {
	// test it hangs indefinitely:
	b := Background()

	go b.Close()
	go func() {
		b.Close()
	}()

	select {
	case <-b.Closing():
		t.Error("b.Closing() closed :(")
	default:
	}
}

func TestWithSignals(t *testing.T) {
	p := WithSignals(syscall.SIGABRT)
	testNotClosed(t, p)

	syscall.Kill(syscall.Getpid(), syscall.SIGABRT)
	testClosed(t, p)
}

func testClosing(t *testing.T, p Process) {
	select {
	case <-p.Closing():
	case <-time.After(50 * time.Millisecond):
		t.Fatal("should be closing")
	}
}

func testNotClosing(t *testing.T, p Process) {
	select {
	case <-p.Closing():
		t.Fatal("should not be closing")
	case <-p.Closed():
		t.Fatal("should not be closed")
	default:
	}
}

func testClosed(t *testing.T, p Process) {
	select {
	case <-p.Closed():
	case <-time.After(50 * time.Millisecond):
		t.Fatal("should be closed")
	}
}

func testNotClosed(t *testing.T, p Process) {
	select {
	case <-p.Closed():
		t.Fatal("should not be closed")
	case <-time.After(50 * time.Millisecond):
	}
}

func testNone(t *testing.T, c <-chan string) {
	select {
	case <-c:
		t.Fatal("none should be closed")
	default:
	}
}

func testStrs(t *testing.T, Q <-chan string, ss ...string) {
	s1 := <-Q
	for _, s2 := range ss {
		if s1 == s2 {
			return
		}
	}
	t.Error("context not in group", s1, ss)
}

func onClosedStr(Q chan<- string, s string, p Process) {
	<-p.Closed()
	Q <- s
}
<|endoftext|>


<commit_before>package livesql

import (
	"database/sql/driver"
	"errors"
	"fmt"
	"reflect"
	"time"

	"github.com/samsarahq/thunder/internal/fields"
	"github.com/samsarahq/thunder/sqlgen"
	"github.com/samsarahq/thunder/thunderpb"
)

// valueToField converts a driver.Value into a thunderpb.Field.
// driver.Value must be one of the following:
//   int64
//   float64
//   bool
//   []byte
//   string
//   time.Time
func valueToField(value driver.Value) (*thunderpb.Field, error) {
	switch column := value.(type) {
	case nil:
		return &thunderpb.Field{Kind: thunderpb.FieldKind_Null}, nil
	case int64:
		return &thunderpb.Field{Kind: thunderpb.FieldKind_Int, Value: &thunderpb.Field_Int{Int: column}}, nil
	case float64:
		return &thunderpb.Field{Kind: thunderpb.FieldKind_Float64, Value: &thunderpb.Field_Float64{Float64: column}}, nil
	case bool:
		return &thunderpb.Field{Kind: thunderpb.FieldKind_Bool, Value: &thunderpb.Field_Bool{Bool: column}}, nil
	case []byte:
		return &thunderpb.Field{Kind: thunderpb.FieldKind_Bytes, Value: &thunderpb.Field_Bytes{Bytes: column}}, nil
	case string:
		return &thunderpb.Field{Kind: thunderpb.FieldKind_String, Value: &thunderpb.Field_String_{String_: column}}, nil
	case time.Time:
		return &thunderpb.Field{Kind: thunderpb.FieldKind_Time, Value: &thunderpb.Field_Time{Time: &column}}, nil
	default:
		return nil, fmt.Errorf("unknown type %s", reflect.TypeOf(column))
	}
}

func fieldToValue(field *thunderpb.Field) (driver.Value, error) {
	switch field.Kind {
	case thunderpb.FieldKind_Null:
		return nil, nil
	case thunderpb.FieldKind_Bool:
		return field.Value.(*thunderpb.Field_Bool).Bool, nil
	case thunderpb.FieldKind_Int:
		return field.Value.(*thunderpb.Field_Int).Int, nil
	case thunderpb.FieldKind_Uint:
		return field.Value.(*thunderpb.Field_Uint).Uint, nil
	case thunderpb.FieldKind_String:
		return field.Value.(*thunderpb.Field_String_).String_, nil
	case thunderpb.FieldKind_Bytes:
		return field.Value.(*thunderpb.Field_Bytes).Bytes, nil
	case thunderpb.FieldKind_Float64:
		return field.Value.(*thunderpb.Field_Float64).Float64, nil
	case thunderpb.FieldKind_Time:
		return *field.Value.(*thunderpb.Field_Time).Time, nil
	default:
		return nil, fmt.Errorf("unknown kind %s", field.Kind.String())
	}
}

// filterToProto takes a sqlgen.Filter, runs Valuer on each filter value, and returns a thunderpb.SQLFilter.
func filterToProto(schema *sqlgen.Schema, tableName string, filter sqlgen.Filter) (*thunderpb.SQLFilter, error) {
	table, ok := schema.ByName[tableName]
	if !ok {
		return nil, fmt.Errorf("unknown table: %s", tableName)
	}

	if filter == nil {
		return &thunderpb.SQLFilter{Table: tableName}, nil
	}

	fields := make(map[string]*thunderpb.Field, len(filter))
	for col, val := range filter {
		column, ok := table.ColumnsByName[col]
		if !ok {
			return nil, fmt.Errorf("unknown column %s", col)
		}

		val, err := column.Descriptor.Valuer(reflect.ValueOf(val)).Value()
		if err != nil {
			return nil, err
		}

		field, err := valueToField(val)
		if err != nil {
			return nil, err
		}
		fields[col] = field
	}
	return &thunderpb.SQLFilter{Table: tableName, Fields: fields}, nil
}

// filterFromProto takes a thunderpb.SQLFilter, runs Scanner on each field value, and returns a sqlgen.Filter.
func filterFromProto(schema *sqlgen.Schema, proto *thunderpb.SQLFilter) (string, sqlgen.Filter, error) {
	table, ok := schema.ByName[proto.Table]
	if !ok {
		return "", nil, fmt.Errorf("unknown table: %s", proto.Table)
	}

	scanners := table.Scanners.Get().([]interface{})
	defer table.Scanners.Put(scanners)

	filter := make(sqlgen.Filter, len(proto.Fields))
	for col, field := range proto.Fields {
		val, err := fieldToValue(field)
		if err != nil {
			return "", nil, err
		}

		column, ok := table.ColumnsByName[col]
		if !ok {
			return "", nil, fmt.Errorf("unknown column %s", col)
		}

		if !column.Descriptor.Ptr && val == nil {
			return "", nil, errors.New("cannot unmarshal nil into non-pointer type")
		}

		scanner := scanners[column.Order].(*fields.Scanner)

		// target is always a pointer.
		var target, ptrptr reflect.Value
		if column.Descriptor.Ptr {
			// We need to hold onto this pointer-pointer in order to make the value addressable.
			ptrptr = reflect.New(reflect.PtrTo(column.Descriptor.Type))
			target = ptrptr.Elem()
		} else {
			target = reflect.New(column.Descriptor.Type)
		}
		scanner.Target(target)

		if err := scanner.Scan(val); err != nil {
			return "", nil, err
		}

		if column.Descriptor.Ptr {
			filter[col] = target.Interface()
		} else {
			// Dereference pointer if column type is not a pointer.
			filter[col] = target.Elem().Interface()
		}
	}
	return proto.Table, filter, nil
}
<commit_msg>go livesql: use nil-safe accessors for thunderpb.Field values<commit_after>package livesql

import (
	"database/sql/driver"
	"errors"
	"fmt"
	"reflect"
	"time"

	"github.com/samsarahq/thunder/internal/fields"
	"github.com/samsarahq/thunder/sqlgen"
	"github.com/samsarahq/thunder/thunderpb"
)

// valueToField converts a driver.Value into a thunderpb.Field.
// driver.Value must be one of the following:
//   int64
//   float64
//   bool
//   []byte
//   string
//   time.Time
func valueToField(value driver.Value) (*thunderpb.Field, error) {
	switch column := value.(type) {
	case nil:
		return &thunderpb.Field{Kind: thunderpb.FieldKind_Null}, nil
	case int64:
		return &thunderpb.Field{Kind: thunderpb.FieldKind_Int, Value: &thunderpb.Field_Int{Int: column}}, nil
	case float64:
		return &thunderpb.Field{Kind: thunderpb.FieldKind_Float64, Value: &thunderpb.Field_Float64{Float64: column}}, nil
	case bool:
		return &thunderpb.Field{Kind: thunderpb.FieldKind_Bool, Value: &thunderpb.Field_Bool{Bool: column}}, nil
	case []byte:
		return &thunderpb.Field{Kind: thunderpb.FieldKind_Bytes, Value: &thunderpb.Field_Bytes{Bytes: column}}, nil
	case string:
		return &thunderpb.Field{Kind: thunderpb.FieldKind_String, Value: &thunderpb.Field_String_{String_: column}}, nil
	case time.Time:
		return &thunderpb.Field{Kind: thunderpb.FieldKind_Time, Value: &thunderpb.Field_Time{Time: &column}}, nil
	default:
		return nil, fmt.Errorf("unknown type %s", reflect.TypeOf(column))
	}
}

func fieldToValue(field *thunderpb.Field) (driver.Value, error) {
	switch field.Kind {
	case thunderpb.FieldKind_Null:
		return nil, nil
	case thunderpb.FieldKind_Bool:
		return field.GetBool(), nil
	case thunderpb.FieldKind_Int:
		return field.GetInt(), nil
	case thunderpb.FieldKind_Uint:
		return field.GetUint(), nil
	case thunderpb.FieldKind_String:
		return field.GetString_(), nil
	case thunderpb.FieldKind_Bytes:
		return field.GetBytes(), nil
	case thunderpb.FieldKind_Float64:
		return field.GetFloat64(), nil
	case thunderpb.FieldKind_Time:
		ptr := field.GetTime()
		if ptr == nil {
			return time.Time{}, nil
		}
		return *ptr, nil
	default:
		return nil, fmt.Errorf("unknown kind %s", field.Kind.String())
	}
}

// filterToProto takes a sqlgen.Filter, runs Valuer on each filter value, and returns a thunderpb.SQLFilter.
func filterToProto(schema *sqlgen.Schema, tableName string, filter sqlgen.Filter) (*thunderpb.SQLFilter, error) {
	table, ok := schema.ByName[tableName]
	if !ok {
		return nil, fmt.Errorf("unknown table: %s", tableName)
	}

	if filter == nil {
		return &thunderpb.SQLFilter{Table: tableName}, nil
	}

	fields := make(map[string]*thunderpb.Field, len(filter))
	for col, val := range filter {
		column, ok := table.ColumnsByName[col]
		if !ok {
			return nil, fmt.Errorf("unknown column %s", col)
		}

		val, err := column.Descriptor.Valuer(reflect.ValueOf(val)).Value()
		if err != nil {
			return nil, err
		}

		field, err := valueToField(val)
		if err != nil {
			return nil, err
		}
		fields[col] = field
	}
	return &thunderpb.SQLFilter{Table: tableName, Fields: fields}, nil
}

// filterFromProto takes a thunderpb.SQLFilter, runs Scanner on each field value, and returns a sqlgen.Filter.
func filterFromProto(schema *sqlgen.Schema, proto *thunderpb.SQLFilter) (string, sqlgen.Filter, error) {
	table, ok := schema.ByName[proto.Table]
	if !ok {
		return "", nil, fmt.Errorf("unknown table: %s", proto.Table)
	}

	scanners := table.Scanners.Get().([]interface{})
	defer table.Scanners.Put(scanners)

	filter := make(sqlgen.Filter, len(proto.Fields))
	for col, field := range proto.Fields {
		val, err := fieldToValue(field)
		if err != nil {
			return "", nil, err
		}

		column, ok := table.ColumnsByName[col]
		if !ok {
			return "", nil, fmt.Errorf("unknown column %s", col)
		}

		if !column.Descriptor.Ptr && val == nil {
			return "", nil, errors.New("cannot unmarshal nil into non-pointer type")
		}

		scanner := scanners[column.Order].(*fields.Scanner)

		// target is always a pointer.
		var target, ptrptr reflect.Value
		if column.Descriptor.Ptr {
			// We need to hold onto this pointer-pointer in order to make the value addressable.
			ptrptr = reflect.New(reflect.PtrTo(column.Descriptor.Type))
			target = ptrptr.Elem()
		} else {
			target = reflect.New(column.Descriptor.Type)
		}
		scanner.Target(target)

		if err := scanner.Scan(val); err != nil {
			return "", nil, err
		}

		if column.Descriptor.Ptr {
			filter[col] = target.Interface()
		} else {
			// Dereference pointer if column type is not a pointer.
			filter[col] = target.Elem().Interface()
		}
	}
	return proto.Table, filter, nil
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"bytes"
	"fmt"
	"go/build"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"sort"
	"strings"

	"golang.org/x/tools/go/vcs"
)

// Import defines an import dependency
type Import struct {
	// ie, golang.org/x/tools/go/vcs
	ImportPath string

	// ie, 759e96ebaffb01c3cba0e8b129ef29f56507b323
	Rev string

	// Controls verbosity of output
	Verbose bool

	// see https://godoc.org/golang.org/x/tools/go/vcs#RepoRoot
	Repo *vcs.RepoRoot
}

// RestoreImport takes the import and restores it at the given GOPATH.
// There are four steps to this:
//   1. cd $GOPATH/src/<import_path>
//   2. Checkout default branch (ie, git checkout master)
//   3. Download changes (ie, git pull --ff-only)
//   4. Checkout revision (ie, git checkout 759e96ebaffb01c3cba0e8b129ef29f56507b323)
func (i *Import) RestoreImport(gopath string) error {
	vcs.ShowCmd = i.Verbose
	fullpath := filepath.Join(gopath, "src", i.ImportPath)
	fmt.Printf("> Restoring %s to %s\n", fullpath, i.Rev)

	// If the repo doesn't exist already, create it
	_, err := os.Stat(fullpath)
	if err != nil && os.IsNotExist(err) {
		if i.Verbose {
			fmt.Printf("> Repo %s not found, creating at rev %s\n", fullpath, i.Rev)
		}
		rootpath := filepath.Join(gopath, "src", i.Repo.Root)
		err = i.Repo.VCS.CreateAtRev(rootpath, i.Repo.Repo, i.Rev)
		if err != nil {
			return fmt.Errorf("Error creating repo at %s, %s\n",
				fullpath, err.Error())
		}
		return nil
	}

	// Checkout default branch
	_, err = runInDir(i.Repo.VCS.Cmd, strings.Fields(i.Repo.VCS.TagSyncDefault),
		fullpath, i.Verbose)
	if err != nil {
		return err
	}

	// Download changes
	err = i.Repo.VCS.Download(fullpath)
	if err != nil {
		return fmt.Errorf("Error downloading changes to %s, %s\n",
			fullpath, err.Error())
	}

	// Checkout revision
	cmdString := i.Repo.VCS.TagSyncCmd
	cmdString = strings.Replace(cmdString, "{tag}", i.Rev, 1)
	_, err = runInDir(i.Repo.VCS.Cmd, strings.Fields(cmdString),
		fullpath, i.Verbose)
	if err != nil {
		return err
	}

	return nil
}

// ImportsFromFile reads the given file and returns Import structs.
func ImportsFromFile(filename string) []*Import {
	content, err := ioutil.ReadFile(filename)
	if err != nil {
		panic(err)
	}
	lines := strings.Split(string(content), "\n")
	imports := []*Import{}
	roots := make(map[string]bool)
	for _, line := range lines {
		line = strings.TrimSpace(line)
		if strings.HasPrefix(line, "#") || len(line) == 0 {
			// Skip commented line
			continue
		} else if strings.Contains(line, "#") {
			// in-line comment
			line = strings.TrimSpace(strings.Split(line, "#")[0])
		}
		parts := strings.Fields(line)
		if len(parts) != 2 {
			fmt.Fprintf(os.Stderr, "Invalid line: %s\n", line)
			os.Exit(1)
		}
		path := parts[0]
		rev := parts[1]
		root, err := getRepoRoot(path)
		if err != nil {
			fmt.Fprintf(os.Stderr, "Error getting VCS info for %s\n", path)
			os.Exit(1)
		}
		if _, ok := roots[root.Root]; !ok {
			roots[root.Root] = true
			imports = append(imports, &Import{
				Rev:        rev,
				ImportPath: path,
				Repo:       root,
			})
		}
	}
	return imports
}

// ImportsFromPath looks in the given working directory and finds all 3rd-party
// imports, and returns Import structs
func ImportsFromPath(wd, gopath string, verbose bool) ([]*Import, error) {
	// Get a set of transitive dependencies (package import paths) for the
	// specified package.
	depsOutput, err := runInDir("go",
		[]string{"list", "-f", `{{range .Deps}}{{.}}{{"\n"}}{{end}}`, "./..."},
		wd, verbose)
	if err != nil {
		return nil, err
	}
	// filter out standard library
	deps := filterPackages(depsOutput, nil)

	// List dependencies of test files, which are not included in the go list .Deps
	// Also, ignore any dependencies that are already covered.
	testDepsOutput, err := runInDir("go",
		[]string{"list", "-f",
			`{{join .TestImports "\n"}}{{"\n"}}{{join .XTestImports "\n"}}`, "./..."},
		wd, verbose)
	if err != nil {
		return nil, err
	}
	// filter out stdlib and existing deps
	testDeps := filterPackages(testDepsOutput, deps)
	for dep := range testDeps {
		deps[dep] = true
	}

	// Sort the import set into a list of string paths
	sortedImportPaths := []string{}
	repoRoot := getImportPath(wd)
	for path, _ := range deps {
		// Do not vendor the repo that we are vendoring
		if path == repoRoot {
			continue
		}
		sortedImportPaths = append(sortedImportPaths, path)
	}
	sort.Strings(sortedImportPaths)

	// Iterate through imports, creating a list of Import structs
	result := []*Import{}
	for _, importpath := range sortedImportPaths {
		root, err := getRepoRoot(importpath)
		if err != nil {
			fmt.Fprintf(os.Stderr, "Error getting VCS info for %s, skipping\n", importpath)
			continue
		}
		_, ok := deps[root.Root]
		if root.Root == importpath || !ok {
			// Use the repo root as importpath if it's a usable go VCS repo
			if _, err := getRepoRoot(root.Root); err == nil {
				deps[root.Root] = true
				importpath = root.Root
			}
			// If this is the repo root, or root is not already imported
			fullpath := filepath.Join(gopath, "src", importpath)
			rev := getRevisionFromPath(fullpath, root)
			result = append(result, &Import{
				Rev:        rev,
				ImportPath: importpath,
				Repo:       root,
			})
		}
	}
	return result, nil
}

// getImportPath takes a path like /home/csparr/go/src/github.com/sparrc/gdm
// and returns the import path, ie, github.com/sparrc/gdm
func getImportPath(fullpath string) string {
	p, err := build.ImportDir(fullpath, 0)
	if err != nil {
		return ""
	}
	return p.ImportPath
}

// getRepoRoot takes an import path like github.com/sparrc/gdm
// and returns the VCS Repository information for it.
func getRepoRoot(importpath string) (*vcs.RepoRoot, error) {
	repo, err := vcs.RepoRootForImportPath(importpath, false)
	if err != nil {
		return nil, err
	}
	return repo, nil
}

// getRevisionFromPath takes a path like /home/csparr/go/src/github.com/sparrc/gdm
// and the VCS Repository information and returns the currently checked out
// revision, ie, 759e96ebaffb01c3cba0e8b129ef29f56507b323
func getRevisionFromPath(fullpath string, root *vcs.RepoRoot) string {
	// Check that we have the executable available
	_, err := exec.LookPath(root.VCS.Cmd)
	if err != nil {
		fmt.Fprintf(os.Stderr, "gdm missing %s command.\n", root.VCS.Name)
		os.Exit(1)
	}

	// Determine command to get the current hash
	var cmd *exec.Cmd
	switch root.VCS.Cmd {
	case "git":
		cmd = exec.Command("git", "rev-parse", "HEAD")
	case "hg":
		cmd = exec.Command("hg", "id", "-i")
	case "bzr":
		cmd = exec.Command("bzr", "revno")
	default:
		fmt.Fprintf(os.Stderr, "gdm does not support %s\n", root.VCS.Cmd)
		os.Exit(1)
	}
	cmd.Dir = fullpath

	output, err := cmd.Output()
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error getting revision hash at %s, %s\n",
			fullpath, err.Error())
		os.Exit(1)
	}
	return strings.TrimSpace(string(output))
}

// filterPackages accepts the output of a go list comment (one package per line)
// and returns a set of package import paths, excluding standard library.
// Additionally, any packages present in the "exclude" set will be excluded.
func filterPackages(output []byte, exclude map[string]bool) map[string]bool {
	var scanner = bufio.NewScanner(bytes.NewReader(output))
	var deps = map[string]bool{}
	for scanner.Scan() {
		var (
			pkg    = scanner.Text()
			slash  = strings.Index(pkg, "/")
			stdLib = slash == -1 || strings.Index(pkg[:slash], ".") == -1
		)
		if stdLib {
			continue
		}
		if _, ok := exclude[pkg]; ok {
			continue
		}
		deps[pkg] = true
	}
	return deps
}

// runInDir runs the given command (name) with args, in the given directory.
// if verbose, prints out the command and dir it is executing.
// This function exits the whole program if it fails.
// Returns output of the command.
func runInDir(name string, args []string, dir string, verbose bool) ([]byte, error) {
	cmd := exec.Command(name, args...)
	cmd.Dir = dir
	if verbose {
		fmt.Printf("cd %s\n%s %s\n", dir, name, strings.Join(args, " "))
	}
	output, err := cmd.Output()
	if err != nil {
		fmt.Errorf("Error running %s %s in dir %s, %s\n",
			name, strings.Join(args, " "), dir, err.Error())
		return output, err
	}
	return output, nil
}
<commit_msg>Attempt a checkout before pulling remote<commit_after>package main

import (
	"bufio"
	"bytes"
	"fmt"
	"go/build"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"sort"
	"strings"

	"golang.org/x/tools/go/vcs"
)

// Import defines an import dependency
type Import struct {
	// ie, golang.org/x/tools/go/vcs
	ImportPath string

	// ie, 759e96ebaffb01c3cba0e8b129ef29f56507b323
	Rev string

	// Controls verbosity of output
	Verbose bool

	// see https://godoc.org/golang.org/x/tools/go/vcs#RepoRoot
	Repo *vcs.RepoRoot
}

// RestoreImport takes the import and restores it at the given GOPATH.
// There are four steps to this:
//   1. cd $GOPATH/src/<import_path>
//   2. Checkout default branch (ie, git checkout master)
//   3. Download changes (ie, git pull --ff-only)
//   4. Checkout revision (ie, git checkout 759e96ebaffb01c3cba0e8b129ef29f56507b323)
func (i *Import) RestoreImport(gopath string) error {
	vcs.ShowCmd = i.Verbose
	fullpath := filepath.Join(gopath, "src", i.ImportPath)
	fmt.Printf("> Restoring %s to %s\n", fullpath, i.Rev)

	// If the repo doesn't exist already, create it
	_, err := os.Stat(fullpath)
	if err != nil && os.IsNotExist(err) {
		if i.Verbose {
			fmt.Printf("> Repo %s not found, creating at rev %s\n", fullpath, i.Rev)
		}
		rootpath := filepath.Join(gopath, "src", i.Repo.Root)
		err = i.Repo.VCS.CreateAtRev(rootpath, i.Repo.Repo, i.Rev)
		if err != nil {
			return fmt.Errorf("Error creating repo at %s, %s\n",
				fullpath, err.Error())
		}
		return nil
	}

	// Checkout default branch
	_, err = runInDir(i.Repo.VCS.Cmd, strings.Fields(i.Repo.VCS.TagSyncDefault),
		fullpath, i.Verbose)
	if err != nil {
		return err
	}

	// Attempt to checkout revision.
	cmdString := i.Repo.VCS.TagSyncCmd
	cmdString = strings.Replace(cmdString, "{tag}", i.Rev, 1)
	if _, err = runInDir(i.Repo.VCS.Cmd, strings.Fields(cmdString), fullpath, i.Verbose); err == nil {
		return nil
	}

	// There was an error checking out revision: download changes and
	// re-checkout revision
	err = i.Repo.VCS.Download(fullpath)
	if err != nil {
		return fmt.Errorf("Error downloading changes to %s, %s\n",
			fullpath, err.Error())
	}
	_, err = runInDir(i.Repo.VCS.Cmd, strings.Fields(cmdString), fullpath, i.Verbose)
	return err
}

// ImportsFromFile reads the given file and returns Import structs.
func ImportsFromFile(filename string) []*Import {
	content, err := ioutil.ReadFile(filename)
	if err != nil {
		panic(err)
	}
	lines := strings.Split(string(content), "\n")
	imports := []*Import{}
	roots := make(map[string]bool)
	for _, line := range lines {
		line = strings.TrimSpace(line)
		if strings.HasPrefix(line, "#") || len(line) == 0 {
			// Skip commented line
			continue
		} else if strings.Contains(line, "#") {
			// in-line comment
			line = strings.TrimSpace(strings.Split(line, "#")[0])
		}
		parts := strings.Fields(line)
		if len(parts) != 2 {
			fmt.Fprintf(os.Stderr, "Invalid line: %s\n", line)
			os.Exit(1)
		}
		path := parts[0]
		rev := parts[1]
		root, err := getRepoRoot(path)
		if err != nil {
			fmt.Fprintf(os.Stderr, "Error getting VCS info for %s\n", path)
			os.Exit(1)
		}
		if _, ok := roots[root.Root]; !ok {
			roots[root.Root] = true
			imports = append(imports, &Import{
				Rev:        rev,
				ImportPath: path,
				Repo:       root,
			})
		}
	}
	return imports
}

// ImportsFromPath looks in the given working directory and finds all 3rd-party
// imports, and returns Import structs
func ImportsFromPath(wd, gopath string, verbose bool) ([]*Import, error) {
	// Get a set of transitive dependencies (package import paths) for the
	// specified package.
	depsOutput, err := runInDir("go",
		[]string{"list", "-f", `{{range .Deps}}{{.}}{{"\n"}}{{end}}`, "./..."},
		wd, verbose)
	if err != nil {
		return nil, err
	}
	// filter out standard library
	deps := filterPackages(depsOutput, nil)

	// List dependencies of test files, which are not included in the go list .Deps
	// Also, ignore any dependencies that are already covered.
	testDepsOutput, err := runInDir("go",
		[]string{"list", "-f",
			`{{join .TestImports "\n"}}{{"\n"}}{{join .XTestImports "\n"}}`, "./..."},
		wd, verbose)
	if err != nil {
		return nil, err
	}
	// filter out stdlib and existing deps
	testDeps := filterPackages(testDepsOutput, deps)
	for dep := range testDeps {
		deps[dep] = true
	}

	// Sort the import set into a list of string paths
	sortedImportPaths := []string{}
	repoRoot := getImportPath(wd)
	for path, _ := range deps {
		// Do not vendor the repo that we are vendoring
		if path == repoRoot {
			continue
		}
		sortedImportPaths = append(sortedImportPaths, path)
	}
	sort.Strings(sortedImportPaths)

	// Iterate through imports, creating a list of Import structs
	result := []*Import{}
	for _, importpath := range sortedImportPaths {
		root, err := getRepoRoot(importpath)
		if err != nil {
			fmt.Fprintf(os.Stderr, "Error getting VCS info for %s, skipping\n", importpath)
			continue
		}
		_, ok := deps[root.Root]
		if root.Root == importpath || !ok {
			// Use the repo root as importpath if it's a usable go VCS repo
			if _, err := getRepoRoot(root.Root); err == nil {
				deps[root.Root] = true
				importpath = root.Root
			}
			// If this is the repo root, or root is not already imported
			fullpath := filepath.Join(gopath, "src", importpath)
			rev := getRevisionFromPath(fullpath, root)
			result = append(result, &Import{
				Rev:        rev,
				ImportPath: importpath,
				Repo:       root,
			})
		}
	}
	return result, nil
}

// getImportPath takes a path like /home/csparr/go/src/github.com/sparrc/gdm
// and returns the import path, ie, github.com/sparrc/gdm
func getImportPath(fullpath string) string {
	p, err := build.ImportDir(fullpath, 0)
	if err != nil {
		return ""
	}
	return p.ImportPath
}

// getRepoRoot takes an import path like github.com/sparrc/gdm
// and returns the VCS Repository information for it.
func getRepoRoot(importpath string) (*vcs.RepoRoot, error) {
	repo, err := vcs.RepoRootForImportPath(importpath, false)
	if err != nil {
		return nil, err
	}
	return repo, nil
}

// getRevisionFromPath takes a path like /home/csparr/go/src/github.com/sparrc/gdm
// and the VCS Repository information and returns the currently checked out
// revision, ie, 759e96ebaffb01c3cba0e8b129ef29f56507b323
func getRevisionFromPath(fullpath string, root *vcs.RepoRoot) string {
	// Check that we have the executable available
	_, err := exec.LookPath(root.VCS.Cmd)
	if err != nil {
		fmt.Fprintf(os.Stderr, "gdm missing %s command.\n", root.VCS.Name)
		os.Exit(1)
	}

	// Determine command to get the current hash
	var cmd *exec.Cmd
	switch root.VCS.Cmd {
	case "git":
		cmd = exec.Command("git", "rev-parse", "HEAD")
	case "hg":
		cmd = exec.Command("hg", "id", "-i")
	case "bzr":
		cmd = exec.Command("bzr", "revno")
	default:
		fmt.Fprintf(os.Stderr, "gdm does not support %s\n", root.VCS.Cmd)
		os.Exit(1)
	}
	cmd.Dir = fullpath

	output, err := cmd.Output()
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error getting revision hash at %s, %s\n",
			fullpath, err.Error())
		os.Exit(1)
	}
	return strings.TrimSpace(string(output))
}

// filterPackages accepts the output of a go list comment (one package per line)
// and returns a set of package import paths, excluding standard library.
// Additionally, any packages present in the "exclude" set will be excluded.
func filterPackages(output []byte, exclude map[string]bool) map[string]bool {
	var scanner = bufio.NewScanner(bytes.NewReader(output))
	var deps = map[string]bool{}
	for scanner.Scan() {
		var (
			pkg    = scanner.Text()
			slash  = strings.Index(pkg, "/")
			stdLib = slash == -1 || strings.Index(pkg[:slash], ".") == -1
		)
		if stdLib {
			continue
		}
		if _, ok := exclude[pkg]; ok {
			continue
		}
		deps[pkg] = true
	}
	return deps
}

// runInDir runs the given command (name) with args, in the given directory.
// if verbose, prints out the command and dir it is executing.
// This function exits the whole program if it fails.
// Returns output of the command.
func runInDir(name string, args []string, dir string, verbose bool) ([]byte, error) {
	cmd := exec.Command(name, args...)
	cmd.Dir = dir
	if verbose {
		fmt.Printf("cd %s\n%s %s\n", dir, name, strings.Join(args, " "))
	}
	output, err := cmd.Output()
	if err != nil {
		fmt.Errorf("Error running %s %s in dir %s, %s\n",
			name, strings.Join(args, " "), dir, err.Error())
		return output, err
	}
	return output, nil
}
<|endoftext|>