<commit_before>package horenso

import (
	"encoding/json"
	"io/ioutil"
	"reflect"
	"testing"
)

func temp() string {
	f, err := ioutil.TempFile("", "")
	if err != nil {
		panic(err)
	}
	defer f.Close()
	return f.Name()
}

func parseReport(fname string) Report {
	byt, err := ioutil.ReadFile(fname)
	if err != nil {
		panic("failed to read " + fname)
	}
	r := Report{}
	json.Unmarshal(byt, &r)
	return r
}

func TestRun(t *testing.T) {
	noticeReport := temp()
	fname := temp()
	fname2 := temp()
	_, o, cmdArgs, err := parseArgs([]string{
		"--noticer",
		"go run _testdata/reporter.go " + noticeReport,
		"-n", "invalid",
		"--reporter",
		"go run _testdata/reporter.go " + fname,
		"-r",
		"go run _testdata/reporter.go " + fname2,
		"--",
		"go", "run", "_testdata/run.go",
	})
	if err != nil {
		t.Errorf("err should be nil but: %s", err)
	}
	r, err := o.run(cmdArgs)
	if err != nil {
		t.Errorf("err should be nil but: %s", err)
	}

	if *r.ExitCode != 0 {
		t.Errorf("exit code should be 0 but: %d", r.ExitCode)
	}

	expect := "1\n2\n3\n"
	if r.Output != expect {
		t.Errorf("output should be %s but: %s", expect, r.Output)
	}
	if r.Stdout != expect {
		t.Errorf("output should be %s but: %s", expect, r.Stdout)
	}
	if r.Stderr != "" {
		t.Errorf("output should be empty but: %s", r.Stderr)
	}
	if r.StartAt == nil {
		t.Errorf("StartAt shouldn't be nil")
	}
	if r.EndAt == nil {
		t.Errorf("EtartAt shouldn't be nil")
	}

	rr := parseReport(fname)
	if !reflect.DeepEqual(r, rr) {
		t.Errorf("something went wrong. expect: %#v, got: %#v", r, rr)
	}
	rr2 := parseReport(fname2)
	if !reflect.DeepEqual(r, rr2) {
		t.Errorf("something went wrong. expect: %#v, got: %#v", r, rr2)
	}

	nr := parseReport(noticeReport)
	if nr.Pid != r.Pid {
		t.Errorf("something went wrong")
	}
	if nr.Output != "" {
		t.Errorf("something went wrong")
	}
	if nr.StartAt == nil {
		t.Errorf("StartAt shouldn't be nil")
	}
	if nr.EndAt != nil {
		t.Errorf("EndAt should be nil")
	}
	if nr.ExitCode != nil {
		t.Errorf("ExitCode should be nil")
	}
}
<commit_msg>define deepEqual(r1, r2)<commit_after>package horenso

import (
	"encoding/json"
	"io/ioutil"
	"reflect"
	"testing"
)

func temp() string {
	f, err := ioutil.TempFile("", "")
	if err != nil {
		panic(err)
	}
	defer f.Close()
	return f.Name()
}

func parseReport(fname string) Report {
	byt, err := ioutil.ReadFile(fname)
	if err != nil {
		panic("failed to read " + fname)
	}
	r := Report{}
	json.Unmarshal(byt, &r)
	return r
}

func TestRun(t *testing.T) {
	noticeReport := temp()
	fname := temp()
	fname2 := temp()
	_, o, cmdArgs, err := parseArgs([]string{
		"--noticer",
		"go run _testdata/reporter.go " + noticeReport,
		"-n", "invalid",
		"--reporter",
		"go run _testdata/reporter.go " + fname,
		"-r",
		"go run _testdata/reporter.go " + fname2,
		"--",
		"go", "run", "_testdata/run.go",
	})
	if err != nil {
		t.Errorf("err should be nil but: %s", err)
	}
	r, err := o.run(cmdArgs)
	if err != nil {
		t.Errorf("err should be nil but: %s", err)
	}

	if *r.ExitCode != 0 {
		t.Errorf("exit code should be 0 but: %d", r.ExitCode)
	}

	expect := "1\n2\n3\n"
	if r.Output != expect {
		t.Errorf("output should be %s but: %s", expect, r.Output)
	}
	if r.Stdout != expect {
		t.Errorf("output should be %s but: %s", expect, r.Stdout)
	}
	if r.Stderr != "" {
		t.Errorf("output should be empty but: %s", r.Stderr)
	}
	if r.StartAt == nil {
		t.Errorf("StartAt shouldn't be nil")
	}
	if r.EndAt == nil {
		t.Errorf("EtartAt shouldn't be nil")
	}

	rr := parseReport(fname)
	if !deepEqual(r, rr) {
		t.Errorf("something went wrong. expect: %#v, got: %#v", r, rr)
	}
	rr2 := parseReport(fname2)
	if !deepEqual(r, rr2) {
		t.Errorf("something went wrong. expect: %#v, got: %#v", r, rr2)
	}

	nr := parseReport(noticeReport)
	if nr.Pid != r.Pid {
		t.Errorf("something went wrong")
	}
	if nr.Output != "" {
		t.Errorf("something went wrong")
	}
	if nr.StartAt == nil {
		t.Errorf("StartAt shouldn't be nil")
	}
	if nr.EndAt != nil {
		t.Errorf("EndAt should be nil")
	}
	if nr.ExitCode != nil {
		t.Errorf("ExitCode should be nil")
	}
}

func deepEqual(r1, r2 Report) bool {
	return r1.Command == r2.Command &&
		reflect.DeepEqual(r1.CommandArgs, r2.CommandArgs) &&
		r1.Tag == r2.Tag &&
		r1.Output == r2.Output &&
		r1.Stdout == r2.Stdout &&
		r1.Stderr == r2.Stderr &&
		*r1.ExitCode == *r2.ExitCode &&
		r1.Result == r2.Result &&
		r1.Pid == r2.Pid &&
		*r1.StartAt == *r2.StartAt &&
		*r2.EndAt == *r2.EndAt
}
<|endoftext|>


<commit_before>package elastic

import (
	"strconv"

	"github.com/astaxie/beego"

	"gopkg.in/olivere/elastic.v3"
)

var ESClient *elastic.Client

func InitElastic(url string) error {
	var err error
	ESClient, err = elastic.NewClient(elastic.SetURL(url))
	if err != nil {
		return err
	}

	info, code, err := ESClient.Ping(url).Do()
	if err != nil {
		return err
	}
	beego.Trace("Elasticsearch returned with code " + strconv.Itoa(code) + "and version " + info.Version.Number)

	esversion, err := ESClient.ElasticsearchVersion(url)
	if err != nil {
		return err
	}
	beego.Trace("Elasticsearch version " + esversion)

	return nil
}

func CreateIndexElastic(index string) error {
	exists, err := ESClient.IndexExists(index).Do()
	if err != nil {
		return err
	}
	if !exists {
		createIndex, err := ESClient.CreateIndex(index).Do()
		if err != nil {
			return err
		}
		if !createIndex.Acknowledged {
			// Not acknowledged
			beego.Trace("Not acknowledged: " + strconv.FormatBool(createIndex.Acknowledged))
		}
	}
	return nil
}
<commit_msg>增加Elasticsearch用例<commit_after>package elastic

import (
	"strconv"

	"github.com/astaxie/beego"

	"gopkg.in/olivere/elastic.v3"
)

var ESClient *elastic.Client

func InitElastic(url string) error {
	var err error
	ESClient, err = elastic.NewClient(elastic.SetURL(url))
	if err != nil {
		return err
	}

	info, code, err := ESClient.Ping(url).Do()
	if err != nil {
		return err
	}
	beego.Trace("Elasticsearch returned with code " + strconv.Itoa(code) + "and version " + info.Version.Number)

	esversion, err := ESClient.ElasticsearchVersion(url)
	if err != nil {
		return err
	}
	beego.Trace("Elasticsearch version " + esversion)

	return nil
}

func CreateIndexElastic(index string) error {
	exists, err := ESClient.IndexExists(index).Do()
	if err != nil {
		return err
	}
	if !exists {
		createIndex, err := ESClient.CreateIndex(index).Do()
		if err != nil {
			return err
		}
		if !createIndex.Acknowledged {
			// Not acknowledged
			beego.Trace("Not acknowledged: " + strconv.FormatBool(createIndex.Acknowledged))
		}
	}
	return nil
}

/*
type Member struct {
	User     string `json:"user"`
	Nickname string `json:"nickname"`
}

func main() {
	err := core.InitElastic("http://127.0.0.1:9200")
	if err != nil {
		panic(err)
	}

	err = core.CreateIndexElastic("member")
	if err != nil {
		panic(err)
	}

			member1 := Member{User: "jream", Nickname: "fjh"}
			put1, err := core.ESClient.Index().
				Index("member").
				Type("family").
				Id("1").
				BodyJson(member1).
				Do()
			if err != nil {
				// Handle error
				panic(err)
			}

			member2 := Member{User: "jream", Nickname: "fjh34D"}
			put2, err := core.ESClient.Index().
				Index("member").
				Type("family").
				Id("2").
				BodyJson(member2).
				Do()
			if err != nil {
				// Handle error
				panic(err)
			}
			fmt.Printf("Indexed tweet %s to index %s, type %s\n", put1.Id, put1.Index, put1.Type)
			fmt.Printf("Indexed tweet %s to index %s, type %s\n", put1.Id, put2.Index, put2.Type)

		get1, err := core.ESClient.Get().
			Index("member").
			Type("family").
			Id("1").
			Do()
		if err != nil {
			// Handle error
			panic(err)
		}
		if get1.Found {
			fmt.Printf("Got document %s in version %d from index %s, type %s\n", get1.Id, get1.Version, get1.Index, get1.Type)
		}

	termQuery := elastic.NewTermQuery("user", "jream")
	searchResult, err := core.ESClient.Search().
		Index("member").    // search in index "twitter"
		Query(termQuery).   // specify the query
		Sort("user", true). // sort by "user" field, ascending
		From(0).Size(10).   // take documents 0-9
		Pretty(true).       // pretty print request and response JSON
		Do()                // execute
	if err != nil {
		// Handle error
		panic(err)
	}
	fmt.Printf("Query took %d milliseconds\n", searchResult.TookInMillis)

	var mem Member
	for _, item := range searchResult.Each(reflect.TypeOf(mem)) {
		m := item.(Member)
		fmt.Printf("Member by %s: %s\n", m.User, m.Nickname)
	}

	fmt.Printf("Found a total of %d family\n", searchResult.TotalHits())

	if searchResult.Hits.TotalHits > 0 {
		fmt.Printf("Found a total of %d family\n", searchResult.Hits.TotalHits)

		for _, hit := range searchResult.Hits.Hits {
			var m Member
			err := json.Unmarshal(*hit.Source, &m)
			if err != nil {
			}

			fmt.Printf("Member by %s: %s\n", m.User, m.Nickname)
		}
	} else {
		fmt.Print("Found no family\n")
	}

	script := elastic.NewScript("ctx._source.nickname = _nickname").Param("_nickname", "fjh301")
	update, err := core.ESClient.Update().Index("member").Type("family").Id("1").
		Script(script).
		Upsert(map[string]interface{}{"nickname": "fjh"}).
		Do()
	if err != nil {
		// Handle error
		panic(err)
	}
	fmt.Printf("New version of member %q is now %d", update.Id, update.Version)

	_, err = core.ESClient.Flush().Index("member").Do()
	if err != nil {
		panic(err)
	}

		deleteIndex, err := core.ESClient.DeleteIndex("member").Do()
		if err != nil {
			panic(err)
		}
		if !deleteIndex.Acknowledged {
			fmt.Println("Not acknowledged: ", deleteIndex.Acknowledged)
		}
}
*/
<|endoftext|>


<commit_before>package store

import (
	"encoding/json"
	"errors"
	"io"
	"os"
	"sync"
	"time"

	"github.com/cloudfoundry-incubator/executor"
	"github.com/cloudfoundry-incubator/executor/depot/log_streamer"
	"github.com/cloudfoundry-incubator/executor/depot/steps"
	"github.com/cloudfoundry-incubator/executor/depot/transformer"
	garden "github.com/cloudfoundry-incubator/garden/api"
	"github.com/cloudfoundry/gunk/timeprovider"
	"github.com/pivotal-golang/lager"
	"github.com/tedsuo/ifrit"
)

var (
	ErrContainerNotFound = errors.New("container not found")
)

type GardenStore struct {
	gardenClient       garden.Client
	exchanger          Exchanger
	containerOwnerName string

	healthyMonitoringInterval   time.Duration
	unhealthyMonitoringInterval time.Duration

	transformer  *transformer.Transformer
	timeProvider timeprovider.TimeProvider

	tracker InitializedTracker

	eventEmitter EventEmitter

	runningProcesses map[string]ifrit.Process
	processesL       sync.Mutex
}

type InitializedTracker interface {
	Initialize(executor.Container)
	Deinitialize(string)
	SyncInitialized([]executor.Container)
}

func NewGardenStore(
	gardenClient garden.Client,
	containerOwnerName string,
	containerMaxCPUShares uint64,
	containerInodeLimit uint64,
	healthyMonitoringInterval time.Duration,
	unhealthyMonitoringInterval time.Duration,
	transformer *transformer.Transformer,
	timeProvider timeprovider.TimeProvider,
	tracker InitializedTracker,
	eventEmitter EventEmitter,
) *GardenStore {
	return &GardenStore{
		gardenClient:       gardenClient,
		exchanger:          NewExchanger(containerOwnerName, containerMaxCPUShares, containerInodeLimit),
		containerOwnerName: containerOwnerName,

		healthyMonitoringInterval:   healthyMonitoringInterval,
		unhealthyMonitoringInterval: unhealthyMonitoringInterval,

		transformer:  transformer,
		timeProvider: timeProvider,

		tracker: tracker,

		eventEmitter: eventEmitter,

		runningProcesses: map[string]ifrit.Process{},
	}
}

func (store *GardenStore) Lookup(guid string) (executor.Container, error) {
	gardenContainer, err := store.gardenClient.Lookup(guid)
	if err != nil {
		return executor.Container{}, ErrContainerNotFound
	}

	return store.exchanger.Garden2Executor(gardenContainer)
}

func (store *GardenStore) List(tags executor.Tags) ([]executor.Container, error) {
	filter := garden.Properties{
		ContainerOwnerProperty: store.containerOwnerName,
	}

	for k, v := range tags {
		filter[tagPropertyPrefix+k] = v
	}

	gardenContainers, err := store.gardenClient.Containers(filter)
	if err != nil {
		return nil, ErrContainerNotFound
	}

	result := make([]executor.Container, 0, len(gardenContainers))

	for _, gardenContainer := range gardenContainers {
		container, err := store.exchanger.Garden2Executor(gardenContainer)
		if err != nil {
			continue
		}

		result = append(result, container)
	}

	return result, nil
}

func (store *GardenStore) Create(container executor.Container) (executor.Container, error) {
	container.State = executor.StateCreated

	container, err := store.exchanger.CreateInGarden(store.gardenClient, container)
	if err != nil {
		return executor.Container{}, err
	}

	store.tracker.Initialize(container)

	return container, nil
}

func (store *GardenStore) Stop(guid string) error {
	store.processesL.Lock()
	process, found := store.runningProcesses[guid]
	delete(store.runningProcesses, guid)
	store.processesL.Unlock()

	if !found {
		return ErrContainerNotFound
	}

	process.Signal(os.Interrupt)
	<-process.Wait()

	return nil
}

func (store *GardenStore) Destroy(guid string) error {
	err := store.gardenClient.Destroy(guid)
	if err != nil {
		return err
	}

	store.tracker.Deinitialize(guid)

	return nil
}

func (store *GardenStore) GetFiles(guid, sourcePath string) (io.ReadCloser, error) {
	container, err := store.gardenClient.Lookup(guid)
	if err != nil {
		return nil, ErrContainerNotFound
	}

	return container.StreamOut(sourcePath)
}

func (store *GardenStore) Ping() error {
	return store.gardenClient.Ping()
}

func (store *GardenStore) Run(container executor.Container, logger lager.Logger) {
	logger = logger.WithData(lager.Data{
		"container": container.Guid,
	})

	gardenContainer, err := store.gardenClient.Lookup(container.Guid)
	if err != nil {
		logger.Debug("container-deleted-just-before-running")
		return
	}

	logStreamer := log_streamer.New(
		container.Log.Guid,
		container.Log.SourceName,
		container.Log.Index,
	)

	seq := []steps.Step{}

	if container.Setup != nil {
		seq = append(seq, store.transformer.StepFor(
			logStreamer,
			container.Setup,
			gardenContainer,
			container.ExternalIP,
			container.Ports,
			logger,
		))
	}

	parallelSequence := []steps.Step{
		store.transformer.StepFor(
			logStreamer,
			container.Action,
			gardenContainer,
			container.ExternalIP,
			container.Ports,
			logger,
		),
	}

	hasStartedRunning := make(chan struct{}, 1)

	var monitorStep steps.Step

	if container.Monitor != nil {
		monitoredStep := store.transformer.StepFor(
			logStreamer,
			container.Monitor,
			gardenContainer,
			container.ExternalIP,
			container.Ports,
			logger,
		)

		monitorStep = steps.NewMonitor(
			monitoredStep,
			hasStartedRunning,
			logger.Session("monitor"),
			store.timeProvider,
			time.Duration(container.StartTimeout)*time.Second,
			store.healthyMonitoringInterval,
			store.unhealthyMonitoringInterval,
		)
		parallelSequence = append(parallelSequence, monitorStep)
	} else {
		// this container isn't monitored, so we mark it running right away
		hasStartedRunning <- struct{}{}
	}

	seq = append(seq, steps.NewCodependent(parallelSequence))

	step := steps.NewSerial(seq)

	process := ifrit.Invoke(ifrit.RunFunc(func(signals <-chan os.Signal, ready chan<- struct{}) error {
		seqComplete := make(chan error)

		close(ready)

		go func() {
			seqComplete <- step.Perform()
		}()

		result := executor.ContainerRunResult{}

	OUTER_LOOP:
		for {
			select {
			case <-signals:
				signals = nil
				if monitorStep != nil {
					monitorStep.Cancel()
				}
				step.Cancel()

			case <-hasStartedRunning:
				hasStartedRunning = nil
				err := store.transitionToRunning(gardenContainer)
				if err != nil {
					logger.Error("failed-to-transition-to-running", err)
					result.Failed = true
					result.FailureReason = err.Error()
					break OUTER_LOOP
				}

			case err := <-seqComplete:
				if err != nil {
					result.Failed = true
					result.FailureReason = err.Error()
				}
				break OUTER_LOOP
			}
		}

		err := store.transitionToComplete(gardenContainer, result)

		if err != nil {
			logger.Error("failed-to-transition-to-complete", err)
		}

		return nil
	}))

	store.processesL.Lock()
	store.runningProcesses[container.Guid] = process
	store.processesL.Unlock()
}

func (store *GardenStore) transitionToRunning(gardenContainer garden.Container) error {
	err := gardenContainer.SetProperty(ContainerStateProperty, string(executor.StateRunning))
	if err != nil {
		return err
	}

	executorContainer, err := store.exchanger.Garden2Executor(gardenContainer)
	if err != nil {
		return err
	}

	store.eventEmitter.EmitEvent(executor.ContainerRunningEvent{
		Container: executorContainer,
	})

	return nil
}

func (store *GardenStore) transitionToComplete(gardenContainer garden.Container, result executor.ContainerRunResult) error {
	resultJson, err := json.Marshal(result)
	if err != nil {
		return err
	}

	err = gardenContainer.SetProperty(ContainerResultProperty, string(resultJson))
	if err != nil {
		return err
	}

	err = gardenContainer.SetProperty(ContainerStateProperty, string(executor.StateCompleted))
	if err != nil {
		return err
	}

	executorContainer, err := store.exchanger.Garden2Executor(gardenContainer)
	if err != nil {
		return err
	}

	store.eventEmitter.EmitEvent(executor.ContainerCompleteEvent{
		Container: executorContainer,
	})

	return nil
}

func (store *GardenStore) TrackContainers(interval time.Duration, logger lager.Logger) ifrit.Runner {
	logger = logger.Session("garden-store-track-containers")

	return ifrit.RunFunc(func(signals <-chan os.Signal, ready chan<- struct{}) error {
		ticker := store.timeProvider.NewTicker(interval)
		defer ticker.Stop()

		close(ready)

	dance:
		for {
			select {
			case <-signals:
				break dance

			case <-ticker.C():
				containers, err := store.List(nil)
				if err != nil {
					logger.Error("failed-to-list-containers", err)
					continue
				}

				store.tracker.SyncInitialized(containers)
			}
		}

		return nil
	})
}
<commit_msg>Garden store Destroy cleans up step process<commit_after>package store

import (
	"encoding/json"
	"errors"
	"io"
	"os"
	"sync"
	"time"

	"github.com/cloudfoundry-incubator/executor"
	"github.com/cloudfoundry-incubator/executor/depot/log_streamer"
	"github.com/cloudfoundry-incubator/executor/depot/steps"
	"github.com/cloudfoundry-incubator/executor/depot/transformer"
	garden "github.com/cloudfoundry-incubator/garden/api"
	"github.com/cloudfoundry/gunk/timeprovider"
	"github.com/pivotal-golang/lager"
	"github.com/tedsuo/ifrit"
)

var (
	ErrContainerNotFound = errors.New("container not found")
)

type GardenStore struct {
	gardenClient       garden.Client
	exchanger          Exchanger
	containerOwnerName string

	healthyMonitoringInterval   time.Duration
	unhealthyMonitoringInterval time.Duration

	transformer  *transformer.Transformer
	timeProvider timeprovider.TimeProvider

	tracker InitializedTracker

	eventEmitter EventEmitter

	runningProcesses map[string]ifrit.Process
	processesL       sync.Mutex
}

type InitializedTracker interface {
	Initialize(executor.Container)
	Deinitialize(string)
	SyncInitialized([]executor.Container)
}

func NewGardenStore(
	gardenClient garden.Client,
	containerOwnerName string,
	containerMaxCPUShares uint64,
	containerInodeLimit uint64,
	healthyMonitoringInterval time.Duration,
	unhealthyMonitoringInterval time.Duration,
	transformer *transformer.Transformer,
	timeProvider timeprovider.TimeProvider,
	tracker InitializedTracker,
	eventEmitter EventEmitter,
) *GardenStore {
	return &GardenStore{
		gardenClient:       gardenClient,
		exchanger:          NewExchanger(containerOwnerName, containerMaxCPUShares, containerInodeLimit),
		containerOwnerName: containerOwnerName,

		healthyMonitoringInterval:   healthyMonitoringInterval,
		unhealthyMonitoringInterval: unhealthyMonitoringInterval,

		transformer:  transformer,
		timeProvider: timeProvider,

		tracker: tracker,

		eventEmitter: eventEmitter,

		runningProcesses: map[string]ifrit.Process{},
	}
}

func (store *GardenStore) Lookup(guid string) (executor.Container, error) {
	gardenContainer, err := store.gardenClient.Lookup(guid)
	if err != nil {
		return executor.Container{}, ErrContainerNotFound
	}

	return store.exchanger.Garden2Executor(gardenContainer)
}

func (store *GardenStore) List(tags executor.Tags) ([]executor.Container, error) {
	filter := garden.Properties{
		ContainerOwnerProperty: store.containerOwnerName,
	}

	for k, v := range tags {
		filter[tagPropertyPrefix+k] = v
	}

	gardenContainers, err := store.gardenClient.Containers(filter)
	if err != nil {
		return nil, ErrContainerNotFound
	}

	result := make([]executor.Container, 0, len(gardenContainers))

	for _, gardenContainer := range gardenContainers {
		container, err := store.exchanger.Garden2Executor(gardenContainer)
		if err != nil {
			continue
		}

		result = append(result, container)
	}

	return result, nil
}

func (store *GardenStore) Create(container executor.Container) (executor.Container, error) {
	container.State = executor.StateCreated

	container, err := store.exchanger.CreateInGarden(store.gardenClient, container)
	if err != nil {
		return executor.Container{}, err
	}

	store.tracker.Initialize(container)

	return container, nil
}

func (store *GardenStore) freeStepProcess(guid string) (ifrit.Process, bool) {
	store.processesL.Lock()
	process, found := store.runningProcesses[guid]
	delete(store.runningProcesses, guid)
	store.processesL.Unlock()

	if !found {
		return nil, false
	}

	process.Signal(os.Interrupt)
	return process, true
}

func (store *GardenStore) Stop(guid string) error {
	process, found := store.freeStepProcess(guid)
	if !found {
		return ErrContainerNotFound
	}

	<-process.Wait()

	return nil
}

func (store *GardenStore) Destroy(guid string) error {
	store.freeStepProcess(guid)

	err := store.gardenClient.Destroy(guid)
	if err != nil {
		return err
	}

	store.tracker.Deinitialize(guid)

	return nil
}

func (store *GardenStore) GetFiles(guid, sourcePath string) (io.ReadCloser, error) {
	container, err := store.gardenClient.Lookup(guid)
	if err != nil {
		return nil, ErrContainerNotFound
	}

	return container.StreamOut(sourcePath)
}

func (store *GardenStore) Ping() error {
	return store.gardenClient.Ping()
}

func (store *GardenStore) Run(container executor.Container, logger lager.Logger) {
	logger = logger.WithData(lager.Data{
		"container": container.Guid,
	})

	gardenContainer, err := store.gardenClient.Lookup(container.Guid)
	if err != nil {
		logger.Debug("container-deleted-just-before-running")
		return
	}

	logStreamer := log_streamer.New(
		container.Log.Guid,
		container.Log.SourceName,
		container.Log.Index,
	)

	seq := []steps.Step{}

	if container.Setup != nil {
		seq = append(seq, store.transformer.StepFor(
			logStreamer,
			container.Setup,
			gardenContainer,
			container.ExternalIP,
			container.Ports,
			logger,
		))
	}

	parallelSequence := []steps.Step{
		store.transformer.StepFor(
			logStreamer,
			container.Action,
			gardenContainer,
			container.ExternalIP,
			container.Ports,
			logger,
		),
	}

	hasStartedRunning := make(chan struct{}, 1)

	var monitorStep steps.Step

	if container.Monitor != nil {
		monitoredStep := store.transformer.StepFor(
			logStreamer,
			container.Monitor,
			gardenContainer,
			container.ExternalIP,
			container.Ports,
			logger,
		)

		monitorStep = steps.NewMonitor(
			monitoredStep,
			hasStartedRunning,
			logger.Session("monitor"),
			store.timeProvider,
			time.Duration(container.StartTimeout)*time.Second,
			store.healthyMonitoringInterval,
			store.unhealthyMonitoringInterval,
		)
		parallelSequence = append(parallelSequence, monitorStep)
	} else {
		// this container isn't monitored, so we mark it running right away
		hasStartedRunning <- struct{}{}
	}

	seq = append(seq, steps.NewCodependent(parallelSequence))

	step := steps.NewSerial(seq)

	store.runStepProcess(logger, step, monitorStep, hasStartedRunning, gardenContainer, container.Guid)
}

func (store *GardenStore) runStepProcess(
	logger lager.Logger,
	step steps.Step,
	monitorStep steps.Step,
	hasStartedRunning <-chan struct{},
	gardenContainer garden.Container,
	guid string,
) {
	process := ifrit.Invoke(ifrit.RunFunc(func(signals <-chan os.Signal, ready chan<- struct{}) error {
		seqComplete := make(chan error)

		close(ready)

		go func() {
			seqComplete <- step.Perform()
		}()

		result := executor.ContainerRunResult{}

	OUTER_LOOP:
		for {
			select {
			case <-signals:
				signals = nil
				if monitorStep != nil {
					monitorStep.Cancel()
				}
				step.Cancel()

			case <-hasStartedRunning:
				hasStartedRunning = nil
				err := store.transitionToRunning(gardenContainer)
				if err != nil {
					logger.Error("failed-to-transition-to-running", err)
					result.Failed = true
					result.FailureReason = err.Error()
					break OUTER_LOOP
				}

			case err := <-seqComplete:
				if err != nil {
					result.Failed = true
					result.FailureReason = err.Error()
				}
				break OUTER_LOOP
			}
		}

		err := store.transitionToComplete(gardenContainer, result)

		if err != nil {
			logger.Error("failed-to-transition-to-complete", err)
		}

		return nil
	}))

	store.processesL.Lock()
	store.runningProcesses[guid] = process
	store.processesL.Unlock()
}

func (store *GardenStore) transitionToRunning(gardenContainer garden.Container) error {
	err := gardenContainer.SetProperty(ContainerStateProperty, string(executor.StateRunning))
	if err != nil {
		return err
	}

	executorContainer, err := store.exchanger.Garden2Executor(gardenContainer)
	if err != nil {
		return err
	}

	store.eventEmitter.EmitEvent(executor.ContainerRunningEvent{
		Container: executorContainer,
	})

	return nil
}

func (store *GardenStore) transitionToComplete(gardenContainer garden.Container, result executor.ContainerRunResult) error {
	resultJson, err := json.Marshal(result)
	if err != nil {
		return err
	}

	err = gardenContainer.SetProperty(ContainerResultProperty, string(resultJson))
	if err != nil {
		return err
	}

	err = gardenContainer.SetProperty(ContainerStateProperty, string(executor.StateCompleted))
	if err != nil {
		return err
	}

	executorContainer, err := store.exchanger.Garden2Executor(gardenContainer)
	if err != nil {
		return err
	}

	store.eventEmitter.EmitEvent(executor.ContainerCompleteEvent{
		Container: executorContainer,
	})

	return nil
}

func (store *GardenStore) TrackContainers(interval time.Duration, logger lager.Logger) ifrit.Runner {
	logger = logger.Session("garden-store-track-containers")

	return ifrit.RunFunc(func(signals <-chan os.Signal, ready chan<- struct{}) error {
		ticker := store.timeProvider.NewTicker(interval)
		defer ticker.Stop()

		close(ready)

	dance:
		for {
			select {
			case <-signals:
				break dance

			case <-ticker.C():
				containers, err := store.List(nil)
				if err != nil {
					logger.Error("failed-to-list-containers", err)
					continue
				}

				store.tracker.SyncInitialized(containers)
			}
		}

		return nil
	})
}
<|endoftext|>


<commit_before>package directory

import (
	"context"
	"errors"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"runtime"

	"github.com/containers/image/v5/internal/putblobdigest"
	"github.com/containers/image/v5/types"
	"github.com/opencontainers/go-digest"
	perrors "github.com/pkg/errors"
	"github.com/sirupsen/logrus"
)

const version = "Directory Transport Version: 1.1\n"

// ErrNotContainerImageDir indicates that the directory doesn't match the expected contents of a directory created
// using the 'dir' transport
var ErrNotContainerImageDir = errors.New("not a containers image directory, don't want to overwrite important data")

type dirImageDestination struct {
	ref                     dirReference
	desiredLayerCompression types.LayerCompression
}

// newImageDestination returns an ImageDestination for writing to a directory.
func newImageDestination(sys *types.SystemContext, ref dirReference) (types.ImageDestination, error) {
	desiredLayerCompression := types.PreserveOriginal
	if sys != nil {
		if sys.DirForceCompress {
			desiredLayerCompression = types.Compress

			if sys.DirForceDecompress {
				return nil, fmt.Errorf("Cannot compress and decompress at the same time")
			}
		}
		if sys.DirForceDecompress {
			desiredLayerCompression = types.Decompress
		}
	}
	d := &dirImageDestination{ref: ref, desiredLayerCompression: desiredLayerCompression}

	// If directory exists check if it is empty
	// if not empty, check whether the contents match that of a container image directory and overwrite the contents
	// if the contents don't match throw an error
	dirExists, err := pathExists(d.ref.resolvedPath)
	if err != nil {
		return nil, perrors.Wrapf(err, "checking for path %q", d.ref.resolvedPath)
	}
	if dirExists {
		isEmpty, err := isDirEmpty(d.ref.resolvedPath)
		if err != nil {
			return nil, err
		}

		if !isEmpty {
			versionExists, err := pathExists(d.ref.versionPath())
			if err != nil {
				return nil, perrors.Wrapf(err, "checking if path exists %q", d.ref.versionPath())
			}
			if versionExists {
				contents, err := os.ReadFile(d.ref.versionPath())
				if err != nil {
					return nil, err
				}
				// check if contents of version file is what we expect it to be
				if string(contents) != version {
					return nil, ErrNotContainerImageDir
				}
			} else {
				return nil, ErrNotContainerImageDir
			}
			// delete directory contents so that only one image is in the directory at a time
			if err = removeDirContents(d.ref.resolvedPath); err != nil {
				return nil, perrors.Wrapf(err, "erasing contents in %q", d.ref.resolvedPath)
			}
			logrus.Debugf("overwriting existing container image directory %q", d.ref.resolvedPath)
		}
	} else {
		// create directory if it doesn't exist
		if err := os.MkdirAll(d.ref.resolvedPath, 0755); err != nil {
			return nil, perrors.Wrapf(err, "unable to create directory %q", d.ref.resolvedPath)
		}
	}
	// create version file
	err = os.WriteFile(d.ref.versionPath(), []byte(version), 0644)
	if err != nil {
		return nil, perrors.Wrapf(err, "creating version file %q", d.ref.versionPath())
	}
	return d, nil
}

// Reference returns the reference used to set up this destination.  Note that this should directly correspond to user's intent,
// e.g. it should use the public hostname instead of the result of resolving CNAMEs or following redirects.
func (d *dirImageDestination) Reference() types.ImageReference {
	return d.ref
}

// Close removes resources associated with an initialized ImageDestination, if any.
func (d *dirImageDestination) Close() error {
	return nil
}

func (d *dirImageDestination) SupportedManifestMIMETypes() []string {
	return nil
}

// SupportsSignatures returns an error (to be displayed to the user) if the destination certainly can't store signatures.
// Note: It is still possible for PutSignatures to fail if SupportsSignatures returns nil.
func (d *dirImageDestination) SupportsSignatures(ctx context.Context) error {
	return nil
}

func (d *dirImageDestination) DesiredLayerCompression() types.LayerCompression {
	return d.desiredLayerCompression
}

// AcceptsForeignLayerURLs returns false iff foreign layers in manifest should be actually
// uploaded to the image destination, true otherwise.
func (d *dirImageDestination) AcceptsForeignLayerURLs() bool {
	return false
}

// MustMatchRuntimeOS returns true iff the destination can store only images targeted for the current runtime architecture and OS. False otherwise.
func (d *dirImageDestination) MustMatchRuntimeOS() bool {
	return false
}

// IgnoresEmbeddedDockerReference returns true iff the destination does not care about Image.EmbeddedDockerReferenceConflicts(),
// and would prefer to receive an unmodified manifest instead of one modified for the destination.
// Does not make a difference if Reference().DockerReference() is nil.
func (d *dirImageDestination) IgnoresEmbeddedDockerReference() bool {
	return false // N/A, DockerReference() returns nil.
}

// HasThreadSafePutBlob indicates whether PutBlob can be executed concurrently.
func (d *dirImageDestination) HasThreadSafePutBlob() bool {
	return false
}

// PutBlob writes contents of stream and returns data representing the result (with all data filled in).
// inputInfo.Digest can be optionally provided if known; if provided, and stream is read to the end without error, the digest MUST match the stream contents.
// inputInfo.Size is the expected length of stream, if known.
// May update cache.
// WARNING: The contents of stream are being verified on the fly.  Until stream.Read() returns io.EOF, the contents of the data SHOULD NOT be available
// to any other readers for download using the supplied digest.
// If stream.Read() at any time, ESPECIALLY at end of input, returns an error, PutBlob MUST 1) fail, and 2) delete any data stored so far.
func (d *dirImageDestination) PutBlob(ctx context.Context, stream io.Reader, inputInfo types.BlobInfo, cache types.BlobInfoCache, isConfig bool) (types.BlobInfo, error) {
	blobFile, err := os.CreateTemp(d.ref.path, "dir-put-blob")
	if err != nil {
		return types.BlobInfo{}, err
	}
	succeeded := false
	explicitClosed := false
	defer func() {
		if !explicitClosed {
			blobFile.Close()
		}
		if !succeeded {
			os.Remove(blobFile.Name())
		}
	}()

	digester, stream := putblobdigest.DigestIfCanonicalUnknown(stream, inputInfo)
	// TODO: This can take quite some time, and should ideally be cancellable using ctx.Done().
	size, err := io.Copy(blobFile, stream)
	if err != nil {
		return types.BlobInfo{}, err
	}
	blobDigest := digester.Digest()
	if inputInfo.Size != -1 && size != inputInfo.Size {
		return types.BlobInfo{}, fmt.Errorf("Size mismatch when copying %s, expected %d, got %d", blobDigest, inputInfo.Size, size)
	}
	if err := blobFile.Sync(); err != nil {
		return types.BlobInfo{}, err
	}

	// On POSIX systems, blobFile was created with mode 0600, so we need to make it readable.
	// On Windows, the “permissions of newly created files” argument to syscall.Open is
	// ignored and the file is already readable; besides, blobFile.Chmod, i.e. syscall.Fchmod,
	// always fails on Windows.
	if runtime.GOOS != "windows" {
		if err := blobFile.Chmod(0644); err != nil {
			return types.BlobInfo{}, err
		}
	}

	blobPath := d.ref.layerPath(blobDigest)
	// need to explicitly close the file, since a rename won't otherwise not work on Windows
	blobFile.Close()
	explicitClosed = true
	if err := os.Rename(blobFile.Name(), blobPath); err != nil {
		return types.BlobInfo{}, err
	}
	succeeded = true
	return types.BlobInfo{Digest: blobDigest, Size: size}, nil
}

// TryReusingBlob checks whether the transport already contains, or can efficiently reuse, a blob, and if so, applies it to the current destination
// (e.g. if the blob is a filesystem layer, this signifies that the changes it describes need to be applied again when composing a filesystem tree).
// info.Digest must not be empty.
// If canSubstitute, TryReusingBlob can use an equivalent equivalent of the desired blob; in that case the returned info may not match the input.
// If the blob has been successfully reused, returns (true, info, nil); info must contain at least a digest and size, and may
// include CompressionOperation and CompressionAlgorithm fields to indicate that a change to the compression type should be
// reflected in the manifest that will be written.
// If the transport can not reuse the requested blob, TryReusingBlob returns (false, {}, nil); it returns a non-nil error only on an unexpected failure.
// May use and/or update cache.
func (d *dirImageDestination) TryReusingBlob(ctx context.Context, info types.BlobInfo, cache types.BlobInfoCache, canSubstitute bool) (bool, types.BlobInfo, error) {
	if info.Digest == "" {
		return false, types.BlobInfo{}, fmt.Errorf("Can not check for a blob with unknown digest")
	}
	blobPath := d.ref.layerPath(info.Digest)
	finfo, err := os.Stat(blobPath)
	if err != nil && os.IsNotExist(err) {
		return false, types.BlobInfo{}, nil
	}
	if err != nil {
		return false, types.BlobInfo{}, err
	}
	return true, types.BlobInfo{Digest: info.Digest, Size: finfo.Size()}, nil
}

// PutManifest writes manifest to the destination.
// If instanceDigest is not nil, it contains a digest of the specific manifest instance to write the manifest for (when
// the primary manifest is a manifest list); this should always be nil if the primary manifest is not a manifest list.
// It is expected but not enforced that the instanceDigest, when specified, matches the digest of `manifest` as generated
// by `manifest.Digest()`.
// FIXME? This should also receive a MIME type if known, to differentiate between schema versions.
// If the destination is in principle available, refuses this manifest type (e.g. it does not recognize the schema),
// but may accept a different manifest type, the returned error must be an ManifestTypeRejectedError.
func (d *dirImageDestination) PutManifest(ctx context.Context, manifest []byte, instanceDigest *digest.Digest) error {
	return os.WriteFile(d.ref.manifestPath(instanceDigest), manifest, 0644)
}

// PutSignatures writes a set of signatures to the destination.
// If instanceDigest is not nil, it contains a digest of the specific manifest instance to write or overwrite the signatures for
// (when the primary manifest is a manifest list); this should always be nil if the primary manifest is not a manifest list.
func (d *dirImageDestination) PutSignatures(ctx context.Context, signatures [][]byte, instanceDigest *digest.Digest) error {
	for i, sig := range signatures {
		if err := os.WriteFile(d.ref.signaturePath(i, instanceDigest), sig, 0644); err != nil {
			return err
		}
	}
	return nil
}

// Commit marks the process of storing the image as successful and asks for the image to be persisted.
// unparsedToplevel contains data about the top-level manifest of the source (which may be a single-arch image or a manifest list
// if PutManifest was only called for the single-arch image with instanceDigest == nil), primarily to allow lookups by the
// original manifest list digest, if desired.
// WARNING: This does not have any transactional semantics:
// - Uploaded data MAY be visible to others before Commit() is called
// - Uploaded data MAY be removed or MAY remain around if Close() is called without Commit() (i.e. rollback is allowed but not guaranteed)
func (d *dirImageDestination) Commit(context.Context, types.UnparsedImage) error {
	return nil
}

// returns true if path exists
func pathExists(path string) (bool, error) {
	_, err := os.Stat(path)
	if err == nil {
		return true, nil
	}
	if os.IsNotExist(err) {
		return false, nil
	}
	return false, err
}

// returns true if directory is empty
func isDirEmpty(path string) (bool, error) {
	files, err := os.ReadDir(path)
	if err != nil {
		return false, err
	}
	return len(files) == 0, nil
}

// deletes the contents of a directory
func removeDirContents(path string) error {
	files, err := os.ReadDir(path)
	if err != nil {
		return err
	}

	for _, file := range files {
		if err := os.RemoveAll(filepath.Join(path, file.Name())); err != nil {
			return err
		}
	}
	return nil
}
<commit_msg>Only create dirImageDestination when all checks are done<commit_after>package directory

import (
	"context"
	"errors"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"runtime"

	"github.com/containers/image/v5/internal/putblobdigest"
	"github.com/containers/image/v5/types"
	"github.com/opencontainers/go-digest"
	perrors "github.com/pkg/errors"
	"github.com/sirupsen/logrus"
)

const version = "Directory Transport Version: 1.1\n"

// ErrNotContainerImageDir indicates that the directory doesn't match the expected contents of a directory created
// using the 'dir' transport
var ErrNotContainerImageDir = errors.New("not a containers image directory, don't want to overwrite important data")

type dirImageDestination struct {
	ref                     dirReference
	desiredLayerCompression types.LayerCompression
}

// newImageDestination returns an ImageDestination for writing to a directory.
func newImageDestination(sys *types.SystemContext, ref dirReference) (types.ImageDestination, error) {
	desiredLayerCompression := types.PreserveOriginal
	if sys != nil {
		if sys.DirForceCompress {
			desiredLayerCompression = types.Compress

			if sys.DirForceDecompress {
				return nil, fmt.Errorf("Cannot compress and decompress at the same time")
			}
		}
		if sys.DirForceDecompress {
			desiredLayerCompression = types.Decompress
		}
	}

	// If directory exists check if it is empty
	// if not empty, check whether the contents match that of a container image directory and overwrite the contents
	// if the contents don't match throw an error
	dirExists, err := pathExists(ref.resolvedPath)
	if err != nil {
		return nil, perrors.Wrapf(err, "checking for path %q", ref.resolvedPath)
	}
	if dirExists {
		isEmpty, err := isDirEmpty(ref.resolvedPath)
		if err != nil {
			return nil, err
		}

		if !isEmpty {
			versionExists, err := pathExists(ref.versionPath())
			if err != nil {
				return nil, perrors.Wrapf(err, "checking if path exists %q", ref.versionPath())
			}
			if versionExists {
				contents, err := os.ReadFile(ref.versionPath())
				if err != nil {
					return nil, err
				}
				// check if contents of version file is what we expect it to be
				if string(contents) != version {
					return nil, ErrNotContainerImageDir
				}
			} else {
				return nil, ErrNotContainerImageDir
			}
			// delete directory contents so that only one image is in the directory at a time
			if err = removeDirContents(ref.resolvedPath); err != nil {
				return nil, perrors.Wrapf(err, "erasing contents in %q", ref.resolvedPath)
			}
			logrus.Debugf("overwriting existing container image directory %q", ref.resolvedPath)
		}
	} else {
		// create directory if it doesn't exist
		if err := os.MkdirAll(ref.resolvedPath, 0755); err != nil {
			return nil, perrors.Wrapf(err, "unable to create directory %q", ref.resolvedPath)
		}
	}
	// create version file
	err = os.WriteFile(ref.versionPath(), []byte(version), 0644)
	if err != nil {
		return nil, perrors.Wrapf(err, "creating version file %q", ref.versionPath())
	}

	d := &dirImageDestination{ref: ref, desiredLayerCompression: desiredLayerCompression}
	return d, nil
}

// Reference returns the reference used to set up this destination.  Note that this should directly correspond to user's intent,
// e.g. it should use the public hostname instead of the result of resolving CNAMEs or following redirects.
func (d *dirImageDestination) Reference() types.ImageReference {
	return d.ref
}

// Close removes resources associated with an initialized ImageDestination, if any.
func (d *dirImageDestination) Close() error {
	return nil
}

func (d *dirImageDestination) SupportedManifestMIMETypes() []string {
	return nil
}

// SupportsSignatures returns an error (to be displayed to the user) if the destination certainly can't store signatures.
// Note: It is still possible for PutSignatures to fail if SupportsSignatures returns nil.
func (d *dirImageDestination) SupportsSignatures(ctx context.Context) error {
	return nil
}

func (d *dirImageDestination) DesiredLayerCompression() types.LayerCompression {
	return d.desiredLayerCompression
}

// AcceptsForeignLayerURLs returns false iff foreign layers in manifest should be actually
// uploaded to the image destination, true otherwise.
func (d *dirImageDestination) AcceptsForeignLayerURLs() bool {
	return false
}

// MustMatchRuntimeOS returns true iff the destination can store only images targeted for the current runtime architecture and OS. False otherwise.
func (d *dirImageDestination) MustMatchRuntimeOS() bool {
	return false
}

// IgnoresEmbeddedDockerReference returns true iff the destination does not care about Image.EmbeddedDockerReferenceConflicts(),
// and would prefer to receive an unmodified manifest instead of one modified for the destination.
// Does not make a difference if Reference().DockerReference() is nil.
func (d *dirImageDestination) IgnoresEmbeddedDockerReference() bool {
	return false // N/A, DockerReference() returns nil.
}

// HasThreadSafePutBlob indicates whether PutBlob can be executed concurrently.
func (d *dirImageDestination) HasThreadSafePutBlob() bool {
	return false
}

// PutBlob writes contents of stream and returns data representing the result (with all data filled in).
// inputInfo.Digest can be optionally provided if known; if provided, and stream is read to the end without error, the digest MUST match the stream contents.
// inputInfo.Size is the expected length of stream, if known.
// May update cache.
// WARNING: The contents of stream are being verified on the fly.  Until stream.Read() returns io.EOF, the contents of the data SHOULD NOT be available
// to any other readers for download using the supplied digest.
// If stream.Read() at any time, ESPECIALLY at end of input, returns an error, PutBlob MUST 1) fail, and 2) delete any data stored so far.
func (d *dirImageDestination) PutBlob(ctx context.Context, stream io.Reader, inputInfo types.BlobInfo, cache types.BlobInfoCache, isConfig bool) (types.BlobInfo, error) {
	blobFile, err := os.CreateTemp(d.ref.path, "dir-put-blob")
	if err != nil {
		return types.BlobInfo{}, err
	}
	succeeded := false
	explicitClosed := false
	defer func() {
		if !explicitClosed {
			blobFile.Close()
		}
		if !succeeded {
			os.Remove(blobFile.Name())
		}
	}()

	digester, stream := putblobdigest.DigestIfCanonicalUnknown(stream, inputInfo)
	// TODO: This can take quite some time, and should ideally be cancellable using ctx.Done().
	size, err := io.Copy(blobFile, stream)
	if err != nil {
		return types.BlobInfo{}, err
	}
	blobDigest := digester.Digest()
	if inputInfo.Size != -1 && size != inputInfo.Size {
		return types.BlobInfo{}, fmt.Errorf("Size mismatch when copying %s, expected %d, got %d", blobDigest, inputInfo.Size, size)
	}
	if err := blobFile.Sync(); err != nil {
		return types.BlobInfo{}, err
	}

	// On POSIX systems, blobFile was created with mode 0600, so we need to make it readable.
	// On Windows, the “permissions of newly created files” argument to syscall.Open is
	// ignored and the file is already readable; besides, blobFile.Chmod, i.e. syscall.Fchmod,
	// always fails on Windows.
	if runtime.GOOS != "windows" {
		if err := blobFile.Chmod(0644); err != nil {
			return types.BlobInfo{}, err
		}
	}

	blobPath := d.ref.layerPath(blobDigest)
	// need to explicitly close the file, since a rename won't otherwise not work on Windows
	blobFile.Close()
	explicitClosed = true
	if err := os.Rename(blobFile.Name(), blobPath); err != nil {
		return types.BlobInfo{}, err
	}
	succeeded = true
	return types.BlobInfo{Digest: blobDigest, Size: size}, nil
}

// TryReusingBlob checks whether the transport already contains, or can efficiently reuse, a blob, and if so, applies it to the current destination
// (e.g. if the blob is a filesystem layer, this signifies that the changes it describes need to be applied again when composing a filesystem tree).
// info.Digest must not be empty.
// If canSubstitute, TryReusingBlob can use an equivalent equivalent of the desired blob; in that case the returned info may not match the input.
// If the blob has been successfully reused, returns (true, info, nil); info must contain at least a digest and size, and may
// include CompressionOperation and CompressionAlgorithm fields to indicate that a change to the compression type should be
// reflected in the manifest that will be written.
// If the transport can not reuse the requested blob, TryReusingBlob returns (false, {}, nil); it returns a non-nil error only on an unexpected failure.
// May use and/or update cache.
func (d *dirImageDestination) TryReusingBlob(ctx context.Context, info types.BlobInfo, cache types.BlobInfoCache, canSubstitute bool) (bool, types.BlobInfo, error) {
	if info.Digest == "" {
		return false, types.BlobInfo{}, fmt.Errorf("Can not check for a blob with unknown digest")
	}
	blobPath := d.ref.layerPath(info.Digest)
	finfo, err := os.Stat(blobPath)
	if err != nil && os.IsNotExist(err) {
		return false, types.BlobInfo{}, nil
	}
	if err != nil {
		return false, types.BlobInfo{}, err
	}
	return true, types.BlobInfo{Digest: info.Digest, Size: finfo.Size()}, nil
}

// PutManifest writes manifest to the destination.
// If instanceDigest is not nil, it contains a digest of the specific manifest instance to write the manifest for (when
// the primary manifest is a manifest list); this should always be nil if the primary manifest is not a manifest list.
// It is expected but not enforced that the instanceDigest, when specified, matches the digest of `manifest` as generated
// by `manifest.Digest()`.
// FIXME? This should also receive a MIME type if known, to differentiate between schema versions.
// If the destination is in principle available, refuses this manifest type (e.g. it does not recognize the schema),
// but may accept a different manifest type, the returned error must be an ManifestTypeRejectedError.
func (d *dirImageDestination) PutManifest(ctx context.Context, manifest []byte, instanceDigest *digest.Digest) error {
	return os.WriteFile(d.ref.manifestPath(instanceDigest), manifest, 0644)
}

// PutSignatures writes a set of signatures to the destination.
// If instanceDigest is not nil, it contains a digest of the specific manifest instance to write or overwrite the signatures for
// (when the primary manifest is a manifest list); this should always be nil if the primary manifest is not a manifest list.
func (d *dirImageDestination) PutSignatures(ctx context.Context, signatures [][]byte, instanceDigest *digest.Digest) error {
	for i, sig := range signatures {
		if err := os.WriteFile(d.ref.signaturePath(i, instanceDigest), sig, 0644); err != nil {
			return err
		}
	}
	return nil
}

// Commit marks the process of storing the image as successful and asks for the image to be persisted.
// unparsedToplevel contains data about the top-level manifest of the source (which may be a single-arch image or a manifest list
// if PutManifest was only called for the single-arch image with instanceDigest == nil), primarily to allow lookups by the
// original manifest list digest, if desired.
// WARNING: This does not have any transactional semantics:
// - Uploaded data MAY be visible to others before Commit() is called
// - Uploaded data MAY be removed or MAY remain around if Close() is called without Commit() (i.e. rollback is allowed but not guaranteed)
func (d *dirImageDestination) Commit(context.Context, types.UnparsedImage) error {
	return nil
}

// returns true if path exists
func pathExists(path string) (bool, error) {
	_, err := os.Stat(path)
	if err == nil {
		return true, nil
	}
	if os.IsNotExist(err) {
		return false, nil
	}
	return false, err
}

// returns true if directory is empty
func isDirEmpty(path string) (bool, error) {
	files, err := os.ReadDir(path)
	if err != nil {
		return false, err
	}
	return len(files) == 0, nil
}

// deletes the contents of a directory
func removeDirContents(path string) error {
	files, err := os.ReadDir(path)
	if err != nil {
		return err
	}

	for _, file := range files {
		if err := os.RemoveAll(filepath.Join(path, file.Name())); err != nil {
			return err
		}
	}
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2015 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package storage

import (
	"github.com/juju/errors"
	"github.com/juju/names"
	"gopkg.in/juju/charm.v4/hooks"
	"launchpad.net/tomb"

	apiwatcher "github.com/juju/juju/api/watcher"
	"github.com/juju/juju/apiserver/params"
	"github.com/juju/juju/state/watcher"
	"github.com/juju/juju/storage"
	"github.com/juju/juju/worker/uniter/hook"
	"github.com/juju/juju/worker/uniter/runner/jujuc"
)

// storageSource is a hook source that generates storage hooks for
// a single storage attachment.
type storageSource struct {
	tomb tomb.Tomb

	*storageHookQueue
	st      StorageAccessor
	watcher apiwatcher.NotifyWatcher
	changes chan hook.SourceChange
}

// storageHookQueue implements a subset of hook.Source, separated from
// storageSource for simpler testing.
type storageHookQueue struct {
	unitTag    names.UnitTag
	storageTag names.StorageTag

	// attached records whether or not the storage-attached
	// hook has been executed.
	attached bool

	// hookInfo is the next hook.Info to return, if non-nil.
	hookInfo *hook.Info

	// context contains the details of the storage attachment.
	context *contextStorage
}

// newStorageSource creates a hook source that watches for changes to,
// and generates storage hooks for, a single storage attachment.
func newStorageSource(
	st StorageAccessor,
	unitTag names.UnitTag,
	storageTag names.StorageTag,
	attached bool,
) (*storageSource, error) {
	w, err := st.WatchStorageAttachment(storageTag, unitTag)
	if err != nil {
		return nil, errors.Annotate(err, "watching storage attachment")
	}
	s := &storageSource{
		storageHookQueue: &storageHookQueue{
			unitTag:    unitTag,
			storageTag: storageTag,
			attached:   attached,
		},
		st:      st,
		watcher: w,
		changes: make(chan hook.SourceChange),
	}
	go func() {
		defer s.tomb.Done()
		defer watcher.Stop(w, &s.tomb)
		s.tomb.Kill(s.loop())
	}()
	return s, nil
}

func (s *storageSource) loop() error {
	defer close(s.changes)

	var inChanges <-chan struct{}
	var outChanges chan<- hook.SourceChange
	var outChange hook.SourceChange
	ready := make(chan struct{}, 1)
	ready <- struct{}{}
	for {
		select {
		case <-s.tomb.Dying():
			return tomb.ErrDying
		case <-ready:
			inChanges = s.watcher.Changes()
		case _, ok := <-inChanges:
			logger.Debugf("got storage attachment change")
			if !ok {
				return tomb.ErrDying
			}
			inChanges = nil
			outChanges = s.changes
			outChange = func() error {
				defer func() {
					ready <- struct{}{}
				}()
				logger.Debugf("processing storage source change")
				return s.update()
			}
		case outChanges <- outChange:
			logger.Debugf("sent storage source change")
			outChanges = nil
			outChange = nil
		}
	}
}

// Changes is part of the hook.Source interface.
func (s *storageSource) Changes() <-chan hook.SourceChange {
	return s.changes
}

// Stop is part of the hook.Source interface.
func (s *storageSource) Stop() error {
	s.tomb.Kill(nil)
	watcher.Stop(s.watcher, &s.tomb)
	return s.tomb.Wait()
}

// update is called when hook.SourceChanges are applied.
func (s *storageSource) update() error {
	attachment, err := s.st.StorageAttachment(s.storageTag, s.unitTag)
	if params.IsCodeNotFound(err) {
		// The storage attachment was removed from state, which
		// implies that the storage has been detached already.
		logger.Debugf("storage attachment %q not found", s.storageTag.Id())
		return nil
	} else if params.IsCodeNotProvisioned(err) {
		logger.Debugf("storage attachment %q not provisioned yet", s.storageTag.Id())
		return nil
	} else if err != nil {
		logger.Debugf("error refreshing storage details: %v", err)
		return errors.Annotate(err, "refreshing storage details")
	}
	return s.storageHookQueue.Update(attachment)
}

// Empty is part of the hook.Source interface.
func (s *storageHookQueue) Empty() bool {
	return s.hookInfo == nil
}

// Next is part of the hook.Source interface.
func (s *storageHookQueue) Next() hook.Info {
	if s.Empty() {
		panic("source is empty")
	}
	return *s.hookInfo
}

// Pop is part of the hook.Source interface.
func (s *storageHookQueue) Pop() {
	if s.Empty() {
		panic("source is empty")
	}
	if s.hookInfo.Kind == hooks.StorageAttached {
		s.attached = true
	}
	s.hookInfo = nil
}

// Update updates the hook queue with the freshly acquired information about
// the storage attachment.
func (s *storageHookQueue) Update(attachment params.StorageAttachment) error {
	switch attachment.Life {
	case params.Alive:
		if s.attached {
			// Storage attachments currently do not change
			// (apart from lifecycle) after being provisioned.
			// We don't process unprovisioned storage here,
			// so there's nothing to do.
			return nil
		}
	case params.Dying:
		if !s.attached {
			// Nothing to do: attachment is dying, but
			// the storage-attached hook has not been
			// consumed.
			s.hookInfo = nil
			return nil
		}
	case params.Dead:
		// Storage must been Dying to become Dead;
		// no further action is required.
		return nil
	}

	// Set the storage context when the first hook is generated
	// for this storager. Later, when we need to handle changing
	// storage, we'll need to have a cache in the runner like
	// we have for relations.
	if s.context == nil {
		s.context = &contextStorage{
			tag:      s.storageTag,
			kind:     storage.StorageKind(attachment.Kind),
			location: attachment.Location,
		}
	}

	if s.hookInfo == nil {
		s.hookInfo = &hook.Info{
			StorageId: s.storageTag.Id(),
		}
	}
	if attachment.Life == params.Alive {
		s.hookInfo.Kind = hooks.StorageAttached
	} else {
		// TODO(axw) this should be Detaching, not Detached.
		s.hookInfo.Kind = hooks.StorageDetached
	}
	logger.Debugf("queued hook: %v", s.hookInfo)
	return nil
}

func (s *storageHookQueue) Context() (jujuc.ContextStorage, bool) {
	if s.context != nil {
		return s.context, true
	}
	return nil, false
}
<commit_msg>worker/uniter/storage: add comment<commit_after>// Copyright 2015 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package storage

import (
	"github.com/juju/errors"
	"github.com/juju/names"
	"gopkg.in/juju/charm.v4/hooks"
	"launchpad.net/tomb"

	apiwatcher "github.com/juju/juju/api/watcher"
	"github.com/juju/juju/apiserver/params"
	"github.com/juju/juju/state/watcher"
	"github.com/juju/juju/storage"
	"github.com/juju/juju/worker/uniter/hook"
	"github.com/juju/juju/worker/uniter/runner/jujuc"
)

// storageSource is a hook source that generates storage hooks for
// a single storage attachment.
type storageSource struct {
	tomb tomb.Tomb

	*storageHookQueue
	st      StorageAccessor
	watcher apiwatcher.NotifyWatcher
	changes chan hook.SourceChange
}

// storageHookQueue implements a subset of hook.Source, separated from
// storageSource for simpler testing.
type storageHookQueue struct {
	unitTag    names.UnitTag
	storageTag names.StorageTag

	// attached records whether or not the storage-attached
	// hook has been executed.
	attached bool

	// hookInfo is the next hook.Info to return, if non-nil.
	hookInfo *hook.Info

	// context contains the details of the storage attachment.
	context *contextStorage
}

// newStorageSource creates a hook source that watches for changes to,
// and generates storage hooks for, a single storage attachment.
func newStorageSource(
	st StorageAccessor,
	unitTag names.UnitTag,
	storageTag names.StorageTag,
	attached bool,
) (*storageSource, error) {
	w, err := st.WatchStorageAttachment(storageTag, unitTag)
	if err != nil {
		return nil, errors.Annotate(err, "watching storage attachment")
	}
	s := &storageSource{
		storageHookQueue: &storageHookQueue{
			unitTag:    unitTag,
			storageTag: storageTag,
			attached:   attached,
		},
		st:      st,
		watcher: w,
		changes: make(chan hook.SourceChange),
	}
	go func() {
		defer s.tomb.Done()
		defer watcher.Stop(w, &s.tomb)
		s.tomb.Kill(s.loop())
	}()
	return s, nil
}

func (s *storageSource) loop() error {
	defer close(s.changes)

	var inChanges <-chan struct{}
	var outChanges chan<- hook.SourceChange
	var outChange hook.SourceChange
	ready := make(chan struct{}, 1)
	ready <- struct{}{}
	for {
		select {
		case <-s.tomb.Dying():
			return tomb.ErrDying
		case <-ready:
			inChanges = s.watcher.Changes()
		case _, ok := <-inChanges:
			logger.Debugf("got storage attachment change")
			if !ok {
				return tomb.ErrDying
			}
			inChanges = nil
			outChanges = s.changes
			outChange = func() error {
				defer func() {
					ready <- struct{}{}
				}()
				logger.Debugf("processing storage source change")
				return s.update()
			}
		case outChanges <- outChange:
			logger.Debugf("sent storage source change")
			outChanges = nil
			outChange = nil
		}
	}
}

// Changes is part of the hook.Source interface.
func (s *storageSource) Changes() <-chan hook.SourceChange {
	return s.changes
}

// Stop is part of the hook.Source interface.
func (s *storageSource) Stop() error {
	s.tomb.Kill(nil)
	watcher.Stop(s.watcher, &s.tomb)
	return s.tomb.Wait()
}

// update is called when hook.SourceChanges are applied.
func (s *storageSource) update() error {
	attachment, err := s.st.StorageAttachment(s.storageTag, s.unitTag)
	if params.IsCodeNotFound(err) {
		// The storage attachment was removed from state, which
		// implies that the storage has been detached already.
		logger.Debugf("storage attachment %q not found", s.storageTag.Id())
		return nil
	} else if params.IsCodeNotProvisioned(err) {
		logger.Debugf("storage attachment %q not provisioned yet", s.storageTag.Id())
		return nil
	} else if err != nil {
		logger.Debugf("error refreshing storage details: %v", err)
		return errors.Annotate(err, "refreshing storage details")
	}
	return s.storageHookQueue.Update(attachment)
}

// Empty is part of the hook.Source interface.
func (s *storageHookQueue) Empty() bool {
	return s.hookInfo == nil
}

// Next is part of the hook.Source interface.
func (s *storageHookQueue) Next() hook.Info {
	if s.Empty() {
		panic("source is empty")
	}
	return *s.hookInfo
}

// Pop is part of the hook.Source interface.
func (s *storageHookQueue) Pop() {
	if s.Empty() {
		panic("source is empty")
	}
	if s.hookInfo.Kind == hooks.StorageAttached {
		s.attached = true
	}
	s.hookInfo = nil
}

// Update updates the hook queue with the freshly acquired information about
// the storage attachment.
func (s *storageHookQueue) Update(attachment params.StorageAttachment) error {
	switch attachment.Life {
	case params.Alive:
		if s.attached {
			// Storage attachments currently do not change
			// (apart from lifecycle) after being provisioned.
			// We don't process unprovisioned storage here,
			// so there's nothing to do.
			return nil
		}
	case params.Dying:
		if !s.attached {
			// Nothing to do: attachment is dying, but
			// the storage-attached hook has not been
			// consumed.
			s.hookInfo = nil
			return nil
		}
	case params.Dead:
		// Storage must been Dying to become Dead;
		// no further action is required.
		return nil
	}

	// Set the storage context when the first hook is generated
	// for this storager. Later, when we need to handle changing
	// storage, we'll need to have a cache in the runner like
	// we have for relations.
	if s.context == nil {
		s.context = &contextStorage{
			tag:      s.storageTag,
			kind:     storage.StorageKind(attachment.Kind),
			location: attachment.Location,
		}
	}

	if s.hookInfo == nil {
		s.hookInfo = &hook.Info{
			StorageId: s.storageTag.Id(),
		}
	}
	if attachment.Life == params.Alive {
		s.hookInfo.Kind = hooks.StorageAttached
	} else {
		// TODO(axw) this should be Detaching, not Detached.
		s.hookInfo.Kind = hooks.StorageDetached
	}
	logger.Debugf("queued hook: %v", s.hookInfo)
	return nil
}

// Context returns the ContextStorage for the storage that this hook queue
// corresponds to, and whether there is any context available yet. There
// will be context beginning from when the first hook is queued.
func (s *storageHookQueue) Context() (jujuc.ContextStorage, bool) {
	if s.context != nil {
		return s.context, true
	}
	return nil, false
}
<|endoftext|>


<commit_before>// Copyright 2016 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package translate_test

import (
	"context"
	"fmt"

	"cloud.google.com/go/translate"
	"golang.org/x/text/language"
)

func Example_NewClient() {
	ctx := context.Background()
	client, err := translate.NewClient(ctx)
	if err != nil {
		// TODO: handle error.
	}
	// Use the client.

	// Close the client when finished.
	if err := client.Close(); err != nil {
		// TODO: handle error.
	}
}

func Example_Translate() {
	ctx := context.Background()
	client, err := translate.NewClient(ctx)
	if err != nil {
		// TODO: handle error.
	}
	translations, err := client.Translate(ctx,
		[]string{"Le singe est sur la branche"}, language.English,
		&translate.Options{
			Source: language.French,
			Format: translate.Text,
		})
	if err != nil {
		// TODO: handle error.
	}
	fmt.Println(translations[0].Text)
}

func Example_DetectLanguage() {
	ctx := context.Background()
	client, err := translate.NewClient(ctx)
	if err != nil {
		// TODO: handle error.
	}
	ds, err := client.DetectLanguage(ctx, []string{"Today is Monday"})
	if err != nil {
		// TODO: handle error.
	}
	fmt.Println(ds)
}

func Example_SupportedLanguages() {
	ctx := context.Background()
	client, err := translate.NewClient(ctx)
	if err != nil {
		// TODO: handle error.
	}
	langs, err := client.SupportedLanguages(ctx, language.English)
	if err != nil {
		// TODO: handle error.
	}
	fmt.Println(langs)
}
<commit_msg>fix(translate): properly name examples (#2892)<commit_after>// Copyright 2016 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package translate_test

import (
	"context"
	"fmt"

	"cloud.google.com/go/translate"
	"golang.org/x/text/language"
)

func ExampleNewClient() {
	ctx := context.Background()
	client, err := translate.NewClient(ctx)
	if err != nil {
		// TODO: handle error.
	}
	// Use the client.

	// Close the client when finished.
	if err := client.Close(); err != nil {
		// TODO: handle error.
	}
}

func ExampleClient_Translate() {
	ctx := context.Background()
	client, err := translate.NewClient(ctx)
	if err != nil {
		// TODO: handle error.
	}
	translations, err := client.Translate(ctx,
		[]string{"Le singe est sur la branche"}, language.English,
		&translate.Options{
			Source: language.French,
			Format: translate.Text,
		})
	if err != nil {
		// TODO: handle error.
	}
	fmt.Println(translations[0].Text)
}

func ExampleClient_DetectLanguage() {
	ctx := context.Background()
	client, err := translate.NewClient(ctx)
	if err != nil {
		// TODO: handle error.
	}
	ds, err := client.DetectLanguage(ctx, []string{"Today is Monday"})
	if err != nil {
		// TODO: handle error.
	}
	fmt.Println(ds)
}

func ExampleClient_SupportedLanguages() {
	ctx := context.Background()
	client, err := translate.NewClient(ctx)
	if err != nil {
		// TODO: handle error.
	}
	langs, err := client.SupportedLanguages(ctx, language.English)
	if err != nil {
		// TODO: handle error.
	}
	fmt.Println(langs)
}
<|endoftext|>


<commit_before>package design

import (
	. "github.com/goadesign/goa/design"
	. "github.com/goadesign/goa/design/apidsl"
)

var User = Type("User", func() {
	Attribute("id", String, "ID of logged user")
	Attribute("subscriber", Boolean, "Flag whether user is subscriber (has paid for access)")
	Attribute("browser_id", String, "Anonymized ID of user's browser")
	Attribute("url", String, "URL of the content/conversion point", func() {
		Format("uri")
	})
	Attribute("adblock", Boolean, "Flag whether user has adblock enabled")
	Attribute("window_height", Number, "Height of the users browser window")
	Attribute("window_width", Number, "Width of the users browser window")
	Attribute("cookies", Boolean, "Flag whether user has cookies enabled")
	Attribute("websockets", Boolean, "Flag whether user has websockets supported browser")
	Attribute("user_agent", String, "User agent of client")
	Attribute("ip_address", String, "IP address of client", func() {
		Format("ip")
	})
	Attribute("source", Source, "UTM and social source metadata")
	Attribute("remp_session_id", String, "ID of reader's session")
	Attribute("remp_pageview_id", String, "ID of pageview")
	Attribute("referer", String, "Value of HTTP referer header (if present)")
})

var System = Type("System", func() {
	Attribute("property_token", UUID, "Property token")
	Attribute("time", DateTime, "Time of occurrence")
	Required("property_token", "time")
})

var Pageview = Type("Pageview", func() {
	Description("Pageview is the payload for tracking pageview event")

	Attribute("action", String, func() {
		Enum("load", "timespent", "progress")
	})
	Attribute("system", System)
	Attribute("user", User)
	Attribute("article", Article)
	Attribute("timespent", TimeSpent)
	Attribute("progress", Progress)

	Required("system", "user", "action")
})

var Commerce = Type("Commerce", func() {
	Description("Commerce is the payload for tracking commerce event")

	Attribute("system", System)
	Attribute("user", User)
	Attribute("article", Article)
	Attribute("step", String, func() {
		Enum("checkout", "payment", "purchase", "refund")
	})
	Attribute("remp_commerce_id", String, "ID of event")
	Attribute("checkout", CommerceCheckout, "Used when user enters the checkout process (reviews the cart)")
	Attribute("payment", CommercePayment, "Used when user confirmed checkout and was redirected to the payment processor")
	Attribute("purchase", CommercePayment, "Used when payment processor confirms the payment")
	Attribute("refund", CommercePayment, "Used when refund is issued. Revenue should contain refunded amount of money.")

	Required("system", "user", "step")
})

var Event = Type("Event", func() {
	Description("TrackEvent is the payload for tracking generic event")

	Attribute("system", System)
	Attribute("user", User)

	Attribute("category", String, "Category of event (time, video, comment)")
	Attribute("action", String, "Specific action (read, pause, reply)")
	Attribute("value", Number, "Numeric value of event (read 60 seconds, paused after 200 seconds, 3rd comment")
	Attribute("tags", HashOf(String, String), "Custom filtering tags")
	Attribute("fields", HashOf(String, Any), "Additinal key-value data")
	Attribute("remp_event_id", String, "ID of event")

	Required("system", "category", "action")
})

var Article = Type("Article", func() {
	Description("Article is the payload for tracking article-related data")

	Attribute("id", String, "ID of article")
	Attribute("category", String, "Page category (homepage, world news...")
	Attribute("tags", ArrayOf(String), "List of tags (breaking news, trump...")
	Attribute("author_id", String, "ID of author")
	Attribute("locked", Boolean, "Flag whether content was locked for the visitor")
	Attribute("variants", HashOf(String, String), "Hash of key-value pairs bearing A/B test variant information (what's A/B-tested / variant label)")

	Required("id")
})

var TimeSpent = Type("TimeSpent", func() {
	Description("Update time spent on page")

	Attribute("seconds", Integer, "Number of seconds from previous update of time spent on page")
	Attribute("unload", Boolean, "Flag to indicate last update of time spent on page before unload event")

	Required("seconds")
})

var Progress = Type("Progress", func() {
	Description("Update reading progress on pageview")

	Attribute("page_ratio", Number, "Whole page reading ratio", func() {
		Minimum(0)
		Maximum(1)
	})
	Attribute("article_ratio", Number, "Article reading ratio", func() {
		Minimum(0)
		Maximum(1)
	})
	Attribute("unload", Boolean, "Flag to indicate last update of progress on page before unload event")

	Required("page_ratio")
})

var Source = Type("source", func() {
	Description("User-source related data")

	Attribute("utm_source", String, "Origin of user (e.g. remp_campaign)")
	Attribute("utm_medium", String, "Medium through which the came (e.g. overlay, inline)")
	Attribute("utm_campaign", String, "Reference to specific campaign (e.g. campaign ID")
	Attribute("utm_content", String, "Reference to specific campaign mean (e.g. banner ID)")
	Attribute("banner_variant", String, "Reference to specific banner variant (e.g. variant ID)")
})

var CommerceCheckout = Type("CommerceCheckout", func() {
	Attribute("funnel_id", String, "ID of funnel user is being routed trough")

	Required("funnel_id")
})

var CommercePayment = Type("CommercePayment", func() {
	Attribute("funnel_id", String, "ID of funnel user is being routed trough")
	Attribute("transaction_id", String, "Public ID of transaction (variable symbol)")
	Attribute("product_ids", ArrayOf(String), "Public IDs of selected products")
	Attribute("revenue", Revenue, "Amount of money for given payment")

	Required("revenue", "transaction_id", "product_ids")
})

var Revenue = Type("Revenue", func() {
	Attribute("amount", Number, "Numeric amount of money")
	Attribute("currency", String, "ISO 4217 representation of currency")

	Required("amount", "currency")
})

var Entity = Type("Entity", func() {
	Attribute("entity_def", func() {
		Attribute("id", String)
		Attribute("name", String)
		Attribute("data", HashOf(String, Any))

		Required("id", "name", "data")
	})
	Attribute("system", System)
	Required("entity_def", "system")
})
<commit_msg>Explicit_referer_medium user parameter<commit_after>package design

import (
	. "github.com/goadesign/goa/design"
	. "github.com/goadesign/goa/design/apidsl"
)

var User = Type("User", func() {
	Attribute("id", String, "ID of logged user")
	Attribute("subscriber", Boolean, "Flag whether user is subscriber (has paid for access)")
	Attribute("browser_id", String, "Anonymized ID of user's browser")
	Attribute("url", String, "URL of the content/conversion point", func() {
		Format("uri")
	})
	Attribute("adblock", Boolean, "Flag whether user has adblock enabled")
	Attribute("window_height", Number, "Height of the users browser window")
	Attribute("window_width", Number, "Width of the users browser window")
	Attribute("cookies", Boolean, "Flag whether user has cookies enabled")
	Attribute("websockets", Boolean, "Flag whether user has websockets supported browser")
	Attribute("user_agent", String, "User agent of client")
	Attribute("ip_address", String, "IP address of client", func() {
		Format("ip")
	})
	Attribute("source", Source, "UTM and social source metadata")
	Attribute("extras", Extras, "Optional user parameters")
	Attribute("remp_session_id", String, "ID of reader's session")
	Attribute("remp_pageview_id", String, "ID of pageview")
	Attribute("referer", String, "Value of HTTP referer header (if present)")
})

var System = Type("System", func() {
	Attribute("property_token", UUID, "Property token")
	Attribute("time", DateTime, "Time of occurrence")
	Required("property_token", "time")
})

var Pageview = Type("Pageview", func() {
	Description("Pageview is the payload for tracking pageview event")

	Attribute("action", String, func() {
		Enum("load", "timespent", "progress")
	})
	Attribute("system", System)
	Attribute("user", User)
	Attribute("article", Article)
	Attribute("timespent", TimeSpent)
	Attribute("progress", Progress)

	Required("system", "user", "action")
})

var Commerce = Type("Commerce", func() {
	Description("Commerce is the payload for tracking commerce event")

	Attribute("system", System)
	Attribute("user", User)
	Attribute("article", Article)
	Attribute("step", String, func() {
		Enum("checkout", "payment", "purchase", "refund")
	})
	Attribute("remp_commerce_id", String, "ID of event")
	Attribute("checkout", CommerceCheckout, "Used when user enters the checkout process (reviews the cart)")
	Attribute("payment", CommercePayment, "Used when user confirmed checkout and was redirected to the payment processor")
	Attribute("purchase", CommercePayment, "Used when payment processor confirms the payment")
	Attribute("refund", CommercePayment, "Used when refund is issued. Revenue should contain refunded amount of money.")

	Required("system", "user", "step")
})

var Event = Type("Event", func() {
	Description("TrackEvent is the payload for tracking generic event")

	Attribute("system", System)
	Attribute("user", User)

	Attribute("category", String, "Category of event (time, video, comment)")
	Attribute("action", String, "Specific action (read, pause, reply)")
	Attribute("value", Number, "Numeric value of event (read 60 seconds, paused after 200 seconds, 3rd comment")
	Attribute("tags", HashOf(String, String), "Custom filtering tags")
	Attribute("fields", HashOf(String, Any), "Additinal key-value data")
	Attribute("remp_event_id", String, "ID of event")

	Required("system", "category", "action")
})

var Article = Type("Article", func() {
	Description("Article is the payload for tracking article-related data")

	Attribute("id", String, "ID of article")
	Attribute("category", String, "Page category (homepage, world news...")
	Attribute("tags", ArrayOf(String), "List of tags (breaking news, trump...")
	Attribute("author_id", String, "ID of author")
	Attribute("locked", Boolean, "Flag whether content was locked for the visitor")
	Attribute("variants", HashOf(String, String), "Hash of key-value pairs bearing A/B test variant information (what's A/B-tested / variant label)")

	Required("id")
})

var TimeSpent = Type("TimeSpent", func() {
	Description("Update time spent on page")

	Attribute("seconds", Integer, "Number of seconds from previous update of time spent on page")
	Attribute("unload", Boolean, "Flag to indicate last update of time spent on page before unload event")

	Required("seconds")
})

var Progress = Type("Progress", func() {
	Description("Update reading progress on pageview")

	Attribute("page_ratio", Number, "Whole page reading ratio", func() {
		Minimum(0)
		Maximum(1)
	})
	Attribute("article_ratio", Number, "Article reading ratio", func() {
		Minimum(0)
		Maximum(1)
	})
	Attribute("unload", Boolean, "Flag to indicate last update of progress on page before unload event")

	Required("page_ratio")
})

var Source = Type("source", func() {
	Description("User-source related data")

	Attribute("utm_source", String, "Origin of user (e.g. remp_campaign)")
	Attribute("utm_medium", String, "Medium through which the came (e.g. overlay, inline)")
	Attribute("utm_campaign", String, "Reference to specific campaign (e.g. campaign ID")
	Attribute("utm_content", String, "Reference to specific campaign mean (e.g. banner ID)")
	Attribute("banner_variant", String, "Reference to specific banner variant (e.g. variant ID)")
})

var Extras = Type("extras", func() {
	Description("Custom user parameters defined by each site separately")

	Attribute("explicit_referer_medium", String, "Attribute for more specific tracking of referer medium")
})

var CommerceCheckout = Type("CommerceCheckout", func() {
	Attribute("funnel_id", String, "ID of funnel user is being routed trough")

	Required("funnel_id")
})

var CommercePayment = Type("CommercePayment", func() {
	Attribute("funnel_id", String, "ID of funnel user is being routed trough")
	Attribute("transaction_id", String, "Public ID of transaction (variable symbol)")
	Attribute("product_ids", ArrayOf(String), "Public IDs of selected products")
	Attribute("revenue", Revenue, "Amount of money for given payment")

	Required("revenue", "transaction_id", "product_ids")
})

var Revenue = Type("Revenue", func() {
	Attribute("amount", Number, "Numeric amount of money")
	Attribute("currency", String, "ISO 4217 representation of currency")

	Required("amount", "currency")
})

var Entity = Type("Entity", func() {
	Attribute("entity_def", func() {
		Attribute("id", String)
		Attribute("name", String)
		Attribute("data", HashOf(String, Any))

		Required("id", "name", "data")
	})
	Attribute("system", System)
	Required("entity_def", "system")
})
<|endoftext|>


<commit_before>package ibmmq

/*
  Copyright (c) IBM Corporation 2018

  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

  http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License.

   Contributors:
     Mark Taylor - Initial Contribution
*/

/*

#include <stdlib.h>
#include <string.h>
#include <cmqc.h>

*/
import "C"
import (
	"unsafe"
)

/*
MQSTS is a structure containing the MQ Status Information Record
*/
type MQSTS struct {
	CompCode           int32
	Reason             int32
	PutSuccessCount    int32
	PutWarningCount    int32
	PutFailureCount    int32
	ObjectType         int32
	ObjectName         string
	ObjectQMgrName     string
	ResolvedObjectName string
	ResolvedQMgrName   string
	ObjectString       string
	SubName            string
	OpenOptions        int32
	SubOptions         int32
}

/*
NewMQSTS fills in default values for the MQSTS structure
*/
func NewMQSTS() *MQSTS {

	sts := new(MQSTS)
	sts.CompCode = int32(C.MQCC_OK)
	sts.Reason = int32(C.MQRC_NONE)
	sts.PutSuccessCount = 0
	sts.PutWarningCount = 0
	sts.PutFailureCount = 0
	sts.ObjectType = int32(C.MQOT_Q)
	sts.ObjectName = ""
	sts.ObjectQMgrName = ""
	sts.ResolvedObjectName = ""
	sts.ResolvedQMgrName = ""
	sts.ObjectString = ""
	sts.SubName = ""
	sts.OpenOptions = 0
	sts.SubOptions = 0

	return sts
}

func copySTStoC(mqsts *C.MQSTS, gosts *MQSTS) {
	const vsbufsize = C.MQ_TOPIC_STR_LENGTH

	setMQIString((*C.char)(&mqsts.StrucId[0]), "STS ", 4)
	mqsts.Version = C.MQSTS_VERSION_2

	mqsts.PutSuccessCount = C.MQLONG(gosts.PutSuccessCount)
	mqsts.PutWarningCount = C.MQLONG(gosts.PutWarningCount)
	mqsts.PutFailureCount = C.MQLONG(gosts.PutFailureCount)
	mqsts.ObjectType = C.MQLONG(gosts.ObjectType)

	setMQIString((*C.char)(&mqsts.ObjectName[0]), gosts.ObjectName, C.MQ_OBJECT_NAME_LENGTH)
	setMQIString((*C.char)(&mqsts.ObjectQMgrName[0]), gosts.ObjectQMgrName, C.MQ_OBJECT_NAME_LENGTH)
	setMQIString((*C.char)(&mqsts.ResolvedObjectName[0]), gosts.ResolvedObjectName, C.MQ_OBJECT_NAME_LENGTH)
	setMQIString((*C.char)(&mqsts.ResolvedQMgrName[0]), gosts.ResolvedQMgrName, C.MQ_OBJECT_NAME_LENGTH)

	if gosts.ObjectString != "" {

		mqsts.ObjectString.VSLength = (C.MQLONG)(len(gosts.ObjectString))
		mqsts.ObjectString.VSCCSID = C.MQCCSI_APPL
		if mqsts.ObjectString.VSLength == 0 {
			mqsts.ObjectString.VSPtr = C.MQPTR(C.malloc(vsbufsize))
			mqsts.ObjectString.VSBufSize = vsbufsize
		} else {
			mqsts.ObjectString.VSPtr = (C.MQPTR)(C.CString(gosts.ObjectString))
		}
	}
	if gosts.SubName != "" {

		mqsts.SubName.VSLength = (C.MQLONG)(len(gosts.SubName))
		mqsts.SubName.VSCCSID = C.MQCCSI_APPL
		if mqsts.SubName.VSLength == 0 {
			mqsts.SubName.VSPtr = C.MQPTR(C.malloc(vsbufsize))
			mqsts.SubName.VSBufSize = vsbufsize
		} else {
			mqsts.SubName.VSPtr = (C.MQPTR)(C.CString(gosts.SubName))
		}
	}

	mqsts.OpenOptions = C.MQLONG(gosts.OpenOptions)
	mqsts.SubOptions = C.MQLONG(gosts.SubOptions)

	return
}

func copySTSfromC(mqsts *C.MQSTS, gosts *MQSTS) {

	gosts.CompCode = int32(mqsts.CompCode)
	gosts.Reason = int32(mqsts.Reason)
	gosts.PutSuccessCount = int32(mqsts.PutSuccessCount)
	gosts.PutWarningCount = int32(mqsts.PutWarningCount)
	gosts.PutFailureCount = int32(mqsts.PutFailureCount)
	gosts.ObjectType = int32(mqsts.ObjectType)

	gosts.ObjectName = trimStringN((*C.char)(&mqsts.ObjectName[0]), C.MQ_OBJECT_NAME_LENGTH)
	gosts.ObjectQMgrName = trimStringN((*C.char)(&mqsts.ObjectQMgrName[0]), C.MQ_OBJECT_NAME_LENGTH)
	gosts.ResolvedObjectName = trimStringN((*C.char)(&mqsts.ResolvedObjectName[0]), C.MQ_OBJECT_NAME_LENGTH)
	gosts.ResolvedQMgrName = trimStringN((*C.char)(&mqsts.ResolvedQMgrName[0]), C.MQ_OBJECT_NAME_LENGTH)

	if mqsts.Version >= C.MQSTS_VERSION_2 {
		gosts.ObjectString = trimStringN((*C.char)(mqsts.ObjectString.VSPtr), (C.int)(mqsts.ObjectString.VSLength))
		C.free(unsafe.Pointer(mqsts.ObjectString.VSPtr))

		gosts.SubName = trimStringN((*C.char)(mqsts.SubName.VSPtr), (C.int)(mqsts.SubName.VSLength))
		C.free(unsafe.Pointer(mqsts.SubName.VSPtr))

		gosts.OpenOptions = int32(mqsts.OpenOptions)
		gosts.SubOptions = int32(mqsts.SubOptions)
	}

	return
}
<commit_msg>Correct strucId for STS (#146)<commit_after>package ibmmq

/*
  Copyright (c) IBM Corporation 2018

  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

  http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License.

   Contributors:
     Mark Taylor - Initial Contribution
*/

/*

#include <stdlib.h>
#include <string.h>
#include <cmqc.h>

*/
import "C"
import (
	"unsafe"
)

/*
MQSTS is a structure containing the MQ Status Information Record
*/
type MQSTS struct {
	CompCode           int32
	Reason             int32
	PutSuccessCount    int32
	PutWarningCount    int32
	PutFailureCount    int32
	ObjectType         int32
	ObjectName         string
	ObjectQMgrName     string
	ResolvedObjectName string
	ResolvedQMgrName   string
	ObjectString       string
	SubName            string
	OpenOptions        int32
	SubOptions         int32
}

/*
NewMQSTS fills in default values for the MQSTS structure
*/
func NewMQSTS() *MQSTS {

	sts := new(MQSTS)
	sts.CompCode = int32(C.MQCC_OK)
	sts.Reason = int32(C.MQRC_NONE)
	sts.PutSuccessCount = 0
	sts.PutWarningCount = 0
	sts.PutFailureCount = 0
	sts.ObjectType = int32(C.MQOT_Q)
	sts.ObjectName = ""
	sts.ObjectQMgrName = ""
	sts.ResolvedObjectName = ""
	sts.ResolvedQMgrName = ""
	sts.ObjectString = ""
	sts.SubName = ""
	sts.OpenOptions = 0
	sts.SubOptions = 0

	return sts
}

func copySTStoC(mqsts *C.MQSTS, gosts *MQSTS) {
	const vsbufsize = C.MQ_TOPIC_STR_LENGTH

	setMQIString((*C.char)(&mqsts.StrucId[0]), "STAT", 4)
	mqsts.Version = C.MQSTS_VERSION_2

	mqsts.PutSuccessCount = C.MQLONG(gosts.PutSuccessCount)
	mqsts.PutWarningCount = C.MQLONG(gosts.PutWarningCount)
	mqsts.PutFailureCount = C.MQLONG(gosts.PutFailureCount)
	mqsts.ObjectType = C.MQLONG(gosts.ObjectType)

	setMQIString((*C.char)(&mqsts.ObjectName[0]), gosts.ObjectName, C.MQ_OBJECT_NAME_LENGTH)
	setMQIString((*C.char)(&mqsts.ObjectQMgrName[0]), gosts.ObjectQMgrName, C.MQ_OBJECT_NAME_LENGTH)
	setMQIString((*C.char)(&mqsts.ResolvedObjectName[0]), gosts.ResolvedObjectName, C.MQ_OBJECT_NAME_LENGTH)
	setMQIString((*C.char)(&mqsts.ResolvedQMgrName[0]), gosts.ResolvedQMgrName, C.MQ_OBJECT_NAME_LENGTH)

	if gosts.ObjectString != "" {

		mqsts.ObjectString.VSLength = (C.MQLONG)(len(gosts.ObjectString))
		mqsts.ObjectString.VSCCSID = C.MQCCSI_APPL
		if mqsts.ObjectString.VSLength == 0 {
			mqsts.ObjectString.VSPtr = C.MQPTR(C.malloc(vsbufsize))
			mqsts.ObjectString.VSBufSize = vsbufsize
		} else {
			mqsts.ObjectString.VSPtr = (C.MQPTR)(C.CString(gosts.ObjectString))
		}
	}
	if gosts.SubName != "" {

		mqsts.SubName.VSLength = (C.MQLONG)(len(gosts.SubName))
		mqsts.SubName.VSCCSID = C.MQCCSI_APPL
		if mqsts.SubName.VSLength == 0 {
			mqsts.SubName.VSPtr = C.MQPTR(C.malloc(vsbufsize))
			mqsts.SubName.VSBufSize = vsbufsize
		} else {
			mqsts.SubName.VSPtr = (C.MQPTR)(C.CString(gosts.SubName))
		}
	}

	mqsts.OpenOptions = C.MQLONG(gosts.OpenOptions)
	mqsts.SubOptions = C.MQLONG(gosts.SubOptions)

	return
}

func copySTSfromC(mqsts *C.MQSTS, gosts *MQSTS) {

	gosts.CompCode = int32(mqsts.CompCode)
	gosts.Reason = int32(mqsts.Reason)
	gosts.PutSuccessCount = int32(mqsts.PutSuccessCount)
	gosts.PutWarningCount = int32(mqsts.PutWarningCount)
	gosts.PutFailureCount = int32(mqsts.PutFailureCount)
	gosts.ObjectType = int32(mqsts.ObjectType)

	gosts.ObjectName = trimStringN((*C.char)(&mqsts.ObjectName[0]), C.MQ_OBJECT_NAME_LENGTH)
	gosts.ObjectQMgrName = trimStringN((*C.char)(&mqsts.ObjectQMgrName[0]), C.MQ_OBJECT_NAME_LENGTH)
	gosts.ResolvedObjectName = trimStringN((*C.char)(&mqsts.ResolvedObjectName[0]), C.MQ_OBJECT_NAME_LENGTH)
	gosts.ResolvedQMgrName = trimStringN((*C.char)(&mqsts.ResolvedQMgrName[0]), C.MQ_OBJECT_NAME_LENGTH)

	if mqsts.Version >= C.MQSTS_VERSION_2 {
		gosts.ObjectString = trimStringN((*C.char)(mqsts.ObjectString.VSPtr), (C.int)(mqsts.ObjectString.VSLength))
		C.free(unsafe.Pointer(mqsts.ObjectString.VSPtr))

		gosts.SubName = trimStringN((*C.char)(mqsts.SubName.VSPtr), (C.int)(mqsts.SubName.VSLength))
		C.free(unsafe.Pointer(mqsts.SubName.VSPtr))

		gosts.OpenOptions = int32(mqsts.OpenOptions)
		gosts.SubOptions = int32(mqsts.SubOptions)
	}

	return
}
<|endoftext|>