<commit_before>package selenium

import (
	"fmt"
	"github.com/viant/toolbox"
	"strings"
)

const (
	undefined int = iota
	eof
	illegal
	whitespaces
	id
	operator
	selector
	selectorBy
	params
	dot
	method
)

var matchers = map[int]toolbox.Matcher{
	eof:         toolbox.EOFMatcher{},
	whitespaces: toolbox.CharactersMatcher{" \n\t"},

	id: toolbox.NewCustomIdMatcher("_", "-"),
	operator: toolbox.KeywordsMatcher{
		Keywords:      []string{"="},
		CaseSensitive: false,
	},
	selector:   toolbox.BodyMatcher{"(", ")"},
	selectorBy: toolbox.NewSequenceMatcher(":"),
	params:     toolbox.BodyMatcher{"(", ")"},
	dot:        toolbox.CharactersMatcher{Chars: "."},
	method:     toolbox.LiteralMatcher{},
}

//parser represents selenium command action parser
type parser struct{}

func (p *parser) expectOptionalWhitespaceFollowedBy(tokenizer *toolbox.Tokenizer, expectedTokensMessage string, expected ...int) (*toolbox.Token, error) {
	var expectedTokens = make([]int, 0)
	expectedTokens = append(expectedTokens, whitespaces)
	expectedTokens = append(expectedTokens, expected...)

	token := tokenizer.Nexts(expectedTokens...)

	if token.Token == eof && !toolbox.HasSliceAnyElements(expectedTokens, eof) {
		return nil, newIllegalTokenParsingError(tokenizer.Index, token.Token, expectedTokensMessage)
	}

	if token.Token == illegal {
		return nil, newIllegalTokenParsingError(tokenizer.Index, token.Token, expectedTokensMessage)
	}
	if token.Token == whitespaces {
		token = tokenizer.Nexts(expected...)
	}
	if token.Token == illegal {
		return nil, newIllegalTokenParsingError(tokenizer.Index, token.Token, expectedTokensMessage)
	}
	if token.Token == eof && len(token.Matched) > 0 {
		return nil, newIllegalTokenParsingError(tokenizer.Index, token.Token, expectedTokensMessage)
	}
	return token, nil
}

//Parse parses supplied expression. It returns criteria or parsing error.
func (p *parser) Parse(command string) (*Action, error) {
	result := &Action{
		Calls: []*MethodCall{{}},
	}
	tokenizer := toolbox.NewTokenizer(command, illegal, eof, matchers)

	var call = result.Calls[0]
	var webSelector WebSelector
	var callParams = ""

	expectTokens := []int{selector, id}

outer:
	for {

		token, err := p.expectOptionalWhitespaceFollowedBy(tokenizer, "id/method or web element selector", expectTokens...)
		if err != nil {
			return nil, err
		}
		switch token.Token {

		case selector:
			webSelector = WebSelector(token.Matched[1 : len(token.Matched)-1])
			token, err = p.expectOptionalWhitespaceFollowedBy(tokenizer, "'.' method selector", dot)
			if token.Token != dot {
				return nil, err
			}
			token, err = p.expectOptionalWhitespaceFollowedBy(tokenizer, "'.' method", id)
			if token.Token != id {
				return nil, err
			}
			call.Method = token.Matched
			fallthrough
		case id:
			identity := token.Matched
			token, err = p.expectOptionalWhitespaceFollowedBy(tokenizer, "'operator/params", operator, params, eof)
			if err != nil {
				return nil, err
			}
			switch token.Token {
			case params:
				call.Method = identity
				if len(token.Matched) > 2 {
					callParams = string(token.Matched[1 : len(token.Matched)-1])
				}
				break outer
			case operator:
				if result.Key != "" {
					return nil, fmt.Errorf("invalid token operator, expected: params at %d", tokenizer.Index)
				}
				result.Key = identity
				continue
			case eof:
				call.Method = identity
				break outer
			default:

				return nil, fmt.Errorf("invalid token: '%v', %v", token.Token, err)
			}
		}
	}
	var params = strings.TrimSpace(callParams)
	if len(params) > 0 {
		call.Parameters = []interface{}{strings.Trim(callParams, " '\"")}
	}

	if len(webSelector) > 0 {
		result.Selector = &WebElementSelector{}
		result.Selector.By, result.Selector.Value = webSelector.ByAndValue()
		result.Selector.Key = result.Key
	}
	method := call.Method
	if len(method) > 1 {
		call.Method = strings.ToUpper(string(method[0])) + string(method[1:])
	}
	return result, nil
}

//NewParser creates a new criteria parser
func NewParser() *parser {
	return &parser{}
}

type illegalTokenParsingError struct {
	Index    int
	Expected string
	error    string
}

func (e illegalTokenParsingError) Error() string {
	return e.error
}

func newIllegalTokenParsingError(index, token int, expected string) error {
	return &illegalTokenParsingError{Index: index, Expected: expected, error: fmt.Sprintf("illegal token:%v at %v, expected %v", token, index, expected)}
}
<commit_msg>updated parser incompatibility<commit_after>package selenium

import (
	"fmt"
	"github.com/viant/toolbox"
	"strings"
)

const (
	undefined int = iota
	eof
	illegal
	whitespaces
	id
	operator
	selector
	selectorBy
	params
	dot
	method
)

var matchers = map[int]toolbox.Matcher{
	eof:         toolbox.EOFMatcher{},
	whitespaces: toolbox.CharactersMatcher{" \n\t"},

	id: toolbox.NewCustomIdMatcher("_", "-"),
	operator: toolbox.KeywordsMatcher{
		Keywords:      []string{"="},
		CaseSensitive: false,
	},
	selector:   &toolbox.BodyMatcher{"(", ")"},
	selectorBy: toolbox.NewSequenceMatcher(":"),
	params:     &toolbox.BodyMatcher{"(", ")"},
	dot:        toolbox.CharactersMatcher{Chars: "."},
	method:     toolbox.LiteralMatcher{},
}

//parser represents selenium command action parser
type parser struct{}

func (p *parser) expectOptionalWhitespaceFollowedBy(tokenizer *toolbox.Tokenizer, expectedTokensMessage string, expected ...int) (*toolbox.Token, error) {
	var expectedTokens = make([]int, 0)
	expectedTokens = append(expectedTokens, whitespaces)
	expectedTokens = append(expectedTokens, expected...)

	token := tokenizer.Nexts(expectedTokens...)

	if token.Token == eof && !toolbox.HasSliceAnyElements(expectedTokens, eof) {
		return nil, newIllegalTokenParsingError(tokenizer.Index, token.Token, expectedTokensMessage)
	}

	if token.Token == illegal {
		return nil, newIllegalTokenParsingError(tokenizer.Index, token.Token, expectedTokensMessage)
	}
	if token.Token == whitespaces {
		token = tokenizer.Nexts(expected...)
	}
	if token.Token == illegal {
		return nil, newIllegalTokenParsingError(tokenizer.Index, token.Token, expectedTokensMessage)
	}
	if token.Token == eof && len(token.Matched) > 0 {
		return nil, newIllegalTokenParsingError(tokenizer.Index, token.Token, expectedTokensMessage)
	}
	return token, nil
}

//Parse parses supplied expression. It returns criteria or parsing error.
func (p *parser) Parse(command string) (*Action, error) {
	result := &Action{
		Calls: []*MethodCall{{}},
	}
	tokenizer := toolbox.NewTokenizer(command, illegal, eof, matchers)

	var call = result.Calls[0]
	var webSelector WebSelector
	var callParams = ""

	expectTokens := []int{selector, id}

outer:
	for {

		token, err := p.expectOptionalWhitespaceFollowedBy(tokenizer, "id/method or web element selector", expectTokens...)
		if err != nil {
			return nil, err
		}
		switch token.Token {

		case selector:
			webSelector = WebSelector(token.Matched[1 : len(token.Matched)-1])
			token, err = p.expectOptionalWhitespaceFollowedBy(tokenizer, "'.' method selector", dot)
			if token.Token != dot {
				return nil, err
			}
			token, err = p.expectOptionalWhitespaceFollowedBy(tokenizer, "'.' method", id)
			if token.Token != id {
				return nil, err
			}
			call.Method = token.Matched
			fallthrough
		case id:
			identity := token.Matched
			token, err = p.expectOptionalWhitespaceFollowedBy(tokenizer, "'operator/params", operator, params, eof)
			if err != nil {
				return nil, err
			}
			switch token.Token {
			case params:
				call.Method = identity
				if len(token.Matched) > 2 {
					callParams = string(token.Matched[1 : len(token.Matched)-1])
				}
				break outer
			case operator:
				if result.Key != "" {
					return nil, fmt.Errorf("invalid token operator, expected: params at %d", tokenizer.Index)
				}
				result.Key = identity
				continue
			case eof:
				call.Method = identity
				break outer
			default:

				return nil, fmt.Errorf("invalid token: '%v', %v", token.Token, err)
			}
		}
	}
	var params = strings.TrimSpace(callParams)
	if len(params) > 0 {
		call.Parameters = []interface{}{strings.Trim(callParams, " '\"")}
	}

	if len(webSelector) > 0 {
		result.Selector = &WebElementSelector{}
		result.Selector.By, result.Selector.Value = webSelector.ByAndValue()
		result.Selector.Key = result.Key
	}
	method := call.Method
	if len(method) > 1 {
		call.Method = strings.ToUpper(string(method[0])) + string(method[1:])
	}
	return result, nil
}

//NewParser creates a new criteria parser
func NewParser() *parser {
	return &parser{}
}

type illegalTokenParsingError struct {
	Index    int
	Expected string
	error    string
}

func (e illegalTokenParsingError) Error() string {
	return e.error
}

func newIllegalTokenParsingError(index, token int, expected string) error {
	return &illegalTokenParsingError{Index: index, Expected: expected, error: fmt.Sprintf("illegal token:%v at %v, expected %v", token, index, expected)}
}
<|endoftext|>


<commit_before>package container_test

import (
	"io/ioutil"
	. "launchpad.net/gocheck"
	"launchpad.net/juju-core/container"
	"launchpad.net/juju-core/environs"
	"launchpad.net/juju-core/juju/testing"
	"launchpad.net/juju-core/state"
	coretesting "launchpad.net/juju-core/testing"
	"launchpad.net/juju-core/version"
	"os"
	"path/filepath"
	stdtesting "testing"
)

type suite struct {
	testing.JujuConnSuite
}

var _ = Suite(&suite{})

func TestPackage(t *stdtesting.T) {
	coretesting.ZkTestPackage(t)
}

var _ container.Container = (*container.Simple)(nil)

func (s *suite) TestDeploy(c *C) {
	// create a unit to deploy
	dummy := s.AddTestingCharm(c, "dummy")
	service, err := s.State.AddService("dummy", dummy)
	c.Assert(err, IsNil)
	unit, err := service.AddUnit()
	c.Assert(err, IsNil)

	// make sure there's a jujud "executable" in the tools directory
	// for the current version.
	tools := &state.Tools{
		URL:    "unused",
		Binary: version.MustParseBinary("3.2.1-foo-bar"),
	}
	dataDir := c.MkDir()
	toolsDir := environs.ToolsDir(dataDir, tools.Binary)
	err = os.MkdirAll(toolsDir, 0777)
	c.Assert(err, IsNil)
	exe := filepath.Join(toolsDir, "jujud")
	err = ioutil.WriteFile(exe, []byte("#!/bin/sh\n"), 0777)
	c.Assert(err, IsNil)

	initDir := c.MkDir()
	cont := container.Simple{
		DataDir: dataDir,
		InitDir: initDir,
	}

	info := &state.Info{
		Addrs: []string{"a", "b"},
	}

	err = cont.Deploy(cfg, unit, info, tools)
	c.Assert(err, ErrorMatches, `(.|\n)+Unknown job(.|\n)+`)

	upstartScript := filepath.Join(cont.InitDir, "juju-unit-dummy-0.conf")

	data, err := ioutil.ReadFile(upstartScript)
	c.Assert(err, IsNil)

	c.Assert(string(data), Matches, `(.|\n)+`+
		`.*/unit-dummy-0/jujud unit`+
		` --zookeeper-servers 'a,b'`+
		` --log-file /var/log/juju/unit-dummy-0\.log`+
		` --unit-name dummy/0\n`+
		`(.|\n)*`)

	// We can't check that the unit directory is created, because
	// it is removed when the call to Deploy fails, but
	// we can check that it is removed.
	unitDir := filepath.Join(cont.DataDir, "agents", "unit-dummy-0")
	err = os.MkdirAll(filepath.Join(unitDir, "foo"), 0777)
	c.Assert(err, IsNil)

	err = cont.Destroy(unit)
	c.Assert(err, IsNil)

	_, err = os.Stat(unitDir)
	c.Assert(os.IsNotExist(err), Equals, true, Commentf("%v", err))

	_, err = os.Stat(upstartScript)
	c.Assert(err, NotNil)
}
<commit_msg>container: fix test<commit_after>package container_test

import (
	"io/ioutil"
	. "launchpad.net/gocheck"
	"launchpad.net/juju-core/container"
	"launchpad.net/juju-core/environs"
	"launchpad.net/juju-core/juju/testing"
	"launchpad.net/juju-core/state"
	coretesting "launchpad.net/juju-core/testing"
	"launchpad.net/juju-core/version"
	"os"
	"path/filepath"
	stdtesting "testing"
)

type suite struct {
	testing.JujuConnSuite
}

var _ = Suite(&suite{})

func TestPackage(t *stdtesting.T) {
	coretesting.ZkTestPackage(t)
}

var _ container.Container = (*container.Simple)(nil)

func (s *suite) TestDeploy(c *C) {
	// create a unit to deploy
	dummy := s.AddTestingCharm(c, "dummy")
	service, err := s.State.AddService("dummy", dummy)
	c.Assert(err, IsNil)
	unit, err := service.AddUnit()
	c.Assert(err, IsNil)

	// make sure there's a jujud "executable" in the tools directory
	// for the current version.
	tools := &state.Tools{
		URL:    "unused",
		Binary: version.MustParseBinary("3.2.1-foo-bar"),
	}
	dataDir := c.MkDir()
	toolsDir := environs.ToolsDir(dataDir, tools.Binary)
	err = os.MkdirAll(toolsDir, 0777)
	c.Assert(err, IsNil)
	exe := filepath.Join(toolsDir, "jujud")
	err = ioutil.WriteFile(exe, []byte("#!/bin/sh\n"), 0777)
	c.Assert(err, IsNil)

	initDir := c.MkDir()
	cont := container.Simple{
		DataDir: dataDir,
		InitDir: initDir,
	}

	info := &state.Info{
		Addrs: []string{"a", "b"},
	}

	err = cont.Deploy(unit, info, tools)
	c.Assert(err, ErrorMatches, `(.|\n)+Unknown job(.|\n)+`)

	upstartScript := filepath.Join(cont.InitDir, "juju-unit-dummy-0.conf")

	data, err := ioutil.ReadFile(upstartScript)
	c.Assert(err, IsNil)

	c.Assert(string(data), Matches, `(.|\n)+`+
		`.*/unit-dummy-0/jujud unit`+
		` --zookeeper-servers 'a,b'`+
		` --log-file /var/log/juju/unit-dummy-0\.log`+
		` --unit-name dummy/0\n`+
		`(.|\n)*`)

	// We can't check that the unit directory is created, because
	// it is removed when the call to Deploy fails, but
	// we can check that it is removed.
	unitDir := filepath.Join(cont.DataDir, "agents", "unit-dummy-0")
	err = os.MkdirAll(filepath.Join(unitDir, "foo"), 0777)
	c.Assert(err, IsNil)

	err = cont.Destroy(unit)
	c.Assert(err, IsNil)

	_, err = os.Stat(unitDir)
	c.Assert(os.IsNotExist(err), Equals, true, Commentf("%v", err))

	_, err = os.Stat(upstartScript)
	c.Assert(err, NotNil)
}
<|endoftext|>


<commit_before>/*
 * Copyright 2017 Google Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package stackdriver

import (
	"fmt"
	"path"
	"strings"
	"sync"
	"time"

	"github.com/cloudfoundry-community/stackdriver-tools/src/stackdriver-nozzle/messages"
	"github.com/golang/protobuf/ptypes/timestamp"
	labelpb "google.golang.org/genproto/googleapis/api/label"
	metricpb "google.golang.org/genproto/googleapis/api/metric"
	monitoringpb "google.golang.org/genproto/googleapis/monitoring/v3"
)

type MetricAdapter interface {
	PostMetricEvents([]*messages.MetricEvent) error
}

type Heartbeater interface {
	Start()
	Increment(string)
	IncrementBy(string, uint)
	Stop()
}

type metricAdapter struct {
	projectID             string
	client                MetricClient
	descriptors           map[string]struct{}
	createDescriptorMutex *sync.Mutex
	heartbeater           Heartbeater
}

type postMetricErr struct {
	metricDescriptor []error
	postErr          error
	heartbeater      Heartbeater
}

func (pme *postMetricErr) BuildError() error {
	pme.processErrors()
	if pme == nil || pme.postErr == nil && len(pme.metricDescriptor) == 0 {
		return nil
	}

	return fmt.Errorf("PostMetricEvents, PostErr: %v, MetricDescriptor: %v", pme.postErr, pme.metricDescriptor)
}

func (pme *postMetricErr) processErrors() {
	if pme == nil || pme.postErr == nil && len(pme.metricDescriptor) == 0 {
		return
	}

	if pme.postErr != nil {
		pme.heartbeater.Increment("metrics.post.errors")

		// This is an expected error once there is more than a single nozzle writing to Stackdriver.
		// If one nozzle writes a metric occuring at time T2 and this one tries to write at T1 (where T2 later than T1)
		// we will receive this error.
		if strings.Contains(pme.postErr.Error(), `Points must be written in order`) {
			pme.heartbeater.Increment("metrics.post.errors.out_of_order")
			pme.postErr = nil
		} else {
			pme.heartbeater.Increment("metrics.post.errors.unknown")
		}
	}

	metricDescriptorErrs := len(pme.metricDescriptor)
	if metricDescriptorErrs > 0 {
		pme.heartbeater.IncrementBy("metrics.metric_descriptor.errors", uint(metricDescriptorErrs))
	}
}

// NewMetricAdapter returns a MetricAdapater that can write to Stackdriver Monitoring
func NewMetricAdapter(projectID string, client MetricClient, heartbeater Heartbeater) (MetricAdapter, error) {
	ma := &metricAdapter{
		projectID:             projectID,
		client:                client,
		createDescriptorMutex: &sync.Mutex{},
		descriptors:           map[string]struct{}{},
		heartbeater:           heartbeater,
	}

	err := ma.fetchMetricDescriptorNames()
	return ma, err
}

func (ma *metricAdapter) PostMetricEvents(events []*messages.MetricEvent) error {
	var timeSerieses []*monitoringpb.TimeSeries

	compositeErr := postMetricErr{heartbeater: ma.heartbeater}

	for _, event := range events {
		if len(event.Metrics) == 0 {
			continue
		}

		ma.heartbeater.Increment("metrics.events.count")
		for _, metric := range event.Metrics {
			ma.heartbeater.Increment("metrics.timeseries.count")
			err := ma.ensureMetricDescriptor(metric, event.Labels)
			if err != nil {
				compositeErr.metricDescriptor = append(compositeErr.metricDescriptor, err)
				continue
			}

			metricType := path.Join("custom.googleapis.com", metric.Name)
			timeSeries := monitoringpb.TimeSeries{
				Metric: &metricpb.Metric{
					Type:   metricType,
					Labels: event.Labels,
				},
				Points: points(metric.Value, metric.EventTime),
			}
			timeSerieses = append(timeSerieses, &timeSeries)
		}
	}

	projectName := path.Join("projects", ma.projectID)
	request := &monitoringpb.CreateTimeSeriesRequest{
		Name:       projectName,
		TimeSeries: timeSerieses,
	}

	ma.heartbeater.Increment("metrics.requests")
	compositeErr.postErr = ma.client.Post(request)

	return compositeErr.BuildError()
}

func (ma *metricAdapter) CreateMetricDescriptor(metric *messages.Metric, labels map[string]string) error {
	projectName := path.Join("projects", ma.projectID)
	metricType := path.Join("custom.googleapis.com", metric.Name)
	metricName := path.Join(projectName, "metricDescriptors", metricType)

	var labelDescriptors []*labelpb.LabelDescriptor
	for key := range labels {
		labelDescriptors = append(labelDescriptors, &labelpb.LabelDescriptor{
			Key:       key,
			ValueType: labelpb.LabelDescriptor_STRING,
		})
	}

	req := &monitoringpb.CreateMetricDescriptorRequest{
		Name: projectName,
		MetricDescriptor: &metricpb.MetricDescriptor{
			Name:        metricName,
			Type:        metricType,
			Labels:      labelDescriptors,
			MetricKind:  metricpb.MetricDescriptor_GAUGE,
			ValueType:   metricpb.MetricDescriptor_DOUBLE,
			Unit:        metric.Unit,
			Description: "stackdriver-nozzle created custom metric.",
			DisplayName: metric.Name, // TODO
		},
	}

	return ma.client.CreateMetricDescriptor(req)
}

func (ma *metricAdapter) fetchMetricDescriptorNames() error {
	req := &monitoringpb.ListMetricDescriptorsRequest{
		Name:   fmt.Sprintf("projects/%s", ma.projectID),
		Filter: "metric.type = starts_with(\"custom.googleapis.com/\")",
	}

	descriptors, err := ma.client.ListMetricDescriptors(req)
	if err != nil {
		return err
	}

	for _, descriptor := range descriptors {
		ma.descriptors[descriptor.Name] = struct{}{}
	}
	return nil
}

func (ma *metricAdapter) ensureMetricDescriptor(metric *messages.Metric, labels map[string]string) error {
	if metric.Unit == "" {
		return nil
	}

	ma.createDescriptorMutex.Lock()
	defer ma.createDescriptorMutex.Unlock()

	if _, ok := ma.descriptors[metric.Name]; ok {
		return nil
	}

	err := ma.CreateMetricDescriptor(metric, labels)
	if err != nil {
		return err
	}
	ma.descriptors[metric.Name] = struct{}{}
	return nil
}

func points(value float64, eventTime time.Time) []*monitoringpb.Point {
	timeStamp := timestamp.Timestamp{
		Seconds: eventTime.Unix(),
		Nanos:   int32(eventTime.Nanosecond()),
	}
	point := &monitoringpb.Point{
		Interval: &monitoringpb.TimeInterval{
			EndTime:   &timeStamp,
			StartTime: &timeStamp,
		},
		Value: &monitoringpb.TypedValue{
			Value: &monitoringpb.TypedValue_DoubleValue{
				DoubleValue: value,
			},
		},
	}
	return []*monitoringpb.Point{point}
}
<commit_msg>stackdriver/metric_adapter: refactor time series<commit_after>/*
 * Copyright 2017 Google Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package stackdriver

import (
	"fmt"
	"path"
	"strings"
	"sync"
	"time"

	"github.com/cloudfoundry-community/stackdriver-tools/src/stackdriver-nozzle/messages"
	"github.com/golang/protobuf/ptypes/timestamp"
	labelpb "google.golang.org/genproto/googleapis/api/label"
	metricpb "google.golang.org/genproto/googleapis/api/metric"
	monitoringpb "google.golang.org/genproto/googleapis/monitoring/v3"
)

type MetricAdapter interface {
	PostMetricEvents([]*messages.MetricEvent) error
}

type Heartbeater interface {
	Start()
	Increment(string)
	IncrementBy(string, uint)
	Stop()
}

type metricAdapter struct {
	projectID             string
	client                MetricClient
	descriptors           map[string]struct{}
	createDescriptorMutex *sync.Mutex
	heartbeater           Heartbeater
}

type postMetricErr struct {
	metricDescriptor []error
	postErr          error
	heartbeater      Heartbeater
}

func (pme *postMetricErr) BuildError() error {
	pme.processErrors()
	if pme == nil || pme.postErr == nil && len(pme.metricDescriptor) == 0 {
		return nil
	}

	return fmt.Errorf("PostMetricEvents, PostErr: %v, MetricDescriptor: %v", pme.postErr, pme.metricDescriptor)
}

func (pme *postMetricErr) processErrors() {
	if pme == nil || pme.postErr == nil && len(pme.metricDescriptor) == 0 {
		return
	}

	if pme.postErr != nil {
		pme.heartbeater.Increment("metrics.post.errors")

		// This is an expected error once there is more than a single nozzle writing to Stackdriver.
		// If one nozzle writes a metric occuring at time T2 and this one tries to write at T1 (where T2 later than T1)
		// we will receive this error.
		if strings.Contains(pme.postErr.Error(), `Points must be written in order`) {
			pme.heartbeater.Increment("metrics.post.errors.out_of_order")
			pme.postErr = nil
		} else {
			pme.heartbeater.Increment("metrics.post.errors.unknown")
		}
	}

	metricDescriptorErrs := len(pme.metricDescriptor)
	if metricDescriptorErrs > 0 {
		pme.heartbeater.IncrementBy("metrics.metric_descriptor.errors", uint(metricDescriptorErrs))
	}
}

// NewMetricAdapter returns a MetricAdapater that can write to Stackdriver Monitoring
func NewMetricAdapter(projectID string, client MetricClient, heartbeater Heartbeater) (MetricAdapter, error) {
	ma := &metricAdapter{
		projectID:             projectID,
		client:                client,
		createDescriptorMutex: &sync.Mutex{},
		descriptors:           map[string]struct{}{},
		heartbeater:           heartbeater,
	}

	err := ma.fetchMetricDescriptorNames()
	return ma, err
}

func (ma *metricAdapter) PostMetricEvents(events []*messages.MetricEvent) error {
	series, compositeErr := ma.buildTimeSeries(events)

	projectName := path.Join("projects", ma.projectID)
	request := &monitoringpb.CreateTimeSeriesRequest{
		Name:       projectName,
		TimeSeries: series,
	}

	ma.heartbeater.Increment("metrics.requests")
	compositeErr.postErr = ma.client.Post(request)

	return compositeErr.BuildError()
}

func (ma *metricAdapter) buildTimeSeries(events []*messages.MetricEvent) ([]*monitoringpb.TimeSeries, postMetricErr) {
	var timeSerieses []*monitoringpb.TimeSeries

	compositeErr := postMetricErr{heartbeater: ma.heartbeater}

	for _, event := range events {
		if len(event.Metrics) == 0 {
			continue
		}

		ma.heartbeater.Increment("metrics.events.count")
		for _, metric := range event.Metrics {
			ma.heartbeater.Increment("metrics.timeseries.count")
			err := ma.ensureMetricDescriptor(metric, event.Labels)
			if err != nil {
				compositeErr.metricDescriptor = append(compositeErr.metricDescriptor, err)
				continue
			}

			metricType := path.Join("custom.googleapis.com", metric.Name)
			timeSeries := monitoringpb.TimeSeries{
				Metric: &metricpb.Metric{
					Type:   metricType,
					Labels: event.Labels,
				},
				Points: points(metric.Value, metric.EventTime),
			}
			timeSerieses = append(timeSerieses, &timeSeries)
		}
	}

	return timeSerieses, compositeErr
}

func (ma *metricAdapter) CreateMetricDescriptor(metric *messages.Metric, labels map[string]string) error {
	projectName := path.Join("projects", ma.projectID)
	metricType := path.Join("custom.googleapis.com", metric.Name)
	metricName := path.Join(projectName, "metricDescriptors", metricType)

	var labelDescriptors []*labelpb.LabelDescriptor
	for key := range labels {
		labelDescriptors = append(labelDescriptors, &labelpb.LabelDescriptor{
			Key:       key,
			ValueType: labelpb.LabelDescriptor_STRING,
		})
	}

	req := &monitoringpb.CreateMetricDescriptorRequest{
		Name: projectName,
		MetricDescriptor: &metricpb.MetricDescriptor{
			Name:        metricName,
			Type:        metricType,
			Labels:      labelDescriptors,
			MetricKind:  metricpb.MetricDescriptor_GAUGE,
			ValueType:   metricpb.MetricDescriptor_DOUBLE,
			Unit:        metric.Unit,
			Description: "stackdriver-nozzle created custom metric.",
			DisplayName: metric.Name, // TODO
		},
	}

	return ma.client.CreateMetricDescriptor(req)
}

func (ma *metricAdapter) fetchMetricDescriptorNames() error {
	req := &monitoringpb.ListMetricDescriptorsRequest{
		Name:   fmt.Sprintf("projects/%s", ma.projectID),
		Filter: "metric.type = starts_with(\"custom.googleapis.com/\")",
	}

	descriptors, err := ma.client.ListMetricDescriptors(req)
	if err != nil {
		return err
	}

	for _, descriptor := range descriptors {
		ma.descriptors[descriptor.Name] = struct{}{}
	}
	return nil
}

func (ma *metricAdapter) ensureMetricDescriptor(metric *messages.Metric, labels map[string]string) error {
	if metric.Unit == "" {
		return nil
	}

	ma.createDescriptorMutex.Lock()
	defer ma.createDescriptorMutex.Unlock()

	if _, ok := ma.descriptors[metric.Name]; ok {
		return nil
	}

	err := ma.CreateMetricDescriptor(metric, labels)
	if err != nil {
		return err
	}
	ma.descriptors[metric.Name] = struct{}{}
	return nil
}

func points(value float64, eventTime time.Time) []*monitoringpb.Point {
	timeStamp := timestamp.Timestamp{
		Seconds: eventTime.Unix(),
		Nanos:   int32(eventTime.Nanosecond()),
	}
	point := &monitoringpb.Point{
		Interval: &monitoringpb.TimeInterval{
			EndTime:   &timeStamp,
			StartTime: &timeStamp,
		},
		Value: &monitoringpb.TypedValue{
			Value: &monitoringpb.TypedValue_DoubleValue{
				DoubleValue: value,
			},
		},
	}
	return []*monitoringpb.Point{point}
}
<|endoftext|>


<commit_before>package case_0_static

import (
	"errors"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"sort"
	"time"

	entryPkg "github.com/ghthor/journal/entry"
	"github.com/ghthor/journal/git"

	"code.google.com/p/go.tools/godoc/vfs/mapfs"
)

type entriesByDate []string

func (f entriesByDate) Len() int { return len(f) }
func (f entriesByDate) Less(i, j int) bool {
	iTime, err := time.Parse(entryPkg.FilenameLayout, f[i])
	if err != nil {
		panic(err)
	}

	jTime, err := time.Parse(entryPkg.FilenameLayout, f[j])
	if err != nil {
		panic(err)
	}

	return jTime.After(iTime)
}
func (f entriesByDate) Swap(i, j int) { f[i], f[j] = f[j], f[i] }

// This algorithm will only mkdir's of 1 depth
func mkDirsIn(directory string, paths []string) (dirMap map[string][]string, err error) {

	// collect directories in a map as the keys
	dirMap = make(map[string][]string, len(paths))

	for _, path := range paths {
		d := filepath.Dir(path)

		dirMap[d] = append(dirMap[d], filepath.Base(path))
	}

	// make the directories
	for d, _ := range dirMap {
		if d == "." {
			continue
		}

		err := os.Mkdir(filepath.Join(directory, d), 0777)
		if err != nil {
			return nil, err
		}
	}

	return dirMap, nil
}

func NewIn(directory string) (case_0_directory string, entries []string, err error) {
	// Build a slice of the file names from the static mapfs Files
	paths := make([]string, 0, len(Files))
	for path, _ := range Files {
		paths = append(paths, path)
	}

	// Build the directory tree from the files list
	dirMap, err := mkDirsIn(directory, paths)
	if err != nil {
		return "", nil, err
	}

	// Create and copy all the files into the directory
	vfs := mapfs.New(Files)

	for file, _ := range Files {
		fi, err := vfs.Stat(file)
		if err != nil {
			return "", nil, errors.New(fmt.Sprintf("error stating source file: %s %s", file, err))
		}

		if fi.IsDir() {
			continue
		}

		source, err := vfs.Open(file)
		if err != nil {
			return "", nil, errors.New(fmt.Sprintf("error opening source file: %s %s", file, err))
		}
		defer source.Close()

		dest, err := os.OpenFile(filepath.Join(directory, file), os.O_CREATE|os.O_WRONLY, 0666)
		if err != nil {
			return "", nil, errors.New(fmt.Sprintf("error opening dest file: %s %s", file, err))
		}
		defer dest.Close()

		_, err = io.Copy(dest, source)
		if err != nil {
			return "", nil, err
		}
	}

	// Initialize a git repository in the case_0/ directory and commit all the entries
	case_0_directory = filepath.Join(directory, "case_0")

	err = git.Init(case_0_directory)
	if err != nil {
		return "", nil, err
	}

	entries = dirMap["case_0"]
	sort.Sort(entriesByDate(entries))

	// Commit the entries into git
	for i, entryFilename := range entries {
		changes := git.NewChangesIn(case_0_directory)
		changes.Add(git.ChangedFile(entryFilename))
		changes.Msg = fmt.Sprintf("Commit Msg | Entry %d\n", i+1)
		err = changes.Commit()
		if err != nil {
			return "", nil, err
		}
	}

	return case_0_directory, entries, nil
}
<commit_msg>Added godoc comment<commit_after>package case_0_static

import (
	"errors"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"sort"
	"time"

	entryPkg "github.com/ghthor/journal/entry"
	"github.com/ghthor/journal/git"

	"code.google.com/p/go.tools/godoc/vfs/mapfs"
)

type entriesByDate []string

func (f entriesByDate) Len() int { return len(f) }
func (f entriesByDate) Less(i, j int) bool {
	iTime, err := time.Parse(entryPkg.FilenameLayout, f[i])
	if err != nil {
		panic(err)
	}

	jTime, err := time.Parse(entryPkg.FilenameLayout, f[j])
	if err != nil {
		panic(err)
	}

	return jTime.After(iTime)
}
func (f entriesByDate) Swap(i, j int) { f[i], f[j] = f[j], f[i] }

// This algorithm will only mkdir's of 1 depth
func mkDirsIn(directory string, paths []string) (dirMap map[string][]string, err error) {

	// collect directories in a map as the keys
	dirMap = make(map[string][]string, len(paths))

	for _, path := range paths {
		d := filepath.Dir(path)

		dirMap[d] = append(dirMap[d], filepath.Base(path))
	}

	// make the directories
	for d, _ := range dirMap {
		if d == "." {
			continue
		}

		err := os.Mkdir(filepath.Join(directory, d), 0777)
		if err != nil {
			return nil, err
		}
	}

	return dirMap, nil
}

// Copies all the static resources into directory
// with all the entries commited to the repository.
// The directory will also contain a reflog of
// the commits that fix will create to
// update the journal to the current format.
//
// The directory tree will be
/*
	directory/
	├── case_0
	│   ├── 2014-01-01-0000-EST
	│   ├── 2014-01-02-0000-EST
	│   ├── 2014-01-03-0000-EST
	│   ├── 2014-01-04-0000-EST
	│   ├── 2014-01-05-0000-EST
	│   ├── 2014-01-06-0000-EST
	│   └── 2014-01-07-0000-EST
	├── case_0.json
	└── case_0_fix_reflog
	    ├── 0
	    ├── 1
	    ├── 10
	    ├── 11
	    ├── 12
	    ├── 13
	    ├── 14
	    ├── 2
	    ├── 3
	    ├── 4
	    ├── 5
	    ├── 6
	    ├── 7
	    ├── 8
	    └── 9
*/
func NewIn(directory string) (case_0_directory string, entries []string, err error) {
	// Build a slice of the file names from the static mapfs Files
	paths := make([]string, 0, len(Files))
	for path, _ := range Files {
		paths = append(paths, path)
	}

	// Build the directory tree from the files list
	dirMap, err := mkDirsIn(directory, paths)
	if err != nil {
		return "", nil, err
	}

	// Create and copy all the files into the directory
	vfs := mapfs.New(Files)

	for file, _ := range Files {
		fi, err := vfs.Stat(file)
		if err != nil {
			return "", nil, errors.New(fmt.Sprintf("error stating source file: %s %s", file, err))
		}

		if fi.IsDir() {
			continue
		}

		source, err := vfs.Open(file)
		if err != nil {
			return "", nil, errors.New(fmt.Sprintf("error opening source file: %s %s", file, err))
		}
		defer source.Close()

		dest, err := os.OpenFile(filepath.Join(directory, file), os.O_CREATE|os.O_WRONLY, 0666)
		if err != nil {
			return "", nil, errors.New(fmt.Sprintf("error opening dest file: %s %s", file, err))
		}
		defer dest.Close()

		_, err = io.Copy(dest, source)
		if err != nil {
			return "", nil, err
		}
	}

	// Initialize a git repository in the case_0/ directory and commit all the entries
	case_0_directory = filepath.Join(directory, "case_0")

	err = git.Init(case_0_directory)
	if err != nil {
		return "", nil, err
	}

	entries = dirMap["case_0"]
	sort.Sort(entriesByDate(entries))

	// Commit the entries into git
	for i, entryFilename := range entries {
		changes := git.NewChangesIn(case_0_directory)
		changes.Add(git.ChangedFile(entryFilename))
		changes.Msg = fmt.Sprintf("Commit Msg | Entry %d\n", i+1)
		err = changes.Commit()
		if err != nil {
			return "", nil, err
		}
	}

	return case_0_directory, entries, nil
}
<|endoftext|>


<commit_before>// Copyright 2012, 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package watcher_test

import (
	stdtesting "testing"
	"time"

	gc "launchpad.net/gocheck"

	"launchpad.net/juju-core/juju/testing"
	"launchpad.net/juju-core/state"
	"launchpad.net/juju-core/state/api"
	"launchpad.net/juju-core/state/api/params"
	"launchpad.net/juju-core/state/api/watcher"
	statetesting "launchpad.net/juju-core/state/testing"
	coretesting "launchpad.net/juju-core/testing"
)

func TestAll(t *stdtesting.T) {
	coretesting.MgoTestPackage(t)
}

type watcherSuite struct {
	testing.JujuConnSuite

	stateAPI *api.State

	// These are raw State objects. Use them for setup and assertions, but
	// should never be touched by the API calls themselves
	rawMachine *state.Machine
}

var _ = gc.Suite(&watcherSuite{})

func (s *watcherSuite) SetUpTest(c *gc.C) {
	s.JujuConnSuite.SetUpTest(c)

	// Create a machine to work with
	var err error
	s.rawMachine, err = s.State.AddMachine("series", state.JobHostUnits)
	c.Assert(err, gc.IsNil)
	err = s.rawMachine.SetProvisioned("foo", "fake_nonce", nil)
	c.Assert(err, gc.IsNil)
	err = s.rawMachine.SetPassword("test-password")
	c.Assert(err, gc.IsNil)

	// Login as the machine agent of the created machine.
	s.stateAPI = s.OpenAPIAsMachine(c, s.rawMachine.Tag(), "test-password", "fake_nonce")
	c.Assert(s.stateAPI, gc.NotNil)
}

func (s *watcherSuite) TearDownTest(c *gc.C) {
	if s.stateAPI != nil {
		err := s.stateAPI.Close()
		c.Check(err, gc.IsNil)
	}
	s.JujuConnSuite.TearDownTest(c)
}

func (s *watcherSuite) TestWatchInitialEventConsumed(c *gc.C) {
	// Machiner.Watch should send the initial event as part of the Watch
	// call (for NotifyWatchers there is no state to be transmitted). So a
	// call to Next() should not have anything to return.
	var results params.NotifyWatchResults
	args := params.Entities{Entities: []params.Entity{{Tag: s.rawMachine.Tag()}}}
	err := s.stateAPI.Call("Machiner", "", "Watch", args, &results)
	c.Assert(err, gc.IsNil)
	c.Assert(results.Results, gc.HasLen, 1)
	result := results.Results[0]
	c.Assert(result.Error, gc.IsNil)

	// We expect the Call() to "Next" to block, so run it in a goroutine.
	done := make(chan error)
	go func() {
		ignored := struct{}{}
		done <- s.stateAPI.Call("NotifyWatcher", result.NotifyWatcherId, "Next", nil, &ignored)
	}()

	select {
	case err := <-done:
		c.Errorf("Call(Next) did not block immediately after Watch(): err %v", err)
	case <-time.After(coretesting.ShortWait):
	}
}

func (s *watcherSuite) TestWatchMachine(c *gc.C) {
	var results params.NotifyWatchResults
	args := params.Entities{Entities: []params.Entity{{Tag: s.rawMachine.Tag()}}}
	err := s.stateAPI.Call("Machiner", "", "Watch", args, &results)
	c.Assert(err, gc.IsNil)
	c.Assert(results.Results, gc.HasLen, 1)
	result := results.Results[0]
	c.Assert(result.Error, gc.IsNil)

	// params.NotifyWatcher conforms to the state.NotifyWatcher interface
	w := watcher.NewNotifyWatcher(s.stateAPI, result)
	wc := statetesting.NewNotifyWatcherC(c, s.State, w)
	wc.AssertOneChange()
	statetesting.AssertStop(c, w)
	wc.AssertClosed()
}

func (s *watcherSuite) TestNotifyWatcherStopsWithPendingSend(c *gc.C) {
	var results params.NotifyWatchResults
	args := params.Entities{Entities: []params.Entity{{Tag: s.rawMachine.Tag()}}}
	err := s.stateAPI.Call("Machiner", "", "Watch", args, &results)
	c.Assert(err, gc.IsNil)
	c.Assert(results.Results, gc.HasLen, 1)
	result := results.Results[0]
	c.Assert(result.Error, gc.IsNil)

	// params.NotifyWatcher conforms to the state.NotifyWatcher interface
	w := watcher.NewNotifyWatcher(s.stateAPI, result)
	wc := statetesting.NewNotifyWatcherC(c, s.State, w)

	// Now, without reading any changes try stopping the watcher.
	statetesting.AssertCanStopWhenSending(c, w)
	wc.AssertClosed()
}

func (s *watcherSuite) TestWatchUnitsKeepsEvents(c *gc.C) {
	// Create two services, relate them, and add one unit to each - a
	// principal and a subordinate.
	mysql, err := s.State.AddService("mysql", s.AddTestingCharm(c, "mysql"))
	c.Assert(err, gc.IsNil)
	logging, err := s.State.AddService("logging", s.AddTestingCharm(c, "logging"))
	c.Assert(err, gc.IsNil)
	eps, err := s.State.InferEndpoints([]string{"mysql", "logging"})
	c.Assert(err, gc.IsNil)
	rel, err := s.State.AddRelation(eps...)
	c.Assert(err, gc.IsNil)
	principal, err := mysql.AddUnit()
	c.Assert(err, gc.IsNil)
	err = principal.AssignToMachine(s.rawMachine)
	c.Assert(err, gc.IsNil)
	relUnit, err := rel.Unit(principal)
	c.Assert(err, gc.IsNil)
	err = relUnit.EnterScope(nil)
	c.Assert(err, gc.IsNil)
	subordinate, err := logging.Unit("logging/0")
	c.Assert(err, gc.IsNil)

	// Call the Deployer facade's WatchUnits for machine-0.
	var results params.StringsWatchResults
	args := params.Entities{Entities: []params.Entity{{Tag: s.rawMachine.Tag()}}}
	err = s.stateAPI.Call("Deployer", "", "WatchUnits", args, &results)
	c.Assert(err, gc.IsNil)
	c.Assert(results.Results, gc.HasLen, 1)
	result := results.Results[0]
	c.Assert(result.Error, gc.IsNil)

	// Start a StringsWatcher and check the initial event.
	w := watcher.NewStringsWatcher(s.stateAPI, result)
	wc := statetesting.NewStringsWatcherC(c, s.State, w)
	wc.AssertChange("mysql/0", "logging/0")
	wc.AssertNoChange()

	// Now, without reading any changes advance the lifecycle of both
	// units, inducing an update server-side after each two changes to
	// ensure they're reported as separate events over the API.
	err = subordinate.EnsureDead()
	c.Assert(err, gc.IsNil)
	s.BackingState.StartSync()
	err = subordinate.Remove()
	c.Assert(err, gc.IsNil)
	err = principal.EnsureDead()
	c.Assert(err, gc.IsNil)
	s.BackingState.StartSync()

	// Expect these changes as 2 separate events, so that
	// nothing gets lost.
	wc.AssertChange("logging/0")
	wc.AssertChange("mysql/0")
	wc.AssertNoChange()

	statetesting.AssertStop(c, w)
	wc.AssertClosed()
}

func (s *watcherSuite) TestStringsWatcherStopsWithPendingSend(c *gc.C) {
	// Call the Deployer facade's WatchUnits for machine-0.
	var results params.StringsWatchResults
	args := params.Entities{Entities: []params.Entity{{Tag: s.rawMachine.Tag()}}}
	err := s.stateAPI.Call("Deployer", "", "WatchUnits", args, &results)
	c.Assert(err, gc.IsNil)
	c.Assert(results.Results, gc.HasLen, 1)
	result := results.Results[0]
	c.Assert(result.Error, gc.IsNil)

	// Start a StringsWatcher and check the initial event.
	w := watcher.NewStringsWatcher(s.stateAPI, result)
	wc := statetesting.NewStringsWatcherC(c, s.State, w)

	// Create a service, deploy a unit of it on the machine.
	mysql, err := s.State.AddService("mysql", s.AddTestingCharm(c, "mysql"))
	c.Assert(err, gc.IsNil)
	principal, err := mysql.AddUnit()
	c.Assert(err, gc.IsNil)
	err = principal.AssignToMachine(s.rawMachine)
	c.Assert(err, gc.IsNil)

	// Ensure the initial event is delivered. Then test the watcher
	// can be stopped cleanly without reading the pending change.
	s.BackingState.StartSync()
	statetesting.AssertCanStopWhenSending(c, w)
	wc.AssertClosed()
}
<commit_msg>and last one for now<commit_after>// Copyright 2012, 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package watcher_test

import (
	stdtesting "testing"
	"time"

	gc "launchpad.net/gocheck"

	"launchpad.net/juju-core/juju/testing"
	"launchpad.net/juju-core/state"
	"launchpad.net/juju-core/state/api"
	"launchpad.net/juju-core/state/api/params"
	"launchpad.net/juju-core/state/api/watcher"
	statetesting "launchpad.net/juju-core/state/testing"
	coretesting "launchpad.net/juju-core/testing"
)

func TestAll(t *stdtesting.T) {
	coretesting.MgoTestPackage(t)
}

type watcherSuite struct {
	testing.JujuConnSuite

	stateAPI *api.State

	// These are raw State objects. Use them for setup and assertions, but
	// should never be touched by the API calls themselves
	rawMachine *state.Machine
}

var _ = gc.Suite(&watcherSuite{})

func (s *watcherSuite) SetUpTest(c *gc.C) {
	s.JujuConnSuite.SetUpTest(c)
	s.stateAPI, s.rawMachine = s.OpenAPIAsNewMachine(c)
	c.Assert(s.stateAPI, gc.NotNil)
}

func (s *watcherSuite) TearDownTest(c *gc.C) {
	if s.stateAPI != nil {
		err := s.stateAPI.Close()
		c.Check(err, gc.IsNil)
	}
	s.JujuConnSuite.TearDownTest(c)
}

func (s *watcherSuite) TestWatchInitialEventConsumed(c *gc.C) {
	// Machiner.Watch should send the initial event as part of the Watch
	// call (for NotifyWatchers there is no state to be transmitted). So a
	// call to Next() should not have anything to return.
	var results params.NotifyWatchResults
	args := params.Entities{Entities: []params.Entity{{Tag: s.rawMachine.Tag()}}}
	err := s.stateAPI.Call("Machiner", "", "Watch", args, &results)
	c.Assert(err, gc.IsNil)
	c.Assert(results.Results, gc.HasLen, 1)
	result := results.Results[0]
	c.Assert(result.Error, gc.IsNil)

	// We expect the Call() to "Next" to block, so run it in a goroutine.
	done := make(chan error)
	go func() {
		ignored := struct{}{}
		done <- s.stateAPI.Call("NotifyWatcher", result.NotifyWatcherId, "Next", nil, &ignored)
	}()

	select {
	case err := <-done:
		c.Errorf("Call(Next) did not block immediately after Watch(): err %v", err)
	case <-time.After(coretesting.ShortWait):
	}
}

func (s *watcherSuite) TestWatchMachine(c *gc.C) {
	var results params.NotifyWatchResults
	args := params.Entities{Entities: []params.Entity{{Tag: s.rawMachine.Tag()}}}
	err := s.stateAPI.Call("Machiner", "", "Watch", args, &results)
	c.Assert(err, gc.IsNil)
	c.Assert(results.Results, gc.HasLen, 1)
	result := results.Results[0]
	c.Assert(result.Error, gc.IsNil)

	// params.NotifyWatcher conforms to the state.NotifyWatcher interface
	w := watcher.NewNotifyWatcher(s.stateAPI, result)
	wc := statetesting.NewNotifyWatcherC(c, s.State, w)
	wc.AssertOneChange()
	statetesting.AssertStop(c, w)
	wc.AssertClosed()
}

func (s *watcherSuite) TestNotifyWatcherStopsWithPendingSend(c *gc.C) {
	var results params.NotifyWatchResults
	args := params.Entities{Entities: []params.Entity{{Tag: s.rawMachine.Tag()}}}
	err := s.stateAPI.Call("Machiner", "", "Watch", args, &results)
	c.Assert(err, gc.IsNil)
	c.Assert(results.Results, gc.HasLen, 1)
	result := results.Results[0]
	c.Assert(result.Error, gc.IsNil)

	// params.NotifyWatcher conforms to the state.NotifyWatcher interface
	w := watcher.NewNotifyWatcher(s.stateAPI, result)
	wc := statetesting.NewNotifyWatcherC(c, s.State, w)

	// Now, without reading any changes try stopping the watcher.
	statetesting.AssertCanStopWhenSending(c, w)
	wc.AssertClosed()
}

func (s *watcherSuite) TestWatchUnitsKeepsEvents(c *gc.C) {
	// Create two services, relate them, and add one unit to each - a
	// principal and a subordinate.
	mysql, err := s.State.AddService("mysql", s.AddTestingCharm(c, "mysql"))
	c.Assert(err, gc.IsNil)
	logging, err := s.State.AddService("logging", s.AddTestingCharm(c, "logging"))
	c.Assert(err, gc.IsNil)
	eps, err := s.State.InferEndpoints([]string{"mysql", "logging"})
	c.Assert(err, gc.IsNil)
	rel, err := s.State.AddRelation(eps...)
	c.Assert(err, gc.IsNil)
	principal, err := mysql.AddUnit()
	c.Assert(err, gc.IsNil)
	err = principal.AssignToMachine(s.rawMachine)
	c.Assert(err, gc.IsNil)
	relUnit, err := rel.Unit(principal)
	c.Assert(err, gc.IsNil)
	err = relUnit.EnterScope(nil)
	c.Assert(err, gc.IsNil)
	subordinate, err := logging.Unit("logging/0")
	c.Assert(err, gc.IsNil)

	// Call the Deployer facade's WatchUnits for machine-0.
	var results params.StringsWatchResults
	args := params.Entities{Entities: []params.Entity{{Tag: s.rawMachine.Tag()}}}
	err = s.stateAPI.Call("Deployer", "", "WatchUnits", args, &results)
	c.Assert(err, gc.IsNil)
	c.Assert(results.Results, gc.HasLen, 1)
	result := results.Results[0]
	c.Assert(result.Error, gc.IsNil)

	// Start a StringsWatcher and check the initial event.
	w := watcher.NewStringsWatcher(s.stateAPI, result)
	wc := statetesting.NewStringsWatcherC(c, s.State, w)
	wc.AssertChange("mysql/0", "logging/0")
	wc.AssertNoChange()

	// Now, without reading any changes advance the lifecycle of both
	// units, inducing an update server-side after each two changes to
	// ensure they're reported as separate events over the API.
	err = subordinate.EnsureDead()
	c.Assert(err, gc.IsNil)
	s.BackingState.StartSync()
	err = subordinate.Remove()
	c.Assert(err, gc.IsNil)
	err = principal.EnsureDead()
	c.Assert(err, gc.IsNil)
	s.BackingState.StartSync()

	// Expect these changes as 2 separate events, so that
	// nothing gets lost.
	wc.AssertChange("logging/0")
	wc.AssertChange("mysql/0")
	wc.AssertNoChange()

	statetesting.AssertStop(c, w)
	wc.AssertClosed()
}

func (s *watcherSuite) TestStringsWatcherStopsWithPendingSend(c *gc.C) {
	// Call the Deployer facade's WatchUnits for machine-0.
	var results params.StringsWatchResults
	args := params.Entities{Entities: []params.Entity{{Tag: s.rawMachine.Tag()}}}
	err := s.stateAPI.Call("Deployer", "", "WatchUnits", args, &results)
	c.Assert(err, gc.IsNil)
	c.Assert(results.Results, gc.HasLen, 1)
	result := results.Results[0]
	c.Assert(result.Error, gc.IsNil)

	// Start a StringsWatcher and check the initial event.
	w := watcher.NewStringsWatcher(s.stateAPI, result)
	wc := statetesting.NewStringsWatcherC(c, s.State, w)

	// Create a service, deploy a unit of it on the machine.
	mysql, err := s.State.AddService("mysql", s.AddTestingCharm(c, "mysql"))
	c.Assert(err, gc.IsNil)
	principal, err := mysql.AddUnit()
	c.Assert(err, gc.IsNil)
	err = principal.AssignToMachine(s.rawMachine)
	c.Assert(err, gc.IsNil)

	// Ensure the initial event is delivered. Then test the watcher
	// can be stopped cleanly without reading the pending change.
	s.BackingState.StartSync()
	statetesting.AssertCanStopWhenSending(c, w)
	wc.AssertClosed()
}
<|endoftext|>


<commit_before>// This package implements a provisioner for Packer that executes a
// saltstack highstate within the remote machine
package saltmasterless

import (
	"errors"
	"fmt"
	"github.com/mitchellh/packer/common"
	"github.com/mitchellh/packer/packer"
	"os"
	"path/filepath"
	"strings"
)

const DefaultTempConfigDir = "/tmp/salt"

type Config struct {
	common.PackerConfig `mapstructure:",squash"`

	// If true, run the salt-bootstrap script
	SkipBootstrap bool   `mapstructure:"skip_bootstrap"`
	BootstrapArgs string `mapstructure:"bootstrap_args"`

	// Local path to the minion config
	MinionConfig string `mapstructure:"minion_config"`

	// Local path to the salt state tree
	LocalStateTree string `mapstructure:"local_state_tree"`

	// Local path to the salt pillar roots
	LocalPillarRoots string `mapstructure:"local_pillar_roots"`

	// Where files will be copied before moving to the /srv/salt directory
	TempConfigDir string `mapstructure:"temp_config_dir"`

	tpl *packer.ConfigTemplate
}

type Provisioner struct {
	config Config
}

func (p *Provisioner) Prepare(raws ...interface{}) error {
	md, err := common.DecodeConfig(&p.config, raws...)
	if err != nil {
		return err
	}

	p.config.tpl, err = packer.NewConfigTemplate()
	if err != nil {
		return err
	}
	p.config.tpl.UserVars = p.config.PackerUserVars

	if p.config.TempConfigDir == "" {
		p.config.TempConfigDir = DefaultTempConfigDir
	}

	// Accumulate any errors
	errs := common.CheckUnusedConfig(md)

	templates := map[string]*string{
		"bootstrap_args":     &p.config.BootstrapArgs,
		"minion_config":      &p.config.MinionConfig,
		"local_state_tree":   &p.config.LocalStateTree,
		"local_pillar_roots": &p.config.LocalPillarRoots,
		"temp_config_dir":    &p.config.TempConfigDir,
	}

	for n, ptr := range templates {
		var err error
		*ptr, err = p.config.tpl.Process(*ptr, nil)
		if err != nil {
			errs = packer.MultiErrorAppend(
				errs, fmt.Errorf("Error processing %s: %s", n, err))
		}
	}

	if p.config.LocalStateTree != "" {
		if _, err := os.Stat(p.config.LocalStateTree); err != nil {
			errs = packer.MultiErrorAppend(errs,
				errors.New("local_state_tree must exist and be accessible"))
		}
	}

	if p.config.LocalPillarRoots != "" {
		if _, err := os.Stat(p.config.LocalPillarRoots); err != nil {
			errs = packer.MultiErrorAppend(errs,
				errors.New("local_pillar_roots must exist and be accessible"))
		}
	}

	if p.config.MinionConfig != "" {
		if _, err := os.Stat(p.config.MinionConfig); err != nil {
			errs = packer.MultiErrorAppend(errs,
				errors.New("minion_config must exist and be accessible"))
		}
	}

	if errs != nil && len(errs.Errors) > 0 {
		return errs
	}

	return nil
}

func (p *Provisioner) Provision(ui packer.Ui, comm packer.Communicator) error {
	var err error

	ui.Say("Provisioning with Salt...")
	if !p.config.SkipBootstrap {
		cmd := &packer.RemoteCmd{
			Command: fmt.Sprintf("wget -O - http://bootstrap.saltstack.org | sudo sh -s %s", p.config.BootstrapArgs),
		}
		ui.Message(fmt.Sprintf("Installing Salt with command %s", cmd))
		if err = cmd.StartWithUi(comm, ui); err != nil {
			return fmt.Errorf("Unable to install Salt: %d", err)
		}
	}

	ui.Message(fmt.Sprintf("Creating remote directory: %s", p.config.TempConfigDir))
	cmd := &packer.RemoteCmd{Command: fmt.Sprintf("mkdir -p %s", p.config.TempConfigDir)}
	if err = cmd.StartWithUi(comm, ui); err != nil || cmd.ExitStatus != 0 {
		if err == nil {
			err = fmt.Errorf("Bad exit status: %d", cmd.ExitStatus)
		}

		return fmt.Errorf("Error creating remote salt state directory: %s", err)
	}

	if p.config.MinionConfig != "" {
		ui.Message(fmt.Sprintf("Uploading minion config: %s", p.config.MinionConfig))
		if err = uploadMinionConfig(comm, fmt.Sprintf("%s/minion", p.config.TempConfigDir), p.config.MinionConfig); err != nil {
			return fmt.Errorf("Error uploading local minion config file to remote: %s", err)
		}

		ui.Message(fmt.Sprintf("Moving %s/minion to /etc/salt/minion", p.config.TempConfigDir))
		cmd = &packer.RemoteCmd{Command: fmt.Sprintf("sudo mv %s/minion /etc/salt/minion", p.config.TempConfigDir)}
		if err = cmd.StartWithUi(comm, ui); err != nil || cmd.ExitStatus != 0 {
			if err == nil {
				err = fmt.Errorf("Bad exit status: %d", cmd.ExitStatus)
			}

			return fmt.Errorf("Unable to move %s/minion to /etc/salt/minion: %d", p.config.TempConfigDir, err)
		}
	}

	ui.Message(fmt.Sprintf("Uploading local state tree: %s", p.config.LocalStateTree))
	if err = UploadLocalDirectory(p.config.LocalStateTree, fmt.Sprintf("%s/states", p.config.TempConfigDir), comm, ui); err != nil {
		return fmt.Errorf("Error uploading local state tree to remote: %s", err)
	}

	ui.Message(fmt.Sprintf("Moving %s to /srv/salt", p.config.TempConfigDir))
	cmd = &packer.RemoteCmd{Command: fmt.Sprintf("sudo mv %s/states /srv/salt/", p.config.TempConfigDir)}
	if err = cmd.StartWithUi(comm, ui); err != nil || cmd.ExitStatus != 0 {
		if err == nil {
			err = fmt.Errorf("Bad exit status: %d", cmd.ExitStatus)
		}

		return fmt.Errorf("Unable to move %s/states to /srv/salt: %d", p.config.TempConfigDir, err)
	}

	if p.config.LocalPillarRoots != "" {
		ui.Message(fmt.Sprintf("Creating remote pillar directory: %s/pillar", p.config.TempConfigDir))
		cmd := &packer.RemoteCmd{Command: fmt.Sprintf("mkdir -p %s/pillar", p.config.TempConfigDir)}
		if err = cmd.StartWithUi(comm, ui); err != nil || cmd.ExitStatus != 0 {
			if err == nil {
				err = fmt.Errorf("Bad exit status: %d", cmd.ExitStatus)
			}

			return fmt.Errorf("Error creating remote pillar directory: %s", err)
		}

		ui.Message(fmt.Sprintf("Uploading local pillar roots: %s", p.config.LocalPillarRoots))
		if err = UploadLocalDirectory(p.config.LocalPillarRoots, fmt.Sprintf("%s/pillar", p.config.TempConfigDir), comm, ui); err != nil {
			return fmt.Errorf("Error uploading local pillar roots to remote: %s", err)
		}

		ui.Message(fmt.Sprintf("Moving %s/pillar to /srv/pillar", p.config.TempConfigDir))
		cmd = &packer.RemoteCmd{Command: fmt.Sprintf("sudo mv %s/pillar /srv/pillar", p.config.TempConfigDir)}
		if err = cmd.StartWithUi(comm, ui); err != nil || cmd.ExitStatus != 0 {
			if err == nil {
				err = fmt.Errorf("Bad exit status: %d", cmd.ExitStatus)
			}

			return fmt.Errorf("Unable to move %s/pillar to /srv/pillar: %d", p.config.TempConfigDir, err)
		}
	}

	ui.Message("Running highstate")
	cmd = &packer.RemoteCmd{Command: "sudo salt-call --local state.highstate -l info"}
	if err = cmd.StartWithUi(comm, ui); err != nil || cmd.ExitStatus != 0 {
		if err == nil {
			err = fmt.Errorf("Bad exit status: %d", cmd.ExitStatus)
		}

		return fmt.Errorf("Error executing highstate: %s", err)
	}

	return nil
}

func (p *Provisioner) Cancel() {
	// Just hard quit. It isn't a big deal if what we're doing keeps
	// running on the other side.
	os.Exit(0)
}

func UploadLocalDirectory(localDir string, remoteDir string, comm packer.Communicator, ui packer.Ui) (err error) {
	visitPath := func(localPath string, f os.FileInfo, err error) (err2 error) {
		localRelPath := strings.Replace(localPath, localDir, "", 1)
		localRelPath = strings.Replace(localRelPath, "\\", "/", -1)
		remotePath := filepath.Join(remoteDir, localRelPath)
		if f.IsDir() && f.Name() == ".git" {
			return filepath.SkipDir
		}
		if f.IsDir() {
			// Make remote directory
			cmd := &packer.RemoteCmd{Command: fmt.Sprintf("mkdir -p %s", remotePath)}
			if err = cmd.StartWithUi(comm, ui); err != nil {
				return err
			}
		} else {
			// Upload file to existing directory
			file, err := os.Open(localPath)
			if err != nil {
				return fmt.Errorf("Error opening file: %s", err)
			}
			defer file.Close()

			ui.Message(fmt.Sprintf("Uploading file %s: %s", localPath, remotePath))
			if err = comm.Upload(remotePath, file); err != nil {
				return fmt.Errorf("Error uploading file: %s", err)
			}
		}
		return
	}

	err = filepath.Walk(localDir, visitPath)
	if err != nil {
		return fmt.Errorf("Error uploading local directory %s: %s", localDir, err)
	}

	return nil
}

func uploadMinionConfig(comm packer.Communicator, dst string, src string) error {
	f, err := os.Open(src)
	if err != nil {
		return fmt.Errorf("Error opening minion config: %s", err)
	}
	defer f.Close()

	if err = comm.Upload(dst, f); err != nil {
		return fmt.Errorf("Error uploading minion config: %s", err)
	}

	return nil
}
<commit_msg>provisioner/salt-masterless: move states to /srv/salt, not /srv/salt/states<commit_after>// This package implements a provisioner for Packer that executes a
// saltstack highstate within the remote machine
package saltmasterless

import (
	"errors"
	"fmt"
	"github.com/mitchellh/packer/common"
	"github.com/mitchellh/packer/packer"
	"os"
	"path/filepath"
	"strings"
)

const DefaultTempConfigDir = "/tmp/salt"

type Config struct {
	common.PackerConfig `mapstructure:",squash"`

	// If true, run the salt-bootstrap script
	SkipBootstrap bool   `mapstructure:"skip_bootstrap"`
	BootstrapArgs string `mapstructure:"bootstrap_args"`

	// Local path to the minion config
	MinionConfig string `mapstructure:"minion_config"`

	// Local path to the salt state tree
	LocalStateTree string `mapstructure:"local_state_tree"`

	// Local path to the salt pillar roots
	LocalPillarRoots string `mapstructure:"local_pillar_roots"`

	// Where files will be copied before moving to the /srv/salt directory
	TempConfigDir string `mapstructure:"temp_config_dir"`

	tpl *packer.ConfigTemplate
}

type Provisioner struct {
	config Config
}

func (p *Provisioner) Prepare(raws ...interface{}) error {
	md, err := common.DecodeConfig(&p.config, raws...)
	if err != nil {
		return err
	}

	p.config.tpl, err = packer.NewConfigTemplate()
	if err != nil {
		return err
	}
	p.config.tpl.UserVars = p.config.PackerUserVars

	if p.config.TempConfigDir == "" {
		p.config.TempConfigDir = DefaultTempConfigDir
	}

	// Accumulate any errors
	errs := common.CheckUnusedConfig(md)

	templates := map[string]*string{
		"bootstrap_args":     &p.config.BootstrapArgs,
		"minion_config":      &p.config.MinionConfig,
		"local_state_tree":   &p.config.LocalStateTree,
		"local_pillar_roots": &p.config.LocalPillarRoots,
		"temp_config_dir":    &p.config.TempConfigDir,
	}

	for n, ptr := range templates {
		var err error
		*ptr, err = p.config.tpl.Process(*ptr, nil)
		if err != nil {
			errs = packer.MultiErrorAppend(
				errs, fmt.Errorf("Error processing %s: %s", n, err))
		}
	}

	if p.config.LocalStateTree != "" {
		if _, err := os.Stat(p.config.LocalStateTree); err != nil {
			errs = packer.MultiErrorAppend(errs,
				errors.New("local_state_tree must exist and be accessible"))
		}
	}

	if p.config.LocalPillarRoots != "" {
		if _, err := os.Stat(p.config.LocalPillarRoots); err != nil {
			errs = packer.MultiErrorAppend(errs,
				errors.New("local_pillar_roots must exist and be accessible"))
		}
	}

	if p.config.MinionConfig != "" {
		if _, err := os.Stat(p.config.MinionConfig); err != nil {
			errs = packer.MultiErrorAppend(errs,
				errors.New("minion_config must exist and be accessible"))
		}
	}

	if errs != nil && len(errs.Errors) > 0 {
		return errs
	}

	return nil
}

func (p *Provisioner) Provision(ui packer.Ui, comm packer.Communicator) error {
	var err error

	ui.Say("Provisioning with Salt...")
	if !p.config.SkipBootstrap {
		cmd := &packer.RemoteCmd{
			Command: fmt.Sprintf("wget -O - http://bootstrap.saltstack.org | sudo sh -s %s", p.config.BootstrapArgs),
		}
		ui.Message(fmt.Sprintf("Installing Salt with command %s", cmd))
		if err = cmd.StartWithUi(comm, ui); err != nil {
			return fmt.Errorf("Unable to install Salt: %d", err)
		}
	}

	ui.Message(fmt.Sprintf("Creating remote directory: %s", p.config.TempConfigDir))
	cmd := &packer.RemoteCmd{Command: fmt.Sprintf("mkdir -p %s", p.config.TempConfigDir)}
	if err = cmd.StartWithUi(comm, ui); err != nil || cmd.ExitStatus != 0 {
		if err == nil {
			err = fmt.Errorf("Bad exit status: %d", cmd.ExitStatus)
		}

		return fmt.Errorf("Error creating remote salt state directory: %s", err)
	}

	if p.config.MinionConfig != "" {
		ui.Message(fmt.Sprintf("Uploading minion config: %s", p.config.MinionConfig))
		if err = uploadMinionConfig(comm, fmt.Sprintf("%s/minion", p.config.TempConfigDir), p.config.MinionConfig); err != nil {
			return fmt.Errorf("Error uploading local minion config file to remote: %s", err)
		}

		ui.Message(fmt.Sprintf("Moving %s/minion to /etc/salt/minion", p.config.TempConfigDir))
		cmd = &packer.RemoteCmd{Command: fmt.Sprintf("sudo mv %s/minion /etc/salt/minion", p.config.TempConfigDir)}
		if err = cmd.StartWithUi(comm, ui); err != nil || cmd.ExitStatus != 0 {
			if err == nil {
				err = fmt.Errorf("Bad exit status: %d", cmd.ExitStatus)
			}

			return fmt.Errorf("Unable to move %s/minion to /etc/salt/minion: %d", p.config.TempConfigDir, err)
		}
	}

	ui.Message(fmt.Sprintf("Uploading local state tree: %s", p.config.LocalStateTree))
	if err = UploadLocalDirectory(p.config.LocalStateTree, fmt.Sprintf("%s/states", p.config.TempConfigDir), comm, ui); err != nil {
		return fmt.Errorf("Error uploading local state tree to remote: %s", err)
	}

	ui.Message(fmt.Sprintf("Moving %s/states to /srv/salt", p.config.TempConfigDir))
	cmd = &packer.RemoteCmd{Command: fmt.Sprintf("sudo mv %s/states /srv/salt", p.config.TempConfigDir)}
	if err = cmd.StartWithUi(comm, ui); err != nil || cmd.ExitStatus != 0 {
		if err == nil {
			err = fmt.Errorf("Bad exit status: %d", cmd.ExitStatus)
		}

		return fmt.Errorf("Unable to move %s/states to /srv/salt: %d", p.config.TempConfigDir, err)
	}

	if p.config.LocalPillarRoots != "" {
		ui.Message(fmt.Sprintf("Creating remote pillar directory: %s/pillar", p.config.TempConfigDir))
		cmd := &packer.RemoteCmd{Command: fmt.Sprintf("mkdir -p %s/pillar", p.config.TempConfigDir)}
		if err = cmd.StartWithUi(comm, ui); err != nil || cmd.ExitStatus != 0 {
			if err == nil {
				err = fmt.Errorf("Bad exit status: %d", cmd.ExitStatus)
			}

			return fmt.Errorf("Error creating remote pillar directory: %s", err)
		}

		ui.Message(fmt.Sprintf("Uploading local pillar roots: %s", p.config.LocalPillarRoots))
		if err = UploadLocalDirectory(p.config.LocalPillarRoots, fmt.Sprintf("%s/pillar", p.config.TempConfigDir), comm, ui); err != nil {
			return fmt.Errorf("Error uploading local pillar roots to remote: %s", err)
		}

		ui.Message(fmt.Sprintf("Moving %s/pillar to /srv/pillar", p.config.TempConfigDir))
		cmd = &packer.RemoteCmd{Command: fmt.Sprintf("sudo mv %s/pillar /srv/pillar", p.config.TempConfigDir)}
		if err = cmd.StartWithUi(comm, ui); err != nil || cmd.ExitStatus != 0 {
			if err == nil {
				err = fmt.Errorf("Bad exit status: %d", cmd.ExitStatus)
			}

			return fmt.Errorf("Unable to move %s/pillar to /srv/pillar: %d", p.config.TempConfigDir, err)
		}
	}

	ui.Message("Running highstate")
	cmd = &packer.RemoteCmd{Command: "sudo salt-call --local state.highstate -l info"}
	if err = cmd.StartWithUi(comm, ui); err != nil || cmd.ExitStatus != 0 {
		if err == nil {
			err = fmt.Errorf("Bad exit status: %d", cmd.ExitStatus)
		}

		return fmt.Errorf("Error executing highstate: %s", err)
	}

	return nil
}

func (p *Provisioner) Cancel() {
	// Just hard quit. It isn't a big deal if what we're doing keeps
	// running on the other side.
	os.Exit(0)
}

func UploadLocalDirectory(localDir string, remoteDir string, comm packer.Communicator, ui packer.Ui) (err error) {
	visitPath := func(localPath string, f os.FileInfo, err error) (err2 error) {
		localRelPath := strings.Replace(localPath, localDir, "", 1)
		localRelPath = strings.Replace(localRelPath, "\\", "/", -1)
		remotePath := filepath.Join(remoteDir, localRelPath)
		if f.IsDir() && f.Name() == ".git" {
			return filepath.SkipDir
		}
		if f.IsDir() {
			// Make remote directory
			cmd := &packer.RemoteCmd{Command: fmt.Sprintf("mkdir -p %s", remotePath)}
			if err = cmd.StartWithUi(comm, ui); err != nil {
				return err
			}
		} else {
			// Upload file to existing directory
			file, err := os.Open(localPath)
			if err != nil {
				return fmt.Errorf("Error opening file: %s", err)
			}
			defer file.Close()

			ui.Message(fmt.Sprintf("Uploading file %s: %s", localPath, remotePath))
			if err = comm.Upload(remotePath, file); err != nil {
				return fmt.Errorf("Error uploading file: %s", err)
			}
		}
		return
	}

	err = filepath.Walk(localDir, visitPath)
	if err != nil {
		return fmt.Errorf("Error uploading local directory %s: %s", localDir, err)
	}

	return nil
}

func uploadMinionConfig(comm packer.Communicator, dst string, src string) error {
	f, err := os.Open(src)
	if err != nil {
		return fmt.Errorf("Error opening minion config: %s", err)
	}
	defer f.Close()

	if err = comm.Upload(dst, f); err != nil {
		return fmt.Errorf("Error uploading minion config: %s", err)
	}

	return nil
}
<|endoftext|>


<commit_before>package upload

import (
	"io"
	"io/ioutil"
	"os"
	"path/filepath"

	"github.com/inconshreveable/log15"
	"github.com/materials-commons/mcstore/pkg/app"
	"github.com/materials-commons/mcstore/pkg/app/flow"
)

type assembler struct {
	request *flow.Request
	log     log15.Logger
	tracker *uploadTracker
}

func newAssembler(request *flow.Request, tracker *uploadTracker) *assembler {
	return &assembler{
		request: request,
		tracker: tracker,
		log:     app.NewLog("go-routine", "assembler"),
	}
}

func (a *assembler) launch() {
	go func() {
		if a.assembleFile() {
			a.finish()
		}
	}()
}

// assembleFile reassembles a file by combining all of its chunks. This routine
// needs to ensure that all the chunks have arrived. It does this by counting
// the number of entries in the directory vs the expected number
func (a *assembler) assembleFile() bool {
	if fdst, err := a.createUploadPaths(); err == nil {
		defer fdst.Close()
		return a.assembleFromDirectory(fdst)
	}
	return false
}

func (a *assembler) createUploadPaths() (*os.File, error) {
	os.MkdirAll(app.MCDir.FileDir(a.request.FileID), 0700)
	filePath := app.MCDir.FilePath(a.request.FileID)
	fdst, err := os.Create(filePath)
	if err != nil {
		a.log.Error(app.Logf("Error creating assembly file %s: %s", filePath, err))
	}
	return fdst, err
}

func (a *assembler) assembleFromDirectory(fdst *os.File) bool {
	uploadDir := a.request.Dir()
	finfos, err := ioutil.ReadDir(uploadDir)
	if err != nil {
		a.log.Error(app.Logf("Error reading assembly dir %s: %s", uploadDir, err))
		return false
	}

	//sort.Sort(byChunk(finfos))
	for _, finfo := range finfos {
		fsrc, err := os.Open(filepath.Join(uploadDir, finfo.Name()))
		if err != nil {
			a.log.Error(app.Logf("Error reading chunk %s: %s", finfo.Name(), err))
			return false
		}
		io.Copy(fdst, fsrc)
		fsrc.Close()
	}
	os.RemoveAll(uploadDir)
	return true
}

// finish takes care of the final steps. At this point the file
// has been assembled. There are a couple of other checks that
// need to be done:
//    1. Create the file hash and check if it already exists
//    2. If hash exists, delete file and mark its usesid, mark
//       the upload as finished
//    3. If no hash, then mark the upload as finished.
func (a *assembler) finish() {
	//
	a.tracker.clear(a.request.UploadID())
}
<commit_msg>File will be going away once I've extracted all its functionality. Changed/commented out code that relied on old functionality.<commit_after>package upload

import (
	"io"
	"io/ioutil"
	"os"
	"path/filepath"

	"github.com/inconshreveable/log15"
	"github.com/materials-commons/mcstore/pkg/app"
	"github.com/materials-commons/mcstore/pkg/app/flow"
)

type assembler struct {
	request *flow.Request
	log     log15.Logger
	tracker *uploadTracker
}

func newAssembler(request *flow.Request, tracker *uploadTracker) *assembler {
	return &assembler{
		request: request,
		tracker: tracker,
		log:     app.NewLog("go-routine", "assembler"),
	}
}

func (a *assembler) launch() {
	go func() {
		if a.assembleFile() {
			a.finish()
		}
	}()
}

// assembleFile reassembles a file by combining all of its chunks. This routine
// needs to ensure that all the chunks have arrived. It does this by counting
// the number of entries in the directory vs the expected number
func (a *assembler) assembleFile() bool {
	if fdst, err := a.createUploadPaths(); err == nil {
		defer fdst.Close()
		return a.assembleFromDirectory(fdst)
	}
	return false
}

func (a *assembler) createUploadPaths() (*os.File, error) {
	os.MkdirAll(app.MCDir.FileDir(a.request.FileID), 0700)
	filePath := app.MCDir.FilePath(a.request.FileID)
	fdst, err := os.Create(filePath)
	if err != nil {
		a.log.Error(app.Logf("Error creating assembly file %s: %s", filePath, err))
	}
	return fdst, err
}

func (a *assembler) assembleFromDirectory(fdst *os.File) bool {
	uploadDir := "" // a.request.Dir()
	finfos, err := ioutil.ReadDir(uploadDir)
	if err != nil {
		a.log.Error(app.Logf("Error reading assembly dir %s: %s", uploadDir, err))
		return false
	}

	//sort.Sort(byChunk(finfos))
	for _, finfo := range finfos {
		fsrc, err := os.Open(filepath.Join(uploadDir, finfo.Name()))
		if err != nil {
			a.log.Error(app.Logf("Error reading chunk %s: %s", finfo.Name(), err))
			return false
		}
		io.Copy(fdst, fsrc)
		fsrc.Close()
	}
	os.RemoveAll(uploadDir)
	return true
}

// finish takes care of the final steps. At this point the file
// has been assembled. There are a couple of other checks that
// need to be done:
//    1. Create the file hash and check if it already exists
//    2. If hash exists, delete file and mark its usesid, mark
//       the upload as finished
//    3. If no hash, then mark the upload as finished.
func (a *assembler) finish() {
	//
	a.tracker.clear(a.request.UploadID())
}
<|endoftext|>


<commit_before>/*
   Nging is a toolbox for webmasters
   Copyright (C) 2018-present  Wenhui Shen <swh@admpub.com>

   This program is free software: you can redistribute it and/or modify
   it under the terms of the GNU Affero General Public License as published
   by the Free Software Foundation, either version 3 of the License, or
   (at your option) any later version.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU Affero General Public License for more details.

   You should have received a copy of the GNU Affero General Public License
   along with this program.  If not, see <https://www.gnu.org/licenses/>.
*/

package config

import (
	"reflect"
	"strings"

	"github.com/webx-top/echo"
	"github.com/webx-top/echo/subdomains"

	"github.com/admpub/log"
	"github.com/admpub/nging/v3/application/library/common"
	"github.com/admpub/nging/v3/application/library/notice"
	"github.com/admpub/nging/v3/application/registry/settings"
)

var Setting = common.Setting

func NewSettings(config *Config) *Settings {
	c := &Settings{
		config: config,
	}
	return c
}

type Settings struct {
	Email              Email  `json:"email"`
	Log                Log    `json:"log"`
	APIKey             string `json:"-"` //API密钥
	Debug              bool   `json:"debug"`
	MaxRequestBodySize int    `json:"maxRequestBodySize"`
	Base               echo.H `json:"base"`
	config             *Config
}

func (c *Settings) SetBy(r echo.H, defaults echo.H) *Settings {
	if !r.Has(`base`) && defaults != nil {
		r.Set(`base`, defaults.GetStore(`base`))
	}
	c.Base = r.GetStore(`base`)
	c.APIKey = c.Base.String(`apiKey`)
	c.Debug = c.Base.Bool(`debug`)
	c.MaxRequestBodySize = c.Base.Int(`maxRequestBodySize`)
	return c
}

func (c *Settings) SetDebug(on bool) {
	c.Log.Debug = on
	c.Debug = on

	c.config.DB.SetDebug(on)
	notice.SetDebug(on)
	if on {
		log.Info(`Currently running in debug mode`)
		log.SetLevel(`Debug`)
	} else {
		log.Info(`Currently running in normal mode`)
		log.SetLevel(`Info`)
	}
	subdomains.Default.SetDebug(on)
}

var (
	actGroups          = []string{`base`, `smtp`, `log`}
	onKeySetSettings   = map[string][]func(Diff) error{}
	onGroupSetSettings = map[string][]func(Diffs) error{}
)

func OnGroupSetSettings(groupAndKey string, fn func(Diffs) error) {
	if _, ok := onGroupSetSettings[groupAndKey]; !ok {
		onGroupSetSettings[groupAndKey] = []func(Diffs) error{}
	}
	onGroupSetSettings[groupAndKey] = append(onGroupSetSettings[groupAndKey], fn)
}

func OnKeySetSettings(groupAndKey string, fn func(Diff) error) {
	if _, ok := onKeySetSettings[groupAndKey]; !ok {
		onKeySetSettings[groupAndKey] = []func(Diff) error{}
	}
	onKeySetSettings[groupAndKey] = append(onKeySetSettings[groupAndKey], fn)
}

func FireInitSettings(configs echo.H) error {
	for group, fnList := range onGroupSetSettings {
		values := configs.GetStore(group)
		diffs := Diffs{}
		for k, v := range values {
			diffs[k] = &Diff{
				Old: v,
				New: v,
			}
		}
		for _, fn := range fnList {
			if err := fn(diffs); err != nil {
				return err
			}
		}
	}
	for groupAndKey, fnList := range onKeySetSettings {
		args := strings.SplitN(groupAndKey, `.`, 2)
		values := configs.GetStore(args[0])
		var val interface{}
		if len(args) == 2 {
			val = values.Get(args[1])
		} else {
			val = values
		}
		for _, fn := range fnList {
			if err := fn(Diff{
				Old: val,
				New: val,
			}); err != nil {
				return err
			}
		}
	}
	return nil
}

func FireSetSettings(group string, diffs Diffs) error {
	if fnList, ok := onGroupSetSettings[group]; ok {
		for _, fn := range fnList {
			if err := fn(diffs); err != nil {
				return err
			}
		}
	}
	for key, diff := range diffs {
		k := group + `.` + key
		if fnList, ok := onKeySetSettings[k]; ok {
			for _, fn := range fnList {
				if err := fn(*diff); err != nil {
					return err
				}
			}
		}
	}
	return nil
}

func (c *Settings) Init() error {
	defaults := settings.ConfigDefaultsAsStore()
	var configs = defaults
	if IsInstalled() {
		if c.config.ConnectedDB() {
			configs = settings.ConfigAsStore()
		}
	}
	echo.Set(`NgingConfig`, configs)
	for _, group := range actGroups {
		c.SetConfig(group, configs, defaults)
	}
	return FireInitSettings(configs)
}

func (c *Settings) GetConfig() echo.H {
	r, _ := echo.Get(`NgingConfig`).(echo.H)
	return r
}

type Diff struct {
	Old    interface{}
	New    interface{}
	IsDiff bool
}

func (d Diff) String() string {
	s, _ := d.New.(string)
	return s
}

type Diffs map[string]*Diff

func (d Diffs) Get(key string) interface{} {
	return d[key]
}

func (c *Settings) SetConfigs(groups ...string) error {
	newConfigs := settings.ConfigAsStore(groups...)
	oldConfigs := c.GetConfig()
	return c.setConfigs(newConfigs, oldConfigs)
}

func (c *Settings) setConfigs(newConfigs echo.H, oldConfigs echo.H) error {
	for group, conf := range newConfigs {
		keyCfg := conf.(echo.H)
		keyOldCfg := oldConfigs.GetStore(group)
		diffs := Diffs{}
		for k, v := range keyCfg {
			if !reflect.DeepEqual(keyOldCfg.Get(k), v) {
				diffs[k] = &Diff{
					Old:    keyOldCfg.Get(k),
					New:    v,
					IsDiff: true,
				}
			}
		}
		if len(diffs) == 0 {
			continue
		}
		if err := FireSetSettings(group, diffs); err != nil {
			return err
		}
		oldConfigs.Set(group, keyCfg)
		//log.Debug(`Change configuration:`, group, `:`, echo.Dump(conf, false))
		c.SetConfig(group, oldConfigs, nil)
	}
	return nil
}

func (c *Settings) SetConfig(group string, ngingConfig echo.H, defaults echo.H) {
	switch group {
	case `base`:
		c.SetBy(ngingConfig, defaults)
		c.SetDebug(c.Debug)
	case `smtp`:
		c.Email.SetBy(ngingConfig, defaults).Init()
	case `log`:
		c.Log.SetBy(ngingConfig, defaults).Init()
	}
}
<commit_msg>improved<commit_after>/*
   Nging is a toolbox for webmasters
   Copyright (C) 2018-present  Wenhui Shen <swh@admpub.com>

   This program is free software: you can redistribute it and/or modify
   it under the terms of the GNU Affero General Public License as published
   by the Free Software Foundation, either version 3 of the License, or
   (at your option) any later version.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU Affero General Public License for more details.

   You should have received a copy of the GNU Affero General Public License
   along with this program.  If not, see <https://www.gnu.org/licenses/>.
*/

package config

import (
	"reflect"
	"strings"

	"github.com/webx-top/echo"
	"github.com/webx-top/echo/subdomains"

	"github.com/admpub/log"
	"github.com/admpub/nging/v3/application/library/common"
	"github.com/admpub/nging/v3/application/library/notice"
	"github.com/admpub/nging/v3/application/registry/settings"
)

var Setting = common.Setting

func NewSettings(config *Config) *Settings {
	c := &Settings{
		config: config,
	}
	return c
}

type Settings struct {
	Email              Email  `json:"email"`
	Log                Log    `json:"log"`
	APIKey             string `json:"-"` //API密钥
	Debug              bool   `json:"debug"`
	MaxRequestBodySize int    `json:"maxRequestBodySize"`
	Base               echo.H `json:"base"`
	config             *Config
}

func (c *Settings) SetBy(r echo.H, defaults echo.H) *Settings {
	if !r.Has(`base`) && defaults != nil {
		r.Set(`base`, defaults.GetStore(`base`))
	}
	c.Base = r.GetStore(`base`)
	c.APIKey = c.Base.String(`apiKey`)
	c.Debug = c.Base.Bool(`debug`)
	c.MaxRequestBodySize = c.Base.Int(`maxRequestBodySize`)
	return c
}

func (c *Settings) SetDebug(on bool) {
	c.Log.Debug = on
	c.Debug = on

	c.config.DB.SetDebug(on)
	notice.SetDebug(on)
	if on {
		log.Info(`Currently running in debug mode`)
		log.SetLevel(`Debug`)
	} else {
		log.Info(`Currently running in normal mode`)
		log.SetLevel(`Info`)
	}
	subdomains.Default.SetDebug(on)
}

var (
	actGroups          = []string{`base`, `smtp`, `log`}
	onKeySetSettings   = map[string][]func(Diff) error{}
	onGroupSetSettings = map[string][]func(Diffs) error{}
)

func OnGroupSetSettings(groupAndKey string, fn func(Diffs) error) {
	if _, ok := onGroupSetSettings[groupAndKey]; !ok {
		onGroupSetSettings[groupAndKey] = []func(Diffs) error{}
	}
	onGroupSetSettings[groupAndKey] = append(onGroupSetSettings[groupAndKey], fn)
}

func OnKeySetSettings(groupAndKey string, fn func(Diff) error) {
	if _, ok := onKeySetSettings[groupAndKey]; !ok {
		onKeySetSettings[groupAndKey] = []func(Diff) error{}
	}
	onKeySetSettings[groupAndKey] = append(onKeySetSettings[groupAndKey], fn)
}

func FireInitSettings(configs echo.H) error {
	for group, fnList := range onGroupSetSettings {
		values := configs.GetStore(group)
		diffs := Diffs{}
		for k, v := range values {
			diffs[k] = &Diff{
				Old: v,
				New: v,
			}
		}
		for _, fn := range fnList {
			if err := fn(diffs); err != nil {
				return err
			}
		}
	}
	for groupAndKey, fnList := range onKeySetSettings {
		args := strings.SplitN(groupAndKey, `.`, 2)
		values := configs.GetStore(args[0])
		var val interface{}
		if len(args) == 2 {
			val = values.Get(args[1])
		} else {
			val = values
		}
		for _, fn := range fnList {
			if err := fn(Diff{
				Old: val,
				New: val,
			}); err != nil {
				return err
			}
		}
	}
	return nil
}

func FireSetSettings(group string, diffs Diffs) error {
	if fnList, ok := onGroupSetSettings[group]; ok {
		for _, fn := range fnList {
			if err := fn(diffs); err != nil {
				return err
			}
		}
	}
	for key, diff := range diffs {
		k := group + `.` + key
		if fnList, ok := onKeySetSettings[k]; ok {
			for _, fn := range fnList {
				if err := fn(*diff); err != nil {
					return err
				}
			}
		}
	}
	return nil
}

func (c *Settings) Init() error {
	defaults := settings.ConfigDefaultsAsStore()
	var configs = defaults
	if IsInstalled() {
		if c.config.ConnectedDB() {
			configs = settings.ConfigAsStore()
		}
	}
	echo.Set(`NgingConfig`, configs)
	for _, group := range actGroups {
		c.SetConfig(group, configs, defaults)
	}
	return FireInitSettings(configs)
}

func (c *Settings) GetConfig() echo.H {
	r, _ := echo.Get(`NgingConfig`).(echo.H)
	return r
}

type Diff struct {
	Old    interface{}
	New    interface{}
	IsDiff bool
}

func (d Diff) String() string {
	s, _ := d.New.(string)
	return s
}

type Diffs map[string]*Diff

func (d Diffs) Get(key string) interface{} {
	return d[key]
}

func (c *Settings) SetConfigs(groups ...string) error {
	newConfigs := settings.ConfigAsStore(groups...)
	oldConfigs := c.GetConfig()
	return c.setConfigs(newConfigs, oldConfigs)
}

func (c *Settings) setConfigs(newConfigs echo.H, oldConfigs echo.H) error {
	for group, conf := range newConfigs {
		keyCfg := conf.(echo.H)
		keyOldCfg := oldConfigs.GetStore(group)
		diffs := Diffs{}
		for k, v := range keyCfg {
			if !reflect.DeepEqual(keyOldCfg.Get(k), v) {
				diffs[k] = &Diff{
					Old:    keyOldCfg.Get(k),
					New:    v,
					IsDiff: true,
				}
			}
		}
		if len(diffs) == 0 {
			continue
		}
		oldConfigs.Set(group, keyCfg)
		if err := FireSetSettings(group, diffs); err != nil {
			return err
		}
		//log.Debug(`Change configuration:`, group, `:`, echo.Dump(conf, false))
		c.SetConfig(group, oldConfigs, nil)
	}
	return nil
}

func (c *Settings) SetConfig(group string, ngingConfig echo.H, defaults echo.H) {
	switch group {
	case `base`:
		c.SetBy(ngingConfig, defaults)
		c.SetDebug(c.Debug)
	case `smtp`:
		c.Email.SetBy(ngingConfig, defaults).Init()
	case `log`:
		c.Log.SetBy(ngingConfig, defaults).Init()
	}
}
<|endoftext|>