<commit_before>package experimental_test

import (
	"testing"

	"github.com/cf-deployment/units/helpers"
)

const testDirectory = "operations/experimental"

var experimentalTests = map[string]helpers.OpsFileTestParams{
	"add-credhub-lb.yml":         {},
	"add-deployment-updater.yml": {},
	"add-deployment-updater-external-db.yml": {
		Ops:       []string{"add-deployment-updater.yml", "../use-external-dbs.yml", "add-deployment-updater-external-db.yml"},
		VarsFiles: []string{"../example-vars-files/vars-use-external-dbs.yml"},
	},
	"add-deployment-updater-postgres.yml": {
		Ops: []string{"add-deployment-updater.yml", "../use-postgres.yml", "add-deployment-updater-postgres.yml"},
	},
	"add-metric-store.yml": {},
	"add-syslog-agent.yml": {},
	"add-syslog-agent-windows1803.yml": {
		Ops: []string{"../windows1803-cell.yml", "add-syslog-agent.yml", "add-syslog-agent-windows1803.yml"},
	},
	"add-system-metrics-agent.yml": {},
	"add-system-metrics-agent-windows1803.yml": {
		Ops: []string{"../windows1803-cell.yml", "add-system-metrics-agent.yml", "add-system-metrics-agent-windows1803.yml"},
	},
	"deploy-forwarder-agent.yml":               {},
	"disable-interpolate-service-bindings.yml": {},
	"enable-bpm-garden.yml":                    {},
	"enable-iptables-logger.yml":               {},
	"enable-mysql-tls.yml":                     {},
	"enable-nfs-volume-service-credhub.yml":    {},
	"enable-oci-phase-1.yml":                   {},
	"enable-routing-integrity-windows1803.yml": {
		Ops: []string{"../windows1803-cell.yml", "enable-routing-integrity-windows1803.yml"},
	},
	"enable-routing-integrity-windows2016.yml": {
		Ops: []string{"../windows2016-cell.yml", "enable-routing-integrity-windows2016.yml"},
	},
	"enable-smb-volume-service.yml":            {},
	"enable-suspect-actual-lrp-generation.yml": {},
	"enable-tls-cloud-controller-postgres.yml": {
		Ops: []string{"../use-postgres.yml", "enable-tls-cloud-controller-postgres.yml"},
	},
	"enable-traffic-to-internal-networks.yml":  {},
	"fast-deploy-with-downtime-and-danger.yml": {},
	"infrastructure-metrics.yml":               {},
	"migrate-nfsbroker-mysql-to-credhub.yml": {
		Ops:       []string{"../enable-nfs-volume-service.yml", "migrate-nfsbroker-mysql-to-credhub.yml"},
		VarsFiles: []string{"../example-vars-files/vars-migrate-nfsbroker-mysql-to-credhub.yml"},
	},
	"perm-service.yml": {
		Ops:  []string{"enable-mysql-tls.yml", "perm-service.yml"},
		Vars: []string{"perm_uaa_clients_cc_perm_secret=perm_secret", "perm_uaa_clients_perm_monitor_secret=perm_monitor_secret"},
	},
	"perm-service-with-pxc-release.yml": {
		Ops:  []string{"perm-service.yml", "../use-pxc.yml", "perm-service-with-pxc-release.yml"},
		Vars: []string{"perm_uaa_clients_cc_perm_secret=perm_secret", "perm_uaa_clients_perm_monitor_secret=perm_monitor_secret"},
	},
	"perm-service-with-tcp-routing.yml": {
		Ops:  []string{"perm-service.yml", "../use-pxc.yml", "perm-service-with-pxc-release.yml", "perm-service-with-tcp-routing.yml"},
		Vars: []string{"perm_uaa_clients_cc_perm_secret=perm_secret", "perm_uaa_clients_perm_monitor_secret=perm_monitor_secret"},
	},
	"rootless-containers.yml": {},
	"set-cpu-weight.yml":      {},
	"use-compiled-releases-windows.yml": {
		Ops: []string{"../use-compiled-releases.yml", "../windows2012R2-cell.yml", "use-compiled-releases-windows.yml"},
	},
	"use-create-swap-delete-vm-strategy.yml":          {},
	"use-logcache-for-cloud-controller-app-stats.yml": {},
	"use-native-garden-runc-runner.yml":               {},
	"windows-component-syslog-ca.yml": {
		Ops:       []string{"windows-enable-component-syslog.yml", "windows-component-syslog-ca.yml"},
		VarsFiles: []string{"../addons/example-vars-files/vars-enable-component-syslog.yml"},
	},
	"windows-enable-component-syslog.yml": {
		Ops:       []string{"windows-enable-component-syslog.yml"},
		VarsFiles: []string{"../addons/example-vars-files/vars-enable-component-syslog.yml"},
	},
}

func TestExperimental(t *testing.T) {
	cfDeploymentHome, err := helpers.SetPath()
	if err != nil {
		t.Fatalf("setup: %v", err)
	}

	suite := helpers.NewSuiteTest(cfDeploymentHome, testDirectory, experimentalTests)
	suite.EnsureTestCoverage(t)
	suite.ReadmeTest(t)
	suite.InterpolateTest(t)
}
<commit_msg>Add go test for experimental ops.<commit_after>package experimental_test

import (
	"testing"

	"github.com/cf-deployment/units/helpers"
)

const testDirectory = "operations/experimental"

var experimentalTests = map[string]helpers.OpsFileTestParams{
	"add-credhub-lb.yml":         {},
	"add-deployment-updater.yml": {},
	"add-deployment-updater-external-db.yml": {
		Ops:       []string{"add-deployment-updater.yml", "../use-external-dbs.yml", "add-deployment-updater-external-db.yml"},
		VarsFiles: []string{"../example-vars-files/vars-use-external-dbs.yml"},
	},
	"add-deployment-updater-postgres.yml": {
		Ops: []string{"add-deployment-updater.yml", "../use-postgres.yml", "add-deployment-updater-postgres.yml"},
	},
	"add-metric-store.yml": {},
	"add-syslog-agent.yml": {},
	"add-syslog-agent-windows1803.yml": {
		Ops: []string{"../windows1803-cell.yml", "add-syslog-agent.yml", "add-syslog-agent-windows1803.yml"},
	},
	"add-system-metrics-agent.yml": {},
	"add-system-metrics-agent-windows1803.yml": {
		Ops: []string{"../windows1803-cell.yml", "add-system-metrics-agent.yml", "add-system-metrics-agent-windows1803.yml"},
	},
	"deploy-forwarder-agent.yml":               {},
	"disable-interpolate-service-bindings.yml": {},
	"enable-bpm-garden.yml":                    {},
	"enable-iptables-logger.yml":               {},
	"enable-mysql-tls.yml":                     {},
	"enable-nfs-volume-service-credhub.yml":    {},
	"enable-oci-phase-1.yml":                   {},
	"enable-routing-integrity-windows1803.yml": {
		Ops: []string{"../windows1803-cell.yml", "enable-routing-integrity-windows1803.yml"},
	},
	"enable-routing-integrity-windows2016.yml": {
		Ops: []string{"../windows2016-cell.yml", "enable-routing-integrity-windows2016.yml"},
	},
	"enable-smb-volume-service.yml":            {},
	"enable-suspect-actual-lrp-generation.yml": {},
	"enable-tls-cloud-controller-postgres.yml": {
		Ops: []string{"../use-postgres.yml", "enable-tls-cloud-controller-postgres.yml"},
	},
	"enable-traffic-to-internal-networks.yml":  {},
	"fast-deploy-with-downtime-and-danger.yml": {},
	"infrastructure-metrics.yml":               {},
	"migrate-nfsbroker-mysql-to-credhub.yml": {
		Ops:       []string{"../enable-nfs-volume-service.yml", "migrate-nfsbroker-mysql-to-credhub.yml"},
		VarsFiles: []string{"../example-vars-files/vars-migrate-nfsbroker-mysql-to-credhub.yml"},
	},
	"perm-service.yml": {
		Ops:  []string{"enable-mysql-tls.yml", "perm-service.yml"},
		Vars: []string{"perm_uaa_clients_cc_perm_secret=perm_secret", "perm_uaa_clients_perm_monitor_secret=perm_monitor_secret"},
	},
	"perm-service-with-pxc-release.yml": {
		Ops:  []string{"perm-service.yml", "../use-pxc.yml", "perm-service-with-pxc-release.yml"},
		Vars: []string{"perm_uaa_clients_cc_perm_secret=perm_secret", "perm_uaa_clients_perm_monitor_secret=perm_monitor_secret"},
	},
	"perm-service-with-tcp-routing.yml": {
		Ops:  []string{"perm-service.yml", "../use-pxc.yml", "perm-service-with-pxc-release.yml", "perm-service-with-tcp-routing.yml"},
		Vars: []string{"perm_uaa_clients_cc_perm_secret=perm_secret", "perm_uaa_clients_perm_monitor_secret=perm_monitor_secret"},
	},
	"rootless-containers.yml": {},
	"set-cpu-weight.yml":      {},
	"set-cpu-weight-windows1803.yml": {
		Ops: []string{"../windows1803-cell.yml"},
	},
	"set-cpu-weight-windows2019.yml": {
		Ops: []string{"../windows2019-cell.yml"},
	},
	"use-compiled-releases-windows.yml": {
		Ops: []string{"../use-compiled-releases.yml", "../windows2012R2-cell.yml", "use-compiled-releases-windows.yml"},
	},
	"use-create-swap-delete-vm-strategy.yml":          {},
	"use-logcache-for-cloud-controller-app-stats.yml": {},
	"use-native-garden-runc-runner.yml":               {},
	"windows-component-syslog-ca.yml": {
		Ops:       []string{"windows-enable-component-syslog.yml", "windows-component-syslog-ca.yml"},
		VarsFiles: []string{"../addons/example-vars-files/vars-enable-component-syslog.yml"},
	},
	"windows-enable-component-syslog.yml": {
		Ops:       []string{"windows-enable-component-syslog.yml"},
		VarsFiles: []string{"../addons/example-vars-files/vars-enable-component-syslog.yml"},
	},
}

func TestExperimental(t *testing.T) {
	cfDeploymentHome, err := helpers.SetPath()
	if err != nil {
		t.Fatalf("setup: %v", err)
	}

	suite := helpers.NewSuiteTest(cfDeploymentHome, testDirectory, experimentalTests)
	suite.EnsureTestCoverage(t)
	suite.ReadmeTest(t)
	suite.InterpolateTest(t)
}
<|endoftext|>


<commit_before>package helpers

import (
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	. "github.com/onsi/gomega/gexec"
	"time"

	"github.com/pivotal-cf-experimental/cf-test-helpers/cf"
)

var AdminUserContext cf.UserContext
var RegularUserContext cf.UserContext

type SuiteContext interface {
	Setup()
	Teardown()

	AdminUserContext() cf.UserContext
	RegularUserContext() cf.UserContext
}

func SetupEnvironment(context SuiteContext) {
	var originalCfHomeDir, currentCfHomeDir string

	BeforeEach(func() {
		AdminUserContext = context.AdminUserContext()
		RegularUserContext = context.RegularUserContext()

		context.Setup()

		cf.AsUser(AdminUserContext, func() {
			setUpSpaceWithUserAccess(RegularUserContext)
		})

		originalCfHomeDir, currentCfHomeDir = cf.InitiateUserContext(RegularUserContext)
		cf.TargetSpace(RegularUserContext)

	})

	AfterEach(func() {
		cf.RestoreUserContext(RegularUserContext, originalCfHomeDir, currentCfHomeDir)

		context.Teardown()
	})
}

func setUpSpaceWithUserAccess(uc cf.UserContext) {
	Eventually(cf.Cf("create-space", "-o", uc.Org, uc.Space), 60*time.Second).Should(Exit(0))
	Eventually(cf.Cf("set-space-role", uc.Username, uc.Org, uc.Space, "SpaceManager"), 60*time.Second).Should(Exit(0))
	Eventually(cf.Cf("set-space-role", uc.Username, uc.Org, uc.Space, "SpaceDeveloper"), 60*time.Second).Should(Exit(0))
	Eventually(cf.Cf("set-space-role", uc.Username, uc.Org, uc.Space, "SpaceAuditor"), 60*time.Second).Should(Exit(0))
}
<commit_msg>Temporarily prevent user/org deletion after testing for debugging purposes<commit_after>package helpers

import (
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	. "github.com/onsi/gomega/gexec"
	"time"

	"github.com/pivotal-cf-experimental/cf-test-helpers/cf"
)

var AdminUserContext cf.UserContext
var RegularUserContext cf.UserContext

type SuiteContext interface {
	Setup()
	Teardown()

	AdminUserContext() cf.UserContext
	RegularUserContext() cf.UserContext
}

func SetupEnvironment(context SuiteContext) {
	var originalCfHomeDir, currentCfHomeDir string

	BeforeEach(func() {
		AdminUserContext = context.AdminUserContext()
		RegularUserContext = context.RegularUserContext()

		context.Setup()

		cf.AsUser(AdminUserContext, func() {
			setUpSpaceWithUserAccess(RegularUserContext)
		})

		originalCfHomeDir, currentCfHomeDir = cf.InitiateUserContext(RegularUserContext)
		cf.TargetSpace(RegularUserContext)

	})

	AfterEach(func() {
		cf.RestoreUserContext(RegularUserContext, originalCfHomeDir, currentCfHomeDir)

		//		context.Teardown()
	})
}

func setUpSpaceWithUserAccess(uc cf.UserContext) {
	Eventually(cf.Cf("create-space", "-o", uc.Org, uc.Space), 60*time.Second).Should(Exit(0))
	Eventually(cf.Cf("set-space-role", uc.Username, uc.Org, uc.Space, "SpaceManager"), 60*time.Second).Should(Exit(0))
	Eventually(cf.Cf("set-space-role", uc.Username, uc.Org, uc.Space, "SpaceDeveloper"), 60*time.Second).Should(Exit(0))
	Eventually(cf.Cf("set-space-role", uc.Username, uc.Org, uc.Space, "SpaceAuditor"), 60*time.Second).Should(Exit(0))
}
<|endoftext|>


<commit_before><commit_msg>Pretty much working solution. some kinds to work out.<commit_after><|endoftext|>


<commit_before>package gedcom

import (
	"errors"
	"io"
	"strings"

	"github.com/MJKWoolnough/parser"
)

const (
	alpha       = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz_"
	delim       = " "
	digit       = "0123456789"
	alphanum    = alpha + digit
	otherchar   = "!\"$%&'()*+,-./:;<=>?[\\]^`{|}~" // + "\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9\xaa\xab\xac\xad\xae\xaf\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee\xef\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd\xfe" // ANSEL
	non_at      = alpha + digit + otherchar + " #"
	terminators = "\r\n"
	any_char    = alpha + digit + otherchar + "# @"
	levelIgnore = terminators + delim + "	"
	invalidchar = "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\xab\xac\xad\x1e\x1f\x7f" + terminators + "@"
)

type tokenType uint8

const (
	tokenError tokenType = iota
	tokenLevel
	tokenXref
	tokenTag
	tokenPointer
	tokenLine
	tokenEndLine
	tokenDone
)

type token struct {
	typ  tokenType
	data string
}

type stateFn func() (token, stateFn)

type tokeniser struct {
	p     parser.Parser
	state stateFn
	err   error
	options
}

func newTokeniser(r io.Reader, o options) *tokeniser {
	t := &tokeniser{
		p:       parser.NewReaderParser(r),
		options: o,
	}
	t.state = t.level
	return t
}

func (t *tokeniser) GetToken() (token, error) {
	if t.err == io.EOF {
		return token{tokenDone, ""}, io.EOF
	}
	var tk token
	tk, t.state = t.state()
	if t.err == io.EOF {
		if tk.typ == tokenError {
			t.err = io.ErrUnexpectedEOF
		} else {
			return tk, nil
		}
	}
	return tk, t.err
}

func (t *tokeniser) level() (token, stateFn) {
	t.p.AcceptRun(levelIgnore)
	t.p.Get()
	if t.p.Peek() == -1 {
		return t.done()
	}
	if !t.p.Accept(digit) {
		t.err = ErrInvalidLevel
		return t.errorfn()
	}
	t.p.AcceptRun(digit)
	if !t.p.Accept(delim) {
		t.err = ErrMissingDelim
		return t.errorfn()
	}
	return token{
		tokenLevel,
		strings.TrimSpace(t.p.Get()),
	}, t.optionalXrefID
}

func (t *tokeniser) optionalXrefID() (token, stateFn) {
	if t.p.Peek() == '@' {
		return t.xrefID()
	}
	return t.tag()
}

func (t *tokeniser) readPointer() (string, error) {
	if !t.p.Accept(alphanum) {
		return "", ErrInvalidPointer
	}
	t.p.AcceptRun(non_at)
	if !t.p.Accept("@") {
		return "", ErrInvalidPointer
	}
	return t.p.Get(), nil
}

func (t *tokeniser) xrefID() (token, stateFn) {
	t.p.Accept("@")
	pointer, err := t.readPointer()
	if err != nil {
		t.err = err
		return t.errorfn()
	}
	if !t.p.Accept(delim) {
		t.err = ErrMissingDelim
		return t.errorfn()
	}
	t.p.Get()
	return token{
		tokenXref,
		strings.Trim(pointer, "@"),
	}, t.tag
}

func (t *tokeniser) tag() (token, stateFn) {
	if !t.p.Accept(alphanum) {
		t.err = ErrInvalidTag
		return t.errorfn()
	}
	t.p.AcceptRun(alphanum)
	tag := t.p.Get()
	if t.p.Accept(delim) {
		t.p.Get()
		return token{
			tokenTag,
			tag,
		}, t.lineValue
	}
	next := t.endLine
	if t.p.Peek() == -1 {
		next = t.done
	} else {
		if !t.p.Accept(terminators) {
			t.err = ErrInvalidTag
			return t.errorfn()
		}
		t.p.AcceptRun(terminators)
		t.p.Get()
	}
	return token{
		tokenTag,
		tag,
	}, next
}

func (t *tokeniser) endLine() (token, stateFn) {
	return token{
		tokenEndLine,
		"",
	}, t.level
}

func (t *tokeniser) lineValue() (token, stateFn) {
	if t.p.Peek() == '@' {
		t.p.Accept("@")
		if t.p.Peek() != '@' {
			pointer, err := t.readPointer()
			if err != nil {
				if !t.allowInvalidEscape {
					t.err = err
					return t.errorfn()
				}
			} else {
				t.p.AcceptRun(terminators)
				t.p.Get()
				return token{
					tokenPointer,
					pointer,
				}, t.level
			}
		}
	}
	next := t.level
	for {
		for {
			if t.allowUnknownCharset && t.p.Except(invalidchar) {
				t.p.ExceptRun(invalidchar)
			} else if t.p.Accept(non_at) {
				t.p.AcceptRun(non_at)
			} else if t.p.Accept("@") {
				if t.allowInvalidEscape || t.p.Accept("@") {
					continue
				}
				if !t.p.Accept("#") {
					t.err = ErrBadEscape
					return t.errorfn()
				}
				if t.allowUnknownCharset {
					t.p.ExceptRun(invalidchar)
				} else {
					t.p.AcceptRun(non_at)
				}
				if !t.p.Accept("@") {
					t.err = ErrBadEscape
					return t.errorfn()
				}
			} else {
				break
			}
		}
		p := t.p.Peek()
		if p == -1 {
			next = t.done
			break
		}
		if strings.ContainsRune(terminators, p) {
			t.p.AcceptRun(terminators)
			if !t.allowTerminatorsInValue {
				break
			}
			p = t.p.Peek()
			if p == -1 {
				next = t.done
				break
			}
			if strings.ContainsRune(digit, p) {
				break
			}
		} else {
			if !t.allowInvalidChars {
				t.err = ErrBadChar
				return t.errorfn()
			}
			t.p.Except("")
		}
	}
	return token{
		tokenLine,
		strings.TrimSpace(t.p.Get()),
	}, next
}

func (t *tokeniser) done() (token, stateFn) {
	t.err = io.EOF
	return token{
		tokenDone,
		"",
	}, t.done
}

func (t *tokeniser) errorfn() (token, stateFn) {
	return token{
		tokenError,
		t.err.Error(),
	}, t.errorfn
}

// Errors
var (
	ErrInvalidLevel   = errors.New("invalid level num")
	ErrMissingDelim   = errors.New("missing delminitator")
	ErrInvalidPointer = errors.New("invalid pointer string")
	ErrInvalidTag     = errors.New("invalid tag")
	ErrBadEscape      = errors.New("bad escape sequence")
	ErrBadChar        = errors.New("bad character")
)
<commit_msg>Removed the @ symbols from the IDs<commit_after>package gedcom

import (
	"errors"
	"io"
	"strings"

	"github.com/MJKWoolnough/parser"
)

const (
	alpha       = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz_"
	delim       = " "
	digit       = "0123456789"
	alphanum    = alpha + digit
	otherchar   = "!\"$%&'()*+,-./:;<=>?[\\]^`{|}~" // + "\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9\xaa\xab\xac\xad\xae\xaf\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee\xef\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd\xfe" // ANSEL
	non_at      = alpha + digit + otherchar + " #"
	terminators = "\r\n"
	any_char    = alpha + digit + otherchar + "# @"
	levelIgnore = terminators + delim + "	"
	invalidchar = "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\xab\xac\xad\x1e\x1f\x7f" + terminators + "@"
)

type tokenType uint8

const (
	tokenError tokenType = iota
	tokenLevel
	tokenXref
	tokenTag
	tokenPointer
	tokenLine
	tokenEndLine
	tokenDone
)

type token struct {
	typ  tokenType
	data string
}

type stateFn func() (token, stateFn)

type tokeniser struct {
	p     parser.Parser
	state stateFn
	err   error
	options
}

func newTokeniser(r io.Reader, o options) *tokeniser {
	t := &tokeniser{
		p:       parser.NewReaderParser(r),
		options: o,
	}
	t.state = t.level
	return t
}

func (t *tokeniser) GetToken() (token, error) {
	if t.err == io.EOF {
		return token{tokenDone, ""}, io.EOF
	}
	var tk token
	tk, t.state = t.state()
	if t.err == io.EOF {
		if tk.typ == tokenError {
			t.err = io.ErrUnexpectedEOF
		} else {
			return tk, nil
		}
	}
	return tk, t.err
}

func (t *tokeniser) level() (token, stateFn) {
	t.p.AcceptRun(levelIgnore)
	t.p.Get()
	if t.p.Peek() == -1 {
		return t.done()
	}
	if !t.p.Accept(digit) {
		t.err = ErrInvalidLevel
		return t.errorfn()
	}
	t.p.AcceptRun(digit)
	if !t.p.Accept(delim) {
		t.err = ErrMissingDelim
		return t.errorfn()
	}
	return token{
		tokenLevel,
		strings.TrimSpace(t.p.Get()),
	}, t.optionalXrefID
}

func (t *tokeniser) optionalXrefID() (token, stateFn) {
	if t.p.Peek() == '@' {
		return t.xrefID()
	}
	return t.tag()
}

func (t *tokeniser) readPointer() (string, error) {
	if !t.p.Accept(alphanum) {
		return "", ErrInvalidPointer
	}
	t.p.AcceptRun(non_at)
	if !t.p.Accept("@") {
		return "", ErrInvalidPointer
	}
	pointer := t.p.Get()
	return pointer[1 : len(pointer)-1], nil
}

func (t *tokeniser) xrefID() (token, stateFn) {
	t.p.Accept("@")
	pointer, err := t.readPointer()
	if err != nil {
		t.err = err
		return t.errorfn()
	}
	if !t.p.Accept(delim) {
		t.err = ErrMissingDelim
		return t.errorfn()
	}
	t.p.Get()
	return token{
		tokenXref,
		strings.Trim(pointer, "@"),
	}, t.tag
}

func (t *tokeniser) tag() (token, stateFn) {
	if !t.p.Accept(alphanum) {
		t.err = ErrInvalidTag
		return t.errorfn()
	}
	t.p.AcceptRun(alphanum)
	tag := t.p.Get()
	if t.p.Accept(delim) {
		t.p.Get()
		return token{
			tokenTag,
			tag,
		}, t.lineValue
	}
	next := t.endLine
	if t.p.Peek() == -1 {
		next = t.done
	} else {
		if !t.p.Accept(terminators) {
			t.err = ErrInvalidTag
			return t.errorfn()
		}
		t.p.AcceptRun(terminators)
		t.p.Get()
	}
	return token{
		tokenTag,
		tag,
	}, next
}

func (t *tokeniser) endLine() (token, stateFn) {
	return token{
		tokenEndLine,
		"",
	}, t.level
}

func (t *tokeniser) lineValue() (token, stateFn) {
	if t.p.Peek() == '@' {
		t.p.Accept("@")
		if t.p.Peek() != '@' {
			pointer, err := t.readPointer()
			if err != nil {
				if !t.allowInvalidEscape {
					t.err = err
					return t.errorfn()
				}
			} else {
				t.p.AcceptRun(terminators)
				t.p.Get()
				return token{
					tokenPointer,
					pointer,
				}, t.level
			}
		}
	}
	next := t.level
	for {
		for {
			if t.allowUnknownCharset && t.p.Except(invalidchar) {
				t.p.ExceptRun(invalidchar)
			} else if t.p.Accept(non_at) {
				t.p.AcceptRun(non_at)
			} else if t.p.Accept("@") {
				if t.allowInvalidEscape || t.p.Accept("@") {
					continue
				}
				if !t.p.Accept("#") {
					t.err = ErrBadEscape
					return t.errorfn()
				}
				if t.allowUnknownCharset {
					t.p.ExceptRun(invalidchar)
				} else {
					t.p.AcceptRun(non_at)
				}
				if !t.p.Accept("@") {
					t.err = ErrBadEscape
					return t.errorfn()
				}
			} else {
				break
			}
		}
		p := t.p.Peek()
		if p == -1 {
			next = t.done
			break
		}
		if strings.ContainsRune(terminators, p) {
			t.p.AcceptRun(terminators)
			if !t.allowTerminatorsInValue {
				break
			}
			p = t.p.Peek()
			if p == -1 {
				next = t.done
				break
			}
			if strings.ContainsRune(digit, p) {
				break
			}
		} else {
			if !t.allowInvalidChars {
				t.err = ErrBadChar
				return t.errorfn()
			}
			t.p.Except("")
		}
	}
	return token{
		tokenLine,
		strings.TrimSpace(t.p.Get()),
	}, next
}

func (t *tokeniser) done() (token, stateFn) {
	t.err = io.EOF
	return token{
		tokenDone,
		"",
	}, t.done
}

func (t *tokeniser) errorfn() (token, stateFn) {
	return token{
		tokenError,
		t.err.Error(),
	}, t.errorfn
}

// Errors
var (
	ErrInvalidLevel   = errors.New("invalid level num")
	ErrMissingDelim   = errors.New("missing delminitator")
	ErrInvalidPointer = errors.New("invalid pointer string")
	ErrInvalidTag     = errors.New("invalid tag")
	ErrBadEscape      = errors.New("bad escape sequence")
	ErrBadChar        = errors.New("bad character")
)
<|endoftext|>


<commit_before>package mail

import (
	"bytes"
	"fmt"
	"html/template"
	"net/smtp"
	"strings"

	"github.com/cosiner/gohper/bytes2"
	"github.com/cosiner/gohper/errors"
	"github.com/cosiner/gohper/strings2"
	"github.com/cosiner/gohper/unsafe2"
)

const (
	ErrNoTemplate = errors.Err("no template for this type")
)

type mailTemplate struct {
	Subject string
	*template.Template
}

type Mail struct {
	From    string
	To      []string
	Subject string

	Type string
	Data interface{}

	RawContent string
}

type Mailer struct {
	PrintMail bool

	addr     string
	auth     smtp.Auth
	username string

	Templates  map[string]mailTemplate
	bufferPool bytes2.Pool
}

func NewMailer(username, password, addr string) *Mailer {
	mailer := &Mailer{
		addr:       addr,
		username:   username,
		bufferPool: bytes2.NewSyncPool(1024, false),
	}
	auth := smtp.PlainAuth("", username, password, strings.Split(addr, ":")[0])
	mailer.auth = auth
	mailer.Templates = make(map[string]mailTemplate)

	return mailer
}

func (m *Mailer) AddTemplateFile(typ, filename, subject string) error {
	t, err := template.ParseFiles(filename)
	if err != nil {
		return err
	}

	if typ == "" {
		typ = strings.Split(filename, ".")[0]
	}
	m.Templates[typ] = mailTemplate{
		Subject:  subject,
		Template: t,
	}

	return nil
}

func (m *Mailer) Send(mail *Mail) (err error) {
	tmpl, has := m.Templates[mail.Type]
	if !has && mail.RawContent == "" {
		return ErrNoTemplate
	}

	from := mail.From
	if from == "" {
		from = m.username
	}

	buffer := bytes.NewBuffer(m.bufferPool.Get(1024, false))

	buffer.WriteString("To:")
	strings2.WriteStringsToBuffer(buffer, mail.To, ";")

	buffer.WriteString("\r\n")
	buffer.WriteString("From:" + from + "\r\n")

	subject := mail.Subject
	if has && subject == "" {
		subject = tmpl.Subject
	}
	buffer.WriteString("Subject:" + subject + "\r\n")
	buffer.WriteString("Content-Type: text/html;charset=UTF-8\r\n\r\n")
	if mail.RawContent != "" {
		buffer.WriteString(mail.RawContent)
	} else {
		err = tmpl.Execute(buffer, mail.Data)
	}

	data := buffer.Bytes()
	if m.PrintMail {
		fmt.Println(unsafe2.String(data))
	}
	if err == nil {
		err = smtp.SendMail(m.addr, m.auth, from, mail.To, data)
	}
	m.bufferPool.Put(data)

	return
}
<commit_msg>utils/mail: support tls, nickname<commit_after>package mail

import (
	"bytes"
	"crypto/tls"
	"fmt"
	"html/template"
	"net"
	netmail "net/mail"
	"net/smtp"
	"strings"

	"github.com/cosiner/gohper/bytes2"
	"github.com/cosiner/gohper/errors"
	"github.com/cosiner/gohper/strings2"
	"github.com/cosiner/gohper/unsafe2"
)

const (
	ErrNoTemplate = errors.Err("no template for this type")
)

type mailTemplate struct {
	Subject string
	*template.Template
}

type Mail struct {
	Sender  string
	From    string
	To      []string
	Subject string

	Type string
	Data interface{}

	RawContent string
}

type Mailer struct {
	PrintMail bool
	DoHelo    bool

	addr string
	host string
	auth smtp.Auth

	from   string
	sender string

	Templates  map[string]mailTemplate
	bufferPool bytes2.Pool
	tls        bool
}

func NewMailer(from, sender, username, password, addr string, tls bool) (*Mailer, error) {
	host, _, err := net.SplitHostPort(addr)
	if err != nil {
		return nil, err
	}
	if sender == "" {
		sender = from
	}
	mailer := &Mailer{
		addr:       addr,
		host:       host,
		from:       from,
		sender:     sender,
		bufferPool: bytes2.NewSyncPool(1024, false),
	}
	auth := smtp.PlainAuth("", username, password, strings.Split(addr, ":")[0])
	mailer.auth = auth
	mailer.Templates = make(map[string]mailTemplate)
	mailer.tls = tls
	return mailer, nil
}

func (m *Mailer) AddTemplateFile(typ, filename, subject string) error {
	t, err := template.ParseFiles(filename)
	if err != nil {
		return err
	}

	if typ == "" {
		typ = strings.Split(filename, ".")[0]
	}
	m.Templates[typ] = mailTemplate{
		Subject:  subject,
		Template: t,
	}

	return nil
}

func (m *Mailer) Send(mail *Mail) (err error) {
	tmpl, has := m.Templates[mail.Type]
	if !has && mail.RawContent == "" {
		return ErrNoTemplate
	}

	from := mail.From
	if from == "" {
		from = m.from
	}
	sender := mail.Sender
	if sender == "" {
		sender = m.sender
	}

	buffer := bytes.NewBuffer(m.bufferPool.Get(1024, false))

	buffer.WriteString("To:")
	strings2.WriteStringsToBuffer(buffer, mail.To, ";")

	buffer.WriteString("\r\n")
	nm := netmail.Address{
		Address: from,
		Name:    sender,
	}
	buffer.WriteString("From:" + nm.String() + "\r\n")

	subject := mail.Subject
	if has && subject == "" {
		subject = tmpl.Subject
	}
	buffer.WriteString("Subject:" + subject + "\r\n")
	buffer.WriteString("Content-Type: text/html;charset=UTF-8\r\n\r\n")
	if mail.RawContent != "" {
		buffer.WriteString(mail.RawContent)
	} else {
		err = tmpl.Execute(buffer, mail.Data)
	}

	data := buffer.Bytes()
	if m.PrintMail {
		fmt.Println(unsafe2.String(data))
	}
	if err == nil {
		err = m.send(from, mail.To, data)
	}
	m.bufferPool.Put(data)

	return
}

func (m *Mailer) send(from string, to []string, msg []byte) error {
	if !m.tls {
		return smtp.SendMail(m.addr, m.auth, from, to, msg)
	}

	conn, err := tls.Dial("tcp", m.addr, nil)
	if err != nil {
		return err
	}
	c, err := smtp.NewClient(conn, m.host)
	if err != nil {
		conn.Close()
		return err
	}
	defer c.Close()

	if m.DoHelo {
		err = c.Hello("")
		if err != nil {
			return err
		}
	}
	if m.auth != nil {
		if ok, _ := c.Extension("AUTH"); ok {
			if err = c.Auth(m.auth); err != nil {
				return err
			}
		}
	}

	if err = c.Mail(from); err != nil {
		return err
	}
	for _, addr := range to {
		if err = c.Rcpt(addr); err != nil {
			return err
		}
	}
	w, err := c.Data()
	if err != nil {
		return err
	}
	_, err = w.Write(msg)
	if err != nil {
		return err
	}
	err = w.Close()
	if err != nil {
		return err
	}
	return c.Quit()
}
<|endoftext|>


<commit_before>package object

import (
	"encoding/json"
	"fmt"
	"strings"
	"time"

	"github.com/jmoiron/sqlx"

	"github.com/tapglue/snaas/platform/flake"
	"github.com/tapglue/snaas/platform/pg"
)

const (
	orderNone ordering = iota
	orderCreatedAt
)

const (
	pgInsertObject = `INSERT INTO %s.objects(json_data) VALUES($1)`
	pgUpdateObject = `UPDATE %s.objects SET json_data = $1
		WHERE (json_data->>'id')::BIGINT = $2::BIGINT`
	pgDeleteObject = `DELETE FROM %s.objects
		WHERE (json_data->>'id')::BIGINT = $1::BIGINT`

	pgCountObjects = `SELECT count(json_data) FROM %s.objects
		%s`
	pgListObjects = `SELECT json_data FROM %s.objects
		%s`

	pgClauseBefore     = `(json_data->>'created_at') < ?`
	pgClauseDeleted    = `(json_data->>'deleted')::BOOL = ?::BOOL`
	pgClauseExternalID = `(json_data->>'external_id')::TEXT IN (?)`
	pgClauseID         = `(json_data->>'id')::BIGINT = ?::BIGINT`
	pgClauseObjectID   = `(json_data->>'object_id')::BIGINT IN (?)`
	pgClauseOwnerID    = `(json_data->>'owner_id')::BIGINT IN (?)`
	pgClauseOwned      = `(json_data->>'owned')::BOOL = ?::BOOL`
	pgClauseTags       = `(json_data->'tags')::JSONB @> '[%s]'`
	pgClauseType       = `(json_data->>'type')::TEXT IN (?)`
	pgClauseVisibility = `(json_data->>'visibility')::INT IN (?)`
	pgOrderCreatedAt   = `ORDER BY json_data->>'created_at' DESC`

	pgCreateSchema = `CREATE SCHEMA IF NOT EXISTS %s`
	pgCreateTable  = `CREATE TABLE IF NOT EXISTS %s.objects
		(json_data JSONB NOT NULL)`

	pgCreateIndexCreatedAt = `CREATE INDEX %s ON %s.objects
		USING btree ((json_data->>'created_at') DESC)`
	pgCreateIndexExternalID = `CREATE INDEX %s ON %s.objects
		USING btree (((json_data->>'external_id')::TEXT))`
	pgCreateIndexID = `CREATE INDEX %s ON %s.objects
		USING btree (((json_data->>'id')::BIGINT))`
	pgCreateIndexObjectID = `CREATE INDEX %s ON %s.objects
		USING btree (((json_data->>'object_id')::BIGINT))`
	pgCreateIndexOwnerID = `CREATE INDEX %s ON %s.objects
		USING btree (((json_data->>'owner_id')::BIGINT))`
	pgCreateIndexOwned = `CREATE INDEX %s ON %s.objects
		USING btree (((json_data->>'owned')::BOOL))`
	pgCreateIndexTags = `CREATE INDEX %s ON %s.objects
		USING gin ((json_data->'tags'))`
	pgCreateIndexType = `CREATE INDEX %s ON %s.objects
		USING btree (((json_data->>'type')::TEXT))`
	pgCreateIndexVisibility = `CREATE INDEX %s ON %s.objects
		USING btree (((json_data->>'visibility')::INT))`

	pgDropTable = `DROP TABLE IF EXISTS %s.objects`
)

type ordering int

type pgService struct {
	db *sqlx.DB
}

// PostgresService returns a Postgres based Service implementation.
func PostgresService(db *sqlx.DB) Service {
	return &pgService{
		db: db,
	}
}

func (s *pgService) Count(ns string, opts QueryOptions) (int, error) {
	where, params, err := convertOpts(opts, orderNone)
	if err != nil {
		return 0, err
	}

	return s.countObjects(ns, where, params...)
}

func (s *pgService) Put(ns string, object *Object) (*Object, error) {
	var (
		now   = time.Now().UTC()
		query = pgUpdateObject

		params []interface{}
	)

	if err := object.Validate(); err != nil {
		return nil, err
	}

	if object.ObjectID != 0 {
		os, err := s.Query(ns, QueryOptions{
			ID: &object.ObjectID,
		})
		if err != nil {
			return nil, err
		}

		if len(os) != 1 {
			return nil, ErrMissingReference
		}
	}

	if object.ID != 0 {
		params = []interface{}{
			object.ID,
		}

		os, err := s.Query(ns, QueryOptions{
			ID: &object.ID,
		})
		if err != nil {
			return nil, err
		}

		if len(os) == 0 {
			return nil, ErrNotFound
		}

		object.CreatedAt = os[0].CreatedAt
	} else {
		id, err := flake.NextID(flakeNamespace(ns))
		if err != nil {
			return nil, err
		}

		if object.CreatedAt.IsZero() {
			object.CreatedAt = now
		} else {
			object.CreatedAt = object.CreatedAt.UTC()
		}

		object.ID = id
		query = pgInsertObject
	}

	object.UpdatedAt = now

	data, err := json.Marshal(object)
	if err != nil {
		return nil, err
	}

	params = append([]interface{}{data}, params...)

	_, err = s.db.Exec(wrapNamespace(query, ns), params...)
	if err != nil {
		if pg.IsRelationNotFound(pg.WrapError(err)) {
			if err := s.Setup(ns); err != nil {
				return nil, err
			}
			if _, err := s.db.Exec(wrapNamespace(query, ns), params...); err != nil {
				return nil, err
			}
		} else {
			return nil, err
		}
	}

	return object, nil
}

func (s *pgService) Query(ns string, opts QueryOptions) (List, error) {
	where, params, err := convertOpts(opts, orderCreatedAt)
	if err != nil {
		return nil, err
	}

	return s.listObjects(ns, where, params...)
}

func (s *pgService) Setup(ns string) error {
	qs := []string{
		wrapNamespace(pgCreateSchema, ns),
		wrapNamespace(pgCreateTable, ns),
		pg.GuardIndex(ns, "object_created_at", pgCreateIndexCreatedAt),
		pg.GuardIndex(ns, "object_external_id", pgCreateIndexExternalID),
		pg.GuardIndex(ns, "object_id", pgCreateIndexID),
		pg.GuardIndex(ns, "object_object_id", pgCreateIndexObjectID),
		pg.GuardIndex(ns, "object_owned", pgCreateIndexOwned),
		pg.GuardIndex(ns, "object_owned_id", pgCreateIndexOwnerID),
		pg.GuardIndex(ns, "object_tags", pgCreateIndexTags),
		pg.GuardIndex(ns, "object_type", pgCreateIndexType),
		pg.GuardIndex(ns, "object_visibility", pgCreateIndexVisibility),
	}

	for _, query := range qs {
		_, err := s.db.Exec(query)
		if err != nil {
			return fmt.Errorf("query (%s): %s", query, err)
		}
	}

	return nil
}

func (s *pgService) Teardown(namespace string) error {
	qs := []string{
		fmt.Sprintf(pgDropTable, namespace),
	}

	for _, query := range qs {
		_, err := s.db.Exec(query)
		if err != nil {
			return fmt.Errorf("query (%s): %s", query, err)
		}
	}

	return nil
}

func (s *pgService) countObjects(
	ns, where string,
	params ...interface{},
) (int, error) {
	var (
		count = 0
		query = fmt.Sprintf(pgCountObjects, ns, where)
	)

	err := s.db.Get(&count, query, params...)
	if err != nil {
		if pg.IsRelationNotFound(pg.WrapError(err)) {
			if err := s.Setup(ns); err != nil {
				return 0, err
			}

			err = s.db.Get(&count, query, params...)
		} else {
			return 0, err
		}
	}

	return count, err
}

func (s *pgService) listObjects(
	ns, where string,
	params ...interface{},
) (List, error) {
	query := fmt.Sprintf(pgListObjects, ns, where)

	rows, err := s.db.Query(query, params...)
	if err != nil {
		if pg.IsRelationNotFound(pg.WrapError(err)) {
			if err := s.Setup(ns); err != nil {
				return nil, err
			}

			rows, err = s.db.Query(query, params...)
			if err != nil {
				return nil, err
			}
		} else {
			return nil, err
		}
	}
	defer rows.Close()

	os := List{}

	for rows.Next() {
		var (
			object = &Object{}

			raw []byte
		)

		err := rows.Scan(&raw)
		if err != nil {
			return nil, err
		}

		err = json.Unmarshal(raw, object)
		if err != nil {
			return nil, err
		}

		os = append(os, object)
	}

	if err := rows.Err(); err != nil {
		return nil, err
	}

	return os, nil
}

func convertOpts(opts QueryOptions, order ordering) (string, []interface{}, error) {
	var (
		clauses = []string{
			pgClauseDeleted,
		}
		params = []interface{}{
			opts.Deleted,
		}
	)

	if !opts.Before.IsZero() {
		clauses = append(clauses, pgClauseBefore)
		params = append(params, opts.Before.UTC().Format(time.RFC3339Nano))
	}

	if len(opts.ExternalIDs) > 0 {
		ps := []interface{}{}

		for _, id := range opts.ExternalIDs {
			ps = append(ps, id)
		}

		clause, _, err := sqlx.In(pgClauseExternalID, ps)
		if err != nil {
			return "", nil, err
		}

		clauses = append(clauses, clause)
		params = append(params, ps...)
	}

	if opts.ID != nil {
		params = append(params, *opts.ID)
		clauses = append(clauses, pgClauseID)
	}

	if len(opts.OwnerIDs) > 0 {
		ps := []interface{}{}

		for _, id := range opts.OwnerIDs {
			ps = append(ps, id)
		}

		clause, _, err := sqlx.In(pgClauseOwnerID, ps)
		if err != nil {
			return "", nil, err
		}

		clauses = append(clauses, clause)
		params = append(params, ps...)
	}

	if len(opts.ObjectIDs) > 0 {
		ps := []interface{}{}

		for _, id := range opts.ObjectIDs {
			ps = append(ps, id)
		}

		clause, _, err := sqlx.In(pgClauseObjectID, ps)
		if err != nil {
			return "", nil, err
		}

		clauses = append(clauses, clause)
		params = append(params, ps...)
	}

	if opts.Owned != nil {
		clause, _, err := sqlx.In(pgClauseOwned, []interface{}{*opts.Owned})
		if err != nil {
			return "", nil, err
		}

		clauses = append(clauses, clause)
		params = append(params, *opts.Owned)
	}

	if len(opts.Tags) > 0 {
		ts := []string{}

		for _, t := range opts.Tags {
			ts = append(ts, fmt.Sprintf(`"%s"`, t))
		}

		clause := fmt.Sprintf(pgClauseTags, strings.Join(ts, ","))
		clauses = append(clauses, clause)
	}

	if len(opts.Types) > 0 {
		ps := []interface{}{}

		for _, id := range opts.Types {
			ps = append(ps, id)
		}

		clause, _, err := sqlx.In(pgClauseType, ps)
		if err != nil {
			return "", nil, err
		}

		clauses = append(clauses, clause)
		params = append(params, ps...)
	}

	if len(opts.Visibilities) > 0 {
		ps := []interface{}{}

		for _, v := range opts.Visibilities {
			ps = append(ps, v)
		}

		clause, _, err := sqlx.In(pgClauseVisibility, ps)
		if err != nil {
			return "", nil, err
		}

		clauses = append(clauses, clause)
		params = append(params, ps...)
	}

	query := ""

	if len(clauses) > 0 {
		query = sqlx.Rebind(sqlx.DOLLAR, pg.ClausesToWhere(clauses...))
	}

	if !opts.Before.IsZero() && order == orderCreatedAt {
		query = fmt.Sprintf("%s\n%s", query, pgOrderCreatedAt)
	}

	if opts.Limit > 0 {
		query = fmt.Sprintf("%s\nLIMIT %d", query, opts.Limit)
	}

	return query, params, nil
}

func wrapNamespace(query, namespace string) string {
	return fmt.Sprintf(query, namespace)
}
<commit_msg>Remove integrity check in service layer<commit_after>package object

import (
	"encoding/json"
	"fmt"
	"strings"
	"time"

	"github.com/jmoiron/sqlx"

	"github.com/tapglue/snaas/platform/flake"
	"github.com/tapglue/snaas/platform/pg"
)

const (
	orderNone ordering = iota
	orderCreatedAt
)

const (
	pgInsertObject = `INSERT INTO %s.objects(json_data) VALUES($1)`
	pgUpdateObject = `UPDATE %s.objects SET json_data = $1
		WHERE (json_data->>'id')::BIGINT = $2::BIGINT`
	pgDeleteObject = `DELETE FROM %s.objects
		WHERE (json_data->>'id')::BIGINT = $1::BIGINT`

	pgCountObjects = `SELECT count(json_data) FROM %s.objects
		%s`
	pgListObjects = `SELECT json_data FROM %s.objects
		%s`

	pgClauseBefore     = `(json_data->>'created_at') < ?`
	pgClauseDeleted    = `(json_data->>'deleted')::BOOL = ?::BOOL`
	pgClauseExternalID = `(json_data->>'external_id')::TEXT IN (?)`
	pgClauseID         = `(json_data->>'id')::BIGINT = ?::BIGINT`
	pgClauseObjectID   = `(json_data->>'object_id')::BIGINT IN (?)`
	pgClauseOwnerID    = `(json_data->>'owner_id')::BIGINT IN (?)`
	pgClauseOwned      = `(json_data->>'owned')::BOOL = ?::BOOL`
	pgClauseTags       = `(json_data->'tags')::JSONB @> '[%s]'`
	pgClauseType       = `(json_data->>'type')::TEXT IN (?)`
	pgClauseVisibility = `(json_data->>'visibility')::INT IN (?)`
	pgOrderCreatedAt   = `ORDER BY json_data->>'created_at' DESC`

	pgCreateSchema = `CREATE SCHEMA IF NOT EXISTS %s`
	pgCreateTable  = `CREATE TABLE IF NOT EXISTS %s.objects
		(json_data JSONB NOT NULL)`

	pgCreateIndexCreatedAt = `CREATE INDEX %s ON %s.objects
		USING btree ((json_data->>'created_at') DESC)`
	pgCreateIndexExternalID = `CREATE INDEX %s ON %s.objects
		USING btree (((json_data->>'external_id')::TEXT))`
	pgCreateIndexID = `CREATE INDEX %s ON %s.objects
		USING btree (((json_data->>'id')::BIGINT))`
	pgCreateIndexObjectID = `CREATE INDEX %s ON %s.objects
		USING btree (((json_data->>'object_id')::BIGINT))`
	pgCreateIndexOwnerID = `CREATE INDEX %s ON %s.objects
		USING btree (((json_data->>'owner_id')::BIGINT))`
	pgCreateIndexOwned = `CREATE INDEX %s ON %s.objects
		USING btree (((json_data->>'owned')::BOOL))`
	pgCreateIndexTags = `CREATE INDEX %s ON %s.objects
		USING gin ((json_data->'tags'))`
	pgCreateIndexType = `CREATE INDEX %s ON %s.objects
		USING btree (((json_data->>'type')::TEXT))`
	pgCreateIndexVisibility = `CREATE INDEX %s ON %s.objects
		USING btree (((json_data->>'visibility')::INT))`

	pgDropTable = `DROP TABLE IF EXISTS %s.objects`
)

type ordering int

type pgService struct {
	db *sqlx.DB
}

// PostgresService returns a Postgres based Service implementation.
func PostgresService(db *sqlx.DB) Service {
	return &pgService{
		db: db,
	}
}

func (s *pgService) Count(ns string, opts QueryOptions) (int, error) {
	where, params, err := convertOpts(opts, orderNone)
	if err != nil {
		return 0, err
	}

	return s.countObjects(ns, where, params...)
}

func (s *pgService) Put(ns string, object *Object) (*Object, error) {
	var (
		now   = time.Now().UTC()
		query = pgUpdateObject

		params []interface{}
	)

	if err := object.Validate(); err != nil {
		return nil, err
	}

	if object.ID != 0 {
		params = []interface{}{
			object.ID,
		}

		os, err := s.Query(ns, QueryOptions{
			ID: &object.ID,
		})
		if err != nil {
			return nil, err
		}

		if len(os) == 0 {
			return nil, ErrNotFound
		}

		object.CreatedAt = os[0].CreatedAt
	} else {
		id, err := flake.NextID(flakeNamespace(ns))
		if err != nil {
			return nil, err
		}

		if object.CreatedAt.IsZero() {
			object.CreatedAt = now
		} else {
			object.CreatedAt = object.CreatedAt.UTC()
		}

		object.ID = id
		query = pgInsertObject
	}

	object.UpdatedAt = now

	data, err := json.Marshal(object)
	if err != nil {
		return nil, err
	}

	params = append([]interface{}{data}, params...)

	_, err = s.db.Exec(wrapNamespace(query, ns), params...)
	if err != nil {
		if pg.IsRelationNotFound(pg.WrapError(err)) {
			if err := s.Setup(ns); err != nil {
				return nil, err
			}
			if _, err := s.db.Exec(wrapNamespace(query, ns), params...); err != nil {
				return nil, err
			}
		} else {
			return nil, err
		}
	}

	return object, nil
}

func (s *pgService) Query(ns string, opts QueryOptions) (List, error) {
	where, params, err := convertOpts(opts, orderCreatedAt)
	if err != nil {
		return nil, err
	}

	return s.listObjects(ns, where, params...)
}

func (s *pgService) Setup(ns string) error {
	qs := []string{
		wrapNamespace(pgCreateSchema, ns),
		wrapNamespace(pgCreateTable, ns),
		pg.GuardIndex(ns, "object_created_at", pgCreateIndexCreatedAt),
		pg.GuardIndex(ns, "object_external_id", pgCreateIndexExternalID),
		pg.GuardIndex(ns, "object_id", pgCreateIndexID),
		pg.GuardIndex(ns, "object_object_id", pgCreateIndexObjectID),
		pg.GuardIndex(ns, "object_owned", pgCreateIndexOwned),
		pg.GuardIndex(ns, "object_owned_id", pgCreateIndexOwnerID),
		pg.GuardIndex(ns, "object_tags", pgCreateIndexTags),
		pg.GuardIndex(ns, "object_type", pgCreateIndexType),
		pg.GuardIndex(ns, "object_visibility", pgCreateIndexVisibility),
	}

	for _, query := range qs {
		_, err := s.db.Exec(query)
		if err != nil {
			return fmt.Errorf("query (%s): %s", query, err)
		}
	}

	return nil
}

func (s *pgService) Teardown(namespace string) error {
	qs := []string{
		fmt.Sprintf(pgDropTable, namespace),
	}

	for _, query := range qs {
		_, err := s.db.Exec(query)
		if err != nil {
			return fmt.Errorf("query (%s): %s", query, err)
		}
	}

	return nil
}

func (s *pgService) countObjects(
	ns, where string,
	params ...interface{},
) (int, error) {
	var (
		count = 0
		query = fmt.Sprintf(pgCountObjects, ns, where)
	)

	err := s.db.Get(&count, query, params...)
	if err != nil {
		if pg.IsRelationNotFound(pg.WrapError(err)) {
			if err := s.Setup(ns); err != nil {
				return 0, err
			}

			err = s.db.Get(&count, query, params...)
		} else {
			return 0, err
		}
	}

	return count, err
}

func (s *pgService) listObjects(
	ns, where string,
	params ...interface{},
) (List, error) {
	query := fmt.Sprintf(pgListObjects, ns, where)

	rows, err := s.db.Query(query, params...)
	if err != nil {
		if pg.IsRelationNotFound(pg.WrapError(err)) {
			if err := s.Setup(ns); err != nil {
				return nil, err
			}

			rows, err = s.db.Query(query, params...)
			if err != nil {
				return nil, err
			}
		} else {
			return nil, err
		}
	}
	defer rows.Close()

	os := List{}

	for rows.Next() {
		var (
			object = &Object{}

			raw []byte
		)

		err := rows.Scan(&raw)
		if err != nil {
			return nil, err
		}

		err = json.Unmarshal(raw, object)
		if err != nil {
			return nil, err
		}

		os = append(os, object)
	}

	if err := rows.Err(); err != nil {
		return nil, err
	}

	return os, nil
}

func convertOpts(opts QueryOptions, order ordering) (string, []interface{}, error) {
	var (
		clauses = []string{
			pgClauseDeleted,
		}
		params = []interface{}{
			opts.Deleted,
		}
	)

	if !opts.Before.IsZero() {
		clauses = append(clauses, pgClauseBefore)
		params = append(params, opts.Before.UTC().Format(time.RFC3339Nano))
	}

	if len(opts.ExternalIDs) > 0 {
		ps := []interface{}{}

		for _, id := range opts.ExternalIDs {
			ps = append(ps, id)
		}

		clause, _, err := sqlx.In(pgClauseExternalID, ps)
		if err != nil {
			return "", nil, err
		}

		clauses = append(clauses, clause)
		params = append(params, ps...)
	}

	if opts.ID != nil {
		params = append(params, *opts.ID)
		clauses = append(clauses, pgClauseID)
	}

	if len(opts.OwnerIDs) > 0 {
		ps := []interface{}{}

		for _, id := range opts.OwnerIDs {
			ps = append(ps, id)
		}

		clause, _, err := sqlx.In(pgClauseOwnerID, ps)
		if err != nil {
			return "", nil, err
		}

		clauses = append(clauses, clause)
		params = append(params, ps...)
	}

	if len(opts.ObjectIDs) > 0 {
		ps := []interface{}{}

		for _, id := range opts.ObjectIDs {
			ps = append(ps, id)
		}

		clause, _, err := sqlx.In(pgClauseObjectID, ps)
		if err != nil {
			return "", nil, err
		}

		clauses = append(clauses, clause)
		params = append(params, ps...)
	}

	if opts.Owned != nil {
		clause, _, err := sqlx.In(pgClauseOwned, []interface{}{*opts.Owned})
		if err != nil {
			return "", nil, err
		}

		clauses = append(clauses, clause)
		params = append(params, *opts.Owned)
	}

	if len(opts.Tags) > 0 {
		ts := []string{}

		for _, t := range opts.Tags {
			ts = append(ts, fmt.Sprintf(`"%s"`, t))
		}

		clause := fmt.Sprintf(pgClauseTags, strings.Join(ts, ","))
		clauses = append(clauses, clause)
	}

	if len(opts.Types) > 0 {
		ps := []interface{}{}

		for _, id := range opts.Types {
			ps = append(ps, id)
		}

		clause, _, err := sqlx.In(pgClauseType, ps)
		if err != nil {
			return "", nil, err
		}

		clauses = append(clauses, clause)
		params = append(params, ps...)
	}

	if len(opts.Visibilities) > 0 {
		ps := []interface{}{}

		for _, v := range opts.Visibilities {
			ps = append(ps, v)
		}

		clause, _, err := sqlx.In(pgClauseVisibility, ps)
		if err != nil {
			return "", nil, err
		}

		clauses = append(clauses, clause)
		params = append(params, ps...)
	}

	query := ""

	if len(clauses) > 0 {
		query = sqlx.Rebind(sqlx.DOLLAR, pg.ClausesToWhere(clauses...))
	}

	if !opts.Before.IsZero() && order == orderCreatedAt {
		query = fmt.Sprintf("%s\n%s", query, pgOrderCreatedAt)
	}

	if opts.Limit > 0 {
		query = fmt.Sprintf("%s\nLIMIT %d", query, opts.Limit)
	}

	return query, params, nil
}

func wrapNamespace(query, namespace string) string {
	return fmt.Sprintf(query, namespace)
}
<|endoftext|>


<commit_before>package plugin

import (
	"context"
	"encoding/json"
	"fmt"
	"os"
	"os/signal"
	"strings"
	"time"

	"docker.io/go-docker"
	"docker.io/go-docker/api/types"
	"docker.io/go-docker/api/types/container"
	"docker.io/go-docker/api/types/filters"
	"docker.io/go-docker/api/types/mount"
	"docker.io/go-docker/api/types/swarm"
	"github.com/appcelerator/amp/docker/docker/pkg/stdcopy"
)

const (
	InitTimeout       = 10
	ContainerName     = "ampagent"
	ImageName         = "appcelerator/ampagent"
	DockerSocket      = "/var/run/docker.sock"
	DockerSwarmSocket = "/var/run/docker"
	CoreStackName     = "amp"
)

var (
	ContainerLabels = map[string]string{"io.amp.role": "infrastructure"}
)

// RequestOptions stores parameters for the Docker API
type RequestOptions struct {
	InitRequest swarm.InitRequest
	// Node labels
	Labels map[string]string
	// Tag of the ampagent image
	Tag           string
	Registration  string
	Notifications bool
	ForceLeave    bool
	NoLogs        bool
	NoMetrics     bool
	NoProxy       bool
}

type FullSwarmInfo struct {
	Swarm swarm.Swarm
	Node  swarm.Node
}
type ShortSwarmInfo struct {
	SwarmStatus  string `json:"Swarm Status"`
	CoreServices int    `json:"Core Services"`
	UserServices int    `json:"User Services"`
}

// EnsureSwarmExists checks that the Swarm is initialized, and does it if it's not the case
func EnsureSwarmExists(ctx context.Context, c *docker.Client, opts *RequestOptions) error {
	timeout := make(chan bool, 1)
	done := make(chan bool, 1)
	var err error
	// the Init method may freeze if the Docker engine has issues (and it often has)
	go func() {
		time.Sleep(InitTimeout * time.Second)
		timeout <- true
	}()
	go func() {
		_, err = c.SwarmInit(ctx, opts.InitRequest)
		if err != nil {
			// if the swarm is already initialized, ignore the error
			if strings.Contains(fmt.Sprintf("%v", err), "This node is already part of a swarm") {
				err = nil
			} else {
				fmt.Printf("%v\n", err)
			}
		}
		done <- true
	}()
	select {
	case <-done:
		return err
	case <-timeout:
		return fmt.Errorf("Timed out")
	}
}

func LabelNode(ctx context.Context, c *docker.Client, opts *RequestOptions) error {
	node, err := InfoNode(ctx, c)
	if err != nil {
		return err
	}
	version := node.Meta.Version
	nodeSpec := node.Spec
	for label := range opts.Labels {
		node.Spec.Annotations.Labels[label] = opts.Labels[label]
	}
	return c.NodeUpdate(ctx, node.ID, version, nodeSpec)
}

func removeAgent(ctx context.Context, c *docker.Client, cid string, force bool) error {
	return c.ContainerRemove(ctx, cid, types.ContainerRemoveOptions{Force: force})
}

func containerLogs(ctx context.Context, c *docker.Client, id string) {
	reader, err := c.ContainerLogs(ctx, id, types.ContainerLogsOptions{
		ShowStdout: true,
		ShowStderr: true,
		Follow:     true,
	})
	if err != nil {
		return
	}
	defer reader.Close()
	_, err = stdcopy.StdCopy(os.Stdout, os.Stderr, reader)
}

// RunAgent runs the ampagent image to init (action ="install") or destroy (action="uninstall")
func RunAgent(ctx context.Context, c *docker.Client, action string, opts *RequestOptions) error {
	containerName := ContainerName
	image := fmt.Sprintf("%s:%s", ImageName, opts.Tag)
	config := container.Config{
		Image: image,
		Env: []string{
			fmt.Sprintf("TAG=%s", opts.Tag),
			fmt.Sprintf("REGISTRATION=%s", opts.Registration),
			fmt.Sprintf("NOTIFICATIONS=%t", opts.Notifications),
		},
		Labels: ContainerLabels,
		Tty:    false,
	}
	var actionArgs []string
	if opts.NoLogs {
		actionArgs = append(actionArgs, "--no-logs")
	}
	if opts.NoMetrics {
		actionArgs = append(actionArgs, "--no-metrics")
	}
	if opts.NoProxy {
		actionArgs = append(actionArgs, "--no-proxy")
	}
	switch action {
	case "install":
		action = ""
		config.Cmd = actionArgs
	case "uninstall":
		config.Cmd = append([]string{action}, actionArgs...)
	default:
		return fmt.Errorf("action %s is not implemented", action)
	}
	mounts := []mount.Mount{
		{
			Type:   "bind",
			Source: DockerSocket,
			Target: DockerSocket,
		},
		{
			Type:   "bind",
			Source: DockerSwarmSocket,
			Target: DockerSwarmSocket,
		},
	}
	hostConfig := container.HostConfig{
		AutoRemove: false,
		Mounts:     mounts,
	}
	reader, err := c.ImagePull(ctx, image, types.ImagePullOptions{})
	if err != nil {
		// don't exit, if not in the registry we may still want to run the container with a local image
		fmt.Println("ampagent image pull failed, which is expected on a development version")
	} else {
		// wait for the image to be pulled
		data := make([]byte, 1000, 1000)
		for {
			_, err := reader.Read(data)
			if err != nil {
				if err.Error() == "EOF" {
					break
				}
				return err
			}
		}
	}
	r, err := c.ContainerCreate(ctx, &config, &hostConfig, nil, containerName)
	if err != nil {
		return err
	}

	done := make(chan bool, 1)
	interruption := make(chan os.Signal, 1)
	signal.Notify(interruption, os.Interrupt, os.Kill)
	go func() {
		sig := <-interruption
		fmt.Printf("Received signal %s\n", sig.String())
		err = c.ContainerKill(ctx, r.ID, "SIGINT")
		if err != nil {
			fmt.Printf("%v\n", err)
		}
		for i := 0; i < 10; i++ {
			if err := removeAgent(ctx, c, r.ID, false); err == nil {
				break
			}
			time.Sleep(time.Second)
		}
		done <- true
		return
	}()

	if err = c.ContainerStart(ctx, r.ID, types.ContainerStartOptions{}); err != nil {
		_ = removeAgent(ctx, c, r.ID, true)
		return err
	}

	go containerLogs(ctx, c, r.ID)

	go func() {
		for {
			filter := filters.NewArgs()
			filter.Add("id", r.ID)
			l, _ := c.ContainerList(ctx, types.ContainerListOptions{Filters: filter})
			if len(l) == 0 {
				done <- true
				return
			}
			time.Sleep(time.Second)
		}
	}()

	<-done
	_ = removeAgent(ctx, c, r.ID, true)
	return nil
}

// InfoCluster returns the Swarm info
func InfoCluster(ctx context.Context, c *docker.Client) (swarm.Swarm, error) {
	return c.SwarmInspect(ctx)
}

// InfoNode returns the Node info
func InfoNode(ctx context.Context, c *docker.Client) (swarm.Node, error) {
	nodes, err := c.NodeList(ctx, types.NodeListOptions{})
	if len(nodes) != 1 {
		return swarm.Node{}, fmt.Errorf("expected 1 node, got %d", len(nodes))
	}
	node, _, err := c.NodeInspectWithRaw(ctx, nodes[0].ID)
	return node, err
}

// InfoAMPCore returns the number of AMP core services
func InfoAMPCore(ctx context.Context, c *docker.Client) (int, error) {
	var count int
	services, err := c.ServiceList(ctx, types.ServiceListOptions{})
	if err != nil {
		return 0, err
	}
	for _, service := range services {
		if strings.HasPrefix(service.Spec.Name, fmt.Sprintf("%s_", CoreStackName)) {
			count++
		}
	}
	return count, nil
}

// InfoUser returns the number of user services
func InfoUser(ctx context.Context, c *docker.Client) (int, error) {
	var count int
	services, err := c.ServiceList(ctx, types.ServiceListOptions{})
	if err != nil {
		return 0, err
	}
	for _, service := range services {
		if !strings.HasPrefix(service.Spec.Name, CoreStackName) {
			count++
		}
	}
	return count, err
}

func InfoToJSON(status string, csCount int, usCount int) (string, error) {
	// filter the swarm content
	si := ShortSwarmInfo{SwarmStatus: status, CoreServices: csCount, UserServices: usCount}
	j, err := json.Marshal(si)
	if err != nil {
		return "", err
	}
	return string(j), nil
}

// DeleteSwarm starts the delete operation
func DeleteCluster(ctx context.Context, c *docker.Client, opts *RequestOptions) error {
	if opts.ForceLeave {
		return c.SwarmLeave(ctx, true)
	}
	return nil
}

// SwarmNodeStatus returns the swarm status for this node
func SwarmNodeStatus(c *docker.Client) (swarm.LocalNodeState, error) {
	info, err := c.Info(context.Background())
	if err != nil {
		return "", err
	}
	return info.Swarm.LocalNodeState, nil
}
<commit_msg>local plugin: check for exited ampagent containers (#1768)<commit_after>package plugin

import (
	"context"
	"encoding/json"
	"fmt"
	"os"
	"os/signal"
	"strings"
	"time"

	"docker.io/go-docker"
	"docker.io/go-docker/api/types"
	"docker.io/go-docker/api/types/container"
	"docker.io/go-docker/api/types/filters"
	"docker.io/go-docker/api/types/mount"
	"docker.io/go-docker/api/types/swarm"
	"github.com/appcelerator/amp/docker/docker/pkg/stdcopy"
)

const (
	InitTimeout       = 10
	ContainerName     = "ampagent"
	ImageName         = "appcelerator/ampagent"
	DockerSocket      = "/var/run/docker.sock"
	DockerSwarmSocket = "/var/run/docker"
	CoreStackName     = "amp"
)

var (
	ContainerLabels = map[string]string{"io.amp.role": "infrastructure"}
)

// RequestOptions stores parameters for the Docker API
type RequestOptions struct {
	InitRequest swarm.InitRequest
	// Node labels
	Labels map[string]string
	// Tag of the ampagent image
	Tag           string
	Registration  string
	Notifications bool
	ForceLeave    bool
	NoLogs        bool
	NoMetrics     bool
	NoProxy       bool
}

type FullSwarmInfo struct {
	Swarm swarm.Swarm
	Node  swarm.Node
}
type ShortSwarmInfo struct {
	SwarmStatus  string `json:"Swarm Status"`
	CoreServices int    `json:"Core Services"`
	UserServices int    `json:"User Services"`
}

// EnsureSwarmExists checks that the Swarm is initialized, and does it if it's not the case
func EnsureSwarmExists(ctx context.Context, c *docker.Client, opts *RequestOptions) error {
	timeout := make(chan bool, 1)
	done := make(chan bool, 1)
	var err error
	// the Init method may freeze if the Docker engine has issues (and it often has)
	go func() {
		time.Sleep(InitTimeout * time.Second)
		timeout <- true
	}()
	go func() {
		_, err = c.SwarmInit(ctx, opts.InitRequest)
		if err != nil {
			// if the swarm is already initialized, ignore the error
			if strings.Contains(fmt.Sprintf("%v", err), "This node is already part of a swarm") {
				err = nil
			} else {
				fmt.Printf("%v\n", err)
			}
		}
		done <- true
	}()
	select {
	case <-done:
		return err
	case <-timeout:
		return fmt.Errorf("Timed out")
	}
}

func LabelNode(ctx context.Context, c *docker.Client, opts *RequestOptions) error {
	node, err := InfoNode(ctx, c)
	if err != nil {
		return err
	}
	version := node.Meta.Version
	nodeSpec := node.Spec
	for label := range opts.Labels {
		node.Spec.Annotations.Labels[label] = opts.Labels[label]
	}
	return c.NodeUpdate(ctx, node.ID, version, nodeSpec)
}

func removeAgent(ctx context.Context, c *docker.Client, cid string, force bool) error {
	return c.ContainerRemove(ctx, cid, types.ContainerRemoveOptions{Force: force})
}

func containerLogs(ctx context.Context, c *docker.Client, id string) {
	reader, err := c.ContainerLogs(ctx, id, types.ContainerLogsOptions{
		ShowStdout: true,
		ShowStderr: true,
		Follow:     true,
	})
	if err != nil {
		return
	}
	defer reader.Close()
	_, err = stdcopy.StdCopy(os.Stdout, os.Stderr, reader)
}

// RunAgent runs the ampagent image to init (action ="install") or destroy (action="uninstall")
func RunAgent(ctx context.Context, c *docker.Client, action string, opts *RequestOptions) error {
	// first remove any exited or dead container with same name
	// this can happen when Docker crashes while the plugin and the agent are running
	if err := removeExitedAgent(c); err != nil {
		return err
	}
	image := fmt.Sprintf("%s:%s", ImageName, opts.Tag)
	config := container.Config{
		Image: image,
		Env: []string{
			fmt.Sprintf("TAG=%s", opts.Tag),
			fmt.Sprintf("REGISTRATION=%s", opts.Registration),
			fmt.Sprintf("NOTIFICATIONS=%t", opts.Notifications),
		},
		Labels: ContainerLabels,
		Tty:    false,
	}
	var actionArgs []string
	if opts.NoLogs {
		actionArgs = append(actionArgs, "--no-logs")
	}
	if opts.NoMetrics {
		actionArgs = append(actionArgs, "--no-metrics")
	}
	if opts.NoProxy {
		actionArgs = append(actionArgs, "--no-proxy")
	}
	switch action {
	case "install":
		action = ""
		config.Cmd = actionArgs
	case "uninstall":
		config.Cmd = append([]string{action}, actionArgs...)
	default:
		return fmt.Errorf("action %s is not implemented", action)
	}
	mounts := []mount.Mount{
		{
			Type:   "bind",
			Source: DockerSocket,
			Target: DockerSocket,
		},
		{
			Type:   "bind",
			Source: DockerSwarmSocket,
			Target: DockerSwarmSocket,
		},
	}
	hostConfig := container.HostConfig{
		AutoRemove: false,
		Mounts:     mounts,
	}
	reader, err := c.ImagePull(ctx, image, types.ImagePullOptions{})
	if err != nil {
		// don't exit, if not in the registry we may still want to run the container with a local image
		fmt.Println("ampagent image pull failed, which is expected on a development version")
	} else {
		// wait for the image to be pulled
		data := make([]byte, 1000, 1000)
		for {
			_, err := reader.Read(data)
			if err != nil {
				if err.Error() == "EOF" {
					break
				}
				return err
			}
		}
	}
	r, err := c.ContainerCreate(ctx, &config, &hostConfig, nil, ContainerName)
	if err != nil {
		return err
	}

	done := make(chan bool, 1)
	interruption := make(chan os.Signal, 1)
	signal.Notify(interruption, os.Interrupt, os.Kill)
	go func() {
		sig := <-interruption
		fmt.Printf("Received signal %s\n", sig.String())
		err = c.ContainerKill(ctx, r.ID, "SIGINT")
		if err != nil {
			fmt.Printf("%v\n", err)
		}
		for i := 0; i < 10; i++ {
			if err := removeAgent(ctx, c, r.ID, false); err == nil {
				break
			}
			time.Sleep(time.Second)
		}
		done <- true
		return
	}()

	if err = c.ContainerStart(ctx, r.ID, types.ContainerStartOptions{}); err != nil {
		_ = removeAgent(ctx, c, r.ID, true)
		return err
	}

	go containerLogs(ctx, c, r.ID)

	go func() {
		for {
			filter := filters.NewArgs()
			filter.Add("id", r.ID)
			l, _ := c.ContainerList(ctx, types.ContainerListOptions{Filters: filter})
			if len(l) == 0 {
				done <- true
				return
			}
			time.Sleep(time.Second)
		}
	}()

	<-done
	_ = removeAgent(ctx, c, r.ID, true)
	return nil
}

// InfoCluster returns the Swarm info
func InfoCluster(ctx context.Context, c *docker.Client) (swarm.Swarm, error) {
	return c.SwarmInspect(ctx)
}

// InfoNode returns the Node info
func InfoNode(ctx context.Context, c *docker.Client) (swarm.Node, error) {
	nodes, err := c.NodeList(ctx, types.NodeListOptions{})
	if len(nodes) != 1 {
		return swarm.Node{}, fmt.Errorf("expected 1 node, got %d", len(nodes))
	}
	node, _, err := c.NodeInspectWithRaw(ctx, nodes[0].ID)
	return node, err
}

// InfoAMPCore returns the number of AMP core services
func InfoAMPCore(ctx context.Context, c *docker.Client) (int, error) {
	var count int
	services, err := c.ServiceList(ctx, types.ServiceListOptions{})
	if err != nil {
		return 0, err
	}
	for _, service := range services {
		if strings.HasPrefix(service.Spec.Name, fmt.Sprintf("%s_", CoreStackName)) {
			count++
		}
	}
	return count, nil
}

// InfoUser returns the number of user services
func InfoUser(ctx context.Context, c *docker.Client) (int, error) {
	var count int
	services, err := c.ServiceList(ctx, types.ServiceListOptions{})
	if err != nil {
		return 0, err
	}
	for _, service := range services {
		if !strings.HasPrefix(service.Spec.Name, CoreStackName) {
			count++
		}
	}
	return count, err
}

func InfoToJSON(status string, csCount int, usCount int) (string, error) {
	// filter the swarm content
	si := ShortSwarmInfo{SwarmStatus: status, CoreServices: csCount, UserServices: usCount}
	j, err := json.Marshal(si)
	if err != nil {
		return "", err
	}
	return string(j), nil
}

// DeleteSwarm starts the delete operation
func DeleteCluster(ctx context.Context, c *docker.Client, opts *RequestOptions) error {
	if opts.ForceLeave {
		return c.SwarmLeave(ctx, true)
	}
	return nil
}

// SwarmNodeStatus returns the swarm status for this node
func SwarmNodeStatus(c *docker.Client) (swarm.LocalNodeState, error) {
	info, err := c.Info(context.Background())
	if err != nil {
		return "", err
	}
	return info.Swarm.LocalNodeState, nil
}

func removeExitedAgent(c *docker.Client) error {
	filter := filters.NewArgs()
	filter.Add("name", ContainerName)
	l, err := c.ContainerList(context.Background(), types.ContainerListOptions{All: true, Filters: filter})
	if err != nil {
		return err
	}
	if len(l) == 1 {
		switch l[0].State {
		case "exited", "dead":
			fmt.Printf("found an exited ampagent container [%s], removing it\n", l[0].ID)
			return removeAgent(context.Background(), c, l[0].ID, false)
		case "created", "running":
			return fmt.Errorf("ampagent is already %s, this means you probably already have an amp CLI running", l[0].State)
		default:
			return fmt.Errorf("ampagent is in an unexpected state [%s]", l[0].State)
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package server

import (
	"crypto/tls"
	"io"
	"net"
	"net/http"
	"os"
	"sort"
	"strings"
	"time"

	"github.com/carbocation/interpose"
	"github.com/julienschmidt/httprouter"
	"gopkg.in/errgo.v1"
	"gopkg.in/tomb.v2"

	"hockeypuck/hkp"
	"hockeypuck/hkp/sks"
	"hockeypuck/hkp/storage"
	log "hockeypuck/logrus"
	"hockeypuck/mgohkp"
	"hockeypuck/pghkp"
)

type Server struct {
	settings  *Settings
	st        storage.Storage
	middle    *interpose.Middleware
	r         *httprouter.Router
	sksPeer   *sks.Peer
	logWriter io.WriteCloser

	t                 tomb.Tomb
	hkpAddr, hkpsAddr string
}

func NewServer(settings *Settings) (*Server, error) {
	if settings == nil {
		defaults := DefaultSettings()
		settings = &defaults
	}
	s := &Server{
		settings: settings,
		r:        httprouter.New(),
	}

	var err error
	s.st, err = DialStorage(settings)
	if err != nil {
		return nil, err
	}

	s.middle = interpose.New()
	s.middle.Use(func(next http.Handler) http.Handler {
		return http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
			start := time.Now()
			next.ServeHTTP(rw, req)
			log.WithFields(log.Fields{
				req.Method: req.URL.String(),
				"duration": time.Since(start).String(),
				"from":     req.RemoteAddr,
			}).Info()
		})
	})
	s.middle.UseHandler(s.r)

	s.sksPeer, err = sks.NewPeer(s.st, settings.Conflux.Recon.LevelDB.Path, &settings.Conflux.Recon.Settings)
	if err != nil {
		return nil, errgo.Mask(err)
	}

	options := []hkp.HandlerOption{hkp.StatsFunc(s.stats)}
	if settings.IndexTemplate != "" {
		options = append(options, hkp.IndexTemplate(settings.IndexTemplate))
	}
	if settings.VIndexTemplate != "" {
		options = append(options, hkp.VIndexTemplate(settings.VIndexTemplate))
	}
	if settings.StatsTemplate != "" {
		options = append(options, hkp.StatsTemplate(settings.StatsTemplate))
	}
	h, err := hkp.NewHandler(s.st, options...)
	if err != nil {
		return nil, errgo.Mask(err)
	}
	h.Register(s.r)

	if settings.Webroot != "" {
		err := s.registerWebroot(settings.Webroot)
		if err != nil {
			return nil, errgo.Mask(err)
		}
	}

	return s, nil
}

func DialStorage(settings *Settings) (storage.Storage, error) {
	switch settings.OpenPGP.DB.Driver {
	case "mongo":
		var options []mgohkp.Option
		if settings.OpenPGP.DB.Mongo != nil {
			if settings.OpenPGP.DB.Mongo.DB != "" {
				options = append(options, mgohkp.DBName(settings.OpenPGP.DB.Mongo.DB))
			}
			if settings.OpenPGP.DB.Mongo.Collection != "" {
				options = append(options, mgohkp.CollectionName(settings.OpenPGP.DB.Mongo.Collection))
			}
		}
		return mgohkp.Dial(settings.OpenPGP.DB.DSN, options...)
	case "postgres-jsonb":
		return pghkp.Dial(settings.OpenPGP.DB.DSN)
	}
	return nil, errgo.Newf("storage driver %q not supported", settings.OpenPGP.DB.Driver)
}

type stats struct {
	Now       string      `json:"now"`
	Version   string      `json:"version"`
	Hostname  string      `json:"hostname"`
	Nodename  string      `json:"nodename"`
	Contact   string      `json:"server_contact"`
	HTTPAddr  string      `json:"httpAddr"`
	ReconAddr string      `json:"reconAddr"`
	Software  string      `json:"software"`
	Peers     []statsPeer `json:"peers"`

	Total  int
	Hourly []loadStat
	Daily  []loadStat
}

type loadStat struct {
	*sks.LoadStat
	Time time.Time
}

type loadStats []loadStat

func (s loadStats) Len() int           { return len(s) }
func (s loadStats) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }
func (s loadStats) Less(i, j int) bool { return s[i].Time.Before(s[j].Time) }

type statsPeer struct {
	Name      string
	HTTPAddr  string `json:"httpAddr"`
	ReconAddr string `json:"reconAddr"`
}

type statsPeers []statsPeer

func (s statsPeers) Len() int           { return len(s) }
func (s statsPeers) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }
func (s statsPeers) Less(i, j int) bool { return s[i].Name < s[j].Name }

func (s *Server) stats() (interface{}, error) {
	sksStats := s.sksPeer.Stats()

	result := &stats{
		Now:       time.Now().UTC().Format(time.RFC3339),
		Version:   s.settings.Version,
		Contact:   s.settings.Contact,
		HTTPAddr:  s.settings.HKP.Bind,
		ReconAddr: s.settings.Conflux.Recon.Settings.ReconAddr,
		Software:  s.settings.Software,

		Total: sksStats.Total,
	}

	nodename, err := os.Hostname()
	if err != nil {
		log.Warningf("cannot determine local hostname: %v", err)
	} else {
		result.Nodename = nodename
	}

	if s.settings.Hostname != "" {
		result.Hostname = s.settings.Hostname
	} else if nodename != "" {
		result.Hostname = nodename
	}

	for k, v := range sksStats.Hourly {
		result.Hourly = append(result.Hourly, loadStat{LoadStat: v, Time: k})
	}
	sort.Sort(loadStats(result.Hourly))
	for k, v := range sksStats.Daily {
		result.Daily = append(result.Daily, loadStat{LoadStat: v, Time: k})
	}
	sort.Sort(loadStats(result.Daily))
	for k, v := range s.settings.Conflux.Recon.Settings.Partners {
		result.Peers = append(result.Peers, statsPeer{
			Name:      k,
			HTTPAddr:  v.HTTPAddr,
			ReconAddr: v.ReconAddr,
		})
	}
	sort.Sort(statsPeers(result.Peers))
	return result, nil
}

func (s *Server) registerWebroot(webroot string) error {
	fileServer := http.FileServer(http.Dir(webroot))
	d, err := os.Open(webroot)
	if os.IsNotExist(err) {
		log.Errorf("webroot %q not found", webroot)
		// non-fatal error
		return nil
	} else if err != nil {
		return errgo.Mask(err)
	}
	defer d.Close()
	files, err := d.Readdir(0)
	if err != nil {
		return errgo.Mask(err)
	}

	s.r.GET("/", func(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
		fileServer.ServeHTTP(w, req)
	})
	// httprouter needs explicit paths, so we need to set up a route for each
	// path. This will panic if there are any paths that conflict with
	// previously registered routes.
	for _, fi := range files {
		name := fi.Name()
		if !fi.IsDir() {
			s.r.GET("/"+name, func(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
				req.URL.Path = "/" + name
				fileServer.ServeHTTP(w, req)
			})
		} else {
			s.r.GET("/"+name+"/*filepath", func(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
				req.URL.Path = "/" + name + ps.ByName("filepath")
				fileServer.ServeHTTP(w, req)
			})
		}
	}
	return nil
}

func (s *Server) Start() error {
	s.openLog()

	s.t.Go(s.listenAndServeHKP)
	if s.settings.HKPS != nil {
		s.t.Go(s.listenAndServeHKPS)
	}

	if s.sksPeer != nil {
		s.sksPeer.Start()
	}

	return nil
}

type nopCloser struct {
	io.Writer
}

func (nopCloser) Close() error { return nil }

func (s *Server) openLog() {
	defer func() {
		level, err := log.ParseLevel(strings.ToLower(s.settings.LogLevel))
		if err != nil {
			log.Warningf("invalid LogLevel=%q: %v", s.settings.LogLevel, err)
			return
		}
		log.SetLevel(level)
	}()

	s.logWriter = nopCloser{os.Stderr}
	if s.settings.LogFile != "" {
		f, err := os.OpenFile(s.settings.LogFile, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0644)
		if err != nil {
			log.Errorf("failed to open LogFile=%q: %v", s.settings.LogFile, err)
		}
		s.logWriter = f
	}
	log.SetOutput(s.logWriter)
	log.Debug("log opened")
}

func (s *Server) closeLog() {
	log.SetOutput(os.Stderr)
	s.logWriter.Close()
}

func (s *Server) LogRotate() {
	w := s.logWriter
	s.openLog()
	w.Close()
}

func (s *Server) Wait() error {
	return s.t.Wait()
}

func (s *Server) Stop() {
	defer s.closeLog()

	if s.sksPeer != nil {
		s.sksPeer.Stop()
	}
	s.t.Kill(nil)
	s.t.Wait()
}

// tcpKeepAliveListener sets TCP keep-alive timeouts on accepted
// connections. It's used by listenAndServe and listenAndServeTLS so
// dead TCP connections (e.g. closing laptop mid-download) eventually
// go away.
type tcpKeepAliveListener struct {
	*net.TCPListener
}

// Accept implements net.Listener.
func (ln tcpKeepAliveListener) Accept() (net.Conn, error) {
	tc, err := ln.AcceptTCP()
	if err != nil {
		return nil, errgo.Mask(err)
	}
	tc.SetKeepAlive(true)
	tc.SetKeepAlivePeriod(3 * time.Minute)
	return tc, nil
}

var newListener = (*Server).newListener

func (s *Server) newListener(addr string) (net.Listener, error) {
	ln, err := net.Listen("tcp", addr)
	if err != nil {
		return nil, errgo.Mask(err)
	}

	s.t.Go(func() error {
		<-s.t.Dying()
		return ln.Close()
	})
	return tcpKeepAliveListener{ln.(*net.TCPListener)}, nil
}

func (s *Server) listenAndServeHKP() error {
	ln, err := newListener(s, s.settings.HKP.Bind)
	if err != nil {
		return err
	}
	s.hkpAddr = ln.Addr().String()
	return http.Serve(ln, s.middle)
}

func (s *Server) listenAndServeHKPS() error {
	config := &tls.Config{
		NextProtos: []string{"http/1.1"},
	}
	var err error
	config.Certificates = make([]tls.Certificate, 1)
	config.Certificates[0], err = tls.LoadX509KeyPair(s.settings.HKPS.Cert, s.settings.HKPS.Key)
	if err != nil {
		return errgo.Notef(err, "failed to load HKPS certificate=%q key=%q", s.settings.HKPS.Cert, s.settings.HKPS.Key)
	}

	ln, err := newListener(s, s.settings.HKP.Bind)
	if err != nil {
		return errgo.Mask(err)
	}
	s.hkpsAddr = ln.Addr().String()
	ln = tls.NewListener(ln, config)
	return http.Serve(ln, s.middle)
}
<commit_msg>capture and log the HTTP status code, and some other headers<commit_after>package server

import (
	"crypto/tls"
	"io"
	"net"
	"net/http"
	"os"
	"sort"
	"strings"
	"time"

	"github.com/carbocation/interpose"
	"github.com/julienschmidt/httprouter"
	"gopkg.in/errgo.v1"
	"gopkg.in/tomb.v2"

	"hockeypuck/hkp"
	"hockeypuck/hkp/sks"
	"hockeypuck/hkp/storage"
	log "hockeypuck/logrus"
	"hockeypuck/mgohkp"
	"hockeypuck/pghkp"
)

type Server struct {
	settings  *Settings
	st        storage.Storage
	middle    *interpose.Middleware
	r         *httprouter.Router
	sksPeer   *sks.Peer
	logWriter io.WriteCloser

	t                 tomb.Tomb
	hkpAddr, hkpsAddr string
}

type statusCodeResponseWriter struct {
	http.ResponseWriter
	statusCode int
}

func NewStatusCodeResponseWriter(w http.ResponseWriter) *statusCodeResponseWriter {
	// WriteHeader is not called if our response implicitly
	// returns 200 OK, so we default to that status code.
	return &statusCodeResponseWriter{w, http.StatusOK}
}

func (scrw *statusCodeResponseWriter) WriteHeader(code int) {
	scrw.statusCode = code
	scrw.ResponseWriter.WriteHeader(code)
}

func NewServer(settings *Settings) (*Server, error) {
	if settings == nil {
		defaults := DefaultSettings()
		settings = &defaults
	}
	s := &Server{
		settings: settings,
		r:        httprouter.New(),
	}

	var err error
	s.st, err = DialStorage(settings)
	if err != nil {
		return nil, err
	}

	s.middle = interpose.New()
	s.middle.Use(func(next http.Handler) http.Handler {
		return http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
			start := time.Now()
			scrw := NewStatusCodeResponseWriter(rw)
			next.ServeHTTP(scrw, req)
			fields := log.Fields{
				req.Method:    req.URL.String(),
				"duration":    time.Since(start).String(),
				"from":        req.RemoteAddr,
				"host":        req.Host,
				"status-code": scrw.statusCode,
				"user-agent":  req.UserAgent(),
			}
			proxyHeaders := []string{
				"x-forwarded-for",
				"x-forwarded-host",
				"x-forwarded-server",
			}
			for _, ph := range proxyHeaders {
				if v := req.Header.Get(ph); v != "" {
					fields[ph] = v
				}
			}
			log.WithFields(fields).Info()
		})
	})
	s.middle.UseHandler(s.r)

	s.sksPeer, err = sks.NewPeer(s.st, settings.Conflux.Recon.LevelDB.Path, &settings.Conflux.Recon.Settings)
	if err != nil {
		return nil, errgo.Mask(err)
	}

	options := []hkp.HandlerOption{hkp.StatsFunc(s.stats)}
	if settings.IndexTemplate != "" {
		options = append(options, hkp.IndexTemplate(settings.IndexTemplate))
	}
	if settings.VIndexTemplate != "" {
		options = append(options, hkp.VIndexTemplate(settings.VIndexTemplate))
	}
	if settings.StatsTemplate != "" {
		options = append(options, hkp.StatsTemplate(settings.StatsTemplate))
	}
	h, err := hkp.NewHandler(s.st, options...)
	if err != nil {
		return nil, errgo.Mask(err)
	}
	h.Register(s.r)

	if settings.Webroot != "" {
		err := s.registerWebroot(settings.Webroot)
		if err != nil {
			return nil, errgo.Mask(err)
		}
	}

	return s, nil
}

func DialStorage(settings *Settings) (storage.Storage, error) {
	switch settings.OpenPGP.DB.Driver {
	case "mongo":
		var options []mgohkp.Option
		if settings.OpenPGP.DB.Mongo != nil {
			if settings.OpenPGP.DB.Mongo.DB != "" {
				options = append(options, mgohkp.DBName(settings.OpenPGP.DB.Mongo.DB))
			}
			if settings.OpenPGP.DB.Mongo.Collection != "" {
				options = append(options, mgohkp.CollectionName(settings.OpenPGP.DB.Mongo.Collection))
			}
		}
		return mgohkp.Dial(settings.OpenPGP.DB.DSN, options...)
	case "postgres-jsonb":
		return pghkp.Dial(settings.OpenPGP.DB.DSN)
	}
	return nil, errgo.Newf("storage driver %q not supported", settings.OpenPGP.DB.Driver)
}

type stats struct {
	Now       string      `json:"now"`
	Version   string      `json:"version"`
	Hostname  string      `json:"hostname"`
	Nodename  string      `json:"nodename"`
	Contact   string      `json:"server_contact"`
	HTTPAddr  string      `json:"httpAddr"`
	ReconAddr string      `json:"reconAddr"`
	Software  string      `json:"software"`
	Peers     []statsPeer `json:"peers"`

	Total  int
	Hourly []loadStat
	Daily  []loadStat
}

type loadStat struct {
	*sks.LoadStat
	Time time.Time
}

type loadStats []loadStat

func (s loadStats) Len() int           { return len(s) }
func (s loadStats) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }
func (s loadStats) Less(i, j int) bool { return s[i].Time.Before(s[j].Time) }

type statsPeer struct {
	Name      string
	HTTPAddr  string `json:"httpAddr"`
	ReconAddr string `json:"reconAddr"`
}

type statsPeers []statsPeer

func (s statsPeers) Len() int           { return len(s) }
func (s statsPeers) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }
func (s statsPeers) Less(i, j int) bool { return s[i].Name < s[j].Name }

func (s *Server) stats() (interface{}, error) {
	sksStats := s.sksPeer.Stats()

	result := &stats{
		Now:       time.Now().UTC().Format(time.RFC3339),
		Version:   s.settings.Version,
		Contact:   s.settings.Contact,
		HTTPAddr:  s.settings.HKP.Bind,
		ReconAddr: s.settings.Conflux.Recon.Settings.ReconAddr,
		Software:  s.settings.Software,

		Total: sksStats.Total,
	}

	nodename, err := os.Hostname()
	if err != nil {
		log.Warningf("cannot determine local hostname: %v", err)
	} else {
		result.Nodename = nodename
	}

	if s.settings.Hostname != "" {
		result.Hostname = s.settings.Hostname
	} else if nodename != "" {
		result.Hostname = nodename
	}

	for k, v := range sksStats.Hourly {
		result.Hourly = append(result.Hourly, loadStat{LoadStat: v, Time: k})
	}
	sort.Sort(loadStats(result.Hourly))
	for k, v := range sksStats.Daily {
		result.Daily = append(result.Daily, loadStat{LoadStat: v, Time: k})
	}
	sort.Sort(loadStats(result.Daily))
	for k, v := range s.settings.Conflux.Recon.Settings.Partners {
		result.Peers = append(result.Peers, statsPeer{
			Name:      k,
			HTTPAddr:  v.HTTPAddr,
			ReconAddr: v.ReconAddr,
		})
	}
	sort.Sort(statsPeers(result.Peers))
	return result, nil
}

func (s *Server) registerWebroot(webroot string) error {
	fileServer := http.FileServer(http.Dir(webroot))
	d, err := os.Open(webroot)
	if os.IsNotExist(err) {
		log.Errorf("webroot %q not found", webroot)
		// non-fatal error
		return nil
	} else if err != nil {
		return errgo.Mask(err)
	}
	defer d.Close()
	files, err := d.Readdir(0)
	if err != nil {
		return errgo.Mask(err)
	}

	s.r.GET("/", func(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
		fileServer.ServeHTTP(w, req)
	})
	// httprouter needs explicit paths, so we need to set up a route for each
	// path. This will panic if there are any paths that conflict with
	// previously registered routes.
	for _, fi := range files {
		name := fi.Name()
		if !fi.IsDir() {
			s.r.GET("/"+name, func(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
				req.URL.Path = "/" + name
				fileServer.ServeHTTP(w, req)
			})
		} else {
			s.r.GET("/"+name+"/*filepath", func(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
				req.URL.Path = "/" + name + ps.ByName("filepath")
				fileServer.ServeHTTP(w, req)
			})
		}
	}
	return nil
}

func (s *Server) Start() error {
	s.openLog()

	s.t.Go(s.listenAndServeHKP)
	if s.settings.HKPS != nil {
		s.t.Go(s.listenAndServeHKPS)
	}

	if s.sksPeer != nil {
		s.sksPeer.Start()
	}

	return nil
}

type nopCloser struct {
	io.Writer
}

func (nopCloser) Close() error { return nil }

func (s *Server) openLog() {
	defer func() {
		level, err := log.ParseLevel(strings.ToLower(s.settings.LogLevel))
		if err != nil {
			log.Warningf("invalid LogLevel=%q: %v", s.settings.LogLevel, err)
			return
		}
		log.SetLevel(level)
	}()

	s.logWriter = nopCloser{os.Stderr}
	if s.settings.LogFile != "" {
		f, err := os.OpenFile(s.settings.LogFile, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0644)
		if err != nil {
			log.Errorf("failed to open LogFile=%q: %v", s.settings.LogFile, err)
		}
		s.logWriter = f
	}
	log.SetOutput(s.logWriter)
	log.Debug("log opened")
}

func (s *Server) closeLog() {
	log.SetOutput(os.Stderr)
	s.logWriter.Close()
}

func (s *Server) LogRotate() {
	w := s.logWriter
	s.openLog()
	w.Close()
}

func (s *Server) Wait() error {
	return s.t.Wait()
}

func (s *Server) Stop() {
	defer s.closeLog()

	if s.sksPeer != nil {
		s.sksPeer.Stop()
	}
	s.t.Kill(nil)
	s.t.Wait()
}

// tcpKeepAliveListener sets TCP keep-alive timeouts on accepted
// connections. It's used by listenAndServe and listenAndServeTLS so
// dead TCP connections (e.g. closing laptop mid-download) eventually
// go away.
type tcpKeepAliveListener struct {
	*net.TCPListener
}

// Accept implements net.Listener.
func (ln tcpKeepAliveListener) Accept() (net.Conn, error) {
	tc, err := ln.AcceptTCP()
	if err != nil {
		return nil, errgo.Mask(err)
	}
	tc.SetKeepAlive(true)
	tc.SetKeepAlivePeriod(3 * time.Minute)
	return tc, nil
}

var newListener = (*Server).newListener

func (s *Server) newListener(addr string) (net.Listener, error) {
	ln, err := net.Listen("tcp", addr)
	if err != nil {
		return nil, errgo.Mask(err)
	}

	s.t.Go(func() error {
		<-s.t.Dying()
		return ln.Close()
	})
	return tcpKeepAliveListener{ln.(*net.TCPListener)}, nil
}

func (s *Server) listenAndServeHKP() error {
	ln, err := newListener(s, s.settings.HKP.Bind)
	if err != nil {
		return err
	}
	s.hkpAddr = ln.Addr().String()
	return http.Serve(ln, s.middle)
}

func (s *Server) listenAndServeHKPS() error {
	config := &tls.Config{
		NextProtos: []string{"http/1.1"},
	}
	var err error
	config.Certificates = make([]tls.Certificate, 1)
	config.Certificates[0], err = tls.LoadX509KeyPair(s.settings.HKPS.Cert, s.settings.HKPS.Key)
	if err != nil {
		return errgo.Notef(err, "failed to load HKPS certificate=%q key=%q", s.settings.HKPS.Cert, s.settings.HKPS.Key)
	}

	ln, err := newListener(s, s.settings.HKP.Bind)
	if err != nil {
		return errgo.Mask(err)
	}
	s.hkpsAddr = ln.Addr().String()
	ln = tls.NewListener(ln, config)
	return http.Serve(ln, s.middle)
}
<|endoftext|>


<commit_before>package slack

import (
	"fmt"
	"log"

	slack_webhook "github.com/ashwanthkumar/slack-go-webhook"
	"github.com/checkr/codeflow/server/agent"
	"github.com/checkr/codeflow/server/plugins"
	"github.com/spf13/viper"
)

type Slack struct {
}

func init() {
	agent.RegisterPlugin("slack", func() agent.Plugin {
		return &Slack{}
	})
}

func (x *Slack) Description() string {
	return "Send slack events to subscribed plugins"
}

func (x *Slack) SampleConfig() string {
	return ` `
}

func (x *Slack) slack() error {
	return nil
}

func (x *Slack) Start(e chan agent.Event) error {
	go x.slack()
	log.Println("Started Slack")

	return nil
}

func (x *Slack) Stop() {
	log.Println("Stopping Slack")
}

func (x *Slack) Subscribe() []string {
	return []string{
		"plugins.DockerBuild",
		"plugins.DockerDeploy",
		"plugins.Project",
		"plugins.Release",
	}
}

func (x *Slack) Process(e agent.Event) error {
	webhookUrl := viper.GetString("plugins.slack.webhook_url")

	switch e.Name {
	case "plugins.DockerDeploy:create":
		payload := e.Payload.(plugins.DockerDeploy)

		for _, channel := range payload.Project.NotifyChannels {
			project := payload.Project.Slug
			release := payload.Release.HeadFeature.Hash
			message := payload.Release.HeadFeature.Message

			repository := payload.Project.Repository
			tail := payload.Release.TailFeature.Hash
			head := payload.Release.HeadFeature.Hash

			diffUrl := fmt.Sprintf("https://github.com/%s/compare/%s...%s", repository, tail, head)

			attachment1 := slack_webhook.Attachment{}
			attachment1.AddField(slack_webhook.Field{Title: "Commit", Value: message}).AddField(slack_webhook.Field{Title: "Link", Value: diffUrl})

			slackPayload := slack_webhook.Payload{
				Text:        fmt.Sprintf("Deploying %s for %s", release, project),
				Username:    "codeflow-bot",
				Channel:     channel,
				IconEmoji:   ":rocket:",
				Attachments: []slack_webhook.Attachment{attachment1},
			}
			err := slack_webhook.Send(webhookUrl, "", slackPayload)
			if len(err) > 0 {
				log.Println("error: %s\n", err)
			}
		}
	case "plugins.DockerDeploy:status":
		payload := e.Payload.(plugins.DockerDeploy)

		if payload.State != plugins.Complete || payload.State != plugins.Failed {
			return nil
		}

		for _, channel := range payload.Project.NotifyChannels {
			project := payload.Project.Slug
			release := payload.Release.HeadFeature.Hash

			msg := fmt.Sprintf("Deploy %s for %s", release, project)
			color := "#008000"

			if payload.State == plugins.Failed {
				color = "#FF0000"
				msg = fmt.Sprintf("Deploy %s for %s", release, project)
			}

			attachment1 := slack_webhook.Attachment{Color: &color}
			attachment1.AddField(slack_webhook.Field{Title: "Status", Value: string(payload.State)})

			slackPayload := slack_webhook.Payload{
				Text:        msg,
				Username:    "codeflow-bot",
				Channel:     channel,
				IconEmoji:   ":rocket:",
				Attachments: []slack_webhook.Attachment{attachment1},
			}
			err := slack_webhook.Send(webhookUrl, "", slackPayload)
			if len(err) > 0 {
				log.Println("error: %s\n", err)
			}
		}
	}

	return nil
}
<commit_msg>Its hackaton<commit_after>package slack

import (
	"fmt"
	"log"

	slack_webhook "github.com/ashwanthkumar/slack-go-webhook"
	"github.com/checkr/codeflow/server/agent"
	"github.com/checkr/codeflow/server/plugins"
	"github.com/spf13/viper"
)

type Slack struct {
}

func init() {
	agent.RegisterPlugin("slack", func() agent.Plugin {
		return &Slack{}
	})
}

func (x *Slack) Description() string {
	return "Send slack events to subscribed plugins"
}

func (x *Slack) SampleConfig() string {
	return ` `
}

func (x *Slack) slack() error {
	return nil
}

func (x *Slack) Start(e chan agent.Event) error {
	go x.slack()
	log.Println("Started Slack")

	return nil
}

func (x *Slack) Stop() {
	log.Println("Stopping Slack")
}

func (x *Slack) Subscribe() []string {
	return []string{
		"plugins.DockerBuild",
		"plugins.DockerDeploy",
		"plugins.Project",
		"plugins.Release",
	}
}

func (x *Slack) Process(e agent.Event) error {
	webhookUrl := viper.GetString("plugins.slack.webhook_url")

	switch e.Name {
	case "plugins.DockerDeploy:create":
		payload := e.Payload.(plugins.DockerDeploy)

		for _, channel := range payload.Project.NotifyChannels {
			project := payload.Project.Slug
			release := payload.Release.HeadFeature.Hash
			message := payload.Release.HeadFeature.Message

			repository := payload.Project.Repository
			tail := payload.Release.TailFeature.Hash
			head := payload.Release.HeadFeature.Hash

			diffUrl := fmt.Sprintf("https://github.com/%s/compare/%s...%s", repository, tail, head)

			attachment1 := slack_webhook.Attachment{}
			attachment1.AddField(slack_webhook.Field{Title: "Commit", Value: message}).AddField(slack_webhook.Field{Title: "Link", Value: diffUrl})

			slackPayload := slack_webhook.Payload{
				Text:        fmt.Sprintf("Deploying %s for %s", release, project),
				Username:    "codeflow-bot",
				Channel:     channel,
				IconEmoji:   ":rocket:",
				Attachments: []slack_webhook.Attachment{attachment1},
			}
			err := slack_webhook.Send(webhookUrl, "", slackPayload)
			if len(err) > 0 {
				log.Println("error: %s\n", err)
			}
		}
	case "plugins.DockerDeploy:status":
		payload := e.Payload.(plugins.DockerDeploy)

		if payload.State != plugins.Complete && payload.State != plugins.Failed {
			return nil
		}

		for _, channel := range payload.Project.NotifyChannels {
			project := payload.Project.Slug
			release := payload.Release.HeadFeature.Hash

			msg := fmt.Sprintf("Deploy %s for %s", release, project)
			color := "#008000"

			if payload.State == plugins.Failed {
				color = "#FF0000"
				msg = fmt.Sprintf("Deploy %s for %s", release, project)
			}

			attachment1 := slack_webhook.Attachment{Color: &color}
			attachment1.AddField(slack_webhook.Field{Title: "Status", Value: string(payload.State)})

			slackPayload := slack_webhook.Payload{
				Text:        msg,
				Username:    "codeflow-bot",
				Channel:     channel,
				IconEmoji:   ":rocket:",
				Attachments: []slack_webhook.Attachment{attachment1},
			}
			err := slack_webhook.Send(webhookUrl, "", slackPayload)
			if len(err) > 0 {
				log.Println("error: %s\n", err)
			}
		}
	}

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"net/http"
	"encoding/json"
	"io/ioutil"
	"log"
)

func ValidateService(cli *http.Client, url string, token string) string {

	// Set up the HTTP request to validate service/routes
	req, err := http.NewRequest("GET", url, nil)
	req.Header.Add("Authorization", "Bearer "+token)
	resp, err := cli.Do(req)

	if err != nil {
		log.Println("error Health URL = " + url)
		return "ko"
	}

	defer resp.Body.Close()

	if resp.StatusCode == 200 ||  resp.StatusCode == 401 ||resp.StatusCode == 403 {
		return "ok"
	} else {
		return "ko"
	}

}

func ValidateReplicas(cli *http.Client, url string, token string) float64  {

	replicas :=  0.0

	req, err := http.NewRequest("GET", url, nil)
	req.Header.Add("Authorization", "Bearer "+token)
	resp, err := cli.Do(req)

	if err != nil {
		log.Println("error getDC = " + url)
		return replicas
	}

	defer resp.Body.Close()

	dcs, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		panic(err)
	}

	dcsCustom := map[string]interface{}{}
	json.Unmarshal(dcs, &dcsCustom)

	if dcsCustom != nil && len(dcsCustom)> 0 {
		specMap := dcsCustom["spec"].(map[string]interface{})
		replicas =  specMap["replicas"].(float64)
	}

	return replicas
}
<commit_msg>fixing bugs deploymentconfig response<commit_after>package main

import (
	"net/http"
	"encoding/json"
	"io/ioutil"
	"log"
)

func ValidateService(cli *http.Client, url string, token string) string {

	// Set up the HTTP request to validate service/routes
	req, err := http.NewRequest("GET", url, nil)
	req.Header.Add("Authorization", "Bearer "+token)
	resp, err := cli.Do(req)

	if err != nil {
		log.Println("error Health URL = " + url)
		return "ko"
	}

	defer resp.Body.Close()

	if resp.StatusCode == 200 ||  resp.StatusCode == 401 ||resp.StatusCode == 403 {
		return "ok"
	} else {
		return "ko"
	}

}

func ValidateReplicas(cli *http.Client, url string, token string) float64  {

	replicas :=  0.0

	req, err := http.NewRequest("GET", url, nil)
	req.Header.Add("Authorization", "Bearer "+token)
	resp, err := cli.Do(req)

	if err != nil {
		log.Println("error getDC = " + url)
		return replicas
	}

	defer resp.Body.Close()

	dcs, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		panic(err)
	}

	dcsCustom := map[string]interface{}{}
	json.Unmarshal(dcs, &dcsCustom)

	if dcsCustom != nil && len(dcsCustom)> 0 {
		spec := dcsCustom["spec"]
		if spec != nil {
			specMap := spec.(map[string]interface{})
			replicas =  specMap["replicas"].(float64)
		}
	}

	return replicas
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io/ioutil"
	"os"
	"regexp"
	"sort"
	"strings"
)

func WordCount(s string) map[string]int {
	r := regexp.MustCompile(`[a-z\']+`)
	substrings := strings.Fields(s)
	counts := make(map[string]int)

	for _, word := range substrings {
		if w := r.FindString(strings.ToLower(word)); w != "" {
			counts[w] += 1
		}
	}

	return counts
}

func main() {
	if bytes, err := ioutil.ReadAll(os.Stdin); err != nil {
		panic(err)
	} else {
		m := WordCount(string(bytes))
		var words []string
		for word := range m {
			words = append(words, word)
		}
		sort.Strings(words)

		for _, word := range words {
			fmt.Println(word, m[word])
		}
	}
}
<commit_msg>Improve Go word count a little bit<commit_after>package main

import (
	"fmt"
	"io/ioutil"
	"os"
	"regexp"
	"sort"
	"strings"
)

func WordCount(s string) map[string]int {
	r := regexp.MustCompile(`[a-z\']+`)
	counts := make(map[string]int)

	for _, word := range r.FindAllString(strings.ToLower(s), -1) {
		counts[word] += 1
	}

	return counts
}

func main() {
	if bytes, err := ioutil.ReadAll(os.Stdin); err != nil {
		panic(err)
	} else {
		m := WordCount(string(bytes))
		var words []string
		for word := range m {
			words = append(words, word)
		}
		sort.Strings(words)

		for _, word := range words {
			fmt.Println(word, m[word])
		}
	}
}
<|endoftext|>