<commit_before>package sql

import (
	"crypto/tls"
	"crypto/x509"
	"database/sql"
	"fmt"
	"io/ioutil"
	"net"
	"regexp"
	"strconv"
	"strings"
	"time"

	"github.com/go-sql-driver/mysql"
	"github.com/lib/pq"
	sqlite3 "github.com/mattn/go-sqlite3"

	"github.com/dexidp/dex/pkg/log"
	"github.com/dexidp/dex/storage"
)

const (
	// postgres error codes
	pgErrUniqueViolation = "23505" // unique_violation
)

const (
	// MySQL error codes
	mysqlErrDupEntry            = 1062
	mysqlErrDupEntryWithKeyName = 1586
)

// SQLite3 options for creating an SQL db.
type SQLite3 struct {
	// File to
	File string `json:"file"`
}

// Open creates a new storage implementation backed by SQLite3
func (s *SQLite3) Open(logger log.Logger) (storage.Storage, error) {
	conn, err := s.open(logger)
	if err != nil {
		return nil, err
	}
	return conn, nil
}

func (s *SQLite3) open(logger log.Logger) (*conn, error) {
	db, err := sql.Open("sqlite3", s.File)
	if err != nil {
		return nil, err
	}
	if s.File == ":memory:" {
		// sqlite3 uses file locks to coordinate concurrent access. In memory
		// doesn't support this, so limit the number of connections to 1.
		db.SetMaxOpenConns(1)
	}

	errCheck := func(err error) bool {
		sqlErr, ok := err.(sqlite3.Error)
		if !ok {
			return false
		}
		return sqlErr.ExtendedCode == sqlite3.ErrConstraintPrimaryKey
	}

	c := &conn{db, flavorSQLite3, logger, errCheck}
	if _, err := c.migrate(); err != nil {
		return nil, fmt.Errorf("failed to perform migrations: %v", err)
	}
	return c, nil
}

// nolint
const (
	// postgres SSL modes
	pgSSLDisable    = "disable"
	pgSSLRequire    = "require"
	pgSSLVerifyCA   = "verify-ca"
	pgSSLVerifyFull = "verify-full"
)

// nolint
const (
	// MySQL SSL modes
	mysqlSSLTrue       = "true"
	mysqlSSLFalse      = "false"
	mysqlSSLSkipVerify = "skip-verify"
	mysqlSSLCustom     = "custom"
)

// NetworkDB contains options common to SQL databases accessed over network.
type NetworkDB struct {
	Database string
	User     string
	Password string
	Host     string
	Port     uint16

	ConnectionTimeout int // Seconds

	// database/sql tunables, see
	// https://golang.org/pkg/database/sql/#DB.SetConnMaxLifetime and below
	// Note: defaults will be set if these are 0
	MaxOpenConns    int // default: 5
	MaxIdleConns    int // default: 5
	ConnMaxLifetime int // Seconds, default: not set
}

// SSL represents SSL options for network databases.
type SSL struct {
	Mode   string
	CAFile string
	// Files for client auth.
	KeyFile  string
	CertFile string
}

// Postgres options for creating an SQL db.
type Postgres struct {
	NetworkDB

	SSL SSL `json:"ssl" yaml:"ssl"`
}

// Open creates a new storage implementation backed by Postgres.
func (p *Postgres) Open(logger log.Logger) (storage.Storage, error) {
	conn, err := p.open(logger)
	if err != nil {
		return nil, err
	}
	return conn, nil
}

var strEsc = regexp.MustCompile(`([\\'])`)

func dataSourceStr(str string) string {
	return "'" + strEsc.ReplaceAllString(str, `\$1`) + "'"
}

// createDataSourceName takes the configuration provided via the Postgres
// struct to create a data-source name that Go's database/sql package can
// make use of.
func (p *Postgres) createDataSourceName() string {
	parameters := []string{}

	addParam := func(key, val string) {
		parameters = append(parameters, fmt.Sprintf("%s=%s", key, val))
	}

	addParam("connect_timeout", strconv.Itoa(p.ConnectionTimeout))

	// detect host:port for backwards-compatibility
	host, port, err := net.SplitHostPort(p.Host)
	if err != nil {
		// not host:port, probably unix socket or bare address

		host = p.Host

		if p.Port != 0 {
			port = strconv.Itoa(int(p.Port))
		}
	}

	if host != "" {
		addParam("host", dataSourceStr(host))
	}

	if port != "" {
		addParam("port", port)
	}

	if p.User != "" {
		addParam("user", dataSourceStr(p.User))
	}

	if p.Password != "" {
		addParam("password", dataSourceStr(p.Password))
	}

	if p.Database != "" {
		addParam("dbname", dataSourceStr(p.Database))
	}

	if p.SSL.Mode == "" {
		// Assume the strictest mode if unspecified.
		addParam("sslmode", dataSourceStr(pgSSLVerifyFull))
	} else {
		addParam("sslmode", dataSourceStr(p.SSL.Mode))
	}

	if p.SSL.CAFile != "" {
		addParam("sslrootcert", dataSourceStr(p.SSL.CAFile))
	}

	if p.SSL.CertFile != "" {
		addParam("sslcert", dataSourceStr(p.SSL.CertFile))
	}

	if p.SSL.KeyFile != "" {
		addParam("sslkey", dataSourceStr(p.SSL.KeyFile))
	}

	return strings.Join(parameters, " ")
}

func (p *Postgres) open(logger log.Logger) (*conn, error) {
	dataSourceName := p.createDataSourceName()

	db, err := sql.Open("postgres", dataSourceName)
	if err != nil {
		return nil, err
	}

	// set database/sql tunables if configured
	if p.ConnMaxLifetime != 0 {
		db.SetConnMaxLifetime(time.Duration(p.ConnMaxLifetime) * time.Second)
	}

	if p.MaxIdleConns == 0 {
		db.SetMaxIdleConns(5)
	} else {
		db.SetMaxIdleConns(p.MaxIdleConns)
	}

	if p.MaxOpenConns == 0 {
		db.SetMaxOpenConns(5)
	} else {
		db.SetMaxOpenConns(p.MaxOpenConns)
	}

	errCheck := func(err error) bool {
		sqlErr, ok := err.(*pq.Error)
		if !ok {
			return false
		}
		return sqlErr.Code == pgErrUniqueViolation
	}

	c := &conn{db, flavorPostgres, logger, errCheck}
	if _, err := c.migrate(); err != nil {
		return nil, fmt.Errorf("failed to perform migrations: %v", err)
	}
	return c, nil
}

// MySQL options for creating a MySQL db.
type MySQL struct {
	NetworkDB

	SSL SSL `json:"ssl" yaml:"ssl"`

	// TODO(pborzenkov): used by tests to reduce lock wait timeout. Should
	// we make it exported and allow users to provide arbitrary params?
	params map[string]string
}

// Open creates a new storage implementation backed by MySQL.
func (s *MySQL) Open(logger log.Logger) (storage.Storage, error) {
	conn, err := s.open(logger)
	if err != nil {
		return nil, err
	}
	return conn, nil
}

func (s *MySQL) open(logger log.Logger) (*conn, error) {
	cfg := mysql.Config{
		User:                 s.User,
		Passwd:               s.Password,
		DBName:               s.Database,
		AllowNativePasswords: true,

		Timeout: time.Second * time.Duration(s.ConnectionTimeout),

		ParseTime: true,
		Params: map[string]string{
			"transaction_isolation": "'SERIALIZABLE'",
		},
	}
	if s.Host != "" {
		if s.Host[0] != '/' {
			cfg.Net = "tcp"
			cfg.Addr = s.Host
		} else {
			cfg.Net = "unix"
			cfg.Addr = s.Host
		}
	}
	if s.SSL.CAFile != "" || s.SSL.CertFile != "" || s.SSL.KeyFile != "" {
		if err := s.makeTLSConfig(); err != nil {
			return nil, fmt.Errorf("failed to make TLS config: %v", err)
		}
		cfg.TLSConfig = mysqlSSLCustom
	} else if s.SSL.Mode == "" {
		cfg.TLSConfig = mysqlSSLTrue
	} else {
		cfg.TLSConfig = s.SSL.Mode
	}
	for k, v := range s.params {
		cfg.Params[k] = v
	}

	db, err := sql.Open("mysql", cfg.FormatDSN())
	if err != nil {
		return nil, err
	}

	errCheck := func(err error) bool {
		sqlErr, ok := err.(*mysql.MySQLError)
		if !ok {
			return false
		}
		return sqlErr.Number == mysqlErrDupEntry ||
			sqlErr.Number == mysqlErrDupEntryWithKeyName
	}

	c := &conn{db, flavorMySQL, logger, errCheck}
	if _, err := c.migrate(); err != nil {
		return nil, fmt.Errorf("failed to perform migrations: %v", err)
	}
	return c, nil
}

func (s *MySQL) makeTLSConfig() error {
	cfg := &tls.Config{}
	if s.SSL.CAFile != "" {
		rootCertPool := x509.NewCertPool()
		pem, err := ioutil.ReadFile(s.SSL.CAFile)
		if err != nil {
			return err
		}
		if ok := rootCertPool.AppendCertsFromPEM(pem); !ok {
			return fmt.Errorf("failed to append PEM")
		}
		cfg.RootCAs = rootCertPool
	}
	if s.SSL.CertFile != "" && s.SSL.KeyFile != "" {
		clientCert := make([]tls.Certificate, 0, 1)
		certs, err := tls.LoadX509KeyPair(s.SSL.CertFile, s.SSL.KeyFile)
		if err != nil {
			return err
		}
		clientCert = append(clientCert, certs)
		cfg.Certificates = clientCert
	}

	mysql.RegisterTLSConfig(mysqlSSLCustom, cfg)
	return nil
}
<commit_msg>storage/mysql: support pre-5.7.20 instances with tx_isolation only<commit_after>package sql

import (
	"crypto/tls"
	"crypto/x509"
	"database/sql"
	"fmt"
	"io/ioutil"
	"net"
	"regexp"
	"strconv"
	"strings"
	"time"

	"github.com/go-sql-driver/mysql"
	"github.com/lib/pq"
	sqlite3 "github.com/mattn/go-sqlite3"

	"github.com/dexidp/dex/pkg/log"
	"github.com/dexidp/dex/storage"
)

const (
	// postgres error codes
	pgErrUniqueViolation = "23505" // unique_violation
)

const (
	// MySQL error codes
	mysqlErrDupEntry            = 1062
	mysqlErrDupEntryWithKeyName = 1586
	mysqlErrUnknownSysVar       = 1193
)

// SQLite3 options for creating an SQL db.
type SQLite3 struct {
	// File to
	File string `json:"file"`
}

// Open creates a new storage implementation backed by SQLite3
func (s *SQLite3) Open(logger log.Logger) (storage.Storage, error) {
	conn, err := s.open(logger)
	if err != nil {
		return nil, err
	}
	return conn, nil
}

func (s *SQLite3) open(logger log.Logger) (*conn, error) {
	db, err := sql.Open("sqlite3", s.File)
	if err != nil {
		return nil, err
	}
	if s.File == ":memory:" {
		// sqlite3 uses file locks to coordinate concurrent access. In memory
		// doesn't support this, so limit the number of connections to 1.
		db.SetMaxOpenConns(1)
	}

	errCheck := func(err error) bool {
		sqlErr, ok := err.(sqlite3.Error)
		if !ok {
			return false
		}
		return sqlErr.ExtendedCode == sqlite3.ErrConstraintPrimaryKey
	}

	c := &conn{db, flavorSQLite3, logger, errCheck}
	if _, err := c.migrate(); err != nil {
		return nil, fmt.Errorf("failed to perform migrations: %v", err)
	}
	return c, nil
}

// nolint
const (
	// postgres SSL modes
	pgSSLDisable    = "disable"
	pgSSLRequire    = "require"
	pgSSLVerifyCA   = "verify-ca"
	pgSSLVerifyFull = "verify-full"
)

// nolint
const (
	// MySQL SSL modes
	mysqlSSLTrue       = "true"
	mysqlSSLFalse      = "false"
	mysqlSSLSkipVerify = "skip-verify"
	mysqlSSLCustom     = "custom"
)

// NetworkDB contains options common to SQL databases accessed over network.
type NetworkDB struct {
	Database string
	User     string
	Password string
	Host     string
	Port     uint16

	ConnectionTimeout int // Seconds

	// database/sql tunables, see
	// https://golang.org/pkg/database/sql/#DB.SetConnMaxLifetime and below
	// Note: defaults will be set if these are 0
	MaxOpenConns    int // default: 5
	MaxIdleConns    int // default: 5
	ConnMaxLifetime int // Seconds, default: not set
}

// SSL represents SSL options for network databases.
type SSL struct {
	Mode   string
	CAFile string
	// Files for client auth.
	KeyFile  string
	CertFile string
}

// Postgres options for creating an SQL db.
type Postgres struct {
	NetworkDB

	SSL SSL `json:"ssl" yaml:"ssl"`
}

// Open creates a new storage implementation backed by Postgres.
func (p *Postgres) Open(logger log.Logger) (storage.Storage, error) {
	conn, err := p.open(logger)
	if err != nil {
		return nil, err
	}
	return conn, nil
}

var strEsc = regexp.MustCompile(`([\\'])`)

func dataSourceStr(str string) string {
	return "'" + strEsc.ReplaceAllString(str, `\$1`) + "'"
}

// createDataSourceName takes the configuration provided via the Postgres
// struct to create a data-source name that Go's database/sql package can
// make use of.
func (p *Postgres) createDataSourceName() string {
	parameters := []string{}

	addParam := func(key, val string) {
		parameters = append(parameters, fmt.Sprintf("%s=%s", key, val))
	}

	addParam("connect_timeout", strconv.Itoa(p.ConnectionTimeout))

	// detect host:port for backwards-compatibility
	host, port, err := net.SplitHostPort(p.Host)
	if err != nil {
		// not host:port, probably unix socket or bare address

		host = p.Host

		if p.Port != 0 {
			port = strconv.Itoa(int(p.Port))
		}
	}

	if host != "" {
		addParam("host", dataSourceStr(host))
	}

	if port != "" {
		addParam("port", port)
	}

	if p.User != "" {
		addParam("user", dataSourceStr(p.User))
	}

	if p.Password != "" {
		addParam("password", dataSourceStr(p.Password))
	}

	if p.Database != "" {
		addParam("dbname", dataSourceStr(p.Database))
	}

	if p.SSL.Mode == "" {
		// Assume the strictest mode if unspecified.
		addParam("sslmode", dataSourceStr(pgSSLVerifyFull))
	} else {
		addParam("sslmode", dataSourceStr(p.SSL.Mode))
	}

	if p.SSL.CAFile != "" {
		addParam("sslrootcert", dataSourceStr(p.SSL.CAFile))
	}

	if p.SSL.CertFile != "" {
		addParam("sslcert", dataSourceStr(p.SSL.CertFile))
	}

	if p.SSL.KeyFile != "" {
		addParam("sslkey", dataSourceStr(p.SSL.KeyFile))
	}

	return strings.Join(parameters, " ")
}

func (p *Postgres) open(logger log.Logger) (*conn, error) {
	dataSourceName := p.createDataSourceName()

	db, err := sql.Open("postgres", dataSourceName)
	if err != nil {
		return nil, err
	}

	// set database/sql tunables if configured
	if p.ConnMaxLifetime != 0 {
		db.SetConnMaxLifetime(time.Duration(p.ConnMaxLifetime) * time.Second)
	}

	if p.MaxIdleConns == 0 {
		db.SetMaxIdleConns(5)
	} else {
		db.SetMaxIdleConns(p.MaxIdleConns)
	}

	if p.MaxOpenConns == 0 {
		db.SetMaxOpenConns(5)
	} else {
		db.SetMaxOpenConns(p.MaxOpenConns)
	}

	errCheck := func(err error) bool {
		sqlErr, ok := err.(*pq.Error)
		if !ok {
			return false
		}
		return sqlErr.Code == pgErrUniqueViolation
	}

	c := &conn{db, flavorPostgres, logger, errCheck}
	if _, err := c.migrate(); err != nil {
		return nil, fmt.Errorf("failed to perform migrations: %v", err)
	}
	return c, nil
}

// MySQL options for creating a MySQL db.
type MySQL struct {
	NetworkDB

	SSL SSL `json:"ssl" yaml:"ssl"`

	// TODO(pborzenkov): used by tests to reduce lock wait timeout. Should
	// we make it exported and allow users to provide arbitrary params?
	params map[string]string
}

// Open creates a new storage implementation backed by MySQL.
func (s *MySQL) Open(logger log.Logger) (storage.Storage, error) {
	conn, err := s.open(logger)
	if err != nil {
		return nil, err
	}
	return conn, nil
}

func (s *MySQL) open(logger log.Logger) (*conn, error) {
	cfg := mysql.Config{
		User:                 s.User,
		Passwd:               s.Password,
		DBName:               s.Database,
		AllowNativePasswords: true,

		Timeout: time.Second * time.Duration(s.ConnectionTimeout),

		ParseTime: true,
		Params: map[string]string{
			"transaction_isolation": "'SERIALIZABLE'",
		},
	}
	if s.Host != "" {
		if s.Host[0] != '/' {
			cfg.Net = "tcp"
			cfg.Addr = s.Host
		} else {
			cfg.Net = "unix"
			cfg.Addr = s.Host
		}
	}
	if s.SSL.CAFile != "" || s.SSL.CertFile != "" || s.SSL.KeyFile != "" {
		if err := s.makeTLSConfig(); err != nil {
			return nil, fmt.Errorf("failed to make TLS config: %v", err)
		}
		cfg.TLSConfig = mysqlSSLCustom
	} else if s.SSL.Mode == "" {
		cfg.TLSConfig = mysqlSSLTrue
	} else {
		cfg.TLSConfig = s.SSL.Mode
	}
	for k, v := range s.params {
		cfg.Params[k] = v
	}

	db, err := sql.Open("mysql", cfg.FormatDSN())
	if err != nil {
		return nil, err
	}

	err = db.Ping()
	if err != nil {
		if mysqlErr, ok := err.(*mysql.MySQLError); ok && mysqlErr.Number == mysqlErrUnknownSysVar {
			logger.Info("reconnecting with MySQL pre-5.7.20 compatibilty mode")

			// MySQL 5.7.20 introduced transaction_isolation and deprecated tx_isolation.
			// MySQL 8.0 doesn't have tx_isolation at all.
			// https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_transaction_isolation
			delete(cfg.Params, "transaction_isolation")
			cfg.Params["tx_isolation"] = "'SERIALIZABLE'"

			db, err = sql.Open("mysql", cfg.FormatDSN())
			if err != nil {
				return nil, err
			}
		} else {
			return nil, err
		}
	}

	errCheck := func(err error) bool {
		sqlErr, ok := err.(*mysql.MySQLError)
		if !ok {
			return false
		}
		return sqlErr.Number == mysqlErrDupEntry ||
			sqlErr.Number == mysqlErrDupEntryWithKeyName
	}

	c := &conn{db, flavorMySQL, logger, errCheck}
	if _, err := c.migrate(); err != nil {
		return nil, fmt.Errorf("failed to perform migrations: %v", err)
	}
	return c, nil
}

func (s *MySQL) makeTLSConfig() error {
	cfg := &tls.Config{}
	if s.SSL.CAFile != "" {
		rootCertPool := x509.NewCertPool()
		pem, err := ioutil.ReadFile(s.SSL.CAFile)
		if err != nil {
			return err
		}
		if ok := rootCertPool.AppendCertsFromPEM(pem); !ok {
			return fmt.Errorf("failed to append PEM")
		}
		cfg.RootCAs = rootCertPool
	}
	if s.SSL.CertFile != "" && s.SSL.KeyFile != "" {
		clientCert := make([]tls.Certificate, 0, 1)
		certs, err := tls.LoadX509KeyPair(s.SSL.CertFile, s.SSL.KeyFile)
		if err != nil {
			return err
		}
		clientCert = append(clientCert, certs)
		cfg.Certificates = clientCert
	}

	mysql.RegisterTLSConfig(mysqlSSLCustom, cfg)
	return nil
}
<|endoftext|>


<commit_before>package store

import (
	"fmt"
	"testing"
	"time"
)

func TestWatch(t *testing.T) {

	s := CreateStore(100)

	watchers := make([]*Watcher, 10)

	for i, _ := range watchers {

		// create a new watcher
		watchers[i] = NewWatcher()
		// add to the watchers list
		s.AddWatcher("foo", watchers[i], 0)

	}

	s.Set("/foo/foo", "bar", time.Unix(0, 0), 1)

	for _, watcher := range watchers {

		// wait for the notification for any changing
		res := <-watcher.C

		if res == nil {
			t.Fatal("watcher is cleared")
		}
	}

	for i, _ := range watchers {

		// create a new watcher
		watchers[i] = NewWatcher()
		// add to the watchers list
		s.AddWatcher("foo/foo/foo", watchers[i], 0)

	}

	s.watcher.stopWatchers()

	for _, watcher := range watchers {

		// wait for the notification for any changing
		res := <-watcher.C

		if res != nil {
			t.Fatal("watcher is cleared")
		}
	}
}
<commit_msg>add watch benchmark<commit_after>package store

import (
	"math/rand"
	"strconv"
	"testing"
	"time"
)

func TestWatch(t *testing.T) {

	s := CreateStore(100)

	watchers := make([]*Watcher, 10)

	for i, _ := range watchers {

		// create a new watcher
		watchers[i] = NewWatcher()
		// add to the watchers list
		s.AddWatcher("foo", watchers[i], 0)

	}

	s.Set("/foo/foo", "bar", time.Unix(0, 0), 1)

	for _, watcher := range watchers {

		// wait for the notification for any changing
		res := <-watcher.C

		if res == nil {
			t.Fatal("watcher is cleared")
		}
	}

	for i, _ := range watchers {

		// create a new watcher
		watchers[i] = NewWatcher()
		// add to the watchers list
		s.AddWatcher("foo/foo/foo", watchers[i], 0)

	}

	s.watcher.stopWatchers()

	for _, watcher := range watchers {

		// wait for the notification for any changing
		res := <-watcher.C

		if res != nil {
			t.Fatal("watcher is cleared")
		}
	}
}

// BenchmarkWatch creates 10K watchers watch at /foo/[paht] each time.
// Path is randomly chosen with max depth 10.
// It should take less than 15ms to wake up 10K watchers.
func BenchmarkWatch(b *testing.B) {
	s := CreateStore(100)

	key := make([]string, 10000)
	for i := 0; i < 10000; i++ {

		key[i] = "/foo/"
		depth := rand.Intn(10)

		for j := 0; j < depth; j++ {
			key[i] += "/" + strconv.Itoa(rand.Int()%10)
		}
	}

	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		watchers := make([]*Watcher, 10000)
		for i := 0; i < 10000; i++ {
			// create a new watcher
			watchers[i] = NewWatcher()
			// add to the watchers list
			s.AddWatcher(key[i], watchers[i], 0)
		}

		s.watcher.stopWatchers()

		for _, watcher := range watchers {
			// wait for the notification for any changing
			<-watcher.C
		}

		s.watcher = newWatcherHub()
	}
}
<|endoftext|>


<commit_before>// Copyright 2017 The Upspin Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Command camserver is an Upspin Directory and Store server that serves JPEG
// images read from a webcam. It requires an ffmpeg binary be present in PATH.
// It only works with the built in camera on MacOS machines, for now.
package main

// TODO(adg): implement Watch.

import (
	"flag"
	"io/ioutil"
	"mime/multipart"
	"net/http"
	"os/exec"
	"sync"
	"time"

	"upspin.io/access"
	"upspin.io/bind"
	"upspin.io/cache"
	"upspin.io/client"
	"upspin.io/cloud/https"
	"upspin.io/config"
	"upspin.io/errors"
	"upspin.io/flags"
	"upspin.io/key/sha256key"
	"upspin.io/log"
	"upspin.io/pack"
	"upspin.io/path"
	"upspin.io/rpc/dirserver"
	"upspin.io/rpc/storeserver"
	"upspin.io/serverutil"
	"upspin.io/transports"
	"upspin.io/upspin"

	_ "upspin.io/pack/eeintegrity"
)

func main() {
	readers := flag.String("readers", "all", "comma-separated list of users to be given read/list access")
	flags.Parse(flags.Server)

	cfg, err := config.FromFile(flags.Config)
	if err != nil {
		log.Fatal(err)
	}
	transports.Init(cfg)
	addr := upspin.NetAddr(flags.NetAddr)
	ep := upspin.Endpoint{
		Transport: upspin.Remote,
		NetAddr:   addr,
	}

	s, err := newServer(cfg, ep, *readers)
	if err != nil {
		log.Fatal(err)
	}

	http.Handle("/api/Dir/", dirserver.New(cfg, dirServer{server: s}, addr))
	http.Handle("/api/Store/", storeserver.New(cfg, storeServer{server: s}, addr))

	https.ListenAndServeFromFlags(nil)
}

// server is the base for a combined Upspin DirServer and StoreServer
// implementation that serves frames from a webcam.
// Each frame is served as frame.jpg in the root of the tree of cfg's user.
// As each new frame is read from the camera, it replaces frame.jpg.
type server struct {
	// Set by newServer.
	cfg          upspin.Config
	ep           upspin.Endpoint
	accessEntry  *upspin.DirEntry
	accessBytes  []byte
	framePacking upspin.Packing
	readerKeys   []upspin.PublicKey

	// Set by Dial.
	user upspin.UserName

	// state is embedded here as a struct pointer so that the Dial methods
	// do not make a copy of it when they copy the server struct.
	*state
}

// state contains mutable state shared by all users of the server.
type state struct {
	frameData *cache.LRU // map[upspin.Reference][]byte
	sequence  int64      // read/written only by capture method

	mu         sync.Mutex
	frameEntry *upspin.DirEntry // The current frame.
}

// dirServer is a shim around server that implements upspin.DirServer.
type dirServer struct {
	*server
	stubService
}

// storeServer is a shim around server that implements upspin.StoreServer.
type storeServer struct {
	*server
	stubService
}

const (
	accessFileName = access.AccessFile
	accessRef      = upspin.Reference(accessFileName)
	frameFileName  = "frame.jpg"
	numFrames      = 100 // The number of frames to keep in memory.
)

var (
	accessRefdata     = upspin.Refdata{Reference: accessRef}
	errNotImplemented = errors.Str("not implemented")
)

// newServer initializes a server with the given Config and Endpoint,
// and starts ffmpeg to read frames from the built-in webcam.
func newServer(cfg upspin.Config, ep upspin.Endpoint, readers string) (*server, error) {
	s := &server{
		cfg:          cfg,
		ep:           ep,
		framePacking: upspin.EEPack,
		state: &state{
			frameData: cache.NewLRU(numFrames),
		},
	}

	accessFile := []byte("Read, List: " + readers)
	if readers == "all" {
		s.framePacking = upspin.EEIntegrityPack
	}

	var err error
	s.accessEntry, s.accessBytes, err = s.pack(upspin.EEIntegrityPack, accessFileName, accessRef, accessFile, 0)
	if err != nil {
		return nil, err
	}

	if s.framePacking == upspin.EEPack {
		a, err := access.Parse(s.accessEntry.Name, accessFile)
		if err != nil {
			return nil, err
		}
		users, err := a.Users(access.Read, client.New(cfg).Get)
		if err != nil {
			return nil, err
		}
		key, err := bind.KeyServer(cfg, cfg.KeyEndpoint())
		if err != nil {
			return nil, err
		}
		keys := []upspin.PublicKey{cfg.Factotum().PublicKey()}
		for _, name := range users {
			if name == cfg.UserName() {
				continue
			}
			user, err := key.Lookup(name)
			if err != nil {
				return nil, err
			}
			keys = append(keys, user.PublicKey)
		}
		s.readerKeys = keys
	}

	if err := s.capture(); err != nil {
		return nil, err
	}

	return s, nil
}

// pack packs the given file using packing
// and returns the resulting DirEntry and ciphertext.
func (s *server) pack(packing upspin.Packing, filePath string, ref upspin.Reference, data []byte, seq int64) (*upspin.DirEntry, []byte, error) {
	name := upspin.PathName(s.cfg.UserName()) + "/" + upspin.PathName(filePath)
	de := &upspin.DirEntry{
		Writer:     s.cfg.UserName(),
		Name:       name,
		SignedName: name,
		Packing:    packing,
		Time:       upspin.Now(),
		Sequence:   seq,
	}

	bp, err := pack.Lookup(packing).Pack(s.cfg, de)
	if err != nil {
		return nil, nil, err
	}
	cipher, err := bp.Pack(data)
	if err != nil {
		return nil, nil, err
	}
	bp.SetLocation(upspin.Location{
		Endpoint:  s.ep,
		Reference: ref,
	})
	return de, cipher, bp.Close()
}

// capture starts ffmpeg to read a video stream from the built-in webcam,
// packing each frame as a DirEntry and storing it in frameEntry and
// frameBytes. It returns after the first frame has been packed.
func (s *server) capture() error {
	// TODO(adg): make this command line configurable.
	cmd := exec.Command("ffmpeg",
		// Input from the FaceTime webcam (present in most Macs).
		"-f", "avfoundation", "-pix_fmt", "0rgb", "-s", "1280x720", "-r", "30", "-i", "FaceTime",
		// Output Motion JPEG at 2fps at high quality.
		"-f", "mpjpeg", "-r", "2", "-b:v", "1M", "-")
	out, err := cmd.StdoutPipe()
	if err != nil {
		return err
	}
	if err := cmd.Start(); err != nil {
		return err
	}
	mr := multipart.NewReader(out, "ffserver")
	readFrame := func() error {
		p, err := mr.NextPart()
		if err != nil {
			return err
		}
		b, err := ioutil.ReadAll(p)
		if err != nil {
			return err
		}
		ref := upspin.Reference(sha256key.Of(b).String())
		de, cipher, err := s.pack(s.framePacking, frameFileName, ref, b, s.sequence)
		if err != nil {
			return err
		}
		if s.framePacking == upspin.EEPack {
			packdata := make([]*[]byte, 1)
			packdata[0] = &de.Packdata
			pack.Lookup(upspin.EEPack).Share(s.cfg, s.readerKeys, packdata)
		}
		s.sequence++
		s.frameData.Add(ref, cipher)
		s.mu.Lock()
		s.frameEntry = de
		s.mu.Unlock()
		return nil
	}
	// Read the first frame so that when this function returns
	// the frameEntry and frameData fields are initialized.
	// This has the pleasant side effect of making sure that
	// ffmpeg is working correctly.
	if err := readFrame(); err != nil {
		return err
	}
	go func() {
		for {
			if err := readFrame(); err != nil {
				log.Println("readFrame:", err)
				return
			}
		}
	}()
	return nil
}

// upspin.Service and upspin.Dialer methods.

func (s dirServer) Endpoint() upspin.Endpoint { return s.ep }

func (s dirServer) Dial(cfg upspin.Config, ep upspin.Endpoint) (upspin.Service, error) {
	s2 := *s.server
	s2.user = cfg.UserName()
	return dirServer{server: &s2}, nil
}

func (s storeServer) Endpoint() upspin.Endpoint { return s.ep }

func (s storeServer) Dial(cfg upspin.Config, ep upspin.Endpoint) (upspin.Service, error) {
	s2 := *s.server
	s2.user = cfg.UserName()
	return storeServer{server: &s2}, nil
}

// upspin.DirServer methods.

func (s dirServer) Lookup(name upspin.PathName) (*upspin.DirEntry, error) {
	p, err := path.Parse(name)
	if err != nil {
		return nil, err
	}
	if p.User() != s.cfg.UserName() {
		return nil, errors.E(name, errors.NotExist)
	}

	fp := p.FilePath()
	switch fp {
	case "": // Root directory.
		return &upspin.DirEntry{
			Name:       p.Path(),
			SignedName: p.Path(),
			Attr:       upspin.AttrDirectory,
			Time:       upspin.Now(),
		}, nil
	case accessFileName:
		return s.accessEntry, nil
	case frameFileName:
		s.mu.Lock()
		defer s.mu.Unlock()
		return s.frameEntry, nil
	default:
		return nil, errors.E(name, errors.NotExist)
	}
}

func (s dirServer) Glob(pattern string) ([]*upspin.DirEntry, error) {
	return serverutil.Glob(pattern, s.Lookup, s.listDir)
}

func (s dirServer) listDir(name upspin.PathName) ([]*upspin.DirEntry, error) {
	p, err := path.Parse(name)
	if err != nil {
		return nil, err
	}
	if p.User() != s.cfg.UserName() || p.FilePath() != "" {
		return nil, errors.E(name, errors.NotExist)
	}
	s.mu.Lock()
	defer s.mu.Unlock()
	return []*upspin.DirEntry{
		s.accessEntry,
		s.frameEntry,
	}, nil
}

func (s dirServer) WhichAccess(upspin.PathName) (*upspin.DirEntry, error) {
	return s.accessEntry, nil
}

func (s dirServer) Watch(name upspin.PathName, order int64, done <-chan struct{}) (<-chan upspin.Event, error) {
	return nil, upspin.ErrNotSupported
}

func (s dirServer) Put(*upspin.DirEntry) (*upspin.DirEntry, error) {
	return nil, errNotImplemented
}

func (s dirServer) Delete(upspin.PathName) (*upspin.DirEntry, error) {
	return nil, errNotImplemented
}

// upspin.StoreServer methods.

func (s storeServer) Get(ref upspin.Reference) ([]byte, *upspin.Refdata, []upspin.Location, error) {
	if ref == accessRef {
		return s.accessBytes, &accessRefdata, nil, nil
	}
	if b, ok := s.frameData.Get(ref); ok {
		return b.([]byte), &upspin.Refdata{
			Reference: ref,
			Volatile:  true,
			Duration:  time.Second,
		}, nil, nil
	}
	return nil, nil, nil, errors.E(errors.NotExist)
}

func (s storeServer) Put([]byte) (*upspin.Refdata, error) {
	return nil, errNotImplemented
}

func (s storeServer) Delete(upspin.Reference) error {
	return errNotImplemented
}

// stubService provides a stub implementation of upspin.Service.
type stubService struct {
}

func (s stubService) Endpoint() upspin.Endpoint { return upspin.Endpoint{} }
func (s stubService) Ping() bool                { return true }
func (s stubService) Close()                    {}
<commit_msg>exp/cmd/camserver: implement DirServer.Watch<commit_after>// Copyright 2017 The Upspin Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Command camserver is an Upspin Directory and Store server that serves JPEG
// images read from a webcam. It requires an ffmpeg binary be present in PATH.
// It only works with the built in camera on MacOS machines, for now.
package main

import (
	"flag"
	"io/ioutil"
	"mime/multipart"
	"net/http"
	"os/exec"
	"sync"
	"time"

	"upspin.io/access"
	"upspin.io/bind"
	"upspin.io/cache"
	"upspin.io/client"
	"upspin.io/cloud/https"
	"upspin.io/config"
	"upspin.io/errors"
	"upspin.io/flags"
	"upspin.io/key/sha256key"
	"upspin.io/log"
	"upspin.io/pack"
	"upspin.io/path"
	"upspin.io/rpc/dirserver"
	"upspin.io/rpc/storeserver"
	"upspin.io/serverutil"
	"upspin.io/transports"
	"upspin.io/upspin"

	_ "upspin.io/pack/eeintegrity"
)

func main() {
	readers := flag.String("readers", "all", "comma-separated list of users to be given read/list access")
	flags.Parse(flags.Server)

	cfg, err := config.FromFile(flags.Config)
	if err != nil {
		log.Fatal(err)
	}
	transports.Init(cfg)
	addr := upspin.NetAddr(flags.NetAddr)
	ep := upspin.Endpoint{
		Transport: upspin.Remote,
		NetAddr:   addr,
	}

	s, err := newServer(cfg, ep, *readers)
	if err != nil {
		log.Fatal(err)
	}

	http.Handle("/api/Dir/", dirserver.New(cfg, dirServer{server: s}, addr))
	http.Handle("/api/Store/", storeserver.New(cfg, storeServer{server: s}, addr))

	https.ListenAndServeFromFlags(nil)
}

// server is the base for a combined Upspin DirServer and StoreServer
// implementation that serves frames from a webcam.
// Each frame is served as frame.jpg in the root of the tree of cfg's user.
// As each new frame is read from the camera, it replaces frame.jpg.
type server struct {
	// Set by newServer.
	cfg          upspin.Config
	ep           upspin.Endpoint
	rootEntry    *upspin.DirEntry
	accessEntry  *upspin.DirEntry
	accessBytes  []byte
	framePacking upspin.Packing
	readerKeys   []upspin.PublicKey

	// Set by Dial.
	user upspin.UserName

	// state is embedded here as a struct pointer so that the Dial methods
	// do not make a copy of it when they copy the server struct.
	*state
}

// state contains mutable state shared by all users of the server.
type state struct {
	frameData *cache.LRU // map[upspin.Reference][]byte
	sequence  int64      // read/written only by capture method

	mu         sync.Mutex
	update     *sync.Cond
	frameEntry *upspin.DirEntry // The current frame.
}

// dirServer is a shim around server that implements upspin.DirServer.
type dirServer struct {
	*server
	stubService
}

// storeServer is a shim around server that implements upspin.StoreServer.
type storeServer struct {
	*server
	stubService
}

const (
	accessFileName    = access.AccessFile
	accessRef         = upspin.Reference(accessFileName)
	frameFileName     = "frame.jpg"
	numFrames         = 100 // The number of frames to keep in memory.
	watchEventTimeout = 5 * time.Second
)

var (
	accessRefdata     = upspin.Refdata{Reference: accessRef}
	errNotImplemented = errors.Str("not implemented")
)

// newServer initializes a server with the given Config and Endpoint,
// and starts ffmpeg to read frames from the built-in webcam.
func newServer(cfg upspin.Config, ep upspin.Endpoint, readers string) (*server, error) {
	s := &server{
		cfg:          cfg,
		ep:           ep,
		framePacking: upspin.EEPack,
		state: &state{
			frameData: cache.NewLRU(numFrames),
		},
	}
	s.update = sync.NewCond(&s.mu)

	accessFile := []byte("Read, List: " + readers)
	if readers == "all" {
		s.framePacking = upspin.EEIntegrityPack
	}

	rootName := upspin.PathName(cfg.UserName() + "/")
	s.rootEntry = &upspin.DirEntry{
		Name:       rootName,
		SignedName: rootName,
		Attr:       upspin.AttrDirectory,
		Time:       upspin.Now(),
	}

	var err error
	s.accessEntry, s.accessBytes, err = s.pack(upspin.EEIntegrityPack, accessFileName, accessRef, accessFile, 0)
	if err != nil {
		return nil, err
	}

	if s.framePacking == upspin.EEPack {
		a, err := access.Parse(s.accessEntry.Name, accessFile)
		if err != nil {
			return nil, err
		}
		users, err := a.Users(access.Read, client.New(cfg).Get)
		if err != nil {
			return nil, err
		}
		key, err := bind.KeyServer(cfg, cfg.KeyEndpoint())
		if err != nil {
			return nil, err
		}
		keys := []upspin.PublicKey{cfg.Factotum().PublicKey()}
		for _, name := range users {
			if name == cfg.UserName() {
				continue
			}
			user, err := key.Lookup(name)
			if err != nil {
				return nil, err
			}
			keys = append(keys, user.PublicKey)
		}
		s.readerKeys = keys
	}

	if err := s.capture(); err != nil {
		return nil, err
	}

	return s, nil
}

// pack packs the given file using packing
// and returns the resulting DirEntry and ciphertext.
func (s *server) pack(packing upspin.Packing, filePath string, ref upspin.Reference, data []byte, seq int64) (*upspin.DirEntry, []byte, error) {
	name := upspin.PathName(s.cfg.UserName()) + "/" + upspin.PathName(filePath)
	de := &upspin.DirEntry{
		Writer:     s.cfg.UserName(),
		Name:       name,
		SignedName: name,
		Packing:    packing,
		Time:       upspin.Now(),
		Sequence:   seq,
	}

	bp, err := pack.Lookup(packing).Pack(s.cfg, de)
	if err != nil {
		return nil, nil, err
	}
	cipher, err := bp.Pack(data)
	if err != nil {
		return nil, nil, err
	}
	bp.SetLocation(upspin.Location{
		Endpoint:  s.ep,
		Reference: ref,
	})
	return de, cipher, bp.Close()
}

// capture starts ffmpeg to read a video stream from the built-in webcam,
// packing each frame as a DirEntry and storing it in frameEntry and
// frameBytes. It returns after the first frame has been packed.
func (s *server) capture() error {
	// TODO(adg): make this command line configurable.
	cmd := exec.Command("ffmpeg",
		// Input from the FaceTime webcam (present in most Macs).
		"-f", "avfoundation", "-pix_fmt", "0rgb", "-s", "1280x720", "-r", "30", "-i", "FaceTime",
		// Output Motion JPEG at 2fps at high quality.
		"-f", "mpjpeg", "-r", "2", "-b:v", "1M", "-")
	out, err := cmd.StdoutPipe()
	if err != nil {
		return err
	}
	if err := cmd.Start(); err != nil {
		return err
	}
	mr := multipart.NewReader(out, "ffserver")

	readFrame := func() error {
		s.sequence++

		// Read the next frame from the ffmpeg output.
		p, err := mr.NextPart()
		if err != nil {
			return err
		}
		b, err := ioutil.ReadAll(p)
		if err != nil {
			return err
		}

		// Pack the frame.
		ref := upspin.Reference(sha256key.Of(b).String())
		de, cipher, err := s.pack(s.framePacking, frameFileName, ref, b, s.sequence)
		if err != nil {
			return err
		}

		// Share the frame with the configured readers.
		if s.framePacking == upspin.EEPack {
			packdata := make([]*[]byte, 1)
			packdata[0] = &de.Packdata
			pack.Lookup(upspin.EEPack).Share(s.cfg, s.readerKeys, packdata)
		}

		// Update frameData and frameEntry.
		s.frameData.Add(ref, cipher)
		s.mu.Lock()
		s.frameEntry = de
		s.mu.Unlock()

		// Notify any watchers that a new frame is available.
		s.update.Broadcast()
		return nil
	}

	// Read the first frame so that when this function returns
	// the frameEntry and frameData fields are initialized.
	// This has the pleasant side effect of making sure that
	// ffmpeg is working correctly.
	if err := readFrame(); err != nil {
		return err
	}
	go func() {
		// Read frames forever.
		for {
			if err := readFrame(); err != nil {
				log.Println("readFrame:", err)
				return
			}
		}
	}()
	return nil
}

// upspin.Service and upspin.Dialer methods.

func (s dirServer) Endpoint() upspin.Endpoint { return s.ep }

func (s dirServer) Dial(cfg upspin.Config, ep upspin.Endpoint) (upspin.Service, error) {
	s2 := *s.server
	s2.user = cfg.UserName()
	return dirServer{server: &s2}, nil
}

func (s storeServer) Endpoint() upspin.Endpoint { return s.ep }

func (s storeServer) Dial(cfg upspin.Config, ep upspin.Endpoint) (upspin.Service, error) {
	s2 := *s.server
	s2.user = cfg.UserName()
	return storeServer{server: &s2}, nil
}

// upspin.DirServer methods.

func (s dirServer) Lookup(name upspin.PathName) (*upspin.DirEntry, error) {
	p, err := path.Parse(name)
	if err != nil {
		return nil, err
	}
	if p.User() != s.cfg.UserName() {
		return nil, errors.E(name, errors.NotExist)
	}

	fp := p.FilePath()
	switch fp {
	case "": // Root directory.
		return s.rootEntry, nil
	case accessFileName:
		return s.accessEntry, nil
	case frameFileName:
		s.mu.Lock()
		defer s.mu.Unlock()
		return s.frameEntry, nil
	default:
		return nil, errors.E(name, errors.NotExist)
	}
}

func (s dirServer) Glob(pattern string) ([]*upspin.DirEntry, error) {
	return serverutil.Glob(pattern, s.Lookup, s.listDir)
}

func (s dirServer) listDir(name upspin.PathName) ([]*upspin.DirEntry, error) {
	p, err := path.Parse(name)
	if err != nil {
		return nil, err
	}
	if p.User() != s.cfg.UserName() || p.FilePath() != "" {
		return nil, errors.E(name, errors.NotExist)
	}
	s.mu.Lock()
	defer s.mu.Unlock()
	return []*upspin.DirEntry{
		s.accessEntry,
		s.frameEntry,
	}, nil
}

func (s dirServer) WhichAccess(upspin.PathName) (*upspin.DirEntry, error) {
	return s.accessEntry, nil
}

func (s dirServer) Watch(name upspin.PathName, order int64, done <-chan struct{}) (<-chan upspin.Event, error) {
	p, err := path.Parse(name)
	if err != nil {
		return nil, err
	}
	if p.User() != s.cfg.UserName() {
		return nil, errors.E(name, errors.NotExist)
	}

	// Determine which paths the user is interested in watching.
	var (
		sendRoot   = false
		sendAccess = false
		sendFrame  = false
	)
	switch p.FilePath() {
	case "":
		sendRoot = true
		sendAccess = true
		sendFrame = true
	case accessFileName:
		sendAccess = true
	case frameFileName:
		sendFrame = true
	}
	switch order {
	case upspin.WatchStart, upspin.WatchCurrent:
		// OK to send everything.
	default: // order >= 0 (includes upspin.WatchNew)
		sendRoot = false
		sendAccess = false
	}

	ch := make(chan upspin.Event)
	timer := time.NewTimer(watchEventTimeout)
	// send sends de to ch, observing the done channel and timeouts.
	// It returns true if done is closed or the send times out.
	send := func(de *upspin.DirEntry) (isDone bool) {
		if !timer.Stop() {
			<-timer.C
		}
		timer.Reset(watchEventTimeout)
		select {
		case ch <- upspin.Event{Entry: de}:
			return false
		case <-done:
			return true
		case <-timer.C:
			return true
		}
	}
	go func() {
		defer close(ch)
		if sendRoot && send(s.rootEntry) {
			return
		}
		if sendAccess && send(s.accessEntry) {
			return
		}
		if !sendFrame {
			// The watched path will never exist or change.
			<-done
			return
		}
		for {
			// Wait for a new frame to become available.
			// (The frame's Sequence number is its order.)
			s.mu.Lock()
			for s.frameEntry.Sequence <= order {
				s.update.Wait()
			}
			de := s.frameEntry
			s.mu.Unlock()

			// Send the frame and update order.
			if send(de) {
				return
			}
			order = de.Sequence
		}
	}()
	return ch, nil
}

func (s dirServer) Put(*upspin.DirEntry) (*upspin.DirEntry, error) {
	return nil, errNotImplemented
}

func (s dirServer) Delete(upspin.PathName) (*upspin.DirEntry, error) {
	return nil, errNotImplemented
}

// upspin.StoreServer methods.

func (s storeServer) Get(ref upspin.Reference) ([]byte, *upspin.Refdata, []upspin.Location, error) {
	if ref == accessRef {
		return s.accessBytes, &accessRefdata, nil, nil
	}
	if b, ok := s.frameData.Get(ref); ok {
		return b.([]byte), &upspin.Refdata{
			Reference: ref,
			Volatile:  true,
			Duration:  time.Second,
		}, nil, nil
	}
	return nil, nil, nil, errors.E(errors.NotExist)
}

func (s storeServer) Put([]byte) (*upspin.Refdata, error) {
	return nil, errNotImplemented
}

func (s storeServer) Delete(upspin.Reference) error {
	return errNotImplemented
}

// stubService provides a stub implementation of upspin.Service.
type stubService struct {
}

func (s stubService) Endpoint() upspin.Endpoint { return upspin.Endpoint{} }
func (s stubService) Ping() bool                { return true }
func (s stubService) Close()                    {}
<|endoftext|>


<commit_before>package control

import (
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"strings"

	log "github.com/Sirupsen/logrus"
	"gopkg.in/yaml.v2"

	"github.com/codegangsta/cli"
	"github.com/rancherio/os/config"
	"github.com/rancherio/os/docker"
)

func configSubcommands() []cli.Command {
	return []cli.Command{
		{
			Name:   "get",
			Usage:  "get value",
			Action: configGet,
		},
		{
			Name:   "set",
			Usage:  "set a value",
			Action: configSet,
		},
		{
			Name:   "import",
			Usage:  "import configuration from standard in or a file",
			Action: configImport,
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "input, i",
					Usage: "File from which to read",
				},
			},
		},
		{
			Name:  "export",
			Usage: "dump full configuration",
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "output, o",
					Usage: "File to which to save",
				},
				cli.BoolFlag{
					Name:  "full",
					Usage: "Include full configuration, not just writable fields",
				},
			},
			Action: configSave,
		},
	}
}

func getConfigData() (map[interface{}]interface{}, error) {
	cfg, err := config.LoadConfig()
	if err != nil {
		log.Fatal(err)
	}

	content, err := cfg.Dump()
	if err != nil {
		log.Fatal(err)
	}

	data := make(map[interface{}]interface{})
	err = yaml.Unmarshal([]byte(content), data)

	return data, err
}

func configImport(c *cli.Context) {
	var input io.Reader
	var err error
	input = os.Stdin

	inputFile := c.String("input")
	if inputFile != "" {
		input, err = os.Open(inputFile)
		if err != nil {
			log.Fatal(err)
		}
	}

	bytes, err := ioutil.ReadAll(input)
	if err != nil {
		log.Fatal(err)
	}

	err = mergeConfig(bytes)
	if err != nil {
		log.Fatal(err)
	}
}

func mergeConfig(bytes []byte) error {
	var newConfig config.Config

	err := yaml.Unmarshal(bytes, &newConfig)
	if err != nil {
		return err
	}

	cfg, err := config.LoadConfig()
	if err != nil {
		return err
	}

	reboot, err := cfg.Merge(newConfig)
	if err != nil {
		return err
	}

	err = cfg.Save()
	if err != nil {
		return err
	}

	if reboot {
		fmt.Println("Reboot needed")
	}

	return err
}

func configSet(c *cli.Context) {
	key := c.Args().Get(0)
	value := c.Args().Get(1)
	if key == "" {
		return
	}

	data, err := getConfigData()
	getOrSetVal(key, data, value)

	bytes, err := yaml.Marshal(data)
	if err != nil {
		log.Fatal(err)
	}

	err = mergeConfig(bytes)
	if err != nil {
		log.Fatal(err)
	}
}

func configGet(c *cli.Context) {
	arg := c.Args().Get(0)
	if arg == "" {
		return
	}

	data, err := getConfigData()
	if err != nil {
		log.Fatal(err)
	}

	val := getOrSetVal(arg, data, nil)

	printYaml := false
	switch val.(type) {
	case []interface{}:
		printYaml = true
	case map[interface{}]interface{}:
		printYaml = true
	}

	if printYaml {
		bytes, err := yaml.Marshal(val)
		if err != nil {
			log.Fatal(err)
		}
		fmt.Println(string(bytes))
	} else {
		fmt.Println(val)
	}
}

func getOrSetVal(args string, data map[interface{}]interface{}, value interface{}) interface{} {
	parts := strings.Split(args, ".")

	for i, part := range parts {
		val, ok := data[part]
		last := i+1 == len(parts)

		if last && value != nil {
			if s, ok := value.(string); ok {
				value = config.DummyMarshall(s)
			}

			data[part] = value
			return value
		}

		if !ok {
			break
		}

		if last {
			return val
		}

		newData, ok := val.(map[interface{}]interface{})
		if !ok {
			break
		}

		data = newData
	}

	return ""
}

func configSave(c *cli.Context) {
	cfg, err := config.LoadConfig()
	if err != nil {
		log.Fatal(err)
	}

	//TODO: why doesn't this work
	for _, c := range cfg.SystemContainers {
		container := docker.NewContainer("", &c)
		if container.Err != nil {
			log.Fatalf("Failed to parse [%s] : %v", c.Cmd, container.Err)
		}
	}

	if !c.Bool("full") {
		cfg.ClearReadOnly()
	}

	content, err := cfg.Dump()

	output := c.String("output")
	if output == "" {
		fmt.Println(content)
	} else {
		err := ioutil.WriteFile(output, []byte(content), 0400)
		if err != nil {
			log.Fatal(err)
		}
	}

}
<commit_msg>Don't print reboot message<commit_after>package control

import (
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"strings"

	log "github.com/Sirupsen/logrus"
	"gopkg.in/yaml.v2"

	"github.com/codegangsta/cli"
	"github.com/rancherio/os/config"
	"github.com/rancherio/os/docker"
)

func configSubcommands() []cli.Command {
	return []cli.Command{
		{
			Name:   "get",
			Usage:  "get value",
			Action: configGet,
		},
		{
			Name:   "set",
			Usage:  "set a value",
			Action: configSet,
		},
		{
			Name:   "import",
			Usage:  "import configuration from standard in or a file",
			Action: configImport,
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "input, i",
					Usage: "File from which to read",
				},
			},
		},
		{
			Name:  "export",
			Usage: "dump full configuration",
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "output, o",
					Usage: "File to which to save",
				},
				cli.BoolFlag{
					Name:  "full",
					Usage: "Include full configuration, not just writable fields",
				},
			},
			Action: configSave,
		},
	}
}

func getConfigData() (map[interface{}]interface{}, error) {
	cfg, err := config.LoadConfig()
	if err != nil {
		log.Fatal(err)
	}

	content, err := cfg.Dump()
	if err != nil {
		log.Fatal(err)
	}

	data := make(map[interface{}]interface{})
	err = yaml.Unmarshal([]byte(content), data)

	return data, err
}

func configImport(c *cli.Context) {
	var input io.Reader
	var err error
	input = os.Stdin

	inputFile := c.String("input")
	if inputFile != "" {
		input, err = os.Open(inputFile)
		if err != nil {
			log.Fatal(err)
		}
	}

	bytes, err := ioutil.ReadAll(input)
	if err != nil {
		log.Fatal(err)
	}

	err = mergeConfig(bytes)
	if err != nil {
		log.Fatal(err)
	}
}

func mergeConfig(bytes []byte) error {
	var newConfig config.Config

	err := yaml.Unmarshal(bytes, &newConfig)
	if err != nil {
		return err
	}

	cfg, err := config.LoadConfig()
	if err != nil {
		return err
	}

	_, err = cfg.Merge(newConfig)
	if err != nil {
		return err
	}

	err = cfg.Save()
	if err != nil {
		return err
	}

	return err
}

func configSet(c *cli.Context) {
	key := c.Args().Get(0)
	value := c.Args().Get(1)
	if key == "" {
		return
	}

	data, err := getConfigData()
	getOrSetVal(key, data, value)

	bytes, err := yaml.Marshal(data)
	if err != nil {
		log.Fatal(err)
	}

	err = mergeConfig(bytes)
	if err != nil {
		log.Fatal(err)
	}
}

func configGet(c *cli.Context) {
	arg := c.Args().Get(0)
	if arg == "" {
		return
	}

	data, err := getConfigData()
	if err != nil {
		log.Fatal(err)
	}

	val := getOrSetVal(arg, data, nil)

	printYaml := false
	switch val.(type) {
	case []interface{}:
		printYaml = true
	case map[interface{}]interface{}:
		printYaml = true
	}

	if printYaml {
		bytes, err := yaml.Marshal(val)
		if err != nil {
			log.Fatal(err)
		}
		fmt.Println(string(bytes))
	} else {
		fmt.Println(val)
	}
}

func getOrSetVal(args string, data map[interface{}]interface{}, value interface{}) interface{} {
	parts := strings.Split(args, ".")

	for i, part := range parts {
		val, ok := data[part]
		last := i+1 == len(parts)

		if last && value != nil {
			if s, ok := value.(string); ok {
				value = config.DummyMarshall(s)
			}

			data[part] = value
			return value
		}

		if !ok {
			break
		}

		if last {
			return val
		}

		newData, ok := val.(map[interface{}]interface{})
		if !ok {
			break
		}

		data = newData
	}

	return ""
}

func configSave(c *cli.Context) {
	cfg, err := config.LoadConfig()
	if err != nil {
		log.Fatal(err)
	}

	//TODO: why doesn't this work
	for _, c := range cfg.SystemContainers {
		container := docker.NewContainer("", &c)
		if container.Err != nil {
			log.Fatalf("Failed to parse [%s] : %v", c.Cmd, container.Err)
		}
	}

	if !c.Bool("full") {
		cfg.ClearReadOnly()
	}

	content, err := cfg.Dump()

	output := c.String("output")
	if output == "" {
		fmt.Println(content)
	} else {
		err := ioutil.WriteFile(output, []byte(content), 0400)
		if err != nil {
			log.Fatal(err)
		}
	}

}
<|endoftext|>


<commit_before>// Copyright 2009-2015 The freegeoip authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

package main

import (
	"flag"
	"fmt"
	"log"
	"net"
	"net/http"
	"net/url"
	"runtime"
	"strconv"
	"strings"
	"time"

	_ "net/http/pprof"

	"github.com/fiorix/freegeoip"
	"github.com/fiorix/go-redis/redis"
	"github.com/gorilla/context"
)

// Version tag.
var Version = "3.0.4"

var maxmindFile = "http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.mmdb.gz"

func main() {
	addr := flag.String("addr", ":8080", "Address in form of ip:port to listen on")
	certFile := flag.String("cert", "", "X.509 certificate file")
	keyFile := flag.String("key", "", "X.509 key file")
	public := flag.String("public", "", "Public directory to serve at the / endpoint")
	ipdb := flag.String("db", maxmindFile, "IP database file or URL")
	updateIntvl := flag.Duration("update", 24*time.Hour, "Database update check interval")
	retryIntvl := flag.Duration("retry", time.Hour, "Max time to wait before retrying update")
	useXFF := flag.Bool("use-x-forwarded-for", false, "Use the X-Forwarded-For header when available")
	silent := flag.Bool("silent", false, "Do not log requests to stderr")
	redisAddr := flag.String("redis", "127.0.0.1:6379", "Redis address in form of ip:port for quota")
	redisTimeout := flag.Duration("redis-timeout", 500*time.Millisecond, "Redis read/write timeout")
	quotaMax := flag.Int("quota-max", 0, "Max requests per source IP per interval; Set 0 to turn off")
	quotaIntvl := flag.Duration("quota-interval", time.Hour, "Quota expiration interval")
	version := flag.Bool("version", false, "Show version and exit")
	pprof := flag.String("pprof", "", "Address in form of ip:port to listen on for pprof")
	flag.Parse()

	if *version {
		fmt.Printf("freegeoip v%s\n", Version)
		return
	}

	rc, err := redis.Dial(*redisAddr)
	if err != nil {
		log.Fatal(err)
	}
	rc.Timeout = *redisTimeout

	db, err := openDB(*ipdb, *updateIntvl, *retryIntvl)
	if err != nil {
		log.Fatal(err)
	}

	runtime.GOMAXPROCS(runtime.NumCPU())

	encoders := map[string]http.Handler{
		"/csv/":  freegeoip.NewHandler(db, &freegeoip.CSVEncoder{UseCRLF: true}),
		"/xml/":  freegeoip.NewHandler(db, &freegeoip.XMLEncoder{Indent: true}),
		"/json/": freegeoip.NewHandler(db, &freegeoip.JSONEncoder{}),
	}

	if *quotaMax > 0 {
		seconds := int((*quotaIntvl).Seconds())
		for path, f := range encoders {
			encoders[path] = userQuota(rc, *quotaMax, seconds, f, *silent)
		}
	}

	mux := http.NewServeMux()
	for path, handler := range encoders {
		mux.Handle(path, handler)
	}

	if len(*public) > 0 {
		mux.Handle("/", http.FileServer(http.Dir(*public)))
	}

	handler := CORS(mux, "GET", "HEAD")

	if !*silent {
		log.Println("freegeoip server starting on", *addr)
		go logEvents(db)
		handler = logHandler(handler)
	}

	if *useXFF {
		handler = freegeoip.ProxyHandler(handler)
	}

	if len(*pprof) > 0 {
		go func() {
			log.Fatal(http.ListenAndServe(*pprof, nil))
		}()
	}

	if len(*certFile) > 0 && len(*keyFile) > 0 {
		err = http.ListenAndServeTLS(*addr, *certFile, *keyFile, handler)
	} else {
		err = http.ListenAndServe(*addr, handler)
	}
	if err != nil {
		log.Fatal(err)
	}
}

// openDB opens and returns the IP database.
func openDB(dsn string, updateIntvl, maxRetryIntvl time.Duration) (db *freegeoip.DB, err error) {
	u, err := url.Parse(dsn)
	if err != nil || len(u.Scheme) == 0 {
		db, err = freegeoip.Open(dsn)
	} else {
		db, err = freegeoip.OpenURL(dsn, updateIntvl, maxRetryIntvl)
	}
	return
}

// CORS is an http handler that checks for allowed request methods (verbs)
// and adds CORS headers to all http responses.
//
// See http://en.wikipedia.org/wiki/Cross-origin_resource_sharing for details.
func CORS(f http.Handler, allow ...string) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Access-Control-Allow-Origin", "*")
		w.Header().Set("Access-Control-Allow-Method",
			strings.Join(allow, ", ")+", OPTIONS")
		if r.Method == "OPTIONS" {
			w.WriteHeader(200)
			return
		}
		for _, method := range allow {
			if r.Method == method {
				f.ServeHTTP(w, r)
				return
			}
		}
		w.Header().Set("Allow", strings.Join(allow, ", ")+", OPTIONS")
		http.Error(w, http.StatusText(http.StatusMethodNotAllowed),
			http.StatusMethodNotAllowed)
	})
}

// userQuota is a handler that provides a rate limiter to the freegeoip API.
// It allows qmax requests per qintvl, in seconds.
//
// If redis is not available it responds with service unavailable.
func userQuota(rc *redis.Client, qmax int, qintvl int, f http.Handler, silent bool) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		var ip string
		if idx := strings.LastIndex(r.RemoteAddr, ":"); idx != -1 {
			ip = r.RemoteAddr[:idx]
		} else {
			ip = r.RemoteAddr
		}
		sreq, err := rc.Get(ip)
		if err != nil {
			if !silent {
				context.Set(r, "log", err.Error())
			}
			http.Error(w, "Try again later",
				http.StatusServiceUnavailable)
			return
		}
		if len(sreq) == 0 {
			err = rc.SetEx(ip, qintvl, "1")
			if err != nil {
				if !silent {
					context.Set(r, "log", err.Error())
				}
				http.Error(w, "Try again later",
					http.StatusServiceUnavailable)
				return
			}
			f.ServeHTTP(w, r)
			return
		}
		nreq, _ := strconv.Atoi(sreq)
		if nreq >= qmax {
			http.Error(w, "Quota exceeded", http.StatusForbidden)
			return
		}
		_, err = rc.Incr(ip)
		if err != nil && !silent {
			context.Set(r, "log", err.Error())
		}
		f.ServeHTTP(w, r)
	})
}

// logEvents logs database events.
func logEvents(db *freegeoip.DB) {
	for {
		select {
		case file := <-db.NotifyOpen():
			log.Println("database loaded:", file)
		case err := <-db.NotifyError():
			log.Println("database error:", err)
		case <-db.NotifyClose():
			return
		}
	}
}

// logHandler logs http requests.
func logHandler(f http.Handler) http.Handler {
	empty := ""
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		resp := responseWriter{w, http.StatusOK, 0}
		start := time.Now()
		f.ServeHTTP(&resp, r)
		elapsed := time.Since(start)
		extra := context.Get(r, "log")
		if extra != nil {
			defer context.Clear(r)
		} else {
			extra = empty
		}
		log.Printf("%q %d %q %q %s %q %db in %s %q",
			r.Proto,
			resp.status,
			r.Method,
			r.URL.Path,
			remoteIP(r),
			r.Header.Get("User-Agent"),
			resp.bytes,
			elapsed,
			extra,
		)
	})
}

// remoteIP returns the client's address without the port number.
func remoteIP(r *http.Request) string {
	host, _, err := net.SplitHostPort(r.RemoteAddr)
	if err != nil {
		return r.RemoteAddr
	}
	return host
}

// responseWriter is an http.ResponseWriter that records the returned
// status and bytes written to the client.
type responseWriter struct {
	http.ResponseWriter
	status int
	bytes  int
}

// Write implements the http.ResponseWriter interface.
func (f *responseWriter) Write(b []byte) (int, error) {
	n, err := f.ResponseWriter.Write(b)
	if err != nil {
		return 0, err
	}
	f.bytes += n
	return n, nil
}

// WriteHeader implements the http.ResponseWriter interface.
func (f *responseWriter) WriteHeader(code int) {
	f.status = code
	f.ResponseWriter.WriteHeader(code)
}
<commit_msg>Fix name of CORS header<commit_after>// Copyright 2009-2015 The freegeoip authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

package main

import (
	"flag"
	"fmt"
	"log"
	"net"
	"net/http"
	"net/url"
	"runtime"
	"strconv"
	"strings"
	"time"

	_ "net/http/pprof"

	"github.com/fiorix/freegeoip"
	"github.com/fiorix/go-redis/redis"
	"github.com/gorilla/context"
)

// Version tag.
var Version = "3.0.4"

var maxmindFile = "http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.mmdb.gz"

func main() {
	addr := flag.String("addr", ":8080", "Address in form of ip:port to listen on")
	certFile := flag.String("cert", "", "X.509 certificate file")
	keyFile := flag.String("key", "", "X.509 key file")
	public := flag.String("public", "", "Public directory to serve at the / endpoint")
	ipdb := flag.String("db", maxmindFile, "IP database file or URL")
	updateIntvl := flag.Duration("update", 24*time.Hour, "Database update check interval")
	retryIntvl := flag.Duration("retry", time.Hour, "Max time to wait before retrying update")
	useXFF := flag.Bool("use-x-forwarded-for", false, "Use the X-Forwarded-For header when available")
	silent := flag.Bool("silent", false, "Do not log requests to stderr")
	redisAddr := flag.String("redis", "127.0.0.1:6379", "Redis address in form of ip:port for quota")
	redisTimeout := flag.Duration("redis-timeout", 500*time.Millisecond, "Redis read/write timeout")
	quotaMax := flag.Int("quota-max", 0, "Max requests per source IP per interval; Set 0 to turn off")
	quotaIntvl := flag.Duration("quota-interval", time.Hour, "Quota expiration interval")
	version := flag.Bool("version", false, "Show version and exit")
	pprof := flag.String("pprof", "", "Address in form of ip:port to listen on for pprof")
	flag.Parse()

	if *version {
		fmt.Printf("freegeoip v%s\n", Version)
		return
	}

	rc, err := redis.Dial(*redisAddr)
	if err != nil {
		log.Fatal(err)
	}
	rc.Timeout = *redisTimeout

	db, err := openDB(*ipdb, *updateIntvl, *retryIntvl)
	if err != nil {
		log.Fatal(err)
	}

	runtime.GOMAXPROCS(runtime.NumCPU())

	encoders := map[string]http.Handler{
		"/csv/":  freegeoip.NewHandler(db, &freegeoip.CSVEncoder{UseCRLF: true}),
		"/xml/":  freegeoip.NewHandler(db, &freegeoip.XMLEncoder{Indent: true}),
		"/json/": freegeoip.NewHandler(db, &freegeoip.JSONEncoder{}),
	}

	if *quotaMax > 0 {
		seconds := int((*quotaIntvl).Seconds())
		for path, f := range encoders {
			encoders[path] = userQuota(rc, *quotaMax, seconds, f, *silent)
		}
	}

	mux := http.NewServeMux()
	for path, handler := range encoders {
		mux.Handle(path, handler)
	}

	if len(*public) > 0 {
		mux.Handle("/", http.FileServer(http.Dir(*public)))
	}

	handler := CORS(mux, "GET", "HEAD")

	if !*silent {
		log.Println("freegeoip server starting on", *addr)
		go logEvents(db)
		handler = logHandler(handler)
	}

	if *useXFF {
		handler = freegeoip.ProxyHandler(handler)
	}

	if len(*pprof) > 0 {
		go func() {
			log.Fatal(http.ListenAndServe(*pprof, nil))
		}()
	}

	if len(*certFile) > 0 && len(*keyFile) > 0 {
		err = http.ListenAndServeTLS(*addr, *certFile, *keyFile, handler)
	} else {
		err = http.ListenAndServe(*addr, handler)
	}
	if err != nil {
		log.Fatal(err)
	}
}

// openDB opens and returns the IP database.
func openDB(dsn string, updateIntvl, maxRetryIntvl time.Duration) (db *freegeoip.DB, err error) {
	u, err := url.Parse(dsn)
	if err != nil || len(u.Scheme) == 0 {
		db, err = freegeoip.Open(dsn)
	} else {
		db, err = freegeoip.OpenURL(dsn, updateIntvl, maxRetryIntvl)
	}
	return
}

// CORS is an http handler that checks for allowed request methods (verbs)
// and adds CORS headers to all http responses.
//
// See http://en.wikipedia.org/wiki/Cross-origin_resource_sharing for details.
func CORS(f http.Handler, allow ...string) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Access-Control-Allow-Origin", "*")
		w.Header().Set("Access-Control-Allow-Methods",
			strings.Join(allow, ", ")+", OPTIONS")
		if r.Method == "OPTIONS" {
			w.WriteHeader(200)
			return
		}
		for _, method := range allow {
			if r.Method == method {
				f.ServeHTTP(w, r)
				return
			}
		}
		w.Header().Set("Allow", strings.Join(allow, ", ")+", OPTIONS")
		http.Error(w, http.StatusText(http.StatusMethodNotAllowed),
			http.StatusMethodNotAllowed)
	})
}

// userQuota is a handler that provides a rate limiter to the freegeoip API.
// It allows qmax requests per qintvl, in seconds.
//
// If redis is not available it responds with service unavailable.
func userQuota(rc *redis.Client, qmax int, qintvl int, f http.Handler, silent bool) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		var ip string
		if idx := strings.LastIndex(r.RemoteAddr, ":"); idx != -1 {
			ip = r.RemoteAddr[:idx]
		} else {
			ip = r.RemoteAddr
		}
		sreq, err := rc.Get(ip)
		if err != nil {
			if !silent {
				context.Set(r, "log", err.Error())
			}
			http.Error(w, "Try again later",
				http.StatusServiceUnavailable)
			return
		}
		if len(sreq) == 0 {
			err = rc.SetEx(ip, qintvl, "1")
			if err != nil {
				if !silent {
					context.Set(r, "log", err.Error())
				}
				http.Error(w, "Try again later",
					http.StatusServiceUnavailable)
				return
			}
			f.ServeHTTP(w, r)
			return
		}
		nreq, _ := strconv.Atoi(sreq)
		if nreq >= qmax {
			http.Error(w, "Quota exceeded", http.StatusForbidden)
			return
		}
		_, err = rc.Incr(ip)
		if err != nil && !silent {
			context.Set(r, "log", err.Error())
		}
		f.ServeHTTP(w, r)
	})
}

// logEvents logs database events.
func logEvents(db *freegeoip.DB) {
	for {
		select {
		case file := <-db.NotifyOpen():
			log.Println("database loaded:", file)
		case err := <-db.NotifyError():
			log.Println("database error:", err)
		case <-db.NotifyClose():
			return
		}
	}
}

// logHandler logs http requests.
func logHandler(f http.Handler) http.Handler {
	empty := ""
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		resp := responseWriter{w, http.StatusOK, 0}
		start := time.Now()
		f.ServeHTTP(&resp, r)
		elapsed := time.Since(start)
		extra := context.Get(r, "log")
		if extra != nil {
			defer context.Clear(r)
		} else {
			extra = empty
		}
		log.Printf("%q %d %q %q %s %q %db in %s %q",
			r.Proto,
			resp.status,
			r.Method,
			r.URL.Path,
			remoteIP(r),
			r.Header.Get("User-Agent"),
			resp.bytes,
			elapsed,
			extra,
		)
	})
}

// remoteIP returns the client's address without the port number.
func remoteIP(r *http.Request) string {
	host, _, err := net.SplitHostPort(r.RemoteAddr)
	if err != nil {
		return r.RemoteAddr
	}
	return host
}

// responseWriter is an http.ResponseWriter that records the returned
// status and bytes written to the client.
type responseWriter struct {
	http.ResponseWriter
	status int
	bytes  int
}

// Write implements the http.ResponseWriter interface.
func (f *responseWriter) Write(b []byte) (int, error) {
	n, err := f.ResponseWriter.Write(b)
	if err != nil {
		return 0, err
	}
	f.bytes += n
	return n, nil
}

// WriteHeader implements the http.ResponseWriter interface.
func (f *responseWriter) WriteHeader(code int) {
	f.status = code
	f.ResponseWriter.WriteHeader(code)
}
<|endoftext|>


<commit_before>// Copyright 2009-2014 The freegeoip authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

package main

import (
	"flag"
	"log"
	"net"
	"net/http"
	"net/url"
	"runtime"
	"strconv"
	"strings"
	"time"

	"github.com/fiorix/freegeoip"
	"github.com/fiorix/go-redis/redis"
	"github.com/gorilla/context"
)

var maxmindFile = "http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.mmdb.gz"

func main() {
	addr := flag.String("addr", ":8080", "Address in form of ip:port to listen on")
	certFile := flag.String("cert", "", "X.509 certificate file")
	keyFile := flag.String("key", "", "X.509 key file")
	public := flag.String("public", "", "Public directory to serve at the / endpoint")
	ipdb := flag.String("db", maxmindFile, "IP database file or URL")
	updateIntvl := flag.Duration("update", 24*time.Hour, "Database update check interval")
	retryIntvl := flag.Duration("retry", time.Hour, "Max time to wait before retrying update")
	useXFF := flag.Bool("use-x-forwarded-for", false, "Use the X-Forwarded-For header when available")
	silent := flag.Bool("silent", false, "Do not log requests to stderr")
	redisAddr := flag.String("redis", "127.0.0.1:6379", "Redis address in form of ip:port for quota")
	quotaMax := flag.Int("quota-max", 0, "Max requests per source IP per interval; Set 0 to turn off")
	quotaIntvl := flag.Duration("quota-interval", time.Hour, "Quota expiration interval")
	flag.Parse()

	rc, err := redis.Dial(*redisAddr)
	if err != nil {
		log.Fatal(err)
	}

	db, err := openDB(*ipdb, *updateIntvl, *retryIntvl)
	if err != nil {
		log.Fatal(err)
	}

	runtime.GOMAXPROCS(runtime.NumCPU())

	encoders := map[string]http.Handler{
		"/csv/":  freegeoip.NewHandler(db, &freegeoip.CSVEncoder{UseCRLF: true}),
		"/xml/":  freegeoip.NewHandler(db, &freegeoip.XMLEncoder{Indent: true}),
		"/json/": freegeoip.NewHandler(db, &freegeoip.JSONEncoder{}),
	}

	if *quotaMax > 0 {
		seconds := int((*quotaIntvl).Seconds())
		for path, f := range encoders {
			encoders[path] = userQuota(rc, *quotaMax, seconds, f)
		}
	}

	mux := http.NewServeMux()
	for path, handler := range encoders {
		mux.Handle(path, handler)
	}

	if len(*public) > 0 {
		mux.Handle("/", http.FileServer(http.Dir(*public)))
	}

	handler := CORS(mux, "GET", "HEAD")

	if !*silent {
		log.Println("freegeoip server starting on", *addr)
		go logEvents(db)
		handler = logHandler(handler)
	}

	if *useXFF {
		handler = freegeoip.ProxyHandler(handler)
	}

	if len(*certFile) > 0 && len(*keyFile) > 0 {
		err = http.ListenAndServeTLS(*addr, *certFile, *keyFile, handler)
	} else {
		err = http.ListenAndServe(*addr, handler)
	}
	if err != nil {
		log.Fatal(err)
	}
}

// openDB opens and returns the IP database.
func openDB(dsn string, updateIntvl, maxRetryIntvl time.Duration) (db *freegeoip.DB, err error) {
	_, err = url.Parse(dsn)
	if err != nil {
		db, err = freegeoip.Open(dsn)
	} else {
		db, err = freegeoip.OpenURL(dsn, updateIntvl, maxRetryIntvl)
	}
	return
}

// CORS is an http handler that checks for allowed request methods (verbs)
// and adds CORS headers to all http responses.
//
// See http://en.wikipedia.org/wiki/Cross-origin_resource_sharing for details.
func CORS(f http.Handler, allow ...string) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Access-Control-Allow-Origin", "*")
		w.Header().Set("Access-Control-Allow-Method",
			strings.Join(allow, ", ")+", OPTIONS")
		if r.Method == "OPTIONS" {
			w.WriteHeader(200)
			return
		}
		for _, method := range allow {
			if r.Method == method {
				f.ServeHTTP(w, r)
				return
			}
		}
		w.Header().Set("Allow", strings.Join(allow, ", ")+", OPTIONS")
		http.Error(w, http.StatusText(http.StatusMethodNotAllowed),
			http.StatusMethodNotAllowed)
	})
}

// userQuota is a handler that provides a rate limiter to the freegeoip API.
// It allows qmax requests per qintvl, in seconds.
//
// If redis is not available it responds with service unavailable.
func userQuota(rc *redis.Client, qmax int, qintvl int, f http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		var ip string
		if idx := strings.LastIndex(r.RemoteAddr, ":"); idx != -1 {
			ip = r.RemoteAddr[:idx]
		} else {
			ip = r.RemoteAddr
		}
		sreq, err := rc.Get(ip)
		if err != nil {
			serviceUnavailable(w, r, err.Error())
			return
		}
		if len(sreq) == 0 {
			err = rc.SetEx(ip, qintvl, "1")
			if err != nil {
				serviceUnavailable(w, r, err.Error())
				return
			}
			f.ServeHTTP(w, r)
			return
		}
		nreq, _ := strconv.Atoi(sreq)
		if nreq >= qmax {
			http.Error(w, "Quota exceeded", http.StatusForbidden)
			return
		}
		_, err = rc.Incr(ip)
		if err != nil {
			context.Set(r, "log", err.Error())
		}
		f.ServeHTTP(w, r)
	})
}

// serviceUnavailable writes an http error 501 to a client.
func serviceUnavailable(w http.ResponseWriter, r *http.Request, log string) {
	context.Set(r, "log", log)
	http.Error(w, "Try again later", http.StatusServiceUnavailable)
}

// logEvents logs database events.
func logEvents(db *freegeoip.DB) {
	for {
		select {
		case file := <-db.NotifyOpen():
			log.Println("database loaded:", file)
		case err := <-db.NotifyError():
			log.Println("database error:", err)
		case <-db.NotifyClose():
			return
		}
	}
}

// logHandler logs http requests.
func logHandler(f http.Handler) http.Handler {
	empty := ""
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		resp := responseWriter{w, http.StatusOK, 0}
		start := time.Now()
		f.ServeHTTP(&resp, r)
		elapsed := time.Since(start)
		extra := context.Get(r, "log")
		if extra != nil {
			defer context.Clear(r)
		} else {
			extra = empty
		}
		log.Printf("%q %d %q %q %s %q %db in %s %q",
			r.Proto,
			resp.status,
			r.Method,
			r.URL.Path,
			remoteIP(r),
			r.Header.Get("User-Agent"),
			resp.bytes,
			elapsed,
			extra,
		)
	})
}

// remoteIP returns the client's address without the port number.
func remoteIP(r *http.Request) string {
	host, _, err := net.SplitHostPort(r.RemoteAddr)
	if err != nil {
		return r.RemoteAddr
	}
	return host
}

// responseWriter is an http.ResponseWriter that records the returned
// status and bytes written to the client.
type responseWriter struct {
	http.ResponseWriter
	status int
	bytes  int
}

// Write implements the http.ResponseWriter interface.
func (f *responseWriter) Write(b []byte) (int, error) {
	n, err := f.ResponseWriter.Write(b)
	if err != nil {
		return 0, err
	}
	f.bytes += n
	return n, nil
}

// WriteHeader implements the http.ResponseWriter interface.
func (f *responseWriter) WriteHeader(code int) {
	f.status = code
	f.ResponseWriter.WriteHeader(code)
}
<commit_msg>Fix for loading local db<commit_after>// Copyright 2009-2014 The freegeoip authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

package main

import (
	"flag"
	"log"
	"net"
	"net/http"
	"net/url"
	"runtime"
	"strconv"
	"strings"
	"time"

	"github.com/fiorix/freegeoip"
	"github.com/fiorix/go-redis/redis"
	"github.com/gorilla/context"
)

var maxmindFile = "http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.mmdb.gz"

func main() {
	addr := flag.String("addr", ":8080", "Address in form of ip:port to listen on")
	certFile := flag.String("cert", "", "X.509 certificate file")
	keyFile := flag.String("key", "", "X.509 key file")
	public := flag.String("public", "", "Public directory to serve at the / endpoint")
	ipdb := flag.String("db", maxmindFile, "IP database file or URL")
	updateIntvl := flag.Duration("update", 24*time.Hour, "Database update check interval")
	retryIntvl := flag.Duration("retry", time.Hour, "Max time to wait before retrying update")
	useXFF := flag.Bool("use-x-forwarded-for", false, "Use the X-Forwarded-For header when available")
	silent := flag.Bool("silent", false, "Do not log requests to stderr")
	redisAddr := flag.String("redis", "127.0.0.1:6379", "Redis address in form of ip:port for quota")
	quotaMax := flag.Int("quota-max", 0, "Max requests per source IP per interval; Set 0 to turn off")
	quotaIntvl := flag.Duration("quota-interval", time.Hour, "Quota expiration interval")
	flag.Parse()

	rc, err := redis.Dial(*redisAddr)
	if err != nil {
		log.Fatal(err)
	}

	db, err := openDB(*ipdb, *updateIntvl, *retryIntvl)
	if err != nil {
		log.Fatal(err)
	}

	runtime.GOMAXPROCS(runtime.NumCPU())

	encoders := map[string]http.Handler{
		"/csv/":  freegeoip.NewHandler(db, &freegeoip.CSVEncoder{UseCRLF: true}),
		"/xml/":  freegeoip.NewHandler(db, &freegeoip.XMLEncoder{Indent: true}),
		"/json/": freegeoip.NewHandler(db, &freegeoip.JSONEncoder{}),
	}

	if *quotaMax > 0 {
		seconds := int((*quotaIntvl).Seconds())
		for path, f := range encoders {
			encoders[path] = userQuota(rc, *quotaMax, seconds, f)
		}
	}

	mux := http.NewServeMux()
	for path, handler := range encoders {
		mux.Handle(path, handler)
	}

	if len(*public) > 0 {
		mux.Handle("/", http.FileServer(http.Dir(*public)))
	}

	handler := CORS(mux, "GET", "HEAD")

	if !*silent {
		log.Println("freegeoip server starting on", *addr)
		go logEvents(db)
		handler = logHandler(handler)
	}

	if *useXFF {
		handler = freegeoip.ProxyHandler(handler)
	}

	if len(*certFile) > 0 && len(*keyFile) > 0 {
		err = http.ListenAndServeTLS(*addr, *certFile, *keyFile, handler)
	} else {
		err = http.ListenAndServe(*addr, handler)
	}
	if err != nil {
		log.Fatal(err)
	}
}

// openDB opens and returns the IP database.
func openDB(dsn string, updateIntvl, maxRetryIntvl time.Duration) (db *freegeoip.DB, err error) {
	u, err := url.Parse(dsn)
	if err != nil || len(u.Scheme) == 0 {
		db, err = freegeoip.Open(dsn)
	} else {
		db, err = freegeoip.OpenURL(dsn, updateIntvl, maxRetryIntvl)
	}
	return
}

// CORS is an http handler that checks for allowed request methods (verbs)
// and adds CORS headers to all http responses.
//
// See http://en.wikipedia.org/wiki/Cross-origin_resource_sharing for details.
func CORS(f http.Handler, allow ...string) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Access-Control-Allow-Origin", "*")
		w.Header().Set("Access-Control-Allow-Method",
			strings.Join(allow, ", ")+", OPTIONS")
		if r.Method == "OPTIONS" {
			w.WriteHeader(200)
			return
		}
		for _, method := range allow {
			if r.Method == method {
				f.ServeHTTP(w, r)
				return
			}
		}
		w.Header().Set("Allow", strings.Join(allow, ", ")+", OPTIONS")
		http.Error(w, http.StatusText(http.StatusMethodNotAllowed),
			http.StatusMethodNotAllowed)
	})
}

// userQuota is a handler that provides a rate limiter to the freegeoip API.
// It allows qmax requests per qintvl, in seconds.
//
// If redis is not available it responds with service unavailable.
func userQuota(rc *redis.Client, qmax int, qintvl int, f http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		var ip string
		if idx := strings.LastIndex(r.RemoteAddr, ":"); idx != -1 {
			ip = r.RemoteAddr[:idx]
		} else {
			ip = r.RemoteAddr
		}
		sreq, err := rc.Get(ip)
		if err != nil {
			serviceUnavailable(w, r, err.Error())
			return
		}
		if len(sreq) == 0 {
			err = rc.SetEx(ip, qintvl, "1")
			if err != nil {
				serviceUnavailable(w, r, err.Error())
				return
			}
			f.ServeHTTP(w, r)
			return
		}
		nreq, _ := strconv.Atoi(sreq)
		if nreq >= qmax {
			http.Error(w, "Quota exceeded", http.StatusForbidden)
			return
		}
		_, err = rc.Incr(ip)
		if err != nil {
			context.Set(r, "log", err.Error())
		}
		f.ServeHTTP(w, r)
	})
}

// serviceUnavailable writes an http error 501 to a client.
func serviceUnavailable(w http.ResponseWriter, r *http.Request, log string) {
	context.Set(r, "log", log)
	http.Error(w, "Try again later", http.StatusServiceUnavailable)
}

// logEvents logs database events.
func logEvents(db *freegeoip.DB) {
	for {
		select {
		case file := <-db.NotifyOpen():
			log.Println("database loaded:", file)
		case err := <-db.NotifyError():
			log.Println("database error:", err)
		case <-db.NotifyClose():
			return
		}
	}
}

// logHandler logs http requests.
func logHandler(f http.Handler) http.Handler {
	empty := ""
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		resp := responseWriter{w, http.StatusOK, 0}
		start := time.Now()
		f.ServeHTTP(&resp, r)
		elapsed := time.Since(start)
		extra := context.Get(r, "log")
		if extra != nil {
			defer context.Clear(r)
		} else {
			extra = empty
		}
		log.Printf("%q %d %q %q %s %q %db in %s %q",
			r.Proto,
			resp.status,
			r.Method,
			r.URL.Path,
			remoteIP(r),
			r.Header.Get("User-Agent"),
			resp.bytes,
			elapsed,
			extra,
		)
	})
}

// remoteIP returns the client's address without the port number.
func remoteIP(r *http.Request) string {
	host, _, err := net.SplitHostPort(r.RemoteAddr)
	if err != nil {
		return r.RemoteAddr
	}
	return host
}

// responseWriter is an http.ResponseWriter that records the returned
// status and bytes written to the client.
type responseWriter struct {
	http.ResponseWriter
	status int
	bytes  int
}

// Write implements the http.ResponseWriter interface.
func (f *responseWriter) Write(b []byte) (int, error) {
	n, err := f.ResponseWriter.Write(b)
	if err != nil {
		return 0, err
	}
	f.bytes += n
	return n, nil
}

// WriteHeader implements the http.ResponseWriter interface.
func (f *responseWriter) WriteHeader(code int) {
	f.status = code
	f.ResponseWriter.WriteHeader(code)
}
<|endoftext|>


<commit_before>package main

import (
	"os"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/codegangsta/cli"
	librato "github.com/mihasya/go-metrics-librato"
	metrics "github.com/rcrowley/go-metrics"
	travismetrics "github.com/travis-ci/jupiter-brain/metrics"
	"github.com/travis-ci/jupiter-brain/server"
)

func main() {
	app := cli.NewApp()
	app.Usage = "Jupiter Brain API server"
	app.Author = "Travis CI"
	app.Email = "contact+jupiter-brain@travis-ci.org"
	app.Version = VersionString
	app.Compiled = GeneratedTime()
	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:  "addr",
			Usage: "host:port to listen to",
			Value: func() string {
				v := ":" + os.Getenv("PORT")
				if v == ":" {
					v = ":42161"
				}
				return v
			}(),
			EnvVar: "JUPITER_BRAIN_ADDR",
		},
		cli.StringFlag{
			Name:   "auth-token",
			Usage:  "authentication token for the api server",
			EnvVar: "JUPITER_BRAIN_AUTH_TOKEN",
		},
		cli.StringFlag{
			Name:   "vsphere-api-url",
			Usage:  "URL to vSphere API",
			EnvVar: "JUPITER_BRAIN_VSPHERE_API_URL,VSPHERE_API_URL",
		},
		cli.StringFlag{
			Name:   "vsphere-base-path",
			Usage:  "path to folder of base VMs in vSphere inventory",
			EnvVar: "JUPITER_BRAIN_VSPHERE_BASE_PATH",
		},
		cli.StringFlag{
			Name:   "vsphere-vm-path",
			Usage:  "path to folder where VMs will be put in vSphere inventory",
			EnvVar: "JUPITER_BRAIN_VSPHERE_VM_PATH",
		},
		cli.StringFlag{
			Name:   "vsphere-cluster-path",
			Usage:  "path to compute cluster that VMs will be booted in",
			EnvVar: "JUPITER_BRAIN_VSPHERE_CLUSTER_PATH",
		},
		cli.StringFlag{
			Name:   "database-url",
			Usage:  "URL to the PostgreSQL database",
			EnvVar: "JUPITER_BRAIN_DATABASE_URL,DATABASE_URL",
		},
		cli.BoolFlag{
			Name:   "debug",
			Usage:  "enable debug logging",
			EnvVar: "JUPITER_BRAIN_DEBUG,DEBUG",
		},
		cli.StringFlag{
			Name:   "sentry-dsn",
			Usage:  "Sentry DSN to send errors to",
			EnvVar: "JUPITER_BRAIN_SENTRY_DSN,SENTRY_DSN",
		},
		cli.StringFlag{
			Name:   "librato-email",
			Usage:  "Email for Librato account to send metrics to",
			EnvVar: "JUPITER_BRAIN_LIBRATO_EMAIL,LIBRATO_EMAIL",
		},
		cli.StringFlag{
			Name:   "librato-token",
			Usage:  "Token for Librato account to send metrics to",
			EnvVar: "JUPITER_BRAIN_LIBRATO_TOKEN,LIBRATO_TOKEN",
		},
		cli.StringFlag{
			Name:   "librato-source",
			Usage:  "The source to use when sending metrics to Librato",
			EnvVar: "JUPITER_BRAIN_LIBRATO_SOURCE,LIBRATO_SOURCE",
		},
	}
	app.Action = runServer

	app.RunAndExitOnError()
}

func runServer(c *cli.Context) {
	logrus.SetFormatter(&logrus.TextFormatter{DisableColors: true})

	if c.IsSet("librato-email") && c.IsSet("librato-token") && c.IsSet("librato-source") {
		logrus.Info("starting librato metrics reporter")

		go librato.Librato(
			metrics.DefaultRegistry,
			time.Minute,
			c.String("librato-email"),
			c.String("librato-token"),
			c.String("librato-source"),
			[]float64{0.50, 0.75, 0.90, 0.95, 0.99, 0.999, 1.0},
			time.Millisecond,
		)
	}
	go travismetrics.ReportMemstatsMetrics()

	server.Main(&server.Config{
		Addr:      c.String("addr"),
		AuthToken: c.String("auth-token"),
		Debug:     c.Bool("debug"),
		SentryDSN: c.String("sentry-dsn"),

		VSphereURL:         c.String("vsphere-api-url"),
		VSphereBasePath:    c.String("vsphere-base-path"),
		VSphereVMPath:      c.String("vsphere-vm-path"),
		VSphereClusterPath: c.String("vsphere-cluster-path"),

		DatabaseURL: c.String("database-url"),
	})
}
<commit_msg>main: work around bug in cli library for flags<commit_after>package main

import (
	"os"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/codegangsta/cli"
	librato "github.com/mihasya/go-metrics-librato"
	metrics "github.com/rcrowley/go-metrics"
	travismetrics "github.com/travis-ci/jupiter-brain/metrics"
	"github.com/travis-ci/jupiter-brain/server"
)

func main() {
	app := cli.NewApp()
	app.Usage = "Jupiter Brain API server"
	app.Author = "Travis CI"
	app.Email = "contact+jupiter-brain@travis-ci.org"
	app.Version = VersionString
	app.Compiled = GeneratedTime()
	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:  "addr",
			Usage: "host:port to listen to",
			Value: func() string {
				v := ":" + os.Getenv("PORT")
				if v == ":" {
					v = ":42161"
				}
				return v
			}(),
			EnvVar: "JUPITER_BRAIN_ADDR",
		},
		cli.StringFlag{
			Name:   "auth-token",
			Usage:  "authentication token for the api server",
			EnvVar: "JUPITER_BRAIN_AUTH_TOKEN",
		},
		cli.StringFlag{
			Name:   "vsphere-api-url",
			Usage:  "URL to vSphere API",
			EnvVar: "JUPITER_BRAIN_VSPHERE_API_URL,VSPHERE_API_URL",
		},
		cli.StringFlag{
			Name:   "vsphere-base-path",
			Usage:  "path to folder of base VMs in vSphere inventory",
			EnvVar: "JUPITER_BRAIN_VSPHERE_BASE_PATH",
		},
		cli.StringFlag{
			Name:   "vsphere-vm-path",
			Usage:  "path to folder where VMs will be put in vSphere inventory",
			EnvVar: "JUPITER_BRAIN_VSPHERE_VM_PATH",
		},
		cli.StringFlag{
			Name:   "vsphere-cluster-path",
			Usage:  "path to compute cluster that VMs will be booted in",
			EnvVar: "JUPITER_BRAIN_VSPHERE_CLUSTER_PATH",
		},
		cli.StringFlag{
			Name:   "database-url",
			Usage:  "URL to the PostgreSQL database",
			EnvVar: "JUPITER_BRAIN_DATABASE_URL,DATABASE_URL",
		},
		cli.BoolFlag{
			Name:   "debug",
			Usage:  "enable debug logging",
			EnvVar: "JUPITER_BRAIN_DEBUG,DEBUG",
		},
		cli.StringFlag{
			Name:   "sentry-dsn",
			Usage:  "Sentry DSN to send errors to",
			EnvVar: "JUPITER_BRAIN_SENTRY_DSN,SENTRY_DSN",
		},
		cli.StringFlag{
			Name:   "librato-email",
			Usage:  "Email for Librato account to send metrics to",
			EnvVar: "JUPITER_BRAIN_LIBRATO_EMAIL,LIBRATO_EMAIL",
		},
		cli.StringFlag{
			Name:   "librato-token",
			Usage:  "Token for Librato account to send metrics to",
			EnvVar: "JUPITER_BRAIN_LIBRATO_TOKEN,LIBRATO_TOKEN",
		},
		cli.StringFlag{
			Name:   "librato-source",
			Usage:  "The source to use when sending metrics to Librato",
			EnvVar: "JUPITER_BRAIN_LIBRATO_SOURCE,LIBRATO_SOURCE",
		},
	}
	app.Action = runServer

	app.RunAndExitOnError()
}

func runServer(c *cli.Context) {
	logrus.SetFormatter(&logrus.TextFormatter{DisableColors: true})

	if c.String("librato-email") != "" && c.String("librato-token") != "" && c.String("librato-source") != "" {
		logrus.Info("starting librato metrics reporter")

		go librato.Librato(
			metrics.DefaultRegistry,
			time.Minute,
			c.String("librato-email"),
			c.String("librato-token"),
			c.String("librato-source"),
			[]float64{0.50, 0.75, 0.90, 0.95, 0.99, 0.999, 1.0},
			time.Millisecond,
		)
	}
	go travismetrics.ReportMemstatsMetrics()

	server.Main(&server.Config{
		Addr:      c.String("addr"),
		AuthToken: c.String("auth-token"),
		Debug:     c.Bool("debug"),
		SentryDSN: c.String("sentry-dsn"),

		VSphereURL:         c.String("vsphere-api-url"),
		VSphereBasePath:    c.String("vsphere-base-path"),
		VSphereVMPath:      c.String("vsphere-vm-path"),
		VSphereClusterPath: c.String("vsphere-cluster-path"),

		DatabaseURL: c.String("database-url"),
	})
}
<|endoftext|>