<commit_before>package main

func cmdPull() int {
	fetchret := cmdFetch()
	if fetchret != 0 {
		// Fetch failed, abort
		return fetchret
	}
	// Now run checkout but with no args
	oldArgs := GlobalOptions.Args
	GlobalOptions.Args = []string{}
	ret := cmdCheckout()
	GlobalOptions.Args = oldArgs

	return ret

}

func cmdPullHelp() {
	LogConsole(`Usage: git-lob pull [options] [<remote> [<ref>...]]

  Run the 'git lob fetch' command with the same parameters to download binaries
  from a remote store, followed by 'git lob checkout' to populate the working
  copy with any binary content which your working copy references, but which
  wasn't available previously.

  You probably want to run this command after cloning a git-lob enabled git
  repository. You may also need to run it after the standard 'git pull' to
  obtain new binaries you don't already have, or if you check out an old commit
  which was previously outside the 'recent' range that 'git lob fetch'
  would usually download when you're on the default branch.

  See 'git lob fetch --help' for full details of the options & parameters you
  can pass to this command, they are the same. Also see
  'git lob checkout --help' for information on how the second stage works.

`)
}
<commit_msg>When pruning after a pull, do it after the checkout and not at end of fetch<commit_after>package main

func cmdPull() int {
	// extract the 'prune' option & perform it AFTER the checkout instead of in the Fetch
	// this is so that user can abort the prune if they want (or carry on working)
	optPrune := GlobalOptions.BoolOpts.Contains("prune")
	if optPrune {
		GlobalOptions.BoolOpts.Remove("prune")
	}
	fetchret := cmdFetch()
	if fetchret != 0 {
		// Fetch failed, abort
		return fetchret
	}
	// Now run checkout but with no args
	oldArgs := GlobalOptions.Args
	GlobalOptions.Args = []string{}
	ret := cmdCheckout()
	GlobalOptions.Args = oldArgs

	if optPrune && !GlobalOptions.DryRun {
		// NOW do the prune
		LogConsole("Performing post-pull prune...")
		LogConsole("You can abort this process or carry on working now, pull is complete")
		PostFetchPullPrune()
	}

	return ret

}

func cmdPullHelp() {
	LogConsole(`Usage: git-lob pull [options] [<remote> [<ref>...]]

  Run the 'git lob fetch' command with the same parameters to download binaries
  from a remote store, followed by 'git lob checkout' to populate the working
  copy with any binary content which your working copy references, but which
  wasn't available previously.

  You probably want to run this command after cloning a git-lob enabled git
  repository. You may also need to run it after the standard 'git pull' to
  obtain new binaries you don't already have, or if you check out an old commit
  which was previously outside the 'recent' range that 'git lob fetch'
  would usually download when you're on the default branch.

  See 'git lob fetch --help' for full details of the options & parameters you
  can pass to this command, they are the same. Also see
  'git lob checkout --help' for information on how the second stage works.

`)
}
<|endoftext|>


<commit_before>package throttled

import (
	"fmt"
	"time"
)

const (
	// Maximum number of times to retry SetIfNotExists/CompareAndSwap operations
	// before returning an error.
	maxCASAttempts = 10
)

// A RateLimiter manages limiting the rate of actions by key.
type RateLimiter interface {
	// RateLimit checks whether a particular key has exceeded a rate
	// limit. It also returns a RateLimitResult to provide additional
	// information about the state of the RateLimiter.
	//
	// If the rate limit has not been exceeded, the underlying storage
	// is updated by the supplied quantity. For example, a quantity of
	// 1 might be used to rate limit a single request while a greater
	// quantity could rate limit based on the size of a file upload in
	// megabytes. If quantity is 0, no update is performed allowing
	// you to "peek" at the state of the RateLimiter for a given key.
	RateLimit(key string, quantity int) (bool, RateLimitResult, error)
}

// RateLimitResult represents the state of the RateLimiter for a
// given key at the time of the query. This state can be used, for
// example, to communicate information to the client via HTTP
// headers. Negative values indicate that the attribute is not
// relevant to the implementation or state.
type RateLimitResult struct {
	// Limit is the maximum number of requests that could be permitted
	// instantaneously for this key starting from an empty state. For
	// example, if a rate limiter allows 10 requests per second per
	// key, Limit would always be 10.
	Limit int

	// Remaining is the maximum number of requests that could be
	// permitted instantaneously for this key given the current
	// state. For example, if a rate limiter allows 10 requests per
	// second and has already received 6 requests for this key this
	// second, Remaining would be 4.
	Remaining int

	// ResetAfter is the time until the RateLimiter returns to its
	// initial state for a given key. For example, if a rate limiter
	// manages requests per second and received one request 200ms ago,
	// Reset would return 800ms. You can also think of this as the time
	// until Limit and Remaining will be equal.
	ResetAfter time.Duration

	// RetryAfter is the time until the next request will be permitted.
	// It should be -1 unless the rate limit has been exceeded.
	RetryAfter time.Duration
}

type limitResult struct {
	limited bool
}

func (r *limitResult) Limited() bool { return r.limited }

type rateLimitResult struct {
	limitResult

	limit, remaining  int
	reset, retryAfter time.Duration
}

func (r *rateLimitResult) Limit() int                { return r.limit }
func (r *rateLimitResult) Remaining() int            { return r.remaining }
func (r *rateLimitResult) Reset() time.Duration      { return r.reset }
func (r *rateLimitResult) RetryAfter() time.Duration { return r.retryAfter }

// Rate describes a frequency of an activity such as the number of requests
// allowed per minute.
type Rate struct {
	period time.Duration // Time between equally spaced requests at the rate
	count  int           // Used internally for deprecated `RateLimit` interface only
}

// RateQuota describes the number of requests allowed per time period.
// MaxRate specified the maximum sustained rate of requests and must
// be greater than zero. MaxBurst defines the number of requests that
// will be allowed to exceed the rate in a single burst and must be
// greater than or equal to zero.
//
// Rate{PerSec(1), 0} would mean that after each request, no more
// requests will be permitted for that client for one second.
// Rate{PerSec(2), 0} permits one request per 0.5 seconds rather than
// two requests in one second.
// In practice, you probably want to set MaxBurst >0 to provide some
// flexibility to clients that only need to make a handful of
// requests. In fact a MaxBurst of zero will *never* permit a request
// with a quantity greater than one because it will immediately exceed
// the limit.
type RateQuota struct {
	MaxRate  Rate
	MaxBurst int
}

// PerSec represents a number of requests per second.
func PerSec(n int) Rate { return Rate{time.Second / time.Duration(n), n} }

// PerMin represents a number of requests per minute.
func PerMin(n int) Rate { return Rate{time.Minute / time.Duration(n), n} }

// PerHour represents a number of requests per hour.
func PerHour(n int) Rate { return Rate{time.Hour / time.Duration(n), n} }

// PerDay represents a number of requests per day.
func PerDay(n int) Rate { return Rate{24 * time.Hour / time.Duration(n), n} }

// GCRARateLimiter is a RateLimiter that users the generic cell-rate
// algorithm. The algorithm has been slightly modified from its usual
// form to support limiting with an additional quantity parameter, such
// as for limiting the number of bytes uploaded.
type GCRARateLimiter struct {
	limit int

	// Think of the DVT as our flexibility:
	// How far can you deviate from the nominal equally spaced schedule?
	// If you like leaky buckets, think about it as the size of your bucket.
	delayVariationTolerance time.Duration

	// Think of the emission interval as the time between events
	// in the nominal equally spaced schedule. If you like leaky buckets,
	// think of it as how frequently the bucket leaks one unit.
	emissionInterval time.Duration

	store GCRAStore
}

// NewGCRARateLimiter creates a GCRARateLimiter. quota.Count defines
// the maximum number of requests permitted in an instantaneous burst
// and quota.Count / quota.Period defines the maximum sustained
// rate. For example, PerMin(60) permits 60 requests instantly per key
// followed by one request per second indefinitely whereas PerSec(1)
// only permits one request per second with no tolerance for bursts.
func NewGCRARateLimiter(st GCRAStore, quota RateQuota) (*GCRARateLimiter, error) {
	if quota.MaxBurst < 0 {
		return nil, fmt.Errorf("Invalid RateQuota %#v. MaxBurst must be greater than zero.", quota)
	}
	if quota.MaxRate.period <= 0 {
		return nil, fmt.Errorf("Invalid RateQuota %#v. MaxRate must be greater than zero.", quota)
	}

	return &GCRARateLimiter{
		delayVariationTolerance: quota.MaxRate.period * (time.Duration(quota.MaxBurst) + 1),
		emissionInterval:        quota.MaxRate.period,
		limit:                   quota.MaxBurst + 1,
		store:                   st,
	}, nil
}

// RateLimit checks whether a particular key has exceeded a rate
// limit. It also returns a RateLimitResult to provide additional
// information about the state of the RateLimiter.
//
// If the rate limit has not been exceeded, the underlying storage is
// updated by the supplied quantity. For example, a quantity of 1
// might be used to rate limit a single request while a greater
// quantity could rate limit based on the size of a file upload in
// megabytes. If quantity is 0, no update is performed allowing you
// to "peek" at the state of the RateLimiter for a given key.
func (g *GCRARateLimiter) RateLimit(key string, quantity int) (bool, RateLimitResult, error) {
	var tat, newTat, now time.Time
	var ttl time.Duration
	rlc := RateLimitResult{Limit: g.limit, RetryAfter: -1}
	limited := false

	i := 0
	for {
		var err error
		var tatVal int64
		var updated bool

		// tat refers to the theoretical arrival time that would be expected
		// from equally spaced requests at exactly the rate limit.
		tatVal, now, err = g.store.GetWithTime(key)
		if err != nil {
			return false, rlc, err
		}

		if tatVal == -1 {
			tat = now
		} else {
			tat = time.Unix(0, tatVal)
		}

		increment := time.Duration(quantity) * g.emissionInterval
		if now.After(tat) {
			newTat = now.Add(increment)
		} else {
			newTat = tat.Add(increment)
		}

		// Block the request if the next permitted time is in the future
		allowAt := newTat.Add(-(g.delayVariationTolerance))
		if diff := now.Sub(allowAt); diff < 0 {
			if increment <= g.delayVariationTolerance {
				rlc.RetryAfter = -diff
			}
			ttl = tat.Sub(now)
			limited = true
			break
		}

		ttl = newTat.Sub(now)

		if tatVal == -1 {
			updated, err = g.store.SetIfNotExistsWithTTL(key, newTat.UnixNano(), ttl)
		} else {
			updated, err = g.store.CompareAndSwapWithTTL(key, tatVal, newTat.UnixNano(), ttl)
		}

		if err != nil {
			return false, rlc, err
		}
		if updated {
			break
		}

		i++
		if i > maxCASAttempts {
			return false, rlc, fmt.Errorf(
				"Failed to store updated rate limit data for key %s after %d attempts",
				key, i,
			)
		}
	}

	next := g.delayVariationTolerance - ttl
	if next > -g.emissionInterval {
		rlc.Remaining = int(next / g.emissionInterval)
	}
	rlc.ResetAfter = ttl

	return limited, rlc, nil
}
<commit_msg>Reformat documentation for line length<commit_after>package throttled

import (
	"fmt"
	"time"
)

const (
	// Maximum number of times to retry SetIfNotExists/CompareAndSwap operations
	// before returning an error.
	maxCASAttempts = 10
)

// A RateLimiter manages limiting the rate of actions by key.
type RateLimiter interface {
	// RateLimit checks whether a particular key has exceeded a rate
	// limit. It also returns a RateLimitResult to provide additional
	// information about the state of the RateLimiter.
	//
	// If the rate limit has not been exceeded, the underlying storage
	// is updated by the supplied quantity. For example, a quantity of
	// 1 might be used to rate limit a single request while a greater
	// quantity could rate limit based on the size of a file upload in
	// megabytes. If quantity is 0, no update is performed allowing
	// you to "peek" at the state of the RateLimiter for a given key.
	RateLimit(key string, quantity int) (bool, RateLimitResult, error)
}

// RateLimitResult represents the state of the RateLimiter for a
// given key at the time of the query. This state can be used, for
// example, to communicate information to the client via HTTP
// headers. Negative values indicate that the attribute is not
// relevant to the implementation or state.
type RateLimitResult struct {
	// Limit is the maximum number of requests that could be permitted
	// instantaneously for this key starting from an empty state. For
	// example, if a rate limiter allows 10 requests per second per
	// key, Limit would always be 10.
	Limit int

	// Remaining is the maximum number of requests that could be
	// permitted instantaneously for this key given the current
	// state. For example, if a rate limiter allows 10 requests per
	// second and has already received 6 requests for this key this
	// second, Remaining would be 4.
	Remaining int

	// ResetAfter is the time until the RateLimiter returns to its
	// initial state for a given key. For example, if a rate limiter
	// manages requests per second and received one request 200ms ago,
	// Reset would return 800ms. You can also think of this as the time
	// until Limit and Remaining will be equal.
	ResetAfter time.Duration

	// RetryAfter is the time until the next request will be permitted.
	// It should be -1 unless the rate limit has been exceeded.
	RetryAfter time.Duration
}

type limitResult struct {
	limited bool
}

func (r *limitResult) Limited() bool { return r.limited }

type rateLimitResult struct {
	limitResult

	limit, remaining  int
	reset, retryAfter time.Duration
}

func (r *rateLimitResult) Limit() int                { return r.limit }
func (r *rateLimitResult) Remaining() int            { return r.remaining }
func (r *rateLimitResult) Reset() time.Duration      { return r.reset }
func (r *rateLimitResult) RetryAfter() time.Duration { return r.retryAfter }

// Rate describes a frequency of an activity such as the number of requests
// allowed per minute.
type Rate struct {
	period time.Duration // Time between equally spaced requests at the rate
	count  int           // Used internally for deprecated `RateLimit` interface only
}

// RateQuota describes the number of requests allowed per time period.
// MaxRate specified the maximum sustained rate of requests and must
// be greater than zero. MaxBurst defines the number of requests that
// will be allowed to exceed the rate in a single burst and must be
// greater than or equal to zero.
//
// Rate{PerSec(1), 0} would mean that after each request, no more
// requests will be permitted for that client for one second.
// Rate{PerSec(2), 0} permits one request per 0.5 seconds rather than
// two requests in one second. In practice, you probably want to set
// MaxBurst >0 to provide some flexibility to clients that only need
// to make a handful of requests. In fact a MaxBurst of zero will
// *never* permit a request with a quantity greater than one because
// it will immediately exceed the limit.
type RateQuota struct {
	MaxRate  Rate
	MaxBurst int
}

// PerSec represents a number of requests per second.
func PerSec(n int) Rate { return Rate{time.Second / time.Duration(n), n} }

// PerMin represents a number of requests per minute.
func PerMin(n int) Rate { return Rate{time.Minute / time.Duration(n), n} }

// PerHour represents a number of requests per hour.
func PerHour(n int) Rate { return Rate{time.Hour / time.Duration(n), n} }

// PerDay represents a number of requests per day.
func PerDay(n int) Rate { return Rate{24 * time.Hour / time.Duration(n), n} }

// GCRARateLimiter is a RateLimiter that users the generic cell-rate
// algorithm. The algorithm has been slightly modified from its usual
// form to support limiting with an additional quantity parameter, such
// as for limiting the number of bytes uploaded.
type GCRARateLimiter struct {
	limit int

	// Think of the DVT as our flexibility:
	// How far can you deviate from the nominal equally spaced schedule?
	// If you like leaky buckets, think about it as the size of your bucket.
	delayVariationTolerance time.Duration

	// Think of the emission interval as the time between events
	// in the nominal equally spaced schedule. If you like leaky buckets,
	// think of it as how frequently the bucket leaks one unit.
	emissionInterval time.Duration

	store GCRAStore
}

// NewGCRARateLimiter creates a GCRARateLimiter. quota.Count defines
// the maximum number of requests permitted in an instantaneous burst
// and quota.Count / quota.Period defines the maximum sustained
// rate. For example, PerMin(60) permits 60 requests instantly per key
// followed by one request per second indefinitely whereas PerSec(1)
// only permits one request per second with no tolerance for bursts.
func NewGCRARateLimiter(st GCRAStore, quota RateQuota) (*GCRARateLimiter, error) {
	if quota.MaxBurst < 0 {
		return nil, fmt.Errorf("Invalid RateQuota %#v. MaxBurst must be greater than zero.", quota)
	}
	if quota.MaxRate.period <= 0 {
		return nil, fmt.Errorf("Invalid RateQuota %#v. MaxRate must be greater than zero.", quota)
	}

	return &GCRARateLimiter{
		delayVariationTolerance: quota.MaxRate.period * (time.Duration(quota.MaxBurst) + 1),
		emissionInterval:        quota.MaxRate.period,
		limit:                   quota.MaxBurst + 1,
		store:                   st,
	}, nil
}

// RateLimit checks whether a particular key has exceeded a rate
// limit. It also returns a RateLimitResult to provide additional
// information about the state of the RateLimiter.
//
// If the rate limit has not been exceeded, the underlying storage is
// updated by the supplied quantity. For example, a quantity of 1
// might be used to rate limit a single request while a greater
// quantity could rate limit based on the size of a file upload in
// megabytes. If quantity is 0, no update is performed allowing you
// to "peek" at the state of the RateLimiter for a given key.
func (g *GCRARateLimiter) RateLimit(key string, quantity int) (bool, RateLimitResult, error) {
	var tat, newTat, now time.Time
	var ttl time.Duration
	rlc := RateLimitResult{Limit: g.limit, RetryAfter: -1}
	limited := false

	i := 0
	for {
		var err error
		var tatVal int64
		var updated bool

		// tat refers to the theoretical arrival time that would be expected
		// from equally spaced requests at exactly the rate limit.
		tatVal, now, err = g.store.GetWithTime(key)
		if err != nil {
			return false, rlc, err
		}

		if tatVal == -1 {
			tat = now
		} else {
			tat = time.Unix(0, tatVal)
		}

		increment := time.Duration(quantity) * g.emissionInterval
		if now.After(tat) {
			newTat = now.Add(increment)
		} else {
			newTat = tat.Add(increment)
		}

		// Block the request if the next permitted time is in the future
		allowAt := newTat.Add(-(g.delayVariationTolerance))
		if diff := now.Sub(allowAt); diff < 0 {
			if increment <= g.delayVariationTolerance {
				rlc.RetryAfter = -diff
			}
			ttl = tat.Sub(now)
			limited = true
			break
		}

		ttl = newTat.Sub(now)

		if tatVal == -1 {
			updated, err = g.store.SetIfNotExistsWithTTL(key, newTat.UnixNano(), ttl)
		} else {
			updated, err = g.store.CompareAndSwapWithTTL(key, tatVal, newTat.UnixNano(), ttl)
		}

		if err != nil {
			return false, rlc, err
		}
		if updated {
			break
		}

		i++
		if i > maxCASAttempts {
			return false, rlc, fmt.Errorf(
				"Failed to store updated rate limit data for key %s after %d attempts",
				key, i,
			)
		}
	}

	next := g.delayVariationTolerance - ttl
	if next > -g.emissionInterval {
		rlc.Remaining = int(next / g.emissionInterval)
	}
	rlc.ResetAfter = ttl

	return limited, rlc, nil
}
<|endoftext|>


<commit_before>package client

import (
	"net/http"
	boshhttp "github.com/cloudfoundry/bosh-utils/httpclient"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/pivotal-golang/s3cli/config"
)

func NewSDK(c config.S3Cli) (*s3.S3, error) {
	var httpClient *http.Client

	if c.SSLVerifyPeer {
		httpClient = boshhttp.CreateDefaultClient(nil)
	}	else {
		httpClient = boshhttp.CreateDefaultClientInsecureSkipVerify()
	}
	httpClient := http.DefaultClient

	s3Config := aws.NewConfig().
		WithLogLevel(aws.LogOff).
		WithS3ForcePathStyle(true).
		WithDisableSSL(!c.UseSSL).
		WithHTTPClient(httpClient)

	if c.UseRegion() {
		s3Config = s3Config.WithRegion(c.Region).WithEndpoint(c.S3Endpoint())
	} else {
		s3Config = s3Config.WithRegion(config.EmptyRegion).WithEndpoint(c.S3Endpoint())
	}

	if c.CredentialsSource == config.StaticCredentialsSource {
		s3Config = s3Config.WithCredentials(credentials.NewStaticCredentials(c.AccessKeyID, c.SecretAccessKey, ""))
	}

	if c.CredentialsSource == config.NoneCredentialsSource {
		s3Config = s3Config.WithCredentials(credentials.AnonymousCredentials)
	}

	s3Session := session.New(s3Config)
	s3Client := s3.New(s3Session)

	if c.UseV2SigningMethod {
		setv2Handlers(s3Client)
	}

	return s3Client, nil
}
<commit_msg>Remove unneeded line in client/sdk.go<commit_after>package client

import (
	"net/http"
	boshhttp "github.com/cloudfoundry/bosh-utils/httpclient"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/pivotal-golang/s3cli/config"
)

func NewSDK(c config.S3Cli) (*s3.S3, error) {
	var httpClient *http.Client

	if c.SSLVerifyPeer {
		httpClient = boshhttp.CreateDefaultClient(nil)
	}	else {
		httpClient = boshhttp.CreateDefaultClientInsecureSkipVerify()
	}

	s3Config := aws.NewConfig().
		WithLogLevel(aws.LogOff).
		WithS3ForcePathStyle(true).
		WithDisableSSL(!c.UseSSL).
		WithHTTPClient(httpClient)

	if c.UseRegion() {
		s3Config = s3Config.WithRegion(c.Region).WithEndpoint(c.S3Endpoint())
	} else {
		s3Config = s3Config.WithRegion(config.EmptyRegion).WithEndpoint(c.S3Endpoint())
	}

	if c.CredentialsSource == config.StaticCredentialsSource {
		s3Config = s3Config.WithCredentials(credentials.NewStaticCredentials(c.AccessKeyID, c.SecretAccessKey, ""))
	}

	if c.CredentialsSource == config.NoneCredentialsSource {
		s3Config = s3Config.WithCredentials(credentials.AnonymousCredentials)
	}

	s3Session := session.New(s3Config)
	s3Client := s3.New(s3Session)

	if c.UseV2SigningMethod {
		setv2Handlers(s3Client)
	}

	return s3Client, nil
}
<|endoftext|>


<commit_before>package clock

import (
	"testing"
	"time"
)

func TestFakeClockGoldenPath(t *testing.T) {
	clk := NewFake()
	second := NewFake()
	if !clk.Now().Equal(second.Now()) {
		t.Errorf("clocks must start out at the same time but didn't: %#v vs %#v", clk.Now(), second.Now())
	}
	clk.Add(3 * time.Second)
	if clk.Now().Equal(second.Now()) {
		t.Errorf("clocks different must differ: %#v vs %#v", clk.Now(), second.Now())
	}
}
<commit_msg>add small examples for Clock and FakeClock<commit_after>package clock

import (
	"fmt"
	"testing"
	"time"
)

func TestFakeClockGoldenPath(t *testing.T) {
	clk := NewFake()
	second := NewFake()
	if !clk.Now().Equal(second.Now()) {
		t.Errorf("clocks must start out at the same time but didn't: %#v vs %#v", clk.Now(), second.Now())
	}
	clk.Add(3 * time.Second)
	if clk.Now().Equal(second.Now()) {
		t.Errorf("clocks different must differ: %#v vs %#v", clk.Now(), second.Now())
	}
}

func ExampleClock() {
	c := Default()
	now := c.Now()
	fmt.Println(now.UTC().Zone())
	// Output:
	// UTC 0
}

func ExampleFakeClock() {
	c := Default()
	fc := NewFake()
	fc.Add(20 * time.Hour)
	fc.Add(-5 * time.Minute) // negatives work, as well

	if fc.Now().Equal(fc.Now()) {
		fmt.Println("FakeClocks' Times always equal themselves.")
	}
	if !c.Now().Equal(fc.Now()) {
		fmt.Println("Clock and FakeClock can be set to different times.")
	}
	if !fc.Now().Equal(NewFake().Now()) {
		fmt.Println("FakeClocks work independently, too.")
	}
	// Output:
	// FakeClocks' Times always equal themselves.
	// Clock and FakeClock can be set to different times.
	// FakeClocks work independently, too.
}
<|endoftext|>


<commit_before>package cmd

import (
	"bufio"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"github.com/alexzorin/onapp/cmd/log"
	"io/ioutil"
	"os"
	"os/user"
	"path/filepath"
	"reflect"
	"strings"
)

const (
	configCmdDescription = "Configure this tool"
	configCmdHelp        = "This command is an interactive wizard to help you setup this tool.\n" +
		"You will need your OnApp dashboard URL, email address and API key to complete it."
)

type config struct {
	ConfigFile string
	ApiUser    string
	ApiKey     string
	Server     string
	Verbose    bool
}

type configCmd struct {
}

func (c configCmd) Run(args []string, ctx *cli) error {

	log.Infoln("This is the configuration wizard. Please provide the following: \n")
	reader := bufio.NewReader(os.Stdin)

	log.Infof("Hostname of the OnApp dashboard: ")
	host, err := reader.ReadString('\n')
	if err != nil {
		return err
	}
	log.Infof("API Username (generally the email address): ")
	user, err := reader.ReadString('\n')
	if err != nil {
		return err
	}
	log.Infof("API Key: ")
	apiKey, err := reader.ReadString('\n')
	if err != nil {
		return err
	}

	log.Infof("Test these details? [yes/no]: ")
	doTest, err := reader.ReadString('\n')
	if err != nil {
		return err
	}
	if strings.ToLower(doTest)[0] == 'y' {
		log.Warnln("Unimplemented")
	}

	ctx.config.Server = strings.Trim(host, "\r\n")
	ctx.config.ApiUser = strings.Trim(user, "\r\n")
	ctx.config.ApiKey = strings.Trim(apiKey, "\r\n")

	err = ctx.config.save()
	if err != nil {
		log.Errorln(err)
	} else {
		log.Infoln("\nSaved configuration to", ctx.config.ConfigFile)
	}

	return nil
}

func (c configCmd) Description() string {
	return configCmdDescription
}

func (c configCmd) Help(args []string) {
	log.Infoln(configCmdHelp)
}

func (c *config) save() error {
	data, err := json.Marshal(c)
	if err != nil {
		return err
	}
	err = ioutil.WriteFile(c.ConfigFile, data, 0)
	if err != nil {
		return err
	}
	return nil
}

func loadConfig() (*config, error) {
	conf := &config{}

	u, err := user.Current()
	if err != nil {
		return conf, err
	}

	flag.StringVar(&conf.ConfigFile, "configFile", fmt.Sprintf("%s%c.onapp", u.HomeDir, os.PathSeparator), "Path to config file")
	flag.BoolVar(&conf.Verbose, "v", false, "Verbose logging")
	flag.Parse()

	_, err = os.Stat(conf.ConfigFile)
	var merge *config
	if err != nil {
		merge = &config{}
	} else {
		rawConf, err := ioutil.ReadFile(conf.ConfigFile)
		if err != nil {
			return conf, errors.New(fmt.Sprintf("Error reading from %s: %s", conf.ConfigFile, err.Error()))
		}
		confFromFile := &config{}
		err = json.Unmarshal(rawConf, confFromFile)
		if err != nil {
			return conf, errors.New(fmt.Sprintf("Error parsing %s: %s", conf.ConfigFile, err.Error()))
		}
		merge = confFromFile
	}
	merged, err := mergeConfigs(conf, merge)
	if err != nil {
		return conf, err
	}

	if merged.ApiUser == "" || merged.ApiKey == "" || merged.Server == "" {
		log.Warnf("You haven't configured yet. Try `%s config`.\n", filepath.Base(os.Args[0]))
	}

	return merged, nil
}

/* Single depth merging, prefers values in `first` over `second` */
func mergeConfigs(first *config, second *config) (*config, error) {
	merged := &config{}
	r1 := reflect.ValueOf(*first)
	r2 := reflect.ValueOf(*second)
	for i := 0; i < r1.NumField(); i++ {
		f1 := r1.Field(i)
		f2 := r2.Field(i)
		dst := reflect.ValueOf(merged).Elem().Field(i)
		switch f1.Kind() {
		case reflect.String:
			if f1.String() == "" {
				dst.SetString(f2.String())
			} else {
				dst.SetString(f1.String())
			}
		case reflect.Bool:
			if f1.Bool() || f2.Bool() {
				dst.SetBool(true)
			}
		}
	}
	return merged, nil
}
<commit_msg>file perms on new config file<commit_after>package cmd

import (
	"bufio"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"github.com/alexzorin/onapp/cmd/log"
	"io/ioutil"
	"os"
	"os/user"
	"path/filepath"
	"reflect"
	"strings"
)

const (
	configCmdDescription = "Configure this tool"
	configCmdHelp        = "This command is an interactive wizard to help you setup this tool.\n" +
		"You will need your OnApp dashboard URL, email address and API key to complete it."
)

type config struct {
	ConfigFile string
	ApiUser    string
	ApiKey     string
	Server     string
	Verbose    bool
}

type configCmd struct {
}

func (c configCmd) Run(args []string, ctx *cli) error {

	log.Infoln("This is the configuration wizard. Please provide the following: \n")
	reader := bufio.NewReader(os.Stdin)

	log.Infof("Hostname of the OnApp dashboard: ")
	host, err := reader.ReadString('\n')
	if err != nil {
		return err
	}
	log.Infof("API Username (generally the email address): ")
	user, err := reader.ReadString('\n')
	if err != nil {
		return err
	}
	log.Infof("API Key: ")
	apiKey, err := reader.ReadString('\n')
	if err != nil {
		return err
	}

	log.Infof("Test these details? [yes/no]: ")
	doTest, err := reader.ReadString('\n')
	if err != nil {
		return err
	}
	if strings.ToLower(doTest)[0] == 'y' {
		log.Warnln("Unimplemented")
	}

	ctx.config.Server = strings.Trim(host, "\r\n")
	ctx.config.ApiUser = strings.Trim(user, "\r\n")
	ctx.config.ApiKey = strings.Trim(apiKey, "\r\n")

	err = ctx.config.save()
	if err != nil {
		log.Errorln(err)
	} else {
		log.Infoln("\nSaved configuration to", ctx.config.ConfigFile)
	}

	return nil
}

func (c configCmd) Description() string {
	return configCmdDescription
}

func (c configCmd) Help(args []string) {
	log.Infoln(configCmdHelp)
}

func (c *config) save() error {
	data, err := json.Marshal(c)
	if err != nil {
		return err
	}
	err = ioutil.WriteFile(c.ConfigFile, data, 0644)
	if err != nil {
		return err
	}
	return nil
}

func loadConfig() (*config, error) {
	conf := &config{}

	u, err := user.Current()
	if err != nil {
		return conf, err
	}

	flag.StringVar(&conf.ConfigFile, "configFile", fmt.Sprintf("%s%c.onapp", u.HomeDir, os.PathSeparator), "Path to config file")
	flag.BoolVar(&conf.Verbose, "v", false, "Verbose logging")
	flag.Parse()

	_, err = os.Stat(conf.ConfigFile)
	var merge *config
	if err != nil {
		merge = &config{}
	} else {
		rawConf, err := ioutil.ReadFile(conf.ConfigFile)
		if err != nil {
			return conf, errors.New(fmt.Sprintf("Error reading from %s: %s", conf.ConfigFile, err.Error()))
		}
		confFromFile := &config{}
		err = json.Unmarshal(rawConf, confFromFile)
		if err != nil {
			return conf, errors.New(fmt.Sprintf("Error parsing %s: %s", conf.ConfigFile, err.Error()))
		}
		merge = confFromFile
	}
	merged, err := mergeConfigs(conf, merge)
	if err != nil {
		return conf, err
	}

	if merged.ApiUser == "" || merged.ApiKey == "" || merged.Server == "" {
		log.Warnf("You haven't configured yet. Try `%s config`.\n", filepath.Base(os.Args[0]))
	}

	return merged, nil
}

/* Single depth merging, prefers values in `first` over `second` */
func mergeConfigs(first *config, second *config) (*config, error) {
	merged := &config{}
	r1 := reflect.ValueOf(*first)
	r2 := reflect.ValueOf(*second)
	for i := 0; i < r1.NumField(); i++ {
		f1 := r1.Field(i)
		f2 := r2.Field(i)
		dst := reflect.ValueOf(merged).Elem().Field(i)
		switch f1.Kind() {
		case reflect.String:
			if f1.String() == "" {
				dst.SetString(f2.String())
			} else {
				dst.SetString(f1.String())
			}
		case reflect.Bool:
			if f1.Bool() || f2.Bool() {
				dst.SetBool(true)
			}
		}
	}
	return merged, nil
}
<|endoftext|>


<commit_before>// Copyright 2015 ISRG.  All rights reserved
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

package cmd

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"strings"
	"time"

	cfsslConfig "github.com/letsencrypt/boulder/Godeps/_workspace/src/github.com/cloudflare/cfssl/config"
	"github.com/letsencrypt/boulder/core"
	"github.com/letsencrypt/boulder/publisher"
	"github.com/letsencrypt/boulder/va"
)

// Config stores configuration parameters that applications
// will need.  For simplicity, we just lump them all into
// one struct, and use encoding/json to read it from a file.
//
// Note: NO DEFAULTS are provided.
type Config struct {
	ActivityMonitor struct {
		ServiceConfig
	}

	// Default AMQPConfig for services that don't specify one.
	// TODO(jsha): Delete this after a deploy.
	AMQP *AMQPConfig

	WFE struct {
		ServiceConfig
		BaseURL       string
		ListenAddress string

		AllowOrigins []string

		CertCacheDuration           string
		CertNoCacheExpirationWindow string
		IndexCacheDuration          string
		IssuerCacheDuration         string

		ShutdownStopTimeout string
		ShutdownKillTimeout string
	}

	CA CAConfig

	RA struct {
		ServiceConfig

		RateLimitPoliciesFilename string

		MaxConcurrentRPCServerRequests int64

		MaxContactsPerRegistration int

		// UseIsSafeDomain determines whether to call VA.IsSafeDomain
		UseIsSafeDomain bool // TODO(jmhodges): remove after va IsSafeDomain deploy
	}

	SA struct {
		ServiceConfig
		DBConfig

		MaxConcurrentRPCServerRequests int64
	}

	VA struct {
		ServiceConfig

		UserAgent string

		PortConfig va.PortConfig

		MaxConcurrentRPCServerRequests int64

		GoogleSafeBrowsing *GoogleSafeBrowsingConfig
	}

	SQL struct {
		SQLDebug bool
	}

	Statsd StatsdConfig

	Syslog SyslogConfig

	Revoker struct {
		DBConfig
		// The revoker isn't a long running service, so doesn't get a full
		// ServiceConfig, just an AMQPConfig.
		AMQP *AMQPConfig
	}

	Mailer struct {
		ServiceConfig
		DBConfig

		Server   string
		Port     string
		Username string
		Password string

		CertLimit int
		NagTimes  []string
		// How much earlier (than configured nag intervals) to
		// send reminders, to account for the expected delay
		// before the next expiration-mailer invocation.
		NagCheckInterval string
		// Path to a text/template email template
		EmailTemplate string
	}

	OCSPResponder struct {
		ServiceConfig
		DBConfig

		// Source indicates the source of pre-signed OCSP responses to be used. It
		// can be a DBConnect string or a file URL. The file URL style is used
		// when responding from a static file for intermediates and roots.
		// If DBConfig has non-empty fields, it takes precedence over this.
		Source string

		Path          string
		ListenAddress string
		// MaxAge is the max-age to set in the Cache-Controler response
		// header. It is a time.Duration formatted string.
		MaxAge ConfigDuration

		ShutdownStopTimeout string
		ShutdownKillTimeout string
	}

	OCSPUpdater OCSPUpdaterConfig

	Publisher struct {
		ServiceConfig
		MaxConcurrentRPCServerRequests int64
	}

	ExternalCertImporter struct {
		CertsToImportCSVFilename   string
		DomainsToImportCSVFilename string
		CertsToRemoveCSVFilename   string
		StatsdRate                 float32
	}

	PA PAConfig

	Common struct {
		BaseURL string
		// Path to a PEM-encoded copy of the issuer certificate.
		IssuerCert string

		DNSResolver               string
		DNSTimeout                string
		DNSAllowLoopbackAddresses bool

		CT publisher.CTConfig
	}

	CertChecker struct {
		DBConfig

		Workers             int
		ReportDirectoryPath string
	}

	SubscriberAgreementURL string
}

// ServiceConfig contains config items that are common to all our services, to
// be embedded in other config structs.
type ServiceConfig struct {
	// DebugAddr is the address to run the /debug handlers on.
	DebugAddr string
	AMQP      *AMQPConfig
}

type DBConfig struct {
	DBConnect string
	// A file containing a connect URL for the DB.
	DBConnectFile string
}

func (d *DBConfig) URL() (string, error) {
	if c.DBConnectFile != "" {
		return ioutil.ReadFile(c.DBConnectFile)
	}
	return c.DBConnect, nil
}

// AMQPConfig describes how to connect to AMQP, and how to speak to each of the
// RPC services we offer via AMQP.
type AMQPConfig struct {
	// A file containing a connect URL for the server.
	ServerURLFile string

	Server    string
	Insecure  bool
	RA        *RPCServerConfig
	VA        *RPCServerConfig
	SA        *RPCServerConfig
	CA        *RPCServerConfig
	Publisher *RPCServerConfig
	TLS       *TLSConfig
	// Queue name on which to listen, if this is an RPC service (vs acting only as
	// an RPC client).
	ServiceQueue      string
	ReconnectTimeouts struct {
		Base ConfigDuration
		Max  ConfigDuration
	}
}

// CAConfig structs have configuration information for the certificate
// authority, including database parameters as well as controls for
// issued certificates.
type CAConfig struct {
	ServiceConfig
	DBConfig

	Profile      string
	TestMode     bool
	SerialPrefix int
	Key          KeyConfig
	// LifespanOCSP is how long OCSP responses are valid for; It should be longer
	// than the minTimeToExpiry field for the OCSP Updater.
	LifespanOCSP string
	// How long issued certificates are valid for, should match expiry field
	// in cfssl config.
	Expiry string
	// The maximum number of subjectAltNames in a single certificate
	MaxNames int
	CFSSL    cfsslConfig.Config

	MaxConcurrentRPCServerRequests int64

	HSMFaultTimeout ConfigDuration
}

// PAConfig specifies how a policy authority should connect to its
// database, what policies it should enforce, and what challenges
// it should offer.
type PAConfig struct {
	DBConfig
	EnforcePolicyWhitelist bool
	Challenges             map[string]bool
}

// CheckChallenges checks whether the list of challenges in the PA config
// actually contains valid challenge names
func (pc PAConfig) CheckChallenges() error {
	for name := range pc.Challenges {
		if !core.ValidChallenge(name) {
			return fmt.Errorf("Invalid challenge in PA config: %s", name)
		}
	}
	return nil
}

// SetDefaultChallengesIfEmpty sets a default list of challenges if no
// challenges are enabled in the PA config.  The set of challenges specified
// corresponds to the set that was hard-coded before these configuration
// options were added.
func (pc *PAConfig) SetDefaultChallengesIfEmpty() {
	if len(pc.Challenges) == 0 {
		pc.Challenges = map[string]bool{}
		pc.Challenges[core.ChallengeTypeSimpleHTTP] = true
		pc.Challenges[core.ChallengeTypeDVSNI] = true
		pc.Challenges[core.ChallengeTypeHTTP01] = true
		pc.Challenges[core.ChallengeTypeTLSSNI01] = true
	}
}

// KeyConfig should contain either a File path to a PEM-format private key,
// or a PKCS11Config defining how to load a module for an HSM.
type KeyConfig struct {
	File   string
	PKCS11 PKCS11Config
}

// PKCS11Config defines how to load a module for an HSM.
type PKCS11Config struct {
	Module          string
	TokenLabel      string
	PIN             string
	PrivateKeyLabel string
}

// TLSConfig reprents certificates and a key for authenticated TLS.
type TLSConfig struct {
	CertFile   *string
	KeyFile    *string
	CACertFile *string
}

// RPCServerConfig contains configuration particular to a specific RPC server
// type (e.g. RA, SA, etc)
type RPCServerConfig struct {
	Server     string // Queue name where the server receives requests
	RPCTimeout ConfigDuration
}

// OCSPUpdaterConfig provides the various window tick times and batch sizes needed
// for the OCSP (and SCT) updater
type OCSPUpdaterConfig struct {
	ServiceConfig
	DBConfig

	NewCertificateWindow     ConfigDuration
	OldOCSPWindow            ConfigDuration
	MissingSCTWindow         ConfigDuration
	RevokedCertificateWindow ConfigDuration

	NewCertificateBatchSize     int
	OldOCSPBatchSize            int
	MissingSCTBatchSize         int
	RevokedCertificateBatchSize int

	OCSPMinTimeToExpiry ConfigDuration
	OldestIssuedSCT     ConfigDuration

	AkamaiBaseURL           string
	AkamaiClientToken       string
	AkamaiClientSecret      string
	AkamaiAccessToken       string
	AkamaiPurgeRetries      int
	AkamaiPurgeRetryBackoff ConfigDuration

	SignFailureBackoffFactor float64
	SignFailureBackoffMax    ConfigDuration
}

// GoogleSafeBrowsingConfig is the JSON config struct for the VA's use of the
// Google Safe Browsing API.
type GoogleSafeBrowsingConfig struct {
	APIKey  string
	DataDir string
}

// SyslogConfig defines the config for syslogging.
type SyslogConfig struct {
	Network     string
	Server      string
	StdoutLevel *int
}

// StatsdConfig defines the config for Statsd.
type StatsdConfig struct {
	Server string
	Prefix string
}

// ConfigDuration is just an alias for time.Duration that allows
// serialization to YAML as well as JSON.
type ConfigDuration struct {
	time.Duration
}

// ErrDurationMustBeString is returned when a non-string value is
// presented to be deserialized as a ConfigDuration
var ErrDurationMustBeString = errors.New("cannot JSON unmarshal something other than a string into a ConfigDuration")

// UnmarshalJSON parses a string into a ConfigDuration using
// time.ParseDuration.  If the input does not unmarshal as a
// string, then UnmarshalJSON returns ErrDurationMustBeString.
func (d *ConfigDuration) UnmarshalJSON(b []byte) error {
	s := ""
	err := json.Unmarshal(b, &s)
	if err != nil {
		if _, ok := err.(*json.UnmarshalTypeError); ok {
			return ErrDurationMustBeString
		}
		return err
	}
	dd, err := time.ParseDuration(s)
	d.Duration = dd
	return err
}

// MarshalJSON returns the string form of the duration, as a byte array.
func (d ConfigDuration) MarshalJSON() ([]byte, error) {
	return []byte(d.Duration.String()), nil
}

// UnmarshalYAML uses the same frmat as JSON, but is called by the YAML
// parser (vs. the JSON parser).
func (d *ConfigDuration) UnmarshalYAML(unmarshal func(interface{}) error) error {
	var s string
	if err := unmarshal(&s); err != nil {
		return err
	}
	dur, err := time.ParseDuration(s)
	if err != nil {
		return err
	}

	d.Duration = dur
	return nil
}

// A ConfigSecret represents a string-valued config field. It may be specified
// directly in the config or, if it starts with the string "secret:", its
// contents are read from the filename that comes after "secret:", with
// trailing newlines removed.
type ConfigSecret string

var errSecretMustBeString = errors.New("cannot JSON unmarshal something other than a string into a ConfigSecret")

const secretPrefix = "secret:"

// UnmarshalJSON unmarshals a ConfigSecret
func (d *ConfigSecret) UnmarshalJSON(b []byte) error {
	s := ""
	err := json.Unmarshal(b, &s)
	if err != nil {
		if _, ok := err.(*json.UnmarshalTypeError); ok {
			return errSecretMustBeString
		}
		return err
	}
	if !strings.HasPrefix(s, secretPrefix) {
		*d = ConfigSecret(s)
		return nil
	}
	contents, err := ioutil.ReadFile(s[len(secretPrefix):])
	if err != nil {
		return err
	}
	*d = ConfigSecret(strings.TrimRight(string(contents), "\n"))
	return nil
}
<commit_msg>Remove old traces of ConfigSecret<commit_after>// Copyright 2015 ISRG.  All rights reserved
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

package cmd

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"strings"
	"time"

	cfsslConfig "github.com/letsencrypt/boulder/Godeps/_workspace/src/github.com/cloudflare/cfssl/config"
	"github.com/letsencrypt/boulder/core"
	"github.com/letsencrypt/boulder/publisher"
	"github.com/letsencrypt/boulder/va"
)

// Config stores configuration parameters that applications
// will need.  For simplicity, we just lump them all into
// one struct, and use encoding/json to read it from a file.
//
// Note: NO DEFAULTS are provided.
type Config struct {
	ActivityMonitor struct {
		ServiceConfig
	}

	// Default AMQPConfig for services that don't specify one.
	// TODO(jsha): Delete this after a deploy.
	AMQP *AMQPConfig

	WFE struct {
		ServiceConfig
		BaseURL       string
		ListenAddress string

		AllowOrigins []string

		CertCacheDuration           string
		CertNoCacheExpirationWindow string
		IndexCacheDuration          string
		IssuerCacheDuration         string

		ShutdownStopTimeout string
		ShutdownKillTimeout string
	}

	CA CAConfig

	RA struct {
		ServiceConfig

		RateLimitPoliciesFilename string

		MaxConcurrentRPCServerRequests int64

		MaxContactsPerRegistration int

		// UseIsSafeDomain determines whether to call VA.IsSafeDomain
		UseIsSafeDomain bool // TODO(jmhodges): remove after va IsSafeDomain deploy
	}

	SA struct {
		ServiceConfig
		DBConfig

		MaxConcurrentRPCServerRequests int64
	}

	VA struct {
		ServiceConfig

		UserAgent string

		PortConfig va.PortConfig

		MaxConcurrentRPCServerRequests int64

		GoogleSafeBrowsing *GoogleSafeBrowsingConfig
	}

	SQL struct {
		SQLDebug bool
	}

	Statsd StatsdConfig

	Syslog SyslogConfig

	Revoker struct {
		DBConfig
		// The revoker isn't a long running service, so doesn't get a full
		// ServiceConfig, just an AMQPConfig.
		AMQP *AMQPConfig
	}

	Mailer struct {
		ServiceConfig
		DBConfig

		Server   string
		Port     string
		Username string
		Password string

		CertLimit int
		NagTimes  []string
		// How much earlier (than configured nag intervals) to
		// send reminders, to account for the expected delay
		// before the next expiration-mailer invocation.
		NagCheckInterval string
		// Path to a text/template email template
		EmailTemplate string
	}

	OCSPResponder struct {
		ServiceConfig
		DBConfig

		// Source indicates the source of pre-signed OCSP responses to be used. It
		// can be a DBConnect string or a file URL. The file URL style is used
		// when responding from a static file for intermediates and roots.
		// If DBConfig has non-empty fields, it takes precedence over this.
		Source string

		Path          string
		ListenAddress string
		// MaxAge is the max-age to set in the Cache-Controler response
		// header. It is a time.Duration formatted string.
		MaxAge ConfigDuration

		ShutdownStopTimeout string
		ShutdownKillTimeout string
	}

	OCSPUpdater OCSPUpdaterConfig

	Publisher struct {
		ServiceConfig
		MaxConcurrentRPCServerRequests int64
	}

	ExternalCertImporter struct {
		CertsToImportCSVFilename   string
		DomainsToImportCSVFilename string
		CertsToRemoveCSVFilename   string
		StatsdRate                 float32
	}

	PA PAConfig

	Common struct {
		BaseURL string
		// Path to a PEM-encoded copy of the issuer certificate.
		IssuerCert string

		DNSResolver               string
		DNSTimeout                string
		DNSAllowLoopbackAddresses bool

		CT publisher.CTConfig
	}

	CertChecker struct {
		DBConfig

		Workers             int
		ReportDirectoryPath string
	}

	SubscriberAgreementURL string
}

// ServiceConfig contains config items that are common to all our services, to
// be embedded in other config structs.
type ServiceConfig struct {
	// DebugAddr is the address to run the /debug handlers on.
	DebugAddr string
	AMQP      *AMQPConfig
}

type DBConfig struct {
	DBConnect string
	// A file containing a connect URL for the DB.
	DBConnectFile string
}

func (d *DBConfig) URL() (string, error) {
	if c.DBConnectFile != "" {
		return ioutil.ReadFile(c.DBConnectFile)
	}
	return c.DBConnect, nil
}

// AMQPConfig describes how to connect to AMQP, and how to speak to each of the
// RPC services we offer via AMQP.
type AMQPConfig struct {
	Server    string
	Insecure  bool
	RA        *RPCServerConfig
	VA        *RPCServerConfig
	SA        *RPCServerConfig
	CA        *RPCServerConfig
	Publisher *RPCServerConfig
	TLS       *TLSConfig
	// Queue name on which to listen, if this is an RPC service (vs acting only as
	// an RPC client).
	ServiceQueue      string
	ReconnectTimeouts struct {
		Base ConfigDuration
		Max  ConfigDuration
	}
}

// CAConfig structs have configuration information for the certificate
// authority, including database parameters as well as controls for
// issued certificates.
type CAConfig struct {
	ServiceConfig
	DBConfig

	Profile      string
	TestMode     bool
	SerialPrefix int
	Key          KeyConfig
	// LifespanOCSP is how long OCSP responses are valid for; It should be longer
	// than the minTimeToExpiry field for the OCSP Updater.
	LifespanOCSP string
	// How long issued certificates are valid for, should match expiry field
	// in cfssl config.
	Expiry string
	// The maximum number of subjectAltNames in a single certificate
	MaxNames int
	CFSSL    cfsslConfig.Config

	MaxConcurrentRPCServerRequests int64

	HSMFaultTimeout ConfigDuration
}

// PAConfig specifies how a policy authority should connect to its
// database, what policies it should enforce, and what challenges
// it should offer.
type PAConfig struct {
	DBConfig
	EnforcePolicyWhitelist bool
	Challenges             map[string]bool
}

// CheckChallenges checks whether the list of challenges in the PA config
// actually contains valid challenge names
func (pc PAConfig) CheckChallenges() error {
	for name := range pc.Challenges {
		if !core.ValidChallenge(name) {
			return fmt.Errorf("Invalid challenge in PA config: %s", name)
		}
	}
	return nil
}

// SetDefaultChallengesIfEmpty sets a default list of challenges if no
// challenges are enabled in the PA config.  The set of challenges specified
// corresponds to the set that was hard-coded before these configuration
// options were added.
func (pc *PAConfig) SetDefaultChallengesIfEmpty() {
	if len(pc.Challenges) == 0 {
		pc.Challenges = map[string]bool{}
		pc.Challenges[core.ChallengeTypeSimpleHTTP] = true
		pc.Challenges[core.ChallengeTypeDVSNI] = true
		pc.Challenges[core.ChallengeTypeHTTP01] = true
		pc.Challenges[core.ChallengeTypeTLSSNI01] = true
	}
}

// KeyConfig should contain either a File path to a PEM-format private key,
// or a PKCS11Config defining how to load a module for an HSM.
type KeyConfig struct {
	File   string
	PKCS11 PKCS11Config
}

// PKCS11Config defines how to load a module for an HSM.
type PKCS11Config struct {
	Module          string
	TokenLabel      string
	PIN             string
	PrivateKeyLabel string
}

// TLSConfig reprents certificates and a key for authenticated TLS.
type TLSConfig struct {
	CertFile   *string
	KeyFile    *string
	CACertFile *string
}

// RPCServerConfig contains configuration particular to a specific RPC server
// type (e.g. RA, SA, etc)
type RPCServerConfig struct {
	Server     string // Queue name where the server receives requests
	RPCTimeout ConfigDuration
}

// OCSPUpdaterConfig provides the various window tick times and batch sizes needed
// for the OCSP (and SCT) updater
type OCSPUpdaterConfig struct {
	ServiceConfig
	DBConfig

	NewCertificateWindow     ConfigDuration
	OldOCSPWindow            ConfigDuration
	MissingSCTWindow         ConfigDuration
	RevokedCertificateWindow ConfigDuration

	NewCertificateBatchSize     int
	OldOCSPBatchSize            int
	MissingSCTBatchSize         int
	RevokedCertificateBatchSize int

	OCSPMinTimeToExpiry ConfigDuration
	OldestIssuedSCT     ConfigDuration

	AkamaiBaseURL           string
	AkamaiClientToken       string
	AkamaiClientSecret      string
	AkamaiAccessToken       string
	AkamaiPurgeRetries      int
	AkamaiPurgeRetryBackoff ConfigDuration

	SignFailureBackoffFactor float64
	SignFailureBackoffMax    ConfigDuration
}

// GoogleSafeBrowsingConfig is the JSON config struct for the VA's use of the
// Google Safe Browsing API.
type GoogleSafeBrowsingConfig struct {
	APIKey  string
	DataDir string
}

// SyslogConfig defines the config for syslogging.
type SyslogConfig struct {
	Network     string
	Server      string
	StdoutLevel *int
}

// StatsdConfig defines the config for Statsd.
type StatsdConfig struct {
	Server string
	Prefix string
}

// ConfigDuration is just an alias for time.Duration that allows
// serialization to YAML as well as JSON.
type ConfigDuration struct {
	time.Duration
}

// ErrDurationMustBeString is returned when a non-string value is
// presented to be deserialized as a ConfigDuration
var ErrDurationMustBeString = errors.New("cannot JSON unmarshal something other than a string into a ConfigDuration")

// UnmarshalJSON parses a string into a ConfigDuration using
// time.ParseDuration.  If the input does not unmarshal as a
// string, then UnmarshalJSON returns ErrDurationMustBeString.
func (d *ConfigDuration) UnmarshalJSON(b []byte) error {
	s := ""
	err := json.Unmarshal(b, &s)
	if err != nil {
		if _, ok := err.(*json.UnmarshalTypeError); ok {
			return ErrDurationMustBeString
		}
		return err
	}
	dd, err := time.ParseDuration(s)
	d.Duration = dd
	return err
}

// MarshalJSON returns the string form of the duration, as a byte array.
func (d ConfigDuration) MarshalJSON() ([]byte, error) {
	return []byte(d.Duration.String()), nil
}

// UnmarshalYAML uses the same frmat as JSON, but is called by the YAML
// parser (vs. the JSON parser).
func (d *ConfigDuration) UnmarshalYAML(unmarshal func(interface{}) error) error {
	var s string
	if err := unmarshal(&s); err != nil {
		return err
	}
	dur, err := time.ParseDuration(s)
	if err != nil {
		return err
	}

	d.Duration = dur
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2016 CodisLabs. All Rights Reserved.
// Licensed under the MIT (MIT-LICENSE.txt) license.

package main

import (
	"bufio"
	"bytes"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"io"
	"os"
	"time"

	"github.com/CodisLabs/redis-port/pkg/libs/atomic2"
	"github.com/CodisLabs/redis-port/pkg/libs/log"
	"github.com/CodisLabs/redis-port/pkg/rdb"
)

type cmdDecode struct {
	rbytes, wbytes, nentry atomic2.Int64
}

type cmdDecodeStat struct {
	rbytes, wbytes, nentry int64
}

func (cmd *cmdDecode) Stat() *cmdDecodeStat {
	return &cmdDecodeStat{
		rbytes: cmd.rbytes.Get(),
		wbytes: cmd.wbytes.Get(),
		nentry: cmd.nentry.Get(),
	}
}

func (cmd *cmdDecode) Main() {
	input, output := args.input, args.output
	if len(input) == 0 {
		input = "/dev/stdin"
	}
	if len(output) == 0 {
		output = "/dev/stdout"
	}

	log.Infof("decode from '%s' to '%s'\n", input, output)

	var readin io.ReadCloser
	var nsize int64
	if input != "/dev/stdin" {
		readin, nsize = openReadFile(input)
		defer readin.Close()
	} else {
		readin, nsize = os.Stdin, 0
	}

	var saveto io.WriteCloser
	if output != "/dev/stdout" {
		saveto = openWriteFile(output)
		defer saveto.Close()
	} else {
		saveto = os.Stdout
	}

	reader := bufio.NewReaderSize(readin, ReaderBufferSize)
	writer := bufio.NewWriterSize(saveto, WriterBufferSize)

	ipipe := newRDBLoader(reader, &cmd.rbytes, args.parallel*32)
	opipe := make(chan string, cap(ipipe))

	go func() {
		defer close(opipe)
		group := make(chan int, args.parallel)
		for i := 0; i < cap(group); i++ {
			go func() {
				defer func() {
					group <- 0
				}()
				cmd.decoderMain(ipipe, opipe)
			}()
		}
		for i := 0; i < cap(group); i++ {
			<-group
		}
	}()

	wait := make(chan struct{})
	go func() {
		defer close(wait)
		for s := range opipe {
			cmd.wbytes.Add(int64(len(s)))
			if _, err := writer.WriteString(s); err != nil {
				log.PanicError(err, "write string failed")
			}
			flushWriter(writer)
		}
	}()

	for done := false; !done; {
		select {
		case <-wait:
			done = true
		case <-time.After(time.Second):
		}
		stat := cmd.Stat()
		var b bytes.Buffer
		fmt.Fprintf(&b, "decode: ")
		if nsize != 0 {
			fmt.Fprintf(&b, "total = %d - %12d [%3d%%]", nsize, stat.rbytes, 100*stat.rbytes/nsize)
		} else {
			fmt.Fprintf(&b, "total = %12d", stat.rbytes)
		}
		fmt.Fprintf(&b, "  write=%-12d", stat.wbytes)
		fmt.Fprintf(&b, "  entry=%-12d", stat.nentry)
		log.Info(b.String())
	}
	log.Info("decode: done")
}

func (cmd *cmdDecode) decoderMain(ipipe <-chan *rdb.BinEntry, opipe chan<- string) {
	toText := func(p []byte) string {
		var b bytes.Buffer
		for _, c := range p {
			switch {
			case c >= '#' && c <= '~':
				b.WriteByte(c)
			default:
				b.WriteByte('.')
			}
		}
		return b.String()
	}
	toBase64 := func(p []byte) string {
		return base64.StdEncoding.EncodeToString(p)
	}
	toJson := func(o interface{}) string {
		b, err := json.Marshal(o)
		if err != nil {
			log.PanicError(err, "encode to json failed")
		}
		return string(b)
	}
	for e := range ipipe {
		o, err := rdb.DecodeDump(e.Value)
		if err != nil {
			log.PanicError(err, "decode failed")
		}
		var b bytes.Buffer
		switch obj := o.(type) {
		default:
			log.Panicf("unknown object %v", o)
		case rdb.String:
			o := &struct {
				DB       uint32 `json:"db"`
				Type     string `json:"type"`
				ExpireAt uint64 `json:"expireat"`
				Key      string `json:"key"`
				Key64    string `json:"key64"`
				Value64  string `json:"value64"`
			}{
				e.DB, "string", e.ExpireAt, toText(e.Key), toBase64(e.Key),
				toBase64(obj),
			}
			fmt.Fprintf(&b, "%s\n", toJson(o))
		case rdb.List:
			for i, ele := range obj {
				o := &struct {
					DB       uint32 `json:"db"`
					Type     string `json:"type"`
					ExpireAt uint64 `json:"expireat"`
					Key      string `json:"key"`
					Key64    string `json:"key64"`
					Index    int    `json:"index"`
					Value64  string `json:"value64"`
				}{
					e.DB, "list", e.ExpireAt, toText(e.Key), toBase64(e.Key),
					i, toBase64(ele),
				}
				fmt.Fprintf(&b, "%s\n", toJson(o))
			}
		case rdb.Hash:
			for _, ele := range obj {
				o := &struct {
					DB       uint32 `json:"db"`
					Type     string `json:"type"`
					ExpireAt uint64 `json:"expireat"`
					Key      string `json:"key"`
					Key64    string `json:"key64"`
					Field    string `json:"field"`
					Field64  string `json:"field64"`
					Value64  string `json:"value64"`
				}{
					e.DB, "hash", e.ExpireAt, toText(e.Key), toBase64(e.Key),
					toText(ele.Field), toBase64(ele.Field), toBase64(ele.Value),
				}
				fmt.Fprintf(&b, "%s\n", toJson(o))
			}
		case rdb.Set:
			for _, mem := range obj {
				o := &struct {
					DB       uint32 `json:"db"`
					Type     string `json:"type"`
					ExpireAt uint64 `json:"expireat"`
					Key      string `json:"key"`
					Key64    string `json:"key64"`
					Member   string `json:"member"`
					Member64 string `json:"member64"`
				}{
					e.DB, "set", e.ExpireAt, toText(e.Key), toBase64(e.Key),
					toText(mem), toBase64(mem),
				}
				fmt.Fprintf(&b, "%s\n", toJson(o))
			}
		case rdb.ZSet:
			for _, ele := range obj {
				o := &struct {
					DB       uint32  `json:"db"`
					Type     string  `json:"type"`
					ExpireAt uint64  `json:"expireat"`
					Key      string  `json:"key"`
					Key64    string  `json:"key64"`
					Member   string  `json:"member"`
					Member64 string  `json:"member64"`
					Score    float64 `json:"score"`
				}{
					e.DB, "zset", e.ExpireAt, toText(e.Key), toBase64(e.Key),
					toText(ele.Member), toBase64(ele.Member), ele.Score,
				}
				fmt.Fprintf(&b, "%s\n", toJson(o))
			}
		}
		cmd.nentry.Incr()
		opipe <- b.String()
	}
}
<commit_msg>cmd: update format of sub-command 'decode'<commit_after>// Copyright 2016 CodisLabs. All Rights Reserved.
// Licensed under the MIT (MIT-LICENSE.txt) license.

package main

import (
	"bufio"
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"os"
	"time"

	"github.com/CodisLabs/redis-port/pkg/libs/atomic2"
	"github.com/CodisLabs/redis-port/pkg/libs/log"
	"github.com/CodisLabs/redis-port/pkg/rdb"
)

type cmdDecode struct {
	rbytes, wbytes, nentry atomic2.Int64
}

type cmdDecodeStat struct {
	rbytes, wbytes, nentry int64
}

func (cmd *cmdDecode) Stat() *cmdDecodeStat {
	return &cmdDecodeStat{
		rbytes: cmd.rbytes.Get(),
		wbytes: cmd.wbytes.Get(),
		nentry: cmd.nentry.Get(),
	}
}

func (cmd *cmdDecode) Main() {
	input, output := args.input, args.output
	if len(input) == 0 {
		input = "/dev/stdin"
	}
	if len(output) == 0 {
		output = "/dev/stdout"
	}

	log.Infof("decode from '%s' to '%s'\n", input, output)

	var readin io.ReadCloser
	var nsize int64
	if input != "/dev/stdin" {
		readin, nsize = openReadFile(input)
		defer readin.Close()
	} else {
		readin, nsize = os.Stdin, 0
	}

	var saveto io.WriteCloser
	if output != "/dev/stdout" {
		saveto = openWriteFile(output)
		defer saveto.Close()
	} else {
		saveto = os.Stdout
	}

	reader := bufio.NewReaderSize(readin, ReaderBufferSize)
	writer := bufio.NewWriterSize(saveto, WriterBufferSize)

	ipipe := newRDBLoader(reader, &cmd.rbytes, args.parallel*32)
	opipe := make(chan string, cap(ipipe))

	go func() {
		defer close(opipe)
		group := make(chan int, args.parallel)
		for i := 0; i < cap(group); i++ {
			go func() {
				defer func() {
					group <- 0
				}()
				cmd.decoderMain(ipipe, opipe)
			}()
		}
		for i := 0; i < cap(group); i++ {
			<-group
		}
	}()

	wait := make(chan struct{})
	go func() {
		defer close(wait)
		for s := range opipe {
			cmd.wbytes.Add(int64(len(s)))
			if _, err := writer.WriteString(s); err != nil {
				log.PanicError(err, "write string failed")
			}
			flushWriter(writer)
		}
	}()

	for done := false; !done; {
		select {
		case <-wait:
			done = true
		case <-time.After(time.Second):
		}
		stat := cmd.Stat()
		var b bytes.Buffer
		fmt.Fprintf(&b, "decode: ")
		if nsize != 0 {
			fmt.Fprintf(&b, "total = %d - %12d [%3d%%]", nsize, stat.rbytes, 100*stat.rbytes/nsize)
		} else {
			fmt.Fprintf(&b, "total = %12d", stat.rbytes)
		}
		fmt.Fprintf(&b, "  write=%-12d", stat.wbytes)
		fmt.Fprintf(&b, "  entry=%-12d", stat.nentry)
		log.Info(b.String())
	}
	log.Info("decode: done")
}

func (cmd *cmdDecode) decoderMain(ipipe <-chan *rdb.BinEntry, opipe chan<- string) {
	encodeJson := func(w *bytes.Buffer, o interface{}) {
		b, err := json.Marshal(o)
		if err != nil {
			log.PanicError(err, "encode to json failed")
		}
		if _, err := w.Write(b); err != nil {
			log.PanicError(err, "encode to json failed")
		}
		if _, err := w.WriteString("\n"); err != nil {
			log.PanicError(err, "encode to json failed")
		}
	}
	for e := range ipipe {
		o, err := rdb.DecodeDump(e.Value)
		if err != nil {
			log.PanicError(err, "decode failed")
		}
		var b = &bytes.Buffer{}
		switch obj := o.(type) {
		default:
			log.Panicf("unknown object %v", o)
		case rdb.String:
			encodeJson(b, &struct {
				DB    uint32 `json:"db"`
				Type  string `json:"type"`
				Key   string `json:"key"`
				Value string `json:"value"`
			}{
				e.DB, "string", string(e.Key), string(obj),
			})
		case rdb.List:
			for i, ele := range obj {
				encodeJson(b, &struct {
					DB    uint32 `json:"db"`
					Type  string `json:"type"`
					Key   string `json:"key"`
					Index int    `json:"index"`
					Value string `json:"value"`
				}{
					e.DB, "list", string(e.Key), i, string(ele),
				})
			}
		case rdb.Hash:
			for _, ele := range obj {
				encodeJson(b, &struct {
					DB    uint32 `json:"db"`
					Type  string `json:"type"`
					Key   string `json:"key"`
					Field string `json:"field"`
					Value string `json:"value"`
				}{
					e.DB, "hash", string(e.Key), string(ele.Field), string(ele.Value),
				})
			}
		case rdb.Set:
			for _, mem := range obj {
				encodeJson(b, &struct {
					DB     uint32 `json:"db"`
					Type   string `json:"type"`
					Key    string `json:"key"`
					Member string `json:"member"`
				}{
					e.DB, "dict", string(e.Key), string(mem),
				})
			}
		case rdb.ZSet:
			for _, ele := range obj {
				encodeJson(b, &struct {
					DB     uint32  `json:"db"`
					Type   string  `json:"type"`
					Key    string  `json:"key"`
					Member string  `json:"member"`
					Score  float64 `json:"score"`
				}{
					e.DB, "zset", string(e.Key), string(ele.Member), ele.Score,
				})
			}
		}
		if e.ExpireAt != 0 {
			encodeJson(b, &struct {
				DB       uint32 `json:"db"`
				Type     string `json:"type"`
				Key      string `json:"key"`
				ExpireAt uint64 `json:"expireat"`
			}{
				e.DB, "expire", string(e.Key), e.ExpireAt,
			})
		}
		cmd.nentry.Incr()
		opipe <- b.String()
	}
}
<|endoftext|>


<commit_before>package cmd

import (
	"strings"

	"github.com/nextgearcapital/pepper/pkg/device42"
	"github.com/nextgearcapital/pepper/pkg/log"
	"github.com/nextgearcapital/pepper/pkg/salt"
	"github.com/nextgearcapital/pepper/template/vsphere"

	"github.com/spf13/cobra"
)

var (
	profile    string
	role       string
	datacenter string
	osTemplate string
	ipam       bool
)

func init() {
	RootCmd.AddCommand(deployCmd)

	deployCmd.Flags().StringVarP(&profile, "profile", "p", "", "Profile to generate and output to /etc/salt/cloud.profiles.d for salt-cloud to use")
	deployCmd.Flags().StringVarP(&role, "role", "r", "", "Role to assign to the host via grain [eg: kubernetes-master]")
	deployCmd.Flags().StringVarP(&datacenter, "datacenter", "d", "", "Datacenter to assign to the host via grain [eg: us-east]")
	deployCmd.Flags().StringVarP(&osTemplate, "template", "t", "", "Which OS template you want to use [eg: Ubuntu, CentOS, ubuntu_16.04]")
	deployCmd.Flags().BoolVarP(&ipam, "no-ipam", "", false, "Whether or not to use Device42 IPAM [This is only used internally]")
	deployCmd.Flags().BoolVarP(&log.IsDebugging, "debug", "", false, "Turn debugging on")
}

var deployCmd = &cobra.Command{
	Use:   "deploy",
	Short: "Deploy VM's via salt-cloud",
	Long: `pepper is a wrapper around salt-cloud that will generate salt-cloud profiles based on information you provide in profile configs.
Profile configs live in "/etc/pepper/config.d/{platform}/{environment}. Pepper is opinionated and looks at the profile you pass in as it's source
of truth. For example: If you pass in "vmware-dev-large" as the profile, it will look for your profile config in "/etc/pepper/config.d/vmware/large.yaml".
This allows for maximum flexibility due to the fact that everyone has different environments and may have some sort of naming scheme associated with them
so Pepper makes no assumptions on that. Pepper does however make assumptions on your instance type. [eg: nano, micro, small, medium, etc] Although these
options are available to you, you are free to override them as you see fit.
For example:

Provision new host web01 (Ubuntu) in the dev environment from the nano profile using vmware as a provider:

$ pepper deploy -p vmware-dev-nano -t Ubuntu web01

Or alternatively:

$ pepper deploy --profile vmware-dev-nano --template Ubuntu web01

Provision new host web02 (CentOS) in the prd environment from the large profile using vmware as a provider:

$ pepper deploy -p vmware-prd-large -t CentOS web02

Provision new host web03 (Ubuntu) in the uat environment from the mega profile using vmware as a provider:

$ pepper deploy -p vmware-uat-mega -t Ubuntu web03

Are we understanding how this works?

$ pepper deploy -p vmware-prd-mid -t Ubuntu -r kubernetes-master kubernetes01 kubernetes02 kubernetes03

We can also define a role and datacenter via grains

$ pepper deploy -p vmware-prd-mid -t Ubuntu -r kubernetes-master -d us-east kubernetes01 kubernetes02 kubernetes03`,
	Run: func(cmd *cobra.Command, args []string) {
		if profile == "" {
			log.Die("You didn't specify a profile.")
		} else if osTemplate == "" {
			log.Die("You didn't specify an OS template.")
		} else if len(args) == 0 {
			log.Die("You didn't specify any hosts.")
		}

		splitProfile := strings.Split(profile, "-")

		// These will be the basis for how the profile gets generated.
		platform := splitProfile[0]
		environment := splitProfile[1]
		instancetype := splitProfile[2]

		// Nothing really gained here it just makes the code more readable.
		hosts := args

		for _, host := range hosts {
			if ipam != true {
				if err := device42.ReadConfig(environment); err != nil {
					log.Die("%s", err)
				}
				// Get a new IP
				newIP, err := device42.GetNextIP(device42.IPRange)
				if err != nil {
					log.Die("%s", err)
				}
				vsphere.IPAddress = newIP
				// Create the Device
				if err := device42.CreateDevice(host); err != nil {
					if err = device42.DeleteDevice(host); err != nil {
						log.Die("%s", err)
					}
					log.Die("%s", err)
				}
				// Reserve IP
				if err := device42.ReserveIP(vsphere.IPAddress, host); err != nil {
					if err = device42.CleanDeviceAndIP(vsphere.IPAddress, host); err != nil {
						log.Die("%s", err)
					}
					log.Die("%s", err)
				}
			}
			switch platform {
			case "vmware":
				var config vsphere.ProfileConfig

				vsphere.Platform = platform
				vsphere.Environment = environment
				vsphere.InstanceType = instancetype
				vsphere.Template = osTemplate
				vsphere.Role = role
				vsphere.Datacenter = datacenter

				if err := config.Prepare(); err != nil {
					if err = device42.CleanDeviceAndIP(config.IPAddress, host); err != nil {
						log.Die("%s", err)
					}
					log.Die("%s", err)
				}
				if err := config.Generate(); err != nil {
					if err = device42.CleanDeviceAndIP(config.IPAddress, host); err != nil {
						log.Die("%s", err)
					}
					log.Die("%s", err)
				}
				if err := salt.Provision(profile, host); err != nil {
					if err = device42.CleanDeviceAndIP(config.IPAddress, host); err != nil {
						log.Die("%s", err)
					}
					log.Die("%s", err)
				}
			default:
				log.Die("I don't recognize this platform!")
			}
			log.CleanExit("Success!")
		}
	},
}
<commit_msg>Be consistent<commit_after>package cmd

import (
	"strings"

	"github.com/nextgearcapital/pepper/pkg/device42"
	"github.com/nextgearcapital/pepper/pkg/log"
	"github.com/nextgearcapital/pepper/pkg/salt"
	"github.com/nextgearcapital/pepper/template/vsphere"

	"github.com/spf13/cobra"
)

var (
	profile    string
	role       string
	datacenter string
	osTemplate string
	ipam       bool
)

func init() {
	RootCmd.AddCommand(deployCmd)

	deployCmd.Flags().StringVarP(&profile, "profile", "p", "", "Profile to generate and output to /etc/salt/cloud.profiles.d for salt-cloud to use")
	deployCmd.Flags().StringVarP(&role, "role", "r", "", "Role to assign to the host via grain [eg: kubernetes-master]")
	deployCmd.Flags().StringVarP(&datacenter, "datacenter", "d", "", "Datacenter to assign to the host via grain [eg: us-east]")
	deployCmd.Flags().StringVarP(&osTemplate, "template", "t", "", "Which OS template you want to use [eg: Ubuntu, CentOS, ubuntu_16.04]")
	deployCmd.Flags().BoolVarP(&ipam, "no-ipam", "", false, "Whether or not to use Device42 IPAM [This is only used internally]")
	deployCmd.Flags().BoolVarP(&log.IsDebugging, "debug", "", false, "Turn debugging on")
}

var deployCmd = &cobra.Command{
	Use:   "deploy",
	Short: "Deploy VM's via salt-cloud",
	Long: `pepper is a wrapper around salt-cloud that will generate salt-cloud profiles based on information you provide in profile configs.
Profile configs live in "/etc/pepper/config.d/{platform}/{environment}. Pepper is opinionated and looks at the profile you pass in as it's source
of truth. For example: If you pass in "vmware-dev-large" as the profile, it will look for your profile config in "/etc/pepper/config.d/vmware/large.yaml".
This allows for maximum flexibility due to the fact that everyone has different environments and may have some sort of naming scheme associated with them
so Pepper makes no assumptions on that. Pepper does however make assumptions on your instance type. [eg: nano, micro, small, medium, etc] Although these
options are available to you, you are free to override them as you see fit.
For example:

Provision new host web01 (Ubuntu) in the dev environment from the nano profile using vmware as a provider:

$ pepper deploy -p vmware-dev-nano -t Ubuntu web01

Or alternatively:

$ pepper deploy --profile vmware-dev-nano --template Ubuntu web01

Provision new host web02 (CentOS) in the prd environment from the large profile using vmware as a provider:

$ pepper deploy -p vmware-prd-large -t CentOS web02

Provision new host web03 (Ubuntu) in the uat environment from the mega profile using vmware as a provider:

$ pepper deploy -p vmware-uat-mega -t Ubuntu web03

Are we understanding how this works?

$ pepper deploy -p vmware-prd-mid -t Ubuntu -r kubernetes-master kubernetes01 kubernetes02 kubernetes03

We can also define a role and datacenter via grains

$ pepper deploy -p vmware-prd-mid -t Ubuntu -r kubernetes-master -d us-east kubernetes01 kubernetes02 kubernetes03`,
	Run: func(cmd *cobra.Command, args []string) {
		if profile == "" {
			log.Die("You didn't specify a profile.")
		} else if osTemplate == "" {
			log.Die("You didn't specify an OS template.")
		} else if len(args) == 0 {
			log.Die("You didn't specify any hosts.")
		}

		splitProfile := strings.Split(profile, "-")

		// These will be the basis for how the profile gets generated.
		platform := splitProfile[0]
		environment := splitProfile[1]
		instancetype := splitProfile[2]

		// Nothing really gained here it just makes the code more readable.
		hosts := args

		for _, host := range hosts {
			if ipam != true {
				if err := device42.ReadConfig(environment); err != nil {
					log.Die("%s", err)
				}
				// Get a new IP
				newIP, err := device42.GetNextIP(device42.IPRange)
				if err != nil {
					log.Die("%s", err)
				}
				vsphere.IPAddress = newIP
				// Create the Device
				if err := device42.CreateDevice(host); err != nil {
					if err = device42.DeleteDevice(host); err != nil {
						log.Die("%s", err)
					}
					log.Die("%s", err)
				}
				// Reserve IP
				if err := device42.ReserveIP(vsphere.IPAddress, host); err != nil {
					if err = device42.CleanDeviceAndIP(vsphere.IPAddress, host); err != nil {
						log.Die("%s", err)
					}
					log.Die("%s", err)
				}
			}
			switch platform {
			case "vmware":
				var config vsphere.ProfileConfig

				vsphere.Platform = platform
				vsphere.Environment = environment
				vsphere.InstanceType = instancetype
				vsphere.Template = osTemplate
				vsphere.Role = role
				vsphere.Datacenter = datacenter

				if err := config.Prepare(); err != nil {
					if err = device42.CleanDeviceAndIP(vsphere.IPAddress, host); err != nil {
						log.Die("%s", err)
					}
					log.Die("%s", err)
				}
				if err := config.Generate(); err != nil {
					if err = device42.CleanDeviceAndIP(vsphere.IPAddress, host); err != nil {
						log.Die("%s", err)
					}
					log.Die("%s", err)
				}
				if err := salt.Provision(profile, host); err != nil {
					if err = device42.CleanDeviceAndIP(vsphere.IPAddress, host); err != nil {
						log.Die("%s", err)
					}
					log.Die("%s", err)
				}
			default:
				log.Die("I don't recognize this platform!")
			}
			log.CleanExit("Success!")
		}
	},
}
<|endoftext|>


<commit_before>package cmd

import (
	"fmt"

	"github.com/spf13/cobra"
)

var searchCmd = &cobra.Command{
	Use:   "search",
	Short: "Call Api of Qiita",
	Long:  "Call Api of Qiita",
	Run: func(cmd *cobra.Command, args []string) {
		fmt.Println("search called")
	},
}

func init() {
	RootCmd.AddCommand(searchCmd)
}
<commit_msg>Add sample api call<commit_after>package cmd

import (
	"fmt"
	"io/ioutil"
	"net/http"

	"github.com/spf13/cobra"
)

var searchCmd = &cobra.Command{
	Use:   "search",
	Short: "Call Api of Qiita",
	Long:  "Call Api of Qiita",
	Run: func(cmd *cobra.Command, args []string) {
		resp, err := http.Get("http://qiita.com/api/v2/items?page=1&per_page=2&query=ruby")
		if err == nil {
			puts(resp)
		} else {
			fmt.Println(err)
		}
	},
}

func init() {
	RootCmd.AddCommand(searchCmd)
}

func puts(resp *http.Response) {
	defer resp.Body.Close()
	b, err := ioutil.ReadAll(resp.Body)
	if err == nil {
		fmt.Println(string(b))
	} else {
		fmt.Println(err)
	}
}
<|endoftext|>


<commit_before>// Copyright © 2017 Toomore Chiang
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package cmd

import (
	"encoding/hex"
	"fmt"
	"log"
	"net/http"
	"net/url"
	"regexp"

	"github.com/spf13/cobra"
	"github.com/toomore/mailbox/campaign"
	"github.com/toomore/mailbox/mails"
	"github.com/toomore/mailbox/reader"
	"github.com/toomore/mailbox/utils"
)

var (
	servercExpr      = regexp.MustCompile(`/(read|door|washi|vote)/([0-9a-zA-Z]+)`)
	serverhttpPort   *string
	serverlinksCache = make(map[string]string)
)

func serverLog(note string, r *http.Request) {
	log.Printf("%s [%s] \"%+v\" %s\n", note, r.Header.Get("X-Real-Ip"), r, r.Header.Get("User-Agent"))
}

func read(w http.ResponseWriter, r *http.Request) {
	v, _ := url.ParseQuery(r.Header.Get("X-Args"))

	var hm string
	match := servercExpr.FindStringSubmatch(r.Header.Get("X-Uri"))
	if len(match) >= 3 {
		hm = match[2]
	}
	w.WriteHeader(http.StatusNotFound)
	hmbyte, _ := hex.DecodeString(hm)
	if campaign.CheckMac(hmbyte, v.Get("c"), v) {
		serverLog("[read] Pass", r)
		reader.Save(v.Get("c"), v.Get("u"), r.Header.Get("X-Real-Ip"), r.Header.Get("User-Agent"))
	} else {
		serverLog("[read] Hash Fail!!!", r)
	}
}

func washi(v url.Values, url string) []byte {
	washigroup := regexp.MustCompile(`{{WASHI}}(.+){{/WASHI}}`).FindStringSubmatch(url)
	if len(washigroup) > 1 {
		userrows, err := utils.GetConn().Query(`SELECT f_name, l_name FROM user WHERE id=?`, v.Get("u"))
		if err == nil {
			washiURL := []byte(washigroup[1])
			for userrows.Next() {
				var (
					fname string
					lname string
				)
				userrows.Scan(&fname, &lname)
				mails.ReplaceFname(&washiURL, fname)
				mails.ReplaceLname(&washiURL, lname)
				return washiURL
			}
		}
	}
	return nil
}

func door(w http.ResponseWriter, r *http.Request) {
	v, _ := url.ParseQuery(r.Header.Get("X-Args"))

	match := servercExpr.FindStringSubmatch(r.Header.Get("X-Uri"))
	if match[1] == "door" {
		if v.Get("t") != "a" {
			log.Println("No `t`", v.Get("t"))
			return
		}
	}

	var hm string
	if len(match) >= 3 {
		hm = match[2]
	}

	hmbyte, _ := hex.DecodeString(hm)
	if campaign.CheckMac(hmbyte, v.Get("c"), v) {
		utils.GetConn().Query(`INSERT INTO doors(cid,uid,linkid,ip,agent) VALUES(?,?,?,?,?)`,
			v.Get("c"), v.Get("u"), v.Get("l"), r.Header.Get("X-Real-Ip"), r.Header.Get("User-Agent"))
		serverLog("[door] Pass", r)

		var (
			serverlinksCacheKey = fmt.Sprintf("%s|%s", v.Get("c"), v.Get("l"))
			ok                  bool
			url                 string
		)

		if url, ok = serverlinksCache[serverlinksCacheKey]; ok {
			log.Println("Using", match[1], "cache", serverlinksCacheKey, url)
		} else {
			rows, err := utils.GetConn().Query(`SELECT url FROM links WHERE cid=? AND id=?`, v.Get("c"), v.Get("l"))
			if err == nil {
				for rows.Next() {
					rows.Scan(&url)
					serverlinksCache[serverlinksCacheKey] = url
					log.Println("Find", match[1], serverlinksCacheKey, url)
				}
			}
		}

		if url != "" {
			switch match[1] {
			case "door":
				http.Redirect(w, r, url, http.StatusSeeOther)
				return
			case "washi":
				http.Redirect(w, r, string(washi(v, url)), http.StatusSeeOther)
				return
			}
		}
	} else {
		serverLog("[door] Hash Fail!!!", r)
	}
	w.WriteHeader(http.StatusNotFound)
}

func vote(w http.ResponseWriter, r *http.Request) {
	var hm string
	match := servercExpr.FindStringSubmatch(r.Header.Get("X-Uri"))
	if len(match) >= 3 {
		hm = match[2]
	}
	utils.GetConn().Query(`INSERT INTO vote(id,ip,agent) VALUES(?,?,?)`,
		hm, r.Header.Get("X-Real-Ip"), r.Header.Get("User-Agent"))

	w.WriteHeader(http.StatusCreated)
	w.Write([]byte("Thanks"))
}

// serverCmd represents the server command
var serverCmd = &cobra.Command{
	Use:   "server",
	Short: "Run tiny server for open, click trace",
	Long:  `啟動一個 web server，來接收開信、點擊連結紀錄。`,
	Run: func(cmd *cobra.Command, args []string) {
		fmt.Println("Run server ...")
		http.HandleFunc("/read/", read)
		http.HandleFunc("/door/", door)
		http.HandleFunc("/washi/", door)
		http.HandleFunc("/vote/", vote)
		log.Println("HTTP Port:", *serverhttpPort)
		log.Println(http.ListenAndServe(*serverhttpPort, nil))
	},
}

func init() {
	serverhttpPort = serverCmd.Flags().StringP("port", "p", ":8801", "HTTP Port")
	RootCmd.AddCommand(serverCmd)

	// Here you will define your flags and configuration settings.

	// Cobra supports Persistent Flags which will work for this command
	// and all subcommands, e.g.:
	// serverCmd.PersistentFlags().String("foo", "", "A help for foo")

	// Cobra supports local flags which will only run when this command
	// is called directly, e.g.:
	// serverCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle")
}
<commit_msg>Add serverLog at cmd server vote<commit_after>// Copyright © 2017 Toomore Chiang
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package cmd

import (
	"encoding/hex"
	"fmt"
	"log"
	"net/http"
	"net/url"
	"regexp"

	"github.com/spf13/cobra"
	"github.com/toomore/mailbox/campaign"
	"github.com/toomore/mailbox/mails"
	"github.com/toomore/mailbox/reader"
	"github.com/toomore/mailbox/utils"
)

var (
	servercExpr      = regexp.MustCompile(`/(read|door|washi|vote)/([0-9a-zA-Z]+)`)
	serverhttpPort   *string
	serverlinksCache = make(map[string]string)
)

func serverLog(note string, r *http.Request) {
	log.Printf("%s [%s] \"%+v\" %s\n", note, r.Header.Get("X-Real-Ip"), r, r.Header.Get("User-Agent"))
}

func read(w http.ResponseWriter, r *http.Request) {
	v, _ := url.ParseQuery(r.Header.Get("X-Args"))

	var hm string
	match := servercExpr.FindStringSubmatch(r.Header.Get("X-Uri"))
	if len(match) >= 3 {
		hm = match[2]
	}
	w.WriteHeader(http.StatusNotFound)
	hmbyte, _ := hex.DecodeString(hm)
	if campaign.CheckMac(hmbyte, v.Get("c"), v) {
		serverLog("[read] Pass", r)
		reader.Save(v.Get("c"), v.Get("u"), r.Header.Get("X-Real-Ip"), r.Header.Get("User-Agent"))
	} else {
		serverLog("[read] Hash Fail!!!", r)
	}
}

func washi(v url.Values, url string) []byte {
	washigroup := regexp.MustCompile(`{{WASHI}}(.+){{/WASHI}}`).FindStringSubmatch(url)
	if len(washigroup) > 1 {
		userrows, err := utils.GetConn().Query(`SELECT f_name, l_name FROM user WHERE id=?`, v.Get("u"))
		if err == nil {
			washiURL := []byte(washigroup[1])
			for userrows.Next() {
				var (
					fname string
					lname string
				)
				userrows.Scan(&fname, &lname)
				mails.ReplaceFname(&washiURL, fname)
				mails.ReplaceLname(&washiURL, lname)
				return washiURL
			}
		}
	}
	return nil
}

func door(w http.ResponseWriter, r *http.Request) {
	v, _ := url.ParseQuery(r.Header.Get("X-Args"))

	match := servercExpr.FindStringSubmatch(r.Header.Get("X-Uri"))
	if match[1] == "door" {
		if v.Get("t") != "a" {
			log.Println("No `t`", v.Get("t"))
			return
		}
	}

	var hm string
	if len(match) >= 3 {
		hm = match[2]
	}

	hmbyte, _ := hex.DecodeString(hm)
	if campaign.CheckMac(hmbyte, v.Get("c"), v) {
		utils.GetConn().Query(`INSERT INTO doors(cid,uid,linkid,ip,agent) VALUES(?,?,?,?,?)`,
			v.Get("c"), v.Get("u"), v.Get("l"), r.Header.Get("X-Real-Ip"), r.Header.Get("User-Agent"))
		serverLog("[door] Pass", r)

		var (
			serverlinksCacheKey = fmt.Sprintf("%s|%s", v.Get("c"), v.Get("l"))
			ok                  bool
			url                 string
		)

		if url, ok = serverlinksCache[serverlinksCacheKey]; ok {
			log.Println("Using", match[1], "cache", serverlinksCacheKey, url)
		} else {
			rows, err := utils.GetConn().Query(`SELECT url FROM links WHERE cid=? AND id=?`, v.Get("c"), v.Get("l"))
			if err == nil {
				for rows.Next() {
					rows.Scan(&url)
					serverlinksCache[serverlinksCacheKey] = url
					log.Println("Find", match[1], serverlinksCacheKey, url)
				}
			}
		}

		if url != "" {
			switch match[1] {
			case "door":
				http.Redirect(w, r, url, http.StatusSeeOther)
				return
			case "washi":
				http.Redirect(w, r, string(washi(v, url)), http.StatusSeeOther)
				return
			}
		}
	} else {
		serverLog("[door] Hash Fail!!!", r)
	}
	w.WriteHeader(http.StatusNotFound)
}

func vote(w http.ResponseWriter, r *http.Request) {
	serverLog("[vote]", r)
	var hm string
	match := servercExpr.FindStringSubmatch(r.Header.Get("X-Uri"))
	if len(match) >= 3 {
		hm = match[2]
	}
	utils.GetConn().Query(`INSERT INTO vote(id,ip,agent) VALUES(?,?,?)`,
		hm, r.Header.Get("X-Real-Ip"), r.Header.Get("User-Agent"))

	w.WriteHeader(http.StatusCreated)
	w.Write([]byte("Thanks"))
}

// serverCmd represents the server command
var serverCmd = &cobra.Command{
	Use:   "server",
	Short: "Run tiny server for open, click trace",
	Long:  `啟動一個 web server，來接收開信、點擊連結紀錄。`,
	Run: func(cmd *cobra.Command, args []string) {
		fmt.Println("Run server ...")
		http.HandleFunc("/read/", read)
		http.HandleFunc("/door/", door)
		http.HandleFunc("/washi/", door)
		http.HandleFunc("/vote/", vote)
		log.Println("HTTP Port:", *serverhttpPort)
		log.Println(http.ListenAndServe(*serverhttpPort, nil))
	},
}

func init() {
	serverhttpPort = serverCmd.Flags().StringP("port", "p", ":8801", "HTTP Port")
	RootCmd.AddCommand(serverCmd)

	// Here you will define your flags and configuration settings.

	// Cobra supports Persistent Flags which will work for this command
	// and all subcommands, e.g.:
	// serverCmd.PersistentFlags().String("foo", "", "A help for foo")

	// Cobra supports local flags which will only run when this command
	// is called directly, e.g.:
	// serverCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle")
}
<|endoftext|>