<commit_before>// Package pubsub implements a pub-sub model with a single publisher (Server)
// and multiple subscribers (clients).
//
// Though you can have multiple publishers by sharing a pointer to a server or
// by giving the same channel to each publisher and publishing messages from
// that channel (fan-in).
//
// Clients subscribe for messages, which could be of any type, using a query.
// When some message is published, we match it with all queries. If there is a
// match, this message will be pushed to all clients, subscribed to that query.
// See query subpackage for our implementation.
//
// Due to the blocking send implementation, a single subscriber can freeze an
// entire server by not reading messages before it unsubscribes. To avoid such
// scenario, subscribers must either:
//
// a) make sure they continue to read from the out channel until
// Unsubscribe(All) is called
//
//     s.Subscribe(ctx, sub, qry, out)
//     go func() {
//         for msg := range out {
//             // handle msg
//             // will exit automatically when out is closed by Unsubscribe(All)
//         }
//     }()
//     s.UnsubscribeAll(ctx, sub)
//
// b) drain the out channel before calling Unsubscribe(All)
//
//     s.Subscribe(ctx, sub, qry, out)
//     defer func() {
//         // drain out to make sure we don't block
//     LOOP:
//		     for {
// 		     	   select {
// 		     	   case <-out:
// 		     	   default:
// 		     	   	   break LOOP
// 		     	   }
// 		     }
//         s.UnsubscribeAll(ctx, sub)
//     }()
//     for msg := range out {
//         // handle msg
//         if err != nil {
//            return err
//         }
//     }
//
package pubsub

import (
	"context"
	"errors"
	"sync"

	cmn "github.com/tendermint/tendermint/libs/common"
)

type operation int

const (
	sub operation = iota
	pub
	unsub
	shutdown
)

var (
	// ErrSubscriptionNotFound is returned when a client tries to unsubscribe
	// from not existing subscription.
	ErrSubscriptionNotFound = errors.New("subscription not found")

	// ErrAlreadySubscribed is returned when a client tries to subscribe twice or
	// more using the same query.
	ErrAlreadySubscribed = errors.New("already subscribed")
)

type cmd struct {
	op       operation
	query    Query
	ch       chan<- interface{}
	clientID string
	msg      interface{}
	tags     TagMap
}

// Query defines an interface for a query to be used for subscribing.
type Query interface {
	Matches(tags TagMap) bool
	String() string
}

// Server allows clients to subscribe/unsubscribe for messages, publishing
// messages with or without tags, and manages internal state.
type Server struct {
	cmn.BaseService

	cmds    chan cmd
	cmdsCap int

	mtx           sync.RWMutex
	subscriptions map[string]map[string]Query // subscriber -> query (string) -> Query
}

// Option sets a parameter for the server.
type Option func(*Server)

// TagMap is used to associate tags to a message.
// They can be queried by subscribers to choose messages they will received.
type TagMap interface {
	// Get returns the value for a key, or nil if no value is present.
	// The ok result indicates whether value was found in the tags.
	Get(key string) (value string, ok bool)
	// Len returns the number of tags.
	Len() int
}

type tagMap map[string]string

var _ TagMap = (*tagMap)(nil)

// NewTagMap constructs a new immutable tag set from a map.
func NewTagMap(data map[string]string) TagMap {
	return tagMap(data)
}

// Get returns the value for a key, or nil if no value is present.
// The ok result indicates whether value was found in the tags.
func (ts tagMap) Get(key string) (value string, ok bool) {
	value, ok = ts[key]
	return
}

// Len returns the number of tags.
func (ts tagMap) Len() int {
	return len(ts)
}

// NewServer returns a new server. See the commentary on the Option functions
// for a detailed description of how to configure buffering. If no options are
// provided, the resulting server's queue is unbuffered.
func NewServer(options ...Option) *Server {
	s := &Server{
		subscriptions: make(map[string]map[string]Query),
	}
	s.BaseService = *cmn.NewBaseService(nil, "PubSub", s)

	for _, option := range options {
		option(s)
	}

	// if BufferCapacity option was not set, the channel is unbuffered
	s.cmds = make(chan cmd, s.cmdsCap)

	return s
}

// BufferCapacity allows you to specify capacity for the internal server's
// queue. Since the server, given Y subscribers, could only process X messages,
// this option could be used to survive spikes (e.g. high amount of
// transactions during peak hours).
func BufferCapacity(cap int) Option {
	return func(s *Server) {
		if cap > 0 {
			s.cmdsCap = cap
		}
	}
}

// BufferCapacity returns capacity of the internal server's queue.
func (s *Server) BufferCapacity() int {
	return s.cmdsCap
}

// Subscribe creates a subscription for the given client. It accepts a channel
// on which messages matching the given query can be received. An error will be
// returned to the caller if the context is canceled or if subscription already
// exist for pair clientID and query.
func (s *Server) Subscribe(ctx context.Context, clientID string, query Query, out chan<- interface{}) error {
	s.mtx.RLock()
	clientSubscriptions, ok := s.subscriptions[clientID]
	if ok {
		_, ok = clientSubscriptions[query.String()]
	}
	s.mtx.RUnlock()
	if ok {
		return ErrAlreadySubscribed
	}

	select {
	case s.cmds <- cmd{op: sub, clientID: clientID, query: query, ch: out}:
		s.mtx.Lock()
		if _, ok = s.subscriptions[clientID]; !ok {
			s.subscriptions[clientID] = make(map[string]Query)
		}
		// preserve original query
		// see Unsubscribe
		s.subscriptions[clientID][query.String()] = query
		s.mtx.Unlock()
		return nil
	case <-ctx.Done():
		return ctx.Err()
	case <-s.Quit():
		return nil
	}
}

// Unsubscribe removes the subscription on the given query. An error will be
// returned to the caller if the context is canceled or if subscription does
// not exist.
func (s *Server) Unsubscribe(ctx context.Context, clientID string, query Query) error {
	var origQuery Query
	s.mtx.RLock()
	clientSubscriptions, ok := s.subscriptions[clientID]
	if ok {
		origQuery, ok = clientSubscriptions[query.String()]
	}
	s.mtx.RUnlock()
	if !ok {
		return ErrSubscriptionNotFound
	}

	// original query is used here because we're using pointers as map keys
	select {
	case s.cmds <- cmd{op: unsub, clientID: clientID, query: origQuery}:
		s.mtx.Lock()
		delete(clientSubscriptions, query.String())
		s.mtx.Unlock()
		return nil
	case <-ctx.Done():
		return ctx.Err()
	case <-s.Quit():
		return nil
	}
}

// UnsubscribeAll removes all client subscriptions. An error will be returned
// to the caller if the context is canceled or if subscription does not exist.
func (s *Server) UnsubscribeAll(ctx context.Context, clientID string) error {
	s.mtx.RLock()
	_, ok := s.subscriptions[clientID]
	s.mtx.RUnlock()
	if !ok {
		return ErrSubscriptionNotFound
	}

	select {
	case s.cmds <- cmd{op: unsub, clientID: clientID}:
		s.mtx.Lock()
		delete(s.subscriptions, clientID)
		s.mtx.Unlock()
		return nil
	case <-ctx.Done():
		return ctx.Err()
	case <-s.Quit():
		return nil
	}
}

// Publish publishes the given message. An error will be returned to the caller
// if the context is canceled.
func (s *Server) Publish(ctx context.Context, msg interface{}) error {
	return s.PublishWithTags(ctx, msg, NewTagMap(make(map[string]string)))
}

// PublishWithTags publishes the given message with the set of tags. The set is
// matched with clients queries. If there is a match, the message is sent to
// the client.
func (s *Server) PublishWithTags(ctx context.Context, msg interface{}, tags TagMap) error {
	select {
	case s.cmds <- cmd{op: pub, msg: msg, tags: tags}:
		return nil
	case <-ctx.Done():
		return ctx.Err()
	case <-s.Quit():
		return nil
	}
}

// OnStop implements Service.OnStop by shutting down the server.
func (s *Server) OnStop() {
	s.cmds <- cmd{op: shutdown}
}

// NOTE: not goroutine safe
type state struct {
	// query -> client -> ch
	queries map[Query]map[string]chan<- interface{}
	// client -> query -> struct{}
	clients map[string]map[Query]struct{}
}

// OnStart implements Service.OnStart by starting the server.
func (s *Server) OnStart() error {
	go s.loop(state{
		queries: make(map[Query]map[string]chan<- interface{}),
		clients: make(map[string]map[Query]struct{}),
	})
	return nil
}

// OnReset implements Service.OnReset
func (s *Server) OnReset() error {
	return nil
}

func (s *Server) loop(state state) {
loop:
	for cmd := range s.cmds {
		switch cmd.op {
		case unsub:
			if cmd.query != nil {
				state.remove(cmd.clientID, cmd.query)
			} else {
				state.removeAll(cmd.clientID)
			}
		case shutdown:
			for clientID := range state.clients {
				state.removeAll(clientID)
			}
			break loop
		case sub:
			state.add(cmd.clientID, cmd.query, cmd.ch)
		case pub:
			state.send(cmd.msg, cmd.tags)
		}
	}
}

func (state *state) add(clientID string, q Query, ch chan<- interface{}) {

	// initialize clientToChannelMap per query if needed
	if _, ok := state.queries[q]; !ok {
		state.queries[q] = make(map[string]chan<- interface{})
	}

	// create subscription
	state.queries[q][clientID] = ch

	// add client if needed
	if _, ok := state.clients[clientID]; !ok {
		state.clients[clientID] = make(map[Query]struct{})
	}
	state.clients[clientID][q] = struct{}{}
}

func (state *state) remove(clientID string, q Query) {
	clientToChannelMap, ok := state.queries[q]
	if !ok {
		return
	}

	ch, ok := clientToChannelMap[clientID]
	if ok {
		close(ch)

		delete(state.clients[clientID], q)

		// if it not subscribed to anything else, remove the client
		if len(state.clients[clientID]) == 0 {
			delete(state.clients, clientID)
		}

		delete(state.queries[q], clientID)
		if len(state.queries[q]) == 0 {
			delete(state.queries, q)
		}
	}
}

func (state *state) removeAll(clientID string) {
	queryMap, ok := state.clients[clientID]
	if !ok {
		return
	}

	for q := range queryMap {
		ch := state.queries[q][clientID]
		close(ch)

		delete(state.queries[q], clientID)
		if len(state.queries[q]) == 0 {
			delete(state.queries, q)
		}
	}
	delete(state.clients, clientID)
}

func (state *state) send(msg interface{}, tags TagMap) {
	for q, clientToChannelMap := range state.queries {
		if q.Matches(tags) {
			for _, ch := range clientToChannelMap {
				ch <- msg
			}
		}
	}
}
<commit_msg>pubsub: comments<commit_after>// Package pubsub implements a pub-sub model with a single publisher (Server)
// and multiple subscribers (clients).
//
// Though you can have multiple publishers by sharing a pointer to a server or
// by giving the same channel to each publisher and publishing messages from
// that channel (fan-in).
//
// Clients subscribe for messages, which could be of any type, using a query.
// When some message is published, we match it with all queries. If there is a
// match, this message will be pushed to all clients, subscribed to that query.
// See query subpackage for our implementation.
//
// Due to the blocking send implementation, a single subscriber can freeze an
// entire server by not reading messages before it unsubscribes. To avoid such
// scenario, subscribers must either:
//
// a) make sure they continue to read from the out channel until
// Unsubscribe(All) is called
//
//     s.Subscribe(ctx, sub, qry, out)
//     go func() {
//         for msg := range out {
//             // handle msg
//             // will exit automatically when out is closed by Unsubscribe(All)
//         }
//     }()
//     s.UnsubscribeAll(ctx, sub)
//
// b) drain the out channel before calling Unsubscribe(All)
//
//     s.Subscribe(ctx, sub, qry, out)
//     defer func() {
//         // drain out to make sure we don't block
//     LOOP:
//		     for {
// 		     	   select {
// 		     	   case <-out:
// 		     	   default:
// 		     	   	   break LOOP
// 		     	   }
// 		     }
//         s.UnsubscribeAll(ctx, sub)
//     }()
//     for msg := range out {
//         // handle msg
//         if err != nil {
//            return err
//         }
//     }
//
package pubsub

import (
	"context"
	"errors"
	"sync"

	cmn "github.com/tendermint/tendermint/libs/common"
)

type operation int

const (
	sub operation = iota
	pub
	unsub
	shutdown
)

var (
	// ErrSubscriptionNotFound is returned when a client tries to unsubscribe
	// from not existing subscription.
	ErrSubscriptionNotFound = errors.New("subscription not found")

	// ErrAlreadySubscribed is returned when a client tries to subscribe twice or
	// more using the same query.
	ErrAlreadySubscribed = errors.New("already subscribed")
)

type cmd struct {
	op       operation
	query    Query
	ch       chan<- interface{}
	clientID string
	msg      interface{}
	tags     TagMap
}

// Query defines an interface for a query to be used for subscribing.
type Query interface {
	Matches(tags TagMap) bool
	String() string
}

// Server allows clients to subscribe/unsubscribe for messages, publishing
// messages with or without tags, and manages internal state.
type Server struct {
	cmn.BaseService

	cmds    chan cmd
	cmdsCap int

	mtx           sync.RWMutex
	subscriptions map[string]map[string]Query // subscriber -> query (string) -> Query
}

// Option sets a parameter for the server.
type Option func(*Server)

// TagMap is used to associate tags to a message.
// They can be queried by subscribers to choose messages they will received.
type TagMap interface {
	// Get returns the value for a key, or nil if no value is present.
	// The ok result indicates whether value was found in the tags.
	Get(key string) (value string, ok bool)
	// Len returns the number of tags.
	Len() int
}

type tagMap map[string]string

var _ TagMap = (*tagMap)(nil)

// NewTagMap constructs a new immutable tag set from a map.
func NewTagMap(data map[string]string) TagMap {
	return tagMap(data)
}

// Get returns the value for a key, or nil if no value is present.
// The ok result indicates whether value was found in the tags.
func (ts tagMap) Get(key string) (value string, ok bool) {
	value, ok = ts[key]
	return
}

// Len returns the number of tags.
func (ts tagMap) Len() int {
	return len(ts)
}

// NewServer returns a new server. See the commentary on the Option functions
// for a detailed description of how to configure buffering. If no options are
// provided, the resulting server's queue is unbuffered.
func NewServer(options ...Option) *Server {
	s := &Server{
		subscriptions: make(map[string]map[string]Query),
	}
	s.BaseService = *cmn.NewBaseService(nil, "PubSub", s)

	for _, option := range options {
		option(s)
	}

	// if BufferCapacity option was not set, the channel is unbuffered
	s.cmds = make(chan cmd, s.cmdsCap)

	return s
}

// BufferCapacity allows you to specify capacity for the internal server's
// queue. Since the server, given Y subscribers, could only process X messages,
// this option could be used to survive spikes (e.g. high amount of
// transactions during peak hours).
func BufferCapacity(cap int) Option {
	return func(s *Server) {
		if cap > 0 {
			s.cmdsCap = cap
		}
	}
}

// BufferCapacity returns capacity of the internal server's queue.
func (s *Server) BufferCapacity() int {
	return s.cmdsCap
}

// Subscribe creates a subscription for the given client. It accepts a channel
// on which messages matching the given query can be received. An error will be
// returned to the caller if the context is canceled or if subscription already
// exist for pair clientID and query.
func (s *Server) Subscribe(ctx context.Context, clientID string, query Query, out chan<- interface{}) error {
	s.mtx.RLock()
	clientSubscriptions, ok := s.subscriptions[clientID]
	if ok {
		_, ok = clientSubscriptions[query.String()]
	}
	s.mtx.RUnlock()
	if ok {
		return ErrAlreadySubscribed
	}

	select {
	case s.cmds <- cmd{op: sub, clientID: clientID, query: query, ch: out}:
		s.mtx.Lock()
		if _, ok = s.subscriptions[clientID]; !ok {
			s.subscriptions[clientID] = make(map[string]Query)
		}
		// preserve original query
		// see Unsubscribe
		s.subscriptions[clientID][query.String()] = query
		s.mtx.Unlock()
		return nil
	case <-ctx.Done():
		return ctx.Err()
	case <-s.Quit():
		return nil
	}
}

// Unsubscribe removes the subscription on the given query. An error will be
// returned to the caller if the context is canceled or if subscription does
// not exist.
func (s *Server) Unsubscribe(ctx context.Context, clientID string, query Query) error {
	var origQuery Query
	s.mtx.RLock()
	clientSubscriptions, ok := s.subscriptions[clientID]
	if ok {
		origQuery, ok = clientSubscriptions[query.String()]
	}
	s.mtx.RUnlock()
	if !ok {
		return ErrSubscriptionNotFound
	}

	// original query is used here because we're using pointers as map keys
	// ?
	select {
	case s.cmds <- cmd{op: unsub, clientID: clientID, query: origQuery}:
		s.mtx.Lock()
		// if its the only query left, should we also delete the client?
		delete(clientSubscriptions, query.String())
		s.mtx.Unlock()
		return nil
	case <-ctx.Done():
		return ctx.Err()
	case <-s.Quit():
		return nil
	}
}

// UnsubscribeAll removes all client subscriptions. An error will be returned
// to the caller if the context is canceled or if subscription does not exist.
func (s *Server) UnsubscribeAll(ctx context.Context, clientID string) error {
	s.mtx.RLock()
	_, ok := s.subscriptions[clientID]
	s.mtx.RUnlock()
	if !ok {
		return ErrSubscriptionNotFound
	}

	select {
	case s.cmds <- cmd{op: unsub, clientID: clientID}:
		s.mtx.Lock()
		delete(s.subscriptions, clientID)
		s.mtx.Unlock()
		return nil
	case <-ctx.Done():
		return ctx.Err()
	case <-s.Quit():
		return nil
	}
}

// Publish publishes the given message. An error will be returned to the caller
// if the context is canceled.
func (s *Server) Publish(ctx context.Context, msg interface{}) error {
	return s.PublishWithTags(ctx, msg, NewTagMap(make(map[string]string)))
}

// PublishWithTags publishes the given message with the set of tags. The set is
// matched with clients queries. If there is a match, the message is sent to
// the client.
func (s *Server) PublishWithTags(ctx context.Context, msg interface{}, tags TagMap) error {
	select {
	case s.cmds <- cmd{op: pub, msg: msg, tags: tags}:
		return nil
	case <-ctx.Done():
		return ctx.Err()
	case <-s.Quit():
		return nil
	}
}

// OnStop implements Service.OnStop by shutting down the server.
func (s *Server) OnStop() {
	s.cmds <- cmd{op: shutdown}
}

// NOTE: not goroutine safe
type state struct {
	// query -> client -> ch
	queries map[Query]map[string]chan<- interface{}
	// client -> query -> struct{}
	clients map[string]map[Query]struct{}
}

// OnStart implements Service.OnStart by starting the server.
func (s *Server) OnStart() error {
	go s.loop(state{
		queries: make(map[Query]map[string]chan<- interface{}),
		clients: make(map[string]map[Query]struct{}),
	})
	return nil
}

// OnReset implements Service.OnReset
func (s *Server) OnReset() error {
	return nil
}

func (s *Server) loop(state state) {
loop:
	for cmd := range s.cmds {
		switch cmd.op {
		case unsub:
			if cmd.query != nil {
				state.remove(cmd.clientID, cmd.query)
			} else {
				state.removeAll(cmd.clientID)
			}
		case shutdown:
			for clientID := range state.clients {
				state.removeAll(clientID)
			}
			break loop
		case sub:
			state.add(cmd.clientID, cmd.query, cmd.ch)
		case pub:
			state.send(cmd.msg, cmd.tags)
		}
	}
}

func (state *state) add(clientID string, q Query, ch chan<- interface{}) {

	// initialize clientToChannelMap per query if needed
	if _, ok := state.queries[q]; !ok {
		state.queries[q] = make(map[string]chan<- interface{})
	}

	// create subscription
	state.queries[q][clientID] = ch

	// add client if needed
	if _, ok := state.clients[clientID]; !ok {
		state.clients[clientID] = make(map[Query]struct{})
	}
	state.clients[clientID][q] = struct{}{}
}

func (state *state) remove(clientID string, q Query) {
	clientToChannelMap, ok := state.queries[q]
	if !ok {
		return
	}

	ch, ok := clientToChannelMap[clientID]
	if !ok {
		return
	}

	close(ch)

	// remove the query from client map.
	// if client is not subscribed to anything else, remove it.
	delete(state.clients[clientID], q)
	if len(state.clients[clientID]) == 0 {
		delete(state.clients, clientID)
	}

	// remove the client from query map.
	// if query has no other clients subscribed, remove it.
	delete(state.queries[q], clientID)
	if len(state.queries[q]) == 0 {
		delete(state.queries, q)
	}
}

func (state *state) removeAll(clientID string) {
	queryMap, ok := state.clients[clientID]
	if !ok {
		return
	}

	for q := range queryMap {
		ch := state.queries[q][clientID]
		close(ch)

		// remove the client from query map.
		// if query has no other clients subscribed, remove it.
		delete(state.queries[q], clientID)
		if len(state.queries[q]) == 0 {
			delete(state.queries, q)
		}
	}

	// remove the client.
	delete(state.clients, clientID)
}

func (state *state) send(msg interface{}, tags TagMap) {
	for q, clientToChannelMap := range state.queries {
		if q.Matches(tags) {
			for _, ch := range clientToChannelMap {
				ch <- msg
			}
		}
	}
}
<|endoftext|>


<commit_before><commit_msg>Added more tests for the loudness converter.<commit_after><|endoftext|>


<commit_before>package aws

import (
	"fmt"
	"log"
	"os"
	"testing"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session"
	cf "github.com/aws/aws-sdk-go/service/cloudformation"
	"github.com/satori/go.uuid"
)

var (
	keyPair string
	region string

	sess *session.Session
	svc *cf.CloudFormation
)

// REQUIREMENTS:
// AWS credentials in path
// Environment variables set for the following:
//   KEYPAIR=<your-aws-key-pair>
//   REGION=<aws-region>
func init() {
	keyPair = os.Getenv("KEYPAIR")
	if keyPair == "" {
		log.Fatal("KEYPAIR environment variable not set")
	}
	region = os.Getenv("REGION")
	if region == "" {
		log.Fatal("REGION environment variable not set")
	}
}

func TestMain(m *testing.M) {
	setup()
	code := m.Run()
	teardown()
	os.Exit(code)
}

func setup() {
	sess = session.Must(session.NewSession())

	// Create the service's client with the session.
	svc = cf.New(sess,
		aws.NewConfig().WithRegion(region).WithLogLevel(aws.LogOff))
}

func teardown() {
	// TODO destroy test stacks
}

func TestCreate(t *testing.T) {
	stackName := fmt.Sprintf("amp-aws-plugin-test-%s", uuid.NewV4())

	stackSpec := &StackSpec{
		KeyPair: keyPair,
		Region: region,
		StackName: stackName,
		OnFailure: "DELETE",
	}

	stackSpec.Params = []*cf.Parameter{
		{ParameterKey: aws.String("KeyName"), ParameterValue: aws.String(keyPair)},
	}

	_, err := CreateStack(svc, stackSpec, 20)
	if err != nil {
		t.Fatal(err)
	}
}
<commit_msg>add keypair to personally identify test stack names<commit_after>package aws

import (
	"fmt"
	"log"
	"os"
	"testing"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session"
	cf "github.com/aws/aws-sdk-go/service/cloudformation"
	"github.com/satori/go.uuid"
)

var (
	keyPair string
	region string

	sess *session.Session
	svc *cf.CloudFormation
)

// REQUIREMENTS:
// AWS credentials in path
// Environment variables set for the following:
//   KEYPAIR=<your-aws-key-pair>
//   REGION=<aws-region>
func init() {
	keyPair = os.Getenv("KEYPAIR")
	if keyPair == "" {
		log.Fatal("KEYPAIR environment variable not set")
	}
	region = os.Getenv("REGION")
	if region == "" {
		log.Fatal("REGION environment variable not set")
	}
}

func TestMain(m *testing.M) {
	setup()
	code := m.Run()
	teardown()
	os.Exit(code)
}

func setup() {
	sess = session.Must(session.NewSession())

	// Create the service's client with the session.
	svc = cf.New(sess,
		aws.NewConfig().WithRegion(region).WithLogLevel(aws.LogOff))
}

func teardown() {
	// TODO destroy test stacks
}

func TestCreate(t *testing.T) {
	stackName := fmt.Sprintf("%s-plugin-test-%s", keyPair, uuid.NewV4())

	stackSpec := &StackSpec{
		KeyPair: keyPair,
		Region: region,
		StackName: stackName,
		OnFailure: "DELETE",
	}

	stackSpec.Params = []*cf.Parameter{
		{ParameterKey: aws.String("KeyName"), ParameterValue: aws.String(keyPair)},
	}

	_, err := CreateStack(svc, stackSpec, 20)
	if err != nil {
		t.Fatal(err)
	}
}
<|endoftext|>


<commit_before>// Copyright © 2018 Banzai Cloud
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"context"
	"fmt"
	"net/http"
	"os"
	"strconv"
	"strings"

	whhttp "github.com/slok/kubewebhook/pkg/http"
	"github.com/slok/kubewebhook/pkg/log"
	whcontext "github.com/slok/kubewebhook/pkg/webhook/context"
	"github.com/slok/kubewebhook/pkg/webhook/mutating"
	"github.com/spf13/viper"
	corev1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/rest"
)

type vaultConfig struct {
	addr       string
	role       string
	path       string
	skipVerify string
	useAgent   bool
}

var vaultAgentConfig = `
pid_file = "./pidfile"
exit_after_auth = true

auto_auth {
	method "kubernetes" {
		mount_path = "%s"
		config = {
			role = "%s"
		}
	}

	sink "file" {
		config = {
			path = "/vault/token"
		}
	}
}`

func getInitContainers(vaultConfig vaultConfig) []corev1.Container {
	containers := []corev1.Container{}

	if vaultConfig.useAgent {
		containers = append(containers, corev1.Container{
			Name:            "vault-agent",
			Image:           viper.GetString("vault_image"),
			ImagePullPolicy: corev1.PullIfNotPresent,
			Command:         []string{"vault", "agent", "-config=/vault-agent/config.hcl"},
			Env: []corev1.EnvVar{
				{
					Name:  "VAULT_ADDR",
					Value: vaultConfig.addr,
				},
				{
					Name:  "VAULT_SKIP_VERIFY",
					Value: vaultConfig.skipVerify,
				},
			},
			VolumeMounts: []corev1.VolumeMount{
				{
					Name:      "vault-agent-config",
					MountPath: "/vault-agent/",
				},
				{
					Name:      "vault-env",
					MountPath: "/vault/",
				},
			},
		})
	}

	containers = append(containers, corev1.Container{
		Name:            "copy-vault-env",
		Image:           viper.GetString("vault_env_image"),
		ImagePullPolicy: corev1.PullIfNotPresent,
		Command:         []string{"sh", "-c", "cp /usr/local/bin/vault-env /vault/"},
		VolumeMounts: []corev1.VolumeMount{
			{
				Name:      "vault-env",
				MountPath: "/vault/",
			},
		},
	})

	return containers
}

func getVolumes(name string, vaultConfig vaultConfig) []corev1.Volume {
	volumes := []corev1.Volume{
		{
			Name: "vault-env",
			VolumeSource: corev1.VolumeSource{
				EmptyDir: &corev1.EmptyDirVolumeSource{
					Medium: corev1.StorageMediumMemory,
				},
			},
		},
	}

	if vaultConfig.useAgent {
		volumes = append(volumes, corev1.Volume{
			Name: "vault-agent-config",
			VolumeSource: corev1.VolumeSource{
				ConfigMap: &corev1.ConfigMapVolumeSource{
					LocalObjectReference: corev1.LocalObjectReference{
						Name: name + "-vault-agent-config",
					},
				},
			},
		})
	}

	return volumes
}

func vaultSecretsMutator(ctx context.Context, obj metav1.Object) (bool, error) {
	var podSpec *corev1.PodSpec

	switch v := obj.(type) {
	case *corev1.Pod:
		podSpec = &v.Spec
	default:
		return false, nil
	}

	vaultConfig := parseVaultConfig(obj)

	return false, mutatePodSpec(obj, podSpec, vaultConfig, whcontext.GetAdmissionRequest(ctx).Namespace)
}

func parseVaultConfig(obj metav1.Object) vaultConfig {
	var vaultConfig vaultConfig
	annotations := obj.GetAnnotations()
	vaultConfig.addr = annotations["vault.security.banzaicloud.io/vault-addr"]
	vaultConfig.role = annotations["vault.security.banzaicloud.io/vault-role"]
	if vaultConfig.role == "" {
		vaultConfig.role = "default"
	}
	vaultConfig.path = annotations["vault.security.banzaicloud.io/vault-path"]
	if vaultConfig.path == "" {
		vaultConfig.path = "kubernetes"
	}
	vaultConfig.skipVerify = annotations["vault.security.banzaicloud.io/vault-skip-verify"]
	vaultConfig.useAgent, _ = strconv.ParseBool(annotations["vault.security.banzaicloud.io/vault-agent"])
	return vaultConfig
}

func getConfigMapForVaultAgent(obj metav1.Object, vaultConfig vaultConfig) *corev1.ConfigMap {
	return &corev1.ConfigMap{
		ObjectMeta: metav1.ObjectMeta{
			Name: obj.GetName() + "-vault-agent-config",
			// OwnerReferences: []metav1.OwnerReference{
			// 	{
			// 		Name: obj.GetName(),
			// 		// UID:  obj.GetUID(),
			// 	},
			// },
		},
		Data: map[string]string{
			"config.hcl": fmt.Sprintf(vaultAgentConfig, vaultConfig.path, vaultConfig.role),
		},
	}
}

func getDataFromConfigmap(cmName string, ns string) (map[string]string, error) {
	clientset, err := newClientSet()
	if err != nil {
		return nil, err
	}
	configMap, err := clientset.CoreV1().ConfigMaps(ns).Get(cmName, metav1.GetOptions{})
	if err != nil {
		return nil, err
	}
	return configMap.Data, nil
}

func getDataFromSecret(secretName string, ns string) (map[string][]byte, error) {
	clientset, err := newClientSet()
	if err != nil {
		return nil, err
	}
	secret, err := clientset.CoreV1().Secrets(ns).Get(secretName, metav1.GetOptions{})
	if err != nil {
		return nil, err
	}
	return secret.Data, nil
}

func lookForEnvFrom(envFrom []corev1.EnvFromSource, ns string) []corev1.EnvVar {
	var envVars []corev1.EnvVar

	for _, ef := range envFrom {
		if ef.ConfigMapRef != nil {
			data, err := getDataFromConfigmap(ef.ConfigMapRef.Name, ns)
			if err != nil {
				continue
			}
			for key, value := range data {
				if strings.HasPrefix(value, "vault:") {
					envFromCM := corev1.EnvVar{
						Name:  key,
						Value: value,
					}
					envVars = append(envVars, envFromCM)
				}
			}
		}
		if ef.SecretRef != nil {
			data, err := getDataFromSecret(ef.SecretRef.Name, ns)
			if err != nil {
				continue
			}
			for key, value := range data {
				if strings.HasPrefix(string(value), "vault:") {
					envFromSec := corev1.EnvVar{
						Name:  key,
						Value: string(value),
					}
					envVars = append(envVars, envFromSec)
				}
			}
		}
	}
	return envVars
}

func lookForValueFrom(env corev1.EnvVar, ns string) (corev1.EnvVar, error) {
	if env.ValueFrom.ConfigMapKeyRef != nil {
		data, err := getDataFromConfigmap(env.ValueFrom.ConfigMapKeyRef.Name, ns)
		if err != nil {
			return corev1.EnvVar{}, err
		}
		if strings.HasPrefix(data[env.ValueFrom.ConfigMapKeyRef.Key], "vault:") {
			fromCM := corev1.EnvVar{
				Name:  env.Name,
				Value: data[env.ValueFrom.ConfigMapKeyRef.Key],
			}
			return fromCM, nil
		}
	}
	if env.ValueFrom.SecretKeyRef != nil {
		data, err := getDataFromSecret(env.ValueFrom.SecretKeyRef.Name, ns)
		if err != nil {
			return corev1.EnvVar{}, err
		}
		fmt.Println(string(data[env.ValueFrom.SecretKeyRef.Key]))
		if strings.HasPrefix(string(data[env.ValueFrom.SecretKeyRef.Key]), "vault:") {
			fromSecret := corev1.EnvVar{
				Name:  env.Name,
				Value: string(data[env.ValueFrom.SecretKeyRef.Key]),
			}
			return fromSecret, nil
		}
	}
	return corev1.EnvVar{}, nil
}

func mutateContainers(containers []corev1.Container, vaultConfig vaultConfig, ns string) bool {
	mutated := false
	for i, container := range containers {
		var envVars []corev1.EnvVar
		if len(container.EnvFrom) > 0 {
			envFrom := lookForEnvFrom(container.EnvFrom, ns)
			envVars = append(envVars, envFrom...)
		}

		for _, env := range container.Env {
			if strings.HasPrefix(env.Value, "vault:") {
				envVars = append(envVars, env)
			}
			if env.ValueFrom != nil {
				valueFrom, err := lookForValueFrom(env, ns)
				if err != nil {
					continue
				}
				envVars = append(envVars, valueFrom)
			}
		}
		if len(envVars) == 0 {
			continue
		}

		mutated = true

		args := append(container.Command, container.Args...)

		container.Command = []string{"/vault/vault-env"}
		container.Args = args

		container.VolumeMounts = append(container.VolumeMounts, []corev1.VolumeMount{
			{
				Name:      "vault-env",
				MountPath: "/vault/",
			},
		}...)

		container.Env = append(container.Env, []corev1.EnvVar{
			{
				Name:  "VAULT_ADDR",
				Value: vaultConfig.addr,
			},
			{
				Name:  "VAULT_SKIP_VERIFY",
				Value: vaultConfig.skipVerify,
			},
			{
				Name:  "VAULT_PATH",
				Value: vaultConfig.path,
			},
			{
				Name:  "VAULT_ROLE",
				Value: vaultConfig.role,
			},
		}...)

		if vaultConfig.useAgent {
			container.Env = append(container.Env, corev1.EnvVar{
				Name:  "VAULT_TOKEN_FILE",
				Value: "/vault/token",
			})
		}

		containers[i] = container
	}

	return mutated
}
func newClientSet() (*kubernetes.Clientset, error) {
	kubeConfig, err := rest.InClusterConfig()
	if err != nil {
		return nil, err
	}

	clientset, err := kubernetes.NewForConfig(kubeConfig)
	if err != nil {
		return nil, err
	}
	return clientset, nil
}

func mutatePodSpec(obj metav1.Object, podSpec *corev1.PodSpec, vaultConfig vaultConfig, ns string) error {

	clientset, err := newClientSet()
	if err != nil {
		return err
	}

	initContainersMutated := mutateContainers(podSpec.InitContainers, vaultConfig, ns)
	containersMutated := mutateContainers(podSpec.Containers, vaultConfig, ns)

	if initContainersMutated || containersMutated {

		if vaultConfig.useAgent {

			configMap := getConfigMapForVaultAgent(obj, vaultConfig)

			_, err := clientset.CoreV1().ConfigMaps(obj.GetNamespace()).Create(configMap)
			if err != nil {
				if errors.IsAlreadyExists(err) {
					_, err = clientset.CoreV1().ConfigMaps(obj.GetNamespace()).Update(configMap)
					if err != nil {
						return err
					}
				} else {
					return err
				}
			}
		}

		podSpec.InitContainers = append(getInitContainers(vaultConfig), podSpec.InitContainers...)
		podSpec.Volumes = append(podSpec.Volumes, getVolumes(obj.GetName(), vaultConfig)...)
	}

	return nil
}

func initConfig() {
	viper.SetDefault("vault_image", "vault:latest")
	viper.SetDefault("vault_env_image", "banzaicloud/vault-env:latest")
	viper.AutomaticEnv()
}

func handlerFor(config mutating.WebhookConfig, mutator mutating.MutatorFunc, logger log.Logger) http.Handler {
	webhook, err := mutating.NewWebhook(config, mutator, nil, nil, logger)
	if err != nil {
		fmt.Fprintf(os.Stderr, "error creating webhook: %s", err)
		os.Exit(1)
	}

	handler, err := whhttp.HandlerFor(webhook)
	if err != nil {
		fmt.Fprintf(os.Stderr, "error creating webhook: %s", err)
		os.Exit(1)
	}

	return handler
}

func main() {

	initConfig()

	logger := &log.Std{Debug: viper.GetBool("debug")}

	mutator := mutating.MutatorFunc(vaultSecretsMutator)

	podHandler := handlerFor(mutating.WebhookConfig{Name: "vault-secrets-pods", Obj: &corev1.Pod{}}, mutator, logger)

	mux := http.NewServeMux()
	mux.Handle("/pods", podHandler)

	logger.Infof("Listening on :443")
	err := http.ListenAndServeTLS(":443", viper.GetString("tls_cert_file"), viper.GetString("tls_private_key_file"), mux)
	if err != nil {
		fmt.Fprintf(os.Stderr, "error serving webhook: %s", err)
		os.Exit(1)
	}
}
<commit_msg>Fix envFrom empty handling (#338)<commit_after>// Copyright © 2018 Banzai Cloud
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"context"
	"fmt"
	"net/http"
	"os"
	"strconv"
	"strings"

	whhttp "github.com/slok/kubewebhook/pkg/http"
	"github.com/slok/kubewebhook/pkg/log"
	whcontext "github.com/slok/kubewebhook/pkg/webhook/context"
	"github.com/slok/kubewebhook/pkg/webhook/mutating"
	"github.com/spf13/viper"
	corev1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/rest"
)

type vaultConfig struct {
	addr       string
	role       string
	path       string
	skipVerify string
	useAgent   bool
}

var vaultAgentConfig = `
pid_file = "./pidfile"
exit_after_auth = true

auto_auth {
	method "kubernetes" {
		mount_path = "%s"
		config = {
			role = "%s"
		}
	}

	sink "file" {
		config = {
			path = "/vault/token"
		}
	}
}`

func getInitContainers(vaultConfig vaultConfig) []corev1.Container {
	containers := []corev1.Container{}

	if vaultConfig.useAgent {
		containers = append(containers, corev1.Container{
			Name:            "vault-agent",
			Image:           viper.GetString("vault_image"),
			ImagePullPolicy: corev1.PullIfNotPresent,
			Command:         []string{"vault", "agent", "-config=/vault-agent/config.hcl"},
			Env: []corev1.EnvVar{
				{
					Name:  "VAULT_ADDR",
					Value: vaultConfig.addr,
				},
				{
					Name:  "VAULT_SKIP_VERIFY",
					Value: vaultConfig.skipVerify,
				},
			},
			VolumeMounts: []corev1.VolumeMount{
				{
					Name:      "vault-agent-config",
					MountPath: "/vault-agent/",
				},
				{
					Name:      "vault-env",
					MountPath: "/vault/",
				},
			},
		})
	}

	containers = append(containers, corev1.Container{
		Name:            "copy-vault-env",
		Image:           viper.GetString("vault_env_image"),
		ImagePullPolicy: corev1.PullIfNotPresent,
		Command:         []string{"sh", "-c", "cp /usr/local/bin/vault-env /vault/"},
		VolumeMounts: []corev1.VolumeMount{
			{
				Name:      "vault-env",
				MountPath: "/vault/",
			},
		},
	})

	return containers
}

func getVolumes(name string, vaultConfig vaultConfig) []corev1.Volume {
	volumes := []corev1.Volume{
		{
			Name: "vault-env",
			VolumeSource: corev1.VolumeSource{
				EmptyDir: &corev1.EmptyDirVolumeSource{
					Medium: corev1.StorageMediumMemory,
				},
			},
		},
	}

	if vaultConfig.useAgent {
		volumes = append(volumes, corev1.Volume{
			Name: "vault-agent-config",
			VolumeSource: corev1.VolumeSource{
				ConfigMap: &corev1.ConfigMapVolumeSource{
					LocalObjectReference: corev1.LocalObjectReference{
						Name: name + "-vault-agent-config",
					},
				},
			},
		})
	}

	return volumes
}

func vaultSecretsMutator(ctx context.Context, obj metav1.Object) (bool, error) {
	var podSpec *corev1.PodSpec

	switch v := obj.(type) {
	case *corev1.Pod:
		podSpec = &v.Spec
	default:
		return false, nil
	}

	vaultConfig := parseVaultConfig(obj)

	return false, mutatePodSpec(obj, podSpec, vaultConfig, whcontext.GetAdmissionRequest(ctx).Namespace)
}

func parseVaultConfig(obj metav1.Object) vaultConfig {
	var vaultConfig vaultConfig
	annotations := obj.GetAnnotations()
	vaultConfig.addr = annotations["vault.security.banzaicloud.io/vault-addr"]
	vaultConfig.role = annotations["vault.security.banzaicloud.io/vault-role"]
	if vaultConfig.role == "" {
		vaultConfig.role = "default"
	}
	vaultConfig.path = annotations["vault.security.banzaicloud.io/vault-path"]
	if vaultConfig.path == "" {
		vaultConfig.path = "kubernetes"
	}
	vaultConfig.skipVerify = annotations["vault.security.banzaicloud.io/vault-skip-verify"]
	vaultConfig.useAgent, _ = strconv.ParseBool(annotations["vault.security.banzaicloud.io/vault-agent"])
	return vaultConfig
}

func getConfigMapForVaultAgent(obj metav1.Object, vaultConfig vaultConfig) *corev1.ConfigMap {
	return &corev1.ConfigMap{
		ObjectMeta: metav1.ObjectMeta{
			Name: obj.GetName() + "-vault-agent-config",
			// OwnerReferences: []metav1.OwnerReference{
			// 	{
			// 		Name: obj.GetName(),
			// 		// UID:  obj.GetUID(),
			// 	},
			// },
		},
		Data: map[string]string{
			"config.hcl": fmt.Sprintf(vaultAgentConfig, vaultConfig.path, vaultConfig.role),
		},
	}
}

func getDataFromConfigmap(cmName string, ns string) (map[string]string, error) {
	clientset, err := newClientSet()
	if err != nil {
		return nil, err
	}
	configMap, err := clientset.CoreV1().ConfigMaps(ns).Get(cmName, metav1.GetOptions{})
	if err != nil {
		return nil, err
	}
	return configMap.Data, nil
}

func getDataFromSecret(secretName string, ns string) (map[string][]byte, error) {
	clientset, err := newClientSet()
	if err != nil {
		return nil, err
	}
	secret, err := clientset.CoreV1().Secrets(ns).Get(secretName, metav1.GetOptions{})
	if err != nil {
		return nil, err
	}
	return secret.Data, nil
}

func lookForEnvFrom(envFrom []corev1.EnvFromSource, ns string) ([]corev1.EnvVar, error) {
	var envVars []corev1.EnvVar

	for _, ef := range envFrom {
		if ef.ConfigMapRef != nil {
			data, err := getDataFromConfigmap(ef.ConfigMapRef.Name, ns)
			if err != nil {
				return envVars, err
			}
			for key, value := range data {
				if strings.HasPrefix(value, "vault:") {
					envFromCM := corev1.EnvVar{
						Name:  key,
						Value: value,
					}
					envVars = append(envVars, envFromCM)
				}
			}
		}
		if ef.SecretRef != nil {
			data, err := getDataFromSecret(ef.SecretRef.Name, ns)
			if err != nil {
				return envVars, err
			}
			for key, value := range data {
				if strings.HasPrefix(string(value), "vault:") {
					envFromSec := corev1.EnvVar{
						Name:  key,
						Value: string(value),
					}
					envVars = append(envVars, envFromSec)
				}
			}
		}
	}
	return envVars, nil
}

func lookForValueFrom(env corev1.EnvVar, ns string) (*corev1.EnvVar, error) {
	if env.ValueFrom.ConfigMapKeyRef != nil {
		data, err := getDataFromConfigmap(env.ValueFrom.ConfigMapKeyRef.Name, ns)
		if err != nil {
			return nil, err
		}
		if strings.HasPrefix(data[env.ValueFrom.ConfigMapKeyRef.Key], "vault:") {
			fromCM := corev1.EnvVar{
				Name:  env.Name,
				Value: data[env.ValueFrom.ConfigMapKeyRef.Key],
			}
			return &fromCM, nil
		}
	}
	if env.ValueFrom.SecretKeyRef != nil {
		data, err := getDataFromSecret(env.ValueFrom.SecretKeyRef.Name, ns)
		if err != nil {
			return nil, err
		}
		if strings.HasPrefix(string(data[env.ValueFrom.SecretKeyRef.Key]), "vault:") {
			fromSecret := corev1.EnvVar{
				Name:  env.Name,
				Value: string(data[env.ValueFrom.SecretKeyRef.Key]),
			}
			return &fromSecret, nil
		}
	}
	return nil, nil
}

func mutateContainers(containers []corev1.Container, vaultConfig vaultConfig, ns string) (bool, error) {
	mutated := false
	for i, container := range containers {
		var envVars []corev1.EnvVar
		if len(container.EnvFrom) > 0 {
			envFrom, err := lookForEnvFrom(container.EnvFrom, ns)
			if err != nil {
				return false, err
			}
			envVars = append(envVars, envFrom...)
		}

		for _, env := range container.Env {
			if strings.HasPrefix(env.Value, "vault:") {
				envVars = append(envVars, env)
			}
			if env.ValueFrom != nil {
				valueFrom, err := lookForValueFrom(env, ns)
				if err != nil {
					return false, err
				}
				if valueFrom == nil {
					continue
				}
				envVars = append(envVars, *valueFrom)
			}
		}
		if len(envVars) == 0 {
			continue
		}

		mutated = true

		args := append(container.Command, container.Args...)

		container.Command = []string{"/vault/vault-env"}
		container.Args = args

		container.VolumeMounts = append(container.VolumeMounts, []corev1.VolumeMount{
			{
				Name:      "vault-env",
				MountPath: "/vault/",
			},
		}...)

		container.Env = append(container.Env, []corev1.EnvVar{
			{
				Name:  "VAULT_ADDR",
				Value: vaultConfig.addr,
			},
			{
				Name:  "VAULT_SKIP_VERIFY",
				Value: vaultConfig.skipVerify,
			},
			{
				Name:  "VAULT_PATH",
				Value: vaultConfig.path,
			},
			{
				Name:  "VAULT_ROLE",
				Value: vaultConfig.role,
			},
		}...)

		if vaultConfig.useAgent {
			container.Env = append(container.Env, corev1.EnvVar{
				Name:  "VAULT_TOKEN_FILE",
				Value: "/vault/token",
			})
		}

		containers[i] = container
	}

	return mutated, nil
}
func newClientSet() (*kubernetes.Clientset, error) {
	kubeConfig, err := rest.InClusterConfig()
	if err != nil {
		return nil, err
	}

	clientset, err := kubernetes.NewForConfig(kubeConfig)
	if err != nil {
		return nil, err
	}
	return clientset, nil
}

func mutatePodSpec(obj metav1.Object, podSpec *corev1.PodSpec, vaultConfig vaultConfig, ns string) error {

	clientset, err := newClientSet()
	if err != nil {
		return err
	}

	initContainersMutated, err := mutateContainers(podSpec.InitContainers, vaultConfig, ns)
	if err != nil {
		return err
	}
	containersMutated, err := mutateContainers(podSpec.Containers, vaultConfig, ns)
	if err != nil {
		return err
	}

	if initContainersMutated || containersMutated {

		if vaultConfig.useAgent {

			configMap := getConfigMapForVaultAgent(obj, vaultConfig)

			_, err := clientset.CoreV1().ConfigMaps(obj.GetNamespace()).Create(configMap)
			if err != nil {
				if errors.IsAlreadyExists(err) {
					_, err = clientset.CoreV1().ConfigMaps(obj.GetNamespace()).Update(configMap)
					if err != nil {
						return err
					}
				} else {
					return err
				}
			}
		}

		podSpec.InitContainers = append(getInitContainers(vaultConfig), podSpec.InitContainers...)
		podSpec.Volumes = append(podSpec.Volumes, getVolumes(obj.GetName(), vaultConfig)...)
	}

	return nil
}

func initConfig() {
	viper.SetDefault("vault_image", "vault:latest")
	viper.SetDefault("vault_env_image", "banzaicloud/vault-env:latest")
	viper.AutomaticEnv()
}

func handlerFor(config mutating.WebhookConfig, mutator mutating.MutatorFunc, logger log.Logger) http.Handler {
	webhook, err := mutating.NewWebhook(config, mutator, nil, nil, logger)
	if err != nil {
		fmt.Fprintf(os.Stderr, "error creating webhook: %s", err)
		os.Exit(1)
	}

	handler, err := whhttp.HandlerFor(webhook)
	if err != nil {
		fmt.Fprintf(os.Stderr, "error creating webhook: %s", err)
		os.Exit(1)
	}

	return handler
}

func main() {

	initConfig()

	logger := &log.Std{Debug: viper.GetBool("debug")}

	mutator := mutating.MutatorFunc(vaultSecretsMutator)

	podHandler := handlerFor(mutating.WebhookConfig{Name: "vault-secrets-pods", Obj: &corev1.Pod{}}, mutator, logger)

	mux := http.NewServeMux()
	mux.Handle("/pods", podHandler)

	logger.Infof("Listening on :443")
	err := http.ListenAndServeTLS(":443", viper.GetString("tls_cert_file"), viper.GetString("tls_private_key_file"), mux)
	if err != nil {
		fmt.Fprintf(os.Stderr, "error serving webhook: %s", err)
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>package labeler

import (
	"log"
	"regexp"
	"strconv"

	"github.com/google/go-github/github"
	"github.com/parkr/auto-reply/ctx"
)

var fixesIssueMatcher = regexp.MustCompile(`(Close|Closes|Closed|Fix|Fixes|Fixed|Resolve|Resolves|Resolved) #(\d+)`)

var IssueHasPullRequestLabeler = func(context *ctx.Context, payload interface{}) error {
	event, ok := payload.(*github.PullRequestEvent)
	if !ok {
		return context.NewError("IssueHasPullRequestLabeler: not a pull request event")
	}

	if *event.Action != "opened" {
		return nil
	}

	owner, repo, description := *event.Repo.Owner.Login, *event.Repo.Name, *event.PullRequest.Body

	issueNum := issueFixed(description)

	var err error
	if issueNum != -1 {
		err := AddLabels(context.GitHub, owner, repo, issueNum, []string{"has-pull-request"})
		if err != nil {
			log.Printf("error adding the has-pull-request label: %v", err)
		}
	}

	return err
}

func issueFixed(description string) int {
	issueSubmatches := fixesIssueMatcher.FindAllStringSubmatch(description, -1)
	if len(issueSubmatches) == 0 || len(issueSubmatches[0]) < 2 {
		return -1
	}

	issueNum, _ := strconv.Atoi(issueSubmatches[0][1])
	return issueNum
}
<commit_msg>Avoid changing the submatch index<commit_after>package labeler

import (
	"log"
	"regexp"
	"strconv"

	"github.com/google/go-github/github"
	"github.com/parkr/auto-reply/ctx"
)

var fixesIssueMatcher = regexp.MustCompile(`(?:Close|Closes|Closed|Fix|Fixes|Fixed|Resolve|Resolves|Resolved) #(\d+)`)

var IssueHasPullRequestLabeler = func(context *ctx.Context, payload interface{}) error {
	event, ok := payload.(*github.PullRequestEvent)
	if !ok {
		return context.NewError("IssueHasPullRequestLabeler: not a pull request event")
	}

	if *event.Action != "opened" {
		return nil
	}

	owner, repo, description := *event.Repo.Owner.Login, *event.Repo.Name, *event.PullRequest.Body

	issueNum := issueFixed(description)

	var err error
	if issueNum != -1 {
		err := AddLabels(context.GitHub, owner, repo, issueNum, []string{"has-pull-request"})
		if err != nil {
			log.Printf("error adding the has-pull-request label: %v", err)
		}
	}

	return err
}

func issueFixed(description string) int {
	issueSubmatches := fixesIssueMatcher.FindAllStringSubmatch(description, -1)
	if len(issueSubmatches) == 0 || len(issueSubmatches[0]) < 2 {
		return -1
	}

	issueNum, _ := strconv.Atoi(issueSubmatches[0][1])
	return issueNum
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"time"

	"github.com/sayden/simple-triggered-stack"
	"github.com/timjchin/unpuzzled"
)

type config struct {
	port string
}

var conf config

func main() {
	app := unpuzzled.NewApp()

	app.Name = "Server"
	app.Usage = "See --help for info"
	app.Authors = []unpuzzled.Author{{Name: "Mario Castro", Email: "mariocaster@gmail.com"}}

	app.Command = &unpuzzled.Command{
		Name:  "Main",
		Usage: "Launches the server",
		Variables: []unpuzzled.Variable{
			&unpuzzled.StringVariable{
				Name:        "port",
				Destination: &conf.port,
				Default:     "8080",
				Description: "Server port",
			},
		},
		Action: launch,
	}
	app.Run(os.Args)
}

type stackableString struct {
	simple_triggered_stack.Stackable
	data string
}

func launch() {
	msgN := 10

	//Channel that the stack will use to notify us that flushing has complete
	quit := make(chan struct{})

	stackConfig := simple_triggered_stack.Config{
		MaxStack:           3,
		MaxIngestionBuffer: 10,
	}

	stackCallback := func(stack []simple_triggered_stack.Stackable) {
		for _, v := range stack {
			if s, ok := v.(*stackableString); ok {
				fmt.Printf("%s ", s.data	)
			}
		}

		fmt.Println()
	}

	stack := simple_triggered_stack.NewStack(quit, stackConfig, stackCallback)

	for i := 0; i < msgN; i++ {
		time.Sleep(time.Millisecond * 100)
		stack.IngestionCh() <- &stackableString{data: fmt.Sprintf("Hello %d", i)}
	}

	//We have finished pushing, close ingestion channel. Queue will notify of successful flushing by closing quit ch
	close(stack.IngestionCh())

	<-quit
}
<commit_msg>Server code removed<commit_after>package main

import (
	"fmt"
	"time"

	"github.com/sayden/simple-triggered-stack"
)

type stackableString struct {
	simple_triggered_stack.Stackable
	data string
}

func main() {
	msgN := 10

	//Channel that the stack will use to notify us that flushing has complete
	quit := make(chan struct{})

	stackConfig := simple_triggered_stack.Config{
		MaxStack:           3,
		MaxIngestionBuffer: 10,
	}

	stackCallback := func(stack []simple_triggered_stack.Stackable) {
		for _, v := range stack {
			if s, ok := v.(*stackableString); ok {
				fmt.Printf("%s ", s.data)
			}
		}

		fmt.Println()
	}

	stack := simple_triggered_stack.NewStack(quit, stackConfig, stackCallback)

	for i := 0; i < msgN; i++ {
		time.Sleep(time.Millisecond * 100)
		stack.IngestionCh() <- &stackableString{data: fmt.Sprintf("Hello %d", i)}
	}

	//We have finished pushing, close ingestion channel. Queue will notify of successful flushing by closing quit ch
	close(stack.IngestionCh())

	<-quit
}
<|endoftext|>


<commit_before>package models

import (
	"appengine"
	"appengine/datastore"
	"fmt"
	"time"
)

type Mention struct {
	Source   string
	Target   string
	Recieved time.Time
	Verified bool
}

func NewMention(c appengine.Context, source string, target string) *Mention {
	var k *datastore.Key
	var e *Mention

	k = GetMention(c, source, target)
	if k == nil {
		k = datastore.NewIncompleteKey(c, "Mention", nil)
		e = new(Mention)
		e.Source = source
		e.target = target
		e.Recieved = time.Now()
		e.Verified = false
	}

	return e
}

func MentionExists(c appengine.Context, source string, target string) bool {
	return GetMention(c, source, target) != nil
}

func GetMention(c appengine.Context, source string, target string) *datastore.Key {
	q := datastore.NewQuery("Mention").Filter("source =", source).Filter("target =", target).KeysOnly()
	k, err := q.Run(c).Next(nil)
	if err != nil {
		c.Infof("Error getting Mention (%s => %s): %v", source, target, err)
		return nil
	}
	return k
}

func (e *Entry) Save(c appengine.Context) error {
	var k *datastore.Key
	if e.HasId() {
		id, _ := MaxId(c)
		e.Id = id + 1
		k = datastore.NewIncompleteKey(c, "Entry", nil)
	} else {
		k = datastore.NewKey(c, "Entry", fmt.Sprintf("%d", e.Id), 0, nil)
	}

	_, err := datastore.Put(c, k, e)
	if err == nil {
		c.Infof("Wrote %+v", e)
	} else {
		c.Warningf("Error writing entry: %v", e)
	}
	return err
}
<commit_msg>mention<commit_after>package models

import (
	"appengine"
	"appengine/datastore"
	"fmt"
	"net/url"
	"time"
)

type Mention struct {
	Source   url.URL
	Target   url.URL
	Recieved time.Time
	Verified bool
}

func NewMention(c appengine.Context, source string, target string) (*Mention, error) {
	var k *datastore.Key
	var e *Mention

	k = GetMention(c, source, target)
	if k == nil {
		k = datastore.NewIncompleteKey(c, "Mention", nil)
		e = new(Mention)
		src, err := url.Parse(source)
		if err != nil {
			return nil, err
		}

		trg, err := url.Parse(target)
		if err != nil {
			return nil, err
		}

		e.Source = src
		e.target = trg
		e.Recieved = time.Now()
		e.Verified = false

		_, err := datastore.Put(c, k, e)
		if err != nil {
			return nil, err
		}
	} else {
		if err = datastore.Get(c, k, &e); err != nil {
			return nil, err
		}
	}

	return e, nil
}

func MentionExists(c appengine.Context, source string, target string) bool {
	return GetMention(c, source, target) != nil
}

func GetMention(c appengine.Context, source string, target string) *datastore.Key {
	q := datastore.NewQuery("Mention").Filter("source =", source).Filter("target =", target).KeysOnly()
	k, err := q.Run(c).Next(nil)
	if err != nil {
		c.Infof("Error getting Mention (%s => %s): %v", source, target, err)
		return nil
	}
	return k
}
<|endoftext|>


<commit_before>package common

import (
	"fmt"
	"log"
	"os"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/credentials/stscreds"
	"github.com/aws/aws-sdk-go/aws/ec2metadata"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/hashicorp/packer/template/interpolate"
)

// AccessConfig is for common configuration related to AWS access
type AccessConfig struct {
	AccessKey         string `mapstructure:"access_key"`
	AssumeRoleArn     string `mapstructure:"assume_role_arn"`
	CustomEndpointEc2 string `mapstructure:"custom_endpoint_ec2"`
	ExternalID        string `mapstructure:"external_id"`
	MFACode           string `mapstructure:"mfa_code"`
	MFASerial         string `mapstructure:"mfa_serial"`
	ProfileName       string `mapstructure:"profile"`
	RawRegion         string `mapstructure:"region"`
	SecretKey         string `mapstructure:"secret_key"`
	SkipValidation    bool   `mapstructure:"skip_region_validation"`
	Token             string `mapstructure:"token"`
	session           *session.Session
}

// Config returns a valid aws.Config object for access to AWS services, or
// an error if the authentication and region couldn't be resolved
func (c *AccessConfig) Session() (*session.Session, error) {
	if c.session != nil {
		return c.session, nil
	}

	region, err := c.Region()
	if err != nil {
		return nil, err
	}

	if c.AssumeRoleArn != "" {
		var options []func(*stscreds.AssumeRoleProvider)
		if c.MFACode != "" {
			options = append(options, func(p *stscreds.AssumeRoleProvider) {
				p.SerialNumber = aws.String(c.MFASerial)
				p.TokenProvider = func() (string, error) {
					return c.MFACode, nil
				}
			})
		}
	}

	if c.ProfileName != "" {
		err := os.Setenv("AWS_PROFILE", c.ProfileName)
		if err != nil {
			log.Printf("Set env error: %s", err)
		}
	}

	config := aws.NewConfig().WithRegion(region).WithMaxRetries(11).WithCredentialsChainVerboseErrors(true)

	if c.CustomEndpointEc2 != "" {
		config = config.WithEndpoint(aws.String(c.CustomEndpointEc2))
	}

	if c.AccessKey != "" {
		creds := credentials.NewChainCredentials(
			[]credentials.Provider{
				&credentials.StaticProvider{
					Value: credentials.Value{
						AccessKeyID:     c.AccessKey,
						SecretAccessKey: c.SecretKey,
						SessionToken:    c.Token,
					},
				},
			})
		config = config.WithCredentials(creds)
	}

	opts := session.Options{
		SharedConfigState: session.SharedConfigEnable,
		Config:            *config,
	}
	if c.MFACode != "" {
		opts.AssumeRoleTokenProvider = func() (string, error) {
			return c.MFACode, nil
		}
	}
	c.session, err = session.NewSessionWithOptions(opts)
	if err != nil {
		return nil, err
	}

	return c.session, nil
}

// Region returns the aws.Region object for access to AWS services, requesting
// the region from the instance metadata if possible.
func (c *AccessConfig) Region() (string, error) {
	if c.RawRegion != "" {
		if !c.SkipValidation {
			if valid := ValidateRegion(c.RawRegion); !valid {
				return "", fmt.Errorf("Not a valid region: %s", c.RawRegion)
			}
		}
		return c.RawRegion, nil
	}

	sess := session.New()
	ec2meta := ec2metadata.New(sess)
	identity, err := ec2meta.GetInstanceIdentityDocument()
	if err != nil {
		return "", err
	}
	return identity.Region, nil
}

func (c *AccessConfig) Prepare(ctx *interpolate.Context) []error {
	var errs []error
	if c.RawRegion != "" && !c.SkipValidation {
		if valid := ValidateRegion(c.RawRegion); !valid {
			errs = append(errs, fmt.Errorf("Unknown region: %s", c.RawRegion))
		}
	}

	if len(errs) > 0 {
		return errs
	}

	return nil
}
<commit_msg>bugfix<commit_after>package common

import (
	"fmt"
	"log"
	"os"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/credentials/stscreds"
	"github.com/aws/aws-sdk-go/aws/ec2metadata"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/hashicorp/packer/template/interpolate"
)

// AccessConfig is for common configuration related to AWS access
type AccessConfig struct {
	AccessKey         string `mapstructure:"access_key"`
	AssumeRoleArn     string `mapstructure:"assume_role_arn"`
	CustomEndpointEc2 string `mapstructure:"custom_endpoint_ec2"`
	ExternalID        string `mapstructure:"external_id"`
	MFACode           string `mapstructure:"mfa_code"`
	MFASerial         string `mapstructure:"mfa_serial"`
	ProfileName       string `mapstructure:"profile"`
	RawRegion         string `mapstructure:"region"`
	SecretKey         string `mapstructure:"secret_key"`
	SkipValidation    bool   `mapstructure:"skip_region_validation"`
	Token             string `mapstructure:"token"`
	session           *session.Session
}

// Config returns a valid aws.Config object for access to AWS services, or
// an error if the authentication and region couldn't be resolved
func (c *AccessConfig) Session() (*session.Session, error) {
	if c.session != nil {
		return c.session, nil
	}

	region, err := c.Region()
	if err != nil {
		return nil, err
	}

	if c.AssumeRoleArn != "" {
		var options []func(*stscreds.AssumeRoleProvider)
		if c.MFACode != "" {
			options = append(options, func(p *stscreds.AssumeRoleProvider) {
				p.SerialNumber = aws.String(c.MFASerial)
				p.TokenProvider = func() (string, error) {
					return c.MFACode, nil
				}
			})
		}
	}

	if c.ProfileName != "" {
		err := os.Setenv("AWS_PROFILE", c.ProfileName)
		if err != nil {
			log.Printf("Set env error: %s", err)
		}
	}

	config := aws.NewConfig().WithRegion(region).WithMaxRetries(11).WithCredentialsChainVerboseErrors(true)

	if c.CustomEndpointEc2 != "" {
		config = config.WithEndpoint(c.CustomEndpointEc2)
	}

	if c.AccessKey != "" {
		creds := credentials.NewChainCredentials(
			[]credentials.Provider{
				&credentials.StaticProvider{
					Value: credentials.Value{
						AccessKeyID:     c.AccessKey,
						SecretAccessKey: c.SecretKey,
						SessionToken:    c.Token,
					},
				},
			})
		config = config.WithCredentials(creds)
	}

	opts := session.Options{
		SharedConfigState: session.SharedConfigEnable,
		Config:            *config,
	}
	if c.MFACode != "" {
		opts.AssumeRoleTokenProvider = func() (string, error) {
			return c.MFACode, nil
		}
	}
	c.session, err = session.NewSessionWithOptions(opts)
	if err != nil {
		return nil, err
	}

	return c.session, nil
}

// Region returns the aws.Region object for access to AWS services, requesting
// the region from the instance metadata if possible.
func (c *AccessConfig) Region() (string, error) {
	if c.RawRegion != "" {
		if !c.SkipValidation {
			if valid := ValidateRegion(c.RawRegion); !valid {
				return "", fmt.Errorf("Not a valid region: %s", c.RawRegion)
			}
		}
		return c.RawRegion, nil
	}

	sess := session.New()
	ec2meta := ec2metadata.New(sess)
	identity, err := ec2meta.GetInstanceIdentityDocument()
	if err != nil {
		return "", err
	}
	return identity.Region, nil
}

func (c *AccessConfig) Prepare(ctx *interpolate.Context) []error {
	var errs []error
	if c.RawRegion != "" && !c.SkipValidation {
		if valid := ValidateRegion(c.RawRegion); !valid {
			errs = append(errs, fmt.Errorf("Unknown region: %s", c.RawRegion))
		}
	}

	if len(errs) > 0 {
		return errs
	}

	return nil
}
<|endoftext|>


<commit_before>package common

import (
	"fmt"
	"log"
	"strings"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/ec2metadata"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/ec2"
	"github.com/aws/aws-sdk-go/service/ec2/ec2iface"
	"github.com/hashicorp/go-cleanhttp"
	"github.com/hashicorp/packer/template/interpolate"
)

// AccessConfig is for common configuration related to AWS access
type AccessConfig struct {
	AccessKey            string `mapstructure:"access_key"`
	CustomEndpointEc2    string `mapstructure:"custom_endpoint_ec2"`
	DecodeAuthZMessages  bool   `mapstructure:"decode_authorization_messages"`
	MFACode              string `mapstructure:"mfa_code"`
	ProfileName          string `mapstructure:"profile"`
	RawRegion            string `mapstructure:"region"`
	SecretKey            string `mapstructure:"secret_key"`
	SkipValidation       bool   `mapstructure:"skip_region_validation"`
	SkipMetadataApiCheck bool   `mapstructure:"skip_metadata_api_check"`
	Token                string `mapstructure:"token"`
	session              *session.Session

	getEC2Connection func() ec2iface.EC2API
}

// Config returns a valid aws.Config object for access to AWS services, or
// an error if the authentication and region couldn't be resolved
func (c *AccessConfig) Session() (*session.Session, error) {
	if c.session != nil {
		return c.session, nil
	}

	config := aws.NewConfig().WithCredentialsChainVerboseErrors(true)
	staticCreds := credentials.NewStaticCredentials(c.AccessKey, c.SecretKey, c.Token)
	if _, err := staticCreds.Get(); err != credentials.ErrStaticCredentialsEmpty {
		config.WithCredentials(staticCreds)
	}

	// default is 3, and when it was causing failures for users being throttled
	config = config.WithMaxRetries(20)

	region, err := c.region()
	if err != nil {
		return nil, fmt.Errorf("Could not get region, "+
			"probably because it's not set or we're not running on AWS. %s", err)
	}
	config = config.WithRegion(region)

	if c.CustomEndpointEc2 != "" {
		config = config.WithEndpoint(c.CustomEndpointEc2)
	}

	opts := session.Options{
		SharedConfigState: session.SharedConfigEnable,
		Config:            *config,
	}

	if c.ProfileName != "" {
		opts.Profile = c.ProfileName
	}

	if c.MFACode != "" {
		opts.AssumeRoleTokenProvider = func() (string, error) {
			return c.MFACode, nil
		}
	}

	if sess, err := session.NewSessionWithOptions(opts); err != nil {
		return nil, err
	} else if *sess.Config.Region == "" {
		return nil, fmt.Errorf("Could not find AWS region, make sure it's set.")
	} else {
		log.Printf("Found region %s", *sess.Config.Region)
		c.session = sess

		cp, err := c.session.Config.Credentials.Get()
		if err != nil {
			if awsErr, ok := err.(awserr.Error); ok && awsErr.Code() == "NoCredentialProviders" {
				return nil, fmt.Errorf("No valid credential sources found for AWS Builder. " +
					"Please see https://www.packer.io/docs/builders/amazon.html#specifying-amazon-credentials " +
					"for more information on providing credentials for the AWS Builder.")
			} else {
				return nil, fmt.Errorf("Error loading credentials for AWS Provider: %s", err)
			}
		}
		log.Printf("[INFO] AWS Auth provider used: %q", cp.ProviderName)
	}

	if c.DecodeAuthZMessages {
		DecodeAuthZMessages(c.session)
	}

	return c.session, nil
}

func (c *AccessConfig) SessionRegion() string {
	if c.session == nil {
		panic("access config session should be set.")
	}
	return aws.StringValue(c.session.Config.Region)
}

func (c *AccessConfig) IsGovCloud() bool {
	return strings.HasPrefix(c.SessionRegion(), "us-gov-")
}

func (c *AccessConfig) IsChinaCloud() bool {
	return strings.HasPrefix(c.SessionRegion(), "cn-")
}

// metadataRegion returns the region from the metadata service
func (c *AccessConfig) metadataRegion() (string, error) {

	client := cleanhttp.DefaultClient()

	// Keep the default timeout (100ms) low as we don't want to wait in non-EC2 environments
	client.Timeout = 100 * time.Millisecond
	ec2meta := ec2metadata.New(session.New(), &aws.Config{
		HTTPClient: client,
	})
	return ec2meta.Region()
}

func (c *AccessConfig) region() (string, error) {
	if c.RawRegion != "" {
		return c.RawRegion, nil
	}
	return c.metadataRegion()
}

func (c *AccessConfig) Prepare(ctx *interpolate.Context) []error {
	var errs []error

	if c.SkipMetadataApiCheck {
		log.Println("(WARN) skip_metadata_api_check ignored.")
	}
	// Either both access and secret key must be set or neither of them should
	// be.
	if (len(c.AccessKey) > 0) != (len(c.SecretKey) > 0) {
		errs = append(errs,
			fmt.Errorf("`access_key` and `secret_key` must both be either set or not set."))
	}

	if c.RawRegion != "" && !c.SkipValidation {
		err := c.ValidateRegion(c.RawRegion)
		if err != nil {
			errs = append(errs, fmt.Errorf("error validating region: %s", err.Error()))
		}
	}

	return errs
}

func (c *AccessConfig) NewEC2Connection() (ec2iface.EC2API, error) {
	if c.getEC2Connection != nil {
		return c.getEC2Connection(), nil
	}
	sess, err := c.Session()
	if err != nil {
		return nil, err
	}
	return ec2.New(sess), nil
}
<commit_msg>remove duplicate region check error<commit_after>package common

import (
	"fmt"
	"log"
	"strings"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/ec2metadata"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/ec2"
	"github.com/aws/aws-sdk-go/service/ec2/ec2iface"
	"github.com/hashicorp/go-cleanhttp"
	"github.com/hashicorp/packer/template/interpolate"
)

// AccessConfig is for common configuration related to AWS access
type AccessConfig struct {
	AccessKey            string `mapstructure:"access_key"`
	CustomEndpointEc2    string `mapstructure:"custom_endpoint_ec2"`
	DecodeAuthZMessages  bool   `mapstructure:"decode_authorization_messages"`
	MFACode              string `mapstructure:"mfa_code"`
	ProfileName          string `mapstructure:"profile"`
	RawRegion            string `mapstructure:"region"`
	SecretKey            string `mapstructure:"secret_key"`
	SkipValidation       bool   `mapstructure:"skip_region_validation"`
	SkipMetadataApiCheck bool   `mapstructure:"skip_metadata_api_check"`
	Token                string `mapstructure:"token"`
	session              *session.Session

	getEC2Connection func() ec2iface.EC2API
}

// Config returns a valid aws.Config object for access to AWS services, or
// an error if the authentication and region couldn't be resolved
func (c *AccessConfig) Session() (*session.Session, error) {
	if c.session != nil {
		return c.session, nil
	}

	config := aws.NewConfig().WithCredentialsChainVerboseErrors(true)
	staticCreds := credentials.NewStaticCredentials(c.AccessKey, c.SecretKey, c.Token)
	if _, err := staticCreds.Get(); err != credentials.ErrStaticCredentialsEmpty {
		config.WithCredentials(staticCreds)
	}

	// default is 3, and when it was causing failures for users being throttled
	config = config.WithMaxRetries(20)

	region, err := c.region()
	if err != nil {
		return nil, fmt.Errorf("Could not get region, "+
			"probably because it's not set or we're not running on AWS. %s", err)
	}
	config = config.WithRegion(region)

	if c.CustomEndpointEc2 != "" {
		config = config.WithEndpoint(c.CustomEndpointEc2)
	}

	opts := session.Options{
		SharedConfigState: session.SharedConfigEnable,
		Config:            *config,
	}

	if c.ProfileName != "" {
		opts.Profile = c.ProfileName
	}

	if c.MFACode != "" {
		opts.AssumeRoleTokenProvider = func() (string, error) {
			return c.MFACode, nil
		}
	}

	if sess, err := session.NewSessionWithOptions(opts); err != nil {
		return nil, err
	} else {
		log.Printf("Found region %s", *sess.Config.Region)
		c.session = sess

		cp, err := c.session.Config.Credentials.Get()
		if err != nil {
			if awsErr, ok := err.(awserr.Error); ok && awsErr.Code() == "NoCredentialProviders" {
				return nil, fmt.Errorf("No valid credential sources found for AWS Builder. " +
					"Please see https://www.packer.io/docs/builders/amazon.html#specifying-amazon-credentials " +
					"for more information on providing credentials for the AWS Builder.")
			} else {
				return nil, fmt.Errorf("Error loading credentials for AWS Provider: %s", err)
			}
		}
		log.Printf("[INFO] AWS Auth provider used: %q", cp.ProviderName)
	}

	if c.DecodeAuthZMessages {
		DecodeAuthZMessages(c.session)
	}

	return c.session, nil
}

func (c *AccessConfig) SessionRegion() string {
	if c.session == nil {
		panic("access config session should be set.")
	}
	return aws.StringValue(c.session.Config.Region)
}

func (c *AccessConfig) IsGovCloud() bool {
	return strings.HasPrefix(c.SessionRegion(), "us-gov-")
}

func (c *AccessConfig) IsChinaCloud() bool {
	return strings.HasPrefix(c.SessionRegion(), "cn-")
}

// metadataRegion returns the region from the metadata service
func (c *AccessConfig) metadataRegion() (string, error) {

	client := cleanhttp.DefaultClient()

	// Keep the default timeout (100ms) low as we don't want to wait in non-EC2 environments
	client.Timeout = 100 * time.Millisecond
	ec2meta := ec2metadata.New(session.New(), &aws.Config{
		HTTPClient: client,
	})
	return ec2meta.Region()
}

func (c *AccessConfig) region() (string, error) {
	if c.RawRegion != "" {
		return c.RawRegion, nil
	}
	return c.metadataRegion()
}

func (c *AccessConfig) Prepare(ctx *interpolate.Context) []error {
	var errs []error

	if c.SkipMetadataApiCheck {
		log.Println("(WARN) skip_metadata_api_check ignored.")
	}
	// Either both access and secret key must be set or neither of them should
	// be.
	if (len(c.AccessKey) > 0) != (len(c.SecretKey) > 0) {
		errs = append(errs,
			fmt.Errorf("`access_key` and `secret_key` must both be either set or not set."))
	}

	if c.RawRegion != "" && !c.SkipValidation {
		err := c.ValidateRegion(c.RawRegion)
		if err != nil {
			errs = append(errs, fmt.Errorf("error validating region: %s", err.Error()))
		}
	}

	return errs
}

func (c *AccessConfig) NewEC2Connection() (ec2iface.EC2API, error) {
	if c.getEC2Connection != nil {
		return c.getEC2Connection(), nil
	}
	sess, err := c.Session()
	if err != nil {
		return nil, err
	}
	return ec2.New(sess), nil
}
<|endoftext|>


<commit_before>package main

// this takes a list of gzipped Reddit comment-body files
// and returns a frequency count of words
// as a gzipped CSV file
import (
	"compress/gzip"
	"encoding/json"
	"flag"
	"fmt"
	"log"
	"os"
	"path/filepath"
	"runtime"
	"sort"
	"strings"
	"sync"
)

// freqCount is mapping of string->count
type freqCount map[string]int

// make a new counter with some minor preallocation
//  each month has about 2.2M uniques
func newFreqCount() freqCount {
	return make(freqCount, 3000000)
}

// globals used for worker queues and global counts
var wg sync.WaitGroup

// Reddit is struct used to unmarshal the reddit comment
type Reddit struct {
	Body string `json:"body"`
}

// isAZ returns true if input is only consists of letters a-z (case SENSITIVE)
func isAZ(s string) bool {
	// NO: for _, ch := range s {
	//   This iterates over UTF-8 characters
	slen := len(s)
	for i := 0; i < slen; i++ {
		ch := s[i]
		if ch < 'a' || ch > 'z' {
			return false
		}
	}
	return true
}

// text2words takes raw text and extracts English-list words
//   split on whitespace
//   remove starting or ending  `'".,:;-()`
//   lower case
//   check if all a-z
//
// this doesn't catch everything, but that's ok for this purpose.
//
func text2words(raw string) []string {
	parts := strings.Fields(raw)
	out := make([]string, 0, len(parts))
	for _, word := range parts {
		word = strings.Trim(word, `'".,:;-()`)
		word = strings.ToLower(word)
		if isAZ(word) {
			out = append(out, word)
		}
	}
	return out
}

// doit does the following
//   opens a gizpped up reddit file
//   extracts words
//   updates counts
//
// Every 1M lines, it updates the global freq count
//
func doit(prefix, filename string) {
	const maxLines = 1 << 31
	const suffix = "-body.json.gz"
	// generate outputfile name
	// blah/RC_2015-06-body.json.gz --> RC_2015-06-counts.csv.gz
	base := filepath.Base(filename)
	base = base[0:len(base)-len(suffix)] + "-counts.csv.gz"

	log.Printf("[%s] starting. Output file %s", prefix, base)
	counts := newFreqCount()
	fi, err := os.Open(filename)
	if err != nil {
		log.Fatalf("[%s]: %s", prefix, err)
	}

	file, err := gzip.NewReader(fi)
	if err != nil {
		log.Fatalf("[%s]: gzip error %s", prefix, err)
	}

	// no need to buffer this since raw network and bzip2 will
	// naturally buffer the input
	jsonin := json.NewDecoder(file)
	obj := Reddit{}
	lines := 0
	total := 0
	for jsonin.More() && lines < maxLines {
		lines++
		// decode an array value (Message)
		err := jsonin.Decode(&obj)
		if err != nil {
			log.Fatalf("[%s] unable to unmarshal object: %s", prefix, err)
		}

		// turn into words
		words := text2words(obj.Body)

		// take only ones that are reasonable
		// and add to counts
		for _, word := range words {
			if len(word) > 4 && len(word) < 21 {
				counts[word]++
				total++
			}
		}

	}
	fi.Close()

	fo, err := os.Create(base)
	if err != nil {
		log.Fatalf("OH NO, unable to write: %s", err)
	}
	fout := gzip.NewWriter(fo)

	keys := make([]string, 0, len(counts))
	for k, _ := range counts {
		keys = append(keys, k)
	}
	sort.Strings(keys)
	for _, k := range keys {
		fout.Write([]byte(fmt.Sprintf("%s,%d\n", k, counts[k])))
	}
	fout.Close()
	fo.Close()
	log.Printf("[%s] DONE: wrote %s got %d unique words from %d", prefix, base, len(counts), total)
}

func worker(id int, jobs <-chan string) {
	for j := range jobs {
		doit(fmt.Sprintf("%d:%s", id, j), j)
	}
	wg.Done()
}

func main() {
	numWorkers := runtime.NumCPU()

	flag.Parse()
	args := flag.Args()

	jobs := make(chan string, len(args))

	for w := 1; w <= numWorkers; w++ {
		wg.Add(1)
		go worker(w, jobs)
	}

	for _, arg := range args {
		log.Printf("[MASTER}: adding %s", arg)
		jobs <- arg
	}
	close(jobs)
	wg.Wait()
	log.Printf("[MASTER]: done")
}
<commit_msg>close gzip reader<commit_after>package main

// this takes a list of gzipped Reddit comment-body files
// and returns a frequency count of words
// as a gzipped CSV file
import (
	"compress/gzip"
	"encoding/json"
	"flag"
	"fmt"
	"log"
	"os"
	"path/filepath"
	"runtime"
	"sort"
	"strings"
	"sync"
)

// freqCount is mapping of string->count
type freqCount map[string]int

// make a new counter with some minor preallocation
//  each month has about 2.2M uniques
func newFreqCount() freqCount {
	return make(freqCount, 3000000)
}

// globals used for worker queues and global counts
var wg sync.WaitGroup

// Reddit is struct used to unmarshal the reddit comment
type Reddit struct {
	Body string `json:"body"`
}

// isAZ returns true if input is only consists of letters a-z (case SENSITIVE)
func isAZ(s string) bool {
	// NO: for _, ch := range s {
	//   This iterates over UTF-8 characters
	slen := len(s)
	for i := 0; i < slen; i++ {
		ch := s[i]
		if ch < 'a' || ch > 'z' {
			return false
		}
	}
	return true
}

// text2words takes raw text and extracts English-list words
//   split on whitespace
//   remove starting or ending  `'".,:;-()`
//   lower case
//   check if all a-z
//
// this doesn't catch everything, but that's ok for this purpose.
//
func text2words(raw string) []string {
	parts := strings.Fields(raw)
	out := make([]string, 0, len(parts))
	for _, word := range parts {
		word = strings.Trim(word, `'".,:;-()`)
		word = strings.ToLower(word)
		if isAZ(word) {
			out = append(out, word)
		}
	}
	return out
}

// doit does the following
//   opens a gizpped up reddit file
//   extracts words
//   updates counts
//
// Every 1M lines, it updates the global freq count
//
func doit(prefix, filename string) {
	const maxLines = 1 << 31
	const suffix = "-body.json.gz"
	// generate outputfile name
	// blah/RC_2015-06-body.json.gz --> RC_2015-06-counts.csv.gz
	base := filepath.Base(filename)
	base = base[0:len(base)-len(suffix)] + "-counts.csv.gz"

	log.Printf("[%s] starting. Output file %s", prefix, base)
	counts := newFreqCount()
	fi, err := os.Open(filename)
	if err != nil {
		log.Fatalf("[%s]: %s", prefix, err)
	}

	fizip, err := gzip.NewReader(fi)
	if err != nil {
		log.Fatalf("[%s]: gzip error %s", prefix, err)
	}

	// no need to buffer this since raw network and bzip2 will
	// naturally buffer the input
	jsonin := json.NewDecoder(fizip)
	obj := Reddit{}
	lines := 0
	total := 0
	for jsonin.More() && lines < maxLines {
		lines++
		// decode an array value (Message)
		err := jsonin.Decode(&obj)
		if err != nil {
			log.Fatalf("[%s] unable to unmarshal object: %s", prefix, err)
		}

		// turn into words
		words := text2words(obj.Body)

		// take only ones that are reasonable
		// and add to counts
		for _, word := range words {
			if len(word) > 4 && len(word) < 21 {
				counts[word]++
				total++
			}
		}

	}
	fizip.Close()
	fi.Close()

	fo, err := os.Create(base)
	if err != nil {
		log.Fatalf("OH NO, unable to write: %s", err)
	}
	fout := gzip.NewWriter(fo)

	keys := make([]string, 0, len(counts))
	for k, _ := range counts {
		keys = append(keys, k)
	}
	sort.Strings(keys)
	for _, k := range keys {
		fout.Write([]byte(fmt.Sprintf("%s,%d\n", k, counts[k])))
	}
	fout.Close()
	fo.Close()
	log.Printf("[%s] DONE: wrote %s got %d unique words from %d", prefix, base, len(counts), total)
}

func worker(id int, jobs <-chan string) {
	for j := range jobs {
		doit(fmt.Sprintf("%d:%s", id, j), j)
	}
	wg.Done()
}

func main() {
	numWorkers := runtime.NumCPU()

	flag.Parse()
	args := flag.Args()

	jobs := make(chan string, len(args))

	for w := 1; w <= numWorkers; w++ {
		wg.Add(1)
		go worker(w, jobs)
	}

	for _, arg := range args {
		log.Printf("[MASTER}: adding %s", arg)
		jobs <- arg
	}
	close(jobs)
	wg.Wait()
	log.Printf("[MASTER]: done")
}
<|endoftext|>


<commit_before>/**
 * (C) Copyright 2014, Deft Labs
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at:
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package dlshared

import (
	"fmt"
	"strings"
	"reflect"
	"runtime"
)

// Return the function name. This returns packageName.functionName (if available).
// Initial code came from:
// http://stackoverflow.com/questions/7052693/how-to-get-the-name-of-a-function-in-go
//
// This method does not work if you have [ '(' || ')' || '*' ] in your file path. If you use this method
// add a unit test to confirm it works for you needs. If it does not work for your
// application, please reach out and let us know: https://github.com/deftlabs/dlshared/issues
//
// This will likely not work on Window$
func GetFunctionName(i interface{}) string {

	val := runtime.FuncForPC(reflect.ValueOf(i).Pointer()).Name()

	startFunc := strings.Index(val, "(")
	endFunc := strings.Index(val, ")")

	if startFunc == -1 {
		return removeFilePathIfPresent(val)
	}

	funcStr := removeFilePathIfPresent(val[startFunc+1:endFunc])

	if i := strings.LastIndex(funcStr, "."); i > -1 {
		funcStr = funcStr[i+1:len(funcStr)]
	}

	structStr := removeFilePathIfPresent(val[0:startFunc-1])

	return strings.Replace(fmt.Sprintf("%s.%s", structStr, funcStr), "*", "", -1)
}

func removeFilePathIfPresent(val string) string {
	if i := strings.LastIndex(val, "/"); i > -1 { val = val[i+1:len(val)] }

	return val
}

<commit_msg>removed erroneous package information... it isn't available.<commit_after>/**
 * (C) Copyright 2014, Deft Labs
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at:
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package dlshared

import (
	"fmt"
	"strings"
	"reflect"
	"runtime"
)

// Return the function name.
//
// Initial code came from:
// http://stackoverflow.com/questions/7052693/how-to-get-the-name-of-a-function-in-go
//
// This does not include parameter names/types or package, simply the name. If the function
// is attached to a struct, it returns the struct name.
//
// This method does not work if you have [ '(' || ')' || '*' ] in your file path. If you use this method
// add a unit test to confirm it works for you needs. If it does not work for your
// application, please reach out and let us know: https://github.com/deftlabs/dlshared/issues
func GetFunctionName(i interface{}) string {

	val := runtime.FuncForPC(reflect.ValueOf(i).Pointer()).Name()

	// Strip the path if present.
	if pathIdx := strings.Index(val, "."); pathIdx != -1 { val = val[pathIdx+1:len(val)] }

	// Remove the pointer if present.
	if strings.HasPrefix(val, "*") { val = val[1:len(val)] }

	startFunc := strings.Index(val, "(")
	endFunc := strings.Index(val, ")")

	if startFunc == -1 { return removeFilePathIfPresent(val) }

	funcStr := removeFilePathIfPresent(val[startFunc+1:endFunc])

	if i := strings.LastIndex(funcStr, "."); i > -1 { funcStr = funcStr[i+1:len(funcStr)] }

	structStr := removeFilePathIfPresent(val[0:startFunc-1])

	return strings.Replace(fmt.Sprintf("%s.%s", structStr, funcStr), "*", "", -1)
}

func removeFilePathIfPresent(val string) string {
	if i := strings.LastIndex(val, "/"); i > -1 { val = val[i+1:len(val)] }

	return val
}

<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"log"
	"os"
	"strconv"
	"strings"

	"github.com/apex/go-apex"
)

func main() {
	apex.HandleFunc(func(event json.RawMessage, ctx *apex.Context) (interface{}, error) {
		var s Solution
		var rEvent Event
		xborbits := log.Ldate | log.Ltime | log.Lshortfile
		info := log.New(os.Stderr, "AdditionLog", xborbits)
		data, err := event.MarshalJSON()
		jdata := strings.Replace(string(data), `\"`, `"`, -1)
		json.Unmarshal([]byte(jdata), &rEvent)
		//info.Println("PlayerID: ", eType.Params.Querystring["id"])
		//info.Println("Headers:", eType.Params.Header["header1"], eType.Params.Header["header2"])
		info.Println("Path:", rEvent.Params.Path["pid2"])
		var n1 int
		var n2 int

		var nErr error
		for k, v := range rEvent.Params.Path {
			info.Printf("Path: %s %s\n", k, v)
			if k == "num1" {
				n1, nErr = strconv.Atoi(v)
				if nErr != nil {
					s.Message = fmt.Sprintf("%v", err)
					return s, nErr
				}
			}
			if k == "num2" {
				n2, nErr = strconv.Atoi(v)
				if nErr != nil {
					s.Message = fmt.Sprintf("%v", err)
					return s, nErr
				}
			}
		}
		if err != nil {
			s.Message = fmt.Sprintf("%v", err)
		}
		s.Numbers = append(s.Numbers, n1, n2)
		s.Sum = n1 + n2
		return s, nil
	})
}

//Solution contains info about the solved addition problem
type Solution struct {
	Numbers []int  `json:"numbers"`
	Sum     int    `json:"sum"`
	Message string `json:"message,omitempty"`
}

type Event struct {
	Body_json struct{} `json:"body-json"`
	Context   struct {
		Account_id                      string `json:"account-id"`
		API_id                          string `json:"api-id"`
		API_key                         string `json:"api-key"`
		Authorizer_principal_id         string `json:"authorizer-principal-id"`
		Caller                          string `json:"caller"`
		Cognito_authentication_provider string `json:"cognito-authentication-provider"`
		Cognito_authentication_type     string `json:"cognito-authentication-type"`
		Cognito_identity_id             string `json:"cognito-identity-id"`
		Cognito_identity_pool_id        string `json:"cognito-identity-pool-id"`
		HTTP_method                     string `json:"http-method"`
		Request_id                      string `json:"request-id"`
		Resource_id                     string `json:"resource-id"`
		Resource_path                   string `json:"resource-path"`
		Source_ip                       string `json:"source-ip"`
		Stage                           string `json:"stage"`
		User                            string `json:"user"`
		User_agent                      string `json:"user-agent"`
		User_arn                        string `json:"user-arn"`
	} `json:"context"`
	Params struct {
		Header      map[string]string `json:"header"`
		Path        map[string]string `json:"path"`
		Querystring map[string]string `json:"querystring"`
	} `json:"params"`
	Stage_variables map[string]string `json:"stage-variables"`
}
<commit_msg>remove logging<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"log"
	"os"
	"strconv"
	"strings"

	"github.com/apex/go-apex"
)

func main() {
	apex.HandleFunc(func(event json.RawMessage, ctx *apex.Context) (interface{}, error) {
		var s Solution
		var rEvent Event
		xborbits := log.Ldate | log.Ltime | log.Lshortfile
		info := log.New(os.Stderr, "AdditionLog", xborbits)
		data, err := event.MarshalJSON()
		jdata := strings.Replace(string(data), `\"`, `"`, -1)
		json.Unmarshal([]byte(jdata), &rEvent)
		info.Println("Path:", rEvent.Params.Path["pid2"])
		var n1 int
		var n2 int

		var nErr error
		for k, v := range rEvent.Params.Path {
			info.Printf("Path: %s %s\n", k, v)
			if k == "num1" {
				n1, nErr = strconv.Atoi(v)
				if nErr != nil {
					s.Message = fmt.Sprintf("%v", err)
					return s, nErr
				}
			}
			if k == "num2" {
				n2, nErr = strconv.Atoi(v)
				if nErr != nil {
					s.Message = fmt.Sprintf("%v", err)
					return s, nErr
				}
			}
		}
		if err != nil {
			s.Message = fmt.Sprintf("%v", err)
		}
		s.Numbers = append(s.Numbers, n1, n2)
		s.Sum = n1 + n2
		return s, nil
	})
}

//Solution contains info about the solved addition problem
type Solution struct {
	Numbers []int  `json:"numbers"`
	Sum     int    `json:"sum"`
	Message string `json:"message,omitempty"`
}

type Event struct {
	Body_json struct{} `json:"body-json"`
	Context   struct {
		Account_id                      string `json:"account-id"`
		API_id                          string `json:"api-id"`
		API_key                         string `json:"api-key"`
		Authorizer_principal_id         string `json:"authorizer-principal-id"`
		Caller                          string `json:"caller"`
		Cognito_authentication_provider string `json:"cognito-authentication-provider"`
		Cognito_authentication_type     string `json:"cognito-authentication-type"`
		Cognito_identity_id             string `json:"cognito-identity-id"`
		Cognito_identity_pool_id        string `json:"cognito-identity-pool-id"`
		HTTP_method                     string `json:"http-method"`
		Request_id                      string `json:"request-id"`
		Resource_id                     string `json:"resource-id"`
		Resource_path                   string `json:"resource-path"`
		Source_ip                       string `json:"source-ip"`
		Stage                           string `json:"stage"`
		User                            string `json:"user"`
		User_agent                      string `json:"user-agent"`
		User_arn                        string `json:"user-arn"`
	} `json:"context"`
	Params struct {
		Header      map[string]string `json:"header"`
		Path        map[string]string `json:"path"`
		Querystring map[string]string `json:"querystring"`
	} `json:"params"`
	Stage_variables map[string]string `json:"stage-variables"`
}
<|endoftext|>