<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Write out a file of a certain size, close it, then measure the performance
// of doing the following:
//
// 1.  Open the file.
// 2.  Read it from start to end with a configurable buffer size.
//
package main

import (
	"crypto/rand"
	"errors"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"sort"
	"time"
)

var fDir = flag.String("dir", "", "Directory within which to write the file.")
var fDuration = flag.Duration("duration", 5*time.Second, "How long to run.")
var fFileSize = flag.Int64("file_size", 1<<20, "Size of file to use.")
var fReadSize = flag.Int64("read_size", 1<<14, "Size of each call to read(2).")

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

type DurationSlice []time.Duration

func (p DurationSlice) Len() int           { return len(p) }
func (p DurationSlice) Less(i, j int) bool { return p[i] < p[j] }
func (p DurationSlice) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }

// REQUIRES: vals is sorted.
// REQUIRES: len(vals) > 0
// REQUIRES: 0 <= n <= 100
func percentile(
	vals DurationSlice,
	n int) (x time.Duration) {
	// Special cases.
	switch {
	case n == 0:
		x = vals[0]
		return

	case n == 100:
		x = vals[len(vals)-1]
		return
	}

	// Find the nearest, truncating (why not).
	index := int((float64(n) / 100) * float64(len(vals)))
	x = vals[index]

	return
}

func formatBytes(v float64) string {
	switch {
	case v >= 1<<30:
		return fmt.Sprintf("%.2f GiB", v/(1<<30))

	case v >= 1<<20:
		return fmt.Sprintf("%.2f MiB", v/(1<<20))

	case v >= 1<<10:
		return fmt.Sprintf("%.2f KiB", v/(1<<10))

	default:
		return fmt.Sprintf("%.2f bytes", v)
	}
}

////////////////////////////////////////////////////////////////////////
// main logic
////////////////////////////////////////////////////////////////////////

func run() (err error) {
	if *fDir == "" {
		err = errors.New("You must set --dir.")
		return
	}

	// Create a temporary file.
	log.Printf("Creating a temporary file in %s.", *fDir)

	f, err := ioutil.TempFile(*fDir, "sequential_read")
	if err != nil {
		err = fmt.Errorf("TempFile: %v", err)
		return
	}

	path := f.Name()

	// Make sure we clean it up later.
	defer func() {
		log.Printf("Deleting %s.", path)
		os.Remove(path)
	}()

	// Fill it with random content.
	log.Printf("Writing %d random bytes.", *fFileSize)
	_, err = io.Copy(f, io.LimitReader(rand.Reader, *fFileSize))
	if err != nil {
		err = fmt.Errorf("Copying random bytes: %v", err)
		return
	}

	// Finish off the file.
	err = f.Close()
	if err != nil {
		err = fmt.Errorf("Closing file: %v", err)
		return
	}

	// Run several iterations.
	log.Printf("Measuring for %v...", *fDuration)

	var fullFileRead DurationSlice
	var singleReadCall DurationSlice
	buf := make([]byte, *fReadSize)

	overallStartTime := time.Now()
	for len(fullFileRead) == 0 || time.Since(overallStartTime) < *fDuration {
		// Open the file for reading.
		f, err = os.Open(path)
		if err != nil {
			err = fmt.Errorf("Opening file: %v", err)
			return
		}

		// Read the whole thing.
		fileStartTime := time.Now()
		for err == nil {
			readStartTime := time.Now()
			_, err = f.Read(buf)
			singleReadCall = append(singleReadCall, time.Since(readStartTime))
		}

		fullFileRead = append(fullFileRead, time.Since(fileStartTime))

		switch {
		case err == io.EOF:
			err = nil

		case err != nil:
			err = fmt.Errorf("Reading: %v", err)
			return
		}

		// Close the file.
		err = f.Close()
		if err != nil {
			err = fmt.Errorf("Closing file after reading: %v", err)
			return
		}
	}

	sort.Sort(fullFileRead)
	sort.Sort(singleReadCall)

	// Report.
	ptiles := []int{50, 90, 98}

	reportSlice := func(
		name string,
		bytesPerObservation int64,
		observations DurationSlice) {
		fmt.Printf("\n%s:\n", name)
		for _, ptile := range ptiles {
			d := percentile(observations, ptile)
			seconds := float64(d) / float64(time.Second)
			bandwidthBytesPerSec := float64(bytesPerObservation) / seconds

			fmt.Printf(
				"  %02dth ptile: %10v (%s/s)\n",
				ptile,
				d,
				formatBytes(bandwidthBytesPerSec))
		}
	}

	reportSlice("Full-file read times", *fFileSize, fullFileRead)
	reportSlice("read(2) latencies", *fReadSize, singleReadCall)

	fmt.Println()

	return
}

func main() {
	log.SetFlags(log.Lmicroseconds | log.Lshortfile)
	flag.Parse()

	err := run()
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
}
<commit_msg>Report the number of reads.<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Write out a file of a certain size, close it, then measure the performance
// of doing the following:
//
// 1.  Open the file.
// 2.  Read it from start to end with a configurable buffer size.
//
package main

import (
	"crypto/rand"
	"errors"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"sort"
	"time"
)

var fDir = flag.String("dir", "", "Directory within which to write the file.")
var fDuration = flag.Duration("duration", 5*time.Second, "How long to run.")
var fFileSize = flag.Int64("file_size", 1<<20, "Size of file to use.")
var fReadSize = flag.Int64("read_size", 1<<14, "Size of each call to read(2).")

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

type DurationSlice []time.Duration

func (p DurationSlice) Len() int           { return len(p) }
func (p DurationSlice) Less(i, j int) bool { return p[i] < p[j] }
func (p DurationSlice) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }

// REQUIRES: vals is sorted.
// REQUIRES: len(vals) > 0
// REQUIRES: 0 <= n <= 100
func percentile(
	vals DurationSlice,
	n int) (x time.Duration) {
	// Special cases.
	switch {
	case n == 0:
		x = vals[0]
		return

	case n == 100:
		x = vals[len(vals)-1]
		return
	}

	// Find the nearest, truncating (why not).
	index := int((float64(n) / 100) * float64(len(vals)))
	x = vals[index]

	return
}

func formatBytes(v float64) string {
	switch {
	case v >= 1<<30:
		return fmt.Sprintf("%.2f GiB", v/(1<<30))

	case v >= 1<<20:
		return fmt.Sprintf("%.2f MiB", v/(1<<20))

	case v >= 1<<10:
		return fmt.Sprintf("%.2f KiB", v/(1<<10))

	default:
		return fmt.Sprintf("%.2f bytes", v)
	}
}

////////////////////////////////////////////////////////////////////////
// main logic
////////////////////////////////////////////////////////////////////////

func run() (err error) {
	if *fDir == "" {
		err = errors.New("You must set --dir.")
		return
	}

	// Create a temporary file.
	log.Printf("Creating a temporary file in %s.", *fDir)

	f, err := ioutil.TempFile(*fDir, "sequential_read")
	if err != nil {
		err = fmt.Errorf("TempFile: %v", err)
		return
	}

	path := f.Name()

	// Make sure we clean it up later.
	defer func() {
		log.Printf("Deleting %s.", path)
		os.Remove(path)
	}()

	// Fill it with random content.
	log.Printf("Writing %d random bytes.", *fFileSize)
	_, err = io.Copy(f, io.LimitReader(rand.Reader, *fFileSize))
	if err != nil {
		err = fmt.Errorf("Copying random bytes: %v", err)
		return
	}

	// Finish off the file.
	err = f.Close()
	if err != nil {
		err = fmt.Errorf("Closing file: %v", err)
		return
	}

	// Run several iterations.
	log.Printf("Measuring for %v...", *fDuration)

	var fullFileRead DurationSlice
	var singleReadCall DurationSlice
	buf := make([]byte, *fReadSize)

	overallStartTime := time.Now()
	for len(fullFileRead) == 0 || time.Since(overallStartTime) < *fDuration {
		// Open the file for reading.
		f, err = os.Open(path)
		if err != nil {
			err = fmt.Errorf("Opening file: %v", err)
			return
		}

		// Read the whole thing.
		fileStartTime := time.Now()
		for err == nil {
			readStartTime := time.Now()
			_, err = f.Read(buf)
			singleReadCall = append(singleReadCall, time.Since(readStartTime))
		}

		fullFileRead = append(fullFileRead, time.Since(fileStartTime))

		switch {
		case err == io.EOF:
			err = nil

		case err != nil:
			err = fmt.Errorf("Reading: %v", err)
			return
		}

		// Close the file.
		err = f.Close()
		if err != nil {
			err = fmt.Errorf("Closing file after reading: %v", err)
			return
		}
	}

	sort.Sort(fullFileRead)
	sort.Sort(singleReadCall)

	log.Printf(
		"Read the file %d times, using %d calls to read(2).",
		len(fullFileRead),
		len(singleReadCall))

	// Report.
	ptiles := []int{50, 90, 98}

	reportSlice := func(
		name string,
		bytesPerObservation int64,
		observations DurationSlice) {
		fmt.Printf("\n%s:\n", name)
		for _, ptile := range ptiles {
			d := percentile(observations, ptile)
			seconds := float64(d) / float64(time.Second)
			bandwidthBytesPerSec := float64(bytesPerObservation) / seconds

			fmt.Printf(
				"  %02dth ptile: %10v (%s/s)\n",
				ptile,
				d,
				formatBytes(bandwidthBytesPerSec))
		}
	}

	reportSlice("Full-file read times", *fFileSize, fullFileRead)
	reportSlice("read(2) latencies", *fReadSize, singleReadCall)

	fmt.Println()

	return
}

func main() {
	log.SetFlags(log.Lmicroseconds | log.Lshortfile)
	flag.Parse()

	err := run()
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>package common

import (
	gossh "code.google.com/p/go.crypto/ssh"
	"errors"
	"fmt"
	"github.com/mitchellh/multistep"
	"github.com/mitchellh/packer/communicator/ssh"
	"github.com/mitchellh/packer/packer"
	"log"
	"time"
)

// StepConnectSSH is a multistep Step implementation that waits for SSH
// to become available. It gets the connection information from a single
// configuration when creating the step.
//
// Uses:
//   ui packer.Ui
//
// Produces:
//   communicator packer.Communicator
type StepConnectSSH struct {
	// SSHAddress is a function that returns the TCP address to connect to
	// for SSH. This is a function so that you can query information
	// if necessary for this address.
	SSHAddress func(map[string]interface{}) (string, error)

	// SSHConfig is a function that returns the proper client configuration
	// for SSH access.
	SSHConfig func(map[string]interface{}) (*gossh.ClientConfig, error)

	// SSHWaitTimeout is the total timeout to wait for SSH to become available.
	SSHWaitTimeout time.Duration

	cancel bool
	comm   packer.Communicator
}

func (s *StepConnectSSH) Run(state map[string]interface{}) multistep.StepAction {
	ui := state["ui"].(packer.Ui)

	var comm packer.Communicator
	var err error

	waitDone := make(chan bool, 1)
	go func() {
		ui.Say("Waiting for SSH to become available...")
		comm, err = s.waitForSSH(state)
		waitDone <- true
	}()

	log.Printf("Waiting for SSH, up to timeout: %s", s.SSHWaitTimeout)
	timeout := time.After(s.SSHWaitTimeout)
WaitLoop:
	for {
		// Wait for either SSH to become available, a timeout to occur,
		// or an interrupt to come through.
		select {
		case <-waitDone:
			if err != nil {
				ui.Error(fmt.Sprintf("Error waiting for SSH: %s", err))
				return multistep.ActionHalt
			}

			ui.Say("Connected to SSH!")
			s.comm = comm
			state["communicator"] = comm
			break WaitLoop
		case <-timeout:
			ui.Error("Timeout waiting for SSH.")
			s.cancel = true
			return multistep.ActionHalt
		case <-time.After(1 * time.Second):
			if _, ok := state[multistep.StateCancelled]; ok {
				// The step sequence was cancelled, so cancel waiting for SSH
				// and just start the halting process.
				s.cancel = true
				log.Println("Interrupt detected, quitting waiting for SSH.")
				return multistep.ActionHalt
			}
		}
	}

	return multistep.ActionContinue
}

func (s *StepConnectSSH) Cleanup(map[string]interface{}) {
}

func (s *StepConnectSSH) waitForSSH(state map[string]interface{}) (packer.Communicator, error) {
	handshakeAttempts := 0

	var comm packer.Communicator
	for {
		time.Sleep(5 * time.Second)

		if s.cancel {
			log.Println("SSH wait cancelled. Exiting loop.")
			return nil, errors.New("SSH wait cancelled")
		}

		// First we request the TCP connection information
		address, err := s.SSHAddress(state)
		if err != nil {
			log.Printf("Error getting SSH address: %s", err)
			continue
		}

		// Retrieve the SSH configuration
		sshConfig, err := s.SSHConfig(state)
		if err != nil {
			log.Printf("Error getting SSH config: %s", err)
			continue
		}

		// Attempt to connect to SSH port
		connFunc := ssh.ConnectFunc("tcp", address, 5*time.Minute)
		nc, err := connFunc()
		if err != nil {
			log.Printf("TCP connection to SSH ip/port failed: %s", err)
			continue
		}
		nc.Close()

		// Then we attempt to connect via SSH
		config := &ssh.Config{
			Connection: connFunc,
			SSHConfig:  sshConfig,
		}

		comm, err = ssh.New(config)
		if err != nil {
			log.Printf("SSH handshake err: %s", err)

			handshakeAttempts += 1
			if handshakeAttempts < 10 {
				// Try to connect via SSH a handful of times
				continue
			}

			return nil, err
		}

		break
	}

	return comm, nil
}
<commit_msg>builder/common: only count handshake error if auth tried<commit_after>package common

import (
	gossh "code.google.com/p/go.crypto/ssh"
	"errors"
	"fmt"
	"github.com/mitchellh/multistep"
	"github.com/mitchellh/packer/communicator/ssh"
	"github.com/mitchellh/packer/packer"
	"log"
	"strings"
	"time"
)

// StepConnectSSH is a multistep Step implementation that waits for SSH
// to become available. It gets the connection information from a single
// configuration when creating the step.
//
// Uses:
//   ui packer.Ui
//
// Produces:
//   communicator packer.Communicator
type StepConnectSSH struct {
	// SSHAddress is a function that returns the TCP address to connect to
	// for SSH. This is a function so that you can query information
	// if necessary for this address.
	SSHAddress func(map[string]interface{}) (string, error)

	// SSHConfig is a function that returns the proper client configuration
	// for SSH access.
	SSHConfig func(map[string]interface{}) (*gossh.ClientConfig, error)

	// SSHWaitTimeout is the total timeout to wait for SSH to become available.
	SSHWaitTimeout time.Duration

	cancel bool
	comm   packer.Communicator
}

func (s *StepConnectSSH) Run(state map[string]interface{}) multistep.StepAction {
	ui := state["ui"].(packer.Ui)

	var comm packer.Communicator
	var err error

	waitDone := make(chan bool, 1)
	go func() {
		ui.Say("Waiting for SSH to become available...")
		comm, err = s.waitForSSH(state)
		waitDone <- true
	}()

	log.Printf("Waiting for SSH, up to timeout: %s", s.SSHWaitTimeout)
	timeout := time.After(s.SSHWaitTimeout)
WaitLoop:
	for {
		// Wait for either SSH to become available, a timeout to occur,
		// or an interrupt to come through.
		select {
		case <-waitDone:
			if err != nil {
				ui.Error(fmt.Sprintf("Error waiting for SSH: %s", err))
				return multistep.ActionHalt
			}

			ui.Say("Connected to SSH!")
			s.comm = comm
			state["communicator"] = comm
			break WaitLoop
		case <-timeout:
			ui.Error("Timeout waiting for SSH.")
			s.cancel = true
			return multistep.ActionHalt
		case <-time.After(1 * time.Second):
			if _, ok := state[multistep.StateCancelled]; ok {
				// The step sequence was cancelled, so cancel waiting for SSH
				// and just start the halting process.
				s.cancel = true
				log.Println("Interrupt detected, quitting waiting for SSH.")
				return multistep.ActionHalt
			}
		}
	}

	return multistep.ActionContinue
}

func (s *StepConnectSSH) Cleanup(map[string]interface{}) {
}

func (s *StepConnectSSH) waitForSSH(state map[string]interface{}) (packer.Communicator, error) {
	handshakeAttempts := 0

	var comm packer.Communicator
	for {
		time.Sleep(5 * time.Second)

		if s.cancel {
			log.Println("SSH wait cancelled. Exiting loop.")
			return nil, errors.New("SSH wait cancelled")
		}

		// First we request the TCP connection information
		address, err := s.SSHAddress(state)
		if err != nil {
			log.Printf("Error getting SSH address: %s", err)
			continue
		}

		// Retrieve the SSH configuration
		sshConfig, err := s.SSHConfig(state)
		if err != nil {
			log.Printf("Error getting SSH config: %s", err)
			continue
		}

		// Attempt to connect to SSH port
		connFunc := ssh.ConnectFunc("tcp", address, 5*time.Minute)
		nc, err := connFunc()
		if err != nil {
			log.Printf("TCP connection to SSH ip/port failed: %s", err)
			continue
		}
		nc.Close()

		// Then we attempt to connect via SSH
		config := &ssh.Config{
			Connection: connFunc,
			SSHConfig:  sshConfig,
		}

		comm, err = ssh.New(config)
		if err != nil {
			log.Printf("SSH handshake err: %s", err)

			// Only count this as an attempt if we were able to attempt
			// to authenticate. Note this is very brittle since it depends
			// on the string of the error... but I don't see any other way.
			if strings.Contains(err.Error(), "authenticate") {
				log.Printf("Detected authentication error. Increasing handshake attempts.")
				handshakeAttempts += 1
			}

			if handshakeAttempts < 10 {
				// Try to connect via SSH a handful of times
				continue
			}

			return nil, err
		}

		break
	}

	return comm, nil
}
<|endoftext|>


<commit_before>package mongodb

import (
	"fmt"
	"strings"
	"sync"
	"time"

	"github.com/hashicorp/vault/logical"
	"github.com/hashicorp/vault/logical/framework"
	"gopkg.in/mgo.v2"
)

func Factory(conf *logical.BackendConfig) (logical.Backend, error) {
	return Backend().Setup(conf)
}

func Backend() *framework.Backend {
	var b backend
	b.Backend = &framework.Backend{
		Help: strings.TrimSpace(backendHelp),

		Paths: []*framework.Path{
			pathConfigConnection(&b),
			pathConfigLease(&b),
			pathListRoles(&b),
			pathRoles(&b),
			pathCredsCreate(&b),
		},

		Secrets: []*framework.Secret{
			secretCreds(&b),
		},

		Clean: b.ResetSession,
	}

	return b.Backend
}

type backend struct {
	*framework.Backend

	session *mgo.Session
	lock    sync.Mutex
}

// Session returns the database connection.
func (b *backend) Session(s logical.Storage) (*mgo.Session, error) {
	b.lock.Lock()
	defer b.lock.Unlock()

	// If we already have a Session, we got it!
	if b.session != nil {
		return b.session, nil
	}

	// Otherwise, attempt to make connection
	connConfigJSON, err := s.Get("config/connection")
	if err != nil {
		return nil, err
	}
	if connConfigJSON == nil {
		return nil, fmt.Errorf("configure the MongoDB connection with config/connection first")
	}

	var connConfig connectionConfig
	if err := connConfigJSON.DecodeJSON(&connConfig); err != nil {
		return nil, err
	}

	dialInfo, err := parseMongoURI(connConfig.URI)
	if err != nil {
		return nil, err
	}

	b.session, err = mgo.DialWithInfo(dialInfo)
	if err != nil {
		return nil, err
	}
	b.session.SetSyncTimeout(1 * time.Minute)
	b.session.SetSocketTimeout(1 * time.Minute)

	return b.session, nil
}

// ResetSession forces a connection next time Session() is called.
func (b *backend) ResetSession() {
	b.lock.Lock()
	defer b.lock.Unlock()

	if b.session != nil {
		b.session.Close()
	}

	b.session = nil
}

// LeaseConfig returns the lease configuration
func (b *backend) LeaseConfig(s logical.Storage) (*configLease, error) {
	entry, err := s.Get("config/lease")
	if err != nil {
		return nil, err
	}
	if entry == nil {
		return nil, nil
	}

	var result configLease
	if err := entry.DecodeJSON(&result); err != nil {
		return nil, err
	}

	return &result, nil
}

const backendHelp = `
The mongodb backend dynamically generates MongoDB credentials.

After mounting this backend, configure it using the endpoints within
the "config/" path.
`
<commit_msg>mongodb secret backend: Verify existing Session is still working before reusing it<commit_after>package mongodb

import (
	"fmt"
	"strings"
	"sync"
	"time"

	"github.com/hashicorp/vault/logical"
	"github.com/hashicorp/vault/logical/framework"
	"gopkg.in/mgo.v2"
)

func Factory(conf *logical.BackendConfig) (logical.Backend, error) {
	return Backend().Setup(conf)
}

func Backend() *framework.Backend {
	var b backend
	b.Backend = &framework.Backend{
		Help: strings.TrimSpace(backendHelp),

		Paths: []*framework.Path{
			pathConfigConnection(&b),
			pathConfigLease(&b),
			pathListRoles(&b),
			pathRoles(&b),
			pathCredsCreate(&b),
		},

		Secrets: []*framework.Secret{
			secretCreds(&b),
		},

		Clean: b.ResetSession,
	}

	return b.Backend
}

type backend struct {
	*framework.Backend

	session *mgo.Session
	lock    sync.Mutex
}

// Session returns the database connection.
func (b *backend) Session(s logical.Storage) (*mgo.Session, error) {
	b.lock.Lock()
	defer b.lock.Unlock()

	if b.session != nil {
		if err := b.session.Ping(); err == nil {
			return b.session, nil
		}
		b.session.Close()
	}

	connConfigJSON, err := s.Get("config/connection")
	if err != nil {
		return nil, err
	}
	if connConfigJSON == nil {
		return nil, fmt.Errorf("configure the MongoDB connection with config/connection first")
	}

	var connConfig connectionConfig
	if err := connConfigJSON.DecodeJSON(&connConfig); err != nil {
		return nil, err
	}

	dialInfo, err := parseMongoURI(connConfig.URI)
	if err != nil {
		return nil, err
	}

	b.session, err = mgo.DialWithInfo(dialInfo)
	if err != nil {
		return nil, err
	}
	b.session.SetSyncTimeout(1 * time.Minute)
	b.session.SetSocketTimeout(1 * time.Minute)

	return b.session, nil
}

// ResetSession forces creation of a new connection next time Session() is called.
func (b *backend) ResetSession() {
	b.lock.Lock()
	defer b.lock.Unlock()

	if b.session != nil {
		b.session.Close()
	}

	b.session = nil
}

// LeaseConfig returns the lease configuration
func (b *backend) LeaseConfig(s logical.Storage) (*configLease, error) {
	entry, err := s.Get("config/lease")
	if err != nil {
		return nil, err
	}
	if entry == nil {
		return nil, nil
	}

	var result configLease
	if err := entry.DecodeJSON(&result); err != nil {
		return nil, err
	}

	return &result, nil
}

const backendHelp = `
The mongodb backend dynamically generates MongoDB credentials.

After mounting this backend, configure it using the endpoints within
the "config/" path.
`
<|endoftext|>


<commit_before>package linode

import (
	"context"
	"fmt"
	"strconv"

	"github.com/hashicorp/terraform/helper/schema"
	"github.com/linode/linodego"
)

func resourceLinodeDomain() *schema.Resource {
	return &schema.Resource{
		Create: resourceLinodeDomainCreate,
		Read:   resourceLinodeDomainRead,
		Update: resourceLinodeDomainUpdate,
		Delete: resourceLinodeDomainDelete,
		Exists: resourceLinodeDomainExists,
		Importer: &schema.ResourceImporter{
			State: schema.ImportStatePassthrough,
		},
		Schema: map[string]*schema.Schema{
			"domain": &schema.Schema{
				Type:        schema.TypeString,
				Description: "The domain this Domain represents. These must be unique in our system; you cannot have two Domains representing the same domain.",
				Required:    true,
			},
			"type": &schema.Schema{
				Type:         schema.TypeString,
				Description:  "If this Domain represents the authoritative source of information for the domain it describes, or if it is a read-only copy of a master (also called a slave).",
				InputDefault: "master",
				Required:     true,
				ForceNew:     true,
			},
			"group": &schema.Schema{
				Type:        schema.TypeString,
				Description: "The group this Domain belongs to. This is for display purposes only.",
				Optional:    true,
			},
			"status": &schema.Schema{
				Type:         schema.TypeString,
				Description:  "Used to control whether this Domain is currently being rendered.",
				Optional:     true,
				Computed:     true,
				InputDefault: "active",
			},
			"description": &schema.Schema{
				Type:        schema.TypeString,
				Description: "A description for this Domain. This is for display purposes only.",
				Optional:    true,
			},
			"master_ips": &schema.Schema{
				Type: schema.TypeSet,
				Elem: &schema.Schema{
					Type: schema.TypeString,
				},
				Description: "The IP addresses representing the master DNS for this Domain.",
				Optional:    true,
			},
			"axfr_ips": &schema.Schema{
				Type: schema.TypeSet,
				Elem: &schema.Schema{
					Type: schema.TypeString,
				},
				Description: "The list of IPs that may perform a zone transfer for this Domain. This is potentially dangerous, and should be set to an empty list unless you intend to use it.",
				Optional:    true,
			},
			"ttl_sec": &schema.Schema{
				Type:        schema.TypeInt,
				Description: "'Time to Live' - the amount of time in seconds that this Domain's records may be cached by resolvers or other domain servers. Valid values are 300, 3600, 7200, 14400, 28800, 57600, 86400, 172800, 345600, 604800, 1209600, and 2419200 - any other value will be rounded to the nearest valid value.",
				Optional:    true,
			},
			"retry_sec": &schema.Schema{
				Type:        schema.TypeInt,
				Description: "The interval, in seconds, at which a failed refresh should be retried. Valid values are 300, 3600, 7200, 14400, 28800, 57600, 86400, 172800, 345600, 604800, 1209600, and 2419200 - any other value will be rounded to the nearest valid value.",
				Optional:    true,
			},
			"expire_sec": &schema.Schema{
				Type:        schema.TypeInt,
				Description: "The amount of time in seconds that may pass before this Domain is no longer authoritative. Valid values are 300, 3600, 7200, 14400, 28800, 57600, 86400, 172800, 345600, 604800, 1209600, and 2419200 - any other value will be rounded to the nearest valid value.",
				Optional:    true,
			},
			"refresh_sec": &schema.Schema{
				Type:        schema.TypeInt,
				Description: "The amount of time in seconds before this Domain should be refreshed. Valid values are 300, 3600, 7200, 14400, 28800, 57600, 86400, 172800, 345600, 604800, 1209600, and 2419200 - any other value will be rounded to the nearest valid value.",
				Optional:    true,
			},
			"soa_email": &schema.Schema{
				Type:        schema.TypeString,
				Description: "Start of Authority email address. This is required for master Domains.",
				Optional:    true,
			},
		},
	}
}

func resourceLinodeDomainExists(d *schema.ResourceData, meta interface{}) (bool, error) {
	client := meta.(linodego.Client)
	id, err := strconv.ParseInt(d.Id(), 10, 64)
	if err != nil {
		return false, fmt.Errorf("Error parsing Linode Domain ID %s as int: %s", d.Id(), err)
	}

	_, err = client.GetDomain(context.Background(), int(id))
	if err != nil {
		if lerr, ok := err.(*linodego.Error); ok && lerr.Code == 404 {
			d.SetId("")
			return false, nil
		}

		return false, fmt.Errorf("Error getting Linode Domain ID %s: %s", d.Id(), err)
	}
	return true, nil
}

func resourceLinodeDomainRead(d *schema.ResourceData, meta interface{}) error {
	client := meta.(linodego.Client)
	id, err := strconv.ParseInt(d.Id(), 10, 64)
	if err != nil {
		return fmt.Errorf("Error parsing Linode Domain ID %s as int: %s", d.Id(), err)
	}

	domain, err := client.GetDomain(context.Background(), int(id))

	if err != nil {
		return fmt.Errorf("Error finding the specified Linode Domain: %s", err)
	}

	d.Set("domain", domain.Domain)
	d.Set("type", domain.Type)
	d.Set("group", domain.Group)
	d.Set("status", domain.Status)
	d.Set("description", domain.Description)
	d.Set("master_ips", domain.MasterIPs)
	if len(domain.AXfrIPs) > 0 {
		d.Set("afxr_ips", domain.AXfrIPs)
	}
	d.Set("ttl_sec", domain.TTLSec)
	d.Set("retry_sec", domain.RetrySec)
	d.Set("expire_sec", domain.ExpireSec)
	d.Set("refresh_sec", domain.RefreshSec)
	d.Set("soa_email", domain.SOAEmail)

	return nil
}

func resourceLinodeDomainCreate(d *schema.ResourceData, meta interface{}) error {
	client, ok := meta.(linodego.Client)
	if !ok {
		return fmt.Errorf("Invalid Client when creating Linode Domain")
	}

	createOpts := linodego.DomainCreateOptions{
		Domain:      d.Get("domain").(string),
		Type:        linodego.DomainType(d.Get("type").(string)),
		Group:       d.Get("group").(string),
		Description: d.Get("description").(string),
		SOAEmail:    d.Get("soa_email").(string),
		RetrySec:    d.Get("retry_sec").(int),
		ExpireSec:   d.Get("expire_sec").(int),
		RefreshSec:  d.Get("refresh_sec").(int),
		TTLSec:      d.Get("ttl_sec").(int),
	}

	if v, ok := d.GetOk("master_ips"); ok {
		var masterIPS []string
		for _, ip := range v.([]interface{}) {
			masterIPS = append(masterIPS, ip.(string))
		}

		createOpts.MasterIPs = masterIPS
	}

	if v, ok := d.GetOk("axfr_ips"); ok {
		var AXfrIPs []string
		for _, ip := range v.([]interface{}) {
			AXfrIPs = append(AXfrIPs, ip.(string))
		}

		createOpts.AXfrIPs = AXfrIPs
	}

	domain, err := client.CreateDomain(context.Background(), createOpts)
	if err != nil {
		return fmt.Errorf("Error creating a Linode Domain: %s", err)
	}
	d.SetId(fmt.Sprintf("%d", domain.ID))

	return resourceLinodeDomainRead(d, meta)
}

func resourceLinodeDomainUpdate(d *schema.ResourceData, meta interface{}) error {
	client := meta.(linodego.Client)

	id, err := strconv.ParseInt(d.Id(), 10, 64)
	if err != nil {
		return fmt.Errorf("Error parsing Linode Domain id %s as int: %s", d.Id(), err)
	}

	updateOpts := linodego.DomainUpdateOptions{
		Domain:      d.Get("domain").(string),
		Status:      linodego.DomainStatus(d.Get("status").(string)),
		Group:       d.Get("group").(string),
		Description: d.Get("description").(string),
		SOAEmail:    d.Get("soa_email").(string),
		RetrySec:    d.Get("retry_sec").(int),
		ExpireSec:   d.Get("expire_sec").(int),
		RefreshSec:  d.Get("refresh_sec").(int),
		TTLSec:      d.Get("ttl_sec").(int),
	}

	if v, ok := d.GetOk("master_ips"); ok {
		var masterIPS []string
		for _, ip := range v.([]interface{}) {
			masterIPS = append(masterIPS, ip.(string))
		}

		updateOpts.MasterIPs = masterIPS
	}

	if v, ok := d.GetOk("axfr_ips"); ok {
		var AXfrIPs []string
		for _, ip := range v.([]interface{}) {
			AXfrIPs = append(AXfrIPs, ip.(string))
		}

		updateOpts.AXfrIPs = AXfrIPs
	}

	_, err = client.UpdateDomain(context.Background(), int(id), updateOpts)
	if err != nil {
		return fmt.Errorf("Error updating Linode Domain %d: %s", id, err)
	}
	return resourceLinodeDomainRead(d, meta)
}

func resourceLinodeDomainDelete(d *schema.ResourceData, meta interface{}) error {
	client := meta.(linodego.Client)
	id, err := strconv.ParseInt(d.Id(), 10, 64)
	if err != nil {
		return fmt.Errorf("Error parsing Linode Domain id %s as int", d.Id())
	}
	err = client.DeleteDomain(context.Background(), int(id))
	if err != nil {
		return fmt.Errorf("Error deleting Linode Domain %d: %s", id, err)
	}
	d.SetId("")

	return nil
}
<commit_msg>check for errors when d.Set is used on TypeSet fields<commit_after>package linode

import (
	"context"
	"fmt"
	"strconv"

	"github.com/hashicorp/terraform/helper/schema"
	"github.com/linode/linodego"
)

func resourceLinodeDomain() *schema.Resource {
	return &schema.Resource{
		Create: resourceLinodeDomainCreate,
		Read:   resourceLinodeDomainRead,
		Update: resourceLinodeDomainUpdate,
		Delete: resourceLinodeDomainDelete,
		Exists: resourceLinodeDomainExists,
		Importer: &schema.ResourceImporter{
			State: schema.ImportStatePassthrough,
		},
		Schema: map[string]*schema.Schema{
			"domain": &schema.Schema{
				Type:        schema.TypeString,
				Description: "The domain this Domain represents. These must be unique in our system; you cannot have two Domains representing the same domain.",
				Required:    true,
			},
			"type": &schema.Schema{
				Type:         schema.TypeString,
				Description:  "If this Domain represents the authoritative source of information for the domain it describes, or if it is a read-only copy of a master (also called a slave).",
				InputDefault: "master",
				Required:     true,
				ForceNew:     true,
			},
			"group": &schema.Schema{
				Type:        schema.TypeString,
				Description: "The group this Domain belongs to. This is for display purposes only.",
				Optional:    true,
			},
			"status": &schema.Schema{
				Type:         schema.TypeString,
				Description:  "Used to control whether this Domain is currently being rendered.",
				Optional:     true,
				Computed:     true,
				InputDefault: "active",
			},
			"description": &schema.Schema{
				Type:        schema.TypeString,
				Description: "A description for this Domain. This is for display purposes only.",
				Optional:    true,
			},
			"master_ips": &schema.Schema{
				Type: schema.TypeSet,
				Elem: &schema.Schema{
					Type: schema.TypeString,
				},
				Description: "The IP addresses representing the master DNS for this Domain.",
				Optional:    true,
			},
			"axfr_ips": &schema.Schema{
				Type: schema.TypeSet,
				Elem: &schema.Schema{
					Type: schema.TypeString,
				},
				Description: "The list of IPs that may perform a zone transfer for this Domain. This is potentially dangerous, and should be set to an empty list unless you intend to use it.",
				Optional:    true,
			},
			"ttl_sec": &schema.Schema{
				Type:        schema.TypeInt,
				Description: "'Time to Live' - the amount of time in seconds that this Domain's records may be cached by resolvers or other domain servers. Valid values are 300, 3600, 7200, 14400, 28800, 57600, 86400, 172800, 345600, 604800, 1209600, and 2419200 - any other value will be rounded to the nearest valid value.",
				Optional:    true,
			},
			"retry_sec": &schema.Schema{
				Type:        schema.TypeInt,
				Description: "The interval, in seconds, at which a failed refresh should be retried. Valid values are 300, 3600, 7200, 14400, 28800, 57600, 86400, 172800, 345600, 604800, 1209600, and 2419200 - any other value will be rounded to the nearest valid value.",
				Optional:    true,
			},
			"expire_sec": &schema.Schema{
				Type:        schema.TypeInt,
				Description: "The amount of time in seconds that may pass before this Domain is no longer authoritative. Valid values are 300, 3600, 7200, 14400, 28800, 57600, 86400, 172800, 345600, 604800, 1209600, and 2419200 - any other value will be rounded to the nearest valid value.",
				Optional:    true,
			},
			"refresh_sec": &schema.Schema{
				Type:        schema.TypeInt,
				Description: "The amount of time in seconds before this Domain should be refreshed. Valid values are 300, 3600, 7200, 14400, 28800, 57600, 86400, 172800, 345600, 604800, 1209600, and 2419200 - any other value will be rounded to the nearest valid value.",
				Optional:    true,
			},
			"soa_email": &schema.Schema{
				Type:        schema.TypeString,
				Description: "Start of Authority email address. This is required for master Domains.",
				Optional:    true,
			},
		},
	}
}

func resourceLinodeDomainExists(d *schema.ResourceData, meta interface{}) (bool, error) {
	client := meta.(linodego.Client)
	id, err := strconv.ParseInt(d.Id(), 10, 64)
	if err != nil {
		return false, fmt.Errorf("Error parsing Linode Domain ID %s as int: %s", d.Id(), err)
	}

	_, err = client.GetDomain(context.Background(), int(id))
	if err != nil {
		if lerr, ok := err.(*linodego.Error); ok && lerr.Code == 404 {
			d.SetId("")
			return false, nil
		}

		return false, fmt.Errorf("Error getting Linode Domain ID %s: %s", d.Id(), err)
	}
	return true, nil
}

func resourceLinodeDomainRead(d *schema.ResourceData, meta interface{}) error {
	client := meta.(linodego.Client)
	id, err := strconv.ParseInt(d.Id(), 10, 64)
	if err != nil {
		return fmt.Errorf("Error parsing Linode Domain ID %s as int: %s", d.Id(), err)
	}

	domain, err := client.GetDomain(context.Background(), int(id))

	if err != nil {
		return fmt.Errorf("Error finding the specified Linode Domain: %s", err)
	}

	d.Set("domain", domain.Domain)
	d.Set("type", domain.Type)
	d.Set("group", domain.Group)
	d.Set("status", domain.Status)
	d.Set("description", domain.Description)
	if err = d.Set("master_ips", domain.MasterIPs); err != nil {
		return fmt.Errorf("Error setting master_ips: %s", err)
	}
	if len(domain.AXfrIPs) > 0 {
		if err = d.Set("afxr_ips", domain.AXfrIPs); err != nil {
			return fmt.Errorf("Error setting axfr_ips: %s", err)
		}
	}
	d.Set("ttl_sec", domain.TTLSec)
	d.Set("retry_sec", domain.RetrySec)
	d.Set("expire_sec", domain.ExpireSec)
	d.Set("refresh_sec", domain.RefreshSec)
	d.Set("soa_email", domain.SOAEmail)

	return nil
}

func resourceLinodeDomainCreate(d *schema.ResourceData, meta interface{}) error {
	client, ok := meta.(linodego.Client)
	if !ok {
		return fmt.Errorf("Invalid Client when creating Linode Domain")
	}

	createOpts := linodego.DomainCreateOptions{
		Domain:      d.Get("domain").(string),
		Type:        linodego.DomainType(d.Get("type").(string)),
		Group:       d.Get("group").(string),
		Description: d.Get("description").(string),
		SOAEmail:    d.Get("soa_email").(string),
		RetrySec:    d.Get("retry_sec").(int),
		ExpireSec:   d.Get("expire_sec").(int),
		RefreshSec:  d.Get("refresh_sec").(int),
		TTLSec:      d.Get("ttl_sec").(int),
	}

	if v, ok := d.GetOk("master_ips"); ok {
		var masterIPS []string
		for _, ip := range v.([]interface{}) {
			masterIPS = append(masterIPS, ip.(string))
		}

		createOpts.MasterIPs = masterIPS
	}

	if v, ok := d.GetOk("axfr_ips"); ok {
		var AXfrIPs []string
		for _, ip := range v.([]interface{}) {
			AXfrIPs = append(AXfrIPs, ip.(string))
		}

		createOpts.AXfrIPs = AXfrIPs
	}

	domain, err := client.CreateDomain(context.Background(), createOpts)
	if err != nil {
		return fmt.Errorf("Error creating a Linode Domain: %s", err)
	}
	d.SetId(fmt.Sprintf("%d", domain.ID))

	return resourceLinodeDomainRead(d, meta)
}

func resourceLinodeDomainUpdate(d *schema.ResourceData, meta interface{}) error {
	client := meta.(linodego.Client)

	id, err := strconv.ParseInt(d.Id(), 10, 64)
	if err != nil {
		return fmt.Errorf("Error parsing Linode Domain id %s as int: %s", d.Id(), err)
	}

	updateOpts := linodego.DomainUpdateOptions{
		Domain:      d.Get("domain").(string),
		Status:      linodego.DomainStatus(d.Get("status").(string)),
		Group:       d.Get("group").(string),
		Description: d.Get("description").(string),
		SOAEmail:    d.Get("soa_email").(string),
		RetrySec:    d.Get("retry_sec").(int),
		ExpireSec:   d.Get("expire_sec").(int),
		RefreshSec:  d.Get("refresh_sec").(int),
		TTLSec:      d.Get("ttl_sec").(int),
	}

	if v, ok := d.GetOk("master_ips"); ok {
		var masterIPS []string
		for _, ip := range v.([]interface{}) {
			masterIPS = append(masterIPS, ip.(string))
		}

		updateOpts.MasterIPs = masterIPS
	}

	if v, ok := d.GetOk("axfr_ips"); ok {
		var AXfrIPs []string
		for _, ip := range v.([]interface{}) {
			AXfrIPs = append(AXfrIPs, ip.(string))
		}

		updateOpts.AXfrIPs = AXfrIPs
	}

	_, err = client.UpdateDomain(context.Background(), int(id), updateOpts)
	if err != nil {
		return fmt.Errorf("Error updating Linode Domain %d: %s", id, err)
	}
	return resourceLinodeDomainRead(d, meta)
}

func resourceLinodeDomainDelete(d *schema.ResourceData, meta interface{}) error {
	client := meta.(linodego.Client)
	id, err := strconv.ParseInt(d.Id(), 10, 64)
	if err != nil {
		return fmt.Errorf("Error parsing Linode Domain id %s as int", d.Id())
	}
	err = client.DeleteDomain(context.Background(), int(id))
	if err != nil {
		return fmt.Errorf("Error deleting Linode Domain %d: %s", id, err)
	}
	d.SetId("")

	return nil
}
<|endoftext|>


<commit_before>package image

import (
	"context"
	"crypto/rand"
	"crypto/sha256"
	"encoding/hex"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"

	"github.com/docker/cli/cli/command"
	cliconfig "github.com/docker/cli/cli/config"
	"github.com/docker/docker/api/types/versions"
	"github.com/moby/buildkit/session"
	"github.com/pkg/errors"
)

const clientSessionRemote = "client-session"

func isSessionSupported(dockerCli command.Cli, forStream bool) bool {
	if !forStream && versions.GreaterThanOrEqualTo(dockerCli.Client().ClientVersion(), "1.39") {
		return true
	}
	return dockerCli.ServerInfo().HasExperimental && versions.GreaterThanOrEqualTo(dockerCli.Client().ClientVersion(), "1.31")
}

func trySession(dockerCli command.Cli, contextDir string, forStream bool) (*session.Session, error) {
	var s *session.Session
	if isSessionSupported(dockerCli, forStream) {
		sharedKey, err := getBuildSharedKey(contextDir)
		if err != nil {
			return nil, errors.Wrap(err, "failed to get build shared key")
		}
		s, err = session.NewSession(context.Background(), filepath.Base(contextDir), sharedKey)
		if err != nil {
			return nil, errors.Wrap(err, "failed to create session")
		}
	}
	return s, nil
}

func getBuildSharedKey(dir string) (string, error) {
	// build session is hash of build dir with node based randomness
	s := sha256.Sum256([]byte(fmt.Sprintf("%s:%s", tryNodeIdentifier(), dir)))
	return hex.EncodeToString(s[:]), nil
}

func tryNodeIdentifier() string {
	out := cliconfig.Dir() // return config dir as default on permission error
	if err := os.MkdirAll(cliconfig.Dir(), 0700); err == nil {
		sessionFile := filepath.Join(cliconfig.Dir(), ".buildNodeID")
		if _, err := os.Lstat(sessionFile); err != nil {
			if os.IsNotExist(err) { // create a new file with stored randomness
				b := make([]byte, 32)
				if _, err := rand.Read(b); err != nil {
					return out
				}
				if err := ioutil.WriteFile(sessionFile, []byte(hex.EncodeToString(b)), 0600); err != nil {
					return out
				}
			}
		}

		dt, err := ioutil.ReadFile(sessionFile)
		if err == nil {
			return string(dt)
		}
	}
	return out
}
<commit_msg>cli/command/image/build_session.go:133:45: getBuildSharedKey - result 1 (error) is always nil (unparam)<commit_after>package image

import (
	"context"
	"crypto/rand"
	"crypto/sha256"
	"encoding/hex"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"

	"github.com/docker/cli/cli/command"
	cliconfig "github.com/docker/cli/cli/config"
	"github.com/docker/docker/api/types/versions"
	"github.com/moby/buildkit/session"
	"github.com/pkg/errors"
)

const clientSessionRemote = "client-session"

func isSessionSupported(dockerCli command.Cli, forStream bool) bool {
	if !forStream && versions.GreaterThanOrEqualTo(dockerCli.Client().ClientVersion(), "1.39") {
		return true
	}
	return dockerCli.ServerInfo().HasExperimental && versions.GreaterThanOrEqualTo(dockerCli.Client().ClientVersion(), "1.31")
}

func trySession(dockerCli command.Cli, contextDir string, forStream bool) (*session.Session, error) {
	if !isSessionSupported(dockerCli, forStream) {
		return nil, nil
	}
	sharedKey := getBuildSharedKey(contextDir)
	s, err := session.NewSession(context.Background(), filepath.Base(contextDir), sharedKey)
	if err != nil {
		return nil, errors.Wrap(err, "failed to create session")
	}
	return s, nil
}

func getBuildSharedKey(dir string) string {
	// build session is hash of build dir with node based randomness
	s := sha256.Sum256([]byte(fmt.Sprintf("%s:%s", tryNodeIdentifier(), dir)))
	return hex.EncodeToString(s[:])
}

func tryNodeIdentifier() string {
	out := cliconfig.Dir() // return config dir as default on permission error
	if err := os.MkdirAll(cliconfig.Dir(), 0700); err == nil {
		sessionFile := filepath.Join(cliconfig.Dir(), ".buildNodeID")
		if _, err := os.Lstat(sessionFile); err != nil {
			if os.IsNotExist(err) { // create a new file with stored randomness
				b := make([]byte, 32)
				if _, err := rand.Read(b); err != nil {
					return out
				}
				if err := ioutil.WriteFile(sessionFile, []byte(hex.EncodeToString(b)), 0600); err != nil {
					return out
				}
			}
		}

		dt, err := ioutil.ReadFile(sessionFile)
		if err == nil {
			return string(dt)
		}
	}
	return out
}
<|endoftext|>


<commit_before>package client

import (
	"context"
	"errors"

	k8serrors "k8s.io/apimachinery/pkg/api/errors"

	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/runtime/serializer"
	"k8s.io/client-go/rest"

	"github.com/coreos-inc/alm/apis/alphacatalogentry/v1alpha1"
)

type AlphaCatalogEntryInterface interface {
	UpdateEntry(csv *v1alpha1.AlphaCatalogEntry) (*v1alpha1.AlphaCatalogEntry, error)
}

type AlphaCatalogEntryClient struct {
	*rest.RESTClient
}

var _ AlphaCatalogEntryInterface = &AlphaCatalogEntryClient{}

// NewAlphaCatalogEntryClient creates a client that can interact with the AlphaCatalogEntry resource in k8s api
func NewAlphaCatalogEntryClient(kubeconfig string) (client *AlphaCatalogEntryClient, err error) {
	var config *rest.Config
	config, err = getConfig(kubeconfig)
	if err != nil {
		return
	}

	scheme := runtime.NewScheme()
	if err = v1alpha1.AddToScheme(scheme); err != nil {
		return
	}

	config.GroupVersion = &v1alpha1.SchemeGroupVersion
	config.APIPath = "/apis"
	config.ContentType = runtime.ContentTypeJSON
	config.NegotiatedSerializer = serializer.DirectCodecFactory{CodecFactory: serializer.NewCodecFactory(scheme)}

	var restClient *rest.RESTClient
	restClient, err = rest.RESTClientFor(config)
	if err != nil {
		return
	}
	return &AlphaCatalogEntryClient{restClient}, nil
}

func (c *AlphaCatalogEntryClient) UpdateEntry(in *v1alpha1.AlphaCatalogEntry) (result *v1alpha1.AlphaCatalogEntry, err error) {
	result = &v1alpha1.AlphaCatalogEntry{}
	err = c.RESTClient.Post().Context(context.TODO()).
		Namespace(in.Namespace).
		Resource(v1alpha1.AlphaCatalogEntryCRDName).
		Name(in.Name).
		Body(in).
		Do().
		Into(result)
	if err == nil {
		return
	}
	if k8serrors.IsAlreadyExists(err) {
		err = c.RESTClient.Put().Context(context.TODO()).
			Namespace(in.Namespace).
			Resource(v1alpha1.AlphaCatalogEntryCRDName).
			Name(in.Name).
			Body(in).
			Do().
			Into(result)
		if err != nil {
			err = errors.New("failed to update CR status: " + err.Error())
		}
	}
	return
}
<commit_msg>Add logging to alphacatalogclient<commit_after>package client

import (
	"context"
	"errors"

	log "github.com/sirupsen/logrus"
	k8serrors "k8s.io/apimachinery/pkg/api/errors"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/runtime/serializer"
	"k8s.io/client-go/rest"

	"github.com/coreos-inc/alm/apis/alphacatalogentry/v1alpha1"
)

type AlphaCatalogEntryInterface interface {
	UpdateEntry(csv *v1alpha1.AlphaCatalogEntry) (*v1alpha1.AlphaCatalogEntry, error)
}

type AlphaCatalogEntryClient struct {
	*rest.RESTClient
}

var _ AlphaCatalogEntryInterface = &AlphaCatalogEntryClient{}

// NewAlphaCatalogEntryClient creates a client that can interact with the AlphaCatalogEntry resource in k8s api
func NewAlphaCatalogEntryClient(kubeconfig string) (client *AlphaCatalogEntryClient, err error) {
	var config *rest.Config
	config, err = getConfig(kubeconfig)
	if err != nil {
		return
	}

	scheme := runtime.NewScheme()
	if err = v1alpha1.AddToScheme(scheme); err != nil {
		return
	}

	config.GroupVersion = &v1alpha1.SchemeGroupVersion
	config.APIPath = "/apis"
	config.ContentType = runtime.ContentTypeJSON
	config.NegotiatedSerializer = serializer.DirectCodecFactory{CodecFactory: serializer.NewCodecFactory(scheme)}

	var restClient *rest.RESTClient
	restClient, err = rest.RESTClientFor(config)
	if err != nil {
		return
	}
	return &AlphaCatalogEntryClient{restClient}, nil
}

func (c *AlphaCatalogEntryClient) UpdateEntry(in *v1alpha1.AlphaCatalogEntry) (result *v1alpha1.AlphaCatalogEntry, err error) {
	result = &v1alpha1.AlphaCatalogEntry{}
	log.Info(in)
	err = c.RESTClient.Post().Context(context.TODO()).
		Namespace(in.Namespace).
		Resource(v1alpha1.AlphaCatalogEntryCRDName).
		Name(in.Name).
		Body(in).
		Do().
		Into(result)
	if err == nil {
		return
	}
	if k8serrors.IsAlreadyExists(err) {
		err = c.RESTClient.Put().Context(context.TODO()).
			Namespace(in.Namespace).
			Resource(v1alpha1.AlphaCatalogEntryCRDName).
			Name(in.Name).
			Body(in).
			Do().
			Into(result)
		if err != nil {
			err = errors.New("failed to update CR status: " + err.Error())
		}
	}
	return
}
<|endoftext|>


<commit_before>package logs

import (
	"bufio"
	"encoding/json"
	"fmt"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/pkg/errors"
	"github.com/stitchfix/flotilla-os/config"
	"github.com/stitchfix/flotilla-os/state"
	"io"
	"log"
	"net/http"
	"os"
	"time"
)

//
// EKSS3LogsClient corresponds with the aws logs driver
// for ECS and returns logs for runs
//
type EKSS3LogsClient struct {
	logRetentionInDays int64
	logNamespace       string
	s3Client           *s3.S3
	s3Bucket           string
	s3BucketRootDir    string
	logger             *log.Logger
}

type s3Log struct {
	Log    string    `json:"log"`
	Stream string    `json:"stream"`
	Time   time.Time `json:"time"`
}

//
// Name returns the name of the logs client
//
func (lc *EKSS3LogsClient) Name() string {
	return "eks-s3"
}

//
// Initialize sets up the EKSS3LogsClient
//
func (lc *EKSS3LogsClient) Initialize(conf config.Config) error {
	confLogOptions := conf.GetStringMapString("eks.log.driver.options")

	awsRegion := confLogOptions["awslogs-region"]
	if len(awsRegion) == 0 {
		awsRegion = conf.GetString("aws_default_region")
	}

	if len(awsRegion) == 0 {
		return errors.Errorf(
			"EKSS3LogsClient needs one of [eks.log.driver.options.awslogs-region] or [aws_default_region] set in config")
	}

	flotillaMode := conf.GetString("flotilla_mode")
	if flotillaMode != "test" {
		sess := session.Must(session.NewSession(&aws.Config{
			Region: aws.String(awsRegion)}))

		lc.s3Client = s3.New(sess, aws.NewConfig().WithRegion(awsRegion))
	}

	s3BucketName := confLogOptions["s3_bucket_name"]

	if len(s3BucketName) == 0 {
		return errors.Errorf(
			"EKSS3LogsClient needs [eks.log.driver.options.s3_bucket_name] set in config")
	}
	lc.s3Bucket = s3BucketName

	s3BucketRootDir := confLogOptions["s3_bucket_root_dir"]

	if len(s3BucketRootDir) == 0 {
		return errors.Errorf(
			"EKSS3LogsClient needs [eks.log.driver.options.s3_bucket_root_dir] set in config")
	}
	lc.s3BucketRootDir = s3BucketRootDir

	lc.logger = log.New(os.Stderr, "[s3logs] ",
		log.Ldate|log.Ltime|log.Lshortfile)
	return nil
}

func (lc *EKSS3LogsClient) Logs(definition state.Definition, run state.Run, lastSeen *string) (string, *string, error) {
	return "", nil, errors.Errorf("EKSS3LogsClient does not support the Logs method.")
}

//
// Logs returns all logs from the log stream identified by handle since lastSeen
//
func (lc *EKSS3LogsClient) LogsText(definition state.Definition, run state.Run, w http.ResponseWriter) error {
	//Pod isn't there yet - dont return a 404
	if run.PodName == nil {
		return nil
	}
	s3DirName := lc.toS3DirName(run)

	// Get list of S3 objects in the run_id folder.
	result, err := lc.s3Client.ListObjects(&s3.ListObjectsInput{
		Bucket: aws.String(lc.s3Bucket),
		Prefix: aws.String(s3DirName),
	})

	if err != nil {
		return errors.Wrap(err, "problem getting logs")
	}

	// TODO: get latest file.
	if len(result.Contents) == 1 {
		s3Key := result.Contents[0].Key
		result, err := lc.s3Client.GetObject(&s3.GetObjectInput{
			Bucket: aws.String(lc.s3Bucket),
			Key:    aws.String(*s3Key),
		})

		if err != nil {
			return err
		}
		return lc.logsToMessage(result, w)
	}
	return nil
}

func (lc *EKSS3LogsClient) toS3DirName(run state.Run) string {
	return fmt.Sprintf("%s/%s", lc.s3BucketRootDir, run.RunID)
}

func (lc *EKSS3LogsClient) logsToMessage(result *s3.GetObjectOutput, w http.ResponseWriter) error {
	reader := bufio.NewReader(result.Body)
	for {
		line, err := reader.ReadBytes('\n')
		if err != nil {
			if err == io.EOF {
				err = nil
			}
			return err
		} else {
			var parsedLine s3Log
			err := json.Unmarshal(line, &parsedLine)
			if err != nil {
				return err
			}
			_, err = io.WriteString(w, parsedLine.Log)
			if err != nil {
				return err
			}
		}
	}
}
<commit_msg>Logs when pod name changes (#218)<commit_after>package logs

import (
	"bufio"
	"encoding/json"
	"fmt"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/pkg/errors"
	"github.com/stitchfix/flotilla-os/config"
	"github.com/stitchfix/flotilla-os/state"
	"io"
	"log"
	"net/http"
	"os"
	"strings"
	"time"
)

//
// EKSS3LogsClient corresponds with the aws logs driver
// for ECS and returns logs for runs
//
type EKSS3LogsClient struct {
	logRetentionInDays int64
	logNamespace       string
	s3Client           *s3.S3
	s3Bucket           string
	s3BucketRootDir    string
	logger             *log.Logger
}

type s3Log struct {
	Log    string    `json:"log"`
	Stream string    `json:"stream"`
	Time   time.Time `json:"time"`
}

//
// Name returns the name of the logs client
//
func (lc *EKSS3LogsClient) Name() string {
	return "eks-s3"
}

//
// Initialize sets up the EKSS3LogsClient
//
func (lc *EKSS3LogsClient) Initialize(conf config.Config) error {
	confLogOptions := conf.GetStringMapString("eks.log.driver.options")

	awsRegion := confLogOptions["awslogs-region"]
	if len(awsRegion) == 0 {
		awsRegion = conf.GetString("aws_default_region")
	}

	if len(awsRegion) == 0 {
		return errors.Errorf(
			"EKSS3LogsClient needs one of [eks.log.driver.options.awslogs-region] or [aws_default_region] set in config")
	}

	flotillaMode := conf.GetString("flotilla_mode")
	if flotillaMode != "test" {
		sess := session.Must(session.NewSession(&aws.Config{
			Region: aws.String(awsRegion)}))

		lc.s3Client = s3.New(sess, aws.NewConfig().WithRegion(awsRegion))
	}

	s3BucketName := confLogOptions["s3_bucket_name"]

	if len(s3BucketName) == 0 {
		return errors.Errorf(
			"EKSS3LogsClient needs [eks.log.driver.options.s3_bucket_name] set in config")
	}
	lc.s3Bucket = s3BucketName

	s3BucketRootDir := confLogOptions["s3_bucket_root_dir"]

	if len(s3BucketRootDir) == 0 {
		return errors.Errorf(
			"EKSS3LogsClient needs [eks.log.driver.options.s3_bucket_root_dir] set in config")
	}
	lc.s3BucketRootDir = s3BucketRootDir

	lc.logger = log.New(os.Stderr, "[s3logs] ",
		log.Ldate|log.Ltime|log.Lshortfile)
	return nil
}

func (lc *EKSS3LogsClient) Logs(definition state.Definition, run state.Run, lastSeen *string) (string, *string, error) {
	return "", nil, errors.Errorf("EKSS3LogsClient does not support the Logs method.")
}

//
// Logs returns all logs from the log stream identified by handle since lastSeen
//
func (lc *EKSS3LogsClient) LogsText(definition state.Definition, run state.Run, w http.ResponseWriter) error {
	//Pod isn't there yet - dont return a 404
	if run.PodName == nil {
		return nil
	}
	s3DirName := lc.toS3DirName(run)

	// Get list of S3 objects in the run_id folder.
	result, err := lc.s3Client.ListObjects(&s3.ListObjectsInput{
		Bucket: aws.String(lc.s3Bucket),
		Prefix: aws.String(s3DirName),
	})

	if err != nil {
		return errors.Wrap(err, "problem getting logs")
	}

	if result == nil || result.Contents == nil || len(result.Contents) == 0 {
		return nil
	}

	for _, content := range result.Contents {
		if strings.Contains(*content.Key, *run.PodName) {
			s3Key := content.Key
			result, err := lc.s3Client.GetObject(&s3.GetObjectInput{
				Bucket: aws.String(lc.s3Bucket),
				Key:    aws.String(*s3Key),
			})

			if err != nil {
				return err
			}
			return lc.logsToMessage(result, w)
		}
	}

	return nil
}

func (lc *EKSS3LogsClient) toS3DirName(run state.Run) string {
	return fmt.Sprintf("%s/%s", lc.s3BucketRootDir, run.RunID)
}

func (lc *EKSS3LogsClient) logsToMessage(result *s3.GetObjectOutput, w http.ResponseWriter) error {
	reader := bufio.NewReader(result.Body)
	for {
		line, err := reader.ReadBytes('\n')
		if err != nil {
			if err == io.EOF {
				err = nil
			}
			return err
		} else {
			var parsedLine s3Log
			err := json.Unmarshal(line, &parsedLine)
			if err != nil {
				return err
			}
			_, err = io.WriteString(w, parsedLine.Log)
			if err != nil {
				return err
			}
		}
	}
}
<|endoftext|>


<commit_before>package ctpolicy

import (
	"context"
	"errors"
	"math/rand"
	"time"

	"github.com/letsencrypt/boulder/canceled"
	"github.com/letsencrypt/boulder/cmd"
	"github.com/letsencrypt/boulder/core"
	berrors "github.com/letsencrypt/boulder/errors"
	blog "github.com/letsencrypt/boulder/log"
	"github.com/letsencrypt/boulder/metrics"
	pubpb "github.com/letsencrypt/boulder/publisher/proto"
	"github.com/prometheus/client_golang/prometheus"
)

// CTPolicy is used to hold information about SCTs required from various
// groupings
type CTPolicy struct {
	pub           core.Publisher
	groups        []cmd.CTGroup
	informational []cmd.LogDescription
	finalLogs     []cmd.LogDescription
	log           blog.Logger

	winnerCounter *prometheus.CounterVec
}

// New creates a new CTPolicy struct
func New(pub core.Publisher,
	groups []cmd.CTGroup,
	informational []cmd.LogDescription,
	log blog.Logger,
	stats metrics.Scope,
) *CTPolicy {
	var finalLogs []cmd.LogDescription
	for _, group := range groups {
		for _, log := range group.Logs {
			if log.SubmitFinalCert {
				finalLogs = append(finalLogs, log)
			}
		}
	}
	for _, log := range informational {
		if log.SubmitFinalCert {
			finalLogs = append(finalLogs, log)
		}
	}

	winnerCounter := prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Name: "sct_race_winner",
			Help: "Counter of logs that win SCT submission races.",
		},
		[]string{"log", "group"},
	)
	stats.MustRegister(winnerCounter)

	return &CTPolicy{
		pub:           pub,
		groups:        groups,
		informational: informational,
		finalLogs:     finalLogs,
		log:           log,
		winnerCounter: winnerCounter,
	}
}

type result struct {
	sct []byte
	log string
	err error
}

// race submits an SCT to each log in a group and waits for the first response back,
// once it has the first SCT it cancels all of the other submissions and returns.
// It allows up to len(group)-1 of the submissions to fail as we only care about
// getting a single SCT.
func (ctp *CTPolicy) race(ctx context.Context, cert core.CertDER, group cmd.CTGroup) ([]byte, error) {
	results := make(chan result, len(group.Logs))
	isPrecert := true
	// Randomize the order in which we send requests to the logs in a group
	// so we maximize the distribution of logs we get SCTs from.
	for _, i := range rand.Perm(len(group.Logs)) {
		l := group.Logs[i]
		go func(i int, l cmd.LogDescription) {
			// Each submission waits a bit longer than the previous one, to give the
			// previous log a chance to reply. If the context is already done by the
			// time we get here, don't bother submitting. That generally means the
			// context was canceled because another log returned a success already.
			time.Sleep(time.Duration(i) * group.Stagger.Duration)
			if ctx.Err() != nil {
				return
			}
			sct, err := ctp.pub.SubmitToSingleCTWithResult(ctx, &pubpb.Request{
				LogURL:       &l.URI,
				LogPublicKey: &l.Key,
				Der:          cert,
				Precert:      &isPrecert,
			})
			if err != nil {
				// Only log the error if it is not a result of the context being canceled
				if !canceled.Is(err) {
					ctp.log.Warningf("ct submission to %q failed: %s", l.URI, err)
				}
				results <- result{err: err}
				return
			}
			results <- result{sct: sct.Sct, log: l.URI}
		}(i, l)
	}

	for i := 0; i < len(group.Logs); i++ {
		select {
		case <-ctx.Done():
			return nil, ctx.Err()
		case res := <-results:
			if res.sct != nil {
				ctp.winnerCounter.With(prometheus.Labels{"log": res.log, "group": group.Name}).Inc()
				// Return the very first SCT we get back. Returning triggers
				// the defer'd context cancellation method.
				return res.sct, nil
			}
			// We will continue waiting for an SCT until we've seen the same number
			// of errors as there are logs in the group as we may still get a SCT
			// back from another log.
		}
	}
	return nil, errors.New("all submissions failed")
}

// GetSCTs attempts to retrieve a SCT from each configured grouping of logs and returns
// the set of SCTs to the caller.
func (ctp *CTPolicy) GetSCTs(ctx context.Context, cert core.CertDER) (core.SCTDERs, error) {
	results := make(chan result, len(ctp.groups))
	subCtx, cancel := context.WithCancel(ctx)
	defer cancel()
	for i, g := range ctp.groups {
		go func(i int, g cmd.CTGroup) {
			sct, err := ctp.race(subCtx, cert, g)
			// Only one of these will be non-nil
			if err != nil {
				results <- result{err: berrors.MissingSCTsError("CT log group %q: %s", g.Name, err)}
			}
			results <- result{sct: sct}
		}(i, g)
	}
	isPrecert := true
	for _, log := range ctp.informational {
		go func(l cmd.LogDescription) {
			// We use a context.Background() here instead of subCtx because these
			// submissions are running in a goroutine and we don't want them to be
			// cancelled when the caller of CTPolicy.GetSCTs returns and cancels
			// its RPC context.
			_, err := ctp.pub.SubmitToSingleCTWithResult(context.Background(), &pubpb.Request{
				LogURL:       &l.URI,
				LogPublicKey: &l.Key,
				Der:          cert,
				Precert:      &isPrecert,
			})
			if err != nil {
				ctp.log.Warningf("ct submission to informational log %q failed: %s", l.URI, err)
			}
		}(log)
	}

	var ret core.SCTDERs
	for i := 0; i < len(ctp.groups); i++ {
		res := <-results
		// If any one group fails to get a SCT then we fail out immediately
		// cancel any other in progress work as we can't continue
		if res.err != nil {
			// Returning triggers the defer'd context cancellation method
			return nil, res.err
		}
		ret = append(ret, res.sct)
	}
	return ret, nil
}

// SubmitFinalCert submits finalized certificates created from precertificates
// to any configured logs
func (ctp *CTPolicy) SubmitFinalCert(cert []byte) {
	falseVar := false
	for _, log := range ctp.finalLogs {
		go func(l cmd.LogDescription) {
			_, err := ctp.pub.SubmitToSingleCTWithResult(context.Background(), &pubpb.Request{
				LogURL:       &l.URI,
				LogPublicKey: &l.Key,
				Der:          cert,
				Precert:      &falseVar,
				StoreSCT:     &falseVar,
			})
			if err != nil {
				ctp.log.Warningf("ct submission of final cert to log %q failed: %s", l.URI, err)
			}
		}(log)
	}
}
<commit_msg>Fix staggering of log submission. (#3796)<commit_after>package ctpolicy

import (
	"context"
	"errors"
	"math/rand"
	"time"

	"github.com/letsencrypt/boulder/canceled"
	"github.com/letsencrypt/boulder/cmd"
	"github.com/letsencrypt/boulder/core"
	berrors "github.com/letsencrypt/boulder/errors"
	blog "github.com/letsencrypt/boulder/log"
	"github.com/letsencrypt/boulder/metrics"
	pubpb "github.com/letsencrypt/boulder/publisher/proto"
	"github.com/prometheus/client_golang/prometheus"
)

// CTPolicy is used to hold information about SCTs required from various
// groupings
type CTPolicy struct {
	pub           core.Publisher
	groups        []cmd.CTGroup
	informational []cmd.LogDescription
	finalLogs     []cmd.LogDescription
	log           blog.Logger

	winnerCounter *prometheus.CounterVec
}

// New creates a new CTPolicy struct
func New(pub core.Publisher,
	groups []cmd.CTGroup,
	informational []cmd.LogDescription,
	log blog.Logger,
	stats metrics.Scope,
) *CTPolicy {
	var finalLogs []cmd.LogDescription
	for _, group := range groups {
		for _, log := range group.Logs {
			if log.SubmitFinalCert {
				finalLogs = append(finalLogs, log)
			}
		}
	}
	for _, log := range informational {
		if log.SubmitFinalCert {
			finalLogs = append(finalLogs, log)
		}
	}

	winnerCounter := prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Name: "sct_race_winner",
			Help: "Counter of logs that win SCT submission races.",
		},
		[]string{"log", "group"},
	)
	stats.MustRegister(winnerCounter)

	return &CTPolicy{
		pub:           pub,
		groups:        groups,
		informational: informational,
		finalLogs:     finalLogs,
		log:           log,
		winnerCounter: winnerCounter,
	}
}

type result struct {
	sct []byte
	log string
	err error
}

// race submits an SCT to each log in a group and waits for the first response back,
// once it has the first SCT it cancels all of the other submissions and returns.
// It allows up to len(group)-1 of the submissions to fail as we only care about
// getting a single SCT.
func (ctp *CTPolicy) race(ctx context.Context, cert core.CertDER, group cmd.CTGroup) ([]byte, error) {
	results := make(chan result, len(group.Logs))
	isPrecert := true
	// Randomize the order in which we send requests to the logs in a group
	// so we maximize the distribution of logs we get SCTs from.
	for i, logNum := range rand.Perm(len(group.Logs)) {
		ld := group.Logs[logNum]
		go func(i int, ld cmd.LogDescription) {
			// Each submission waits a bit longer than the previous one, to give the
			// previous log a chance to reply. If the context is already done by the
			// time we get here, don't bother submitting. That generally means the
			// context was canceled because another log returned a success already.
			time.Sleep(time.Duration(i) * group.Stagger.Duration)
			if ctx.Err() != nil {
				return
			}
			sct, err := ctp.pub.SubmitToSingleCTWithResult(ctx, &pubpb.Request{
				LogURL:       &ld.URI,
				LogPublicKey: &ld.Key,
				Der:          cert,
				Precert:      &isPrecert,
			})
			if err != nil {
				// Only log the error if it is not a result of the context being canceled
				if !canceled.Is(err) {
					ctp.log.Warningf("ct submission to %q failed: %s", ld.URI, err)
				}
				results <- result{err: err}
				return
			}
			results <- result{sct: sct.Sct, log: ld.URI}
		}(i, ld)
	}

	for i := 0; i < len(group.Logs); i++ {
		select {
		case <-ctx.Done():
			return nil, ctx.Err()
		case res := <-results:
			if res.sct != nil {
				ctp.winnerCounter.With(prometheus.Labels{"log": res.log, "group": group.Name}).Inc()
				// Return the very first SCT we get back. Returning triggers
				// the defer'd context cancellation method.
				return res.sct, nil
			}
			// We will continue waiting for an SCT until we've seen the same number
			// of errors as there are logs in the group as we may still get a SCT
			// back from another log.
		}
	}
	return nil, errors.New("all submissions failed")
}

// GetSCTs attempts to retrieve a SCT from each configured grouping of logs and returns
// the set of SCTs to the caller.
func (ctp *CTPolicy) GetSCTs(ctx context.Context, cert core.CertDER) (core.SCTDERs, error) {
	results := make(chan result, len(ctp.groups))
	subCtx, cancel := context.WithCancel(ctx)
	defer cancel()
	for i, g := range ctp.groups {
		go func(i int, g cmd.CTGroup) {
			sct, err := ctp.race(subCtx, cert, g)
			// Only one of these will be non-nil
			if err != nil {
				results <- result{err: berrors.MissingSCTsError("CT log group %q: %s", g.Name, err)}
			}
			results <- result{sct: sct}
		}(i, g)
	}
	isPrecert := true
	for _, log := range ctp.informational {
		go func(l cmd.LogDescription) {
			// We use a context.Background() here instead of subCtx because these
			// submissions are running in a goroutine and we don't want them to be
			// cancelled when the caller of CTPolicy.GetSCTs returns and cancels
			// its RPC context.
			_, err := ctp.pub.SubmitToSingleCTWithResult(context.Background(), &pubpb.Request{
				LogURL:       &l.URI,
				LogPublicKey: &l.Key,
				Der:          cert,
				Precert:      &isPrecert,
			})
			if err != nil {
				ctp.log.Warningf("ct submission to informational log %q failed: %s", l.URI, err)
			}
		}(log)
	}

	var ret core.SCTDERs
	for i := 0; i < len(ctp.groups); i++ {
		res := <-results
		// If any one group fails to get a SCT then we fail out immediately
		// cancel any other in progress work as we can't continue
		if res.err != nil {
			// Returning triggers the defer'd context cancellation method
			return nil, res.err
		}
		ret = append(ret, res.sct)
	}
	return ret, nil
}

// SubmitFinalCert submits finalized certificates created from precertificates
// to any configured logs
func (ctp *CTPolicy) SubmitFinalCert(cert []byte) {
	falseVar := false
	for _, log := range ctp.finalLogs {
		go func(l cmd.LogDescription) {
			_, err := ctp.pub.SubmitToSingleCTWithResult(context.Background(), &pubpb.Request{
				LogURL:       &l.URI,
				LogPublicKey: &l.Key,
				Der:          cert,
				Precert:      &falseVar,
				StoreSCT:     &falseVar,
			})
			if err != nil {
				ctp.log.Warningf("ct submission of final cert to log %q failed: %s", l.URI, err)
			}
		}(log)
	}
}
<|endoftext|>


<commit_before>// +build !fasthttp

package gateway

import (
	"io"
	"net/http"
	"strconv"
	"time"

	"github.com/funkygao/gafka/cmd/kateway/hh"
	"github.com/funkygao/gafka/cmd/kateway/manager"
	"github.com/funkygao/gafka/cmd/kateway/store"
	"github.com/funkygao/gafka/mpool"
	"github.com/funkygao/httprouter"
	log "github.com/funkygao/log4go"
)

//go:generate goannotation $GOFILE
// @rest POST /v1/msgs/:topic/:ver?key=mykey&async=1&ack=all&hh=n
func (this *pubServer) pubHandler(w http.ResponseWriter, r *http.Request, params httprouter.Params) {
	var (
		appid        string
		topic        string
		ver          string
		tag          string
		partitionKey string
		async        bool
		hhDisabled   bool // hh enabled by default
		t1           = time.Now()
	)

	if !Options.DisableMetrics {
		this.pubMetrics.PubTryQps.Mark(1)
	}

	realIp := getHttpRemoteIp(r)
	if Options.Ratelimit && !this.throttlePub.Pour(realIp, 1) {
		log.Warn("pub[%s] %s(%s) rate limit reached: %d/s", appid, r.RemoteAddr, realIp, Options.PubQpsLimit)

		this.pubMetrics.ClientError.Inc(1)
		writeQuotaExceeded(w)
		return
	}

	appid = r.Header.Get(HttpHeaderAppid)
	topic = params.ByName(UrlParamTopic)
	ver = params.ByName(UrlParamVersion)

	if err := manager.Default.OwnTopic(appid, r.Header.Get(HttpHeaderPubkey), topic); err != nil {
		log.Warn("pub[%s] %s(%s) {topic:%s ver:%s UA:%s} %s",
			appid, r.RemoteAddr, realIp, topic, ver, r.Header.Get("User-Agent"), err)

		this.pubMetrics.ClientError.Inc(1)
		this.respond4XX(appid, w, err.Error(), http.StatusUnauthorized)
		return
	}

	msgLen := int(r.ContentLength)
	switch {
	case int64(msgLen) > Options.MaxPubSize:
		log.Warn("pub[%s] %s(%s) {topic:%s ver:%s UA:%s} too big content length: %d",
			appid, r.RemoteAddr, realIp, topic, ver, r.Header.Get("User-Agent"), msgLen)

		this.pubMetrics.ClientError.Inc(1)
		this.respond4XX(appid, w, ErrTooBigMessage.Error(), http.StatusBadRequest)
		return

	case msgLen < Options.MinPubSize:
		log.Warn("pub[%s] %s(%s) {topic:%s ver:%s UA:%s} too small content length: %d",
			appid, r.RemoteAddr, realIp, topic, ver, r.Header.Get("User-Agent"), msgLen)

		this.pubMetrics.ClientError.Inc(1)
		this.respond4XX(appid, w, ErrTooSmallMessage.Error(), http.StatusBadRequest)
		return
	}

	query := r.URL.Query() // reuse the query will save 100ns

	partitionKey = query.Get("key")
	if len(partitionKey) > MaxPartitionKeyLen {
		log.Warn("pub[%s] %s(%s) {topic:%s ver:%s UA:%s} too big key: %s",
			appid, r.RemoteAddr, realIp, topic, ver,
			r.Header.Get("User-Agent"), partitionKey)

		this.pubMetrics.ClientError.Inc(1)
		this.respond4XX(appid, w, "too big key", http.StatusBadRequest)
		return
	}

	var msg *mpool.Message
	tag = r.Header.Get(HttpHeaderMsgTag)
	if tag != "" {
		if len(tag) > Options.MaxMsgTagLen {
			this.respond4XX(appid, w, "too big tag", http.StatusBadRequest)
			return
		}

		msgSz := tagLen(tag) + msgLen
		msg = mpool.NewMessage(msgSz)
		msg.Body = msg.Body[0:msgSz]
	} else {
		msg = mpool.NewMessage(msgLen)
		msg.Body = msg.Body[0:msgLen]
	}

	// get the raw POST message, if body more than content-length ignore the extra payload
	lbr := io.LimitReader(r.Body, Options.MaxPubSize+1)
	if _, err := io.ReadAtLeast(lbr, msg.Body, msgLen); err != nil {
		msg.Free()

		log.Error("pub[%s] %s(%s) {topic:%s ver:%s UA:%s} %s",
			appid, r.RemoteAddr, realIp, topic, ver, r.Header.Get("User-Agent"), err)

		this.pubMetrics.ClientError.Inc(1)
		this.respond4XX(appid, w, err.Error(), http.StatusBadRequest) // TODO http.StatusRequestEntityTooLarge
		return
	}

	if tag != "" {
		AddTagToMessage(msg, tag)
	}

	if !Options.DisableMetrics {
		this.pubMetrics.PubQps.Mark(1)
		this.pubMetrics.PubMsgSize.Update(int64(len(msg.Body)))
	}

	cluster, found := manager.Default.LookupCluster(appid)
	if !found {
		log.Warn("pub[%s] %s(%s) {topic:%s ver:%s UA:%s} cluster not found",
			appid, r.RemoteAddr, realIp, topic, r.Header.Get("User-Agent"), ver)

		this.pubMetrics.ClientError.Inc(1)
		this.respond4XX(appid, w, "invalid appid", http.StatusBadRequest)
		return
	}

	var (
		partition int32
		offset    int64
		err       error
		rawTopic  = manager.Default.KafkaTopic(appid, topic, ver)
	)

	pubMethod := store.DefaultPubStore.SyncPub
	async = query.Get("async") == "1"
	if async {
		pubMethod = store.DefaultPubStore.AsyncPub
	}

	ackAll := query.Get("ack") == "all"
	if ackAll {
		pubMethod = store.DefaultPubStore.SyncAllPub
	}

	hhDisabled = query.Get("hh") == "n" // yes | no

	msgKey := []byte(partitionKey)
	if ackAll {
		// hh not applied
		partition, offset, err = pubMethod(cluster, rawTopic, msgKey, msg.Body)
	} else if Options.AllwaysHintedHandoff {
		err = hh.Default.Append(cluster, rawTopic, msgKey, msg.Body)
	} else if !hhDisabled && Options.EnableHintedHandoff && !hh.Default.Empty(cluster, rawTopic) {
		err = hh.Default.Append(cluster, rawTopic, msgKey, msg.Body)
	} else if async {
		if !hhDisabled && Options.EnableHintedHandoff {
			// async uses hinted handoff mechanism to save memory overhead
			err = hh.Default.Append(cluster, rawTopic, msgKey, msg.Body)
		} else {
			// message pool can't be applied on async pub because
			// we don't know when to recycle the memory
			body := make([]byte, 0, len(msg.Body))
			copy(body, msg.Body)
			partition, offset, err = pubMethod(cluster, rawTopic, msgKey, body)
		}
	} else {
		// hack byte string conv TODO
		partition, offset, err = pubMethod(cluster, rawTopic, msgKey, msg.Body)
		if err != nil && store.DefaultPubStore.IsSystemError(err) && !hhDisabled && Options.EnableHintedHandoff {
			// resort to hinted handoff
			err = hh.Default.Append(cluster, rawTopic, msgKey, msg.Body)
		}
	}

	// in case of request panic, mem pool leakage
	msg.Free()

	if Options.AuditPub {
		this.auditor.Trace("pub[%s] %s(%s) {topic:%s ver:%s UA:%s} {P:%d O:%d}",
			appid, r.RemoteAddr, realIp, topic, ver, r.Header.Get("User-Agent"), partition, offset)
	}

	if err != nil {
		log.Error("pub[%s] %s(%s) {topic:%s ver:%s} %s", appid, r.RemoteAddr, realIp, topic, ver, err)

		if !Options.DisableMetrics {
			this.pubMetrics.PubFail(appid, topic, ver)
		}

		if store.DefaultPubStore.IsSystemError(err) {
			writeServerError(w, err.Error())
		} else {
			this.respond4XX(appid, w, err.Error(), http.StatusBadRequest)
		}
		return
	}

	w.Header().Set(HttpHeaderPartition, strconv.FormatInt(int64(partition), 10))
	w.Header().Set(HttpHeaderOffset, strconv.FormatInt(offset, 10))
	if async {
		w.WriteHeader(http.StatusAccepted)
	} else {
		w.WriteHeader(http.StatusCreated)
	}

	if _, err = w.Write(ResponseOk); err != nil {
		log.Error("%s: %v", r.RemoteAddr, err)
		this.pubMetrics.ClientError.Inc(1)
	}

	if !Options.DisableMetrics {
		this.pubMetrics.PubOk(appid, topic, ver)
		this.pubMetrics.PubLatency.Update(time.Since(t1).Nanoseconds() / 1e6) // in ms
	}

}
<commit_msg>tweak of the audit log<commit_after>// +build !fasthttp

package gateway

import (
	"io"
	"net/http"
	"strconv"
	"time"

	"github.com/funkygao/gafka/cmd/kateway/hh"
	"github.com/funkygao/gafka/cmd/kateway/manager"
	"github.com/funkygao/gafka/cmd/kateway/store"
	"github.com/funkygao/gafka/mpool"
	"github.com/funkygao/httprouter"
	log "github.com/funkygao/log4go"
)

//go:generate goannotation $GOFILE
// @rest POST /v1/msgs/:topic/:ver?key=mykey&async=1&ack=all&hh=n
func (this *pubServer) pubHandler(w http.ResponseWriter, r *http.Request, params httprouter.Params) {
	var (
		appid        string
		topic        string
		ver          string
		tag          string
		partitionKey string
		async        bool
		hhDisabled   bool // hh enabled by default
		t1           = time.Now()
	)

	if !Options.DisableMetrics {
		this.pubMetrics.PubTryQps.Mark(1)
	}

	realIp := getHttpRemoteIp(r)
	if Options.Ratelimit && !this.throttlePub.Pour(realIp, 1) {
		log.Warn("pub[%s] %s(%s) rate limit reached: %d/s", appid, r.RemoteAddr, realIp, Options.PubQpsLimit)

		this.pubMetrics.ClientError.Inc(1)
		writeQuotaExceeded(w)
		return
	}

	appid = r.Header.Get(HttpHeaderAppid)
	topic = params.ByName(UrlParamTopic)
	ver = params.ByName(UrlParamVersion)

	if err := manager.Default.OwnTopic(appid, r.Header.Get(HttpHeaderPubkey), topic); err != nil {
		log.Warn("pub[%s] %s(%s) {topic:%s ver:%s UA:%s} %s",
			appid, r.RemoteAddr, realIp, topic, ver, r.Header.Get("User-Agent"), err)

		this.pubMetrics.ClientError.Inc(1)
		this.respond4XX(appid, w, err.Error(), http.StatusUnauthorized)
		return
	}

	msgLen := int(r.ContentLength)
	switch {
	case int64(msgLen) > Options.MaxPubSize:
		log.Warn("pub[%s] %s(%s) {topic:%s ver:%s UA:%s} too big content length: %d",
			appid, r.RemoteAddr, realIp, topic, ver, r.Header.Get("User-Agent"), msgLen)

		this.pubMetrics.ClientError.Inc(1)
		this.respond4XX(appid, w, ErrTooBigMessage.Error(), http.StatusBadRequest)
		return

	case msgLen < Options.MinPubSize:
		log.Warn("pub[%s] %s(%s) {topic:%s ver:%s UA:%s} too small content length: %d",
			appid, r.RemoteAddr, realIp, topic, ver, r.Header.Get("User-Agent"), msgLen)

		this.pubMetrics.ClientError.Inc(1)
		this.respond4XX(appid, w, ErrTooSmallMessage.Error(), http.StatusBadRequest)
		return
	}

	query := r.URL.Query() // reuse the query will save 100ns

	partitionKey = query.Get("key")
	if len(partitionKey) > MaxPartitionKeyLen {
		log.Warn("pub[%s] %s(%s) {topic:%s ver:%s UA:%s} too big key: %s",
			appid, r.RemoteAddr, realIp, topic, ver,
			r.Header.Get("User-Agent"), partitionKey)

		this.pubMetrics.ClientError.Inc(1)
		this.respond4XX(appid, w, "too big key", http.StatusBadRequest)
		return
	}

	var msg *mpool.Message
	tag = r.Header.Get(HttpHeaderMsgTag)
	if tag != "" {
		if len(tag) > Options.MaxMsgTagLen {
			this.respond4XX(appid, w, "too big tag", http.StatusBadRequest)
			return
		}

		msgSz := tagLen(tag) + msgLen
		msg = mpool.NewMessage(msgSz)
		msg.Body = msg.Body[0:msgSz]
	} else {
		msg = mpool.NewMessage(msgLen)
		msg.Body = msg.Body[0:msgLen]
	}

	// get the raw POST message, if body more than content-length ignore the extra payload
	lbr := io.LimitReader(r.Body, Options.MaxPubSize+1)
	if _, err := io.ReadAtLeast(lbr, msg.Body, msgLen); err != nil {
		msg.Free()

		log.Error("pub[%s] %s(%s) {topic:%s ver:%s UA:%s} %s",
			appid, r.RemoteAddr, realIp, topic, ver, r.Header.Get("User-Agent"), err)

		this.pubMetrics.ClientError.Inc(1)
		this.respond4XX(appid, w, err.Error(), http.StatusBadRequest) // TODO http.StatusRequestEntityTooLarge
		return
	}

	if tag != "" {
		AddTagToMessage(msg, tag)
	}

	if !Options.DisableMetrics {
		this.pubMetrics.PubQps.Mark(1)
		this.pubMetrics.PubMsgSize.Update(int64(len(msg.Body)))
	}

	cluster, found := manager.Default.LookupCluster(appid)
	if !found {
		log.Warn("pub[%s] %s(%s) {topic:%s ver:%s UA:%s} cluster not found",
			appid, r.RemoteAddr, realIp, topic, r.Header.Get("User-Agent"), ver)

		this.pubMetrics.ClientError.Inc(1)
		this.respond4XX(appid, w, "invalid appid", http.StatusBadRequest)
		return
	}

	var (
		partition int32
		offset    int64
		err       error
		rawTopic  = manager.Default.KafkaTopic(appid, topic, ver)
	)

	pubMethod := store.DefaultPubStore.SyncPub
	async = query.Get("async") == "1"
	if async {
		pubMethod = store.DefaultPubStore.AsyncPub
	}

	ackAll := query.Get("ack") == "all"
	if ackAll {
		pubMethod = store.DefaultPubStore.SyncAllPub
	}

	hhDisabled = query.Get("hh") == "n" // yes | no

	msgKey := []byte(partitionKey)
	if ackAll {
		// hh not applied
		partition, offset, err = pubMethod(cluster, rawTopic, msgKey, msg.Body)
	} else if Options.AllwaysHintedHandoff {
		err = hh.Default.Append(cluster, rawTopic, msgKey, msg.Body)
	} else if !hhDisabled && Options.EnableHintedHandoff && !hh.Default.Empty(cluster, rawTopic) {
		err = hh.Default.Append(cluster, rawTopic, msgKey, msg.Body)
	} else if async {
		if !hhDisabled && Options.EnableHintedHandoff {
			// async uses hinted handoff mechanism to save memory overhead
			err = hh.Default.Append(cluster, rawTopic, msgKey, msg.Body)
		} else {
			// message pool can't be applied on async pub because
			// we don't know when to recycle the memory
			body := make([]byte, 0, len(msg.Body))
			copy(body, msg.Body)
			partition, offset, err = pubMethod(cluster, rawTopic, msgKey, body)
		}
	} else {
		// hack byte string conv TODO
		partition, offset, err = pubMethod(cluster, rawTopic, msgKey, msg.Body)
		if err != nil && store.DefaultPubStore.IsSystemError(err) && !hhDisabled && Options.EnableHintedHandoff {
			log.Warn("pub[%s] %s(%s) {%s.%s.%s UA:%s} resort hh", appid, r.RemoteAddr, realIp,
				appid, topic, ver, r.Header.Get("User-Agent"))
			err = hh.Default.Append(cluster, rawTopic, msgKey, msg.Body)
		}
	}

	// in case of request panic, mem pool leakage
	msg.Free()

	if Options.AuditPub {
		this.auditor.Trace("pub[%s] %s(%s) {%s.%s.%s UA:%s} {P:%d O:%d}",
			appid, r.RemoteAddr, realIp, appid, topic, ver, r.Header.Get("User-Agent"), partition, offset)
	}

	if err != nil {
		log.Error("pub[%s] %s(%s) {topic:%s ver:%s} %s", appid, r.RemoteAddr, realIp, topic, ver, err)

		if !Options.DisableMetrics {
			this.pubMetrics.PubFail(appid, topic, ver)
		}

		if store.DefaultPubStore.IsSystemError(err) {
			writeServerError(w, err.Error())
		} else {
			this.respond4XX(appid, w, err.Error(), http.StatusBadRequest)
		}
		return
	}

	w.Header().Set(HttpHeaderPartition, strconv.FormatInt(int64(partition), 10))
	w.Header().Set(HttpHeaderOffset, strconv.FormatInt(offset, 10))
	if async {
		w.WriteHeader(http.StatusAccepted)
	} else {
		w.WriteHeader(http.StatusCreated)
	}

	if _, err = w.Write(ResponseOk); err != nil {
		log.Error("%s: %v", r.RemoteAddr, err)
		this.pubMetrics.ClientError.Inc(1)
	}

	if !Options.DisableMetrics {
		this.pubMetrics.PubOk(appid, topic, ver)
		this.pubMetrics.PubLatency.Update(time.Since(t1).Nanoseconds() / 1e6) // in ms
	}

}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"fmt"

	"github.com/mattn/go-mastodon"
	"github.com/urfave/cli"
)

func cmdInstanceActivity(c *cli.Context) error {
	client := c.App.Metadata["client"].(*mastodon.Client)
	activities, err := client.GetInstanceActivity(context.Background())
	if err != nil {
		return err
	}
	for _, activity := range activities {
		fmt.Fprintf(c.App.Writer, "Logins        : %v\n", activity.Logins)
		fmt.Fprintf(c.App.Writer, "Registrations : %v\n", activity.Registrations)
		fmt.Fprintf(c.App.Writer, "Statuses      : %v\n", activity.Statuses)
		fmt.Fprintf(c.App.Writer, "EMail         : %v\n", activity.Week)
	}
	return nil
}
<commit_msg>fix typo<commit_after>package main

import (
	"context"
	"fmt"

	"github.com/mattn/go-mastodon"
	"github.com/urfave/cli"
)

func cmdInstanceActivity(c *cli.Context) error {
	client := c.App.Metadata["client"].(*mastodon.Client)
	activities, err := client.GetInstanceActivity(context.Background())
	if err != nil {
		return err
	}
	for _, activity := range activities {
		fmt.Fprintf(c.App.Writer, "Logins        : %v\n", activity.Logins)
		fmt.Fprintf(c.App.Writer, "Registrations : %v\n", activity.Registrations)
		fmt.Fprintf(c.App.Writer, "Statuses      : %v\n", activity.Statuses)
		fmt.Fprintf(c.App.Writer, "Week          : %v\n", activity.Week)
	}
	return nil
}
<|endoftext|>


<commit_before>// Copyright (c) 2017 Pani Networks
// All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

package main

import (
	"flag"
	"os"
	"strings"
	"time"

	"github.com/romana/core/agent/router/bird"
	"github.com/romana/core/agent/router/publisher"
	"github.com/romana/core/common"
	"github.com/romana/core/common/client"
	log "github.com/romana/rlog"
)

// GetGroupByHost finds all groups on IPAM which have host with given hostname,
// returns map[NetworkName]Group. If host not found in any group of a network
// then the network isn't mentioned in return map.
func GetGroupByBost(ipam *client.IPAM, hostname string) map[string]*client.Group {

	hostInList := func(hosts []*client.Host, hostname string) bool {
		for _, host := range hosts {
			if host.Name == hostname {
				return true
			}
		}

		return false
	}

	// hostInGroup is a recursive function, currently in Go anonymous functions
	// can't be recursive, which is why this variable exists to make the function
	// _less_ anonymous.
	var hostInGroup func(group *client.Group, hostname string) *client.Group

	hostInGroup = func(group *client.Group, hostname string) *client.Group {
		if hostInList(group.Hosts, hostname) {
			return group
		}

		if group.Groups == nil {
			return nil
		}

		for _, nestedGroup := range group.Groups {
			g := hostInGroup(nestedGroup, hostname)
			if g != nil {
				return g
			}
		}

		return nil
	}

	res := make(map[string]*client.Group)

	for netName, net := range ipam.Networks {
		group := hostInGroup(net.Group, hostname)
		if group != nil {
			res[netName] = group
		}
	}

	return res
}

func main() {
	var err error

	etcdEndpoints := flag.String("endpoints", "", "csv list of etcd endpoints to romana storage")
	etcdPrefix := flag.String("prefix", "", "string that prefixes all romana keys in etcd")
	hostname := flag.String("hostname", "", "name of the host in romana database")
	flagTemplateFile := flag.String("template", "/etc/bird/bird.conf.t", "template file for bird config")
	flagBirdConfigFile := flag.String("config", "/etc/bird/bird.conf", "location of the bird config file")
	flagBirdPidFile := flag.String("pid", "/var/run/bird.pid", "location of bird pid file")
	flagDebug := flag.String("debug", "", "set to yes or true to enable debug output")
	flagLocalAS := flag.String("as", "65534", "local as number")
	flag.Parse()

	config := make(map[string]string)
	config["templateFileName"] = *flagTemplateFile
	config["birdConfigName"] = *flagBirdConfigFile
	config["pidFile"] = *flagBirdPidFile
	config["localAS"] = *flagLocalAS
	config["debug"] = *flagDebug

	bird, err := bird.New(publisher.Config(config))
	if err != nil {
		panic(err)
	}

	romanaConfig := common.Config{
		EtcdEndpoints: strings.Split(*etcdEndpoints, ","),
		EtcdPrefix:    *etcdPrefix,
	}

	if *hostname == "" {
		*hostname, err = os.Hostname()
		if err != nil {
			panic(err)
		}
	}

	romanaClient, err := client.NewClient(&romanaConfig)

	stopCh := make(chan struct{})
	defer close(stopCh)

	// blocksChannel := WatchBlocks(ctx, romanaClient)
	blocksChannel, err := romanaClient.WatchBlocks(stopCh)
	if err != nil {
		log.Errorf("Failed to start watching for blocks, %s", err)
		os.Exit(2)
	}

	for {
		select {
		case blocks := <-blocksChannel:
			startTime := time.Now()

			hostGroups := GetGroupGyHost(romanaClient.IPAM, *hostname)
			args := make(map[string]interface{})

			if len(hostGroups) > 0 {
				args["HostGroups"] = hostGroups
			}

			createRouteToBlocks(blocks.Blocks, args, *hostname, bird)
			runTime := time.Now().Sub(startTime)
			log.Tracef(4, "Time between route table flush and route table rebuild %s", runTime)

		}
	}
}
<commit_msg>Fix misspel<commit_after>// Copyright (c) 2017 Pani Networks
// All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

package main

import (
	"flag"
	"os"
	"strings"
	"time"

	"github.com/romana/core/agent/router/bird"
	"github.com/romana/core/agent/router/publisher"
	"github.com/romana/core/common"
	"github.com/romana/core/common/client"
	log "github.com/romana/rlog"
)

// GetGroupByHost finds all groups on IPAM which have host with given hostname,
// returns map[NetworkName]Group. If host not found in any group of a network
// then the network isn't mentioned in return map.
func GetGroupByHost(ipam *client.IPAM, hostname string) map[string]*client.Group {

	hostInList := func(hosts []*client.Host, hostname string) bool {
		for _, host := range hosts {
			if host.Name == hostname {
				return true
			}
		}

		return false
	}

	// hostInGroup is a recursive function, currently in Go anonymous functions
	// can't be recursive, which is why this variable exists to make the function
	// _less_ anonymous.
	var hostInGroup func(group *client.Group, hostname string) *client.Group

	hostInGroup = func(group *client.Group, hostname string) *client.Group {
		if hostInList(group.Hosts, hostname) {
			return group
		}

		if group.Groups == nil {
			return nil
		}

		for _, nestedGroup := range group.Groups {
			g := hostInGroup(nestedGroup, hostname)
			if g != nil {
				return g
			}
		}

		return nil
	}

	res := make(map[string]*client.Group)

	for netName, net := range ipam.Networks {
		group := hostInGroup(net.Group, hostname)
		if group != nil {
			res[netName] = group
		}
	}

	return res
}

func main() {
	var err error

	etcdEndpoints := flag.String("endpoints", "", "csv list of etcd endpoints to romana storage")
	etcdPrefix := flag.String("prefix", "", "string that prefixes all romana keys in etcd")
	hostname := flag.String("hostname", "", "name of the host in romana database")
	flagTemplateFile := flag.String("template", "/etc/bird/bird.conf.t", "template file for bird config")
	flagBirdConfigFile := flag.String("config", "/etc/bird/bird.conf", "location of the bird config file")
	flagBirdPidFile := flag.String("pid", "/var/run/bird.pid", "location of bird pid file")
	flagDebug := flag.String("debug", "", "set to yes or true to enable debug output")
	flagLocalAS := flag.String("as", "65534", "local as number")
	flag.Parse()

	config := make(map[string]string)
	config["templateFileName"] = *flagTemplateFile
	config["birdConfigName"] = *flagBirdConfigFile
	config["pidFile"] = *flagBirdPidFile
	config["localAS"] = *flagLocalAS
	config["debug"] = *flagDebug

	bird, err := bird.New(publisher.Config(config))
	if err != nil {
		panic(err)
	}

	romanaConfig := common.Config{
		EtcdEndpoints: strings.Split(*etcdEndpoints, ","),
		EtcdPrefix:    *etcdPrefix,
	}

	if *hostname == "" {
		*hostname, err = os.Hostname()
		if err != nil {
			panic(err)
		}
	}

	romanaClient, err := client.NewClient(&romanaConfig)

	stopCh := make(chan struct{})
	defer close(stopCh)

	// blocksChannel := WatchBlocks(ctx, romanaClient)
	blocksChannel, err := romanaClient.WatchBlocks(stopCh)
	if err != nil {
		log.Errorf("Failed to start watching for blocks, %s", err)
		os.Exit(2)
	}

	for {
		select {
		case blocks := <-blocksChannel:
			startTime := time.Now()

			hostGroups := GetGroupByHost(romanaClient.IPAM, *hostname)
			args := make(map[string]interface{})

			if len(hostGroups) > 0 {
				args["HostGroups"] = hostGroups
			}

			createRouteToBlocks(blocks.Blocks, args, *hostname, bird)
			runTime := time.Now().Sub(startTime)
			log.Tracef(4, "Time between route table flush and route table rebuild %s", runTime)

		}
	}
}
<|endoftext|>