<commit_before>package rpc_client

import (
	"crypto/tls"
	"fmt"
	"io"
	"net"
	netrpc "net/rpc"
	"sync"
	"sync/atomic"
	"time"

	"github.com/stackengine/selog"
	"github.com/stackengine/serpc"
	"github.com/stackengine/ssltls"
)

var sLog = selog.Register("clntrpc", 0)

type NewClientCodec func(conn io.ReadWriteCloser) netrpc.ClientCodec

type Conn struct {
	sync.Mutex

	addr           net.Addr
	key            string
	lastUsed       time.Time
	newClientCodec NewClientCodec
	net_con        net.Conn
	pool           *ConnPool
	refCount       int32
	rpc_clnt       *netrpc.Client
	shutdown       int32
	stream_type    string
	version        int
}

func (c *Conn) String() string {
	return fmt.Sprintf("Conn:%p type: %s ref: %d key: %s addr: %s shutdown: %d",
		c, c.stream_type, c.refCount, c.key, c.addr.String(), c.shutdown)
}

func NewConn(newClientCodec NewClientCodec,
	addr net.Addr,
	stream_type string,
	key string,
	timo time.Duration,
	tlsConfig *tls.Config) (*Conn, error) {

	//	sLog.Printf("New Connection: addr -> %s stream -> '%s' key -> %s", addr, stream_type, key)
	// Try to dial the conn
	conn, err := net.DialTimeout("tcp", addr.String(), timo)
	if err != nil {
		return nil, err
	}

	// Cast to TCPConn
	if tcp, ok := conn.(*net.TCPConn); ok {
		tcp.SetKeepAlive(true)
		tcp.SetNoDelay(true)
	}

	// write stream mux version byte
	if _, err := conn.Write([]byte{byte(rpc_stream.Mux_v2)}); err != nil {
		conn.Close()
		return nil, err
	}

	// Check if TLS is enabled
	if tlsConfig != nil {
		// Switch the connection into TLS mode
		//		sLog.Println("Switch Connection for: ", rpc_stream.RpcTLS)
		if _, err := conn.Write([]byte(rpc_stream.Nameify(rpc_stream.RpcTLS))); err != nil {
			conn.Close()
			return nil, err
		}
		// Wrap the connection in a TLS client
		tconn, err := ssltls.WrapTLSClient(conn, tlsConfig)
		if err != nil {
			conn.Close()
			return nil, err
		}
		conn = tconn
	}

	st := rpc_stream.Nameify(stream_type)

	// write stream type bytes
	if _, err := conn.Write([]byte(st)); err != nil {
		conn.Close()
		return nil, err
	}

	//	sLog.Printf("Wrote stream type for: '%s'", stream_type)
	var clnt *netrpc.Client

	if newClientCodec != nil {
		clnt = netrpc.NewClientWithCodec(newClientCodec(conn))
	} else {
		clnt = netrpc.NewClient(conn)
	}

	// build Conn
	c := &Conn{
		refCount:    1,
		addr:        addr,
		net_con:     conn,
		rpc_clnt:    clnt,
		lastUsed:    time.Now(),
		key:         key,
		stream_type: st,
	}
	return c, nil
}

func (c *Conn) Close() {
	// net connection  clean up
	if c.net_con != nil {
		c.net_con.Close()
		c.net_con = nil
	}

	// rcp clnt clean up
	if c.rpc_clnt != nil {
		c.rpc_clnt.Close()
		c.rpc_clnt = nil
	}
}

func (c *Conn) Release() {
	refCount := atomic.AddInt32(&c.refCount, -1)
	if refCount == 0 && atomic.LoadInt32(&c.shutdown) == 1 {
		sLog.Printf("Release: calling Close() %p %s\n", c, c)
		c.Close()
	}
}

func (c *Conn) Hold() {
	atomic.AddInt32(&c.refCount, 1)
	c.lastUsed = time.Now()
}

func (c *Conn) Shutdown() {
	atomic.StoreInt32(&c.shutdown, 1)
}
<commit_msg>update to be on a single line<commit_after>package rpc_client

import (
	"crypto/tls"
	"fmt"
	"io"
	"net"
	netrpc "net/rpc"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	"github.com/stackengine/selog"
	"github.com/stackengine/serpc"
	"github.com/stackengine/ssltls"
)

var sLog = selog.Register("clntrpc", 0)

type NewClientCodec func(conn io.ReadWriteCloser) netrpc.ClientCodec

type Conn struct {
	sync.Mutex

	addr           net.Addr
	key            string
	lastUsed       time.Time
	newClientCodec NewClientCodec
	net_con        net.Conn
	pool           *ConnPool
	refCount       int32
	rpc_clnt       *netrpc.Client
	shutdown       int32
	stream_type    string
	version        int
}

func (c *Conn) String() string {
	return fmt.Sprintf("Conn:%p type: %s ref: %d key: %s addr: %s shutdown: %d",
		c, strings.TrimSuffix(c.stream_type, "\n"), c.refCount, c.key, c.addr.String(), c.shutdown)
}

func NewConn(newClientCodec NewClientCodec,
	addr net.Addr,
	stream_type string,
	key string,
	timo time.Duration,
	tlsConfig *tls.Config) (*Conn, error) {

	sLog.Printf("New Connection: addr -> %s stream -> '%s' key -> %s", addr, stream_type, key)
	// Try to dial the conn
	conn, err := net.DialTimeout("tcp", addr.String(), timo)
	if err != nil {
		return nil, err
	}

	// Cast to TCPConn
	if tcp, ok := conn.(*net.TCPConn); ok {
		tcp.SetKeepAlive(true)
		tcp.SetNoDelay(true)
	}

	// write stream mux version byte
	if _, err := conn.Write([]byte{byte(rpc_stream.Mux_v2)}); err != nil {
		conn.Close()
		return nil, err
	}

	// Check if TLS is enabled
	if tlsConfig != nil {
		// Switch the connection into TLS mode
		//		sLog.Println("Switch Connection for: ", rpc_stream.RpcTLS)
		if _, err := conn.Write([]byte(rpc_stream.Nameify(rpc_stream.RpcTLS))); err != nil {
			conn.Close()
			return nil, err
		}
		// Wrap the connection in a TLS client
		tconn, err := ssltls.WrapTLSClient(conn, tlsConfig)
		if err != nil {
			conn.Close()
			return nil, err
		}
		conn = tconn
	}

	st := rpc_stream.Nameify(stream_type)

	// write stream type bytes
	if _, err := conn.Write([]byte(st)); err != nil {
		conn.Close()
		return nil, err
	}

	//	sLog.Printf("Wrote stream type for: '%s'", stream_type)
	var clnt *netrpc.Client

	if newClientCodec != nil {
		clnt = netrpc.NewClientWithCodec(newClientCodec(conn))
	} else {
		clnt = netrpc.NewClient(conn)
	}

	// build Conn
	c := &Conn{
		refCount:    1,
		addr:        addr,
		net_con:     conn,
		rpc_clnt:    clnt,
		lastUsed:    time.Now(),
		key:         key,
		stream_type: st,
	}
	return c, nil
}

func (c *Conn) Close() {
	// net connection  clean up
	if c.net_con != nil {
		c.net_con.Close()
		c.net_con = nil
	}

	// rcp clnt clean up
	if c.rpc_clnt != nil {
		c.rpc_clnt.Close()
		c.rpc_clnt = nil
	}
}

func (c *Conn) Release() {
	refCount := atomic.AddInt32(&c.refCount, -1)
	if refCount == 0 && atomic.LoadInt32(&c.shutdown) == 1 {
		sLog.Printf("Release: calling Close() %p %s\n", c, c)
		c.Close()
	}
}

func (c *Conn) Hold() {
	atomic.AddInt32(&c.refCount, 1)
	c.lastUsed = time.Now()
}

func (c *Conn) Shutdown() {
	atomic.StoreInt32(&c.shutdown, 1)
}
<|endoftext|>


<commit_before>package client

import (
	"errors"
	"fmt"
)

// ErrConnectionFailed is an error raised when the connection between the client and the server failed.
var ErrConnectionFailed = errors.New("Cannot connect to the Docker daemon. Is the docker daemon running on this host?")

type notFound interface {
	error
	NotFound() bool // Is the error a NotFound error
}

// IsErrNotFound returns true if the error is caused with an
// object (image, container, network, volume, …) is not found in the docker host.
func IsErrNotFound(err error) bool {
	te, ok := err.(notFound)
	return ok && te.NotFound()
}

// imageNotFoundError implements an error returned when an image is not in the docker host.
type imageNotFoundError struct {
	imageID string
}

// NoFound indicates that this error type is of NotFound
func (e imageNotFoundError) NotFound() bool {
	return true
}

// Error returns a string representation of an imageNotFoundError
func (e imageNotFoundError) Error() string {
	return fmt.Sprintf("Error: No such image: %s", e.imageID)
}

// IsErrImageNotFound returns true if the error is caused
// when an image is not found in the docker host.
func IsErrImageNotFound(err error) bool {
	return IsErrNotFound(err)
}

// containerNotFoundError implements an error returned when a container is not in the docker host.
type containerNotFoundError struct {
	containerID string
}

// NoFound indicates that this error type is of NotFound
func (e containerNotFoundError) NotFound() bool {
	return true
}

// Error returns a string representation of a containerNotFoundError
func (e containerNotFoundError) Error() string {
	return fmt.Sprintf("Error: No such container: %s", e.containerID)
}

// IsErrContainerNotFound returns true if the error is caused
// when a container is not found in the docker host.
func IsErrContainerNotFound(err error) bool {
	return IsErrNotFound(err)
}

// networkNotFoundError implements an error returned when a network is not in the docker host.
type networkNotFoundError struct {
	networkID string
}

// NoFound indicates that this error type is of NotFound
func (e networkNotFoundError) NotFound() bool {
	return true
}

// Error returns a string representation of a networkNotFoundError
func (e networkNotFoundError) Error() string {
	return fmt.Sprintf("Error: No such network: %s", e.networkID)
}

// IsErrNetworkNotFound returns true if the error is caused
// when a network is not found in the docker host.
func IsErrNetworkNotFound(err error) bool {
	return IsErrNotFound(err)
}

// volumeNotFoundError implements an error returned when a volume is not in the docker host.
type volumeNotFoundError struct {
	volumeID string
}

// NoFound indicates that this error type is of NotFound
func (e volumeNotFoundError) NotFound() bool {
	return true
}

// Error returns a string representation of a networkNotFoundError
func (e volumeNotFoundError) Error() string {
	return fmt.Sprintf("Error: No such volume: %s", e.volumeID)
}

// IsErrVolumeNotFound returns true if the error is caused
// when a volume is not found in the docker host.
func IsErrVolumeNotFound(err error) bool {
	return IsErrNotFound(err)
}

// unauthorizedError represents an authorization error in a remote registry.
type unauthorizedError struct {
	cause error
}

// Error returns a string representation of an unauthorizedError
func (u unauthorizedError) Error() string {
	return u.cause.Error()
}

// IsErrUnauthorized returns true if the error is caused
// when a remote registry authentication fails
func IsErrUnauthorized(err error) bool {
	_, ok := err.(unauthorizedError)
	return ok
}

// nodeNotFoundError implements an error returned when a node is not found.
type nodeNotFoundError struct {
	nodeID string
}

// Error returns a string representation of a nodeNotFoundError
func (e nodeNotFoundError) Error() string {
	return fmt.Sprintf("Error: No such node: %s", e.nodeID)
}

// IsErrNodeNotFound returns true if the error is caused
// when a node is not found.
func IsErrNodeNotFound(err error) bool {
	_, ok := err.(nodeNotFoundError)
	return ok
}

// serviceNotFoundError implements an error returned when a service is not found.
type serviceNotFoundError struct {
	serviceID string
}

// Error returns a string representation of a serviceNotFoundError
func (e serviceNotFoundError) Error() string {
	return fmt.Sprintf("Error: No such service: %s", e.serviceID)
}

// NoFound indicates that this error type is of NotFound
func (e serviceNotFoundError) NotFound() bool {
	return true
}

// IsErrServiceNotFound returns true if the error is caused
// when a service is not found.
func IsErrServiceNotFound(err error) bool {
	_, ok := err.(serviceNotFoundError)
	return ok
}

// taskNotFoundError implements an error returned when a task is not found.
type taskNotFoundError struct {
	taskID string
}

// Error returns a string representation of a taskNotFoundError
func (e taskNotFoundError) Error() string {
	return fmt.Sprintf("Error: No such task: %s", e.taskID)
}

// NoFound indicates that this error type is of NotFound
func (e taskNotFoundError) NotFound() bool {
	return true
}

// IsErrTaskNotFound returns true if the error is caused
// when a task is not found.
func IsErrTaskNotFound(err error) bool {
	_, ok := err.(taskNotFoundError)
	return ok
}

type pluginPermissionDenied struct {
	name string
}

func (e pluginPermissionDenied) Error() string {
	return "Permission denied while installing plugin " + e.name
}

// IsErrPluginPermissionDenied returns true if the error is caused
// when a user denies a plugin's permissions
func IsErrPluginPermissionDenied(err error) bool {
	_, ok := err.(pluginPermissionDenied)
	return ok
}
<commit_msg>Add nodeNotFound NotFound()<commit_after>package client

import (
	"errors"
	"fmt"
)

// ErrConnectionFailed is an error raised when the connection between the client and the server failed.
var ErrConnectionFailed = errors.New("Cannot connect to the Docker daemon. Is the docker daemon running on this host?")

type notFound interface {
	error
	NotFound() bool // Is the error a NotFound error
}

// IsErrNotFound returns true if the error is caused with an
// object (image, container, network, volume, …) is not found in the docker host.
func IsErrNotFound(err error) bool {
	te, ok := err.(notFound)
	return ok && te.NotFound()
}

// imageNotFoundError implements an error returned when an image is not in the docker host.
type imageNotFoundError struct {
	imageID string
}

// NoFound indicates that this error type is of NotFound
func (e imageNotFoundError) NotFound() bool {
	return true
}

// Error returns a string representation of an imageNotFoundError
func (e imageNotFoundError) Error() string {
	return fmt.Sprintf("Error: No such image: %s", e.imageID)
}

// IsErrImageNotFound returns true if the error is caused
// when an image is not found in the docker host.
func IsErrImageNotFound(err error) bool {
	return IsErrNotFound(err)
}

// containerNotFoundError implements an error returned when a container is not in the docker host.
type containerNotFoundError struct {
	containerID string
}

// NoFound indicates that this error type is of NotFound
func (e containerNotFoundError) NotFound() bool {
	return true
}

// Error returns a string representation of a containerNotFoundError
func (e containerNotFoundError) Error() string {
	return fmt.Sprintf("Error: No such container: %s", e.containerID)
}

// IsErrContainerNotFound returns true if the error is caused
// when a container is not found in the docker host.
func IsErrContainerNotFound(err error) bool {
	return IsErrNotFound(err)
}

// networkNotFoundError implements an error returned when a network is not in the docker host.
type networkNotFoundError struct {
	networkID string
}

// NoFound indicates that this error type is of NotFound
func (e networkNotFoundError) NotFound() bool {
	return true
}

// Error returns a string representation of a networkNotFoundError
func (e networkNotFoundError) Error() string {
	return fmt.Sprintf("Error: No such network: %s", e.networkID)
}

// IsErrNetworkNotFound returns true if the error is caused
// when a network is not found in the docker host.
func IsErrNetworkNotFound(err error) bool {
	return IsErrNotFound(err)
}

// volumeNotFoundError implements an error returned when a volume is not in the docker host.
type volumeNotFoundError struct {
	volumeID string
}

// NoFound indicates that this error type is of NotFound
func (e volumeNotFoundError) NotFound() bool {
	return true
}

// Error returns a string representation of a networkNotFoundError
func (e volumeNotFoundError) Error() string {
	return fmt.Sprintf("Error: No such volume: %s", e.volumeID)
}

// IsErrVolumeNotFound returns true if the error is caused
// when a volume is not found in the docker host.
func IsErrVolumeNotFound(err error) bool {
	return IsErrNotFound(err)
}

// unauthorizedError represents an authorization error in a remote registry.
type unauthorizedError struct {
	cause error
}

// Error returns a string representation of an unauthorizedError
func (u unauthorizedError) Error() string {
	return u.cause.Error()
}

// IsErrUnauthorized returns true if the error is caused
// when a remote registry authentication fails
func IsErrUnauthorized(err error) bool {
	_, ok := err.(unauthorizedError)
	return ok
}

// nodeNotFoundError implements an error returned when a node is not found.
type nodeNotFoundError struct {
	nodeID string
}

// Error returns a string representation of a nodeNotFoundError
func (e nodeNotFoundError) Error() string {
	return fmt.Sprintf("Error: No such node: %s", e.nodeID)
}

// NoFound indicates that this error type is of NotFound
func (e nodeNotFoundError) NotFound() bool {
	return true
}

// IsErrNodeNotFound returns true if the error is caused
// when a node is not found.
func IsErrNodeNotFound(err error) bool {
	_, ok := err.(nodeNotFoundError)
	return ok
}

// serviceNotFoundError implements an error returned when a service is not found.
type serviceNotFoundError struct {
	serviceID string
}

// Error returns a string representation of a serviceNotFoundError
func (e serviceNotFoundError) Error() string {
	return fmt.Sprintf("Error: No such service: %s", e.serviceID)
}

// NoFound indicates that this error type is of NotFound
func (e serviceNotFoundError) NotFound() bool {
	return true
}

// IsErrServiceNotFound returns true if the error is caused
// when a service is not found.
func IsErrServiceNotFound(err error) bool {
	_, ok := err.(serviceNotFoundError)
	return ok
}

// taskNotFoundError implements an error returned when a task is not found.
type taskNotFoundError struct {
	taskID string
}

// Error returns a string representation of a taskNotFoundError
func (e taskNotFoundError) Error() string {
	return fmt.Sprintf("Error: No such task: %s", e.taskID)
}

// NoFound indicates that this error type is of NotFound
func (e taskNotFoundError) NotFound() bool {
	return true
}

// IsErrTaskNotFound returns true if the error is caused
// when a task is not found.
func IsErrTaskNotFound(err error) bool {
	_, ok := err.(taskNotFoundError)
	return ok
}

type pluginPermissionDenied struct {
	name string
}

func (e pluginPermissionDenied) Error() string {
	return "Permission denied while installing plugin " + e.name
}

// IsErrPluginPermissionDenied returns true if the error is caused
// when a user denies a plugin's permissions
func IsErrPluginPermissionDenied(err error) bool {
	_, ok := err.(pluginPermissionDenied)
	return ok
}
<|endoftext|>


<commit_before>//Project Euler Problem 551
//Currently using an n^3 algorithm -- fail
package main

import "fmt"
import "math/big"

func SumBig(n *big.Int, base int) int {
	var sum int
	i := new(big.Int).Set(n)
	b := new(big.Int).SetUint64(uint64(base))
	r := new(big.Int)
	for i.BitLen() > 0 {
		i.DivMod(i, b, r)
		sum += int(r.Uint64())
	}
	return sum
}

func sumSlice(slice []int) int {
	var sum int
	for _, v := range slice {
		sum += v
	}
	return sum
}

func sumDigits(n int, base int) int {
	var sum int
	for n > 0 {
		sum += n % base
		n /= base
	}
	return sum
}

func sumSliceDigits(slice []int) int {
	var sum int
	for _, v := range slice {
		sum += sumDigits(v, 10)
	}
	return sum
}

func euler551(n int) int {
	slice := make([]int, 1, 1e6)
	slice = append(slice, 1)
	for i := 1; i < 10e6; i++ {
		slice = append(slice, sumSliceDigits(slice))
	}
	return slice[len(slice)-1]
}

func main() {
	fmt.Println(euler551(0))
}
<commit_msg>missed an instance of 10e<commit_after>//Project Euler Problem 551
//Currently using an n^3 algorithm -- fail
package main

import "fmt"
import "math/big"

func SumBig(n *big.Int, base int) int {
	var sum int
	i := new(big.Int).Set(n)
	b := new(big.Int).SetUint64(uint64(base))
	r := new(big.Int)
	for i.BitLen() > 0 {
		i.DivMod(i, b, r)
		sum += int(r.Uint64())
	}
	return sum
}

func sumSlice(slice []int) int {
	var sum int
	for _, v := range slice {
		sum += v
	}
	return sum
}

func sumDigits(n int, base int) int {
	var sum int
	for n > 0 {
		sum += n % base
		n /= base
	}
	return sum
}

func sumSliceDigits(slice []int) int {
	var sum int
	for _, v := range slice {
		sum += sumDigits(v, 10)
	}
	return sum
}

func euler551(n int) int {
	slice := make([]int, 1, 1e6)
	slice = append(slice, 1)
	for i := 1; i < 1e6; i++ {
		slice = append(slice, sumSliceDigits(slice))
	}
	return slice[len(slice)-1]
}

func main() {
	fmt.Println(euler551(0))
}
<|endoftext|>


<commit_before>package atlas

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"reflect"
	"strconv"
)

// MeasurementResp contains all the results of the measurements
type MeasurementResp struct {
	Measurements []int
}

var (
	// If nothing is specified, use this
	defProbeSet = ProbeSet{
		{
			Requested: 10,
			Type:      "area",
			Value:     "WW",
			Tags:      nil,
		},
	}
)

// NewMeasurement create a new MeasurementRequest and fills some fields
func (client *Client) NewMeasurement() (req *MeasurementRequest) {
	var defs []Definition

	req = &MeasurementRequest{
		Definitions: defs,
		IsOneoff:    true,
		Probes:      defProbeSet,
	}
	return
}

// NewProbeSet create a set of probes for later requests
func NewProbeSet(howmany int) (ps *ProbeSet) {
	ps = &ProbeSet{
		{
			Requested: howmany,
			Type:      "area",
			Value:     "WW",
			Tags:      nil,
		},
	}
	return
}

// SetParams set a few parameters in a definition list
func (d *Definition) setParams(fields map[string]string) {
	sdef := reflect.ValueOf(d).Elem()
	typeOfDef := sdef.Type()
	for k, v := range fields {
		// Check the field is present
		if f, ok := typeOfDef.FieldByName(k); ok {
			// Use the right type
			switch f.Type.Name() {
			case "float":
				vf, _ := strconv.ParseFloat(v, 32)
				sdef.FieldByName(k).SetFloat(vf)
			case "int":
				vi, _ := strconv.ParseInt(v, 10, 32)
				sdef.FieldByName(k).SetInt(vi)
			case "string":
				sdef.FieldByName(k).SetString(v)
			case "bool":
				vb, _ := strconv.ParseBool(v)
				sdef.FieldByName(k).SetBool(vb)
			default:
				log.Printf("Unsupported type: %s", f.Type.Name())
			}
		}
	}
}

// AddDefinition create a new MeasurementRequest and fills some fields
func (m *MeasurementRequest) AddDefinition(fields map[string]string) *MeasurementRequest {
	def := new(Definition)
	def.setParams(fields)
	m.Definitions = append(m.Definitions, *def)

	return m
}

// createMeasurement creates a measurement for all types
func (client *Client) createMeasurement(t string, d *MeasurementRequest) (m *MeasurementResp, err error) {
	opts := make(map[string]string)
	req := client.prepareRequest("POST", fmt.Sprintf("measurements/%s", t), opts)

	body, err := json.Marshal(d)
	if err != nil {
		return
	}

	buf := bytes.NewReader(body)
	req.Body = ioutil.NopCloser(buf)

	log.Printf("body: %s", body)
	resp, err := client.call(req)
	log.Printf("resp: %v", resp)
	if err != nil {
		log.Printf("err: %v", err)
		//return
	}

	err = handleAPIResponse(resp)
	if err != nil {
		return
	}

	m = &MeasurementResp{}
	rbody, _ := ioutil.ReadAll(resp.Body)
	defer resp.Body.Close()

	err = json.Unmarshal(rbody, m)
	//r, err := api.Res(base, &resp).Post(d)
	fmt.Printf("m: %v\nresp: %#v\nd: %v\n", m, string(rbody), d)
	if err != nil {
		err = fmt.Errorf("err: %v - m:%v", err, m)
		return
	}

	return
}

// DNS creates a measurement
func (client *Client) DNS(d *MeasurementRequest) (m *MeasurementResp, err error) {
	return client.createMeasurement("dns", d)
}

// HTTP creates a measurement
func (client *Client) HTTP(d *MeasurementRequest) (m *MeasurementResp, err error) {
	return client.createMeasurement("http", d)
}

// NTP creates a measurement
func (client *Client) NTP(d *MeasurementRequest) (m *MeasurementResp, err error) {
	return client.createMeasurement("ntp", d)
}

// Ping creates a measurement
func (client *Client) Ping(d *MeasurementRequest) (m *MeasurementResp, err error) {
	return client.createMeasurement("ping", d)
}

// SSLCert creates a measurement
func (client *Client) SSLCert(d *MeasurementRequest) (m *MeasurementResp, err error) {
	return client.createMeasurement("sslcert", d)
}

// Traceroute creates a measurement
func (client *Client) Traceroute(d *MeasurementRequest) (m *MeasurementResp, err error) {
	return client.createMeasurement("traceroute", d)
}
<commit_msg>When denug is set, display everything.<commit_after>package atlas

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"reflect"
	"strconv"
)

// MeasurementResp contains all the results of the measurements
type MeasurementResp struct {
	Measurements []int
}

var (
	// If nothing is specified, use this
	defProbeSet = ProbeSet{
		{
			Requested: 10,
			Type:      "area",
			Value:     "WW",
			Tags:      nil,
		},
	}
)

// NewMeasurement create a new MeasurementRequest and fills some fields
func (client *Client) NewMeasurement() (req *MeasurementRequest) {
	var defs []Definition

	req = &MeasurementRequest{
		Definitions: defs,
		IsOneoff:    true,
		Probes:      defProbeSet,
	}
	return
}

// NewProbeSet create a set of probes for later requests
func NewProbeSet(howmany int) (ps *ProbeSet) {
	ps = &ProbeSet{
		{
			Requested: howmany,
			Type:      "area",
			Value:     "WW",
			Tags:      nil,
		},
	}
	return
}

// SetParams set a few parameters in a definition list
func (d *Definition) setParams(fields map[string]string) {
	sdef := reflect.ValueOf(d).Elem()
	typeOfDef := sdef.Type()
	for k, v := range fields {
		// Check the field is present
		if f, ok := typeOfDef.FieldByName(k); ok {
			// Use the right type
			switch f.Type.Name() {
			case "float":
				vf, _ := strconv.ParseFloat(v, 32)
				sdef.FieldByName(k).SetFloat(vf)
			case "int":
				vi, _ := strconv.ParseInt(v, 10, 32)
				sdef.FieldByName(k).SetInt(vi)
			case "string":
				sdef.FieldByName(k).SetString(v)
			case "bool":
				vb, _ := strconv.ParseBool(v)
				sdef.FieldByName(k).SetBool(vb)
			default:
				log.Printf("Unsupported type: %s", f.Type.Name())
			}
		}
	}
}

// AddDefinition create a new MeasurementRequest and fills some fields
func (m *MeasurementRequest) AddDefinition(fields map[string]string) *MeasurementRequest {
	def := new(Definition)
	def.setParams(fields)
	m.Definitions = append(m.Definitions, *def)

	return m
}

// createMeasurement creates a measurement for all types
func (client *Client) createMeasurement(t string, d *MeasurementRequest) (m *MeasurementResp, err error) {
	opts := make(map[string]string)
	req := client.prepareRequest("POST", fmt.Sprintf("measurements/%s", t), opts)

	body, err := json.Marshal(d)
	if err != nil {
		return
	}

	buf := bytes.NewReader(body)
	req.Body = ioutil.NopCloser(buf)
	req.ContentLength = int64(buf.Len())

	if client.config.Verbose {
		log.Printf("req: %#v", req)
		log.Printf("body: %s", body)
	}
	resp, err := client.call(req)
	if client.config.Verbose {
		log.Printf("resp: %v", resp)
	}
	if err != nil {
		log.Printf("err: %v", err)
		//return
	}

	err = handleAPIResponse(resp)
	if err != nil {
		return
	}

	m = &MeasurementResp{}
	rbody, _ := ioutil.ReadAll(resp.Body)
	defer resp.Body.Close()

	err = json.Unmarshal(rbody, m)
	//r, err := api.Res(base, &resp).Post(d)
	fmt.Printf("m: %v\nresp: %#v\nd: %v\n", m, string(rbody), d)
	if err != nil {
		err = fmt.Errorf("err: %v - m:%v", err, m)
		return
	}

	return
}

// DNS creates a measurement
func (client *Client) DNS(d *MeasurementRequest) (m *MeasurementResp, err error) {
	return client.createMeasurement("dns", d)
}

// HTTP creates a measurement
func (client *Client) HTTP(d *MeasurementRequest) (m *MeasurementResp, err error) {
	return client.createMeasurement("http", d)
}

// NTP creates a measurement
func (client *Client) NTP(d *MeasurementRequest) (m *MeasurementResp, err error) {
	return client.createMeasurement("ntp", d)
}

// Ping creates a measurement
func (client *Client) Ping(d *MeasurementRequest) (m *MeasurementResp, err error) {
	return client.createMeasurement("ping", d)
}

// SSLCert creates a measurement
func (client *Client) SSLCert(d *MeasurementRequest) (m *MeasurementResp, err error) {
	return client.createMeasurement("sslcert", d)
}

// Traceroute creates a measurement
func (client *Client) Traceroute(d *MeasurementRequest) (m *MeasurementResp, err error) {
	return client.createMeasurement("traceroute", d)
}
<|endoftext|>


<commit_before>package post

import (
	"bufio"
	"bytes"
	"encoding/json"
	"errors"
	"github.com/russross/blackfriday"
	"html/template"
	"io/ioutil"
	"log"
	"path/filepath"
	"strings"
	"time"
)

type BPost struct {
	Title           string        `json:"title"`
	Description     string        `json:"description"`
	Author          string        `json:"author"`
	UrlPermalink    string        `json:"url"`
	DateCreated     time.Time     `json:"date_created"`
	DateEdited      time.Time     `json:"date_edited"`
	ContentMarkdown string        `json:"content_markdown"`
	ContentHtml     template.HTML `json:"content_html"`
}

// ByDate implements sort.Interface for []BPost based on
// the DateCreated field.
type ByDate []*BPost

func (a ByDate) Len() int      { return len(a) }
func (a ByDate) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
func (a ByDate) Less(i, j int) bool {
	if a[i].DateCreated.Unix() > a[j].DateCreated.Unix() {
		return true
	}
	if a[i].DateCreated.Unix() < a[j].DateCreated.Unix() {
		return false
	}
	return a[i].DateEdited.Unix() >= a[j].DateEdited.Unix()
}

func (p *BPost) FormattedEditedTimeLong() string {
	return p.DateEdited.Format("January 02, 2006 | Monday -- 15:04PM")
}

func (p *BPost) FormattedCreatedTimeLong() string {
	return p.DateCreated.Format("January 02, 2006 | Monday -- 15:04PM")
}

func (p *BPost) FormattedCreatedTime() string {
	return p.DateCreated.Format("January 02, 2006 | Monday")
}

func (p *BPost) HTML5CreatedTime() string {
	return p.DateCreated.Format("2006-01-02")
}

func FromJson(b []byte) *BPost {
	bp := &BPost{}
	if err := json.Unmarshal(b, bp); err != nil {
		return nil
	}
	return bp
}

// FromMarkdown reads a post folder (that follows our special structure) and creates a new
// post structure with fields filled from the file loaded.
func FromMarkdown(pathname string) (*BPost, error) {
	bp := &BPost{}

	var err error

	// extract the date from the filename
	dateStr := filepath.Base(pathname)[:10]
	bp.DateCreated, err = time.Parse("2006-01-02", dateStr)
	if err != nil {
		return nil, err
	}

	// assign the Url to the current pathname but might be overwritten
	bp.UrlPermalink = filepath.Base(pathname)[11:]
	bp.UrlPermalink = bp.UrlPermalink[:len(bp.UrlPermalink)-3]

	markdown, err := ioutil.ReadFile(pathname)
	if err != nil {
		return nil, err
	}
	bytesRead, err := parseFrontMatter(bp, markdown)

	remBytes := markdown[bytesRead:]
	bp.ContentMarkdown = string(remBytes)
	bp.ContentHtml = template.HTML(string(blackfriday.MarkdownCommon(remBytes)))

	return bp, nil
}

func parseFrontMatter(bp *BPost, markdown []byte) (int, error) {
	bRead := 0
	scanner := bufio.NewScanner(bytes.NewReader(markdown))
	if scanner == nil {
		return -1, errors.New("Could not create reader from the markdown bytes: " + string(markdown))
	}

	foundDateEdited := false

	lines := 0
	for scanner.Scan() {
		bytesScanned := scanner.Bytes()
		bRead += len(bytesScanned)
		line := string(bytesScanned)
		lines++
		if line == "---" {
			if lines > 1 {
				// We found the second line of the matter so stop the reading
				break
			} else {
				// the first '---'
				continue
			}
		} else if lines == 1 {
			// no front-matter is defined - should start from the first line
			return 0, nil
		}
		segments := strings.Split(line, ":")
		switch segments[0] {
		case "title":
			bp.Title = strings.Trim(segments[1], " ")
			break
		case "url":
			bp.UrlPermalink = strings.Trim(segments[1], " ")
			break
		case "description":
			bp.Description = strings.Trim(segments[1], " ")
			break
		case "date-edited":
			// Ignore invalid dates.
			bp.DateEdited, _ = time.Parse("2006-01-02", strings.Trim(segments[1], " "))
			foundDateEdited = true
			break
		default:
			log.Println(errors.New("Wrong property defined in the front-matter: " + line))
		}
	}
	// add the new line bytes since they are discarded by the scanner
	bRead += len([]byte("\n")) * lines
	// Set the date edited to those created.
	if !foundDateEdited {
		bp.DateEdited = bp.DateCreated
	}
	return bRead, nil
}
<commit_msg>Fix a bug that does not split properly the front matter<commit_after>package post

import (
	"bufio"
	"bytes"
	"encoding/json"
	"errors"
	"html/template"
	"io/ioutil"
	"log"
	"path/filepath"
	"strings"
	"time"

	"github.com/russross/blackfriday"
)

type BPost struct {
	Title           string        `json:"title"`
	Description     string        `json:"description"`
	Author          string        `json:"author"`
	UrlPermalink    string        `json:"url"`
	DateCreated     time.Time     `json:"date_created"`
	DateEdited      time.Time     `json:"date_edited"`
	ContentMarkdown string        `json:"content_markdown"`
	ContentHtml     template.HTML `json:"content_html"`
}

// ByDate implements sort.Interface for []BPost based on
// the DateCreated field.
type ByDate []*BPost

func (a ByDate) Len() int      { return len(a) }
func (a ByDate) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
func (a ByDate) Less(i, j int) bool {
	if a[i].DateCreated.Unix() > a[j].DateCreated.Unix() {
		return true
	}
	if a[i].DateCreated.Unix() < a[j].DateCreated.Unix() {
		return false
	}
	return a[i].DateEdited.Unix() >= a[j].DateEdited.Unix()
}

func (p *BPost) FormattedEditedTimeLong() string {
	return p.DateEdited.Format("January 02, 2006 | Monday -- 15:04PM")
}

func (p *BPost) FormattedCreatedTimeLong() string {
	return p.DateCreated.Format("January 02, 2006 | Monday -- 15:04PM")
}

func (p *BPost) FormattedCreatedTime() string {
	return p.DateCreated.Format("January 02, 2006 | Monday")
}

func (p *BPost) HTML5CreatedTime() string {
	return p.DateCreated.Format("2006-01-02")
}

func FromJson(b []byte) *BPost {
	bp := &BPost{}
	if err := json.Unmarshal(b, bp); err != nil {
		return nil
	}
	return bp
}

// FromMarkdown reads a post folder (that follows our special structure) and creates a new
// post structure with fields filled from the file loaded.
func FromMarkdown(pathname string) (*BPost, error) {
	bp := &BPost{}

	var err error

	// extract the date from the filename
	dateStr := filepath.Base(pathname)[:10]
	bp.DateCreated, err = time.Parse("2006-01-02", dateStr)
	if err != nil {
		return nil, err
	}

	// assign the Url to the current pathname but might be overwritten
	bp.UrlPermalink = filepath.Base(pathname)[11:]
	bp.UrlPermalink = bp.UrlPermalink[:len(bp.UrlPermalink)-3]

	markdown, err := ioutil.ReadFile(pathname)
	if err != nil {
		return nil, err
	}
	bytesRead, err := parseFrontMatter(bp, markdown)

	remBytes := markdown[bytesRead:]
	bp.ContentMarkdown = string(remBytes)
	bp.ContentHtml = template.HTML(string(blackfriday.MarkdownCommon(remBytes)))

	return bp, nil
}

func parseFrontMatter(bp *BPost, markdown []byte) (int, error) {
	bRead := 0
	scanner := bufio.NewScanner(bytes.NewReader(markdown))
	if scanner == nil {
		return -1, errors.New("Could not create reader from the markdown bytes: " + string(markdown))
	}

	foundDateEdited := false

	lines := 0
	for scanner.Scan() {
		bytesScanned := scanner.Bytes()
		bRead += len(bytesScanned)
		line := string(bytesScanned)
		lines++
		if line == "---" {
			if lines > 1 {
				// We found the second line of the matter so stop the reading
				break
			} else {
				// the first '---'
				continue
			}
		} else if lines == 1 {
			// no front-matter is defined - should start from the first line
			return 0, nil
		}
		segments := strings.SplitN(line, ":", 2)
		switch segments[0] {
		case "title":
			bp.Title = strings.Trim(segments[1], " ")
			break
		case "url":
			bp.UrlPermalink = strings.Trim(segments[1], " ")
			break
		case "description":
			bp.Description = strings.Trim(segments[1], " ")
			break
		case "date-edited":
			// Ignore invalid dates.
			bp.DateEdited, _ = time.Parse("2006-01-02", strings.Trim(segments[1], " "))
			foundDateEdited = true
			break
		default:
			log.Println(errors.New("Wrong property defined in the front-matter: " + line))
		}
	}
	// add the new line bytes since they are discarded by the scanner
	bRead += len([]byte("\n")) * lines
	// Set the date edited to those created.
	if !foundDateEdited {
		bp.DateEdited = bp.DateCreated
	}
	return bRead, nil
}
<|endoftext|>


<commit_before>package daemon

import (
	"fmt"
	"os"
	"syscall"

	"github.com/Sirupsen/logrus"
	psignal "github.com/docker/docker/pkg/signal"
	"github.com/docker/docker/pkg/system"
)

func setupDumpStackTrap() {
	// Windows does not support signals like *nix systems. So instead of
	// trapping on SIGUSR1 to dump stacks, we wait on a Win32 event to be
	// signaled.
	go func() {
		sa := syscall.SecurityAttributes{
			Length: 0,
		}
		ev := "Global\\docker-daemon-" + fmt.Sprint(os.Getpid())
		if h, _ := system.CreateEvent(&sa, false, false, ev); h != 0 {
			logrus.Debugf("Stackdump - waiting signal at %s", ev)
			for {
				syscall.WaitForSingleObject(h, syscall.INFINITE)
				psignal.DumpStacks()
			}
		}
	}()
}
<commit_msg>Windows: psignal alias not required<commit_after>package daemon

import (
	"fmt"
	"os"
	"syscall"

	"github.com/Sirupsen/logrus"
	"github.com/docker/docker/pkg/signal"
	"github.com/docker/docker/pkg/system"
)

func setupDumpStackTrap() {
	// Windows does not support signals like *nix systems. So instead of
	// trapping on SIGUSR1 to dump stacks, we wait on a Win32 event to be
	// signaled.
	go func() {
		sa := syscall.SecurityAttributes{
			Length: 0,
		}
		ev := "Global\\docker-daemon-" + fmt.Sprint(os.Getpid())
		if h, _ := system.CreateEvent(&sa, false, false, ev); h != 0 {
			logrus.Debugf("Stackdump - waiting signal at %s", ev)
			for {
				syscall.WaitForSingleObject(h, syscall.INFINITE)
				signal.DumpStacks()
			}
		}
	}()
}
<|endoftext|>


<commit_before>// Copyright 2012 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cmac

import (
	"crypto/aes"
	"fmt"
	"math"
)

func padBlock(block []byte) []byte {
	blockLen := len(block)
	if blockLen == 16 {
		return block
	}

	result := make([]byte, 16)
	copy(result, block)
	result[blockLen] = 0x80

	return result
}

// Given a 128-bit key and a message, return a MAC that can be used to validate
// the input message. This is the AES-CMAC function of RFC 4493.
func generateCmac(key []byte, msg []byte) ([]byte, error) {
	if len(key) != 16 {
		return nil, fmt.Errorf("generateCmac requires a 16-byte key.")
	}

	msgLen := len(msg)

	// Generate subkeys.
	k1, k2 := generateSubkey(key)

	// Calculate the number of rounds.
	lastBlockComplete := false
	n := uint64(math.Ceil(float64(msgLen) / 16))
	if n == 0 {
		n = 1
	} else {
		if msgLen % 16 == 0 {
			lastBlockComplete = true
		}
	}

	// Calculate M_last.
	lastBlock := msg[16*(n-1):]
	var mLast []byte
	if lastBlockComplete {
		mLast = xor(lastBlock, k1)
	} else {
		mLast = xor(padBlock(lastBlock), k2)
	}

	// Create a cipher.
	c, err := aes.NewCipher(key)
	if err != nil {
		return nil, fmt.Errorf("aes.NewCipher: %v", err)
	}

	// Run the rounds.
	x := make([]byte, 16)
	for i := uint64(0); i < n-1; i++ {
		block := msg[16*i:16*(i+1)]
		y := xor(x, block)
		c.Encrypt(x, y)
	}

	y := xor(mLast, x)
	result := make([]byte, 16)
	c.Encrypt(result, y)

	return result, nil
}
<commit_msg>Started on a streaming implementation.<commit_after>// Copyright 2012 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cmac

import (
	"crypto/aes"
	"crypto/cipher"
	"fmt"
	"math"
)

func padBlock(block []byte) []byte {
	blockLen := len(block)
	if blockLen == 16 {
		return block
	}

	result := make([]byte, 16)
	copy(result, block)
	result[blockLen] = 0x80

	return result
}

type cmacHash struct {
	// An AES cipher configured with the original key.
	ciph cipher.Block

	// Generated sub-keys.
	k1 []byte
	k2 []byte

	// Data that has been seen since the last block was disposed of (i.e. since
	// we finished an iteration of the for loop in RFC 4493's AES-CMAC algorithm
	// and were sure we were going into a new one).
	data []byte

	// The current value of X, as defined in the AES-CMAC algorithm in RFC 4493.
	// Initially this is a 128-bit zero, and it is updated with the current block
	// when we're sure it's not the last one.
	x []byte
}

func (h *cmacHash) Write(p []byte) (n int, err error) {
	// Consume the data.
	n = len(p)
	h.data = append(h.data, p...)

	// Consume any blocks that we're sure aren't the last.
	blocksToProcess := len(h.data) / 16
	if blocksToProcess > 0 && len(h.data) % 16 == 0 {
		blocksToProcess--
	}

	for i := 0; i < blocksToProcess; i++ {
		block := h.data[16*i:16*(i+1)]
		y := xor(h.x, block)
		h.ciph.Encrypt(h.x, y)
	}

	h.data = h.data[16*blocksToProcess:]

	return
}

// Given a 128-bit key and a message, return a MAC that can be used to validate
// the input message. This is the AES-CMAC function of RFC 4493.
func generateCmac(key []byte, msg []byte) ([]byte, error) {
	if len(key) != 16 {
		return nil, fmt.Errorf("generateCmac requires a 16-byte key.")
	}

	msgLen := len(msg)

	// Generate subkeys.
	k1, k2 := generateSubkey(key)

	// Calculate the number of rounds.
	lastBlockComplete := false
	n := uint64(math.Ceil(float64(msgLen) / 16))
	if n == 0 {
		n = 1
	} else {
		if msgLen % 16 == 0 {
			lastBlockComplete = true
		}
	}

	// Calculate M_last.
	lastBlock := msg[16*(n-1):]
	var mLast []byte
	if lastBlockComplete {
		mLast = xor(lastBlock, k1)
	} else {
		mLast = xor(padBlock(lastBlock), k2)
	}

	// Create a cipher.
	c, err := aes.NewCipher(key)
	if err != nil {
		return nil, fmt.Errorf("aes.NewCipher: %v", err)
	}

	// Run the rounds.
	x := make([]byte, 16)
	for i := uint64(0); i < n-1; i++ {
		block := msg[16*i:16*(i+1)]
		y := xor(x, block)
		c.Encrypt(x, y)
	}

	y := xor(mLast, x)
	result := make([]byte, 16)
	c.Encrypt(result, y)

	return result, nil
}
<|endoftext|>


<commit_before>// Copyright 2017 Istio Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package framework

import (
	"bufio"
	"context"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"regexp"
	"strings"
	"sync"
	"time"

	logging "cloud.google.com/go/logging/apiv2"
	"cloud.google.com/go/storage"
	"github.com/golang/glog"
	"github.com/google/uuid"
	multierror "github.com/hashicorp/go-multierror"
	"google.golang.org/api/iterator"
	loggingpb "google.golang.org/genproto/googleapis/logging/v2"

	"istio.io/istio/tests/e2e/util"
)

var (
	logsBucketPath = flag.String("logs_bucket_path", "", "Cloud Storage Bucket path to use to store logs")
	projectID      = flag.String("project_id", "", "Project ID")
	resources      = []string{
		"pod",
		"service",
		"ingress",
	}
	testLogsPath = flag.String("test_logs_path", "", "Local path to store logs in")
	logIDs       = []string{
		"apiserver",
		"discovery",
		"istio-ingress",
		"mixer",
		"prometheus",
		"statesd-to-prometheus",
	}
)

const (
	tmpPrefix            = "istio.e2e."
	idMaxLength          = 36
	pageSize             = 1000 // number of log entries for each paginated request to fetch logs
	maxConcurrentWorkers = 2    //avoid overloading stackdriver api
)

// TestInfo gathers Test Information
type testInfo struct {
	RunID         string
	TestID        string
	Bucket        string
	LogBucketPath string
	LogsPath      string
}

// Test information
type testStatus struct {
	TestID string    `json:"test_id"`
	Status int       `json:"status"`
	RunID  string    `json:"run_id"`
	Date   time.Time `json:"date"`
}

// NewTestInfo creates a TestInfo given a test id.
func newTestInfo(testID string) (*testInfo, error) {
	id, err := generateRunID(testID)
	if err != nil {
		return nil, err
	}
	bucket := ""
	logsPath := ""
	var tmpDir string
	// testLogsPath will be used when called by Prow.
	// Bootstrap already gather stdout and stdin so we don't need to keep the logs from glog.
	if *testLogsPath != "" {
		tmpDir = path.Join(*testLogsPath, id)
		if err = os.MkdirAll(tmpDir, 0777); err != nil {
			return nil, err
		}
	} else {
		f := flag.Lookup("log_dir")
		tmpDir = f.Value.String()
		if tmpDir == "" {
			tmpDir, err = ioutil.TempDir(os.TempDir(), tmpPrefix)
			if err != nil {
				return nil, err
			}
		}
	}
	glog.Infof("Using log dir %s", tmpDir)

	if *logsBucketPath != "" {
		r := regexp.MustCompile(`gs://(?P<bucket>[^\/]+)/(?P<path>.+)`)
		m := r.FindStringSubmatch(*logsBucketPath)
		if m != nil {
			bucket = m[1]
			logsPath = m[2]
		} else {
			return nil, errors.New("cannot parse logBucketPath flag")
		}
	}
	if err != nil {
		return nil, errors.New("could not create a temporary dir")
	}
	// Need to setup logging here
	return &testInfo{
		TestID:        testID,
		RunID:         id,
		Bucket:        bucket,
		LogBucketPath: filepath.Join(logsPath, id),
		LogsPath:      tmpDir,
	}, nil
}

func (t testInfo) Setup() error {
	return nil
}

// Update sets the test status.
func (t testInfo) Update(r int) error {
	return t.createStatusFile(r)
}

func (t testInfo) FetchAndSaveClusterLogs(namespace string) error {
	if *projectID == "" {
		return nil
	}
	// connect to stackdriver
	ctx := context.Background()
	glog.Info("Fetching cluster logs")
	loggingClient, err := logging.NewClient(ctx)
	if err != nil {
		return err
	}

	fetchAndWrite := func(logID string) error {
		// fetch logs from pods created for this run only
		filter := fmt.Sprintf(
			`logName = "projects/%s/logs/%s" AND
			resource.labels.namespace_id = "%s"`,
			*projectID, logID, namespace)
		req := &loggingpb.ListLogEntriesRequest{
			ResourceNames: []string{"projects/" + *projectID},
			Filter:        filter,
		}
		it := loggingClient.ListLogEntries(ctx, req)
		// create log file in append mode
		path := filepath.Join(t.LogsPath, fmt.Sprintf("%s.log", logID))
		f, err := os.OpenFile(path, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0600)
		if err != nil {
			return err
		}
		defer func() {
			if err := f.Close(); err != nil {
				glog.Warningf("Error during closing file: %v\n", err)
			}
		}()
		// fetch log entries with pagination
		var entries []*loggingpb.LogEntry
		pager := iterator.NewPager(it, pageSize, "")
		for page := 0; ; page++ {
			pageToken, err := pager.NextPage(&entries)
			if err != nil {
				glog.Warningf("%s Iterator paging stops: %v", logID, err)
				return err
			}
			// append logs to file
			for _, logEntry := range entries {
				fmtTime := time.Unix(logEntry.GetTimestamp().Seconds, 0)
				timestamp := fmt.Sprintf("[%02d:%02d:%02d] ",
					fmtTime.Hour(), fmtTime.Minute(), fmtTime.Second())
				if _, err = f.WriteString(timestamp); err != nil {
					return err
				}
				log := logEntry.GetTextPayload()
				if _, err = f.WriteString(log); err != nil {
					return err
				}
				if len(log) == 0 || log[len(log)-1] != '\n' {
					if _, err = f.WriteString("\n"); err != nil {
						return err
					}
				}
			}
			if pageToken == "" {
				break
			}
		}
		return nil
	}

	var multiErr error
	var wg sync.WaitGroup
	// limit number of concurrent jobs to stay in stackdriver api quota
	jobQue := make(chan string, maxConcurrentWorkers)
	for _, logID := range logIDs {
		wg.Add(1)
		jobQue <- logID // blocked if jobQue channel is already filled
		// fetch logs in another go routine
		go func(logID string) {
			glog.Infof("Fetching logs on %s", logID)
			if err := fetchAndWrite(logID); err != nil {
				multiErr = multierror.Append(multiErr, err)
			}
			<-jobQue
			wg.Done()
		}(logID)
	}
	wg.Wait()

	for _, resrc := range resources {
		glog.Info(fmt.Sprintf("Fetching deployment info on %s\n", resrc))
		path := filepath.Join(t.LogsPath, fmt.Sprintf("%s.yaml", resrc))
		if yaml, err0 := util.Shell(fmt.Sprintf("kubectl get %s -n %s -o yaml", resrc, namespace)); err0 != nil {
			multiErr = multierror.Append(multiErr, err0)
		} else {
			if f, err1 := os.Create(path); err1 != nil {
				multiErr = multierror.Append(multiErr, err1)
			} else {
				if _, err2 := f.WriteString(fmt.Sprintf("%s\n", yaml)); err2 != nil {
					multiErr = multierror.Append(multiErr, err2)
				}
			}
		}
	}
	return multiErr
}

func (t testInfo) createStatusFile(r int) error {
	glog.Info("Creating status file")
	ts := testStatus{
		Status: r,
		Date:   time.Now(),
		TestID: t.TestID,
		RunID:  t.RunID,
	}
	fp := filepath.Join(t.LogsPath, fmt.Sprintf("%s.json", t.TestID))
	f, err := os.Create(fp)
	if err != nil {
		glog.Errorf("Could not create %s. Error %s", fp, err)
		return err
	}
	w := bufio.NewWriter(f)
	e := json.NewEncoder(w)
	e.SetIndent("", "  ")
	if err = e.Encode(ts); err == nil {
		if err = w.Flush(); err == nil {
			if err = f.Close(); err == nil {
				glog.Infof("Created Status file %s", fp)
			}
		}
	}
	return err
}

func (t testInfo) uploadDir() error {
	ctx := context.Background()
	client, err := storage.NewClient(ctx)
	if err != nil {
		glog.Errorf("Could not set Storage client. Error %s", err)
		return err
	}
	bkt := client.Bucket(t.Bucket)

	uploadFileFn := func(p string) error {
		// Relative Path
		rp, err := filepath.Rel(t.LogsPath, p)
		if err != nil {
			return err
		}
		rp = filepath.Join(t.LogBucketPath, rp)
		glog.Infof("Uploading %s to gs://%s/%s", p, rp, t.Bucket)
		o := bkt.Object(rp)
		w := o.NewWriter(ctx)
		var b []byte
		if b, err = ioutil.ReadFile(p); err == nil {
			if _, err = w.Write(b); err == nil {
				if err = w.Close(); err == nil {
					glog.Infof("Uploaded %s to gs://%s/%s", p, rp, t.Bucket)
				}
			}
		}
		return err
	}

	walkFn := func(path string, info os.FileInfo, err error) error {
		// We are filtering all errors here as we want filepath.Walk to go over all the files.
		if err != nil {
			glog.Warningf("Skipping %s", path, err)
			return filepath.SkipDir
		}
		if !info.IsDir() {
			if uploadFileFn(path) != nil {
				glog.Warningf("An error occurred when upload %s %s", path, err)
			}
		}
		return nil
	}
	return filepath.Walk(t.LogsPath, walkFn)
}

func (t testInfo) Teardown() error {
	if t.Bucket != "" {
		glog.Info("Uploading logs remotely")
		glog.Flush()
		return t.uploadDir()
	}
	return nil
}

func generateRunID(t string) (string, error) {
	u := uuid.New().String()
	u = strings.Replace(u, "-", "", -1)
	t = strings.Replace(t, "_", "-", -1)
	// We want at least 6 characters of uuid padding
	padding := idMaxLength - len(t)
	if padding < 6 {
		return "", fmt.Errorf("test name should be less that %d characters", idMaxLength-6)
	}
	return fmt.Sprintf("%s-%s", t, u[0:padding]), nil
}
<commit_msg>More logs (#670)<commit_after>// Copyright 2017 Istio Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package framework

import (
	"bufio"
	"context"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"regexp"
	"strings"
	"sync"
	"time"

	logging "cloud.google.com/go/logging/apiv2"
	"cloud.google.com/go/storage"
	"github.com/golang/glog"
	"github.com/google/uuid"
	multierror "github.com/hashicorp/go-multierror"
	"google.golang.org/api/iterator"
	loggingpb "google.golang.org/genproto/googleapis/logging/v2"

	"istio.io/istio/tests/e2e/util"
)

var (
	logsBucketPath = flag.String("logs_bucket_path", "", "Cloud Storage Bucket path to use to store logs")
	projectID      = flag.String("project_id", "", "Project ID")
	resources      = []string{
		"pod",
		"service",
		"ingress",
	}
	testLogsPath = flag.String("test_logs_path", "", "Local path to store logs in")
	logIDs       = []string{
		"app",
		"autoscaler",
		"calico-typha",
		"details",
		"discovery",
		"dnsmasq",
		"event-exporter",
		"fluentd-gcp",
		"heapster",
		"heapster-nanny",
		"istio-ca",
		"istio-ca-container",
		"istio-ingress",
		"istio-proxy",
		"mixer",
		"kubedns",
		"mongodb",
		"mysqldb",
		"proxy",
		"ratings",
		"reviews",
		"sidecar",
		"sidecar-initializer",
		"zipkin",
	}
)

const (
	tmpPrefix            = "istio.e2e."
	idMaxLength          = 36
	pageSize             = 1000 // number of log entries for each paginated request to fetch logs
	maxConcurrentWorkers = 2    //avoid overloading stackdriver api
)

// TestInfo gathers Test Information
type testInfo struct {
	RunID         string
	TestID        string
	Bucket        string
	LogBucketPath string
	LogsPath      string
}

// Test information
type testStatus struct {
	TestID string    `json:"test_id"`
	Status int       `json:"status"`
	RunID  string    `json:"run_id"`
	Date   time.Time `json:"date"`
}

// NewTestInfo creates a TestInfo given a test id.
func newTestInfo(testID string) (*testInfo, error) {
	id, err := generateRunID(testID)
	if err != nil {
		return nil, err
	}
	bucket := ""
	logsPath := ""
	var tmpDir string
	// testLogsPath will be used when called by Prow.
	// Bootstrap already gather stdout and stdin so we don't need to keep the logs from glog.
	if *testLogsPath != "" {
		tmpDir = path.Join(*testLogsPath, id)
		if err = os.MkdirAll(tmpDir, 0777); err != nil {
			return nil, err
		}
	} else {
		f := flag.Lookup("log_dir")
		tmpDir = f.Value.String()
		if tmpDir == "" {
			tmpDir, err = ioutil.TempDir(os.TempDir(), tmpPrefix)
			if err != nil {
				return nil, err
			}
		}
	}
	glog.Infof("Using log dir %s", tmpDir)

	if *logsBucketPath != "" {
		r := regexp.MustCompile(`gs://(?P<bucket>[^\/]+)/(?P<path>.+)`)
		m := r.FindStringSubmatch(*logsBucketPath)
		if m != nil {
			bucket = m[1]
			logsPath = m[2]
		} else {
			return nil, errors.New("cannot parse logBucketPath flag")
		}
	}
	if err != nil {
		return nil, errors.New("could not create a temporary dir")
	}
	// Need to setup logging here
	return &testInfo{
		TestID:        testID,
		RunID:         id,
		Bucket:        bucket,
		LogBucketPath: filepath.Join(logsPath, id),
		LogsPath:      tmpDir,
	}, nil
}

func (t testInfo) Setup() error {
	return nil
}

// Update sets the test status.
func (t testInfo) Update(r int) error {
	return t.createStatusFile(r)
}

func (t testInfo) FetchAndSaveClusterLogs(namespace string) error {
	if *projectID == "" {
		return nil
	}
	// connect to stackdriver
	ctx := context.Background()
	glog.Info("Fetching cluster logs")
	loggingClient, err := logging.NewClient(ctx)
	if err != nil {
		return err
	}

	fetchAndWrite := func(logID string) error {
		// fetch logs from pods created for this run only
		filter := fmt.Sprintf(
			`logName = "projects/%s/logs/%s" AND
			resource.labels.namespace_id = "%s"`,
			*projectID, logID, namespace)
		req := &loggingpb.ListLogEntriesRequest{
			ResourceNames: []string{"projects/" + *projectID},
			Filter:        filter,
		}
		it := loggingClient.ListLogEntries(ctx, req)
		// create log file in append mode
		path := filepath.Join(t.LogsPath, fmt.Sprintf("%s.log", logID))
		f, err := os.OpenFile(path, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0600)
		if err != nil {
			return err
		}
		defer func() {
			if err := f.Close(); err != nil {
				glog.Warningf("Error during closing file: %v\n", err)
			}
		}()
		// fetch log entries with pagination
		var entries []*loggingpb.LogEntry
		pager := iterator.NewPager(it, pageSize, "")
		for page := 0; ; page++ {
			pageToken, err := pager.NextPage(&entries)
			if err != nil {
				glog.Warningf("%s Iterator paging stops: %v", logID, err)
				return err
			}
			// append logs to file
			for _, logEntry := range entries {
				fmtTime := time.Unix(logEntry.GetTimestamp().Seconds, 0)
				timestamp := fmt.Sprintf("[%02d:%02d:%02d] ",
					fmtTime.Hour(), fmtTime.Minute(), fmtTime.Second())
				if _, err = f.WriteString(timestamp); err != nil {
					return err
				}
				log := logEntry.GetTextPayload()
				if _, err = f.WriteString(log); err != nil {
					return err
				}
				if len(log) == 0 || log[len(log)-1] != '\n' {
					if _, err = f.WriteString("\n"); err != nil {
						return err
					}
				}
			}
			if pageToken == "" {
				break
			}
		}
		return nil
	}

	var multiErr error
	var wg sync.WaitGroup
	// limit number of concurrent jobs to stay in stackdriver api quota
	jobQue := make(chan string, maxConcurrentWorkers)
	for _, logID := range logIDs {
		wg.Add(1)
		jobQue <- logID // blocked if jobQue channel is already filled
		// fetch logs in another go routine
		go func(logID string) {
			glog.Infof("Fetching logs on %s", logID)
			if err := fetchAndWrite(logID); err != nil {
				multiErr = multierror.Append(multiErr, err)
			}
			<-jobQue
			wg.Done()
		}(logID)
	}
	wg.Wait()

	for _, resrc := range resources {
		glog.Info(fmt.Sprintf("Fetching deployment info on %s\n", resrc))
		path := filepath.Join(t.LogsPath, fmt.Sprintf("%s.yaml", resrc))
		if yaml, err0 := util.Shell(fmt.Sprintf("kubectl get %s -n %s -o yaml", resrc, namespace)); err0 != nil {
			multiErr = multierror.Append(multiErr, err0)
		} else {
			if f, err1 := os.Create(path); err1 != nil {
				multiErr = multierror.Append(multiErr, err1)
			} else {
				if _, err2 := f.WriteString(fmt.Sprintf("%s\n", yaml)); err2 != nil {
					multiErr = multierror.Append(multiErr, err2)
				}
			}
		}
	}
	return multiErr
}

func (t testInfo) createStatusFile(r int) error {
	glog.Info("Creating status file")
	ts := testStatus{
		Status: r,
		Date:   time.Now(),
		TestID: t.TestID,
		RunID:  t.RunID,
	}
	fp := filepath.Join(t.LogsPath, fmt.Sprintf("%s.json", t.TestID))
	f, err := os.Create(fp)
	if err != nil {
		glog.Errorf("Could not create %s. Error %s", fp, err)
		return err
	}
	w := bufio.NewWriter(f)
	e := json.NewEncoder(w)
	e.SetIndent("", "  ")
	if err = e.Encode(ts); err == nil {
		if err = w.Flush(); err == nil {
			if err = f.Close(); err == nil {
				glog.Infof("Created Status file %s", fp)
			}
		}
	}
	return err
}

func (t testInfo) uploadDir() error {
	ctx := context.Background()
	client, err := storage.NewClient(ctx)
	if err != nil {
		glog.Errorf("Could not set Storage client. Error %s", err)
		return err
	}
	bkt := client.Bucket(t.Bucket)

	uploadFileFn := func(p string) error {
		// Relative Path
		rp, err := filepath.Rel(t.LogsPath, p)
		if err != nil {
			return err
		}
		rp = filepath.Join(t.LogBucketPath, rp)
		glog.Infof("Uploading %s to gs://%s/%s", p, rp, t.Bucket)
		o := bkt.Object(rp)
		w := o.NewWriter(ctx)
		var b []byte
		if b, err = ioutil.ReadFile(p); err == nil {
			if _, err = w.Write(b); err == nil {
				if err = w.Close(); err == nil {
					glog.Infof("Uploaded %s to gs://%s/%s", p, rp, t.Bucket)
				}
			}
		}
		return err
	}

	walkFn := func(path string, info os.FileInfo, err error) error {
		// We are filtering all errors here as we want filepath.Walk to go over all the files.
		if err != nil {
			glog.Warningf("Skipping %s", path, err)
			return filepath.SkipDir
		}
		if !info.IsDir() {
			if uploadFileFn(path) != nil {
				glog.Warningf("An error occurred when upload %s %s", path, err)
			}
		}
		return nil
	}
	return filepath.Walk(t.LogsPath, walkFn)
}

func (t testInfo) Teardown() error {
	if t.Bucket != "" {
		glog.Info("Uploading logs remotely")
		glog.Flush()
		return t.uploadDir()
	}
	return nil
}

func generateRunID(t string) (string, error) {
	u := uuid.New().String()
	u = strings.Replace(u, "-", "", -1)
	t = strings.Replace(t, "_", "-", -1)
	// We want at least 6 characters of uuid padding
	padding := idMaxLength - len(t)
	if padding < 6 {
		return "", fmt.Errorf("test name should be less that %d characters", idMaxLength-6)
	}
	return fmt.Sprintf("%s-%s", t, u[0:padding]), nil
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"net/http"
	"time"

	log "github.com/Sirupsen/logrus"

	"github.com/rvolosatovs/systemgo/config"
	"github.com/rvolosatovs/systemgo/system"
	"github.com/rvolosatovs/systemgo/systemctl"
)

// Initializes the system, sets the default paths, as specified in configuration and attempts to start the default target, falls back to "rescue.target", if it fails
func main() {
	// Initialize system
	System = system.New()
	System.SetPaths(config.Paths...)

	go Serve()

	// Start the default target
	if err := System.Start(config.Target); err != nil {
		log.Errorf("Error starting default target %s: %s", config.Target, err)
		if err = System.Start(config.RESCUE_TARGET); err != nil {
			log.Errorf("Error starting rescue target %s: %s", config.RESCUE_TARGET, err)
		}
	}

	select {}
}

// Instance of a system
var System *system.Daemon

// Listen for systemctl requests
func Serve() {
	for {
		if err := listenHTTP(config.Port.String()); err != nil {
			log.Errorf("Error listening on %v: %s", config.Port, err)
		}
		log.Infof("Retrying in %v seconds", config.Retry)
		time.Sleep(config.Retry)
	}
}

// Handle systemctl requests using HTTP
func listenHTTP(addr string) (err error) {
	server := http.NewServeMux()

	for name, handler := range systemctl.Handlers(System) {
		func(handler systemctl.Handler) {
			server.HandleFunc("/"+name, func(w http.ResponseWriter, req *http.Request) {
				//msg := []systemctl.Response{}
				v := req.URL.Query()

				names, ok := v["unit"]
				if !ok {
					names = []string{}
				}

				var resp []byte
				if resp, err = json.Marshal(handler(names...)); err != nil {
					log.Printf("json.Marshal(result): %s", err)
					return
				}

				if _, err = w.Write(resp); err != nil {
					log.Printf("Write(resp): %s", err)
				}
			})
		}(handler)
	}

	log.Infof("Listening on http://localhost%s", config.Port)
	return http.ListenAndServe(addr, server)
}
<commit_msg>init: debugging statements<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"net/http"
	"time"

	log "github.com/Sirupsen/logrus"

	"github.com/rvolosatovs/systemgo/config"
	"github.com/rvolosatovs/systemgo/system"
	"github.com/rvolosatovs/systemgo/systemctl"
)

// Initializes the system, sets the default paths, as specified in configuration and attempts to start the default target, falls back to "rescue.target", if it fails
func main() {
	// Initialize system
	System = system.New()
	System.SetPaths(config.Paths...)

	go Serve()

	// Start the default target
	if err := System.Start(config.Target); err != nil {
		log.Errorf("Error starting default target %s: %s", config.Target, err)
		if err = System.Start(config.RESCUE_TARGET); err != nil {
			log.Errorf("Error starting rescue target %s: %s", config.RESCUE_TARGET, err)
		}
	}

	if log.GetLevel() == log.DebugLevel {
		go func() {
			for range time.Tick(5 * time.Second) {
				for _, u := range System.Units() {
					st, err := System.StatusOf(u.Name())
					if err != nil {
						panic(err)
					}
					fmt.Println("********************************************************************************")
					fmt.Println("\t\t", u.Name())
					fmt.Println("********************************************************************************")
					fmt.Printf("->Status:\n%s\n", st)
					fmt.Println("--------------------------------------------------------------------------------")
					fmt.Printf("->Unit:\n%+v\n", u)
					fmt.Println("********************************************************************************")
				}
				fmt.Println("********************************************************************************")
				fmt.Println("********************************************************************************")
				fmt.Println("********************************************************************************")
			}
		}()
	}

	select {}
}

// Instance of a system
var System *system.Daemon

// Listen for systemctl requests
func Serve() {
	for {
		if err := listenHTTP(config.Port.String()); err != nil {
			log.Errorf("Error listening on %v: %s", config.Port, err)
		}
		log.Infof("Retrying in %v seconds", config.Retry)
		time.Sleep(config.Retry)
	}
}

// Handle systemctl requests using HTTP
func listenHTTP(addr string) (err error) {
	server := http.NewServeMux()

	for name, handler := range systemctl.Handlers(System) {
		func(handler systemctl.Handler) {
			server.HandleFunc("/"+name, func(w http.ResponseWriter, req *http.Request) {
				//msg := []systemctl.Response{}
				v := req.URL.Query()

				names, ok := v["unit"]
				if !ok {
					names = []string{}
				}

				var resp []byte
				if resp, err = json.Marshal(handler(names...)); err != nil {
					log.Printf("json.Marshal(result): %s", err)
					return
				}

				if _, err = w.Write(resp); err != nil {
					log.Printf("Write(resp): %s", err)
				}
			})
		}(handler)
	}

	log.Infof("Listening on http://localhost%s", config.Port)
	return http.ListenAndServe(addr, server)
}
<|endoftext|>


<commit_before>package main

import (
	"os"
	"os/signal"
	"path/filepath"
	"time"

	"context"

	netContext "golang.org/x/net/context"
	kingpin "gopkg.in/alecthomas/kingpin.v2"

	"github.com/keel-hq/keel/approvals"
	"github.com/keel-hq/keel/bot"
	"github.com/keel-hq/keel/cache/kubekv"

	"github.com/keel-hq/keel/constants"
	"github.com/keel-hq/keel/extension/credentialshelper"
	"github.com/keel-hq/keel/extension/notification"
	"github.com/keel-hq/keel/internal/k8s"
	"github.com/keel-hq/keel/internal/workgroup"
	"github.com/keel-hq/keel/provider"
	"github.com/keel-hq/keel/provider/helm"
	"github.com/keel-hq/keel/provider/kubernetes"
	"github.com/keel-hq/keel/registry"
	"github.com/keel-hq/keel/secrets"
	"github.com/keel-hq/keel/trigger/http"
	"github.com/keel-hq/keel/trigger/poll"
	"github.com/keel-hq/keel/trigger/pubsub"
	"github.com/keel-hq/keel/types"
	"github.com/keel-hq/keel/util/codecs"
	"github.com/keel-hq/keel/version"

	// notification extensions
	_ "github.com/keel-hq/keel/extension/notification/hipchat"
	_ "github.com/keel-hq/keel/extension/notification/mattermost"
	_ "github.com/keel-hq/keel/extension/notification/slack"
	_ "github.com/keel-hq/keel/extension/notification/webhook"

	// credentials helpers
	_ "github.com/keel-hq/keel/extension/credentialshelper/aws"
	secretsCredentialsHelper "github.com/keel-hq/keel/extension/credentialshelper/secrets"

	// bots
	_ "github.com/keel-hq/keel/bot/hipchat"
	_ "github.com/keel-hq/keel/bot/slack"

	log "github.com/sirupsen/logrus"
)

// gcloud pubsub related config
const (
	EnvTriggerPubSub = "PUBSUB" // set to 1 or something to enable pub/sub trigger
	EnvTriggerPoll   = "POLL"   // set to 1 or something to enable poll trigger
	EnvProjectID     = "PROJECT_ID"

	EnvNamespace = "NAMESPACE" // Keel's namespace

	EnvHelmProvider      = "HELM_PROVIDER"  // helm provider
	EnvHelmTillerAddress = "TILLER_ADDRESS" // helm provider

	// EnvDefaultDockerRegistryCfg - default registry configuration that can be passed into
	// keel for polling trigger
	EnvDefaultDockerRegistryCfg = "DOCKER_REGISTRY_CFG"
)

// kubernetes config, if empty - will default to InCluster
const (
	EnvKubernetesConfig = "KUBERNETES_CONFIG"
)

// EnvDebug - set to 1 or anything else to enable debug logging
const EnvDebug = "DEBUG"

func main() {
	ver := version.GetKeelVersion()

	inCluster := kingpin.Flag("incluster", "use in cluster configuration (defaults to 'true'), use '--no-incluster' if running outside of the cluster").Default("true").Bool()
	kubeconfig := kingpin.Flag("kubeconfig", "path to kubeconfig (if not in running inside a cluster)").Default(filepath.Join(os.Getenv("HOME"), ".kube", "config")).String()

	kingpin.UsageTemplate(kingpin.CompactUsageTemplate).Version(ver.Version)
	kingpin.CommandLine.Help = "Automated Kubernetes deployment updates. Learn more on https://keel.sh."
	kingpin.Parse()

	log.WithFields(log.Fields{
		"os":         ver.OS,
		"build_date": ver.BuildDate,
		"revision":   ver.Revision,
		"version":    ver.Version,
		"go_version": ver.GoVersion,
		"arch":       ver.Arch,
	}).Info("keel starting...")

	if os.Getenv(EnvDebug) != "" {
		log.SetLevel(log.DebugLevel)
	}

	// setting up triggers
	ctx, cancel := netContext.WithCancel(context.Background())
	defer cancel()

	notificationLevel := types.LevelInfo
	if os.Getenv(constants.EnvNotificationLevel) != "" {
		parsedLevel, err := types.ParseLevel(os.Getenv(constants.EnvNotificationLevel))
		if err != nil {
			log.WithFields(log.Fields{
				"error": err,
			}).Errorf("main: got error while parsing notification level, defaulting to: %s", notificationLevel)
		} else {
			notificationLevel = parsedLevel
		}
	}

	notifCfg := &notification.Config{
		Attempts: 10,
		Level:    notificationLevel,
	}
	sender := notification.New(ctx)

	_, err := sender.Configure(notifCfg)
	if err != nil {
		log.WithFields(log.Fields{
			"error": err,
		}).Fatal("main: failed to configure notification sender manager")
	}

	// getting k8s provider
	k8sCfg := &kubernetes.Opts{
		ConfigPath: *kubeconfig,
	}

	if os.Getenv(EnvKubernetesConfig) != "" {
		k8sCfg.ConfigPath = os.Getenv(EnvKubernetesConfig)
	}

	k8sCfg.InCluster = *inCluster

	implementer, err := kubernetes.NewKubernetesImplementer(k8sCfg)
	if err != nil {
		log.WithFields(log.Fields{
			"error":  err,
			"config": k8sCfg,
		}).Fatal("main: failed to create kubernetes implementer")
	}

	var g workgroup.Group

	t := &k8s.Translator{
		FieldLogger: log.WithField("context", "translator"),
	}

	buf := k8s.NewBuffer(&g, t, log.StandardLogger(), 128)
	wl := log.WithField("context", "watch")
	k8s.WatchDeployments(&g, implementer.Client(), wl, buf)
	k8s.WatchStatefulSets(&g, implementer.Client(), wl, buf)
	k8s.WatchDaemonSets(&g, implementer.Client(), wl, buf)

	keelsNamespace := constants.DefaultNamespace
	if os.Getenv(EnvNamespace) != "" {
		keelsNamespace = os.Getenv(EnvNamespace)
	}

	kkv, err := kubekv.New(implementer.ConfigMaps(keelsNamespace), "approvals")
	if err != nil {
		log.WithFields(log.Fields{
			"error":     err,
			"namespace": keelsNamespace,
		}).Fatal("main: failed to initialise kube-kv")
	}

	serializer := codecs.DefaultSerializer()
	approvalsManager := approvals.New(kkv, serializer)

	go approvalsManager.StartExpiryService(ctx)

	// setting up providers
	providers := setupProviders(implementer, sender, approvalsManager, &t.GenericResourceCache)

	// registering secrets based credentials helper
	dockerConfig := make(secrets.DockerCfg)
	if os.Getenv(EnvDefaultDockerRegistryCfg) != "" {
		dockerConfigStr := os.Getenv(EnvDefaultDockerRegistryCfg)
		dockerConfig, err = secrets.DecodeDockerCfgJson([]byte(dockerConfigStr))
		if err != nil {
			log.WithFields(log.Fields{
				"error": err,
			}).Fatalf("failed to decode secret provided in %s env variable", EnvDefaultDockerRegistryCfg)
		}
	}
	secretsGetter := secrets.NewGetter(implementer, dockerConfig)

	ch := secretsCredentialsHelper.New(secretsGetter)
	credentialshelper.RegisterCredentialsHelper("secrets", ch)

	// trigger setup
	teardownTriggers := setupTriggers(ctx, providers, approvalsManager)

	bot.Run(implementer, approvalsManager)

	signalChan := make(chan os.Signal, 1)
	cleanupDone := make(chan bool)
	signal.Notify(signalChan, os.Interrupt)
	g.Add(func(stop <-chan struct{}) {
		go func() {
			for range signalChan {
				log.Info("received an interrupt, shutting down...")
				go func() {
					select {
					case <-time.After(10 * time.Second):
						log.Info("connection shutdown took too long, exiting... ")
						close(cleanupDone)
						return
					case <-cleanupDone:
						return
					}
				}()
				providers.Stop()
				teardownTriggers()
				bot.Stop()

				cleanupDone <- true
			}
		}()
		<-cleanupDone
	})
	g.Run()
}

// setupProviders - setting up available providers. New providers should be initialised here and added to
// provider map
func setupProviders(k8sImplementer kubernetes.Implementer, sender notification.Sender, approvalsManager approvals.Manager, grc *k8s.GenericResourceCache) (providers provider.Providers) {
	var enabledProviders []provider.Provider

	k8sProvider, err := kubernetes.NewProvider(k8sImplementer, sender, approvalsManager, grc)
	if err != nil {
		log.WithFields(log.Fields{
			"error": err,
		}).Fatal("main.setupProviders: failed to create kubernetes provider")
	}
	go k8sProvider.Start()
	enabledProviders = append(enabledProviders, k8sProvider)

	if os.Getenv(EnvHelmProvider) == "1" {
		tillerAddr := os.Getenv(EnvHelmTillerAddress)
		helmImplementer := helm.NewHelmImplementer(tillerAddr)
		helmProvider := helm.NewProvider(helmImplementer, sender, approvalsManager)

		go helmProvider.Start()
		enabledProviders = append(enabledProviders, helmProvider)
	}

	providers = provider.New(enabledProviders, approvalsManager)

	return providers
}

// setupTriggers - setting up triggers. New triggers should be added to this function. Each trigger
// should go through all providers (or not if there is a reason) and submit events)
func setupTriggers(ctx context.Context, providers provider.Providers, approvalsManager approvals.Manager) (teardown func()) {

	// setting up generic http webhook server
	whs := http.NewTriggerServer(&http.Opts{
		Port:            types.KeelDefaultPort,
		Providers:       providers,
		ApprovalManager: approvalsManager,
	})

	go whs.Start()

	// checking whether pubsub (GCR) trigger is enabled
	if os.Getenv(EnvTriggerPubSub) != "" {
		projectID := os.Getenv(EnvProjectID)
		if projectID == "" {
			log.Fatalf("main.setupTriggers: project ID env variable not set")
			return
		}

		ps, err := pubsub.NewPubsubSubscriber(&pubsub.Opts{
			ProjectID: projectID,
			Providers: providers,
		})
		if err != nil {
			log.WithFields(log.Fields{
				"error": err,
			}).Fatal("main.setupTriggers: failed to create gcloud pubsub subscriber")
			return
		}

		subManager := pubsub.NewDefaultManager(projectID, providers, ps)
		go subManager.Start(ctx)
	}

	if os.Getenv(EnvTriggerPoll) != "0" {

		registryClient := registry.New()
		watcher := poll.NewRepositoryWatcher(providers, registryClient)
		pollManager := poll.NewPollManager(providers, watcher)

		// start poll manager, will finish with ctx
		go watcher.Start(ctx)
		go pollManager.Start(ctx)
	}

	teardown = func() {
		whs.Stop()
	}

	return teardown
}
<commit_msg>switching to in-memory cache for approvals<commit_after>package main

import (
	"os"
	"os/signal"
	"path/filepath"
	"time"

	"context"

	netContext "golang.org/x/net/context"
	kingpin "gopkg.in/alecthomas/kingpin.v2"

	"github.com/keel-hq/keel/approvals"
	"github.com/keel-hq/keel/bot"
	"github.com/keel-hq/keel/cache/memory"

	"github.com/keel-hq/keel/constants"
	"github.com/keel-hq/keel/extension/credentialshelper"
	"github.com/keel-hq/keel/extension/notification"
	"github.com/keel-hq/keel/internal/k8s"
	"github.com/keel-hq/keel/internal/workgroup"
	"github.com/keel-hq/keel/provider"
	"github.com/keel-hq/keel/provider/helm"
	"github.com/keel-hq/keel/provider/kubernetes"
	"github.com/keel-hq/keel/registry"
	"github.com/keel-hq/keel/secrets"
	"github.com/keel-hq/keel/trigger/http"
	"github.com/keel-hq/keel/trigger/poll"
	"github.com/keel-hq/keel/trigger/pubsub"
	"github.com/keel-hq/keel/types"
	"github.com/keel-hq/keel/util/codecs"
	"github.com/keel-hq/keel/version"

	// notification extensions
	_ "github.com/keel-hq/keel/extension/notification/hipchat"
	_ "github.com/keel-hq/keel/extension/notification/mattermost"
	_ "github.com/keel-hq/keel/extension/notification/slack"
	_ "github.com/keel-hq/keel/extension/notification/webhook"

	// credentials helpers
	_ "github.com/keel-hq/keel/extension/credentialshelper/aws"
	secretsCredentialsHelper "github.com/keel-hq/keel/extension/credentialshelper/secrets"

	// bots
	_ "github.com/keel-hq/keel/bot/hipchat"
	_ "github.com/keel-hq/keel/bot/slack"

	log "github.com/sirupsen/logrus"
)

// gcloud pubsub related config
const (
	EnvTriggerPubSub = "PUBSUB" // set to 1 or something to enable pub/sub trigger
	EnvTriggerPoll   = "POLL"   // set to 1 or something to enable poll trigger
	EnvProjectID     = "PROJECT_ID"

	EnvNamespace = "NAMESPACE" // Keel's namespace

	EnvHelmProvider      = "HELM_PROVIDER"  // helm provider
	EnvHelmTillerAddress = "TILLER_ADDRESS" // helm provider

	// EnvDefaultDockerRegistryCfg - default registry configuration that can be passed into
	// keel for polling trigger
	EnvDefaultDockerRegistryCfg = "DOCKER_REGISTRY_CFG"
)

// kubernetes config, if empty - will default to InCluster
const (
	EnvKubernetesConfig = "KUBERNETES_CONFIG"
)

// EnvDebug - set to 1 or anything else to enable debug logging
const EnvDebug = "DEBUG"

func main() {
	ver := version.GetKeelVersion()

	inCluster := kingpin.Flag("incluster", "use in cluster configuration (defaults to 'true'), use '--no-incluster' if running outside of the cluster").Default("true").Bool()
	kubeconfig := kingpin.Flag("kubeconfig", "path to kubeconfig (if not in running inside a cluster)").Default(filepath.Join(os.Getenv("HOME"), ".kube", "config")).String()

	kingpin.UsageTemplate(kingpin.CompactUsageTemplate).Version(ver.Version)
	kingpin.CommandLine.Help = "Automated Kubernetes deployment updates. Learn more on https://keel.sh."
	kingpin.Parse()

	log.WithFields(log.Fields{
		"os":         ver.OS,
		"build_date": ver.BuildDate,
		"revision":   ver.Revision,
		"version":    ver.Version,
		"go_version": ver.GoVersion,
		"arch":       ver.Arch,
	}).Info("keel starting...")

	if os.Getenv(EnvDebug) != "" {
		log.SetLevel(log.DebugLevel)
	}

	// setting up triggers
	ctx, cancel := netContext.WithCancel(context.Background())
	defer cancel()

	notificationLevel := types.LevelInfo
	if os.Getenv(constants.EnvNotificationLevel) != "" {
		parsedLevel, err := types.ParseLevel(os.Getenv(constants.EnvNotificationLevel))
		if err != nil {
			log.WithFields(log.Fields{
				"error": err,
			}).Errorf("main: got error while parsing notification level, defaulting to: %s", notificationLevel)
		} else {
			notificationLevel = parsedLevel
		}
	}

	notifCfg := &notification.Config{
		Attempts: 10,
		Level:    notificationLevel,
	}
	sender := notification.New(ctx)

	_, err := sender.Configure(notifCfg)
	if err != nil {
		log.WithFields(log.Fields{
			"error": err,
		}).Fatal("main: failed to configure notification sender manager")
	}

	// getting k8s provider
	k8sCfg := &kubernetes.Opts{
		ConfigPath: *kubeconfig,
	}

	if os.Getenv(EnvKubernetesConfig) != "" {
		k8sCfg.ConfigPath = os.Getenv(EnvKubernetesConfig)
	}

	k8sCfg.InCluster = *inCluster

	implementer, err := kubernetes.NewKubernetesImplementer(k8sCfg)
	if err != nil {
		log.WithFields(log.Fields{
			"error":  err,
			"config": k8sCfg,
		}).Fatal("main: failed to create kubernetes implementer")
	}

	var g workgroup.Group

	t := &k8s.Translator{
		FieldLogger: log.WithField("context", "translator"),
	}

	buf := k8s.NewBuffer(&g, t, log.StandardLogger(), 128)
	wl := log.WithField("context", "watch")
	k8s.WatchDeployments(&g, implementer.Client(), wl, buf)
	k8s.WatchStatefulSets(&g, implementer.Client(), wl, buf)
	k8s.WatchDaemonSets(&g, implementer.Client(), wl, buf)

	approvalsCache := memory.NewMemoryCache(24*time.Hour, 24*time.Hour, 10*time.Second)

	serializer := codecs.DefaultSerializer()
	approvalsManager := approvals.New(approvalsCache, serializer)

	go approvalsManager.StartExpiryService(ctx)

	// setting up providers
	providers := setupProviders(implementer, sender, approvalsManager, &t.GenericResourceCache)

	// registering secrets based credentials helper
	dockerConfig := make(secrets.DockerCfg)
	if os.Getenv(EnvDefaultDockerRegistryCfg) != "" {
		dockerConfigStr := os.Getenv(EnvDefaultDockerRegistryCfg)
		dockerConfig, err = secrets.DecodeDockerCfgJson([]byte(dockerConfigStr))
		if err != nil {
			log.WithFields(log.Fields{
				"error": err,
			}).Fatalf("failed to decode secret provided in %s env variable", EnvDefaultDockerRegistryCfg)
		}
	}
	secretsGetter := secrets.NewGetter(implementer, dockerConfig)

	ch := secretsCredentialsHelper.New(secretsGetter)
	credentialshelper.RegisterCredentialsHelper("secrets", ch)

	// trigger setup
	teardownTriggers := setupTriggers(ctx, providers, approvalsManager)

	bot.Run(implementer, approvalsManager)

	signalChan := make(chan os.Signal, 1)
	cleanupDone := make(chan bool)
	signal.Notify(signalChan, os.Interrupt)
	g.Add(func(stop <-chan struct{}) {
		go func() {
			for range signalChan {
				log.Info("received an interrupt, shutting down...")
				go func() {
					select {
					case <-time.After(10 * time.Second):
						log.Info("connection shutdown took too long, exiting... ")
						close(cleanupDone)
						return
					case <-cleanupDone:
						return
					}
				}()
				providers.Stop()
				teardownTriggers()
				bot.Stop()

				cleanupDone <- true
			}
		}()
		<-cleanupDone
	})
	g.Run()
}

// setupProviders - setting up available providers. New providers should be initialised here and added to
// provider map
func setupProviders(k8sImplementer kubernetes.Implementer, sender notification.Sender, approvalsManager approvals.Manager, grc *k8s.GenericResourceCache) (providers provider.Providers) {
	var enabledProviders []provider.Provider

	k8sProvider, err := kubernetes.NewProvider(k8sImplementer, sender, approvalsManager, grc)
	if err != nil {
		log.WithFields(log.Fields{
			"error": err,
		}).Fatal("main.setupProviders: failed to create kubernetes provider")
	}
	go k8sProvider.Start()
	enabledProviders = append(enabledProviders, k8sProvider)

	if os.Getenv(EnvHelmProvider) == "1" {
		tillerAddr := os.Getenv(EnvHelmTillerAddress)
		helmImplementer := helm.NewHelmImplementer(tillerAddr)
		helmProvider := helm.NewProvider(helmImplementer, sender, approvalsManager)

		go helmProvider.Start()
		enabledProviders = append(enabledProviders, helmProvider)
	}

	providers = provider.New(enabledProviders, approvalsManager)

	return providers
}

// setupTriggers - setting up triggers. New triggers should be added to this function. Each trigger
// should go through all providers (or not if there is a reason) and submit events)
func setupTriggers(ctx context.Context, providers provider.Providers, approvalsManager approvals.Manager) (teardown func()) {

	// setting up generic http webhook server
	whs := http.NewTriggerServer(&http.Opts{
		Port:            types.KeelDefaultPort,
		Providers:       providers,
		ApprovalManager: approvalsManager,
	})

	go whs.Start()

	// checking whether pubsub (GCR) trigger is enabled
	if os.Getenv(EnvTriggerPubSub) != "" {
		projectID := os.Getenv(EnvProjectID)
		if projectID == "" {
			log.Fatalf("main.setupTriggers: project ID env variable not set")
			return
		}

		ps, err := pubsub.NewPubsubSubscriber(&pubsub.Opts{
			ProjectID: projectID,
			Providers: providers,
		})
		if err != nil {
			log.WithFields(log.Fields{
				"error": err,
			}).Fatal("main.setupTriggers: failed to create gcloud pubsub subscriber")
			return
		}

		subManager := pubsub.NewDefaultManager(projectID, providers, ps)
		go subManager.Start(ctx)
	}

	if os.Getenv(EnvTriggerPoll) != "0" {

		registryClient := registry.New()
		watcher := poll.NewRepositoryWatcher(providers, registryClient)
		pollManager := poll.NewPollManager(providers, watcher)

		// start poll manager, will finish with ctx
		go watcher.Start(ctx)
		go pollManager.Start(ctx)
	}

	teardown = func() {
		whs.Stop()
	}

	return teardown
}
<|endoftext|>