<commit_before>// Copyright (c) 2016 ECS Team, Inc. - All Rights Reserved
// https://github.com/ECSTeam/cloudfoundry-top-plugin
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package top

import (
	"bufio"
	"crypto/tls"
	"fmt"
	"os"
	"strings"
	"time"

	"github.com/cloudfoundry/cli/cf/terminal"
	"github.com/cloudfoundry/cli/plugin"
	"github.com/cloudfoundry/noaa/consumer"
	"github.com/cloudfoundry/sonde-go/events"
	"github.com/gorilla/websocket"

	"github.com/ecsteam/cloudfoundry-top-plugin/eventrouting"
	"github.com/ecsteam/cloudfoundry-top-plugin/toplog"
	"github.com/ecsteam/cloudfoundry-top-plugin/ui"
	gops "github.com/mitchellh/go-ps"
)

type Client struct {
	options       *ClientOptions
	ui            terminal.UI
	cliConnection plugin.CliConnection
	eventrouting  *eventrouting.EventRouter
	router        *eventrouting.EventRouter
}

type ClientOptions struct {
	AppGUID    string
	Debug      bool
	NoTopCheck bool
	Cygwin     bool
	Nozzles    int
}

func NewClient(cliConnection plugin.CliConnection, options *ClientOptions, ui terminal.UI) *Client {

	return &Client{
		options:       options,
		ui:            ui,
		cliConnection: cliConnection,
	}
}

// Created mine own Ask func instead of the CF provided one because the CF version adds
// a call to PromptColor(">") which does not work cleanly on MS-Windows
func (c *Client) Ask(prompt string) string {
	fmt.Printf("%s ", prompt)
	rd := bufio.NewReader(os.Stdin)
	line, err := rd.ReadString('\n')
	if err == nil {
		return strings.TrimSpace(line)
	}
	return ""
}

func (c *Client) Start() {

	if !c.options.NoTopCheck && c.shouldExitTop() {
		// There are other instances of top running and user requested to exit
		return
	}

	toplog.SetDebugEnabled(c.options.Debug)

	conn := c.cliConnection

	isLoggedIn, err := conn.IsLoggedIn()
	if err != nil {
		c.ui.Failed(err.Error())
		return
	}
	if !isLoggedIn {
		c.ui.Failed("Must login first")
		return
	}

	fmt.Printf("Loading...")
	// We request an access token to confirm that authentication has not expired
	_, err = conn.AccessToken()
	if err != nil {
		c.ui.Failed("AccessToken failed: %v", err)
		return
	}
	fmt.Printf("\r           \r")

	ui := ui.NewMasterUI(conn)

	c.router = ui.GetRouter()

	toplog.Info("Top started at " + time.Now().Format("01-02-2006 15:04:05"))

	apps, err := c.cliConnection.GetApps()
	if err != nil {
		c.ui.Failed("Fetching all Apps failed: %v", err)
		return
	}

	for i, application := range apps {
		c.createNozzles(application.Guid, application.Name, i)
	}

	ui.Start()

}

func (c *Client) createNozzles(appGUID, appName string, instanceID int) {
	go c.createAndKeepAliveNozzle(appGUID, instanceID)
	toplog.Info("Starting nozzle #%v instance for App %s", instanceID, appName)
}

func (c *Client) createAndKeepAliveNozzle(appGUID string, instanceID int) error {

	minRetrySeconds := (2 * time.Second)

	for {
		// This is a blocking call if no error
		startTime := time.Now()
		err := c.createNozzle(appGUID, instanceID)
		if err != nil {
			errMsg := err.Error()
			notAuthorized := strings.Contains(errMsg, "authorized")
			if websocket.IsCloseError(err, websocket.CloseNormalClosure) || notAuthorized {
				toplog.Error("Nozzle #%v - Stopped with error: %v", instanceID, err)
				if notAuthorized {
					toplog.Error("Are you sure you have 'admin' privileges on foundation?")
					toplog.Error("See needed permissions for this plugin here:")
					toplog.Error("https://github.com/ECSTeam/cloudfoundry-top-plugin")
				}
				break
			}
			toplog.Warn("Nozzle #%v - error: %v", instanceID, err)
		}
		toplog.Warn("Nozzle #%v - Shutdown. Nozzle instance will be restarted", instanceID)
		lastRetry := time.Now().Sub(startTime)
		if lastRetry < minRetrySeconds {
			toplog.Info("Nozzle #%v - Nozzle instance restart too fast, delaying for %v", instanceID, minRetrySeconds)
			time.Sleep(minRetrySeconds)
		}
	}
	return nil
}

func (c *Client) createNozzle(appGUID string, instanceID int) error {

	// Delay each nozzle instance creation by 1 second
	time.Sleep(time.Duration(instanceID) * time.Second)

	conn := c.cliConnection
	dopplerEndpoint, err := conn.DopplerEndpoint()
	if err != nil {
		return err
	}

	skipVerifySSL, err := conn.IsSSLDisabled()
	if err != nil {
		return err
	}

	dopplerConnection := consumer.New(dopplerEndpoint, &tls.Config{InsecureSkipVerify: skipVerifySSL}, nil)

	tokenRefresher := NewTokenRefresher(conn, instanceID)
	dopplerConnection.RefreshTokenFrom(tokenRefresher)

	authToken, err := conn.AccessToken()
	if err != nil {
		return err
	}

	messages, errors := dopplerConnection.StreamWithoutReconnect(appGUID, authToken)
	defer dopplerConnection.Close()

	toplog.Info("Nozzle #%v for %s - Started", instanceID, appGUID)

	eventError := c.routeEvents(instanceID, messages, errors)
	if eventError != nil {
		msg := eventError.Error()
		if strings.Contains(msg, "Invalid authorization") {
			return eventError
		}
	}
	return nil
}

func (c *Client) routeEvents(instanceID int, messages <-chan *events.Envelope, errors <-chan error) error {
	for {
		select {
		case envelope := <-messages:
			c.router.Route(instanceID, envelope)
		case err := <-errors:
			c.handleError(instanceID, err)
			// Nozzle connection does not seem to recover from errors well, so
			// return here so it can be closed and a new instanced opened
			return err
		}
	}
}

func (c *Client) handleError(instanceID int, err error) {

	switch {
	case websocket.IsCloseError(err, websocket.CloseNormalClosure):
		msg := fmt.Sprintf("Nozzle #%v - Normal Websocket Closure: %v", instanceID, err)
		toplog.Error(msg)
	case websocket.IsCloseError(err, websocket.ClosePolicyViolation):
		msg := fmt.Sprintf("Nozzle #%v - Disconnected because nozzle couldn't keep up (CloseError): %v", instanceID, err)
		toplog.Error(msg)
	default:
		msg := fmt.Sprintf("Nozzle #%v - Error reading firehose: %v", instanceID, err)
		toplog.Error(msg)
	}
}

func (c *Client) shouldExitTop() bool {

	numRunning := c.getNumberOfTopPluginsRunning() - 1
	if numRunning > 0 {
		plural := ""
		if numRunning > 1 {
			plural = "s"
		}
		fmt.Printf("Currently %v instance%v of cf top already running on this OS.\n", numRunning, plural)
		for {
			response := c.Ask("Do you want to continue and start top? (y/n)")
			response = strings.ToLower(response)
			if response == "y" || response == "yes" {
				break
			} else if response == "n" || response == "no" {
				fmt.Printf("Top will not be started\n")
				return true
			}
		}
	}
	return false
}

// Check how many instance of this plugin are running on the currently
// OS.  This is based on checking the process list for processes that
// have the same name as our plugin process name.
func (c *Client) getNumberOfTopPluginsRunning() int {

	p, err := gops.Processes()
	if err != nil {
		fmt.Printf("err: %s", err)
	}

	if len(p) <= 0 {
		fmt.Printf("should have processes")
	}

	processName := ""
	for _, p1 := range p {
		if os.Getpid() == p1.Pid() {
			processName = p1.Executable()
		}
	}

	numberRunning := 0
	for _, p1 := range p {
		if p1.Executable() == processName {
			//fmt.Printf("Found: %v  PID: %v\n", p1.Executable(), p1.Pid())
			numberRunning++
		}
	}

	//fmt.Printf("Number of programs running with my same name: %v\n", numberRunning)
	return numberRunning
}
<commit_msg>Toggle firhose opening based on privileges<commit_after>// Copyright (c) 2016 ECS Team, Inc. - All Rights Reserved
// https://github.com/ECSTeam/cloudfoundry-top-plugin
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package top

import (
	"bufio"
	"crypto/tls"
	"encoding/base64"
	"fmt"
	"os"
	"strings"
	"time"

	"github.com/Jeffail/gabs"
	"github.com/cloudfoundry/cli/cf/terminal"
	"github.com/cloudfoundry/cli/plugin"
	"github.com/cloudfoundry/noaa/consumer"
	"github.com/cloudfoundry/sonde-go/events"
	"github.com/gorilla/websocket"

	"github.com/ecsteam/cloudfoundry-top-plugin/eventrouting"
	"github.com/ecsteam/cloudfoundry-top-plugin/toplog"
	"github.com/ecsteam/cloudfoundry-top-plugin/ui"
	"github.com/ecsteam/cloudfoundry-top-plugin/util"
	gops "github.com/mitchellh/go-ps"
)

// Client struct
type Client struct {
	options       *ClientOptions
	ui            terminal.UI
	cliConnection plugin.CliConnection
	eventrouting  *eventrouting.EventRouter
	router        *eventrouting.EventRouter
}

// ClientOptions needed to start the Client
type ClientOptions struct {
	Debug      bool
	NoTopCheck bool
	Cygwin     bool
	Nozzles    int
}

// NewClient instantiating the top client
func NewClient(cliConnection plugin.CliConnection, options *ClientOptions, ui terminal.UI) *Client {

	return &Client{
		options:       options,
		ui:            ui,
		cliConnection: cliConnection,
	}
}

// Ask Created my own Ask func instead of the CF provided one because the CF version adds
// a call to PromptColor(">") which does not work cleanly on MS-Windows
func (c *Client) Ask(prompt string) string {
	fmt.Printf("%s ", prompt)
	rd := bufio.NewReader(os.Stdin)
	line, err := rd.ReadString('\n')
	if err == nil {
		return strings.TrimSpace(line)
	}
	return ""
}

// Start starting the client
func (c *Client) Start() {

	if !c.options.NoTopCheck && c.shouldExitTop() {
		// There are other instances of top running and user requested to exit
		return
	}

	toplog.SetDebugEnabled(c.options.Debug)

	conn := c.cliConnection

	isLoggedIn, err := conn.IsLoggedIn()
	if err != nil {
		c.ui.Failed(err.Error())
		return
	}
	if !isLoggedIn {
		c.ui.Failed("Must login first")
		return
	}

	fmt.Printf("Loading...")
	fmt.Printf("\r           \r")

	ui := ui.NewMasterUI(conn)

	c.router = ui.GetRouter()

	toplog.Info("Top started at " + time.Now().Format("01-02-2006 15:04:05"))

	c.setupFirehoseConnections()

	ui.Start()
}

func (c *Client) setupFirehoseConnections() {
	privileged, err := c.hasDopplerFirehoseScope()
	if err != nil {
		c.ui.Failed("Could not determine privileges. Are you logged in?", err)
		return
	}

	if privileged {
		toplog.Info("Running with doppler.firehose privileges - opening %v nozzles", c.options.Nozzles)
		subscriptionID := "TopPlugin_" + util.Pseudo_uuid()
		for i := 0; i < c.options.Nozzles; i++ {
			go c.createAndKeepAliveNozzle(subscriptionID, "", i)
		}
		toplog.Info("Starting %v firehose nozzle instances", c.options.Nozzles)
		return
	}

	apps, err := c.cliConnection.GetApps()
	if err != nil {
		c.ui.Failed("Fetching all Apps failed: %v", err)
		return
	}
	toplog.Info("Running without doppler.firehose privileges - opening %v nozzles", len(apps))
	for i, application := range apps {
		go c.createAndKeepAliveNozzle("", application.Guid, i)
		toplog.Info("Starting app nozzle #%v instance for App %s", i, application.Name)
	}
}

func (c *Client) createAndKeepAliveNozzle(subscriptionID, appGUID string, instanceID int) error {
	minRetrySeconds := (2 * time.Second)

	for {
		// This is a blocking call if no error
		startTime := time.Now()
		var err error
		if len(subscriptionID) > 0 {
			err = c.createNozzle(subscriptionID, instanceID)
		} else {
			err = c.createAppNozzle(appGUID, instanceID)
		}
		if err != nil {
			errMsg := err.Error()
			notAuthorized := strings.Contains(errMsg, "authorized")
			if websocket.IsCloseError(err, websocket.CloseNormalClosure) || notAuthorized {
				toplog.Error("Nozzle #%v - Stopped with error: %v", instanceID, err)
				if notAuthorized {
					toplog.Error("Are you sure you have 'admin' privileges on foundation?")
					toplog.Error("See needed permissions for this plugin here:")
					toplog.Error("https://github.com/ECSTeam/cloudfoundry-top-plugin")
				}
				break
			}
			toplog.Warn("Nozzle #%v - error: %v", instanceID, err)
		}
		toplog.Warn("Nozzle #%v - Shutdown. Nozzle instance will be restarted", instanceID)
		lastRetry := time.Now().Sub(startTime)
		if lastRetry < minRetrySeconds {
			toplog.Info("Nozzle #%v - Nozzle instance restart too fast, delaying for %v", instanceID, minRetrySeconds)
			time.Sleep(minRetrySeconds)
		}
	}
	return nil
}

func (c *Client) createNozzle(subscriptionID string, instanceID int) error {
	// Delay each nozzle instance creation by 1 second
	time.Sleep(time.Duration(instanceID) * time.Second)

	conn := c.cliConnection
	dopplerEndpoint, err := conn.DopplerEndpoint()
	if err != nil {
		return err
	}

	skipVerifySSL, err := conn.IsSSLDisabled()
	if err != nil {
		return err
	}

	dopplerConnection := consumer.New(dopplerEndpoint, &tls.Config{InsecureSkipVerify: skipVerifySSL}, nil)

	tokenRefresher := NewTokenRefresher(conn, instanceID)
	dopplerConnection.RefreshTokenFrom(tokenRefresher)
	dopplerConnection.SetMinRetryDelay(500 * time.Millisecond)
	dopplerConnection.SetMaxRetryDelay(15 * time.Second)
	dopplerConnection.SetIdleTimeout(15 * time.Second)

	authToken, err := conn.AccessToken()
	if err != nil {
		return err
	}

	messages, errors := dopplerConnection.Firehose(subscriptionID, authToken)
	defer dopplerConnection.Close()

	toplog.Info("Nozzle #%v - Started", instanceID)

	eventError := c.routeEvents(instanceID, messages, errors)
	if eventError != nil {
		msg := eventError.Error()
		if strings.Contains(msg, "Invalid authorization") {
			return eventError
		}
	}
	return nil
}

func (c *Client) createAppNozzle(appGUID string, instanceID int) error {

	// Delay each nozzle instance creation by 1 second
	time.Sleep(time.Duration(instanceID) * time.Second)

	conn := c.cliConnection
	dopplerEndpoint, err := conn.DopplerEndpoint()
	if err != nil {
		return err
	}

	skipVerifySSL, err := conn.IsSSLDisabled()
	if err != nil {
		return err
	}

	dopplerConnection := consumer.New(dopplerEndpoint, &tls.Config{InsecureSkipVerify: skipVerifySSL}, nil)

	tokenRefresher := NewTokenRefresher(conn, instanceID)
	dopplerConnection.RefreshTokenFrom(tokenRefresher)

	authToken, err := conn.AccessToken()
	if err != nil {
		return err
	}

	messages, errors := dopplerConnection.StreamWithoutReconnect(appGUID, authToken)
	defer dopplerConnection.Close()

	toplog.Info("Nozzle #%v for %s - Started", instanceID, appGUID)

	eventError := c.routeEvents(instanceID, messages, errors)
	if eventError != nil {
		msg := eventError.Error()
		if strings.Contains(msg, "Invalid authorization") {
			return eventError
		}
	}
	return nil
}

func (c *Client) routeEvents(instanceID int, messages <-chan *events.Envelope, errors <-chan error) error {
	for {
		select {
		case envelope := <-messages:
			c.router.Route(instanceID, envelope)
		case err := <-errors:
			c.handleError(instanceID, err)
			// Nozzle connection does not seem to recover from errors well, so
			// return here so it can be closed and a new instanced opened
			return err
		}
	}
}

func (c *Client) handleError(instanceID int, err error) {

	switch {
	case websocket.IsCloseError(err, websocket.CloseNormalClosure):
		msg := fmt.Sprintf("Nozzle #%v - Normal Websocket Closure: %v", instanceID, err)
		toplog.Error(msg)
	case websocket.IsCloseError(err, websocket.ClosePolicyViolation):
		msg := fmt.Sprintf("Nozzle #%v - Disconnected because nozzle couldn't keep up (CloseError): %v", instanceID, err)
		toplog.Error(msg)
	default:
		msg := fmt.Sprintf("Nozzle #%v - Error reading firehose: %v", instanceID, err)
		toplog.Error(msg)
	}
}

func (c *Client) shouldExitTop() bool {
	numRunning := c.getNumberOfTopPluginsRunning() - 1
	if numRunning > 0 {
		plural := ""
		if numRunning > 1 {
			plural = "s"
		}
		fmt.Printf("Currently %v instance%v of cf top already running on this OS.\n", numRunning, plural)
		for {
			response := c.Ask("Do you want to continue and start top? (y/n)")
			response = strings.ToLower(response)
			if response == "y" || response == "yes" {
				break
			} else if response == "n" || response == "no" {
				fmt.Printf("Top will not be started\n")
				return true
			}
		}
	}
	return false
}

// Check how many instance of this plugin are running on the currently
// OS.  This is based on checking the process list for processes that
// have the same name as our plugin process name.
func (c *Client) getNumberOfTopPluginsRunning() int {

	p, err := gops.Processes()
	if err != nil {
		fmt.Printf("err: %s", err)
	}

	if len(p) <= 0 {
		fmt.Printf("should have processes")
	}

	processName := ""
	for _, p1 := range p {
		if os.Getpid() == p1.Pid() {
			processName = p1.Executable()
		}
	}

	numberRunning := 0
	for _, p1 := range p {
		if p1.Executable() == processName {
			//fmt.Printf("Found: %v  PID: %v\n", p1.Executable(), p1.Pid())
			numberRunning++
		}
	}

	//fmt.Printf("Number of programs running with my same name: %v\n", numberRunning)
	return numberRunning
}

func (c *Client) hasDopplerFirehoseScope() (bool, error) {
	authToken, err := c.cliConnection.AccessToken()
	if err != nil {
		return false, err
	}

	decodedAccessToken, err := decodeAccessToken(authToken)
	if err != nil {
		return false, err
	}

	jsonParsed, err := gabs.ParseJSON(decodedAccessToken)
	if err != nil {
		return false, err
	}

	scopes, err := jsonParsed.Search("scope").Children()
	if err != nil {
		return false, err
	}

	for _, scope := range scopes {
		if scope.Data().(string) == "doppler.firehose" {
			return true, nil
		}
	}

	return false, nil
}

func decodeAccessToken(accessToken string) (tokenJSON []byte, err error) {
	tokenParts := strings.Split(accessToken, " ")

	if len(tokenParts) < 2 {
		return
	}

	token := tokenParts[1]
	encodedParts := strings.Split(token, ".")

	if len(encodedParts) < 3 {
		return
	}

	encodedTokenJSON := encodedParts[1]
	return base64Decode(encodedTokenJSON)
}

func base64Decode(encodedData string) ([]byte, error) {
	return base64.StdEncoding.DecodeString(restorePadding(encodedData))
}

func restorePadding(seg string) string {
	switch len(seg) % 4 {
	case 2:
		seg = seg + "=="
	case 3:
		seg = seg + "="
	}
	return seg
}
<|endoftext|>


<commit_before>/**
 * Package validator
 *
 * MISC:
 * - anonymous structs - they don't have names so expect the Struct name within StructErrors to be blank
 *
 */

package validator

import (
	"bytes"
	"errors"
	"fmt"
	"reflect"
	"strings"
	"time"
	"unicode"
)

const (
	tagSeparator    = ","
	orSeparator     = "|"
	noValidationTag = "-"
	tagKeySeparator = "="
	structOnlyTag   = "structonly"
	omitempty       = "omitempty"
	fieldErrMsg     = "Field validation for \"%s\" failed on the \"%s\" tag"
	structErrMsg    = "Struct:%s\n"
)

type cachedTags struct {
	keyVals [][]string
	isOrVal bool
}

type cachedField struct {
	index  int
	name   string
	tags   []*cachedTags
	tag    string
	kind   reflect.Kind
	typ    reflect.Type
	isTime bool
}

type cachedStruct struct {
	children int
	name     string
	kind     reflect.Kind
	fields   []*cachedField
}

var structCache = map[reflect.Type]*cachedStruct{}
var fieldsCache = map[string][]*cachedTags{}

// FieldError contains a single field's validation error along
// with other properties that may be needed for error message creation
type FieldError struct {
	Field string
	Tag   string
	Kind  reflect.Kind
	Type  reflect.Type
	Param string
	Value interface{}
}

// This is intended for use in development + debugging and not intended to be a production error message.
// it also allows FieldError to be used as an Error interface
func (e *FieldError) Error() string {
	return fmt.Sprintf(fieldErrMsg, e.Field, e.Tag)
}

// StructErrors is hierarchical list of field and struct validation errors
// for a non hierarchical representation please see the Flatten method for StructErrors
type StructErrors struct {
	// Name of the Struct
	Struct string
	// Struct Field Errors
	Errors map[string]*FieldError
	// Struct Fields of type struct and their errors
	// key = Field Name of current struct, but internally Struct will be the actual struct name unless anonymous struct, it will be blank
	StructErrors map[string]*StructErrors
}

// This is intended for use in development + debugging and not intended to be a production error message.
// it also allows StructErrors to be used as an Error interface
func (e *StructErrors) Error() string {
	buff := bytes.NewBufferString(fmt.Sprintf(structErrMsg, e.Struct))

	for _, err := range e.Errors {
		buff.WriteString(err.Error())
		buff.WriteString("\n")
	}

	for _, err := range e.StructErrors {
		buff.WriteString(err.Error())
	}

	return buff.String()
}

// Flatten flattens the StructErrors hierarchical structure into a flat namespace style field name
// for those that want/need it
func (e *StructErrors) Flatten() map[string]*FieldError {

	if e == nil {
		return nil
	}

	errs := map[string]*FieldError{}

	for _, f := range e.Errors {

		errs[f.Field] = f
	}

	for key, val := range e.StructErrors {

		otherErrs := val.Flatten()

		for _, f2 := range otherErrs {

			f2.Field = fmt.Sprintf("%s.%s", key, f2.Field)
			errs[f2.Field] = f2
		}
	}

	return errs
}

// Func accepts all values needed for file and cross field validation
// top     = top level struct when validating by struct otherwise nil
// current = current level struct when validating by struct otherwise optional comparison value
// f       = field value for validation
// param   = parameter used in validation i.e. gt=0 param would be 0
type Func func(top interface{}, current interface{}, f interface{}, param string) bool

// Validate implements the Validate Struct
// NOTE: Fields within are not thread safe and that is on purpose
// Functions and Tags should all be predifined before use, so subscribe to the philosiphy
// or make it thread safe on your end
type Validate struct {
	// tagName being used.
	tagName string
	// validateFuncs is a map of validation functions and the tag keys
	validationFuncs map[string]Func
}

// New creates a new Validate instance for use.
func New(tagName string, funcs map[string]Func) *Validate {
	return &Validate{
		tagName:         tagName,
		validationFuncs: funcs,
	}
}

// SetTag sets tagName of the Validator to one of your choosing after creation
// perhaps to dodge a tag name conflict in a specific section of code
func (v *Validate) SetTag(tagName string) {
	v.tagName = tagName
}

// AddFunction adds a validation Func to a Validate's map of validators denoted by the key
// NOTE: if the key already exists, it will get replaced.
func (v *Validate) AddFunction(key string, f Func) error {

	if len(key) == 0 {
		return errors.New("Function Key cannot be empty")
	}

	if f == nil {
		return errors.New("Function cannot be empty")
	}

	v.validationFuncs[key] = f

	return nil
}

// Struct validates a struct, even it's nested structs, and returns a struct containing the errors
// NOTE: Nested Arrays, or Maps of structs do not get validated only the Array or Map itself; the reason is that there is no good
// way to represent or report which struct within the array has the error, besides can validate the struct prior to adding it to
// the Array or Map.
func (v *Validate) Struct(s interface{}) *StructErrors {

	return v.structRecursive(s, s, s)
}

// structRecursive validates a struct recursivly and passes the top level and current struct around for use in validator functions and returns a struct containing the errors
func (v *Validate) structRecursive(top interface{}, current interface{}, s interface{}) *StructErrors {

	structValue := reflect.ValueOf(s)

	if structValue.Kind() == reflect.Ptr && !structValue.IsNil() {
		return v.structRecursive(top, current, structValue.Elem().Interface())
	}

	if structValue.Kind() != reflect.Struct && structValue.Kind() != reflect.Interface {
		panic("interface passed for validation is not a struct")
	}

	structType := reflect.TypeOf(s)

	var structName string
	var numFields int

	cs, isCached := structCache[structType]

	if isCached {
		structName = cs.name
		numFields = cs.children
	} else {
		structName = structType.Name()
		numFields = structValue.NumField()
		cs = &cachedStruct{name: structName, children: numFields}
		structCache[structType] = cs
	}

	validationErrors := &StructErrors{
		Struct:       structName,
		Errors:       map[string]*FieldError{},
		StructErrors: map[string]*StructErrors{},
	}

	for i := 0; i < numFields; i++ {

		var valueField reflect.Value
		var cField *cachedField
		var typeField reflect.StructField

		if isCached {
			cField = cs.fields[i]
			valueField = structValue.Field(cField.index)

			if valueField.Kind() == reflect.Ptr && !valueField.IsNil() {
				valueField = valueField.Elem()
			}
		} else {
			valueField = structValue.Field(i)

			if valueField.Kind() == reflect.Ptr && !valueField.IsNil() {
				valueField = valueField.Elem()
			}

			typeField = structType.Field(i)

			cField = &cachedField{index: i, tag: typeField.Tag.Get(v.tagName)}

			if cField.tag == noValidationTag {
				cs.children--
				continue
			}

			// if no validation and not a struct (which may containt fields for validation)
			if cField.tag == "" && ((valueField.Kind() != reflect.Struct && valueField.Kind() != reflect.Interface) || valueField.Type() == reflect.TypeOf(time.Time{})) {
				cs.children--
				continue
			}

			cField.name = typeField.Name
			cField.kind = valueField.Kind()
			cField.typ = valueField.Type()
		}

		// this can happen if the first cache value was nil
		// but the second actually has a value
		if cField.kind == reflect.Ptr {
			cField.kind = valueField.Kind()
		}

		switch cField.kind {

		case reflect.Struct, reflect.Interface:

			if !unicode.IsUpper(rune(cField.name[0])) {
				cs.children--
				continue
			}

			if cField.isTime || valueField.Type() == reflect.TypeOf(time.Time{}) {

				cField.isTime = true

				if fieldError := v.fieldWithNameAndValue(top, current, valueField.Interface(), cField.tag, cField.name, false, cField); fieldError != nil {
					validationErrors.Errors[fieldError.Field] = fieldError
					// free up memory reference
					fieldError = nil
				}

			} else {

				if strings.Contains(cField.tag, structOnlyTag) {
					cs.children--
					continue
				}

				if structErrors := v.structRecursive(top, valueField.Interface(), valueField.Interface()); structErrors != nil {
					validationErrors.StructErrors[cField.name] = structErrors
					// free up memory map no longer needed
					structErrors = nil
				}
			}

		default:

			if fieldError := v.fieldWithNameAndValue(top, current, valueField.Interface(), cField.tag, cField.name, false, cField); fieldError != nil {
				validationErrors.Errors[fieldError.Field] = fieldError
				// free up memory reference
				fieldError = nil
			}
		}

		if !isCached {
			cs.fields = append(cs.fields, cField)
		}
	}

	if len(validationErrors.Errors) == 0 && len(validationErrors.StructErrors) == 0 {
		return nil
	}

	return validationErrors
}

// Field allows validation of a single field, still using tag style validation to check multiple errors
func (v *Validate) Field(f interface{}, tag string) *FieldError {

	return v.FieldWithValue(nil, f, tag)
}

// FieldWithValue allows validation of a single field, possibly even against another fields value, still using tag style validation to check multiple errors
func (v *Validate) FieldWithValue(val interface{}, f interface{}, tag string) *FieldError {

	return v.fieldWithNameAndValue(nil, val, f, tag, "", true, nil)
}

func (v *Validate) fieldWithNameAndValue(val interface{}, current interface{}, f interface{}, tag string, name string, isSingleField bool, cacheField *cachedField) *FieldError {

	var cField *cachedField
	var isCached bool

	// This is a double check if coming from validate.Struct but need to be here in case function is called directly
	if tag == noValidationTag {
		return nil
	}

	if strings.Contains(tag, omitempty) && !hasValue(val, current, f, "") {
		return nil
	}

	if cacheField == nil {
		valueField := reflect.ValueOf(f)

		if valueField.Kind() == reflect.Ptr && !valueField.IsNil() {
			valueField = valueField.Elem()
			f = valueField.Interface()
		}

		cField = &cachedField{name: name, kind: valueField.Kind(), tag: tag, typ: valueField.Type()}
	} else {
		cField = cacheField
	}

	switch cField.kind {

	case reflect.Struct, reflect.Interface, reflect.Invalid:

		if cField.typ != reflect.TypeOf(time.Time{}) {
			panic("Invalid field passed to ValidateFieldWithTag")
		}
	}

	if len(cField.tags) == 0 {

		if isSingleField {
			cField.tags, isCached = fieldsCache[tag]
		}

		if !isCached {

			for _, t := range strings.Split(tag, tagSeparator) {

				orVals := strings.Split(t, orSeparator)
				cTag := &cachedTags{isOrVal: len(orVals) > 1, keyVals: make([][]string, len(orVals))}
				cField.tags = append(cField.tags, cTag)

				for i, val := range orVals {
					vals := strings.Split(val, tagKeySeparator)

					key := strings.TrimSpace(vals[0])

					if len(key) == 0 {
						panic(fmt.Sprintf("Invalid validation tag on field %s", name))
					}

					param := ""
					if len(vals) > 1 {
						param = strings.TrimSpace(vals[1])
					}

					cTag.keyVals[i] = []string{key, param}
				}
			}

			if isSingleField {
				fieldsCache[cField.tag] = cField.tags
			}
		}
	}

	var fieldErr *FieldError
	var err error

	for _, cTag := range cField.tags {

		if cTag.isOrVal {

			errTag := ""

			for _, val := range cTag.keyVals {

				fieldErr, err = v.fieldWithNameAndSingleTag(val, current, f, val[0], val[1], name)

				if err == nil {
					return nil
				}

				errTag += orSeparator + fieldErr.Tag
			}

			errTag = strings.TrimLeft(errTag, orSeparator)

			fieldErr.Tag = errTag
			fieldErr.Kind = cField.kind
			fieldErr.Type = cField.typ

			return fieldErr
		}

		if fieldErr, err = v.fieldWithNameAndSingleTag(val, current, f, cTag.keyVals[0][0], cTag.keyVals[0][1], name); err != nil {

			fieldErr.Kind = cField.kind
			fieldErr.Type = cField.typ

			return fieldErr
		}
	}

	return nil
}

func (v *Validate) fieldWithNameAndSingleTag(val interface{}, current interface{}, f interface{}, key string, param string, name string) (*FieldError, error) {

	// OK to continue because we checked it's existance before getting into this loop
	if key == omitempty {
		return nil, nil
	}

	valFunc, ok := v.validationFuncs[key]
	if !ok {
		panic(fmt.Sprintf("Undefined validation function on field %s", name))
	}

	if err := valFunc(val, current, f, param); err {
		return nil, nil
	} else {
		return &FieldError{
			Field: name,
			Tag:   key,
			Value: f,
			Param: param,
		}, errors.New(key)
	}
}
<commit_msg>add map thread safety<commit_after>/**
 * Package validator
 *
 * MISC:
 * - anonymous structs - they don't have names so expect the Struct name within StructErrors to be blank
 *
 */

package validator

import (
	"bytes"
	"errors"
	"fmt"
	"reflect"
	"strings"
	"sync"
	"time"
	"unicode"
)

const (
	tagSeparator    = ","
	orSeparator     = "|"
	noValidationTag = "-"
	tagKeySeparator = "="
	structOnlyTag   = "structonly"
	omitempty       = "omitempty"
	fieldErrMsg     = "Field validation for \"%s\" failed on the \"%s\" tag"
	structErrMsg    = "Struct:%s\n"
)

type cachedTags struct {
	keyVals [][]string
	isOrVal bool
}

type cachedField struct {
	index  int
	name   string
	tags   []*cachedTags
	tag    string
	kind   reflect.Kind
	typ    reflect.Type
	isTime bool
}

type cachedStruct struct {
	children int
	name     string
	kind     reflect.Kind
	fields   []*cachedField
}

type structsCacheMap struct {
	lock sync.RWMutex
	m    map[reflect.Type]*cachedStruct
}

func (s *structsCacheMap) Get(key reflect.Type) (*cachedStruct, bool) {
	s.lock.RLock()
	defer s.lock.RUnlock()
	value, ok := s.m[key]
	return value, ok
}

func (s *structsCacheMap) Set(key reflect.Type, value *cachedStruct) {
	s.lock.Lock()
	defer s.lock.Unlock()
	s.m[key] = value
}

var structCache = &structsCacheMap{m: map[reflect.Type]*cachedStruct{}}

type fieldsCacheMap struct {
	lock sync.RWMutex
	m    map[string][]*cachedTags
}

func (s *fieldsCacheMap) Get(key string) ([]*cachedTags, bool) {
	s.lock.RLock()
	defer s.lock.RUnlock()
	value, ok := s.m[key]
	return value, ok
}

func (s *fieldsCacheMap) Set(key string, value []*cachedTags) {
	s.lock.Lock()
	defer s.lock.Unlock()
	s.m[key] = value
}

var fieldsCache = &fieldsCacheMap{m: map[string][]*cachedTags{}}

// FieldError contains a single field's validation error along
// with other properties that may be needed for error message creation
type FieldError struct {
	Field string
	Tag   string
	Kind  reflect.Kind
	Type  reflect.Type
	Param string
	Value interface{}
}

// This is intended for use in development + debugging and not intended to be a production error message.
// it also allows FieldError to be used as an Error interface
func (e *FieldError) Error() string {
	return fmt.Sprintf(fieldErrMsg, e.Field, e.Tag)
}

// StructErrors is hierarchical list of field and struct validation errors
// for a non hierarchical representation please see the Flatten method for StructErrors
type StructErrors struct {
	// Name of the Struct
	Struct string
	// Struct Field Errors
	Errors map[string]*FieldError
	// Struct Fields of type struct and their errors
	// key = Field Name of current struct, but internally Struct will be the actual struct name unless anonymous struct, it will be blank
	StructErrors map[string]*StructErrors
}

// This is intended for use in development + debugging and not intended to be a production error message.
// it also allows StructErrors to be used as an Error interface
func (e *StructErrors) Error() string {
	buff := bytes.NewBufferString(fmt.Sprintf(structErrMsg, e.Struct))

	for _, err := range e.Errors {
		buff.WriteString(err.Error())
		buff.WriteString("\n")
	}

	for _, err := range e.StructErrors {
		buff.WriteString(err.Error())
	}

	return buff.String()
}

// Flatten flattens the StructErrors hierarchical structure into a flat namespace style field name
// for those that want/need it
func (e *StructErrors) Flatten() map[string]*FieldError {

	if e == nil {
		return nil
	}

	errs := map[string]*FieldError{}

	for _, f := range e.Errors {

		errs[f.Field] = f
	}

	for key, val := range e.StructErrors {

		otherErrs := val.Flatten()

		for _, f2 := range otherErrs {

			f2.Field = fmt.Sprintf("%s.%s", key, f2.Field)
			errs[f2.Field] = f2
		}
	}

	return errs
}

// Func accepts all values needed for file and cross field validation
// top     = top level struct when validating by struct otherwise nil
// current = current level struct when validating by struct otherwise optional comparison value
// f       = field value for validation
// param   = parameter used in validation i.e. gt=0 param would be 0
type Func func(top interface{}, current interface{}, f interface{}, param string) bool

// Validate implements the Validate Struct
// NOTE: Fields within are not thread safe and that is on purpose
// Functions and Tags should all be predifined before use, so subscribe to the philosiphy
// or make it thread safe on your end
type Validate struct {
	// tagName being used.
	tagName string
	// validateFuncs is a map of validation functions and the tag keys
	validationFuncs map[string]Func
}

// New creates a new Validate instance for use.
func New(tagName string, funcs map[string]Func) *Validate {
	return &Validate{
		tagName:         tagName,
		validationFuncs: funcs,
	}
}

// SetTag sets tagName of the Validator to one of your choosing after creation
// perhaps to dodge a tag name conflict in a specific section of code
func (v *Validate) SetTag(tagName string) {
	v.tagName = tagName
}

// AddFunction adds a validation Func to a Validate's map of validators denoted by the key
// NOTE: if the key already exists, it will get replaced.
func (v *Validate) AddFunction(key string, f Func) error {

	if len(key) == 0 {
		return errors.New("Function Key cannot be empty")
	}

	if f == nil {
		return errors.New("Function cannot be empty")
	}

	v.validationFuncs[key] = f

	return nil
}

// Struct validates a struct, even it's nested structs, and returns a struct containing the errors
// NOTE: Nested Arrays, or Maps of structs do not get validated only the Array or Map itself; the reason is that there is no good
// way to represent or report which struct within the array has the error, besides can validate the struct prior to adding it to
// the Array or Map.
func (v *Validate) Struct(s interface{}) *StructErrors {

	return v.structRecursive(s, s, s)
}

// structRecursive validates a struct recursivly and passes the top level and current struct around for use in validator functions and returns a struct containing the errors
func (v *Validate) structRecursive(top interface{}, current interface{}, s interface{}) *StructErrors {

	structValue := reflect.ValueOf(s)

	if structValue.Kind() == reflect.Ptr && !structValue.IsNil() {
		return v.structRecursive(top, current, structValue.Elem().Interface())
	}

	if structValue.Kind() != reflect.Struct && structValue.Kind() != reflect.Interface {
		panic("interface passed for validation is not a struct")
	}

	structType := reflect.TypeOf(s)

	var structName string
	var numFields int

	cs, isCached := structCache.Get(structType)

	if isCached {
		structName = cs.name
		numFields = cs.children
	} else {
		structName = structType.Name()
		numFields = structValue.NumField()
		cs = &cachedStruct{name: structName, children: numFields}
		structCache.Set(structType, cs)
	}

	validationErrors := &StructErrors{
		Struct:       structName,
		Errors:       map[string]*FieldError{},
		StructErrors: map[string]*StructErrors{},
	}

	for i := 0; i < numFields; i++ {

		var valueField reflect.Value
		var cField *cachedField
		var typeField reflect.StructField

		if isCached {
			cField = cs.fields[i]
			valueField = structValue.Field(cField.index)

			if valueField.Kind() == reflect.Ptr && !valueField.IsNil() {
				valueField = valueField.Elem()
			}
		} else {
			valueField = structValue.Field(i)

			if valueField.Kind() == reflect.Ptr && !valueField.IsNil() {
				valueField = valueField.Elem()
			}

			typeField = structType.Field(i)

			cField = &cachedField{index: i, tag: typeField.Tag.Get(v.tagName)}

			if cField.tag == noValidationTag {
				cs.children--
				continue
			}

			// if no validation and not a struct (which may containt fields for validation)
			if cField.tag == "" && ((valueField.Kind() != reflect.Struct && valueField.Kind() != reflect.Interface) || valueField.Type() == reflect.TypeOf(time.Time{})) {
				cs.children--
				continue
			}

			cField.name = typeField.Name
			cField.kind = valueField.Kind()
			cField.typ = valueField.Type()
		}

		// this can happen if the first cache value was nil
		// but the second actually has a value
		if cField.kind == reflect.Ptr {
			cField.kind = valueField.Kind()
		}

		switch cField.kind {

		case reflect.Struct, reflect.Interface:

			if !unicode.IsUpper(rune(cField.name[0])) {
				cs.children--
				continue
			}

			if cField.isTime || valueField.Type() == reflect.TypeOf(time.Time{}) {

				cField.isTime = true

				if fieldError := v.fieldWithNameAndValue(top, current, valueField.Interface(), cField.tag, cField.name, false, cField); fieldError != nil {
					validationErrors.Errors[fieldError.Field] = fieldError
					// free up memory reference
					fieldError = nil
				}

			} else {

				if strings.Contains(cField.tag, structOnlyTag) {
					cs.children--
					continue
				}

				if structErrors := v.structRecursive(top, valueField.Interface(), valueField.Interface()); structErrors != nil {
					validationErrors.StructErrors[cField.name] = structErrors
					// free up memory map no longer needed
					structErrors = nil
				}
			}

		default:

			if fieldError := v.fieldWithNameAndValue(top, current, valueField.Interface(), cField.tag, cField.name, false, cField); fieldError != nil {
				validationErrors.Errors[fieldError.Field] = fieldError
				// free up memory reference
				fieldError = nil
			}
		}

		if !isCached {
			cs.fields = append(cs.fields, cField)
		}
	}

	if len(validationErrors.Errors) == 0 && len(validationErrors.StructErrors) == 0 {
		return nil
	}

	return validationErrors
}

// Field allows validation of a single field, still using tag style validation to check multiple errors
func (v *Validate) Field(f interface{}, tag string) *FieldError {

	return v.FieldWithValue(nil, f, tag)
}

// FieldWithValue allows validation of a single field, possibly even against another fields value, still using tag style validation to check multiple errors
func (v *Validate) FieldWithValue(val interface{}, f interface{}, tag string) *FieldError {

	return v.fieldWithNameAndValue(nil, val, f, tag, "", true, nil)
}

func (v *Validate) fieldWithNameAndValue(val interface{}, current interface{}, f interface{}, tag string, name string, isSingleField bool, cacheField *cachedField) *FieldError {

	var cField *cachedField
	var isCached bool

	// This is a double check if coming from validate.Struct but need to be here in case function is called directly
	if tag == noValidationTag {
		return nil
	}

	if strings.Contains(tag, omitempty) && !hasValue(val, current, f, "") {
		return nil
	}

	if cacheField == nil {
		valueField := reflect.ValueOf(f)

		if valueField.Kind() == reflect.Ptr && !valueField.IsNil() {
			valueField = valueField.Elem()
			f = valueField.Interface()
		}

		cField = &cachedField{name: name, kind: valueField.Kind(), tag: tag, typ: valueField.Type()}
	} else {
		cField = cacheField
	}

	switch cField.kind {

	case reflect.Struct, reflect.Interface, reflect.Invalid:

		if cField.typ != reflect.TypeOf(time.Time{}) {
			panic("Invalid field passed to ValidateFieldWithTag")
		}
	}

	if len(cField.tags) == 0 {

		if isSingleField {
			cField.tags, isCached = fieldsCache.Get(tag)
		}

		if !isCached {

			for _, t := range strings.Split(tag, tagSeparator) {

				orVals := strings.Split(t, orSeparator)
				cTag := &cachedTags{isOrVal: len(orVals) > 1, keyVals: make([][]string, len(orVals))}
				cField.tags = append(cField.tags, cTag)

				for i, val := range orVals {
					vals := strings.Split(val, tagKeySeparator)

					key := strings.TrimSpace(vals[0])

					if len(key) == 0 {
						panic(fmt.Sprintf("Invalid validation tag on field %s", name))
					}

					param := ""
					if len(vals) > 1 {
						param = strings.TrimSpace(vals[1])
					}

					cTag.keyVals[i] = []string{key, param}
				}
			}

			if isSingleField {
				fieldsCache.Set(cField.tag, cField.tags)
			}
		}
	}

	var fieldErr *FieldError
	var err error

	for _, cTag := range cField.tags {

		if cTag.isOrVal {

			errTag := ""

			for _, val := range cTag.keyVals {

				fieldErr, err = v.fieldWithNameAndSingleTag(val, current, f, val[0], val[1], name)

				if err == nil {
					return nil
				}

				errTag += orSeparator + fieldErr.Tag
			}

			errTag = strings.TrimLeft(errTag, orSeparator)

			fieldErr.Tag = errTag
			fieldErr.Kind = cField.kind
			fieldErr.Type = cField.typ

			return fieldErr
		}

		if fieldErr, err = v.fieldWithNameAndSingleTag(val, current, f, cTag.keyVals[0][0], cTag.keyVals[0][1], name); err != nil {

			fieldErr.Kind = cField.kind
			fieldErr.Type = cField.typ

			return fieldErr
		}
	}

	return nil
}

func (v *Validate) fieldWithNameAndSingleTag(val interface{}, current interface{}, f interface{}, key string, param string, name string) (*FieldError, error) {

	// OK to continue because we checked it's existance before getting into this loop
	if key == omitempty {
		return nil, nil
	}

	valFunc, ok := v.validationFuncs[key]
	if !ok {
		panic(fmt.Sprintf("Undefined validation function on field %s", name))
	}

	if err := valFunc(val, current, f, param); err {
		return nil, nil
	} else {
		return &FieldError{
			Field: name,
			Tag:   key,
			Value: f,
			Param: param,
		}, errors.New(key)
	}
}
<|endoftext|>


<commit_before>// Package cloudflare implements the Cloudflare v4 API.
package cloudflare

import (
	"bytes"
	"context"
	"encoding/json"
	"io"
	"io/ioutil"
	"log"
	"math"
	"net/http"
	"net/url"
	"strconv"
	"strings"
	"time"

	"github.com/pkg/errors"
	"golang.org/x/time/rate"
)

const apiURL = "https://api.cloudflare.com/client/v4"

const (
	// AuthKeyEmail specifies that we should authenticate with API key and email address
	AuthKeyEmail = 1 << iota
	// AuthUserService specifies that we should authenticate with a User-Service key
	AuthUserService
	// AuthToken specifies that we should authenticate with an API Token
	AuthToken
)

// API holds the configuration for the current API client. A client should not
// be modified concurrently.
type API struct {
	APIKey            string
	APIEmail          string
	APIUserServiceKey string
	APIToken          string
	BaseURL           string
	OrganizationID    string
	UserAgent         string
	headers           http.Header
	httpClient        *http.Client
	authType          int
	rateLimiter       *rate.Limiter
	retryPolicy       RetryPolicy
	logger            Logger
}

// newClient provides shared logic for New and NewWithUserServiceKey
func newClient(opts ...Option) (*API, error) {
	silentLogger := log.New(ioutil.Discard, "", log.LstdFlags)

	api := &API{
		BaseURL:     apiURL,
		headers:     make(http.Header),
		rateLimiter: rate.NewLimiter(rate.Limit(4), 1), // 4rps equates to default api limit (1200 req/5 min)
		retryPolicy: RetryPolicy{
			MaxRetries:    3,
			MinRetryDelay: time.Duration(1) * time.Second,
			MaxRetryDelay: time.Duration(30) * time.Second,
		},
		logger: silentLogger,
	}

	err := api.parseOptions(opts...)
	if err != nil {
		return nil, errors.Wrap(err, "options parsing failed")
	}

	// Fall back to http.DefaultClient if the package user does not provide
	// their own.
	if api.httpClient == nil {
		api.httpClient = http.DefaultClient
	}

	return api, nil
}

// New creates a new Cloudflare v4 API client.
func New(key, email string, opts ...Option) (*API, error) {
	if key == "" || email == "" {
		return nil, errors.New(errEmptyCredentials)
	}

	api, err := newClient(opts...)
	if err != nil {
		return nil, err
	}

	api.APIKey = key
	api.APIEmail = email
	api.authType = AuthKeyEmail

	return api, nil
}

// NewWithAPIToken creates a new Cloudflare v4 API client using API Tokens
func NewWithAPIToken(token string, opts ...Option) (*API, error) {
	if token == "" {
		return nil, errors.New(errEmptyAPIToken)
	}

	api, err := newClient(opts...)
	if err != nil {
		return nil, err
	}

	api.APIToken = token
	api.authType = AuthToken

	return api, nil
}

// NewWithUserServiceKey creates a new Cloudflare v4 API client using service key authentication.
func NewWithUserServiceKey(key string, opts ...Option) (*API, error) {
	if key == "" {
		return nil, errors.New(errEmptyCredentials)
	}

	api, err := newClient(opts...)
	if err != nil {
		return nil, err
	}

	api.APIUserServiceKey = key
	api.authType = AuthUserService

	return api, nil
}

// SetAuthType sets the authentication method (AuthKeyEmail, AuthToken, or AuthUserService).
func (api *API) SetAuthType(authType int) {
	api.authType = authType
}

// ZoneIDByName retrieves a zone's ID from the name.
func (api *API) ZoneIDByName(zoneName string) (string, error) {
	res, err := api.ListZonesContext(context.TODO(), WithZoneFilter(zoneName))
	if err != nil {
		return "", errors.Wrap(err, "ListZonesContext command failed")
	}

	if len(res.Result) > 1 && api.OrganizationID == "" {
		return "", errors.New("ambiguous zone name used without an account ID")
	}

	for _, zone := range res.Result {
		if api.OrganizationID != "" {
			if zone.Name == zoneName && api.OrganizationID == zone.Account.ID {
				return zone.ID, nil
			}
		} else {
			if zone.Name == zoneName {
				return zone.ID, nil
			}
		}
	}

	return "", errors.New("Zone could not be found")
}

// makeRequest makes a HTTP request and returns the body as a byte slice,
// closing it before returning. params will be serialized to JSON.
func (api *API) makeRequest(method, uri string, params interface{}) ([]byte, error) {
	return api.makeRequestWithAuthType(context.TODO(), method, uri, params, api.authType)
}

func (api *API) makeRequestContext(ctx context.Context, method, uri string, params interface{}) ([]byte, error) {
	return api.makeRequestWithAuthType(ctx, method, uri, params, api.authType)
}

func (api *API) makeRequestWithHeaders(method, uri string, params interface{}, headers http.Header) ([]byte, error) {
	return api.makeRequestWithAuthTypeAndHeaders(context.TODO(), method, uri, params, api.authType, headers)
}

func (api *API) makeRequestWithAuthType(ctx context.Context, method, uri string, params interface{}, authType int) ([]byte, error) {
	return api.makeRequestWithAuthTypeAndHeaders(ctx, method, uri, params, authType, nil)
}

func (api *API) makeRequestWithAuthTypeAndHeaders(ctx context.Context, method, uri string, params interface{}, authType int, headers http.Header) ([]byte, error) {
	// Replace nil with a JSON object if needed
	var jsonBody []byte
	var err error

	if params != nil {
		if paramBytes, ok := params.([]byte); ok {
			jsonBody = paramBytes
		} else {
			jsonBody, err = json.Marshal(params)
			if err != nil {
				return nil, errors.Wrap(err, "error marshalling params to JSON")
			}
		}
	} else {
		jsonBody = nil
	}

	var resp *http.Response
	var respErr error
	var reqBody io.Reader
	var respBody []byte
	for i := 0; i <= api.retryPolicy.MaxRetries; i++ {
		if jsonBody != nil {
			reqBody = bytes.NewReader(jsonBody)
		}
		if i > 0 {
			// expect the backoff introduced here on errored requests to dominate the effect of rate limiting
			// don't need a random component here as the rate limiter should do something similar
			// nb time duration could truncate an arbitrary float. Since our inputs are all ints, we should be ok
			sleepDuration := time.Duration(math.Pow(2, float64(i-1)) * float64(api.retryPolicy.MinRetryDelay))

			if sleepDuration > api.retryPolicy.MaxRetryDelay {
				sleepDuration = api.retryPolicy.MaxRetryDelay
			}
			// useful to do some simple logging here, maybe introduce levels later
			api.logger.Printf("Sleeping %s before retry attempt number %d for request %s %s", sleepDuration.String(), i, method, uri)
			time.Sleep(sleepDuration)

		}
		api.rateLimiter.Wait(context.TODO())
		if err != nil {
			return nil, errors.Wrap(err, "Error caused by request rate limiting")
		}
		resp, respErr = api.request(ctx, method, uri, reqBody, authType, headers)

		// retry if the server is rate limiting us or if it failed
		// assumes server operations are rolled back on failure
		if respErr != nil || resp.StatusCode == http.StatusTooManyRequests || resp.StatusCode >= 500 {
			// if we got a valid http response, try to read body so we can reuse the connection
			// see https://golang.org/pkg/net/http/#Client.Do
			if respErr == nil {
				respBody, err = ioutil.ReadAll(resp.Body)
				resp.Body.Close()

				respErr = errors.Wrap(err, "could not read response body")

				api.logger.Printf("Request: %s %s got an error response %d: %s\n", method, uri, resp.StatusCode,
					strings.Replace(strings.Replace(string(respBody), "\n", "", -1), "\t", "", -1))
			} else {
				api.logger.Printf("Error performing request: %s %s : %s \n", method, uri, respErr.Error())
			}
			continue
		} else {
			respBody, err = ioutil.ReadAll(resp.Body)
			defer resp.Body.Close()
			if err != nil {
				return nil, errors.Wrap(err, "could not read response body")
			}
			break
		}
	}
	if respErr != nil {
		return nil, respErr
	}

	switch {
	case resp.StatusCode >= http.StatusOK && resp.StatusCode < http.StatusMultipleChoices:
	case resp.StatusCode == http.StatusUnauthorized:
		return nil, errors.Errorf("HTTP status %d: invalid credentials", resp.StatusCode)
	case resp.StatusCode == http.StatusForbidden:
		return nil, errors.Errorf("HTTP status %d: insufficient permissions", resp.StatusCode)
	case resp.StatusCode == http.StatusServiceUnavailable,
		resp.StatusCode == http.StatusBadGateway,
		resp.StatusCode == http.StatusGatewayTimeout,
		resp.StatusCode == 522,
		resp.StatusCode == 523,
		resp.StatusCode == 524:
		return nil, errors.Errorf("HTTP status %d: service failure", resp.StatusCode)
	// This isn't a great solution due to the way the `default` case is
	// a catch all and that the `filters/validate-expr` returns a HTTP 400
	// yet the clients need to use the HTTP body as a JSON string.
	case resp.StatusCode == 400 && strings.HasSuffix(resp.Request.URL.Path, "/filters/validate-expr"):
		return nil, errors.Errorf("%s", respBody)
	default:
		var s string
		if respBody != nil {
			s = string(respBody)
		}
		return nil, errors.Errorf("HTTP status %d: content %q", resp.StatusCode, s)
	}

	return respBody, nil
}

// request makes a HTTP request to the given API endpoint, returning the raw
// *http.Response, or an error if one occurred. The caller is responsible for
// closing the response body.
func (api *API) request(ctx context.Context, method, uri string, reqBody io.Reader, authType int, headers http.Header) (*http.Response, error) {
	req, err := http.NewRequest(method, api.BaseURL+uri, reqBody)
	if err != nil {
		return nil, errors.Wrap(err, "HTTP request creation failed")
	}
	req.WithContext(ctx)

	combinedHeaders := make(http.Header)
	copyHeader(combinedHeaders, api.headers)
	copyHeader(combinedHeaders, headers)
	req.Header = combinedHeaders

	if authType&AuthKeyEmail != 0 {
		req.Header.Set("X-Auth-Key", api.APIKey)
		req.Header.Set("X-Auth-Email", api.APIEmail)
	}
	if authType&AuthUserService != 0 {
		req.Header.Set("X-Auth-User-Service-Key", api.APIUserServiceKey)
	}
	if authType&AuthToken != 0 {
		req.Header.Set("Authorization", "Bearer "+api.APIToken)
	}

	if api.UserAgent != "" {
		req.Header.Set("User-Agent", api.UserAgent)
	}

	if req.Header.Get("Content-Type") == "" {
		req.Header.Set("Content-Type", "application/json")
	}

	resp, err := api.httpClient.Do(req)
	if err != nil {
		return nil, errors.Wrap(err, "HTTP request failed")
	}

	return resp, nil
}

// Returns the base URL to use for API endpoints that exist for both accounts and organizations.
// If an Organization option was used when creating the API instance, returns the org URL.
//
// accountBase is the base URL for endpoints referring to the current user. It exists as a
// parameter because it is not consistent across APIs.
func (api *API) userBaseURL(accountBase string) string {
	if api.OrganizationID != "" {
		return "/accounts/" + api.OrganizationID
	}
	return accountBase
}

// copyHeader copies all headers for `source` and sets them on `target`.
// based on https://godoc.org/github.com/golang/gddo/httputil/header#Copy
func copyHeader(target, source http.Header) {
	for k, vs := range source {
		target[k] = vs
	}
}

// ResponseInfo contains a code and message returned by the API as errors or
// informational messages inside the response.
type ResponseInfo struct {
	Code    int    `json:"code"`
	Message string `json:"message"`
}

// Response is a template.  There will also be a result struct.  There will be a
// unique response type for each response, which will include this type.
type Response struct {
	Success  bool           `json:"success"`
	Errors   []ResponseInfo `json:"errors"`
	Messages []ResponseInfo `json:"messages"`
}

// ResultInfo contains metadata about the Response.
type ResultInfo struct {
	Page       int `json:"page"`
	PerPage    int `json:"per_page"`
	TotalPages int `json:"total_pages"`
	Count      int `json:"count"`
	Total      int `json:"total_count"`
}

// RawResponse keeps the result as JSON form
type RawResponse struct {
	Response
	Result json.RawMessage `json:"result"`
}

// Raw makes a HTTP request with user provided params and returns the
// result as untouched JSON.
func (api *API) Raw(method, endpoint string, data interface{}) (json.RawMessage, error) {
	res, err := api.makeRequest(method, endpoint, data)
	if err != nil {
		return nil, errors.Wrap(err, errMakeRequestError)
	}

	var r RawResponse
	if err := json.Unmarshal(res, &r); err != nil {
		return nil, errors.Wrap(err, errUnmarshalError)
	}
	return r.Result, nil
}

// PaginationOptions can be passed to a list request to configure paging
// These values will be defaulted if omitted, and PerPage has min/max limits set by resource
type PaginationOptions struct {
	Page    int `json:"page,omitempty"`
	PerPage int `json:"per_page,omitempty"`
}

// RetryPolicy specifies number of retries and min/max retry delays
// This config is used when the client exponentially backs off after errored requests
type RetryPolicy struct {
	MaxRetries    int
	MinRetryDelay time.Duration
	MaxRetryDelay time.Duration
}

// Logger defines the interface this library needs to use logging
// This is a subset of the methods implemented in the log package
type Logger interface {
	Printf(format string, v ...interface{})
}

// ReqOption is a functional option for configuring API requests
type ReqOption func(opt *reqOption)
type reqOption struct {
	params url.Values
}

// WithZoneFilter applies a filter based on zone name.
func WithZoneFilter(zone string) ReqOption {
	return func(opt *reqOption) {
		opt.params.Set("name", zone)
	}
}

// WithPagination configures the pagination for a response.
func WithPagination(opts PaginationOptions) ReqOption {
	return func(opt *reqOption) {
		opt.params.Set("page", strconv.Itoa(opts.Page))
		opt.params.Set("per_page", strconv.Itoa(opts.PerPage))
	}
}
<commit_msg>Actually check if rate limit is hit (#325)<commit_after>// Package cloudflare implements the Cloudflare v4 API.
package cloudflare

import (
	"bytes"
	"context"
	"encoding/json"
	"io"
	"io/ioutil"
	"log"
	"math"
	"net/http"
	"net/url"
	"strconv"
	"strings"
	"time"

	"github.com/pkg/errors"
	"golang.org/x/time/rate"
)

const apiURL = "https://api.cloudflare.com/client/v4"

const (
	// AuthKeyEmail specifies that we should authenticate with API key and email address
	AuthKeyEmail = 1 << iota
	// AuthUserService specifies that we should authenticate with a User-Service key
	AuthUserService
	// AuthToken specifies that we should authenticate with an API Token
	AuthToken
)

// API holds the configuration for the current API client. A client should not
// be modified concurrently.
type API struct {
	APIKey            string
	APIEmail          string
	APIUserServiceKey string
	APIToken          string
	BaseURL           string
	OrganizationID    string
	UserAgent         string
	headers           http.Header
	httpClient        *http.Client
	authType          int
	rateLimiter       *rate.Limiter
	retryPolicy       RetryPolicy
	logger            Logger
}

// newClient provides shared logic for New and NewWithUserServiceKey
func newClient(opts ...Option) (*API, error) {
	silentLogger := log.New(ioutil.Discard, "", log.LstdFlags)

	api := &API{
		BaseURL:     apiURL,
		headers:     make(http.Header),
		rateLimiter: rate.NewLimiter(rate.Limit(4), 1), // 4rps equates to default api limit (1200 req/5 min)
		retryPolicy: RetryPolicy{
			MaxRetries:    3,
			MinRetryDelay: time.Duration(1) * time.Second,
			MaxRetryDelay: time.Duration(30) * time.Second,
		},
		logger: silentLogger,
	}

	err := api.parseOptions(opts...)
	if err != nil {
		return nil, errors.Wrap(err, "options parsing failed")
	}

	// Fall back to http.DefaultClient if the package user does not provide
	// their own.
	if api.httpClient == nil {
		api.httpClient = http.DefaultClient
	}

	return api, nil
}

// New creates a new Cloudflare v4 API client.
func New(key, email string, opts ...Option) (*API, error) {
	if key == "" || email == "" {
		return nil, errors.New(errEmptyCredentials)
	}

	api, err := newClient(opts...)
	if err != nil {
		return nil, err
	}

	api.APIKey = key
	api.APIEmail = email
	api.authType = AuthKeyEmail

	return api, nil
}

// NewWithAPIToken creates a new Cloudflare v4 API client using API Tokens
func NewWithAPIToken(token string, opts ...Option) (*API, error) {
	if token == "" {
		return nil, errors.New(errEmptyAPIToken)
	}

	api, err := newClient(opts...)
	if err != nil {
		return nil, err
	}

	api.APIToken = token
	api.authType = AuthToken

	return api, nil
}

// NewWithUserServiceKey creates a new Cloudflare v4 API client using service key authentication.
func NewWithUserServiceKey(key string, opts ...Option) (*API, error) {
	if key == "" {
		return nil, errors.New(errEmptyCredentials)
	}

	api, err := newClient(opts...)
	if err != nil {
		return nil, err
	}

	api.APIUserServiceKey = key
	api.authType = AuthUserService

	return api, nil
}

// SetAuthType sets the authentication method (AuthKeyEmail, AuthToken, or AuthUserService).
func (api *API) SetAuthType(authType int) {
	api.authType = authType
}

// ZoneIDByName retrieves a zone's ID from the name.
func (api *API) ZoneIDByName(zoneName string) (string, error) {
	res, err := api.ListZonesContext(context.TODO(), WithZoneFilter(zoneName))
	if err != nil {
		return "", errors.Wrap(err, "ListZonesContext command failed")
	}

	if len(res.Result) > 1 && api.OrganizationID == "" {
		return "", errors.New("ambiguous zone name used without an account ID")
	}

	for _, zone := range res.Result {
		if api.OrganizationID != "" {
			if zone.Name == zoneName && api.OrganizationID == zone.Account.ID {
				return zone.ID, nil
			}
		} else {
			if zone.Name == zoneName {
				return zone.ID, nil
			}
		}
	}

	return "", errors.New("Zone could not be found")
}

// makeRequest makes a HTTP request and returns the body as a byte slice,
// closing it before returning. params will be serialized to JSON.
func (api *API) makeRequest(method, uri string, params interface{}) ([]byte, error) {
	return api.makeRequestWithAuthType(context.TODO(), method, uri, params, api.authType)
}

func (api *API) makeRequestContext(ctx context.Context, method, uri string, params interface{}) ([]byte, error) {
	return api.makeRequestWithAuthType(ctx, method, uri, params, api.authType)
}

func (api *API) makeRequestWithHeaders(method, uri string, params interface{}, headers http.Header) ([]byte, error) {
	return api.makeRequestWithAuthTypeAndHeaders(context.TODO(), method, uri, params, api.authType, headers)
}

func (api *API) makeRequestWithAuthType(ctx context.Context, method, uri string, params interface{}, authType int) ([]byte, error) {
	return api.makeRequestWithAuthTypeAndHeaders(ctx, method, uri, params, authType, nil)
}

func (api *API) makeRequestWithAuthTypeAndHeaders(ctx context.Context, method, uri string, params interface{}, authType int, headers http.Header) ([]byte, error) {
	// Replace nil with a JSON object if needed
	var jsonBody []byte
	var err error

	if params != nil {
		if paramBytes, ok := params.([]byte); ok {
			jsonBody = paramBytes
		} else {
			jsonBody, err = json.Marshal(params)
			if err != nil {
				return nil, errors.Wrap(err, "error marshalling params to JSON")
			}
		}
	} else {
		jsonBody = nil
	}

	var resp *http.Response
	var respErr error
	var reqBody io.Reader
	var respBody []byte
	for i := 0; i <= api.retryPolicy.MaxRetries; i++ {
		if jsonBody != nil {
			reqBody = bytes.NewReader(jsonBody)
		}
		if i > 0 {
			// expect the backoff introduced here on errored requests to dominate the effect of rate limiting
			// don't need a random component here as the rate limiter should do something similar
			// nb time duration could truncate an arbitrary float. Since our inputs are all ints, we should be ok
			sleepDuration := time.Duration(math.Pow(2, float64(i-1)) * float64(api.retryPolicy.MinRetryDelay))

			if sleepDuration > api.retryPolicy.MaxRetryDelay {
				sleepDuration = api.retryPolicy.MaxRetryDelay
			}
			// useful to do some simple logging here, maybe introduce levels later
			api.logger.Printf("Sleeping %s before retry attempt number %d for request %s %s", sleepDuration.String(), i, method, uri)
			time.Sleep(sleepDuration)

		}
		err = api.rateLimiter.Wait(context.TODO())
		if err != nil {
			return nil, errors.Wrap(err, "Error caused by request rate limiting")
		}
		resp, respErr = api.request(ctx, method, uri, reqBody, authType, headers)

		// retry if the server is rate limiting us or if it failed
		// assumes server operations are rolled back on failure
		if respErr != nil || resp.StatusCode == http.StatusTooManyRequests || resp.StatusCode >= 500 {
			// if we got a valid http response, try to read body so we can reuse the connection
			// see https://golang.org/pkg/net/http/#Client.Do
			if respErr == nil {
				respBody, err = ioutil.ReadAll(resp.Body)
				resp.Body.Close()

				respErr = errors.Wrap(err, "could not read response body")

				api.logger.Printf("Request: %s %s got an error response %d: %s\n", method, uri, resp.StatusCode,
					strings.Replace(strings.Replace(string(respBody), "\n", "", -1), "\t", "", -1))
			} else {
				api.logger.Printf("Error performing request: %s %s : %s \n", method, uri, respErr.Error())
			}
			continue
		} else {
			respBody, err = ioutil.ReadAll(resp.Body)
			defer resp.Body.Close()
			if err != nil {
				return nil, errors.Wrap(err, "could not read response body")
			}
			break
		}
	}
	if respErr != nil {
		return nil, respErr
	}

	switch {
	case resp.StatusCode >= http.StatusOK && resp.StatusCode < http.StatusMultipleChoices:
	case resp.StatusCode == http.StatusUnauthorized:
		return nil, errors.Errorf("HTTP status %d: invalid credentials", resp.StatusCode)
	case resp.StatusCode == http.StatusForbidden:
		return nil, errors.Errorf("HTTP status %d: insufficient permissions", resp.StatusCode)
	case resp.StatusCode == http.StatusServiceUnavailable,
		resp.StatusCode == http.StatusBadGateway,
		resp.StatusCode == http.StatusGatewayTimeout,
		resp.StatusCode == 522,
		resp.StatusCode == 523,
		resp.StatusCode == 524:
		return nil, errors.Errorf("HTTP status %d: service failure", resp.StatusCode)
	// This isn't a great solution due to the way the `default` case is
	// a catch all and that the `filters/validate-expr` returns a HTTP 400
	// yet the clients need to use the HTTP body as a JSON string.
	case resp.StatusCode == 400 && strings.HasSuffix(resp.Request.URL.Path, "/filters/validate-expr"):
		return nil, errors.Errorf("%s", respBody)
	default:
		var s string
		if respBody != nil {
			s = string(respBody)
		}
		return nil, errors.Errorf("HTTP status %d: content %q", resp.StatusCode, s)
	}

	return respBody, nil
}

// request makes a HTTP request to the given API endpoint, returning the raw
// *http.Response, or an error if one occurred. The caller is responsible for
// closing the response body.
func (api *API) request(ctx context.Context, method, uri string, reqBody io.Reader, authType int, headers http.Header) (*http.Response, error) {
	req, err := http.NewRequest(method, api.BaseURL+uri, reqBody)
	if err != nil {
		return nil, errors.Wrap(err, "HTTP request creation failed")
	}
	req.WithContext(ctx)

	combinedHeaders := make(http.Header)
	copyHeader(combinedHeaders, api.headers)
	copyHeader(combinedHeaders, headers)
	req.Header = combinedHeaders

	if authType&AuthKeyEmail != 0 {
		req.Header.Set("X-Auth-Key", api.APIKey)
		req.Header.Set("X-Auth-Email", api.APIEmail)
	}
	if authType&AuthUserService != 0 {
		req.Header.Set("X-Auth-User-Service-Key", api.APIUserServiceKey)
	}
	if authType&AuthToken != 0 {
		req.Header.Set("Authorization", "Bearer "+api.APIToken)
	}

	if api.UserAgent != "" {
		req.Header.Set("User-Agent", api.UserAgent)
	}

	if req.Header.Get("Content-Type") == "" {
		req.Header.Set("Content-Type", "application/json")
	}

	resp, err := api.httpClient.Do(req)
	if err != nil {
		return nil, errors.Wrap(err, "HTTP request failed")
	}

	return resp, nil
}

// Returns the base URL to use for API endpoints that exist for both accounts and organizations.
// If an Organization option was used when creating the API instance, returns the org URL.
//
// accountBase is the base URL for endpoints referring to the current user. It exists as a
// parameter because it is not consistent across APIs.
func (api *API) userBaseURL(accountBase string) string {
	if api.OrganizationID != "" {
		return "/accounts/" + api.OrganizationID
	}
	return accountBase
}

// copyHeader copies all headers for `source` and sets them on `target`.
// based on https://godoc.org/github.com/golang/gddo/httputil/header#Copy
func copyHeader(target, source http.Header) {
	for k, vs := range source {
		target[k] = vs
	}
}

// ResponseInfo contains a code and message returned by the API as errors or
// informational messages inside the response.
type ResponseInfo struct {
	Code    int    `json:"code"`
	Message string `json:"message"`
}

// Response is a template.  There will also be a result struct.  There will be a
// unique response type for each response, which will include this type.
type Response struct {
	Success  bool           `json:"success"`
	Errors   []ResponseInfo `json:"errors"`
	Messages []ResponseInfo `json:"messages"`
}

// ResultInfo contains metadata about the Response.
type ResultInfo struct {
	Page       int `json:"page"`
	PerPage    int `json:"per_page"`
	TotalPages int `json:"total_pages"`
	Count      int `json:"count"`
	Total      int `json:"total_count"`
}

// RawResponse keeps the result as JSON form
type RawResponse struct {
	Response
	Result json.RawMessage `json:"result"`
}

// Raw makes a HTTP request with user provided params and returns the
// result as untouched JSON.
func (api *API) Raw(method, endpoint string, data interface{}) (json.RawMessage, error) {
	res, err := api.makeRequest(method, endpoint, data)
	if err != nil {
		return nil, errors.Wrap(err, errMakeRequestError)
	}

	var r RawResponse
	if err := json.Unmarshal(res, &r); err != nil {
		return nil, errors.Wrap(err, errUnmarshalError)
	}
	return r.Result, nil
}

// PaginationOptions can be passed to a list request to configure paging
// These values will be defaulted if omitted, and PerPage has min/max limits set by resource
type PaginationOptions struct {
	Page    int `json:"page,omitempty"`
	PerPage int `json:"per_page,omitempty"`
}

// RetryPolicy specifies number of retries and min/max retry delays
// This config is used when the client exponentially backs off after errored requests
type RetryPolicy struct {
	MaxRetries    int
	MinRetryDelay time.Duration
	MaxRetryDelay time.Duration
}

// Logger defines the interface this library needs to use logging
// This is a subset of the methods implemented in the log package
type Logger interface {
	Printf(format string, v ...interface{})
}

// ReqOption is a functional option for configuring API requests
type ReqOption func(opt *reqOption)
type reqOption struct {
	params url.Values
}

// WithZoneFilter applies a filter based on zone name.
func WithZoneFilter(zone string) ReqOption {
	return func(opt *reqOption) {
		opt.params.Set("name", zone)
	}
}

// WithPagination configures the pagination for a response.
func WithPagination(opts PaginationOptions) ReqOption {
	return func(opt *reqOption) {
		opt.params.Set("page", strconv.Itoa(opts.Page))
		opt.params.Set("per_page", strconv.Itoa(opts.PerPage))
	}
}
<|endoftext|>


<commit_before>// +build windows

package vmware

import (
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"syscall"
	"unsafe"
)

func workstationCheckLicense() error {
	// Not implemented on Windows
	return nil
}

func workstationFindVdiskManager() (string, error) {
	path, _ := exec.LookPath("vmware-vdiskmanager.exe")
	if fileExists(path) {
		return path, nil
	}

	return findProgramFile("vmware-vdiskmanager.exe"), nil
}

func workstationFindVMware() (string, error) {
	path, _ := exec.LookPath("vmware.exe")
	if fileExists(path) {
		return path, nil
	}

	return findProgramFile("vmware.exe"), nil
}

func workstationFindVmrun() (string, error) {
	path, _ := exec.LookPath("vmrun.exe")
	if fileExists(path) {
		return path, nil
	}

	return findProgramFile("vmrun.exe"), nil
}

func workstationToolsIsoPath(flavor string) string {
	return findProgramFile(flavor + ".iso")
}

func workstationDhcpLeasesPath(device string) string {
	path, _ := workstationVmnetDhcpLeasesPathFromRegistry()

	if fileExists(path) {
		return path
	}

	return findDataFile("vmnetdhcp.leases")
}

// See http://blog.natefinch.com/2012/11/go-win-stuff.html
//
// This is used by workstationVMwareRoot in order to read some registry data.
func readRegString(hive syscall.Handle, subKeyPath, valueName string) (value string, err error) {
	var h syscall.Handle
	err = syscall.RegOpenKeyEx(hive, syscall.StringToUTF16Ptr(subKeyPath), 0, syscall.KEY_READ, &h)
	if err != nil {
		return
	}
	defer syscall.RegCloseKey(h)

	var typ uint32
	var bufSize uint32
	err = syscall.RegQueryValueEx(
		h,
		syscall.StringToUTF16Ptr(valueName),
		nil,
		&typ,
		nil,
		&bufSize)
	if err != nil {
		return
	}

	data := make([]uint16, bufSize/2+1)
	err = syscall.RegQueryValueEx(
		h,
		syscall.StringToUTF16Ptr(valueName),
		nil,
		&typ,
		(*byte)(unsafe.Pointer(&data[0])),
		&bufSize)
	if err != nil {
		return
	}

	return syscall.UTF16ToString(data), nil
}

// This reads the VMware installation path from the Windows registry.
func workstationVMwareRoot() (s string, err error) {
	key := `SOFTWARE\Microsoft\Windows\CurrentVersion\App Paths\vmware.exe`
	subkey := "Path"
	s, err = readRegString(syscall.HKEY_LOCAL_MACHINE, key, subkey)
	if err != nil {
		log.Printf(`Unable to read registry key %s\%s`, key, subkey)
		return
	}

	return normalizePath(s), nil
}

// This reads the VMware DHCP leases path from the Windows registry.
func workstationVmnetDhcpLeasesPathFromRegistry() (s string, err error) {
	key := "SYSTEM\\CurrentControlSet\\services\\VMnetDHCP\\Parameters"
	subkey := "LeaseFile"
	s, err = readRegString(syscall.HKEY_LOCAL_MACHINE, key, subkey)
	if err != nil {
		log.Printf(`Unable to read registry key %s\%s`, key, subkey)
		return
	}

	return normalizePath(s), nil
}

func fileExists(file string) bool {
	if file == "" {
		return false
	}

	if _, err := os.Stat(file); err != nil {
		if os.IsNotExist(err) {
			return false
		}
		log.Println(err.Error())
	}

	return true
}

func normalizePath(path string) string {
	path = strings.Replace(path, "\\", "/", -1)
	path = strings.Replace(path, "//", "/", -1)
	path = strings.TrimRight(path, "/")
	return path
}

type Paths [][]string

func findFile(file string, paths Paths) string {
	for _, a := range paths {
		if a[0] == "" {
			continue
		}

		path := filepath.Join(a[0], a[1], file)

		path = normalizePath(path)

		log.Printf("Searching for file '%s'", path)

		if fileExists(path) {
			log.Printf("Found file '%s'", path)
			return path
		}
	}

	log.Printf("File not found: '%s'", file)

	return ""
}

func findProgramFile(file string) string {
	path, _ := workstationVMwareRoot()

	paths := Paths{
		[]string{os.Getenv("VMWARE_HOME"), ""},
		[]string{path, ""},
		[]string{os.Getenv("ProgramFiles(x86)"), "/VMware/VMware Workstation"},
		[]string{os.Getenv("ProgramFiles"), "/VMware/VMware Workstation"},
	}

	return findFile(file, paths)
}

func findDataFile(file string) string {
	path, _ := workstationVmnetDhcpLeasesPathFromRegistry()

	if path != "" {
		path = filepath.Dir(path)
	}

	paths := Paths{
		[]string{os.Getenv("VMWARE_DATA"), ""},
		[]string{path, ""},
		[]string{os.Getenv("ProgramData"), "/VMWare"},
		[]string{os.Getenv("ALLUSERSPROFILE"), "/VMWare"},
	}

	return findFile(file, paths)
}

func workstationVmnetnatConfPath() string {
	const VMNETNAT_CONF = "vmnetnat.conf"

	return findDataFile(VMNETNAT_CONF)
}
<commit_msg>fixed typo in vmware path on Windows XP, fixes #237/#238<commit_after>// +build windows

package vmware

import (
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"syscall"
	"unsafe"
)

func workstationCheckLicense() error {
	// Not implemented on Windows
	return nil
}

func workstationFindVdiskManager() (string, error) {
	path, _ := exec.LookPath("vmware-vdiskmanager.exe")
	if fileExists(path) {
		return path, nil
	}

	return findProgramFile("vmware-vdiskmanager.exe"), nil
}

func workstationFindVMware() (string, error) {
	path, _ := exec.LookPath("vmware.exe")
	if fileExists(path) {
		return path, nil
	}

	return findProgramFile("vmware.exe"), nil
}

func workstationFindVmrun() (string, error) {
	path, _ := exec.LookPath("vmrun.exe")
	if fileExists(path) {
		return path, nil
	}

	return findProgramFile("vmrun.exe"), nil
}

func workstationToolsIsoPath(flavor string) string {
	return findProgramFile(flavor + ".iso")
}

func workstationDhcpLeasesPath(device string) string {
	path, _ := workstationVmnetDhcpLeasesPathFromRegistry()

	if fileExists(path) {
		return path
	}

	return findDataFile("vmnetdhcp.leases")
}

// See http://blog.natefinch.com/2012/11/go-win-stuff.html
//
// This is used by workstationVMwareRoot in order to read some registry data.
func readRegString(hive syscall.Handle, subKeyPath, valueName string) (value string, err error) {
	var h syscall.Handle
	err = syscall.RegOpenKeyEx(hive, syscall.StringToUTF16Ptr(subKeyPath), 0, syscall.KEY_READ, &h)
	if err != nil {
		return
	}
	defer syscall.RegCloseKey(h)

	var typ uint32
	var bufSize uint32
	err = syscall.RegQueryValueEx(
		h,
		syscall.StringToUTF16Ptr(valueName),
		nil,
		&typ,
		nil,
		&bufSize)
	if err != nil {
		return
	}

	data := make([]uint16, bufSize/2+1)
	err = syscall.RegQueryValueEx(
		h,
		syscall.StringToUTF16Ptr(valueName),
		nil,
		&typ,
		(*byte)(unsafe.Pointer(&data[0])),
		&bufSize)
	if err != nil {
		return
	}

	return syscall.UTF16ToString(data), nil
}

// This reads the VMware installation path from the Windows registry.
func workstationVMwareRoot() (s string, err error) {
	key := `SOFTWARE\Microsoft\Windows\CurrentVersion\App Paths\vmware.exe`
	subkey := "Path"
	s, err = readRegString(syscall.HKEY_LOCAL_MACHINE, key, subkey)
	if err != nil {
		log.Printf(`Unable to read registry key %s\%s`, key, subkey)
		return
	}

	return normalizePath(s), nil
}

// This reads the VMware DHCP leases path from the Windows registry.
func workstationVmnetDhcpLeasesPathFromRegistry() (s string, err error) {
	key := "SYSTEM\\CurrentControlSet\\services\\VMnetDHCP\\Parameters"
	subkey := "LeaseFile"
	s, err = readRegString(syscall.HKEY_LOCAL_MACHINE, key, subkey)
	if err != nil {
		log.Printf(`Unable to read registry key %s\%s`, key, subkey)
		return
	}

	return normalizePath(s), nil
}

func fileExists(file string) bool {
	if file == "" {
		return false
	}

	if _, err := os.Stat(file); err != nil {
		if os.IsNotExist(err) {
			return false
		}
		log.Println(err.Error())
	}

	return true
}

func normalizePath(path string) string {
	path = strings.Replace(path, "\\", "/", -1)
	path = strings.Replace(path, "//", "/", -1)
	path = strings.TrimRight(path, "/")
	return path
}

type Paths [][]string

func findFile(file string, paths Paths) string {
	for _, a := range paths {
		if a[0] == "" {
			continue
		}

		path := filepath.Join(a[0], a[1], file)

		path = normalizePath(path)

		log.Printf("Searching for file '%s'", path)

		if fileExists(path) {
			log.Printf("Found file '%s'", path)
			return path
		}
	}

	log.Printf("File not found: '%s'", file)

	return ""
}

func findProgramFile(file string) string {
	path, _ := workstationVMwareRoot()

	paths := Paths{
		[]string{os.Getenv("VMWARE_HOME"), ""},
		[]string{path, ""},
		[]string{os.Getenv("ProgramFiles(x86)"), "/VMware/VMware Workstation"},
		[]string{os.Getenv("ProgramFiles"), "/VMware/VMware Workstation"},
	}

	return findFile(file, paths)
}

func findDataFile(file string) string {
	path, _ := workstationVmnetDhcpLeasesPathFromRegistry()

	if path != "" {
		path = filepath.Dir(path)
	}

	paths := Paths{
		[]string{os.Getenv("VMWARE_DATA"), ""},
		[]string{path, ""},
		[]string{os.Getenv("ProgramData"), "/VMWare"},
		[]string{os.Getenv("ALLUSERSPROFILE"), "/Application Data/VMWare"},
	}

	return findFile(file, paths)
}

func workstationVmnetnatConfPath() string {
	const VMNETNAT_CONF = "vmnetnat.conf"

	return findDataFile(VMNETNAT_CONF)
}
<|endoftext|>


<commit_before>package vapidkeys

import (
	"crypto/elliptic"
	"encoding/base64"
	"github.com/enceve/crypto/dh/ecdh"
)

// This function generates a new set of vapid keys and returns them along with the error message
func Generate() (vapidPrivateKey string, vapidPublicKey string, err error) {
	curve := ecdh.GenericCurve(elliptic.P256())
	privateKey, publicKey, err := curve.GenerateKey(nil)
	if err != nil {
		return "", "", err
	}
	privKey := base64.RawURLEncoding.EncodeToString(privateKey)
	pubKey := base64.RawURLEncoding.EncodeToString(publicKey)
	return privKey, pubKey, nil
}
<commit_msg>Update vapidkeys.go<commit_after>package vapidkeys

import (
	"crypto/elliptic"
	"encoding/base64"
	"github.com/enceve/crypto/dh/ecdh"
)

// Generate generates a new set of vapid keys and returns them along with the error message
func Generate() (vapidPrivateKey string, vapidPublicKey string, err error) {
	curve := ecdh.GenericCurve(elliptic.P256())
	privateKey, publicKey, err := curve.GenerateKey(nil)
	if err != nil {
		return "", "", err
	}
	privKey := base64.RawURLEncoding.EncodeToString(privateKey)
	pubKey := base64.RawURLEncoding.EncodeToString(publicKey)
	return privKey, pubKey, nil
}
<|endoftext|>


<commit_before>package cmd

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"

	"github.com/Sirupsen/logrus"
	"github.com/spf13/cobra"
	"gopkg.in/yaml.v2"

	"net/url"

	"github.com/emc-advanced-dev/pkg/errors"
	"github.com/emc-advanced-dev/unik/pkg/config"
	"github.com/emc-advanced-dev/unik/pkg/daemon"
	unikutil "github.com/emc-advanced-dev/unik/pkg/util"
)

var daemonRuntimeFolder, daemonConfigFile, logFile string
var debugMode, trace bool

var daemonCmd = &cobra.Command{
	Use:   "daemon",
	Short: "Runs the unik backend (daemon)",
	Long: `Run this command to start the unik daemon process.
This should normally be left running as a long-running background process.
The daemon requires that docker is installed and running on the your system.
Necessary docker containers must be built for the daemon to work properly;
Run 'make' in the unik root directory to build all necessary containers.

Daemon also requires a configuration file with credentials and configuration info
for desired providers.

Example usage:
	unik daemon --f ./my-config.yaml --port 12345 --debug --trace --logfile logs.txt

	 # will start the daemon using config file at my-config.yaml
	 # running on port 12345
	 # debug mode activated
	 # trace mode activated
	 # outputting logs to logs.txt
`,
	Run: func(cmd *cobra.Command, args []string) {
		if err := func() error {

			// set unik home
			config.Internal.UnikHome = daemonRuntimeFolder

			if daemonConfigFile == "" {
				daemonConfigFile = filepath.Join(config.Internal.UnikHome, "daemon-config.yaml")
			}

			if err := readDaemonConfig(); err != nil {
				return err
			}

			//don't print vsphere password
			redactions := []string{}
			for _, vsphereConfig := range daemonConfig.Providers.Vsphere {
				redactions = append(redactions, vsphereConfig.VspherePassword, url.QueryEscape(vsphereConfig.VspherePassword))
			}
			logrus.SetFormatter(&unikutil.RedactedTextFormatter{
				Redactions: redactions,
			})

			if debugMode {
				logrus.SetLevel(logrus.DebugLevel)
			}
			if trace {
				logrus.AddHook(&unikutil.AddTraceHook{true})
			}
			if logFile != "" {
				os.Create(logFile)
				f, err := os.OpenFile(logFile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0777)
				if err != nil {
					return errors.New(fmt.Sprintf("failed to open log file %s for writing", logFile), err)
				}
				logrus.AddHook(&unikutil.TeeHook{f})
			}

			logrus.WithField("config", daemonConfig).Info("daemon started")
			d, err := daemon.NewUnikDaemon(daemonConfig)
			if err != nil {
				return errors.New("daemon failed to initialize", err)
			}
			d.Run(port)
			return nil
		}(); err != nil {
			logrus.Errorf("running daemon failed: %v", err)
			os.Exit(-1)
		}
	},
}

func init() {
	RootCmd.AddCommand(daemonCmd)
	daemonCmd.Flags().StringVar(&daemonRuntimeFolder, "d", os.Getenv("HOME")+"/.unik/", "daemon runtime folder - where state is stored. (default is $HOME/.unik/)")
	daemonCmd.Flags().StringVar(&daemonConfigFile, "f", "", "daemon config file (default is {RuntimeFolder}/daemon-config.yaml)")
	daemonCmd.Flags().IntVar(&port, "port", 3000, "<int, optional> listening port for daemon")
	daemonCmd.Flags().BoolVar(&debugMode, "debug", false, "<bool, optional> more verbose logging for the daemon")
	daemonCmd.Flags().BoolVar(&trace, "trace", false, "<bool, optional> add stack trace to daemon logs")
	daemonCmd.Flags().StringVar(&logFile, "logfile", "", "<string, optional> output logs to file (in addition to stdout)")
}

var daemonConfig config.DaemonConfig

func readDaemonConfig() error {
	data, err := ioutil.ReadFile(daemonConfigFile)
	if err != nil {
		logrus.WithError(err).Errorf("failed to read daemon configuration file at " + daemonConfigFile + `\n
		See documentation at http://github.com/emc-advanced-dev/unik for creating daemon config.'`)
		return err
	}
	if err := yaml.Unmarshal(data, &daemonConfig); err != nil {
		logrus.WithError(err).Errorf("failed to parse daemon configuration yaml at " + daemonConfigFile + `\n
		Please ensure config file contains valid yaml.'`)
		return err
	}
	return nil
}
<commit_msg>mute err msg<commit_after>package cmd

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"

	"github.com/Sirupsen/logrus"
	"github.com/spf13/cobra"
	"gopkg.in/yaml.v2"

	"net/url"

	"github.com/emc-advanced-dev/pkg/errors"
	"github.com/emc-advanced-dev/unik/pkg/config"
	"github.com/emc-advanced-dev/unik/pkg/daemon"
	unikutil "github.com/emc-advanced-dev/unik/pkg/util"
)

var daemonRuntimeFolder, daemonConfigFile, logFile string
var debugMode, trace bool

var daemonCmd = &cobra.Command{
	Use:   "daemon",
	Short: "Runs the unik backend (daemon)",
	Long: `Run this command to start the unik daemon process.
This should normally be left running as a long-running background process.
The daemon requires that docker is installed and running on the your system.
Necessary docker containers must be built for the daemon to work properly;
Run 'make' in the unik root directory to build all necessary containers.

Daemon also requires a configuration file with credentials and configuration info
for desired providers.

Example usage:
	unik daemon --f ./my-config.yaml --port 12345 --debug --trace --logfile logs.txt

	 # will start the daemon using config file at my-config.yaml
	 # running on port 12345
	 # debug mode activated
	 # trace mode activated
	 # outputting logs to logs.txt
`,
	Run: func(cmd *cobra.Command, args []string) {
		if err := func() error {

			// set unik home
			config.Internal.UnikHome = daemonRuntimeFolder

			if daemonConfigFile == "" {
				daemonConfigFile = filepath.Join(config.Internal.UnikHome, "daemon-config.yaml")
			}

			if err := readDaemonConfig(); err != nil {
				return err
			}

			//don't print vsphere password
			redactions := []string{}
			for _, vsphereConfig := range daemonConfig.Providers.Vsphere {
				redactions = append(redactions, vsphereConfig.VspherePassword, url.QueryEscape(vsphereConfig.VspherePassword))
			}
			logrus.SetFormatter(&unikutil.RedactedTextFormatter{
				Redactions: redactions,
			})

			if debugMode {
				logrus.SetLevel(logrus.DebugLevel)
			}
			if trace {
				logrus.AddHook(&unikutil.AddTraceHook{true})
			}
			if logFile != "" {
				os.Create(logFile)
				f, err := os.OpenFile(logFile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0777)
				if err != nil {
					return errors.New(fmt.Sprintf("failed to open log file %s for writing", logFile), err)
				}
				logrus.AddHook(&unikutil.TeeHook{f})
			}

			logrus.WithField("config", daemonConfig).Info("daemon started")
			d, err := daemon.NewUnikDaemon(daemonConfig)
			if err != nil {
				return errors.New("daemon failed to initialize", err)
			}
			d.Run(port)
			return nil
		}(); err != nil {
			logrus.Errorf("running daemon failed: %v", err)
			os.Exit(-1)
		}
	},
}

func init() {
	RootCmd.AddCommand(daemonCmd)
	daemonCmd.Flags().StringVar(&daemonRuntimeFolder, "d", os.Getenv("HOME")+"/.unik/", "daemon runtime folder - where state is stored. (default is $HOME/.unik/)")
	daemonCmd.Flags().StringVar(&daemonConfigFile, "f", "", "daemon config file (default is {RuntimeFolder}/daemon-config.yaml)")
	daemonCmd.Flags().IntVar(&port, "port", 3000, "<int, optional> listening port for daemon")
	daemonCmd.Flags().BoolVar(&debugMode, "debug", false, "<bool, optional> more verbose logging for the daemon")
	daemonCmd.Flags().BoolVar(&trace, "trace", false, "<bool, optional> add stack trace to daemon logs")
	daemonCmd.Flags().StringVar(&logFile, "logfile", "", "<string, optional> output logs to file (in addition to stdout)")
}

var daemonConfig config.DaemonConfig

func readDaemonConfig() error {
	data, err := ioutil.ReadFile(daemonConfigFile)
	if err != nil {
		errMsg := fmt.Sprintf("failed to read daemon configuration file at " + daemonConfigFile + `\n
		See documentation at http://github.com/emc-advanced-dev/unik for creating daemon config.'`)
		return errors.New(errMsg, err)
	}
	if err := yaml.Unmarshal(data, &daemonConfig); err != nil {
		errMsg := fmt.Sprintf("failed to parse daemon configuration yaml at " + daemonConfigFile + `\n
		Please ensure config file contains valid yaml.'`)
		return errors.New(errMsg, err)
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"io"
	"log"
	"log/syslog"
	"os"
	"sync"
)

// With Go 1.1, we can specify the syslog facility in addition to the priority:
// https://code.google.com/p/go/source/browse/src/pkg/log/syslog/syslog.go?name=go1.1
// To stay compatible with older Go versions, set the facility by hand for now.
const LOG_DAEMON = (3 << 3)

// derive logger
func getLogger(useSyslog bool) (logger io.Writer, err error) {
	if useSyslog {
		logger, err = syslog.New(LOG_DAEMON|syslog.LOG_NOTICE, monitoringEvent)
	} else {
		logger = os.Stderr
		log.SetPrefix(monitoringEvent + ": ")
	}
	if err == nil {
		log.SetOutput(logger)
	}
	return &LineWriter{w: logger}, err
}

// pipe r to logger in the background
func logStream(r io.Reader, logger io.Writer, wg *sync.WaitGroup) {
	wg.Add(1)

	go func() {
		for {
			if _, err := io.Copy(logger, r); err != nil {
				break
			}
		}
		wg.Done()
	}()
}
<commit_msg>Use upstream log package allowing to set syslog facility<commit_after>package main

import (
	// With Go 1.1, we can specify the syslog facility in addition to the priority:
	// https://code.google.com/p/go/source/browse/src/pkg/log/syslog/syslog.go?name=go1.1
	// To be able to set the facility with older versions of Go, we simply
	// added the log package from Go 1.1 to periodicnoise.
	"github.com/Jimdo/periodicnoise/log/syslog"
	"io"
	"log"
	"os"
	"sync"
)

// derive logger
func getLogger(useSyslog bool) (logger io.Writer, err error) {
	if useSyslog {
		logger, err = syslog.New(syslog.LOG_DAEMON|syslog.LOG_NOTICE, monitoringEvent)
	} else {
		logger = os.Stderr
		log.SetPrefix(monitoringEvent + ": ")
	}
	if err == nil {
		log.SetOutput(logger)
	}
	return &LineWriter{w: logger}, err
}

// pipe r to logger in the background
func logStream(r io.Reader, logger io.Writer, wg *sync.WaitGroup) {
	wg.Add(1)

	go func() {
		for {
			if _, err := io.Copy(logger, r); err != nil {
				break
			}
		}
		wg.Done()
	}()
}
<|endoftext|>