<commit_before>package cloudpelican

// @author Robin Verlangen
// @todo Support bulk index requests
// Tool for logging data to CloudPelican directly from Go

// Imports
import (
    "net"
    "net/http"
    "net/url"
    "log"
    "sync"
    "time"
)

// Settings
var ENDPOINT string = "https://app.cloudpelican.com/api"
var TOKEN string = ""
var backendTimeout = time.Duration(5 * time.Second)
var debugMode = false

// Monitor drain status
var startCounter uint64 = uint64(0)
var startCounterMux sync.Mutex
var doneCounter uint64 = uint64(0)
var doneCounterMux sync.Mutex

// Log queue
var writeAheadBufferSize int = 1000
var writeAhead chan map[string]string = make(chan map[string]string, writeAheadBufferSize)
var writeAheadInit bool
var dropOnFullWriteAheadBuffer bool = true

// Set token
func SetToken(t string) {
    // Validate before setting
    validateToken(t)

    // Store
    TOKEN = t
}

// Set endpoint
func SetEndpoint(e string) {
    // Store
    ENDPOINT = e
}

// Set timeout
func SetBackendTimeout(to time.Duration) {
    backendTimeout = to
}

// Debug
func SetDebugMode(b bool) {
    debugMode = b
}

// Write a message
func LogMessage(msg string) bool {
    // Create fields map
    params := make(map[string]string)
    params["__token__"] = TOKEN
    params["msg"] = msg

    // Push to channel
    return requestAsync(params)
}

// Request async
func requestAsync(params map[string]string) bool {
    // Check amount of open items in the channel, if the channel is full, return false and drop this message
    if dropOnFullWriteAheadBuffer {
        var lwa int = len(writeAhead)
        if lwa == writeAheadBufferSize {
            log.Printf("Write ahead buffer is full and contains %d items. Dropping current log message", lwa)
        }
    }

    // Add counter
    startCounterMux.Lock()
    startCounter++
    startCounterMux.Unlock()

    // Do we have to start a writer?
    if writeAheadInit == false {
        writeAheadInit = true
        backendWriter()
    }

    // Insert into channel
    writeAhead <- params

    // OK
    return true
}

// Backend writer
func backendWriter() {
    go func() {
        // Client
        transport := &http.Transport{
            Dial: func(netw, addr string) (net.Conn, error) {
                    deadline := time.Now().Add(backendTimeout)
                    c, err := net.DialTimeout(netw, addr, time.Second)
                    if err != nil {
                            return nil, err
                    }
                    c.SetDeadline(deadline)
                    return c, nil
            }}
        httpclient := &http.Client{Transport: transport}

        // Wait for messages
        var urlParams url.Values = nil
        var currentEventCount int = 0
        for {
            // Read from channel
            var fields map[string]string
            fields = <- writeAhead

            // Populate url params
            if urlParams == nil {
                urlParams := url.Values{}
            }
            for k, _ := range fields {
                if k == "__token__" {
                    // Token
                    urlParams.Add("t", fields[k]);
                } else {
                    // Field
                    urlParams.Add("f[" + strconv.Itoa(currentEventCount) + "][" + k + "]", fields[k]);
                }
            }

            // Queue length
            var qLen = len(writeAhead)
            if qLen > 0 {
                // There is more in the current queue, bulk request
            }

            // Assemble url
            var url string = ENDPOINT + "/push/bulk"

            // Make request
            if debugMode {
                log.Printf("Write ahead queue %d\n", qLen)
                log.Println(urlParams.Encode())
            }
            resp, err := httpclient.PostForm(url, urlParams)
            if err != nil {
                log.Printf("Error while forwarding data: %s\n", err)
            } else {
                defer resp.Body.Close()
            }

            // Done counter
            doneCounterMux.Lock()
            doneCounter++
            doneCounterMux.Unlock()
        }
        log.Printf("Stopping backend writer")
    }()
}

// Timeout helper
func dialTimeout(network, addr string) (net.Conn, error) {
    return net.DialTimeout(network, addr, backendTimeout)
}

// Validate the token
func validateToken(t string) {
    if len(t) == 0 {
        log.Println("Please set a valid token with cloudpelican.SetToken(token string)")
    }
}<commit_msg>Import<commit_after>package cloudpelican

// @author Robin Verlangen
// @todo Support bulk index requests
// Tool for logging data to CloudPelican directly from Go

// Imports
import (
    "net"
    "net/http"
    "net/url"
    "log"
    "sync"
    "time"
    "strconv"
)

// Settings
var ENDPOINT string = "https://app.cloudpelican.com/api"
var TOKEN string = ""
var backendTimeout = time.Duration(5 * time.Second)
var debugMode = false

// Monitor drain status
var startCounter uint64 = uint64(0)
var startCounterMux sync.Mutex
var doneCounter uint64 = uint64(0)
var doneCounterMux sync.Mutex

// Log queue
var writeAheadBufferSize int = 1000
var writeAhead chan map[string]string = make(chan map[string]string, writeAheadBufferSize)
var writeAheadInit bool
var dropOnFullWriteAheadBuffer bool = true

// Set token
func SetToken(t string) {
    // Validate before setting
    validateToken(t)

    // Store
    TOKEN = t
}

// Set endpoint
func SetEndpoint(e string) {
    // Store
    ENDPOINT = e
}

// Set timeout
func SetBackendTimeout(to time.Duration) {
    backendTimeout = to
}

// Debug
func SetDebugMode(b bool) {
    debugMode = b
}

// Write a message
func LogMessage(msg string) bool {
    // Create fields map
    params := make(map[string]string)
    params["__token__"] = TOKEN
    params["msg"] = msg

    // Push to channel
    return requestAsync(params)
}

// Request async
func requestAsync(params map[string]string) bool {
    // Check amount of open items in the channel, if the channel is full, return false and drop this message
    if dropOnFullWriteAheadBuffer {
        var lwa int = len(writeAhead)
        if lwa == writeAheadBufferSize {
            log.Printf("Write ahead buffer is full and contains %d items. Dropping current log message", lwa)
        }
    }

    // Add counter
    startCounterMux.Lock()
    startCounter++
    startCounterMux.Unlock()

    // Do we have to start a writer?
    if writeAheadInit == false {
        writeAheadInit = true
        backendWriter()
    }

    // Insert into channel
    writeAhead <- params

    // OK
    return true
}

// Backend writer
func backendWriter() {
    go func() {
        // Client
        transport := &http.Transport{
            Dial: func(netw, addr string) (net.Conn, error) {
                    deadline := time.Now().Add(backendTimeout)
                    c, err := net.DialTimeout(netw, addr, time.Second)
                    if err != nil {
                            return nil, err
                    }
                    c.SetDeadline(deadline)
                    return c, nil
            }}
        httpclient := &http.Client{Transport: transport}

        // Wait for messages
        var urlParams url.Values = nil
        var currentEventCount int = 0
        for {
            // Read from channel
            var fields map[string]string
            fields = <- writeAhead

            // Populate url params
            if urlParams == nil {
                urlParams := url.Values{}
            }
            for k, _ := range fields {
                if k == "__token__" {
                    // Token
                    urlParams.Add("t", fields[k]);
                } else {
                    // Field
                    urlParams.Add("f[" + strconv.Itoa(currentEventCount) + "][" + k + "]", fields[k]);
                }
            }

            // Queue length
            var qLen = len(writeAhead)
            if qLen > 0 {
                // There is more in the current queue, bulk request
            }

            // Assemble url
            var url string = ENDPOINT + "/push/bulk"

            // Make request
            if debugMode {
                log.Printf("Write ahead queue %d\n", qLen)
                log.Println(urlParams.Encode())
            }
            resp, err := httpclient.PostForm(url, urlParams)
            if err != nil {
                log.Printf("Error while forwarding data: %s\n", err)
            } else {
                defer resp.Body.Close()
            }

            // Done counter
            doneCounterMux.Lock()
            doneCounter++
            doneCounterMux.Unlock()
        }
        log.Printf("Stopping backend writer")
    }()
}

// Timeout helper
func dialTimeout(network, addr string) (net.Conn, error) {
    return net.DialTimeout(network, addr, backendTimeout)
}

// Validate the token
func validateToken(t string) {
    if len(t) == 0 {
        log.Println("Please set a valid token with cloudpelican.SetToken(token string)")
    }
}<|endoftext|>


<commit_before>package utils

import (
	"errors"
	"reflect"
	"strings"
)

func StringInArray(key string, array []string) bool {
	if key == "" || len(array) == 0 {
		return false
	}
	for _, v := range array {
		if strings.ToLower(key) == strings.ToLower(v) {
			return true
		}
	}
	return false
}

func StructToMap(data interface{}, lowerKey bool, except ...string) (m map[string]interface{}, err error) {
	m = make(map[string]interface{}, 0)
	if data == nil {
		err = errors.New("struct to map error data is nil")
		return
	}
	v := reflect.ValueOf(data)
	if v.Kind() == reflect.Ptr {
		v = v.Elem()
	}
	if v.Kind() != reflect.Struct {
		err = errors.New("data must be struct")
		return
	}
	t := v.Type()
	for i := 0; i < v.NumField(); i++ {
		f := v.Field(i)
		if StringInArray(t.Field(i).Name, except) {
			continue
		}
		if lowerKey {
			m[strings.ToLower(t.Field(i).Name)] = f.Interface()
		} else {
			m[t.Field(i).Name] = f.Interface()
		}
	}
	return
}
<commit_msg>modify method StructToMap<commit_after>package utils

import (
	"errors"
	"reflect"
	"strings"
)

func StringInArray(key string, array []string) bool {
	if key == "" || len(array) == 0 {
		return false
	}
	for _, v := range array {
		if strings.ToLower(key) == strings.ToLower(v) {
			return true
		}
	}
	return false
}

func StructToMap(data interface{}, lowerKey, bExcept bool, fields ...string) (m map[string]interface{}, err error) {
	m = make(map[string]interface{}, 0)
	if data == nil {
		err = errors.New("struct to map error data is nil")
		return
	}
	v := reflect.ValueOf(data)
	if v.Kind() == reflect.Ptr {
		v = v.Elem()
	}
	if v.Kind() != reflect.Struct {
		err = errors.New("data must be struct")
		return
	}
	t := v.Type()
	for i := 0; i < v.NumField(); i++ {
		f := v.Field(i)
		if bExcept {
			if StringInArray(t.Field(i).Name, fields) {
				continue
			}
		} else {
			if !StringInArray(t.Field(i).Name, fields) {
				continue
			}
		}
		if lowerKey {
			m[strings.ToLower(t.Field(i).Name)] = f.Interface()
		} else {
			m[t.Field(i).Name] = f.Interface()
		}
	}
	return
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

// This file contains code for supporting local sockets for the Cloud SQL Auth proxy.

import (
	"bytes"
	"errors"
	"fmt"
	"net"
	"net/http"
	"os"
	"path/filepath"
	"runtime"
	"strings"
	"time"

	"github.com/GoogleCloudPlatform/cloudsql-proxy/logging"
	"github.com/GoogleCloudPlatform/cloudsql-proxy/proxy/fuse"
	"github.com/GoogleCloudPlatform/cloudsql-proxy/proxy/proxy"
	"github.com/GoogleCloudPlatform/cloudsql-proxy/proxy/util"
	sqladmin "google.golang.org/api/sqladmin/v1beta4"
)

// WatchInstances handles the lifecycle of local sockets used for proxying
// local connections.  Values received from the updates channel are
// interpretted as a comma-separated list of instances.  The set of sockets in
// 'dir' is the union of 'instances' and the most recent list from 'updates'.
func WatchInstances(dir string, cfgs []instanceConfig, updates <-chan string, cl *http.Client) (<-chan proxy.Conn, error) {
	ch := make(chan proxy.Conn, 1)

	// Instances specified statically (e.g. as flags to the binary) will always
	// be available. They are ignored if also returned by the GCE metadata since
	// the socket will already be open.
	staticInstances := make(map[string]net.Listener, len(cfgs))
	for _, v := range cfgs {
		l, err := listenInstance(ch, v)
		if err != nil {
			return nil, err
		}
		staticInstances[v.Instance] = l
	}

	if updates != nil {
		go watchInstancesLoop(dir, ch, updates, staticInstances, cl)
	}
	return ch, nil
}

func watchInstancesLoop(dir string, dst chan<- proxy.Conn, updates <-chan string, static map[string]net.Listener, cl *http.Client) {
	dynamicInstances := make(map[string]net.Listener)
	for instances := range updates {
		// All instances were legal when we started, so we pass false below to ensure we don't skip them
		// later if they became unhealthy for some reason; this would be a serious enough problem.
		list, err := parseInstanceConfigs(dir, strings.Split(instances, ","), cl, false)
		if err != nil {
			logging.Errorf("%v", err)
			// If we do not have a valid list of instances, skip this update
			continue
		}

		stillOpen := make(map[string]net.Listener)
		for _, cfg := range list {
			instance := cfg.Instance

			// If the instance is specified in the static list don't do anything:
			// it's already open and should stay open forever.
			if _, ok := static[instance]; ok {
				continue
			}

			if l, ok := dynamicInstances[instance]; ok {
				delete(dynamicInstances, instance)
				stillOpen[instance] = l
				continue
			}

			l, err := listenInstance(dst, cfg)
			if err != nil {
				logging.Errorf("Couldn't open socket for %q: %v", instance, err)
				continue
			}
			stillOpen[instance] = l
		}

		// Any instance in dynamicInstances was not in the most recent metadata
		// update. Clean up those instances' sockets by closing them; note that
		// this does not affect any existing connections instance.
		for instance, listener := range dynamicInstances {
			logging.Infof("Closing socket for instance %v", instance)
			listener.Close()
		}

		dynamicInstances = stillOpen
	}

	for _, v := range static {
		if err := v.Close(); err != nil {
			logging.Errorf("Error closing %q: %v", v.Addr(), err)
		}
	}
	for _, v := range dynamicInstances {
		if err := v.Close(); err != nil {
			logging.Errorf("Error closing %q: %v", v.Addr(), err)
		}
	}
}

func remove(path string) {
	if err := os.Remove(path); err != nil && !os.IsNotExist(err) {
		logging.Infof("Remove(%q) error: %v", path, err)
	}
}

// listenInstance starts listening on a new unix socket in dir to connect to the
// specified instance. New connections to this socket are sent to dst.
func listenInstance(dst chan<- proxy.Conn, cfg instanceConfig) (net.Listener, error) {
	unix := cfg.Network == "unix"
	if unix {
		remove(cfg.Address)
	}
	l, err := net.Listen(cfg.Network, cfg.Address)
	if err != nil {
		return nil, err
	}
	if unix {
		if err := os.Chmod(cfg.Address, 0777|os.ModeSocket); err != nil {
			logging.Errorf("couldn't update permissions for socket file %q: %v; other users may not be unable to connect", cfg.Address, err)
		}
	}

	go func() {
		for {
			start := time.Now()
			c, err := l.Accept()
			if err != nil {
				logging.Errorf("Error in accept for %q on %v: %v", cfg, cfg.Address, err)
				if nerr, ok := err.(net.Error); ok && nerr.Temporary() {
					d := 10*time.Millisecond - time.Since(start)
					if d > 0 {
						time.Sleep(d)
					}
					continue
				}
				l.Close()
				return
			}
			logging.Verbosef("New connection for %q", cfg.Instance)

			switch clientConn := c.(type) {
			case *net.TCPConn:
				clientConn.SetKeepAlive(true)
				clientConn.SetKeepAlivePeriod(1 * time.Minute)

			}
			dst <- proxy.Conn{cfg.Instance, c}
		}
	}()

	logging.Infof("Listening on %s for %s", cfg.Address, cfg.Instance)
	return l, nil
}

type instanceConfig struct {
	Instance         string
	Network, Address string
}

// loopbackForNet maps a network (e.g. tcp6) to the loopback address for that
// network. It is updated during the initialization of validNets to include a
// valid loopback address for "tcp".
var loopbackForNet = map[string]string{
	"tcp4": "127.0.0.1",
	"tcp6": "::1",
}

// validNets tracks the networks that are valid for this platform and machine.
var validNets = func() map[string]bool {
	m := map[string]bool{
		"unix": runtime.GOOS != "windows",
	}

	anyTCP := false
	for _, n := range []string{"tcp4", "tcp6"} {
		host, ok := loopbackForNet[n]
		if !ok {
			// This is effectively a compile-time error.
			panic(fmt.Sprintf("no loopback address found for %v", n))
		}
		// Open any port to see if the net is valid.
		x, err := net.Listen(n, net.JoinHostPort(host, "0"))
		if err != nil {
			// Error is too verbose to be useful.
			continue
		}
		x.Close()
		m[n] = true

		if !anyTCP {
			anyTCP = true
			// Set the loopback value for generic tcp if it hasn't already been
			// set. (If both tcp4/tcp6 are supported the first one in the list
			// (tcp4's 127.0.0.1) is used.
			loopbackForNet["tcp"] = host
		}
	}
	if anyTCP {
		m["tcp"] = true
	}
	return m
}()

func parseInstanceConfig(dir, instance string, cl *http.Client) (instanceConfig, error) {
	var ret instanceConfig
	args := strings.Split(instance, "=")
	if len(args) > 2 {
		return instanceConfig{}, fmt.Errorf("invalid instance argument: must be either form - `<instance_connection_string>` or `<instance_connection_string>=<options>`; invalid arg was %q", instance)
	}
	// Parse the instance connection name - everything before the "=".
	ret.Instance = args[0]
	proj, region, name := util.SplitName(ret.Instance)
	regionName := fmt.Sprintf("%s~%s", region, name)
	if proj == "" || region == "" || name == "" {
		return instanceConfig{}, fmt.Errorf("invalid instance connection string: must be in the form `project:region:instance-name`; invalid name was %q", args[0])
	}
	if len(args) == 1 {
		// Default to listening via unix socket in specified directory
		ret.Network = "unix"
		ret.Address = filepath.Join(dir, instance)
	} else {
		// Parse the instance options if present.
		opts := strings.SplitN(args[1], ":", 2)
		if len(opts) != 2 {
			return instanceConfig{}, fmt.Errorf("invalid instance options: must be in the form `unix:/path/to/socket`, `tcp:port`, `tcp:host:port`; invalid option was %q", strings.Join(opts, ":"))
		}
		ret.Network = opts[0]
		var err error
		if ret.Network == "unix" {
			if strings.HasPrefix(opts[1], "/") {
				ret.Address = opts[1] // Root path.
			} else {
				ret.Address = filepath.Join(dir, opts[1])
			}
		} else {
			ret.Address, err = parseTCPOpts(opts[0], opts[1])
		}
		if err != nil {
			return instanceConfig{}, err
		}
	}

	// Use the SQL Admin API to verify compatibility with the instance.
	sql, err := sqladmin.New(cl)
	if err != nil {
		return instanceConfig{}, err
	}
	if *host != "" {
		sql.BasePath = *host
	}
	inst, err := sql.Instances.Get(proj, regionName).Do()
	if err != nil {
		return instanceConfig{}, err
	}
	if inst.BackendType == "FIRST_GEN" {
		logging.Errorf("WARNING: proxy client does not support first generation Cloud SQL instances.")
		return instanceConfig{}, fmt.Errorf("%q is a first generation instance", instance)
	}
	// Postgres instances use a special suffix on the unix socket.
	// See https://www.postgresql.org/docs/11/runtime-config-connection.html
	if ret.Network == "unix" && strings.HasPrefix(strings.ToLower(inst.DatabaseVersion), "postgres") {
		// Verify the directory exists.
		if err := os.MkdirAll(ret.Address, 0755); err != nil {
			return instanceConfig{}, err
		}
		ret.Address = filepath.Join(ret.Address, ".s.PGSQL.5432")
	}

	if !validNets[ret.Network] {
		return ret, fmt.Errorf("invalid %q: unsupported network: %v", instance, ret.Network)
	}
	return ret, nil
}

// parseTCPOpts parses the instance options when specifying tcp port options.
func parseTCPOpts(ntwk, addrOpt string) (string, error) {
	if strings.Contains(addrOpt, ":") {
		return addrOpt, nil // User provided a host and port; use that.
	}
	// No "host" part of the address. Be safe and assume that they want a loopback address.
	addr, ok := loopbackForNet[ntwk]
	if !ok {
		return "", fmt.Errorf("invalid %q:%q: unrecognized network %v", ntwk, addrOpt, ntwk)
	}
	return net.JoinHostPort(addr, addrOpt), nil
}

// parseInstanceConfigs calls parseInstanceConfig for each instance in the
// provided slice, collecting errors along the way. There may be valid
// instanceConfigs returned even if there's an error.
func parseInstanceConfigs(dir string, instances []string, cl *http.Client, skipFailedInstanceConfigs bool) ([]instanceConfig, error) {
	errs := new(bytes.Buffer)
	var cfg []instanceConfig
	for _, v := range instances {
		if v == "" {
			continue
		}
		if c, err := parseInstanceConfig(dir, v, cl); err != nil {
			if skipFailedInstanceConfigs {
				logging.Infof("There was a problem when parsing a instance configuration but ignoring due to the configuration. Error: %v", err)
			} else {
				fmt.Fprintf(errs, "\n\t%v", err)
			}

		} else {
			cfg = append(cfg, c)
		}
	}

	var err error
	if errs.Len() > 0 {
		err = fmt.Errorf("errors parsing config:%s", errs)
	}
	return cfg, err
}

// CreateInstanceConfigs verifies that the parameters passed to it are valid
// for the proxy for the platform and system and then returns a slice of valid
// instanceConfig. It is possible for the instanceConfig to be empty if no valid
// configurations were specified, however `err` will be set.
func CreateInstanceConfigs(dir string, useFuse bool, instances []string, instancesSrc string, cl *http.Client, skipFailedInstanceConfigs bool) ([]instanceConfig, error) {
	if useFuse && !fuse.Supported() {
		return nil, errors.New("FUSE not supported on this system")
	}

	cfgs, err := parseInstanceConfigs(dir, instances, cl, skipFailedInstanceConfigs)
	if err != nil {
		return nil, err
	}

	if dir == "" {
		// Reasons to set '-dir':
		//    - Using -fuse
		//    - Using the metadata to get a list of instances
		//    - Having an instance that uses a 'unix' network
		if useFuse {
			return nil, errors.New("must set -dir because -fuse was set")
		} else if instancesSrc != "" {
			return nil, errors.New("must set -dir because -instances_metadata was set")
		} else {
			for _, v := range cfgs {
				if v.Network == "unix" {
					return nil, fmt.Errorf("must set -dir: using a unix socket for %v", v.Instance)
				}
			}
		}
		// Otherwise it's safe to not set -dir
	}

	if useFuse {
		if len(instances) != 0 || instancesSrc != "" {
			return nil, errors.New("-fuse is not compatible with -projects, -instances, or -instances_metadata")
		}
		return nil, nil
	}
	// FUSE disabled.
	if len(instances) == 0 && instancesSrc == "" {
		// Failure to specifying instance can be caused by following reasons.
		// 1. not enough information is provided by flags
		// 2. failed to invoke gcloud
		var flags string
		if fuse.Supported() {
			flags = "-projects, -fuse, -instances or -instances_metadata"
		} else {
			flags = "-projects, -instances or -instances_metadata"
		}

		errStr := fmt.Sprintf("no instance selected because none of %s is specified", flags)
		return nil, errors.New(errStr)
	}
	return cfgs, nil
}
<commit_msg>chore: correct typo (#819)<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

// This file contains code for supporting local sockets for the Cloud SQL Auth proxy.

import (
	"bytes"
	"errors"
	"fmt"
	"net"
	"net/http"
	"os"
	"path/filepath"
	"runtime"
	"strings"
	"time"

	"github.com/GoogleCloudPlatform/cloudsql-proxy/logging"
	"github.com/GoogleCloudPlatform/cloudsql-proxy/proxy/fuse"
	"github.com/GoogleCloudPlatform/cloudsql-proxy/proxy/proxy"
	"github.com/GoogleCloudPlatform/cloudsql-proxy/proxy/util"
	sqladmin "google.golang.org/api/sqladmin/v1beta4"
)

// WatchInstances handles the lifecycle of local sockets used for proxying
// local connections.  Values received from the updates channel are
// interpretted as a comma-separated list of instances.  The set of sockets in
// 'dir' is the union of 'instances' and the most recent list from 'updates'.
func WatchInstances(dir string, cfgs []instanceConfig, updates <-chan string, cl *http.Client) (<-chan proxy.Conn, error) {
	ch := make(chan proxy.Conn, 1)

	// Instances specified statically (e.g. as flags to the binary) will always
	// be available. They are ignored if also returned by the GCE metadata since
	// the socket will already be open.
	staticInstances := make(map[string]net.Listener, len(cfgs))
	for _, v := range cfgs {
		l, err := listenInstance(ch, v)
		if err != nil {
			return nil, err
		}
		staticInstances[v.Instance] = l
	}

	if updates != nil {
		go watchInstancesLoop(dir, ch, updates, staticInstances, cl)
	}
	return ch, nil
}

func watchInstancesLoop(dir string, dst chan<- proxy.Conn, updates <-chan string, static map[string]net.Listener, cl *http.Client) {
	dynamicInstances := make(map[string]net.Listener)
	for instances := range updates {
		// All instances were legal when we started, so we pass false below to ensure we don't skip them
		// later if they became unhealthy for some reason; this would be a serious enough problem.
		list, err := parseInstanceConfigs(dir, strings.Split(instances, ","), cl, false)
		if err != nil {
			logging.Errorf("%v", err)
			// If we do not have a valid list of instances, skip this update
			continue
		}

		stillOpen := make(map[string]net.Listener)
		for _, cfg := range list {
			instance := cfg.Instance

			// If the instance is specified in the static list don't do anything:
			// it's already open and should stay open forever.
			if _, ok := static[instance]; ok {
				continue
			}

			if l, ok := dynamicInstances[instance]; ok {
				delete(dynamicInstances, instance)
				stillOpen[instance] = l
				continue
			}

			l, err := listenInstance(dst, cfg)
			if err != nil {
				logging.Errorf("Couldn't open socket for %q: %v", instance, err)
				continue
			}
			stillOpen[instance] = l
		}

		// Any instance in dynamicInstances was not in the most recent metadata
		// update. Clean up those instances' sockets by closing them; note that
		// this does not affect any existing connections instance.
		for instance, listener := range dynamicInstances {
			logging.Infof("Closing socket for instance %v", instance)
			listener.Close()
		}

		dynamicInstances = stillOpen
	}

	for _, v := range static {
		if err := v.Close(); err != nil {
			logging.Errorf("Error closing %q: %v", v.Addr(), err)
		}
	}
	for _, v := range dynamicInstances {
		if err := v.Close(); err != nil {
			logging.Errorf("Error closing %q: %v", v.Addr(), err)
		}
	}
}

func remove(path string) {
	if err := os.Remove(path); err != nil && !os.IsNotExist(err) {
		logging.Infof("Remove(%q) error: %v", path, err)
	}
}

// listenInstance starts listening on a new unix socket in dir to connect to the
// specified instance. New connections to this socket are sent to dst.
func listenInstance(dst chan<- proxy.Conn, cfg instanceConfig) (net.Listener, error) {
	unix := cfg.Network == "unix"
	if unix {
		remove(cfg.Address)
	}
	l, err := net.Listen(cfg.Network, cfg.Address)
	if err != nil {
		return nil, err
	}
	if unix {
		if err := os.Chmod(cfg.Address, 0777|os.ModeSocket); err != nil {
			logging.Errorf("couldn't update permissions for socket file %q: %v; other users may not be unable to connect", cfg.Address, err)
		}
	}

	go func() {
		for {
			start := time.Now()
			c, err := l.Accept()
			if err != nil {
				logging.Errorf("Error in accept for %q on %v: %v", cfg, cfg.Address, err)
				if nerr, ok := err.(net.Error); ok && nerr.Temporary() {
					d := 10*time.Millisecond - time.Since(start)
					if d > 0 {
						time.Sleep(d)
					}
					continue
				}
				l.Close()
				return
			}
			logging.Verbosef("New connection for %q", cfg.Instance)

			switch clientConn := c.(type) {
			case *net.TCPConn:
				clientConn.SetKeepAlive(true)
				clientConn.SetKeepAlivePeriod(1 * time.Minute)

			}
			dst <- proxy.Conn{cfg.Instance, c}
		}
	}()

	logging.Infof("Listening on %s for %s", cfg.Address, cfg.Instance)
	return l, nil
}

type instanceConfig struct {
	Instance         string
	Network, Address string
}

// loopbackForNet maps a network (e.g. tcp6) to the loopback address for that
// network. It is updated during the initialization of validNets to include a
// valid loopback address for "tcp".
var loopbackForNet = map[string]string{
	"tcp4": "127.0.0.1",
	"tcp6": "::1",
}

// validNets tracks the networks that are valid for this platform and machine.
var validNets = func() map[string]bool {
	m := map[string]bool{
		"unix": runtime.GOOS != "windows",
	}

	anyTCP := false
	for _, n := range []string{"tcp4", "tcp6"} {
		host, ok := loopbackForNet[n]
		if !ok {
			// This is effectively a compile-time error.
			panic(fmt.Sprintf("no loopback address found for %v", n))
		}
		// Open any port to see if the net is valid.
		x, err := net.Listen(n, net.JoinHostPort(host, "0"))
		if err != nil {
			// Error is too verbose to be useful.
			continue
		}
		x.Close()
		m[n] = true

		if !anyTCP {
			anyTCP = true
			// Set the loopback value for generic tcp if it hasn't already been
			// set. (If both tcp4/tcp6 are supported the first one in the list
			// (tcp4's 127.0.0.1) is used.
			loopbackForNet["tcp"] = host
		}
	}
	if anyTCP {
		m["tcp"] = true
	}
	return m
}()

func parseInstanceConfig(dir, instance string, cl *http.Client) (instanceConfig, error) {
	var ret instanceConfig
	args := strings.Split(instance, "=")
	if len(args) > 2 {
		return instanceConfig{}, fmt.Errorf("invalid instance argument: must be either form - `<instance_connection_string>` or `<instance_connection_string>=<options>`; invalid arg was %q", instance)
	}
	// Parse the instance connection name - everything before the "=".
	ret.Instance = args[0]
	proj, region, name := util.SplitName(ret.Instance)
	regionName := fmt.Sprintf("%s~%s", region, name)
	if proj == "" || region == "" || name == "" {
		return instanceConfig{}, fmt.Errorf("invalid instance connection string: must be in the form `project:region:instance-name`; invalid name was %q", args[0])
	}
	if len(args) == 1 {
		// Default to listening via unix socket in specified directory
		ret.Network = "unix"
		ret.Address = filepath.Join(dir, instance)
	} else {
		// Parse the instance options if present.
		opts := strings.SplitN(args[1], ":", 2)
		if len(opts) != 2 {
			return instanceConfig{}, fmt.Errorf("invalid instance options: must be in the form `unix:/path/to/socket`, `tcp:port`, `tcp:host:port`; invalid option was %q", strings.Join(opts, ":"))
		}
		ret.Network = opts[0]
		var err error
		if ret.Network == "unix" {
			if strings.HasPrefix(opts[1], "/") {
				ret.Address = opts[1] // Root path.
			} else {
				ret.Address = filepath.Join(dir, opts[1])
			}
		} else {
			ret.Address, err = parseTCPOpts(opts[0], opts[1])
		}
		if err != nil {
			return instanceConfig{}, err
		}
	}

	// Use the SQL Admin API to verify compatibility with the instance.
	sql, err := sqladmin.New(cl)
	if err != nil {
		return instanceConfig{}, err
	}
	if *host != "" {
		sql.BasePath = *host
	}
	inst, err := sql.Instances.Get(proj, regionName).Do()
	if err != nil {
		return instanceConfig{}, err
	}
	if inst.BackendType == "FIRST_GEN" {
		logging.Errorf("WARNING: proxy client does not support first generation Cloud SQL instances.")
		return instanceConfig{}, fmt.Errorf("%q is a first generation instance", instance)
	}
	// Postgres instances use a special suffix on the unix socket.
	// See https://www.postgresql.org/docs/11/runtime-config-connection.html
	if ret.Network == "unix" && strings.HasPrefix(strings.ToLower(inst.DatabaseVersion), "postgres") {
		// Verify the directory exists.
		if err := os.MkdirAll(ret.Address, 0755); err != nil {
			return instanceConfig{}, err
		}
		ret.Address = filepath.Join(ret.Address, ".s.PGSQL.5432")
	}

	if !validNets[ret.Network] {
		return ret, fmt.Errorf("invalid %q: unsupported network: %v", instance, ret.Network)
	}
	return ret, nil
}

// parseTCPOpts parses the instance options when specifying tcp port options.
func parseTCPOpts(ntwk, addrOpt string) (string, error) {
	if strings.Contains(addrOpt, ":") {
		return addrOpt, nil // User provided a host and port; use that.
	}
	// No "host" part of the address. Be safe and assume that they want a loopback address.
	addr, ok := loopbackForNet[ntwk]
	if !ok {
		return "", fmt.Errorf("invalid %q:%q: unrecognized network %v", ntwk, addrOpt, ntwk)
	}
	return net.JoinHostPort(addr, addrOpt), nil
}

// parseInstanceConfigs calls parseInstanceConfig for each instance in the
// provided slice, collecting errors along the way. There may be valid
// instanceConfigs returned even if there's an error.
func parseInstanceConfigs(dir string, instances []string, cl *http.Client, skipFailedInstanceConfigs bool) ([]instanceConfig, error) {
	errs := new(bytes.Buffer)
	var cfg []instanceConfig
	for _, v := range instances {
		if v == "" {
			continue
		}
		if c, err := parseInstanceConfig(dir, v, cl); err != nil {
			if skipFailedInstanceConfigs {
				logging.Infof("There was a problem when parsing an instance configuration but ignoring due to the configuration. Error: %v", err)
			} else {
				fmt.Fprintf(errs, "\n\t%v", err)
			}

		} else {
			cfg = append(cfg, c)
		}
	}

	var err error
	if errs.Len() > 0 {
		err = fmt.Errorf("errors parsing config:%s", errs)
	}
	return cfg, err
}

// CreateInstanceConfigs verifies that the parameters passed to it are valid
// for the proxy for the platform and system and then returns a slice of valid
// instanceConfig. It is possible for the instanceConfig to be empty if no valid
// configurations were specified, however `err` will be set.
func CreateInstanceConfigs(dir string, useFuse bool, instances []string, instancesSrc string, cl *http.Client, skipFailedInstanceConfigs bool) ([]instanceConfig, error) {
	if useFuse && !fuse.Supported() {
		return nil, errors.New("FUSE not supported on this system")
	}

	cfgs, err := parseInstanceConfigs(dir, instances, cl, skipFailedInstanceConfigs)
	if err != nil {
		return nil, err
	}

	if dir == "" {
		// Reasons to set '-dir':
		//    - Using -fuse
		//    - Using the metadata to get a list of instances
		//    - Having an instance that uses a 'unix' network
		if useFuse {
			return nil, errors.New("must set -dir because -fuse was set")
		} else if instancesSrc != "" {
			return nil, errors.New("must set -dir because -instances_metadata was set")
		} else {
			for _, v := range cfgs {
				if v.Network == "unix" {
					return nil, fmt.Errorf("must set -dir: using a unix socket for %v", v.Instance)
				}
			}
		}
		// Otherwise it's safe to not set -dir
	}

	if useFuse {
		if len(instances) != 0 || instancesSrc != "" {
			return nil, errors.New("-fuse is not compatible with -projects, -instances, or -instances_metadata")
		}
		return nil, nil
	}
	// FUSE disabled.
	if len(instances) == 0 && instancesSrc == "" {
		// Failure to specifying instance can be caused by following reasons.
		// 1. not enough information is provided by flags
		// 2. failed to invoke gcloud
		var flags string
		if fuse.Supported() {
			flags = "-projects, -fuse, -instances or -instances_metadata"
		} else {
			flags = "-projects, -instances or -instances_metadata"
		}

		errStr := fmt.Sprintf("no instance selected because none of %s is specified", flags)
		return nil, errors.New(errStr)
	}
	return cfgs, nil
}
<|endoftext|>


<commit_before>package upcloud

import (
	"errors"
	"fmt"
	"github.com/UpCloudLtd/upcloud-go-api/upcloud/client"
	"github.com/UpCloudLtd/upcloud-go-api/upcloud/service"
	"github.com/hashicorp/packer/common"
	"github.com/hashicorp/packer/helper/communicator"
	"github.com/hashicorp/packer/helper/config"
	"github.com/hashicorp/packer/packer"
	"github.com/hashicorp/packer/template/interpolate"
	"time"
)

// Config represents the configuration for this builder
type Config struct {
	common.PackerConfig `mapstructure:",squash"`
	Comm                communicator.Config `mapstructure:",squash"`

	// Required configuration values
	Username       string `mapstructure:"username"`
	Password       string `mapstructure:"password"`
	Zone           string `mapstructure:"zone"`
	StorageUUID    string `mapstructure:"storage_uuid"`
	TemplatePrefix string `mapstructure:"template_prefix"`

	// Optional configuration values
	StorageSize             int    `mapstructure:"storage_size"`
	RawStateTimeoutDuration string `mapstructure:"state_timeout_duration"`

	StateTimeoutDuration time.Duration
	ctx                  interpolate.Context
}

// GetService returns a service object using the credentials specified in the configuration
func (c *Config) GetService() *service.Service {
	t := client.New(c.Username, c.Password)
	return service.New(t)
}

// NewConfig creates a new configuration, setting default values and validating it along the way
func NewConfig(raws ...interface{}) (*Config, error) {
	c := new(Config)

	err := config.Decode(c, &config.DecodeOpts{
		Interpolate:        true,
		InterpolateContext: &c.ctx,
	}, raws...)

	if err != nil {
		return nil, err
	}

	// Assign default values if possible
	c.Comm.SSHUsername = "root"

	if c.StorageSize == 0 {
		c.StorageSize = 30
	}

	if c.RawStateTimeoutDuration == "" {
		c.RawStateTimeoutDuration = "5m"
	}

	// Validation
	var errs *packer.MultiError
	errs = packer.MultiErrorAppend(errs, c.Comm.Prepare(&c.ctx)...)

	// Check for required configurations that will display errors if not set
	if c.Username == "" {
		errs = packer.MultiErrorAppend(
			errs, errors.New("\"username\" must be specified"))
	}

	if c.Password == "" {
		errs = packer.MultiErrorAppend(
			errs, errors.New("\"password\" must be specified"))
	}

	if c.Zone == "" {
		errs = packer.MultiErrorAppend(
			errs, errors.New("\"zone\" must be specified"))
	}

	if c.StorageUUID == "" {
		errs = packer.MultiErrorAppend(
			errs, errors.New("\"storage_uuid\" must be specified"))
	}

	stateTimeout, err := time.ParseDuration(c.RawStateTimeoutDuration)
	if err != nil {
		errs = packer.MultiErrorAppend(
			errs, fmt.Errorf("Failed to parse state_timeout_duration: %s", err))
	}
	c.StateTimeoutDuration = stateTimeout

	if len(errs.Errors) > 0 {
		return nil, errors.New(errs.Error())
	}

	return c, nil
}
<commit_msg>Refactored<commit_after>package upcloud

import (
	"errors"
	"fmt"
	"github.com/UpCloudLtd/upcloud-go-api/upcloud/client"
	"github.com/UpCloudLtd/upcloud-go-api/upcloud/service"
	"github.com/hashicorp/packer/common"
	"github.com/hashicorp/packer/helper/communicator"
	"github.com/hashicorp/packer/helper/config"
	"github.com/hashicorp/packer/packer"
	"github.com/hashicorp/packer/template/interpolate"
	"time"
)

// Config represents the configuration for this builder
type Config struct {
	common.PackerConfig `mapstructure:",squash"`
	Comm                communicator.Config `mapstructure:",squash"`

	// Required configuration values
	Username       string `mapstructure:"username"`
	Password       string `mapstructure:"password"`
	Zone           string `mapstructure:"zone"`
	StorageUUID    string `mapstructure:"storage_uuid"`
	TemplatePrefix string `mapstructure:"template_prefix"`

	// Optional configuration values
	StorageSize             int    `mapstructure:"storage_size"`
	RawStateTimeoutDuration string `mapstructure:"state_timeout_duration"`

	StateTimeoutDuration time.Duration
	ctx                  interpolate.Context
}

// GetService returns a service object using the credentials specified in the configuration
func (c *Config) GetService() *service.Service {
	t := client.New(c.Username, c.Password)
	return service.New(t)
}

// NewConfig creates a new configuration, setting default values and validating it along the way
func NewConfig(raws ...interface{}) (*Config, error) {
	c := new(Config)

	err := config.Decode(c, &config.DecodeOpts{
		Interpolate:        true,
		InterpolateContext: &c.ctx,
	}, raws...)

	if err != nil {
		return nil, err
	}

	// Assign default values if possible
	c.Comm.SSHUsername = "root"

	if c.StorageSize == 0 {
		c.StorageSize = 30
	}

	if c.RawStateTimeoutDuration == "" {
		c.RawStateTimeoutDuration = "5m"
	}

	// Validation
	var errs *packer.MultiError
	errs = packer.MultiErrorAppend(errs, c.Comm.Prepare(&c.ctx)...)

	// Check for required configurations that will display errors if not set
	if c.Username == "" {
		errs = packer.MultiErrorAppend(
			errs, errors.New("\"username\" must be specified"))
	}

	if c.Password == "" {
		errs = packer.MultiErrorAppend(
			errs, errors.New("\"password\" must be specified"))
	}

	if c.Zone == "" {
		errs = packer.MultiErrorAppend(
			errs, errors.New("\"zone\" must be specified"))
	}

	if c.StorageUUID == "" {
		errs = packer.MultiErrorAppend(
			errs, errors.New("\"storage_uuid\" must be specified"))
	}

	stateTimeout, err := time.ParseDuration(c.RawStateTimeoutDuration)
	if err != nil {
		errs = packer.MultiErrorAppend(
			errs, fmt.Errorf("Failed to parse state_timeout_duration: %s", err))
	}
	c.StateTimeoutDuration = stateTimeout

	if len(errs.Errors) > 0 {
		return nil, errs
	}

	return c, nil
}
<|endoftext|>


<commit_before>package air

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net"
	"os"
	"time"
)

// Config is a global set of configurations that for an instance of `Air`
// for customization.
type Config struct {
	// AppName represens the name of current `Air` instance.
	AppName string

	// DebugMode represents the state of `Air`'s debug mode. Default
	// value is "false".
	// It's called "debug_mode" in the config file.
	DebugMode bool

	// LogFormat represents the format of `Logger`'s output content.
	// Default value is:
	// `{"app_name":"{{.app_name}}","time":"{{.time_rfc3339}}",` +
	// `"level":"{{.level}}","file":"{{.short_file}}","line":"{{.line}}"}`
	// It's called "log_format" in the config file.
	LogFormat string

	// Address represents the TCP address that `Server` to listen
	// on. Default value is "localhost:8080".
	// It's called "address" in the config file.
	Address string

	// Listener represens the custom `net.Listener`. If set, `Server`
	// accepts connections on it. Default value is "nil".
	Listener net.Listener

	// TLSCertFile represents the TLS certificate file path. Default
	// value is "".
	// It's called "tls_cert_file" in the config file.
	TLSCertFile string

	// TLSKeyFile represents the TLS key file path. Default value
	// is "".
	// It's called "tls_key_file" in the config file.
	TLSKeyFile string

	// ReadTimeout represents the maximum duration before timing out
	// read of the request. Default value is "0".
	// It's called "read_timeout" in the config file.
	// *It's unit in the config file is SECONDS.*
	ReadTimeout time.Duration

	// WriteTimeout represents the maximum duration before timing
	// out write of the response. Default value is "0".
	// It's called "write_timeout" in the config file.
	// *It's unit in the config file is SECONDS.*
	WriteTimeout time.Duration

	// TemplatesRoot represents the root directory of the html templates.
	// It will be parsed into `Renderer`.
	// Default value is "templates" that means a subdirectory of the
	// runtime directory.
	// It's called "templates_root" in the config file.
	TemplatesRoot string

	// Data represents the data that parsing from config file.
	Data JSONMap
}

// defaultConfig is the default instance of `Config`.
var defaultConfig Config

// configs is the JSON map that stored all the configs parsing from
// config file.
var configs JSONMap

func init() {
	defaultConfig = Config{
		LogFormat: `{"app_name":"{{.app_name}}","time":"{{.time_rfc3339}}",` +
			`"level":"{{.level}}","file":"{{.short_file}}","line":"{{.line}}"}`,
		Address:       "localhost:8080",
		TemplatesRoot: "templates",
	}

	var cfn = "config.json"
	_, err := os.Stat(cfn)
	if err == nil || os.IsExist(err) {
		bytes, err := ioutil.ReadFile(cfn)
		if err != nil {
			panic(err)
		}
		err = json.Unmarshal(bytes, &configs)
		if err != nil {
			panic(err)
		}
		if len(configs) == 0 {
			panic("need at least one app in the config file or remove the config file")
		}
	}
}

// NewConfig returns a new instance of `Config` with a appName by parsing
// the config file that in the rumtime directory named "config.json".
// NewConfig returns the defaultConfig(field "AppName" be setted to provided
// appName) if the config file or appName doesn't exist.
func NewConfig(appName string) *Config {
	c := defaultConfig
	switch {
	case configs == nil:
		c.AppName = appName
	case len(configs) == 1:
		for k, v := range configs {
			c.AppName = k
			c.Data = v.(map[string]interface{})
			c.fill()
		}
	case configs[appName] == nil:
		panic(fmt.Sprintf("app %s does not exist in the config file", appName))
	default:
		c.AppName = appName
		c.Data = configs[appName].(map[string]interface{})
		c.fill()
	}
	return &c
}

// fill fills field's value from field `Data` of c. It fills field's
// value from defaultConfig if target value in `Data` of c does not
// exist.
func (c *Config) fill() {
	if dm, ok := c.Data["debug_mode"]; ok {
		c.DebugMode = dm.(bool)
	}
	if lf, ok := c.Data["log_format"]; ok {
		c.LogFormat = lf.(string)
	}
	if addr, ok := c.Data["address"]; ok {
		c.Address = addr.(string)
	}
	if tlscf, ok := c.Data["tls_cert_file"]; ok {
		c.TLSCertFile = tlscf.(string)
	}
	if tlskf, ok := c.Data["tls_key_file"]; ok {
		c.TLSKeyFile = tlskf.(string)
	}
	if rt, ok := c.Data["read_timeout"]; ok {
		c.ReadTimeout = time.Duration(rt.(float64)) * time.Second
	}
	if wt, ok := c.Data["write_timeout"]; ok {
		c.WriteTimeout = time.Duration(wt.(float64)) * time.Second
	}
	if tr, ok := c.Data["templates_root"]; ok {
		c.TemplatesRoot = tr.(string)
	}
}
<commit_msg>docs: unify comments in config.go<commit_after>package air

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net"
	"os"
	"time"
)

// Config is a global set of configurations that for an instance of `Air`
// for customization.
type Config struct {
	// AppName represens the name of current `Air` instance.
	AppName string

	// DebugMode represents the state of `Air`'s debug mode.
	//
	// Default value is false.
	//
	// It's called "debug_mode" in the config file.
	DebugMode bool

	// LogFormat represents the format of `Logger`'s output content.
	//
	// Default value is:
	// `{"app_name":"{{.app_name}}","time":"{{.time_rfc3339}}",` +
	// `"level":"{{.level}}","file":"{{.short_file}}","line":"{{.line}}"}`
	//
	// It's called "log_format" in the config file.
	LogFormat string

	// Address represents the TCP address that `Server` to listen
	// on.
	//
	// Default value is "localhost:8080".
	//
	// It's called "address" in the config file.
	Address string

	// Listener represens the custom `net.Listener`. If set, `Server`
	// accepts connections on it.
	//
	// Default value is nil.
	Listener net.Listener

	// TLSCertFile represents the TLS certificate file path.
	//
	// Default value is "".
	//
	// It's called "tls_cert_file" in the config file.
	TLSCertFile string

	// TLSKeyFile represents the TLS key file path.
	//
	// Default value is "".
	//
	// It's called "tls_key_file" in the config file.
	TLSKeyFile string

	// ReadTimeout represents the maximum duration before timing out
	// read of the request.
	//
	// Default value is 0.
	//
	// It's called "read_timeout" in the config file.
	//
	// *It's unit in the config file is SECONDS.*
	ReadTimeout time.Duration

	// WriteTimeout represents the maximum duration before timing
	// out write of the response.
	//
	// Default value is 0.
	//
	// It's called "write_timeout" in the config file.
	//
	// *It's unit in the config file is SECONDS.*
	WriteTimeout time.Duration

	// TemplatesRoot represents the root directory of the html templates.
	// It will be parsed into `Renderer`.
	//
	// Default value is "templates" that means a subdirectory of the
	// runtime directory.
	//
	// It's called "templates_root" in the config file.
	TemplatesRoot string

	// Data represents the data that parsing from config file. You can
	// use it to access the values in the config file.
	//
	// e.g. c.Data["mysql_user_name"] will accesses the value in config
	// file called "mysql_user_name".
	Data JSONMap
}

// defaultConfig is the default instance of `Config`.
var defaultConfig Config

// configs is the JSON map that stored all the configs parsing from
// config file.
var configs JSONMap

func init() {
	defaultConfig = Config{
		LogFormat: `{"app_name":"{{.app_name}}","time":"{{.time_rfc3339}}",` +
			`"level":"{{.level}}","file":"{{.short_file}}","line":"{{.line}}"}`,
		Address:       "localhost:8080",
		TemplatesRoot: "templates",
	}

	var cfn = "config.json"
	_, err := os.Stat(cfn)
	if err == nil || os.IsExist(err) {
		bytes, err := ioutil.ReadFile(cfn)
		if err != nil {
			panic(err)
		}
		err = json.Unmarshal(bytes, &configs)
		if err != nil {
			panic(err)
		}
		if len(configs) == 0 {
			panic("need at least one app in the config file or remove the config file")
		}
	}
}

// NewConfig returns a new instance of `Config` with a appName by parsing
// the config file that in the rumtime directory named "config.json".
// NewConfig returns the defaultConfig(field "AppName" be setted to provided
// appName) if the config file or appName doesn't exist.
func NewConfig(appName string) *Config {
	c := defaultConfig
	switch {
	case configs == nil:
		c.AppName = appName
	case len(configs) == 1:
		for k, v := range configs {
			c.AppName = k
			c.Data = v.(map[string]interface{})
			c.fill()
		}
	case configs[appName] == nil:
		panic(fmt.Sprintf("app %s does not exist in the config file", appName))
	default:
		c.AppName = appName
		c.Data = configs[appName].(map[string]interface{})
		c.fill()
	}
	return &c
}

// fill fills field's value from field `Data` of c. It fills field's
// value from defaultConfig if target value in `Data` of c does not
// exist.
func (c *Config) fill() {
	if dm, ok := c.Data["debug_mode"]; ok {
		c.DebugMode = dm.(bool)
	}
	if lf, ok := c.Data["log_format"]; ok {
		c.LogFormat = lf.(string)
	}
	if addr, ok := c.Data["address"]; ok {
		c.Address = addr.(string)
	}
	if tlscf, ok := c.Data["tls_cert_file"]; ok {
		c.TLSCertFile = tlscf.(string)
	}
	if tlskf, ok := c.Data["tls_key_file"]; ok {
		c.TLSKeyFile = tlskf.(string)
	}
	if rt, ok := c.Data["read_timeout"]; ok {
		c.ReadTimeout = time.Duration(rt.(float64)) * time.Second
	}
	if wt, ok := c.Data["write_timeout"]; ok {
		c.WriteTimeout = time.Duration(wt.(float64)) * time.Second
	}
	if tr, ok := c.Data["templates_root"]; ok {
		c.TemplatesRoot = tr.(string)
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 Robert S. Gerus. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package config

import (
	"encoding/json"
	"io/ioutil"
	"log"
	"os"
	"sort"
	"sync"
	"time"
)

type cacheEntry struct {
	modTime  time.Time
	contents interface{}
}

type config struct {
	cache         map[string]cacheEntry
	buildFileList func(map[string]string) []string
	l             sync.Mutex
}

var c config

func init() {
	log.Println("Initializing configuration cache")
	c.cache = make(map[string]cacheEntry)
}

func lookupvar(key, path string) interface{} {
	var f interface{}
	i, err := os.Stat(path)
	_, ok := c.cache[path]
	if os.IsNotExist(err) {
		log.Println("Config does not exist", path)
		if ok {
			log.Println("Purging", path, "from cache")
			delete(c.cache, path)
		}
	}
	if err != nil {
		return nil
	}

	if c.cache[path].modTime.Before(i.ModTime()) || !ok {
		log.Println("Stale cache for", path)
		data, err := ioutil.ReadFile(path)
		if err != nil {
			return nil
		}

		err = json.Unmarshal(data, &f)
		if err != nil {
			log.Println("Cannot parse", path)
			return nil
		}

		log.Println("Updating cache for", path)
		c.cache[path] = cacheEntry{
			modTime:  i.ModTime(),
			contents: f,
		}

		return f.(map[string]interface{})[key]
	}

	log.Println("Found cache for", path)
	return c.cache[path].contents.(map[string]interface{})[key]
}

// SetFileListBuilder registers file list builder function.
//
// Registered function takes context (key-value map[string]string) as the only
// argument and return a slice of strings - file paths.
func SetFileListBuilder(f func(map[string]string) []string) {
	c.buildFileList = f
}

// Lookup searches for requested configuration key in file list built using
// context.
func Lookup(context map[string]string, key string) interface{} {
	var value interface{}

	c.l.Lock()
	defer c.l.Unlock()

	for _, fpath := range c.buildFileList(context) {
		log.Println("Context:", context, "Looking up", key, "in", fpath)
		value = lookupvar(key, fpath)
		if value != nil {
			log.Println("Context:", context, "Found", key, value)
			return value
		}
	}

	log.Println("Context:", context, "Key", key, "not found")
	return nil
}

// LookupString is analogous to Lookup(), but does the cast to string.
func LookupString(context map[string]string, key string) string {
	var value interface{}

	c.l.Lock()
	defer c.l.Unlock()

	for _, fpath := range c.buildFileList(context) {
		log.Println("Context:", context, "Looking up", key, "in", fpath)
		value = lookupvar(key, fpath)
		if value != nil {
			log.Println("Context:", context, "Found", key, value)
			return value.(string)
		}
	}

	log.Println("Context:", context, "Key", key, "not found")
	return ""
}

// LookupStringSlice is analogous to Lookup(), but does the cast to []string
func LookupStringSlice(context map[string]string, key string) (retval []string) {
	var value interface{}

	c.l.Lock()
	defer c.l.Unlock()

	for _, fpath := range c.buildFileList(context) {
		log.Println("Context:", context, "Looking up", key, "in", fpath)
		value = lookupvar(key, fpath)
		if value != nil {
			log.Println("Context:", context, "Found", key, value)
			for _, s := range value.([]interface{}) {
				retval = append(retval, s.(string))
			}
			sort.Strings(retval)
			return retval
		}
	}

	log.Println("Context:", context, "Key", key, "not found")
	return []string{""}
}

// LookupStringMap is analogous to Lookup(), but does the cast to
// map[string]bool for optimised lookups.
func LookupStringMap(context map[string]string, key string) (retval map[string]bool) {
	var value interface{}

	c.l.Lock()
	defer c.l.Unlock()

	for _, fpath := range c.buildFileList(context) {
		log.Println("Context:", context, "Looking up", key, "in", fpath)
		value = lookupvar(key, fpath)
		if value != nil {
			log.Println("Context:", context, "Found", key, value)
			for _, s := range value.([]interface{}) {
				retval[s.(string)] = true
			}
			return retval
		}
	}

	log.Println("Context:", context, "Key", key, "not found")
	return retval
}
<commit_msg>Fix a bug in config.LookupStringMap.<commit_after>// Copyright 2015 Robert S. Gerus. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package config

import (
	"encoding/json"
	"io/ioutil"
	"log"
	"os"
	"sort"
	"sync"
	"time"
)

type cacheEntry struct {
	modTime  time.Time
	contents interface{}
}

type config struct {
	cache         map[string]cacheEntry
	buildFileList func(map[string]string) []string
	l             sync.Mutex
}

var c config

func init() {
	log.Println("Initializing configuration cache")
	c.cache = make(map[string]cacheEntry)
}

func lookupvar(key, path string) interface{} {
	var f interface{}
	i, err := os.Stat(path)
	_, ok := c.cache[path]
	if os.IsNotExist(err) {
		log.Println("Config does not exist", path)
		if ok {
			log.Println("Purging", path, "from cache")
			delete(c.cache, path)
		}
	}
	if err != nil {
		return nil
	}

	if c.cache[path].modTime.Before(i.ModTime()) || !ok {
		log.Println("Stale cache for", path)
		data, err := ioutil.ReadFile(path)
		if err != nil {
			return nil
		}

		err = json.Unmarshal(data, &f)
		if err != nil {
			log.Println("Cannot parse", path)
			return nil
		}

		log.Println("Updating cache for", path)
		c.cache[path] = cacheEntry{
			modTime:  i.ModTime(),
			contents: f,
		}

		return f.(map[string]interface{})[key]
	}

	log.Println("Found cache for", path)
	return c.cache[path].contents.(map[string]interface{})[key]
}

// SetFileListBuilder registers file list builder function.
//
// Registered function takes context (key-value map[string]string) as the only
// argument and return a slice of strings - file paths.
func SetFileListBuilder(f func(map[string]string) []string) {
	c.buildFileList = f
}

// Lookup searches for requested configuration key in file list built using
// context.
func Lookup(context map[string]string, key string) interface{} {
	var value interface{}

	c.l.Lock()
	defer c.l.Unlock()

	for _, fpath := range c.buildFileList(context) {
		log.Println("Context:", context, "Looking up", key, "in", fpath)
		value = lookupvar(key, fpath)
		if value != nil {
			log.Println("Context:", context, "Found", key, value)
			return value
		}
	}

	log.Println("Context:", context, "Key", key, "not found")
	return nil
}

// LookupString is analogous to Lookup(), but does the cast to string.
func LookupString(context map[string]string, key string) string {
	var value interface{}

	c.l.Lock()
	defer c.l.Unlock()

	for _, fpath := range c.buildFileList(context) {
		log.Println("Context:", context, "Looking up", key, "in", fpath)
		value = lookupvar(key, fpath)
		if value != nil {
			log.Println("Context:", context, "Found", key, value)
			return value.(string)
		}
	}

	log.Println("Context:", context, "Key", key, "not found")
	return ""
}

// LookupStringSlice is analogous to Lookup(), but does the cast to []string
func LookupStringSlice(context map[string]string, key string) (retval []string) {
	var value interface{}

	c.l.Lock()
	defer c.l.Unlock()

	for _, fpath := range c.buildFileList(context) {
		log.Println("Context:", context, "Looking up", key, "in", fpath)
		value = lookupvar(key, fpath)
		if value != nil {
			log.Println("Context:", context, "Found", key, value)
			for _, s := range value.([]interface{}) {
				retval = append(retval, s.(string))
			}
			sort.Strings(retval)
			return retval
		}
	}

	log.Println("Context:", context, "Key", key, "not found")
	return []string{""}
}

// LookupStringMap is analogous to Lookup(), but does the cast to
// map[string]bool for optimised lookups.
func LookupStringMap(context map[string]string, key string) (retval map[string]bool) {
	var value interface{}
	retval = make(map[string]bool)

	c.l.Lock()
	defer c.l.Unlock()

	for _, fpath := range c.buildFileList(context) {
		log.Println("Context:", context, "Looking up", key, "in", fpath)
		value = lookupvar(key, fpath)
		if value != nil {
			log.Println("Context:", context, "Found", key, value)
			for _, s := range value.([]interface{}) {
				retval[s.(string)] = true
			}
			return retval
		}
	}

	log.Println("Context:", context, "Key", key, "not found")
	return retval
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"io/ioutil"
	"os"
	"strings"

	yaml "gopkg.in/yaml.v2"
)

type LocalBlobstoreConfig struct {
	PathPrefix string `yaml:"path_prefix"`
}

type S3BlobstoreConfig struct {
	Bucket         string
	AccessKey      string `yaml:"access_key"`
	SecretAccesKey string `yaml:"secret_access_key"`
}

type BlobstoreConfig struct {
	BlobstoreType string                `yaml:"blobstore_type"`
	LocalConfig   *LocalBlobstoreConfig `yaml:"local_config"`
	S3Config      *S3BlobstoreConfig    `yaml:"s3_config"`
}

type LoggingConfig struct {
	File   string
	Syslog string
	Level  string
}

type Config struct {
	Buildpacks      BlobstoreConfig
	Droplets        BlobstoreConfig
	Packages        BlobstoreConfig
	AppStash        BlobstoreConfig
	Logging         LoggingConfig
	PublicEndpoint  string `yaml:"public_endpoint"`
	PrivateEndpoint string `yaml:"private_endpoint"`
	Secret          string
	Port            int
}

func LoadConfig(filename string) (config Config, err error) {
	file, e := os.Open(filename)
	if e != nil {
		return Config{}, errors.New("error opening config. Caused by: " + e.Error())
	}
	content, e := ioutil.ReadAll(file)
	if e != nil {
		return Config{}, errors.New("error reading config. Caused by: " + e.Error())
	}
	e = yaml.Unmarshal(content, &config)
	if e != nil {
		return Config{}, errors.New("error parsing config. Caused by: " + e.Error())
	}
	var errs []string
	if config.Port == 0 {
		errs = append(errs, "port must be an integer > 0")
	}
	if config.PublicEndpoint == "" {
		errs = append(errs, "public_endpoint must not be empty")
	}
	if config.PrivateEndpoint == "" {
		errs = append(errs, "private_endpoint must not be empty")
	}
	if len(errs) > 0 {
		return Config{}, errors.New("error in config values: " + strings.Join(errs, "; "))
	}
	return
}
<commit_msg>Cleanup: adjust order of config structs<commit_after>package main

import (
	"errors"
	"io/ioutil"
	"os"
	"strings"

	yaml "gopkg.in/yaml.v2"
)

type Config struct {
	Buildpacks      BlobstoreConfig
	Droplets        BlobstoreConfig
	Packages        BlobstoreConfig
	AppStash        BlobstoreConfig
	Logging         LoggingConfig
	PublicEndpoint  string `yaml:"public_endpoint"`
	PrivateEndpoint string `yaml:"private_endpoint"`
	Secret          string
	Port            int
}

type BlobstoreConfig struct {
	BlobstoreType string                `yaml:"blobstore_type"`
	LocalConfig   *LocalBlobstoreConfig `yaml:"local_config"`
	S3Config      *S3BlobstoreConfig    `yaml:"s3_config"`
}

type LocalBlobstoreConfig struct {
	PathPrefix string `yaml:"path_prefix"`
}

type S3BlobstoreConfig struct {
	Bucket         string
	AccessKey      string `yaml:"access_key"`
	SecretAccesKey string `yaml:"secret_access_key"`
}

type LoggingConfig struct {
	File   string
	Syslog string
	Level  string
}

func LoadConfig(filename string) (config Config, err error) {
	file, e := os.Open(filename)
	if e != nil {
		return Config{}, errors.New("error opening config. Caused by: " + e.Error())
	}
	content, e := ioutil.ReadAll(file)
	if e != nil {
		return Config{}, errors.New("error reading config. Caused by: " + e.Error())
	}
	e = yaml.Unmarshal(content, &config)
	if e != nil {
		return Config{}, errors.New("error parsing config. Caused by: " + e.Error())
	}
	var errs []string
	if config.Port == 0 {
		errs = append(errs, "port must be an integer > 0")
	}
	if config.PublicEndpoint == "" {
		errs = append(errs, "public_endpoint must not be empty")
	}
	if config.PrivateEndpoint == "" {
		errs = append(errs, "private_endpoint must not be empty")
	}
	if len(errs) > 0 {
		return Config{}, errors.New("error in config values: " + strings.Join(errs, "; "))
	}
	return
}
<|endoftext|>


<commit_before>package config

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"os/signal"
	"strconv"
	"syscall"
)

var cache map[string]map[string]interface{}

func Get(file, key string, v interface{}) error {
	var val interface{}

	env, set := os.LookupEnv(key)

	if set {
		switch v.(type) {
		case *float64:
			val, err := strconv.ParseFloat(env, 64)

			if err != nil {
				return err
			}

			*v.(*float64) = val
			return nil
		case *int:
			val, err := strconv.ParseInt(env, 10, 64)

			if err != nil {
				return err
			}

			*v.(*int) = int(val)
			return nil
		default:
			val = env
		}
	} else {
		if cache[file] == nil {
			if err := primeCacheFromFile(file); err != nil {
				return err
			}
		}

		val = cache[file][key]
	}

	// Cast JSON values
	switch v.(type) {
	case *string:
		if val == nil {
			val = ""
		}

		*v.(*string) = val.(string)
	case *bool:
		switch val {
		case nil, 0, false, "", "0", "false":
			// fasley
			val = false
		default:
			// truthy
			val = true
		}

		*v.(*bool) = val.(bool)
	case *float64:
		if val == nil {
			val = float64(0)
		}

		*v.(*float64) = val.(float64)
	case *int:
		if val == nil {
			val = float64(0)
		}

		*v.(*int) = int(val.(float64))
	default:
		return errors.New("Type not supported")
	}

	return nil
}

func Reload() {
	// Empty the cache
	cache = make(map[string]map[string]interface{})
}

func primeCacheFromFile(file string) error {
	// File exists?
	if _, err := os.Stat(file); os.IsNotExist(err) {
		return err
	}

	// Read file
	raw, err := ioutil.ReadFile(file)
	if err != nil {
		return err
	}

	// Unmarshal
	var config map[string]interface{}
	if err := json.Unmarshal(raw, &config); err != nil {
		return err
	}

	// prime cache
	cache[file] = config
	return nil
}

func init() {
	Reload()

	// Catch SIGHUP to automatically reload cache
	sighup := make(chan os.Signal, 1)
	signal.Notify(sighup, syscall.SIGHUP)

	go func() {
		for sig := range sighup {
			switch sig {
			case syscall.SIGHUP:
				fmt.Println("Caught SIGHUP, reloading config...")
				Reload()
			}
		}
	}()
}
<commit_msg>Simplify SIGHUP logic<commit_after>package config

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"os/signal"
	"strconv"
	"syscall"
)

var cache map[string]map[string]interface{}

func Get(file, key string, v interface{}) error {
	var val interface{}

	env, set := os.LookupEnv(key)

	if set {
		switch v.(type) {
		case *float64:
			val, err := strconv.ParseFloat(env, 64)

			if err != nil {
				return err
			}

			*v.(*float64) = val
			return nil
		case *int:
			val, err := strconv.ParseInt(env, 10, 64)

			if err != nil {
				return err
			}

			*v.(*int) = int(val)
			return nil
		default:
			val = env
		}
	} else {
		if cache[file] == nil {
			if err := primeCacheFromFile(file); err != nil {
				return err
			}
		}

		val = cache[file][key]
	}

	// Cast JSON values
	switch v.(type) {
	case *string:
		if val == nil {
			val = ""
		}

		*v.(*string) = val.(string)
	case *bool:
		switch val {
		case nil, 0, false, "", "0", "false":
			// fasley
			val = false
		default:
			// truthy
			val = true
		}

		*v.(*bool) = val.(bool)
	case *float64:
		if val == nil {
			val = float64(0)
		}

		*v.(*float64) = val.(float64)
	case *int:
		if val == nil {
			val = float64(0)
		}

		*v.(*int) = int(val.(float64))
	default:
		return errors.New("Type not supported")
	}

	return nil
}

func Reload() {
	// Empty the cache
	cache = make(map[string]map[string]interface{})
}

func primeCacheFromFile(file string) error {
	// File exists?
	if _, err := os.Stat(file); os.IsNotExist(err) {
		return err
	}

	// Read file
	raw, err := ioutil.ReadFile(file)
	if err != nil {
		return err
	}

	// Unmarshal
	var config map[string]interface{}
	if err := json.Unmarshal(raw, &config); err != nil {
		return err
	}

	// prime cache
	cache[file] = config
	return nil
}

func init() {
	Reload()

	// Catch SIGHUP to automatically reload cache
	sighup := make(chan os.Signal, 1)
	signal.Notify(sighup, syscall.SIGHUP)

	go func() {
		for {
			<-sighup
			fmt.Println("Caught SIGHUP, reloading config...")
			Reload()
		}
	}()
}
<|endoftext|>


<commit_before>package openstack

import (
	"crypto/tls"
	"crypto/x509"
	"fmt"
	"io/ioutil"
	"net/http"

	"github.com/rackspace/gophercloud"
	"github.com/rackspace/gophercloud/openstack"
)

type Config struct {
	Username         string
	UserID           string
	Password         string
	Token            string
	APIKey           string
	IdentityEndpoint string
	TenantID         string
	TenantName       string
	DomainID         string
	DomainName       string
	Insecure         bool
	EndpointType     string
	CACertFile       string
	ClientCertFile   string
	ClientKeyFile    string

	osClient *gophercloud.ProviderClient
}

func (c *Config) loadAndValidate() error {

	if c.EndpointType != "internal" && c.EndpointType != "internalURL" &&
		c.EndpointType != "admin" && c.EndpointType != "adminURL" &&
		c.EndpointType != "public" && c.EndpointType != "publicURL" &&
		c.EndpointType != "" {
		return fmt.Errorf("Invalid endpoint type provided")
	}

	ao := gophercloud.AuthOptions{
		Username:         c.Username,
		UserID:           c.UserID,
		Password:         c.Password,
		TokenID:          c.Token,
		APIKey:           c.APIKey,
		IdentityEndpoint: c.IdentityEndpoint,
		TenantID:         c.TenantID,
		TenantName:       c.TenantName,
		DomainID:         c.DomainID,
		DomainName:       c.DomainName,
	}

	client, err := openstack.NewClient(ao.IdentityEndpoint)
	if err != nil {
		return err
	}

	config := &tls.Config{}
	if c.CACertFile != "" {

		caCert, err := ioutil.ReadFile(c.CACertFile)
		if err != nil {
			return err
		}

		caCertPool := x509.NewCertPool()
		caCertPool.AppendCertsFromPEM(caCert)
		config.RootCAs = caCertPool
	}
	if c.Insecure {
		config.InsecureSkipVerify = true
	}

	if c.ClientCertFile != "" && c.ClientKeyFile != "" {
		cert, err := tls.LoadX509KeyPair(c.ClientCertFile, c.ClientKeyFile)
		if err != nil {
			return err
		}

		config.Certificates = []tls.Certificate{cert}
		config.BuildNameToCertificate()
	}
	transport := &http.Transport{Proxy: http.ProxyFromEnvironment, TLSClientConfig: config}
	client.HTTPClient.Transport = transport

	err = openstack.Authenticate(client, ao)
	if err != nil {
		return err
	}

	c.osClient = client

	return nil
}

func (c *Config) blockStorageV1Client(region string) (*gophercloud.ServiceClient, error) {
	return openstack.NewBlockStorageV1(c.osClient, gophercloud.EndpointOpts{
		Region:       region,
		Availability: c.getEndpointType(),
	})
}

func (c *Config) blockStorageV2Client(region string) (*gophercloud.ServiceClient, error) {
	return openstack.NewBlockStorageV2(c.osClient, gophercloud.EndpointOpts{
		Region:       region,
		Availability: c.getEndpointType(),
	})
}

func (c *Config) computeV2Client(region string) (*gophercloud.ServiceClient, error) {
	return openstack.NewComputeV2(c.osClient, gophercloud.EndpointOpts{
		Region:       region,
		Availability: c.getEndpointType(),
	})
}

func (c *Config) networkingV2Client(region string) (*gophercloud.ServiceClient, error) {
	return openstack.NewNetworkV2(c.osClient, gophercloud.EndpointOpts{
		Region:       region,
		Availability: c.getEndpointType(),
	})
}

func (c *Config) objectStorageV1Client(region string) (*gophercloud.ServiceClient, error) {
	return openstack.NewObjectStorageV1(c.osClient, gophercloud.EndpointOpts{
		Region:       region,
		Availability: c.getEndpointType(),
	})
}

func (c *Config) getEndpointType() gophercloud.Availability {
	if c.EndpointType == "internal" || c.EndpointType == "internalURL" {
		return gophercloud.AvailabilityInternal
	}
	if c.EndpointType == "admin" || c.EndpointType == "adminURL" {
		return gophercloud.AvailabilityAdmin
	}
	return gophercloud.AvailabilityPublic
}
<commit_msg>provider/openstack: gophercloud migration: Removing APIKey Attribute<commit_after>package openstack

import (
	"crypto/tls"
	"crypto/x509"
	"fmt"
	"io/ioutil"
	"net/http"

	"github.com/gophercloud/gophercloud"
	"github.com/gophercloud/gophercloud/openstack"
)

type Config struct {
	Username         string
	UserID           string
	Password         string
	Token            string
	APIKey           string
	IdentityEndpoint string
	TenantID         string
	TenantName       string
	DomainID         string
	DomainName       string
	Insecure         bool
	EndpointType     string
	CACertFile       string
	ClientCertFile   string
	ClientKeyFile    string

	osClient *gophercloud.ProviderClient
}

func (c *Config) loadAndValidate() error {

	if c.EndpointType != "internal" && c.EndpointType != "internalURL" &&
		c.EndpointType != "admin" && c.EndpointType != "adminURL" &&
		c.EndpointType != "public" && c.EndpointType != "publicURL" &&
		c.EndpointType != "" {
		return fmt.Errorf("Invalid endpoint type provided")
	}

	ao := gophercloud.AuthOptions{
		Username:         c.Username,
		UserID:           c.UserID,
		Password:         c.Password,
		TokenID:          c.Token,
		IdentityEndpoint: c.IdentityEndpoint,
		TenantID:         c.TenantID,
		TenantName:       c.TenantName,
		DomainID:         c.DomainID,
		DomainName:       c.DomainName,
	}

	client, err := openstack.NewClient(ao.IdentityEndpoint)
	if err != nil {
		return err
	}

	config := &tls.Config{}
	if c.CACertFile != "" {

		caCert, err := ioutil.ReadFile(c.CACertFile)
		if err != nil {
			return err
		}

		caCertPool := x509.NewCertPool()
		caCertPool.AppendCertsFromPEM(caCert)
		config.RootCAs = caCertPool
	}
	if c.Insecure {
		config.InsecureSkipVerify = true
	}

	if c.ClientCertFile != "" && c.ClientKeyFile != "" {
		cert, err := tls.LoadX509KeyPair(c.ClientCertFile, c.ClientKeyFile)
		if err != nil {
			return err
		}

		config.Certificates = []tls.Certificate{cert}
		config.BuildNameToCertificate()
	}
	transport := &http.Transport{Proxy: http.ProxyFromEnvironment, TLSClientConfig: config}
	client.HTTPClient.Transport = transport

	err = openstack.Authenticate(client, ao)
	if err != nil {
		return err
	}

	c.osClient = client

	return nil
}

func (c *Config) blockStorageV1Client(region string) (*gophercloud.ServiceClient, error) {
	return openstack.NewBlockStorageV1(c.osClient, gophercloud.EndpointOpts{
		Region:       region,
		Availability: c.getEndpointType(),
	})
}

func (c *Config) blockStorageV2Client(region string) (*gophercloud.ServiceClient, error) {
	return openstack.NewBlockStorageV2(c.osClient, gophercloud.EndpointOpts{
		Region:       region,
		Availability: c.getEndpointType(),
	})
}

func (c *Config) computeV2Client(region string) (*gophercloud.ServiceClient, error) {
	return openstack.NewComputeV2(c.osClient, gophercloud.EndpointOpts{
		Region:       region,
		Availability: c.getEndpointType(),
	})
}

func (c *Config) networkingV2Client(region string) (*gophercloud.ServiceClient, error) {
	return openstack.NewNetworkV2(c.osClient, gophercloud.EndpointOpts{
		Region:       region,
		Availability: c.getEndpointType(),
	})
}

func (c *Config) objectStorageV1Client(region string) (*gophercloud.ServiceClient, error) {
	return openstack.NewObjectStorageV1(c.osClient, gophercloud.EndpointOpts{
		Region:       region,
		Availability: c.getEndpointType(),
	})
}

func (c *Config) getEndpointType() gophercloud.Availability {
	if c.EndpointType == "internal" || c.EndpointType == "internalURL" {
		return gophercloud.AvailabilityInternal
	}
	if c.EndpointType == "admin" || c.EndpointType == "adminURL" {
		return gophercloud.AvailabilityAdmin
	}
	return gophercloud.AvailabilityPublic
}
<|endoftext|>


<commit_before>// Copyright 2016 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package repl contains the implementation of the command that prints the
// BQL version.
package repl

import (
	"bufio"
	"fmt"
	"os"
	"strings"
	"time"

	"golang.org/x/net/context"

	"github.com/google/badwolf/bql/grammar"
	"github.com/google/badwolf/bql/planner"
	"github.com/google/badwolf/bql/semantic"
	"github.com/google/badwolf/bql/table"
	"github.com/google/badwolf/bql/version"
	"github.com/google/badwolf/storage"
	"github.com/google/badwolf/tools/vcli/bw/command"
	"github.com/google/badwolf/tools/vcli/bw/export"
	"github.com/google/badwolf/tools/vcli/bw/io"
	"github.com/google/badwolf/tools/vcli/bw/load"
)

const prompt = "bql> "

// New create the version command.
func New(driver storage.Store, chanSize, bulkSize, builderSize int) *command.Command {
	return &command.Command{
		Run: func(ctx context.Context, args []string) int {
			REPL(driver, os.Stdin, simpleReadLine, chanSize, bulkSize, builderSize)
			return 0
		},
		UsageLine: "bql",
		Short:     "starts a REPL to run BQL statements.",
		Long:      "Starts a REPL from the command line to accept BQL statements. Type quit; to leave the REPL.",
	}
}

type readLiner func(*os.File) <-chan string

// simpleReadline reads a line from the provided file. This does not support
// any advanced terminal functionalities.
//
// TODO(xllora): Replace simple reader for function that supports advanced
// teminal input.
func simpleReadLine(f *os.File) <-chan string {
	c := make(chan string)
	go func() {
		defer close(c)
		scanner := bufio.NewScanner(os.Stdin)
		for scanner.Scan() {
			c <- strings.TrimSpace(scanner.Text())
		}
	}()
	return c
}

// REPL starts a read-evaluation-print-loop to run BQL commands.
func REPL(driver storage.Store, input *os.File, rl readLiner, chanSize, bulkSize, builderSize int) int {
	ctx := context.Background()
	fmt.Printf("Welcome to BadWolf vCli (%d.%d.%d-%s)\n", version.Major, version.Minor, version.Patch, version.Release)
	fmt.Printf("Using driver %q. Type quit; to exit\n", driver.Name(ctx))
	fmt.Printf("Session started at %v\n\n", time.Now())
	defer func() {
		fmt.Printf("\n\nThanks for all those BQL queries!\n\n")
	}()
	fmt.Print(prompt)
	l := ""
	for line := range rl(input) {
		nl := strings.TrimSpace(line)
		if nl == "" {
			fmt.Print(prompt)
			continue
		}
		if l != "" {
			l = l + " " + nl
		} else {
			l = nl
		}
		if !strings.HasSuffix(nl, ";") {
			// Not done with the statement.
			continue
		}
		if strings.HasPrefix(l, "quit") {
			break
		}
		if strings.HasPrefix(l, "help") {
			printHelp()
			fmt.Print(prompt)
			l = ""
			continue
		}
		if strings.HasPrefix(l, "export") {
			args := strings.Split("bw "+l[:len(l)-1], " ")
			usage := "Wrong syntax\n\n\tload <graph_names_separated_by_commas> <file_path>\n"
			export.Eval(ctx, usage, args, driver, bulkSize)
			fmt.Print(prompt)
			l = ""
			continue
		}
		if strings.HasPrefix(l, "load") {
			args := strings.Split("bw "+l[:len(l)-1], " ")
			usage := "Wrong syntax\n\n\tload <file_path> <graph_names_separated_by_commas>\n"
			load.Eval(ctx, usage, args, driver, bulkSize, builderSize)
			fmt.Print(prompt)
			l = ""
			continue
		}
		if strings.HasPrefix(l, "run") {
			path, cmds, err := runBQLFromFile(ctx, driver, chanSize, l[:len(l)-1])
			if err != nil {
				fmt.Printf("[ERROR] %s\n\n", err)
			} else {
				fmt.Printf("Loaded %q and run %d BQL commands successfully\n\n", path, cmds)
			}
			fmt.Print(prompt)
			l = ""
			continue
		}

		table, err := runBQL(ctx, l, driver, chanSize)
		l = ""
		if err != nil {
			fmt.Printf("[ERROR] %s\n\n", err)
		} else {
			if len(table.Bindings()) > 0 {
				fmt.Println(table.String())
			}
			fmt.Println("[OK]")
		}
		fmt.Print(prompt)
	}
	return 0
}

// printHelp prints help for the console commands.
func printHelp() {
	fmt.Println("help                                                  - prints help for the bw console.")
	fmt.Println("export <graph_names_separated_by_commas> <file_path>  - dumps triples from graphs into a file path.")
	fmt.Println("load <file_path> <graph_names_separated_by_commas>    - load triples into the specified graphs.")
	fmt.Println("run <file_with_bql_statements>                        - runs all the BQL statements in the file.")
	fmt.Println("quit                                                  - quits the console.")
	fmt.Println()
}

// runBQLFromFile loads all the statements in the file and runs them.
func runBQLFromFile(ctx context.Context, driver storage.Store, chanSize int, line string) (string, int, error) {
	ss := strings.Split(strings.TrimSpace(line), " ")
	if len(ss) != 2 {
		return "", 0, fmt.Errorf("wrong syntax: run <file_with_bql_statements>")
	}
	path := ss[1]
	lines, err := io.GetStatementsFromFile(path)
	if err != nil {
		return "", 0, fmt.Errorf("failed to read file %q with error %v on\n", path, err)
	}
	for idx, stm := range lines {
		fmt.Printf("Processing statement (%d/%d)\n", idx+1, len(lines))
		_, err := runBQL(ctx, stm, driver, chanSize)
		if err != nil {
			return "", 0, fmt.Errorf("%v on\n%s\n", err, stm)
		}
	}
	fmt.Println()
	return path, len(lines), nil
}

// runBQL attemps to excecute the provided query against the given store.
func runBQL(ctx context.Context, bql string, s storage.Store, chanSize int) (*table.Table, error) {
	p, err := grammar.NewParser(grammar.SemanticBQL())
	if err != nil {
		return nil, fmt.Errorf("failed to initilize a valid BQL parser")
	}
	stm := &semantic.Statement{}
	if err := p.Parse(grammar.NewLLk(bql, 1), stm); err != nil {
		return nil, fmt.Errorf("failed to parse BQL statement with error %v", err)
	}
	pln, err := planner.New(ctx, s, stm, chanSize)
	if err != nil {
		return nil, fmt.Errorf("should have not failed to create a plan using memory.DefaultStorage for statement %v with error %v", stm, err)
	}
	res, err := pln.Excecute(ctx)
	if err != nil {
		return nil, fmt.Errorf("planner.Execute: failed to execute insert plan with error %v", err)
	}
	return res, nil
}
<commit_msg>Clean up multiline composite sentence end<commit_after>// Copyright 2016 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package repl contains the implementation of the command that prints the
// BQL version.
package repl

import (
	"bufio"
	"fmt"
	"os"
	"strings"
	"time"

	"golang.org/x/net/context"

	"github.com/google/badwolf/bql/grammar"
	"github.com/google/badwolf/bql/planner"
	"github.com/google/badwolf/bql/semantic"
	"github.com/google/badwolf/bql/table"
	"github.com/google/badwolf/bql/version"
	"github.com/google/badwolf/storage"
	"github.com/google/badwolf/tools/vcli/bw/command"
	"github.com/google/badwolf/tools/vcli/bw/export"
	"github.com/google/badwolf/tools/vcli/bw/io"
	"github.com/google/badwolf/tools/vcli/bw/load"
)

const prompt = "bql> "

// New create the version command.
func New(driver storage.Store, chanSize, bulkSize, builderSize int) *command.Command {
	return &command.Command{
		Run: func(ctx context.Context, args []string) int {
			REPL(driver, os.Stdin, simpleReadLine, chanSize, bulkSize, builderSize)
			return 0
		},
		UsageLine: "bql",
		Short:     "starts a REPL to run BQL statements.",
		Long:      "Starts a REPL from the command line to accept BQL statements. Type quit; to leave the REPL.",
	}
}

type readLiner func(*os.File) <-chan string

// simpleReadline reads a line from the provided file. This does not support
// any advanced terminal functionalities.
//
// TODO(xllora): Replace simple reader for function that supports advanced
// teminal input.
func simpleReadLine(f *os.File) <-chan string {
	c := make(chan string)
	go func() {
		defer close(c)
		scanner := bufio.NewScanner(os.Stdin)
		for scanner.Scan() {
			c <- strings.TrimSpace(scanner.Text())
		}
	}()
	return c
}

// REPL starts a read-evaluation-print-loop to run BQL commands.
func REPL(driver storage.Store, input *os.File, rl readLiner, chanSize, bulkSize, builderSize int) int {
	ctx := context.Background()
	fmt.Printf("Welcome to BadWolf vCli (%d.%d.%d-%s)\n", version.Major, version.Minor, version.Patch, version.Release)
	fmt.Printf("Using driver %q. Type quit; to exit\n", driver.Name(ctx))
	fmt.Printf("Session started at %v\n\n", time.Now())
	defer func() {
		fmt.Printf("\n\nThanks for all those BQL queries!\n\n")
	}()
	fmt.Print(prompt)
	l := ""
	for line := range rl(input) {
		nl := strings.TrimSpace(line)
		if nl == "" {
			fmt.Print(prompt)
			continue
		}
		if l != "" {
			l = l + " " + nl
		} else {
			l = nl
		}
		if !strings.HasSuffix(nl, ";") {
			// Not done with the statement.
			continue
		}
		if strings.HasPrefix(l, "quit") {
			break
		}
		if strings.HasPrefix(l, "help") {
			printHelp()
			fmt.Print(prompt)
			l = ""
			continue
		}
		if strings.HasPrefix(l, "export") {
			args := strings.Split("bw "+strings.TrimSpace(l[:len(l)-1]), " ")
			usage := "Wrong syntax\n\n\tload <graph_names_separated_by_commas> <file_path>\n"
			export.Eval(ctx, usage, args, driver, bulkSize)
			fmt.Print(prompt)
			l = ""
			continue
		}
		if strings.HasPrefix(l, "load") {
			args := strings.Split("bw "+strings.TrimSpace(l[:len(l)-1]), " ")
			usage := "Wrong syntax\n\n\tload <file_path> <graph_names_separated_by_commas>\n"
			load.Eval(ctx, usage, args, driver, bulkSize, builderSize)
			fmt.Print(prompt)
			l = ""
			continue
		}
		if strings.HasPrefix(l, "run") {
			path, cmds, err := runBQLFromFile(ctx, driver, chanSize, strings.TrimSpace(l[:len(l)-1]))
			if err != nil {
				fmt.Printf("[ERROR] %s\n\n", err)
			} else {
				fmt.Printf("Loaded %q and run %d BQL commands successfully\n\n", path, cmds)
			}
			fmt.Print(prompt)
			l = ""
			continue
		}

		table, err := runBQL(ctx, l, driver, chanSize)
		l = ""
		if err != nil {
			fmt.Printf("[ERROR] %s\n\n", err)
		} else {
			if len(table.Bindings()) > 0 {
				fmt.Println(table.String())
			}
			fmt.Println("[OK]")
		}
		fmt.Print(prompt)
	}
	return 0
}

// printHelp prints help for the console commands.
func printHelp() {
	fmt.Println("help                                                  - prints help for the bw console.")
	fmt.Println("export <graph_names_separated_by_commas> <file_path>  - dumps triples from graphs into a file path.")
	fmt.Println("load <file_path> <graph_names_separated_by_commas>    - load triples into the specified graphs.")
	fmt.Println("run <file_with_bql_statements>                        - runs all the BQL statements in the file.")
	fmt.Println("quit                                                  - quits the console.")
	fmt.Println()
}

// runBQLFromFile loads all the statements in the file and runs them.
func runBQLFromFile(ctx context.Context, driver storage.Store, chanSize int, line string) (string, int, error) {
	ss := strings.Split(strings.TrimSpace(line), " ")
	if len(ss) != 2 {
		return "", 0, fmt.Errorf("wrong syntax: run <file_with_bql_statements>")
	}
	path := ss[1]
	lines, err := io.GetStatementsFromFile(path)
	if err != nil {
		return "", 0, fmt.Errorf("failed to read file %q with error %v on\n", path, err)
	}
	for idx, stm := range lines {
		fmt.Printf("Processing statement (%d/%d)\n", idx+1, len(lines))
		_, err := runBQL(ctx, stm, driver, chanSize)
		if err != nil {
			return "", 0, fmt.Errorf("%v on\n%s\n", err, stm)
		}
	}
	fmt.Println()
	return path, len(lines), nil
}

// runBQL attemps to excecute the provided query against the given store.
func runBQL(ctx context.Context, bql string, s storage.Store, chanSize int) (*table.Table, error) {
	p, err := grammar.NewParser(grammar.SemanticBQL())
	if err != nil {
		return nil, fmt.Errorf("failed to initilize a valid BQL parser")
	}
	stm := &semantic.Statement{}
	if err := p.Parse(grammar.NewLLk(bql, 1), stm); err != nil {
		return nil, fmt.Errorf("failed to parse BQL statement with error %v", err)
	}
	pln, err := planner.New(ctx, s, stm, chanSize)
	if err != nil {
		return nil, fmt.Errorf("should have not failed to create a plan using memory.DefaultStorage for statement %v with error %v", stm, err)
	}
	res, err := pln.Excecute(ctx)
	if err != nil {
		return nil, fmt.Errorf("planner.Execute: failed to execute insert plan with error %v", err)
	}
	return res, nil
}
<|endoftext|>


<commit_before><commit_msg>[BZ #2048441] Fix a bug in deletion of webhook service for replacement (#2606)<commit_after><|endoftext|>


<commit_before>package main

import (
	"bufio"
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"strconv"
	"strings"
)

//The IP struct
type IP struct {
	Address    string
	ServerName string
	Delay      int
}

const (
	tmpOkIPFileName string = "ip_tmpok.txt"
	jsonIPFileName  string = "ip_output.txt"
)

func main() {
	tips()

}
func tips() {

	fmt.Print(`请选择需要处理的操作, 输入对应的数字并按下回车:

1. 提取 ip_tmpok.txt 中的IP, 用｜分隔以及 json 格式, 并生成ip_output.txt

2. IP格式互转 GoAgent <==> GoProxy, 并生成 ip_output.txt

请输入对应的数字：`)

	switch getInputFromCommand() {
	case "1":
		convertIP2JSON()
	case "2":
		goagent2goproxy()
	default:
		tips()

	}
}
func convertIP2JSON() {
	fmt.Print("\n请输入最大延迟（以毫秒计算），否则提取所有IP：")
	input := getInputFromCommand()
	var delay int
	var err error
	isAll := true
	if len(input) > 0 {
		delay, err = strconv.Atoi(input)
		if err != nil {
			fmt.Println("\n输入不正确，请重新输入。")
			convertIP2JSON()
			return
		}
		isAll = false
	}
	gws, gvs := writeJSONIP2File(delay, isAll)
	fmt.Printf("\ndelay: %dms, ip count: %d(gws: %d, gvs: %d)\n", delay, gws+gvs, gws, gvs)

	fmt.Println("\npress Enter to continue...")
	fmt.Scanln()
	tips()
}
func goagent2goproxy() {
	fmt.Println("请输入需要转换的IP, 可使用右键->粘贴：")
	rawips := getInputFromCommand()
	rawips = strings.TrimSpace(rawips)
	var ips string
	if strings.Contains(rawips, "|") {
		ips = strings.Trim(rawips, "|")
		ips = strings.Replace(ips, "|", "\",\"", -1)
		ips = "\"" + ips + "\""
	} else {
		ips = strings.Replace(rawips, "\",\"", "|", -1)
		ips = rawips[1 : len(rawips)-1]
	}
	fmt.Println()
	fmt.Println(ips)
	fmt.Println("\npress Enter to continue...")
	fmt.Scanln()
	tips()
}
func getInputFromCommand() string {
	reader := bufio.NewReader(os.Stdin)
	input, _ := reader.ReadString('\n')
	input = input[:len(input)-2]

	return input
}

//get last ok ip
func getLastOkIP() []IP {
	m := make(map[string]IP)
	var checkedip IP
	var ips []IP
	if isFileExist(tmpOkIPFileName) {
		bytes, err := ioutil.ReadFile(tmpOkIPFileName)
		if err != nil {
			fmt.Printf("read file %s error: %v", tmpOkIPFileName, err)
		}
		lines := strings.Split(string(bytes), "\n")
		for _, line := range lines {
			ipInfo := strings.Split(line, " ")
			if len(ipInfo) == 6 {
				delay, _ := strconv.Atoi(ipInfo[1][:len(ipInfo[1])-2])
				checkedip = IP{
					Address:    ipInfo[0],
					Delay:      delay,
					ServerName: ipInfo[3],
				}
				m[ipInfo[0]] = checkedip
			}
		}
	}
	for _, v := range m {
		ips = append(ips, v)
	}
	return ips
}

/**
writeJSONIP2File: sorting ip, ridding duplicate ip, generating json ip and
bar-separated ip
*/
func writeJSONIP2File(delay int, isAll bool) (gws, gvs int) {
	okIPs := getLastOkIP()
	_, err := os.Create(jsonIPFileName)
	if err != nil {
		fmt.Printf("create file %s error: %v", jsonIPFileName, err)
	}
	var gaipbuf, gpipbuf bytes.Buffer
	for _, ip := range okIPs {

		if isAll {
			if ip.ServerName == "gws" {
				gws++
			}
			if ip.ServerName == "gvs" {
				gvs++
			}
			gaipbuf.WriteString(ip.Address)
			gaipbuf.WriteString("|")
			gpipbuf.WriteString("\"")
			gpipbuf.WriteString(ip.Address)
			gpipbuf.WriteString("\",")
		} else {
			if ip.Delay <= delay {
				if ip.ServerName == "gws" {
					gws++
				}
				if ip.ServerName == "gvs" {
					gvs++
				}
				gaipbuf.WriteString(ip.Address)
				gaipbuf.WriteString("|")
				gpipbuf.WriteString("\"")
				gpipbuf.WriteString(ip.Address)
				gpipbuf.WriteString("\",")
			}
		}
	}
	gaip := gaipbuf.String()
	gpip := gpipbuf.String()

	if len(gaip) > 0 {
		gaip = gaip[:len(gaip)-1]
	}
	if len(gpip) > 0 {
		gpip = gpip[:len(gpip)-1]
	}
	err = ioutil.WriteFile(jsonIPFileName, []byte(gaip+"\n\n\n"+gpip), 0755)
	if err != nil {
		fmt.Printf("write ip to file %s error: %v", jsonIPFileName, err)
	}
	return gws, gvs
}

//Whether file exists.
func isFileExist(file string) bool {
	_, err := os.Stat(file)
	if err != nil && os.IsNotExist(err) {
		return false
	}
	return true
}
<commit_msg>add gws ip filter<commit_after>package main

import (
	"bufio"
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"strconv"
	"strings"
)

//The IP struct
type IP struct {
	Address    string
	ServerName string
	Delay      int
}

const (
	tmpOkIPFileName string = "ip_tmpok.txt"
	jsonIPFileName  string = "ip_output.txt"
)

func main() {
	tips()

}
func tips() {

	fmt.Print(`请选择需要处理的操作, 输入对应的数字并按下回车:

1. 提取 ip_tmpok.txt 中的IP, 用｜分隔以及 json 格式, 并生成ip_output.txt

2. IP格式互转 GoAgent <==> GoProxy, 并生成 ip_output.txt

请输入对应的数字：`)

	switch getInputFromCommand() {
	case "1":
		convertIP2JSON()
	case "2":
		goagent2goproxy()
	default:
		tips()
	}
}
func convertIP2JSON() {
	fmt.Print("\n请输入最大延迟（以毫秒计算），否则提取所有IP：")
	input := getInputFromCommand()
	var delay int
	var err error
	isAll := true
	isGWS := false
	if len(input) > 0 {
		delay, err = strconv.Atoi(input)
		if err != nil {
			fmt.Println("\n输入不正确，请重新输入。")
			convertIP2JSON()
			return
		}
		isAll = false
	}
	fmt.Print("\n是否只提取gws的IP，是请输入y，否请直接按回车键：")
	isgws := getInputFromCommand()
	if isgws == "y" || isgws == "Y" {
		isGWS = true
	}
	gws, gvs := writeJSONIP2File(delay, isGWS, isAll)
	fmt.Printf("\ndelay: %dms, ip count: %d(gws: %d, gvs: %d)\n", delay, gws+gvs, gws, gvs)

	fmt.Println("\npress Enter to continue...")
	fmt.Scanln()
	tips()
}
func goagent2goproxy() {
	fmt.Println("请输入需要转换的IP, 可使用右键->粘贴：")
	rawips := getInputFromCommand()
	rawips = strings.TrimSpace(rawips)
	var ips string
	if strings.Contains(rawips, "|") {
		ips = strings.Trim(rawips, "|")
		ips = strings.Replace(ips, "|", "\",\"", -1)
		ips = "\"" + ips + "\""
	} else {
		ips = strings.Replace(rawips, "\",\"", "|", -1)
		ips = rawips[1 : len(rawips)-1]
	}
	fmt.Println()
	fmt.Println(ips)
	fmt.Println("\npress Enter to continue...")
	fmt.Scanln()
	tips()
}
func getInputFromCommand() string {
	reader := bufio.NewReader(os.Stdin)
	input, _ := reader.ReadString('\n')
	input = input[:len(input)-2]

	return input
}

//get last ok ip
func getLastOkIP() []IP {
	m := make(map[string]IP)
	var checkedip IP
	var ips []IP
	if isFileExist(tmpOkIPFileName) {
		bytes, err := ioutil.ReadFile(tmpOkIPFileName)
		if err != nil {
			fmt.Printf("read file %s error: %v", tmpOkIPFileName, err)
		}
		lines := strings.Split(string(bytes), "\n")
		for _, line := range lines {
			ipInfo := strings.Split(line, " ")
			if len(ipInfo) == 6 {
				delay, _ := strconv.Atoi(ipInfo[1][:len(ipInfo[1])-2])
				checkedip = IP{
					Address:    ipInfo[0],
					Delay:      delay,
					ServerName: ipInfo[3],
				}
				m[ipInfo[0]] = checkedip
			}
		}
	}
	for _, v := range m {
		ips = append(ips, v)
	}
	return ips
}

/**
writeJSONIP2File: sorting ip, ridding duplicate ip, generating json ip and
bar-separated ip
*/
func writeJSONIP2File(delay int, isGWS, isAll bool) (gws, gvs int) {
	okIPs := getLastOkIP()
	_, err := os.Create(jsonIPFileName)
	if err != nil {
		fmt.Printf("create file %s error: %v", jsonIPFileName, err)
	}
	var gaipbuf, gpipbuf bytes.Buffer
	for _, ip := range okIPs {
		if isGWS {
			if ip.ServerName == "gws" {
				if isAll {

					gws++
					gaipbuf.WriteString(ip.Address)
					gaipbuf.WriteString("|")
					gpipbuf.WriteString("\"")
					gpipbuf.WriteString(ip.Address)
					gpipbuf.WriteString("\",")
				} else {
					if ip.Delay <= delay {
						gws++
						gaipbuf.WriteString(ip.Address)
						gaipbuf.WriteString("|")
						gpipbuf.WriteString("\"")
						gpipbuf.WriteString(ip.Address)
						gpipbuf.WriteString("\",")
					}
				}
			}
		} else {
			if isAll {
				if ip.ServerName == "gws" {
					gws++
				}
				if ip.ServerName == "gvs" {
					gvs++
				}
				gaipbuf.WriteString(ip.Address)
				gaipbuf.WriteString("|")
				gpipbuf.WriteString("\"")
				gpipbuf.WriteString(ip.Address)
				gpipbuf.WriteString("\",")
			} else {
				if ip.Delay <= delay {
					if ip.ServerName == "gws" {
						gws++
					}
					if ip.ServerName == "gvs" {
						gvs++
					}
					gaipbuf.WriteString(ip.Address)
					gaipbuf.WriteString("|")
					gpipbuf.WriteString("\"")
					gpipbuf.WriteString(ip.Address)
					gpipbuf.WriteString("\",")
				}
			}
		}
	}

	gaip := gaipbuf.String()
	gpip := gpipbuf.String()

	if len(gaip) > 0 {
		gaip = gaip[:len(gaip)-1]
	}
	if len(gpip) > 0 {
		gpip = gpip[:len(gpip)-1]
	}
	err = ioutil.WriteFile(jsonIPFileName, []byte(gaip+"\n\n\n"+gpip), 0755)
	if err != nil {
		fmt.Printf("write ip to file %s error: %v", jsonIPFileName, err)
	}
	return gws, gvs
}

//Whether file exists.
func isFileExist(file string) bool {
	_, err := os.Stat(file)
	if err != nil && os.IsNotExist(err) {
		return false
	}
	return true
}
<|endoftext|>