<commit_before>/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements. See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership. The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License. You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied. See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

package thrift

import (
	"bytes"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"strconv"
)

// Default to using the shared http client. Library users are
// free to change this global client or specify one through
// THttpClientOptions.
var DefaultHttpClient *http.Client = http.DefaultClient

type THttpClient struct {
	client             *http.Client
	response           *http.Response
	url                *url.URL
	requestBuffer      *bytes.Buffer
	header             http.Header
	nsecConnectTimeout int64
	nsecReadTimeout    int64
}

type THttpClientTransportFactory struct {
	options THttpClientOptions
	url     string
	isPost  bool
}

func (p *THttpClientTransportFactory) GetTransport(trans TTransport) (TTransport, error) {
	if trans != nil {
		t, ok := trans.(*THttpClient)
		if ok && t.url != nil {
			if t.requestBuffer != nil {
				return NewTHttpPostClientWithOptions(t.url.String(), p.options)
			}
			return NewTHttpClientWithOptions(t.url.String(), p.options)
		}
	}
	if p.isPost {
		return NewTHttpPostClientWithOptions(p.url, p.options)
	}
	return NewTHttpClientWithOptions(p.url, p.options)
}

type THttpClientOptions struct {
	// If nil, DefaultHttpClient is used
	Client *http.Client
}

func NewTHttpClientTransportFactory(url string) *THttpClientTransportFactory {
	return NewTHttpClientTransportFactoryWithOptions(url, THttpClientOptions{})
}

func NewTHttpClientTransportFactoryWithOptions(url string, options THttpClientOptions) *THttpClientTransportFactory {
	return &THttpClientTransportFactory{url: url, isPost: false, options: options}
}

func NewTHttpPostClientTransportFactory(url string) *THttpClientTransportFactory {
	return NewTHttpPostClientTransportFactoryWithOptions(url, THttpClientOptions{})
}

func NewTHttpPostClientTransportFactoryWithOptions(url string, options THttpClientOptions) *THttpClientTransportFactory {
	return &THttpClientTransportFactory{url: url, isPost: true, options: options}
}

func NewTHttpClientWithOptions(urlstr string, options THttpClientOptions) (TTransport, error) {
	parsedURL, err := url.Parse(urlstr)
	if err != nil {
		return nil, err
	}
	response, err := http.Get(urlstr)
	if err != nil {
		return nil, err
	}
	client := options.Client
	if client == nil {
		client = DefaultHttpClient
	}
	httpHeader := map[string][]string{"Content-Type": {"application/x-thrift"}}
	return &THttpClient{client: client, response: response, url: parsedURL, header: httpHeader}, nil
}

func NewTHttpClient(urlstr string) (TTransport, error) {
	return NewTHttpClientWithOptions(urlstr, THttpClientOptions{})
}

func NewTHttpPostClientWithOptions(urlstr string, options THttpClientOptions) (TTransport, error) {
	parsedURL, err := url.Parse(urlstr)
	if err != nil {
		return nil, err
	}
	buf := make([]byte, 0, 1024)
	client := options.Client
	if client == nil {
		client = DefaultHttpClient
	}
	httpHeader := map[string][]string{"Content-Type": {"application/x-thrift"}}
	return &THttpClient{client: client, url: parsedURL, requestBuffer: bytes.NewBuffer(buf), header: httpHeader}, nil
}

func NewTHttpPostClient(urlstr string) (TTransport, error) {
	return NewTHttpPostClientWithOptions(urlstr, THttpClientOptions{})
}

// Set the HTTP Header for this specific Thrift Transport
// It is important that you first assert the TTransport as a THttpClient type
// like so:
//
// httpTrans := trans.(THttpClient)
// httpTrans.SetHeader("User-Agent","Thrift Client 1.0")
func (p *THttpClient) SetHeader(key string, value string) {
	p.header.Add(key, value)
}

// Get the HTTP Header represented by the supplied Header Key for this specific Thrift Transport
// It is important that you first assert the TTransport as a THttpClient type
// like so:
//
// httpTrans := trans.(THttpClient)
// hdrValue := httpTrans.GetHeader("User-Agent")
func (p *THttpClient) GetHeader(key string) string {
	return p.header.Get(key)
}

// Deletes the HTTP Header given a Header Key for this specific Thrift Transport
// It is important that you first assert the TTransport as a THttpClient type
// like so:
//
// httpTrans := trans.(THttpClient)
// httpTrans.DelHeader("User-Agent")
func (p *THttpClient) DelHeader(key string) {
	p.header.Del(key)
}

func (p *THttpClient) Open() error {
	// do nothing
	return nil
}

func (p *THttpClient) IsOpen() bool {
	return p.response != nil || p.requestBuffer != nil
}

func (p *THttpClient) closeResponse() error {
	var err error
	if p.response != nil && p.response.Body != nil {
		// The docs specify that if keepalive is enabled and the response body is not
		// read to completion the connection will never be returned to the pool and
		// reused. Errors are being ignored here because if the connection is invalid
		// and this fails for some reason, the Close() method will do any remaining
		// cleanup.
		io.Copy(ioutil.Discard, p.response.Body)

		err = p.response.Body.Close()
	}

	p.response = nil
	return err
}

func (p *THttpClient) Close() error {
	if p.requestBuffer != nil {
		p.requestBuffer.Reset()
		p.requestBuffer = nil
	}
	return p.closeResponse()
}

func (p *THttpClient) Read(buf []byte) (int, error) {
	if p.response == nil {
		return 0, NewTTransportException(NOT_OPEN, "Response buffer is empty, no request.")
	}
	n, err := p.response.Body.Read(buf)
	if n > 0 && (err == nil || err == io.EOF) {
		return n, nil
	}
	return n, NewTTransportExceptionFromError(err)
}

func (p *THttpClient) ReadByte() (c byte, err error) {
	return readByte(p.response.Body)
}

func (p *THttpClient) Write(buf []byte) (int, error) {
	n, err := p.requestBuffer.Write(buf)
	return n, err
}

func (p *THttpClient) WriteByte(c byte) error {
	return p.requestBuffer.WriteByte(c)
}

func (p *THttpClient) WriteString(s string) (n int, err error) {
	return p.requestBuffer.WriteString(s)
}

func (p *THttpClient) Flush() error {
	// Close any previous response body to avoid leaking connections.
	p.closeResponse()

	req, err := http.NewRequest("POST", p.url.String(), p.requestBuffer)
	if err != nil {
		return NewTTransportExceptionFromError(err)
	}
	req.Header = p.header
	response, err := p.client.Do(req)
	if err != nil {
		return NewTTransportExceptionFromError(err)
	}
	if response.StatusCode != http.StatusOK {
		// Close the response to avoid leaking file descriptors. closeResponse does
		// more than just call Close(), so temporarily assign it and reuse the logic.
		p.response = response
		p.closeResponse()

		// TODO(pomack) log bad response
		return NewTTransportException(UNKNOWN_TRANSPORT_EXCEPTION, "HTTP Response code: "+strconv.Itoa(response.StatusCode))
	}
	p.response = response
	return nil
}

func (p *THttpClient) RemainingBytes() (num_bytes uint64) {
	len := p.response.ContentLength
	if len >= 0 {
		return uint64(len)
	}

	const maxSize = ^uint64(0)
	return maxSize // the thruth is, we just don't know unless framed is used
}
<commit_msg>THRIFT-4219 remove http GET functions, Thrift only uses POST Client: Go Patch: D. Can Celasun <dcelasun@gmail.com><commit_after>/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements. See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership. The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License. You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied. See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

package thrift

import (
	"bytes"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"strconv"
)

// Default to using the shared http client. Library users are
// free to change this global client or specify one through
// THttpClientOptions.
var DefaultHttpClient *http.Client = http.DefaultClient

type THttpClient struct {
	client             *http.Client
	response           *http.Response
	url                *url.URL
	requestBuffer      *bytes.Buffer
	header             http.Header
	nsecConnectTimeout int64
	nsecReadTimeout    int64
}

type THttpClientTransportFactory struct {
	options THttpClientOptions
	url     string
}

func (p *THttpClientTransportFactory) GetTransport(trans TTransport) (TTransport, error) {
	if trans != nil {
		t, ok := trans.(*THttpClient)
		if ok && t.url != nil {
			return NewTHttpClientWithOptions(t.url.String(), p.options)
		}
	}
	return NewTHttpClientWithOptions(p.url, p.options)
}

type THttpClientOptions struct {
	// If nil, DefaultHttpClient is used
	Client *http.Client
}

func NewTHttpClientTransportFactory(url string) *THttpClientTransportFactory {
	return NewTHttpClientTransportFactoryWithOptions(url, THttpClientOptions{})
}

func NewTHttpClientTransportFactoryWithOptions(url string, options THttpClientOptions) *THttpClientTransportFactory {
	return &THttpClientTransportFactory{url: url, options: options}
}

func NewTHttpClientWithOptions(urlstr string, options THttpClientOptions) (TTransport, error) {
	parsedURL, err := url.Parse(urlstr)
	if err != nil {
		return nil, err
	}
	buf := make([]byte, 0, 1024)
	client := options.Client
	if client == nil {
		client = DefaultHttpClient
	}
	httpHeader := map[string][]string{"Content-Type": {"application/x-thrift"}}
	return &THttpClient{client: client, url: parsedURL, requestBuffer: bytes.NewBuffer(buf), header: httpHeader}, nil
}

func NewTHttpClient(urlstr string) (TTransport, error) {
	return NewTHttpClientWithOptions(urlstr, THttpClientOptions{})
}

// Set the HTTP Header for this specific Thrift Transport
// It is important that you first assert the TTransport as a THttpClient type
// like so:
//
// httpTrans := trans.(THttpClient)
// httpTrans.SetHeader("User-Agent","Thrift Client 1.0")
func (p *THttpClient) SetHeader(key string, value string) {
	p.header.Add(key, value)
}

// Get the HTTP Header represented by the supplied Header Key for this specific Thrift Transport
// It is important that you first assert the TTransport as a THttpClient type
// like so:
//
// httpTrans := trans.(THttpClient)
// hdrValue := httpTrans.GetHeader("User-Agent")
func (p *THttpClient) GetHeader(key string) string {
	return p.header.Get(key)
}

// Deletes the HTTP Header given a Header Key for this specific Thrift Transport
// It is important that you first assert the TTransport as a THttpClient type
// like so:
//
// httpTrans := trans.(THttpClient)
// httpTrans.DelHeader("User-Agent")
func (p *THttpClient) DelHeader(key string) {
	p.header.Del(key)
}

func (p *THttpClient) Open() error {
	// do nothing
	return nil
}

func (p *THttpClient) IsOpen() bool {
	return p.response != nil || p.requestBuffer != nil
}

func (p *THttpClient) closeResponse() error {
	var err error
	if p.response != nil && p.response.Body != nil {
		// The docs specify that if keepalive is enabled and the response body is not
		// read to completion the connection will never be returned to the pool and
		// reused. Errors are being ignored here because if the connection is invalid
		// and this fails for some reason, the Close() method will do any remaining
		// cleanup.
		io.Copy(ioutil.Discard, p.response.Body)

		err = p.response.Body.Close()
	}

	p.response = nil
	return err
}

func (p *THttpClient) Close() error {
	if p.requestBuffer != nil {
		p.requestBuffer.Reset()
		p.requestBuffer = nil
	}
	return p.closeResponse()
}

func (p *THttpClient) Read(buf []byte) (int, error) {
	if p.response == nil {
		return 0, NewTTransportException(NOT_OPEN, "Response buffer is empty, no request.")
	}
	n, err := p.response.Body.Read(buf)
	if n > 0 && (err == nil || err == io.EOF) {
		return n, nil
	}
	return n, NewTTransportExceptionFromError(err)
}

func (p *THttpClient) ReadByte() (c byte, err error) {
	return readByte(p.response.Body)
}

func (p *THttpClient) Write(buf []byte) (int, error) {
	n, err := p.requestBuffer.Write(buf)
	return n, err
}

func (p *THttpClient) WriteByte(c byte) error {
	return p.requestBuffer.WriteByte(c)
}

func (p *THttpClient) WriteString(s string) (n int, err error) {
	return p.requestBuffer.WriteString(s)
}

func (p *THttpClient) Flush() error {
	// Close any previous response body to avoid leaking connections.
	p.closeResponse()

	req, err := http.NewRequest("POST", p.url.String(), p.requestBuffer)
	if err != nil {
		return NewTTransportExceptionFromError(err)
	}
	req.Header = p.header
	response, err := p.client.Do(req)
	if err != nil {
		return NewTTransportExceptionFromError(err)
	}
	if response.StatusCode != http.StatusOK {
		// Close the response to avoid leaking file descriptors. closeResponse does
		// more than just call Close(), so temporarily assign it and reuse the logic.
		p.response = response
		p.closeResponse()

		// TODO(pomack) log bad response
		return NewTTransportException(UNKNOWN_TRANSPORT_EXCEPTION, "HTTP Response code: "+strconv.Itoa(response.StatusCode))
	}
	p.response = response
	return nil
}

func (p *THttpClient) RemainingBytes() (num_bytes uint64) {
	len := p.response.ContentLength
	if len >= 0 {
		return uint64(len)
	}

	const maxSize = ^uint64(0)
	return maxSize // the thruth is, we just don't know unless framed is used
}

// Deprecated: Use NewTHttpClientTransportFactory instead.
func NewTHttpPostClientTransportFactory(url string) *THttpClientTransportFactory {
	return NewTHttpClientTransportFactoryWithOptions(url, THttpClientOptions{})
}

// Deprecated: Use NewTHttpClientTransportFactoryWithOptions instead.
func NewTHttpPostClientTransportFactoryWithOptions(url string, options THttpClientOptions) *THttpClientTransportFactory {
	return NewTHttpClientTransportFactoryWithOptions(url, options)
}

// Deprecated: Use NewTHttpClientWithOptions instead.
func NewTHttpPostClientWithOptions(urlstr string, options THttpClientOptions) (TTransport, error) {
	return NewTHttpClientWithOptions(urlstr, options)
}

// Deprecated: Use NewTHttpClient instead.
func NewTHttpPostClient(urlstr string) (TTransport, error) {
	return NewTHttpClientWithOptions(urlstr, THttpClientOptions{})
}
<|endoftext|>


<commit_before>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package flag

import (
	"bytes"
	"fmt"
	"io"
	"strings"

	"github.com/spf13/pflag"
)

// NamedFlagSets stores named flag sets in the order of calling FlagSet.
type NamedFlagSets struct {
	// Order is an ordered list of flag set names.
	Order []string
	// FlagSets stores the flag sets by name.
	FlagSets map[string]*pflag.FlagSet
}

// FlagSet returns the flag set with the given name and adds it to the
// ordered name list if it is not in there yet.
func (nfs *NamedFlagSets) FlagSet(name string) *pflag.FlagSet {
	if nfs.FlagSets == nil {
		nfs.FlagSets = map[string]*pflag.FlagSet{}
	}
	if _, ok := nfs.FlagSets[name]; !ok {
		nfs.FlagSets[name] = pflag.NewFlagSet(name, pflag.ExitOnError)
		nfs.Order = append(nfs.Order, name)
	}
	return nfs.FlagSets[name]
}

// PrintSections prints the given names flag sets in sections, with the maximal given column number.
// If cols is zero, lines are not wrapped.
func PrintSections(w io.Writer, fss NamedFlagSets, cols int) {
	for _, name := range fss.Order {
		fs := fss.FlagSets[name]
		if !fs.HasFlags() {
			continue
		}

		wideFS := pflag.NewFlagSet("", pflag.ExitOnError)
		wideFS.AddFlagSet(fs)

		var zzz string
		if cols > 24 {
			zzz = strings.Repeat("z", cols-24)
			wideFS.Int(zzz, 0, strings.Repeat("z", cols-24))
		}

		var buf bytes.Buffer
		fmt.Fprintf(&buf, "\n%s flags:\n\n%s", strings.ToUpper(name[:1])+name[1:], wideFS.FlagUsagesWrapped(cols))

		if cols > 24 {
			i := strings.Index(buf.String(), zzz)
			lines := strings.Split(buf.String()[:i], "\n")
			fmt.Fprint(w, strings.Join(lines[:len(lines)-1], "\n"))
			fmt.Fprintln(w)
		} else {
			fmt.Fprint(w, buf.String())
		}
	}
}
<commit_msg>add normalize function to global FlagSet<commit_after>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package flag

import (
	"bytes"
	"fmt"
	"io"
	"strings"

	"github.com/spf13/pflag"
)

// NamedFlagSets stores named flag sets in the order of calling FlagSet.
type NamedFlagSets struct {
	// Order is an ordered list of flag set names.
	Order []string
	// FlagSets stores the flag sets by name.
	FlagSets map[string]*pflag.FlagSet
	// NormalizeNameFunc is the normalize function which used to initialize FlagSets created by NamedFlagSets.
	NormalizeNameFunc func(f *pflag.FlagSet, name string) pflag.NormalizedName
}

// FlagSet returns the flag set with the given name and adds it to the
// ordered name list if it is not in there yet.
func (nfs *NamedFlagSets) FlagSet(name string) *pflag.FlagSet {
	if nfs.FlagSets == nil {
		nfs.FlagSets = map[string]*pflag.FlagSet{}
	}
	if _, ok := nfs.FlagSets[name]; !ok {
		flagSet := pflag.NewFlagSet(name, pflag.ExitOnError)
		flagSet.SetNormalizeFunc(pflag.CommandLine.GetNormalizeFunc())
		if nfs.NormalizeNameFunc != nil {
			flagSet.SetNormalizeFunc(nfs.NormalizeNameFunc)
		}
		nfs.FlagSets[name] = flagSet
		nfs.Order = append(nfs.Order, name)
	}
	return nfs.FlagSets[name]
}

// PrintSections prints the given names flag sets in sections, with the maximal given column number.
// If cols is zero, lines are not wrapped.
func PrintSections(w io.Writer, fss NamedFlagSets, cols int) {
	for _, name := range fss.Order {
		fs := fss.FlagSets[name]
		if !fs.HasFlags() {
			continue
		}

		wideFS := pflag.NewFlagSet("", pflag.ExitOnError)
		wideFS.AddFlagSet(fs)

		var zzz string
		if cols > 24 {
			zzz = strings.Repeat("z", cols-24)
			wideFS.Int(zzz, 0, strings.Repeat("z", cols-24))
		}

		var buf bytes.Buffer
		fmt.Fprintf(&buf, "\n%s flags:\n\n%s", strings.ToUpper(name[:1])+name[1:], wideFS.FlagUsagesWrapped(cols))

		if cols > 24 {
			i := strings.Index(buf.String(), zzz)
			lines := strings.Split(buf.String()[:i], "\n")
			fmt.Fprint(w, strings.Join(lines[:len(lines)-1], "\n"))
			fmt.Fprintln(w)
		} else {
			fmt.Fprint(w, buf.String())
		}
	}
}
<|endoftext|>


<commit_before>package client // import "github.com/docker/docker/client"

import (
	"bytes"
	"context"
	"io"
	"net/http"
	"net/url"
	"os"
	"runtime"
	"strings"
	"testing"

	"github.com/docker/docker/api"
	"github.com/docker/docker/api/types"
	"gotest.tools/v3/assert"
	is "gotest.tools/v3/assert/cmp"
	"gotest.tools/v3/env"
	"gotest.tools/v3/skip"
)

func TestNewClientWithOpsFromEnv(t *testing.T) {
	skip.If(t, runtime.GOOS == "windows")

	testcases := []struct {
		doc             string
		envs            map[string]string
		expectedError   string
		expectedVersion string
	}{
		{
			doc:             "default api version",
			envs:            map[string]string{},
			expectedVersion: api.DefaultVersion,
		},
		{
			doc: "invalid cert path",
			envs: map[string]string{
				"DOCKER_CERT_PATH": "invalid/path",
			},
			expectedError: "Could not load X509 key pair: open invalid/path/cert.pem: no such file or directory",
		},
		{
			doc: "default api version with cert path",
			envs: map[string]string{
				"DOCKER_CERT_PATH": "testdata/",
			},
			expectedVersion: api.DefaultVersion,
		},
		{
			doc: "default api version with cert path and tls verify",
			envs: map[string]string{
				"DOCKER_CERT_PATH":  "testdata/",
				"DOCKER_TLS_VERIFY": "1",
			},
			expectedVersion: api.DefaultVersion,
		},
		{
			doc: "default api version with cert path and host",
			envs: map[string]string{
				"DOCKER_CERT_PATH": "testdata/",
				"DOCKER_HOST":      "https://notaunixsocket",
			},
			expectedVersion: api.DefaultVersion,
		},
		{
			doc: "invalid docker host",
			envs: map[string]string{
				"DOCKER_HOST": "host",
			},
			expectedError: "unable to parse docker host `host`",
		},
		{
			doc: "invalid docker host, with good format",
			envs: map[string]string{
				"DOCKER_HOST": "invalid://url",
			},
			expectedVersion: api.DefaultVersion,
		},
		{
			doc: "override api version",
			envs: map[string]string{
				"DOCKER_API_VERSION": "1.22",
			},
			expectedVersion: "1.22",
		},
	}

	defer env.PatchAll(t, nil)()
	for _, c := range testcases {
		env.PatchAll(t, c.envs)
		apiclient, err := NewClientWithOpts(FromEnv)
		if c.expectedError != "" {
			assert.Check(t, is.Error(err, c.expectedError), c.doc)
		} else {
			assert.Check(t, err, c.doc)
			version := apiclient.ClientVersion()
			assert.Check(t, is.Equal(c.expectedVersion, version), c.doc)
		}

		if c.envs["DOCKER_TLS_VERIFY"] != "" {
			// pedantic checking that this is handled correctly
			tr := apiclient.client.Transport.(*http.Transport)
			assert.Assert(t, tr.TLSClientConfig != nil, c.doc)
			assert.Check(t, is.Equal(tr.TLSClientConfig.InsecureSkipVerify, false), c.doc)
		}
	}
}

func TestGetAPIPath(t *testing.T) {
	testcases := []struct {
		version  string
		path     string
		query    url.Values
		expected string
	}{
		{"", "/containers/json", nil, "/containers/json"},
		{"", "/containers/json", url.Values{}, "/containers/json"},
		{"", "/containers/json", url.Values{"s": []string{"c"}}, "/containers/json?s=c"},
		{"1.22", "/containers/json", nil, "/v1.22/containers/json"},
		{"1.22", "/containers/json", url.Values{}, "/v1.22/containers/json"},
		{"1.22", "/containers/json", url.Values{"s": []string{"c"}}, "/v1.22/containers/json?s=c"},
		{"v1.22", "/containers/json", nil, "/v1.22/containers/json"},
		{"v1.22", "/containers/json", url.Values{}, "/v1.22/containers/json"},
		{"v1.22", "/containers/json", url.Values{"s": []string{"c"}}, "/v1.22/containers/json?s=c"},
		{"v1.22", "/networks/kiwl$%^", nil, "/v1.22/networks/kiwl$%25%5E"},
	}

	ctx := context.TODO()
	for _, testcase := range testcases {
		c := Client{version: testcase.version, basePath: "/"}
		actual := c.getAPIPath(ctx, testcase.path, testcase.query)
		assert.Check(t, is.Equal(actual, testcase.expected))
	}
}

func TestParseHostURL(t *testing.T) {
	testcases := []struct {
		host        string
		expected    *url.URL
		expectedErr string
	}{
		{
			host:        "",
			expectedErr: "unable to parse docker host",
		},
		{
			host:        "foobar",
			expectedErr: "unable to parse docker host",
		},
		{
			host:     "foo://bar",
			expected: &url.URL{Scheme: "foo", Host: "bar"},
		},
		{
			host:     "tcp://localhost:2476",
			expected: &url.URL{Scheme: "tcp", Host: "localhost:2476"},
		},
		{
			host:     "tcp://localhost:2476/path",
			expected: &url.URL{Scheme: "tcp", Host: "localhost:2476", Path: "/path"},
		},
	}

	for _, testcase := range testcases {
		actual, err := ParseHostURL(testcase.host)
		if testcase.expectedErr != "" {
			assert.Check(t, is.ErrorContains(err, testcase.expectedErr))
		}
		assert.Check(t, is.DeepEqual(testcase.expected, actual))
	}
}

func TestNewClientWithOpsFromEnvSetsDefaultVersion(t *testing.T) {
	defer env.PatchAll(t, map[string]string{
		"DOCKER_HOST":        "",
		"DOCKER_API_VERSION": "",
		"DOCKER_TLS_VERIFY":  "",
		"DOCKER_CERT_PATH":   "",
	})()

	client, err := NewClientWithOpts(FromEnv)
	if err != nil {
		t.Fatal(err)
	}
	assert.Check(t, is.Equal(client.version, api.DefaultVersion))

	expected := "1.22"
	os.Setenv("DOCKER_API_VERSION", expected)
	client, err = NewClientWithOpts(FromEnv)
	if err != nil {
		t.Fatal(err)
	}
	assert.Check(t, is.Equal(expected, client.version))
}

// TestNegotiateAPIVersionEmpty asserts that client.Client can
// negotiate a compatible APIVersion when omitted
func TestNegotiateAPIVersionEmpty(t *testing.T) {
	defer env.PatchAll(t, map[string]string{"DOCKER_API_VERSION": ""})()

	client, err := NewClientWithOpts(FromEnv)
	assert.NilError(t, err)

	ping := types.Ping{
		APIVersion:   "",
		OSType:       "linux",
		Experimental: false,
	}

	// set our version to something new
	client.version = "1.25"

	// if no version from server, expect the earliest
	// version before APIVersion was implemented
	expected := "1.24"

	// test downgrade
	client.NegotiateAPIVersionPing(ping)
	assert.Check(t, is.Equal(expected, client.version))
}

// TestNegotiateAPIVersion asserts that client.Client can
// negotiate a compatible APIVersion with the server
func TestNegotiateAPIVersion(t *testing.T) {
	client, err := NewClientWithOpts(FromEnv)
	assert.NilError(t, err)

	expected := "1.21"
	ping := types.Ping{
		APIVersion:   expected,
		OSType:       "linux",
		Experimental: false,
	}

	// set our version to something new
	client.version = "1.22"

	// test downgrade
	client.NegotiateAPIVersionPing(ping)
	assert.Check(t, is.Equal(expected, client.version))

	// set the client version to something older, and verify that we keep the
	// original setting.
	expected = "1.20"
	client.version = expected
	client.NegotiateAPIVersionPing(ping)
	assert.Check(t, is.Equal(expected, client.version))

}

// TestNegotiateAPIVersionOverride asserts that we honor
// the environment variable DOCKER_API_VERSION when negotiating versions
func TestNegotiateAPVersionOverride(t *testing.T) {
	expected := "9.99"
	defer env.PatchAll(t, map[string]string{"DOCKER_API_VERSION": expected})()

	client, err := NewClientWithOpts(FromEnv)
	assert.NilError(t, err)

	ping := types.Ping{
		APIVersion:   "1.24",
		OSType:       "linux",
		Experimental: false,
	}

	// test that we honored the env var
	client.NegotiateAPIVersionPing(ping)
	assert.Check(t, is.Equal(expected, client.version))
}

func TestNegotiateAPIVersionAutomatic(t *testing.T) {
	var pingVersion string
	httpClient := newMockClient(func(req *http.Request) (*http.Response, error) {
		resp := &http.Response{StatusCode: http.StatusOK, Header: http.Header{}}
		resp.Header.Set("API-Version", pingVersion)
		resp.Body = io.NopCloser(strings.NewReader("OK"))
		return resp, nil
	})

	client, err := NewClientWithOpts(
		WithHTTPClient(httpClient),
		WithAPIVersionNegotiation(),
	)
	assert.NilError(t, err)

	ctx := context.Background()
	assert.Equal(t, client.ClientVersion(), api.DefaultVersion)

	// First request should trigger negotiation
	pingVersion = "1.35"
	_, _ = client.Info(ctx)
	assert.Equal(t, client.ClientVersion(), "1.35")

	// Once successfully negotiated, subsequent requests should not re-negotiate
	pingVersion = "1.25"
	_, _ = client.Info(ctx)
	assert.Equal(t, client.ClientVersion(), "1.35")
}

// TestNegotiateAPIVersionWithEmptyVersion asserts that initializing a client
// with an empty version string does still allow API-version negotiation
func TestNegotiateAPIVersionWithEmptyVersion(t *testing.T) {
	client, err := NewClientWithOpts(WithVersion(""))
	assert.NilError(t, err)

	client.NegotiateAPIVersionPing(types.Ping{APIVersion: "1.35"})
	assert.Equal(t, client.version, "1.35")
}

// TestNegotiateAPIVersionWithFixedVersion asserts that initializing a client
// with an fixed version disables API-version negotiation
func TestNegotiateAPIVersionWithFixedVersion(t *testing.T) {
	client, err := NewClientWithOpts(WithVersion("1.35"))
	assert.NilError(t, err)

	client.NegotiateAPIVersionPing(types.Ping{APIVersion: "1.31"})
	assert.Equal(t, client.version, "1.35")
}

type roundTripFunc func(*http.Request) (*http.Response, error)

func (rtf roundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) {
	return rtf(req)
}

type bytesBufferClose struct {
	*bytes.Buffer
}

func (bbc bytesBufferClose) Close() error {
	return nil
}

func TestClientRedirect(t *testing.T) {
	client := &http.Client{
		CheckRedirect: CheckRedirect,
		Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
			if req.URL.String() == "/bla" {
				return &http.Response{StatusCode: 404}, nil
			}
			return &http.Response{
				StatusCode: 301,
				Header:     map[string][]string{"Location": {"/bla"}},
				Body:       bytesBufferClose{bytes.NewBuffer(nil)},
			}, nil
		}),
	}

	cases := []struct {
		httpMethod  string
		expectedErr *url.Error
		statusCode  int
	}{
		{http.MethodGet, nil, 301},
		{http.MethodPost, &url.Error{Op: "Post", URL: "/bla", Err: ErrRedirect}, 301},
		{http.MethodPut, &url.Error{Op: "Put", URL: "/bla", Err: ErrRedirect}, 301},
		{http.MethodDelete, &url.Error{Op: "Delete", URL: "/bla", Err: ErrRedirect}, 301},
	}

	for _, tc := range cases {
		req, err := http.NewRequest(tc.httpMethod, "/redirectme", nil)
		assert.Check(t, err)
		resp, err := client.Do(req)
		assert.Check(t, is.Equal(tc.statusCode, resp.StatusCode))
		if tc.expectedErr == nil {
			assert.Check(t, is.Nil(err))
		} else {
			urlError, ok := err.(*url.Error)
			assert.Assert(t, ok, "%T is not *url.Error", err)
			assert.Check(t, is.Equal(*tc.expectedErr, *urlError))
		}
	}
}
<commit_msg>client: various small test-improvements<commit_after>package client // import "github.com/docker/docker/client"

import (
	"bytes"
	"context"
	"io"
	"net/http"
	"net/url"
	"os"
	"runtime"
	"strings"
	"testing"

	"github.com/docker/docker/api"
	"github.com/docker/docker/api/types"
	"gotest.tools/v3/assert"
	is "gotest.tools/v3/assert/cmp"
	"gotest.tools/v3/env"
	"gotest.tools/v3/skip"
)

func TestNewClientWithOpsFromEnv(t *testing.T) {
	skip.If(t, runtime.GOOS == "windows")

	testcases := []struct {
		doc             string
		envs            map[string]string
		expectedError   string
		expectedVersion string
	}{
		{
			doc:             "default api version",
			envs:            map[string]string{},
			expectedVersion: api.DefaultVersion,
		},
		{
			doc: "invalid cert path",
			envs: map[string]string{
				"DOCKER_CERT_PATH": "invalid/path",
			},
			expectedError: "Could not load X509 key pair: open invalid/path/cert.pem: no such file or directory",
		},
		{
			doc: "default api version with cert path",
			envs: map[string]string{
				"DOCKER_CERT_PATH": "testdata/",
			},
			expectedVersion: api.DefaultVersion,
		},
		{
			doc: "default api version with cert path and tls verify",
			envs: map[string]string{
				"DOCKER_CERT_PATH":  "testdata/",
				"DOCKER_TLS_VERIFY": "1",
			},
			expectedVersion: api.DefaultVersion,
		},
		{
			doc: "default api version with cert path and host",
			envs: map[string]string{
				"DOCKER_CERT_PATH": "testdata/",
				"DOCKER_HOST":      "https://notaunixsocket",
			},
			expectedVersion: api.DefaultVersion,
		},
		{
			doc: "invalid docker host",
			envs: map[string]string{
				"DOCKER_HOST": "host",
			},
			expectedError: "unable to parse docker host `host`",
		},
		{
			doc: "invalid docker host, with good format",
			envs: map[string]string{
				"DOCKER_HOST": "invalid://url",
			},
			expectedVersion: api.DefaultVersion,
		},
		{
			doc: "override api version",
			envs: map[string]string{
				"DOCKER_API_VERSION": "1.22",
			},
			expectedVersion: "1.22",
		},
	}

	defer env.PatchAll(t, nil)()
	for _, tc := range testcases {
		env.PatchAll(t, tc.envs)
		client, err := NewClientWithOpts(FromEnv)
		if tc.expectedError != "" {
			assert.Check(t, is.Error(err, tc.expectedError), tc.doc)
		} else {
			assert.Check(t, err, tc.doc)
			assert.Check(t, is.Equal(client.ClientVersion(), tc.expectedVersion), tc.doc)
		}

		if tc.envs["DOCKER_TLS_VERIFY"] != "" {
			// pedantic checking that this is handled correctly
			tr := client.client.Transport.(*http.Transport)
			assert.Assert(t, tr.TLSClientConfig != nil, tc.doc)
			assert.Check(t, is.Equal(tr.TLSClientConfig.InsecureSkipVerify, false), tc.doc)
		}
	}
}

func TestGetAPIPath(t *testing.T) {
	testcases := []struct {
		version  string
		path     string
		query    url.Values
		expected string
	}{
		{"", "/containers/json", nil, "/containers/json"},
		{"", "/containers/json", url.Values{}, "/containers/json"},
		{"", "/containers/json", url.Values{"s": []string{"c"}}, "/containers/json?s=c"},
		{"1.22", "/containers/json", nil, "/v1.22/containers/json"},
		{"1.22", "/containers/json", url.Values{}, "/v1.22/containers/json"},
		{"1.22", "/containers/json", url.Values{"s": []string{"c"}}, "/v1.22/containers/json?s=c"},
		{"v1.22", "/containers/json", nil, "/v1.22/containers/json"},
		{"v1.22", "/containers/json", url.Values{}, "/v1.22/containers/json"},
		{"v1.22", "/containers/json", url.Values{"s": []string{"c"}}, "/v1.22/containers/json?s=c"},
		{"v1.22", "/networks/kiwl$%^", nil, "/v1.22/networks/kiwl$%25%5E"},
	}

	ctx := context.TODO()
	for _, tc := range testcases {
		client := Client{
			version:  tc.version,
			basePath: "/",
		}
		actual := client.getAPIPath(ctx, tc.path, tc.query)
		assert.Check(t, is.Equal(actual, tc.expected))
	}
}

func TestParseHostURL(t *testing.T) {
	testcases := []struct {
		host        string
		expected    *url.URL
		expectedErr string
	}{
		{
			host:        "",
			expectedErr: "unable to parse docker host",
		},
		{
			host:        "foobar",
			expectedErr: "unable to parse docker host",
		},
		{
			host:     "foo://bar",
			expected: &url.URL{Scheme: "foo", Host: "bar"},
		},
		{
			host:     "tcp://localhost:2476",
			expected: &url.URL{Scheme: "tcp", Host: "localhost:2476"},
		},
		{
			host:     "tcp://localhost:2476/path",
			expected: &url.URL{Scheme: "tcp", Host: "localhost:2476", Path: "/path"},
		},
	}

	for _, testcase := range testcases {
		actual, err := ParseHostURL(testcase.host)
		if testcase.expectedErr != "" {
			assert.Check(t, is.ErrorContains(err, testcase.expectedErr))
		}
		assert.Check(t, is.DeepEqual(actual, testcase.expected))
	}
}

func TestNewClientWithOpsFromEnvSetsDefaultVersion(t *testing.T) {
	defer env.PatchAll(t, map[string]string{
		"DOCKER_HOST":        "",
		"DOCKER_API_VERSION": "",
		"DOCKER_TLS_VERIFY":  "",
		"DOCKER_CERT_PATH":   "",
	})()

	client, err := NewClientWithOpts(FromEnv)
	if err != nil {
		t.Fatal(err)
	}
	assert.Check(t, is.Equal(client.ClientVersion(), api.DefaultVersion))

	const expected = "1.22"
	_ = os.Setenv("DOCKER_API_VERSION", expected)
	client, err = NewClientWithOpts(FromEnv)
	if err != nil {
		t.Fatal(err)
	}
	assert.Check(t, is.Equal(client.ClientVersion(), expected))
}

// TestNegotiateAPIVersionEmpty asserts that client.Client version negotiation
// downgrades to the correct API version if the API's ping response does not
// return an API version.
func TestNegotiateAPIVersionEmpty(t *testing.T) {
	defer env.PatchAll(t, map[string]string{"DOCKER_API_VERSION": ""})()

	client, err := NewClientWithOpts(FromEnv)
	assert.NilError(t, err)

	// set our version to something new
	client.version = "1.25"

	// if no version from server, expect the earliest
	// version before APIVersion was implemented
	const expected = "1.24"

	// test downgrade
	client.NegotiateAPIVersionPing(types.Ping{})
	assert.Equal(t, client.ClientVersion(), expected)
}

// TestNegotiateAPIVersion asserts that client.Client can
// negotiate a compatible APIVersion with the server
func TestNegotiateAPIVersion(t *testing.T) {
	client, err := NewClientWithOpts(FromEnv)
	assert.NilError(t, err)

	expected := "1.21"
	ping := types.Ping{
		APIVersion:   expected,
		OSType:       "linux",
		Experimental: false,
	}

	// set our version to something new
	client.version = "1.22"

	// test downgrade
	client.NegotiateAPIVersionPing(ping)
	assert.Check(t, is.Equal(expected, client.version))

	// set the client version to something older, and verify that we keep the
	// original setting.
	expected = "1.20"
	client.version = expected
	client.NegotiateAPIVersionPing(ping)
	assert.Check(t, is.Equal(client.version, expected))

}

// TestNegotiateAPIVersionOverride asserts that we honor the DOCKER_API_VERSION
// environment variable when negotiating versions.
func TestNegotiateAPVersionOverride(t *testing.T) {
	const expected = "9.99"
	defer env.PatchAll(t, map[string]string{"DOCKER_API_VERSION": expected})()

	client, err := NewClientWithOpts(FromEnv)
	assert.NilError(t, err)

	// test that we honored the env var
	client.NegotiateAPIVersionPing(types.Ping{APIVersion: "1.24"})
	assert.Equal(t, client.ClientVersion(), expected)
}

func TestNegotiateAPIVersionAutomatic(t *testing.T) {
	var pingVersion string
	httpClient := newMockClient(func(req *http.Request) (*http.Response, error) {
		resp := &http.Response{StatusCode: http.StatusOK, Header: http.Header{}}
		resp.Header.Set("API-Version", pingVersion)
		resp.Body = io.NopCloser(strings.NewReader("OK"))
		return resp, nil
	})

	ctx := context.Background()
	client, err := NewClientWithOpts(
		WithHTTPClient(httpClient),
		WithAPIVersionNegotiation(),
	)
	assert.NilError(t, err)

	// Client defaults to use api.DefaultVersion before version-negotiation.
	expected := api.DefaultVersion
	assert.Equal(t, client.ClientVersion(), expected)

	// First request should trigger negotiation
	pingVersion = "1.35"
	expected = "1.35"
	_, _ = client.Info(ctx)
	assert.Equal(t, client.ClientVersion(), expected)

	// Once successfully negotiated, subsequent requests should not re-negotiate
	pingVersion = "1.25"
	expected = "1.35"
	_, _ = client.Info(ctx)
	assert.Equal(t, client.ClientVersion(), expected)
}

// TestNegotiateAPIVersionWithEmptyVersion asserts that initializing a client
// with an empty version string does still allow API-version negotiation
func TestNegotiateAPIVersionWithEmptyVersion(t *testing.T) {
	client, err := NewClientWithOpts(WithVersion(""))
	assert.NilError(t, err)

	const expected = "1.35"
	client.NegotiateAPIVersionPing(types.Ping{APIVersion: expected})
	assert.Equal(t, client.ClientVersion(), expected)
}

// TestNegotiateAPIVersionWithFixedVersion asserts that initializing a client
// with a fixed version disables API-version negotiation
func TestNegotiateAPIVersionWithFixedVersion(t *testing.T) {
	const customVersion = "1.35"
	client, err := NewClientWithOpts(WithVersion(customVersion))
	assert.NilError(t, err)

	client.NegotiateAPIVersionPing(types.Ping{APIVersion: "1.31"})
	assert.Equal(t, client.ClientVersion(), customVersion)
}

type roundTripFunc func(*http.Request) (*http.Response, error)

func (rtf roundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) {
	return rtf(req)
}

type bytesBufferClose struct {
	*bytes.Buffer
}

func (bbc bytesBufferClose) Close() error {
	return nil
}

func TestClientRedirect(t *testing.T) {
	client := &http.Client{
		CheckRedirect: CheckRedirect,
		Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
			if req.URL.String() == "/bla" {
				return &http.Response{StatusCode: 404}, nil
			}
			return &http.Response{
				StatusCode: 301,
				Header:     map[string][]string{"Location": {"/bla"}},
				Body:       bytesBufferClose{bytes.NewBuffer(nil)},
			}, nil
		}),
	}

	cases := []struct {
		httpMethod  string
		expectedErr *url.Error
		statusCode  int
	}{
		{http.MethodGet, nil, 301},
		{http.MethodPost, &url.Error{Op: "Post", URL: "/bla", Err: ErrRedirect}, 301},
		{http.MethodPut, &url.Error{Op: "Put", URL: "/bla", Err: ErrRedirect}, 301},
		{http.MethodDelete, &url.Error{Op: "Delete", URL: "/bla", Err: ErrRedirect}, 301},
	}

	for _, tc := range cases {
		req, err := http.NewRequest(tc.httpMethod, "/redirectme", nil)
		assert.Check(t, err)
		resp, err := client.Do(req)
		assert.Check(t, is.Equal(resp.StatusCode, tc.statusCode))
		if tc.expectedErr == nil {
			assert.Check(t, is.Nil(err))
		} else {
			urlError, ok := err.(*url.Error)
			assert.Assert(t, ok, "%T is not *url.Error", err)
			assert.Check(t, is.Equal(*urlError, *tc.expectedErr))
		}
	}
}
<|endoftext|>


<commit_before>package lxd

import (
	"fmt"

	"github.com/lxc/lxd/shared/api"
)

// GetCluster returns information about a cluster
//
// If this client is not trusted, the password must be supplied
func (r *ProtocolLXD) GetCluster() (*api.Cluster, string, error) {
	if !r.HasExtension("clustering") {
		return nil, "", fmt.Errorf("The server is missing the required \"clustering\" API extension")
	}

	cluster := &api.Cluster{}
	etag, err := r.queryStruct("GET", "/cluster", nil, "", &cluster)
	if err != nil {
		return nil, "", err
	}

	return cluster, etag, nil
}

// UpdateCluster requests to bootstrap a new cluster or join an existing one.
func (r *ProtocolLXD) UpdateCluster(cluster api.ClusterPut, ETag string) (Operation, error) {
	if !r.HasExtension("clustering") {
		return nil, fmt.Errorf("The server is missing the required \"clustering\" API extension")
	}

	if cluster.ServerAddress != "" || cluster.ClusterPassword != "" || len(cluster.MemberConfig) > 0 {
		if !r.HasExtension("clustering_join") {
			return nil, fmt.Errorf("The server is missing the required \"clustering_join\" API extension")
		}
	}

	op, _, err := r.queryOperation("PUT", "/cluster", cluster, "")
	if err != nil {
		return nil, err
	}

	return op, nil
}

// DeleteClusterMember makes the given member leave the cluster (gracefully or not,
// depending on the force flag)
func (r *ProtocolLXD) DeleteClusterMember(name string, force bool) error {
	if !r.HasExtension("clustering") {
		return fmt.Errorf("The server is missing the required \"clustering\" API extension")
	}

	params := ""
	if force {
		params += "?force=1"
	}

	_, err := r.queryStruct("DELETE", fmt.Sprintf("/cluster/members/%s%s", name, params), nil, "", nil)
	if err != nil {
		return err
	}

	return nil
}

// GetClusterMemberNames returns the URLs of the current members in the cluster
func (r *ProtocolLXD) GetClusterMemberNames() ([]string, error) {
	if !r.HasExtension("clustering") {
		return nil, fmt.Errorf("The server is missing the required \"clustering\" API extension")
	}

	urls := []string{}
	_, err := r.queryStruct("GET", "/cluster/members", nil, "", &urls)
	if err != nil {
		return nil, err
	}

	return urls, nil
}

// GetClusterMembers returns the current members of the cluster
func (r *ProtocolLXD) GetClusterMembers() ([]api.ClusterMember, error) {
	if !r.HasExtension("clustering") {
		return nil, fmt.Errorf("The server is missing the required \"clustering\" API extension")
	}

	members := []api.ClusterMember{}
	_, err := r.queryStruct("GET", "/cluster/members?recursion=1", nil, "", &members)
	if err != nil {
		return nil, err
	}

	return members, nil
}

// GetClusterMember returns information about the given member
func (r *ProtocolLXD) GetClusterMember(name string) (*api.ClusterMember, string, error) {
	if !r.HasExtension("clustering") {
		return nil, "", fmt.Errorf("The server is missing the required \"clustering\" API extension")
	}

	member := api.ClusterMember{}
	etag, err := r.queryStruct("GET", fmt.Sprintf("/cluster/members/%s", name), nil, "", &member)
	if err != nil {
		return nil, "", err
	}

	return &member, etag, nil
}

// UpdateClusterMember returns information about the given member
func (r *ProtocolLXD) UpdateClusterMember(name string, member api.ClusterMemberPut, ETag string) error {
	if !r.HasExtension("clustering_edit_roles") {
		return fmt.Errorf("The server is missing the required \"clustering_edit_roles\" API extension")
	}

	// Send the request
	_, _, err := r.query("PUT", fmt.Sprintf("/cluster/members/%s", name), member, ETag)
	if err != nil {
		return err
	}

	return nil
}

// RenameClusterMember changes the name of an existing member
func (r *ProtocolLXD) RenameClusterMember(name string, member api.ClusterMemberPost) error {
	if !r.HasExtension("clustering") {
		return fmt.Errorf("The server is missing the required \"clustering\" API extension")
	}

	_, _, err := r.query("POST", fmt.Sprintf("/cluster/members/%s", name), member, "")
	if err != nil {
		return err
	}

	return nil
}
<commit_msg>client: Fix bad description for UpdateClusterMember<commit_after>package lxd

import (
	"fmt"

	"github.com/lxc/lxd/shared/api"
)

// GetCluster returns information about a cluster
//
// If this client is not trusted, the password must be supplied
func (r *ProtocolLXD) GetCluster() (*api.Cluster, string, error) {
	if !r.HasExtension("clustering") {
		return nil, "", fmt.Errorf("The server is missing the required \"clustering\" API extension")
	}

	cluster := &api.Cluster{}
	etag, err := r.queryStruct("GET", "/cluster", nil, "", &cluster)
	if err != nil {
		return nil, "", err
	}

	return cluster, etag, nil
}

// UpdateCluster requests to bootstrap a new cluster or join an existing one.
func (r *ProtocolLXD) UpdateCluster(cluster api.ClusterPut, ETag string) (Operation, error) {
	if !r.HasExtension("clustering") {
		return nil, fmt.Errorf("The server is missing the required \"clustering\" API extension")
	}

	if cluster.ServerAddress != "" || cluster.ClusterPassword != "" || len(cluster.MemberConfig) > 0 {
		if !r.HasExtension("clustering_join") {
			return nil, fmt.Errorf("The server is missing the required \"clustering_join\" API extension")
		}
	}

	op, _, err := r.queryOperation("PUT", "/cluster", cluster, "")
	if err != nil {
		return nil, err
	}

	return op, nil
}

// DeleteClusterMember makes the given member leave the cluster (gracefully or not,
// depending on the force flag)
func (r *ProtocolLXD) DeleteClusterMember(name string, force bool) error {
	if !r.HasExtension("clustering") {
		return fmt.Errorf("The server is missing the required \"clustering\" API extension")
	}

	params := ""
	if force {
		params += "?force=1"
	}

	_, err := r.queryStruct("DELETE", fmt.Sprintf("/cluster/members/%s%s", name, params), nil, "", nil)
	if err != nil {
		return err
	}

	return nil
}

// GetClusterMemberNames returns the URLs of the current members in the cluster
func (r *ProtocolLXD) GetClusterMemberNames() ([]string, error) {
	if !r.HasExtension("clustering") {
		return nil, fmt.Errorf("The server is missing the required \"clustering\" API extension")
	}

	urls := []string{}
	_, err := r.queryStruct("GET", "/cluster/members", nil, "", &urls)
	if err != nil {
		return nil, err
	}

	return urls, nil
}

// GetClusterMembers returns the current members of the cluster
func (r *ProtocolLXD) GetClusterMembers() ([]api.ClusterMember, error) {
	if !r.HasExtension("clustering") {
		return nil, fmt.Errorf("The server is missing the required \"clustering\" API extension")
	}

	members := []api.ClusterMember{}
	_, err := r.queryStruct("GET", "/cluster/members?recursion=1", nil, "", &members)
	if err != nil {
		return nil, err
	}

	return members, nil
}

// GetClusterMember returns information about the given member
func (r *ProtocolLXD) GetClusterMember(name string) (*api.ClusterMember, string, error) {
	if !r.HasExtension("clustering") {
		return nil, "", fmt.Errorf("The server is missing the required \"clustering\" API extension")
	}

	member := api.ClusterMember{}
	etag, err := r.queryStruct("GET", fmt.Sprintf("/cluster/members/%s", name), nil, "", &member)
	if err != nil {
		return nil, "", err
	}

	return &member, etag, nil
}

// UpdateClusterMember updates information about the given member
func (r *ProtocolLXD) UpdateClusterMember(name string, member api.ClusterMemberPut, ETag string) error {
	if !r.HasExtension("clustering_edit_roles") {
		return fmt.Errorf("The server is missing the required \"clustering_edit_roles\" API extension")
	}

	// Send the request
	_, _, err := r.query("PUT", fmt.Sprintf("/cluster/members/%s", name), member, ETag)
	if err != nil {
		return err
	}

	return nil
}

// RenameClusterMember changes the name of an existing member
func (r *ProtocolLXD) RenameClusterMember(name string, member api.ClusterMemberPost) error {
	if !r.HasExtension("clustering") {
		return fmt.Errorf("The server is missing the required \"clustering\" API extension")
	}

	_, _, err := r.query("POST", fmt.Sprintf("/cluster/members/%s", name), member, "")
	if err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>package effio

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path"
	"path/filepath"
	"strconv"
	"strings"
)

func (cmd *Cmd) Inventory() {
	devs := InventoryDevs()
	js, err := json.MarshalIndent(devs, "", "  ")
	if err != nil {
		log.Fatalf("Failed to encode inventory JSON: %s\n", err)
	}

	fmt.Println(string(js))
}

// in my tests I use whole devices with a single GPT partition and the
// ext4 filesystem (for now). This finds all the devices and grabs most
// of the info needed for the device JSON file and dumps it to stdout
// so it can be put in a file and edited to taste.
func InventoryDevs() (devs Devices) {
	visitor := func(dpath string, f os.FileInfo, err error) error {
		if err != nil {
			log.Fatalf("Encountered an error while inventorying devices '%s': %s", dpath, err)
		}

		fi, err := os.Stat(dpath)
		if err != nil {
			log.Fatal(err)
		}

		// ignore anything that's not a device, os.Stat seems to follow the
		// link and set mode device which is perfect for this case
		if fi.Mode()&os.ModeDevice == 0 {
			return nil
		}

		device := path.Base(dpath)
		reldev, err := os.Readlink(dpath)
		if err != nil {
			log.Fatal(err)
		}
		letter := path.Base(reldev)

		// only consider devices with a partition table and only
		// consider partition 1
		if !strings.HasSuffix(device, "-part1") {
			return nil
		}

		bdev := strings.TrimRight(letter, "1234567890")
		model := GetSysBlockString(bdev, "device/model")
		size := GetSysBlockInt(bdev, "size")
		bsize := GetSysBlockInt(bdev, "queue/hw_sector_size")
		rotational := GetSysBlockInt(bdev, "queue/rotational")
		brand := GuessBrand(model)
		name := strings.Replace(strings.Replace(strings.ToLower(model), " ", "_", -1), "-", "_", -1)

		d := Device{
			Name:       name,
			Device:     dpath,
			Mountpoint: path.Join("/mnt", bdev),
			Filesystem: "ext4",
			Brand:      brand,
			Series:     model,
			Capacity:   size,
			Rotational: (rotational == 1),
			Transport:  "", // can be detected but it's a lot of work
			HBA:        "", // ditto
			Media:      "", // no way to detect reliably
			BlockSize:  int(bsize),
		}

		devs = append(devs, d)

		return nil
	}

	err := filepath.Walk("/dev/disk/by-id", visitor)
	if err != nil {
		log.Fatalf("Could not inventory devices in /dev/disk/by-id: %s", err)
	}

	return devs
}

func GetSysBlockString(device string, fpath string) string {
	sbpath := path.Join("/sys/block", device, fpath)
	data, err := ioutil.ReadFile(sbpath)
	if err != nil {
		log.Fatal(err)
	}

	return strings.TrimRight(string(data), " \t\r\n")
}

func GetSysBlockInt(device string, fpath string) int64 {
	str := GetSysBlockString(device, fpath)
	out, err := strconv.ParseInt(str, 10, 64)
	if err != nil {
		log.Fatal(err)
	}
	return out
}

func GuessBrand(model string) string {
	if strings.HasPrefix(model, "Samsung") {
		return "Samsung"
	} else if strings.HasPrefix(model, "ST") {
		return "Seagate"
	} else if strings.HasPrefix(model, "WD") {
		return "Western Digital"
	} else if strings.HasPrefix(model, "MRD") {
		return "I/O Switch"
	} else if strings.HasPrefix(model, "SSD") {
		return "PNY"
	} else {
		return "Generic"
	}
}
<commit_msg>Fix device size.<commit_after>package effio

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path"
	"path/filepath"
	"strconv"
	"strings"
)

func (cmd *Cmd) Inventory() {
	devs := InventoryDevs()
	js, err := json.MarshalIndent(devs, "", "  ")
	if err != nil {
		log.Fatalf("Failed to encode inventory JSON: %s\n", err)
	}

	fmt.Println(string(js))
}

// in my tests I use whole devices with a single GPT partition and the
// ext4 filesystem (for now). This finds all the devices and grabs most
// of the info needed for the device JSON file and dumps it to stdout
// so it can be put in a file and edited to taste.
func InventoryDevs() (devs Devices) {
	visitor := func(dpath string, f os.FileInfo, err error) error {
		if err != nil {
			log.Fatalf("Encountered an error while inventorying devices '%s': %s", dpath, err)
		}

		fi, err := os.Stat(dpath)
		if err != nil {
			log.Fatal(err)
		}

		// ignore anything that's not a device, os.Stat seems to follow the
		// link and set mode device which is perfect for this case
		if fi.Mode()&os.ModeDevice == 0 {
			return nil
		}

		device := path.Base(dpath)
		reldev, err := os.Readlink(dpath)
		if err != nil {
			log.Fatal(err)
		}
		letter := path.Base(reldev)

		// only consider devices with a partition table and only
		// consider partition 1
		if !strings.HasSuffix(device, "-part1") {
			return nil
		}

		bdev := strings.TrimRight(letter, "1234567890")
		model := GetSysBlockString(bdev, "device/model")
		bsize := GetSysBlockInt(bdev, "queue/hw_sector_size")
		size := GetSysBlockInt(bdev, "size") * bsize
		rotational := GetSysBlockInt(bdev, "queue/rotational")
		brand := GuessBrand(model)
		// lower case, replace spaces and dashes with underscore
		name := strings.Replace(strings.Replace(strings.ToLower(model), " ", "_", -1), "-", "_", -1)

		d := Device{
			Name:       name,
			Device:     dpath,
			Mountpoint: path.Join("/mnt", bdev),
			Filesystem: "ext4",
			Brand:      brand,
			Series:     model,
			Capacity:   size,
			Rotational: (rotational == 1),
			Transport:  "", // can be detected but it's a lot of work
			HBA:        "", // ditto
			Media:      "", // no way to detect reliably
			BlockSize:  int(bsize),
		}

		devs = append(devs, d)

		return nil
	}

	err := filepath.Walk("/dev/disk/by-id", visitor)
	if err != nil {
		log.Fatalf("Could not inventory devices in /dev/disk/by-id: %s", err)
	}

	return devs
}

func GetSysBlockString(device string, fpath string) string {
	sbpath := path.Join("/sys/block", device, fpath)
	data, err := ioutil.ReadFile(sbpath)
	if err != nil {
		log.Fatal(err)
	}

	return strings.TrimRight(string(data), " \t\r\n")
}

func GetSysBlockInt(device string, fpath string) int64 {
	str := GetSysBlockString(device, fpath)
	out, err := strconv.ParseInt(str, 10, 64)
	if err != nil {
		log.Fatal(err)
	}
	return out
}

func GuessBrand(model string) string {
	if strings.HasPrefix(model, "Samsung") {
		return "Samsung"
	} else if strings.HasPrefix(model, "ST") {
		return "Seagate"
	} else if strings.HasPrefix(model, "WD") {
		return "Western Digital"
	} else if strings.HasPrefix(model, "MRD") {
		return "I/O Switch"
	} else if strings.HasPrefix(model, "SSD") {
		return "PNY"
	} else {
		return "Generic"
	}
}
<|endoftext|>


<commit_before>package main

import (
	"io/ioutil"
	"os"
	"path/filepath"

	"github.com/opencontainers/runtime-tools/validation/util"
)

func main() {
	g, err := util.GetDefaultGenerator()
	if err != nil {
		util.Fatal(err)
	}
	g.AddLinuxMaskedPaths("/masked-dir")
	g.AddLinuxMaskedPaths("/masked-file")
	err = util.RuntimeInsideValidate(g, func(path string) error {
		testDir := filepath.Join(path, "masked-dir")
		err = os.MkdirAll(testDir, 0777)
		if err != nil {
			return err
		}
		// create a temp file to make testDir non-empty
		tmpfile, err := ioutil.TempFile(testDir, "tmp")
		if err != nil {
			return err
		}
		defer os.Remove(tmpfile.Name())

		testFile := filepath.Join(path, "masked-file")

		// runtimetest cannot check the readability of empty files, so
		// write something.
		return ioutil.WriteFile(testFile, []byte("secrets"), 0777)
	})
	if err != nil {
		util.Fatal(err)
	}
}
<commit_msg>validation: more test cases for masked paths tests<commit_after>package main

import (
	"io/ioutil"
	"os"
	"path/filepath"

	"github.com/opencontainers/runtime-tools/validation/util"
)

func checkMaskedPaths() error {
	g, err := util.GetDefaultGenerator()
	if err != nil {
		return err
	}

	maskedDir := "masked-dir"
	maskedSubDir := "masked-subdir"
	maskedFile := "masked-file"

	maskedDirTop := filepath.Join("/", maskedDir)
	maskedFileTop := filepath.Join("/", maskedFile)

	maskedDirSub := filepath.Join(maskedDirTop, maskedSubDir)
	maskedFileSub := filepath.Join(maskedDirTop, maskedFile)
	maskedFileSubSub := filepath.Join(maskedDirSub, maskedFile)

	g.AddLinuxMaskedPaths(maskedDirTop)
	g.AddLinuxMaskedPaths(maskedFileTop)
	g.AddLinuxMaskedPaths(maskedDirSub)
	g.AddLinuxMaskedPaths(maskedFileSub)
	g.AddLinuxMaskedPaths(maskedFileSubSub)
	err = util.RuntimeInsideValidate(g, func(path string) error {
		testDir := filepath.Join(path, maskedDirSub)
		err = os.MkdirAll(testDir, 0777)
		if err != nil {
			return err
		}
		// create a temp file to make testDir non-empty
		tmpfile, err := ioutil.TempFile(testDir, "tmp")
		if err != nil {
			return err
		}
		defer os.Remove(tmpfile.Name())

		// runtimetest cannot check the readability of empty files, so
		// write something.
		testSubSubFile := filepath.Join(path, maskedFileSubSub)
		if err := ioutil.WriteFile(testSubSubFile, []byte("secrets"), 0777); err != nil {
			return err
		}

		testSubFile := filepath.Join(path, maskedFileSub)
		if err := ioutil.WriteFile(testSubFile, []byte("secrets"), 0777); err != nil {
			return err
		}

		testFile := filepath.Join(path, maskedFile)
		return ioutil.WriteFile(testFile, []byte("secrets"), 0777)
	})
	return err
}

func main() {
	if err := checkMaskedPaths(); err != nil {
		util.Fatal(err)
	}
}
<|endoftext|>


<commit_before>package cmd

import (
	"bufio"
	"bytes"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"strings"
	"text/tabwriter"
	"time"

	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/api/unversioned"
	"k8s.io/kubernetes/pkg/apis/batch"

	"github.com/gogo/protobuf/types"
	"github.com/pachyderm/pachyderm/src/client"
	"github.com/pachyderm/pachyderm/src/client/version"
	"github.com/pachyderm/pachyderm/src/client/version/versionpb"
	pfscmds "github.com/pachyderm/pachyderm/src/server/pfs/cmds"
	"github.com/pachyderm/pachyderm/src/server/pkg/cmdutil"
	deploycmds "github.com/pachyderm/pachyderm/src/server/pkg/deploy/cmds"
	"github.com/pachyderm/pachyderm/src/server/pkg/metrics"
	ppscmds "github.com/pachyderm/pachyderm/src/server/pps/cmds"

	log "github.com/Sirupsen/logrus"
	"github.com/spf13/cobra"
	"github.com/ugorji/go/codec"
	"golang.org/x/net/context"
	"golang.org/x/sync/errgroup"
	"google.golang.org/grpc"
	"google.golang.org/grpc/grpclog"
)

// PachctlCmd takes a pachd host-address and creates a cobra.Command
// which may interact with the host.
func PachctlCmd(address string) (*cobra.Command, error) {
	var verbose bool
	var noMetrics bool
	rootCmd := &cobra.Command{
		Use: os.Args[0],
		Long: `Access the Pachyderm API.

Environment variables:
  ADDRESS=<host>:<port>, the pachd server to connect to (e.g. 127.0.0.1:30650).
`,
		PersistentPreRun: func(cmd *cobra.Command, args []string) {
			if !verbose {
				// Silence any grpc logs
				l := log.New()
				l.Level = log.FatalLevel
				grpclog.SetLogger(l)
			}
		},
	}
	rootCmd.PersistentFlags().BoolVarP(&verbose, "verbose", "v", false, "Output verbose logs")
	rootCmd.PersistentFlags().BoolVarP(&noMetrics, "no-metrics", "", false, "Don't report user metrics for this command")

	pfsCmds := pfscmds.Cmds(address, &noMetrics)
	for _, cmd := range pfsCmds {
		rootCmd.AddCommand(cmd)
	}
	ppsCmds, err := ppscmds.Cmds(address, &noMetrics)
	if err != nil {
		return nil, sanitizeErr(err)
	}
	for _, cmd := range ppsCmds {
		rootCmd.AddCommand(cmd)
	}
	deployCmds := deploycmds.Cmds(&noMetrics)
	for _, cmd := range deployCmds {
		rootCmd.AddCommand(cmd)
	}

	versionCmd := &cobra.Command{
		Use:   "version",
		Short: "Return version information.",
		Long:  "Return version information.",
		Run: cmdutil.RunFixedArgs(0, func(args []string) (retErr error) {
			if !noMetrics {
				metricsFn := metrics.ReportAndFlushUserAction("Version")
				defer func(start time.Time) { metricsFn(start, retErr) }(time.Now())
			}
			writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0)
			printVersionHeader(writer)
			printVersion(writer, "pachctl", version.Version)
			writer.Flush()

			versionClient, err := getVersionAPIClient(address)
			if err != nil {
				return sanitizeErr(err)
			}
			ctx, _ := context.WithTimeout(context.Background(), time.Second)
			version, err := versionClient.GetVersion(ctx, &types.Empty{})

			if err != nil {
				buf := bytes.NewBufferString("")
				errWriter := tabwriter.NewWriter(buf, 20, 1, 3, ' ', 0)
				fmt.Fprintf(errWriter, "pachd\t(version unknown) : error connecting to pachd server at address (%v): %v\n\nplease make sure pachd is up (`kubectl get all`) and portforwarding is enabled\n", address, sanitizeErr(err))
				errWriter.Flush()
				return errors.New(buf.String())
			}

			printVersion(writer, "pachd", version)
			return writer.Flush()
		}),
	}
	deleteAll := &cobra.Command{
		Use:   "delete-all",
		Short: "Delete everything.",
		Long: `Delete all repos, commits, files, pipelines and jobs.
This resets the cluster to its initial state.`,
		Run: cmdutil.RunFixedArgs(0, func(args []string) error {
			client, err := client.NewMetricsClientFromAddress(address, !noMetrics, "user")
			if err != nil {
				return sanitizeErr(err)
			}
			fmt.Printf("Are you sure you want to delete all repos, commits, files, pipelines and jobs? yN\n")
			r := bufio.NewReader(os.Stdin)
			bytes, err := r.ReadBytes('\n')
			if err != nil {
				return err
			}
			if bytes[0] == 'y' || bytes[0] == 'Y' {
				return client.DeleteAll()
			}
			return nil
		}),
	}
	var port int
	var uiPort int
	var uiWebsocketPort int
	var kubeCtlFlags string
	portForward := &cobra.Command{
		Use:   "port-forward",
		Short: "Forward a port on the local machine to pachd. This command blocks.",
		Long:  "Forward a port on the local machine to pachd. This command blocks.",
		Run: cmdutil.RunFixedArgs(0, func(args []string) error {

			var eg errgroup.Group

			eg.Go(func() error {
				stdin := strings.NewReader(fmt.Sprintf(`
pod=$(kubectl %v get pod -l app=pachd | awk '{if (NR!=1) { print $1; exit 0 }}')
kubectl %v port-forward "$pod" %d:650
`, kubeCtlFlags, kubeCtlFlags, port))
				return cmdutil.RunIO(cmdutil.IO{
					Stdin:  stdin,
					Stderr: os.Stderr,
				}, "sh")
			})

			eg.Go(func() error {
				stdin := strings.NewReader(fmt.Sprintf(`
pod=$(kubectl %v get pod -l app=dash | awk '{if (NR!=1) { print $1; exit 0 }}')
kubectl %v port-forward "$pod" %d:8080
`, kubeCtlFlags, kubeCtlFlags, uiPort))
				if err := cmdutil.RunIO(cmdutil.IO{
					Stdin: stdin,
				}, "sh"); err != nil {
					return fmt.Errorf("UI not enabled, deploy with --dashboard")
				}
				return nil
			})

			eg.Go(func() error {
				stdin := strings.NewReader(fmt.Sprintf(`
pod=$(kubectl %v get pod -l app=dash | awk '{if (NR!=1) { print $1; exit 0 }}')
kubectl %v port-forward "$pod" %d:8081
`, kubeCtlFlags, kubeCtlFlags, uiWebsocketPort))
				cmdutil.RunIO(cmdutil.IO{
					Stdin: stdin,
				}, "sh")
				return nil
			})

			fmt.Printf("Pachd port forwarded\nDash websocket port forwarded\nDash UI port forwarded, navigate to localhost:%v\nCTRL-C to exit", uiPort)
			return eg.Wait()
		}),
	}
	portForward.Flags().IntVarP(&port, "port", "p", 30650, "The local port to bind to.")
	portForward.Flags().IntVarP(&uiPort, "ui-port", "u", 38080, "The local port to bind to.")
	portForward.Flags().IntVarP(&uiWebsocketPort, "proxy-port", "x", 38081, "The local port to bind to.")
	portForward.Flags().StringVarP(&kubeCtlFlags, "kubectlflags", "k", "", "Any kubectl flags to proxy, e.g. --kubectlflags='--kubeconfig /some/path/kubeconfig'")

	garbageCollect := &cobra.Command{
		Use:   "garbage-collect",
		Short: "Garbage collect unused data.",
		Long: `Garbage collect unused data.

When a file/commit/repo is deleted, the data is not immediately removed from the underlying storage system (e.g. S3) for performance and architectural reasons.  This is similar to how when you delete a file on your computer, the file is not necessarily wiped from disk immediately.

To actually remove the data, you will need to manually invoke garbage collection.  The easiest way to do it is through "pachctl garbage-collecth".

Currently "pachctl garbage-collect" can only be started when there are no active jobs running.  You also need to ensure that there's no ongoing "put-file".  Garbage collection puts the cluster into a readonly mode where no new jobs can be created and no data can be added.
`,
		Run: cmdutil.RunFixedArgs(0, func(args []string) (retErr error) {
			client, err := client.NewMetricsClientFromAddress(address, !noMetrics, "user")
			if err != nil {
				return err
			}

			return client.GarbageCollect()
		}),
	}

	var from, to, namespace string
	migrate := &cobra.Command{
		Use:   "migrate",
		Short: "Migrate the internal state of Pachyderm from one version to another.",
		Long: `Migrate the internal state of Pachyderm from one version to
another.  Note that most of the time updating Pachyderm doesn't
require a migration.  Refer to the docs for your specific version
to find out if it requires a migration.

It's highly recommended that you only run migrations when there are no
activities in your cluster, e.g. no jobs should be running.

The migration command takes the general form:

$ pachctl migrate --from <FROM_VERSION> --to <TO_VERSION>

If "--from" is not provided, pachctl will attempt to discover the current
version of the cluster.  If "--to" is not provided, pachctl will use the
version of pachctl itself.

Example:

# Migrate Pachyderm from 1.4.8 to 1.5.0
$ pachctl migrate --from 1.4.8 --to 1.5.0
`,
		Run: cmdutil.RunFixedArgs(0, func(args []string) (retErr error) {
			jobSpec := batch.Job{
				TypeMeta: unversioned.TypeMeta{
					Kind:       "Job",
					APIVersion: "batch/v1",
				},
				ObjectMeta: api.ObjectMeta{
					Name: "pach-migration",
					Labels: map[string]string{
						"app": "pachyderm",
					},
				},
				Spec: batch.JobSpec{
					Template: api.PodTemplateSpec{
						Spec: api.PodSpec{
							Containers: []api.Container{
								{
									Name:    "migration",
									Image:   fmt.Sprintf("pachyderm/pachd:%v", version.PrettyPrintVersion(version.Version)),
									Command: []string{"/pachd", fmt.Sprintf("--migrate=%v-%v", from, to)},
								},
							},
							RestartPolicy: "OnFailure",
						},
					},
				},
			}

			tmpFile, err := ioutil.TempFile("", "")
			if err != nil {
				return err
			}
			defer os.Remove(tmpFile.Name())

			jsonEncoderHandle := &codec.JsonHandle{
				BasicHandle: codec.BasicHandle{
					EncodeOptions: codec.EncodeOptions{Canonical: true},
				},
				Indent: 2,
			}
			encoder := codec.NewEncoder(tmpFile, jsonEncoderHandle)
			jobSpec.CodecEncodeSelf(encoder)
			tmpFile.Close()

			cmd := exec.Command("kubectl", "create", "--validate=false", "-f", tmpFile.Name())
			out, err := cmd.CombinedOutput()
			fmt.Println(string(out))
			if err != nil {
				return err
			}
			fmt.Println("Successfully launched migration.  To see the progress, use `kubectl logs <job ID>`")
			return nil
		}),
	}
	migrate.Flags().StringVar(&from, "from", "", "The current version of the cluster.  If not specified, pachctl will attempt to discover the version of the cluster.")
	migrate.Flags().StringVar(&to, "to", "", "The version of Pachyderm to migrate to.  If not specified, pachctl will use its own version.")
	migrate.Flags().StringVar(&namespace, "namespace", "default", "The kubernetes namespace under which Pachyderm is deployed.")

	rootCmd.AddCommand(versionCmd)
	rootCmd.AddCommand(deleteAll)
	rootCmd.AddCommand(portForward)
	rootCmd.AddCommand(garbageCollect)
	rootCmd.AddCommand(migrate)
	return rootCmd, nil
}

func getVersionAPIClient(address string) (versionpb.APIClient, error) {
	clientConn, err := grpc.Dial(address, client.PachDialOptions()...)
	if err != nil {
		return nil, err
	}
	return versionpb.NewAPIClient(clientConn), nil
}

func printVersionHeader(w io.Writer) {
	fmt.Fprintf(w, "COMPONENT\tVERSION\t\n")
}

func printVersion(w io.Writer, component string, v *versionpb.Version) {
	fmt.Fprintf(w, "%s\t%s\t\n", component, version.PrettyPrintVersion(v))
}

func sanitizeErr(err error) error {
	if err == nil {
		return nil
	}

	return errors.New(grpc.ErrorDesc(err))
}
<commit_msg>Label the migration job correctly<commit_after>package cmd

import (
	"bufio"
	"bytes"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"strings"
	"text/tabwriter"
	"time"

	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/api/unversioned"
	"k8s.io/kubernetes/pkg/apis/batch"

	"github.com/gogo/protobuf/types"
	"github.com/pachyderm/pachyderm/src/client"
	"github.com/pachyderm/pachyderm/src/client/version"
	"github.com/pachyderm/pachyderm/src/client/version/versionpb"
	pfscmds "github.com/pachyderm/pachyderm/src/server/pfs/cmds"
	"github.com/pachyderm/pachyderm/src/server/pkg/cmdutil"
	deploycmds "github.com/pachyderm/pachyderm/src/server/pkg/deploy/cmds"
	"github.com/pachyderm/pachyderm/src/server/pkg/metrics"
	ppscmds "github.com/pachyderm/pachyderm/src/server/pps/cmds"

	log "github.com/Sirupsen/logrus"
	"github.com/spf13/cobra"
	"github.com/ugorji/go/codec"
	"golang.org/x/net/context"
	"golang.org/x/sync/errgroup"
	"google.golang.org/grpc"
	"google.golang.org/grpc/grpclog"
)

// PachctlCmd takes a pachd host-address and creates a cobra.Command
// which may interact with the host.
func PachctlCmd(address string) (*cobra.Command, error) {
	var verbose bool
	var noMetrics bool
	rootCmd := &cobra.Command{
		Use: os.Args[0],
		Long: `Access the Pachyderm API.

Environment variables:
  ADDRESS=<host>:<port>, the pachd server to connect to (e.g. 127.0.0.1:30650).
`,
		PersistentPreRun: func(cmd *cobra.Command, args []string) {
			if !verbose {
				// Silence any grpc logs
				l := log.New()
				l.Level = log.FatalLevel
				grpclog.SetLogger(l)
			}
		},
	}
	rootCmd.PersistentFlags().BoolVarP(&verbose, "verbose", "v", false, "Output verbose logs")
	rootCmd.PersistentFlags().BoolVarP(&noMetrics, "no-metrics", "", false, "Don't report user metrics for this command")

	pfsCmds := pfscmds.Cmds(address, &noMetrics)
	for _, cmd := range pfsCmds {
		rootCmd.AddCommand(cmd)
	}
	ppsCmds, err := ppscmds.Cmds(address, &noMetrics)
	if err != nil {
		return nil, sanitizeErr(err)
	}
	for _, cmd := range ppsCmds {
		rootCmd.AddCommand(cmd)
	}
	deployCmds := deploycmds.Cmds(&noMetrics)
	for _, cmd := range deployCmds {
		rootCmd.AddCommand(cmd)
	}

	versionCmd := &cobra.Command{
		Use:   "version",
		Short: "Return version information.",
		Long:  "Return version information.",
		Run: cmdutil.RunFixedArgs(0, func(args []string) (retErr error) {
			if !noMetrics {
				metricsFn := metrics.ReportAndFlushUserAction("Version")
				defer func(start time.Time) { metricsFn(start, retErr) }(time.Now())
			}
			writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0)
			printVersionHeader(writer)
			printVersion(writer, "pachctl", version.Version)
			writer.Flush()

			versionClient, err := getVersionAPIClient(address)
			if err != nil {
				return sanitizeErr(err)
			}
			ctx, _ := context.WithTimeout(context.Background(), time.Second)
			version, err := versionClient.GetVersion(ctx, &types.Empty{})

			if err != nil {
				buf := bytes.NewBufferString("")
				errWriter := tabwriter.NewWriter(buf, 20, 1, 3, ' ', 0)
				fmt.Fprintf(errWriter, "pachd\t(version unknown) : error connecting to pachd server at address (%v): %v\n\nplease make sure pachd is up (`kubectl get all`) and portforwarding is enabled\n", address, sanitizeErr(err))
				errWriter.Flush()
				return errors.New(buf.String())
			}

			printVersion(writer, "pachd", version)
			return writer.Flush()
		}),
	}
	deleteAll := &cobra.Command{
		Use:   "delete-all",
		Short: "Delete everything.",
		Long: `Delete all repos, commits, files, pipelines and jobs.
This resets the cluster to its initial state.`,
		Run: cmdutil.RunFixedArgs(0, func(args []string) error {
			client, err := client.NewMetricsClientFromAddress(address, !noMetrics, "user")
			if err != nil {
				return sanitizeErr(err)
			}
			fmt.Printf("Are you sure you want to delete all repos, commits, files, pipelines and jobs? yN\n")
			r := bufio.NewReader(os.Stdin)
			bytes, err := r.ReadBytes('\n')
			if err != nil {
				return err
			}
			if bytes[0] == 'y' || bytes[0] == 'Y' {
				return client.DeleteAll()
			}
			return nil
		}),
	}
	var port int
	var uiPort int
	var uiWebsocketPort int
	var kubeCtlFlags string
	portForward := &cobra.Command{
		Use:   "port-forward",
		Short: "Forward a port on the local machine to pachd. This command blocks.",
		Long:  "Forward a port on the local machine to pachd. This command blocks.",
		Run: cmdutil.RunFixedArgs(0, func(args []string) error {

			var eg errgroup.Group

			eg.Go(func() error {
				stdin := strings.NewReader(fmt.Sprintf(`
pod=$(kubectl %v get pod -l app=pachd | awk '{if (NR!=1) { print $1; exit 0 }}')
kubectl %v port-forward "$pod" %d:650
`, kubeCtlFlags, kubeCtlFlags, port))
				return cmdutil.RunIO(cmdutil.IO{
					Stdin:  stdin,
					Stderr: os.Stderr,
				}, "sh")
			})

			eg.Go(func() error {
				stdin := strings.NewReader(fmt.Sprintf(`
pod=$(kubectl %v get pod -l app=dash | awk '{if (NR!=1) { print $1; exit 0 }}')
kubectl %v port-forward "$pod" %d:8080
`, kubeCtlFlags, kubeCtlFlags, uiPort))
				if err := cmdutil.RunIO(cmdutil.IO{
					Stdin: stdin,
				}, "sh"); err != nil {
					return fmt.Errorf("UI not enabled, deploy with --dashboard")
				}
				return nil
			})

			eg.Go(func() error {
				stdin := strings.NewReader(fmt.Sprintf(`
pod=$(kubectl %v get pod -l app=dash | awk '{if (NR!=1) { print $1; exit 0 }}')
kubectl %v port-forward "$pod" %d:8081
`, kubeCtlFlags, kubeCtlFlags, uiWebsocketPort))
				cmdutil.RunIO(cmdutil.IO{
					Stdin: stdin,
				}, "sh")
				return nil
			})

			fmt.Printf("Pachd port forwarded\nDash websocket port forwarded\nDash UI port forwarded, navigate to localhost:%v\nCTRL-C to exit", uiPort)
			return eg.Wait()
		}),
	}
	portForward.Flags().IntVarP(&port, "port", "p", 30650, "The local port to bind to.")
	portForward.Flags().IntVarP(&uiPort, "ui-port", "u", 38080, "The local port to bind to.")
	portForward.Flags().IntVarP(&uiWebsocketPort, "proxy-port", "x", 38081, "The local port to bind to.")
	portForward.Flags().StringVarP(&kubeCtlFlags, "kubectlflags", "k", "", "Any kubectl flags to proxy, e.g. --kubectlflags='--kubeconfig /some/path/kubeconfig'")

	garbageCollect := &cobra.Command{
		Use:   "garbage-collect",
		Short: "Garbage collect unused data.",
		Long: `Garbage collect unused data.

When a file/commit/repo is deleted, the data is not immediately removed from the underlying storage system (e.g. S3) for performance and architectural reasons.  This is similar to how when you delete a file on your computer, the file is not necessarily wiped from disk immediately.

To actually remove the data, you will need to manually invoke garbage collection.  The easiest way to do it is through "pachctl garbage-collecth".

Currently "pachctl garbage-collect" can only be started when there are no active jobs running.  You also need to ensure that there's no ongoing "put-file".  Garbage collection puts the cluster into a readonly mode where no new jobs can be created and no data can be added.
`,
		Run: cmdutil.RunFixedArgs(0, func(args []string) (retErr error) {
			client, err := client.NewMetricsClientFromAddress(address, !noMetrics, "user")
			if err != nil {
				return err
			}

			return client.GarbageCollect()
		}),
	}

	var from, to, namespace string
	migrate := &cobra.Command{
		Use:   "migrate",
		Short: "Migrate the internal state of Pachyderm from one version to another.",
		Long: `Migrate the internal state of Pachyderm from one version to
another.  Note that most of the time updating Pachyderm doesn't
require a migration.  Refer to the docs for your specific version
to find out if it requires a migration.

It's highly recommended that you only run migrations when there are no
activities in your cluster, e.g. no jobs should be running.

The migration command takes the general form:

$ pachctl migrate --from <FROM_VERSION> --to <TO_VERSION>

If "--from" is not provided, pachctl will attempt to discover the current
version of the cluster.  If "--to" is not provided, pachctl will use the
version of pachctl itself.

Example:

# Migrate Pachyderm from 1.4.8 to 1.5.0
$ pachctl migrate --from 1.4.8 --to 1.5.0
`,
		Run: cmdutil.RunFixedArgs(0, func(args []string) (retErr error) {
			jobSpec := batch.Job{
				TypeMeta: unversioned.TypeMeta{
					Kind:       "Job",
					APIVersion: "batch/v1",
				},
				ObjectMeta: api.ObjectMeta{
					Name: "pach-migration",
					Labels: map[string]string{
						"suite": "pachyderm",
					},
				},
				Spec: batch.JobSpec{
					Template: api.PodTemplateSpec{
						Spec: api.PodSpec{
							Containers: []api.Container{
								{
									Name:    "migration",
									Image:   fmt.Sprintf("pachyderm/pachd:%v", version.PrettyPrintVersion(version.Version)),
									Command: []string{"/pachd", fmt.Sprintf("--migrate=%v-%v", from, to)},
								},
							},
							RestartPolicy: "OnFailure",
						},
					},
				},
			}

			tmpFile, err := ioutil.TempFile("", "")
			if err != nil {
				return err
			}
			defer os.Remove(tmpFile.Name())

			jsonEncoderHandle := &codec.JsonHandle{
				BasicHandle: codec.BasicHandle{
					EncodeOptions: codec.EncodeOptions{Canonical: true},
				},
				Indent: 2,
			}
			encoder := codec.NewEncoder(tmpFile, jsonEncoderHandle)
			jobSpec.CodecEncodeSelf(encoder)
			tmpFile.Close()

			cmd := exec.Command("kubectl", "create", "--validate=false", "-f", tmpFile.Name())
			out, err := cmd.CombinedOutput()
			fmt.Println(string(out))
			if err != nil {
				return err
			}
			fmt.Println("Successfully launched migration.  To see the progress, use `kubectl logs <job ID>`")
			return nil
		}),
	}
	migrate.Flags().StringVar(&from, "from", "", "The current version of the cluster.  If not specified, pachctl will attempt to discover the version of the cluster.")
	migrate.Flags().StringVar(&to, "to", "", "The version of Pachyderm to migrate to.  If not specified, pachctl will use its own version.")
	migrate.Flags().StringVar(&namespace, "namespace", "default", "The kubernetes namespace under which Pachyderm is deployed.")

	rootCmd.AddCommand(versionCmd)
	rootCmd.AddCommand(deleteAll)
	rootCmd.AddCommand(portForward)
	rootCmd.AddCommand(garbageCollect)
	rootCmd.AddCommand(migrate)
	return rootCmd, nil
}

func getVersionAPIClient(address string) (versionpb.APIClient, error) {
	clientConn, err := grpc.Dial(address, client.PachDialOptions()...)
	if err != nil {
		return nil, err
	}
	return versionpb.NewAPIClient(clientConn), nil
}

func printVersionHeader(w io.Writer) {
	fmt.Fprintf(w, "COMPONENT\tVERSION\t\n")
}

func printVersion(w io.Writer, component string, v *versionpb.Version) {
	fmt.Fprintf(w, "%s\t%s\t\n", component, version.PrettyPrintVersion(v))
}

func sanitizeErr(err error) error {
	if err == nil {
		return nil
	}

	return errors.New(grpc.ErrorDesc(err))
}
<|endoftext|>


<commit_before>package metrics

import (
	"fmt"
	"time"

	"github.com/pachyderm/pachyderm/src/client/pkg/uuid"
	"github.com/pachyderm/pachyderm/src/client/version"
	db "github.com/pachyderm/pachyderm/src/server/pfs/db"

	"github.com/dancannon/gorethink"
	"go.pedge.io/lion/proto"
	"golang.org/x/net/context"
	"google.golang.org/grpc/metadata"
	kube "k8s.io/kubernetes/pkg/client/unversioned"
)

type Reporter struct {
	clusterID  string
	kubeClient *kube.Client
	dbClient   *gorethink.Session
	pfsDbName  string
	ppsDbName  string
}

func NewReporter(clusterID string, kubeClient *kube.Client, address string, pfsDbName string, ppsDbName string) (*Reporter, error) {
	dbClient, err := db.DbConnect(address)
	if err != nil {
		return nil, fmt.Errorf("Error connected to DB when reporting metrics: %v\n", err)
	}
	return &Reporter{
		clusterID:  clusterID,
		kubeClient: kubeClient,
		dbClient:   dbClient,
		pfsDbName:  pfsDbName,
		ppsDbName:  ppsDbName,
	}, nil
}

// Segment API allows for map[string]interface{} for a single user's traits
// But we only care about things that are countable for the moment
// map userID -> action name -> count
type countableActions map[string]interface{}
type countableUserActions map[string]countableActions

type incrementUserAction struct {
	action string
	user   string
}

var userActions = make(countableUserActions)
var incrementActionChannel = make(chan *incrementUserAction, 0)

//IncrementUserAction updates a counter per user per action for an API method by name
func IncrementUserAction(ctx context.Context, action string) {
	fmt.Printf("!!! trying to increment user actionw ctx: [%v]\n", ctx)
	md, ok := metadata.FromContext(ctx)
	fmt.Printf("!!! metadata: %v\n", md)
	if ok && md["userid"] != nil && len(md["userid"]) > 0 {
		userID := md["userid"][0]
		fmt.Printf("!!! incrementing user action: %v, %v\n", userID, action)
		incrementActionChannel <- &incrementUserAction{
			action: action,
			user:   userID,
		}
		fmt.Printf("!!! incremented user action!\n")
	}
}

func (r *Reporter) dbMetrics(metrics *Metrics) {
	cursor, err := gorethink.Object(
		"Repos",
		gorethink.DB(r.pfsDbName).Table("Repos").Count(),
		"Commits",
		gorethink.DB(r.pfsDbName).Table("Commits").Count(),
		"ArchivedCommits",
		gorethink.DB(r.pfsDbName).Table("Commits").Filter(
			map[string]interface{}{
				"Archived": true,
			},
		).Count(),
		"CancelledCommits",
		gorethink.DB(r.pfsDbName).Table("Commits").Filter(
			map[string]interface{}{
				"Cancelled": true,
			},
		).Count(),
		"Files",
		gorethink.DB(r.pfsDbName).Table("Diffs").Group("Path").Ungroup().Count(),
		"Jobs",
		gorethink.DB(r.ppsDbName).Table("JobInfos").Count(),
		"Pipelines",
		gorethink.DB(r.ppsDbName).Table("PipelineInfos").Count(),
	).Run(r.dbClient)
	if err != nil {
		protolion.Errorf("Error Fetching Metrics:%+v", err)
	}
	cursor.One(&metrics)
}

// ReportMetrics blocks and reports metrics every 15 seconds
func (r *Reporter) ReportMetrics() {
	reportingTicker := time.NewTicker(time.Second * 15)
	for {
		select {
		case incrementAction := <-incrementActionChannel:
			fmt.Printf("incrementing action in map!\n")
			if userActions[incrementAction.user] == nil {
				userActions[incrementAction.user] = make(countableActions)
			}
			val, ok := userActions[incrementAction.user][incrementAction.action]
			if ok {
				userActions[incrementAction.user][incrementAction.action] = val.(uint64) + uint64(1)
			} else {
				userActions[incrementAction.user][incrementAction.action] = uint64(0)
			}
			break
		case <-reportingTicker.C:
			fmt.Printf("!!! TICK - reporting to segment\n")
			r.reportToSegment()
			fmt.Printf("!!! kicked off segment reporting\n")
		}
	}
}

func (r *Reporter) reportToSegment() {
	fmt.Printf("!!! Reporting to segment, user actions: [%v]\n", userActions)
	if len(userActions) > 0 {
		batchOfUserActions := make(countableUserActions)
		// copy the existing stats into a new object so we can make the segment
		// request asynchronously
		for user, actions := range userActions {
			singleUserActions := make(countableActions)
			for name, count := range actions {
				singleUserActions[name] = count
			}
			batchOfUserActions[user] = singleUserActions
		}
		go r.reportUserMetrics(batchOfUserActions)
		userActions = make(countableUserActions)
	}
	go r.reportClusterMetrics()
}

func (r *Reporter) reportUserMetrics(batchOfUserActions countableUserActions) {
	if len(batchOfUserActions) > 0 {
		reportUserMetricsToSegment(batchOfUserActions)
	}
}

func (r *Reporter) reportClusterMetrics() {
	metrics := &Metrics{}
	r.dbMetrics(metrics)
	externalMetrics(r.kubeClient, metrics)
	metrics.ID = r.clusterID
	metrics.PodID = uuid.NewWithoutDashes()
	metrics.Version = version.PrettyPrintVersion(version.Version)
	reportClusterMetricsToSegment(metrics)
}
<commit_msg>Don't try to increment metrics if they're disabled<commit_after>package metrics

import (
	"fmt"
	"time"

	"github.com/pachyderm/pachyderm/src/client/pkg/uuid"
	"github.com/pachyderm/pachyderm/src/client/version"
	db "github.com/pachyderm/pachyderm/src/server/pfs/db"

	"github.com/dancannon/gorethink"
	"go.pedge.io/lion/proto"
	"golang.org/x/net/context"
	"google.golang.org/grpc/metadata"
	kube "k8s.io/kubernetes/pkg/client/unversioned"
)

type Reporter struct {
	clusterID  string
	kubeClient *kube.Client
	dbClient   *gorethink.Session
	pfsDbName  string
	ppsDbName  string
}

func NewReporter(clusterID string, kubeClient *kube.Client, address string, pfsDbName string, ppsDbName string) (*Reporter, error) {
	dbClient, err := db.DbConnect(address)
	if err != nil {
		return nil, fmt.Errorf("Error connected to DB when reporting metrics: %v\n", err)
	}
	return &Reporter{
		clusterID:  clusterID,
		kubeClient: kubeClient,
		dbClient:   dbClient,
		pfsDbName:  pfsDbName,
		ppsDbName:  ppsDbName,
	}, nil
}

// If we're not reporting metrics, incrementing should do nothing
var metricsEnabled = false

// Segment API allows for map[string]interface{} for a single user's traits
// But we only care about things that are countable for the moment
// map userID -> action name -> count
type countableActions map[string]interface{}
type countableUserActions map[string]countableActions

type incrementUserAction struct {
	action string
	user   string
}

var userActions = make(countableUserActions)
var incrementActionChannel = make(chan *incrementUserAction, 0)

//IncrementUserAction updates a counter per user per action for an API method by name
func IncrementUserAction(ctx context.Context, action string) {
	if !metricsEnabled {
		return
	}
	fmt.Printf("!!! trying to increment user actionw ctx: [%v]\n", ctx)
	md, ok := metadata.FromContext(ctx)
	fmt.Printf("!!! metadata: %v\n", md)
	if ok && md["userid"] != nil && len(md["userid"]) > 0 {
		userID := md["userid"][0]
		fmt.Printf("!!! incrementing user action: %v, %v\n", userID, action)
		incrementActionChannel <- &incrementUserAction{
			action: action,
			user:   userID,
		}
		fmt.Printf("!!! incremented user action!\n")
	}
}

func (r *Reporter) dbMetrics(metrics *Metrics) {
	cursor, err := gorethink.Object(
		"Repos",
		gorethink.DB(r.pfsDbName).Table("Repos").Count(),
		"Commits",
		gorethink.DB(r.pfsDbName).Table("Commits").Count(),
		"ArchivedCommits",
		gorethink.DB(r.pfsDbName).Table("Commits").Filter(
			map[string]interface{}{
				"Archived": true,
			},
		).Count(),
		"CancelledCommits",
		gorethink.DB(r.pfsDbName).Table("Commits").Filter(
			map[string]interface{}{
				"Cancelled": true,
			},
		).Count(),
		"Files",
		gorethink.DB(r.pfsDbName).Table("Diffs").Group("Path").Ungroup().Count(),
		"Jobs",
		gorethink.DB(r.ppsDbName).Table("JobInfos").Count(),
		"Pipelines",
		gorethink.DB(r.ppsDbName).Table("PipelineInfos").Count(),
	).Run(r.dbClient)
	if err != nil {
		protolion.Errorf("Error Fetching Metrics:%+v", err)
	}
	cursor.One(&metrics)
}

// ReportMetrics blocks and reports metrics every 15 seconds
func (r *Reporter) ReportMetrics() {
	metricsEnabled = true
	reportingTicker := time.NewTicker(time.Second * 15)
	for {
		select {
		case incrementAction := <-incrementActionChannel:
			fmt.Printf("incrementing action in map!\n")
			if userActions[incrementAction.user] == nil {
				userActions[incrementAction.user] = make(countableActions)
			}
			val, ok := userActions[incrementAction.user][incrementAction.action]
			if ok {
				userActions[incrementAction.user][incrementAction.action] = val.(uint64) + uint64(1)
			} else {
				userActions[incrementAction.user][incrementAction.action] = uint64(0)
			}
			break
		case <-reportingTicker.C:
			fmt.Printf("!!! TICK - reporting to segment\n")
			r.reportToSegment()
			fmt.Printf("!!! kicked off segment reporting\n")
		}
	}
}

func (r *Reporter) reportToSegment() {
	fmt.Printf("!!! Reporting to segment, user actions: [%v]\n", userActions)
	if len(userActions) > 0 {
		batchOfUserActions := make(countableUserActions)
		// copy the existing stats into a new object so we can make the segment
		// request asynchronously
		for user, actions := range userActions {
			singleUserActions := make(countableActions)
			for name, count := range actions {
				singleUserActions[name] = count
			}
			batchOfUserActions[user] = singleUserActions
			go identifyUser(user)
		}
		go r.reportUserMetrics(batchOfUserActions)
		userActions = make(countableUserActions)
	}
	go r.reportClusterMetrics()
}

func (r *Reporter) reportUserMetrics(batchOfUserActions countableUserActions) {
	if len(batchOfUserActions) > 0 {
		reportUserMetricsToSegment(batchOfUserActions)
	}
}

func (r *Reporter) reportClusterMetrics() {
	metrics := &Metrics{}
	r.dbMetrics(metrics)
	externalMetrics(r.kubeClient, metrics)
	metrics.ID = r.clusterID
	metrics.PodID = uuid.NewWithoutDashes()
	metrics.Version = version.PrettyPrintVersion(version.Version)
	reportClusterMetricsToSegment(metrics)
}
<|endoftext|>


<commit_before>// Copyright 2015 Davis Webb
// Copyright 2015 Luke Shumaker

package handlers

import (
	"github.com/jinzhu/gorm"
	"io"
	"io/ioutil"
	"net/mail"
	"net/smtp"
	"periwinkle/store"
)

func HandleEmail(r io.Reader, name string, db *gorm.DB) int {

	msg, err := mail.ReadMessage(r)
	if err != nil {
		panic(err)
	}
	header := msg.Header

	group := store.GetGroupById(db, name)
	// TODO: check if group == nil
	address_ids := make([]int64, len(group.Subscriptions))
	for i := range group.Subscriptions {
		address_ids[i] = group.Subscriptions[i].AddressId
	}
	var address_list []store.UserAddress
	db.Where("id in (?)", address_ids).Find(&address_list)
	// TODO: error handling
	// convert forward_ary into a set
	forward_set := make(map[string]bool, len(address_list))
	for _, addr := range address_list {
		var str string
		if addr.Medium == "email" {
			str = addr.Address
		} else {
			str = addr.Address + "@" + addr.Medium + ".gateway"
		}
		forward_set[str] = true
	}
	/////////////////////////////////////////////////////////////////////
	addresses, err := mail.ParseAddressList(header.Get("To"))
	if err != nil {
		panic(err)
	}
	for _, addr := range addresses {
		delete(forward_set, addr.Address)
	}
	/////////////////////////////////////////////////////////////////////
	addresses, err = mail.ParseAddressList(header.Get("From"))
	if err != nil {
		panic(err)
	}
	for _, addr := range addresses {
		delete(forward_set, addr.Address)
	}
	/////////////////////////////////////////////////////////////////////
	addresses, err = mail.ParseAddressList(header.Get("Cc"))
	if err != nil {
		panic(err)
	}
	for _, addr := range addresses {
		delete(forward_set, addr.Address)
	}
	// TODO: remove things from the set
	// to add something:
	//     forward_set[something] = true
	// to remove something:
	//     delete(forward_set, something)
	// convert the set back in to an array
	forward_ary := make([]string, len(forward_set))
	i := uint(0)
	for addr := range forward_set {
		forward_ary[i] = addr
		i++
	}
	// msg.Bcc = format_to
	auth := smtp.PlainAuth("", "", "", "")
	from := header.Get("From")
	body, _ := ioutil.ReadAll(msg.Body)
	err = smtp.SendMail("localhost:25", auth, from, forward_ary, body)
	if err != nil {
		panic(err)
	}
	return 0
}
<commit_msg>DavisLWebb - HandleEmail<commit_after>// Copyright 2015 Davis Webb
// Copyright 2015 Luke Shumaker

package handlers

import (
	"github.com/jinzhu/gorm"
	"io"
	"io/ioutil"
	"net/mail"
	"net/smtp"
	"periwinkle/store"
)

func HandleEmail(r io.Reader, name string, db *gorm.DB) int {

	msg, err := mail.ReadMessage(r)

	if err != nil {
		panic(err)
	}
	header := msg.Header

	group := store.GetGroupById(db, name)
	if group == nil {
		panic("No group: " + name)
	}
	// TODO: check if group == nil
	address_ids := make([]int64, len(group.Subscriptions))
	for i := range group.Subscriptions {
		address_ids[i] = group.Subscriptions[i].AddressId
	}
	var address_list []store.UserAddress
	db.Where("id in (?)", address_ids).Find(&address_list)
	// TODO: error handling
	// convert forward_ary into a set
	forward_set := make(map[string]bool, len(address_list))
	for _, addr := range address_list {
		var str string
		if addr.Medium == "email" {
			str = addr.Address
		} else {
			str = addr.Address + "@" + addr.Medium + ".gateway"
		}
		forward_set[str] = true
	}
	/////////////////////////////////////////////////////////////////////
	addresses, err := mail.ParseAddressList(header.Get("To"))
	if err != nil {
		panic(err)
	}
	for _, addr := range addresses {
		delete(forward_set, addr.Address)
	}
	/////////////////////////////////////////////////////////////////////
	addresses, err = mail.ParseAddressList(header.Get("From"))
	if err != nil {
		panic(err)
	}
	for _, addr := range addresses {
		delete(forward_set, addr.Address)
	}
	/////////////////////////////////////////////////////////////////////
	addresses, err = mail.ParseAddressList(header.Get("Cc"))
	if err != nil {
		panic(err)
	}
	for _, addr := range addresses {
		delete(forward_set, addr.Address)
	}

	forward_ary := make([]string, len(forward_set))
	i := uint(0)
	for addr := range forward_set {
		forward_ary[i] = addr
		i++
	}

	auth := smtp.PlainAuth("", "", "", "")
	from := header.Get("From")
	body, _ := ioutil.ReadAll(msg.Body)
	err = smtp.SendMail("localhost:25", auth, from, forward_ary, body)
	if err != nil {
		panic(err)
	}
	return 0
}
<|endoftext|>


<commit_before>// Copyright 2014 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package debug

import (
	"io/ioutil"
	"os"
	"testing"
)

func TestWriteHeapDumpNonempty(t *testing.T) {
	f, err := ioutil.TempFile("", "heapdumptest")
	if err != nil {
		t.Fatalf("TempFile failed: %v", err)
	}
	defer os.Remove(f.Name())
	defer f.Close()
	WriteHeapDump(f.Fd())
	fi, err := f.Stat()
	if err != nil {
		t.Fatalf("Stat failed: %v", err)
	}
	const minSize = 1
	if size := fi.Size(); size < minSize {
		t.Fatalf("Heap dump size %d bytes, expected at least %d bytes", size, minSize)
	}
}
<commit_msg>runtime/debug: skip TestWriteHeapDumpNonempty on NaCl. TestWriteHeap is useless on NaCl anyway.<commit_after>// Copyright 2014 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package debug

import (
	"io/ioutil"
	"os"
	"runtime"
	"testing"
)

func TestWriteHeapDumpNonempty(t *testing.T) {
	if runtime.GOOS == "nacl" {
		t.Skip("WriteHeapDump is not available on NaCl.")
	}
	f, err := ioutil.TempFile("", "heapdumptest")
	if err != nil {
		t.Fatalf("TempFile failed: %v", err)
	}
	defer os.Remove(f.Name())
	defer f.Close()
	WriteHeapDump(f.Fd())
	fi, err := f.Stat()
	if err != nil {
		t.Fatalf("Stat failed: %v", err)
	}
	const minSize = 1
	if size := fi.Size(); size < minSize {
		t.Fatalf("Heap dump size %d bytes, expected at least %d bytes", size, minSize)
	}
}
<|endoftext|>


<commit_before>package network

import (
	"net"

	"github.com/lxc/lxd/lxd/cluster"
	"github.com/lxc/lxd/lxd/cluster/request"
	"github.com/lxc/lxd/lxd/db"
	"github.com/lxc/lxd/lxd/state"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/api"
)

// Type represents a LXD network driver type.
type Type interface {
	FillConfig(config map[string]string) error
	Info() Info
	ValidateName(name string) error
	Type() string
	DBType() db.NetworkType
}

// Network represents an instantiated LXD network.
type Network interface {
	Type

	// Load.
	init(state *state.State, id int64, projectName string, netInfo *api.Network, netNodes map[int64]db.NetworkNode)

	// Config.
	Validate(config map[string]string) error
	ID() int64
	Name() string
	Project() string
	Description() string
	Status() string
	LocalStatus() string
	Config() map[string]string
	IsUsed() (bool, error)
	IsManaged() bool
	DHCPv4Subnet() *net.IPNet
	DHCPv6Subnet() *net.IPNet
	DHCPv4Ranges() []shared.IPRange
	DHCPv6Ranges() []shared.IPRange

	// Actions.
	Create(clientType request.ClientType) error
	Start() error
	Stop() error
	Rename(name string) error
	Update(newNetwork api.NetworkPut, targetNode string, clientType request.ClientType) error
	HandleHeartbeat(heartbeatData *cluster.APIHeartbeat) error
	Delete(clientType request.ClientType) error
	handleDependencyChange(netName string, netConfig map[string]string, changedKeys []string) error
}
<commit_msg>lxd/network/network/interface: Adds network forward management definitions<commit_after>package network

import (
	"net"

	"github.com/lxc/lxd/lxd/cluster"
	"github.com/lxc/lxd/lxd/cluster/request"
	"github.com/lxc/lxd/lxd/db"
	"github.com/lxc/lxd/lxd/state"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/api"
)

// Type represents a LXD network driver type.
type Type interface {
	FillConfig(config map[string]string) error
	Info() Info
	ValidateName(name string) error
	Type() string
	DBType() db.NetworkType
}

// Network represents an instantiated LXD network.
type Network interface {
	Type

	// Load.
	init(state *state.State, id int64, projectName string, netInfo *api.Network, netNodes map[int64]db.NetworkNode)

	// Config.
	Validate(config map[string]string) error
	ID() int64
	Name() string
	Project() string
	Description() string
	Status() string
	LocalStatus() string
	Config() map[string]string
	IsUsed() (bool, error)
	IsManaged() bool
	DHCPv4Subnet() *net.IPNet
	DHCPv6Subnet() *net.IPNet
	DHCPv4Ranges() []shared.IPRange
	DHCPv6Ranges() []shared.IPRange

	// Actions.
	Create(clientType request.ClientType) error
	Start() error
	Stop() error
	Rename(name string) error
	Update(newNetwork api.NetworkPut, targetNode string, clientType request.ClientType) error
	HandleHeartbeat(heartbeatData *cluster.APIHeartbeat) error
	Delete(clientType request.ClientType) error
	handleDependencyChange(netName string, netConfig map[string]string, changedKeys []string) error

	// Address Forwards.
	ForwardCreate(forward api.NetworkForwardsPost) error
	ForwardUpdate(listenAddress string, newForward api.NetworkForwardPut) error
	ForwardDelete(listenAddress string) error
}
<|endoftext|>