<commit_before>package net

import (
	"bytes"
	"crypto/tls"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"net/url"
	"os"
	"runtime"
	"strings"
	"time"

	"github.com/cloudfoundry/cli/cf"
	"github.com/cloudfoundry/cli/cf/configuration/core_config"
	"github.com/cloudfoundry/cli/cf/errors"
	. "github.com/cloudfoundry/cli/cf/i18n"
	"github.com/cloudfoundry/cli/cf/terminal"
)

const (
	JOB_FINISHED             = "finished"
	JOB_FAILED               = "failed"
	DEFAULT_POLLING_THROTTLE = 5 * time.Second
)

type JobResource struct {
	Entity struct {
		Status       string
		ErrorDetails struct {
			Description string
		} `json:"error_details"`
	}
}

type AsyncResource struct {
	Metadata struct {
		URL string
	}
}

type apiErrorHandler func(statusCode int, body []byte) error

type tokenRefresher interface {
	RefreshAuthToken() (string, error)
}

type Request struct {
	HttpReq      *http.Request
	SeekableBody io.ReadSeeker
}

type Gateway struct {
	authenticator   tokenRefresher
	errHandler      apiErrorHandler
	PollingEnabled  bool
	PollingThrottle time.Duration
	trustedCerts    []tls.Certificate
	config          core_config.Reader
	warnings        *[]string
	Clock           func() time.Time
	transport       *http.Transport
	ui              terminal.UI
}

func newGateway(errHandler apiErrorHandler, config core_config.Reader, ui terminal.UI) (gateway Gateway) {
	gateway.errHandler = errHandler
	gateway.config = config
	gateway.PollingThrottle = DEFAULT_POLLING_THROTTLE
	gateway.warnings = &[]string{}
	gateway.Clock = time.Now
	gateway.ui = ui

	return
}

func (gateway *Gateway) AsyncTimeout() time.Duration {
	if gateway.config.AsyncTimeout() > 0 {
		return time.Duration(gateway.config.AsyncTimeout()) * time.Minute
	}

	return 0
}

func (gateway *Gateway) SetTokenRefresher(auth tokenRefresher) {
	gateway.authenticator = auth
}

func (gateway Gateway) GetResource(url string, resource interface{}) (err error) {
	request, err := gateway.NewRequest("GET", url, gateway.config.AccessToken(), nil)
	if err != nil {
		return
	}

	_, err = gateway.PerformRequestForJSONResponse(request, resource)
	return
}

func (gateway Gateway) CreateResourceFromStruct(endpoint, url string, resource interface{}) error {
	bytes, err := json.Marshal(resource)
	if err != nil {
		return err
	}

	return gateway.CreateResource(endpoint, url, strings.NewReader(string(bytes)))
}

func (gateway Gateway) UpdateResourceFromStruct(endpoint, apiUrl string, resource interface{}) error {
	bytes, err := json.Marshal(resource)
	if err != nil {
		return err
	}

	return gateway.UpdateResource(endpoint, apiUrl, strings.NewReader(string(bytes)))
}

func (gateway Gateway) CreateResource(endpoint, apiUrl string, body io.ReadSeeker, optionalResource ...interface{}) error {
	request, err := gateway.NewRequest("POST", endpoint+apiUrl, gateway.config.AccessToken(), body)
	if err != nil {
		return err
	}

	var resource interface{}
	if len(optionalResource) > 0 {
		resource = optionalResource[0]
	}

	if resource == nil {
		_, err = gateway.PerformRequest(request)
		return err
	}

	if gateway.PollingEnabled {
		_, err = gateway.PerformPollingRequestForJSONResponse(endpoint, request, resource, gateway.AsyncTimeout())
		return err
	}

	_, err = gateway.PerformRequestForJSONResponse(request, resource)
	return err
}

func (gateway Gateway) UpdateResource(endpoint, apiUrl string, body io.ReadSeeker, optionalResource ...interface{}) error {
	request, err := gateway.NewRequest("PUT", endpoint+apiUrl, gateway.config.AccessToken(), body)
	if err != nil {
		return err
	}

	var resource interface{}
	if len(optionalResource) > 0 {
		resource = optionalResource[0]
	}

	if resource == nil {
		_, err = gateway.PerformRequest(request)
		return err
	}

	if gateway.PollingEnabled {
		_, err = gateway.PerformPollingRequestForJSONResponse(endpoint, request, resource, gateway.AsyncTimeout())
		return err
	}

	_, err = gateway.PerformRequestForJSONResponse(request, resource)
	return err
}

func (gateway Gateway) UpdateResourceSync(endpoint, apiUrl string, body io.ReadSeeker, optionalResource ...interface{}) error {
	request, err := gateway.NewRequest("PUT", endpoint+apiUrl, gateway.config.AccessToken(), body)
	if err != nil {
		return err
	}

	var resource interface{}
	if len(optionalResource) > 0 {
		resource = optionalResource[0]
	}

	if resource == nil {
		_, err = gateway.PerformRequest(request)
		return err
	}

	_, err = gateway.PerformRequestForJSONResponse(request, resource)
	return err
}

func (gateway Gateway) DeleteResourceSynchronously(endpoint, apiUrl string) error {
	request, err := gateway.NewRequest("DELETE", endpoint+apiUrl, gateway.config.AccessToken(), nil)
	if err != nil {
		return err
	}

	_, err = gateway.PerformRequestForJSONResponse(request, nil)
	return err
}

func (gateway Gateway) DeleteResource(endpoint, apiUrl string) (apiErr error) {
	request, err := gateway.NewRequest("DELETE", endpoint+apiUrl, gateway.config.AccessToken(), nil)
	if err != nil {
		return err
	}

	if gateway.PollingEnabled {
		_, err = gateway.PerformPollingRequestForJSONResponse(endpoint, request, nil, gateway.AsyncTimeout())
		return err
	}

	_, err = gateway.PerformRequestForJSONResponse(request, nil)
	return err
}

func (gateway Gateway) ListPaginatedResources(target string,
	path string,
	resource interface{},
	cb func(interface{}) bool) (apiErr error) {
	for path != "" {
		pagination := NewPaginatedResources(resource)

		apiErr = gateway.GetResource(fmt.Sprintf("%s%s", target, path), &pagination)
		if apiErr != nil {
			return
		}

		resources, err := pagination.Resources()
		if err != nil {
			return fmt.Errorf("%s: %s", T("Error parsing JSON"), err.Error())
		}

		for _, resource := range resources {
			if !cb(resource) {
				return
			}
		}

		path = pagination.NextURL
	}

	return
}

func (gateway Gateway) newRequest(request *http.Request, accessToken string, body io.ReadSeeker) (*Request, error) {
	if accessToken != "" {
		request.Header.Set("Authorization", accessToken)
	}

	request.Header.Set("accept", "application/json")
	request.Header.Set("content-type", "application/json")
	request.Header.Set("User-Agent", "go-cli "+cf.Version+" / "+runtime.GOOS)
	return &Request{HttpReq: request, SeekableBody: body}, nil
}

func (gateway Gateway) NewRequestForFile(method, fullUrl, accessToken string, body *os.File) (req *Request, apiErr error) {
	progressReader := NewProgressReader(body, gateway.ui, 5*time.Second)
	progressReader.Seek(0, 0)
	fileStats, err := body.Stat()

	if err != nil {
		apiErr = fmt.Errorf("%s: %s", T("Error getting file info"), err.Error())
		return
	}

	request, err := http.NewRequest(method, fullUrl, progressReader)
	if err != nil {
		apiErr = fmt.Errorf("%s: %s", T("Error building request"), err.Error())
		return
	}

	fileSize := fileStats.Size()
	progressReader.SetTotalSize(fileSize)
	request.ContentLength = fileSize

	if err != nil {
		apiErr = fmt.Errorf("%s: %s", T("Error building request"), err.Error())
		return
	}

	return gateway.newRequest(request, accessToken, progressReader)
}

func (gateway Gateway) NewRequest(method, path, accessToken string, body io.ReadSeeker) (req *Request, apiErr error) {
	request, err := http.NewRequest(method, path, body)
	if err != nil {
		apiErr = fmt.Errorf("%s: %s", T("Error building request"), err.Error())
		return
	}
	return gateway.newRequest(request, accessToken, body)
}

func (gateway Gateway) PerformRequest(request *Request) (rawResponse *http.Response, apiErr error) {
	return gateway.doRequestHandlingAuth(request)
}

func (gateway Gateway) performRequestForResponseBytes(request *Request) (bytes []byte, headers http.Header, rawResponse *http.Response, apiErr error) {
	rawResponse, apiErr = gateway.doRequestHandlingAuth(request)
	if apiErr != nil {
		return
	}
	defer rawResponse.Body.Close()

	bytes, err := ioutil.ReadAll(rawResponse.Body)
	if err != nil {
		apiErr = fmt.Errorf("%s: %s", T("Error reading response"), err.Error())
	}

	headers = rawResponse.Header
	return
}

func (gateway Gateway) PerformRequestForTextResponse(request *Request) (response string, headers http.Header, apiErr error) {
	bytes, headers, _, apiErr := gateway.performRequestForResponseBytes(request)
	response = string(bytes)
	return
}

func (gateway Gateway) PerformRequestForJSONResponse(request *Request, response interface{}) (headers http.Header, apiErr error) {
	bytes, headers, rawResponse, apiErr := gateway.performRequestForResponseBytes(request)
	if apiErr != nil {
		return
	}

	if rawResponse.StatusCode > 203 || strings.TrimSpace(string(bytes)) == "" {
		return
	}

	err := json.Unmarshal(bytes, &response)
	if err != nil {
		apiErr = fmt.Errorf("%s: %s", T("Invalid JSON response from server"), err.Error())
	}
	return
}

func (gateway Gateway) PerformPollingRequestForJSONResponse(endpoint string, request *Request, response interface{}, timeout time.Duration) (headers http.Header, apiErr error) {
	query := request.HttpReq.URL.Query()
	query.Add("async", "true")
	request.HttpReq.URL.RawQuery = query.Encode()

	bytes, headers, rawResponse, apiErr := gateway.performRequestForResponseBytes(request)
	if apiErr != nil {
		return
	}
	defer rawResponse.Body.Close()

	if rawResponse.StatusCode > 203 || strings.TrimSpace(string(bytes)) == "" {
		return
	}

	err := json.Unmarshal(bytes, &response)
	if err != nil {
		apiErr = fmt.Errorf("%s: %s", T("Invalid JSON response from server"), err.Error())
		return
	}

	asyncResource := &AsyncResource{}
	err = json.Unmarshal(bytes, &asyncResource)
	if err != nil {
		apiErr = fmt.Errorf("%s: %s", T("Invalid async response from server"), err.Error())
		return
	}

	jobUrl := asyncResource.Metadata.URL
	if jobUrl == "" {
		return
	}

	if !strings.Contains(jobUrl, "/jobs/") {
		return
	}

	apiErr = gateway.waitForJob(endpoint+jobUrl, request.HttpReq.Header.Get("Authorization"), timeout)

	return
}

func (gateway Gateway) Warnings() []string {
	return *gateway.warnings
}

func (gateway Gateway) waitForJob(jobUrl, accessToken string, timeout time.Duration) (err error) {
	startTime := gateway.Clock()
	for true {
		if gateway.Clock().Sub(startTime) > timeout && timeout != 0 {
			return errors.NewAsyncTimeoutError(jobUrl)
		}
		var request *Request
		request, err = gateway.NewRequest("GET", jobUrl, accessToken, nil)
		response := &JobResource{}
		_, err = gateway.PerformRequestForJSONResponse(request, response)
		if err != nil {
			return
		}

		switch response.Entity.Status {
		case JOB_FINISHED:
			return
		case JOB_FAILED:
			err = errors.New(response.Entity.ErrorDetails.Description)
			return
		}

		accessToken = request.HttpReq.Header.Get("Authorization")

		time.Sleep(gateway.PollingThrottle)
	}
	return
}

func (gateway Gateway) doRequestHandlingAuth(request *Request) (rawResponse *http.Response, err error) {
	httpReq := request.HttpReq

	if request.SeekableBody != nil {
		httpReq.Body = ioutil.NopCloser(request.SeekableBody)
	}

	// perform request
	rawResponse, err = gateway.doRequestAndHandlerError(request)
	if err == nil || gateway.authenticator == nil {
		return
	}

	switch err.(type) {
	case *errors.InvalidTokenError:
		// refresh the auth token
		var newToken string
		newToken, err = gateway.authenticator.RefreshAuthToken()
		if err != nil {
			return
		}

		// reset the auth token and request body
		httpReq.Header.Set("Authorization", newToken)
		if request.SeekableBody != nil {
			request.SeekableBody.Seek(0, 0)
			httpReq.Body = ioutil.NopCloser(request.SeekableBody)
		}

		// make the request again
		rawResponse, err = gateway.doRequestAndHandlerError(request)
	}

	return
}

func (gateway Gateway) doRequestAndHandlerError(request *Request) (rawResponse *http.Response, err error) {
	rawResponse, err = gateway.doRequest(request.HttpReq)
	if err != nil {
		err = WrapNetworkErrors(request.HttpReq.URL.Host, err)
		return
	}

	if rawResponse.StatusCode > 299 {
		jsonBytes, _ := ioutil.ReadAll(rawResponse.Body)
		rawResponse.Body.Close()
		rawResponse.Body = ioutil.NopCloser(bytes.NewBuffer(jsonBytes))
		err = gateway.errHandler(rawResponse.StatusCode, jsonBytes)
	}

	return
}

func (gateway Gateway) doRequest(request *http.Request) (response *http.Response, err error) {
	if gateway.transport == nil {
		makeHttpTransport(&gateway)
	}

	httpClient := NewHttpClient(gateway.transport)

	dumpRequest(request)

	for i := 0; i < 3; i++ {
		response, err = httpClient.Do(request)
		if response == nil && err != nil {
			continue
		} else {
			break
		}
	}

	if err != nil {
		return
	}

	dumpResponse(response)

	header := http.CanonicalHeaderKey("X-Cf-Warnings")
	raw_warnings := response.Header[header]
	for _, raw_warning := range raw_warnings {
		warning, _ := url.QueryUnescape(raw_warning)
		*gateway.warnings = append(*gateway.warnings, warning)
	}

	return
}

func makeHttpTransport(gateway *Gateway) {
	gateway.transport = &http.Transport{
		Dial:            (&net.Dialer{Timeout: 5 * time.Second}).Dial,
		TLSClientConfig: NewTLSConfig(gateway.trustedCerts, gateway.config.IsSSLDisabled()),
		Proxy:           http.ProxyFromEnvironment,
	}
}

func (gateway *Gateway) SetTrustedCerts(certificates []tls.Certificate) {
	gateway.trustedCerts = certificates
	makeHttpTransport(gateway)
}
<commit_msg>Gateway's newRequest method doesn't return an error<commit_after>package net

import (
	"bytes"
	"crypto/tls"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"net/url"
	"os"
	"runtime"
	"strings"
	"time"

	"github.com/cloudfoundry/cli/cf"
	"github.com/cloudfoundry/cli/cf/configuration/core_config"
	"github.com/cloudfoundry/cli/cf/errors"
	. "github.com/cloudfoundry/cli/cf/i18n"
	"github.com/cloudfoundry/cli/cf/terminal"
)

const (
	JOB_FINISHED             = "finished"
	JOB_FAILED               = "failed"
	DEFAULT_POLLING_THROTTLE = 5 * time.Second
)

type JobResource struct {
	Entity struct {
		Status       string
		ErrorDetails struct {
			Description string
		} `json:"error_details"`
	}
}

type AsyncResource struct {
	Metadata struct {
		URL string
	}
}

type apiErrorHandler func(statusCode int, body []byte) error

type tokenRefresher interface {
	RefreshAuthToken() (string, error)
}

type Request struct {
	HttpReq      *http.Request
	SeekableBody io.ReadSeeker
}

type Gateway struct {
	authenticator   tokenRefresher
	errHandler      apiErrorHandler
	PollingEnabled  bool
	PollingThrottle time.Duration
	trustedCerts    []tls.Certificate
	config          core_config.Reader
	warnings        *[]string
	Clock           func() time.Time
	transport       *http.Transport
	ui              terminal.UI
}

func newGateway(errHandler apiErrorHandler, config core_config.Reader, ui terminal.UI) (gateway Gateway) {
	gateway.errHandler = errHandler
	gateway.config = config
	gateway.PollingThrottle = DEFAULT_POLLING_THROTTLE
	gateway.warnings = &[]string{}
	gateway.Clock = time.Now
	gateway.ui = ui

	return
}

func (gateway *Gateway) AsyncTimeout() time.Duration {
	if gateway.config.AsyncTimeout() > 0 {
		return time.Duration(gateway.config.AsyncTimeout()) * time.Minute
	}

	return 0
}

func (gateway *Gateway) SetTokenRefresher(auth tokenRefresher) {
	gateway.authenticator = auth
}

func (gateway Gateway) GetResource(url string, resource interface{}) (err error) {
	request, err := gateway.NewRequest("GET", url, gateway.config.AccessToken(), nil)
	if err != nil {
		return
	}

	_, err = gateway.PerformRequestForJSONResponse(request, resource)
	return
}

func (gateway Gateway) CreateResourceFromStruct(endpoint, url string, resource interface{}) error {
	bytes, err := json.Marshal(resource)
	if err != nil {
		return err
	}

	return gateway.CreateResource(endpoint, url, strings.NewReader(string(bytes)))
}

func (gateway Gateway) UpdateResourceFromStruct(endpoint, apiUrl string, resource interface{}) error {
	bytes, err := json.Marshal(resource)
	if err != nil {
		return err
	}

	return gateway.UpdateResource(endpoint, apiUrl, strings.NewReader(string(bytes)))
}

func (gateway Gateway) CreateResource(endpoint, apiUrl string, body io.ReadSeeker, optionalResource ...interface{}) error {
	request, err := gateway.NewRequest("POST", endpoint+apiUrl, gateway.config.AccessToken(), body)
	if err != nil {
		return err
	}

	var resource interface{}
	if len(optionalResource) > 0 {
		resource = optionalResource[0]
	}

	if resource == nil {
		_, err = gateway.PerformRequest(request)
		return err
	}

	if gateway.PollingEnabled {
		_, err = gateway.PerformPollingRequestForJSONResponse(endpoint, request, resource, gateway.AsyncTimeout())
		return err
	}

	_, err = gateway.PerformRequestForJSONResponse(request, resource)
	return err
}

func (gateway Gateway) UpdateResource(endpoint, apiUrl string, body io.ReadSeeker, optionalResource ...interface{}) error {
	request, err := gateway.NewRequest("PUT", endpoint+apiUrl, gateway.config.AccessToken(), body)
	if err != nil {
		return err
	}

	var resource interface{}
	if len(optionalResource) > 0 {
		resource = optionalResource[0]
	}

	if resource == nil {
		_, err = gateway.PerformRequest(request)
		return err
	}

	if gateway.PollingEnabled {
		_, err = gateway.PerformPollingRequestForJSONResponse(endpoint, request, resource, gateway.AsyncTimeout())
		return err
	}

	_, err = gateway.PerformRequestForJSONResponse(request, resource)
	return err
}

func (gateway Gateway) UpdateResourceSync(endpoint, apiUrl string, body io.ReadSeeker, optionalResource ...interface{}) error {
	request, err := gateway.NewRequest("PUT", endpoint+apiUrl, gateway.config.AccessToken(), body)
	if err != nil {
		return err
	}

	var resource interface{}
	if len(optionalResource) > 0 {
		resource = optionalResource[0]
	}

	if resource == nil {
		_, err = gateway.PerformRequest(request)
		return err
	}

	_, err = gateway.PerformRequestForJSONResponse(request, resource)
	return err
}

func (gateway Gateway) DeleteResourceSynchronously(endpoint, apiUrl string) error {
	request, err := gateway.NewRequest("DELETE", endpoint+apiUrl, gateway.config.AccessToken(), nil)
	if err != nil {
		return err
	}

	_, err = gateway.PerformRequestForJSONResponse(request, nil)
	return err
}

func (gateway Gateway) DeleteResource(endpoint, apiUrl string) (apiErr error) {
	request, err := gateway.NewRequest("DELETE", endpoint+apiUrl, gateway.config.AccessToken(), nil)
	if err != nil {
		return err
	}

	if gateway.PollingEnabled {
		_, err = gateway.PerformPollingRequestForJSONResponse(endpoint, request, nil, gateway.AsyncTimeout())
		return err
	}

	_, err = gateway.PerformRequestForJSONResponse(request, nil)
	return err
}

func (gateway Gateway) ListPaginatedResources(target string,
	path string,
	resource interface{},
	cb func(interface{}) bool) (apiErr error) {
	for path != "" {
		pagination := NewPaginatedResources(resource)

		apiErr = gateway.GetResource(fmt.Sprintf("%s%s", target, path), &pagination)
		if apiErr != nil {
			return
		}

		resources, err := pagination.Resources()
		if err != nil {
			return fmt.Errorf("%s: %s", T("Error parsing JSON"), err.Error())
		}

		for _, resource := range resources {
			if !cb(resource) {
				return
			}
		}

		path = pagination.NextURL
	}

	return
}

func (gateway Gateway) newRequest(request *http.Request, accessToken string, body io.ReadSeeker) *Request {
	if accessToken != "" {
		request.Header.Set("Authorization", accessToken)
	}

	request.Header.Set("accept", "application/json")
	request.Header.Set("content-type", "application/json")
	request.Header.Set("User-Agent", "go-cli "+cf.Version+" / "+runtime.GOOS)

	return &Request{HttpReq: request, SeekableBody: body}
}

func (gateway Gateway) NewRequestForFile(method, fullUrl, accessToken string, body *os.File) (*Request, error) {
	progressReader := NewProgressReader(body, gateway.ui, 5*time.Second)
	progressReader.Seek(0, 0)
	fileStats, err := body.Stat()

	if err != nil {
		return nil, fmt.Errorf("%s: %s", T("Error getting file info"), err.Error())
	}

	request, err := http.NewRequest(method, fullUrl, progressReader)
	if err != nil {
		return nil, fmt.Errorf("%s: %s", T("Error building request"), err.Error())
	}

	fileSize := fileStats.Size()
	progressReader.SetTotalSize(fileSize)
	request.ContentLength = fileSize

	if err != nil {
		return nil, fmt.Errorf("%s: %s", T("Error building request"), err.Error())
	}

	return gateway.newRequest(request, accessToken, progressReader), nil
}

func (gateway Gateway) NewRequest(method, path, accessToken string, body io.ReadSeeker) (*Request, error) {
	request, err := http.NewRequest(method, path, body)
	if err != nil {
		return nil, fmt.Errorf("%s: %s", T("Error building request"), err.Error())
	}
	return gateway.newRequest(request, accessToken, body), nil
}

func (gateway Gateway) PerformRequest(request *Request) (rawResponse *http.Response, apiErr error) {
	return gateway.doRequestHandlingAuth(request)
}

func (gateway Gateway) performRequestForResponseBytes(request *Request) (bytes []byte, headers http.Header, rawResponse *http.Response, apiErr error) {
	rawResponse, apiErr = gateway.doRequestHandlingAuth(request)
	if apiErr != nil {
		return
	}
	defer rawResponse.Body.Close()

	bytes, err := ioutil.ReadAll(rawResponse.Body)
	if err != nil {
		apiErr = fmt.Errorf("%s: %s", T("Error reading response"), err.Error())
	}

	headers = rawResponse.Header
	return
}

func (gateway Gateway) PerformRequestForTextResponse(request *Request) (response string, headers http.Header, apiErr error) {
	bytes, headers, _, apiErr := gateway.performRequestForResponseBytes(request)
	response = string(bytes)
	return
}

func (gateway Gateway) PerformRequestForJSONResponse(request *Request, response interface{}) (headers http.Header, apiErr error) {
	bytes, headers, rawResponse, apiErr := gateway.performRequestForResponseBytes(request)
	if apiErr != nil {
		return
	}

	if rawResponse.StatusCode > 203 || strings.TrimSpace(string(bytes)) == "" {
		return
	}

	err := json.Unmarshal(bytes, &response)
	if err != nil {
		apiErr = fmt.Errorf("%s: %s", T("Invalid JSON response from server"), err.Error())
	}
	return
}

func (gateway Gateway) PerformPollingRequestForJSONResponse(endpoint string, request *Request, response interface{}, timeout time.Duration) (headers http.Header, apiErr error) {
	query := request.HttpReq.URL.Query()
	query.Add("async", "true")
	request.HttpReq.URL.RawQuery = query.Encode()

	bytes, headers, rawResponse, apiErr := gateway.performRequestForResponseBytes(request)
	if apiErr != nil {
		return
	}
	defer rawResponse.Body.Close()

	if rawResponse.StatusCode > 203 || strings.TrimSpace(string(bytes)) == "" {
		return
	}

	err := json.Unmarshal(bytes, &response)
	if err != nil {
		apiErr = fmt.Errorf("%s: %s", T("Invalid JSON response from server"), err.Error())
		return
	}

	asyncResource := &AsyncResource{}
	err = json.Unmarshal(bytes, &asyncResource)
	if err != nil {
		apiErr = fmt.Errorf("%s: %s", T("Invalid async response from server"), err.Error())
		return
	}

	jobUrl := asyncResource.Metadata.URL
	if jobUrl == "" {
		return
	}

	if !strings.Contains(jobUrl, "/jobs/") {
		return
	}

	apiErr = gateway.waitForJob(endpoint+jobUrl, request.HttpReq.Header.Get("Authorization"), timeout)

	return
}

func (gateway Gateway) Warnings() []string {
	return *gateway.warnings
}

func (gateway Gateway) waitForJob(jobUrl, accessToken string, timeout time.Duration) (err error) {
	startTime := gateway.Clock()
	for true {
		if gateway.Clock().Sub(startTime) > timeout && timeout != 0 {
			return errors.NewAsyncTimeoutError(jobUrl)
		}
		var request *Request
		request, err = gateway.NewRequest("GET", jobUrl, accessToken, nil)
		response := &JobResource{}
		_, err = gateway.PerformRequestForJSONResponse(request, response)
		if err != nil {
			return
		}

		switch response.Entity.Status {
		case JOB_FINISHED:
			return
		case JOB_FAILED:
			err = errors.New(response.Entity.ErrorDetails.Description)
			return
		}

		accessToken = request.HttpReq.Header.Get("Authorization")

		time.Sleep(gateway.PollingThrottle)
	}
	return
}

func (gateway Gateway) doRequestHandlingAuth(request *Request) (rawResponse *http.Response, err error) {
	httpReq := request.HttpReq

	if request.SeekableBody != nil {
		httpReq.Body = ioutil.NopCloser(request.SeekableBody)
	}

	// perform request
	rawResponse, err = gateway.doRequestAndHandlerError(request)
	if err == nil || gateway.authenticator == nil {
		return
	}

	switch err.(type) {
	case *errors.InvalidTokenError:
		// refresh the auth token
		var newToken string
		newToken, err = gateway.authenticator.RefreshAuthToken()
		if err != nil {
			return
		}

		// reset the auth token and request body
		httpReq.Header.Set("Authorization", newToken)
		if request.SeekableBody != nil {
			request.SeekableBody.Seek(0, 0)
			httpReq.Body = ioutil.NopCloser(request.SeekableBody)
		}

		// make the request again
		rawResponse, err = gateway.doRequestAndHandlerError(request)
	}

	return
}

func (gateway Gateway) doRequestAndHandlerError(request *Request) (rawResponse *http.Response, err error) {
	rawResponse, err = gateway.doRequest(request.HttpReq)
	if err != nil {
		err = WrapNetworkErrors(request.HttpReq.URL.Host, err)
		return
	}

	if rawResponse.StatusCode > 299 {
		jsonBytes, _ := ioutil.ReadAll(rawResponse.Body)
		rawResponse.Body.Close()
		rawResponse.Body = ioutil.NopCloser(bytes.NewBuffer(jsonBytes))
		err = gateway.errHandler(rawResponse.StatusCode, jsonBytes)
	}

	return
}

func (gateway Gateway) doRequest(request *http.Request) (response *http.Response, err error) {
	if gateway.transport == nil {
		makeHttpTransport(&gateway)
	}

	httpClient := NewHttpClient(gateway.transport)

	dumpRequest(request)

	for i := 0; i < 3; i++ {
		response, err = httpClient.Do(request)
		if response == nil && err != nil {
			continue
		} else {
			break
		}
	}

	if err != nil {
		return
	}

	dumpResponse(response)

	header := http.CanonicalHeaderKey("X-Cf-Warnings")
	raw_warnings := response.Header[header]
	for _, raw_warning := range raw_warnings {
		warning, _ := url.QueryUnescape(raw_warning)
		*gateway.warnings = append(*gateway.warnings, warning)
	}

	return
}

func makeHttpTransport(gateway *Gateway) {
	gateway.transport = &http.Transport{
		Dial:            (&net.Dialer{Timeout: 5 * time.Second}).Dial,
		TLSClientConfig: NewTLSConfig(gateway.trustedCerts, gateway.config.IsSSLDisabled()),
		Proxy:           http.ProxyFromEnvironment,
	}
}

func (gateway *Gateway) SetTrustedCerts(certificates []tls.Certificate) {
	gateway.trustedCerts = certificates
	makeHttpTransport(gateway)
}
<|endoftext|>


<commit_before>//
// DISCLAIMER
//
// Copyright 2017 ArangoDB GmbH, Cologne, Germany
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Copyright holder is ArangoDB GmbH, Cologne, Germany
//
// Author Ewout Prangsma
//

package test

import "testing"

// BenchmarkCreateDocument measures the CreateDocument operation for a simple document.
func BenchmarkCreateDocument(b *testing.B) {
	c := createClientFromEnv(b, true)
	db := ensureDatabase(nil, c, "document_test", nil, b)
	col := ensureCollection(nil, db, "document_test", nil, b)

	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		doc := UserDoc{
			"Jan",
			40 + i,
		}
		if _, err := col.CreateDocument(nil, doc); err != nil {
			b.Fatalf("Failed to create new document: %s", describe(err))
		}
	}
}

// BenchmarkRemoveDocument measures the RemoveDocument operation for a simple document.
func BenchmarkRemoveDocument(b *testing.B) {
	c := createClientFromEnv(b, true)
	db := ensureDatabase(nil, c, "document_test", nil, b)
	col := ensureCollection(nil, db, "document_test", nil, b)

	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		// Create document (we don't measure that)
		b.StopTimer()
		doc := UserDoc{
			"Jan",
			40 + i,
		}
		meta, err := col.CreateDocument(nil, doc)
		if err != nil {
			b.Fatalf("Failed to create new document: %s", describe(err))
		}

		// Now do the real test
		b.StartTimer()
		if _, err := col.RemoveDocument(nil, meta.Key); err != nil {
			b.Errorf("Failed to remove document: %s", describe(err))
		}
	}
}
<commit_msg>Added ReadDocument benchmark<commit_after>//
// DISCLAIMER
//
// Copyright 2017 ArangoDB GmbH, Cologne, Germany
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Copyright holder is ArangoDB GmbH, Cologne, Germany
//
// Author Ewout Prangsma
//

package test

import "testing"

// BenchmarkCreateDocument measures the CreateDocument operation for a simple document.
func BenchmarkCreateDocument(b *testing.B) {
	c := createClientFromEnv(b, true)
	db := ensureDatabase(nil, c, "document_test", nil, b)
	col := ensureCollection(nil, db, "document_test", nil, b)

	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		doc := UserDoc{
			"Jan",
			40 + i,
		}
		if _, err := col.CreateDocument(nil, doc); err != nil {
			b.Fatalf("Failed to create new document: %s", describe(err))
		}
	}
}

// BenchmarkReadDocument measures the ReadDocument operation for a simple document.
func BenchmarkReadDocument(b *testing.B) {
	c := createClientFromEnv(b, true)
	db := ensureDatabase(nil, c, "document_test", nil, b)
	col := ensureCollection(nil, db, "document_test", nil, b)
	doc := UserDoc{
		"Jan",
		40,
	}
	meta, err := col.CreateDocument(nil, doc)
	if err != nil {
		b.Fatalf("Failed to create new document: %s", describe(err))
	}

	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		var result UserDoc
		if _, err := col.ReadDocument(nil, meta.Key, &result); err != nil {
			b.Errorf("Failed to read document: %s", describe(err))
		}
	}
}

// BenchmarkRemoveDocument measures the RemoveDocument operation for a simple document.
func BenchmarkRemoveDocument(b *testing.B) {
	c := createClientFromEnv(b, true)
	db := ensureDatabase(nil, c, "document_test", nil, b)
	col := ensureCollection(nil, db, "document_test", nil, b)

	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		// Create document (we don't measure that)
		b.StopTimer()
		doc := UserDoc{
			"Jan",
			40 + i,
		}
		meta, err := col.CreateDocument(nil, doc)
		if err != nil {
			b.Fatalf("Failed to create new document: %s", describe(err))
		}

		// Now do the real test
		b.StartTimer()
		if _, err := col.RemoveDocument(nil, meta.Key); err != nil {
			b.Errorf("Failed to remove document: %s", describe(err))
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"fmt"
	"strconv"
	"encoding/json"
	"github.com/hyperledger/fabric/core/chaincode/shim"
	"github.com/hyperledger/fabric/core/util"
)

type Chaincode struct { }

type ChaincodeFunctions struct {
	stub shim.ChaincodeStubInterface
}

func main() {
	err := shim.Start(new(Chaincode))
	if err != nil {
		fmt.Printf("Error starting Simple chaincode: %s", err)
	}
}

// Init resets all the things
func (t Chaincode) Init(stub shim.ChaincodeStubInterface, function string, args []string) ([]byte, error) {
	_ = stub.PutState("dealStatus", []byte("draft")) // Possible Values [draft, open, closed, allocated]
	_ = stub.PutState("orderbook", []byte("{}"))
	return nil, nil
}

func (t Chaincode) Invoke(stub shim.ChaincodeStubInterface, function string, args []string) ([]byte, error) {
	fmt.Println("invoke is running " + function)
	fns := ChaincodeFunctions{stub}
	if function == "addOrder" {
		investor := args[0]
		ioi, err := strconv.ParseFloat(args[1], 64)
		if err != nil {
	        return nil, errors.New("Failed to parse " + args[1] + " as a float64")
    	}
		return fns.AddOrder(investor, ioi)
	} else if function == "allocateOrder" {
		investor := args[0]
		alloc, err := strconv.ParseFloat(args[1], 64)
		if err != nil {
	        return nil, errors.New("Failed to parse " + args[1] + " as a float64")
    	}
		return fns.AllocateOrder(investor, alloc)
	} else if function == "updateDealStatus" {
		status := args[0]
		return fns.UpdateDealStatus(status)
	}
	fmt.Println("invoke did not find func: " + function)

	return nil, errors.New("Received unknown function invocation: " + function)
}

func (t Chaincode) Query(stub shim.ChaincodeStubInterface, function string, args []string) ([]byte, error) {
	fmt.Println("query is running " + function)
	fns := ChaincodeFunctions{stub}
	if function == "getOrder" {
		investor := args[0]
		return fns.GetOrder(investor)
	} else if function == "getOrderbook" {
		return fns.GetOrderbook()
	} else if function == "echo" {
		address := args[0]
		f := "echo"
		arg := args[1]
		return fns.Echo(address, f, arg)
	} else if function == "getDealStatus" {
		return fns.GetDealStatus()
	}
	fmt.Println("query did not find func: " + function)

	return nil, errors.New("Received unknown function query: " + function)
}

type Order struct {
	Investor	string 		`json:"investor"`
	Ioi 		float64 	`json:"ioi"`
	Alloc 		float64		`json:"alloc"`
}

// Public Functions

func (c ChaincodeFunctions) GetDealStatus() ([]byte, error)  {
	dealStatus, _ := c.stub.GetState("dealStatus")
	return dealStatus, nil
}

func (c ChaincodeFunctions) UpdateDealStatus(dealStatus string) ([]byte, error)  {
	_ = c.stub.PutState("dealStatus", []byte(dealStatus))
	c.stub.SetEvent("Book Status Change", []byte("{\"status\":\"" + dealStatus + "\"}"))
	return nil, nil
}

func (c ChaincodeFunctions) AddOrder(investor string, ioi float64) ([]byte, error)  {
	dealStatus, _ := c.GetDealStatus()
	if(string(dealStatus) != "open") {
		c.stub.SetEvent("Permission Denied", []byte("{\"reason\":\"book is not open\"}"))
		return nil, errors.New("Orders cannot be placed unless deal status is 'Open'")
	}
	order := Order{Investor: investor, Ioi: ioi, Alloc: 0.0}
	c.saveOrderToBlockChain(order)
	c.stub.SetEvent("Order Added", []byte("{\"investor\":\"" + investor + "\"}"))
	return nil, nil
}

func (c ChaincodeFunctions) GetOrder(investor string) ([]byte, error) {
	orderJson := c.getOrderAsJsonFromBlockchain(investor)
	return []byte(orderJson), nil
}

func (c ChaincodeFunctions) GetOrderbook() ([]byte, error) {
	orderbookJson, _ := c.stub.GetState("orderbook")
	return []byte(orderbookJson), nil
}

func (c ChaincodeFunctions) AllocateOrder(investor string, alloc float64) ([]byte, error) {
	order := c.getOrderFromBlockChain(investor)
	order.Alloc = alloc
	c.saveOrderToBlockChain(order)
	return nil, nil
}

func (c ChaincodeFunctions) Echo(address string, f string, arg string) ([]byte, error) {
	invokeArgs := util.ToChaincodeArgs(f, arg)
	return c.stub.QueryChaincode(address, invokeArgs)
}

// Private Functions

func (c ChaincodeFunctions) getOrderAsJsonFromBlockchain(investor string) string {
	order := c.getOrderFromBlockChain(investor)
	orderJson, _ := json.Marshal(order)
	return string(orderJson)
}

func (c ChaincodeFunctions) getOrderFromBlockChain(investor string) Order {
	orderbook := c.getOrderbookFromBlockChain()
	return orderbook[investor]
}

func (c ChaincodeFunctions) saveOrderToBlockChain(order Order) {
	orderbook := c.getOrderbookFromBlockChain()
	orderbook[order.Investor] = order
	c.saveOrderbookToBlockChain(orderbook)
}

func (c ChaincodeFunctions) getOrderbookFromBlockChain() map[string]Order {
	orderbookJson, _ := c.stub.GetState("orderbook")
	var orderbook map[string]Order
	_ = json.Unmarshal(orderbookJson, &orderbook)
	return orderbook
}

func (c ChaincodeFunctions) saveOrderbookToBlockChain(orderbook map[string]Order) {
	orderbookJson, _ := json.Marshal(orderbook)
	_ = c.stub.PutState("orderbook", []byte(orderbookJson))
}<commit_msg>batch check-in<commit_after>package main

import (
	"errors"
	"fmt"
	"strconv"
	"encoding/json"
	"github.com/hyperledger/fabric/core/chaincode/shim"
	"github.com/hyperledger/fabric/core/util"
)

type Chaincode struct { }

type ChaincodeFunctions struct {
	stub shim.ChaincodeStubInterface
}

func main() {
	err := shim.Start(new(Chaincode))
	if err != nil {
		fmt.Printf("Error starting Simple chaincode: %s", err)
	}
}

// Init resets all the things
func (t Chaincode) Init(stub shim.ChaincodeStubInterface, function string, args []string) ([]byte, error) {
	_ = stub.PutState("dealStatus", []byte("draft")) // Possible Values [draft, open, closed, allocated]
	_ = stub.PutState("orderbook", []byte("{}"))
	return nil, nil
}

func (t Chaincode) Invoke(stub shim.ChaincodeStubInterface, function string, args []string) ([]byte, error) {
	fmt.Println("invoke is running " + function)
	fns := ChaincodeFunctions{stub}
	if function == "addOrder" {
		investor := args[0]
		ioi, err := strconv.ParseFloat(args[1], 64)
		if err != nil {
	        return nil, errors.New("Failed to parse " + args[1] + " as a float64")
    	}
		return fns.AddOrder(investor, ioi)
	} else if function == "allocateOrder" {
		investor := args[0]
		alloc, err := strconv.ParseFloat(args[1], 64)
		if err != nil {
	        return nil, errors.New("Failed to parse " + args[1] + " as a float64")
    	}
		return fns.AllocateOrder(investor, alloc)
	} else if function == "updateDealStatus" {
		status := args[0]
		return fns.UpdateDealStatus(status)
	}
	fmt.Println("invoke did not find func: " + function)

	return nil, errors.New("Received unknown function invocation: " + function)
}

func (t Chaincode) Query(stub shim.ChaincodeStubInterface, function string, args []string) ([]byte, error) {
	fmt.Println("query is running " + function)
	fns := ChaincodeFunctions{stub}
	if function == "getOrder" {
		investor := args[0]
		return fns.GetOrder(investor)
	} else if function == "getOrderbook" {
		return fns.GetOrderbook()
	} else if function == "echo" {
		address := args[0]
		f := "echo"
		arg := args[1]
		return fns.Echo(address, f, arg)
	} else if function == "getDealStatus" {
		return fns.GetDealStatus()
	}
	fmt.Println("query did not find func: " + function)

	return nil, errors.New("Received unknown function query: " + function)
}

type Order struct {
	Investor	string 		`json:"investor"`
	Ioi 		float64 	`json:"ioi"`
	Alloc 		float64		`json:"alloc"`
}

// Public Functions

func (c ChaincodeFunctions) GetDealStatus() ([]byte, error)  {
	dealStatus, _ := c.stub.GetState("dealStatus")
	return dealStatus, nil
}

func (c ChaincodeFunctions) UpdateDealStatus(dealStatus string) ([]byte, error)  {
	_ = c.stub.PutState("dealStatus", []byte(dealStatus))
	c.stub.SetEvent("Book Status Change", []byte("{\"status\":\"" + dealStatus + "\"}"))
	return nil, nil
}

func (c ChaincodeFunctions) AddOrder(investor string, ioi float64) ([]byte, error)  {
	dealStatus, _ := c.GetDealStatus()
	if(string(dealStatus) != "open") {
		c.stub.SetEvent("Permission Denied", []byte("{\"reason\":\"book is not open\"}"))
		return nil, errors.New("Orders cannot be placed unless deal status is 'Open'")
	}
	order := Order{Investor: investor, Ioi: ioi, Alloc: 0.0}
	c.saveOrderToBlockChain(order)
	c.stub.SetEvent("Order Added", []byte("{\"investor\":\"" + investor + "\"}"))
	return nil, nil
}

func (c ChaincodeFunctions) GetOrder(investor string) ([]byte, error) {
	orderJson := c.getOrderAsJsonFromBlockchain(investor)
	return []byte(orderJson), nil
}

func (c ChaincodeFunctions) GetOrderbook() ([]byte, error) {
	orderbookJson, _ := c.stub.GetState("orderbook")
	return []byte(orderbookJson), nil
}

func (c ChaincodeFunctions) AllocateOrder(investor string, alloc float64) ([]byte, error) {
	order := c.getOrderFromBlockChain(investor)
	order.Alloc = alloc
	c.saveOrderToBlockChain(order)
	c.stub.SetEvent("Order Allocated", []byte("{\"investor\":\"" + investor + "\"}"))
	return nil, nil
}

func (c ChaincodeFunctions) Echo(address string, f string, arg string) ([]byte, error) {
	invokeArgs := util.ToChaincodeArgs(f, arg)
	return c.stub.QueryChaincode(address, invokeArgs)
}

// Private Functions

func (c ChaincodeFunctions) getOrderAsJsonFromBlockchain(investor string) string {
	order := c.getOrderFromBlockChain(investor)
	orderJson, _ := json.Marshal(order)
	return string(orderJson)
}

func (c ChaincodeFunctions) getOrderFromBlockChain(investor string) Order {
	orderbook := c.getOrderbookFromBlockChain()
	return orderbook[investor]
}

func (c ChaincodeFunctions) saveOrderToBlockChain(order Order) {
	orderbook := c.getOrderbookFromBlockChain()
	orderbook[order.Investor] = order
	c.saveOrderbookToBlockChain(orderbook)
}

func (c ChaincodeFunctions) getOrderbookFromBlockChain() map[string]Order {
	orderbookJson, _ := c.stub.GetState("orderbook")
	var orderbook map[string]Order
	_ = json.Unmarshal(orderbookJson, &orderbook)
	return orderbook
}

func (c ChaincodeFunctions) saveOrderbookToBlockChain(orderbook map[string]Order) {
	orderbookJson, _ := json.Marshal(orderbook)
	_ = c.stub.PutState("orderbook", []byte(orderbookJson))
}<|endoftext|>


<commit_before>/*
Copyright 2020 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

	http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package config

import (
	"io/ioutil"
	"time"

	"golang.org/x/oauth2"
	"golang.org/x/oauth2/google"
	"google.golang.org/api/dns/v1"
)

type EnvConfig struct {
	FullHostName                  string `envconfig:"full_host_name" required: "true"`
	DNSZone                       string `envconfig:"dns_zone" required:"true"`
	CloudDNSServiceAccountKeyFile string `envconfig:"cloud_dns_service_account_key_file" required:"true"`
	CloudDNSProject               string `envconfig:"cloud_dns_project" required:"true"`
	IngressIP                     string `envconfig:"ingress_ip" required:"true"`
}

type DNSRecord struct {
	IP     string
	Domain string
}

func MakeRecordSet(record *DNSRecord) *dns.ResourceRecordSet {
	dnsName := record.Domain + "."
	return &dns.ResourceRecordSet{
		Name:    dnsName,
		Rrdatas: []string{record.IP},
		// Setting TTL of DNS record to 5 second to make DNS become effective more quickly.
		Ttl:  int64(5),
		Type: "A",
	}
}

func DeleteDNSRecord(record *DNSRecord, svcAccountKeyFile, dnsProject, dnsZone string) error {
	rec := MakeRecordSet(record)
	svc, err := GetCloudDNSSvc(svcAccountKeyFile)
	if err != nil {
		return err
	}
	deletion := &dns.Change{
		Deletions: []*dns.ResourceRecordSet{rec},
	}
	return ChangeDNSRecord(deletion, svc, dnsProject, dnsZone)
}

func ChangeDNSRecord(change *dns.Change, svc *dns.Service, dnsProject, dnsZone string) error {
	chg, err := svc.Changes.Create(dnsProject, dnsZone, change).Do()
	if err != nil {
		return err
	}
	// wait for change to be acknowledged
	for chg.Status == "pending" {
		time.Sleep(time.Second)
		chg, err = svc.Changes.Get(dnsProject, dnsZone, chg.Id).Do()
		if err != nil {
			return err
		}
	}
	return nil
}

// reference: https://github.com/jetstack/cert-manager/blob/master/pkg/issuer/acme/dns/clouddns/clouddns.go
func GetCloudDNSSvc(svcAccountKeyFile string) (*dns.Service, error) {
	data, err := ioutil.ReadFile(svcAccountKeyFile)
	if err != nil {
		return nil, err
	}
	conf, err := google.JWTConfigFromJSON(data, dns.NdevClouddnsReadwriteScope)
	if err != nil {
		return nil, err
	}
	client := conf.Client(oauth2.NoContext)
	svc, err := dns.New(client)
	if err != nil {
		return nil, err
	}
	return svc, nil
}
<commit_msg>Fix the nits that I sent too late for review. (#6894)<commit_after>/*
Copyright 2020 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

	http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package config

import (
	"io/ioutil"
	"time"

	"golang.org/x/oauth2"
	"golang.org/x/oauth2/google"
	"google.golang.org/api/dns/v1"
	"k8s.io/apimachinery/pkg/util/wait"
)

type EnvConfig struct {
	FullHostName                  string `envconfig:"full_host_name" required: "true"`
	DNSZone                       string `envconfig:"dns_zone" required:"true"`
	CloudDNSServiceAccountKeyFile string `envconfig:"cloud_dns_service_account_key_file" required:"true"`
	CloudDNSProject               string `envconfig:"cloud_dns_project" required:"true"`
	IngressIP                     string `envconfig:"ingress_ip" required:"true"`
}

type DNSRecord struct {
	IP     string
	Domain string
}

func MakeRecordSet(record *DNSRecord) *dns.ResourceRecordSet {
	dnsName := record.Domain + "."
	return &dns.ResourceRecordSet{
		Name:    dnsName,
		Rrdatas: []string{record.IP},
		// Setting TTL of DNS record to 5 seconds to make DNS become effective more quickly.
		Ttl:  int64(5),
		Type: "A",
	}
}

// DeleteDNSRecord deletes the given DNS record.
func DeleteDNSRecord(record *DNSRecord, svcAccountKeyFile, dnsProject, dnsZone string) error {
	rec := MakeRecordSet(record)
	svc, err := GetCloudDNSSvc(svcAccountKeyFile)
	if err != nil {
		return err
	}
	deletion := &dns.Change{
		Deletions: []*dns.ResourceRecordSet{rec},
	}
	return ChangeDNSRecord(deletion, svc, dnsProject, dnsZone)
}

// ChangeDNSRecord changes the given DNS record.
func ChangeDNSRecord(change *dns.Change, svc *dns.Service, dnsProject, dnsZone string) error {
	// Wait for change to be acknowledged.
	return wait.PollImmediate(time.Second, 5*time.Minute, func() (bool, error) {
		chg, err := svc.Changes.Create(dnsProject, dnsZone, change).Do()
		if err != nil {
			return false, err
		}
		return chg.Status != "pending", nil
	})
}

// GetCloudDNSSvc returns the Cloud DNS Service stub.
// reference: https://github.com/jetstack/cert-manager/blob/master/pkg/issuer/acme/dns/clouddns/clouddns.go
func GetCloudDNSSvc(svcAccountKeyFile string) (*dns.Service, error) {
	data, err := ioutil.ReadFile(svcAccountKeyFile)
	if err != nil {
		return nil, err
	}
	conf, err := google.JWTConfigFromJSON(data, dns.NdevClouddnsReadwriteScope)
	if err != nil {
		return nil, err
	}
	client := conf.Client(oauth2.NoContext)
	return dns.New(client)
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
	"fmt"
	"os/exec"
	"strconv"
	"strings"
	"time"

	"k8s.io/kubernetes/pkg/api/v1"
	"k8s.io/kubernetes/pkg/util/json"
	"k8s.io/kubernetes/test/e2e/framework"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = framework.KubeDescribe("Cluster level logging using GCL", func() {
	f := framework.NewDefaultFramework("gcl-logging")

	BeforeEach(func() {
		// TODO (crassirostris): Expand to GKE once the test is stable
		framework.SkipUnlessProviderIs("gce")
	})

	It("should check that logs from containers are ingested in GCL", func() {
		By("Running synthetic logger")
		createSynthLogger(f, expectedLinesCount)
		defer f.PodClient().Delete(synthLoggerPodName, &v1.DeleteOptions{})
		err := framework.WaitForPodSuccessInNamespace(f.ClientSet, synthLoggerPodName, f.Namespace.Name)
		framework.ExpectNoError(err, fmt.Sprintf("Should've successfully waited for pod %s to succeed", synthLoggerPodName))

		By("Waiting for logs to ingest")
		totalMissing := expectedLinesCount
		for start := time.Now(); time.Since(start) < ingestionTimeout; time.Sleep(ingestionRetryDelay) {
			var err error
			totalMissing, err = getMissingLinesCountGcl(f, synthLoggerPodName, expectedLinesCount)
			if err != nil {
				framework.Logf("Failed to get missing lines count due to %v", err)
				totalMissing = expectedLinesCount
			} else if totalMissing > 0 {
				framework.Logf("Still missing %d lines", totalMissing)
			}

			if totalMissing == 0 {
				break
			}
		}

		if totalMissing > 0 {
			if err := reportLogsFromFluentdPod(f); err != nil {
				framework.Logf("Failed to report logs from fluentd pod due to %v", err)
			}
		}

		Expect(totalMissing).To(Equal(0), "Some log lines are still missing")
	})
})

func getMissingLinesCountGcl(f *framework.Framework, podName string, expectedCount int) (int, error) {
	gclFilter := fmt.Sprintf("resource.labels.pod_id:%s AND resource.labels.namespace_id:%s", podName, f.Namespace.Name)
	entries, err := readFilteredEntriesFromGcl(gclFilter)
	if err != nil {
		return 0, err
	}

	occurrences := make(map[int]int)
	for _, entry := range entries {
		lineNumber, err := strconv.Atoi(strings.TrimSpace(entry))
		if err != nil {
			continue
		}
		if lineNumber < 0 || lineNumber >= expectedCount {
			framework.Logf("Unexpected line number: %d", lineNumber)
		} else {
			// Duplicates are possible and fine, fluentd has at-least-once delivery
			occurrences[lineNumber]++
		}
	}

	return expectedCount - len(occurrences), nil
}

type LogEntry struct {
	TextPayload string
}

// Since GCL API is not easily available from the outside of cluster
// we use gcloud command to perform search with filter
func readFilteredEntriesFromGcl(filter string) ([]string, error) {
	framework.Logf("Reading entries from GCL with filter '%v'", filter)
	argList := []string{"beta",
		"logging",
		"read",
		filter,
		"--format",
		"json",
		"--project",
		framework.TestContext.CloudConfig.ProjectID,
	}
	output, err := exec.Command("gcloud", argList...).CombinedOutput()
	if err != nil {
		return nil, err
	}

	var entries []*LogEntry
	if err = json.Unmarshal(output, &entries); err != nil {
		return nil, err
	}
	framework.Logf("Read %d entries from GCL", len(entries))

	var result []string
	for _, entry := range entries {
		if entry.TextPayload != "" {
			result = append(result, entry.TextPayload)
		}
	}

	return result, nil
}
<commit_msg>Make logging for gcl e2e test more verbose<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
	"fmt"
	"os/exec"
	"strconv"
	"strings"
	"time"

	"k8s.io/kubernetes/pkg/api/v1"
	"k8s.io/kubernetes/pkg/util/json"
	"k8s.io/kubernetes/test/e2e/framework"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

const (
	increaseFluentdVerbosityCommand = "sudo sed -i 's/-q/-vv/g' /etc/kubernetes/manifests/fluentd-gcp.yaml;" +
		" sudo sed -i 's/-q/-vv/g' /etc/kubernetes/manifests/fluentd-gcp-gci.yaml"
)

var _ = framework.KubeDescribe("Cluster level logging using GCL", func() {
	f := framework.NewDefaultFramework("gcl-logging")

	BeforeEach(func() {
		// TODO (crassirostris): Expand to GKE once the test is stable
		framework.SkipUnlessProviderIs("gce")

		// TODO (crassirostris): Remove once the test is stable
		increaseFluentdVerbosity(f)
	})

	It("should check that logs from containers are ingested in GCL", func() {
		By("Running synthetic logger")
		createSynthLogger(f, expectedLinesCount)
		defer f.PodClient().Delete(synthLoggerPodName, &v1.DeleteOptions{})
		err := framework.WaitForPodSuccessInNamespace(f.ClientSet, synthLoggerPodName, f.Namespace.Name)
		framework.ExpectNoError(err, fmt.Sprintf("Should've successfully waited for pod %s to succeed", synthLoggerPodName))

		By("Waiting for logs to ingest")
		totalMissing := expectedLinesCount
		for start := time.Now(); time.Since(start) < ingestionTimeout; time.Sleep(ingestionRetryDelay) {
			var err error
			totalMissing, err = getMissingLinesCountGcl(f, synthLoggerPodName, expectedLinesCount)
			if err != nil {
				framework.Logf("Failed to get missing lines count due to %v", err)
				totalMissing = expectedLinesCount
			} else if totalMissing > 0 {
				framework.Logf("Still missing %d lines", totalMissing)
			}

			if totalMissing == 0 {
				break
			}
		}

		if totalMissing > 0 {
			if err := reportLogsFromFluentdPod(f); err != nil {
				framework.Logf("Failed to report logs from fluentd pod due to %v", err)
			}
		}

		Expect(totalMissing).To(Equal(0), "Some log lines are still missing")
	})
})

func getMissingLinesCountGcl(f *framework.Framework, podName string, expectedCount int) (int, error) {
	gclFilter := fmt.Sprintf("resource.labels.pod_id:%s AND resource.labels.namespace_id:%s", podName, f.Namespace.Name)
	entries, err := readFilteredEntriesFromGcl(gclFilter)
	if err != nil {
		return 0, err
	}

	occurrences := make(map[int]int)
	for _, entry := range entries {
		lineNumber, err := strconv.Atoi(strings.TrimSpace(entry))
		if err != nil {
			continue
		}
		if lineNumber < 0 || lineNumber >= expectedCount {
			framework.Logf("Unexpected line number: %d", lineNumber)
		} else {
			// Duplicates are possible and fine, fluentd has at-least-once delivery
			occurrences[lineNumber]++
		}
	}

	return expectedCount - len(occurrences), nil
}

type LogEntry struct {
	TextPayload string
}

// Since GCL API is not easily available from the outside of cluster
// we use gcloud command to perform search with filter
func readFilteredEntriesFromGcl(filter string) ([]string, error) {
	framework.Logf("Reading entries from GCL with filter '%v'", filter)
	argList := []string{"beta",
		"logging",
		"read",
		filter,
		"--format",
		"json",
		"--project",
		framework.TestContext.CloudConfig.ProjectID,
	}
	output, err := exec.Command("gcloud", argList...).CombinedOutput()
	if err != nil {
		return nil, err
	}

	var entries []*LogEntry
	if err = json.Unmarshal(output, &entries); err != nil {
		return nil, err
	}
	framework.Logf("Read %d entries from GCL", len(entries))

	var result []string
	for _, entry := range entries {
		if entry.TextPayload != "" {
			result = append(result, entry.TextPayload)
		}
	}

	return result, nil
}

func increaseFluentdVerbosity(f *framework.Framework) error {
	masters, nodeList := framework.GetMasterAndWorkerNodesOrDie(f.ClientSet)

	for master := range masters {
		if err := increaseFluentdVerbosityOnNode(f, master); err != nil {
			return err
		}
	}

	for _, node := range nodeList.Items {
		if err := increaseFluentdVerbosityOnNode(f, node.Name); err != nil {
			return err
		}
	}

	return nil
}

func increaseFluentdVerbosityOnNode(f *framework.Framework, nodeName string) error {
	argList := []string{"compute",
		"ssh",
		nodeName,
		"--project",
		framework.TestContext.CloudConfig.ProjectID,
		"--zone",
		framework.TestContext.CloudConfig.Zone,
		"--command",
		increaseFluentdVerbosityCommand,
	}

	return exec.Command("gcloud", argList...).Run()
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"os"
)

const VERSION="0.0.1"

type configuration struct {
	host      string
	port      int
	namespace string
	subsystem string
	logger    struct {
		adapter string
		format  string
		level   int
	}
	mnemosyne struct {
		address string
	}
	password struct {
		strategy string
		bcrypt   struct {
			cost int
		}
	}
	monitoring struct {
		engine string
	}
	postgres struct {
		connectionString string
	}
	superuser struct {
		username  string
		password  string
		firstName string
		lastName  string
	}
}

func (c *configuration) init() {
	if c == nil {
		*c = configuration{}
	}

	flag.StringVar(&c.host, "host", "127.0.0.1", "host")
	flag.IntVar(&c.port, "port", 8080, "port")
	flag.StringVar(&c.namespace, "namespace", "", "namespace")
	flag.StringVar(&c.subsystem, "subsystem", "charon", "subsystem")
	flag.StringVar(&c.logger.adapter, "l.adapter", loggerAdapterStdOut, "logger adapter")
	flag.StringVar(&c.logger.format, "l.format", loggerFormatJSON, "logger format")
	flag.IntVar(&c.logger.level, "l.level", 6, "logger level")
	flag.StringVar(&c.mnemosyne.address, "mnemo.address", "", "mnemosyne session store connection address")
	flag.StringVar(&c.password.strategy, "pwd.strategy", "bcrypt", "strategy how password will be stored")
	flag.IntVar(&c.password.bcrypt.cost, "pwd.bcryptcost", 10, "bcrypt cost, bigget than safer (and longer to create)")
	flag.StringVar(&c.monitoring.engine, "m.engine", monitoringEnginePrometheus, "monitoring engine")
	flag.StringVar(&c.postgres.connectionString, "ps.connectionstring", "postgres://localhost:5432?sslmode=disable", "storage postgres connection string")

	// Superuser configuration
	flag.StringVar(&c.superuser.username, "su.username", "", "superuser username")
	flag.StringVar(&c.superuser.password, "su.password", "", "superuser password")
	flag.StringVar(&c.superuser.firstName, "su.firstname", "", "superuser first name")
	flag.StringVar(&c.superuser.lastName, "su.lastname", "", "superuser last name")
}

func (c *configuration) parse() {
	ver := flag.Bool("version", false, "Print version and exit")
	if !flag.Parsed() {
		flag.Parse()
	}
	if *ver {
		fmt.Printf("%s", VERSION)
		os.Exit(0)
	}
}
<commit_msg>Fix for re-defined flag under test<commit_after>package main

import (
	"flag"
	"fmt"
	"os"
)

const VERSION="0.0.1"

type configuration struct {
	host      string
	port      int
	namespace string
	subsystem string
	logger    struct {
		adapter string
		format  string
		level   int
	}
	mnemosyne struct {
		address string
	}
	password struct {
		strategy string
		bcrypt   struct {
			cost int
		}
	}
	monitoring struct {
		engine string
	}
	postgres struct {
		connectionString string
	}
	superuser struct {
		username  string
		password  string
		firstName string
		lastName  string
	}
}

func (c *configuration) init() {
	if c == nil {
		*c = configuration{}
	}

	flag.StringVar(&c.host, "host", "127.0.0.1", "host")
	flag.IntVar(&c.port, "port", 8080, "port")
	flag.StringVar(&c.namespace, "namespace", "", "namespace")
	flag.StringVar(&c.subsystem, "subsystem", "charon", "subsystem")
	flag.StringVar(&c.logger.adapter, "l.adapter", loggerAdapterStdOut, "logger adapter")
	flag.StringVar(&c.logger.format, "l.format", loggerFormatJSON, "logger format")
	flag.IntVar(&c.logger.level, "l.level", 6, "logger level")
	flag.StringVar(&c.mnemosyne.address, "mnemo.address", "", "mnemosyne session store connection address")
	flag.StringVar(&c.password.strategy, "pwd.strategy", "bcrypt", "strategy how password will be stored")
	flag.IntVar(&c.password.bcrypt.cost, "pwd.bcryptcost", 10, "bcrypt cost, bigget than safer (and longer to create)")
	flag.StringVar(&c.monitoring.engine, "m.engine", monitoringEnginePrometheus, "monitoring engine")
	flag.StringVar(&c.postgres.connectionString, "ps.connectionstring", "postgres://localhost:5432?sslmode=disable", "storage postgres connection string")

	// Superuser configuration
	flag.StringVar(&c.superuser.username, "su.username", "", "superuser username")
	flag.StringVar(&c.superuser.password, "su.password", "", "superuser password")
	flag.StringVar(&c.superuser.firstName, "su.firstname", "", "superuser first name")
	flag.StringVar(&c.superuser.lastName, "su.lastname", "", "superuser last name")
}

func (c *configuration) parse() {

	if !flag.Parsed() {
		flag.Parse()
		ver := flag.Bool("version", false, "Print version and exit")
		if *ver {
			fmt.Printf("%s", VERSION)
			os.Exit(0)
		}
	}
}
<|endoftext|>


<commit_before>package generic

import (
	"errors"
	"github.com/jfrog/jfrog-cli-go/jfrog-cli/artifactory/spec"
	"github.com/jfrog/jfrog-cli-go/jfrog-cli/artifactory/utils"
	"github.com/jfrog/jfrog-cli-go/jfrog-cli/utils/config"
	"github.com/jfrog/jfrog-cli-go/jfrog-client/artifactory"
	"github.com/jfrog/jfrog-cli-go/jfrog-client/artifactory/buildinfo"
	"github.com/jfrog/jfrog-cli-go/jfrog-client/artifactory/services"
	clientutils "github.com/jfrog/jfrog-cli-go/jfrog-client/artifactory/services/utils"
	"github.com/jfrog/jfrog-cli-go/jfrog-client/utils/io/fileutils"
	"github.com/jfrog/jfrog-cli-go/jfrog-client/utils/log"
	"strconv"
)

func Download(downloadSpec *spec.SpecFiles, configuration *DownloadConfiguration) (successCount, failCount int, err error) {
	servicesManager, err := createDownloadServiceManager(configuration.ArtDetails, configuration)
	if err != nil {
		return 0, 0, err
	}
	isCollectBuildInfo := len(configuration.BuildName) > 0 && len(configuration.BuildNumber) > 0
	if isCollectBuildInfo && !configuration.DryRun {
		if err = utils.SaveBuildGeneralDetails(configuration.BuildName, configuration.BuildNumber); err != nil {
			return 0, 0, err
		}
	}
	if !configuration.DryRun {
		err = fileutils.CreateTempDirPath()
		if err != nil {
			return 0, 0, err
		}
		defer fileutils.RemoveTempDir()
	}
	var filesInfo []clientutils.FileInfo
	var totalExpected int
	var errorOccurred = false
	for i := 0; i < len(downloadSpec.Files); i++ {
		params, err := downloadSpec.Get(i).ToArtifatoryDownloadParams()
		if err != nil {
			errorOccurred = true
			log.Error(err)
			continue
		}
		flat, err := downloadSpec.Get(i).IsFlat(false)
		if err != nil {
			errorOccurred = true
			log.Error(err)
			continue
		}

		explode, err := downloadSpec.Get(i).IsExplode(false)
		if err != nil {
			errorOccurred = true
			log.Error(err)
			continue
		}

		currentBuildDependencies, expected, err := servicesManager.DownloadFiles(&services.DownloadParamsImpl{ArtifactoryCommonParams: params, ValidateSymlink: configuration.ValidateSymlink, Symlink: configuration.Symlink, Flat: flat, Explode: explode, Retries: configuration.Retries})
		totalExpected += expected
		filesInfo = append(filesInfo, currentBuildDependencies...)
		if err != nil {
			errorOccurred = true
			log.Error(err)
			continue
		}
	}
	if errorOccurred {
		return len(filesInfo), totalExpected - len(filesInfo), errors.New("Download finished with errors. Please review the logs")
	}
	if configuration.DryRun {
		return totalExpected, 0, err
	}
	log.Debug("Downloaded", strconv.Itoa(len(filesInfo)), "artifacts.")
	buildDependencies := convertFileInfoToBuildDependencies(filesInfo)
	if isCollectBuildInfo {
		populateFunc := func(partial *buildinfo.Partial) {
			partial.Dependencies = buildDependencies
		}
		err = utils.SavePartialBuildInfo(configuration.BuildName, configuration.BuildNumber, populateFunc)
	}

	return len(filesInfo), totalExpected - len(filesInfo), err
}

func convertFileInfoToBuildDependencies(filesInfo []clientutils.FileInfo) []buildinfo.Dependency {
	buildDependecies := make([]buildinfo.Dependency, len(filesInfo))
	for i, fileInfo := range filesInfo {
		dependency := buildinfo.Dependency{Checksum: &buildinfo.Checksum{}}
		dependency.Md5 = fileInfo.Md5
		dependency.Sha1 = fileInfo.Sha1
		filename, _ := fileutils.GetFileAndDirFromPath(fileInfo.ArtifactoryPath)
		dependency.Id = filename
		buildDependecies[i] = dependency
	}
	return buildDependecies
}

type DownloadConfiguration struct {
	Threads         int
	SplitCount      int
	MinSplitSize    int64
	BuildName       string
	BuildNumber     string
	DryRun          bool
	Symlink         bool
	ValidateSymlink bool
	ArtDetails      *config.ArtifactoryDetails
	Retries         int
}

func createDownloadServiceManager(artDetails *config.ArtifactoryDetails, flags *DownloadConfiguration) (*artifactory.ArtifactoryServicesManager, error) {
	certPath, err := utils.GetJfrogSecurityDir()
	if err != nil {
		return nil, err
	}
	artAuth, err := artDetails.CreateArtAuthConfig()
	if err != nil {
		return nil, err
	}
	serviceConfig, err := artifactory.NewConfigBuilder().
		SetArtDetails(artAuth).
		SetDryRun(flags.DryRun).
		SetCertificatesPath(certPath).
		SetSplitCount(flags.SplitCount).
		SetMinSplitSize(flags.MinSplitSize).
		SetThreads(flags.Threads).
		SetLogger(log.Logger).
		Build()
	if err != nil {
		return nil, err
	}
	return artifactory.New(serviceConfig)
}
<commit_msg>Revert "Downloading with dry-run always returns exit code 1 #219 (#230)" (#233)<commit_after>package generic

import (
	"errors"
	"github.com/jfrog/jfrog-cli-go/jfrog-cli/artifactory/utils"
	"github.com/jfrog/jfrog-cli-go/jfrog-cli/artifactory/spec"
	"github.com/jfrog/jfrog-cli-go/jfrog-cli/utils/config"
	"github.com/jfrog/jfrog-cli-go/jfrog-client/artifactory"
	"github.com/jfrog/jfrog-cli-go/jfrog-client/artifactory/services"
	clientutils "github.com/jfrog/jfrog-cli-go/jfrog-client/artifactory/services/utils"
	"github.com/jfrog/jfrog-cli-go/jfrog-client/utils/io/fileutils"
	"github.com/jfrog/jfrog-cli-go/jfrog-client/utils/log"
	"strconv"
	"github.com/jfrog/jfrog-cli-go/jfrog-client/artifactory/buildinfo"
)

func Download(downloadSpec *spec.SpecFiles, configuration *DownloadConfiguration) (successCount, failCount int, err error) {
	servicesManager, err := createDownloadServiceManager(configuration.ArtDetails, configuration)
	if err != nil {
		return 0, 0, err
	}
	isCollectBuildInfo := len(configuration.BuildName) > 0 && len(configuration.BuildNumber) > 0
	if isCollectBuildInfo && !configuration.DryRun {
		if err = utils.SaveBuildGeneralDetails(configuration.BuildName, configuration.BuildNumber); err != nil {
			return 0, 0, err
		}
	}
	if !configuration.DryRun {
		err = fileutils.CreateTempDirPath()
		if err != nil {
			return 0, 0, err
		}
		defer fileutils.RemoveTempDir()
	}
	var filesInfo []clientutils.FileInfo
	var totalExpected int
	var errorOccurred = false
	for i := 0; i < len(downloadSpec.Files); i++ {
		params, err := downloadSpec.Get(i).ToArtifatoryDownloadParams()
		if err != nil {
			errorOccurred = true
			log.Error(err)
			continue
		}
		flat, err := downloadSpec.Get(i).IsFlat(false)
		if err != nil {
			errorOccurred = true
			log.Error(err)
			continue
		}

		explode, err := downloadSpec.Get(i).IsExplode(false)
		if err != nil {
			errorOccurred = true
			log.Error(err)
			continue
		}

		currentBuildDependencies, expected, err := servicesManager.DownloadFiles(&services.DownloadParamsImpl{ArtifactoryCommonParams: params, ValidateSymlink: configuration.ValidateSymlink, Symlink: configuration.Symlink, Flat: flat, Explode: explode, Retries: configuration.Retries})
		totalExpected += expected
		filesInfo = append(filesInfo, currentBuildDependencies...)
		if err != nil {
			errorOccurred = true
			log.Error(err)
			continue
		}
	}
	if errorOccurred {
		return len(filesInfo), totalExpected - len(filesInfo), errors.New("Download finished with errors. Please review the logs")
	}
	log.Debug("Downloaded", strconv.Itoa(len(filesInfo)), "artifacts.")
	buildDependencies := convertFileInfoToBuildDependencies(filesInfo)
	if isCollectBuildInfo && !configuration.DryRun {
		populateFunc := func(partial *buildinfo.Partial) {
			partial.Dependencies = buildDependencies
		}
		err = utils.SavePartialBuildInfo(configuration.BuildName, configuration.BuildNumber, populateFunc)
	}

	return len(filesInfo), totalExpected - len(filesInfo), err
}

func convertFileInfoToBuildDependencies(filesInfo []clientutils.FileInfo) []buildinfo.Dependency {
	buildDependecies := make([]buildinfo.Dependency, len(filesInfo))
	for i, fileInfo := range filesInfo {
		dependency := buildinfo.Dependency{Checksum: &buildinfo.Checksum{}}
		dependency.Md5 = fileInfo.Md5
		dependency.Sha1 = fileInfo.Sha1
		filename, _ := fileutils.GetFileAndDirFromPath(fileInfo.ArtifactoryPath)
		dependency.Id = filename
		buildDependecies[i] = dependency
	}
	return buildDependecies
}

type DownloadConfiguration struct {
	Threads         int
	SplitCount      int
	MinSplitSize    int64
	BuildName       string
	BuildNumber     string
	DryRun          bool
	Symlink         bool
	ValidateSymlink bool
	ArtDetails      *config.ArtifactoryDetails
	Retries         int
}

func createDownloadServiceManager(artDetails *config.ArtifactoryDetails, flags *DownloadConfiguration) (*artifactory.ArtifactoryServicesManager, error) {
	certPath, err := utils.GetJfrogSecurityDir()
	if err != nil {
		return nil, err
	}
	artAuth, err := artDetails.CreateArtAuthConfig()
	if err != nil {
		return nil, err
	}
	serviceConfig, err := artifactory.NewConfigBuilder().
		SetArtDetails(artAuth).
		SetDryRun(flags.DryRun).
		SetCertificatesPath(certPath).
		SetSplitCount(flags.SplitCount).
		SetMinSplitSize(flags.MinSplitSize).
		SetThreads(flags.Threads).
		SetLogger(log.Logger).
		Build()
	if err != nil {
		return nil, err
	}
	return artifactory.New(serviceConfig)
}
<|endoftext|>


<commit_before>//
// Copyright (c) 2016 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

package main

import (
	"flag"
	"fmt"
	"os"
	"text/template"

	"github.com/ciao-project/ciao/ciao-controller/types"
	"github.com/intel/tfortools"
	"github.com/pkg/errors"
)

var imageCommand = &command{
	SubCommands: map[string]subCommand{
		"add":    new(imageAddCommand),
		"show":   new(imageShowCommand),
		"list":   new(imageListCommand),
		"delete": new(imageDeleteCommand),
	},
}

type imageAddCommand struct {
	Flag       flag.FlagSet
	name       string
	id         string
	file       string
	template   string
	visibility string
}

func (cmd *imageAddCommand) usage(...string) {
	fmt.Fprintf(os.Stderr, `usage: ciao-cli [options] image add [flags]

Creates a new image

The add flags are:

`)
	cmd.Flag.PrintDefaults()
	fmt.Fprintf(os.Stderr, "\n%s", tfortools.GenerateUsageDecorated("f", types.Image{}, nil))
	os.Exit(2)
}

func (cmd *imageAddCommand) parseArgs(args []string) []string {
	cmd.Flag.StringVar(&cmd.name, "name", "", "Image Name")
	cmd.Flag.StringVar(&cmd.id, "id", "", "Image UUID")
	cmd.Flag.StringVar(&cmd.file, "file", "", "Image file to upload")
	cmd.Flag.StringVar(&cmd.template, "f", "", "Template used to format output")
	cmd.Flag.StringVar(&cmd.visibility, "visibility", string(types.Private),
		"Image visibility (internal,public,private)")
	cmd.Flag.Usage = func() { cmd.usage() }
	cmd.Flag.Parse(args)
	return cmd.Flag.Args()
}

func (cmd *imageAddCommand) run(args []string) error {
	if cmd.name == "" {
		return errors.New("Missing required -name parameter")
	}

	if cmd.file == "" {
		return errors.New("Missing required -file parameter")
	}

	f, err := os.Open(cmd.file)
	if err != nil {
		fatalf("Could not open %s [%s]\n", cmd.file, err)
	}
	defer func() { _ = f.Close() }()

	imageVisibility := types.Private
	if cmd.visibility != "" {
		imageVisibility = types.Visibility(cmd.visibility)
		switch imageVisibility {
		case types.Public, types.Private, types.Internal:
		default:
			fatalf("Invalid image visibility [%v]", imageVisibility)
		}
	}

	id, err := c.CreateImage(cmd.name, imageVisibility, cmd.id, f)
	if err != nil {
		return errors.Wrap(err, "Error creating image")
	}

	image, err := c.GetImage(id)
	if err != nil {
		return errors.Wrap(err, "Error getting image")
	}

	if cmd.template != "" {
		return tfortools.OutputToTemplate(os.Stdout, "image-add", cmd.template, image, nil)
	}

	fmt.Printf("Created image:\n")
	dumpImage(&image)
	return nil
}

type imageShowCommand struct {
	Flag     flag.FlagSet
	image    string
	template string
}

func (cmd *imageShowCommand) usage(...string) {
	fmt.Fprintf(os.Stderr, `usage: ciao-cli [options] image show

Show images
`)
	cmd.Flag.PrintDefaults()
	fmt.Fprintf(os.Stderr, "\n%s", tfortools.GenerateUsageDecorated("f", types.Image{}, nil))
	os.Exit(2)
}

func (cmd *imageShowCommand) parseArgs(args []string) []string {
	cmd.Flag.StringVar(&cmd.template, "f", "", "Template used to format output")
	cmd.Flag.StringVar(&cmd.image, "image", "", "Image UUID")
	cmd.Flag.Usage = func() { cmd.usage() }
	cmd.Flag.Parse(args)
	return cmd.Flag.Args()
}

func (cmd *imageShowCommand) run(args []string) error {
	if cmd.image == "" {
		return errors.New("Missing required -image parameter")
	}

	i, err := c.GetImage(cmd.image)
	if err != nil {
		return errors.Wrap(err, "Error getting image")
	}

	if cmd.template != "" {
		return tfortools.OutputToTemplate(os.Stdout, "image-show", cmd.template, i, nil)
	}

	dumpImage(&i)

	return nil
}

type imageListCommand struct {
	Flag     flag.FlagSet
	template string
}

func (cmd *imageListCommand) usage(...string) {
	fmt.Fprintf(os.Stderr, `usage: ciao-cli [options] image list

List images
`)
	cmd.Flag.PrintDefaults()
	fmt.Fprintf(os.Stderr, `
The template passed to the -f option operates on a

%s

As images are retrieved in pages, the template may be applied multiple
times.  You can not therefore rely on the length of the slice passed
to the template to determine the total number of images.
`, tfortools.GenerateUsageUndecorated([]types.Image{}))
	fmt.Fprintln(os.Stderr, tfortools.TemplateFunctionHelp(nil))
	os.Exit(2)
}

func (cmd *imageListCommand) parseArgs(args []string) []string {
	cmd.Flag.StringVar(&cmd.template, "f", "", "Template used to format output")
	cmd.Flag.Usage = func() { cmd.usage() }
	cmd.Flag.Parse(args)
	return cmd.Flag.Args()
}

func (cmd *imageListCommand) run(args []string) error {
	var t *template.Template
	var err error
	if cmd.template != "" {
		t, err = tfortools.CreateTemplate("image-list", cmd.template, nil)
		if err != nil {
			fatalf(err.Error())
		}
	}

	images, err := c.ListImages()
	if err != nil {
		return errors.Wrap(err, "Error listing images")
	}

	if t != nil {
		if err = t.Execute(os.Stdout, &images); err != nil {
			fatalf(err.Error())
		}
		return nil
	}

	for k, i := range images {
		fmt.Printf("Image #%d\n", k+1)
		dumpImage(&i)
		fmt.Printf("\n")
	}

	return err
}

type imageDeleteCommand struct {
	Flag  flag.FlagSet
	image string
}

func (cmd *imageDeleteCommand) usage(...string) {
	fmt.Fprintf(os.Stderr, `usage: ciao-cli [options] image delete [flags]

Deletes an image

The delete flags are:

`)
	cmd.Flag.PrintDefaults()
	os.Exit(2)
}

func (cmd *imageDeleteCommand) parseArgs(args []string) []string {
	cmd.Flag.StringVar(&cmd.image, "image", "", "Image UUID")
	cmd.Flag.Usage = func() { cmd.usage() }
	cmd.Flag.Parse(args)
	return cmd.Flag.Args()
}

func (cmd *imageDeleteCommand) run(args []string) error {
	err := c.DeleteImage(cmd.image)
	if err != nil {
		return errors.Wrap(err, "Error deleting image")
	}

	fmt.Printf("Deleted image %s\n", cmd.image)

	return nil
}

func dumpImage(i *types.Image) {
	fmt.Printf("\tName\t\t[%s]\n", i.Name)
	fmt.Printf("\tSize\t\t[%d bytes]\n", i.Size)
	fmt.Printf("\tID\t\t[%s]\n", i.ID)
	fmt.Printf("\tState\t\t[%s]\n", i.State)
	fmt.Printf("\tVisibility\t[%s]\n", i.Visibility)
	fmt.Printf("\tCreateTime\t[%s]\n", i.CreateTime)
}
<commit_msg>ciao-cli: enforce image name regexp in ciao-cli<commit_after>//
// Copyright (c) 2016 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

package main

import (
	"flag"
	"fmt"
	"os"
	"regexp"
	"text/template"

	"github.com/ciao-project/ciao/ciao-controller/types"
	"github.com/intel/tfortools"
	"github.com/pkg/errors"
)

var imageCommand = &command{
	SubCommands: map[string]subCommand{
		"add":    new(imageAddCommand),
		"show":   new(imageShowCommand),
		"list":   new(imageListCommand),
		"delete": new(imageDeleteCommand),
	},
}

type imageAddCommand struct {
	Flag       flag.FlagSet
	name       string
	id         string
	file       string
	template   string
	visibility string
}

func (cmd *imageAddCommand) usage(...string) {
	fmt.Fprintf(os.Stderr, `usage: ciao-cli [options] image add [flags]

Creates a new image

The add flags are:

`)
	cmd.Flag.PrintDefaults()
	fmt.Fprintf(os.Stderr, "\n%s", tfortools.GenerateUsageDecorated("f", types.Image{}, nil))
	os.Exit(2)
}

func (cmd *imageAddCommand) parseArgs(args []string) []string {
	cmd.Flag.StringVar(&cmd.name, "name", "", "Image Name")
	cmd.Flag.StringVar(&cmd.id, "id", "", "Image UUID")
	cmd.Flag.StringVar(&cmd.file, "file", "", "Image file to upload")
	cmd.Flag.StringVar(&cmd.template, "f", "", "Template used to format output")
	cmd.Flag.StringVar(&cmd.visibility, "visibility", string(types.Private),
		"Image visibility (internal,public,private)")
	cmd.Flag.Usage = func() { cmd.usage() }
	cmd.Flag.Parse(args)
	return cmd.Flag.Args()
}

func (cmd *imageAddCommand) run(args []string) error {
	if cmd.name == "" {
		return errors.New("Missing required -name parameter")
	}

	r := regexp.MustCompile("^[a-z0-9-.]{1,64}$")
	if !r.MatchString(cmd.name) {
		return errors.New("Requested name must be between 1 and 64 lowercase letters, numbers, hyphens and dots")
	}

	if cmd.file == "" {
		return errors.New("Missing required -file parameter")
	}

	f, err := os.Open(cmd.file)
	if err != nil {
		fatalf("Could not open %s [%s]\n", cmd.file, err)
	}
	defer func() { _ = f.Close() }()

	imageVisibility := types.Private
	if cmd.visibility != "" {
		imageVisibility = types.Visibility(cmd.visibility)
		switch imageVisibility {
		case types.Public, types.Private, types.Internal:
		default:
			fatalf("Invalid image visibility [%v]", imageVisibility)
		}
	}

	id, err := c.CreateImage(cmd.name, imageVisibility, cmd.id, f)
	if err != nil {
		return errors.Wrap(err, "Error creating image")
	}

	image, err := c.GetImage(id)
	if err != nil {
		return errors.Wrap(err, "Error getting image")
	}

	if cmd.template != "" {
		return tfortools.OutputToTemplate(os.Stdout, "image-add", cmd.template, image, nil)
	}

	fmt.Printf("Created image:\n")
	dumpImage(&image)
	return nil
}

type imageShowCommand struct {
	Flag     flag.FlagSet
	image    string
	template string
}

func (cmd *imageShowCommand) usage(...string) {
	fmt.Fprintf(os.Stderr, `usage: ciao-cli [options] image show

Show images
`)
	cmd.Flag.PrintDefaults()
	fmt.Fprintf(os.Stderr, "\n%s", tfortools.GenerateUsageDecorated("f", types.Image{}, nil))
	os.Exit(2)
}

func (cmd *imageShowCommand) parseArgs(args []string) []string {
	cmd.Flag.StringVar(&cmd.template, "f", "", "Template used to format output")
	cmd.Flag.StringVar(&cmd.image, "image", "", "Image UUID")
	cmd.Flag.Usage = func() { cmd.usage() }
	cmd.Flag.Parse(args)
	return cmd.Flag.Args()
}

func (cmd *imageShowCommand) run(args []string) error {
	if cmd.image == "" {
		return errors.New("Missing required -image parameter")
	}

	i, err := c.GetImage(cmd.image)
	if err != nil {
		return errors.Wrap(err, "Error getting image")
	}

	if cmd.template != "" {
		return tfortools.OutputToTemplate(os.Stdout, "image-show", cmd.template, i, nil)
	}

	dumpImage(&i)

	return nil
}

type imageListCommand struct {
	Flag     flag.FlagSet
	template string
}

func (cmd *imageListCommand) usage(...string) {
	fmt.Fprintf(os.Stderr, `usage: ciao-cli [options] image list

List images
`)
	cmd.Flag.PrintDefaults()
	fmt.Fprintf(os.Stderr, `
The template passed to the -f option operates on a

%s

As images are retrieved in pages, the template may be applied multiple
times.  You can not therefore rely on the length of the slice passed
to the template to determine the total number of images.
`, tfortools.GenerateUsageUndecorated([]types.Image{}))
	fmt.Fprintln(os.Stderr, tfortools.TemplateFunctionHelp(nil))
	os.Exit(2)
}

func (cmd *imageListCommand) parseArgs(args []string) []string {
	cmd.Flag.StringVar(&cmd.template, "f", "", "Template used to format output")
	cmd.Flag.Usage = func() { cmd.usage() }
	cmd.Flag.Parse(args)
	return cmd.Flag.Args()
}

func (cmd *imageListCommand) run(args []string) error {
	var t *template.Template
	var err error
	if cmd.template != "" {
		t, err = tfortools.CreateTemplate("image-list", cmd.template, nil)
		if err != nil {
			fatalf(err.Error())
		}
	}

	images, err := c.ListImages()
	if err != nil {
		return errors.Wrap(err, "Error listing images")
	}

	if t != nil {
		if err = t.Execute(os.Stdout, &images); err != nil {
			fatalf(err.Error())
		}
		return nil
	}

	for k, i := range images {
		fmt.Printf("Image #%d\n", k+1)
		dumpImage(&i)
		fmt.Printf("\n")
	}

	return err
}

type imageDeleteCommand struct {
	Flag  flag.FlagSet
	image string
}

func (cmd *imageDeleteCommand) usage(...string) {
	fmt.Fprintf(os.Stderr, `usage: ciao-cli [options] image delete [flags]

Deletes an image

The delete flags are:

`)
	cmd.Flag.PrintDefaults()
	os.Exit(2)
}

func (cmd *imageDeleteCommand) parseArgs(args []string) []string {
	cmd.Flag.StringVar(&cmd.image, "image", "", "Image UUID")
	cmd.Flag.Usage = func() { cmd.usage() }
	cmd.Flag.Parse(args)
	return cmd.Flag.Args()
}

func (cmd *imageDeleteCommand) run(args []string) error {
	err := c.DeleteImage(cmd.image)
	if err != nil {
		return errors.Wrap(err, "Error deleting image")
	}

	fmt.Printf("Deleted image %s\n", cmd.image)

	return nil
}

func dumpImage(i *types.Image) {
	fmt.Printf("\tName\t\t[%s]\n", i.Name)
	fmt.Printf("\tSize\t\t[%d bytes]\n", i.Size)
	fmt.Printf("\tID\t\t[%s]\n", i.ID)
	fmt.Printf("\tState\t\t[%s]\n", i.State)
	fmt.Printf("\tVisibility\t[%s]\n", i.Visibility)
	fmt.Printf("\tCreateTime\t[%s]\n", i.CreateTime)
}
<|endoftext|>