<commit_before>// Copyright 2015 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package service

import (
	"fmt"
	"io"
	"net/url"
	"os"

	"github.com/juju/errors"
	"github.com/juju/utils"
	"github.com/juju/version"
	"gopkg.in/juju/charm.v6-unstable"
	"gopkg.in/juju/charmrepo.v2-unstable"
	"gopkg.in/juju/charmrepo.v2-unstable/csclient"
	csparams "gopkg.in/juju/charmrepo.v2-unstable/csclient/params"
	"gopkg.in/macaroon-bakery.v1/httpbakery"
	"gopkg.in/macaroon.v1"

	"github.com/juju/juju/apiserver/params"
	"github.com/juju/juju/environs/config"
	"github.com/juju/juju/state"
	jujuversion "github.com/juju/juju/version"
)

// TODO - we really want to avoid this, which we can do by refactoring code requiring this
// to use interfaces.
// NewCharmStore instantiates a new charm store repository.
// It is defined at top level for testing purposes.
var NewCharmStore = newCharmStore

func newCharmStore(p charmrepo.NewCharmStoreParams) charmrepo.Interface {
	return charmrepo.NewCharmStore(p)
}

// AddCharmWithAuthorization adds the given charm URL (which must include revision) to
// the environment, if it does not exist yet. Local charms are not
// supported, only charm store URLs. See also AddLocalCharm().
//
// The authorization macaroon, args.CharmStoreMacaroon, may be
// omitted, in which case this call is equivalent to AddCharm.
func AddCharmWithAuthorization(st *state.State, args params.AddCharmWithAuthorization) error {
	charmURL, err := charm.ParseURL(args.URL)
	if err != nil {
		return err
	}
	if charmURL.Schema != "cs" {
		return fmt.Errorf("only charm store charm URLs are supported, with cs: schema")
	}
	if charmURL.Revision < 0 {
		return fmt.Errorf("charm URL must include revision")
	}

	// First, check if a pending or a real charm exists in state.
	stateCharm, err := st.PrepareStoreCharmUpload(charmURL)
	if err != nil {
		return err
	}
	if stateCharm.IsUploaded() {
		// Charm already in state (it was uploaded already).
		return nil
	}

	// Open a charm store client.
	envConfig, err := st.ModelConfig()
	if err != nil {
		return err
	}
	csURL, err := url.Parse(csclient.ServerURL)
	if err != nil {
		return err
	}
	csParams := csclient.Params{
		URL:        csURL.String(),
		HTTPClient: httpbakery.NewHTTPClient(),
	}

	if args.CharmStoreMacaroon != nil {
		// Set the provided charmstore authorizing macaroon
		// as a cookie in the HTTP client.
		// TODO(cmars) discharge any third party caveats in the macaroon.
		ms := []*macaroon.Macaroon{args.CharmStoreMacaroon}
		httpbakery.SetCookie(csParams.HTTPClient.Jar, csURL, ms)
	}
	csClient := csclient.New(csParams)
	channel := csparams.Channel(args.Channel)
	if channel != csparams.NoChannel {
		csClient = csClient.WithChannel(channel)
	}
	repo := charmrepo.NewCharmStoreFromClient(csClient)
	repo = config.SpecializeCharmRepo(repo, envConfig).(*charmrepo.CharmStore)

	// Get the charm and its information from the store.
	downloadedCharm, err := repo.Get(charmURL)
	if err != nil {
		cause := errors.Cause(err)
		if httpbakery.IsDischargeError(cause) || httpbakery.IsInteractionError(cause) {
			return errors.NewUnauthorized(err, "")
		}
		return errors.Trace(err)
	}

	if err := checkMinVersion(downloadedCharm); err != nil {
		return errors.Trace(err)
	}

	// Open it and calculate the SHA256 hash.
	downloadedBundle, ok := downloadedCharm.(*charm.CharmArchive)
	if !ok {
		return errors.Errorf("expected a charm archive, got %T", downloadedCharm)
	}
	archive, err := os.Open(downloadedBundle.Path)
	if err != nil {
		return errors.Annotate(err, "cannot read downloaded charm")
	}
	defer archive.Close()
	bundleSHA256, size, err := utils.ReadSHA256(archive)
	if err != nil {
		return errors.Annotate(err, "cannot calculate SHA256 hash of charm")
	}
	if _, err := archive.Seek(0, 0); err != nil {
		return errors.Annotate(err, "cannot rewind charm archive")
	}

	// Store the charm archive in environment storage.
	return StoreCharmArchive(
		st,
		CharmArchive{
			ID:     charmURL,
			Charm:  downloadedCharm,
			Data:   archive,
			Size:   size,
			SHA256: bundleSHA256,
		},
	)
}

func checkMinVersion(ch charm.Charm) error {
	minver := ch.Meta().MinJujuVersion
	if minver != version.Zero && minver.Compare(jujuversion.Current) > 0 {
		return minVersionError(minver, jujuversion.Current)
	}
	return nil
}

type minJujuVersionErr struct {
	*errors.Err
}

func minVersionError(minver, jujuver version.Number) error {
	err := errors.NewErr("charm's min version (%s) is higher than this juju environment's version (%s)",
		minver, jujuver)
	err.SetLocation(1)
	return minJujuVersionErr{&err}
}

// CharmArchive is the data that needs to be stored for a charm archive in
// state.
type CharmArchive struct {
	// ID is the charm URL for which we're storing the archive.
	ID *charm.URL

	// Charm is the metadata about the charm for the archive.
	Charm charm.Charm

	// Data contains the bytes of the archive.
	Data io.Reader

	// Size is the number of bytes in Data.
	Size int64

	// SHA256 is the hash of the bytes in Data.
	SHA256 string
}

// StoreCharmArchive stores a charm archive in environment storage.
func StoreCharmArchive(st *state.State, archive CharmArchive) error {
	storage := newStateStorage(st.ModelUUID(), st.MongoSession())
	storagePath, err := charmArchiveStoragePath(archive.ID)
	if err != nil {
		return errors.Annotate(err, "cannot generate charm archive name")
	}
	if err := storage.Put(storagePath, archive.Data, archive.Size); err != nil {
		return errors.Annotate(err, "cannot add charm to storage")
	}

	info := state.CharmInfo{
		Charm:       archive.Charm,
		ID:          archive.ID,
		StoragePath: storagePath,
		SHA256:      archive.SHA256,
	}

	// Now update the charm data in state and mark it as no longer pending.
	_, err = st.UpdateUploadedCharm(info)
	if err != nil {
		alreadyUploaded := err == state.ErrCharmRevisionAlreadyModified ||
			errors.Cause(err) == state.ErrCharmRevisionAlreadyModified ||
			state.IsCharmAlreadyUploadedError(err)
		if err := storage.Remove(storagePath); err != nil {
			if alreadyUploaded {
				logger.Errorf("cannot remove duplicated charm archive from storage: %v", err)
			} else {
				logger.Errorf("cannot remove unsuccessfully recorded charm archive from storage: %v", err)
			}
		}
		if alreadyUploaded {
			// Somebody else managed to upload and update the charm in
			// state before us. This is not an error.
			return nil
		}
	}
	return nil
}

// charmArchiveStoragePath returns a string that is suitable as a
// storage path, using a random UUID to avoid colliding with concurrent
// uploads.
func charmArchiveStoragePath(curl *charm.URL) (string, error) {
	uuid, err := utils.NewUUID()
	if err != nil {
		return "", err
	}
	return fmt.Sprintf("charms/%s-%s", curl.String(), uuid), nil
}

// ResolveCharm resolves the best available charm URLs with series, for charm
// locations without a series specified.
func ResolveCharms(st *state.State, args params.ResolveCharms) (params.ResolveCharmResults, error) {
	var results params.ResolveCharmResults

	envConfig, err := st.ModelConfig()
	if err != nil {
		return params.ResolveCharmResults{}, err
	}
	repo := config.SpecializeCharmRepo(
		NewCharmStore(charmrepo.NewCharmStoreParams{}),
		envConfig)

	for _, ref := range args.References {
		result := params.ResolveCharmResult{}
		curl, err := resolveCharm(&ref, repo)
		if err != nil {
			result.Error = err.Error()
		} else {
			result.URL = curl
		}
		results.URLs = append(results.URLs, result)
	}
	return results, nil
}

func resolveCharm(ref *charm.URL, repo charmrepo.Interface) (*charm.URL, error) {
	if ref.Schema != "cs" {
		return nil, fmt.Errorf("only charm store charm references are supported, with cs: schema")
	}

	// Resolve the charm location with the repository.
	resolved, _, err := repo.Resolve(ref)
	if err != nil {
		return nil, err
	}
	if resolved.Series == "" {
		return nil, errors.Errorf("no series found in charm URL %q", resolved)
	}
	return resolved.WithRevision(ref.Revision), nil
}
<commit_msg>Factor out openCSRepo().<commit_after>// Copyright 2015 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package service

import (
	"fmt"
	"io"
	"net/url"
	"os"

	"github.com/juju/errors"
	"github.com/juju/utils"
	"github.com/juju/version"
	"gopkg.in/juju/charm.v6-unstable"
	"gopkg.in/juju/charmrepo.v2-unstable"
	"gopkg.in/juju/charmrepo.v2-unstable/csclient"
	"gopkg.in/macaroon-bakery.v1/httpbakery"
	"gopkg.in/macaroon.v1"

	"github.com/juju/juju/apiserver/params"
	"github.com/juju/juju/environs/config"
	"github.com/juju/juju/state"
	jujuversion "github.com/juju/juju/version"
)

// TODO - we really want to avoid this, which we can do by refactoring code requiring this
// to use interfaces.
// NewCharmStore instantiates a new charm store repository.
// It is exported for testing purposes.
var NewCharmStore = newCharmStore

func newCharmStore(p charmrepo.NewCharmStoreParams) charmrepo.Interface {
	return charmrepo.NewCharmStore(p)
}

// AddCharmWithAuthorization adds the given charm URL (which must include revision) to
// the environment, if it does not exist yet. Local charms are not
// supported, only charm store URLs. See also AddLocalCharm().
//
// The authorization macaroon, args.CharmStoreMacaroon, may be
// omitted, in which case this call is equivalent to AddCharm.
func AddCharmWithAuthorization(st *state.State, args params.AddCharmWithAuthorization) error {
	charmURL, err := charm.ParseURL(args.URL)
	if err != nil {
		return err
	}
	if charmURL.Schema != "cs" {
		return fmt.Errorf("only charm store charm URLs are supported, with cs: schema")
	}
	if charmURL.Revision < 0 {
		return fmt.Errorf("charm URL must include revision")
	}

	// First, check if a pending or a real charm exists in state.
	stateCharm, err := st.PrepareStoreCharmUpload(charmURL)
	if err != nil {
		return err
	}
	if stateCharm.IsUploaded() {
		// Charm already in state (it was uploaded already).
		return nil
	}

	// Open a charm store client.
	repo, err := openCSRepo(args)
	if err != nil {
		return err
	}
	envConfig, err := st.ModelConfig()
	if err != nil {
		return err
	}
	repo = config.SpecializeCharmRepo(repo, envConfig).(*charmrepo.CharmStore)

	// Get the charm and its information from the store.
	downloadedCharm, err := repo.Get(charmURL)
	if err != nil {
		cause := errors.Cause(err)
		if httpbakery.IsDischargeError(cause) || httpbakery.IsInteractionError(cause) {
			return errors.NewUnauthorized(err, "")
		}
		return errors.Trace(err)
	}

	if err := checkMinVersion(downloadedCharm); err != nil {
		return errors.Trace(err)
	}

	// Open it and calculate the SHA256 hash.
	downloadedBundle, ok := downloadedCharm.(*charm.CharmArchive)
	if !ok {
		return errors.Errorf("expected a charm archive, got %T", downloadedCharm)
	}
	archive, err := os.Open(downloadedBundle.Path)
	if err != nil {
		return errors.Annotate(err, "cannot read downloaded charm")
	}
	defer archive.Close()
	bundleSHA256, size, err := utils.ReadSHA256(archive)
	if err != nil {
		return errors.Annotate(err, "cannot calculate SHA256 hash of charm")
	}
	if _, err := archive.Seek(0, 0); err != nil {
		return errors.Annotate(err, "cannot rewind charm archive")
	}

	// Store the charm archive in environment storage.
	return StoreCharmArchive(
		st,
		CharmArchive{
			ID:     charmURL,
			Charm:  downloadedCharm,
			Data:   archive,
			Size:   size,
			SHA256: bundleSHA256,
		},
	)
}

func openCSRepo(args params.AddCharmWithAuthorization) (charmrepo.Interface, error) {
	csURL, err := url.Parse(csclient.ServerURL)
	if err != nil {
		return nil, err
	}
	csParams := charmrepo.NewCharmStoreParams{
		URL:        csURL.String(),
		HTTPClient: httpbakery.NewHTTPClient(),
	}

	if args.CharmStoreMacaroon != nil {
		// Set the provided charmstore authorizing macaroon
		// as a cookie in the HTTP client.
		// TODO(cmars) discharge any third party caveats in the macaroon.
		ms := []*macaroon.Macaroon{args.CharmStoreMacaroon}
		httpbakery.SetCookie(csParams.HTTPClient.Jar, csURL, ms)
	}

	repo := NewCharmStore(csParams)
	return repo, nil
}

func checkMinVersion(ch charm.Charm) error {
	minver := ch.Meta().MinJujuVersion
	if minver != version.Zero && minver.Compare(jujuversion.Current) > 0 {
		return minVersionError(minver, jujuversion.Current)
	}
	return nil
}

type minJujuVersionErr struct {
	*errors.Err
}

func minVersionError(minver, jujuver version.Number) error {
	err := errors.NewErr("charm's min version (%s) is higher than this juju environment's version (%s)",
		minver, jujuver)
	err.SetLocation(1)
	return minJujuVersionErr{&err}
}

// CharmArchive is the data that needs to be stored for a charm archive in
// state.
type CharmArchive struct {
	// ID is the charm URL for which we're storing the archive.
	ID *charm.URL

	// Charm is the metadata about the charm for the archive.
	Charm charm.Charm

	// Data contains the bytes of the archive.
	Data io.Reader

	// Size is the number of bytes in Data.
	Size int64

	// SHA256 is the hash of the bytes in Data.
	SHA256 string
}

// StoreCharmArchive stores a charm archive in environment storage.
func StoreCharmArchive(st *state.State, archive CharmArchive) error {
	storage := newStateStorage(st.ModelUUID(), st.MongoSession())
	storagePath, err := charmArchiveStoragePath(archive.ID)
	if err != nil {
		return errors.Annotate(err, "cannot generate charm archive name")
	}
	if err := storage.Put(storagePath, archive.Data, archive.Size); err != nil {
		return errors.Annotate(err, "cannot add charm to storage")
	}

	info := state.CharmInfo{
		Charm:       archive.Charm,
		ID:          archive.ID,
		StoragePath: storagePath,
		SHA256:      archive.SHA256,
	}

	// Now update the charm data in state and mark it as no longer pending.
	_, err = st.UpdateUploadedCharm(info)
	if err != nil {
		alreadyUploaded := err == state.ErrCharmRevisionAlreadyModified ||
			errors.Cause(err) == state.ErrCharmRevisionAlreadyModified ||
			state.IsCharmAlreadyUploadedError(err)
		if err := storage.Remove(storagePath); err != nil {
			if alreadyUploaded {
				logger.Errorf("cannot remove duplicated charm archive from storage: %v", err)
			} else {
				logger.Errorf("cannot remove unsuccessfully recorded charm archive from storage: %v", err)
			}
		}
		if alreadyUploaded {
			// Somebody else managed to upload and update the charm in
			// state before us. This is not an error.
			return nil
		}
	}
	return nil
}

// charmArchiveStoragePath returns a string that is suitable as a
// storage path, using a random UUID to avoid colliding with concurrent
// uploads.
func charmArchiveStoragePath(curl *charm.URL) (string, error) {
	uuid, err := utils.NewUUID()
	if err != nil {
		return "", err
	}
	return fmt.Sprintf("charms/%s-%s", curl.String(), uuid), nil
}

// ResolveCharm resolves the best available charm URLs with series, for charm
// locations without a series specified.
func ResolveCharms(st *state.State, args params.ResolveCharms) (params.ResolveCharmResults, error) {
	var results params.ResolveCharmResults

	envConfig, err := st.ModelConfig()
	if err != nil {
		return params.ResolveCharmResults{}, err
	}
	repo := config.SpecializeCharmRepo(
		NewCharmStore(charmrepo.NewCharmStoreParams{}),
		envConfig)

	for _, ref := range args.References {
		result := params.ResolveCharmResult{}
		curl, err := resolveCharm(&ref, repo)
		if err != nil {
			result.Error = err.Error()
		} else {
			result.URL = curl
		}
		results.URLs = append(results.URLs, result)
	}
	return results, nil
}

func resolveCharm(ref *charm.URL, repo charmrepo.Interface) (*charm.URL, error) {
	if ref.Schema != "cs" {
		return nil, fmt.Errorf("only charm store charm references are supported, with cs: schema")
	}

	// Resolve the charm location with the repository.
	resolved, _, err := repo.Resolve(ref)
	if err != nil {
		return nil, err
	}
	if resolved.Series == "" {
		return nil, errors.Errorf("no series found in charm URL %q", resolved)
	}
	return resolved.WithRevision(ref.Revision), nil
}
<|endoftext|>


<commit_before>package evaluation

import (
	"github.com/sjwhitworth/golearn/base"
	"github.com/sjwhitworth/golearn/trees"
	. "github.com/smartystreets/goconvey/convey"
	"testing"
)

func TestCrossFold(t *testing.T) {
	Convey("Cross Fold Evaluation", t, func() {
		iris, _ := base.ParseCSVToInstances("../examples/datasets/iris_headers.csv", true)
		tree := trees.NewID3DecisionTree(0.6)
		cfs, _ := GenerateCrossFoldValidationConfusionMatrices(iris, tree, 5)
		Convey("Cross Fold Validation Confusion Matrices", func() {
			So(cfs, ShouldNotBeEmpty)
		})
		Convey("Cross Validated Metric", func() {
			mean, variance := GetCrossValidatedMetric(cfs, GetAccuracy)
			So(mean, ShouldBeBetween, 0.8, 1)
			So(variance, ShouldBeBetween, 0, 0.03)
		})
	})
}
<commit_msg>Fix import cycle<commit_after>package evaluation

import (
	"github.com/sjwhitworth/golearn/base"
	"github.com/sjwhitworth/golearn/knn"
	. "github.com/smartystreets/goconvey/convey"
	"testing"
)

func TestCrossFold(t *testing.T) {
	Convey("Cross Fold Evaluation", t, func() {
		iris, _ := base.ParseCSVToInstances("../examples/datasets/iris_headers.csv", true)
                cls := knn.NewKnnClassifier("euclidean", "linear", 2)
		cfs, _ := GenerateCrossFoldValidationConfusionMatrices(iris, cls, 5)
		Convey("Cross Fold Validation Confusion Matrices", func() {
			So(cfs, ShouldNotBeEmpty)
		})
		Convey("Cross Validated Metric", func() {
			mean, variance := GetCrossValidatedMetric(cfs, GetAccuracy)
			So(mean, ShouldBeBetween, 0.8, 1)
			So(variance, ShouldBeBetween, 0, 0.03)
		})
	})
}
<|endoftext|>


<commit_before>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"flag"
	"fmt"
	"os"
	"path"
	"time"

	"github.com/golang/glog"
	ginkgoconfig "github.com/onsi/ginkgo/config"
	ginkgoreporters "github.com/onsi/ginkgo/reporters"
	ginkgotypes "github.com/onsi/ginkgo/types"
	"github.com/spf13/pflag"
	"k8s.io/perf-tests/clusterloader2/pkg/config"
	"k8s.io/perf-tests/clusterloader2/pkg/errors"
	"k8s.io/perf-tests/clusterloader2/pkg/framework"
	"k8s.io/perf-tests/clusterloader2/pkg/test"
	"k8s.io/perf-tests/clusterloader2/pkg/util"

	_ "k8s.io/perf-tests/clusterloader2/pkg/measurement/common/bundle"
	_ "k8s.io/perf-tests/clusterloader2/pkg/measurement/common/simple"
	_ "k8s.io/perf-tests/clusterloader2/pkg/measurement/common/slos"
)

var (
	clusterLoaderConfig config.ClusterLoaderConfig
	testConfigPaths     []string
)

func initClusterFlags() {
	pflag.StringVar(&clusterLoaderConfig.ClusterConfig.KubeConfigPath, "kubeconfig", "", "Path to the kubeconfig file")
	pflag.IntVar(&clusterLoaderConfig.ClusterConfig.Nodes, "nodes", 0, "number of nodes")
	pflag.StringVar(&clusterLoaderConfig.ClusterConfig.Provider, "provider", "", "Cluster provider")
}

func validateClusterFlags() *errors.ErrorList {
	errList := errors.NewErrorList()
	if clusterLoaderConfig.ClusterConfig.KubeConfigPath == "" {
		errList.Append(fmt.Errorf("no kubeconfig path specified"))
	}
	return errList
}

func initFlags() {
	pflag.CommandLine = pflag.NewFlagSet(os.Args[0], pflag.ContinueOnError)
	pflag.CommandLine.AddGoFlagSet(flag.CommandLine)
	pflag.StringVar(&clusterLoaderConfig.ReportDir, "report-dir", "", "Path to the directory where the reports should be saved. Default is empty, which cause reports being written to standard output.")
	pflag.StringArrayVar(&testConfigPaths, "testconfig", []string{}, "Paths to the test config files")
	pflag.StringVar(&clusterLoaderConfig.TestOverridesPath, "testoverrides", "", "Path to the config overrides file")
	initClusterFlags()
}

func validateFlags() *errors.ErrorList {
	errList := errors.NewErrorList()
	if len(testConfigPaths) < 1 {
		errList.Append(fmt.Errorf("no test config path specified"))
	}
	errList.Concat(validateClusterFlags())
	return errList
}

func completeConfig(f *framework.Framework) error {
	if clusterLoaderConfig.ClusterConfig.Nodes == 0 {
		nodes, err := util.GetSchedulableUntainedNodesNumber(f.GetClientSet())
		if err != nil {
			return fmt.Errorf("getting number of nodes error: %v", err)
		}
		clusterLoaderConfig.ClusterConfig.Nodes = nodes
		glog.Infof("ClusterConfig.Nodes set to %v", nodes)
	}
	if clusterLoaderConfig.ClusterConfig.MasterName == "" {
		masterName, err := util.GetMasterName(f.GetClientSet())
		if err == nil {
			clusterLoaderConfig.ClusterConfig.MasterName = masterName
			glog.Infof("ClusterConfig.MasterName set to %v", masterName)
		} else {
			glog.Errorf("Getting master name error: %v", err)
		}
	}
	if clusterLoaderConfig.ClusterConfig.MasterIP == "" {
		masterIP, err := util.GetMasterExternalIP(f.GetClientSet())
		if err == nil {
			clusterLoaderConfig.ClusterConfig.MasterIP = masterIP
			glog.Infof("ClusterConfig.MasterIP set to %v", masterIP)
		} else {
			glog.Errorf("Getting master ip error: %v", err)
		}
	}
	return nil
}

func createReportDir() error {
	if clusterLoaderConfig.ReportDir != "" {
		if _, err := os.Stat(clusterLoaderConfig.ReportDir); err != nil {
			if !os.IsNotExist(err) {
				return err
			}
			if err = os.Mkdir(clusterLoaderConfig.ReportDir, 0755); err != nil {
				return fmt.Errorf("report directory creation error: %v", err)
			}
		}
	}
	return nil
}

func main() {
	defer glog.Flush()
	initFlags()
	if err := pflag.CommandLine.Parse(os.Args[1:]); err != nil {
		glog.Fatalf("Flag parse failed: %v", err)
	}
	if errList := validateFlags(); !errList.IsEmpty() {
		glog.Fatalf("Parsing flags error: %v", errList.String())
	}

	f, err := framework.NewFramework(clusterLoaderConfig.ClusterConfig.KubeConfigPath)
	if err != nil {
		glog.Fatalf("Framework creation error: %v", err)
	}

	if err = completeConfig(f); err != nil {
		glog.Fatalf("Config completing error: %v", err)
	}

	if err = createReportDir(); err != nil {
		glog.Fatalf("Cannot create report directory: %v", err)
	}

	if err = util.LogClusterNodes(f.GetClientSet()); err != nil {
		glog.Errorf("Nodes info logging error: %v", err)
	}

	suiteSummary := &ginkgotypes.SuiteSummary{
		SuiteDescription:           "ClusterLoaderV2",
		NumberOfSpecsThatWillBeRun: len(testConfigPaths),
	}
	junitReporter := ginkgoreporters.NewJUnitReporter(path.Join(clusterLoaderConfig.ReportDir, "junit.xml"))
	junitReporter.SpecSuiteWillBegin(ginkgoconfig.GinkgoConfig, suiteSummary)
	testsStart := time.Now()
	for _, clusterLoaderConfig.TestConfigPath = range testConfigPaths {
		testStart := time.Now()
		specSummary := &ginkgotypes.SpecSummary{
			ComponentTexts: []string{suiteSummary.SuiteDescription, clusterLoaderConfig.TestConfigPath},
		}
		glog.Infof("Running %v", clusterLoaderConfig.TestConfigPath)
		if errList := test.RunTest(f, &clusterLoaderConfig); !errList.IsEmpty() {
			suiteSummary.NumberOfFailedSpecs++
			specSummary.State = ginkgotypes.SpecStateFailed
			specSummary.Failure = ginkgotypes.SpecFailure{
				Message: errList.String(),
			}
			glog.Errorf("Test execution failed: %v", errList.String())
		} else {
			specSummary.State = ginkgotypes.SpecStatePassed
			glog.Infof("Test %v ran successfully!", clusterLoaderConfig.TestConfigPath)
		}
		specSummary.RunTime = time.Since(testStart)
		junitReporter.SpecDidComplete(specSummary)
	}
	suiteSummary.RunTime = time.Since(testsStart)
	junitReporter.SpecSuiteDidEnd(suiteSummary)
	if suiteSummary.NumberOfFailedSpecs > 0 {
		glog.Fatalf("%d tests have failed!", suiteSummary.NumberOfFailedSpecs)
	}
}
<commit_msg>pretty printing test results<commit_after>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"flag"
	"fmt"
	"os"
	"path"
	"time"

	"github.com/golang/glog"
	ginkgoconfig "github.com/onsi/ginkgo/config"
	ginkgoreporters "github.com/onsi/ginkgo/reporters"
	ginkgotypes "github.com/onsi/ginkgo/types"
	"github.com/spf13/pflag"
	"k8s.io/perf-tests/clusterloader2/pkg/config"
	"k8s.io/perf-tests/clusterloader2/pkg/errors"
	"k8s.io/perf-tests/clusterloader2/pkg/framework"
	"k8s.io/perf-tests/clusterloader2/pkg/test"
	"k8s.io/perf-tests/clusterloader2/pkg/util"

	_ "k8s.io/perf-tests/clusterloader2/pkg/measurement/common/bundle"
	_ "k8s.io/perf-tests/clusterloader2/pkg/measurement/common/simple"
	_ "k8s.io/perf-tests/clusterloader2/pkg/measurement/common/slos"
)

const (
	dashLine = "--------------------------------------------------------------------------------"
)

var (
	clusterLoaderConfig config.ClusterLoaderConfig
	testConfigPaths     []string
)

func initClusterFlags() {
	pflag.StringVar(&clusterLoaderConfig.ClusterConfig.KubeConfigPath, "kubeconfig", "", "Path to the kubeconfig file")
	pflag.IntVar(&clusterLoaderConfig.ClusterConfig.Nodes, "nodes", 0, "number of nodes")
	pflag.StringVar(&clusterLoaderConfig.ClusterConfig.Provider, "provider", "", "Cluster provider")
}

func validateClusterFlags() *errors.ErrorList {
	errList := errors.NewErrorList()
	if clusterLoaderConfig.ClusterConfig.KubeConfigPath == "" {
		errList.Append(fmt.Errorf("no kubeconfig path specified"))
	}
	return errList
}

func initFlags() {
	pflag.CommandLine = pflag.NewFlagSet(os.Args[0], pflag.ContinueOnError)
	pflag.CommandLine.AddGoFlagSet(flag.CommandLine)
	pflag.StringVar(&clusterLoaderConfig.ReportDir, "report-dir", "", "Path to the directory where the reports should be saved. Default is empty, which cause reports being written to standard output.")
	pflag.StringArrayVar(&testConfigPaths, "testconfig", []string{}, "Paths to the test config files")
	pflag.StringVar(&clusterLoaderConfig.TestOverridesPath, "testoverrides", "", "Path to the config overrides file")
	initClusterFlags()
}

func validateFlags() *errors.ErrorList {
	errList := errors.NewErrorList()
	if len(testConfigPaths) < 1 {
		errList.Append(fmt.Errorf("no test config path specified"))
	}
	errList.Concat(validateClusterFlags())
	return errList
}

func completeConfig(f *framework.Framework) error {
	if clusterLoaderConfig.ClusterConfig.Nodes == 0 {
		nodes, err := util.GetSchedulableUntainedNodesNumber(f.GetClientSet())
		if err != nil {
			return fmt.Errorf("getting number of nodes error: %v", err)
		}
		clusterLoaderConfig.ClusterConfig.Nodes = nodes
		glog.Infof("ClusterConfig.Nodes set to %v", nodes)
	}
	if clusterLoaderConfig.ClusterConfig.MasterName == "" {
		masterName, err := util.GetMasterName(f.GetClientSet())
		if err == nil {
			clusterLoaderConfig.ClusterConfig.MasterName = masterName
			glog.Infof("ClusterConfig.MasterName set to %v", masterName)
		} else {
			glog.Errorf("Getting master name error: %v", err)
		}
	}
	if clusterLoaderConfig.ClusterConfig.MasterIP == "" {
		masterIP, err := util.GetMasterExternalIP(f.GetClientSet())
		if err == nil {
			clusterLoaderConfig.ClusterConfig.MasterIP = masterIP
			glog.Infof("ClusterConfig.MasterIP set to %v", masterIP)
		} else {
			glog.Errorf("Getting master ip error: %v", err)
		}
	}
	return nil
}

func createReportDir() error {
	if clusterLoaderConfig.ReportDir != "" {
		if _, err := os.Stat(clusterLoaderConfig.ReportDir); err != nil {
			if !os.IsNotExist(err) {
				return err
			}
			if err = os.Mkdir(clusterLoaderConfig.ReportDir, 0755); err != nil {
				return fmt.Errorf("report directory creation error: %v", err)
			}
		}
	}
	return nil
}

func printTestStart(name string) {
	glog.Infof(dashLine)
	glog.Infof("Running %v", name)
	glog.Infof(dashLine)
}

func printTestResult(name, status, errors string) {
	logf := glog.Infof
	if errors != "" {
		logf = glog.Errorf
	}
	logf(dashLine)
	logf("Test Finished")
	logf("  Test: %v", name)
	logf("  Status: %v", status)
	if errors != "" {
		logf("  Errors: %v", errors)
	}
	logf(dashLine)
}

func main() {
	defer glog.Flush()
	initFlags()
	if err := pflag.CommandLine.Parse(os.Args[1:]); err != nil {
		glog.Fatalf("Flag parse failed: %v", err)
	}
	if errList := validateFlags(); !errList.IsEmpty() {
		glog.Fatalf("Parsing flags error: %v", errList.String())
	}

	f, err := framework.NewFramework(clusterLoaderConfig.ClusterConfig.KubeConfigPath)
	if err != nil {
		glog.Fatalf("Framework creation error: %v", err)
	}

	if err = completeConfig(f); err != nil {
		glog.Fatalf("Config completing error: %v", err)
	}

	if err = createReportDir(); err != nil {
		glog.Fatalf("Cannot create report directory: %v", err)
	}

	if err = util.LogClusterNodes(f.GetClientSet()); err != nil {
		glog.Errorf("Nodes info logging error: %v", err)
	}

	suiteSummary := &ginkgotypes.SuiteSummary{
		SuiteDescription:           "ClusterLoaderV2",
		NumberOfSpecsThatWillBeRun: len(testConfigPaths),
	}
	junitReporter := ginkgoreporters.NewJUnitReporter(path.Join(clusterLoaderConfig.ReportDir, "junit.xml"))
	junitReporter.SpecSuiteWillBegin(ginkgoconfig.GinkgoConfig, suiteSummary)
	testsStart := time.Now()
	for _, clusterLoaderConfig.TestConfigPath = range testConfigPaths {
		testStart := time.Now()
		specSummary := &ginkgotypes.SpecSummary{
			ComponentTexts: []string{suiteSummary.SuiteDescription, clusterLoaderConfig.TestConfigPath},
		}
		printTestStart(clusterLoaderConfig.TestConfigPath)
		if errList := test.RunTest(f, &clusterLoaderConfig); !errList.IsEmpty() {
			suiteSummary.NumberOfFailedSpecs++
			specSummary.State = ginkgotypes.SpecStateFailed
			specSummary.Failure = ginkgotypes.SpecFailure{
				Message: errList.String(),
			}
			printTestResult(clusterLoaderConfig.TestConfigPath, "Fail", errList.String())
		} else {
			specSummary.State = ginkgotypes.SpecStatePassed
			printTestResult(clusterLoaderConfig.TestConfigPath, "Success", "")
		}
		specSummary.RunTime = time.Since(testStart)
		junitReporter.SpecDidComplete(specSummary)
	}
	suiteSummary.RunTime = time.Since(testsStart)
	junitReporter.SpecSuiteDidEnd(suiteSummary)
	if suiteSummary.NumberOfFailedSpecs > 0 {
		glog.Fatalf("%d tests have failed!", suiteSummary.NumberOfFailedSpecs)
	}
}
<|endoftext|>


<commit_before>// (C) Copyright 2016 Hewlett Packard Enterprise Development LP
//
// Licensed under the Apache License, Version 2.0 (the "License");
// You may not use this file except in compliance with the License.
// You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed
// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
// CONDITIONS OF ANY KIND, either express or implied. See the License for the
// specific language governing permissions and limitations under the License.

package oneview

import (
	"github.com/hashicorp/terraform/helper/schema"
)

func dataSourceLogicalInterconnect() *schema.Resource {
	return &schema.Resource{
		Read: dataSourceLogicalInterconnectRead,

		Schema: map[string]*schema.Schema{
			"category": {
				Type:     schema.TypeString,
				Optional: true,
			},
			"consistency_status": {
				Type:     schema.TypeString,
				Optional: true,
			},
			"created": {
				Type:     schema.TypeString,
				Optional: true,
			},
			"description": {
				Type:     schema.TypeString,
				Optional: true,
			},
			"eTag": {
				Type:     schema.TypeString,
				Optional: true,
			},
			"enclosure_uris": {
				Type:     schema.TypeSet,
				Optional: true,
				Elem:     &schema.Schema{Type: schema.TypeString},
				Set:      schema.HashString,
			},
			"ethernet_settings": {
				Type:     schema.TypeList,
				Optional: true,
				Elem: &schema.Resource{
					Schema: map[string]*schema.Schema{
						"category": {
							Type:     schema.TypeString,
							Optional: true,
						},
						"created": {
							Type:     schema.TypeString,
							Optional: true,
						},
						"dependent_resource_uri": {
							Type:     schema.TypeString,
							Optional: true,
						},
						"description": {
							Type:     schema.TypeString,
							Optional: true,
						},
						"eTag": {
							Type:     schema.TypeString,
							Optional: true,
						},
						"enable_fast_mac_cache_failover": {
							Type:     schema.TypeBool,
							Optional: true,
						},
						"enable_igmp_snooping": {
							Type:     schema.TypeBool,
							Optional: true,
						},
						"enable_network_loop_protection": {
							Type:     schema.TypeBool,
							Optional: true,
						},
						"id": {
							Type:     schema.TypeString,
							Optional: true,
						},
						"igmp_idle_timeout_interval": {
							Type:     schema.TypeInt,
							Optional: true,
						},
						"interconnect_type": {
							Type:     schema.TypeString,
							Optional: true,
						},
						"mac_refresh_interval": {
							Type:     schema.TypeInt,
							Optional: true,
						},
						"modified": {
							Type:     schema.TypeString,
							Optional: true,
						},
						"name": {
							Type:     schema.TypeString,
							Optional: true,
						},
						"state": {
							Type:     schema.TypeString,
							Optional: true,
						},
						"status": {
							Type:     schema.TypeString,
							Optional: true,
						},
						"type": {
							Type:     schema.TypeString,
							Optional: true,
						},
						"uri": {
							Type:     schema.TypeString,
							Optional: true,
						},
					},
				},
			},
			"fusion_domain_uri": {
				Type:     schema.TypeString,
				Optional: true,
			},
			"interconnect_map": {
				Type:     schema.TypeList,
				Optional: true,
				Elem: &schema.Resource{
					Schema: map[string]*schema.Schema{
						"interconnect_map_entries": {
							Type:     schema.TypeList,
							Optional: true,
							Elem: &schema.Resource{
								Schema: map[string]*schema.Schema{
									"enclosure_index": {
										Type:     schema.TypeInt,
										Optional: true,
									},
									"interconnect_uri": {
										Type:     schema.TypeString,
										Optional: true,
									},
									"location": {
										Type:     schema.TypeList,
										Optional: true,
										Elem: &schema.Resource{
											Schema: map[string]*schema.Schema{
												"location_entries": {
													Type:     schema.TypeList,
													Optional: true,
													Elem: &schema.Resource{
														Schema: map[string]*schema.Schema{
															"type": {
																Type:     schema.TypeString,
																Optional: true,
															},
															"value": {
																Type:     schema.TypeString,
																Optional: true,
															},
														},
													},
												},
											},
										},
									},
									"logical_downlink_uri": {
										Type:     schema.TypeString,
										Optional: true,
									},
									"permitted_interconnect_type_uri": {
										Type:     schema.TypeString,
										Optional: true,
									},
								},
							},
						},
					},
				},
			},
			"interconnects": {
				Type:     schema.TypeSet,
				Optional: true,
				Elem:     &schema.Schema{Type: schema.TypeString},
				Set:      schema.HashString,
			},
			"logical_interconnect_group_uri": {
				Type:     schema.TypeString,
				Optional: true,
			},
			"modified": {
				Type:     schema.TypeString,
				Optional: true,
			},
			"name": {
				Type:     schema.TypeString,
				Optional: true,
			},
			"port_monitor": {
				Type:     schema.TypeList,
				Optional: true,
				Elem: &schema.Resource{
					Schema: map[string]*schema.Schema{
						"analyzer_port": {
							Type:     schema.TypeString,
							Optional: true,
						},
						"category": {
							Type:     schema.TypeString,
							Optional: true,
						},
						"created": {
							Type:     schema.TypeString,
							Optional: true,
						},
						"description": {
							Type:     schema.TypeString,
							Optional: true,
						},
						"eTag": {
							Type:     schema.TypeString,
							Optional: true,
						},
						"enable_port_monitor": {
							Type:     schema.TypeBool,
							Optional: true,
						},
						"modified": {
							Type:     schema.TypeString,
							Optional: true,
						},
						"name": {
							Type:     schema.TypeString,
							Optional: true,
						},
						"state": {
							Type:     schema.TypeString,
							Optional: true,
						},
						"status": {
							Type:     schema.TypeString,
							Optional: true,
						},
						"type": {
							Type:     schema.TypeString,
							Optional: true,
						},
						"uri": {
							Type:     schema.TypeString,
							Optional: true,
						},
					},
				},
			},
			"snmp_configuration": {
				Type:     schema.TypeList,
				Optional: true,
				Elem: &schema.Resource{
					Schema: map[string]*schema.Schema{
						"category": {
							Type:     schema.TypeString,
							Optional: true,
						},
						"created": {
							Type:     schema.TypeString,
							Optional: true,
						},
						"description": {
							Type:     schema.TypeString,
							Optional: true,
						},
						"eTag": {
							Type:     schema.TypeString,
							Optional: true,
						},
						"enabled": {
							Type:     schema.TypeBool,
							Optional: true,
						},
						"modified": {
							Type:     schema.TypeString,
							Optional: true,
						},
						"name": {
							Type:     schema.TypeString,
							Optional: true,
						},
						"read_community": {
							Type:     schema.TypeString,
							Optional: true,
						},
						"state": {
							Type:     schema.TypeString,
							Optional: true,
						},
						"status": {
							Type:     schema.TypeString,
							Optional: true,
						},
						"system_contact": {
							Type:     schema.TypeString,
							Optional: true,
						},
						"type": {
							Type:     schema.TypeString,
							Optional: true,
						},
						"uri": {
							Type:     schema.TypeString,
							Optional: true,
						},
					},
				},
			},
			"stacking_health": {
				Type:     schema.TypeString,
				Optional: true,
			},
			"state": {
				Type:     schema.TypeString,
				Optional: true,
			},
			"status": {
				Type:     schema.TypeString,
				Optional: true,
			},
			"telemetry_configuration": {
				Type:     schema.TypeList,
				Optional: true,
				Elem: &schema.Resource{
					Schema: map[string]*schema.Schema{
						"category": {
							Type:     schema.TypeString,
							Optional: true,
						},
						"created": {
							Type:     schema.TypeString,
							Optional: true,
						},
						"description": {
							Type:     schema.TypeString,
							Optional: true,
						},
						"eTag": {
							Type:     schema.TypeString,
							Optional: true,
						},
						"enable_telemetry": {
							Type:     schema.TypeBool,
							Optional: true,
						},
						"modified": {
							Type:     schema.TypeString,
							Optional: true,
						},
						"name": {
							Type:     schema.TypeString,
							Optional: true,
						},
						"sample_count": {
							Type:     schema.TypeInt,
							Optional: true,
						},
						"sample_interval": {
							Type:     schema.TypeInt,
							Optional: true,
						},
						"state": {
							Type:     schema.TypeString,
							Optional: true,
						},
						"status": {
							Type:     schema.TypeString,
							Optional: true,
						},
						"type": {
							Type:     schema.TypeString,
							Optional: true,
						},
						"uri": {
							Type:     schema.TypeString,
							Optional: true,
						},
					},
				},
			},
			"type": {
				Type:     schema.TypeString,
				Optional: true,
			},
			"uri": {
				Type:     schema.TypeString,
				Optional: true,
			},
		},
	}
}

func dataSourceLogicalInterconnectRead(d *schema.ResourceData, meta interface{}) error {
	config := meta.(*Config)
	id := d.Get("name").(string)
	logInt, err := config.ovClient.GetLogicalInterconnectById(id)

	if err != nil || logInt.URI.IsNil() {
		d.SetId("")
		return nil
	}

	d.SetId(id)

	d.Set("category", logInt.Category)
	d.Set("consistency_status", logInt.ConsistencyStatus)
	d.Set("created", logInt.Created)
	d.Set("description", logInt.Description)
	d.Set("eTag", logInt.ETAG)
	d.Set("fusion_domain_uri", logInt.FusionDomainUri)
	d.Set("logical_interconnect_group_uri", logInt.LogicalInterconnectGroupUri)
	d.Set("modified", logInt.Modified)
	d.Set("name", logInt.Name)
	d.Set("stacking_health", logInt.StackingHealth)
	d.Set("state", logInt.State)
	d.Set("status", logInt.Status)
	d.Set("type", logInt.Type)
	d.Set("uri", logInt.URI)

	return nil
}
<commit_msg>Go fmt.<commit_after>// (C) Copyright 2016 Hewlett Packard Enterprise Development LP
//
// Licensed under the Apache License, Version 2.0 (the "License");
// You may not use this file except in compliance with the License.
// You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed
// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
// CONDITIONS OF ANY KIND, either express or implied. See the License for the
// specific language governing permissions and limitations under the License.

package oneview

import (
	"github.com/hashicorp/terraform/helper/schema"
)

func dataSourceLogicalInterconnect() *schema.Resource {
	return &schema.Resource{
		Read: dataSourceLogicalInterconnectRead,

		Schema: map[string]*schema.Schema{
			"category": {
				Type:     schema.TypeString,
				Optional: true,
			},
			"consistency_status": {
				Type:     schema.TypeString,
				Optional: true,
			},
			"created": {
				Type:     schema.TypeString,
				Optional: true,
			},
			"description": {
				Type:     schema.TypeString,
				Optional: true,
			},
			"eTag": {
				Type:     schema.TypeString,
				Optional: true,
			},
			"enclosure_uris": {
				Type:     schema.TypeSet,
				Optional: true,
				Elem:     &schema.Schema{Type: schema.TypeString},
				Set:      schema.HashString,
			},
			"ethernet_settings": {
				Type:     schema.TypeList,
				Optional: true,
				Elem: &schema.Resource{
					Schema: map[string]*schema.Schema{
						"category": {
							Type:     schema.TypeString,
							Optional: true,
						},
						"created": {
							Type:     schema.TypeString,
							Optional: true,
						},
						"dependent_resource_uri": {
							Type:     schema.TypeString,
							Optional: true,
						},
						"description": {
							Type:     schema.TypeString,
							Optional: true,
						},
						"eTag": {
							Type:     schema.TypeString,
							Optional: true,
						},
						"enable_fast_mac_cache_failover": {
							Type:     schema.TypeBool,
							Optional: true,
						},
						"enable_igmp_snooping": {
							Type:     schema.TypeBool,
							Optional: true,
						},
						"enable_network_loop_protection": {
							Type:     schema.TypeBool,
							Optional: true,
						},
						"id": {
							Type:     schema.TypeString,
							Optional: true,
						},
						"igmp_idle_timeout_interval": {
							Type:     schema.TypeInt,
							Optional: true,
						},
						"interconnect_type": {
							Type:     schema.TypeString,
							Optional: true,
						},
						"mac_refresh_interval": {
							Type:     schema.TypeInt,
							Optional: true,
						},
						"modified": {
							Type:     schema.TypeString,
							Optional: true,
						},
						"name": {
							Type:     schema.TypeString,
							Optional: true,
						},
						"state": {
							Type:     schema.TypeString,
							Optional: true,
						},
						"status": {
							Type:     schema.TypeString,
							Optional: true,
						},
						"type": {
							Type:     schema.TypeString,
							Optional: true,
						},
						"uri": {
							Type:     schema.TypeString,
							Optional: true,
						},
					},
				},
			},
			"fusion_domain_uri": {
				Type:     schema.TypeString,
				Optional: true,
			},
			"interconnect_map": {
				Type:     schema.TypeList,
				Optional: true,
				Elem: &schema.Resource{
					Schema: map[string]*schema.Schema{
						"interconnect_map_entries": {
							Type:     schema.TypeList,
							Optional: true,
							Elem: &schema.Resource{
								Schema: map[string]*schema.Schema{
									"enclosure_index": {
										Type:     schema.TypeInt,
										Optional: true,
									},
									"interconnect_uri": {
										Type:     schema.TypeString,
										Optional: true,
									},
									"location": {
										Type:     schema.TypeList,
										Optional: true,
										Elem: &schema.Resource{
											Schema: map[string]*schema.Schema{
												"location_entries": {
													Type:     schema.TypeList,
													Optional: true,
													Elem: &schema.Resource{
														Schema: map[string]*schema.Schema{
															"type": {
																Type:     schema.TypeString,
																Optional: true,
															},
															"value": {
																Type:     schema.TypeString,
																Optional: true,
															},
														},
													},
												},
											},
										},
									},
									"logical_downlink_uri": {
										Type:     schema.TypeString,
										Optional: true,
									},
									"permitted_interconnect_type_uri": {
										Type:     schema.TypeString,
										Optional: true,
									},
								},
							},
						},
					},
				},
			},
			"interconnects": {
				Type:     schema.TypeSet,
				Optional: true,
				Elem:     &schema.Schema{Type: schema.TypeString},
				Set:      schema.HashString,
			},
			"logical_interconnect_group_uri": {
				Type:     schema.TypeString,
				Optional: true,
			},
			"modified": {
				Type:     schema.TypeString,
				Optional: true,
			},
			"name": {
				Type:     schema.TypeString,
				Optional: true,
			},
			"port_monitor": {
				Type:     schema.TypeList,
				Optional: true,
				Elem: &schema.Resource{
					Schema: map[string]*schema.Schema{
						"analyzer_port": {
							Type:     schema.TypeString,
							Optional: true,
						},
						"category": {
							Type:     schema.TypeString,
							Optional: true,
						},
						"created": {
							Type:     schema.TypeString,
							Optional: true,
						},
						"description": {
							Type:     schema.TypeString,
							Optional: true,
						},
						"eTag": {
							Type:     schema.TypeString,
							Optional: true,
						},
						"enable_port_monitor": {
							Type:     schema.TypeBool,
							Optional: true,
						},
						"modified": {
							Type:     schema.TypeString,
							Optional: true,
						},
						"name": {
							Type:     schema.TypeString,
							Optional: true,
						},
						"state": {
							Type:     schema.TypeString,
							Optional: true,
						},
						"status": {
							Type:     schema.TypeString,
							Optional: true,
						},
						"type": {
							Type:     schema.TypeString,
							Optional: true,
						},
						"uri": {
							Type:     schema.TypeString,
							Optional: true,
						},
					},
				},
			},
			"snmp_configuration": {
				Type:     schema.TypeList,
				Optional: true,
				Elem: &schema.Resource{
					Schema: map[string]*schema.Schema{
						"category": {
							Type:     schema.TypeString,
							Optional: true,
						},
						"created": {
							Type:     schema.TypeString,
							Optional: true,
						},
						"description": {
							Type:     schema.TypeString,
							Optional: true,
						},
						"eTag": {
							Type:     schema.TypeString,
							Optional: true,
						},
						"enabled": {
							Type:     schema.TypeBool,
							Optional: true,
						},
						"modified": {
							Type:     schema.TypeString,
							Optional: true,
						},
						"name": {
							Type:     schema.TypeString,
							Optional: true,
						},
						"read_community": {
							Type:     schema.TypeString,
							Optional: true,
						},
						"state": {
							Type:     schema.TypeString,
							Optional: true,
						},
						"status": {
							Type:     schema.TypeString,
							Optional: true,
						},
						"system_contact": {
							Type:     schema.TypeString,
							Optional: true,
						},
						"type": {
							Type:     schema.TypeString,
							Optional: true,
						},
						"uri": {
							Type:     schema.TypeString,
							Optional: true,
						},
					},
				},
			},
			"stacking_health": {
				Type:     schema.TypeString,
				Optional: true,
			},
			"state": {
				Type:     schema.TypeString,
				Optional: true,
			},
			"status": {
				Type:     schema.TypeString,
				Optional: true,
			},
			"telemetry_configuration": {
				Type:     schema.TypeList,
				Optional: true,
				Elem: &schema.Resource{
					Schema: map[string]*schema.Schema{
						"category": {
							Type:     schema.TypeString,
							Optional: true,
						},
						"created": {
							Type:     schema.TypeString,
							Optional: true,
						},
						"description": {
							Type:     schema.TypeString,
							Optional: true,
						},
						"eTag": {
							Type:     schema.TypeString,
							Optional: true,
						},
						"enable_telemetry": {
							Type:     schema.TypeBool,
							Optional: true,
						},
						"modified": {
							Type:     schema.TypeString,
							Optional: true,
						},
						"name": {
							Type:     schema.TypeString,
							Optional: true,
						},
						"sample_count": {
							Type:     schema.TypeInt,
							Optional: true,
						},
						"sample_interval": {
							Type:     schema.TypeInt,
							Optional: true,
						},
						"state": {
							Type:     schema.TypeString,
							Optional: true,
						},
						"status": {
							Type:     schema.TypeString,
							Optional: true,
						},
						"type": {
							Type:     schema.TypeString,
							Optional: true,
						},
						"uri": {
							Type:     schema.TypeString,
							Optional: true,
						},
					},
				},
			},
			"type": {
				Type:     schema.TypeString,
				Optional: true,
			},
			"uri": {
				Type:     schema.TypeString,
				Optional: true,
			},
		},
	}
}

func dataSourceLogicalInterconnectRead(d *schema.ResourceData, meta interface{}) error {
	config := meta.(*Config)
	id := d.Get("name").(string)
	logInt, err := config.ovClient.GetLogicalInterconnectById(id)

	if err != nil || logInt.URI.IsNil() {
		d.SetId("")
		return nil
	}

	d.SetId(id)

	d.Set("category", logInt.Category)
	d.Set("consistency_status", logInt.ConsistencyStatus)
	d.Set("created", logInt.Created)
	d.Set("description", logInt.Description)
	d.Set("eTag", logInt.ETAG)
	d.Set("fusion_domain_uri", logInt.FusionDomainUri)
	d.Set("logical_interconnect_group_uri", logInt.LogicalInterconnectGroupUri)
	d.Set("modified", logInt.Modified)
	d.Set("name", logInt.Name)
	d.Set("stacking_health", logInt.StackingHealth)
	d.Set("state", logInt.State)
	d.Set("status", logInt.Status)
	d.Set("type", logInt.Type)
	d.Set("uri", logInt.URI)

	return nil
}
<|endoftext|>


<commit_before>/*
Copyright 2017 by the contributors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"flag"
	"fmt"
	"strings"

	"k8s.io/sample-controller/pkg/signals"
	"sigs.k8s.io/aws-iam-authenticator/pkg/mapper"
	"sigs.k8s.io/aws-iam-authenticator/pkg/server"

	"github.com/sirupsen/logrus"
	"github.com/spf13/cobra"
	"github.com/spf13/viper"
)

// DefaultPort is the default localhost port (chosen randomly).
const DefaultPort = 21362

// serverCmd represents the server command
var serverCmd = &cobra.Command{
	Use:   "server",
	Short: "Run a webhook validation server suitable that validates tokens using AWS IAM",
	Long:  ``,
	Run: func(cmd *cobra.Command, args []string) {
		var err error

		stopCh := signals.SetupSignalHandler()

		cfg, err := getConfig()
		if err != nil {
			logrus.Fatalf("%s", err)
		}

		mappers := server.BuildMapperChain(cfg)
		for _, m := range mappers {
			logrus.Infof("starting mapper %q", m.Name())
			if err := m.Start(stopCh); err != nil {
				logrus.Fatalf("start mapper %q failed", m.Name())
			}
		}

		httpServer := server.New(cfg, mappers)
		httpServer.Run(stopCh)
	},
}

func init() {
	viper.SetDefault("server.port", DefaultPort)

	serverCmd.Flags().String("generate-kubeconfig",
		"/etc/kubernetes/aws-iam-authenticator/kubeconfig.yaml",
		"Output `path` where a generated webhook kubeconfig (for `--authentication-token-webhook-config-file`) will be stored (should be a hostPath mount).")
	viper.BindPFlag("server.generateKubeconfig", serverCmd.Flags().Lookup("generate-kubeconfig"))

	serverCmd.Flags().Bool("kubeconfig-pregenerated",
		false,
		"set to `true` when a webhook kubeconfig is pre-generated by running the `init` command, and therefore the `server` shouldn't unnecessarily re-generate a new one.")
	viper.BindPFlag("server.kubeconfigPregenerated", serverCmd.Flags().Lookup("kubeconfig-pregenerated"))

	serverCmd.Flags().String("state-dir",
		"/var/aws-iam-authenticator",
		"State `directory` for generated certificate and private key (should be a hostPath mount).")
	viper.BindPFlag("server.stateDir", serverCmd.Flags().Lookup("state-dir"))

	serverCmd.Flags().String("kubeconfig",
		"",
		"kubeconfig file path for using a local kubeconfig to configure the client to talk to the API server for the IAMIdentityMappings.")
	viper.BindPFlag("server.kubeconfig", serverCmd.Flags().Lookup("kubeconfig"))
	serverCmd.Flags().String("master",
		"",
		"master is the URL to the api server")
	viper.BindPFlag("server.master", serverCmd.Flags().Lookup("master"))

	serverCmd.Flags().String("address",
		"127.0.0.1",
		"IP Address to bind the server to listen to. (should be a 127.0.0.1 or 0.0.0.0)")
	viper.BindPFlag("server.address", serverCmd.Flags().Lookup("address"))

	serverCmd.Flags().StringSlice("backend-mode",
		[]string{mapper.ModeCRD, mapper.ModeConfigMap, mapper.ModeFile},
		fmt.Sprintf("Ordered list of backends to get mappings from. The first one that returns a matching mapping wins. Comma-delimited list of: %s", strings.Join(mapper.BackendModeChoices, ",")))
	viper.BindPFlag("server.backendMode", serverCmd.Flags().Lookup("backend-mode"))

	fs := flag.NewFlagSet("", flag.ContinueOnError)
	_ = fs.Parse([]string{})
	flag.CommandLine = fs

	rootCmd.AddCommand(serverCmd)
}
<commit_msg>Allow server port customization<commit_after>/*
Copyright 2017 by the contributors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"flag"
	"fmt"
	"strings"

	"k8s.io/sample-controller/pkg/signals"
	"sigs.k8s.io/aws-iam-authenticator/pkg/mapper"
	"sigs.k8s.io/aws-iam-authenticator/pkg/server"

	"github.com/sirupsen/logrus"
	"github.com/spf13/cobra"
	"github.com/spf13/viper"
)

// DefaultPort is the default localhost port (chosen randomly).
const DefaultPort = 21362

// serverCmd represents the server command
var serverCmd = &cobra.Command{
	Use:   "server",
	Short: "Run a webhook validation server suitable that validates tokens using AWS IAM",
	Long:  ``,
	Run: func(cmd *cobra.Command, args []string) {
		var err error

		stopCh := signals.SetupSignalHandler()

		cfg, err := getConfig()
		if err != nil {
			logrus.Fatalf("%s", err)
		}

		mappers := server.BuildMapperChain(cfg)
		for _, m := range mappers {
			logrus.Infof("starting mapper %q", m.Name())
			if err := m.Start(stopCh); err != nil {
				logrus.Fatalf("start mapper %q failed", m.Name())
			}
		}

		httpServer := server.New(cfg, mappers)
		httpServer.Run(stopCh)
	},
}

func init() {
	serverCmd.Flags().String("generate-kubeconfig",
		"/etc/kubernetes/aws-iam-authenticator/kubeconfig.yaml",
		"Output `path` where a generated webhook kubeconfig (for `--authentication-token-webhook-config-file`) will be stored (should be a hostPath mount).")
	viper.BindPFlag("server.generateKubeconfig", serverCmd.Flags().Lookup("generate-kubeconfig"))

	serverCmd.Flags().Bool("kubeconfig-pregenerated",
		false,
		"set to `true` when a webhook kubeconfig is pre-generated by running the `init` command, and therefore the `server` shouldn't unnecessarily re-generate a new one.")
	viper.BindPFlag("server.kubeconfigPregenerated", serverCmd.Flags().Lookup("kubeconfig-pregenerated"))

	serverCmd.Flags().String("state-dir",
		"/var/aws-iam-authenticator",
		"State `directory` for generated certificate and private key (should be a hostPath mount).")
	viper.BindPFlag("server.stateDir", serverCmd.Flags().Lookup("state-dir"))

	serverCmd.Flags().String("kubeconfig",
		"",
		"kubeconfig file path for using a local kubeconfig to configure the client to talk to the API server for the IAMIdentityMappings.")
	viper.BindPFlag("server.kubeconfig", serverCmd.Flags().Lookup("kubeconfig"))
	serverCmd.Flags().String("master",
		"",
		"master is the URL to the api server")
	viper.BindPFlag("server.master", serverCmd.Flags().Lookup("master"))

	serverCmd.Flags().String("address",
		"127.0.0.1",
		"IP Address to bind the server to listen to. (should be a 127.0.0.1 or 0.0.0.0)")
	viper.BindPFlag("server.address", serverCmd.Flags().Lookup("address"))

	serverCmd.Flags().StringSlice("backend-mode",
		[]string{mapper.ModeCRD, mapper.ModeConfigMap, mapper.ModeFile},
		fmt.Sprintf("Ordered list of backends to get mappings from. The first one that returns a matching mapping wins. Comma-delimited list of: %s", strings.Join(mapper.BackendModeChoices, ",")))
	viper.BindPFlag("server.backendMode", serverCmd.Flags().Lookup("backend-mode"))

	serverCmd.Flags().Int(
		"port",
		DefaultPort,
		"Port to bind the server to listen to")
	viper.BindPFlag("server.port", serverCmd.Flags().Lookup("port"))

	fs := flag.NewFlagSet("", flag.ContinueOnError)
	_ = fs.Parse([]string{})
	flag.CommandLine = fs

	rootCmd.AddCommand(serverCmd)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"time"

	"golang.org/x/crypto/ssh/terminal"

	"github.com/Symantec/Dominator/lib/log"
	"github.com/Symantec/Dominator/lib/srpc"
)

func installerShellSubcommand(args []string, logger log.DebugLogger) {
	err := installerShell(args[0], logger)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error talking to installer shell: %s\n", err)
		os.Exit(1)
	}
	os.Exit(0)
}

func installerShell(hostname string, logger log.DebugLogger) error {
	var client *srpc.Client
	fmt.Fprintf(os.Stderr, "trying to connect")
	for ; ; time.Sleep(time.Second * 5) {
		var err error
		client, err = srpc.DialHTTP("tcp", fmt.Sprintf("%s:%d",
			hostname, *installerPortNum), time.Second*15)
		if err == nil {
			break
		}
		fmt.Fprintf(os.Stderr, ".")
	}
	defer client.Close()
	conn, err := client.Call("Installer.Shell")
	if err != nil {
		return err
	}
	defer conn.Close()
	closed := false
	defer func() {
		closed = true
	}()
	fmt.Fprintf(os.Stderr, " connected...\n")
	oldState, err := terminal.MakeRaw(int(os.Stdin.Fd()))
	if err != nil {
		return err
	}
	defer func() {
		terminal.Restore(int(os.Stdin.Fd()), oldState)
		fmt.Println()
	}()
	go func() { // Read from connection, write to stdout.
		for {
			buffer := make([]byte, 256)
			if nRead, err := conn.Read(buffer); err != nil {
				if closed {
					return
				}
				logger.Printf("error reading from remote shell: %s\n", err)
				return
			} else {
				os.Stderr.Write(buffer[:nRead])
			}
		}
	}()
	// Read from stdin until control-d (EOT).
	for {
		buffer := make([]byte, 256)
		if nRead, err := os.Stdin.Read(buffer); err != nil {
			return fmt.Errorf("error reading from stdin: %s", err)
		} else {
			if buffer[0] == '\x04' { // Control-d: EndOfTransmission.
				return nil
			}
			if _, err := conn.Write(buffer[:nRead]); err != nil {
				return fmt.Errorf("error writing to connection: %s", err)
			}
			if err := conn.Flush(); err != nil {
				return fmt.Errorf("error flushing connection: %s", err)
			}
		}
	}
}
<commit_msg>Use new lib/net/terminal/client package for hyper-control installer-shell.<commit_after>package main

import (
	"fmt"
	"os"
	"time"

	"github.com/Symantec/Dominator/lib/log"
	terminalclient "github.com/Symantec/Dominator/lib/net/terminal/client"
	"github.com/Symantec/Dominator/lib/srpc"
)

func installerShellSubcommand(args []string, logger log.DebugLogger) {
	err := installerShell(args[0], logger)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error talking to installer shell: %s\n", err)
		os.Exit(1)
	}
	os.Exit(0)
}

func installerShell(hostname string, logger log.DebugLogger) error {
	var client *srpc.Client
	fmt.Fprintf(os.Stderr, "trying to connect")
	for ; ; time.Sleep(time.Second * 5) {
		var err error
		client, err = srpc.DialHTTP("tcp", fmt.Sprintf("%s:%d",
			hostname, *installerPortNum), time.Second*15)
		if err == nil {
			break
		}
		fmt.Fprintf(os.Stderr, ".")
	}
	defer client.Close()
	conn, err := client.Call("Installer.Shell")
	if err != nil {
		return err
	}
	defer conn.Close()
	fmt.Fprintf(os.Stderr, " connected...\n")
	if err := terminalclient.StartTerminal(conn); err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
	fmt.Fprint(os.Stderr, "\r")
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"net/http"
	_ "net/http/pprof"
	"sync"
	"time"

	"github.com/funkygao/gafka/cmd/kateway/hh/disk"
	log "github.com/funkygao/log4go"
)

func main() {
	go http.ListenAndServe("localhost:6786", nil)
	log.Info("pprf ready on http://localhost:6786/debug/pprof")

	cfg := disk.DefaultConfig()
	cfg.Dir = "hh"
	s := disk.New(cfg)
	if err := s.Start(); err != nil {
		panic(err)
	}

	i, j := 5, 5
	var wg sync.WaitGroup

	for seq := 0; seq < i; seq++ {
		wg.Add(1)
		go func(seq int) {
			defer wg.Done()

			for loops := 0; loops < j; loops++ {
				if err := s.Append("me", "app1.foobar.v1", []byte("key"),
					[]byte(fmt.Sprintf("<#%d/%d sent at: %s>", seq, loops+1, time.Now()))); err != nil {
					panic(err)
				}
			}
		}(seq)
	}

	log.Info("%d sent, waiting Append finish...", i*j)
	wg.Wait()
	log.Info("all Append done")

	s.Stop()
	log.Info("Did you see %d messages?", i*j)
	log.Info("bye!")
	log.Close()
}
<commit_msg>show if queue is empty before exit<commit_after>package main

import (
	"fmt"
	"net/http"
	_ "net/http/pprof"
	"strings"
	"sync"
	"time"

	"github.com/funkygao/gafka/cmd/kateway/hh/disk"
	log "github.com/funkygao/log4go"
)

func main() {
	go http.ListenAndServe("localhost:6786", nil)
	log.Info("pprf ready on http://localhost:6786/debug/pprof")

	cfg := disk.DefaultConfig()
	cfg.Dir = "hh"
	s := disk.New(cfg)
	if err := s.Start(); err != nil {
		panic(err)
	}

	placeholder := strings.Repeat(".", 1<<10)

	i, j := 5, 1000
	cluster, topic := "me", "app1.foobar.v1"
	var wg sync.WaitGroup

	for seq := 0; seq < i; seq++ {
		wg.Add(1)
		go func(seq int) {
			defer wg.Done()

			for loops := 0; loops < j; loops++ {
				if err := s.Append(cluster, topic, []byte("key"),
					[]byte(fmt.Sprintf("<#%d/%d sent at: %s %s>", seq, loops+1, time.Now(), placeholder))); err != nil {
					panic(err)
				}
			}
		}(seq)
	}

	log.Info("%d sent, waiting Append finish...", i*j)
	wg.Wait()
	log.Info("all Append done")
	time.Sleep(time.Second * 5)

	s.Stop()
	log.Info("Did you see %d messages? Inflight empty: %v", i*j, s.Empty(cluster, topic))
	log.Info("bye!")
	log.Close()
}
<|endoftext|>


<commit_before>// query_benchmarker_mongo speed tests Mongo using requests from stdin.
//
// It reads encoded Query objects from stdin, and makes concurrent requests
// to the provided Mongo endpoint using mgo.
//
// TODO(rw): On my machine, this only decodes 700k/sec messages from stdin.
package main

import (
	"bufio"
	"encoding/gob"
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"runtime/pprof"
	"sort"
	"sync"
	"time"

	"gopkg.in/mgo.v2"
)

// Program option vars:
var (
	daemonUrl            string
	workers              int
	debug                int
	prettyPrintResponses bool
	limit                int64
	burnIn               uint64
	printInterval        uint64
	memProfile           string
	doQueries            bool
)

// Global vars:
var (
	queryPool    sync.Pool
	queryChan    chan *Query
	statPool     sync.Pool
	statChan     chan *Stat
	workersGroup sync.WaitGroup
	statGroup    sync.WaitGroup
)

type S []interface{}
type M map[string]interface{}

// Parse args:
func init() {
	// needed for deserializing the mongo query from gob
	gob.Register(S{})
	gob.Register(M{})
	gob.Register([]M{})

	flag.StringVar(&daemonUrl, "url", "mongodb://localhost:27017", "Daemon URL.")
	flag.IntVar(&workers, "workers", 1, "Number of concurrent requests to make.")
	flag.IntVar(&debug, "debug", 0, "Whether to print debug messages.")
	flag.Int64Var(&limit, "limit", -1, "Limit the number of queries to send.")
	flag.Uint64Var(&burnIn, "burn-in", 0, "Number of queries to ignore before collecting statistics.")
	flag.Uint64Var(&printInterval, "print-interval", 100, "Print timing stats to stderr after this many queries (0 to disable)")
	flag.BoolVar(&prettyPrintResponses, "print-responses", false, "Pretty print JSON response bodies (for correctness checking) (default false).")
	flag.StringVar(&memProfile, "memprofile", "", "Write a memory profile to this file.")
	flag.BoolVar(&doQueries, "do-queries", true, "Whether to perform queries (useful for benchmarking the query executor.)")

	flag.Parse()
}

func main() {
	// Make pools to minimize heap usage:
	queryPool = sync.Pool{
		New: func() interface{} {
			return &Query{
				HumanLabel:       make([]byte, 0, 1024),
				HumanDescription: make([]byte, 0, 1024),
				BsonDoc:          nil,
			}
		},
	}

	statPool = sync.Pool{
		New: func() interface{} {
			return &Stat{
				Label: make([]byte, 0, 1024),
				Value: 0.0,
			}
		},
	}

	// Make data and control channels:
	queryChan = make(chan *Query, workers)
	statChan = make(chan *Stat, workers)

	// Launch the stats processor:
	statGroup.Add(1)
	go processStats()

	// Establish the connection pool:
	session, err := mgo.Dial(daemonUrl)
	if err != nil {
		log.Fatal(err)
	}

	// Launch the query processors:
	for i := 0; i < workers; i++ {
		workersGroup.Add(1)
		go processQueries(session)
	}

	// Read in jobs, closing the job channel when done:
	input := bufio.NewReaderSize(os.Stdin, 1<<20)
	wallStart := time.Now()
	scan(input)
	close(queryChan)

	// Block for workers to finish sending requests, closing the stats
	// channel when done:
	workersGroup.Wait()
	close(statChan)

	// Wait on the stat collector to finish (and print its results):
	statGroup.Wait()

	wallEnd := time.Now()
	wallTook := wallEnd.Sub(wallStart)
	_, err = fmt.Printf("wall clock time: %fsec\n", float64(wallTook.Nanoseconds())/1e9)
	if err != nil {
		log.Fatal(err)
	}

	// (Optional) create a memory profile:
	if memProfile != "" {
		f, err := os.Create(memProfile)
		if err != nil {
			log.Fatal(err)
		}
		pprof.WriteHeapProfile(f)
		f.Close()
	}
}

// scan reads encoded Queries and places them onto the workqueue.
func scan(r io.Reader) {
	dec := gob.NewDecoder(r)

	n := int64(0)
	for {
		if limit >= 0 && n >= limit {
			break
		}

		q := queryPool.Get().(*Query)
		err := dec.Decode(q)
		if err == io.EOF {
			break
		}
		if err != nil {
			log.Fatal("decoder", err)
		}

		q.ID = n

		queryChan <- q

		n++

	}
}

// processQueries reads byte buffers from queryChan and writes them to the
// target server, while tracking latency.
func processQueries(session *mgo.Session) {
	for q := range queryChan {
		lag, err := oneQuery(session, q)

		stat := statPool.Get().(*Stat)
		stat.Init(q.HumanLabel, lag)
		statChan <- stat

		queryPool.Put(q)
		if err != nil {
			log.Fatalf("Error during request: %s\n", err.Error())
		}
	}
	workersGroup.Done()
}

// oneQuery executes on Query
func oneQuery(session *mgo.Session, q *Query) (float64, error) {
	start := time.Now().UnixNano()
	var err error
	if doQueries {
		db := session.DB(unsafeBytesToString(q.DatabaseName))
		//fmt.Printf("db: %#v\n", db)
		collection := db.C(unsafeBytesToString(q.CollectionName))
		//fmt.Printf("collection: %#v\n", collection)
		pipe := collection.Pipe(q.BsonDoc)
		iter := pipe.Iter()
		type Result struct {
			Id struct {
				TimeBucket int64 `bson:"time_bucket"`
			} `bson:"_id"`
			Value float64 `bson:"max_value"`
		}

		result := Result{}
		for iter.Next(&result) {
			if prettyPrintResponses {
				t := time.Unix(0, result.Id.TimeBucket).UTC()
				fmt.Printf("ID %d: %s, %f\n", q.ID, t, result.Value)
			}
		}

		err = iter.Close()
	}

	took := time.Now().UnixNano() - start
	lag := float64(took) / 1e6 // milliseconds
	return lag, err
}

// processStats collects latency results, aggregating them into summary
// statistics. Optionally, they are printed to stderr at regular intervals.
func processStats() {
	const allQueriesLabel = "all queries"
	statMapping := map[string]*StatGroup{
		allQueriesLabel: &StatGroup{},
	}

	i := uint64(0)
	for stat := range statChan {
		if i < burnIn {
			i++
			statPool.Put(stat)
			continue
		} else if i == burnIn && burnIn > 0 {
			_, err := fmt.Fprintf(os.Stderr, "burn-in complete after %d queries with %d workers\n", burnIn, workers)
			if err != nil {
				log.Fatal(err)
			}
		}

		if _, ok := statMapping[string(stat.Label)]; !ok {
			statMapping[string(stat.Label)] = &StatGroup{}
		}

		statMapping[allQueriesLabel].Push(stat.Value)
		statMapping[string(stat.Label)].Push(stat.Value)

		statPool.Put(stat)

		i++

		// print stats to stderr (if printInterval is greater than zero):
		if printInterval > 0 && i > 0 && i%printInterval == 0 && (int64(i) < limit || limit < 0) {
			_, err := fmt.Fprintf(os.Stderr, "after %d queries with %d workers:\n", i - burnIn, workers)
			if err != nil {
				log.Fatal(err)
			}
			fprintStats(os.Stderr, statMapping)
			_, err = fmt.Fprintf(os.Stderr, "\n")
			if err != nil {
				log.Fatal(err)
			}
		}
	}

	// the final stats output goes to stdout:
	_, err := fmt.Printf("run complete after %d queries with %d workers:\n", i - burnIn, workers)
	if err != nil {
		log.Fatal(err)
	}
	fprintStats(os.Stdout, statMapping)
	statGroup.Done()
}

// fprintStats pretty-prints stats to the given writer.
func fprintStats(w io.Writer, statGroups map[string]*StatGroup) {
	maxKeyLength := 0
	keys := make([]string, 0, len(statGroups))
	for k := range statGroups {
		if len(k) > maxKeyLength {
			maxKeyLength = len(k)
		}
		keys = append(keys, k)
	}
	sort.Strings(keys)
	for _, k := range keys {
		v := statGroups[k]
		minRate := 1e3 / v.Min
		meanRate := 1e3 / v.Mean
		maxRate := 1e3 / v.Max
		paddedKey := fmt.Sprintf("%s", k)
		for len(paddedKey) < maxKeyLength {
			paddedKey += " "
		}
		_, err := fmt.Fprintf(w, "%s : min: %8.2fms (%7.2f/sec), mean: %8.2fms (%7.2f/sec), max: %7.2fms (%6.2f/sec), count: %8d, sum: %5.1fsec \n", paddedKey, v.Min, minRate, v.Mean, meanRate, v.Max, maxRate, v.Count, v.Sum/1e3)
		if err != nil {
			log.Fatal(err)
		}
	}

}
<commit_msg>Added support for reporting to bulk query tool for MongoDB<commit_after>// query_benchmarker_mongo speed tests Mongo using requests from stdin.
//
// It reads encoded Query objects from stdin, and makes concurrent requests
// to the provided Mongo endpoint using mgo.
//
// TODO(rw): On my machine, this only decodes 700k/sec messages from stdin.
package main

import (
	"bufio"
	"encoding/gob"
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"runtime/pprof"
	"sort"
	"sync"
	"time"

	"github.com/influxdata/influxdb-comparisons/util/report"
	"gopkg.in/mgo.v2"
	"strings"
)

// Program option vars:
var (
	daemonUrl            string
	workers              int
	debug                int
	prettyPrintResponses bool
	limit                int64
	burnIn               uint64
	printInterval        uint64
	memProfile           string
	doQueries            bool
	reportDatabase       string
	reportHost           string
	reportUser           string
	reportPassword       string
	reportTagsCSV        string
)

// Global vars:
var (
	queryPool      sync.Pool
	queryChan      chan *Query
	statPool       sync.Pool
	statChan       chan *Stat
	workersGroup   sync.WaitGroup
	statGroup      sync.WaitGroup
	statMapping    statsMap
	reportTags     [][2]string
	reportHostname string
)

type S []interface{}
type M map[string]interface{}
type statsMap map[string]*StatGroup

const allQueriesLabel = "all queries"

// Parse args:
func init() {
	// needed for deserializing the mongo query from gob
	gob.Register(S{})
	gob.Register(M{})
	gob.Register([]M{})

	flag.StringVar(&daemonUrl, "url", "mongodb://localhost:27017", "Daemon URL.")
	flag.IntVar(&workers, "workers", 1, "Number of concurrent requests to make.")
	flag.IntVar(&debug, "debug", 0, "Whether to print debug messages.")
	flag.Int64Var(&limit, "limit", -1, "Limit the number of queries to send.")
	flag.Uint64Var(&burnIn, "burn-in", 0, "Number of queries to ignore before collecting statistics.")
	flag.Uint64Var(&printInterval, "print-interval", 100, "Print timing stats to stderr after this many queries (0 to disable)")
	flag.BoolVar(&prettyPrintResponses, "print-responses", false, "Pretty print JSON response bodies (for correctness checking) (default false).")
	flag.StringVar(&memProfile, "memprofile", "", "Write a memory profile to this file.")
	flag.BoolVar(&doQueries, "do-queries", true, "Whether to perform queries (useful for benchmarking the query executor.)")
	flag.StringVar(&reportDatabase, "report-database", "database_benchmarks", "Database name where to store result metrics.")
	flag.StringVar(&reportHost, "report-host", "", "Host to send result metrics.")
	flag.StringVar(&reportUser, "report-user", "", "User for Host to send result metrics.")
	flag.StringVar(&reportPassword, "report-password", "", "User password for Host to send result metrics.")
	flag.StringVar(&reportTagsCSV, "report-tags", "", "Comma separated k:v tags to send  alongside result metrics.")

	flag.Parse()

	if reportHost != "" {
		fmt.Printf("results report destination: %v\n", reportHost)
		fmt.Printf("results report database: %v\n", reportDatabase)

		var err error
		reportHostname, err = os.Hostname()
		if err != nil {
			log.Fatalf("os.Hostname() error: %s", err.Error())
		}
		fmt.Printf("hostname for results report: %v\n", reportHostname)

		if reportTagsCSV != "" {
			pairs := strings.Split(reportTagsCSV, ",")
			for _, pair := range pairs {
				fields := strings.SplitN(pair, ":", 2)
				tagpair := [2]string{fields[0], fields[1]}
				reportTags = append(reportTags, tagpair)
			}
		}
		fmt.Printf("results report tags: %v\n", reportTags)
	}
}

func main() {
	// Make pools to minimize heap usage:
	queryPool = sync.Pool{
		New: func() interface{} {
			return &Query{
				HumanLabel:       make([]byte, 0, 1024),
				HumanDescription: make([]byte, 0, 1024),
				BsonDoc:          nil,
			}
		},
	}

	statPool = sync.Pool{
		New: func() interface{} {
			return &Stat{
				Label: make([]byte, 0, 1024),
				Value: 0.0,
			}
		},
	}

	// Make data and control channels:
	queryChan = make(chan *Query, workers)
	statChan = make(chan *Stat, workers)

	// Launch the stats processor:
	statGroup.Add(1)
	go processStats()

	// Establish the connection pool:
	session, err := mgo.Dial(daemonUrl)
	if err != nil {
		log.Fatal(err)
	}

	// Launch the query processors:
	for i := 0; i < workers; i++ {
		workersGroup.Add(1)
		go processQueries(session)
	}

	// Read in jobs, closing the job channel when done:
	input := bufio.NewReaderSize(os.Stdin, 1<<20)
	wallStart := time.Now()
	scan(input)
	close(queryChan)

	// Block for workers to finish sending requests, closing the stats
	// channel when done:
	workersGroup.Wait()
	close(statChan)

	// Wait on the stat collector to finish (and print its results):
	statGroup.Wait()

	wallEnd := time.Now()
	wallTook := wallEnd.Sub(wallStart)
	_, err = fmt.Printf("wall clock time: %fsec\n", float64(wallTook.Nanoseconds())/1e9)
	if err != nil {
		log.Fatal(err)
	}

	// (Optional) create a memory profile:
	if memProfile != "" {
		f, err := os.Create(memProfile)
		if err != nil {
			log.Fatal(err)
		}
		pprof.WriteHeapProfile(f)
		f.Close()
	}
	if reportHost != "" {

		reportParams := &report.QueryReportParams{
			ReportParams: report.ReportParams{
				DBType:             "MongoDB",
				ReportDatabaseName: reportDatabase,
				ReportHost:         reportHost,
				ReportUser:         reportUser,
				ReportPassword:     reportPassword,
				ReportTags:         reportTags,
				Hostname:           reportHostname,
				DestinationUrl:     daemonUrl,
				Workers:            workers,
				ItemLimit:          int(limit),
			},
			BurnIn: int64(burnIn),
		}

		stat := statMapping[allQueriesLabel]
		err = report.ReportQueryResult(reportParams, stat.Min, stat.Mean, stat.Max, stat.Count, wallTook)

		if err != nil {
			log.Fatal(err)
		}
	}
}

// scan reads encoded Queries and places them onto the workqueue.
func scan(r io.Reader) {
	dec := gob.NewDecoder(r)

	n := int64(0)
	for {
		if limit >= 0 && n >= limit {
			break
		}

		q := queryPool.Get().(*Query)
		err := dec.Decode(q)
		if err == io.EOF {
			break
		}
		if err != nil {
			log.Fatal("decoder", err)
		}

		q.ID = n

		queryChan <- q

		n++

	}
}

// processQueries reads byte buffers from queryChan and writes them to the
// target server, while tracking latency.
func processQueries(session *mgo.Session) {
	for q := range queryChan {
		lag, err := oneQuery(session, q)

		stat := statPool.Get().(*Stat)
		stat.Init(q.HumanLabel, lag)
		statChan <- stat

		queryPool.Put(q)
		if err != nil {
			log.Fatalf("Error during request: %s\n", err.Error())
		}
	}
	workersGroup.Done()
}

// oneQuery executes on Query
func oneQuery(session *mgo.Session, q *Query) (float64, error) {
	start := time.Now().UnixNano()
	var err error
	if doQueries {
		db := session.DB(unsafeBytesToString(q.DatabaseName))
		//fmt.Printf("db: %#v\n", db)
		collection := db.C(unsafeBytesToString(q.CollectionName))
		//fmt.Printf("collection: %#v\n", collection)
		pipe := collection.Pipe(q.BsonDoc)
		iter := pipe.Iter()
		type Result struct {
			Id struct {
				TimeBucket int64 `bson:"time_bucket"`
			} `bson:"_id"`
			Value float64 `bson:"max_value"`
		}

		result := Result{}
		for iter.Next(&result) {
			if prettyPrintResponses {
				t := time.Unix(0, result.Id.TimeBucket).UTC()
				fmt.Printf("ID %d: %s, %f\n", q.ID, t, result.Value)
			}
		}

		err = iter.Close()
	}

	took := time.Now().UnixNano() - start
	lag := float64(took) / 1e6 // milliseconds
	return lag, err
}

// processStats collects latency results, aggregating them into summary
// statistics. Optionally, they are printed to stderr at regular intervals.
func processStats() {
	statMapping = statsMap{
		allQueriesLabel: &StatGroup{},
	}

	i := uint64(0)
	for stat := range statChan {
		if i < burnIn {
			i++
			statPool.Put(stat)
			continue
		} else if i == burnIn && burnIn > 0 {
			_, err := fmt.Fprintf(os.Stderr, "burn-in complete after %d queries with %d workers\n", burnIn, workers)
			if err != nil {
				log.Fatal(err)
			}
		}

		if _, ok := statMapping[string(stat.Label)]; !ok {
			statMapping[string(stat.Label)] = &StatGroup{}
		}

		statMapping[allQueriesLabel].Push(stat.Value)
		statMapping[string(stat.Label)].Push(stat.Value)

		statPool.Put(stat)

		i++

		// print stats to stderr (if printInterval is greater than zero):
		if printInterval > 0 && i > 0 && i%printInterval == 0 && (int64(i) < limit || limit < 0) {
			_, err := fmt.Fprintf(os.Stderr, "after %d queries with %d workers:\n", i-burnIn, workers)
			if err != nil {
				log.Fatal(err)
			}
			fprintStats(os.Stderr, statMapping)
			_, err = fmt.Fprintf(os.Stderr, "\n")
			if err != nil {
				log.Fatal(err)
			}
		}
	}

	// the final stats output goes to stdout:
	_, err := fmt.Printf("run complete after %d queries with %d workers:\n", i-burnIn, workers)
	if err != nil {
		log.Fatal(err)
	}
	fprintStats(os.Stdout, statMapping)
	statGroup.Done()
}

// fprintStats pretty-prints stats to the given writer.
func fprintStats(w io.Writer, statGroups statsMap) {
	maxKeyLength := 0
	keys := make([]string, 0, len(statGroups))
	for k := range statGroups {
		if len(k) > maxKeyLength {
			maxKeyLength = len(k)
		}
		keys = append(keys, k)
	}
	sort.Strings(keys)
	for _, k := range keys {
		v := statGroups[k]
		minRate := 1e3 / v.Min
		meanRate := 1e3 / v.Mean
		maxRate := 1e3 / v.Max
		paddedKey := fmt.Sprintf("%s", k)
		for len(paddedKey) < maxKeyLength {
			paddedKey += " "
		}
		_, err := fmt.Fprintf(w, "%s : min: %8.2fms (%7.2f/sec), mean: %8.2fms (%7.2f/sec), max: %7.2fms (%6.2f/sec), count: %8d, sum: %5.1fsec \n", paddedKey, v.Min, minRate, v.Mean, meanRate, v.Max, maxRate, v.Count, v.Sum/1e3)
		if err != nil {
			log.Fatal(err)
		}
	}

}
<|endoftext|>


<commit_before>// Copyright (C) 2015 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.

package cli

import (
	"container/heap"
	"encoding/binary"
	"fmt"

	"github.com/urfave/cli"

	"github.com/syncthing/syncthing/lib/db"
)

type SizedElement struct {
	key  string
	size int
}

type ElementHeap []SizedElement

func (h ElementHeap) Len() int           { return len(h) }
func (h ElementHeap) Less(i, j int) bool { return h[i].size > h[j].size }
func (h ElementHeap) Swap(i, j int)      { h[i], h[j] = h[j], h[i] }

func (h *ElementHeap) Push(x interface{}) {
	*h = append(*h, x.(SizedElement))
}

func (h *ElementHeap) Pop() interface{} {
	old := *h
	n := len(old)
	x := old[n-1]
	*h = old[0 : n-1]
	return x
}

func indexDumpSize(*cli.Context) error {
	ldb, err := getDB()
	if err != nil {
		return err
	}

	h := &ElementHeap{}
	heap.Init(h)

	it, err := ldb.NewPrefixIterator(nil)
	if err != nil {
		return err
	}
	var ele SizedElement
	for it.Next() {
		key := it.Key()
		switch key[0] {
		case db.KeyTypeDevice:
			folder := binary.BigEndian.Uint32(key[1:])
			device := binary.BigEndian.Uint32(key[1+4:])
			name := nulString(key[1+4+4:])
			ele.key = fmt.Sprintf("DEVICE:%d:%d:%s", folder, device, name)

		case db.KeyTypeGlobal:
			folder := binary.BigEndian.Uint32(key[1:])
			name := nulString(key[1+4:])
			ele.key = fmt.Sprintf("GLOBAL:%d:%s", folder, name)

		case db.KeyTypeBlock:
			folder := binary.BigEndian.Uint32(key[1:])
			hash := key[1+4 : 1+4+32]
			name := nulString(key[1+4+32:])
			ele.key = fmt.Sprintf("BLOCK:%d:%x:%s", folder, hash, name)

		case db.KeyTypeDeviceStatistic:
			ele.key = fmt.Sprintf("DEVICESTATS:%s", key[1:])

		case db.KeyTypeFolderStatistic:
			ele.key = fmt.Sprintf("FOLDERSTATS:%s", key[1:])

		case db.KeyTypeVirtualMtime:
			ele.key = fmt.Sprintf("MTIME:%s", key[1:])

		case db.KeyTypeFolderIdx:
			id := binary.BigEndian.Uint32(key[1:])
			ele.key = fmt.Sprintf("FOLDERIDX:%d", id)

		case db.KeyTypeDeviceIdx:
			id := binary.BigEndian.Uint32(key[1:])
			ele.key = fmt.Sprintf("DEVICEIDX:%d", id)

		default:
			ele.key = fmt.Sprintf("UNKNOWN:%x", key)
		}
		ele.size = len(it.Value())
		heap.Push(h, ele)
	}

	for h.Len() > 0 {
		ele = heap.Pop(h).(SizedElement)
		fmt.Println(ele.key, ele.size)
	}

	return nil
}
<commit_msg>cmd/syncthing/cli: indexDumpSize doesn't need a heap (#8024)<commit_after>// Copyright (C) 2015 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.

package cli

import (
	"encoding/binary"
	"fmt"
	"sort"

	"github.com/urfave/cli"

	"github.com/syncthing/syncthing/lib/db"
)

func indexDumpSize(*cli.Context) error {
	type sizedElement struct {
		key  string
		size int
	}

	ldb, err := getDB()
	if err != nil {
		return err
	}

	it, err := ldb.NewPrefixIterator(nil)
	if err != nil {
		return err
	}

	var elems []sizedElement
	for it.Next() {
		var ele sizedElement

		key := it.Key()
		switch key[0] {
		case db.KeyTypeDevice:
			folder := binary.BigEndian.Uint32(key[1:])
			device := binary.BigEndian.Uint32(key[1+4:])
			name := nulString(key[1+4+4:])
			ele.key = fmt.Sprintf("DEVICE:%d:%d:%s", folder, device, name)

		case db.KeyTypeGlobal:
			folder := binary.BigEndian.Uint32(key[1:])
			name := nulString(key[1+4:])
			ele.key = fmt.Sprintf("GLOBAL:%d:%s", folder, name)

		case db.KeyTypeBlock:
			folder := binary.BigEndian.Uint32(key[1:])
			hash := key[1+4 : 1+4+32]
			name := nulString(key[1+4+32:])
			ele.key = fmt.Sprintf("BLOCK:%d:%x:%s", folder, hash, name)

		case db.KeyTypeDeviceStatistic:
			ele.key = fmt.Sprintf("DEVICESTATS:%s", key[1:])

		case db.KeyTypeFolderStatistic:
			ele.key = fmt.Sprintf("FOLDERSTATS:%s", key[1:])

		case db.KeyTypeVirtualMtime:
			ele.key = fmt.Sprintf("MTIME:%s", key[1:])

		case db.KeyTypeFolderIdx:
			id := binary.BigEndian.Uint32(key[1:])
			ele.key = fmt.Sprintf("FOLDERIDX:%d", id)

		case db.KeyTypeDeviceIdx:
			id := binary.BigEndian.Uint32(key[1:])
			ele.key = fmt.Sprintf("DEVICEIDX:%d", id)

		default:
			ele.key = fmt.Sprintf("UNKNOWN:%x", key)
		}
		ele.size = len(it.Value())
		elems = append(elems, ele)
	}

	sort.Slice(elems, func(i, j int) bool {
		return elems[i].size > elems[j].size
	})
	for _, ele := range elems {
		fmt.Println(ele.key, ele.size)
	}

	return nil
}
<|endoftext|>


<commit_before>package paypal_rest

import (
	"database/sql"
	"errors"
	"time"

	"github.com/fritzpay/paymentd/pkg/paymentd/payment"
)

var (
	ErrTransactionNotFound = errors.New("transaction not found")
)

const selectTransaction = `
SELECT
	t.project_id,
	t.payment_id,
	t.timestamp,
	t.type,
	t.intent,
	t.paypal_id,
	t.payer_id,
	t.paypal_create_time,
	t.paypal_state,
	t.paypal_update_time,
	t.links,
	t.data
FROM provider_paypal_transaction AS t
`

const selectTransactionCurrentByPaymentID = selectTransaction + `
WHERE
	t.project_id = ?
	AND
	t.payment_id = ?
	AND
	t.timestamp = (
		SELECT MAX(timestamp) FROM provider_paypal_transaction
		WHERE
			project_id = t.project_id
			AND
			payment_id = t.payment_id
	)
`

func TransactionCurrentByPaymentIDTx(db *sql.Tx, paymentID payment.PaymentID) (*Transaction, error) {
	row := db.QueryRow(selectTransactionCurrentByPaymentID, paymentID.ProjectID, paymentID.PaymentID)
	t := &Transaction{}
	var ts int64
	err := row.Scan(
		&t.ProjectID,
		&t.PaymentID,
		&ts,
		&t.Type,
		&t.Intent,
		&t.PaypalID,
		&t.PayerID,
		&t.PaypalCreateTime,
		&t.PaypalState,
		&t.PaypalUpdateTime,
		&t.Links,
		&t.Data,
	)
	if err != nil {
		if err == sql.ErrNoRows {
			return t, ErrTransactionNotFound
		}
		return t, err
	}
	t.Timestamp = time.Unix(0, ts)
	return t, nil
}

const insertTransaction = `
INSERT INTO provider_paypal_transaction
(project_id, payment_id, timestamp, type, intent, paypal_id, payer_id, paypal_create_time, paypal_state, paypal_update_time, links, data)
VALUES
(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
`

func doInsertTransaction(stmt *sql.Stmt, t *Transaction) error {
	_, err := stmt.Exec(
		t.ProjectID,
		t.PaymentID,
		t.Timestamp.UnixNano(),
		t.Type,
		t.Intent,
		t.PaypalID,
		t.PayerID,
		t.PaypalCreateTime,
		t.PaypalState,
		t.PaypalUpdateTime,
		t.Links,
		t.Data,
	)
	stmt.Close()
	return err
}

func InsertTransactionTx(db *sql.Tx, t *Transaction) error {
	stmt, err := db.Prepare(insertTransaction)
	if err != nil {
		return err
	}
	return doInsertTransaction(stmt, t)
}

func InsertTransactionDB(db *sql.DB, t *Transaction) error {
	stmt, err := db.Prepare(insertTransaction)
	if err != nil {
		return err
	}
	return doInsertTransaction(stmt, t)
}
<commit_msg>select transaction on DB<commit_after>package paypal_rest

import (
	"database/sql"
	"errors"
	"time"

	"github.com/fritzpay/paymentd/pkg/paymentd/payment"
)

var (
	ErrTransactionNotFound = errors.New("transaction not found")
)

const selectTransaction = `
SELECT
	t.project_id,
	t.payment_id,
	t.timestamp,
	t.type,
	t.intent,
	t.paypal_id,
	t.payer_id,
	t.paypal_create_time,
	t.paypal_state,
	t.paypal_update_time,
	t.links,
	t.data
FROM provider_paypal_transaction AS t
`

const selectTransactionCurrentByPaymentID = selectTransaction + `
WHERE
	t.project_id = ?
	AND
	t.payment_id = ?
	AND
	t.timestamp = (
		SELECT MAX(timestamp) FROM provider_paypal_transaction
		WHERE
			project_id = t.project_id
			AND
			payment_id = t.payment_id
	)
`

func scanTransactionRow(row *sql.Row) (*Transaction, error) {
	t := &Transaction{}
	var ts int64
	err := row.Scan(
		&t.ProjectID,
		&t.PaymentID,
		&ts,
		&t.Type,
		&t.Intent,
		&t.PaypalID,
		&t.PayerID,
		&t.PaypalCreateTime,
		&t.PaypalState,
		&t.PaypalUpdateTime,
		&t.Links,
		&t.Data,
	)
	if err != nil {
		if err == sql.ErrNoRows {
			return t, ErrTransactionNotFound
		}
		return t, err
	}
	t.Timestamp = time.Unix(0, ts)
	return t, nil
}

func TransactionCurrentByPaymentIDTx(db *sql.Tx, paymentID payment.PaymentID) (*Transaction, error) {
	row := db.QueryRow(selectTransactionCurrentByPaymentID, paymentID.ProjectID, paymentID.PaymentID)
	return scanTransactionRow(row)
}

func TransactionCurrentByPaymentIDDB(db *sql.DB, paymentID payment.PaymentID) (*Transaction, error) {
	row := db.QueryRow(selectTransactionCurrentByPaymentID, paymentID.ProjectID, paymentID.PaymentID)
	return scanTransactionRow(row)
}

const insertTransaction = `
INSERT INTO provider_paypal_transaction
(project_id, payment_id, timestamp, type, intent, paypal_id, payer_id, paypal_create_time, paypal_state, paypal_update_time, links, data)
VALUES
(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
`

func doInsertTransaction(stmt *sql.Stmt, t *Transaction) error {
	_, err := stmt.Exec(
		t.ProjectID,
		t.PaymentID,
		t.Timestamp.UnixNano(),
		t.Type,
		t.Intent,
		t.PaypalID,
		t.PayerID,
		t.PaypalCreateTime,
		t.PaypalState,
		t.PaypalUpdateTime,
		t.Links,
		t.Data,
	)
	stmt.Close()
	return err
}

func InsertTransactionTx(db *sql.Tx, t *Transaction) error {
	stmt, err := db.Prepare(insertTransaction)
	if err != nil {
		return err
	}
	return doInsertTransaction(stmt, t)
}

func InsertTransactionDB(db *sql.DB, t *Transaction) error {
	stmt, err := db.Prepare(insertTransaction)
	if err != nil {
		return err
	}
	return doInsertTransaction(stmt, t)
}
<|endoftext|>