<commit_before>// Copyright (c) 2017 Cisco and/or its affiliates.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at:
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

//go:generate protoc -I ./model/cni --go_out=plugins=grpc:./model/cni ./model/cni/cni.proto
//go:generate protoc -I ./model/uid --go_out=plugins=grpc:./model/uid ./model/uid/uid.proto

package contiv

import (
	"context"
	"fmt"
	"net"

	"git.fd.io/govpp.git/api"
	"github.com/contiv/vpp/plugins/contiv/containeridx"
	"github.com/contiv/vpp/plugins/contiv/ipam"
	"github.com/contiv/vpp/plugins/contiv/model/cni"
	"github.com/contiv/vpp/plugins/kvdbproxy"
	"github.com/ligato/cn-infra/datasync"
	"github.com/ligato/cn-infra/datasync/resync"
	"github.com/ligato/cn-infra/db/keyval/etcdv3"
	"github.com/ligato/cn-infra/flavors/local"
	"github.com/ligato/cn-infra/logging"
	"github.com/ligato/cn-infra/rpc/grpc"
	"github.com/ligato/cn-infra/utils/safeclose"
	"github.com/ligato/vpp-agent/clientv1/linux"
	linuxlocalclient "github.com/ligato/vpp-agent/clientv1/linux/localclient"
	"github.com/ligato/vpp-agent/plugins/defaultplugins"
	"github.com/ligato/vpp-agent/plugins/govppmux"
)

// Plugin transforms GRPC requests into configuration for the VPP in order
// to connect a container into the network.
type Plugin struct {
	Deps
	govppCh *api.Channel

	configuredContainers *containeridx.ConfigIndex
	cniServer            *remoteCNIserver

	nodeIDAllocator   *idAllocator
	nodeIDsresyncChan chan datasync.ResyncEvent
	nodeIDSchangeChan chan datasync.ChangeEvent
	nodeIDwatchReg    datasync.WatchRegistration

	ctx           context.Context
	ctxCancelFunc context.CancelFunc

	Config *Config
}

// Deps groups the dependencies of the Plugin.
type Deps struct {
	local.PluginInfraDeps
	GRPC    grpc.Server
	Proxy   *kvdbproxy.Plugin
	VPP     *defaultplugins.Plugin
	GoVPP   govppmux.API
	Resync  resync.Subscriber
	ETCD    *etcdv3.Plugin
	Watcher datasync.KeyValProtoWatcher
}

// Config represents configuration for Contiv plugin.
// It can be injected or loaded from external config file. Injection has priority to external config. To use external
// config file, add `-contiv-config="<path to config>` argument when running the contiv-agent.
type Config struct {
	TCPChecksumOffloadDisabled bool
	TCPstackDisabled           bool
	UseTAPInterfaces           bool
	TAPInterfaceVersion        uint8
	TAPv2RxRingSize            uint16
	TAPv2TxRingSize            uint16
	IPAMConfig                 ipam.Config
	NodeConfig                 []OneNodeConfig
}

// OneNodeConfig represents configuration for one node. It contains only settings specific to given node.
type OneNodeConfig struct {
	NodeName             string
	MainVppInterfaceName string
	OtherVPPInterfaces   []InterfaceWithIP // other configured interfaces get only ip address assigned in vpp
}

// InterfaceWithIP binds interface name with ip for configuration purposes.
type InterfaceWithIP struct {
	InterfaceName string
	IP            string
}

// Init initializes the grpc server handling the request from the CNI.
func (plugin *Plugin) Init() error {
	plugin.configuredContainers = containeridx.NewConfigIndex(plugin.Log, plugin.PluginName, "containers")

	plugin.ctx, plugin.ctxCancelFunc = context.WithCancel(context.Background())
	if plugin.Config == nil {
		if err := plugin.applyExternalConfig(); err != nil {
			return err
		}
	}

	var err error
	plugin.govppCh, err = plugin.GoVPP.NewAPIChannel()
	if err != nil {
		return err
	}

	plugin.nodeIDAllocator = newIDAllocator(plugin.ETCD, plugin.ServiceLabel.GetAgentLabel())
	nodeID, err := plugin.nodeIDAllocator.getID()
	if err != nil {
		return err
	}
	plugin.Log.Infof("ID of the node is %v", nodeID)

	plugin.nodeIDsresyncChan = make(chan datasync.ResyncEvent)
	plugin.nodeIDSchangeChan = make(chan datasync.ChangeEvent)

	plugin.nodeIDwatchReg, err = plugin.Watcher.Watch("contiv-plugin", plugin.nodeIDSchangeChan, plugin.nodeIDsresyncChan, allocatedIDsKeyPrefix)
	if err != nil {
		return err
	}

	plugin.cniServer, err = newRemoteCNIServer(plugin.Log,
		func() linux.DataChangeDSL {
			return linuxlocalclient.DataChangeRequest(plugin.PluginName)
		},
		plugin.Proxy,
		plugin.configuredContainers,
		plugin.govppCh,
		plugin.VPP.GetSwIfIndexes(),
		plugin.ServiceLabel.GetAgentLabel(),
		plugin.Config,
		nodeID)
	if err != nil {
		return fmt.Errorf("Can't create new remote CNI server due to error: %v ", err)
	}
	cni.RegisterRemoteCNIServer(plugin.GRPC.Server(), plugin.cniServer)

	go plugin.cniServer.handleNodeEvents(plugin.ctx, plugin.nodeIDsresyncChan, plugin.nodeIDSchangeChan)

	return nil
}

func (plugin *Plugin) applyExternalConfig() error {
	externalCfg := &Config{}
	found, err := plugin.PluginConfig.GetValue(externalCfg) // It tries to lookup `PluginName + "-config"` in go run command flags.
	if err != nil {
		return fmt.Errorf("External Contiv plugin configuration could not load or other problem happened: %v", err)
	}
	if !found {
		return fmt.Errorf("External Contiv plugin configuration was not found")
	}
	plugin.Config = externalCfg
	return nil
}

// AfterInit registers to the ResyncOrchestrator. The registration is done in this phase
// in order to trigger the resync for this plugin once the resync of VPP plugins is finished.
func (plugin *Plugin) AfterInit() error {
	if plugin.Resync != nil {
		reg := plugin.Resync.Register(string(plugin.PluginName))
		go plugin.handleResync(reg.StatusChan())
	}
	return nil
}

// Close cleans up the resources allocated by the plugin
func (plugin *Plugin) Close() error {
	plugin.ctxCancelFunc()
	plugin.cniServer.close()
	plugin.nodeIDAllocator.releaseID()
	_, err := safeclose.CloseAll(plugin.govppCh, plugin.nodeIDwatchReg)
	return err
}

// getContainerConfig return the configuration of the container associated
// with the given pod.
func (plugin *Plugin) getContainerConfig(podNamespace string, podName string) *containeridx.Config {
	podNamesMatch := plugin.configuredContainers.LookupPodName(podName)
	podNamespacesMatch := plugin.configuredContainers.LookupPodNamespace(podNamespace)

	for _, pod1 := range podNamespacesMatch {
		for _, pod2 := range podNamesMatch {
			if pod1 == pod2 {
				data, found := plugin.configuredContainers.LookupContainer(pod1)
				if found {
					return data
				}
			}
		}
	}

	return nil
}

// GetIfName looks up logical interface name that corresponds to the interface associated with the given pod.
func (plugin *Plugin) GetIfName(podNamespace string, podName string) (name string, exists bool) {
	config := plugin.getContainerConfig(podNamespace, podName)
	if config != nil && config.VppIf != nil {
		return config.VppIf.Name, true
	}
	plugin.Log.WithFields(logging.Fields{"podNamespace": podNamespace, "podName": podName}).Warn("No matching result found")
	return "", false
}

// GetNsIndex returns the index of the VPP session namespace associated
// with the given pod.
func (plugin *Plugin) GetNsIndex(podNamespace string, podName string) (nsIndex uint32, exists bool) {
	config := plugin.getContainerConfig(podNamespace, podName)
	if config != nil {
		nsIndex, _, exists = plugin.VPP.GetAppNsIndexes().LookupIdx(config.AppNamespace.NamespaceId)
		return nsIndex, exists
	}
	plugin.Log.WithFields(logging.Fields{"podNamespace": podNamespace, "podName": podName}).Warn("No matching result found")
	return 0, false
}

// GetPodNetwork provides subnet used for allocating pod IP addresses on this host node.
func (plugin *Plugin) GetPodNetwork() *net.IPNet {
	return plugin.cniServer.ipam.PodNetwork()
}

// IsTCPstackDisabled returns true if the tcp stack is disabled and only VETHs/TAPs are configured
func (plugin *Plugin) IsTCPstackDisabled() bool {
	return plugin.Config.TCPstackDisabled
}

func (plugin *Plugin) handleResync(resyncChan chan resync.StatusEvent) {
	for {
		select {
		case ev := <-resyncChan:
			status := ev.ResyncStatus()
			if status == resync.Started {
				err := plugin.cniServer.resync()
				if err != nil {
					plugin.Log.Error(err)
				}
			}
			ev.Ack()
		case <-plugin.ctx.Done():
			return
		}
	}
}
<commit_msg>Tap v2 is default is UseTAPInterfaces is set to true. In order to use legacy tap specify TAPInterfaceVersion = 1<commit_after>// Copyright (c) 2017 Cisco and/or its affiliates.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at:
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

//go:generate protoc -I ./model/cni --go_out=plugins=grpc:./model/cni ./model/cni/cni.proto
//go:generate protoc -I ./model/uid --go_out=plugins=grpc:./model/uid ./model/uid/uid.proto

package contiv

import (
	"context"
	"fmt"
	"net"

	"git.fd.io/govpp.git/api"
	"github.com/contiv/vpp/plugins/contiv/containeridx"
	"github.com/contiv/vpp/plugins/contiv/ipam"
	"github.com/contiv/vpp/plugins/contiv/model/cni"
	"github.com/contiv/vpp/plugins/kvdbproxy"
	"github.com/ligato/cn-infra/datasync"
	"github.com/ligato/cn-infra/datasync/resync"
	"github.com/ligato/cn-infra/db/keyval/etcdv3"
	"github.com/ligato/cn-infra/flavors/local"
	"github.com/ligato/cn-infra/logging"
	"github.com/ligato/cn-infra/rpc/grpc"
	"github.com/ligato/cn-infra/utils/safeclose"
	"github.com/ligato/vpp-agent/clientv1/linux"
	linuxlocalclient "github.com/ligato/vpp-agent/clientv1/linux/localclient"
	"github.com/ligato/vpp-agent/plugins/defaultplugins"
	"github.com/ligato/vpp-agent/plugins/govppmux"
)

// Plugin transforms GRPC requests into configuration for the VPP in order
// to connect a container into the network.
type Plugin struct {
	Deps
	govppCh *api.Channel

	configuredContainers *containeridx.ConfigIndex
	cniServer            *remoteCNIserver

	nodeIDAllocator   *idAllocator
	nodeIDsresyncChan chan datasync.ResyncEvent
	nodeIDSchangeChan chan datasync.ChangeEvent
	nodeIDwatchReg    datasync.WatchRegistration

	ctx           context.Context
	ctxCancelFunc context.CancelFunc

	Config *Config
}

// Deps groups the dependencies of the Plugin.
type Deps struct {
	local.PluginInfraDeps
	GRPC    grpc.Server
	Proxy   *kvdbproxy.Plugin
	VPP     *defaultplugins.Plugin
	GoVPP   govppmux.API
	Resync  resync.Subscriber
	ETCD    *etcdv3.Plugin
	Watcher datasync.KeyValProtoWatcher
}

// Config represents configuration for Contiv plugin.
// It can be injected or loaded from external config file. Injection has priority to external config. To use external
// config file, add `-contiv-config="<path to config>` argument when running the contiv-agent.
type Config struct {
	TCPChecksumOffloadDisabled bool
	TCPstackDisabled           bool
	UseTAPInterfaces           bool
	TAPInterfaceVersion        uint8
	TAPv2RxRingSize            uint16
	TAPv2TxRingSize            uint16
	IPAMConfig                 ipam.Config
	NodeConfig                 []OneNodeConfig
}

// OneNodeConfig represents configuration for one node. It contains only settings specific to given node.
type OneNodeConfig struct {
	NodeName             string
	MainVppInterfaceName string
	OtherVPPInterfaces   []InterfaceWithIP // other configured interfaces get only ip address assigned in vpp
}

// InterfaceWithIP binds interface name with ip for configuration purposes.
type InterfaceWithIP struct {
	InterfaceName string
	IP            string
}

// Init initializes the grpc server handling the request from the CNI.
func (plugin *Plugin) Init() error {
	plugin.configuredContainers = containeridx.NewConfigIndex(plugin.Log, plugin.PluginName, "containers")

	plugin.ctx, plugin.ctxCancelFunc = context.WithCancel(context.Background())
	if plugin.Config == nil {
		if err := plugin.applyExternalConfig(); err != nil {
			return err
		}
	}

	var err error
	plugin.govppCh, err = plugin.GoVPP.NewAPIChannel()
	if err != nil {
		return err
	}

	plugin.nodeIDAllocator = newIDAllocator(plugin.ETCD, plugin.ServiceLabel.GetAgentLabel())
	nodeID, err := plugin.nodeIDAllocator.getID()
	if err != nil {
		return err
	}
	plugin.Log.Infof("ID of the node is %v", nodeID)

	plugin.nodeIDsresyncChan = make(chan datasync.ResyncEvent)
	plugin.nodeIDSchangeChan = make(chan datasync.ChangeEvent)

	plugin.nodeIDwatchReg, err = plugin.Watcher.Watch("contiv-plugin", plugin.nodeIDSchangeChan, plugin.nodeIDsresyncChan, allocatedIDsKeyPrefix)
	if err != nil {
		return err
	}

	plugin.cniServer, err = newRemoteCNIServer(plugin.Log,
		func() linux.DataChangeDSL {
			return linuxlocalclient.DataChangeRequest(plugin.PluginName)
		},
		plugin.Proxy,
		plugin.configuredContainers,
		plugin.govppCh,
		plugin.VPP.GetSwIfIndexes(),
		plugin.ServiceLabel.GetAgentLabel(),
		plugin.Config,
		nodeID)
	if err != nil {
		return fmt.Errorf("Can't create new remote CNI server due to error: %v ", err)
	}
	cni.RegisterRemoteCNIServer(plugin.GRPC.Server(), plugin.cniServer)

	go plugin.cniServer.handleNodeEvents(plugin.ctx, plugin.nodeIDsresyncChan, plugin.nodeIDSchangeChan)

	return nil
}

func (plugin *Plugin) applyExternalConfig() error {
	externalCfg := &Config{}
	found, err := plugin.PluginConfig.GetValue(externalCfg) // It tries to lookup `PluginName + "-config"` in go run command flags.
	if err != nil {
		return fmt.Errorf("External Contiv plugin configuration could not load or other problem happened: %v", err)
	}
	if !found {
		return fmt.Errorf("External Contiv plugin configuration was not found")
	}
	plugin.Config = externalCfg

	//use tap version 2 as default
	if plugin.Config.TAPInterfaceVersion == 0 {
		plugin.Config.TAPInterfaceVersion = 2
	}
	return nil
}

// AfterInit registers to the ResyncOrchestrator. The registration is done in this phase
// in order to trigger the resync for this plugin once the resync of VPP plugins is finished.
func (plugin *Plugin) AfterInit() error {
	if plugin.Resync != nil {
		reg := plugin.Resync.Register(string(plugin.PluginName))
		go plugin.handleResync(reg.StatusChan())
	}
	return nil
}

// Close cleans up the resources allocated by the plugin
func (plugin *Plugin) Close() error {
	plugin.ctxCancelFunc()
	plugin.cniServer.close()
	plugin.nodeIDAllocator.releaseID()
	_, err := safeclose.CloseAll(plugin.govppCh, plugin.nodeIDwatchReg)
	return err
}

// getContainerConfig return the configuration of the container associated
// with the given pod.
func (plugin *Plugin) getContainerConfig(podNamespace string, podName string) *containeridx.Config {
	podNamesMatch := plugin.configuredContainers.LookupPodName(podName)
	podNamespacesMatch := plugin.configuredContainers.LookupPodNamespace(podNamespace)

	for _, pod1 := range podNamespacesMatch {
		for _, pod2 := range podNamesMatch {
			if pod1 == pod2 {
				data, found := plugin.configuredContainers.LookupContainer(pod1)
				if found {
					return data
				}
			}
		}
	}

	return nil
}

// GetIfName looks up logical interface name that corresponds to the interface associated with the given pod.
func (plugin *Plugin) GetIfName(podNamespace string, podName string) (name string, exists bool) {
	config := plugin.getContainerConfig(podNamespace, podName)
	if config != nil && config.VppIf != nil {
		return config.VppIf.Name, true
	}
	plugin.Log.WithFields(logging.Fields{"podNamespace": podNamespace, "podName": podName}).Warn("No matching result found")
	return "", false
}

// GetNsIndex returns the index of the VPP session namespace associated
// with the given pod.
func (plugin *Plugin) GetNsIndex(podNamespace string, podName string) (nsIndex uint32, exists bool) {
	config := plugin.getContainerConfig(podNamespace, podName)
	if config != nil {
		nsIndex, _, exists = plugin.VPP.GetAppNsIndexes().LookupIdx(config.AppNamespace.NamespaceId)
		return nsIndex, exists
	}
	plugin.Log.WithFields(logging.Fields{"podNamespace": podNamespace, "podName": podName}).Warn("No matching result found")
	return 0, false
}

// GetPodNetwork provides subnet used for allocating pod IP addresses on this host node.
func (plugin *Plugin) GetPodNetwork() *net.IPNet {
	return plugin.cniServer.ipam.PodNetwork()
}

// IsTCPstackDisabled returns true if the tcp stack is disabled and only VETHs/TAPs are configured
func (plugin *Plugin) IsTCPstackDisabled() bool {
	return plugin.Config.TCPstackDisabled
}

func (plugin *Plugin) handleResync(resyncChan chan resync.StatusEvent) {
	for {
		select {
		case ev := <-resyncChan:
			status := ev.ResyncStatus()
			if status == resync.Started {
				err := plugin.cniServer.resync()
				if err != nil {
					plugin.Log.Error(err)
				}
			}
			ev.Ack()
		case <-plugin.ctx.Done():
			return
		}
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2018 The Jetstack cert-manager contributors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
	"flag"
	"fmt"
	"path"
	"testing"
	"time"

	"github.com/onsi/ginkgo"
	ginkgoconfig "github.com/onsi/ginkgo/config"
	"github.com/onsi/ginkgo/reporters"
	"github.com/onsi/gomega"
	"k8s.io/apimachinery/pkg/util/wait"

	"github.com/jetstack/cert-manager/test/e2e/framework"
	_ "github.com/jetstack/cert-manager/test/e2e/suite"
)

func init() {
	framework.DefaultConfig.AddFlags(flag.CommandLine)
	flag.Parse()

	// Turn on verbose by default to get spec names
	ginkgoconfig.DefaultReporterConfig.Verbose = true
	// Turn on EmitSpecProgress to get spec progress (especially on interrupt)
	ginkgoconfig.GinkgoConfig.EmitSpecProgress = true
	// Randomize specs as well as suites
	ginkgoconfig.GinkgoConfig.RandomizeAllSpecs = true

	wait.ForeverTestTimeout = time.Second * 60
}

func TestE2E(t *testing.T) {
	if err := framework.DefaultConfig.Validate(); err != nil {
		t.Errorf("Invalid test config: %v", err)
		t.Fail()
	}

	gomega.RegisterFailHandler(ginkgo.Fail)

	// Disable skipped tests unless they are explicitly requested.
	if ginkgoconfig.GinkgoConfig.FocusString == "" && ginkgoconfig.GinkgoConfig.SkipString == "" {
		ginkgoconfig.GinkgoConfig.SkipString = `\[Flaky\]|\[Feature:.+\]`
	}

	var r []ginkgo.Reporter
	if framework.DefaultConfig.Ginkgo.ReportDirectory != "" {
		r = append(r, reporters.NewJUnitReporter(path.Join(framework.DefaultConfig.Ginkgo.ReportDirectory,
			fmt.Sprintf("junit_%s_%02d.xml",
				framework.DefaultConfig.Ginkgo.ReportPrefix,
				ginkgoconfig.GinkgoConfig.ParallelNode))))
	}

	ginkgo.RunSpecsWithDefaultAndCustomReporters(t, "cert-manager e2e suite", r)
}
<commit_msg>Remove default SkipString<commit_after>/*
Copyright 2018 The Jetstack cert-manager contributors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
	"flag"
	"fmt"
	"path"
	"testing"
	"time"

	"github.com/onsi/ginkgo"
	ginkgoconfig "github.com/onsi/ginkgo/config"
	"github.com/onsi/ginkgo/reporters"
	"github.com/onsi/gomega"
	"k8s.io/apimachinery/pkg/util/wait"

	"github.com/jetstack/cert-manager/test/e2e/framework"
	_ "github.com/jetstack/cert-manager/test/e2e/suite"
)

func init() {
	framework.DefaultConfig.AddFlags(flag.CommandLine)
	flag.Parse()

	// Turn on verbose by default to get spec names
	ginkgoconfig.DefaultReporterConfig.Verbose = true
	// Turn on EmitSpecProgress to get spec progress (especially on interrupt)
	ginkgoconfig.GinkgoConfig.EmitSpecProgress = true
	// Randomize specs as well as suites
	ginkgoconfig.GinkgoConfig.RandomizeAllSpecs = true

	wait.ForeverTestTimeout = time.Second * 60
}

func TestE2E(t *testing.T) {
	if err := framework.DefaultConfig.Validate(); err != nil {
		t.Errorf("Invalid test config: %v", err)
		t.Fail()
	}

	gomega.RegisterFailHandler(ginkgo.Fail)

	// TODO: properly make use of default SkipString
	// Disable skipped tests unless they are explicitly requested.
	// if ginkgoconfig.GinkgoConfig.FocusString == "" && ginkgoconfig.GinkgoConfig.SkipString == "" {
	// 	ginkgoconfig.GinkgoConfig.SkipString = `\[Flaky\]|\[Feature:.+\]`
	// }

	var r []ginkgo.Reporter
	if framework.DefaultConfig.Ginkgo.ReportDirectory != "" {
		r = append(r, reporters.NewJUnitReporter(path.Join(framework.DefaultConfig.Ginkgo.ReportDirectory,
			fmt.Sprintf("junit_%s_%02d.xml",
				framework.DefaultConfig.Ginkgo.ReportPrefix,
				ginkgoconfig.GinkgoConfig.ParallelNode))))
	}

	ginkgo.RunSpecsWithDefaultAndCustomReporters(t, "cert-manager e2e suite", r)
}
<|endoftext|>


<commit_before>package sat

import (
	"github.com/mitchellh/go-sat/cnf"
	"github.com/mitchellh/go-sat/packed"
)

// Solver is a SAT solver. This should be created with New to get
// the proper internal memory allocations. Using a manually allocated
// Solver will probably crash.
type Solver struct {
	// Trace, if set to true, will output trace debugging information
	// via the standard library `log` package. If true, Tracer must also
	// be set to a non-nil value. The easiest implmentation is a logger
	// created with log.NewLogger.
	Trace  bool
	Tracer Tracer

	// decideLiterals is to be set by tests to force a certain decision
	// literal ordering. This can be used to exercise specific solver
	// behavior being tested.
	decideLiterals []int

	//---------------------------------------------------------------
	// Internal fields, do not set
	//---------------------------------------------------------------
	result satResult

	f         cnf.Formula // formula we're solving
	reasonMap map[cnf.Literal]cnf.Clause

	// problem
	clauses []packed.Clause  // clauses to solve
	vars    map[int]struct{} // list of available vars

	// conflict clause caching
	c  cnf.Clause
	cH map[cnf.Literal]struct{} // literals in C
	cP map[cnf.Literal]struct{} // literals in lower decision levels of C
	cL cnf.Literal              // last asserted literal in C
	cN int                      // number of literals in the highest decision level of C

	//---------------------------------------------------------------
	// trail
	//---------------------------------------------------------------

	// trail is the actual trail of assigned literals. The value assigned
	// is in the assigns map.
	trail []packed.Lit

	// trailIdx keeps track of the indices for new decision levels.
	// trailIdx[level] = index to the start of that level in trail
	trailIdx []int

	// assigns keeps track of variable assignment values. unassigned variables
	// are never present in assigns.
	assigns map[int]Tribool

	// varinfo holds information about an assigned variable. unassigned
	// variables may be present here but their resulting information is
	// garbage.
	varinfo map[int]varinfo
}

// New creates a new solver and allocates the basics for it.
func New() *Solver {
	return &Solver{
		result: satResultUndef,

		reasonMap: make(map[cnf.Literal]cnf.Clause),

		// problem
		vars: make(map[int]struct{}),

		// trail
		assigns: make(map[int]Tribool),
		varinfo: make(map[int]varinfo),
	}
}

// Solve finds a solution for the formula, returning true on satisfiability.
func (s *Solver) Solve() bool {
	if s.Trace {
		s.Tracer.Printf("[TRACE] sat: starting solve()")
	}

	// Check the result. This can be set already by a prior call to Solve
	// or via the AddClause process.
	if s.result != satResultUndef {
		if s.Trace {
			s.Tracer.Printf(
				"[TRACE] sat: result is already available: %s", s.result)
		}

		return s.result == satResultSat
	}

	for {
		// Perform unit propagation
		s.unitPropagate()

		conflictC := s.isFormulaFalse()
		if !conflictC.IsZero() {
			if s.Trace {
				s.Tracer.Printf("[TRACE] sat: current trail contains negated formula: %#v", s.trail)
				s.Tracer.Printf("[TRACE] sat: conflict clause: %#v", conflictC)
			}

			// Set our conflict clause
			s.applyConflict(conflictC)

			// If we have no more decisions within the trail, then we've
			// failed finding a satisfying value.
			if s.decisionLevel() == 0 {
				return false
			}

			// Explain to learn our conflict clause
			s.applyExplainUIP()
			if len(s.c) > 1 {
				if s.Trace {
					s.Tracer.Printf("[TRACE] sat: learned clause: %#v", s.c)
				}
				s.f = append(s.f, s.c)
			}
			s.applyBackjump()
		} else {
			// Choose a literal to assert.
			lit := s.selectLiteral()

			// If it is undef it means there are no more literals which means
			// we have solved the formula
			if lit == packed.LitUndef {
				if s.Trace {
					s.Tracer.Printf("[TRACE] sat: solver found solution: %#v", s.trail)
				}

				return true
			}

			if s.Trace {
				s.Tracer.Printf("[TRACE] sat: assert: %s (decision)", lit)
			}
			s.newDecisionLevel()
			s.assertLiteral(lit)
		}
	}

	return false
}

func (s *Solver) selectLiteral() packed.Lit {
	for raw, _ := range s.vars {
		if _, ok := s.assigns[raw]; !ok {
			return packed.NewLit(raw, false)
		}
	}

	return packed.LitUndef
}

//-------------------------------------------------------------------
// Unit Propagation
//-------------------------------------------------------------------

func (s *Solver) unitPropagate() {
	for {
		for _, c := range s.f {
			for _, l := range c {
				if s.isUnit(c, l) {
					if s.Trace {
						s.Tracer.Printf(
							"[TRACE] sat: found unit clause %v with literal %d in trail %#v",
							c, l, s.trail)
					}

					s.assertLiteral(l.Pack())
					s.reasonMap[l] = c
					goto UNIT_REPEAT
				}
			}
		}

		// We didn't find a unit clause, close it out
		return

	UNIT_REPEAT:
		// We found a unit clause but we have to check if we violated
		// constraints in the trail.
		if !s.isFormulaFalse().IsZero() {
			return
		}
	}
}

//-------------------------------------------------------------------
// Conflict Clause Learning
//-------------------------------------------------------------------

func (s *Solver) applyConflict(c cnf.Clause) {
	// Build up our lookup caches for the conflict data to optimize
	// the conflict learning process.
	s.cH = make(map[cnf.Literal]struct{})
	s.cP = make(map[cnf.Literal]struct{})
	s.cN = 0
	for _, l := range c {
		s.addConflictLiteral(l)
	}

	// Find the last asserted literal using the cache
	for i := len(s.trail) - 1; i >= 0; i-- {
		s.cL = cnf.Literal(s.trail[i].Int())
		if _, ok := s.cH[s.cL.Negate()]; ok {
			break
		}
	}

	if s.Trace {
		s.Tracer.Printf(
			"[TRACE] sat: applyConflict. cH = %v; cP = %v; cL = %d; cN = %d",
			s.cH, s.cP, s.cL, s.cN)
	}
}

func (s *Solver) addConflictLiteral(l cnf.Literal) {
	if _, ok := s.cH[l]; !ok {
		level := s.level(l.Pack().Var())
		if level > 0 {
			s.cH[l] = struct{}{}
			if level == s.decisionLevel() {
				s.cN++
			} else {
				s.cP[l] = struct{}{}
			}
		}
	}
}

func (s *Solver) removeConflictLiteral(l cnf.Literal) {
	delete(s.cH, l)

	if s.level(l.Pack().Var()) == s.decisionLevel() {
		s.cN--
	} else {
		delete(s.cP, l)
	}
}

func (s *Solver) applyExplain(lit cnf.Literal) {
	s.removeConflictLiteral(lit.Negate())

	reason := s.reasonMap[lit]
	for _, l := range reason {
		if l != lit {
			s.addConflictLiteral(l)
		}
	}

	// Find the last asserted literal using the cache
	for i := len(s.trail) - 1; i >= 0; i-- {
		s.cL = cnf.Literal(s.trail[i].Int())
		if _, ok := s.cH[s.cL.Negate()]; ok {
			break
		}
	}

	if s.Trace {
		s.Tracer.Printf("[TRACE] sat: applyExplain: lit = %d, reason = %#v", lit, reason)
		s.Tracer.Printf(
			"[TRACE] sat: applyExplain. cH = %v; cP = %v; cL = %d; cN = %d",
			s.cH, s.cP, s.cL, s.cN)
	}
}

func (s *Solver) applyExplainUIP() {
	for s.cN != 1 { // !isUIP
		s.applyExplain(s.cL)
	}

	// buildC
	c := make([]cnf.Literal, 0, len(s.cP)+1)
	for l, _ := range s.cP {
		c = append(c, l)
	}
	c = append(c, s.cL.Negate())
	s.c = c
}

func (s *Solver) isUIP() bool {
	return s.cN == 1
}

//-------------------------------------------------------------------
// Backjumping
//-------------------------------------------------------------------

func (s *Solver) applyBackjump() {
	level := 0
	if len(s.cP) > 0 {
		for l, _ := range s.cP {
			if v := s.level(l.Pack().Var()); v > level {
				level = v
			}
		}
	}

	if s.Trace {
		s.Tracer.Printf(
			"[TRACE] sat: backjump. C = %#v; l = %d; level = %d",
			s.c, s.cL, level)
	}

	s.trimToDecisionLevel(level)

	lit := s.cL.Negate()
	s.assertLiteral(lit.Pack())
	s.reasonMap[lit] = s.c

	if s.Trace {
		s.Tracer.Printf("[TRACE] sat: backjump. M = %#v", s.trail)
	}
}

//-------------------------------------------------------------------
// Private types
//-------------------------------------------------------------------

type satResult byte

const (
	satResultUndef satResult = iota
	satResultUnsat
	satResultSat
)

type varinfo struct {
	level int
}

// Tribool is a tri-state boolean with undefined as the 3rd state.
type Tribool uint8

const (
	True  Tribool = 0
	False         = 1
	Undef         = 2
)

func BoolToTri(b bool) Tribool {
	if b {
		return True
	}

	return False
}
<commit_msg>fix output<commit_after>package sat

import (
	"github.com/mitchellh/go-sat/cnf"
	"github.com/mitchellh/go-sat/packed"
)

// Solver is a SAT solver. This should be created with New to get
// the proper internal memory allocations. Using a manually allocated
// Solver will probably crash.
type Solver struct {
	// Trace, if set to true, will output trace debugging information
	// via the standard library `log` package. If true, Tracer must also
	// be set to a non-nil value. The easiest implmentation is a logger
	// created with log.NewLogger.
	Trace  bool
	Tracer Tracer

	// decideLiterals is to be set by tests to force a certain decision
	// literal ordering. This can be used to exercise specific solver
	// behavior being tested.
	decideLiterals []int

	//---------------------------------------------------------------
	// Internal fields, do not set
	//---------------------------------------------------------------
	result satResult

	f         cnf.Formula // formula we're solving
	reasonMap map[cnf.Literal]cnf.Clause

	// problem
	clauses []packed.Clause  // clauses to solve
	vars    map[int]struct{} // list of available vars

	// conflict clause caching
	c  cnf.Clause
	cH map[cnf.Literal]struct{} // literals in C
	cP map[cnf.Literal]struct{} // literals in lower decision levels of C
	cL cnf.Literal              // last asserted literal in C
	cN int                      // number of literals in the highest decision level of C

	//---------------------------------------------------------------
	// trail
	//---------------------------------------------------------------

	// trail is the actual trail of assigned literals. The value assigned
	// is in the assigns map.
	trail []packed.Lit

	// trailIdx keeps track of the indices for new decision levels.
	// trailIdx[level] = index to the start of that level in trail
	trailIdx []int

	// assigns keeps track of variable assignment values. unassigned variables
	// are never present in assigns.
	assigns map[int]Tribool

	// varinfo holds information about an assigned variable. unassigned
	// variables may be present here but their resulting information is
	// garbage.
	varinfo map[int]varinfo
}

// New creates a new solver and allocates the basics for it.
func New() *Solver {
	return &Solver{
		result: satResultUndef,

		reasonMap: make(map[cnf.Literal]cnf.Clause),

		// problem
		vars: make(map[int]struct{}),

		// trail
		assigns: make(map[int]Tribool),
		varinfo: make(map[int]varinfo),
	}
}

// Solve finds a solution for the formula, returning true on satisfiability.
func (s *Solver) Solve() bool {
	if s.Trace {
		s.Tracer.Printf("[TRACE] sat: starting solve()")
	}

	// Check the result. This can be set already by a prior call to Solve
	// or via the AddClause process.
	if s.result != satResultUndef {
		if s.Trace {
			s.Tracer.Printf(
				"[TRACE] sat: result is already available: %s", s.result)
		}

		return s.result == satResultSat
	}

	for {
		// Perform unit propagation
		s.unitPropagate()

		conflictC := s.isFormulaFalse()
		if !conflictC.IsZero() {
			if s.Trace {
				s.Tracer.Printf("[TRACE] sat: current trail contains negated formula: %s", s.trail)
				s.Tracer.Printf("[TRACE] sat: conflict clause: %#v", conflictC)
			}

			// Set our conflict clause
			s.applyConflict(conflictC)

			// If we have no more decisions within the trail, then we've
			// failed finding a satisfying value.
			if s.decisionLevel() == 0 {
				return false
			}

			// Explain to learn our conflict clause
			s.applyExplainUIP()
			if len(s.c) > 1 {
				if s.Trace {
					s.Tracer.Printf("[TRACE] sat: learned clause: %#v", s.c)
				}
				s.f = append(s.f, s.c)
			}
			s.applyBackjump()
		} else {
			// Choose a literal to assert.
			lit := s.selectLiteral()

			// If it is undef it means there are no more literals which means
			// we have solved the formula
			if lit == packed.LitUndef {
				if s.Trace {
					s.Tracer.Printf("[TRACE] sat: solver found solution: %s", s.trail)
				}

				return true
			}

			if s.Trace {
				s.Tracer.Printf("[TRACE] sat: assert: %s (decision)", lit)
			}
			s.newDecisionLevel()
			s.assertLiteral(lit)
		}
	}

	return false
}

func (s *Solver) selectLiteral() packed.Lit {
	for raw, _ := range s.vars {
		if _, ok := s.assigns[raw]; !ok {
			return packed.NewLit(raw, false)
		}
	}

	return packed.LitUndef
}

//-------------------------------------------------------------------
// Unit Propagation
//-------------------------------------------------------------------

func (s *Solver) unitPropagate() {
	for {
		for _, c := range s.f {
			for _, l := range c {
				if s.isUnit(c, l) {
					if s.Trace {
						s.Tracer.Printf(
							"[TRACE] sat: found unit clause %v with literal %d in trail %s",
							c, l, s.trail)
					}

					s.assertLiteral(l.Pack())
					s.reasonMap[l] = c
					goto UNIT_REPEAT
				}
			}
		}

		// We didn't find a unit clause, close it out
		return

	UNIT_REPEAT:
		// We found a unit clause but we have to check if we violated
		// constraints in the trail.
		if !s.isFormulaFalse().IsZero() {
			return
		}
	}
}

//-------------------------------------------------------------------
// Conflict Clause Learning
//-------------------------------------------------------------------

func (s *Solver) applyConflict(c cnf.Clause) {
	// Build up our lookup caches for the conflict data to optimize
	// the conflict learning process.
	s.cH = make(map[cnf.Literal]struct{})
	s.cP = make(map[cnf.Literal]struct{})
	s.cN = 0
	for _, l := range c {
		s.addConflictLiteral(l)
	}

	// Find the last asserted literal using the cache
	for i := len(s.trail) - 1; i >= 0; i-- {
		s.cL = cnf.Literal(s.trail[i].Int())
		if _, ok := s.cH[s.cL.Negate()]; ok {
			break
		}
	}

	if s.Trace {
		s.Tracer.Printf(
			"[TRACE] sat: applyConflict. cH = %v; cP = %v; cL = %d; cN = %d",
			s.cH, s.cP, s.cL, s.cN)
	}
}

func (s *Solver) addConflictLiteral(l cnf.Literal) {
	if _, ok := s.cH[l]; !ok {
		level := s.level(l.Pack().Var())
		if level > 0 {
			s.cH[l] = struct{}{}
			if level == s.decisionLevel() {
				s.cN++
			} else {
				s.cP[l] = struct{}{}
			}
		}
	}
}

func (s *Solver) removeConflictLiteral(l cnf.Literal) {
	delete(s.cH, l)

	if s.level(l.Pack().Var()) == s.decisionLevel() {
		s.cN--
	} else {
		delete(s.cP, l)
	}
}

func (s *Solver) applyExplain(lit cnf.Literal) {
	s.removeConflictLiteral(lit.Negate())

	reason := s.reasonMap[lit]
	for _, l := range reason {
		if l != lit {
			s.addConflictLiteral(l)
		}
	}

	// Find the last asserted literal using the cache
	for i := len(s.trail) - 1; i >= 0; i-- {
		s.cL = cnf.Literal(s.trail[i].Int())
		if _, ok := s.cH[s.cL.Negate()]; ok {
			break
		}
	}

	if s.Trace {
		s.Tracer.Printf("[TRACE] sat: applyExplain: lit = %d, reason = %#v", lit, reason)
		s.Tracer.Printf(
			"[TRACE] sat: applyExplain. cH = %v; cP = %v; cL = %d; cN = %d",
			s.cH, s.cP, s.cL, s.cN)
	}
}

func (s *Solver) applyExplainUIP() {
	for s.cN != 1 { // !isUIP
		s.applyExplain(s.cL)
	}

	// buildC
	c := make([]cnf.Literal, 0, len(s.cP)+1)
	for l, _ := range s.cP {
		c = append(c, l)
	}
	c = append(c, s.cL.Negate())
	s.c = c
}

func (s *Solver) isUIP() bool {
	return s.cN == 1
}

//-------------------------------------------------------------------
// Backjumping
//-------------------------------------------------------------------

func (s *Solver) applyBackjump() {
	level := 0
	if len(s.cP) > 0 {
		for l, _ := range s.cP {
			if v := s.level(l.Pack().Var()); v > level {
				level = v
			}
		}
	}

	if s.Trace {
		s.Tracer.Printf(
			"[TRACE] sat: backjump. C = %#v; l = %d; level = %d",
			s.c, s.cL, level)
	}

	s.trimToDecisionLevel(level)

	lit := s.cL.Negate()
	s.assertLiteral(lit.Pack())
	s.reasonMap[lit] = s.c

	if s.Trace {
		s.Tracer.Printf("[TRACE] sat: backjump. M = %s", s.trail)
	}
}

//-------------------------------------------------------------------
// Private types
//-------------------------------------------------------------------

type satResult byte

const (
	satResultUndef satResult = iota
	satResultUnsat
	satResultSat
)

type varinfo struct {
	level int
}

// Tribool is a tri-state boolean with undefined as the 3rd state.
type Tribool uint8

const (
	True  Tribool = 0
	False         = 1
	Undef         = 2
)

func BoolToTri(b bool) Tribool {
	if b {
		return True
	}

	return False
}
<|endoftext|>


<commit_before>// Copyright 2017 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package k8sTest

import (
	"fmt"
	"sync"
	"time"

	. "github.com/cilium/cilium/test/ginkgo-ext"
	"github.com/cilium/cilium/test/helpers"

	. "github.com/onsi/gomega"
	"github.com/sirupsen/logrus"
)

var _ = Describe("K8sValidatedTunnelTest", func() {

	var kubectl *helpers.Kubectl
	var demoDSPath string
	var once sync.Once
	var logger *logrus.Entry

	initialize := func() {
		logger = log.WithFields(logrus.Fields{"testName": "K8sTunnelTest"})
		logger.Info("Starting")

		kubectl = helpers.CreateKubectl(helpers.K8s1VMName(), logger)
		demoDSPath = kubectl.ManifestGet("demo_ds.yaml")
		kubectl.Exec("kubectl -n kube-system delete ds cilium")
		// Expect(res.Correct()).Should(BeTrue())

		waitToDeleteCilium(kubectl, logger)
	}

	BeforeEach(func() {
		once.Do(initialize)
		kubectl.NodeCleanMetadata()
		kubectl.Apply(demoDSPath)
	}, 600)

	AfterFailed(func() {
		kubectl.CiliumReport(helpers.KubeSystemNamespace, []string{
			"cilium bpf tunnel list",
			"cilium endpoint list"})
	})

	JustAfterEach(func() {
		kubectl.ValidateNoErrorsOnLogs(CurrentGinkgoTestDescription().Duration)
	})

	AfterEach(func() {
		kubectl.Delete(demoDSPath)
		err := kubectl.WaitCleanAllTerminatingPods()
		Expect(err).To(BeNil(), "Terminating containers are not deleted after timeout")
	})

	It("Check VXLAN mode", func() {
		path := kubectl.ManifestGet("cilium_ds.yaml")
		kubectl.Apply(path)
		_, err := kubectl.WaitforPods(helpers.KubeSystemNamespace, "-l k8s-app=cilium", 500)
		Expect(err).Should(BeNil())

		ciliumPod, err := kubectl.GetCiliumPodOnNode(helpers.KubeSystemNamespace, helpers.K8s1)
		Expect(err).Should(BeNil())

		_, err = kubectl.CiliumNodesWait()
		Expect(err).Should(BeNil())
		//Make sure that we delete the ds in case of fail
		defer kubectl.Delete(path)

		By("Checking that BPF tunnels are in place")
		status := kubectl.CiliumExec(ciliumPod, "cilium bpf tunnel list | wc -l")
		if tunnels, _ := status.IntOutput(); tunnels != 4 {
			cmds := []string{
				"cilium bpf tunnel list",
			}
			kubectl.CiliumReport(helpers.KubeSystemNamespace, cmds)
		}

		status.ExpectSuccess()
		Expect(status.IntOutput()).Should(Equal(5))

		By("Checking that BPF tunnels are working correctly")
		tunnStatus := isNodeNetworkingWorking(kubectl, "zgroup=testDS")
		Expect(tunnStatus).Should(BeTrue())
		kubectl.Delete(path)
		waitToDeleteCilium(kubectl, logger)
	}, 600)

	Context("Geneve", func() {

		var (
			geneveDSPath string
		)

		BeforeEach(func() {
			geneveDSPath = kubectl.ManifestGet("cilium_ds_geneve.yaml")
		})

		AfterEach(func() {
			// Do not assert on success in AfterEach intentionally to avoid
			// incomplete teardown.
			_ = kubectl.Delete(geneveDSPath)
			waitToDeleteCilium(kubectl, logger)
		})

		It("Check Geneve mode", func() {
			res := kubectl.Apply(geneveDSPath)
			res.ExpectSuccess("unable to apply %s: %s", geneveDSPath, res.CombineOutput())
			_, err := kubectl.WaitforPods(helpers.KubeSystemNamespace, "-l k8s-app=cilium", 500)
			Expect(err).Should(BeNil())

			ciliumPod, err := kubectl.GetCiliumPodOnNode(helpers.KubeSystemNamespace, helpers.K8s1)
			Expect(err).Should(BeNil())

			_, err = kubectl.CiliumNodesWait()
			Expect(err).Should(BeNil())

			//Check that cilium detects a
			By("Checking that BPF tunnels are in place")
			status := kubectl.CiliumExec(ciliumPod, "cilium bpf tunnel list | wc -l")
			status.ExpectSuccess()
			if tunnels, _ := status.IntOutput(); tunnels != 4 {
				cmds := []string{
					"cilium bpf tunnel list",
				}
				kubectl.CiliumReport(helpers.KubeSystemNamespace, cmds)
			}
			Expect(status.IntOutput()).Should(Equal(5))

			By("Checking that BPF tunnels are working correctly")
			tunnStatus := isNodeNetworkingWorking(kubectl, "zgroup=testDS")
			Expect(tunnStatus).Should(BeTrue())
			//FIXME: Maybe added here a cilium bpf tunnel status?
		}, 600)

	})

})

func isNodeNetworkingWorking(kubectl *helpers.Kubectl, filter string) bool {
	waitReady, _ := kubectl.WaitforPods(helpers.DefaultNamespace, fmt.Sprintf("-l %s", filter), 3000)
	Expect(waitReady).Should(BeTrue())
	pods, err := kubectl.GetPodNames(helpers.DefaultNamespace, filter)
	Expect(err).Should(BeNil())
	podIP, err := kubectl.Get(
		helpers.DefaultNamespace,
		fmt.Sprintf("pod %s -o json", pods[1])).Filter("{.status.podIP}")
	Expect(err).Should(BeNil())
	res := kubectl.ExecPodCmd(helpers.DefaultNamespace, pods[0], helpers.Ping(podIP.String()))
	return res.WasSuccessful()
}

func waitToDeleteCilium(kubectl *helpers.Kubectl, logger *logrus.Entry) {
	status := 1
	for status > 0 {
		pods, err := kubectl.GetCiliumPods(helpers.KubeSystemNamespace)
		status := len(pods)
		logger.Infof("Cilium pods terminating '%d' err='%v' pods='%v'", status, err, pods)
		if status == 0 {
			return
		}
		time.Sleep(1 * time.Second)
	}
}
<commit_msg>test/k8sT: wrap vxlan test in `Context`<commit_after>// Copyright 2017 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package k8sTest

import (
	"fmt"
	"sync"
	"time"

	. "github.com/cilium/cilium/test/ginkgo-ext"
	"github.com/cilium/cilium/test/helpers"

	. "github.com/onsi/gomega"
	"github.com/sirupsen/logrus"
)

var _ = Describe("K8sValidatedTunnelTest", func() {

	var kubectl *helpers.Kubectl
	var demoDSPath string
	var once sync.Once
	var logger *logrus.Entry

	initialize := func() {
		logger = log.WithFields(logrus.Fields{"testName": "K8sTunnelTest"})
		logger.Info("Starting")

		kubectl = helpers.CreateKubectl(helpers.K8s1VMName(), logger)
		demoDSPath = kubectl.ManifestGet("demo_ds.yaml")
		kubectl.Exec("kubectl -n kube-system delete ds cilium")
		// Expect(res.Correct()).Should(BeTrue())

		waitToDeleteCilium(kubectl, logger)
	}

	BeforeEach(func() {
		once.Do(initialize)
		kubectl.NodeCleanMetadata()
		kubectl.Apply(demoDSPath)
	}, 600)

	AfterFailed(func() {
		kubectl.CiliumReport(helpers.KubeSystemNamespace, []string{
			"cilium bpf tunnel list",
			"cilium endpoint list"})
	})

	JustAfterEach(func() {
		kubectl.ValidateNoErrorsOnLogs(CurrentGinkgoTestDescription().Duration)
	})

	AfterEach(func() {
		kubectl.Delete(demoDSPath)
		err := kubectl.WaitCleanAllTerminatingPods()
		Expect(err).To(BeNil(), "Terminating containers are not deleted after timeout")
	})

	Context("VXLan", func() {
		It("Check VXLAN mode", func() {
			path := kubectl.ManifestGet("cilium_ds.yaml")
			kubectl.Apply(path)
			_, err := kubectl.WaitforPods(helpers.KubeSystemNamespace, "-l k8s-app=cilium", 500)
			Expect(err).Should(BeNil())

			ciliumPod, err := kubectl.GetCiliumPodOnNode(helpers.KubeSystemNamespace, helpers.K8s1)
			Expect(err).Should(BeNil())

			_, err = kubectl.CiliumNodesWait()
			Expect(err).Should(BeNil())
			//Make sure that we delete the ds in case of fail
			defer kubectl.Delete(path)

			By("Checking that BPF tunnels are in place")
			status := kubectl.CiliumExec(ciliumPod, "cilium bpf tunnel list | wc -l")
			if tunnels, _ := status.IntOutput(); tunnels != 4 {
				cmds := []string{
					"cilium bpf tunnel list",
				}
				kubectl.CiliumReport(helpers.KubeSystemNamespace, cmds)
			}

			status.ExpectSuccess()
			Expect(status.IntOutput()).Should(Equal(5))

			By("Checking that BPF tunnels are working correctly")
			tunnStatus := isNodeNetworkingWorking(kubectl, "zgroup=testDS")
			Expect(tunnStatus).Should(BeTrue())
			kubectl.Delete(path)
			waitToDeleteCilium(kubectl, logger)
		}, 600)
	})

	Context("Geneve", func() {

		var (
			geneveDSPath string
		)

		BeforeEach(func() {
			geneveDSPath = kubectl.ManifestGet("cilium_ds_geneve.yaml")
		})

		AfterEach(func() {
			// Do not assert on success in AfterEach intentionally to avoid
			// incomplete teardown.
			_ = kubectl.Delete(geneveDSPath)
			waitToDeleteCilium(kubectl, logger)
		})

		It("Check Geneve mode", func() {
			res := kubectl.Apply(geneveDSPath)
			res.ExpectSuccess("unable to apply %s: %s", geneveDSPath, res.CombineOutput())
			_, err := kubectl.WaitforPods(helpers.KubeSystemNamespace, "-l k8s-app=cilium", 500)
			Expect(err).Should(BeNil())

			ciliumPod, err := kubectl.GetCiliumPodOnNode(helpers.KubeSystemNamespace, helpers.K8s1)
			Expect(err).Should(BeNil())

			_, err = kubectl.CiliumNodesWait()
			Expect(err).Should(BeNil())

			//Check that cilium detects a
			By("Checking that BPF tunnels are in place")
			status := kubectl.CiliumExec(ciliumPod, "cilium bpf tunnel list | wc -l")
			status.ExpectSuccess()
			if tunnels, _ := status.IntOutput(); tunnels != 4 {
				cmds := []string{
					"cilium bpf tunnel list",
				}
				kubectl.CiliumReport(helpers.KubeSystemNamespace, cmds)
			}
			Expect(status.IntOutput()).Should(Equal(5))

			By("Checking that BPF tunnels are working correctly")
			tunnStatus := isNodeNetworkingWorking(kubectl, "zgroup=testDS")
			Expect(tunnStatus).Should(BeTrue())
			//FIXME: Maybe added here a cilium bpf tunnel status?
		}, 600)

	})

})

func isNodeNetworkingWorking(kubectl *helpers.Kubectl, filter string) bool {
	waitReady, _ := kubectl.WaitforPods(helpers.DefaultNamespace, fmt.Sprintf("-l %s", filter), 3000)
	Expect(waitReady).Should(BeTrue())
	pods, err := kubectl.GetPodNames(helpers.DefaultNamespace, filter)
	Expect(err).Should(BeNil())
	podIP, err := kubectl.Get(
		helpers.DefaultNamespace,
		fmt.Sprintf("pod %s -o json", pods[1])).Filter("{.status.podIP}")
	Expect(err).Should(BeNil())
	res := kubectl.ExecPodCmd(helpers.DefaultNamespace, pods[0], helpers.Ping(podIP.String()))
	return res.WasSuccessful()
}

func waitToDeleteCilium(kubectl *helpers.Kubectl, logger *logrus.Entry) {
	status := 1
	for status > 0 {
		pods, err := kubectl.GetCiliumPods(helpers.KubeSystemNamespace)
		status := len(pods)
		logger.Infof("Cilium pods terminating '%d' err='%v' pods='%v'", status, err, pods)
		if status == 0 {
			return
		}
		time.Sleep(1 * time.Second)
	}
}
<|endoftext|>


<commit_before>// Copyright 2019 Istio Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package spanner

import (
	"context"
	"fmt"
	"time"

	"cloud.google.com/go/spanner"
)

type getActivityResults struct {
	LastIssueEvent time.Time
	Actor          string
}

type getCommentResults struct {
	LastIssueCommentEvent time.Time
	Actor                 string
}

func (s store) GetLatestIssueMemberActivity(context context.Context, orgLogin string, repoName string, issueNumber int) (time.Time, error) {
	iter := s.client.Single().Query(context, spanner.Statement{SQL: fmt.Sprintf(
		`SELECT CreatedAt as LastIssueEvent, Actor FROM
		(SELECT CreatedAt, Actor FROM IssueEvents WHERE IssueNumber = %d AND OrgLogin = "%s" AND RepoName = "%s"
			UNION ALL
			SELECT CreatedAt, Actor FROM IssueCommentEvents WHERE IssueNumber = %d AND OrgLogin="%s" AND RepoName = "%s")
			LEFT JOIN Members ON Actor = UserLogin
			WHERE OrgLogin is not null
			ORDER BY LastIssueEvent DESC
			LIMIT 1;`, issueNumber, orgLogin, repoName, issueNumber, orgLogin, repoName)})

	var result getActivityResults
	err := iter.Do(func(row *spanner.Row) error {
		return rowToStruct(row, &result)
	})

	return result.LastIssueEvent, err
}

func (s store) GetLatestIssueMemberComment(context context.Context, orgLogin string, repoName string, issueNumber int) (time.Time, error) {
	iter := s.client.Single().Query(context, spanner.Statement{SQL: fmt.Sprintf(
		`SELECT CreatedAt as LastIssueCommentEvent, Actor FROM
		(SELECT CreatedAt, Actor FROM IssueCommentEvents WHERE IssueNumber = %d AND OrgLogin = "%s" AND RepoName = "%s")
		LEFT JOIN Members ON Actor = UserLogin
		WHERE OrgLogin is not null
		ORDER BY LastIssueCommentEvent DESC
		LIMIT 1;`, issueNumber, orgLogin, repoName)})

	var result getCommentResults
	var result2 getCommentResults

	err := iter.Do(func(row *spanner.Row) error {
		return rowToStruct(row, &result)
	})

	// Include reopening issues as a form of member comment activity.
	if err == nil {
		iter = s.client.Single().Query(context, spanner.Statement{SQL: fmt.Sprintf(
			`SELECT CreatedAt as LastIssueCommentEvent, Actor FROM
			(SELECT CreatedAt, Actor FROM IssueEvents WHERE IssueNumber = %d AND OrgLogin = "%s" AND RepoName = "%s" AND Action = "reopened")
			LEFT JOIN Members ON Actor = UserLogin
			WHERE OrgLogin is not null
			ORDER BY LastIssueCommentEvent DESC
			LIMIT 1;`, issueNumber, orgLogin, repoName)})

		err = iter.Do(func(row *spanner.Row) error {
			return rowToStruct(row, &result2)
		})

		if err == nil {
			if result2.LastIssueCommentEvent.After(result.LastIssueCommentEvent) {
				result = result2
			}
		}
	}

	if err == nil {
		iter = s.client.Single().Query(context, spanner.Statement{SQL: fmt.Sprintf(
			`SELECT CreatedAt as LastIssueCommentEvent, Actor FROM
			(SELECT CreatedAt, Actor FROM PullRequestReviewEvents WHERE PullRequestNumber = %d AND OrgLogin = "%s" AND RepoName = "%s")
			LEFT JOIN Members ON Actor = UserLogin
			WHERE OrgLogin is not null
			ORDER BY LastIssueCommentEvent DESC
			LIMIT 1;`, issueNumber, orgLogin, repoName)})

		err = iter.Do(func(row *spanner.Row) error {
			return rowToStruct(row, &result2)
		})

		if err == nil {
			if result2.LastIssueCommentEvent.After(result.LastIssueCommentEvent) {
				result = result2
			}
		}
	}

	if err == nil {
		iter = s.client.Single().Query(context, spanner.Statement{SQL: fmt.Sprintf(
			`SELECT CreatedAt as LastIssueCommentEvent, Actor FROM
			(SELECT CreatedAt, Actor FROM PullRequestReviewCommentEvents WHERE PullRequestNumber = %d AND OrgLogin = "%s" AND RepoName = "%s")
			LEFT JOIN Members ON Actor = UserLogin
			WHERE OrgLogin is not null
			ORDER BY LastIssueCommentEvent DESC
			LIMIT 1;`, issueNumber, orgLogin, repoName)})

		err = iter.Do(func(row *spanner.Row) error {
			return rowToStruct(row, &result2)
		})

		if err == nil {
			if result2.LastIssueCommentEvent.After(result.LastIssueCommentEvent) {
				result = result2
			}
		}
	}

	return result.LastIssueCommentEvent, err
}
<commit_msg>Include reopening PR as a form of member comment activity (#163)<commit_after>// Copyright 2019 Istio Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package spanner

import (
	"context"
	"fmt"
	"time"

	"cloud.google.com/go/spanner"
)

type getActivityResults struct {
	LastIssueEvent time.Time
	Actor          string
}

type getCommentResults struct {
	LastIssueCommentEvent time.Time
	Actor                 string
}

func (s store) GetLatestIssueMemberActivity(context context.Context, orgLogin string, repoName string, issueNumber int) (time.Time, error) {
	iter := s.client.Single().Query(context, spanner.Statement{SQL: fmt.Sprintf(
		`SELECT CreatedAt as LastIssueEvent, Actor FROM
		(SELECT CreatedAt, Actor FROM IssueEvents WHERE IssueNumber = %d AND OrgLogin = "%s" AND RepoName = "%s"
			UNION ALL
			SELECT CreatedAt, Actor FROM IssueCommentEvents WHERE IssueNumber = %d AND OrgLogin="%s" AND RepoName = "%s")
			LEFT JOIN Members ON Actor = UserLogin
			WHERE OrgLogin is not null
			ORDER BY LastIssueEvent DESC
			LIMIT 1;`, issueNumber, orgLogin, repoName, issueNumber, orgLogin, repoName)})

	var result getActivityResults
	err := iter.Do(func(row *spanner.Row) error {
		return rowToStruct(row, &result)
	})

	return result.LastIssueEvent, err
}

func (s store) GetLatestIssueMemberComment(context context.Context, orgLogin string, repoName string, issueNumber int) (time.Time, error) {
	iter := s.client.Single().Query(context, spanner.Statement{SQL: fmt.Sprintf(
		`SELECT CreatedAt as LastIssueCommentEvent, Actor FROM
		(SELECT CreatedAt, Actor FROM IssueCommentEvents WHERE IssueNumber = %d AND OrgLogin = "%s" AND RepoName = "%s")
		LEFT JOIN Members ON Actor = UserLogin
		WHERE OrgLogin is not null
		ORDER BY LastIssueCommentEvent DESC
		LIMIT 1;`, issueNumber, orgLogin, repoName)})

	var result getCommentResults
	var result2 getCommentResults

	err := iter.Do(func(row *spanner.Row) error {
		return rowToStruct(row, &result)
	})

	// Include reopening issues as a form of member comment activity.
	if err == nil {
		iter = s.client.Single().Query(context, spanner.Statement{SQL: fmt.Sprintf(
			`SELECT CreatedAt as LastIssueCommentEvent, Actor FROM
			(SELECT CreatedAt, Actor FROM IssueEvents WHERE IssueNumber = %d AND OrgLogin = "%s" AND RepoName = "%s" AND Action = "reopened")
			LEFT JOIN Members ON Actor = UserLogin
			WHERE OrgLogin is not null
			ORDER BY LastIssueCommentEvent DESC
			LIMIT 1;`, issueNumber, orgLogin, repoName)})

		err = iter.Do(func(row *spanner.Row) error {
			return rowToStruct(row, &result2)
		})

		if err == nil {
			if result2.LastIssueCommentEvent.After(result.LastIssueCommentEvent) {
				result = result2
			}
		}
	}

	// Include reopening pull requests as a form of member comment activity.
	if err == nil {
		iter = s.client.Single().Query(context, spanner.Statement{SQL: fmt.Sprintf(
			`SELECT CreatedAt as LastIssueCommentEvent, Actor FROM
			(SELECT CreatedAt, Actor FROM PullRequestEvents WHERE PullRequestNumber = %d AND OrgLogin = "%s" AND RepoName = "%s" AND Action = "reopened")
			LEFT JOIN Members ON Actor = UserLogin
			WHERE OrgLogin is not null
			ORDER BY LastIssueCommentEvent DESC
			LIMIT 1;`, issueNumber, orgLogin, repoName)})

		err = iter.Do(func(row *spanner.Row) error {
			return rowToStruct(row, &result2)
		})

		if err == nil {
			if result2.LastIssueCommentEvent.After(result.LastIssueCommentEvent) {
				result = result2
			}
		}
	}

	if err == nil {
		iter = s.client.Single().Query(context, spanner.Statement{SQL: fmt.Sprintf(
			`SELECT CreatedAt as LastIssueCommentEvent, Actor FROM
			(SELECT CreatedAt, Actor FROM PullRequestReviewEvents WHERE PullRequestNumber = %d AND OrgLogin = "%s" AND RepoName = "%s")
			LEFT JOIN Members ON Actor = UserLogin
			WHERE OrgLogin is not null
			ORDER BY LastIssueCommentEvent DESC
			LIMIT 1;`, issueNumber, orgLogin, repoName)})

		err = iter.Do(func(row *spanner.Row) error {
			return rowToStruct(row, &result2)
		})

		if err == nil {
			if result2.LastIssueCommentEvent.After(result.LastIssueCommentEvent) {
				result = result2
			}
		}
	}

	if err == nil {
		iter = s.client.Single().Query(context, spanner.Statement{SQL: fmt.Sprintf(
			`SELECT CreatedAt as LastIssueCommentEvent, Actor FROM
			(SELECT CreatedAt, Actor FROM PullRequestReviewCommentEvents WHERE PullRequestNumber = %d AND OrgLogin = "%s" AND RepoName = "%s")
			LEFT JOIN Members ON Actor = UserLogin
			WHERE OrgLogin is not null
			ORDER BY LastIssueCommentEvent DESC
			LIMIT 1;`, issueNumber, orgLogin, repoName)})

		err = iter.Do(func(row *spanner.Row) error {
			return rowToStruct(row, &result2)
		})

		if err == nil {
			if result2.LastIssueCommentEvent.After(result.LastIssueCommentEvent) {
				result = result2
			}
		}
	}

	return result.LastIssueCommentEvent, err
}
<|endoftext|>


<commit_before>package main

import (
	"sync"
	"time"

	"github.com/grafana/grafana/pkg/log"
)

type AggMetrics struct {
	store Store
	sync.RWMutex
	Metrics        map[string]*AggMetric
	chunkSpan      uint32
	numChunks      uint32
	aggSettings    []aggSetting // for now we apply the same settings to all AggMetrics. later we may want to have different settings.
	chunkMaxStale  uint32
	metricMaxStale uint32
	ttl            uint32
	gcInterval     time.Duration
}

var totalPoints chan int

func init() {
	// measurements can lag a bit, that's ok
	totalPoints = make(chan int, 1000)
}

func NewAggMetrics(store Store, chunkSpan, numChunks, chunkMaxStale, metricMaxStale uint32, ttl uint32, gcInterval time.Duration, aggSettings []aggSetting) *AggMetrics {
	ms := AggMetrics{
		store:          store,
		Metrics:        make(map[string]*AggMetric),
		chunkSpan:      chunkSpan,
		numChunks:      numChunks,
		aggSettings:    aggSettings,
		chunkMaxStale:  chunkMaxStale,
		metricMaxStale: metricMaxStale,
		ttl:            ttl,
		gcInterval:     gcInterval,
	}

	go ms.stats()
	go ms.GC()
	return &ms
}

// periodically scan chunks and close any that have not received data in a while
func (ms *AggMetrics) GC() {
	for {
		unix := time.Duration(time.Now().UnixNano())
		diff := ms.gcInterval - (unix % ms.gcInterval)
		time.Sleep(diff + time.Minute)
		if !clusterStatus.IsPrimary() {
			continue
		}
		log.Info("checking for stale chunks that need persisting.")
		now := uint32(time.Now().Unix())
		chunkMinTs := now - (now % ms.chunkSpan) - uint32(ms.chunkMaxStale)
		metricMinTs := now - (now % ms.chunkSpan) - uint32(ms.metricMaxStale)

		// as this is the only goroutine that can delete from ms.Metrics
		// we only need to lock long enough to get the list of actives metrics.
		// it doesnt matter if new metrics are added while we iterate this list.
		ms.RLock()
		keys := make([]string, 0, len(ms.Metrics))
		for k := range ms.Metrics {
			keys = append(keys, k)
		}
		ms.RUnlock()
		for _, key := range keys {
			gcMetric.Inc(1)
			ms.RLock()
			a := ms.Metrics[key]
			ms.RUnlock()
			if stale := a.GC(chunkMinTs, metricMinTs); stale {
				log.Info("metric %s is stale. Purging data from memory.", key)
				ms.Lock()
				delete(ms.Metrics, key)
				ms.Unlock()
			}
		}

	}
}

func (ms *AggMetrics) stats() {
	for range time.Tick(time.Duration(1) * time.Second) {
		ms.RLock()
		metricsActive.Value(int64(len(ms.Metrics)))
		ms.RUnlock()
	}
}

func (ms *AggMetrics) Get(key string) (Metric, bool) {
	ms.RLock()
	m, ok := ms.Metrics[key]
	ms.RUnlock()
	return m, ok
}

func (ms *AggMetrics) GetOrCreate(key string) Metric {
	ms.Lock()
	m, ok := ms.Metrics[key]
	if !ok {
		m = NewAggMetric(ms.store, key, ms.chunkSpan, ms.numChunks, ms.ttl, ms.aggSettings...)
		ms.Metrics[key] = m
	}
	ms.Unlock()
	return m
}
<commit_msg>fix for benchmarks<commit_after>package main

import (
	"sync"
	"time"

	"github.com/grafana/grafana/pkg/log"
)

type AggMetrics struct {
	store Store
	sync.RWMutex
	Metrics        map[string]*AggMetric
	chunkSpan      uint32
	numChunks      uint32
	aggSettings    []aggSetting // for now we apply the same settings to all AggMetrics. later we may want to have different settings.
	chunkMaxStale  uint32
	metricMaxStale uint32
	ttl            uint32
	gcInterval     time.Duration
}

var totalPoints chan int

func init() {
	// measurements can lag a bit, that's ok
	totalPoints = make(chan int, 1000)
}

func NewAggMetrics(store Store, chunkSpan, numChunks, chunkMaxStale, metricMaxStale uint32, ttl uint32, gcInterval time.Duration, aggSettings []aggSetting) *AggMetrics {
	ms := AggMetrics{
		store:          store,
		Metrics:        make(map[string]*AggMetric),
		chunkSpan:      chunkSpan,
		numChunks:      numChunks,
		aggSettings:    aggSettings,
		chunkMaxStale:  chunkMaxStale,
		metricMaxStale: metricMaxStale,
		ttl:            ttl,
		gcInterval:     gcInterval,
	}

	go ms.stats()
	// gcInterval = 0 can be useful in tests
	if gcInterval > 0 {
		go ms.GC()
	}
	return &ms
}

// periodically scan chunks and close any that have not received data in a while
func (ms *AggMetrics) GC() {
	for {
		unix := time.Duration(time.Now().UnixNano())
		diff := ms.gcInterval - (unix % ms.gcInterval)
		time.Sleep(diff + time.Minute)
		if !clusterStatus.IsPrimary() {
			continue
		}
		log.Info("checking for stale chunks that need persisting.")
		now := uint32(time.Now().Unix())
		chunkMinTs := now - (now % ms.chunkSpan) - uint32(ms.chunkMaxStale)
		metricMinTs := now - (now % ms.chunkSpan) - uint32(ms.metricMaxStale)

		// as this is the only goroutine that can delete from ms.Metrics
		// we only need to lock long enough to get the list of actives metrics.
		// it doesnt matter if new metrics are added while we iterate this list.
		ms.RLock()
		keys := make([]string, 0, len(ms.Metrics))
		for k := range ms.Metrics {
			keys = append(keys, k)
		}
		ms.RUnlock()
		for _, key := range keys {
			gcMetric.Inc(1)
			ms.RLock()
			a := ms.Metrics[key]
			ms.RUnlock()
			if stale := a.GC(chunkMinTs, metricMinTs); stale {
				log.Info("metric %s is stale. Purging data from memory.", key)
				ms.Lock()
				delete(ms.Metrics, key)
				ms.Unlock()
			}
		}

	}
}

func (ms *AggMetrics) stats() {
	for range time.Tick(time.Duration(1) * time.Second) {
		ms.RLock()
		metricsActive.Value(int64(len(ms.Metrics)))
		ms.RUnlock()
	}
}

func (ms *AggMetrics) Get(key string) (Metric, bool) {
	ms.RLock()
	m, ok := ms.Metrics[key]
	ms.RUnlock()
	return m, ok
}

func (ms *AggMetrics) GetOrCreate(key string) Metric {
	ms.Lock()
	m, ok := ms.Metrics[key]
	if !ok {
		m = NewAggMetric(ms.store, key, ms.chunkSpan, ms.numChunks, ms.ttl, ms.aggSettings...)
		ms.Metrics[key] = m
	}
	ms.Unlock()
	return m
}
<|endoftext|>


<commit_before>// +build freebsd

package freebsd

import (
	"fmt"
	"os/exec"
	"regexp"
	"strconv"
	"strings"

	"github.com/mackerelio/mackerel-agent/logging"
	"github.com/mackerelio/mackerel-agent/metrics"
)

// MemoryGenerator XXX
type MemoryGenerator struct {
}

var memoryLogger = logging.GetLogger("metrics.memory")

// Generate generate metrics values
func (g *MemoryGenerator) Generate() (metrics.Values, error) {
	var errRet error
	outBytes, err := exec.Command("top", "-bn", "1").Output()
	if err != nil {
		logger.Warningf("'top -bn 1' command exited with a non-zero status: '%s'", err)
		errRet = err
	}
	ret := make(map[string]float64)

	out := string(outBytes)
	lines := strings.Split(out, "\n")

	for _, line := range lines {
		fields := strings.Fields(line)
		if len(fields) < 1 {
			continue
		}

		if fields[0] == "Mem:" {
			for i := 1; i < len(fields); i += 2 {
				v, err := getValue(fields[i])
				if err == nil {
					k := fields[i+1]
					if strings.HasSuffix(k, ",") {
						k = k[0 : len(k)-1]
					}
					switch k {
					case "Active":
						ret["memory.active"] = v
					case "Inact":
						ret["memory.inactive"] = v
					/*
						case "Wired":
							ret["memory.wired"] = v
					*/
					case "Cache":
						ret["memory.cached"] = v
					case "Buf":
						ret["memory.buffers"] = v
					case "Free":
						ret["memory.free"] = v
					}
				} else {
					errRet = err
				}
			}
		}
		if fields[0] == "Swap:" {
			if v, err := getValue(fields[1]); err == nil {
				ret["memory.swap_total"] = v
			} else {
				errRet = err
			}
			swapFreeIndex := 5
			if len(fields) == 5 {
				swapFreeIndex = 3
			}
			if v, err := getValue(fields[swapFreeIndex]); err == nil {
				ret["memory.swap_free"] = v
			} else {
				errRet = err
			}
		}
	}

	v, err := getTotalMem()
	if err != nil {
		return nil, err
	}
	ret["memory.total"] = v

	if errRet == nil {
		return metrics.Values(ret), nil
	}
	return nil, errRet
}

func getValue(strValue string) (float64, error) {
	parseRegexp := regexp.MustCompile(`^(\d+)(.?)$`)
	match := parseRegexp.FindStringSubmatch(strValue)
	var unit float64 = 1
	switch {
	case match[2] == "G":
		unit = 1024 * 1024 * 1024
	case match[2] == "M":
		unit = 1024 * 1024
	case match[2] == "K":
		unit = 1024
	}
	value, err := strconv.ParseFloat(strings.TrimSpace(match[1]), 64)
	if err != nil {
		return 0, fmt.Errorf("while parsing %q: %s", match[1], err)
	}
	return value * unit, nil
}

func getTotalMem() (float64, error) {
	cmd := exec.Command("sysctl", "-n", "hw.physmem")
	outputBytes, err := cmd.Output()
	if err != nil {
		return 0, fmt.Errorf("sysctl -n hw.physmem: %s", err)
	}

	output := string(outputBytes)

	memsizeInBytes, err := strconv.ParseFloat(strings.TrimSpace(output), 64)
	if err != nil {
		return 0, fmt.Errorf("while parsing %q: %s", output, err)
	}

	return memsizeInBytes, nil
}
<commit_msg>trivial fix<commit_after>// +build freebsd

package freebsd

import (
	"fmt"
	"os/exec"
	"regexp"
	"strconv"
	"strings"

	"github.com/mackerelio/mackerel-agent/logging"
	"github.com/mackerelio/mackerel-agent/metrics"
)

// MemoryGenerator XXX
type MemoryGenerator struct {
}

var memoryLogger = logging.GetLogger("metrics.memory")

// Generate generate metrics values
func (g *MemoryGenerator) Generate() (metrics.Values, error) {
	var errRet error
	outBytes, err := exec.Command("top", "-bn", "1").Output()
	if err != nil {
		memorylogger.Warningf("'top -bn 1' command exited with a non-zero status: '%s'", err)
		errRet = err
	}
	ret := make(map[string]float64)

	out := string(outBytes)
	lines := strings.Split(out, "\n")

	for _, line := range lines {
		fields := strings.Fields(line)
		if len(fields) < 1 {
			continue
		}

		if fields[0] == "Mem:" {
			for i := 1; i < len(fields); i += 2 {
				v, err := getValue(fields[i])
				if err == nil {
					k := fields[i+1]
					if strings.HasSuffix(k, ",") {
						k = k[0 : len(k)-1]
					}
					switch k {
					case "Active":
						ret["memory.active"] = v
					case "Inact":
						ret["memory.inactive"] = v
					/*
						case "Wired":
							ret["memory.wired"] = v
					*/
					case "Cache":
						ret["memory.cached"] = v
					case "Buf":
						ret["memory.buffers"] = v
					case "Free":
						ret["memory.free"] = v
					}
				} else {
					errRet = err
				}
			}
		}
		if fields[0] == "Swap:" {
			if v, err := getValue(fields[1]); err == nil {
				ret["memory.swap_total"] = v
			} else {
				errRet = err
			}
			swapFreeIndex := 5
			if len(fields) == 5 {
				swapFreeIndex = 3
			}
			if v, err := getValue(fields[swapFreeIndex]); err == nil {
				ret["memory.swap_free"] = v
			} else {
				errRet = err
			}
		}
	}

	v, err := getTotalMem()
	if err != nil {
		return nil, err
	}
	ret["memory.total"] = v

	if errRet == nil {
		return metrics.Values(ret), nil
	}
	return nil, errRet
}

func getValue(strValue string) (float64, error) {
	parseRegexp := regexp.MustCompile(`^(\d+)(.?)$`)
	match := parseRegexp.FindStringSubmatch(strValue)
	var unit float64 = 1
	switch {
	case match[2] == "G":
		unit = 1024 * 1024 * 1024
	case match[2] == "M":
		unit = 1024 * 1024
	case match[2] == "K":
		unit = 1024
	}
	value, err := strconv.ParseFloat(strings.TrimSpace(match[1]), 64)
	if err != nil {
		return 0, fmt.Errorf("while parsing %q: %s", match[1], err)
	}
	return value * unit, nil
}

func getTotalMem() (float64, error) {
	cmd := exec.Command("sysctl", "-n", "hw.physmem")
	outputBytes, err := cmd.Output()
	if err != nil {
		return 0, fmt.Errorf("sysctl -n hw.physmem: %s", err)
	}

	output := string(outputBytes)

	memsizeInBytes, err := strconv.ParseFloat(strings.TrimSpace(output), 64)
	if err != nil {
		return 0, fmt.Errorf("while parsing %q: %s", output, err)
	}

	return memsizeInBytes, nil
}
<|endoftext|>


<commit_before><commit_msg>Rename function container (#9456)<commit_after><|endoftext|>


<commit_before>package spotmc

import (
	"fmt"
	"github.com/pivotal-golang/archiver/compressor"
	"github.com/pivotal-golang/archiver/extractor"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/exec"
	"strconv"
	"strings"
)

var AWS_RETRY = 3
var JAR_PATH_DIR = ""
var JAR_PATH_PREFIX = "mcjar"
var DATA_PATH_DIR = ""
var DATA_PATH_PREFIX = "mcdata"

// Defaults
var DEFAULT_KILL_INSTANCE_MODE = "false"
var DEFAULT_SHUTDOWN_CMD = "/sbin/shutdown -h now"
var DEFAULT_REGION = "ap-northeast-1"
var DEFAULT_MAX_UPTIME = 43200
var DEFAULT_MAX_IDLE_TIME = 14400
var DEFAULT_IDLE_WATCH_PATH = "world/playerdata"

type SpotMC struct {
	JarFileURL       string
	EULAFileURL      string
	DataFileURL      string
	JavaPath         string
	JavaArgs         string
	serverPath       string
	dataDirPath      string
	ddnsURL          string
	killInstanceMode string
	maxIdleTime      int
	maxUptime        int
	shutdownCommand  string
	idleWatchPath    string
	autoScalingGroup string
}

func NewSpotMC() (*SpotMC, error) {
	// Check mandatory environment variables
	for _, k := range []string{
		"SPOTMC_SERVER_JAR_URL",
		"SPOTMC_SERVER_EULA_URL",
		"SPOTMC_DATA_URL",
		"SPOTMC_JAVA_PATH",
	} {
		s := os.Getenv(k)
		if s == "" {
			return nil, fmt.Errorf("set valid env vars")
		}
	}

	// Kill instance mode
	// "shutdown" or "false"
	killInstanceMode := DEFAULT_KILL_INSTANCE_MODE
	s := os.Getenv("SPOTMC_KILL_INSTANCE_MODE")
	if s != "" {
		killInstanceMode = s
	}
	shutdownCommand := DEFAULT_SHUTDOWN_CMD
	s = os.Getenv("SPOTMC_SHUTDOWN_CMD")
	if s != "" {
		shutdownCommand = s
	}

	// Max uptime and max idle time
	maxIdleTime := DEFAULT_MAX_IDLE_TIME
	s = os.Getenv("SPOTMC_MAX_IDLE_TIME")
	if s != "" {
		i, err := strconv.Atoi(s)
		if err == nil {
			maxIdleTime = i
		}
	}

	maxUptime := DEFAULT_MAX_UPTIME
	s = os.Getenv("SPOTMC_MAX_UPTIME")
	if s != "" {
		i, err := strconv.Atoi(s)
		if err == nil {
			maxUptime = i
		}
	}

	// Idle watch path
	idleWatchPath := DEFAULT_IDLE_WATCH_PATH
	s = os.Getenv("SPOTMC_IDLE_WATCH_PATH")
	if s != "" {
		idleWatchPath = s
	}

	// DDNS Update URL
	ddnsURL := os.Getenv("SPOTMC_DDNS_UPDATE_URL")

	// AutoScaling Group
	autoScalingGroup := os.Getenv("SPOTMC_AUTOSCALING_GROUP")

	smc := &SpotMC{
		JarFileURL:       os.Getenv("SPOTMC_SERVER_JAR_URL"),
		EULAFileURL:      os.Getenv("SPOTMC_SERVER_EULA_URL"),
		DataFileURL:      os.Getenv("SPOTMC_DATA_URL"),
		JavaPath:         os.Getenv("SPOTMC_JAVA_PATH"),
		JavaArgs:         os.Getenv("SPOTMC_JAVA_ARGS"),
		ddnsURL:          ddnsURL,
		killInstanceMode: killInstanceMode,
		maxIdleTime:      maxIdleTime,
		maxUptime:        maxUptime,
		shutdownCommand:  shutdownCommand,
		idleWatchPath:    idleWatchPath,
		autoScalingGroup: autoScalingGroup,
	}

	return smc, nil
}

func (smc *SpotMC) getJarFile() (serverPath string, err error) {
	dir, err := ioutil.TempDir(JAR_PATH_DIR, JAR_PATH_PREFIX)
	if err != nil {
		return "", err
	}
	serverPath = dir + "/server.jar"

	err = s3Get(smc.JarFileURL, serverPath)
	if err != nil {
		return "", err
	}

	smc.serverPath = serverPath
	return serverPath, nil
}

func (smc *SpotMC) getDataDir() (dataDirPath string, err error) {
	// Create the data dir
	dataDirPath, err = ioutil.TempDir(DATA_PATH_DIR, DATA_PATH_PREFIX)
	if err != nil {
		return "", err
	}

	// Get tgz file and uncompress it to the data dir
	tgzFile, err := ioutil.TempFile("", "")
	if err != nil {
		return "", err
	}
	tgzFile.Close()

	err = s3Get(smc.DataFileURL, tgzFile.Name())
	if err != nil {
		// Maybe the first time, it's ok.
		// Populate the data dir with user-provided eula.txt
		log.Printf("downloading EULA file: %s", smc.EULAFileURL)
		eulaFilePath := dataDirPath + "/eula.txt"
		err2 := s3Get(smc.EULAFileURL, eulaFilePath)
		if err2 != nil {
			return "", err2
		}
		log.Printf("EULA file path: %s", eulaFilePath)
	} else {
		tgz := extractor.NewTgz()
		err = tgz.Extract(tgzFile.Name(), dataDirPath)
	}

	smc.dataDirPath = dataDirPath
	return dataDirPath, nil
}

func (smc *SpotMC) putDataDir() error {
	// Create a tempfile
	tgzFile, err := ioutil.TempFile("", "")
	if err != nil {
		return err
	}
	tgzFile.Close()
	defer func() { os.Remove(tgzFile.Name()) }()

	// Compress dir to tgz
	tgz := compressor.NewTgz()
	err = tgz.Compress(smc.dataDirPath+"/", tgzFile.Name())
	if err != nil {
		return err
	}

	// Put tgz to S3
	err = s3Put(smc.DataFileURL, tgzFile.Name())
	return err
}

func (smc *SpotMC) updateDDNS() {
	if smc.ddnsURL != "" {
		log.Printf("issuing DDNS update query")
		resp, err := http.Get(smc.ddnsURL)
		if err != nil {
			log.Printf("DDNS update query failed: %s", err)
		} else {
			log.Printf("DDNS update query result: %s", resp.Status)
		}
	}
}

func (smc *SpotMC) StartServer() (exec.Cmd, error) {
	args := []string{smc.JavaPath}
	if smc.JavaArgs != "" {
		extraArgs := strings.Split(smc.JavaArgs, " ")
		args = append(args, extraArgs...)
	}
	args = append(args, "-jar", smc.serverPath, "nogui")
	log.Printf("Command to execute:%s args:%s length:%d", args[0], args, len(args))

	cmd := exec.Cmd{
		Path: args[0],
		Args: args,
		Dir:  smc.dataDirPath,
		//Stdin:  os.Stdin,
		Stdout: os.Stdout,
		Stderr: os.Stderr,
	}
	err := cmd.Start()
	return cmd, err
}

func (smc *SpotMC) KillInstance() error {
	log.Printf("KillInstance invoked")
	log.Printf("killInstanceMode: %s", smc.killInstanceMode)

	if smc.killInstanceMode == "false" {
		// False mode. Don't do anything
		return nil
	}

	if smc.killInstanceMode == "shutdown" {
		cmdArgs := strings.Split(smc.shutdownCommand, " ")
		cmd := exec.Command(cmdArgs[0], cmdArgs[1:]...)
		err := cmd.Wait()
		return err
	}

	return nil
}

func (smc *SpotMC) ShutdownCluster() error {
	log.Printf("ShutDownCluster invoked")
	log.Printf("autoScalingGroup: %s", smc.autoScalingGroup)
	var err error
	if smc.autoScalingGroup != "" {
		log.Printf("setting cluster capacity to 0")
		for i := 0; i < AWS_RETRY; i++ {
			log.Printf("failed to set cluster capacity, retrying...")
			err = setDesiredCapacity(smc.autoScalingGroup, 0)
			if err == nil {
				break
			}
		}
		log.Fatal("setDesiredCapacity Failed!")
	}
	return err
}
<commit_msg>Run the kill instance command properly<commit_after>package spotmc

import (
	"fmt"
	"github.com/pivotal-golang/archiver/compressor"
	"github.com/pivotal-golang/archiver/extractor"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/exec"
	"strconv"
	"strings"
)

var AWS_RETRY = 3
var JAR_PATH_DIR = ""
var JAR_PATH_PREFIX = "mcjar"
var DATA_PATH_DIR = ""
var DATA_PATH_PREFIX = "mcdata"

// Defaults
var DEFAULT_KILL_INSTANCE_MODE = "false"
var DEFAULT_SHUTDOWN_CMD = "/sbin/shutdown -h now"
var DEFAULT_REGION = "ap-northeast-1"
var DEFAULT_MAX_UPTIME = 43200
var DEFAULT_MAX_IDLE_TIME = 14400
var DEFAULT_IDLE_WATCH_PATH = "world/playerdata"

type SpotMC struct {
	JarFileURL       string
	EULAFileURL      string
	DataFileURL      string
	JavaPath         string
	JavaArgs         string
	serverPath       string
	dataDirPath      string
	ddnsURL          string
	killInstanceMode string
	maxIdleTime      int
	maxUptime        int
	shutdownCommand  string
	idleWatchPath    string
	autoScalingGroup string
}

func NewSpotMC() (*SpotMC, error) {
	// Check mandatory environment variables
	for _, k := range []string{
		"SPOTMC_SERVER_JAR_URL",
		"SPOTMC_SERVER_EULA_URL",
		"SPOTMC_DATA_URL",
		"SPOTMC_JAVA_PATH",
	} {
		s := os.Getenv(k)
		if s == "" {
			return nil, fmt.Errorf("set valid env vars")
		}
	}

	// Kill instance mode
	// "shutdown" or "false"
	killInstanceMode := DEFAULT_KILL_INSTANCE_MODE
	s := os.Getenv("SPOTMC_KILL_INSTANCE_MODE")
	if s != "" {
		killInstanceMode = s
	}
	shutdownCommand := DEFAULT_SHUTDOWN_CMD
	s = os.Getenv("SPOTMC_SHUTDOWN_CMD")
	if s != "" {
		shutdownCommand = s
	}

	// Max uptime and max idle time
	maxIdleTime := DEFAULT_MAX_IDLE_TIME
	s = os.Getenv("SPOTMC_MAX_IDLE_TIME")
	if s != "" {
		i, err := strconv.Atoi(s)
		if err == nil {
			maxIdleTime = i
		}
	}

	maxUptime := DEFAULT_MAX_UPTIME
	s = os.Getenv("SPOTMC_MAX_UPTIME")
	if s != "" {
		i, err := strconv.Atoi(s)
		if err == nil {
			maxUptime = i
		}
	}

	// Idle watch path
	idleWatchPath := DEFAULT_IDLE_WATCH_PATH
	s = os.Getenv("SPOTMC_IDLE_WATCH_PATH")
	if s != "" {
		idleWatchPath = s
	}

	// DDNS Update URL
	ddnsURL := os.Getenv("SPOTMC_DDNS_UPDATE_URL")

	// AutoScaling Group
	autoScalingGroup := os.Getenv("SPOTMC_AUTOSCALING_GROUP")

	smc := &SpotMC{
		JarFileURL:       os.Getenv("SPOTMC_SERVER_JAR_URL"),
		EULAFileURL:      os.Getenv("SPOTMC_SERVER_EULA_URL"),
		DataFileURL:      os.Getenv("SPOTMC_DATA_URL"),
		JavaPath:         os.Getenv("SPOTMC_JAVA_PATH"),
		JavaArgs:         os.Getenv("SPOTMC_JAVA_ARGS"),
		ddnsURL:          ddnsURL,
		killInstanceMode: killInstanceMode,
		maxIdleTime:      maxIdleTime,
		maxUptime:        maxUptime,
		shutdownCommand:  shutdownCommand,
		idleWatchPath:    idleWatchPath,
		autoScalingGroup: autoScalingGroup,
	}

	return smc, nil
}

func (smc *SpotMC) getJarFile() (serverPath string, err error) {
	dir, err := ioutil.TempDir(JAR_PATH_DIR, JAR_PATH_PREFIX)
	if err != nil {
		return "", err
	}
	serverPath = dir + "/server.jar"

	err = s3Get(smc.JarFileURL, serverPath)
	if err != nil {
		return "", err
	}

	smc.serverPath = serverPath
	return serverPath, nil
}

func (smc *SpotMC) getDataDir() (dataDirPath string, err error) {
	// Create the data dir
	dataDirPath, err = ioutil.TempDir(DATA_PATH_DIR, DATA_PATH_PREFIX)
	if err != nil {
		return "", err
	}

	// Get tgz file and uncompress it to the data dir
	tgzFile, err := ioutil.TempFile("", "")
	if err != nil {
		return "", err
	}
	tgzFile.Close()

	err = s3Get(smc.DataFileURL, tgzFile.Name())
	if err != nil {
		// Maybe the first time, it's ok.
		// Populate the data dir with user-provided eula.txt
		log.Printf("downloading EULA file: %s", smc.EULAFileURL)
		eulaFilePath := dataDirPath + "/eula.txt"
		err2 := s3Get(smc.EULAFileURL, eulaFilePath)
		if err2 != nil {
			return "", err2
		}
		log.Printf("EULA file path: %s", eulaFilePath)
	} else {
		tgz := extractor.NewTgz()
		err = tgz.Extract(tgzFile.Name(), dataDirPath)
	}

	smc.dataDirPath = dataDirPath
	return dataDirPath, nil
}

func (smc *SpotMC) putDataDir() error {
	// Create a tempfile
	tgzFile, err := ioutil.TempFile("", "")
	if err != nil {
		return err
	}
	tgzFile.Close()
	defer func() { os.Remove(tgzFile.Name()) }()

	// Compress dir to tgz
	tgz := compressor.NewTgz()
	err = tgz.Compress(smc.dataDirPath+"/", tgzFile.Name())
	if err != nil {
		return err
	}

	// Put tgz to S3
	err = s3Put(smc.DataFileURL, tgzFile.Name())
	return err
}

func (smc *SpotMC) updateDDNS() {
	if smc.ddnsURL != "" {
		log.Printf("issuing DDNS update query")
		resp, err := http.Get(smc.ddnsURL)
		if err != nil {
			log.Printf("DDNS update query failed: %s", err)
		} else {
			log.Printf("DDNS update query result: %s", resp.Status)
		}
	}
}

func (smc *SpotMC) StartServer() (exec.Cmd, error) {
	args := []string{smc.JavaPath}
	if smc.JavaArgs != "" {
		extraArgs := strings.Split(smc.JavaArgs, " ")
		args = append(args, extraArgs...)
	}
	args = append(args, "-jar", smc.serverPath, "nogui")
	log.Printf("Command to execute:%s args:%s length:%d", args[0], args, len(args))

	cmd := exec.Cmd{
		Path: args[0],
		Args: args,
		Dir:  smc.dataDirPath,
		//Stdin:  os.Stdin,
		Stdout: os.Stdout,
		Stderr: os.Stderr,
	}
	err := cmd.Start()
	return cmd, err
}

func (smc *SpotMC) KillInstance() error {
	log.Printf("KillInstance invoked")
	log.Printf("killInstanceMode: %s", smc.killInstanceMode)

	if smc.killInstanceMode == "false" {
		// False mode. Don't do anything
		return nil
	}

	if smc.killInstanceMode == "shutdown" {
		cmdArgs := strings.Split(smc.shutdownCommand, " ")
		cmd := exec.Command(cmdArgs[0], cmdArgs[1:]...)
		err := cmd.Run()
		return err
	}

	return nil
}

func (smc *SpotMC) ShutdownCluster() error {
	log.Printf("ShutDownCluster invoked")
	log.Printf("autoScalingGroup: %s", smc.autoScalingGroup)
	var err error
	if smc.autoScalingGroup != "" {
		log.Printf("setting cluster capacity to 0")
		for i := 0; i < AWS_RETRY; i++ {
			log.Printf("failed to set cluster capacity, retrying...")
			err = setDesiredCapacity(smc.autoScalingGroup, 0)
			if err == nil {
				break
			}
		}
		log.Fatal("setDesiredCapacity Failed!")
	}
	return err
}
<|endoftext|>


<commit_before>package genflowtypes

import (
	"bytes"
	"fmt"
	"strings"
	"text/template"

	"github.com/golang/protobuf/proto"
	pbdescriptor "github.com/golang/protobuf/protoc-gen-go/descriptor"
	"github.com/grpc-ecosystem/grpc-gateway/protoc-gen-grpc-gateway/descriptor"
	"github.com/mwitkow/go-proto-validators"
)

// FlowTyper is a flow language type
type FlowTyper interface {
	FlowType() string
	IsRequired() bool
}

// NamedFlowTyper is a FlowTyper with a name
type NamedFlowTyper interface {
	FlowTyper
	Name() string
}

func newSimpleType(typeString string, required bool) *primitiveType {
	return &primitiveType{typeString, required}
}

type primitiveType struct {
	typeString string
	required   bool
}

func (s primitiveType) FlowType() string { return s.typeString }
func (s primitiveType) IsRequired() bool { return s.required }

type messageType struct {
	*primitiveType
}

func newMessageFlowType(typeName string, required bool) *messageType {
	s := newSimpleType(typeName, required)
	return &messageType{s}
}

type repeatedType struct {
	FlowTyper
	required bool
}

func newRepeatedFlowType(underlying FlowTyper, required bool) *repeatedType {
	return &repeatedType{FlowTyper: underlying, required: required}
}

func (r repeatedType) FlowType() string { return fmt.Sprintf("Array<%s>", r.FlowTyper.FlowType()) }

type namedType struct {
	FlowTyper
	name     string
	required bool
}

func (t *namedType) Name() string {
	return t.name
}

type objectFlowType struct {
	Fields   []NamedFlowTyper
	Options  Options
	required bool
}

func (t *objectFlowType) FlowType() string {
	fields := []string{}
	for _, f := range t.Fields {
		optionalIndicator := "?"
		if f.IsRequired() {
			optionalIndicator = ""
		}
		fields = append(fields, fmt.Sprintf("  %s%s: %s", f.Name(), optionalIndicator, f.FlowType()))
	}
	return fmt.Sprintf("{\n%s\n}", strings.Join(fields, ",\n"))
}

func (t *objectFlowType) IsRequired() bool { return t.required }

func (cfg Options) fqmnToType(fqmn string, registry *descriptor.Registry) (FlowTyper, error) {
	m, err := registry.LookupMsg("", fqmn)
	if err != nil {
		return nil, err
	}
	return cfg.messageToFlowType(m, registry)
}

func (cfg Options) fieldToType(f *descriptor.Field, reg *descriptor.Registry, required bool) (NamedFlowTyper, error) {
	// FieldMessage
	var fieldType FlowTyper = newSimpleType("any", required)
	switch f.GetType() {
	case pbdescriptor.FieldDescriptorProto_TYPE_DOUBLE:
		fallthrough
	case pbdescriptor.FieldDescriptorProto_TYPE_FLOAT:
		fallthrough
	case pbdescriptor.FieldDescriptorProto_TYPE_INT64:
		fallthrough
	case pbdescriptor.FieldDescriptorProto_TYPE_UINT64:
		fallthrough
	case pbdescriptor.FieldDescriptorProto_TYPE_INT32:
		fallthrough
	case pbdescriptor.FieldDescriptorProto_TYPE_FIXED64:
		fallthrough
	case pbdescriptor.FieldDescriptorProto_TYPE_FIXED32:
		fallthrough
	case pbdescriptor.FieldDescriptorProto_TYPE_UINT32:
		fallthrough
	case pbdescriptor.FieldDescriptorProto_TYPE_SFIXED32:
		fallthrough
	case pbdescriptor.FieldDescriptorProto_TYPE_SFIXED64:
		fallthrough
	case pbdescriptor.FieldDescriptorProto_TYPE_SINT32:
		fallthrough
	case pbdescriptor.FieldDescriptorProto_TYPE_SINT64:
		fieldType = newSimpleType("number", required)
	case pbdescriptor.FieldDescriptorProto_TYPE_BOOL:
		fieldType = newSimpleType("boolean", required)
	case pbdescriptor.FieldDescriptorProto_TYPE_STRING:
		fieldType = newSimpleType("string", required)
	case pbdescriptor.FieldDescriptorProto_TYPE_GROUP:
		fieldType = newSimpleType("any", required) // , required?
	case pbdescriptor.FieldDescriptorProto_TYPE_MESSAGE:
		// TODO: should resolve type name relative to this type
		ft, err := reg.LookupMsg("", f.GetTypeName())
		if err != nil {
			return nil, err
		}
		fieldType = newMessageFlowType(cfg.messageTypeName(ft), required)
	case pbdescriptor.FieldDescriptorProto_TYPE_BYTES:
		fieldType = newSimpleType("string", required) // could be more correct
	case pbdescriptor.FieldDescriptorProto_TYPE_ENUM:
		e, err := reg.LookupEnum("", f.GetTypeName())
		if err != nil {
			return nil, err
		}

		if cfg.EmbedEnums {
			fieldType, err = cfg.enumToFlowType(e, reg)
			if err != nil {
				return nil, err
			}
		} else {
			name := cfg.enumTypeName(e)
			fieldType = newSimpleType(name, required)
		}
	}
	if f.GetLabel() == pbdescriptor.FieldDescriptorProto_LABEL_REPEATED {
		fieldType = newRepeatedFlowType(fieldType, required)
	}
	return &namedType{FlowTyper: fieldType, name: f.GetName(), required: required}, nil
}

// this is a hack to use the FieldOptions descriptor from golang/proto instead of gogo/proto
var E_Field = &proto.ExtensionDesc{
	ExtendedType:  (*pbdescriptor.FieldOptions)(nil),
	ExtensionType: validator.E_Field.ExtensionType,
	Field:         validator.E_Field.Field,
	Name:          validator.E_Field.Name,
	Tag:           validator.E_Field.Tag,
	Filename:      validator.E_Field.Filename,
}

func init() {
	proto.RegisterType((*validator.FieldValidator)(nil), "validator.FieldValidator")
	proto.RegisterExtension(E_Field)
}

func getFieldValidatorIfAny(field *pbdescriptor.FieldDescriptorProto) *validator.FieldValidator {
	if field.Options != nil {
		v, err := proto.GetExtension(field.Options, E_Field)
		if err == nil && v.(*validator.FieldValidator) != nil {
			return (v.(*validator.FieldValidator))
		}
	}
	return nil
}

func (cfg Options) messageToFlowType(m *descriptor.Message, reg *descriptor.Registry) (FlowTyper, error) {
	t := &objectFlowType{
		Fields:  []NamedFlowTyper{},
		Options: cfg,
	}
	for _, f := range m.Fields {
		required := false
		if validatorOptions := getFieldValidatorIfAny(f.FieldDescriptorProto); validatorOptions != nil {
			required = *validatorOptions.MsgExists
		}
		field, err := cfg.fieldToType(f, reg, required)
		if err != nil {
			return nil, err
		}
		t.Fields = append(t.Fields, field)
	}
	return &namedType{FlowTyper: t, name: cfg.messageTypeName(m)}, nil
}

func (cfg Options) enumTypeName(e *descriptor.Enum) string {
	name := strings.Replace(e.FQEN(), ".", "", -1)
	if !cfg.AlwaysQualifyTypes {
		if strings.HasPrefix(name, e.File.GoPkg.Name) {
			name = name[len(e.File.GoPkg.Name):]
		}
	}
	return name
}

func (cfg Options) messageTypeName(m *descriptor.Message) string {
	name := strings.Replace(m.FQMN(), ".", "", -1)
	if !cfg.AlwaysQualifyTypes {
		if strings.HasPrefix(name, m.File.GoPkg.Name) {
			name = name[len(m.File.GoPkg.Name):]
		}
	}
	return name
}

func (cfg Options) enumToFlowType(e *descriptor.Enum, reg *descriptor.Registry) (FlowTyper, error) {
	options := []string{}
	for _, v := range e.Value {
		if !cfg.EmitEnumZeros && v.GetNumber() == 0 {
			continue
		}
		options = append(options, fmt.Sprintf(`"%s"`, v.GetName()))
	}
	name := cfg.enumTypeName(e)
	return &namedType{
		FlowTyper: newSimpleType(strings.Join(options, " | "), false),
		name:      name,
	}, nil
}

func generateFlowTypes(file *descriptor.File, registry *descriptor.Registry, opts Options) (string, error) {
	result := []FlowTyper{}
	f, err := registry.LookupFile(file.GetName())
	if err != nil {
		return "", err
	}
	for _, enum := range f.Enums {
		t, err := opts.enumToFlowType(enum, registry)
		if err != nil {
			return "", err
		}
		result = append(result, t)
	}
	for _, message := range f.Messages {
		t, err := opts.messageToFlowType(message, registry)
		if err != nil {
			return "", err
		}
		result = append(result, t)
	}

	buf := new(bytes.Buffer)
	tmpl, err := template.New("").Parse(`/* @flow */
/* eslint-disable */
// Code generated by protoc-gen-flowtypes DO NOT EDIT.
// InputID: {{.InputID}}

{{range .Result}}export type {{.Name}} = {{.FlowType}};

{{end}}
`)
	if err != nil {
		return "", err
	}
	err = tmpl.Execute(buf, struct {
		Options
		Result []FlowTyper
	}{Options: opts, Result: result})
	if err != nil {
		return "", err
	}
	return string(buf.Bytes()), nil
}
<commit_msg>genflowtypes: handle google.protobuf.Timestamp<commit_after>package genflowtypes

import (
	"bytes"
	"fmt"
	"strings"
	"text/template"

	"github.com/golang/protobuf/proto"
	pbdescriptor "github.com/golang/protobuf/protoc-gen-go/descriptor"
	"github.com/grpc-ecosystem/grpc-gateway/protoc-gen-grpc-gateway/descriptor"
	"github.com/mwitkow/go-proto-validators"
)

// FlowTyper is a flow language type
type FlowTyper interface {
	FlowType() string
	IsRequired() bool
}

// NamedFlowTyper is a FlowTyper with a name
type NamedFlowTyper interface {
	FlowTyper
	Name() string
}

// knownTypeMap is a map of paths for known proto types to their desired
// flowtypes.
var knownTypeMap = map[string]string{
	".google.protobuf.Timestamp": "string",
}

func newSimpleType(typeString string, required bool) *primitiveType {
	return &primitiveType{typeString, required}
}

type primitiveType struct {
	typeString string
	required   bool
}

func (s primitiveType) FlowType() string { return s.typeString }
func (s primitiveType) IsRequired() bool { return s.required }

type messageType struct {
	*primitiveType
}

func newMessageFlowType(typeName string, required bool) *messageType {
	s := newSimpleType(typeName, required)
	return &messageType{s}
}

type repeatedType struct {
	FlowTyper
	required bool
}

func newRepeatedFlowType(underlying FlowTyper, required bool) *repeatedType {
	return &repeatedType{FlowTyper: underlying, required: required}
}

func (r repeatedType) FlowType() string { return fmt.Sprintf("Array<%s>", r.FlowTyper.FlowType()) }

type namedType struct {
	FlowTyper
	name     string
	required bool
}

func (t *namedType) Name() string {
	return t.name
}

type objectFlowType struct {
	Fields   []NamedFlowTyper
	Options  Options
	required bool
}

func (t *objectFlowType) FlowType() string {
	fields := []string{}
	for _, f := range t.Fields {
		optionalIndicator := "?"
		if f.IsRequired() {
			optionalIndicator = ""
		}
		fields = append(fields, fmt.Sprintf("  %s%s: %s", f.Name(), optionalIndicator, f.FlowType()))
	}
	return fmt.Sprintf("{\n%s\n}", strings.Join(fields, ",\n"))
}

func (t *objectFlowType) IsRequired() bool { return t.required }

func (cfg Options) fqmnToType(fqmn string, registry *descriptor.Registry) (FlowTyper, error) {
	m, err := registry.LookupMsg("", fqmn)
	if err != nil {
		return nil, err
	}
	return cfg.messageToFlowType(m, registry)
}

func (cfg Options) fieldToType(f *descriptor.Field, reg *descriptor.Registry, required bool) (NamedFlowTyper, error) {
	// FieldMessage
	var fieldType FlowTyper = newSimpleType("any", required)
	switch f.GetType() {
	case pbdescriptor.FieldDescriptorProto_TYPE_DOUBLE:
		fallthrough
	case pbdescriptor.FieldDescriptorProto_TYPE_FLOAT:
		fallthrough
	case pbdescriptor.FieldDescriptorProto_TYPE_INT64:
		fallthrough
	case pbdescriptor.FieldDescriptorProto_TYPE_UINT64:
		fallthrough
	case pbdescriptor.FieldDescriptorProto_TYPE_INT32:
		fallthrough
	case pbdescriptor.FieldDescriptorProto_TYPE_FIXED64:
		fallthrough
	case pbdescriptor.FieldDescriptorProto_TYPE_FIXED32:
		fallthrough
	case pbdescriptor.FieldDescriptorProto_TYPE_UINT32:
		fallthrough
	case pbdescriptor.FieldDescriptorProto_TYPE_SFIXED32:
		fallthrough
	case pbdescriptor.FieldDescriptorProto_TYPE_SFIXED64:
		fallthrough
	case pbdescriptor.FieldDescriptorProto_TYPE_SINT32:
		fallthrough
	case pbdescriptor.FieldDescriptorProto_TYPE_SINT64:
		fieldType = newSimpleType("number", required)
	case pbdescriptor.FieldDescriptorProto_TYPE_BOOL:
		fieldType = newSimpleType("boolean", required)
	case pbdescriptor.FieldDescriptorProto_TYPE_STRING:
		fieldType = newSimpleType("string", required)
	case pbdescriptor.FieldDescriptorProto_TYPE_GROUP:
		fieldType = newSimpleType("any", required) // , required?
	case pbdescriptor.FieldDescriptorProto_TYPE_MESSAGE:
		// TODO: should resolve type name relative to this type
		ft, err := reg.LookupMsg("", f.GetTypeName())
		if err != nil {
			return nil, err
		}

		if flowType, present := knownTypeMap[ft.FQMN()]; present {
			fieldType = newSimpleType(flowType, required)
		} else {
			fieldType = newMessageFlowType(cfg.messageTypeName(ft), required)
		}
	case pbdescriptor.FieldDescriptorProto_TYPE_BYTES:
		fieldType = newSimpleType("string", required) // could be more correct
	case pbdescriptor.FieldDescriptorProto_TYPE_ENUM:
		e, err := reg.LookupEnum("", f.GetTypeName())
		if err != nil {
			return nil, err
		}

		if cfg.EmbedEnums {
			fieldType, err = cfg.enumToFlowType(e, reg)
			if err != nil {
				return nil, err
			}
		} else {
			name := cfg.enumTypeName(e)
			fieldType = newSimpleType(name, required)
		}
	}
	if f.GetLabel() == pbdescriptor.FieldDescriptorProto_LABEL_REPEATED {
		fieldType = newRepeatedFlowType(fieldType, required)
	}
	return &namedType{FlowTyper: fieldType, name: f.GetName(), required: required}, nil
}

// this is a hack to use the FieldOptions descriptor from golang/proto instead of gogo/proto
var E_Field = &proto.ExtensionDesc{
	ExtendedType:  (*pbdescriptor.FieldOptions)(nil),
	ExtensionType: validator.E_Field.ExtensionType,
	Field:         validator.E_Field.Field,
	Name:          validator.E_Field.Name,
	Tag:           validator.E_Field.Tag,
	Filename:      validator.E_Field.Filename,
}

func init() {
	proto.RegisterType((*validator.FieldValidator)(nil), "validator.FieldValidator")
	proto.RegisterExtension(E_Field)
}

func getFieldValidatorIfAny(field *pbdescriptor.FieldDescriptorProto) *validator.FieldValidator {
	if field.Options != nil {
		v, err := proto.GetExtension(field.Options, E_Field)
		if err == nil && v.(*validator.FieldValidator) != nil {
			return (v.(*validator.FieldValidator))
		}
	}
	return nil
}

func (cfg Options) messageToFlowType(m *descriptor.Message, reg *descriptor.Registry) (FlowTyper, error) {
	t := &objectFlowType{
		Fields:  []NamedFlowTyper{},
		Options: cfg,
	}
	for _, f := range m.Fields {
		required := false
		if validatorOptions := getFieldValidatorIfAny(f.FieldDescriptorProto); validatorOptions != nil {
			required = *validatorOptions.MsgExists
		}
		field, err := cfg.fieldToType(f, reg, required)
		if err != nil {
			return nil, err
		}
		t.Fields = append(t.Fields, field)
	}
	return &namedType{FlowTyper: t, name: cfg.messageTypeName(m)}, nil
}

func (cfg Options) enumTypeName(e *descriptor.Enum) string {
	name := strings.Replace(e.FQEN(), ".", "", -1)
	if !cfg.AlwaysQualifyTypes {
		if strings.HasPrefix(name, e.File.GoPkg.Name) {
			name = name[len(e.File.GoPkg.Name):]
		}
	}
	return name
}

func (cfg Options) messageTypeName(m *descriptor.Message) string {
	name := strings.Replace(m.FQMN(), ".", "", -1)
	if !cfg.AlwaysQualifyTypes {
		if strings.HasPrefix(name, m.File.GoPkg.Name) {
			name = name[len(m.File.GoPkg.Name):]
		}
	}
	return name
}

func (cfg Options) enumToFlowType(e *descriptor.Enum, reg *descriptor.Registry) (FlowTyper, error) {
	options := []string{}
	for _, v := range e.Value {
		if !cfg.EmitEnumZeros && v.GetNumber() == 0 {
			continue
		}
		options = append(options, fmt.Sprintf(`"%s"`, v.GetName()))
	}
	name := cfg.enumTypeName(e)
	return &namedType{
		FlowTyper: newSimpleType(strings.Join(options, " | "), false),
		name:      name,
	}, nil
}

func generateFlowTypes(file *descriptor.File, registry *descriptor.Registry, opts Options) (string, error) {
	result := []FlowTyper{}
	f, err := registry.LookupFile(file.GetName())
	if err != nil {
		return "", err
	}
	for _, enum := range f.Enums {
		t, err := opts.enumToFlowType(enum, registry)
		if err != nil {
			return "", err
		}
		result = append(result, t)
	}
	for _, message := range f.Messages {
		t, err := opts.messageToFlowType(message, registry)
		if err != nil {
			return "", err
		}
		result = append(result, t)
	}

	buf := new(bytes.Buffer)
	tmpl, err := template.New("").Parse(`/* @flow */
/* eslint-disable */
// Code generated by protoc-gen-flowtypes DO NOT EDIT.
// InputID: {{.InputID}}

{{range .Result}}export type {{.Name}} = {{.FlowType}};

{{end}}
`)
	if err != nil {
		return "", err
	}
	err = tmpl.Execute(buf, struct {
		Options
		Result []FlowTyper
	}{Options: opts, Result: result})
	if err != nil {
		return "", err
	}
	return string(buf.Bytes()), nil
}
<|endoftext|>