<commit_before>package controller

import (
	"fmt"
	"os"
	"sync"

	"github.com/Sirupsen/logrus"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/types"
	"k8s.io/client-go/pkg/api/v1"
	"k8s.io/client-go/tools/cache"

	"github.com/zalando-incubator/postgres-operator/pkg/apiserver"
	"github.com/zalando-incubator/postgres-operator/pkg/cluster"
	"github.com/zalando-incubator/postgres-operator/pkg/spec"
	"github.com/zalando-incubator/postgres-operator/pkg/util/config"
	"github.com/zalando-incubator/postgres-operator/pkg/util/constants"
	"github.com/zalando-incubator/postgres-operator/pkg/util/k8sutil"
	"github.com/zalando-incubator/postgres-operator/pkg/util/ringlog"
)

// Controller represents operator controller
type Controller struct {
	config   spec.ControllerConfig
	opConfig *config.Config

	logger     *logrus.Entry
	KubeClient k8sutil.KubernetesClient
	apiserver  *apiserver.Server

	stopCh chan struct{}

	curWorkerID      uint32 //initialized with 0
	curWorkerCluster sync.Map
	clusterWorkers   map[spec.NamespacedName]uint32
	clustersMu       sync.RWMutex
	clusters         map[spec.NamespacedName]*cluster.Cluster
	clusterLogs      map[spec.NamespacedName]ringlog.RingLogger
	clusterHistory   map[spec.NamespacedName]ringlog.RingLogger // history of the cluster changes
	teamClusters     map[string][]spec.NamespacedName

	postgresqlInformer cache.SharedIndexInformer
	podInformer        cache.SharedIndexInformer
	nodesInformer      cache.SharedIndexInformer
	podCh              chan spec.PodEvent

	clusterEventQueues  []*cache.FIFO // [workerID]Queue
	lastClusterSyncTime int64

	workerLogs map[uint32]ringlog.RingLogger
}

// NewController creates a new controller
func NewController(controllerConfig *spec.ControllerConfig) *Controller {
	logger := logrus.New()

	c := &Controller{
		config:           *controllerConfig,
		opConfig:         &config.Config{},
		logger:           logger.WithField("pkg", "controller"),
		curWorkerCluster: sync.Map{},
		clusterWorkers:   make(map[spec.NamespacedName]uint32),
		clusters:         make(map[spec.NamespacedName]*cluster.Cluster),
		clusterLogs:      make(map[spec.NamespacedName]ringlog.RingLogger),
		clusterHistory:   make(map[spec.NamespacedName]ringlog.RingLogger),
		teamClusters:     make(map[string][]spec.NamespacedName),
		stopCh:           make(chan struct{}),
		podCh:            make(chan spec.PodEvent),
	}
	logger.Hooks.Add(c)

	return c
}

func (c *Controller) initClients() {
	var err error

	c.KubeClient, err = k8sutil.NewFromConfig(c.config.RestConfig)
	if err != nil {
		c.logger.Fatalf("could not create kubernetes clients: %v", err)
	}
}

func (c *Controller) initOperatorConfig() {
	configMapData := make(map[string]string)

	if c.config.ConfigMapName != (spec.NamespacedName{}) {
		configMap, err := c.KubeClient.ConfigMaps(c.config.ConfigMapName.Namespace).
			Get(c.config.ConfigMapName.Name, metav1.GetOptions{})
		if err != nil {
			panic(err)
		}

		configMapData = configMap.Data
	} else {
		c.logger.Infoln("no ConfigMap specified. Loading default values")
	}

	// by default, the operator listens to all namespaces
	// by setting the env variable, one can restrict the operator to a single namespace
	watchedNamespace, isPresentInEnv := os.LookupEnv("WATCHED_NAMESPACE")
	if isPresentInEnv {
		// special case: v1.NamespaceAll currently also evaluates to the empty string
		// so when the env var is set to the empty string, use the default ns
		// since the meaning of this env var is only one namespace
		if watchedNamespace == "" {
			c.logger.Infof("The WATCHED_NAMESPACE env var evaluates to the empty string, falling back to watching the 'default' namespace.\n", watchedNamespace)
			configMapData["watched_namespace"] = v1.NamespaceDefault
		} else {
			c.logger.Infof("Watch the %q namespace specified in the env variable WATCHED_NAMESPACE\n", watchedNamespace)
			configMapData["watched_namespace"] = watchedNamespace
		}

	} else {
		c.logger.Infof("Watch all namespaces. Set the WATCHED_NAMESPACE env var  to restrict to a single namespace.\n", watchedNamespace)
		configMapData["watched_namespace"] = v1.NamespaceAll
	}

	/*
		// env var takes priority over the same param from the operator ConfigMap

			if configMapData["watched_namespace"] == "" {
				c.logger.Infoln("No namespace to watch specified. Fall back to watching the 'default' namespace.")
				configMapData["watched_namespace"] = v1.NamespaceDefault
			}
	*/
	if c.config.NoDatabaseAccess {
		configMapData["enable_database_access"] = "false"
	}
	if c.config.NoTeamsAPI {
		configMapData["enable_teams_api"] = "false"
	}

	c.opConfig = config.NewFromMap(configMapData)

	scalyrAPIKey := os.Getenv("SCALYR_API_KEY")
	if scalyrAPIKey != "" {
		c.opConfig.ScalyrAPIKey = scalyrAPIKey
	}
}

func (c *Controller) initController() {
	c.initClients()
	c.initOperatorConfig()

	// earliest point where we can check if the namespace to watch actually exists
	if c.opConfig.WatchedNamespace != v1.NamespaceAll {
		_, err := c.KubeClient.Namespaces().Get(c.opConfig.WatchedNamespace, metav1.GetOptions{})
		if err != nil {
			c.logger.Fatalf("Operator was told to watch the %q namespace but was unable to find it via Kubernetes API.", c.opConfig.WatchedNamespace)
		}

	}

	c.initSharedInformers()

	c.logger.Infof("config: %s", c.opConfig.MustMarshal())

	if c.opConfig.DebugLogging {
		c.logger.Logger.Level = logrus.DebugLevel
	}

	if err := c.createCRD(); err != nil {
		c.logger.Fatalf("could not register CustomResourceDefinition: %v", err)
	}

	if infraRoles, err := c.getInfrastructureRoles(&c.opConfig.InfrastructureRolesSecretName); err != nil {
		c.logger.Warningf("could not get infrastructure roles: %v", err)
	} else {
		c.config.InfrastructureRoles = infraRoles
	}

	c.clusterEventQueues = make([]*cache.FIFO, c.opConfig.Workers)
	c.workerLogs = make(map[uint32]ringlog.RingLogger, c.opConfig.Workers)
	for i := range c.clusterEventQueues {
		c.clusterEventQueues[i] = cache.NewFIFO(func(obj interface{}) (string, error) {
			e, ok := obj.(spec.ClusterEvent)
			if !ok {
				return "", fmt.Errorf("could not cast to ClusterEvent")
			}

			return queueClusterKey(e.EventType, e.UID), nil
		})
	}

	c.apiserver = apiserver.New(c, c.opConfig.APIPort, c.logger.Logger)
}

func (c *Controller) initSharedInformers() {
	// Postgresqls
	c.postgresqlInformer = cache.NewSharedIndexInformer(
		&cache.ListWatch{
			ListFunc:  c.clusterListFunc,
			WatchFunc: c.clusterWatchFunc,
		},
		&spec.Postgresql{},
		constants.QueueResyncPeriodTPR,
		cache.Indexers{})

	c.postgresqlInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{
		AddFunc:    c.postgresqlAdd,
		UpdateFunc: c.postgresqlUpdate,
		DeleteFunc: c.postgresqlDelete,
	})

	// Pods
	podLw := &cache.ListWatch{
		ListFunc:  c.podListFunc,
		WatchFunc: c.podWatchFunc,
	}

	c.podInformer = cache.NewSharedIndexInformer(
		podLw,
		&v1.Pod{},
		constants.QueueResyncPeriodPod,
		cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc})

	c.podInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{
		AddFunc:    c.podAdd,
		UpdateFunc: c.podUpdate,
		DeleteFunc: c.podDelete,
	})

	// Kubernetes Nodes
	nodeLw := &cache.ListWatch{
		ListFunc:  c.nodeListFunc,
		WatchFunc: c.nodeWatchFunc,
	}

	c.nodesInformer = cache.NewSharedIndexInformer(
		nodeLw,
		&v1.Node{},
		constants.QueueResyncPeriodNode,
		cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc})

	c.nodesInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{
		AddFunc:    c.nodeAdd,
		UpdateFunc: c.nodeUpdate,
		DeleteFunc: c.nodeDelete,
	})
}

// Run starts background controller processes
func (c *Controller) Run(stopCh <-chan struct{}, wg *sync.WaitGroup) {
	c.initController()

	wg.Add(5)
	go c.runPodInformer(stopCh, wg)
	go c.runPostgresqlInformer(stopCh, wg)
	go c.clusterResync(stopCh, wg)
	go c.apiserver.Run(stopCh, wg)
	go c.kubeNodesInformer(stopCh, wg)

	for i := range c.clusterEventQueues {
		wg.Add(1)
		c.workerLogs[uint32(i)] = ringlog.New(c.opConfig.RingLogLines)
		go c.processClusterEventsQueue(i, stopCh, wg)
	}

	c.logger.Info("started working in background")
}

func (c *Controller) runPodInformer(stopCh <-chan struct{}, wg *sync.WaitGroup) {
	defer wg.Done()

	c.podInformer.Run(stopCh)
}

func (c *Controller) runPostgresqlInformer(stopCh <-chan struct{}, wg *sync.WaitGroup) {
	defer wg.Done()

	c.postgresqlInformer.Run(stopCh)
}

func queueClusterKey(eventType spec.EventType, uid types.UID) string {
	return fmt.Sprintf("%s-%s", eventType, uid)
}

func (c *Controller) kubeNodesInformer(stopCh <-chan struct{}, wg *sync.WaitGroup) {
	defer wg.Done()

	c.nodesInformer.Run(stopCh)
}
<commit_msg>Handle watched namespace set in operator config map<commit_after>package controller

import (
	"fmt"
	"os"
	"sync"

	"github.com/Sirupsen/logrus"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/types"
	"k8s.io/client-go/pkg/api/v1"
	"k8s.io/client-go/tools/cache"

	"github.com/zalando-incubator/postgres-operator/pkg/apiserver"
	"github.com/zalando-incubator/postgres-operator/pkg/cluster"
	"github.com/zalando-incubator/postgres-operator/pkg/spec"
	"github.com/zalando-incubator/postgres-operator/pkg/util/config"
	"github.com/zalando-incubator/postgres-operator/pkg/util/constants"
	"github.com/zalando-incubator/postgres-operator/pkg/util/k8sutil"
	"github.com/zalando-incubator/postgres-operator/pkg/util/ringlog"
)

// Controller represents operator controller
type Controller struct {
	config   spec.ControllerConfig
	opConfig *config.Config

	logger     *logrus.Entry
	KubeClient k8sutil.KubernetesClient
	apiserver  *apiserver.Server

	stopCh chan struct{}

	curWorkerID      uint32 //initialized with 0
	curWorkerCluster sync.Map
	clusterWorkers   map[spec.NamespacedName]uint32
	clustersMu       sync.RWMutex
	clusters         map[spec.NamespacedName]*cluster.Cluster
	clusterLogs      map[spec.NamespacedName]ringlog.RingLogger
	clusterHistory   map[spec.NamespacedName]ringlog.RingLogger // history of the cluster changes
	teamClusters     map[string][]spec.NamespacedName

	postgresqlInformer cache.SharedIndexInformer
	podInformer        cache.SharedIndexInformer
	nodesInformer      cache.SharedIndexInformer
	podCh              chan spec.PodEvent

	clusterEventQueues  []*cache.FIFO // [workerID]Queue
	lastClusterSyncTime int64

	workerLogs map[uint32]ringlog.RingLogger
}

// NewController creates a new controller
func NewController(controllerConfig *spec.ControllerConfig) *Controller {
	logger := logrus.New()

	c := &Controller{
		config:           *controllerConfig,
		opConfig:         &config.Config{},
		logger:           logger.WithField("pkg", "controller"),
		curWorkerCluster: sync.Map{},
		clusterWorkers:   make(map[spec.NamespacedName]uint32),
		clusters:         make(map[spec.NamespacedName]*cluster.Cluster),
		clusterLogs:      make(map[spec.NamespacedName]ringlog.RingLogger),
		clusterHistory:   make(map[spec.NamespacedName]ringlog.RingLogger),
		teamClusters:     make(map[string][]spec.NamespacedName),
		stopCh:           make(chan struct{}),
		podCh:            make(chan spec.PodEvent),
	}
	logger.Hooks.Add(c)

	return c
}

func (c *Controller) initClients() {
	var err error

	c.KubeClient, err = k8sutil.NewFromConfig(c.config.RestConfig)
	if err != nil {
		c.logger.Fatalf("could not create kubernetes clients: %v", err)
	}
}

func (c *Controller) initOperatorConfig() {
	configMapData := make(map[string]string)

	if c.config.ConfigMapName != (spec.NamespacedName{}) {
		configMap, err := c.KubeClient.ConfigMaps(c.config.ConfigMapName.Namespace).
			Get(c.config.ConfigMapName.Name, metav1.GetOptions{})
		if err != nil {
			panic(err)
		}

		configMapData = configMap.Data
	} else {
		c.logger.Infoln("no ConfigMap specified. Loading default values")
	}

	// by default, the operator listens to all namespaces
	if configMapData["watched_namespace"] == "" {
		c.logger.Infoln("The operator config map specifies no namespace to watch. Falling back to watching all namespaces.")
		configMapData["watched_namespace"] = v1.NamespaceAll
	}

	watchedNsEnvVar, isPresentInEnv := os.LookupEnv("WATCHED_NAMESPACE")
	if isPresentInEnv {
		c.logger.Infoln("The WATCHED_NAMESPACE env variable takes priority over the same param from the operator configMap\n")
		// special case: v1.NamespaceAll currently also evaluates to the empty string
		// so when the env var is set to the empty string, use the default ns
		// since the meaning of this env var is only one namespace
		if watchedNsEnvVar == "" {
			c.logger.Infof("The WATCHED_NAMESPACE env var evaluates to the empty string, falling back to watching the 'default' namespace.\n", watchedNsEnvVar)
			configMapData["watched_namespace"] = v1.NamespaceDefault
		} else {
			c.logger.Infof("Watch the %q namespace specified in the env variable WATCHED_NAMESPACE\n", watchedNsEnvVar)
			configMapData["watched_namespace"] = watchedNsEnvVar
		}
	}

	if c.config.NoDatabaseAccess {
		configMapData["enable_database_access"] = "false"
	}
	if c.config.NoTeamsAPI {
		configMapData["enable_teams_api"] = "false"
	}

	c.opConfig = config.NewFromMap(configMapData)

	scalyrAPIKey := os.Getenv("SCALYR_API_KEY")
	if scalyrAPIKey != "" {
		c.opConfig.ScalyrAPIKey = scalyrAPIKey
	}
}

func (c *Controller) initController() {
	c.initClients()
	c.initOperatorConfig()

	// earliest point where we can check if the namespace to watch actually exists
	if c.opConfig.WatchedNamespace != v1.NamespaceAll {
		_, err := c.KubeClient.Namespaces().Get(c.opConfig.WatchedNamespace, metav1.GetOptions{})
		if err != nil {
			c.logger.Fatalf("Operator was told to watch the %q namespace but was unable to find it via Kubernetes API.", c.opConfig.WatchedNamespace)
		}

	}

	c.initSharedInformers()

	c.logger.Infof("config: %s", c.opConfig.MustMarshal())

	if c.opConfig.DebugLogging {
		c.logger.Logger.Level = logrus.DebugLevel
	}

	if err := c.createCRD(); err != nil {
		c.logger.Fatalf("could not register CustomResourceDefinition: %v", err)
	}

	if infraRoles, err := c.getInfrastructureRoles(&c.opConfig.InfrastructureRolesSecretName); err != nil {
		c.logger.Warningf("could not get infrastructure roles: %v", err)
	} else {
		c.config.InfrastructureRoles = infraRoles
	}

	c.clusterEventQueues = make([]*cache.FIFO, c.opConfig.Workers)
	c.workerLogs = make(map[uint32]ringlog.RingLogger, c.opConfig.Workers)
	for i := range c.clusterEventQueues {
		c.clusterEventQueues[i] = cache.NewFIFO(func(obj interface{}) (string, error) {
			e, ok := obj.(spec.ClusterEvent)
			if !ok {
				return "", fmt.Errorf("could not cast to ClusterEvent")
			}

			return queueClusterKey(e.EventType, e.UID), nil
		})
	}

	c.apiserver = apiserver.New(c, c.opConfig.APIPort, c.logger.Logger)
}

func (c *Controller) initSharedInformers() {
	// Postgresqls
	c.postgresqlInformer = cache.NewSharedIndexInformer(
		&cache.ListWatch{
			ListFunc:  c.clusterListFunc,
			WatchFunc: c.clusterWatchFunc,
		},
		&spec.Postgresql{},
		constants.QueueResyncPeriodTPR,
		cache.Indexers{})

	c.postgresqlInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{
		AddFunc:    c.postgresqlAdd,
		UpdateFunc: c.postgresqlUpdate,
		DeleteFunc: c.postgresqlDelete,
	})

	// Pods
	podLw := &cache.ListWatch{
		ListFunc:  c.podListFunc,
		WatchFunc: c.podWatchFunc,
	}

	c.podInformer = cache.NewSharedIndexInformer(
		podLw,
		&v1.Pod{},
		constants.QueueResyncPeriodPod,
		cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc})

	c.podInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{
		AddFunc:    c.podAdd,
		UpdateFunc: c.podUpdate,
		DeleteFunc: c.podDelete,
	})

	// Kubernetes Nodes
	nodeLw := &cache.ListWatch{
		ListFunc:  c.nodeListFunc,
		WatchFunc: c.nodeWatchFunc,
	}

	c.nodesInformer = cache.NewSharedIndexInformer(
		nodeLw,
		&v1.Node{},
		constants.QueueResyncPeriodNode,
		cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc})

	c.nodesInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{
		AddFunc:    c.nodeAdd,
		UpdateFunc: c.nodeUpdate,
		DeleteFunc: c.nodeDelete,
	})
}

// Run starts background controller processes
func (c *Controller) Run(stopCh <-chan struct{}, wg *sync.WaitGroup) {
	c.initController()

	wg.Add(5)
	go c.runPodInformer(stopCh, wg)
	go c.runPostgresqlInformer(stopCh, wg)
	go c.clusterResync(stopCh, wg)
	go c.apiserver.Run(stopCh, wg)
	go c.kubeNodesInformer(stopCh, wg)

	for i := range c.clusterEventQueues {
		wg.Add(1)
		c.workerLogs[uint32(i)] = ringlog.New(c.opConfig.RingLogLines)
		go c.processClusterEventsQueue(i, stopCh, wg)
	}

	c.logger.Info("started working in background")
}

func (c *Controller) runPodInformer(stopCh <-chan struct{}, wg *sync.WaitGroup) {
	defer wg.Done()

	c.podInformer.Run(stopCh)
}

func (c *Controller) runPostgresqlInformer(stopCh <-chan struct{}, wg *sync.WaitGroup) {
	defer wg.Done()

	c.postgresqlInformer.Run(stopCh)
}

func queueClusterKey(eventType spec.EventType, uid types.UID) string {
	return fmt.Sprintf("%s-%s", eventType, uid)
}

func (c *Controller) kubeNodesInformer(stopCh <-chan struct{}, wg *sync.WaitGroup) {
	defer wg.Done()

	c.nodesInformer.Run(stopCh)
}
<|endoftext|>


<commit_before>/*
Copyright 2017 Mirantis

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package flexvolume

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"strings"

	uuid "github.com/nu7hatch/gouuid"
)

type volumeOpts struct {
	Monitor  string `json:"monitor"`
	Pool     string `json:"pool"`
	Volume   string `json:"volume"`
	Secret   string `json:"secret"`
	User     string `json:user`
	Protocol string `json:protocol`
}

func newUuid() string {
	u, err := uuid.NewV4()
	if err != nil {
		panic("can't generate UUID")
	}
	return u.String()
}

type UuidGen func() string

type FlexVolumeDriver struct {
	uuidGen UuidGen
}

func NewFlexVolumeDriver(uuidGen UuidGen) *FlexVolumeDriver {
	if uuidGen == nil {
		uuidGen = newUuid
	}
	return &FlexVolumeDriver{uuidGen: uuidGen}
}

func (d *FlexVolumeDriver) addVolumeDefinitions(target string, opts volumeOpts) error {
	uuid := d.uuidGen()

	secretXML := `
<secret ephemeral='no' private='no'>
  <uuid>%s</uuid>
  <usage type='ceph'>
    <name>%s</name>
  </usage>
</secret>
`
	if err := ioutil.WriteFile(target+"/secret.xml", []byte(fmt.Sprintf(secretXML, uuid, opts.User)), 0644); err != nil {
		return err
	}

	// Will be removed right after creating appropriate secret in libvirt
	if err := ioutil.WriteFile(target+"/key", []byte(opts.Secret), 0644); err != nil {
		return err
	}

	diskXML := `
<disk type="network" device="disk">
  <driver name="qemu" type="raw"/>
  <auth username="%s">
    <secret type="ceph" uuid="%s"/>
  </auth>
  <source protocol="rbd" name="%s/%s">
    <host name="%s" port="%s"/>
  </source>
  <target dev="%s" bus="virtio"/>
</disk>
`
	pairIPPort := strings.Split(opts.Monitor, ":")
	if len(pairIPPort) != 2 {
		return fmt.Errorf("invalid format of ceph monitor setting: %s. Expected in form ip:port", opts.Monitor)
	}
	// Note: target dev name wiil be specified later by virtlet diring combining domain xml definition
	if err := ioutil.WriteFile(target+"/disk.xml", []byte(fmt.Sprintf(diskXML, opts.User, uuid, opts.Pool, opts.Volume, pairIPPort[0], pairIPPort[1], "%s")), 0644); err != nil {
		return err
	}
	return nil
}

// The following functions are not currently needed, but still
// keeping them to make it easier to actually implement them

// Invocation: <driver executable> init
func (d *FlexVolumeDriver) init() (map[string]interface{}, error) {
	return nil, nil
}

// Invocation: <driver executable> attach <json options> <node name>
func (d *FlexVolumeDriver) attach(jsonOptions, nodeName string) (map[string]interface{}, error) {
	return nil, nil
}

// Invocation: <driver executable> detach <mount device> <node name>
func (d *FlexVolumeDriver) detach(mountDev, nodeName string) (map[string]interface{}, error) {
	return nil, nil
}

// Invocation: <driver executable> waitforattach <mount device> <json options>
func (d *FlexVolumeDriver) waitForAttach(mountDev, jsonOptions string) (map[string]interface{}, error) {
	return map[string]interface{}{"device": mountDev}, nil
}

// Invocation: <driver executable> isattached <json options> <node name>
func (d *FlexVolumeDriver) isAttached(jsonOptions, nodeName string) (map[string]interface{}, error) {
	return map[string]interface{}{"attached": true}, nil
}

//Invocation: <driver executable> mount <target mount dir> <mount device> <json options>
func (d *FlexVolumeDriver) mount(targetMountDir, jsonOptions string) (map[string]interface{}, error) {
	var opts volumeOpts
	if err := json.Unmarshal([]byte(jsonOptions), &opts); err != nil {
		return nil, fmt.Errorf("failed to unmarshal json options: %v", err)
	}
	if err := os.MkdirAll(targetMountDir, 0700); err != nil {
		return nil, fmt.Errorf("os.MkDirAll(): %v", err)
	}
	if err := d.addVolumeDefinitions(targetMountDir, opts); err != nil {
		return nil, err
	}
	return nil, nil
}

// Invocation: <driver executable> unmount <mount dir>
func (d *FlexVolumeDriver) unmount(targetMountDir string) (map[string]interface{}, error) {
	if err := os.RemoveAll(targetMountDir); err != nil {
		return nil, fmt.Errorf("os.RemoveAll(): %v", err.Error())
	}

	return nil, nil
}

// Invocation: <driver executable> getvolumename <json options>
func (d *FlexVolumeDriver) getVolumeName(jsonOptions string) (map[string]interface{}, error) {
	var opts volumeOpts
	if err := json.Unmarshal([]byte(jsonOptions), &opts); err != nil {
		return nil, fmt.Errorf("failed to unmarshal json options: %v", err)
	}
	r := []string{}
	monStr := strings.Replace(opts.Monitor, ":", "/", -1)
	for _, s := range []string{monStr, opts.Pool, opts.Volume, opts.User, opts.Protocol} {
		if s != "" {
			r = append(r, s)
		}
	}
	if len(r) == 0 {
		return nil, fmt.Errorf("invalid flexvolume definition")
	}
	return map[string]interface{}{"volumeName": strings.Join(r, "/")}, nil
}

type driverOp func(*FlexVolumeDriver, []string) (map[string]interface{}, error)

type cmdInfo struct {
	numArgs int
	run     driverOp
}

var commands = map[string]cmdInfo{
	"init": cmdInfo{
		0, func(d *FlexVolumeDriver, args []string) (map[string]interface{}, error) {
			return d.init()
		},
	},
	"attach": cmdInfo{
		2, func(d *FlexVolumeDriver, args []string) (map[string]interface{}, error) {
			return d.attach(args[0], args[1])
		},
	},
	"detach": cmdInfo{
		2, func(d *FlexVolumeDriver, args []string) (map[string]interface{}, error) {
			return d.detach(args[0], args[1])
		},
	},
	"waitforattach": cmdInfo{
		2, func(d *FlexVolumeDriver, args []string) (map[string]interface{}, error) {
			return d.waitForAttach(args[0], args[1])
		},
	},
	"isattached": cmdInfo{
		2, func(d *FlexVolumeDriver, args []string) (map[string]interface{}, error) {
			return d.isAttached(args[0], args[1])
		},
	},
	"getvolumename": cmdInfo{
		1, func(d *FlexVolumeDriver, args []string) (map[string]interface{}, error) {
			return d.getVolumeName(args[0])
		},
	},
	"mount": cmdInfo{
		2, func(d *FlexVolumeDriver, args []string) (map[string]interface{}, error) {
			return d.mount(args[0], args[1])
		},
	},
	"unmount": cmdInfo{
		1, func(d *FlexVolumeDriver, args []string) (map[string]interface{}, error) {
			return d.unmount(args[0])
		},
	},
}

func (d *FlexVolumeDriver) doRun(args []string) (map[string]interface{}, error) {
	if len(args) == 0 {
		return nil, errors.New("no arguments passed to flexvolume driver")
	}
	nArgs := len(args) - 1
	op := args[0]
	if cmdInfo, found := commands[op]; found {
		if cmdInfo.numArgs == nArgs {
			return cmdInfo.run(d, args[1:])
		} else {
			return nil, fmt.Errorf("unexpected number of args %d (expected %d) for operation %q", nArgs, cmdInfo.numArgs, op)
		}
	} else {
		return map[string]interface{}{
			"status": "Not supported",
		}, nil
	}
}

func (d *FlexVolumeDriver) Run(args []string) string {
	r := formatResult(d.doRun(args))

	f, err := os.OpenFile("/tmp/flexvolume.log", os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0666)
	if err != nil {
		panic(err)
	}

	defer f.Close()

	fmt.Fprintf(f, "flexvolume %s -> %s\n", strings.Join(args, " "), r)
	return r
}

func formatResult(fields map[string]interface{}, err error) string {
	var data map[string]interface{}
	if err != nil {
		data = map[string]interface{}{
			"status":  "Failure",
			"message": err.Error(),
		}
	} else {
		data = map[string]interface{}{
			"status": "Success",
		}
		for k, v := range fields {
			data[k] = v
		}
	}
	s, err := json.Marshal(data)
	if err != nil {
		panic("error marshalling the data")
	}
	return string(s) + "\n"
}
<commit_msg>Disable flexvolume debugging for now<commit_after>/*
Copyright 2017 Mirantis

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package flexvolume

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"strings"

	uuid "github.com/nu7hatch/gouuid"
)

type volumeOpts struct {
	Monitor  string `json:"monitor"`
	Pool     string `json:"pool"`
	Volume   string `json:"volume"`
	Secret   string `json:"secret"`
	User     string `json:user`
	Protocol string `json:protocol`
}

func newUuid() string {
	u, err := uuid.NewV4()
	if err != nil {
		panic("can't generate UUID")
	}
	return u.String()
}

type UuidGen func() string

type FlexVolumeDriver struct {
	uuidGen UuidGen
}

func NewFlexVolumeDriver(uuidGen UuidGen) *FlexVolumeDriver {
	if uuidGen == nil {
		uuidGen = newUuid
	}
	return &FlexVolumeDriver{uuidGen: uuidGen}
}

func (d *FlexVolumeDriver) addVolumeDefinitions(target string, opts volumeOpts) error {
	uuid := d.uuidGen()

	secretXML := `
<secret ephemeral='no' private='no'>
  <uuid>%s</uuid>
  <usage type='ceph'>
    <name>%s</name>
  </usage>
</secret>
`
	if err := ioutil.WriteFile(target+"/secret.xml", []byte(fmt.Sprintf(secretXML, uuid, opts.User)), 0644); err != nil {
		return err
	}

	// Will be removed right after creating appropriate secret in libvirt
	if err := ioutil.WriteFile(target+"/key", []byte(opts.Secret), 0644); err != nil {
		return err
	}

	diskXML := `
<disk type="network" device="disk">
  <driver name="qemu" type="raw"/>
  <auth username="%s">
    <secret type="ceph" uuid="%s"/>
  </auth>
  <source protocol="rbd" name="%s/%s">
    <host name="%s" port="%s"/>
  </source>
  <target dev="%s" bus="virtio"/>
</disk>
`
	pairIPPort := strings.Split(opts.Monitor, ":")
	if len(pairIPPort) != 2 {
		return fmt.Errorf("invalid format of ceph monitor setting: %s. Expected in form ip:port", opts.Monitor)
	}
	// Note: target dev name wiil be specified later by virtlet diring combining domain xml definition
	if err := ioutil.WriteFile(target+"/disk.xml", []byte(fmt.Sprintf(diskXML, opts.User, uuid, opts.Pool, opts.Volume, pairIPPort[0], pairIPPort[1], "%s")), 0644); err != nil {
		return err
	}
	return nil
}

// The following functions are not currently needed, but still
// keeping them to make it easier to actually implement them

// Invocation: <driver executable> init
func (d *FlexVolumeDriver) init() (map[string]interface{}, error) {
	return nil, nil
}

// Invocation: <driver executable> attach <json options> <node name>
func (d *FlexVolumeDriver) attach(jsonOptions, nodeName string) (map[string]interface{}, error) {
	return nil, nil
}

// Invocation: <driver executable> detach <mount device> <node name>
func (d *FlexVolumeDriver) detach(mountDev, nodeName string) (map[string]interface{}, error) {
	return nil, nil
}

// Invocation: <driver executable> waitforattach <mount device> <json options>
func (d *FlexVolumeDriver) waitForAttach(mountDev, jsonOptions string) (map[string]interface{}, error) {
	return map[string]interface{}{"device": mountDev}, nil
}

// Invocation: <driver executable> isattached <json options> <node name>
func (d *FlexVolumeDriver) isAttached(jsonOptions, nodeName string) (map[string]interface{}, error) {
	return map[string]interface{}{"attached": true}, nil
}

//Invocation: <driver executable> mount <target mount dir> <mount device> <json options>
func (d *FlexVolumeDriver) mount(targetMountDir, jsonOptions string) (map[string]interface{}, error) {
	var opts volumeOpts
	if err := json.Unmarshal([]byte(jsonOptions), &opts); err != nil {
		return nil, fmt.Errorf("failed to unmarshal json options: %v", err)
	}
	if err := os.MkdirAll(targetMountDir, 0700); err != nil {
		return nil, fmt.Errorf("os.MkDirAll(): %v", err)
	}
	if err := d.addVolumeDefinitions(targetMountDir, opts); err != nil {
		return nil, err
	}
	return nil, nil
}

// Invocation: <driver executable> unmount <mount dir>
func (d *FlexVolumeDriver) unmount(targetMountDir string) (map[string]interface{}, error) {
	if err := os.RemoveAll(targetMountDir); err != nil {
		return nil, fmt.Errorf("os.RemoveAll(): %v", err.Error())
	}

	return nil, nil
}

// Invocation: <driver executable> getvolumename <json options>
func (d *FlexVolumeDriver) getVolumeName(jsonOptions string) (map[string]interface{}, error) {
	var opts volumeOpts
	if err := json.Unmarshal([]byte(jsonOptions), &opts); err != nil {
		return nil, fmt.Errorf("failed to unmarshal json options: %v", err)
	}
	r := []string{}
	monStr := strings.Replace(opts.Monitor, ":", "/", -1)
	for _, s := range []string{monStr, opts.Pool, opts.Volume, opts.User, opts.Protocol} {
		if s != "" {
			r = append(r, s)
		}
	}
	if len(r) == 0 {
		return nil, fmt.Errorf("invalid flexvolume definition")
	}
	return map[string]interface{}{"volumeName": strings.Join(r, "/")}, nil
}

type driverOp func(*FlexVolumeDriver, []string) (map[string]interface{}, error)

type cmdInfo struct {
	numArgs int
	run     driverOp
}

var commands = map[string]cmdInfo{
	"init": cmdInfo{
		0, func(d *FlexVolumeDriver, args []string) (map[string]interface{}, error) {
			return d.init()
		},
	},
	"attach": cmdInfo{
		2, func(d *FlexVolumeDriver, args []string) (map[string]interface{}, error) {
			return d.attach(args[0], args[1])
		},
	},
	"detach": cmdInfo{
		2, func(d *FlexVolumeDriver, args []string) (map[string]interface{}, error) {
			return d.detach(args[0], args[1])
		},
	},
	"waitforattach": cmdInfo{
		2, func(d *FlexVolumeDriver, args []string) (map[string]interface{}, error) {
			return d.waitForAttach(args[0], args[1])
		},
	},
	"isattached": cmdInfo{
		2, func(d *FlexVolumeDriver, args []string) (map[string]interface{}, error) {
			return d.isAttached(args[0], args[1])
		},
	},
	"getvolumename": cmdInfo{
		1, func(d *FlexVolumeDriver, args []string) (map[string]interface{}, error) {
			return d.getVolumeName(args[0])
		},
	},
	"mount": cmdInfo{
		2, func(d *FlexVolumeDriver, args []string) (map[string]interface{}, error) {
			return d.mount(args[0], args[1])
		},
	},
	"unmount": cmdInfo{
		1, func(d *FlexVolumeDriver, args []string) (map[string]interface{}, error) {
			return d.unmount(args[0])
		},
	},
}

func (d *FlexVolumeDriver) doRun(args []string) (map[string]interface{}, error) {
	if len(args) == 0 {
		return nil, errors.New("no arguments passed to flexvolume driver")
	}
	nArgs := len(args) - 1
	op := args[0]
	if cmdInfo, found := commands[op]; found {
		if cmdInfo.numArgs == nArgs {
			return cmdInfo.run(d, args[1:])
		} else {
			return nil, fmt.Errorf("unexpected number of args %d (expected %d) for operation %q", nArgs, cmdInfo.numArgs, op)
		}
	} else {
		return map[string]interface{}{
			"status": "Not supported",
		}, nil
	}
}

func (d *FlexVolumeDriver) Run(args []string) string {
	r := formatResult(d.doRun(args))

	// Uncomment the following for debugging.
	// TODO: make this configurable somehow.
	// The problem is that kubelet grabs CombinedOutput() from the process
	// and tries to parse it as JSON (need to recheck this,
	// maybe submit a PS to fix it)

	// f, err := os.OpenFile("/tmp/flexvolume.log", os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0666)
	// if err != nil {
	// 	panic(err)
	// }

	// defer f.Close()

	// fmt.Fprintf(f, "flexvolume %s -> %s\n", strings.Join(args, " "), r)

	return r
}

func formatResult(fields map[string]interface{}, err error) string {
	var data map[string]interface{}
	if err != nil {
		data = map[string]interface{}{
			"status":  "Failure",
			"message": err.Error(),
		}
	} else {
		data = map[string]interface{}{
			"status": "Success",
		}
		for k, v := range fields {
			data[k] = v
		}
	}
	s, err := json.Marshal(data)
	if err != nil {
		panic("error marshalling the data")
	}
	return string(s) + "\n"
}
<|endoftext|>


<commit_before>package oauth

import (
	"strings"

	"github.com/hellofresh/janus/pkg/session"
	"github.com/hellofresh/janus/pkg/store"
)

const (
	// Storage enables you to store the tokens in a cache (This way you don't need to validate the token against
	// the auth provider on every request)
	Storage ManagerType = iota
	// JWT provides a way to check the `exp` field on the JWT and make sure the token is still valid. This is
	// probably the most versatile way to check for tokens, since it doesn't require any storage or extra calls in
	// each request.
	JWT
	// Auth strategy makes sure to validade the provided token on every request against the athentication provider.
	Auth
)

// ParseType takes a string type and returns the Manager type constant.
func ParseType(lvl string) (ManagerType, error) {
	switch strings.ToLower(lvl) {
	case "storage":
		return Storage, nil
	case "jwt":
		return JWT, nil
	case "auth":
		return Auth, nil
	}

	var m ManagerType
	return m, ErrUnknownManager
}

// ManagerType type
type ManagerType uint8

// Manager holds the methods to handle tokens
type Manager interface {
	Set(accessToken string, session session.SessionState, resetTTLTo int64) error
	Remove(accessToken string) error
	IsKeyAuthorised(accessToken string) (session.SessionState, bool)
}

type ManagerFactory struct {
	Storage store.Store
	Secret  string
}

func NewManagerFactory(storage store.Store, secret string) *ManagerFactory {
	return &ManagerFactory{storage, secret}
}

func (f *ManagerFactory) Build(t ManagerType) (Manager, error) {
	switch t {
	case Storage:
		return &StorageTokenManager{Storage: f.Storage}, nil
	case JWT:
		if f.Secret == "" {
			return nil, ErrJWTSecretMissing
		}
		return &JWTManager{Secret: f.Secret}, nil
	case Auth:
		// TODO: Create an Auth Manager that always validated tokens against an auth provider
	}

	return nil, ErrUnknownManager
}
<commit_msg>Fixed wrong type of error<commit_after>package oauth

import (
	"strings"

	"github.com/hellofresh/janus/pkg/session"
	"github.com/hellofresh/janus/pkg/store"
)

const (
	// Storage enables you to store the tokens in a cache (This way you don't need to validate the token against
	// the auth provider on every request)
	Storage ManagerType = iota
	// JWT provides a way to check the `exp` field on the JWT and make sure the token is still valid. This is
	// probably the most versatile way to check for tokens, since it doesn't require any storage or extra calls in
	// each request.
	JWT
	// Auth strategy makes sure to validade the provided token on every request against the athentication provider.
	Auth
)

// ParseType takes a string type and returns the Manager type constant.
func ParseType(lvl string) (ManagerType, error) {
	switch strings.ToLower(lvl) {
	case "storage":
		return Storage, nil
	case "jwt":
		return JWT, nil
	case "auth":
		return Auth, nil
	}

	var m ManagerType
	return m, ErrUnknownStrategy
}

// ManagerType type
type ManagerType uint8

// Manager holds the methods to handle tokens
type Manager interface {
	Set(accessToken string, session session.SessionState, resetTTLTo int64) error
	Remove(accessToken string) error
	IsKeyAuthorised(accessToken string) (session.SessionState, bool)
}

type ManagerFactory struct {
	Storage store.Store
	Secret  string
}

func NewManagerFactory(storage store.Store, secret string) *ManagerFactory {
	return &ManagerFactory{storage, secret}
}

func (f *ManagerFactory) Build(t ManagerType) (Manager, error) {
	switch t {
	case Storage:
		return &StorageTokenManager{Storage: f.Storage}, nil
	case JWT:
		if f.Secret == "" {
			return nil, ErrJWTSecretMissing
		}
		return &JWTManager{Secret: f.Secret}, nil
	case Auth:
		// TODO: Create an Auth Manager that always validated tokens against an auth provider
	}

	return nil, ErrUnknownManager
}
<|endoftext|>


<commit_before>/*******************************************************************************
*
* Copyright 2017 SAP SE
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You should have received a copy of the License along with this
* program. If not, you may obtain a copy of the License at
*
*     http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*******************************************************************************/

package plugins

import (
	"math"

	"github.com/gophercloud/gophercloud"
	"github.com/gophercloud/gophercloud/openstack"
	"github.com/gophercloud/gophercloud/openstack/compute/v2/flavors"
	"github.com/gophercloud/gophercloud/pagination"
	"github.com/sapcc/limes/pkg/limes"
	"github.com/sapcc/limes/pkg/util"
)

type capacityNovaPlugin struct {
	cfg limes.CapacitorConfiguration
}

type extraSpecs struct {
	ExtraSpecs map[string]string `json:"extra_specs"`
}

func init() {
	limes.RegisterCapacityPlugin(func(c limes.CapacitorConfiguration) limes.CapacityPlugin {
		return &capacityNovaPlugin{c}
	})
}

func (p *capacityNovaPlugin) Client(driver limes.Driver) (*gophercloud.ServiceClient, error) {
	return openstack.NewComputeV2(driver.Client(),
		gophercloud.EndpointOpts{Availability: gophercloud.AvailabilityPublic},
	)
}

func (p *capacityNovaPlugin) ID() string {
	return "nova"
}

//Scrape implements the limes.CapacityPlugin interface.
func (p *capacityNovaPlugin) Scrape(driver limes.Driver) (map[string]map[string]uint64, error) {
	client, err := p.Client(driver)
	if err != nil {
		return nil, err
	}

	var result gophercloud.Result

	//Get absolute limits for a tenant
	url := client.ServiceURL("os-hypervisors", "statistics")
	_, err = client.Get(url, &result.Body, nil)
	if err != nil {
		return nil, err
	}
	var hypervisorData struct {
		HypervisorStatistics struct {
			Vcpus    int `json:"vcpus"`
			MemoryMb int `json:"memory_mb"`
			LocalGb  int `json:"local_gb"`
		} `json:"hypervisor_statistics"`
	}
	err = result.ExtractInto(&hypervisorData)
	if err != nil {
		return nil, err
	}

	//Get availability zones
	url = client.ServiceURL("os-availability-zone")
	_, err = client.Get(url, &result.Body, nil)
	if err != nil {
		return nil, err
	}
	var availabilityZoneData struct {
		AvailabilityZoneInfo []struct {
			ZoneName  string `json:"zoneName"`
			ZoneState struct {
				Available bool `json:"available"`
			} `json:"zoneState"`
		} `json:"availabilityZoneInfo"`
	}
	err = result.ExtractInto(&availabilityZoneData)
	if err != nil {
		return nil, err
	}

	//list all flavors and get max(flavor_size)
	pages, maxFlavorSize := 0, 0.0
	err = flavors.ListDetail(client, nil).EachPage(func(page pagination.Page) (bool, error) {
		pages++
		f, err := flavors.ExtractFlavors(page)
		if err != nil {
			return false, err
		}
		for _, element := range f {
			extras, err := getFlavorExtras(client, element.ID)
			if err != nil {
				util.LogDebug("Failed to get extra specs for flavor: %s.", element.ID)
				return false, err
			}

			//necessary to be able to ignore huge baremetal flavors
			//consider only flavors as defined in extra specs
			var extraSpecs = map[string]string{}
			if p.cfg.Nova.ExtraSpecs != nil {
				extraSpecs = p.cfg.Nova.ExtraSpecs
			}

			matches := true
			for key, value := range extraSpecs {
				if value != extras.ExtraSpecs[key] {
					matches = false
					break
				}
			}
			if matches {
				maxFlavorSize = math.Max(maxFlavorSize, float64(element.Disk))
			}
		}

		return true, nil
	})
	if err != nil {
		return nil, err
	}

	var azCount int

	//count availability zones
	for _, element := range availabilityZoneData.AvailabilityZoneInfo {
		if element.ZoneState.Available == true {
			azCount++
		}
	}

	//get overcommit factor from configuration (hypervisor stats unfortunately is
	//stupid and does not include this factor even though it is in the nova.conf)
	var vcpuOvercommitFactor uint64 = 1
	if p.cfg.Nova.VCPUOvercommitFactor != nil {
		vcpuOvercommitFactor = *p.cfg.Nova.VCPUOvercommitFactor
	}

	//returns something like
	//"volumev2": {
	//	"cores": total_vcpus,
	//	"instances": min(10000 per Availability Zone, local_gb/max(flavor size)),
	//	"ram": total_memory_mb,
	//}

	capacity := map[string]map[string]uint64{
		"compute": {
			"cores": uint64(hypervisorData.HypervisorStatistics.Vcpus) * vcpuOvercommitFactor,
			"ram":   uint64(hypervisorData.HypervisorStatistics.MemoryMb)},
	}

	if maxFlavorSize != 0 {
		capacity["compute"]["instances"] = uint64(math.Min(float64(10000*azCount), float64(hypervisorData.HypervisorStatistics.LocalGb)/maxFlavorSize))
	} else {
		util.LogError("Nova Capacity: Maximal flavor size is 0. Not reporting instances.")
	}

	return capacity, nil

}

//get flavor extra-specs
//result contains
//{ "vmware:hv_enabled" : 'True' }
//which identifies a VM flavor
func getFlavorExtras(client *gophercloud.ServiceClient, flavorUUID string) (*extraSpecs, error) {
	var result gophercloud.Result
	var extraSpecsData extraSpecs

	url := client.ServiceURL("flavors", flavorUUID, "os-extra_specs")
	_, err := client.Get(url, &result.Body, nil)
	if err != nil {
		return nil, err
	}

	err = result.ExtractInto(&extraSpecsData)
	if err != nil {
		return nil, err
	}

	return &extraSpecsData, nil
}
<commit_msg>add debug log<commit_after>/*******************************************************************************
*
* Copyright 2017 SAP SE
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You should have received a copy of the License along with this
* program. If not, you may obtain a copy of the License at
*
*     http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*******************************************************************************/

package plugins

import (
	"math"

	"github.com/gophercloud/gophercloud"
	"github.com/gophercloud/gophercloud/openstack"
	"github.com/gophercloud/gophercloud/openstack/compute/v2/flavors"
	"github.com/gophercloud/gophercloud/pagination"
	"github.com/sapcc/limes/pkg/limes"
	"github.com/sapcc/limes/pkg/util"
)

type capacityNovaPlugin struct {
	cfg limes.CapacitorConfiguration
}

type extraSpecs struct {
	ExtraSpecs map[string]string `json:"extra_specs"`
}

func init() {
	limes.RegisterCapacityPlugin(func(c limes.CapacitorConfiguration) limes.CapacityPlugin {
		return &capacityNovaPlugin{c}
	})
}

func (p *capacityNovaPlugin) Client(driver limes.Driver) (*gophercloud.ServiceClient, error) {
	return openstack.NewComputeV2(driver.Client(),
		gophercloud.EndpointOpts{Availability: gophercloud.AvailabilityPublic},
	)
}

func (p *capacityNovaPlugin) ID() string {
	return "nova"
}

//Scrape implements the limes.CapacityPlugin interface.
func (p *capacityNovaPlugin) Scrape(driver limes.Driver) (map[string]map[string]uint64, error) {
	client, err := p.Client(driver)
	if err != nil {
		return nil, err
	}

	var result gophercloud.Result

	//Get absolute limits for a tenant
	url := client.ServiceURL("os-hypervisors", "statistics")
	_, err = client.Get(url, &result.Body, nil)
	if err != nil {
		return nil, err
	}
	var hypervisorData struct {
		HypervisorStatistics struct {
			Vcpus    int `json:"vcpus"`
			MemoryMb int `json:"memory_mb"`
			LocalGb  int `json:"local_gb"`
		} `json:"hypervisor_statistics"`
	}
	err = result.ExtractInto(&hypervisorData)
	if err != nil {
		return nil, err
	}

	//Get availability zones
	url = client.ServiceURL("os-availability-zone")
	_, err = client.Get(url, &result.Body, nil)
	if err != nil {
		return nil, err
	}
	var availabilityZoneData struct {
		AvailabilityZoneInfo []struct {
			ZoneName  string `json:"zoneName"`
			ZoneState struct {
				Available bool `json:"available"`
			} `json:"zoneState"`
		} `json:"availabilityZoneInfo"`
	}
	err = result.ExtractInto(&availabilityZoneData)
	if err != nil {
		return nil, err
	}

	//list all flavors and get max(flavor_size)
	pages, maxFlavorSize := 0, 0.0
	err = flavors.ListDetail(client, nil).EachPage(func(page pagination.Page) (bool, error) {
		pages++
		f, err := flavors.ExtractFlavors(page)
		if err != nil {
			return false, err
		}
		for _, element := range f {
			extras, err := getFlavorExtras(client, element.ID)
			if err != nil {
				util.LogDebug("Failed to get extra specs for flavor: %s.", element.ID)
				return false, err
			}

			//necessary to be able to ignore huge baremetal flavors
			//consider only flavors as defined in extra specs
			var extraSpecs = map[string]string{}
			if p.cfg.Nova.ExtraSpecs != nil {
				extraSpecs = p.cfg.Nova.ExtraSpecs
			}

			matches := true
			for key, value := range extraSpecs {
				if value != extras.ExtraSpecs[key] {
					matches = false
					break
				}
			}
			if matches {
				util.LogDebug("FlavorName: %s, FlavorID: %s, FlavorSize: %d GiB", element.Name, element.ID, element.Disk)
				maxFlavorSize = math.Max(maxFlavorSize, float64(element.Disk))
			}
		}

		return true, nil
	})
	if err != nil {
		return nil, err
	}

	var azCount int

	//count availability zones
	for _, element := range availabilityZoneData.AvailabilityZoneInfo {
		if element.ZoneState.Available == true {
			azCount++
		}
	}

	//get overcommit factor from configuration (hypervisor stats unfortunately is
	//stupid and does not include this factor even though it is in the nova.conf)
	var vcpuOvercommitFactor uint64 = 1
	if p.cfg.Nova.VCPUOvercommitFactor != nil {
		vcpuOvercommitFactor = *p.cfg.Nova.VCPUOvercommitFactor
	}

	//returns something like
	//"volumev2": {
	//	"cores": total_vcpus,
	//	"instances": min(10000 per Availability Zone, local_gb/max(flavor size)),
	//	"ram": total_memory_mb,
	//}

	capacity := map[string]map[string]uint64{
		"compute": {
			"cores": uint64(hypervisorData.HypervisorStatistics.Vcpus) * vcpuOvercommitFactor,
			"ram":   uint64(hypervisorData.HypervisorStatistics.MemoryMb)},
	}

	if maxFlavorSize != 0 {
		capacity["compute"]["instances"] = uint64(math.Min(float64(10000*azCount), float64(hypervisorData.HypervisorStatistics.LocalGb)/maxFlavorSize))
	} else {
		util.LogError("Nova Capacity: Maximal flavor size is 0. Not reporting instances.")
	}

	return capacity, nil

}

//get flavor extra-specs
//result contains
//{ "vmware:hv_enabled" : 'True' }
//which identifies a VM flavor
func getFlavorExtras(client *gophercloud.ServiceClient, flavorUUID string) (*extraSpecs, error) {
	var result gophercloud.Result
	var extraSpecsData extraSpecs

	url := client.ServiceURL("flavors", flavorUUID, "os-extra_specs")
	_, err := client.Get(url, &result.Body, nil)
	if err != nil {
		return nil, err
	}

	err = result.ExtractInto(&extraSpecsData)
	if err != nil {
		return nil, err
	}

	return &extraSpecsData, nil
}
<|endoftext|>


<commit_before>// Copyright 2018-2020 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package groups

import (
	"context"
	"fmt"

	"github.com/cilium/cilium/pkg/k8s"
	cilium_v2 "github.com/cilium/cilium/pkg/k8s/apis/cilium.io/v2"
	slimv1 "github.com/cilium/cilium/pkg/k8s/slim/k8s/apis/meta/v1"
	"github.com/cilium/cilium/pkg/policy/api"

	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

const (
	cnpKindName = "derivative"
	parentCNP   = "io.cilium.network.policy.parent.uuid"
	cnpKindKey  = "io.cilium.network.policy.kind"
)

var (
	blockOwnerDeletionPtr = true
)

func getDerivativeName(obj v1.Object) string {
	return fmt.Sprintf("%s-togroups-%s",
		obj.GetName(),
		obj.GetUID())
}

// createDerivativeCNP will return a new CNP based on the given rule.
func createDerivativeCNP(ctx context.Context, cnp *cilium_v2.CiliumNetworkPolicy) (*cilium_v2.CiliumNetworkPolicy, error) {
	// CNP informer may provide a CNP object without APIVersion or Kind.
	// Setting manually to make sure that the derivative policy works ok.
	derivativeCNP := &cilium_v2.CiliumNetworkPolicy{
		ObjectMeta: v1.ObjectMeta{
			Name:      getDerivativeName(cnp),
			Namespace: cnp.ObjectMeta.Namespace,
			OwnerReferences: []v1.OwnerReference{{
				APIVersion:         cilium_v2.SchemeGroupVersion.String(),
				Kind:               cilium_v2.CNPKindDefinition,
				Name:               cnp.ObjectMeta.Name,
				UID:                cnp.ObjectMeta.UID,
				BlockOwnerDeletion: &blockOwnerDeletionPtr,
			}},
			Labels: map[string]string{
				parentCNP:  string(cnp.ObjectMeta.UID),
				cnpKindKey: cnpKindName,
			},
		},
	}

	var (
		rules api.Rules
		err   error
	)

	rules, err = cnp.Parse()

	if err != nil {
		// We return a valid pointer for derivative policy here instead of nil.
		// This object is used to get generated name for the derivative policy
		// when updating the status of the network policy.
		return derivativeCNP, fmt.Errorf("cannot parse CNP: %v", err)
	}

	derivativeCNP.Specs = make(api.Rules, len(rules))
	for i, rule := range rules {
		if rule.RequiresDerivative() {
			derivativeCNP.Specs[i] = denyEgressRule()
		}
	}

	for i, rule := range rules {
		if !rule.RequiresDerivative() {
			derivativeCNP.Specs[i] = rule
			continue
		}
		newRule, err := rule.CreateDerivative(ctx)
		if err != nil {
			return derivativeCNP, err
		}
		derivativeCNP.Specs[i] = newRule
	}
	return derivativeCNP, nil
}

// createDerivativeCCNP will return a new CCNP based on the given rule.
func createDerivativeCCNP(ctx context.Context, cnp *cilium_v2.CiliumNetworkPolicy) (*cilium_v2.CiliumClusterwideNetworkPolicy, error) {
	ccnp := &cilium_v2.CiliumClusterwideNetworkPolicy{
		TypeMeta:   cnp.TypeMeta,
		ObjectMeta: cnp.ObjectMeta,
		Spec:       cnp.Spec,
		Specs:      cnp.Specs,
		Status:     cnp.Status,
	}

	// CCNP informer may provide a CCNP object without APIVersion or Kind.
	// Setting manually to make sure that the derivative policy works ok.
	derivativeCCNP := &cilium_v2.CiliumClusterwideNetworkPolicy{
		ObjectMeta: v1.ObjectMeta{
			Name:      getDerivativeName(ccnp),
			Namespace: ccnp.ObjectMeta.Namespace,
			OwnerReferences: []v1.OwnerReference{{
				APIVersion:         cilium_v2.SchemeGroupVersion.String(),
				Kind:               cilium_v2.CCNPKindDefinition,
				Name:               ccnp.ObjectMeta.Name,
				UID:                ccnp.ObjectMeta.UID,
				BlockOwnerDeletion: &blockOwnerDeletionPtr,
			}},
			Labels: map[string]string{
				parentCNP:  string(ccnp.ObjectMeta.UID),
				cnpKindKey: cnpKindName,
			},
		},
	}

	var (
		rules api.Rules
		err   error
	)

	rules, err = ccnp.Parse()

	if err != nil {
		// We return a valid pointer for derivative policy here instead of nil.
		// This object is used to get generated name for the derivative policy
		// when updating the status of the network policy.
		return derivativeCCNP, fmt.Errorf("cannot parse CCNP: %v", err)
	}

	derivativeCCNP.Specs = make(api.Rules, len(rules))
	for i, rule := range rules {
		if rule.RequiresDerivative() {
			derivativeCCNP.Specs[i] = denyEgressRule()
		}
	}

	for i, rule := range rules {
		if !rule.RequiresDerivative() {
			derivativeCCNP.Specs[i] = rule
			continue
		}
		newRule, err := rule.CreateDerivative(ctx)
		if err != nil {
			return derivativeCCNP, err
		}
		derivativeCCNP.Specs[i] = newRule
	}
	return derivativeCCNP, nil
}

func denyEgressRule() *api.Rule {
	return &api.Rule{
		Egress: []api.EgressRule{},
	}
}

func updateOrCreateCNP(cnp *cilium_v2.CiliumNetworkPolicy) (*cilium_v2.CiliumNetworkPolicy, error) {
	k8sCNP, err := k8s.CiliumClient().CiliumV2().CiliumNetworkPolicies(cnp.ObjectMeta.Namespace).
		Get(context.TODO(), cnp.ObjectMeta.Name, v1.GetOptions{})
	if err == nil {
		k8sCNP.ObjectMeta.Labels = cnp.ObjectMeta.Labels
		k8sCNP.Spec = cnp.Spec
		k8sCNP.Specs = cnp.Specs
		k8sCNP.Status = cilium_v2.CiliumNetworkPolicyStatus{}
		return k8s.CiliumClient().CiliumV2().CiliumNetworkPolicies(cnp.ObjectMeta.Namespace).Update(context.TODO(), k8sCNP, v1.UpdateOptions{})
	}
	return k8s.CiliumClient().CiliumV2().CiliumNetworkPolicies(cnp.ObjectMeta.Namespace).Create(context.TODO(), cnp, v1.CreateOptions{})
}

func updateOrCreateCCNP(ccnp *cilium_v2.CiliumClusterwideNetworkPolicy) (*cilium_v2.CiliumClusterwideNetworkPolicy, error) {
	k8sCCNP, err := k8s.CiliumClient().CiliumV2().CiliumClusterwideNetworkPolicies().
		Get(context.TODO(), ccnp.ObjectMeta.Name, v1.GetOptions{})
	if err == nil {
		k8sCCNP.ObjectMeta.Labels = ccnp.ObjectMeta.Labels
		k8sCCNP.Spec = ccnp.Spec
		k8sCCNP.Specs = ccnp.Specs
		k8sCCNP.Status = cilium_v2.CiliumNetworkPolicyStatus{}

		return k8s.CiliumClient().CiliumV2().CiliumClusterwideNetworkPolicies().Update(context.TODO(), k8sCCNP, v1.UpdateOptions{})
	}

	return k8s.CiliumClient().CiliumV2().CiliumClusterwideNetworkPolicies().
		Create(context.TODO(), ccnp, v1.CreateOptions{})
}

func updateDerivativeStatus(cnp *cilium_v2.CiliumNetworkPolicy, derivativeName string, err error, clusterScoped bool) error {
	status := cilium_v2.CiliumNetworkPolicyNodeStatus{
		LastUpdated: slimv1.Now(),
		Enforcing:   false,
	}

	if err != nil {
		status.OK = false
		status.Error = err.Error()
	} else {
		status.OK = true
	}

	if clusterScoped {
		return updateDerivativeCCNPStatus(cnp, status, derivativeName)
	}

	return updateDerivativeCNPStatus(cnp, status, derivativeName)
}

func updateDerivativeCNPStatus(cnp *cilium_v2.CiliumNetworkPolicy, status cilium_v2.CiliumNetworkPolicyNodeStatus,
	derivativeName string) error {
	// This CNP can be modified by cilium agent or operator. To be able to push
	// the status correctly fetch the last version to avoid updates issues.
	k8sCNP, clientErr := k8s.CiliumClient().CiliumV2().CiliumNetworkPolicies(cnp.ObjectMeta.Namespace).
		Get(context.TODO(), cnp.ObjectMeta.Name, v1.GetOptions{})

	if clientErr != nil {
		return fmt.Errorf("cannot get Kubernetes policy: %v", clientErr)
	}

	if k8sCNP.ObjectMeta.UID != cnp.ObjectMeta.UID {
		// This case should not happen, but if the UID does not match make sure
		// that the new policy is not in the cache to not loop over it. The
		// kubernetes watcher should take care about that.
		groupsCNPCache.DeleteCNP(k8sCNP)
		return fmt.Errorf("policy UID mistmatch")
	}

	k8sCNP.SetDerivedPolicyStatus(derivativeName, status)
	groupsCNPCache.UpdateCNP(k8sCNP)

	// TODO: Switch to JSON patch.
	_, err := k8s.CiliumClient().CiliumV2().CiliumNetworkPolicies(cnp.ObjectMeta.Namespace).
		UpdateStatus(context.TODO(), k8sCNP, v1.UpdateOptions{})

	return err
}

func updateDerivativeCCNPStatus(cnp *cilium_v2.CiliumNetworkPolicy, status cilium_v2.CiliumNetworkPolicyNodeStatus,
	derivativeName string) error {
	k8sCCNP, clientErr := k8s.CiliumClient().CiliumV2().CiliumClusterwideNetworkPolicies().
		Get(context.TODO(), cnp.ObjectMeta.Name, v1.GetOptions{})

	if clientErr != nil {
		return fmt.Errorf("cannot get Kubernetes policy: %v", clientErr)
	}

	if k8sCCNP.ObjectMeta.UID != cnp.ObjectMeta.UID {
		// This case should not happen, but if the UID does not match make sure
		// that the new policy is not in the cache to not loop over it. The
		// kubernetes watcher should take care of that.
		groupsCNPCache.DeleteCNP(&cilium_v2.CiliumNetworkPolicy{
			ObjectMeta: k8sCCNP.ObjectMeta,
		})
		return fmt.Errorf("policy UID mistmatch")
	}

	k8sCCNP.SetDerivedPolicyStatus(derivativeName, status)
	groupsCNPCache.UpdateCNP(&cilium_v2.CiliumNetworkPolicy{
		TypeMeta:   k8sCCNP.TypeMeta,
		ObjectMeta: k8sCCNP.ObjectMeta,
		Spec:       k8sCCNP.Spec,
		Specs:      k8sCCNP.Specs,
		Status:     k8sCCNP.Status,
	})

	// TODO: Switch to JSON patch
	_, err := k8s.CiliumClient().CiliumV2().CiliumClusterwideNetworkPolicies().
		UpdateStatus(context.TODO(), k8sCCNP, v1.UpdateOptions{})

	return err

}
<commit_msg>policy/groups:refactor api rules modification in createDerivative{CNP,CCNP}<commit_after>// Copyright 2018-2020 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package groups

import (
	"context"
	"fmt"

	"github.com/cilium/cilium/pkg/k8s"
	cilium_v2 "github.com/cilium/cilium/pkg/k8s/apis/cilium.io/v2"
	slimv1 "github.com/cilium/cilium/pkg/k8s/slim/k8s/apis/meta/v1"
	"github.com/cilium/cilium/pkg/policy/api"

	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

const (
	cnpKindName = "derivative"
	parentCNP   = "io.cilium.network.policy.parent.uuid"
	cnpKindKey  = "io.cilium.network.policy.kind"
)

var (
	blockOwnerDeletionPtr = true
)

func getDerivativeName(obj v1.Object) string {
	return fmt.Sprintf("%s-togroups-%s",
		obj.GetName(),
		obj.GetUID())
}

// createDerivativeCNP will return a new CNP based on the given rule.
func createDerivativeCNP(ctx context.Context, cnp *cilium_v2.CiliumNetworkPolicy) (*cilium_v2.CiliumNetworkPolicy, error) {
	// CNP informer may provide a CNP object without APIVersion or Kind.
	// Setting manually to make sure that the derivative policy works ok.
	derivativeCNP := &cilium_v2.CiliumNetworkPolicy{
		ObjectMeta: v1.ObjectMeta{
			Name:      getDerivativeName(cnp),
			Namespace: cnp.ObjectMeta.Namespace,
			OwnerReferences: []v1.OwnerReference{{
				APIVersion:         cilium_v2.SchemeGroupVersion.String(),
				Kind:               cilium_v2.CNPKindDefinition,
				Name:               cnp.ObjectMeta.Name,
				UID:                cnp.ObjectMeta.UID,
				BlockOwnerDeletion: &blockOwnerDeletionPtr,
			}},
			Labels: map[string]string{
				parentCNP:  string(cnp.ObjectMeta.UID),
				cnpKindKey: cnpKindName,
			},
		},
	}

	var (
		rules api.Rules
		err   error
	)

	rules, err = cnp.Parse()

	if err != nil {
		// We return a valid pointer for derivative policy here instead of nil.
		// This object is used to get generated name for the derivative policy
		// when updating the status of the network policy.
		return derivativeCNP, fmt.Errorf("cannot parse CNP: %v", err)
	}

	derivativeCNP.Specs, err = createAPIRules(ctx, rules)

	return derivativeCNP, err
}

// createDerivativeCCNP will return a new CCNP based on the given rule.
func createDerivativeCCNP(ctx context.Context, cnp *cilium_v2.CiliumNetworkPolicy) (*cilium_v2.CiliumClusterwideNetworkPolicy, error) {
	ccnp := &cilium_v2.CiliumClusterwideNetworkPolicy{
		TypeMeta:   cnp.TypeMeta,
		ObjectMeta: cnp.ObjectMeta,
		Spec:       cnp.Spec,
		Specs:      cnp.Specs,
		Status:     cnp.Status,
	}

	// CCNP informer may provide a CCNP object without APIVersion or Kind.
	// Setting manually to make sure that the derivative policy works ok.
	derivativeCCNP := &cilium_v2.CiliumClusterwideNetworkPolicy{
		ObjectMeta: v1.ObjectMeta{
			Name:      getDerivativeName(ccnp),
			Namespace: ccnp.ObjectMeta.Namespace,
			OwnerReferences: []v1.OwnerReference{{
				APIVersion:         cilium_v2.SchemeGroupVersion.String(),
				Kind:               cilium_v2.CCNPKindDefinition,
				Name:               ccnp.ObjectMeta.Name,
				UID:                ccnp.ObjectMeta.UID,
				BlockOwnerDeletion: &blockOwnerDeletionPtr,
			}},
			Labels: map[string]string{
				parentCNP:  string(ccnp.ObjectMeta.UID),
				cnpKindKey: cnpKindName,
			},
		},
	}

	var (
		rules api.Rules
		err   error
	)

	rules, err = ccnp.Parse()

	if err != nil {
		// We return a valid pointer for derivative policy here instead of nil.
		// This object is used to get generated name for the derivative policy
		// when updating the status of the network policy.
		return derivativeCCNP, fmt.Errorf("cannot parse CCNP: %v", err)
	}

	derivativeCCNP.Specs, err = createAPIRules(ctx, rules)

	return derivativeCCNP, err
}

func createAPIRules(ctx context.Context, rules api.Rules) (api.Rules, error) {
	specRules := make(api.Rules, len(rules))
	for i, rule := range rules {
		if rule.RequiresDerivative() {
			specRules[i] = denyEgressRule()
		}
	}

	for i, rule := range rules {
		if !rule.RequiresDerivative() {
			specRules[i] = rule
			continue
		}
		newRule, err := rule.CreateDerivative(ctx)
		if err != nil {
			return specRules, err
		}
		specRules[i] = newRule
	}
	return specRules, nil
}

func denyEgressRule() *api.Rule {
	return &api.Rule{
		Egress: []api.EgressRule{},
	}
}

func updateOrCreateCNP(cnp *cilium_v2.CiliumNetworkPolicy) (*cilium_v2.CiliumNetworkPolicy, error) {
	k8sCNP, err := k8s.CiliumClient().CiliumV2().CiliumNetworkPolicies(cnp.ObjectMeta.Namespace).
		Get(context.TODO(), cnp.ObjectMeta.Name, v1.GetOptions{})
	if err == nil {
		k8sCNP.ObjectMeta.Labels = cnp.ObjectMeta.Labels
		k8sCNP.Spec = cnp.Spec
		k8sCNP.Specs = cnp.Specs
		k8sCNP.Status = cilium_v2.CiliumNetworkPolicyStatus{}
		return k8s.CiliumClient().CiliumV2().CiliumNetworkPolicies(cnp.ObjectMeta.Namespace).Update(context.TODO(), k8sCNP, v1.UpdateOptions{})
	}
	return k8s.CiliumClient().CiliumV2().CiliumNetworkPolicies(cnp.ObjectMeta.Namespace).Create(context.TODO(), cnp, v1.CreateOptions{})
}

func updateOrCreateCCNP(ccnp *cilium_v2.CiliumClusterwideNetworkPolicy) (*cilium_v2.CiliumClusterwideNetworkPolicy, error) {
	k8sCCNP, err := k8s.CiliumClient().CiliumV2().CiliumClusterwideNetworkPolicies().
		Get(context.TODO(), ccnp.ObjectMeta.Name, v1.GetOptions{})
	if err == nil {
		k8sCCNP.ObjectMeta.Labels = ccnp.ObjectMeta.Labels
		k8sCCNP.Spec = ccnp.Spec
		k8sCCNP.Specs = ccnp.Specs
		k8sCCNP.Status = cilium_v2.CiliumNetworkPolicyStatus{}

		return k8s.CiliumClient().CiliumV2().CiliumClusterwideNetworkPolicies().Update(context.TODO(), k8sCCNP, v1.UpdateOptions{})
	}

	return k8s.CiliumClient().CiliumV2().CiliumClusterwideNetworkPolicies().
		Create(context.TODO(), ccnp, v1.CreateOptions{})
}

func updateDerivativeStatus(cnp *cilium_v2.CiliumNetworkPolicy, derivativeName string, err error, clusterScoped bool) error {
	status := cilium_v2.CiliumNetworkPolicyNodeStatus{
		LastUpdated: slimv1.Now(),
		Enforcing:   false,
	}

	if err != nil {
		status.OK = false
		status.Error = err.Error()
	} else {
		status.OK = true
	}

	if clusterScoped {
		return updateDerivativeCCNPStatus(cnp, status, derivativeName)
	}

	return updateDerivativeCNPStatus(cnp, status, derivativeName)
}

func updateDerivativeCNPStatus(cnp *cilium_v2.CiliumNetworkPolicy, status cilium_v2.CiliumNetworkPolicyNodeStatus,
	derivativeName string) error {
	// This CNP can be modified by cilium agent or operator. To be able to push
	// the status correctly fetch the last version to avoid updates issues.
	k8sCNP, clientErr := k8s.CiliumClient().CiliumV2().CiliumNetworkPolicies(cnp.ObjectMeta.Namespace).
		Get(context.TODO(), cnp.ObjectMeta.Name, v1.GetOptions{})

	if clientErr != nil {
		return fmt.Errorf("cannot get Kubernetes policy: %v", clientErr)
	}

	if k8sCNP.ObjectMeta.UID != cnp.ObjectMeta.UID {
		// This case should not happen, but if the UID does not match make sure
		// that the new policy is not in the cache to not loop over it. The
		// kubernetes watcher should take care about that.
		groupsCNPCache.DeleteCNP(k8sCNP)
		return fmt.Errorf("policy UID mistmatch")
	}

	k8sCNP.SetDerivedPolicyStatus(derivativeName, status)
	groupsCNPCache.UpdateCNP(k8sCNP)

	// TODO: Switch to JSON patch.
	_, err := k8s.CiliumClient().CiliumV2().CiliumNetworkPolicies(cnp.ObjectMeta.Namespace).
		UpdateStatus(context.TODO(), k8sCNP, v1.UpdateOptions{})

	return err
}

func updateDerivativeCCNPStatus(cnp *cilium_v2.CiliumNetworkPolicy, status cilium_v2.CiliumNetworkPolicyNodeStatus,
	derivativeName string) error {
	k8sCCNP, clientErr := k8s.CiliumClient().CiliumV2().CiliumClusterwideNetworkPolicies().
		Get(context.TODO(), cnp.ObjectMeta.Name, v1.GetOptions{})

	if clientErr != nil {
		return fmt.Errorf("cannot get Kubernetes policy: %v", clientErr)
	}

	if k8sCCNP.ObjectMeta.UID != cnp.ObjectMeta.UID {
		// This case should not happen, but if the UID does not match make sure
		// that the new policy is not in the cache to not loop over it. The
		// kubernetes watcher should take care of that.
		groupsCNPCache.DeleteCNP(&cilium_v2.CiliumNetworkPolicy{
			ObjectMeta: k8sCCNP.ObjectMeta,
		})
		return fmt.Errorf("policy UID mistmatch")
	}

	k8sCCNP.SetDerivedPolicyStatus(derivativeName, status)
	groupsCNPCache.UpdateCNP(&cilium_v2.CiliumNetworkPolicy{
		TypeMeta:   k8sCCNP.TypeMeta,
		ObjectMeta: k8sCCNP.ObjectMeta,
		Spec:       k8sCCNP.Spec,
		Specs:      k8sCCNP.Specs,
		Status:     k8sCCNP.Status,
	})

	// TODO: Switch to JSON patch
	_, err := k8s.CiliumClient().CiliumV2().CiliumClusterwideNetworkPolicies().
		UpdateStatus(context.TODO(), k8sCCNP, v1.UpdateOptions{})

	return err

}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package routes

import (
	restful "github.com/emicklei/go-restful"
	"k8s.io/klog"

	"k8s.io/apiserver/pkg/server/mux"
	"k8s.io/kube-openapi/pkg/common"
	"k8s.io/kube-openapi/pkg/handler"
)

// OpenAPI installs spec endpoints for each web service.
type OpenAPI struct {
	Config *common.Config
}

// Install adds the SwaggerUI webservice to the given mux.
func (oa OpenAPI) Install(c *restful.Container, mux *mux.PathRecorderMux) {
	// NOTE: [DEPRECATION] We will announce deprecation for format-separated endpoints for OpenAPI spec,
	// and switch to a single /openapi/v2 endpoint in Kubernetes 1.10. The design doc and deprecation process
	// are tracked at: https://docs.google.com/document/d/19lEqE9lc4yHJ3WJAJxS_G7TcORIJXGHyq3wpwcH28nU.
	_, err := handler.BuildAndRegisterOpenAPIService("/swagger.json", c.RegisteredWebServices(), oa.Config, mux)
	if err != nil {
		klog.Fatalf("Failed to register open api spec for root: %v", err)
	}
	_, err = handler.BuildAndRegisterOpenAPIVersionedService("/openapi/v2", c.RegisteredWebServices(), oa.Config, mux)
	if err != nil {
		klog.Fatalf("Failed to register versioned open api spec for root: %v", err)
	}
}
<commit_msg>remove deprecated openapi paths in favor of /openapi/v2<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package routes

import (
	restful "github.com/emicklei/go-restful"
	"k8s.io/klog"

	"k8s.io/apiserver/pkg/server/mux"
	"k8s.io/kube-openapi/pkg/common"
	"k8s.io/kube-openapi/pkg/handler"
)

// OpenAPI installs spec endpoints for each web service.
type OpenAPI struct {
	Config *common.Config
}

// Install adds the SwaggerUI webservice to the given mux.
func (oa OpenAPI) Install(c *restful.Container, mux *mux.PathRecorderMux) {
	_, err := handler.BuildAndRegisterOpenAPIVersionedService("/openapi/v2", c.RegisteredWebServices(), oa.Config, mux)
	if err != nil {
		klog.Fatalf("Failed to register versioned open api spec for root: %v", err)
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2018 Google LLC

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package docker

import (
	"context"
	"encoding/json"
	"fmt"
	"io"
	"os"
	"path"
	"path/filepath"
	"sort"
	"strings"
	"sync"

	"github.com/GoogleCloudPlatform/skaffold/pkg/skaffold/config"
	"github.com/GoogleCloudPlatform/skaffold/pkg/skaffold/constants"
	"github.com/GoogleCloudPlatform/skaffold/pkg/skaffold/util"
	"github.com/containers/image/docker"
	"github.com/containers/image/manifest"
	"github.com/containers/image/types"
	"github.com/docker/docker/builder/dockerignore"
	"github.com/docker/docker/pkg/fileutils"
	"github.com/moby/moby/builder/dockerfile/parser"
	"github.com/moby/moby/builder/dockerfile/shell"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
)

const (
	add    = "add"
	copy   = "copy"
	env    = "env"
	from   = "from"
	expose = "expose"
)

// For testing.
var RetrieveImage = retrieveImage

type DockerfileDepResolver struct{}

var DefaultDockerfileDepResolver = &DockerfileDepResolver{}

func (*DockerfileDepResolver) GetDependencies(a *config.Artifact) ([]string, error) {
	dockerfilePath := a.DockerfilePath
	if a.DockerfilePath == "" {
		dockerfilePath = constants.DefaultDockerfilePath
	}
	dockerfileAbsPath, err := filepath.Abs(filepath.Join(a.Workspace, dockerfilePath))
	if err != nil {
		return nil, errors.Wrap(err, "getting absolute path of dockerfile")
	}
	f, err := util.Fs.Open(a.DockerfilePath)
	if err != nil {
		return nil, errors.Wrap(err, "opening dockerfile")
	}
	defer f.Close()
	deps, err := GetDockerfileDependencies(a.Workspace, f)
	if err != nil {
		return nil, errors.Wrap(err, "getting dockerfile dependencies")
	}
	deps = append(deps, dockerfileAbsPath)
	return deps, nil
}

// GetDockerfileDependencies parses a dockerfile and returns the full paths
// of all the source files that the resulting docker image depends on.
func GetDockerfileDependencies(workspace string, r io.Reader) ([]string, error) {
	res, err := parser.Parse(r)
	if err != nil {
		return nil, errors.Wrap(err, "parsing dockerfile")
	}
	envs := map[string]string{}
	depMap := map[string]struct{}{}
	// First process onbuilds, if present.
	onbuilds := []string{}
	for _, value := range res.AST.Children {
		switch value.Value {
		case from:
			onbuilds, err = processBaseImage(value)
			if err != nil {
				logrus.Warnf("Error processing base image for onbuild triggers: %s. Dependencies may be incomplete.", err)
			}
		}
	}

	var dispatchInstructions = func(r *parser.Result) {
		for _, value := range r.AST.Children {
			switch value.Value {
			case add, copy:
				processCopy(workspace, value, depMap, envs)
			case env:
				envs[value.Next.Value] = value.Next.Next.Value
			}
		}
	}
	for _, ob := range onbuilds {
		obRes, err := parser.Parse(strings.NewReader(ob))
		if err != nil {
			return nil, err
		}
		dispatchInstructions(obRes)
	}

	dispatchInstructions(res)

	deps := []string{}
	for dep := range depMap {
		deps = append(deps, dep)
	}
	logrus.Infof("Found dependencies for dockerfile %s", deps)
	expandedDeps, err := util.ExpandPaths(workspace, deps)
	if err != nil {
		return nil, errors.Wrap(err, "expanding dockerfile paths")
	}

	logrus.Infof("deps %s", expandedDeps)

	// Look for .dockerignore.
	ignorePath := filepath.Join(workspace, ".dockerignore")
	filteredDeps, err := ApplyDockerIgnore(expandedDeps, ignorePath)
	if err != nil {
		return nil, errors.Wrap(err, "applying dockerignore")
	}

	return filteredDeps, nil
}

func PortsFromDockerfile(r io.Reader) ([]string, error) {
	res, err := parser.Parse(r)
	if err != nil {
		return nil, errors.Wrap(err, "parsing dockerfile")
	}

	// Check the dockerfile and the base.
	ports := []string{}
	for _, value := range res.AST.Children {
		switch value.Value {
		case from:
			base := value.Next.Value
			if strings.ToLower(base) == "scratch" {
				logrus.Debug("Skipping port check in SCRATCH base image.")
				continue
			}
			img, err := RetrieveImage(value.Next.Value)
			if err != nil {
				logrus.Warnf("Error checking base image for ports: %s", err)
				continue
			}
			for port := range img.Config.ExposedPorts {
				logrus.Debugf("Found port %s in base image", port)
				ports = append(ports, string(port))
			}
		case expose:
			// There can be multiple ports per line.
			for {
				if value.Next == nil {
					break
				}
				port := value.Next.Value
				logrus.Debugf("Found port %s in Dockerfile", port)
				ports = append(ports, port)
				value = value.Next
			}
		}
	}
	// Sort ports for consistency in tests.
	sort.Strings(ports)
	return ports, nil
}

func processBaseImage(value *parser.Node) ([]string, error) {
	base := value.Next.Value
	logrus.Debugf("Checking base image %s for ONBUILD triggers.", base)
	if strings.ToLower(base) == "scratch" {
		logrus.Debugf("SCRATCH base image found, skipping check: %s", base)
		return nil, nil
	}
	img, err := RetrieveImage(base)
	if err != nil {
		return nil, err
	}
	logrus.Debugf("Found onbuild triggers %v in image %s", img.Config.OnBuild, base)
	return img.Config.OnBuild, nil
}

var imageCache sync.Map

func retrieveImage(image string) (*manifest.Schema2Image, error) {
	cachedCfg, present := imageCache.Load(image)
	if present {
		return cachedCfg.(*manifest.Schema2Image), nil
	}

	client, err := NewDockerAPIClient()
	if err != nil {
		return nil, err
	}

	raw, err := retrieveLocalImage(client, image)
	if err != nil {
		raw, err = retrieveRemoteImage(image)
		if err != nil {
			return nil, err
		}
	}

	cfg := &manifest.Schema2Image{}
	if err := json.Unmarshal(raw, cfg); err != nil {
		return nil, err
	}

	imageCache.Store(image, cfg)

	return cfg, nil
}

func retrieveLocalImage(client DockerAPIClient, image string) ([]byte, error) {
	_, raw, err := client.ImageInspectWithRaw(context.Background(), image)
	if err != nil {
		return nil, err
	}

	return raw, nil
}

func retrieveRemoteImage(image string) ([]byte, error) {
	context := &types.SystemContext{
		OSChoice:           "linux",
		ArchitectureChoice: "amd64",
	}

	ref, err := docker.ParseReference("//" + image)
	if err != nil {
		return nil, err
	}

	img, err := ref.NewImage(context)
	if err != nil {
		return nil, err
	}
	defer img.Close()

	return img.ConfigBlob()
}

func processCopy(workspace string, value *parser.Node, paths map[string]struct{}, envs map[string]string) error {
	slex := shell.NewLex('\\')
	for {
		// Skip last node, since it is the destination, and stop if we arrive at a comment
		if value.Next.Next == nil || strings.HasPrefix(value.Next.Next.Value, "#") {
			break
		}
		src, err := processShellWord(slex, value.Next.Value, envs)
		if err != nil {
			return errors.Wrap(err, "processing word")
		}
		// If the --from flag is provided, we are dealing with a multi-stage dockerfile
		// Adding a dependency from a different stage does not imply a source dependency
		if hasMultiStageFlag(value.Flags) {
			return nil
		}
		if !strings.HasPrefix(src, "http://") && !strings.HasPrefix(src, "https://") {
			dep := path.Join(workspace, src)
			paths[dep] = struct{}{}
		} else {
			logrus.Debugf("Skipping watch on remote dependency %s", src)
		}

		value = value.Next
	}
	return nil
}

func processShellWord(lex *shell.Lex, word string, envs map[string]string) (string, error) {
	envSlice := []string{}
	for envKey, envVal := range envs {
		envSlice = append(envSlice, fmt.Sprintf("%s=%s", envKey, envVal))
	}
	return lex.ProcessWord(word, envSlice)
}

func hasMultiStageFlag(flags []string) bool {
	for _, f := range flags {
		if strings.HasPrefix(f, "--from=") {
			return true
		}
	}
	return false
}

func ApplyDockerIgnore(paths []string, dockerIgnorePath string) ([]string, error) {
	absPaths, err := util.RelPathToAbsPath(paths)
	if err != nil {
		return nil, errors.Wrap(err, "getting absolute path of dependencies")
	}
	excludes := []string{}
	if _, err := util.Fs.Stat(dockerIgnorePath); !os.IsNotExist(err) {
		r, err := util.Fs.Open(dockerIgnorePath)
		defer r.Close()
		if err != nil {
			return nil, err
		}
		excludes, err = dockerignore.ReadAll(r)
		if err != nil {
			return nil, err
		}
		excludes = append(excludes, ".dockerignore")
	}

	absPathExcludes, err := util.RelPathToAbsPath(excludes)
	if err != nil {
		return nil, errors.Wrap(err, "getting absolute path of docker ignored paths")
	}

	filteredDeps := []string{}
	for _, d := range absPaths {
		m, err := fileutils.Matches(d, absPathExcludes)
		if err != nil {
			return nil, err
		}
		if !m {
			filteredDeps = append(filteredDeps, d)
		}
	}
	sort.Strings(filteredDeps)
	return filteredDeps, nil
}
<commit_msg>Fix opening default Dockerfile<commit_after>/*
Copyright 2018 Google LLC

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package docker

import (
	"context"
	"encoding/json"
	"fmt"
	"io"
	"os"
	"path"
	"path/filepath"
	"sort"
	"strings"
	"sync"

	"github.com/GoogleCloudPlatform/skaffold/pkg/skaffold/config"
	"github.com/GoogleCloudPlatform/skaffold/pkg/skaffold/constants"
	"github.com/GoogleCloudPlatform/skaffold/pkg/skaffold/util"
	"github.com/containers/image/docker"
	"github.com/containers/image/manifest"
	"github.com/containers/image/types"
	"github.com/docker/docker/builder/dockerignore"
	"github.com/docker/docker/pkg/fileutils"
	"github.com/moby/moby/builder/dockerfile/parser"
	"github.com/moby/moby/builder/dockerfile/shell"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
)

const (
	add    = "add"
	copy   = "copy"
	env    = "env"
	from   = "from"
	expose = "expose"
)

// For testing.
var RetrieveImage = retrieveImage

type DockerfileDepResolver struct{}

var DefaultDockerfileDepResolver = &DockerfileDepResolver{}

func (*DockerfileDepResolver) GetDependencies(a *config.Artifact) ([]string, error) {
	dockerfilePath := a.DockerfilePath
	if a.DockerfilePath == "" {
		dockerfilePath = constants.DefaultDockerfilePath
	}
	dockerfileAbsPath, err := filepath.Abs(filepath.Join(a.Workspace, dockerfilePath))
	if err != nil {
		return nil, errors.Wrap(err, "getting absolute path of dockerfile")
	}
	f, err := util.Fs.Open(dockerfileAbsPath)
	if err != nil {
		return nil, errors.Wrap(err, "opening dockerfile")
	}
	defer f.Close()
	deps, err := GetDockerfileDependencies(a.Workspace, f)
	if err != nil {
		return nil, errors.Wrap(err, "getting dockerfile dependencies")
	}
	deps = append(deps, dockerfileAbsPath)
	return deps, nil
}

// GetDockerfileDependencies parses a dockerfile and returns the full paths
// of all the source files that the resulting docker image depends on.
func GetDockerfileDependencies(workspace string, r io.Reader) ([]string, error) {
	res, err := parser.Parse(r)
	if err != nil {
		return nil, errors.Wrap(err, "parsing dockerfile")
	}
	envs := map[string]string{}
	depMap := map[string]struct{}{}
	// First process onbuilds, if present.
	onbuilds := []string{}
	for _, value := range res.AST.Children {
		switch value.Value {
		case from:
			onbuilds, err = processBaseImage(value)
			if err != nil {
				logrus.Warnf("Error processing base image for onbuild triggers: %s. Dependencies may be incomplete.", err)
			}
		}
	}

	var dispatchInstructions = func(r *parser.Result) {
		for _, value := range r.AST.Children {
			switch value.Value {
			case add, copy:
				processCopy(workspace, value, depMap, envs)
			case env:
				envs[value.Next.Value] = value.Next.Next.Value
			}
		}
	}
	for _, ob := range onbuilds {
		obRes, err := parser.Parse(strings.NewReader(ob))
		if err != nil {
			return nil, err
		}
		dispatchInstructions(obRes)
	}

	dispatchInstructions(res)

	deps := []string{}
	for dep := range depMap {
		deps = append(deps, dep)
	}
	logrus.Infof("Found dependencies for dockerfile %s", deps)
	expandedDeps, err := util.ExpandPaths(workspace, deps)
	if err != nil {
		return nil, errors.Wrap(err, "expanding dockerfile paths")
	}

	logrus.Infof("deps %s", expandedDeps)

	// Look for .dockerignore.
	ignorePath := filepath.Join(workspace, ".dockerignore")
	filteredDeps, err := ApplyDockerIgnore(expandedDeps, ignorePath)
	if err != nil {
		return nil, errors.Wrap(err, "applying dockerignore")
	}

	return filteredDeps, nil
}

func PortsFromDockerfile(r io.Reader) ([]string, error) {
	res, err := parser.Parse(r)
	if err != nil {
		return nil, errors.Wrap(err, "parsing dockerfile")
	}

	// Check the dockerfile and the base.
	ports := []string{}
	for _, value := range res.AST.Children {
		switch value.Value {
		case from:
			base := value.Next.Value
			if strings.ToLower(base) == "scratch" {
				logrus.Debug("Skipping port check in SCRATCH base image.")
				continue
			}
			img, err := RetrieveImage(value.Next.Value)
			if err != nil {
				logrus.Warnf("Error checking base image for ports: %s", err)
				continue
			}
			for port := range img.Config.ExposedPorts {
				logrus.Debugf("Found port %s in base image", port)
				ports = append(ports, string(port))
			}
		case expose:
			// There can be multiple ports per line.
			for {
				if value.Next == nil {
					break
				}
				port := value.Next.Value
				logrus.Debugf("Found port %s in Dockerfile", port)
				ports = append(ports, port)
				value = value.Next
			}
		}
	}
	// Sort ports for consistency in tests.
	sort.Strings(ports)
	return ports, nil
}

func processBaseImage(value *parser.Node) ([]string, error) {
	base := value.Next.Value
	logrus.Debugf("Checking base image %s for ONBUILD triggers.", base)
	if strings.ToLower(base) == "scratch" {
		logrus.Debugf("SCRATCH base image found, skipping check: %s", base)
		return nil, nil
	}
	img, err := RetrieveImage(base)
	if err != nil {
		return nil, err
	}
	logrus.Debugf("Found onbuild triggers %v in image %s", img.Config.OnBuild, base)
	return img.Config.OnBuild, nil
}

var imageCache sync.Map

func retrieveImage(image string) (*manifest.Schema2Image, error) {
	cachedCfg, present := imageCache.Load(image)
	if present {
		return cachedCfg.(*manifest.Schema2Image), nil
	}

	client, err := NewDockerAPIClient()
	if err != nil {
		return nil, err
	}

	raw, err := retrieveLocalImage(client, image)
	if err != nil {
		raw, err = retrieveRemoteImage(image)
		if err != nil {
			return nil, err
		}
	}

	cfg := &manifest.Schema2Image{}
	if err := json.Unmarshal(raw, cfg); err != nil {
		return nil, err
	}

	imageCache.Store(image, cfg)

	return cfg, nil
}

func retrieveLocalImage(client DockerAPIClient, image string) ([]byte, error) {
	_, raw, err := client.ImageInspectWithRaw(context.Background(), image)
	if err != nil {
		return nil, err
	}

	return raw, nil
}

func retrieveRemoteImage(image string) ([]byte, error) {
	context := &types.SystemContext{
		OSChoice:           "linux",
		ArchitectureChoice: "amd64",
	}

	ref, err := docker.ParseReference("//" + image)
	if err != nil {
		return nil, err
	}

	img, err := ref.NewImage(context)
	if err != nil {
		return nil, err
	}
	defer img.Close()

	return img.ConfigBlob()
}

func processCopy(workspace string, value *parser.Node, paths map[string]struct{}, envs map[string]string) error {
	slex := shell.NewLex('\\')
	for {
		// Skip last node, since it is the destination, and stop if we arrive at a comment
		if value.Next.Next == nil || strings.HasPrefix(value.Next.Next.Value, "#") {
			break
		}
		src, err := processShellWord(slex, value.Next.Value, envs)
		if err != nil {
			return errors.Wrap(err, "processing word")
		}
		// If the --from flag is provided, we are dealing with a multi-stage dockerfile
		// Adding a dependency from a different stage does not imply a source dependency
		if hasMultiStageFlag(value.Flags) {
			return nil
		}
		if !strings.HasPrefix(src, "http://") && !strings.HasPrefix(src, "https://") {
			dep := path.Join(workspace, src)
			paths[dep] = struct{}{}
		} else {
			logrus.Debugf("Skipping watch on remote dependency %s", src)
		}

		value = value.Next
	}
	return nil
}

func processShellWord(lex *shell.Lex, word string, envs map[string]string) (string, error) {
	envSlice := []string{}
	for envKey, envVal := range envs {
		envSlice = append(envSlice, fmt.Sprintf("%s=%s", envKey, envVal))
	}
	return lex.ProcessWord(word, envSlice)
}

func hasMultiStageFlag(flags []string) bool {
	for _, f := range flags {
		if strings.HasPrefix(f, "--from=") {
			return true
		}
	}
	return false
}

func ApplyDockerIgnore(paths []string, dockerIgnorePath string) ([]string, error) {
	absPaths, err := util.RelPathToAbsPath(paths)
	if err != nil {
		return nil, errors.Wrap(err, "getting absolute path of dependencies")
	}
	excludes := []string{}
	if _, err := util.Fs.Stat(dockerIgnorePath); !os.IsNotExist(err) {
		r, err := util.Fs.Open(dockerIgnorePath)
		defer r.Close()
		if err != nil {
			return nil, err
		}
		excludes, err = dockerignore.ReadAll(r)
		if err != nil {
			return nil, err
		}
		excludes = append(excludes, ".dockerignore")
	}

	absPathExcludes, err := util.RelPathToAbsPath(excludes)
	if err != nil {
		return nil, errors.Wrap(err, "getting absolute path of docker ignored paths")
	}

	filteredDeps := []string{}
	for _, d := range absPaths {
		m, err := fileutils.Matches(d, absPathExcludes)
		if err != nil {
			return nil, err
		}
		if !m {
			filteredDeps = append(filteredDeps, d)
		}
	}
	sort.Strings(filteredDeps)
	return filteredDeps, nil
}
<|endoftext|>


<commit_before>package forward

import (
	"reflect"
	"strings"
	"testing"

	"github.com/mholt/caddy"
)

func TestSetup(t *testing.T) {
	tests := []struct {
		input           string
		shouldErr       bool
		expectedFrom    string
		expectedIgnored []string
		expectedFails   uint32
		expectedOpts    options
		expectedErr     string
	}{
		// positive
		{"forward . 127.0.0.1", false, ".", nil, 2, options{}, ""},
		{"forward . 127.0.0.1 {\nexcept miek.nl\n}\n", false, ".", nil, 2, options{}, ""},
		{"forward . 127.0.0.1 {\nmax_fails 3\n}\n", false, ".", nil, 3, options{}, ""},
		{"forward . 127.0.0.1 {\nforce_tcp\n}\n", false, ".", nil, 2, options{forceTCP: true}, ""},
		{"forward . 127.0.0.1 {\nprefer_udp\n}\n", false, ".", nil, 2, options{preferUDP: true}, ""},
		{"forward . 127.0.0.1 {\nforce_tcp\nprefer_udp\n}\n", false, ".", nil, 2, options{preferUDP: true, forceTCP: true}, ""},
		{"forward . 127.0.0.1:53", false, ".", nil, 2, options{}, ""},
		{"forward . 127.0.0.1:8080", false, ".", nil, 2, options{}, ""},
		{"forward . [::1]:53", false, ".", nil, 2, options{}, ""},
		{"forward . [2003::1]:53", false, ".", nil, 2, options{}, ""},
		// negative
		{"forward . a27.0.0.1", true, "", nil, 0, options{}, "not an IP"},
		{"forward . 127.0.0.1 {\nblaatl\n}\n", true, "", nil, 0, options{}, "unknown property"},
		{`forward . ::1
		forward com ::2`, true, "", nil, 0, options{}, "plugin"},
	}

	for i, test := range tests {
		c := caddy.NewTestController("dns", test.input)
		f, err := parseForward(c)

		if test.shouldErr && err == nil {
			t.Errorf("Test %d: expected error but found %s for input %s", i, err, test.input)
		}

		if err != nil {
			if !test.shouldErr {
				t.Errorf("Test %d: expected no error but found one for input %s, got: %v", i, test.input, err)
			}

			if !strings.Contains(err.Error(), test.expectedErr) {
				t.Errorf("Test %d: expected error to contain: %v, found error: %v, input: %s", i, test.expectedErr, err, test.input)
			}
		}

		if !test.shouldErr && f.from != test.expectedFrom {
			t.Errorf("Test %d: expected: %s, got: %s", i, test.expectedFrom, f.from)
		}
		if !test.shouldErr && test.expectedIgnored != nil {
			if !reflect.DeepEqual(f.ignored, test.expectedIgnored) {
				t.Errorf("Test %d: expected: %q, actual: %q", i, test.expectedIgnored, f.ignored)
			}
		}
		if !test.shouldErr && f.maxfails != test.expectedFails {
			t.Errorf("Test %d: expected: %d, got: %d", i, test.expectedFails, f.maxfails)
		}
		if !test.shouldErr && f.opts != test.expectedOpts {
			t.Errorf("Test %d: expected: %v, got: %v", i, test.expectedOpts, f.opts)
		}
	}
}

func TestSetupTLS(t *testing.T) {
	tests := []struct {
		input              string
		shouldErr          bool
		expectedServerName string
		expectedErr        string
	}{
		// positive
		{`forward . tls://127.0.0.1 {
				tls_servername dns
			}`, false, "dns", ""},
		{`forward . 127.0.0.1 {
				tls_servername dns
			}`, false, "", ""},
		{`forward . 127.0.0.1 {
				tls
			}`, false, "", ""},
		{`forward . tls://127.0.0.1`, false, "", ""},
	}

	for i, test := range tests {
		c := caddy.NewTestController("dns", test.input)
		f, err := parseForward(c)

		if test.shouldErr && err == nil {
			t.Errorf("Test %d: expected error but found %s for input %s", i, err, test.input)
		}

		if err != nil {
			if !test.shouldErr {
				t.Errorf("Test %d: expected no error but found one for input %s, got: %v", i, test.input, err)
			}

			if !strings.Contains(err.Error(), test.expectedErr) {
				t.Errorf("Test %d: expected error to contain: %v, found error: %v, input: %s", i, test.expectedErr, err, test.input)
			}
		}

		if !test.shouldErr && test.expectedServerName != "" && test.expectedServerName != f.tlsConfig.ServerName {
			t.Errorf("Test %d: expected: %q, actual: %q", i, test.expectedServerName, f.tlsConfig.ServerName)
		}

		if !test.shouldErr && test.expectedServerName != "" && test.expectedServerName != f.proxies[0].health.(*dnsHc).c.TLSConfig.ServerName {
			t.Errorf("Test %d: expected: %q, actual: %q", i, test.expectedServerName, f.proxies[0].health.(*dnsHc).c.TLSConfig.ServerName)
		}
	}
}
<commit_msg>Add test for #2003 (#2115)<commit_after>package forward

import (
	"io/ioutil"
	"os"
	"reflect"
	"strings"
	"testing"

	"github.com/mholt/caddy"
)

func TestSetup(t *testing.T) {
	tests := []struct {
		input           string
		shouldErr       bool
		expectedFrom    string
		expectedIgnored []string
		expectedFails   uint32
		expectedOpts    options
		expectedErr     string
	}{
		// positive
		{"forward . 127.0.0.1", false, ".", nil, 2, options{}, ""},
		{"forward . 127.0.0.1 {\nexcept miek.nl\n}\n", false, ".", nil, 2, options{}, ""},
		{"forward . 127.0.0.1 {\nmax_fails 3\n}\n", false, ".", nil, 3, options{}, ""},
		{"forward . 127.0.0.1 {\nforce_tcp\n}\n", false, ".", nil, 2, options{forceTCP: true}, ""},
		{"forward . 127.0.0.1 {\nprefer_udp\n}\n", false, ".", nil, 2, options{preferUDP: true}, ""},
		{"forward . 127.0.0.1 {\nforce_tcp\nprefer_udp\n}\n", false, ".", nil, 2, options{preferUDP: true, forceTCP: true}, ""},
		{"forward . 127.0.0.1:53", false, ".", nil, 2, options{}, ""},
		{"forward . 127.0.0.1:8080", false, ".", nil, 2, options{}, ""},
		{"forward . [::1]:53", false, ".", nil, 2, options{}, ""},
		{"forward . [2003::1]:53", false, ".", nil, 2, options{}, ""},
		// negative
		{"forward . a27.0.0.1", true, "", nil, 0, options{}, "not an IP"},
		{"forward . 127.0.0.1 {\nblaatl\n}\n", true, "", nil, 0, options{}, "unknown property"},
		{`forward . ::1
		forward com ::2`, true, "", nil, 0, options{}, "plugin"},
	}

	for i, test := range tests {
		c := caddy.NewTestController("dns", test.input)
		f, err := parseForward(c)

		if test.shouldErr && err == nil {
			t.Errorf("Test %d: expected error but found %s for input %s", i, err, test.input)
		}

		if err != nil {
			if !test.shouldErr {
				t.Errorf("Test %d: expected no error but found one for input %s, got: %v", i, test.input, err)
			}

			if !strings.Contains(err.Error(), test.expectedErr) {
				t.Errorf("Test %d: expected error to contain: %v, found error: %v, input: %s", i, test.expectedErr, err, test.input)
			}
		}

		if !test.shouldErr && f.from != test.expectedFrom {
			t.Errorf("Test %d: expected: %s, got: %s", i, test.expectedFrom, f.from)
		}
		if !test.shouldErr && test.expectedIgnored != nil {
			if !reflect.DeepEqual(f.ignored, test.expectedIgnored) {
				t.Errorf("Test %d: expected: %q, actual: %q", i, test.expectedIgnored, f.ignored)
			}
		}
		if !test.shouldErr && f.maxfails != test.expectedFails {
			t.Errorf("Test %d: expected: %d, got: %d", i, test.expectedFails, f.maxfails)
		}
		if !test.shouldErr && f.opts != test.expectedOpts {
			t.Errorf("Test %d: expected: %v, got: %v", i, test.expectedOpts, f.opts)
		}
	}
}

func TestSetupTLS(t *testing.T) {
	tests := []struct {
		input              string
		shouldErr          bool
		expectedServerName string
		expectedErr        string
	}{
		// positive
		{`forward . tls://127.0.0.1 {
				tls_servername dns
			}`, false, "dns", ""},
		{`forward . 127.0.0.1 {
				tls_servername dns
			}`, false, "", ""},
		{`forward . 127.0.0.1 {
				tls
			}`, false, "", ""},
		{`forward . tls://127.0.0.1`, false, "", ""},
	}

	for i, test := range tests {
		c := caddy.NewTestController("dns", test.input)
		f, err := parseForward(c)

		if test.shouldErr && err == nil {
			t.Errorf("Test %d: expected error but found %s for input %s", i, err, test.input)
		}

		if err != nil {
			if !test.shouldErr {
				t.Errorf("Test %d: expected no error but found one for input %s, got: %v", i, test.input, err)
			}

			if !strings.Contains(err.Error(), test.expectedErr) {
				t.Errorf("Test %d: expected error to contain: %v, found error: %v, input: %s", i, test.expectedErr, err, test.input)
			}
		}

		if !test.shouldErr && test.expectedServerName != "" && test.expectedServerName != f.tlsConfig.ServerName {
			t.Errorf("Test %d: expected: %q, actual: %q", i, test.expectedServerName, f.tlsConfig.ServerName)
		}

		if !test.shouldErr && test.expectedServerName != "" && test.expectedServerName != f.proxies[0].health.(*dnsHc).c.TLSConfig.ServerName {
			t.Errorf("Test %d: expected: %q, actual: %q", i, test.expectedServerName, f.proxies[0].health.(*dnsHc).c.TLSConfig.ServerName)
		}
	}
}

func TestSetupResolvconf(t *testing.T) {
	const resolv = "resolv.conf"
	if err := ioutil.WriteFile(resolv,
		[]byte(`nameserver 10.10.255.252
nameserver 10.10.255.253`), 0666); err != nil {
		t.Fatalf("Failed to write resolv.conf file: %s", err)
	}
	defer os.Remove(resolv)

	tests := []struct {
		input         string
		shouldErr     bool
		expectedErr   string
		expectedNames []string
	}{
		// pass
		{`forward . ` + resolv, false, "", []string{"10.10.255.252:53", "10.10.255.253:53"}},
	}

	for i, test := range tests {
		c := caddy.NewTestController("dns", test.input)
		f, err := parseForward(c)

		if test.shouldErr && err == nil {
			t.Errorf("Test %d: expected error but found %s for input %s", i, err, test.input)
			continue
		}

		if err != nil {
			if !test.shouldErr {
				t.Errorf("Test %d: expected no error but found one for input %s, got: %v", i, test.input, err)
			}

			if !strings.Contains(err.Error(), test.expectedErr) {
				t.Errorf("Test %d: expected error to contain: %v, found error: %v, input: %s", i, test.expectedErr, err, test.input)
			}
		}

		if !test.shouldErr {
			for j, n := range test.expectedNames {
				addr := f.proxies[j].addr
				if n != addr {
					t.Errorf("Test %d, expected %q, got %q", j, n, addr)
				}
			}
		}
		for _, p := range f.proxies {
			p.health.Check(p) // this should almost always err, we don't care it shoulnd't crash
		}
	}
}
<|endoftext|>