<commit_before>// Copyright 2016 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"encoding/json"
	"fmt"
	"log"
	"os"
	"sort"
	"strings"
	"time"

	"github.com/maruel/subcommands"

	"go.chromium.org/luci/auth"
	"go.chromium.org/luci/client/internal/common"
	"go.chromium.org/luci/common/api/swarming/swarming/v1"
	"go.chromium.org/luci/common/data/text/units"
	"go.chromium.org/luci/common/errors"
	"go.chromium.org/luci/common/flag/stringmapflag"
)

func cmdTrigger(defaultAuthOpts auth.Options) *subcommands.Command {
	return &subcommands.Command{
		UsageLine: "trigger <options>",
		ShortDesc: "Triggers a Swarming task",
		LongDesc:  "Triggers a Swarming task.",
		CommandRun: func() subcommands.CommandRun {
			r := &triggerRun{}
			r.Init(defaultAuthOpts)
			return r
		},
	}
}

type array []*swarming.SwarmingRpcsStringPair

func (a array) Len() int { return len(a) }
func (a array) Less(i, j int) bool {
	return (a[i].Key < a[j].Key) ||
		(a[i].Key == a[j].Key && a[i].Value < a[j].Value)
}
func (a array) Swap(i, j int) {
	a[i], a[j] = a[j], a[i]
}

// mapToArray converts a stringmapflag.Value into an array of
// swarming.SwarmingRpcsStringPair, sorted by key and then value.
func mapToArray(m stringmapflag.Value) []*swarming.SwarmingRpcsStringPair {
	a := make([]*swarming.SwarmingRpcsStringPair, 0, len(m))
	for k, v := range m {
		a = append(a, &swarming.SwarmingRpcsStringPair{Key: k, Value: v})
	}

	sort.Sort(array(a))
	return a
}

// namePartFromDimensions creates a string from a map of dimensions that can
// be used as part of the task name.  The dimensions are first sorted as
// described in mapToArray().
func namePartFromDimensions(m stringmapflag.Value) string {
	a := mapToArray(m)
	pairs := make([]string, 0, len(a))
	for i := 0; i < len(a); i++ {
		pairs = append(pairs, fmt.Sprintf("%s=%s", a[i].Key, a[i].Value))
	}
	return strings.Join(pairs, "_")
}

type triggerRun struct {
	commonFlags

	// TODO(rogerta): move these flags to swarming/common.go once other commands
	// are written and I see what parts are common.

	// Isolate server.
	isolateServer string
	namespace     string

	// Task group.
	isolated    string
	dimensions  stringmapflag.Value
	env         stringmapflag.Value
	priority    int64
	taskName    string
	tags        common.Strings
	user        string
	idempotent  bool
	expiration  int
	deadline    int
	hardTimeout int64
	ioTimeout   int64
	rawCmd      bool
	dumpJSON    string
	cipdPackage stringmapflag.Value
	outputs     common.Strings
}

func (c *triggerRun) Init(defaultAuthOpts auth.Options) {
	c.commonFlags.Init(defaultAuthOpts)

	// Isolate server.
	c.Flags.StringVar(&c.isolateServer, "isolate-server", "", "URL of the Isolate Server to use.")
	c.Flags.StringVar(&c.namespace, "namespace", "default-gzip", "The namespace to use on the Isolate Server.")

	// Task group.
	c.Flags.StringVar(&c.isolated, "isolated", "", "Hash of the .isolated to grab from the isolate server.")
	c.Flags.Var(&c.dimensions, "dimension", "Dimension to filter slaves on.")
	c.Flags.Var(&c.env, "env", "Environment variables to set.")
	c.Flags.Int64Var(&c.priority, "priority", 100, "The lower value, the more important the task.")
	c.Flags.StringVar(&c.taskName, "task-name", "", "Display name of the task. Defaults to <base_name>/<dimensions>/<isolated hash>/<timestamp> if an  isolated file is provided, if a hash is provided, it defaults to <user>/<dimensions>/<isolated hash>/<timestamp>")
	c.Flags.Var(&c.tags, "tag", "Tags to assign to the task.")
	c.Flags.StringVar(&c.user, "user", "", "User associated with the task. Defaults to authenticated user on the server.")
	c.Flags.Var(&c.outputs, "output", "(repeatable) Specify an output file or directory that can be retrieved via collect.")
	c.Flags.BoolVar(&c.idempotent, "idempotent", false, "When set, the server will actively try to find a previous task with the same parameter and return this result instead if possible.")
	c.Flags.IntVar(&c.expiration, "expiration", 6*60*60, "Seconds to allow the task to be pending for a bot to run before this task request expires.")
	c.Flags.IntVar(&c.deadline, "deadline", 0, "TODO(rogerta)")
	c.Flags.Int64Var(&c.hardTimeout, "hard-timeout", 60*60, "Seconds to allow the task to complete.")
	c.Flags.Int64Var(&c.ioTimeout, "io-timeout", 20*60, "Seconds to allow the task to be silent.")
	c.Flags.BoolVar(&c.rawCmd, "raw-cmd", false, "When set, the command after -- is run on the bot. Note that this overrides any command in the .isolated file.")
	c.Flags.StringVar(&c.dumpJSON, "dump-json", "", "Dump details about the triggered task(s) to this file as json.")
	c.Flags.Var(&c.cipdPackage, "cipd-package",
		"(repeatable) CIPD packages to install on the swarming bot. This takes a parameter of `[subdir:]pkgname=version`. "+
			"Using an empty version will remove the package. The subdir is optional and defaults to '.'.")
}

func (c *triggerRun) Parse(args []string) error {
	var err error
	if err := c.commonFlags.Parse(); err != nil {
		return err
	}

	// Validate options and args.
	if c.dimensions == nil {
		return errors.Reason("please at least specify one dimension").Err()
	}

	if c.rawCmd && len(args) == 0 {
		return errors.Reason("arguments with -raw-cmd should be passed after -- as command delimiter").Err()
	} else if !c.rawCmd && len(c.isolated) == 0 {
		return errors.Reason("please use -isolated to specify hash or -raw-cmd").Err()
	}

	if len(c.user) == 0 {
		c.user = os.Getenv("USER")
	}

	return err
}

func (c *triggerRun) Run(a subcommands.Application, args []string, env subcommands.Env) int {
	if err := c.Parse(args); err != nil {
		printError(a, err)
		return 1
	}
	cl, err := c.defaultFlags.StartTracing()
	if err != nil {
		printError(a, err)
		return 1
	}
	defer cl.Close()

	if err := c.main(a, args, env); err != nil {
		printError(a, err)
		return 1
	}
	return 0
}

func (c *triggerRun) main(a subcommands.Application, args []string, env subcommands.Env) error {
	start := time.Now()
	request, err := c.processTriggerOptions(args, env)
	if err != nil {
		return err
	}

	result, err := c.createNewTask(request)
	if err != nil {
		return err
	}

	fmt.Printf("Triggered task: %s\n\n", result.TaskId)
	viewURL := fmt.Sprintf("%s/user/task/%s", c.serverURL, result.TaskId)

	if len(c.dumpJSON) > 0 {
		dump, err := os.Create(c.dumpJSON)
		if err != nil {
			return err
		}
		defer dump.Close()

		data := jsonDump{
			TaskID:  result.TaskId,
			ViewURL: viewURL,
			Request: *request,
		}

		b, err := json.MarshalIndent(&data, "", "  ")
		if err != nil {
			return errors.Annotate(err, "marshalling trigger result").Err()
		}

		_, err = dump.Write(b)
		if err != nil {
			return errors.Annotate(err, "writing json dump").Err()
		}

		if !c.defaultFlags.Quiet {
			fmt.Println("To collect results use:")
			fmt.Printf("  swarming collect -server %s -requests-json %s\n", c.serverURL, c.dumpJSON)
		}
	} else if !c.defaultFlags.Quiet {
		fmt.Println("To collect results use:")
		fmt.Printf("  swarming collect -server %s %s\n", c.serverURL, result.TaskId)
	}

	if !c.defaultFlags.Quiet {
		fmt.Printf("or visit: %s\n", viewURL)
	}

	duration := time.Since(start)
	log.Printf("Duration: %s\n", units.Round(duration, time.Millisecond))
	return err
}

func (c *triggerRun) processTriggerOptions(args []string, env subcommands.Env) (*swarming.SwarmingRpcsNewTaskRequest, error) {
	var inputsRefs *swarming.SwarmingRpcsFilesRef
	var commands []string
	var extraArgs []string

	if c.rawCmd {
		commands = args
	} else {
		extraArgs = args
	}

	if len(c.taskName) == 0 {
		c.taskName = fmt.Sprintf("%s/%s", c.user, namePartFromDimensions(c.dimensions))
	}

	if len(c.isolated) > 0 {
		if len(c.taskName) == 0 {
			c.taskName = fmt.Sprintf("%s/%s", c.taskName, c.isolated)
		}
		inputsRefs = &swarming.SwarmingRpcsFilesRef{
			Isolated:       c.isolated,
			Isolatedserver: c.isolateServer,
			Namespace:      c.namespace,
		}
	}

	properties := swarming.SwarmingRpcsTaskProperties{
		Command:              commands,
		Dimensions:           mapToArray(c.dimensions),
		Env:                  mapToArray(c.env),
		ExecutionTimeoutSecs: c.hardTimeout,
		ExtraArgs:            extraArgs,
		GracePeriodSecs:      30,
		Idempotent:           c.idempotent,
		InputsRef:            inputsRefs,
		Outputs:              c.outputs,
		IoTimeoutSecs:        c.ioTimeout,
	}

	if len(c.cipdPackage) > 0 {
		pkgs := []*swarming.SwarmingRpcsCipdPackage{}
		for k, v := range c.cipdPackage {
			s := strings.SplitN(k, ":", 2)
			pkg := swarming.SwarmingRpcsCipdPackage{
				PackageName: s[len(s)-1],
				Version:     v,
			}
			if len(s) > 1 {
				pkg.Path = s[0]
			}
			pkgs = append(pkgs, &pkg)
		}
		properties.CipdInput = &swarming.SwarmingRpcsCipdInput{Packages: pkgs}
	}

	request := swarming.SwarmingRpcsNewTaskRequest{
		ExpirationSecs: c.hardTimeout,
		Name:           c.taskName,
		ParentTaskId:   env["SWARMING_TASK_ID"].Value,
		Priority:       c.priority,
		Properties:     &properties,
		Tags:           c.tags,
		User:           c.user,
	}
	return &request, nil
}

func (c *triggerRun) createNewTask(request *swarming.SwarmingRpcsNewTaskRequest) (*swarming.SwarmingRpcsTaskRequestMetadata, error) {
	client, err := c.createAuthClient()
	if err != nil {
		return &swarming.SwarmingRpcsTaskRequestMetadata{}, err
	}

	s, err := swarming.New(client)
	if err != nil {
		return &swarming.SwarmingRpcsTaskRequestMetadata{}, err
	}
	s.BasePath = c.commonFlags.serverURL + swarmingAPISuffix

	call := s.Tasks.New(request).Fields("task_result")
	result, err := call.Do()
	if err != nil {
		return &swarming.SwarmingRpcsTaskRequestMetadata{}, err
	}

	// Recursively look at error and log.

	return result, nil
}
<commit_msg>[swarming] lower default task priority to 200<commit_after>// Copyright 2016 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"encoding/json"
	"fmt"
	"log"
	"os"
	"sort"
	"strings"
	"time"

	"github.com/maruel/subcommands"

	"go.chromium.org/luci/auth"
	"go.chromium.org/luci/client/internal/common"
	"go.chromium.org/luci/common/api/swarming/swarming/v1"
	"go.chromium.org/luci/common/data/text/units"
	"go.chromium.org/luci/common/errors"
	"go.chromium.org/luci/common/flag/stringmapflag"
)

func cmdTrigger(defaultAuthOpts auth.Options) *subcommands.Command {
	return &subcommands.Command{
		UsageLine: "trigger <options>",
		ShortDesc: "Triggers a Swarming task",
		LongDesc:  "Triggers a Swarming task.",
		CommandRun: func() subcommands.CommandRun {
			r := &triggerRun{}
			r.Init(defaultAuthOpts)
			return r
		},
	}
}

type array []*swarming.SwarmingRpcsStringPair

func (a array) Len() int { return len(a) }
func (a array) Less(i, j int) bool {
	return (a[i].Key < a[j].Key) ||
		(a[i].Key == a[j].Key && a[i].Value < a[j].Value)
}
func (a array) Swap(i, j int) {
	a[i], a[j] = a[j], a[i]
}

// mapToArray converts a stringmapflag.Value into an array of
// swarming.SwarmingRpcsStringPair, sorted by key and then value.
func mapToArray(m stringmapflag.Value) []*swarming.SwarmingRpcsStringPair {
	a := make([]*swarming.SwarmingRpcsStringPair, 0, len(m))
	for k, v := range m {
		a = append(a, &swarming.SwarmingRpcsStringPair{Key: k, Value: v})
	}

	sort.Sort(array(a))
	return a
}

// namePartFromDimensions creates a string from a map of dimensions that can
// be used as part of the task name.  The dimensions are first sorted as
// described in mapToArray().
func namePartFromDimensions(m stringmapflag.Value) string {
	a := mapToArray(m)
	pairs := make([]string, 0, len(a))
	for i := 0; i < len(a); i++ {
		pairs = append(pairs, fmt.Sprintf("%s=%s", a[i].Key, a[i].Value))
	}
	return strings.Join(pairs, "_")
}

type triggerRun struct {
	commonFlags

	// TODO(rogerta): move these flags to swarming/common.go once other commands
	// are written and I see what parts are common.

	// Isolate server.
	isolateServer string
	namespace     string

	// Task group.
	isolated    string
	dimensions  stringmapflag.Value
	env         stringmapflag.Value
	priority    int64
	taskName    string
	tags        common.Strings
	user        string
	idempotent  bool
	expiration  int
	deadline    int
	hardTimeout int64
	ioTimeout   int64
	rawCmd      bool
	dumpJSON    string
	cipdPackage stringmapflag.Value
	outputs     common.Strings
}

func (c *triggerRun) Init(defaultAuthOpts auth.Options) {
	c.commonFlags.Init(defaultAuthOpts)

	// Isolate server.
	c.Flags.StringVar(&c.isolateServer, "isolate-server", "", "URL of the Isolate Server to use.")
	c.Flags.StringVar(&c.namespace, "namespace", "default-gzip", "The namespace to use on the Isolate Server.")

	// Task group.
	c.Flags.StringVar(&c.isolated, "isolated", "", "Hash of the .isolated to grab from the isolate server.")
	c.Flags.Var(&c.dimensions, "dimension", "Dimension to filter slaves on.")
	c.Flags.Var(&c.env, "env", "Environment variables to set.")
	c.Flags.Int64Var(&c.priority, "priority", 200, "The lower value, the more important the task.")
	c.Flags.StringVar(&c.taskName, "task-name", "", "Display name of the task. Defaults to <base_name>/<dimensions>/<isolated hash>/<timestamp> if an  isolated file is provided, if a hash is provided, it defaults to <user>/<dimensions>/<isolated hash>/<timestamp>")
	c.Flags.Var(&c.tags, "tag", "Tags to assign to the task.")
	c.Flags.StringVar(&c.user, "user", "", "User associated with the task. Defaults to authenticated user on the server.")
	c.Flags.Var(&c.outputs, "output", "(repeatable) Specify an output file or directory that can be retrieved via collect.")
	c.Flags.BoolVar(&c.idempotent, "idempotent", false, "When set, the server will actively try to find a previous task with the same parameter and return this result instead if possible.")
	c.Flags.IntVar(&c.expiration, "expiration", 6*60*60, "Seconds to allow the task to be pending for a bot to run before this task request expires.")
	c.Flags.IntVar(&c.deadline, "deadline", 0, "TODO(rogerta)")
	c.Flags.Int64Var(&c.hardTimeout, "hard-timeout", 60*60, "Seconds to allow the task to complete.")
	c.Flags.Int64Var(&c.ioTimeout, "io-timeout", 20*60, "Seconds to allow the task to be silent.")
	c.Flags.BoolVar(&c.rawCmd, "raw-cmd", false, "When set, the command after -- is run on the bot. Note that this overrides any command in the .isolated file.")
	c.Flags.StringVar(&c.dumpJSON, "dump-json", "", "Dump details about the triggered task(s) to this file as json.")
	c.Flags.Var(&c.cipdPackage, "cipd-package",
		"(repeatable) CIPD packages to install on the swarming bot. This takes a parameter of `[subdir:]pkgname=version`. "+
			"Using an empty version will remove the package. The subdir is optional and defaults to '.'.")
}

func (c *triggerRun) Parse(args []string) error {
	var err error
	if err := c.commonFlags.Parse(); err != nil {
		return err
	}

	// Validate options and args.
	if c.dimensions == nil {
		return errors.Reason("please at least specify one dimension").Err()
	}

	if c.rawCmd && len(args) == 0 {
		return errors.Reason("arguments with -raw-cmd should be passed after -- as command delimiter").Err()
	} else if !c.rawCmd && len(c.isolated) == 0 {
		return errors.Reason("please use -isolated to specify hash or -raw-cmd").Err()
	}

	if len(c.user) == 0 {
		c.user = os.Getenv("USER")
	}

	return err
}

func (c *triggerRun) Run(a subcommands.Application, args []string, env subcommands.Env) int {
	if err := c.Parse(args); err != nil {
		printError(a, err)
		return 1
	}
	cl, err := c.defaultFlags.StartTracing()
	if err != nil {
		printError(a, err)
		return 1
	}
	defer cl.Close()

	if err := c.main(a, args, env); err != nil {
		printError(a, err)
		return 1
	}
	return 0
}

func (c *triggerRun) main(a subcommands.Application, args []string, env subcommands.Env) error {
	start := time.Now()
	request, err := c.processTriggerOptions(args, env)
	if err != nil {
		return err
	}

	result, err := c.createNewTask(request)
	if err != nil {
		return err
	}

	fmt.Printf("Triggered task: %s\n\n", result.TaskId)
	viewURL := fmt.Sprintf("%s/user/task/%s", c.serverURL, result.TaskId)

	if len(c.dumpJSON) > 0 {
		dump, err := os.Create(c.dumpJSON)
		if err != nil {
			return err
		}
		defer dump.Close()

		data := jsonDump{
			TaskID:  result.TaskId,
			ViewURL: viewURL,
			Request: *request,
		}

		b, err := json.MarshalIndent(&data, "", "  ")
		if err != nil {
			return errors.Annotate(err, "marshalling trigger result").Err()
		}

		_, err = dump.Write(b)
		if err != nil {
			return errors.Annotate(err, "writing json dump").Err()
		}

		if !c.defaultFlags.Quiet {
			fmt.Println("To collect results use:")
			fmt.Printf("  swarming collect -server %s -requests-json %s\n", c.serverURL, c.dumpJSON)
		}
	} else if !c.defaultFlags.Quiet {
		fmt.Println("To collect results use:")
		fmt.Printf("  swarming collect -server %s %s\n", c.serverURL, result.TaskId)
	}

	if !c.defaultFlags.Quiet {
		fmt.Printf("or visit: %s\n", viewURL)
	}

	duration := time.Since(start)
	log.Printf("Duration: %s\n", units.Round(duration, time.Millisecond))
	return err
}

func (c *triggerRun) processTriggerOptions(args []string, env subcommands.Env) (*swarming.SwarmingRpcsNewTaskRequest, error) {
	var inputsRefs *swarming.SwarmingRpcsFilesRef
	var commands []string
	var extraArgs []string

	if c.rawCmd {
		commands = args
	} else {
		extraArgs = args
	}

	if len(c.taskName) == 0 {
		c.taskName = fmt.Sprintf("%s/%s", c.user, namePartFromDimensions(c.dimensions))
	}

	if len(c.isolated) > 0 {
		if len(c.taskName) == 0 {
			c.taskName = fmt.Sprintf("%s/%s", c.taskName, c.isolated)
		}
		inputsRefs = &swarming.SwarmingRpcsFilesRef{
			Isolated:       c.isolated,
			Isolatedserver: c.isolateServer,
			Namespace:      c.namespace,
		}
	}

	properties := swarming.SwarmingRpcsTaskProperties{
		Command:              commands,
		Dimensions:           mapToArray(c.dimensions),
		Env:                  mapToArray(c.env),
		ExecutionTimeoutSecs: c.hardTimeout,
		ExtraArgs:            extraArgs,
		GracePeriodSecs:      30,
		Idempotent:           c.idempotent,
		InputsRef:            inputsRefs,
		Outputs:              c.outputs,
		IoTimeoutSecs:        c.ioTimeout,
	}

	if len(c.cipdPackage) > 0 {
		pkgs := []*swarming.SwarmingRpcsCipdPackage{}
		for k, v := range c.cipdPackage {
			s := strings.SplitN(k, ":", 2)
			pkg := swarming.SwarmingRpcsCipdPackage{
				PackageName: s[len(s)-1],
				Version:     v,
			}
			if len(s) > 1 {
				pkg.Path = s[0]
			}
			pkgs = append(pkgs, &pkg)
		}
		properties.CipdInput = &swarming.SwarmingRpcsCipdInput{Packages: pkgs}
	}

	request := swarming.SwarmingRpcsNewTaskRequest{
		ExpirationSecs: c.hardTimeout,
		Name:           c.taskName,
		ParentTaskId:   env["SWARMING_TASK_ID"].Value,
		Priority:       c.priority,
		Properties:     &properties,
		Tags:           c.tags,
		User:           c.user,
	}
	return &request, nil
}

func (c *triggerRun) createNewTask(request *swarming.SwarmingRpcsNewTaskRequest) (*swarming.SwarmingRpcsTaskRequestMetadata, error) {
	client, err := c.createAuthClient()
	if err != nil {
		return &swarming.SwarmingRpcsTaskRequestMetadata{}, err
	}

	s, err := swarming.New(client)
	if err != nil {
		return &swarming.SwarmingRpcsTaskRequestMetadata{}, err
	}
	s.BasePath = c.commonFlags.serverURL + swarmingAPISuffix

	call := s.Tasks.New(request).Fields("task_result")
	result, err := call.Do()
	if err != nil {
		return &swarming.SwarmingRpcsTaskRequestMetadata{}, err
	}

	// Recursively look at error and log.

	return result, nil
}
<|endoftext|>


<commit_before>package elasticthought

import (
	"fmt"
	"sync"

	"github.com/couchbaselabs/logg"
)

// Worker job that downloads a datafile url contents to cbfs
type DatafileDownloader struct {
	Configuration Configuration
	Datafile      Datafile
}

// Run this job
func (d DatafileDownloader) Run(wg *sync.WaitGroup) {

	defer wg.Done()

	logg.LogTo("DATAFILE_DOWNLOADER", "datafile downloader run()")

	db := d.Configuration.DbConnection()
	datafile := &d.Datafile
	updatedState, err := datafile.UpdateProcessingState(Processing)
	if err != nil {
		d.recordProcessingError(err)
		return
	}
	if !updatedState {
		logg.LogTo("DATAFILE_DOWNLOADER", "%+v already processed.  Ignoring.", d)
		return
	}

	// create a new cbfs client
	cbfs, err := d.Configuration.NewCbfsClient()
	if err != nil {
		errMsg := fmt.Errorf("Error creating cbfs client: %v", err)
		d.recordProcessingError(errMsg)
		return
	}

	// copy url contents to cbfs
	logg.LogTo("DATAFILE_DOWNLOADER", "Put to CBFS: %+v %v %v", d, db, cbfs)
	cbfsDestPath, err := d.Datafile.CopyToCBFS(db, cbfs)
	if err != nil {
		d.recordProcessingError(err)
		return
	}

	// TODO:
	// figure out what kind datafile this is and update fields
	// layers_type=DATA layers_data_param=LEVELDB

	// build a url to the cbfs file
	cbfsUrl := fmt.Sprintf("%v/%v", d.Configuration.CbfsUrl, cbfsDestPath)
	d.Datafile.Url = cbfsUrl

	// Update the state of the dataset to be finished
	if err := d.Datafile.FinishedSuccessfully(db); err != nil {
		errMsg := fmt.Errorf("Error marking datafile %+v finished: %v", d, err)
		d.recordProcessingError(errMsg)
		return
	}

}

// Codereview: de-dupe -- dataset_splitter has same method
func (d DatafileDownloader) recordProcessingError(err error) {
	logg.LogError(err)
	db := d.Configuration.DbConnection()
	if err := d.Datafile.Failed(db, err); err != nil {
		errMsg := fmt.Errorf("Error setting datafile as failed: %v", err)
		logg.LogError(errMsg)
	}
}
<commit_msg>remove TODO<commit_after>package elasticthought

import (
	"fmt"
	"sync"

	"github.com/couchbaselabs/logg"
)

// Worker job that downloads a datafile url contents to cbfs
type DatafileDownloader struct {
	Configuration Configuration
	Datafile      Datafile
}

// Run this job
func (d DatafileDownloader) Run(wg *sync.WaitGroup) {

	defer wg.Done()

	logg.LogTo("DATAFILE_DOWNLOADER", "datafile downloader run()")

	db := d.Configuration.DbConnection()
	datafile := &d.Datafile
	updatedState, err := datafile.UpdateProcessingState(Processing)
	if err != nil {
		d.recordProcessingError(err)
		return
	}
	if !updatedState {
		logg.LogTo("DATAFILE_DOWNLOADER", "%+v already processed.  Ignoring.", d)
		return
	}

	// create a new cbfs client
	cbfs, err := d.Configuration.NewCbfsClient()
	if err != nil {
		errMsg := fmt.Errorf("Error creating cbfs client: %v", err)
		d.recordProcessingError(errMsg)
		return
	}

	// copy url contents to cbfs
	logg.LogTo("DATAFILE_DOWNLOADER", "Put to CBFS: %+v %v %v", d, db, cbfs)
	cbfsDestPath, err := d.Datafile.CopyToCBFS(db, cbfs)
	if err != nil {
		d.recordProcessingError(err)
		return
	}

	// build a url to the cbfs file
	cbfsUrl := fmt.Sprintf("%v/%v", d.Configuration.CbfsUrl, cbfsDestPath)
	d.Datafile.Url = cbfsUrl

	// Update the state of the dataset to be finished
	if err := d.Datafile.FinishedSuccessfully(db); err != nil {
		errMsg := fmt.Errorf("Error marking datafile %+v finished: %v", d, err)
		d.recordProcessingError(errMsg)
		return
	}

}

// Codereview: de-dupe -- dataset_splitter has same method
func (d DatafileDownloader) recordProcessingError(err error) {
	logg.LogError(err)
	db := d.Configuration.DbConnection()
	if err := d.Datafile.Failed(db, err); err != nil {
		errMsg := fmt.Errorf("Error setting datafile as failed: %v", err)
		logg.LogError(errMsg)
	}
}
<|endoftext|>


<commit_before><commit_msg>Remove GenericError.<commit_after><|endoftext|>


<commit_before>/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
    http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package backends

import (
	"fmt"
	"net/http"

	"github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud/meta"
	"google.golang.org/api/compute/v1"
	"k8s.io/ingress-gce/pkg/backends/features"
	"k8s.io/ingress-gce/pkg/composite"
	"k8s.io/ingress-gce/pkg/utils"
	"k8s.io/ingress-gce/pkg/utils/namer"
	"k8s.io/klog"
	"k8s.io/legacy-cloud-providers/gce"
)

// Backends handles CRUD operations for backends.
type Backends struct {
	cloud *gce.Cloud
	namer *namer.Namer
}

// Backends is a Pool.
var _ Pool = (*Backends)(nil)

// NewPool returns a new backend pool.
// - cloud: implements BackendServices
// - namer: produces names for backends.
func NewPool(cloud *gce.Cloud, namer *namer.Namer) *Backends {
	return &Backends{
		cloud: cloud,
		namer: namer,
	}
}

// ensureDescription updates the BackendService Description with the expected value
func ensureDescription(be *composite.BackendService, sp *utils.ServicePort) (needsUpdate bool) {
	desc := sp.GetDescription()
	features.SetDescription(&desc, sp)
	descString := desc.String()
	if be.Description == descString {
		return false
	}
	be.Description = descString
	return true
}

// Create implements Pool.
func (b *Backends) Create(sp utils.ServicePort, hcLink string) (*composite.BackendService, error) {
	name := sp.BackendName(b.namer)
	namedPort := &compute.NamedPort{
		Name: b.namer.NamedPort(sp.NodePort),
		Port: sp.NodePort,
	}

	version := features.VersionFromServicePort(&sp)
	be := &composite.BackendService{
		Version:      version,
		Name:         name,
		Protocol:     string(sp.Protocol),
		Port:         namedPort.Port,
		PortName:     namedPort.Name,
		HealthChecks: []string{hcLink},
	}

	if sp.L7ILBEnabled {
		be.LoadBalancingScheme = "INTERNAL"
	}

	ensureDescription(be, &sp)
	scope := features.ScopeFromServicePort(&sp)
	key, err := composite.CreateKey(b.cloud, name, scope)
	if err != nil {
		return nil, err
	}

	if err := composite.CreateBackendService(b.cloud, key, be); err != nil {
		return nil, err
	}
	// Note: We need to perform a GCE call to re-fetch the object we just created
	// so that the "Fingerprint" field is filled in. This is needed to update the
	// object without error.
	return b.Get(name, version, scope)
}

// Update implements Pool.
func (b *Backends) Update(be *composite.BackendService) error {
	// Ensure the backend service has the proper version before updating.
	be.Version = features.VersionFromDescription(be.Description)
	scope, err := composite.ScopeFromSelfLink(be.SelfLink)
	if err != nil {
		return err
	}

	key, err := composite.CreateKey(b.cloud, be.Name, scope)
	if err != nil {
		return err
	}
	if err := composite.UpdateBackendService(b.cloud, key, be); err != nil {
		return err
	}
	return nil
}

// Get implements Pool.
func (b *Backends) Get(name string, version meta.Version, scope meta.KeyType) (*composite.BackendService, error) {
	key, err := composite.CreateKey(b.cloud, name, scope)
	if err != nil {
		return nil, err
	}
	be, err := composite.GetBackendService(b.cloud, key, version)
	if err != nil {
		return nil, err
	}
	// Evaluate the existing features from description to see if a lower
	// API version is required so that we don't lose information from
	// the existing backend service.
	versionRequired := features.VersionFromDescription(be.Description)

	if features.IsLowerVersion(versionRequired, version) {
		be, err = composite.GetBackendService(b.cloud, key, versionRequired)
		if err != nil {
			return nil, err
		}
	}
	return be, nil
}

// Delete implements Pool.
func (b *Backends) Delete(name string, version meta.Version, scope meta.KeyType) error {
	klog.V(2).Infof("Deleting backend service %v", name)

	key, err := composite.CreateKey(b.cloud, name, scope)
	if err != nil {
		return err
	}
	err = composite.DeleteBackendService(b.cloud, key, version)
	if err != nil {
		if utils.IsHTTPErrorCode(err, http.StatusNotFound) || utils.IsInUsedByError(err) {
			klog.Infof("DeleteBackendService(_, %v, %v) = %v; ignorable error", key, version, err)
			return nil
		}
		klog.Errorf("DeleteBackendService(_, %v, %v) = %v", key, version, err)
		return err
	}
	klog.V(2).Infof("DeleteBackendService(_, %v, %v) ok", key, version)
	return nil
}

// Health implements Pool.
func (b *Backends) Health(name string, version meta.Version, scope meta.KeyType) (string, error) {
	be, err := b.Get(name, version, scope)
	if err != nil || len(be.Backends) == 0 {
		return "Unknown", fmt.Errorf("error getting backend %q: %v", name, err)
	}

	// TODO: Look at more than one backend's status
	// TODO: Include port, ip in the status, since it's in the health info.
	// TODO (shance) convert to composite types
	var hs *compute.BackendServiceGroupHealth
	switch scope {
	case meta.Global:
		hs, err = b.cloud.GetGlobalBackendServiceHealth(name, be.Backends[0].Group)
	case meta.Regional:
		hs, err = b.cloud.GetRegionalBackendServiceHealth(name, b.cloud.Region(), be.Backends[0].Group)
	default:
		return "Unknown", fmt.Errorf("invalid scope for Health(): %s", scope)
	}

	if err != nil {
		return "Unknown", fmt.Errorf("error getting health for backend %q: %v", name, err)
	}
	if len(hs.HealthStatus) == 0 || hs.HealthStatus[0] == nil {
		klog.V(3).Infof("backend service %q does not have health status: %v", name, hs.HealthStatus)
		return "Unknown", nil
	}

	// TODO: State transition are important, not just the latest.
	return hs.HealthStatus[0].HealthState, nil
}

// List lists all backends managed by this controller.
func (b *Backends) List(key *meta.Key, version meta.Version) ([]*composite.BackendService, error) {
	// TODO: for consistency with the rest of this sub-package this method
	// should return a list of backend ports.
	var backends []*composite.BackendService
	var err error

	backends, err = composite.ListBackendServices(b.cloud, key, version)
	if err != nil {
		return nil, err
	}

	var clusterBackends []*composite.BackendService

	for _, bs := range backends {
		if b.namer.NameBelongsToCluster(bs.Name) {
			scope, err := composite.ScopeFromSelfLink(bs.SelfLink)
			if err != nil {
				return nil, err
			}
			bs.Scope = scope

			clusterBackends = append(clusterBackends, bs)
		}
	}
	return clusterBackends, nil
}
<commit_msg>Separate out health check errors for Backends<commit_after>/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
    http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package backends

import (
	"fmt"
	"net/http"

	"github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud/meta"
	"google.golang.org/api/compute/v1"
	"k8s.io/ingress-gce/pkg/backends/features"
	"k8s.io/ingress-gce/pkg/composite"
	"k8s.io/ingress-gce/pkg/utils"
	"k8s.io/ingress-gce/pkg/utils/namer"
	"k8s.io/klog"
	"k8s.io/legacy-cloud-providers/gce"
)

// Backends handles CRUD operations for backends.
type Backends struct {
	cloud *gce.Cloud
	namer *namer.Namer
}

// Backends is a Pool.
var _ Pool = (*Backends)(nil)

// NewPool returns a new backend pool.
// - cloud: implements BackendServices
// - namer: produces names for backends.
func NewPool(cloud *gce.Cloud, namer *namer.Namer) *Backends {
	return &Backends{
		cloud: cloud,
		namer: namer,
	}
}

// ensureDescription updates the BackendService Description with the expected value
func ensureDescription(be *composite.BackendService, sp *utils.ServicePort) (needsUpdate bool) {
	desc := sp.GetDescription()
	features.SetDescription(&desc, sp)
	descString := desc.String()
	if be.Description == descString {
		return false
	}
	be.Description = descString
	return true
}

// Create implements Pool.
func (b *Backends) Create(sp utils.ServicePort, hcLink string) (*composite.BackendService, error) {
	name := sp.BackendName(b.namer)
	namedPort := &compute.NamedPort{
		Name: b.namer.NamedPort(sp.NodePort),
		Port: sp.NodePort,
	}

	version := features.VersionFromServicePort(&sp)
	be := &composite.BackendService{
		Version:      version,
		Name:         name,
		Protocol:     string(sp.Protocol),
		Port:         namedPort.Port,
		PortName:     namedPort.Name,
		HealthChecks: []string{hcLink},
	}

	if sp.L7ILBEnabled {
		be.LoadBalancingScheme = "INTERNAL"
	}

	ensureDescription(be, &sp)
	scope := features.ScopeFromServicePort(&sp)
	key, err := composite.CreateKey(b.cloud, name, scope)
	if err != nil {
		return nil, err
	}

	if err := composite.CreateBackendService(b.cloud, key, be); err != nil {
		return nil, err
	}
	// Note: We need to perform a GCE call to re-fetch the object we just created
	// so that the "Fingerprint" field is filled in. This is needed to update the
	// object without error.
	return b.Get(name, version, scope)
}

// Update implements Pool.
func (b *Backends) Update(be *composite.BackendService) error {
	// Ensure the backend service has the proper version before updating.
	be.Version = features.VersionFromDescription(be.Description)
	scope, err := composite.ScopeFromSelfLink(be.SelfLink)
	if err != nil {
		return err
	}

	key, err := composite.CreateKey(b.cloud, be.Name, scope)
	if err != nil {
		return err
	}
	if err := composite.UpdateBackendService(b.cloud, key, be); err != nil {
		return err
	}
	return nil
}

// Get implements Pool.
func (b *Backends) Get(name string, version meta.Version, scope meta.KeyType) (*composite.BackendService, error) {
	key, err := composite.CreateKey(b.cloud, name, scope)
	if err != nil {
		return nil, err
	}
	be, err := composite.GetBackendService(b.cloud, key, version)
	if err != nil {
		return nil, err
	}
	// Evaluate the existing features from description to see if a lower
	// API version is required so that we don't lose information from
	// the existing backend service.
	versionRequired := features.VersionFromDescription(be.Description)

	if features.IsLowerVersion(versionRequired, version) {
		be, err = composite.GetBackendService(b.cloud, key, versionRequired)
		if err != nil {
			return nil, err
		}
	}
	return be, nil
}

// Delete implements Pool.
func (b *Backends) Delete(name string, version meta.Version, scope meta.KeyType) error {
	klog.V(2).Infof("Deleting backend service %v", name)

	key, err := composite.CreateKey(b.cloud, name, scope)
	if err != nil {
		return err
	}
	err = composite.DeleteBackendService(b.cloud, key, version)
	if err != nil {
		if utils.IsHTTPErrorCode(err, http.StatusNotFound) || utils.IsInUsedByError(err) {
			klog.Infof("DeleteBackendService(_, %v, %v) = %v; ignorable error", key, version, err)
			return nil
		}
		klog.Errorf("DeleteBackendService(_, %v, %v) = %v", key, version, err)
		return err
	}
	klog.V(2).Infof("DeleteBackendService(_, %v, %v) ok", key, version)
	return nil
}

// Health implements Pool.
func (b *Backends) Health(name string, version meta.Version, scope meta.KeyType) (string, error) {
	be, err := b.Get(name, version, scope)
	if err != nil {
		return "Unknown", fmt.Errorf("error getting backend service %s: %v", name, err)
	}
	if len(be.Backends) == 0 {
		return "Unknown", fmt.Errorf("no backends found for backend service %q", name)
	}

	// TODO: Look at more than one backend's status
	// TODO: Include port, ip in the status, since it's in the health info.
	// TODO (shance) convert to composite types
	var hs *compute.BackendServiceGroupHealth
	switch scope {
	case meta.Global:
		hs, err = b.cloud.GetGlobalBackendServiceHealth(name, be.Backends[0].Group)
	case meta.Regional:
		hs, err = b.cloud.GetRegionalBackendServiceHealth(name, b.cloud.Region(), be.Backends[0].Group)
	default:
		return "Unknown", fmt.Errorf("invalid scope for Health(): %s", scope)
	}

	if err != nil {
		return "Unknown", fmt.Errorf("error getting health for backend %q: %v", name, err)
	}
	if len(hs.HealthStatus) == 0 || hs.HealthStatus[0] == nil {
		klog.V(3).Infof("backend service %q does not have health status: %v", name, hs.HealthStatus)
		return "Unknown", nil
	}

	// TODO: State transition are important, not just the latest.
	return hs.HealthStatus[0].HealthState, nil
}

// List lists all backends managed by this controller.
func (b *Backends) List(key *meta.Key, version meta.Version) ([]*composite.BackendService, error) {
	// TODO: for consistency with the rest of this sub-package this method
	// should return a list of backend ports.
	var backends []*composite.BackendService
	var err error

	backends, err = composite.ListBackendServices(b.cloud, key, version)
	if err != nil {
		return nil, err
	}

	var clusterBackends []*composite.BackendService

	for _, bs := range backends {
		if b.namer.NameBelongsToCluster(bs.Name) {
			scope, err := composite.ScopeFromSelfLink(bs.SelfLink)
			if err != nil {
				return nil, err
			}
			bs.Scope = scope

			clusterBackends = append(clusterBackends, bs)
		}
	}
	return clusterBackends, nil
}
<|endoftext|>


<commit_before>package cmd

import (
	"errors"
	"fmt"
	"strings"

	"github.com/adamdecaf/cert-manage/pkg/whitelist/gen"
)

func GenerateWhitelist(from, file string) error {
	if from == "" && file == "" {
		return errors.New("you need to specify either -from or -file")
	}
	choices := getChoices(from, file)
	for i := range choices {
		opt := strings.ToLower(strings.TrimSpace(choices[i]))
		switch opt {
		case "browser", "browsers":
			urls, err := gen.FromAllBrowsers()
			if err != nil {
				return err
			}
			fmt.Printf("urls=%d\n", len(urls))
		case "chrome", "edge", "firefox", "ie", "opera", "safari":
			urls, err := gen.FromBrowser(opt)
			if err != nil {
				return err
			}
			fmt.Printf("urls=%d\n", len(urls))
		case "file":
			urls, err := gen.FromFile(file)
			if err != nil {
				return err
			}
			fmt.Printf("urls=%q\n", urls)
		}
	}

	return nil
}

func getChoices(from, file string) []string {
	if !strings.Contains(from, "file") && file != "" {
		if from != "" {
			from += ",file"
		}
		from += "file"
	}
	return strings.Split(from, ",")
}
<commit_msg>cmd/gen-whitelist: cleanup url accumulation<commit_after>package cmd

import (
	"errors"
	"fmt"
	"net/url"
	"os"
	"strings"
	"sync"

	"github.com/adamdecaf/cert-manage/pkg/whitelist/gen"
)

var (
	debug = os.Getenv("DEBUG") != ""
)

func GenerateWhitelist(from, file string) error {
	if from == "" && file == "" {
		return errors.New("you need to specify either -from or -file")
	}

	var accum []*url.URL
	var err error
	var mu sync.Mutex
	uacc := make(chan []*url.URL)
	eacc := make(chan error)

	choices := getChoices(from, file)
	debugLog("running %d choices\n", len(choices))
	for i := range choices {
		opt := strings.ToLower(strings.TrimSpace(choices[i]))
		switch opt {
		case "browser", "browsers":
			debugLog("starting browser url retrieval")
			go accumulateUrls(gen.FromAllBrowsers, uacc, eacc)

		case "chrome", "edge", "firefox", "ie", "opera", "safari":
			debugLog("starting %s url retrieval", opt)
			go accumulateUrls(func() ([]*url.URL, error) {
				return gen.FromBrowser(opt)
			}, uacc, eacc)
		case "file":
			debugLog("grabbing urls from %s", file)
			go accumulateUrls(func() ([]*url.URL, error) {
				return gen.FromFile(file)
			}, uacc, eacc)
		default:
			fmt.Printf("WARNING: Unknown -from option %q\n", opt)
		}
	}

	for _ = range choices {
		select {
		case urls := <-uacc:
			mu.Lock()
			debugLog("accumulating %d urls", len(urls))
			accum = append(accum, urls...)
			mu.Unlock()
		case err1 := <-eacc:
			mu.Lock()
			if err1 != nil {
				err = err1
			}
			mu.Unlock()
		}
	}
	debugLog("cleaning up")
	close(uacc)
	close(eacc)
	fmt.Printf("out=%q\n", accum)
	return err
}

func getChoices(from, file string) []string {
	if !strings.Contains(from, "file") && file != "" {
		if from != "" {
			from += ","
		}
		from += "file"
	}
	return strings.Split(from, ",")
}

func accumulateUrls(f func() ([]*url.URL, error), u chan []*url.URL, e chan error) {
	urls, err := f() // often long blocking call
	if err != nil {
		e <- err
	} else {
		debugLog("adding %d urls", len(urls))
		u <- urls
	}
}

func debugLog(msg string, args ...interface{}) {
	if debug {
		fmt.Printf("cmd/gen-whitelist: "+msg+"\n", args...)
	}
}
<|endoftext|>


<commit_before>package envoy

import (
	"testing"

	v2 "github.com/envoyproxy/go-control-plane/envoy/api/v2"
	route "github.com/envoyproxy/go-control-plane/envoy/api/v2/route"
	"github.com/envoyproxy/go-control-plane/pkg/cache"
	"gotest.tools/assert"
	kubev1 "k8s.io/api/core/v1"
	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/util/intstr"
	"knative.dev/serving/pkg/apis/networking/v1alpha1"
)

// Tests that when there is a traffic split defined in the ingress:
// - Creates a route with weighted clusters defined.
// - There's one weighted cluster for each traffic split and each contains:
// 		- The traffic percentage.
//		- The headers to add (namespace and revision name).
//		- The cluster name, based on the revision name plus the path.
// - The weighted clusters exists also in the clusters cache with the same
//   name.
//
// Note: for now, the name of the cluster is the name of the revision plus the
// path. That might change in the future.
func TestTrafficSplits(t *testing.T) {
	ingress := v1alpha1.Ingress{
		TypeMeta: v1.TypeMeta{
			Kind:       "Ingress",
			APIVersion: "networking.internal.knative.dev/v1alpha1",
		},
		ObjectMeta: v1.ObjectMeta{
			Name: "hello-world",
		},
		Spec: v1alpha1.IngressSpec{
			Rules: []v1alpha1.IngressRule{
				{
					HTTP: &v1alpha1.HTTPIngressRuleValue{
						Paths: []v1alpha1.HTTPIngressPath{
							{
								Splits: []v1alpha1.IngressBackendSplit{
									{
										IngressBackend: v1alpha1.IngressBackend{
											ServiceNamespace: "default",
											ServiceName:      "hello-world-rev1",
											ServicePort: intstr.IntOrString{
												Type:   intstr.Int,
												IntVal: 80,
											},
										},
										Percent: 60,
										AppendHeaders: map[string]string{
											"Knative-Serving-Namespace": "default",
											"Knative-Serving-Revision":  "hello-world-rev1",
										},
									},
									{
										IngressBackend: v1alpha1.IngressBackend{
											ServiceNamespace: "default",
											ServiceName:      "hello-world-rev2",
											ServicePort: intstr.IntOrString{
												Type:   intstr.Int,
												IntVal: 80,
											},
										},
										Percent: 40,
										AppendHeaders: map[string]string{
											"Knative-Serving-Namespace": "default",
											"Knative-Serving-Revision":  "hello-world-rev2",
										},
									},
								},
							},
						},
					},
				},
			},
		},
		Status: v1alpha1.IngressStatus{},
	}

	// Check that there is one route in the result
	caches := CachesForClusterIngresses(
		[]v1alpha1.IngressAccessor{v1alpha1.IngressAccessor(&ingress)},
		newMockedKubeClient(),
		"cluster.local",
	)
	assert.Equal(t, 1, len(caches.routes))

	// Check that there are 2 weighted clusters for the route
	envoyRoute := caches.routes[0].(*route.Route)
	weightedClusters := envoyRoute.GetRoute().GetWeightedClusters().Clusters
	assert.Equal(t, 2, len(weightedClusters))

	// Check the first weighted cluster
	assertWeightedClusterCorrect(
		t,
		weightedClusters[0],
		"hello-world-rev1/",
		uint32(60),
		map[string]string{
			"Knative-Serving-Namespace": "default",
			"Knative-Serving-Revision":  "hello-world-rev1",
		},
	)

	// Check the second weighted cluster
	assertWeightedClusterCorrect(
		t,
		weightedClusters[1],
		"hello-world-rev2/",
		uint32(40),
		map[string]string{
			"Knative-Serving-Namespace": "default",
			"Knative-Serving-Revision":  "hello-world-rev2",
		},
	)

	// Check the clusters cache
	assert.Equal(
		t,
		true,
		clustersExist([]string{"hello-world-rev1/", "hello-world-rev2/"}, caches.clusters),
	)
}

type mockedKubeClient struct{}

func (kubeClient *mockedKubeClient) EndpointsForRevision(namespace string, serviceName string) (*kubev1.EndpointsList, error) {
	list := kubev1.EndpointsList{
		Items: []kubev1.Endpoints{},
	}

	return &list, nil
}

func (kubeClient *mockedKubeClient) ServiceForRevision(namespace string, serviceName string) (*kubev1.Service, error) {
	service := kubev1.Service{
		Spec: kubev1.ServiceSpec{},
	}
	return &service, nil
}

func (kubeClient *mockedKubeClient) GetSecret(namespace string, secretName string) (*kubev1.Secret, error) {
	return nil, nil
}

func newMockedKubeClient() *mockedKubeClient {
	return new(mockedKubeClient)
}

func clustersExist(names []string, clustersCache []cache.Resource) bool {
	// Cast Resources to Clusters
	var clusters []*v2.Cluster
	for _, cacheCluster := range clustersCache {
		clusters = append(clusters, cacheCluster.(*v2.Cluster))
	}

	// Create map that contains names that are present
	present := make(map[string]bool)
	for _, cacheCluster := range clusters {
		present[cacheCluster.Name] = true
	}

	// Verify if the received names are present
	for _, name := range names {
		if !present[name] {
			return false
		}
	}

	return true
}

// Checks whether the weightedCluster received has the received name, traffic
// percentage and headers to add
func assertWeightedClusterCorrect(t *testing.T,
	weightedCluster *route.WeightedCluster_ClusterWeight,
	name string,
	trafficPerc uint32,
	headersToAdd map[string]string) {

	assert.Equal(t, name, weightedCluster.Name)

	assert.Equal(t, trafficPerc, weightedCluster.Weight.Value)

	// Collect headers for easier comparison
	clusterHeaders := make(map[string]string)
	for _, header := range weightedCluster.RequestHeadersToAdd {
		clusterHeaders[header.Header.Key] = header.Header.Value
	}
	assert.DeepEqual(t, clusterHeaders, headersToAdd)
}
<commit_msg>Adapt unit tests to latest changes<commit_after>package envoy

import (
	"testing"

	v2 "github.com/envoyproxy/go-control-plane/envoy/api/v2"
	route "github.com/envoyproxy/go-control-plane/envoy/api/v2/route"
	"github.com/envoyproxy/go-control-plane/pkg/cache"
	"gotest.tools/assert"
	kubev1 "k8s.io/api/core/v1"
	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/util/intstr"
	"knative.dev/serving/pkg/apis/networking/v1alpha1"
)

// Tests that when there is a traffic split defined in the ingress:
// - Creates a route with weighted clusters defined.
// - There's one weighted cluster for each traffic split and each contains:
// 		- The traffic percentage.
//		- The headers to add (namespace and revision name).
//		- The cluster name, based on the revision name plus the path.
// - The weighted clusters exists also in the clusters cache with the same
//   name.
//
// Note: for now, the name of the cluster is the name of the revision plus the
// path. That might change in the future.
func TestTrafficSplits(t *testing.T) {
	ingress := v1alpha1.Ingress{
		TypeMeta: v1.TypeMeta{
			Kind:       "Ingress",
			APIVersion: "networking.internal.knative.dev/v1alpha1",
		},
		ObjectMeta: v1.ObjectMeta{
			Name: "hello-world",
		},
		Spec: v1alpha1.IngressSpec{
			Rules: []v1alpha1.IngressRule{
				{
					HTTP: &v1alpha1.HTTPIngressRuleValue{
						Paths: []v1alpha1.HTTPIngressPath{
							{
								Splits: []v1alpha1.IngressBackendSplit{
									{
										IngressBackend: v1alpha1.IngressBackend{
											ServiceNamespace: "default",
											ServiceName:      "hello-world-rev1",
											ServicePort: intstr.IntOrString{
												Type:   intstr.Int,
												IntVal: 80,
											},
										},
										Percent: 60,
										AppendHeaders: map[string]string{
											"Knative-Serving-Namespace": "default",
											"Knative-Serving-Revision":  "hello-world-rev1",
										},
									},
									{
										IngressBackend: v1alpha1.IngressBackend{
											ServiceNamespace: "default",
											ServiceName:      "hello-world-rev2",
											ServicePort: intstr.IntOrString{
												Type:   intstr.Int,
												IntVal: 80,
											},
										},
										Percent: 40,
										AppendHeaders: map[string]string{
											"Knative-Serving-Namespace": "default",
											"Knative-Serving-Revision":  "hello-world-rev2",
										},
									},
								},
							},
						},
					},
				},
			},
		},
		Status: v1alpha1.IngressStatus{},
	}

	// Check that there is one route in the result
	caches := CachesForClusterIngresses(
		[]v1alpha1.IngressAccessor{v1alpha1.IngressAccessor(&ingress)},
		newMockedKubeClient(),
		"cluster.local",
	)
	assert.Equal(t, 1, len(caches.routes))

	// Check that there are 2 weighted clusters for the route
	envoyRoute := caches.routes[0].(*route.Route)
	weightedClusters := envoyRoute.GetRoute().GetWeightedClusters().Clusters
	assert.Equal(t, 2, len(weightedClusters))

	// Check the first weighted cluster
	assertWeightedClusterCorrect(
		t,
		weightedClusters[0],
		"hello-world0/",
		uint32(60),
		map[string]string{
			"Knative-Serving-Namespace": "default",
			"Knative-Serving-Revision":  "hello-world-rev1",
		},
	)

	// Check the second weighted cluster
	assertWeightedClusterCorrect(
		t,
		weightedClusters[1],
		"hello-world1/",
		uint32(40),
		map[string]string{
			"Knative-Serving-Namespace": "default",
			"Knative-Serving-Revision":  "hello-world-rev2",
		},
	)

	// Check the clusters cache
	assert.Equal(
		t,
		true,
		clustersExist([]string{"hello-world0/", "hello-world1/"}, caches.clusters),
	)
}

type mockedKubeClient struct{}

func (kubeClient *mockedKubeClient) EndpointsForRevision(namespace string, serviceName string) (*kubev1.EndpointsList, error) {
	list := kubev1.EndpointsList{
		Items: []kubev1.Endpoints{},
	}

	return &list, nil
}

func (kubeClient *mockedKubeClient) ServiceForRevision(namespace string, serviceName string) (*kubev1.Service, error) {
	service := kubev1.Service{
		Spec: kubev1.ServiceSpec{},
	}
	return &service, nil
}

func (kubeClient *mockedKubeClient) GetSecret(namespace string, secretName string) (*kubev1.Secret, error) {
	return nil, nil
}

func newMockedKubeClient() *mockedKubeClient {
	return new(mockedKubeClient)
}

func clustersExist(names []string, clustersCache []cache.Resource) bool {
	// Cast Resources to Clusters
	var clusters []*v2.Cluster
	for _, cacheCluster := range clustersCache {
		clusters = append(clusters, cacheCluster.(*v2.Cluster))
	}

	// Create map that contains names that are present
	present := make(map[string]bool)
	for _, cacheCluster := range clusters {
		present[cacheCluster.Name] = true
	}

	// Verify if the received names are present
	for _, name := range names {
		if !present[name] {
			return false
		}
	}

	return true
}

// Checks whether the weightedCluster received has the received name, traffic
// percentage and headers to add
func assertWeightedClusterCorrect(t *testing.T,
	weightedCluster *route.WeightedCluster_ClusterWeight,
	name string,
	trafficPerc uint32,
	headersToAdd map[string]string) {

	assert.Equal(t, name, weightedCluster.Name)

	assert.Equal(t, trafficPerc, weightedCluster.Weight.Value)

	// Collect headers for easier comparison
	clusterHeaders := make(map[string]string)
	for _, header := range weightedCluster.RequestHeadersToAdd {
		clusterHeaders[header.Header.Key] = header.Header.Value
	}
	assert.DeepEqual(t, clusterHeaders, headersToAdd)
}
<|endoftext|>


<commit_before>package helm

import (
	"testing"

	chartutil "github.com/kubernetes/helm/pkg/chart"
	"gopkg.in/yaml.v2"
)

func TestInstallReleaseOverrides(t *testing.T) {
	vals := `name = "mariner"`
	ch := "./testdata/albatross"
	ir, err := InstallRelease([]byte(vals), ch, true)
	if err != nil {
		t.Fatalf("Failed to release: %s", err)
	}

	if len(ir.Release.Manifest) == 0 {
		t.Fatalf("Expected a manifest.")
	}

	// Parse the result and see if the override worked
	d := map[string]interface{}{}
	if err := yaml.Unmarshal([]byte(ir.Release.Manifest), d); err != nil {
		t.Fatalf("Failed to unmarshal manifest: %s", err)
	}

	if d["name"] != "mariner" {
		t.Errorf("Unexpected name %q", d["name"])
	}

	if d["home"] != "nest" {
		t.Errorf("Unexpected home %q", d["home"])
	}
}

func TestOverridesToProto(t *testing.T) {
	override := []byte(`test = "foo"`)
	c := OverridesToProto(override)
	if c.Raw != string(override) {
		t.Errorf("Expected %q to match %q", c.Raw, override)
	}
}

func TestChartToProto(t *testing.T) {
	c, err := chartutil.LoadDir("./testdata/albatross")
	if err != nil {
		t.Fatalf("failed to load testdata chart: %s", err)
	}

	p, err := ChartToProto(c)
	if err != nil {
		t.Fatalf("failed to conver chart to proto: %s", err)
	}

	if p.Metadata.Name != c.Chartfile().Name {
		t.Errorf("Expected names to match.")
	}
}
<commit_msg>fix(pkg/helm): skip test that can't run in Circle<commit_after>package helm

import (
	"testing"

	chartutil "github.com/kubernetes/helm/pkg/chart"
	"gopkg.in/yaml.v2"
)

func TestInstallReleaseOverrides(t *testing.T) {
	// FIXME: This can't currently run unless a Tiller server is running, simply
	// because --dry-run still uses the server. There's already a WIP for a
	// testing harness, so this can be ported when that is done.
	t.Skip()

	vals := `name = "mariner"`
	ch := "./testdata/albatross"
	ir, err := InstallRelease([]byte(vals), ch, true)
	if err != nil {
		t.Fatalf("Failed to release: %s", err)
	}

	if len(ir.Release.Manifest) == 0 {
		t.Fatalf("Expected a manifest.")
	}

	// Parse the result and see if the override worked
	d := map[string]interface{}{}
	if err := yaml.Unmarshal([]byte(ir.Release.Manifest), d); err != nil {
		t.Fatalf("Failed to unmarshal manifest: %s", err)
	}

	if d["name"] != "mariner" {
		t.Errorf("Unexpected name %q", d["name"])
	}

	if d["home"] != "nest" {
		t.Errorf("Unexpected home %q", d["home"])
	}
}

func TestOverridesToProto(t *testing.T) {
	override := []byte(`test = "foo"`)
	c := OverridesToProto(override)
	if c.Raw != string(override) {
		t.Errorf("Expected %q to match %q", c.Raw, override)
	}
}

func TestChartToProto(t *testing.T) {
	c, err := chartutil.LoadDir("./testdata/albatross")
	if err != nil {
		t.Fatalf("failed to load testdata chart: %s", err)
	}

	p, err := ChartToProto(c)
	if err != nil {
		t.Fatalf("failed to conver chart to proto: %s", err)
	}

	if p.Metadata.Name != c.Chartfile().Name {
		t.Errorf("Expected names to match.")
	}
}
<|endoftext|>


<commit_before>package acme

import (
	"context"
	"crypto/rsa"
	"fmt"
	"strings"

	"golang.org/x/crypto/acme"
	"k8s.io/api/core/v1"
	k8sErrors "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

	"github.com/golang/glog"
	"github.com/jetstack-experimental/cert-manager/pkg/apis/certmanager/v1alpha1"
	"github.com/jetstack-experimental/cert-manager/pkg/util/errors"
	"github.com/jetstack-experimental/cert-manager/pkg/util/kube"
	"github.com/jetstack-experimental/cert-manager/pkg/util/pki"
)

const (
	errorAccountRegistrationFailed = "ErrRegisterACMEAccount"
	errorAccountVerificationFailed = "ErrVerifyACMEAccount"

	successAccountRegistered = "ACMEAccountRegistered"
	successAccountVerified   = "ACMEAccountVerified"

	messageAccountRegistrationFailed = "Failed to register ACME account: "
	messageAccountVerificationFailed = "Failed to verify ACME account: "
	messageAccountRegistered         = "The ACME account was registered with the ACME server"
	messageAccountVerified           = "The ACME account was verified with the ACME server"
)

func (a *Acme) Setup(ctx context.Context) error {
	glog.V(4).Infof("%s: getting acme account private key '%s/%s'", a.issuer.GetObjectMeta().Name, a.resourceNamespace, a.issuer.GetSpec().ACME.PrivateKey.Name)
	cl, err := a.acmeClient()
	if k8sErrors.IsNotFound(err) || errors.IsInvalidData(err) {
		glog.V(4).Infof("%s: generating acme account private key '%s/%s'", a.issuer.GetObjectMeta().Name, a.resourceNamespace, a.issuer.GetSpec().ACME.PrivateKey.Name)
		var accountPrivKey *rsa.PrivateKey
		accountPrivKey, err = a.createAccountPrivateKey()
		if err != nil {
			s := messageAccountRegistrationFailed + err.Error()
			a.issuer.UpdateStatusCondition(v1alpha1.IssuerConditionReady, v1alpha1.ConditionFalse, errorAccountRegistrationFailed, s)
			return fmt.Errorf(s)
		}
		cl = &acme.Client{
			Key:          accountPrivKey,
			DirectoryURL: a.issuer.GetSpec().ACME.Server,
		}
	}
	if err != nil {
		s := messageAccountVerificationFailed + err.Error()
		glog.V(4).Infof("%s: %s", a.issuer.GetObjectMeta().Name, s)
		a.recorder.Event(a.issuer, v1.EventTypeWarning, errorAccountVerificationFailed, s)
		return err
	}

	glog.V(4).Infof("Verifying ")
	glog.V(4).Infof("%s: verifying existing registration with ACME server", a.issuer.GetObjectMeta().Name)
	_, err = cl.GetReg(ctx, a.issuer.GetStatus().ACMEStatus().URI)

	if err == nil {
		glog.V(4).Infof("%s: verified existing registration with ACME server", a.issuer.GetObjectMeta().Name)
		a.issuer.UpdateStatusCondition(v1alpha1.IssuerConditionReady, v1alpha1.ConditionTrue, successAccountVerified, messageAccountVerified)
		return nil
	}

	s := messageAccountVerificationFailed + err.Error()
	glog.V(4).Infof("%s: %s", a.issuer.GetObjectMeta().Name, s)
	a.recorder.Event(a.issuer, v1.EventTypeWarning, errorAccountVerificationFailed, s)

	acc := &acme.Account{
		Contact: []string{fmt.Sprintf("mailto:%s", strings.ToLower(a.issuer.GetSpec().ACME.Email))},
	}

	account, err := cl.Register(ctx, acc, acme.AcceptTOS)
	if err != nil {
		s := messageAccountRegistrationFailed + err.Error()
		a.issuer.UpdateStatusCondition(v1alpha1.IssuerConditionReady, v1alpha1.ConditionFalse, errorAccountRegistrationFailed, s)
		return err
	}

	a.issuer.UpdateStatusCondition(v1alpha1.IssuerConditionReady, v1alpha1.ConditionTrue, successAccountRegistered, messageAccountRegistered)
	a.issuer.GetStatus().ACMEStatus().URI = account.URI

	return nil
}

func (a *Acme) createAccountPrivateKey() (*rsa.PrivateKey, error) {
	secretName, secretKey := a.acmeAccountPrivateKeyMeta()
	accountPrivKey, err := pki.GenerateRSAPrivateKey(2048)
	if err != nil {
		return nil, err
	}

	_, err = kube.EnsureSecret(a.client, &v1.Secret{
		ObjectMeta: metav1.ObjectMeta{
			Name:      secretName,
			Namespace: a.resourceNamespace,
		},
		Data: map[string][]byte{
			secretKey: pki.EncodePKCS1PrivateKey(accountPrivKey),
		},
	})

	if err != nil {
		return nil, err
	}

	return accountPrivKey, err
}
<commit_msg>Allow rediscovering lost/not set ACME account URIs<commit_after>package acme

import (
	"context"
	"crypto/rsa"
	"fmt"
	"net/http"
	"strings"

	"golang.org/x/crypto/acme"
	"k8s.io/api/core/v1"
	k8sErrors "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

	"github.com/golang/glog"
	"github.com/jetstack-experimental/cert-manager/pkg/apis/certmanager/v1alpha1"
	"github.com/jetstack-experimental/cert-manager/pkg/util/errors"
	"github.com/jetstack-experimental/cert-manager/pkg/util/kube"
	"github.com/jetstack-experimental/cert-manager/pkg/util/pki"
)

const (
	errorAccountRegistrationFailed = "ErrRegisterACMEAccount"
	errorAccountVerificationFailed = "ErrVerifyACMEAccount"

	successAccountRegistered = "ACMEAccountRegistered"
	successAccountVerified   = "ACMEAccountVerified"

	messageAccountRegistrationFailed = "Failed to register ACME account: "
	messageAccountVerificationFailed = "Failed to verify ACME account: "
	messageAccountRegistered         = "The ACME account was registered with the ACME server"
	messageAccountVerified           = "The ACME account was verified with the ACME server"
)

// Setup will verify an existing ACME registration, or create one if not
// already registered.
func (a *Acme) Setup(ctx context.Context) error {
	glog.V(4).Infof("%s: getting acme account private key '%s/%s'", a.issuer.GetObjectMeta().Name, a.resourceNamespace, a.issuer.GetSpec().ACME.PrivateKey.Name)
	cl, err := a.acmeClient()
	if k8sErrors.IsNotFound(err) || errors.IsInvalidData(err) {
		glog.V(4).Infof("%s: generating acme account private key '%s/%s'", a.issuer.GetObjectMeta().Name, a.resourceNamespace, a.issuer.GetSpec().ACME.PrivateKey.Name)
		accountPrivKey, err := a.createAccountPrivateKey()
		if err != nil {
			s := messageAccountRegistrationFailed + err.Error()
			a.issuer.UpdateStatusCondition(v1alpha1.IssuerConditionReady, v1alpha1.ConditionFalse, errorAccountRegistrationFailed, s)
			return fmt.Errorf(s)
		}
		a.issuer.GetStatus().ACMEStatus().URI = ""
		cl = &acme.Client{
			Key:          accountPrivKey,
			DirectoryURL: a.issuer.GetSpec().ACME.Server,
		}
	} else if err != nil {
		s := messageAccountVerificationFailed + err.Error()
		glog.V(4).Infof("%s: %s", a.issuer.GetObjectMeta().Name, s)
		a.recorder.Event(a.issuer, v1.EventTypeWarning, errorAccountVerificationFailed, s)
		return err
	}

	account, err := a.registerAccount(ctx, cl)
	if err != nil {
		s := messageAccountVerificationFailed + err.Error()
		glog.V(4).Infof("%s: %s", a.issuer.GetObjectMeta().Name, s)
		a.recorder.Event(a.issuer, v1.EventTypeWarning, errorAccountVerificationFailed, s)
		a.issuer.UpdateStatusCondition(v1alpha1.IssuerConditionReady, v1alpha1.ConditionFalse, errorAccountRegistrationFailed, s)
		return err
	}

	glog.V(4).Infof("%s: verified existing registration with ACME server", a.issuer.GetObjectMeta().Name)
	a.issuer.UpdateStatusCondition(v1alpha1.IssuerConditionReady, v1alpha1.ConditionTrue, successAccountRegistered, messageAccountRegistered)
	a.issuer.GetStatus().ACMEStatus().URI = account.URI

	return nil
}

// registerAccount will register a new ACME account with the server. If an
// account with the clients private key already exists, it will attempt to look
// up and verify the corresponding account, and will return that. If this fails
// it will return the
func (a *Acme) registerAccount(ctx context.Context, cl *acme.Client) (*acme.Account, error) {
	acc := &acme.Account{
		Contact: []string{fmt.Sprintf("mailto:%s", strings.ToLower(a.issuer.GetSpec().ACME.Email))},
	}
	acc, err := cl.Register(ctx, acc, acme.AcceptTOS)
	if err != nil {
		typedErr, ok := err.(*acme.Error)
		// if this isn't an ACME error, we should just return it
		if !ok {
			return nil, err
		}
		if typedErr.StatusCode == http.StatusConflict {
			accountUri := typedErr.Header.Get("Location")
			if accountUri == "" {
				return nil, fmt.Errorf("unexpected error - 409 Conflict error returned, but no Location header set: %s", typedErr.Error())
			}
			return a.verifyAccount(ctx, cl, accountUri)
		}
	}
	return acc, nil
}

// verifyAccount will verify an ACME account with the given URI.
func (a *Acme) verifyAccount(ctx context.Context, cl *acme.Client, uri string) (*acme.Account, error) {
	acc, err := cl.GetReg(ctx, uri)
	if err != nil {
		return nil, err
	}
	return acc, nil
}

func (a *Acme) createAccountPrivateKey() (*rsa.PrivateKey, error) {
	secretName, secretKey := a.acmeAccountPrivateKeyMeta()
	accountPrivKey, err := pki.GenerateRSAPrivateKey(2048)
	if err != nil {
		return nil, err
	}

	_, err = kube.EnsureSecret(a.client, &v1.Secret{
		ObjectMeta: metav1.ObjectMeta{
			Name:      secretName,
			Namespace: a.resourceNamespace,
		},
		Data: map[string][]byte{
			secretKey: pki.EncodePKCS1PrivateKey(accountPrivKey),
		},
	})

	if err != nil {
		return nil, err
	}

	return accountPrivKey, err
}
<|endoftext|>


<commit_before>// Copyright 2020 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package watchers

import (
	"github.com/cilium/cilium/pkg/comparator"
	"github.com/cilium/cilium/pkg/k8s"
	"github.com/cilium/cilium/pkg/k8s/informer"
	slim_corev1 "github.com/cilium/cilium/pkg/k8s/slim/k8s/api/core/v1"
	"github.com/cilium/cilium/pkg/node"
	nodeTypes "github.com/cilium/cilium/pkg/node/types"
	"github.com/cilium/cilium/pkg/option"

	v1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/fields"
	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/tools/cache"
)

func (k *K8sWatcher) nodesInit(k8sClient kubernetes.Interface) {
	_, nodeController := informer.NewInformer(
		cache.NewListWatchFromClient(k8sClient.CoreV1().RESTClient(),
			"nodes", v1.NamespaceAll, fields.ParseSelectorOrDie("metadata.name="+nodeTypes.GetName())),
		&slim_corev1.Node{},
		0,
		cache.ResourceEventHandlerFuncs{
			AddFunc: func(obj interface{}) {
				var valid bool
				if node := k8s.ObjToV1Node(obj); node != nil {
					valid = true
					err := k.updateK8sNodeV1(nil, node)
					k.K8sEventProcessed(metricNode, metricCreate, err == nil)
				}
				k.K8sEventReceived(metricNode, metricCreate, valid, false)
			},
			UpdateFunc: func(oldObj, newObj interface{}) {
				var valid, equal bool
				if oldNode := k8s.ObjToV1Node(oldObj); oldNode != nil {
					valid = true
					if newNode := k8s.ObjToV1Node(newObj); newNode != nil {
						oldNodeLabels := oldNode.GetLabels()
						newNodeLabels := newNode.GetLabels()
						if comparator.MapStringEquals(oldNodeLabels, newNodeLabels) {
							equal = true
						} else {
							err := k.updateK8sNodeV1(oldNode, newNode)
							k.K8sEventProcessed(metricNode, metricUpdate, err == nil)
						}
					}
				}
				k.K8sEventReceived(metricNode, metricUpdate, valid, equal)
			},
		},
		nil,
	)

	k.blockWaitGroupToSyncResources(wait.NeverStop, nil, nodeController.HasSynced, k8sAPIGroupNodeV1Core)
	go nodeController.Run(wait.NeverStop)
	k.k8sAPIGroups.AddAPI(k8sAPIGroupNodeV1Core)
}

func (k *K8sWatcher) updateK8sNodeV1(oldK8sNode, newK8sNode *slim_corev1.Node) error {
	var oldNodeLabels map[string]string
	if oldK8sNode != nil {
		oldNodeLabels = oldK8sNode.GetLabels()
	}
	newNodeLabels := newK8sNode.GetLabels()

	nodeEP := k.endpointManager.GetHostEndpoint()
	if nodeEP == nil {
		log.Debug("Host endpoint not found, updating node labels")
		node.SetLabels(newNodeLabels)
		return nil
	}

	err := updateEndpointLabels(nodeEP, oldNodeLabels, newNodeLabels)
	if err != nil {
		return err
	}

	if option.Config.BGPAnnounceLBIP {
		k.bgpSpeakerManager.OnUpdateNode(newK8sNode)
	}

	return nil
}
<commit_msg>watchers: Fix BGP subscriber potentially getting skipped<commit_after>// Copyright 2020 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package watchers

import (
	"github.com/cilium/cilium/pkg/comparator"
	"github.com/cilium/cilium/pkg/k8s"
	"github.com/cilium/cilium/pkg/k8s/informer"
	slim_corev1 "github.com/cilium/cilium/pkg/k8s/slim/k8s/api/core/v1"
	"github.com/cilium/cilium/pkg/node"
	nodeTypes "github.com/cilium/cilium/pkg/node/types"
	"github.com/cilium/cilium/pkg/option"

	v1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/fields"
	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/tools/cache"
)

func (k *K8sWatcher) nodesInit(k8sClient kubernetes.Interface) {
	_, nodeController := informer.NewInformer(
		cache.NewListWatchFromClient(k8sClient.CoreV1().RESTClient(),
			"nodes", v1.NamespaceAll, fields.ParseSelectorOrDie("metadata.name="+nodeTypes.GetName())),
		&slim_corev1.Node{},
		0,
		cache.ResourceEventHandlerFuncs{
			AddFunc: func(obj interface{}) {
				var valid bool
				if node := k8s.ObjToV1Node(obj); node != nil {
					valid = true
					err := k.updateK8sNodeV1(nil, node)
					k.K8sEventProcessed(metricNode, metricCreate, err == nil)
				}
				k.K8sEventReceived(metricNode, metricCreate, valid, false)
			},
			UpdateFunc: func(oldObj, newObj interface{}) {
				var valid, equal bool
				if oldNode := k8s.ObjToV1Node(oldObj); oldNode != nil {
					valid = true
					if newNode := k8s.ObjToV1Node(newObj); newNode != nil {
						oldNodeLabels := oldNode.GetLabels()
						newNodeLabels := newNode.GetLabels()
						if comparator.MapStringEquals(oldNodeLabels, newNodeLabels) {
							equal = true
						} else {
							err := k.updateK8sNodeV1(oldNode, newNode)
							k.K8sEventProcessed(metricNode, metricUpdate, err == nil)
						}
					}
				}
				k.K8sEventReceived(metricNode, metricUpdate, valid, equal)
			},
		},
		nil,
	)

	k.blockWaitGroupToSyncResources(wait.NeverStop, nil, nodeController.HasSynced, k8sAPIGroupNodeV1Core)
	go nodeController.Run(wait.NeverStop)
	k.k8sAPIGroups.AddAPI(k8sAPIGroupNodeV1Core)
}

func (k *K8sWatcher) updateK8sNodeV1(oldK8sNode, newK8sNode *slim_corev1.Node) error {
	var oldNodeLabels map[string]string
	if oldK8sNode != nil {
		oldNodeLabels = oldK8sNode.GetLabels()
	}
	newNodeLabels := newK8sNode.GetLabels()

	if option.Config.BGPAnnounceLBIP {
		k.bgpSpeakerManager.OnUpdateNode(newK8sNode)
	}

	nodeEP := k.endpointManager.GetHostEndpoint()
	if nodeEP == nil {
		log.Debug("Host endpoint not found, updating node labels")
		node.SetLabels(newNodeLabels)
		return nil
	}

	err := updateEndpointLabels(nodeEP, oldNodeLabels, newNodeLabels)
	if err != nil {
		return err
	}
	return nil
}
<|endoftext|>


<commit_before>/*
Copyright The Helm Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package fake

import (
	"io"
	"strings"
	"time"

	v1 "k8s.io/api/core/v1"
	"k8s.io/cli-runtime/pkg/resource"

	"helm.sh/helm/v3/pkg/kube"
)

// PrintingKubeClient implements KubeClient, but simply prints the reader to
// the given output.
type PrintingKubeClient struct {
	Out io.Writer
}

// IsReachable checks if the cluster is reachable
func (p *PrintingKubeClient) IsReachable() error {
	return nil
}

// Create prints the values of what would be created with a real KubeClient.
func (p *PrintingKubeClient) Create(resources kube.ResourceList) (*kube.Result, error) {
	_, err := io.Copy(p.Out, bufferize(resources))
	if err != nil {
		return nil, err
	}
	return &kube.Result{Created: resources}, nil
}

func (p *PrintingKubeClient) Wait(resources kube.ResourceList, d time.Duration) error {
	_, err := io.Copy(p.Out, bufferize(resources))
	return err
}

func (p *PrintingKubeClient) WaitWithJobs(resources kube.ResourceList, _ time.Duration) error {
	_, err := io.Copy(p.Out, bufferize(resources))
	return err
}

// Delete implements KubeClient delete.
//
// It only prints out the content to be deleted.
func (p *PrintingKubeClient) Delete(resources kube.ResourceList) (*kube.Result, []error) {
	_, err := io.Copy(p.Out, bufferize(resources))
	if err != nil {
		return nil, []error{err}
	}
	return &kube.Result{Deleted: resources}, nil
}

// WatchUntilReady implements KubeClient WatchUntilReady.
func (p *PrintingKubeClient) WatchUntilReady(resources kube.ResourceList, _ time.Duration) error {
	_, err := io.Copy(p.Out, bufferize(resources))
	return err
}

// Update implements KubeClient Update.
func (p *PrintingKubeClient) Update(_, modified kube.ResourceList, _ bool) (*kube.Result, error) {
	_, err := io.Copy(p.Out, bufferize(modified))
	if err != nil {
		return nil, err
	}
	// TODO: This doesn't completely mock out have some that get created,
	// updated, and deleted. I don't think these are used in any unit tests, but
	// we may want to refactor a way to handle future tests
	return &kube.Result{Updated: modified}, nil
}

// Build implements KubeClient Build.
func (p *PrintingKubeClient) Build(_ io.Reader, _ bool) (kube.ResourceList, error) {
	return []*resource.Info{}, nil
}

// WaitAndGetCompletedPodPhase implements KubeClient WaitAndGetCompletedPodPhase.
func (p *PrintingKubeClient) WaitAndGetCompletedPodPhase(_ string, _ time.Duration) (v1.PodPhase, error) {
	return v1.PodSucceeded, nil
}

func bufferize(resources kube.ResourceList) io.Reader {
	var builder strings.Builder
	for _, info := range resources {
		builder.WriteString(info.String() + "\n")
	}
	return strings.NewReader(builder.String())
}
<commit_msg>Revert change in PrintingKubeClient<commit_after>/*
Copyright The Helm Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package fake

import (
	"io"
	"strings"
	"time"

	v1 "k8s.io/api/core/v1"
	"k8s.io/cli-runtime/pkg/resource"

	"helm.sh/helm/v3/pkg/kube"
)

// PrintingKubeClient implements KubeClient, but simply prints the reader to
// the given output.
type PrintingKubeClient struct {
	Out io.Writer
}

// IsReachable checks if the cluster is reachable
func (p *PrintingKubeClient) IsReachable() error {
	return nil
}

// Create prints the values of what would be created with a real KubeClient.
func (p *PrintingKubeClient) Create(resources kube.ResourceList) (*kube.Result, error) {
	_, err := io.Copy(p.Out, bufferize(resources))
	if err != nil {
		return nil, err
	}
	return &kube.Result{Created: resources}, nil
}

func (p *PrintingKubeClient) Wait(resources kube.ResourceList, _ time.Duration) error {
	_, err := io.Copy(p.Out, bufferize(resources))
	return err
}

func (p *PrintingKubeClient) WaitWithJobs(resources kube.ResourceList, _ time.Duration) error {
	_, err := io.Copy(p.Out, bufferize(resources))
	return err
}

// Delete implements KubeClient delete.
//
// It only prints out the content to be deleted.
func (p *PrintingKubeClient) Delete(resources kube.ResourceList) (*kube.Result, []error) {
	_, err := io.Copy(p.Out, bufferize(resources))
	if err != nil {
		return nil, []error{err}
	}
	return &kube.Result{Deleted: resources}, nil
}

// WatchUntilReady implements KubeClient WatchUntilReady.
func (p *PrintingKubeClient) WatchUntilReady(resources kube.ResourceList, _ time.Duration) error {
	_, err := io.Copy(p.Out, bufferize(resources))
	return err
}

// Update implements KubeClient Update.
func (p *PrintingKubeClient) Update(_, modified kube.ResourceList, _ bool) (*kube.Result, error) {
	_, err := io.Copy(p.Out, bufferize(modified))
	if err != nil {
		return nil, err
	}
	// TODO: This doesn't completely mock out have some that get created,
	// updated, and deleted. I don't think these are used in any unit tests, but
	// we may want to refactor a way to handle future tests
	return &kube.Result{Updated: modified}, nil
}

// Build implements KubeClient Build.
func (p *PrintingKubeClient) Build(_ io.Reader, _ bool) (kube.ResourceList, error) {
	return []*resource.Info{}, nil
}

// WaitAndGetCompletedPodPhase implements KubeClient WaitAndGetCompletedPodPhase.
func (p *PrintingKubeClient) WaitAndGetCompletedPodPhase(_ string, _ time.Duration) (v1.PodPhase, error) {
	return v1.PodSucceeded, nil
}

func bufferize(resources kube.ResourceList) io.Reader {
	var builder strings.Builder
	for _, info := range resources {
		builder.WriteString(info.String() + "\n")
	}
	return strings.NewReader(builder.String())
}
<|endoftext|>


<commit_before>package s3

import (
	"context"
	"fmt"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/libopenstorage/secrets/aws/credentials"
	stork_api "github.com/libopenstorage/stork/pkg/apis/stork/v1alpha1"
	"github.com/libopenstorage/stork/pkg/objectstore/common"
	"gocloud.dev/blob"
	"gocloud.dev/blob/s3blob"
)

func getSession(backupLocation *stork_api.BackupLocation) (*session.Session, error) {
	// AWS SDK fetches the correct endpoint based on region provided if endpoint is passed empty
	var endpoint string
	if backupLocation.Location.S3Config.Endpoint == "s3.amazonaws.com" {
		endpoint = ""
	} else {
		endpoint = backupLocation.Location.S3Config.Endpoint
	}
	awsCreds, err := credentials.NewAWSCredentials(
		backupLocation.Location.S3Config.AccessKeyID,
		backupLocation.Location.S3Config.SecretAccessKey,
		"",
		backupLocation.Location.S3Config.UseIam, // runningOnEc2 when set ec2 role credentials will be used
	)
	if err != nil {
		return nil, err
	}
	creds, err := awsCreds.Get()
	if err != nil {
		return nil, err
	}

	return session.NewSession(&aws.Config{
		Endpoint:         aws.String(endpoint),
		Credentials:      creds,
		Region:           aws.String(backupLocation.Location.S3Config.Region),
		DisableSSL:       aws.Bool(backupLocation.Location.S3Config.DisableSSL),
		S3ForcePathStyle: aws.Bool(true),
	})
}

// GetBucket gets a reference to the bucket for that backup location
func GetBucket(backupLocation *stork_api.BackupLocation) (*blob.Bucket, error) {
	sess, err := getSession(backupLocation)
	if err != nil {
		return nil, err
	}
	return s3blob.OpenBucket(context.Background(), sess, backupLocation.Location.Path, nil)
}

// CreateBucket creates a bucket for the bucket location
func CreateBucket(backupLocation *stork_api.BackupLocation) error {
	sess, err := getSession(backupLocation)
	if err != nil {
		return err
	}

	input := &s3.CreateBucketInput{
		Bucket: &backupLocation.Location.Path,
	}
	_, err = s3.New(sess).CreateBucket(input)
	if err != nil {
		if awsErr, ok := err.(awserr.Error); ok {
			if awsErr.Code() == s3.ErrCodeBucketAlreadyOwnedByYou {
				return nil
			}
		}
	}
	return err
}

// GetObjLockInfo fetches the object lock configuration of a S3 bucket
func GetObjLockInfo(backupLocation *stork_api.BackupLocation) (*common.ObjLockInfo, error) {
	sess, err := getSession(backupLocation)
	if err != nil {
		return nil, err
	}

	input := &s3.GetObjectLockConfigurationInput{
		Bucket: &backupLocation.Location.Path,
	}

	objLockInfo := &common.ObjLockInfo{}
	out, err := s3.New(sess).GetObjectLockConfiguration(input)
	if err != nil {
		if awsErr, ok := err.(awserr.Error); ok {
			// When a Minio server doesn't have object-lock implemented then above API
			// throws following error codes depending on version it runs for normal buckets
			//	1. "ObjectLockConfigurationNotFoundError"
			//  2. "MethodNotAllowed"
			// Similarly in case of AWS, we need to ignore "NoSuchBucket" so that
			// px-backup/stork can create the bucket on behalf user when validation flag is not set.
			if awsErr.Code() == "ObjectLockConfigurationNotFoundError" ||
				awsErr.Code() == "MethodNotAllowed" ||
				awsErr.Code() == "NoSuchBucket" {
				// for a non-objectlocked bucket we needn't throw error
				return objLockInfo, nil
			}
		}
		return nil, err
	}
	if (out != nil) && (out.ObjectLockConfiguration != nil) {
		if aws.StringValue(out.ObjectLockConfiguration.ObjectLockEnabled) == "Enabled" {
			objLockInfo.LockEnabled = true
		}
		if out.ObjectLockConfiguration.Rule != nil &&
			out.ObjectLockConfiguration.Rule.DefaultRetention != nil {
			objLockInfo.LockMode = aws.StringValue(out.ObjectLockConfiguration.Rule.DefaultRetention.Mode)
			objLockInfo.RetentionPeriodYears = aws.Int64Value(out.ObjectLockConfiguration.Rule.DefaultRetention.Years)
			objLockInfo.RetentionPeriodDays = aws.Int64Value(out.ObjectLockConfiguration.Rule.DefaultRetention.Days)
		} else {
			//This is an invalid object-lock config, no default-retention but object-loc enabled
			objLockInfo.LockEnabled = false
			return nil, fmt.Errorf("invalid config: object lock is enabled but default retention period is not set on the bucket")
		}
	}
	return objLockInfo, err
}
<commit_msg>pb-2375: Fixed issue in handling the return value of GetObjectLockConfiguration call from FB and Dell ECS objecstore, when bucket is not enabled with lock<commit_after>package s3

import (
	"context"
	"fmt"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/libopenstorage/secrets/aws/credentials"
	stork_api "github.com/libopenstorage/stork/pkg/apis/stork/v1alpha1"
	"github.com/libopenstorage/stork/pkg/objectstore/common"
	"gocloud.dev/blob"
	"gocloud.dev/blob/s3blob"
)

func getSession(backupLocation *stork_api.BackupLocation) (*session.Session, error) {
	// AWS SDK fetches the correct endpoint based on region provided if endpoint is passed empty
	var endpoint string
	if backupLocation.Location.S3Config.Endpoint == "s3.amazonaws.com" {
		endpoint = ""
	} else {
		endpoint = backupLocation.Location.S3Config.Endpoint
	}
	awsCreds, err := credentials.NewAWSCredentials(
		backupLocation.Location.S3Config.AccessKeyID,
		backupLocation.Location.S3Config.SecretAccessKey,
		"",
		backupLocation.Location.S3Config.UseIam, // runningOnEc2 when set ec2 role credentials will be used
	)
	if err != nil {
		return nil, err
	}
	creds, err := awsCreds.Get()
	if err != nil {
		return nil, err
	}

	return session.NewSession(&aws.Config{
		Endpoint:         aws.String(endpoint),
		Credentials:      creds,
		Region:           aws.String(backupLocation.Location.S3Config.Region),
		DisableSSL:       aws.Bool(backupLocation.Location.S3Config.DisableSSL),
		S3ForcePathStyle: aws.Bool(true),
	})
}

// GetBucket gets a reference to the bucket for that backup location
func GetBucket(backupLocation *stork_api.BackupLocation) (*blob.Bucket, error) {
	sess, err := getSession(backupLocation)
	if err != nil {
		return nil, err
	}
	return s3blob.OpenBucket(context.Background(), sess, backupLocation.Location.Path, nil)
}

// CreateBucket creates a bucket for the bucket location
func CreateBucket(backupLocation *stork_api.BackupLocation) error {
	sess, err := getSession(backupLocation)
	if err != nil {
		return err
	}

	input := &s3.CreateBucketInput{
		Bucket: &backupLocation.Location.Path,
	}
	_, err = s3.New(sess).CreateBucket(input)
	if err != nil {
		if awsErr, ok := err.(awserr.Error); ok {
			if awsErr.Code() == s3.ErrCodeBucketAlreadyOwnedByYou {
				return nil
			}
		}
	}
	return err
}

// GetObjLockInfo fetches the object lock configuration of a S3 bucket
func GetObjLockInfo(backupLocation *stork_api.BackupLocation) (*common.ObjLockInfo, error) {
	sess, err := getSession(backupLocation)
	if err != nil {
		return nil, err
	}

	input := &s3.GetObjectLockConfigurationInput{
		Bucket: &backupLocation.Location.Path,
	}

	objLockInfo := &common.ObjLockInfo{}
	out, err := s3.New(sess).GetObjectLockConfiguration(input)
	if err != nil {
		if awsErr, ok := err.(awserr.Error); ok {
			// When a Minio server doesn't have object-lock implemented then above API
			// throws following error codes depending on version it runs for normal buckets
			//	1. "ObjectLockConfigurationNotFoundError"
			//  2. "MethodNotAllowed"
			// Similarly in case of AWS, we need to ignore "NoSuchBucket" so that
			// px-backup/stork can create the bucket on behalf user when validation flag is not set.
			if awsErr.Code() == "ObjectLockConfigurationNotFoundError" ||
				awsErr.Code() == "MethodNotAllowed" ||
				awsErr.Code() == "NoSuchBucket" {
				// for a non-objectlocked bucket we needn't throw error
				return objLockInfo, nil
			}
		}
		return nil, err
	}
	if (out != nil) && (out.ObjectLockConfiguration != nil) {
		if aws.StringValue(out.ObjectLockConfiguration.ObjectLockEnabled) == "Enabled" {
			objLockInfo.LockEnabled = true
		} else {
			// For some of the objectstore like FB and dell ECS, GetObjectLockConfiguration
			// will return empty objectlockconfiguration instead of nil or error
			return objLockInfo, nil
		}
		if out.ObjectLockConfiguration.Rule != nil &&
			out.ObjectLockConfiguration.Rule.DefaultRetention != nil {
			objLockInfo.LockMode = aws.StringValue(out.ObjectLockConfiguration.Rule.DefaultRetention.Mode)
			objLockInfo.RetentionPeriodYears = aws.Int64Value(out.ObjectLockConfiguration.Rule.DefaultRetention.Years)
			objLockInfo.RetentionPeriodDays = aws.Int64Value(out.ObjectLockConfiguration.Rule.DefaultRetention.Days)
		} else {
			//This is an invalid object-lock config, no default-retention but object-loc enabled
			objLockInfo.LockEnabled = false
			return nil, fmt.Errorf("invalid config: object lock is enabled but default retention period is not set on the bucket")
		}
	}
	return objLockInfo, err
}
<|endoftext|>


<commit_before>package paxos

import (
    "fmt"
    "strings"
    "strconv"

    "borg/assert"
    "testing"
    "container/vector"
)

const (
    iFrom = iota
    iTo
    iCmd
    iBody
    iNumParts
)

const (
    nRnd = iota
    nVal
    nNumParts
)

func accept(me uint64, ins, outs chan string) {
    var rnd, vrnd uint64
    var vval string

    ch, sent := make(chan int), 0
    for in := range ins {
        parts := strings.Split(in, ":", iNumParts)
        if len(parts) != iNumParts {
            continue
        }

        inTo, _ := strconv.Btoui64(parts[iTo], 10)
        if inTo != me && parts[iTo] != "*" {
            continue
        }

        switch parts[iCmd] {
        case "INVITE":
            i, err := strconv.Btoui64(parts[iBody], 10)
            if err != nil { continue }

            inFrom, err := strconv.Btoui64(parts[iFrom], 10)
            if err != nil { continue }

            switch {
                case i <= rnd:
                case i > rnd:
                    rnd = i

                    outTo := inFrom // reply to the sender
                    msg := fmt.Sprintf("%d:%d:ACCEPT:%d:%d:%s",
                        me,
                        outTo,
                        i,
                        vrnd,
                        vval,
                    )
                    go func(msg string) { outs <- msg ; ch <- 1 }(msg)
                    sent++
            }
        case "NOMINATE":
            nominateParts := strings.Split(parts[iBody], ":", nNumParts)
            if len(nominateParts) != nNumParts {
                continue
            }

            i, err := strconv.Btoui64(nominateParts[nRnd], 10)
            if err != nil {
                continue
            }

            // SUPER IMPT MAD PAXOS
            if i < rnd || i == vrnd {
                continue
            }

            rnd = i
            vrnd = i
            vval = nominateParts[nVal]

            msg := fmt.Sprintf("%d:*:VOTE:%d:%s", me, i, vval)
            go func(msg string) { outs <- msg ; ch <- 1 }(msg)
            sent++
        }
    }

    for x := 0; x < sent; x++ {
        <-ch
    }

    close(outs)
}

// TESTING

func gather(ch chan string) (got []string) {
    for x := range ch { (*vector.StringVector)(&got).Push(x) }
    return
}

func TestAcceptsInvite(t *testing.T) {
    ins := make(chan string)
    outs := make(chan string)

    go accept(2, ins, outs)
    ins <- "1:*:INVITE:1"
    close(ins)

    exp := []string{"2:1:ACCEPT:1:0:"}

    // outs was closed; therefore all messages have been processed
    assert.Equal(t, exp, gather(outs), "")
}

func TestInvitesAfterNewInvitesAreStaleAndIgnored(t *testing.T) {
    ins := make(chan string)
    outs := make(chan string)

    go accept(2, ins, outs)
    ins <- "1:*:INVITE:2"
    ins <- "1:*:INVITE:1"
    close(ins)

    exp := []string{"2:1:ACCEPT:2:0:"}

    // outs was closed; therefore all messages have been processed
    assert.Equal(t, exp, gather(outs), "")
}

func TestIgnoresMalformedMessages(t *testing.T) {
    totest := []string{
        "x", // too few separators
        "x:x", // too few separators
        "x:x:x", // too few separators
        "x:x:x:x:x", // too many separators
        "1:*:INVITE:x", // invalid round number
        "1:*:x:1", // unknown command
        "1:x:INVITE:1", // invalid to address
        "1:7:INVITE:1", // valid but incorrect to address
        "X:*:INVITE:1", // invalid from address

        "1:*:NOMINATE:x", // too few separators in nominate body
        "1:*:NOMINATE:x:foo", // invalid round number
    }

    for _, msg := range(totest) {
        ins := make(chan string)
        outs := make(chan string)

        go accept(2, ins, outs)
        ins <- msg
        close(ins)

        exp := []string{}

        // outs was closed; therefore all messages have been processed
        assert.Equal(t, exp, gather(outs), msg)
    }
}

func TestItVotes(t *testing.T) {
    ins := make(chan string)
    outs := make(chan string)

    val := "foo"

    go accept(2, ins, outs)
    // According to paxos, we can omit Phase 1 in round 1
    ins <- "1:*:NOMINATE:1:" + val
    close(ins)

    exp := []string{"2:*:VOTE:1:" + val}

    // outs was closed; therefore all messages have been processed
    assert.Equal(t, exp, gather(outs), "")
}

func TestItVotesWithAnotherValue(t *testing.T) {
    ins := make(chan string)
    outs := make(chan string)

    val := "bar"

    go accept(2, ins, outs)
    // According to paxos, we can omit Phase 1 in round 1
    ins <- "1:*:NOMINATE:1:" + val
    close(ins)

    exp := []string{"2:*:VOTE:1:" + val}

    // outs was closed; therefore all messages have been processed
    assert.Equal(t, exp, gather(outs), "")
}

func TestItVotesWithAnotherRound(t *testing.T) {
    ins := make(chan string)
    outs := make(chan string)

    val := "bar"

    go accept(2, ins, outs)
    // According to paxos, we can omit Phase 1 in the first round
    ins <- "1:*:NOMINATE:2:" + val
    close(ins)

    exp := []string{"2:*:VOTE:2:" + val}

    // outs was closed; therefore all messages have been processed
    assert.Equal(t, exp, gather(outs), "")
}

func TestItVotesWithAnotherSelf(t *testing.T) {
    ins := make(chan string)
    outs := make(chan string)

    val := "bar"

    go accept(3, ins, outs)
    // According to paxos, we can omit Phase 1 in the first round
    ins <- "1:*:NOMINATE:2:" + val
    close(ins)

    exp := []string{"3:*:VOTE:2:" + val}

    // outs was closed; therefore all messages have been processed
    assert.Equal(t, exp, gather(outs), "")
}

func TestItIgnoresOldNominations(t *testing.T) {
    ins := make(chan string)
    outs := make(chan string)

    val := "bar"

    go accept(3, ins, outs)
    // According to paxos, we can omit Phase 1 in the first round
    ins <- "1:*:INVITE:2"
    <-outs // throw away ACCEPT message
    ins <- "1:*:NOMINATE:1:" + val
    close(ins)

    exp := []string{}

    // outs was closed; therefore all messages have been processed
    assert.Equal(t, exp, gather(outs), "")
}

func TestInvitesAfterNewNominationsAreStaleAndIgnored(t *testing.T) {
    ins := make(chan string)
    outs := make(chan string)

    go accept(2, ins, outs)
    ins <- "1:*:NOMINATE:2:v"
    <-outs // throw away VOTE message
    ins <- "1:*:INVITE:1"
    close(ins)

    exp := []string{}

    // outs was closed; therefore all messages have been processed
    assert.Equal(t, exp, gather(outs), "")
}

func TestVotedRoundsAndValuesAreTracked(t *testing.T) {
    ins := make(chan string)
    outs := make(chan string)

    go accept(2, ins, outs)
    ins <- "1:*:NOMINATE:1:v"
    <-outs // throw away VOTE message
    ins <- "1:*:INVITE:2"
    close(ins)

    exp := []string{"2:1:ACCEPT:2:1:v"}

    // outs was closed; therefore all messages have been processed
    assert.Equal(t, exp, gather(outs), "")
}

func TestVotesOnlyOncePerRound(t *testing.T) {
    ins := make(chan string)
    outs := make(chan string)

    go accept(2, ins, outs)
    ins <- "1:*:NOMINATE:1:v"
    ins <- "1:*:NOMINATE:1:v"
    close(ins)

    exp := []string{"2:*:VOTE:1:v"}

    // outs was closed; therefore all messages have been processed
    assert.Equal(t, exp, gather(outs), "")
}
<commit_msg>gofmt<commit_after>package paxos

import (
	"fmt"
	"strings"
	"strconv"

	"borg/assert"
	"testing"
	"container/vector"
)

const (
	iFrom = iota
	iTo
	iCmd
	iBody
	iNumParts
)

const (
	nRnd = iota
	nVal
	nNumParts
)

func accept(me uint64, ins, outs chan string) {
	var rnd, vrnd uint64
	var vval string

	ch, sent := make(chan int), 0
	for in := range ins {
		parts := strings.Split(in, ":", iNumParts)
		if len(parts) != iNumParts {
			continue
		}

		inTo, _ := strconv.Btoui64(parts[iTo], 10)
		if inTo != me && parts[iTo] != "*" {
			continue
		}

		switch parts[iCmd] {
		case "INVITE":
			i, err := strconv.Btoui64(parts[iBody], 10)
			if err != nil {
				continue
			}

			inFrom, err := strconv.Btoui64(parts[iFrom], 10)
			if err != nil {
				continue
			}

			switch {
			case i <= rnd:
			case i > rnd:
				rnd = i

				outTo := inFrom // reply to the sender
				msg := fmt.Sprintf("%d:%d:ACCEPT:%d:%d:%s",
					me,
					outTo,
					i,
					vrnd,
					vval,
				)
				go func(msg string) { outs <- msg; ch <- 1 }(msg)
				sent++
			}
		case "NOMINATE":
			nominateParts := strings.Split(parts[iBody], ":", nNumParts)
			if len(nominateParts) != nNumParts {
				continue
			}

			i, err := strconv.Btoui64(nominateParts[nRnd], 10)
			if err != nil {
				continue
			}

			// SUPER IMPT MAD PAXOS
			if i < rnd || i == vrnd {
				continue
			}

			rnd = i
			vrnd = i
			vval = nominateParts[nVal]

			msg := fmt.Sprintf("%d:*:VOTE:%d:%s", me, i, vval)
			go func(msg string) { outs <- msg; ch <- 1 }(msg)
			sent++
		}
	}

	for x := 0; x < sent; x++ {
		<-ch
	}

	close(outs)
}

// TESTING

func gather(ch chan string) (got []string) {
	for x := range ch {
		(*vector.StringVector)(&got).Push(x)
	}
	return
}

func TestAcceptsInvite(t *testing.T) {
	ins := make(chan string)
	outs := make(chan string)

	go accept(2, ins, outs)
	ins <- "1:*:INVITE:1"
	close(ins)

	exp := []string{"2:1:ACCEPT:1:0:"}

	// outs was closed; therefore all messages have been processed
	assert.Equal(t, exp, gather(outs), "")
}

func TestInvitesAfterNewInvitesAreStaleAndIgnored(t *testing.T) {
	ins := make(chan string)
	outs := make(chan string)

	go accept(2, ins, outs)
	ins <- "1:*:INVITE:2"
	ins <- "1:*:INVITE:1"
	close(ins)

	exp := []string{"2:1:ACCEPT:2:0:"}

	// outs was closed; therefore all messages have been processed
	assert.Equal(t, exp, gather(outs), "")
}

func TestIgnoresMalformedMessages(t *testing.T) {
	totest := []string{
		"x",            // too few separators
		"x:x",          // too few separators
		"x:x:x",        // too few separators
		"x:x:x:x:x",    // too many separators
		"1:*:INVITE:x", // invalid round number
		"1:*:x:1",      // unknown command
		"1:x:INVITE:1", // invalid to address
		"1:7:INVITE:1", // valid but incorrect to address
		"X:*:INVITE:1", // invalid from address

		"1:*:NOMINATE:x",     // too few separators in nominate body
		"1:*:NOMINATE:x:foo", // invalid round number
	}

	for _, msg := range totest {
		ins := make(chan string)
		outs := make(chan string)

		go accept(2, ins, outs)
		ins <- msg
		close(ins)

		exp := []string{}

		// outs was closed; therefore all messages have been processed
		assert.Equal(t, exp, gather(outs), msg)
	}
}

func TestItVotes(t *testing.T) {
	ins := make(chan string)
	outs := make(chan string)

	val := "foo"

	go accept(2, ins, outs)
	// According to paxos, we can omit Phase 1 in round 1
	ins <- "1:*:NOMINATE:1:"+val
	close(ins)

	exp := []string{"2:*:VOTE:1:" + val}

	// outs was closed; therefore all messages have been processed
	assert.Equal(t, exp, gather(outs), "")
}

func TestItVotesWithAnotherValue(t *testing.T) {
	ins := make(chan string)
	outs := make(chan string)

	val := "bar"

	go accept(2, ins, outs)
	// According to paxos, we can omit Phase 1 in round 1
	ins <- "1:*:NOMINATE:1:"+val
	close(ins)

	exp := []string{"2:*:VOTE:1:" + val}

	// outs was closed; therefore all messages have been processed
	assert.Equal(t, exp, gather(outs), "")
}

func TestItVotesWithAnotherRound(t *testing.T) {
	ins := make(chan string)
	outs := make(chan string)

	val := "bar"

	go accept(2, ins, outs)
	// According to paxos, we can omit Phase 1 in the first round
	ins <- "1:*:NOMINATE:2:"+val
	close(ins)

	exp := []string{"2:*:VOTE:2:" + val}

	// outs was closed; therefore all messages have been processed
	assert.Equal(t, exp, gather(outs), "")
}

func TestItVotesWithAnotherSelf(t *testing.T) {
	ins := make(chan string)
	outs := make(chan string)

	val := "bar"

	go accept(3, ins, outs)
	// According to paxos, we can omit Phase 1 in the first round
	ins <- "1:*:NOMINATE:2:"+val
	close(ins)

	exp := []string{"3:*:VOTE:2:" + val}

	// outs was closed; therefore all messages have been processed
	assert.Equal(t, exp, gather(outs), "")
}

func TestItIgnoresOldNominations(t *testing.T) {
	ins := make(chan string)
	outs := make(chan string)

	val := "bar"

	go accept(3, ins, outs)
	// According to paxos, we can omit Phase 1 in the first round
	ins <- "1:*:INVITE:2"
	<-outs // throw away ACCEPT message
	ins <- "1:*:NOMINATE:1:"+val
	close(ins)

	exp := []string{}

	// outs was closed; therefore all messages have been processed
	assert.Equal(t, exp, gather(outs), "")
}

func TestInvitesAfterNewNominationsAreStaleAndIgnored(t *testing.T) {
	ins := make(chan string)
	outs := make(chan string)

	go accept(2, ins, outs)
	ins <- "1:*:NOMINATE:2:v"
	<-outs // throw away VOTE message
	ins <- "1:*:INVITE:1"
	close(ins)

	exp := []string{}

	// outs was closed; therefore all messages have been processed
	assert.Equal(t, exp, gather(outs), "")
}

func TestVotedRoundsAndValuesAreTracked(t *testing.T) {
	ins := make(chan string)
	outs := make(chan string)

	go accept(2, ins, outs)
	ins <- "1:*:NOMINATE:1:v"
	<-outs // throw away VOTE message
	ins <- "1:*:INVITE:2"
	close(ins)

	exp := []string{"2:1:ACCEPT:2:1:v"}

	// outs was closed; therefore all messages have been processed
	assert.Equal(t, exp, gather(outs), "")
}

func TestVotesOnlyOncePerRound(t *testing.T) {
	ins := make(chan string)
	outs := make(chan string)

	go accept(2, ins, outs)
	ins <- "1:*:NOMINATE:1:v"
	ins <- "1:*:NOMINATE:1:v"
	close(ins)

	exp := []string{"2:*:VOTE:1:v"}

	// outs was closed; therefore all messages have been processed
	assert.Equal(t, exp, gather(outs), "")
}
<|endoftext|>