<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"github.com/ziutek/mymysql/mysql"
	_ "github.com/ziutek/mymysql/native"
	"log"
	"os"
	"sort"
)

const _DB_CONFIG_FILENAME = "db_config.SECRET.json"

type dbConfig struct {
	Url, Username, Password, DbName, SolvesTable, SolvesID, SolvesPuzzleID, SolvesTotalTime, SolvesUser string
}

type solve struct {
	puzzleID  int
	totalTime int
}

type userSolvesCollection struct {
	solves []*solve
	max    int
	min    int
}

type puzzle struct {
	id                     int
	userRelativeDifficulty float32
}

type puzzles []puzzle

type byUserRelativeDifficulty struct {
	puzzles
}

func (self puzzles) Len() int {
	return len(self)
}

func (self puzzles) Swap(i, j int) {
	self[i], self[j] = self[j], self[i]
}

func (self byUserRelativeDifficulty) Less(i, j int) bool {
	return self.puzzles[i].userRelativeDifficulty < self.puzzles[j].userRelativeDifficulty
}

func (self *userSolvesCollection) addSolve(solve *solve) {
	self.solves = append(self.solves, solve)
	if len(self.solves) == 1 {
		self.max = solve.totalTime
		self.min = solve.totalTime
	} else {
		if self.max < solve.totalTime {
			self.max = solve.totalTime
		}
		if self.min > solve.totalTime {
			self.min = solve.totalTime
		}
	}
}

func (self *userSolvesCollection) relativeDifficulties() map[int]float32 {
	//Returns a map of puzzle id to relative difficulty, normalized by our max and min.
	avgSolveTimes := make(map[int]float32)
	//Keep track of how many times we've seen each puzzle solved by this user so we can do correct averaging.
	avgSolveTimesCount := make(map[int]int)

	//First, collect the average solve time (in case the same user has solved more than once the same puzzle)

	for _, solve := range self.solves {
		currentAvgSolveTime := avgSolveTimes[solve.puzzleID]

		avgSolveTimes[solve.puzzleID] = (currentAvgSolveTime*float32(avgSolveTimesCount[solve.puzzleID]) + float32(solve.totalTime)) / float32(avgSolveTimesCount[solve.puzzleID]+1)

		avgSolveTimesCount[solve.puzzleID]++
	}

	//Now, relativize all of the scores.

	result := make(map[int]float32)

	for puzzleID, avgSolveTime := range avgSolveTimes {
		result[puzzleID] = (avgSolveTime - float32(self.min)) / float32(self.max-self.min)
	}

	return result
}

func main() {
	file, err := os.Open(_DB_CONFIG_FILENAME)
	if err != nil {
		log.Fatal("Could not find the config file at ", _DB_CONFIG_FILENAME, ". You should copy the SAMPLE one to that filename and configure.")
		os.Exit(1)
	}
	defer file.Close()
	decoder := json.NewDecoder(file)
	var config dbConfig
	if err := decoder.Decode(&config); err != nil {
		log.Fatal("There was an error parsing JSON from the config file: ", err)
		os.Exit(1)
	}

	db := mysql.New("tcp", "", config.Url, config.Username, config.Password, config.DbName)

	if err := db.Connect(); err != nil {
		log.Fatal(err)
		os.Exit(1)
	}

	res, err := db.Start("select %s, %s, %s from %s limit 100", config.SolvesUser, config.SolvesPuzzleID, config.SolvesTotalTime, config.SolvesTable)

	if err != nil {
		log.Fatal(err)
		os.Exit(1)
	}

	solvesByUser := make(map[string]*userSolvesCollection)

	var userSolves *userSolvesCollection
	var ok bool
	var i int

	//First, process all user records in the DB to collect all solves by userName.
	for {

		row, _ := res.GetRow()

		if row == nil {
			break
		}

		userSolves, ok = solvesByUser[row.Str(0)]

		if !ok {
			userSolves = new(userSolvesCollection)
			solvesByUser[row.Str(0)] = userSolves
		}

		userSolves.addSolve(&solve{row.Int(1), row.Int(2)})
		i++
	}

	fmt.Println("Processed ", i, " solves by ", len(solvesByUser), " users.")

	//Now get the relative difficulty for each user's puzzles, and collect them.

	relativeDifficultiesByPuzzle := make(map[int][]float32)

	var skippedUsers int

	for _, collection := range solvesByUser {

		if len(collection.solves) < 2 {
			skippedUsers++
			continue
		}

		for puzzleID, relativeDifficulty := range collection.relativeDifficulties() {
			relativeDifficultiesByPuzzle[puzzleID] = append(relativeDifficultiesByPuzzle[puzzleID], relativeDifficulty)
		}

	}

	fmt.Println("Skipped ", skippedUsers, " users because they had only solved one unique puzzle.")

	puzzles := make([]puzzle, len(relativeDifficultiesByPuzzle))

	var index int

	for puzzleID, difficulties := range relativeDifficultiesByPuzzle {
		var sum float32
		for _, difficulty := range difficulties {
			sum += difficulty
		}
		puzzles[index] = puzzle{puzzleID, sum / float32(len(difficulties))}
		index++
	}

	//Sort the puzzles by relative user difficulty
	//We actually don't need the wrapper, since it will modify the underlying slice.
	sort.Sort(byUserRelativeDifficulty{puzzles})
}
<commit_msg>Added a flag for whether or not to run the query without a limit.<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"github.com/ziutek/mymysql/mysql"
	_ "github.com/ziutek/mymysql/native"
	"log"
	"os"
	"sort"
	"strconv"
)

const _DB_CONFIG_FILENAME = "db_config.SECRET.json"
const QUERY_LIMIT = 100

var noLimitFlag bool

func init() {
	flag.BoolVar(&noLimitFlag, "a", false, "Specify to execute the solves query with no limit.")
}

type dbConfig struct {
	Url, Username, Password, DbName, SolvesTable, SolvesID, SolvesPuzzleID, SolvesTotalTime, SolvesUser string
}

type solve struct {
	puzzleID  int
	totalTime int
}

type userSolvesCollection struct {
	solves []*solve
	max    int
	min    int
}

type puzzle struct {
	id                     int
	userRelativeDifficulty float32
}

type puzzles []puzzle

type byUserRelativeDifficulty struct {
	puzzles
}

func (self puzzles) Len() int {
	return len(self)
}

func (self puzzles) Swap(i, j int) {
	self[i], self[j] = self[j], self[i]
}

func (self byUserRelativeDifficulty) Less(i, j int) bool {
	return self.puzzles[i].userRelativeDifficulty < self.puzzles[j].userRelativeDifficulty
}

func (self *userSolvesCollection) addSolve(solve *solve) {
	self.solves = append(self.solves, solve)
	if len(self.solves) == 1 {
		self.max = solve.totalTime
		self.min = solve.totalTime
	} else {
		if self.max < solve.totalTime {
			self.max = solve.totalTime
		}
		if self.min > solve.totalTime {
			self.min = solve.totalTime
		}
	}
}

func (self *userSolvesCollection) relativeDifficulties() map[int]float32 {
	//Returns a map of puzzle id to relative difficulty, normalized by our max and min.
	avgSolveTimes := make(map[int]float32)
	//Keep track of how many times we've seen each puzzle solved by this user so we can do correct averaging.
	avgSolveTimesCount := make(map[int]int)

	//First, collect the average solve time (in case the same user has solved more than once the same puzzle)

	for _, solve := range self.solves {
		currentAvgSolveTime := avgSolveTimes[solve.puzzleID]

		avgSolveTimes[solve.puzzleID] = (currentAvgSolveTime*float32(avgSolveTimesCount[solve.puzzleID]) + float32(solve.totalTime)) / float32(avgSolveTimesCount[solve.puzzleID]+1)

		avgSolveTimesCount[solve.puzzleID]++
	}

	//Now, relativize all of the scores.

	result := make(map[int]float32)

	for puzzleID, avgSolveTime := range avgSolveTimes {
		result[puzzleID] = (avgSolveTime - float32(self.min)) / float32(self.max-self.min)
	}

	return result
}

func main() {

	flag.Parse()

	file, err := os.Open(_DB_CONFIG_FILENAME)
	if err != nil {
		log.Fatal("Could not find the config file at ", _DB_CONFIG_FILENAME, ". You should copy the SAMPLE one to that filename and configure.")
		os.Exit(1)
	}
	defer file.Close()
	decoder := json.NewDecoder(file)
	var config dbConfig
	if err := decoder.Decode(&config); err != nil {
		log.Fatal("There was an error parsing JSON from the config file: ", err)
		os.Exit(1)
	}

	db := mysql.New("tcp", "", config.Url, config.Username, config.Password, config.DbName)

	if err := db.Connect(); err != nil {
		log.Fatal(err)
		os.Exit(1)
	}

	var solvesQuery string

	if noLimitFlag {
		fmt.Println("Running without a limit for number of solves to retrieve.")
		solvesQuery = "select %s, %s, %s from %s"
	} else {
		fmt.Println("Running with a limit of ", QUERY_LIMIT, " for number of solves to retrieve.")
		solvesQuery = "select %s, %s, %s from %s limit " + strconv.Itoa(QUERY_LIMIT)
	}

	res, err := db.Start(solvesQuery, config.SolvesUser, config.SolvesPuzzleID, config.SolvesTotalTime, config.SolvesTable)

	if err != nil {
		log.Fatal(err)
		os.Exit(1)
	}

	solvesByUser := make(map[string]*userSolvesCollection)

	var userSolves *userSolvesCollection
	var ok bool
	var i int

	//First, process all user records in the DB to collect all solves by userName.
	for {

		row, _ := res.GetRow()

		if row == nil {
			break
		}

		userSolves, ok = solvesByUser[row.Str(0)]

		if !ok {
			userSolves = new(userSolvesCollection)
			solvesByUser[row.Str(0)] = userSolves
		}

		userSolves.addSolve(&solve{row.Int(1), row.Int(2)})
		i++
	}

	fmt.Println("Processed ", i, " solves by ", len(solvesByUser), " users.")

	//Now get the relative difficulty for each user's puzzles, and collect them.

	relativeDifficultiesByPuzzle := make(map[int][]float32)

	var skippedUsers int

	for _, collection := range solvesByUser {

		if len(collection.solves) < 2 {
			skippedUsers++
			continue
		}

		for puzzleID, relativeDifficulty := range collection.relativeDifficulties() {
			relativeDifficultiesByPuzzle[puzzleID] = append(relativeDifficultiesByPuzzle[puzzleID], relativeDifficulty)
		}

	}

	fmt.Println("Skipped ", skippedUsers, " users because they had only solved one unique puzzle.")

	puzzles := make([]puzzle, len(relativeDifficultiesByPuzzle))

	var index int

	for puzzleID, difficulties := range relativeDifficultiesByPuzzle {
		var sum float32
		for _, difficulty := range difficulties {
			sum += difficulty
		}
		puzzles[index] = puzzle{puzzleID, sum / float32(len(difficulties))}
		index++
	}

	//Sort the puzzles by relative user difficulty
	//We actually don't need the wrapper, since it will modify the underlying slice.
	sort.Sort(byUserRelativeDifficulty{puzzles})
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"reflect"
	"testing"
)

type testUnmarshalCase struct {
	ok   bool
	in   []byte
	want interface{}
}

func unOneTestCases() map[string]testUnmarshalCase {
	c := map[string]testUnmarshalCase{
		"ok": {
			ok:   true,
			in:   []byte(`{"i":123, "f":123.456, "b":true,"s":"abc"}`),
			want: One{I: int64(123), F: float64(123.456), B: true, S: "abc"},
		},
	}
	return c
}

func TestUnmarshal(t *testing.T) {
	for name, tc := range unOneTestCases() {
		t.Run(name, func(t *testing.T) {
			testUnmarshal(t, tc)
		})
	}
}

func testUnmarshal(t *testing.T, tc testUnmarshalCase) {
	var got One
	err := json.Unmarshal(tc.in, &got)
	if !tc.ok {
		if !reflect.DeepEqual(got, tc.want) || err == nil {
			t.Errorf("got %+v, %v; want %+v, <error>", got, err, tc.want)
		}
		return
	}
	if !reflect.DeepEqual(got, tc.want) || err != nil {
		t.Errorf("got %+v, %+v; want %+v, <nil>", got, err, tc.want)
	}
}
<commit_msg>jj: rename unOneTestCase<commit_after>package main

import (
	"encoding/json"
	"reflect"
	"testing"
)

type unOneTestCase struct {
	ok   bool
	in   []byte
	want interface{}
}

func unOneTestCases() map[string]unOneTestCase {
	c := map[string]unOneTestCase{
		"ok": {
			ok:   true,
			in:   []byte(`{"i":123, "f":123.456, "b":true,"s":"abc"}`),
			want: One{I: int64(123), F: float64(123.456), B: true, S: "abc"},
		},
	}
	return c
}

func TestUnmarshal(t *testing.T) {
	for name, tc := range unOneTestCases() {
		t.Run(name, func(t *testing.T) {
			testUnmarshal(t, tc)
		})
	}
}

func testUnmarshal(t *testing.T, tc unOneTestCase) {
	var got One
	err := json.Unmarshal(tc.in, &got)
	if !tc.ok {
		if !reflect.DeepEqual(got, tc.want) || err == nil {
			t.Errorf("got %+v, %v; want %+v, <error>", got, err, tc.want)
		}
		return
	}
	if !reflect.DeepEqual(got, tc.want) || err != nil {
		t.Errorf("got %+v, %+v; want %+v, <nil>", got, err, tc.want)
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package journal provides write bindings to the systemd journal
package journal

import (
	"bytes"
	"encoding/binary"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"os"
	"strconv"
	"strings"
	"syscall"
)

// Priority of a journal message
type Priority int

const (
	PriEmerg Priority = iota
	PriAlert
	PriCrit
	PriErr
	PriWarning
	PriNotice
	PriInfo
	PriDebug
)

var conn net.Conn

func init() {
	var err error
	conn, err = net.Dial("unixgram", "/run/systemd/journal/socket")
	if err != nil {
		conn = nil
	}
}

// Enabled returns true iff the systemd journal is available for logging
func Enabled() bool {
	return conn != nil
}

// Send a message to the systemd journal. vars is a map of journald fields to
// values.  Fields must be composed of uppercase letters, numbers, and
// underscores, but must not start with an underscore. Within these
// restrictions, any arbitrary field name may be used.  Some names have special
// significance: see the journalctl documentation
// (http://www.freedesktop.org/software/systemd/man/systemd.journal-fields.html)
// for more details.  vars may be nil.
func Send(message string, priority Priority, vars map[string]string) error {
	if conn == nil {
		return journalError("could not connect to journald socket")
	}

	data := new(bytes.Buffer)
	appendVariable(data, "PRIORITY", strconv.Itoa(int(priority)))
	appendVariable(data, "MESSAGE", message)
	for k, v := range vars {
		appendVariable(data, k, v)
	}

	_, err := io.Copy(conn, data)
	if err != nil && isSocketSpaceError(err) {
		file, err := tempFd()
		if err != nil {
			return journalError(err.Error())
		}
		_, err = io.Copy(file, data)
		if err != nil {
			return journalError(err.Error())
		}

		rights := syscall.UnixRights(int(file.Fd()))

		/* this connection should always be a UnixConn, but better safe than sorry */
		unixConn, ok := conn.(*net.UnixConn)
		if !ok {
			return journalError("can't send file through non-Unix connection")
		}
		unixConn.WriteMsgUnix([]byte{}, rights, nil)
	} else if err != nil {
		return journalError(err.Error())
	}
	return nil
}

func appendVariable(w io.Writer, name, value string) {
	if !validVarName(name) {
		journalError("variable name contains invalid character, ignoring")
	}
	if strings.ContainsRune(value, '\n') {
		/* When the value contains a newline, we write:
		 * - the variable name, followed by a newline
		 * - the size (in 64bit little endian format)
		 * - the data, followed by a newline
		 */
		fmt.Fprintln(w, name)
		binary.Write(w, binary.LittleEndian, uint64(len(value)))
		fmt.Fprintln(w, value)
	} else {
		/* just write the variable and value all on one line */
		fmt.Fprintf(w, "%s=%s\n", name, value)
	}
}

func validVarName(name string) bool {
	/* The variable name must be in uppercase and consist only of characters,
	 * numbers and underscores, and may not begin with an underscore. (from the docs)
	 */

	valid := name[0] != '_'
	for _, c := range name {
		valid = valid && ('A' <= c && c <= 'Z') || ('0' <= c && c <= '9') || c == '_'
	}
	return valid
}

func isSocketSpaceError(err error) bool {
	opErr, ok := err.(*net.OpError)
	if !ok {
		return false
	}

	sysErr, ok := opErr.Err.(syscall.Errno)
	if !ok {
		return false
	}

	return sysErr == syscall.EMSGSIZE || sysErr == syscall.ENOBUFS
}

func tempFd() (*os.File, error) {
	file, err := ioutil.TempFile("/dev/shm/", "journal.XXXXX")
	if err != nil {
		return nil, err
	}
	syscall.Unlink(file.Name())
	if err != nil {
		return nil, err
	}
	return file, nil
}

func journalError(s string) error {
	s = "journal error: " + s
	fmt.Fprintln(os.Stderr, s)
	return errors.New(s)
}
<commit_msg>Typo fix<commit_after>// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package journal provides write bindings to the systemd journal
package journal

import (
	"bytes"
	"encoding/binary"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"os"
	"strconv"
	"strings"
	"syscall"
)

// Priority of a journal message
type Priority int

const (
	PriEmerg Priority = iota
	PriAlert
	PriCrit
	PriErr
	PriWarning
	PriNotice
	PriInfo
	PriDebug
)

var conn net.Conn

func init() {
	var err error
	conn, err = net.Dial("unixgram", "/run/systemd/journal/socket")
	if err != nil {
		conn = nil
	}
}

// Enabled returns true if the systemd journal is available for logging
func Enabled() bool {
	return conn != nil
}

// Send a message to the systemd journal. vars is a map of journald fields to
// values.  Fields must be composed of uppercase letters, numbers, and
// underscores, but must not start with an underscore. Within these
// restrictions, any arbitrary field name may be used.  Some names have special
// significance: see the journalctl documentation
// (http://www.freedesktop.org/software/systemd/man/systemd.journal-fields.html)
// for more details.  vars may be nil.
func Send(message string, priority Priority, vars map[string]string) error {
	if conn == nil {
		return journalError("could not connect to journald socket")
	}

	data := new(bytes.Buffer)
	appendVariable(data, "PRIORITY", strconv.Itoa(int(priority)))
	appendVariable(data, "MESSAGE", message)
	for k, v := range vars {
		appendVariable(data, k, v)
	}

	_, err := io.Copy(conn, data)
	if err != nil && isSocketSpaceError(err) {
		file, err := tempFd()
		if err != nil {
			return journalError(err.Error())
		}
		_, err = io.Copy(file, data)
		if err != nil {
			return journalError(err.Error())
		}

		rights := syscall.UnixRights(int(file.Fd()))

		/* this connection should always be a UnixConn, but better safe than sorry */
		unixConn, ok := conn.(*net.UnixConn)
		if !ok {
			return journalError("can't send file through non-Unix connection")
		}
		unixConn.WriteMsgUnix([]byte{}, rights, nil)
	} else if err != nil {
		return journalError(err.Error())
	}
	return nil
}

func appendVariable(w io.Writer, name, value string) {
	if !validVarName(name) {
		journalError("variable name contains invalid character, ignoring")
	}
	if strings.ContainsRune(value, '\n') {
		/* When the value contains a newline, we write:
		 * - the variable name, followed by a newline
		 * - the size (in 64bit little endian format)
		 * - the data, followed by a newline
		 */
		fmt.Fprintln(w, name)
		binary.Write(w, binary.LittleEndian, uint64(len(value)))
		fmt.Fprintln(w, value)
	} else {
		/* just write the variable and value all on one line */
		fmt.Fprintf(w, "%s=%s\n", name, value)
	}
}

func validVarName(name string) bool {
	/* The variable name must be in uppercase and consist only of characters,
	 * numbers and underscores, and may not begin with an underscore. (from the docs)
	 */

	valid := name[0] != '_'
	for _, c := range name {
		valid = valid && ('A' <= c && c <= 'Z') || ('0' <= c && c <= '9') || c == '_'
	}
	return valid
}

func isSocketSpaceError(err error) bool {
	opErr, ok := err.(*net.OpError)
	if !ok {
		return false
	}

	sysErr, ok := opErr.Err.(syscall.Errno)
	if !ok {
		return false
	}

	return sysErr == syscall.EMSGSIZE || sysErr == syscall.ENOBUFS
}

func tempFd() (*os.File, error) {
	file, err := ioutil.TempFile("/dev/shm/", "journal.XXXXX")
	if err != nil {
		return nil, err
	}
	syscall.Unlink(file.Name())
	if err != nil {
		return nil, err
	}
	return file, nil
}

func journalError(s string) error {
	s = "journal error: " + s
	fmt.Fprintln(os.Stderr, s)
	return errors.New(s)
}
<|endoftext|>


<commit_before>// telnet crates a new Expect sp
package main

import (
	"fmt"
	"net"
	"time"

	expect "github.com/google/goexpect"
	"github.com/google/goterm/term"
	"github.com/ziutek/telnet"
)

func main() {
	fmt.Println(term.Bluef("Telnet spawner example"))

}

func telnetSpawn(network string, addr net.IP, timeout time.Duration) (expect.Expecter, <-chan error, error) {
	conn, err := telnet.Dial(network, addr.String())
	if err != nil {
		return nil, nil, err
	}

	resCh := make(chan error)

	return expect.SpawnGeneric(&expect.GenOptions{
		In:  conn,
		Out: conn,
		Wait: func() error {
			return <-resCh
		},
		Close: func() error {
			close(resCh)
			return conn.Close()
		},
		Check: func() bool { return true },
	}, timeout)
}
<commit_msg>Could this possibly work?<commit_after>// telnet crates a new Expect spawner for Telnet.
package main

import (
	"fmt"
	"time"

	expect "github.com/google/goexpect"

	"github.com/golang/glog"
	"github.com/google/goterm/term"
	"github.com/ziutek/telnet"
)

const (
	network = "tcp"
	address = "telehack.com:23"
	timeout = 10 * time.Second
	command = "geoip"
)

func main() {
	fmt.Println(term.Bluef("Telnet spawner example"))
	exp, _, err := telnetSpawn(address, timeout)
	if err != nil {
		glog.Exitf("telnetSpawn(%q,%v) failed: %v", address, timeout, err)
	}
	res, err := exp.ExpectBatch([]expect.Batcher{
		&expect.BExp{R: `\n\.`},
		&expect.BSnd{S: command + "\n"},
		&expect.BExp{R: `\n\.`},
	}, timeout)
	if err != nil {
		glog.Exit(err)
	}
	fmt.Println(term.Greenf("Res: %s", res[len(res)-1].Output))

}

func telnetSpawn(addr string, timeout time.Duration) (expect.Expecter, <-chan error, error) {
	conn, err := telnet.Dial(network, addr)
	if err != nil {
		return nil, nil, err
	}

	resCh := make(chan error)

	return expect.SpawnGeneric(&expect.GenOptions{
		In:  conn,
		Out: conn,
		Wait: func() error {
			return <-resCh
		},
		Close: func() error {
			close(resCh)
			return conn.Close()
		},
		Check: func() bool { return true },
	}, timeout)
}
<|endoftext|>


<commit_before>// Package aphgrpc provides various interfaces, functions, types
// for building and working with gRPC services.
package aphgrpc

import (
	"encoding/json"
	"net/http"
	"strconv"

	context "golang.org/x/net/context"
	dat "gopkg.in/mgutz/dat.v1"

	"github.com/grpc-ecosystem/grpc-gateway/runtime"
	"github.com/manyminds/api2go"
	"github.com/pkg/errors"
	"google.golang.org/grpc"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/metadata"
	"google.golang.org/grpc/status"
)

const (
	// MetaKey is the key used for storing all metadata
	MetaKey = "error"
)

var (
	//ErrDatabaseQuery represents database query related errors
	ErrDatabaseQuery = newError("Database query error")
	//ErrDatabaseInsert represents database insert related errors
	ErrDatabaseInsert = newError("Database insert error")
	//ErrDatabaseUpdate represents database update related errors
	ErrDatabaseUpdate = newError("Database update error")
	//ErrDatabaseDelete represents database update delete errors
	ErrDatabaseDelete = newError("Database delete error")
	//ErrNotFound represents the absence of an HTTP resource
	ErrNotFound = newError("Resource not found")
	//ErrJSONEncoding represents any json encoding error
	ErrJSONEncoding = newError("Json encoding error")
	//ErrStructMarshal represents any error with marshalling structure
	ErrStructMarshal = newError("Structure marshalling error")
	//ErrIncludeParam represents any error with invalid include query parameter
	ErrIncludeParam = newErrorWithParam("Invalid include query parameter", "include")
	//ErrSparseFieldSets represents any error with invalid sparse fieldsets query parameter
	ErrFields = newErrorWithParam("Invalid field query parameter", "field")
	//ErrFilterParam represents any error with invalid filter query paramter
	ErrFilterParam = newErrorWithParam("Invalid filter query parameter", "filter")
	//ErrNotAcceptable represents any error with wrong or inappropriate http Accept header
	ErrNotAcceptable = newError("Accept header is not acceptable")
	//ErrUnsupportedMedia represents any error with unsupported media type in http header
	ErrUnsupportedMedia = newError("Media type is not supported")
	//ErrRetrieveMetadata represents any error to retrieve grpc metadata from the running context
	ErrRetrieveMetadata = errors.New("unable to retrieve metadata")
	//ErrXForwardedHost represents any failure or absence of x-forwarded-host HTTP header in the grpc context
	ErrXForwardedHost = errors.New("x-forwarded-host header is absent")
)

func newErrorWithParam(msg, param string) metadata.MD {
	return metadata.Pairs(MetaKey, msg, param)
}

func newError(msg string) metadata.MD {
	return metadata.Pairs(MetaKey, msg)
}

// CustomHTTPError is a custom error handler for grpc-gateway to generate
// JSONAPI formatted HTTP response.
func CustomHTTPError(ctx context.Context, _ *runtime.ServeMux, marshaler runtime.Marshaler, w http.ResponseWriter, _ *http.Request, err error) {
	md, ok := runtime.ServerMetadataFromContext(ctx)
	if !ok {
		fallbackError(w, getgRPCStatus(errors.Wrap(err, "unable to retrieve metadata")))
		return
	}
	JSONAPIError(w, md.TrailerMD, getgRPCStatus(err))
}

func getgRPCStatus(err error) *status.Status {
	s, ok := status.FromError(err)
	if !ok {
		return status.New(codes.Unknown, err.Error())
	}
	return s
}

// JSONAPIError generates JSONAPI formatted error message
func JSONAPIError(w http.ResponseWriter, md metadata.MD, s *status.Status) {
	status := runtime.HTTPStatusFromCode(s.Code())
	jsnErr := api2go.Error{
		Status: strconv.Itoa(status),
		Title:  md["error"][0],
		Detail: s.Message(),
		Meta: map[string]interface{}{
			"creator": "api error helper",
		},
	}
	w.Header().Set("Content-Type", "application/vnd.api+json")
	w.WriteHeader(status)
	encErr := json.NewEncoder(w).Encode(api2go.HTTPError{Errors: []api2go.Error{jsnErr}})
	if encErr != nil {
		http.Error(w, encErr.Error(), http.StatusInternalServerError)
	}
}

func fallbackError(w http.ResponseWriter, s *status.Status) {
	status := runtime.HTTPStatusFromCode(s.Code())
	jsnErr := api2go.Error{
		Status: strconv.Itoa(status),
		Title:  "gRPC error",
		Detail: s.Message(),
		Meta: map[string]interface{}{
			"creator": "api error helper",
		},
	}
	w.Header().Set("Content-Type", "application/vnd.api+json")
	w.WriteHeader(status)
	encErr := json.NewEncoder(w).Encode(api2go.HTTPError{Errors: []api2go.Error{jsnErr}})
	if encErr != nil {
		http.Error(w, encErr.Error(), http.StatusInternalServerError)
	}
}

func handleError(ctx context.Context, err error) error {
	switch err {
	case dat.ErrNotFound:
		grpc.SetTrailer(ctx, ErrNotFound)
		return status.Error(codes.NotFound, err.Error())
	case ErrRetrieveMetadata:
		grpc.SetTrailer(ctx, newError(err.Error()))
		return status.Error(codes.Internal, err.Error())
	case ErrXForwardedHost:
		grpc.SetTrailer(ctx, newError(err.Error()))
		return status.Error(codes.Internal, err.Error())
	default:
		grpc.SetTrailer(ctx, ErrDatabaseQuery)
		return status.Error(codes.Internal, err.Error())
	}
}
<commit_msg>Exported the function<commit_after>// Package aphgrpc provides various interfaces, functions, types
// for building and working with gRPC services.
package aphgrpc

import (
	"encoding/json"
	"net/http"
	"strconv"

	context "golang.org/x/net/context"
	dat "gopkg.in/mgutz/dat.v1"

	"github.com/grpc-ecosystem/grpc-gateway/runtime"
	"github.com/manyminds/api2go"
	"github.com/pkg/errors"
	"google.golang.org/grpc"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/metadata"
	"google.golang.org/grpc/status"
)

const (
	// MetaKey is the key used for storing all metadata
	MetaKey = "error"
)

var (
	//ErrDatabaseQuery represents database query related errors
	ErrDatabaseQuery = newError("Database query error")
	//ErrDatabaseInsert represents database insert related errors
	ErrDatabaseInsert = newError("Database insert error")
	//ErrDatabaseUpdate represents database update related errors
	ErrDatabaseUpdate = newError("Database update error")
	//ErrDatabaseDelete represents database update delete errors
	ErrDatabaseDelete = newError("Database delete error")
	//ErrNotFound represents the absence of an HTTP resource
	ErrNotFound = newError("Resource not found")
	//ErrJSONEncoding represents any json encoding error
	ErrJSONEncoding = newError("Json encoding error")
	//ErrStructMarshal represents any error with marshalling structure
	ErrStructMarshal = newError("Structure marshalling error")
	//ErrIncludeParam represents any error with invalid include query parameter
	ErrIncludeParam = newErrorWithParam("Invalid include query parameter", "include")
	//ErrSparseFieldSets represents any error with invalid sparse fieldsets query parameter
	ErrFields = newErrorWithParam("Invalid field query parameter", "field")
	//ErrFilterParam represents any error with invalid filter query paramter
	ErrFilterParam = newErrorWithParam("Invalid filter query parameter", "filter")
	//ErrNotAcceptable represents any error with wrong or inappropriate http Accept header
	ErrNotAcceptable = newError("Accept header is not acceptable")
	//ErrUnsupportedMedia represents any error with unsupported media type in http header
	ErrUnsupportedMedia = newError("Media type is not supported")
	//ErrRetrieveMetadata represents any error to retrieve grpc metadata from the running context
	ErrRetrieveMetadata = errors.New("unable to retrieve metadata")
	//ErrXForwardedHost represents any failure or absence of x-forwarded-host HTTP header in the grpc context
	ErrXForwardedHost = errors.New("x-forwarded-host header is absent")
)

func newErrorWithParam(msg, param string) metadata.MD {
	return metadata.Pairs(MetaKey, msg, param)
}

func newError(msg string) metadata.MD {
	return metadata.Pairs(MetaKey, msg)
}

// CustomHTTPError is a custom error handler for grpc-gateway to generate
// JSONAPI formatted HTTP response.
func CustomHTTPError(ctx context.Context, _ *runtime.ServeMux, marshaler runtime.Marshaler, w http.ResponseWriter, _ *http.Request, err error) {
	md, ok := runtime.ServerMetadataFromContext(ctx)
	if !ok {
		fallbackError(w, getgRPCStatus(errors.Wrap(err, "unable to retrieve metadata")))
		return
	}
	JSONAPIError(w, md.TrailerMD, getgRPCStatus(err))
}

func getgRPCStatus(err error) *status.Status {
	s, ok := status.FromError(err)
	if !ok {
		return status.New(codes.Unknown, err.Error())
	}
	return s
}

// JSONAPIError generates JSONAPI formatted error message
func JSONAPIError(w http.ResponseWriter, md metadata.MD, s *status.Status) {
	status := runtime.HTTPStatusFromCode(s.Code())
	jsnErr := api2go.Error{
		Status: strconv.Itoa(status),
		Title:  md["error"][0],
		Detail: s.Message(),
		Meta: map[string]interface{}{
			"creator": "api error helper",
		},
	}
	w.Header().Set("Content-Type", "application/vnd.api+json")
	w.WriteHeader(status)
	encErr := json.NewEncoder(w).Encode(api2go.HTTPError{Errors: []api2go.Error{jsnErr}})
	if encErr != nil {
		http.Error(w, encErr.Error(), http.StatusInternalServerError)
	}
}

func fallbackError(w http.ResponseWriter, s *status.Status) {
	status := runtime.HTTPStatusFromCode(s.Code())
	jsnErr := api2go.Error{
		Status: strconv.Itoa(status),
		Title:  "gRPC error",
		Detail: s.Message(),
		Meta: map[string]interface{}{
			"creator": "api error helper",
		},
	}
	w.Header().Set("Content-Type", "application/vnd.api+json")
	w.WriteHeader(status)
	encErr := json.NewEncoder(w).Encode(api2go.HTTPError{Errors: []api2go.Error{jsnErr}})
	if encErr != nil {
		http.Error(w, encErr.Error(), http.StatusInternalServerError)
	}
}

func HandleError(ctx context.Context, err error) error {
	switch err {
	case dat.ErrNotFound:
		grpc.SetTrailer(ctx, ErrNotFound)
		return status.Error(codes.NotFound, err.Error())
	case ErrRetrieveMetadata:
		grpc.SetTrailer(ctx, newError(err.Error()))
		return status.Error(codes.Internal, err.Error())
	case ErrXForwardedHost:
		grpc.SetTrailer(ctx, newError(err.Error()))
		return status.Error(codes.Internal, err.Error())
	default:
		grpc.SetTrailer(ctx, ErrDatabaseQuery)
		return status.Error(codes.Internal, err.Error())
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"errors"
	"fmt"
	// "log"
	"os"
	"strings"
)

var torrentDB *TorrentDB
var err error

const resultLimit int = 100

func main() {
	// Try to connect to the database
	torrentDB, err = NewTorrentDB("10.0.1.12")
	if err != nil {
		fmt.Println("Couldn't connect to Mongo. Please make sure it is installed and running.")
		return
	}
	defer torrentDB.Close()

	if len(os.Args) > 1 {
		importFile(os.Args[1])
	} else {
		runServer()
	}
}

func runServer() {
	fmt.Println("Starting the API server.")
	api := NewAPI()
	api.AddRoutes()
	api.Run(":1337")
}

func importFile(filename string) {
	fmt.Print("Attempting to parse ")
	fmt.Println(filename)

	file, err := os.Open(filename)
	if err != nil {
		fmt.Println("Error opening the file. Make sure it exists and is readable.")
		return
	}
	defer file.Close()

	scanner := bufio.NewScanner(file)
	imported := 0
	skipped := 0
	for scanner.Scan() {
		status, _ := importLine(scanner.Text())
		if status {
			imported++
		} else {
			skipped++
		}
	}

	fmt.Println("File parsing ended.")
	fmt.Print("New: ")
	fmt.Println(imported)
	fmt.Print("Dup: ")
	fmt.Println(skipped)

	if err := scanner.Err(); err != nil {
		fmt.Println("The program encountered an error while reading the file. Ensure that the file isn't corrupted.")
		return
	}
}

func importLine(line string) (bool, error) {
	if strings.Count(line, "|") != 4 {
		return false, errors.New("Something's up with this torrent. Expected 5 values separated by |.")
	}
	data := strings.Split(line, "|")
	return torrentDB.Insert(data[0], data[1], data[2], data[3], data[4])
}
<commit_msg>Change mongo url back to loopback.<commit_after>package main

import (
	"bufio"
	"errors"
	"fmt"
	// "log"
	"os"
	"strings"
)

var torrentDB *TorrentDB
var err error

const resultLimit int = 100

func main() {
	// Try to connect to the database
	torrentDB, err = NewTorrentDB("127.0.0.1")
	if err != nil {
		fmt.Println("Couldn't connect to Mongo. Please make sure it is installed and running.")
		return
	}
	defer torrentDB.Close()

	if len(os.Args) > 1 {
		importFile(os.Args[1])
	} else {
		runServer()
	}
}

func runServer() {
	fmt.Println("Starting the API server.")
	api := NewAPI()
	api.AddRoutes()
	api.Run(":1337")
}

func importFile(filename string) {
	fmt.Print("Attempting to parse ")
	fmt.Println(filename)

	file, err := os.Open(filename)
	if err != nil {
		fmt.Println("Error opening the file. Make sure it exists and is readable.")
		return
	}
	defer file.Close()

	scanner := bufio.NewScanner(file)
	imported := 0
	skipped := 0
	for scanner.Scan() {
		status, _ := importLine(scanner.Text())
		if status {
			imported++
		} else {
			skipped++
		}
	}

	fmt.Println("File parsing ended.")
	fmt.Print("New: ")
	fmt.Println(imported)
	fmt.Print("Dup: ")
	fmt.Println(skipped)

	if err := scanner.Err(); err != nil {
		fmt.Println("The program encountered an error while reading the file. Ensure that the file isn't corrupted.")
		return
	}
}

func importLine(line string) (bool, error) {
	if strings.Count(line, "|") != 4 {
		return false, errors.New("Something's up with this torrent. Expected 5 values separated by |.")
	}
	data := strings.Split(line, "|")
	return torrentDB.Insert(data[0], data[1], data[2], data[3], data[4])
}
<|endoftext|>


<commit_before>package graph

import (
	"container/heap"
	"github.com/twmb/algoimpl/go/graph/lite"
	"math/rand"
	"sync"
	"time"
)

const (
	unseen = iota
	seen
	queued = 1 << 30 // assume that no weight will be > 2^30
)

// O(V + E). It does not matter to traverse back
// on a bidirectional edge, because any vertex dfs is
// recursing on is marked as visited and won't be visited
// again anyway.
func (g *Graph) dfs(node *node, finishList *[]Node) {
	node.state = seen
	for _, edge := range node.edges {
		if edge.end.state == unseen {
			edge.end.parent = node
			g.dfs(edge.end, finishList)
		}
	}
	*finishList = append(*finishList, node.container)
}

func (g *Graph) dfsReversedEdges(node *node, finishList *[]Node) {
	node.state = seen
	for _, edge := range node.reversedEdges {
		if edge.end.state == unseen {
			edge.end.parent = node
			g.dfsReversedEdges(edge.end, finishList)
		}
	}
	*finishList = append(*finishList, node.container)
}

// Topologically sorts a directed acyclic graph.
// If the graph is cyclic, the sort order will change
// based on which node the sort starts on. O(V+E) complexity.
func (g *Graph) TopologicalSort() []Node {
	sorted := make([]Node, 0, len(g.nodes))
	// sort preorder (first jacket, then shirt)
	for _, node := range g.nodes {
		if node.state == unseen {
			g.dfs(node, &sorted)
		}
	}
	// now make post order for correct sort (jacket follows shirt). O(V)
	length := len(sorted)
	for i := 0; i < length/2; i++ {
		sorted[i], sorted[length-i-1] = sorted[length-i-1], sorted[i]
	}
	return sorted
}

// Returns reversed copy of the directed graph g. O(V+E) complexity.
// This function can be used to copy an undirected graph.
func (g *Graph) Reverse() *Graph {
	reversed := New(Directed)
	if g.kind == Undirected {
		reversed = New(Undirected)
	}
	// O(V)
	for _ = range g.nodes {
		reversed.MakeNode()
	}
	// O(V + E)
	for _, node := range g.nodes {
		for _, edge := range node.edges {
			reversed.MakeEdge(reversed.nodes[edge.end.index].container, reversed.nodes[node.index].container)
		}
	}
	return reversed
}

// Returns a slice of strongly connected nodes on a directed graph.
// If passed an undirected graph, returns nil.
// The returned components have reversed, nonexclusive edges.
// For example, if this is passed the graph
//     a->b, c
//     b->a, c
//     c
// will return components
//     [[c->a, b], [b->a], [a->b]]
// where -> represents the edges that the node contains.
// O(V+E) complexity.
func (g *Graph) StronglyConnectedComponents() [][]Node {
	if g.kind == Undirected {
		return nil
	}
	components := make([][]Node, 0)
	finishOrder := g.TopologicalSort()
	for i := range finishOrder {
		finishOrder[i].node.state = unseen
	}
	for _, sink := range finishOrder {
		if g.nodes[sink.node.index].state == unseen {
			component := make([]Node, 0)
			g.dfsReversedEdges(g.nodes[sink.node.index], &component)
			components = append(components, component)
		}
	}
	return components
}

// RandMinimumCut runs Kargers algorithm to find a random minimum cut
// on the graph. If iterations is < 1, this will return an empty slice.
// Otherwise, it returns a slice of the edges crossing the best minimum
// cut found in any iteration.
//
// This function takes a number of iterations to start concurrently. If
// concurrent is <= 1, it will run only one iteration at a time.
//
// If the graph is Directed, this will return a cut of edges in both directions.
// If the graph is Undirected, this will return a proper min cut.
func (g *Graph) RandMinimumCut(iterations, concurrent int) []Edge {
	var mutex sync.Mutex
	doneChan := make(chan struct{}, iterations)
	if concurrent < 1 {
		concurrent = 1
	}
	sem := make(chan struct{}, concurrent)
	for i := 0; i < concurrent; i++ {
		sem <- struct{}{}
	}
	// make a lite slice of the edges and shuffle for random edge removal order
	rand.Seed(time.Now().Unix())
	var baseAllEdges []lite.Edge
	for n := range g.nodes {
		for _, edge := range g.nodes[n].edges {
			if g.kind == Undirected && n < edge.end.index {
				continue
			}
			baseAllEdges = append(baseAllEdges, lite.Edge{Start: n,
				End: edge.end.index, S: g.nodes[n], E: edge})
		}
	}

	minCutLite := make([]lite.Edge, len(baseAllEdges))

	// reuse lite edges as opposed to rebuild every iteration
	for iter := 0; iter < iterations; iter++ {
		<-sem
		go func() {
			nodecount := len(g.nodes)
			allEdges := make([]lite.Edge, len(baseAllEdges))
			copy(allEdges, baseAllEdges)
			shuffle(allEdges)
			for nodecount > 2 {
				// remove first edge, keep the start node, collapse the end node
				// anything that points to the collapsing node now points to the keep node
				// anything that starts at the collapsing node now starts at the keep node
				keep := allEdges[len(allEdges)-1].Start
				remove := allEdges[len(allEdges)-1].End // deleting this node
				allEdges = allEdges[:len(allEdges)-1]
				for e := 0; e < len(allEdges); e++ {
					if allEdges[e].Start == remove {
						allEdges[e].Start = keep
					}
					if allEdges[e].End == remove {
						allEdges[e].End = keep
					}
					// remove the node if it self looped
					if allEdges[e].Start == allEdges[e].End {
						allEdges[e] = allEdges[len(allEdges)-1]
						allEdges = allEdges[:len(allEdges)-1]
						e--
					}
				}

				nodecount--
			}

			mutex.Lock()
			if iter == 0 || len(allEdges) < len(minCutLite) {
				minCutLite = make([]lite.Edge, len(allEdges))
				copy(minCutLite, allEdges)
			}
			mutex.Unlock()
			doneChan <- struct{}{}
			sem <- struct{}{}
		}()
	}
	for iter := 0; iter < iterations; iter++ {
		<-doneChan
	}

	minCut := make([]Edge, len(minCutLite))
	for i := range minCutLite {
		start := minCutLite[i].S.(*node)
		edge := minCutLite[i].E.(edge)
		minCut[i] = Edge{Weight: edge.weight, Start: start.container,
			End: edge.end.container, Kind: g.kind}
	}
	return minCut
}

// Fischer-Yates shuffle
func shuffle(edges []lite.Edge) {
	for i := len(edges) - 1; i > 0; i-- {
		j := rand.Intn(i + 1)
		edges[j], edges[i] = edges[i], edges[j]
	}
}

// This function will return the edges corresponding to the
// minimum spanning tree in the graph based off of the edge's weight values.
func (g *Graph) MinimumSpanningTree() []Edge {
	// create priority queue for vertices
	nodesBase := nodeSlice(make([]*node, 0))
	nodes := &nodesBase
	heap.Init(nodes)
	for _, node := range g.nodes[1:] {
		node.state = 1<<30 - 1 // bit 31 is for queue testing
		node.state |= queued
		heap.Push(nodes, node)
	}
	g.nodes[0].state = 0
	heap.Push(nodes, g.nodes[0]) // now add root element

	for nodes.Len() > 0 {
		min := heap.Pop(nodes).(*node)
		for _, edge := range min.edges {
			v := edge.end // get the other side of the edge
			if nodes.QueueContains(v) && edge.weight < (v.state & ^queued) {
				v.parent = min
				heap.Remove(nodes, v.data)     // remove it
				v.state = edge.weight | queued // update state, add queue bit
				heap.Push(nodes, v)            // add it back in, WORD, O(lg n) update key time!
			}
		}
	}

	mst := make([]Edge, 0)
	for i := range g.nodes {
		if g.nodes[i].parent != nil {
			mst = append(mst, Edge{Weight: g.edgeWeightBetween(g.nodes[i], g.nodes[i].parent),
				Start: g.nodes[i].container, End: g.nodes[i].parent.container, Kind: g.kind})
		}
	}

	return mst
}

// only called when the graph is guaranteed to have an edge
// between the two nodes
func (g *Graph) edgeWeightBetween(v, u *node) int {
	for _, edge := range u.edges {
		// one of the two is always u
		if edge.end == v {
			return edge.weight
		}
	}
	return 0
}

type nodeSlice []*node

func (n *nodeSlice) Push(x interface{}) {
	p := x.(*node)
	p.data = len(*n) // index into heap
	*n = append(*n, x.(*node))
}
func (n *nodeSlice) Pop() interface{} {
	rNode := (*n)[len(*n)-1]
	rNode.state &= ^queued
	*n = (*n)[0 : len(*n)-1]
	return rNode
}
func (n nodeSlice) Len() int {
	return len(n)
}
func (n nodeSlice) Less(i, j int) bool {
	return n[i].state & ^queued < n[j].state & ^queued
}
func (n nodeSlice) Swap(i, j int) {
	n[j].data, n[i].data = n[i].data, n[j].data // swap data containing indices
	n[j], n[i] = n[i], n[j]
}
func (n nodeSlice) QueueContains(node *node) bool { // extend heap interface
	return node.state&queued > 0
}
<commit_msg>updated documentation<commit_after>package graph

import (
	"container/heap"
	"github.com/twmb/algoimpl/go/graph/lite"
	"math/rand"
	"sync"
	"time"
)

const (
	unseen = iota
	seen
	queued = 1 << 30 // assume that no weight will be > 2^30
)

// O(V + E). It does not matter to traverse back
// on a bidirectional edge, because any vertex dfs is
// recursing on is marked as visited and won't be visited
// again anyway.
func (g *Graph) dfs(node *node, finishList *[]Node) {
	node.state = seen
	for _, edge := range node.edges {
		if edge.end.state == unseen {
			edge.end.parent = node
			g.dfs(edge.end, finishList)
		}
	}
	*finishList = append(*finishList, node.container)
}

func (g *Graph) dfsReversedEdges(node *node, finishList *[]Node) {
	node.state = seen
	for _, edge := range node.reversedEdges {
		if edge.end.state == unseen {
			edge.end.parent = node
			g.dfsReversedEdges(edge.end, finishList)
		}
	}
	*finishList = append(*finishList, node.container)
}

// Topologically sorts a directed acyclic graph.
// If the graph is cyclic, the sort order will change
// based on which node the sort starts on. O(V+E) complexity.
func (g *Graph) TopologicalSort() []Node {
	sorted := make([]Node, 0, len(g.nodes))
	// sort preorder (first jacket, then shirt)
	for _, node := range g.nodes {
		if node.state == unseen {
			g.dfs(node, &sorted)
		}
	}
	// now make post order for correct sort (jacket follows shirt). O(V)
	length := len(sorted)
	for i := 0; i < length/2; i++ {
		sorted[i], sorted[length-i-1] = sorted[length-i-1], sorted[i]
	}
	return sorted
}

// Returns reversed copy of the directed graph g. O(V+E) complexity.
// This function can be used to copy an undirected graph.
func (g *Graph) Reverse() *Graph {
	reversed := New(Directed)
	if g.kind == Undirected {
		reversed = New(Undirected)
	}
	// O(V)
	for _ = range g.nodes {
		reversed.MakeNode()
	}
	// O(V + E)
	for _, node := range g.nodes {
		for _, edge := range node.edges {
			reversed.MakeEdge(reversed.nodes[edge.end.index].container,
				reversed.nodes[node.index].container)
		}
	}
	return reversed
}

// Returns a slice of strongly connected nodes on a directed graph.
// If passed an undirected graph, returns nil.
// The returned components have reversed, nonexclusive edges.
// For example, if this is passed the graph
//     a->b, c
//     b->a, c
//     c
// will return components
//     [[c->a, b], [b->a], [a->b]]
// where -> represents the edges that the node contains.
// O(V+E) complexity.
func (g *Graph) StronglyConnectedComponents() [][]Node {
	if g.kind == Undirected {
		return nil
	}
	components := make([][]Node, 0)
	finishOrder := g.TopologicalSort()
	for i := range finishOrder {
		finishOrder[i].node.state = unseen
	}
	for _, sink := range finishOrder {
		if g.nodes[sink.node.index].state == unseen {
			component := make([]Node, 0)
			g.dfsReversedEdges(g.nodes[sink.node.index], &component)
			components = append(components, component)
		}
	}
	return components
}

// RandMinimumCut runs Kargers algorithm to find a random minimum cut
// on the graph. If iterations is < 1, this will return an empty slice.
// Otherwise, it returns a slice of the edges crossing the best minimum
// cut found in any iteration.
//
// This function takes a number of iterations to start concurrently. If
// concurrent is <= 1, it will run one iteration at a time.
//
// If the graph is Directed, this will return a cut of edges in both directions.
// If the graph is Undirected, this will return a proper min cut.
func (g *Graph) RandMinimumCut(iterations, concurrent int) []Edge {
	var mutex sync.Mutex
	doneChan := make(chan struct{}, iterations)
	if concurrent < 1 {
		concurrent = 1
	}
	sem := make(chan struct{}, concurrent)
	for i := 0; i < concurrent; i++ {
		sem <- struct{}{}
	}
	// make a lite slice of the edges
	rand.Seed(time.Now().Unix())
	var baseAllEdges []lite.Edge
	for n := range g.nodes {
		for _, edge := range g.nodes[n].edges {
			if g.kind == Undirected && n < edge.end.index {
				continue
			}
			baseAllEdges = append(baseAllEdges, lite.Edge{Start: n,
				End: edge.end.index, S: g.nodes[n], E: edge})
		}
	}

	minCutLite := make([]lite.Edge, len(baseAllEdges))

	for iter := 0; iter < iterations; iter++ {
		<-sem
		go func() {
			nodecount := len(g.nodes)
			allEdges := make([]lite.Edge, len(baseAllEdges))
			copy(allEdges, baseAllEdges)
			// shuffle for random edge removal order
			shuffle(allEdges)
			for nodecount > 2 {
				// remove first edge, keep the start node, collapse the end node
				// anything that points to the collapsing node now points to the keep node
				// anything that starts at the collapsing node now starts at the keep node
				keep := allEdges[len(allEdges)-1].Start
				remove := allEdges[len(allEdges)-1].End
				allEdges = allEdges[:len(allEdges)-1]
				for e := 0; e < len(allEdges); e++ {
					if allEdges[e].Start == remove {
						allEdges[e].Start = keep
					}
					if allEdges[e].End == remove {
						allEdges[e].End = keep
					}
					// remove the edge if it self looped
					if allEdges[e].Start == allEdges[e].End {
						allEdges[e] = allEdges[len(allEdges)-1]
						allEdges = allEdges[:len(allEdges)-1]
						e--
					}
				}
				// every edge removed removes a node
				nodecount--
			}

			mutex.Lock()
			if iter == 0 || len(allEdges) < len(minCutLite) {
				minCutLite = make([]lite.Edge, len(allEdges))
				copy(minCutLite, allEdges)
			}
			mutex.Unlock()

			doneChan <- struct{}{}
			sem <- struct{}{}
		}()
	}
	for iter := 0; iter < iterations; iter++ {
		<-doneChan
	}

	minCut := make([]Edge, len(minCutLite))
	for i := range minCutLite {
		start := minCutLite[i].S.(*node)
		edge := minCutLite[i].E.(edge)
		minCut[i] = Edge{Weight: edge.weight, Start: start.container,
			End: edge.end.container, Kind: g.kind}
	}
	return minCut
}

// Fischer-Yates shuffle
func shuffle(edges []lite.Edge) {
	for i := len(edges) - 1; i > 0; i-- {
		j := rand.Intn(i + 1)
		edges[j], edges[i] = edges[i], edges[j]
	}
}

// This function will return the edges corresponding to the
// minimum spanning tree in the graph based off of the edge's weight values.
func (g *Graph) MinimumSpanningTree() []Edge {
	// create priority queue for vertices
	nodesBase := nodeSlice(make([]*node, 0))
	nodes := &nodesBase
	heap.Init(nodes)
	for _, node := range g.nodes[1:] {
		node.state = 1<<30 - 1 // bit 31 is for queue testing
		node.state |= queued
		heap.Push(nodes, node)
	}
	g.nodes[0].state = 0
	heap.Push(nodes, g.nodes[0]) // now add root element

	for nodes.Len() > 0 {
		min := heap.Pop(nodes).(*node)
		for _, edge := range min.edges {
			v := edge.end // get the other side of the edge
			if nodes.QueueContains(v) && edge.weight < (v.state & ^queued) {
				v.parent = min
				heap.Remove(nodes, v.data)     // remove it
				v.state = edge.weight | queued // update state, add queue bit
				heap.Push(nodes, v)            // add it back in, WORD, O(lg n) update key time!
			}
		}
	}

	mst := make([]Edge, 0)
	for i := range g.nodes {
		if g.nodes[i].parent != nil {
			mst = append(mst, Edge{Weight: g.edgeWeightBetween(g.nodes[i], g.nodes[i].parent),
				Start: g.nodes[i].container, End: g.nodes[i].parent.container, Kind: g.kind})
		}
	}

	return mst
}

// only called when the graph is guaranteed to have an edge
// between the two nodes
func (g *Graph) edgeWeightBetween(v, u *node) int {
	for _, edge := range u.edges {
		// one of the two is always u
		if edge.end == v {
			return edge.weight
		}
	}
	return 0
}

type nodeSlice []*node

func (n *nodeSlice) Push(x interface{}) {
	p := x.(*node)
	p.data = len(*n) // index into heap
	*n = append(*n, x.(*node))
}
func (n *nodeSlice) Pop() interface{} {
	rNode := (*n)[len(*n)-1]
	rNode.state &= ^queued
	*n = (*n)[0 : len(*n)-1]
	return rNode
}
func (n nodeSlice) Len() int {
	return len(n)
}
func (n nodeSlice) Less(i, j int) bool {
	return n[i].state & ^queued < n[j].state & ^queued
}
func (n nodeSlice) Swap(i, j int) {
	n[j].data, n[i].data = n[i].data, n[j].data // swap data containing indices
	n[j], n[i] = n[i], n[j]
}
func (n nodeSlice) QueueContains(node *node) bool { // extend heap interface
	return node.state&queued > 0
}
<|endoftext|>


<commit_before>// Copyright 2013 The Pythia Authors.
// This file is part of Pythia.
//
// Pythia is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, version 3 of the License.
//
// Pythia is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with Pythia.  If not, see <http://www.gnu.org/licenses/>.

package pythia

import (
	"encoding/json"
	"io"
	"log"
	"net"
)

// Conn is a wrapper over net.Conn, reading and writing Messages.
type Conn struct {
	// The underlying connection.
	conn net.Conn

	// Message channels.
	input, output chan Message

	// Flag to ignore errors after closing the connection.
	closed bool
}

// WrapConn wraps a stream network connection into a Message-oriented
// connection. The raw conn connection shall not be used by the user anymore.
func WrapConn(conn net.Conn) *Conn {
	c := new(Conn)
	c.conn = conn
	c.input, c.output = make(chan Message), make(chan Message)
	go c.reader()
	go c.writer()
	return c
}

// The reader goroutine parses the Messages and put them in the input channel.
func (c *Conn) reader() {
	defer close(c.input)
	dec := json.NewDecoder(c.conn)
	for {
		var msg Message
		if err := dec.Decode(&msg); err == io.EOF || c.closed {
			break
		} else if err != nil {
			log.Fatal(err)
		}
		c.input <- msg
	}
}

// The writer goroutine fetches Messages from the output channel and sends them
// through the network connection.
func (c *Conn) writer() {
	enc := json.NewEncoder(c.conn)
	for msg := range c.output {
		if err := enc.Encode(msg); err != nil {
			log.Fatal(err)
		}
	}
}

// Dial connects to the address addr and returns a Message-oriented connection.
func Dial(addr net.Addr) (*Conn, error) {
	conn, err := net.Dial(addr.Network(), addr.String())
	if err != nil {
		return nil, err
	}
	return WrapConn(conn), nil
}

// Send sends a message through the connection.
func (c *Conn) Send(msg Message) {
	if c.closed {
		log.Fatal("Try to write message on closed connection.")
	}
	c.output <- msg
}

// Receive returns the channel from which incoming messages can be retrieved.
func (c *Conn) Receive() <-chan Message {
	return c.input
}

// Close closes the connection. The receive channel will also be closed.
func (c *Conn) Close() {
	c.closed = true
	close(c.output)
	if err := c.conn.Close(); err != nil {
		log.Fatal(err)
	}
}

// vim:set sw=4 ts=4 noet:
<commit_msg>Add bug notice for pythia.Conn<commit_after>// Copyright 2013 The Pythia Authors.
// This file is part of Pythia.
//
// Pythia is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, version 3 of the License.
//
// Pythia is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with Pythia.  If not, see <http://www.gnu.org/licenses/>.

package pythia

import (
	"encoding/json"
	"io"
	"log"
	"net"
)

// BUG(vianney): When a connection is closed from the remote side, the writer
// goroutine of pythia.Conn will remain running.

// Conn is a wrapper over net.Conn, reading and writing Messages.
type Conn struct {
	// The underlying connection.
	conn net.Conn

	// Message channels.
	input, output chan Message

	// Flag to ignore errors after closing the connection.
	closed bool
}

// WrapConn wraps a stream network connection into a Message-oriented
// connection. The raw conn connection shall not be used by the user anymore.
func WrapConn(conn net.Conn) *Conn {
	c := new(Conn)
	c.conn = conn
	c.input, c.output = make(chan Message), make(chan Message)
	go c.reader()
	go c.writer()
	return c
}

// The reader goroutine parses the Messages and put them in the input channel.
func (c *Conn) reader() {
	defer close(c.input)
	dec := json.NewDecoder(c.conn)
	for {
		var msg Message
		if err := dec.Decode(&msg); err == io.EOF || c.closed {
			break
		} else if err != nil {
			log.Fatal(err)
		}
		c.input <- msg
	}
}

// The writer goroutine fetches Messages from the output channel and sends them
// through the network connection.
func (c *Conn) writer() {
	enc := json.NewEncoder(c.conn)
	for msg := range c.output {
		if err := enc.Encode(msg); err != nil {
			log.Fatal(err)
		}
	}
}

// Dial connects to the address addr and returns a Message-oriented connection.
func Dial(addr net.Addr) (*Conn, error) {
	conn, err := net.Dial(addr.Network(), addr.String())
	if err != nil {
		return nil, err
	}
	return WrapConn(conn), nil
}

// Send sends a message through the connection.
func (c *Conn) Send(msg Message) {
	if c.closed {
		log.Fatal("Try to write message on closed connection.")
	}
	c.output <- msg
}

// Receive returns the channel from which incoming messages can be retrieved.
func (c *Conn) Receive() <-chan Message {
	return c.input
}

// Close closes the connection. The receive channel will also be closed.
func (c *Conn) Close() {
	c.closed = true
	close(c.output)
	if err := c.conn.Close(); err != nil {
		log.Fatal(err)
	}
}

// vim:set sw=4 ts=4 noet:
<|endoftext|>


<commit_before>// Copyright 2012, Google Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package stats

import (
	"bytes"
	"fmt"
	"sync"
)

// Histogram tracks counts and totals while
// splitting the counts under different buckets
// using specified cutoffs.
type Histogram struct {
	cutoffs    []int64
	labels     []string
	countLabel string
	totalLabel string
	hook       func(int64)

	// mu controls buckets & total
	mu      sync.Mutex
	buckets []int64
	total   int64
}

// NewHistogram creates a histogram with auto-generated labels
// based on the cutoffs. The buckets are categorized using the
// following criterion: cutoff[i-1] <= value < cutoff[i]. Anything
// higher than the highest cutoff is labeled as "inf".
func NewHistogram(name string, cutoffs []int64) *Histogram {
	labels := make([]string, len(cutoffs)+1)
	for i, v := range cutoffs {
		labels[i] = fmt.Sprintf("%d", v)
	}
	labels[len(labels)-1] = "inf"
	return NewGenericHistogram(name, cutoffs, labels, "Count", "Total")
}

// NewGenericHistogram creates a histogram where all the labels are
// supplied by the caller. The number of labels has to be one more than
// the number of cutoffs because the last label captures everything that
// exceeds the highest cutoff.
func NewGenericHistogram(name string, cutoffs []int64, labels []string, countLabel, totalLabel string) *Histogram {
	if len(cutoffs) != len(labels)-1 {
		panic("mismatched cutoff and label lengths")
	}
	h := &Histogram{
		cutoffs:    cutoffs,
		labels:     labels,
		countLabel: countLabel,
		totalLabel: totalLabel,
		buckets:    make([]int64, len(labels)),
		total:      0,
	}
	if name != "" {
		Publish(name, h)
	}
	return h
}

// Add adds a new measurement to the Histogram.
func (h *Histogram) Add(value int64) {
	for i := range h.labels {
		if i == len(h.labels)-1 || value < h.cutoffs[i] {
			h.mu.Lock()
			h.buckets[i]++
			h.total += value
			h.mu.Unlock()
			return
		}
	}
	if h.hook != nil {
		h.hook(value)
	}
}

// String returns a string representation of the Histogram.
func (h *Histogram) String() string {
	b, _ := h.MarshalJSON()
	return string(b)
}

// MarshalJSON returns a JSON representation of the Histogram.
func (h *Histogram) MarshalJSON() ([]byte, error) {
	h.mu.Lock()
	defer h.mu.Unlock()

	b := bytes.NewBuffer(make([]byte, 0, 4096))
	fmt.Fprintf(b, "{")
	totalCount := int64(0)
	for i, label := range h.labels {
		totalCount += h.buckets[i]
		fmt.Fprintf(b, "\"%v\": %v, ", label, totalCount)
	}
	fmt.Fprintf(b, "\"%s\": %v, ", h.countLabel, totalCount)
	fmt.Fprintf(b, "\"%s\": %v", h.totalLabel, h.total)
	fmt.Fprintf(b, "}")
	return b.Bytes(), nil
}

// Counts returns a map from labels to the current count in the Histogram for that label.
func (h *Histogram) Counts() map[string]int64 {
	h.mu.Lock()
	defer h.mu.Unlock()

	counts := make(map[string]int64, len(h.labels))
	for i, label := range h.labels {
		counts[label] = h.buckets[i]
	}
	return counts
}

// CountLabel returns the count label that was set when this Histogram was created.
func (h *Histogram) CountLabel() string {
	return h.countLabel
}

// Count returns the number of times Add has been called.
func (h *Histogram) Count() (count int64) {
	h.mu.Lock()
	defer h.mu.Unlock()

	for _, v := range h.buckets {
		count += v
	}
	return
}

// TotalLabel returns the total label that was set when this Histogram was created.
func (h *Histogram) TotalLabel() string {
	return h.totalLabel
}

// Total returns the sum of all values that have been added to this Histogram.
func (h *Histogram) Total() (total int64) {
	h.mu.Lock()
	defer h.mu.Unlock()
	return h.total
}

// Labels returns the labels that were set when this Histogram was created.
func (h *Histogram) Labels() []string {
	return h.labels
}

// Cutoffs returns the cutoffs that were set when this Histogram was created.
func (h *Histogram) Cutoffs() []int64 {
	return h.cutoffs
}

// Buckets returns a snapshot of the current values in all buckets.
func (h *Histogram) Buckets() []int64 {
	h.mu.Lock()
	defer h.mu.Unlock()
	buckets := make([]int64, len(h.buckets))
	copy(buckets, h.buckets)
	return buckets
}
<commit_msg>Exporting internal changes back to open-source.<commit_after>// Copyright 2012, Google Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package stats

import (
	"bytes"
	"fmt"
	"sync"
)

// Histogram tracks counts and totals while
// splitting the counts under different buckets
// using specified cutoffs.
type Histogram struct {
	cutoffs    []int64
	labels     []string
	countLabel string
	totalLabel string
	hook       func(int64)

	// mu controls buckets & total
	mu      sync.Mutex
	buckets []int64
	total   int64
}

// NewHistogram creates a histogram with auto-generated labels
// based on the cutoffs. The buckets are categorized using the
// following criterion: cutoff[i-1] <= value < cutoff[i]. Anything
// higher than the highest cutoff is labeled as "inf".
func NewHistogram(name string, cutoffs []int64) *Histogram {
	labels := make([]string, len(cutoffs)+1)
	for i, v := range cutoffs {
		labels[i] = fmt.Sprintf("%d", v)
	}
	labels[len(labels)-1] = "inf"
	return NewGenericHistogram(name, cutoffs, labels, "Count", "Total")
}

// NewGenericHistogram creates a histogram where all the labels are
// supplied by the caller. The number of labels has to be one more than
// the number of cutoffs because the last label captures everything that
// exceeds the highest cutoff.
func NewGenericHistogram(name string, cutoffs []int64, labels []string, countLabel, totalLabel string) *Histogram {
	if len(cutoffs) != len(labels)-1 {
		panic("mismatched cutoff and label lengths")
	}
	h := &Histogram{
		cutoffs:    cutoffs,
		labels:     labels,
		countLabel: countLabel,
		totalLabel: totalLabel,
		buckets:    make([]int64, len(labels)),
		total:      0,
	}
	if name != "" {
		Publish(name, h)
	}
	return h
}

// Add adds a new measurement to the Histogram.
func (h *Histogram) Add(value int64) {
	for i := range h.labels {
		if i == len(h.labels)-1 || value < h.cutoffs[i] {
			h.mu.Lock()
			h.buckets[i]++
			h.total += value
			h.mu.Unlock()
			break
		}
	}
	if h.hook != nil {
		h.hook(value)
	}
}

// String returns a string representation of the Histogram.
func (h *Histogram) String() string {
	b, _ := h.MarshalJSON()
	return string(b)
}

// MarshalJSON returns a JSON representation of the Histogram.
func (h *Histogram) MarshalJSON() ([]byte, error) {
	h.mu.Lock()
	defer h.mu.Unlock()

	b := bytes.NewBuffer(make([]byte, 0, 4096))
	fmt.Fprintf(b, "{")
	totalCount := int64(0)
	for i, label := range h.labels {
		totalCount += h.buckets[i]
		fmt.Fprintf(b, "\"%v\": %v, ", label, totalCount)
	}
	fmt.Fprintf(b, "\"%s\": %v, ", h.countLabel, totalCount)
	fmt.Fprintf(b, "\"%s\": %v", h.totalLabel, h.total)
	fmt.Fprintf(b, "}")
	return b.Bytes(), nil
}

// Counts returns a map from labels to the current count in the Histogram for that label.
func (h *Histogram) Counts() map[string]int64 {
	h.mu.Lock()
	defer h.mu.Unlock()

	counts := make(map[string]int64, len(h.labels))
	for i, label := range h.labels {
		counts[label] = h.buckets[i]
	}
	return counts
}

// CountLabel returns the count label that was set when this Histogram was created.
func (h *Histogram) CountLabel() string {
	return h.countLabel
}

// Count returns the number of times Add has been called.
func (h *Histogram) Count() (count int64) {
	h.mu.Lock()
	defer h.mu.Unlock()

	for _, v := range h.buckets {
		count += v
	}
	return
}

// TotalLabel returns the total label that was set when this Histogram was created.
func (h *Histogram) TotalLabel() string {
	return h.totalLabel
}

// Total returns the sum of all values that have been added to this Histogram.
func (h *Histogram) Total() (total int64) {
	h.mu.Lock()
	defer h.mu.Unlock()
	return h.total
}

// Labels returns the labels that were set when this Histogram was created.
func (h *Histogram) Labels() []string {
	return h.labels
}

// Cutoffs returns the cutoffs that were set when this Histogram was created.
func (h *Histogram) Cutoffs() []int64 {
	return h.cutoffs
}

// Buckets returns a snapshot of the current values in all buckets.
func (h *Histogram) Buckets() []int64 {
	h.mu.Lock()
	defer h.mu.Unlock()
	buckets := make([]int64, len(h.buckets))
	copy(buckets, h.buckets)
	return buckets
}
<|endoftext|>


<commit_before>package task

import "github.com/pborman/uuid"

// Task states
const (
	// Unknown state of the task
	// This is the default state of a task
	// when new task is initially created
	TaskStateUnknown = "unknown"

	// Task has been received by the
	// minion and is queued for execution
	TaskStateQueued = "queued"

	// Task is being processed
	TaskStateProcessing = "processing"

	// Task has been processed by the
	// minion and was flagged as successful
	TaskStateSuccess = "success"

	// Task has been processed by the
	// minion and was flagged as failed
	TaskStateFailed = "failed"
)

// Task type represents a task that is processed by minions
type Task struct {
	// Command to be executed by the minion
	Command string `json:"command"`

	// Command arguments
	Args []string `json:"args"`

	// Time when the command was sent for processing
	TimeReceived int64 `json:"timeReceived"`

	// Time when the command was processed
	TimeProcessed int64 `json:"timeProcessed"`

	// Task unique identifier
	TaskID uuid.UUID `json:"taskId"`

	// Result of task after processing
	Result string `json:"result"`

	// If true this task can run concurrently with other tasks
	IsConcurrent bool `json:"isConcurrent"`

	// Task error, if any
	Error string `json:"error"`

	// Task state
	State string `json:"state"`
}

// New creates a new task
func New(command string, args ...string) *Task {
	t := &Task{
		Command: command,
		Args:    args,
		TaskID:  uuid.NewRandom(),
		State:   TaskStateUnknown,
	}

	return t
}
<commit_msg>task.Task contains a catalog to be processed by minions<commit_after>package task

import (
	"github.com/dnaeon/gru/catalog"
	"github.com/pborman/uuid"
)

// Task states
const (
	// Unknown state of the task
	// This is the default state of a task
	// when new task is initially created
	TaskStateUnknown = "unknown"

	// Task has been received by the
	// minion and is queued for execution
	TaskStateQueued = "queued"

	// Task is being processed
	TaskStateProcessing = "processing"

	// Task has been processed by the
	// minion and was flagged as successful
	TaskStateSuccess = "success"

	// Task has been processed by the
	// minion and was flagged as failed
	TaskStateFailed = "failed"
)

// Task type represents a task that is processed by minions
type Task struct {
	// Catalog to be processed
	Catalog *catalog.Catalog `json:"catalog"`

	// Time when the command was sent for processing
	TimeReceived int64 `json:"timeReceived"`

	// Time when the command was processed
	TimeProcessed int64 `json:"timeProcessed"`

	// Task unique identifier
	TaskID uuid.UUID `json:"taskId"`

	// Result of task after processing
	Result string `json:"result"`

	// If true this task can run concurrently with other tasks
	IsConcurrent bool `json:"isConcurrent"`

	// Task state
	State string `json:"state"`
}

// New creates a new task
func New(c *catalog.Catalog) *Task {
	t := &Task{
		Catalog: c,
		TaskID:  uuid.NewRandom(),
		State:   TaskStateUnknown,
	}

	return t
}
<|endoftext|>


<commit_before>package tests

import (
	"context"
	"fmt"
	"os"
	"testing"
	"time"

	"github.com/keel-hq/keel/secrets"
	"github.com/keel-hq/keel/types"
	log "github.com/sirupsen/logrus"
	apps_v1 "k8s.io/api/apps/v1"
	"k8s.io/api/core/v1"
	meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

func TestPollingSemverUpdate(t *testing.T) {

	// stop := make(chan struct{})
	ctx, cancel := context.WithCancel(context.Background())
	// defer close(ctx)
	defer cancel()

	// go startKeel(ctx)
	keel := &KeelCmd{}
	go func() {
		err := keel.Start(ctx)
		if err != nil {
			log.WithFields(log.Fields{
				"error": err,
			}).Error("failed to start Keel process")
		}
	}()

	defer func() {
		err := keel.Stop()
		if err != nil {
			log.WithFields(log.Fields{
				"error": err,
			}).Error("failed to stop Keel process")
		}
	}()

	_, kcs := getKubernetesClient()

	t.Run("UpdateThroughDockerHubPollingA", func(t *testing.T) {
		// UpdateThroughDockerHubPollingA tests a polling trigger when we have a higher version
		// but without a pre-release tag and a lower version with pre-release. The version of the deployment
		// is with pre-prerealse so we should upgrade to that one.

		testNamespace := createNamespaceForTest()
		defer deleteTestNamespace(testNamespace)

		dep := &apps_v1.Deployment{
			meta_v1.TypeMeta{},
			meta_v1.ObjectMeta{
				Name:      "deployment-1",
				Namespace: testNamespace,
				Labels: map[string]string{
					types.KeelPolicyLabel:  "major",
					types.KeelTriggerLabel: "poll",
				},
				Annotations: map[string]string{
					types.KeelPollScheduleAnnotation: "@every 2s",
				},
			},
			apps_v1.DeploymentSpec{
				Selector: &meta_v1.LabelSelector{
					MatchLabels: map[string]string{
						"app": "wd-1",
					},
				},
				Template: v1.PodTemplateSpec{
					ObjectMeta: meta_v1.ObjectMeta{
						Labels: map[string]string{
							"app":     "wd-1",
							"release": "1",
						},
					},
					Spec: v1.PodSpec{
						Containers: []v1.Container{
							v1.Container{
								Name:  "wd-1",
								Image: "keelhq/push-workflow-example:0.1.0-dev",
							},
						},
					},
				},
			},
			apps_v1.DeploymentStatus{},
		}

		_, err := kcs.AppsV1().Deployments(testNamespace).Create(dep)
		if err != nil {
			t.Fatalf("failed to create deployment: %s", err)
		}
		ctx, cancel := context.WithTimeout(context.Background(), time.Second*10)
		defer cancel()

		err = waitFor(ctx, kcs, testNamespace, dep.ObjectMeta.Name, "keelhq/push-workflow-example:0.5.0-dev")
		if err != nil {
			t.Errorf("update failed: %s", err)
		}
	})

	t.Run("UpdateThroughDockerHubPollingB", func(t *testing.T) {
		// UpdateThroughDockerHubPollingA tests a polling trigger when we have a higher version
		// but without a pre-release tag and a lower version with pre-release. The version of the deployment
		// is without pre-prerealse

		testNamespace := createNamespaceForTest()
		defer deleteTestNamespace(testNamespace)

		dep := &apps_v1.Deployment{
			meta_v1.TypeMeta{},
			meta_v1.ObjectMeta{
				Name:      "deployment-2",
				Namespace: testNamespace,
				Labels: map[string]string{
					types.KeelPolicyLabel:  "major",
					types.KeelTriggerLabel: "poll",
				},
				Annotations: map[string]string{
					types.KeelPollScheduleAnnotation: "@every 2s",
				},
			},
			apps_v1.DeploymentSpec{
				Selector: &meta_v1.LabelSelector{
					MatchLabels: map[string]string{
						"app": "wd-1",
					},
				},
				Template: v1.PodTemplateSpec{
					ObjectMeta: meta_v1.ObjectMeta{
						Labels: map[string]string{
							"app":     "wd-1",
							"release": "1",
						},
					},
					Spec: v1.PodSpec{
						Containers: []v1.Container{
							v1.Container{
								Name:  "wd-1",
								Image: "keelhq/push-workflow-example:0.1.0",
							},
						},
					},
				},
			},
			apps_v1.DeploymentStatus{},
		}

		_, err := kcs.AppsV1().Deployments(testNamespace).Create(dep)
		if err != nil {
			t.Fatalf("failed to create deployment: %s", err)
		}
		ctx, cancel := context.WithTimeout(context.Background(), time.Second*10)
		defer cancel()

		err = waitFor(ctx, kcs, testNamespace, dep.ObjectMeta.Name, "keelhq/push-workflow-example:0.10.0")
		if err != nil {
			t.Errorf("update failed: %s", err)
		}
	})
}

func TestPollingPrivateRegistry(t *testing.T) {

	// stop := make(chan struct{})
	ctx, cancel := context.WithCancel(context.Background())
	// defer close(ctx)
	defer cancel()

	// go startKeel(ctx)
	keel := &KeelCmd{}
	go func() {
		err := keel.Start(ctx)
		if err != nil {
			log.WithFields(log.Fields{
				"error": err,
			}).Error("failed to start Keel process")
		}
	}()

	defer func() {
		err := keel.Stop()
		if err != nil {
			log.WithFields(log.Fields{
				"error": err,
			}).Error("failed to stop Keel process")
		}
	}()

	_, kcs := getKubernetesClient()

	t.Run("UpdateThroughPrivateQuayPollingA", func(t *testing.T) {

		user := os.Getenv("DOCKERHUB_USERNAME")
		password := os.Getenv("DOCKERHUB_PASSWORD")

		if user == "" || password == "" {
			fmt.Println("[X] Skipping UpdateThroughPrivateQuayPollingA test since DOCKERHUB_USERNAME and/or DOCKERHUB_PASSWORD env vars not set")
			t.Skip()
		}

		// UpdateThroughDockerHubPollingA tests a polling trigger when we have a higher version
		// but without a pre-release tag and a lower version with pre-release. The version of the deployment
		// is with pre-prerealse so we should upgrade to that one.

		testNamespace := createNamespaceForTest()
		defer deleteTestNamespace(testNamespace)

		payload, err := secrets.EncodeDockerCfgJson(&secrets.DockerCfg{
			"https://index.docker.io/v1/": &secrets.Auth{
				Auth: secrets.EncodeBase64Secret(user, password),
			},
		})
		if err != nil {
			t.Fatalf("failed to encode docker cfg secret payload: %s", err)
		}

		secretName := "verysecret"

		fmt.Println(string(payload))

		secret := &v1.Secret{
			TypeMeta: meta_v1.TypeMeta{},
			ObjectMeta: meta_v1.ObjectMeta{
				Name:        secretName,
				Namespace:   testNamespace,
				Labels:      map[string]string{},
				Annotations: map[string]string{},
			},
			Type: v1.SecretTypeDockerConfigJson,
			Data: map[string][]byte{
				".dockerconfigjson": []byte(payload),
			},
		}

		_, err = kcs.CoreV1().Secrets(testNamespace).Create(secret)
		if err != nil {
			t.Fatalf("failed to create secret: %s", err)
		}

		dep := &apps_v1.Deployment{
			meta_v1.TypeMeta{},
			meta_v1.ObjectMeta{
				Name:      "deployment-1",
				Namespace: testNamespace,
				Labels: map[string]string{
					types.KeelPolicyLabel:  "major",
					types.KeelTriggerLabel: "poll",
				},
				Annotations: map[string]string{
					types.KeelPollScheduleAnnotation: "@every 2s",
				},
			},
			apps_v1.DeploymentSpec{

				Selector: &meta_v1.LabelSelector{
					MatchLabels: map[string]string{
						"app": "wd-1",
					},
				},
				Template: v1.PodTemplateSpec{

					ObjectMeta: meta_v1.ObjectMeta{
						Labels: map[string]string{
							"app":     "wd-1",
							"release": "1",
						},
					},
					Spec: v1.PodSpec{
						ImagePullSecrets: []v1.LocalObjectReference{
							{
								Name: secretName,
							},
						},
						Containers: []v1.Container{
							v1.Container{
								ImagePullPolicy: v1.PullAlways,
								Name:            "wd-1",
								Image:           "karolisr/demo-webhook:0.0.1",
							},
						},
					},
				},
			},
			apps_v1.DeploymentStatus{},
		}

		_, err = kcs.AppsV1().Deployments(testNamespace).Create(dep)
		if err != nil {
			t.Fatalf("failed to create deployment: %s", err)
		}
		ctx, cancel := context.WithTimeout(context.Background(), time.Second*10)
		defer cancel()

		err = waitFor(ctx, kcs, testNamespace, dep.ObjectMeta.Name, "karolisr/demo-webhook:0.0.2")
		if err != nil {
			t.Errorf("update failed: %s", err)
		}
	})

	t.Run("UpdateThroughPrivateGitlabPolling", func(t *testing.T) {

		user := os.Getenv("GITLAB_USERNAME")
		password := os.Getenv("GITLAB_PASSWORD")

		if user == "" || password == "" {
			fmt.Println("[X] Skipping UpdateThroughPrivateGitlabPolling test since GITLAB_USERNAME and/or GITLAB_PASSWORD env vars not set")
			t.Skip()
		}

		testNamespace := createNamespaceForTest()
		defer func() {
			err := deleteTestNamespace(testNamespace)
			if err != nil {
				log.WithFields(log.Fields{
					"error":     err,
					"namespace": testNamespace,
				}).Error("error while deleting test namespace")
			}
		}()

		payload, err := secrets.EncodeDockerCfgJson(&secrets.DockerCfg{
			"registry.gitlab.com": &secrets.Auth{
				Auth: secrets.EncodeBase64Secret(user, password),
			},
		})
		if err != nil {
			t.Fatalf("failed to encode docker cfg secret payload: %s", err)
		}

		secretName := "gitlab-registry-credentials"

		secret := &v1.Secret{
			TypeMeta: meta_v1.TypeMeta{},
			ObjectMeta: meta_v1.ObjectMeta{
				Name:        secretName,
				Namespace:   testNamespace,
				Labels:      map[string]string{},
				Annotations: map[string]string{},
			},
			Type: v1.SecretTypeDockerConfigJson,
			Data: map[string][]byte{
				".dockerconfigjson": []byte(payload),
			},
		}

		_, err = kcs.CoreV1().Secrets(testNamespace).Create(secret)
		if err != nil {
			t.Fatalf("failed to create secret: %s", err)
		} else {
			t.Logf("secret '%s' created in namespace '%s'", secret.Name, secret.Namespace)
		}

		dep := &apps_v1.Deployment{
			meta_v1.TypeMeta{},
			meta_v1.ObjectMeta{
				Name:      "deployment-1",
				Namespace: testNamespace,
				Labels: map[string]string{
					types.KeelPolicyLabel:  "major",
					types.KeelTriggerLabel: "poll",
				},
				Annotations: map[string]string{
					types.KeelPollScheduleAnnotation: "@every 2s",
				},
			},
			apps_v1.DeploymentSpec{
				Selector: &meta_v1.LabelSelector{
					MatchLabels: map[string]string{
						"app": "wd-1",
					},
				},
				Template: v1.PodTemplateSpec{

					ObjectMeta: meta_v1.ObjectMeta{
						Labels: map[string]string{
							"app":     "wd-1",
							"release": "1",
						},
					},
					Spec: v1.PodSpec{
						ImagePullSecrets: []v1.LocalObjectReference{
							{
								Name: secretName,
							},
						},
						Containers: []v1.Container{
							v1.Container{
								ImagePullPolicy: v1.PullAlways,
								Name:            "wd-1",
								Image:           "registry.gitlab.com/karolisr/keel:0.1.0",
							},
						},
					},
				},
			},
			apps_v1.DeploymentStatus{},
		}

		_, err = kcs.AppsV1().Deployments(testNamespace).Create(dep)
		if err != nil {
			t.Fatalf("failed to create deployment: %s", err)
		}
		ctx, cancel := context.WithTimeout(context.Background(), time.Second*10)
		defer cancel()

		err = waitFor(ctx, kcs, testNamespace, dep.ObjectMeta.Name, "registry.gitlab.com/karolisr/keel:0.2.0")
		if err != nil {
			t.Errorf("update failed: %s", err)
		}
	})

}
<commit_msg>additional image test<commit_after>package tests

import (
	"context"
	"fmt"
	"os"
	"testing"
	"time"

	"github.com/keel-hq/keel/secrets"
	"github.com/keel-hq/keel/types"
	log "github.com/sirupsen/logrus"
	apps_v1 "k8s.io/api/apps/v1"
	"k8s.io/api/core/v1"
	meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

func TestPollingSemverUpdate(t *testing.T) {

	// stop := make(chan struct{})
	ctx, cancel := context.WithCancel(context.Background())
	// defer close(ctx)
	defer cancel()

	// go startKeel(ctx)
	keel := &KeelCmd{}
	go func() {
		err := keel.Start(ctx)
		if err != nil {
			log.WithFields(log.Fields{
				"error": err,
			}).Error("failed to start Keel process")
		}
	}()

	defer func() {
		err := keel.Stop()
		if err != nil {
			log.WithFields(log.Fields{
				"error": err,
			}).Error("failed to stop Keel process")
		}
	}()

	_, kcs := getKubernetesClient()

	t.Run("UpdateThroughDockerHubPollingA", func(t *testing.T) {
		// UpdateThroughDockerHubPollingA tests a polling trigger when we have a higher version
		// but without a pre-release tag and a lower version with pre-release. The version of the deployment
		// is with pre-prerealse so we should upgrade to that one.

		testNamespace := createNamespaceForTest()
		defer deleteTestNamespace(testNamespace)

		dep := &apps_v1.Deployment{
			meta_v1.TypeMeta{},
			meta_v1.ObjectMeta{
				Name:      "deployment-1",
				Namespace: testNamespace,
				Labels: map[string]string{
					types.KeelPolicyLabel:  "major",
					types.KeelTriggerLabel: "poll",
				},
				Annotations: map[string]string{
					types.KeelPollScheduleAnnotation: "@every 2s",
				},
			},
			apps_v1.DeploymentSpec{
				Selector: &meta_v1.LabelSelector{
					MatchLabels: map[string]string{
						"app": "wd-1",
					},
				},
				Template: v1.PodTemplateSpec{
					ObjectMeta: meta_v1.ObjectMeta{
						Labels: map[string]string{
							"app":     "wd-1",
							"release": "1",
						},
					},
					Spec: v1.PodSpec{
						Containers: []v1.Container{
							v1.Container{
								Name:  "wd-1",
								Image: "keelhq/push-workflow-example:0.1.0-dev",
							},
						},
					},
				},
			},
			apps_v1.DeploymentStatus{},
		}

		_, err := kcs.AppsV1().Deployments(testNamespace).Create(dep)
		if err != nil {
			t.Fatalf("failed to create deployment: %s", err)
		}
		ctx, cancel := context.WithTimeout(context.Background(), time.Second*10)
		defer cancel()

		err = waitFor(ctx, kcs, testNamespace, dep.ObjectMeta.Name, "keelhq/push-workflow-example:0.5.0-dev")
		if err != nil {
			t.Errorf("update failed: %s", err)
		}
	})

	t.Run("UpdateThroughDockerHubPollingB", func(t *testing.T) {
		// UpdateThroughDockerHubPollingA tests a polling trigger when we have a higher version
		// but without a pre-release tag and a lower version with pre-release. The version of the deployment
		// is without pre-prerealse

		testNamespace := createNamespaceForTest()
		defer deleteTestNamespace(testNamespace)

		dep := &apps_v1.Deployment{
			meta_v1.TypeMeta{},
			meta_v1.ObjectMeta{
				Name:      "deployment-2",
				Namespace: testNamespace,
				Labels: map[string]string{
					types.KeelPolicyLabel:  "major",
					types.KeelTriggerLabel: "poll",
				},
				Annotations: map[string]string{
					types.KeelPollScheduleAnnotation: "@every 2s",
				},
			},
			apps_v1.DeploymentSpec{
				Selector: &meta_v1.LabelSelector{
					MatchLabels: map[string]string{
						"app": "wd-1",
					},
				},
				Template: v1.PodTemplateSpec{
					ObjectMeta: meta_v1.ObjectMeta{
						Labels: map[string]string{
							"app":     "wd-1",
							"release": "1",
						},
					},
					Spec: v1.PodSpec{
						Containers: []v1.Container{
							v1.Container{
								Name:  "wd-1",
								Image: "keelhq/push-workflow-example:0.1.0",
							},
						},
					},
				},
			},
			apps_v1.DeploymentStatus{},
		}

		_, err := kcs.AppsV1().Deployments(testNamespace).Create(dep)
		if err != nil {
			t.Fatalf("failed to create deployment: %s", err)
		}
		ctx, cancel := context.WithTimeout(context.Background(), time.Second*10)
		defer cancel()

		err = waitFor(ctx, kcs, testNamespace, dep.ObjectMeta.Name, "keelhq/push-workflow-example:0.10.0")
		if err != nil {
			t.Errorf("update failed: %s", err)
		}
	})

	t.Run("UpdateThroughDockerHubPollingC", func(t *testing.T) {

		testNamespace := createNamespaceForTest()
		defer deleteTestNamespace(testNamespace)

		dep := &apps_v1.Deployment{
			meta_v1.TypeMeta{},
			meta_v1.ObjectMeta{
				Name:      "deployment-2",
				Namespace: testNamespace,
				Labels: map[string]string{
					types.KeelPolicyLabel:  "major",
					types.KeelTriggerLabel: "poll",
				},
				Annotations: map[string]string{
					types.KeelPollScheduleAnnotation: "@every 2s",
				},
			},
			apps_v1.DeploymentSpec{
				Selector: &meta_v1.LabelSelector{
					MatchLabels: map[string]string{
						"app": "wd-1",
					},
				},
				Template: v1.PodTemplateSpec{
					ObjectMeta: meta_v1.ObjectMeta{
						Labels: map[string]string{
							"app":     "wd-1",
							"release": "1",
						},
					},
					Spec: v1.PodSpec{
						Containers: []v1.Container{
							v1.Container{
								Name:  "wd-1",
								Image: "keelhq/push-workflow-example:0.3.0-alpha",
							},
						},
					},
				},
			},
			apps_v1.DeploymentStatus{},
		}

		_, err := kcs.AppsV1().Deployments(testNamespace).Create(dep)
		if err != nil {
			t.Fatalf("failed to create deployment: %s", err)
		}
		ctx, cancel := context.WithTimeout(context.Background(), time.Second*10)
		defer cancel()

		err = waitFor(ctx, kcs, testNamespace, dep.ObjectMeta.Name, "keelhq/push-workflow-example:0.11.0-alpha")
		if err != nil {
			t.Errorf("update failed: %s", err)
		}
	})
}

func TestPollingPrivateRegistry(t *testing.T) {

	// stop := make(chan struct{})
	ctx, cancel := context.WithCancel(context.Background())
	// defer close(ctx)
	defer cancel()

	// go startKeel(ctx)
	keel := &KeelCmd{}
	go func() {
		err := keel.Start(ctx)
		if err != nil {
			log.WithFields(log.Fields{
				"error": err,
			}).Error("failed to start Keel process")
		}
	}()

	defer func() {
		err := keel.Stop()
		if err != nil {
			log.WithFields(log.Fields{
				"error": err,
			}).Error("failed to stop Keel process")
		}
	}()

	_, kcs := getKubernetesClient()

	t.Run("UpdateThroughPrivateQuayPollingA", func(t *testing.T) {

		user := os.Getenv("DOCKERHUB_USERNAME")
		password := os.Getenv("DOCKERHUB_PASSWORD")

		if user == "" || password == "" {
			fmt.Println("[X] Skipping UpdateThroughPrivateQuayPollingA test since DOCKERHUB_USERNAME and/or DOCKERHUB_PASSWORD env vars not set")
			t.Skip()
		}

		// UpdateThroughDockerHubPollingA tests a polling trigger when we have a higher version
		// but without a pre-release tag and a lower version with pre-release. The version of the deployment
		// is with pre-prerealse so we should upgrade to that one.

		testNamespace := createNamespaceForTest()
		defer deleteTestNamespace(testNamespace)

		payload, err := secrets.EncodeDockerCfgJson(&secrets.DockerCfg{
			"https://index.docker.io/v1/": &secrets.Auth{
				Auth: secrets.EncodeBase64Secret(user, password),
			},
		})
		if err != nil {
			t.Fatalf("failed to encode docker cfg secret payload: %s", err)
		}

		secretName := "verysecret"

		fmt.Println(string(payload))

		secret := &v1.Secret{
			TypeMeta: meta_v1.TypeMeta{},
			ObjectMeta: meta_v1.ObjectMeta{
				Name:        secretName,
				Namespace:   testNamespace,
				Labels:      map[string]string{},
				Annotations: map[string]string{},
			},
			Type: v1.SecretTypeDockerConfigJson,
			Data: map[string][]byte{
				".dockerconfigjson": []byte(payload),
			},
		}

		_, err = kcs.CoreV1().Secrets(testNamespace).Create(secret)
		if err != nil {
			t.Fatalf("failed to create secret: %s", err)
		}

		dep := &apps_v1.Deployment{
			meta_v1.TypeMeta{},
			meta_v1.ObjectMeta{
				Name:      "deployment-1",
				Namespace: testNamespace,
				Labels: map[string]string{
					types.KeelPolicyLabel:  "major",
					types.KeelTriggerLabel: "poll",
				},
				Annotations: map[string]string{
					types.KeelPollScheduleAnnotation: "@every 2s",
				},
			},
			apps_v1.DeploymentSpec{

				Selector: &meta_v1.LabelSelector{
					MatchLabels: map[string]string{
						"app": "wd-1",
					},
				},
				Template: v1.PodTemplateSpec{

					ObjectMeta: meta_v1.ObjectMeta{
						Labels: map[string]string{
							"app":     "wd-1",
							"release": "1",
						},
					},
					Spec: v1.PodSpec{
						ImagePullSecrets: []v1.LocalObjectReference{
							{
								Name: secretName,
							},
						},
						Containers: []v1.Container{
							v1.Container{
								ImagePullPolicy: v1.PullAlways,
								Name:            "wd-1",
								Image:           "karolisr/demo-webhook:0.0.1",
							},
						},
					},
				},
			},
			apps_v1.DeploymentStatus{},
		}

		_, err = kcs.AppsV1().Deployments(testNamespace).Create(dep)
		if err != nil {
			t.Fatalf("failed to create deployment: %s", err)
		}
		ctx, cancel := context.WithTimeout(context.Background(), time.Second*10)
		defer cancel()

		err = waitFor(ctx, kcs, testNamespace, dep.ObjectMeta.Name, "karolisr/demo-webhook:0.0.2")
		if err != nil {
			t.Errorf("update failed: %s", err)
		}
	})

	t.Run("UpdateThroughPrivateGitlabPolling", func(t *testing.T) {

		user := os.Getenv("GITLAB_USERNAME")
		password := os.Getenv("GITLAB_PASSWORD")

		if user == "" || password == "" {
			fmt.Println("[X] Skipping UpdateThroughPrivateGitlabPolling test since GITLAB_USERNAME and/or GITLAB_PASSWORD env vars not set")
			t.Skip()
		}

		testNamespace := createNamespaceForTest()
		defer func() {
			err := deleteTestNamespace(testNamespace)
			if err != nil {
				log.WithFields(log.Fields{
					"error":     err,
					"namespace": testNamespace,
				}).Error("error while deleting test namespace")
			}
		}()

		payload, err := secrets.EncodeDockerCfgJson(&secrets.DockerCfg{
			"registry.gitlab.com": &secrets.Auth{
				Auth: secrets.EncodeBase64Secret(user, password),
			},
		})
		if err != nil {
			t.Fatalf("failed to encode docker cfg secret payload: %s", err)
		}

		secretName := "gitlab-registry-credentials"

		secret := &v1.Secret{
			TypeMeta: meta_v1.TypeMeta{},
			ObjectMeta: meta_v1.ObjectMeta{
				Name:        secretName,
				Namespace:   testNamespace,
				Labels:      map[string]string{},
				Annotations: map[string]string{},
			},
			Type: v1.SecretTypeDockerConfigJson,
			Data: map[string][]byte{
				".dockerconfigjson": []byte(payload),
			},
		}

		_, err = kcs.CoreV1().Secrets(testNamespace).Create(secret)
		if err != nil {
			t.Fatalf("failed to create secret: %s", err)
		} else {
			t.Logf("secret '%s' created in namespace '%s'", secret.Name, secret.Namespace)
		}

		dep := &apps_v1.Deployment{
			meta_v1.TypeMeta{},
			meta_v1.ObjectMeta{
				Name:      "deployment-1",
				Namespace: testNamespace,
				Labels: map[string]string{
					types.KeelPolicyLabel:  "major",
					types.KeelTriggerLabel: "poll",
				},
				Annotations: map[string]string{
					types.KeelPollScheduleAnnotation: "@every 2s",
				},
			},
			apps_v1.DeploymentSpec{
				Selector: &meta_v1.LabelSelector{
					MatchLabels: map[string]string{
						"app": "wd-1",
					},
				},
				Template: v1.PodTemplateSpec{

					ObjectMeta: meta_v1.ObjectMeta{
						Labels: map[string]string{
							"app":     "wd-1",
							"release": "1",
						},
					},
					Spec: v1.PodSpec{
						ImagePullSecrets: []v1.LocalObjectReference{
							{
								Name: secretName,
							},
						},
						Containers: []v1.Container{
							v1.Container{
								ImagePullPolicy: v1.PullAlways,
								Name:            "wd-1",
								Image:           "registry.gitlab.com/karolisr/keel:0.1.0",
							},
						},
					},
				},
			},
			apps_v1.DeploymentStatus{},
		}

		_, err = kcs.AppsV1().Deployments(testNamespace).Create(dep)
		if err != nil {
			t.Fatalf("failed to create deployment: %s", err)
		}
		ctx, cancel := context.WithTimeout(context.Background(), time.Second*10)
		defer cancel()

		err = waitFor(ctx, kcs, testNamespace, dep.ObjectMeta.Name, "registry.gitlab.com/karolisr/keel:0.2.0")
		if err != nil {
			t.Errorf("update failed: %s", err)
		}
	})

}
<|endoftext|>