<commit_before>package pagerduty

/*
 * Copyright 2016 Albert P. Tobey <atobey@netflix.com>
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

import (
	"bytes"
	"fmt"
	"log"
	"sort"
	"strings"
	"time"

	"github.com/netflix/hal-9001/hal"
)

const OncallUsage = `!oncall <alias>

Find out who is oncall. If only one argument is provided, it must match
a known alias for a Pagerduty service. Otherwise, it is expected to be
a subcommand.

!oncall core
`

const DefaultTopicInterval = "10m"

// TODO: add the service key to the output such that someone trying to contact a team
// can page them from within Slack without having to set up a page alias or go out
// to a web page. !page should be able to take a service key so the output can include something
// like: "To page <team> use the command: !page <servicekey> <message>"
func oncall(msg hal.Evt) {
	parts := msg.BodyAsArgv()

	if len(parts) == 1 {
		msg.Reply(OncallUsage)
		return
	}

	// make sure the pagerduty token is setup in hal.Secrets
	token, err := getSecrets()
	if err != nil || token == "" {
		msg.Replyf("pagerduty: %s is not set up in hal.Secrets. Cannot continue.", PagerdutyTokenKey)
		return
	}

	if parts[1] == "cache-now" {
		msg.Reply("Updating Pagerduty policy cache now.")
		cacheNow(token, msg.RoomId)
		msg.Reply("Pagerduty policy cache update complete.")
		return
	} else if parts[1] == "cache-status" {
		age := int(hal.Cache().Age(CacheKey).Seconds())
		next := time.Time{}
		status := "broken"
		pf := hal.GetPeriodicFunc(cacheFuncName(msg.RoomId))
		if pf != nil {
			next = pf.Last().Add(pf.Interval)
			status = pf.Status()
		}
		msg.Replyf("The cache is %d seconds old. Auto-update is %s and its next update is at %s.",
			age, status, next.Format(time.UnixDate))
		return
	} else if len(parts) > 2 {
		// flatten split words back into position 1, parts isn't used after the ToLower
		parts[1] = strings.Join(parts[1:], " ")
	}

	// TODO: look at the aliases set up for !page and try for an exact match
	// before doing fuzzy search -- move fuzzy search to a "search" subcommand
	// so it's clear that it is not precise
	want := strings.ToLower(parts[1])

	// see if there's an exact match on an alias, e.g. "!oncall core" -> alias.core
	/*
		aliasPref := msg.AsPref().SetUser("").FindKey(aliasKey(want)).One()
		if aliasPref.Success {
			svc, err := GetServiceByKey(token, aliasPref.Value)
			if err == nil {
			}
			// all through to search ...
		}
	*/

	// search over all policies looking for matching policy name, escalation
	// rule name, or service name
	matches := make([]Oncall, 0)
	oncalls := getOncallCache(token, false)
	var exactMatchFound bool

	for _, oncall := range oncalls {
		schedSummary := strings.ToLower(oncall.Schedule.Summary)
		if schedSummary == want {
			matches = append(matches, oncall)
			exactMatchFound = true
			continue
		} else if !exactMatchFound && strings.Contains(schedSummary, want) {
			matches = append(matches, oncall)
			continue
		}

		epSummary := strings.ToLower(oncall.EscalationPolicy.Summary)
		if epSummary == want {
			matches = append(matches, oncall)
			exactMatchFound = true
			continue
		} else if !exactMatchFound && strings.Contains(epSummary, want) {
			matches = append(matches, oncall)
			continue
		}
	}

	// check team names if there were no matches
	// TODO: cache some of these results and always check team names
	if len(matches) == 0 {
		log.Printf("GETTING TEAMS!")
		teams, err := GetTeams(token)
		if err != nil {
			log.Printf("REST call to Pagerduty /teams failed: %s", err)
		} else {
			for _, team := range teams {
				ltname := strings.ToLower(team.Name)
				ltdesc := strings.ToLower(team.Description)

				if strings.Contains(ltname, want) || strings.Contains(ltdesc, want) {
					matches = getTeamOncalls(token, team)
				}
			}
		}
	}

	reply := formatOncallReply(want, exactMatchFound, matches)
	msg.Reply(reply)
}

// getTeamOncalls fetches escalation policies for the team then the oncalls for those
// policies and returns a list.
func getTeamOncalls(token string, team Team) []Oncall {
	out := make([]Oncall, 0)
	log.Printf("Returning empty list but should have fetched oncalls for team %+v", team)

	params := map[string]string{"team_ids[]": team.Id}
	policies, err := GetEscalationPolicies(token, params)
	if err != nil {
		log.Printf("Error while fetching escalation policies for team id %q: %s", team.Id, err)
		return out
	}

	policy_ids := make([]string, 0)
	for _, policy := range policies {
		policy_ids = append(policy_ids, policy.Id)
	}

	params = map[string]string{"escalation_policy_ids[]": strings.Join(policy_ids, ",")}
	oncalls, err := GetOncalls(token, params)
	if err != nil {
		log.Printf("Error while fetching oncalls for team id %q's policies: %s", team.Id, err)
	} else {
		return oncalls
	}

	return out
}

func getOncallCache(token string, forceUpdate bool) []Oncall {
	oncalls := []Oncall{}

	// see if there's a copy cached
	if hal.Cache().Exists(CacheKey) {
		ttl, err := hal.Cache().Get(CacheKey, &oncalls)
		if err != nil {
			log.Printf("Error retreiving oncalls from the Hal TTL cache: %s", err)
			oncalls = []Oncall{}
		} else if ttl == 0 || forceUpdate {
			oncalls = []Oncall{}
		}
	}

	// the cache exists and is still valid, return it now
	if len(oncalls) > 0 {
		return oncalls
	}

	// get all of the defined policies
	var err error
	oncalls, err = GetOncalls(token, nil)
	if err != nil {
		log.Printf("Returning empty list. REST call to Pagerduty failed: %s", err)
		return []Oncall{}
	}

	// always update the cache regardless of ttl
	hal.Cache().Set(CacheKey, &oncalls, cacheExpire)

	return oncalls
}

func oncallInit(i *hal.Instance) {
	cacheFreq := hal.GetPref("", "", i.RoomId, "pagerduty", "cache-update-frequency", DefaultCacheInterval)
	cd, err := time.ParseDuration(cacheFreq.Value)
	if err != nil {
		log.Panicf("BUG: could not parse cache update frequency preference: %q", cacheFreq.Value)
	}

	topicFreq := hal.GetPref("", "", i.RoomId, "pagerduty", "topic-update-frequency", DefaultTopicInterval)
	td, err := time.ParseDuration(topicFreq.Value)
	if err != nil {
		log.Panicf("BUG: could not parse topic update frequency preference: %q", topicFreq.Value)
	}

	token, err := getSecrets()
	if err != nil || token == "" {
		return // getSecrets will log the error
	}

	go func() {
		pf := hal.PeriodicFunc{
			Name:     cacheFuncName(i.RoomId),
			Interval: cd,
			Function: func() { cacheNow(token, i.RoomId) },
		}

		pf.Register()
		pf.Start()
	}()

	go func() {
		pf := hal.PeriodicFunc{
			Name:     topicFuncName(i.RoomId),
			Interval: td,
			Function: func() { topicUpdater(token, i.RoomId, i.Broker.Name()) },
		}

		pf.Register()
		pf.Start()
	}()

	// TODO: add a command to stop, etc.
}

func cacheNow(token, roomId string) {
	getOncallCache(token, true)
}

// topicUpdater runs periodically to update the topic in the room
// it's configured in.
// To fully enable it, you need the oncall schedule id from the pagerduty API.
// !prefs set --room * --broker slack --plugin pagerduty --key topic-updater-schedule-id --value <schedule id>
// !prefs set --room * --broker slack --plugin pagerduty --key topic-prefix --value <text>
// !prefs set --room * --broker slack --plugin pagerduty --key topic-suffix --value <text>
// TODO: see if there's a way to also resolve integration keys instead of using the schedule id
func topicUpdater(token, roomId, brokerName string) {
	log.Printf("ENTER topicUpdater(token, %q, %q)", roomId, brokerName)

	pref := hal.GetPref("", brokerName, roomId, "pagerduty", "topic-updater-schedule-id", "-")
	// probably not configured, nothing to see here...
	if !pref.Success || pref.Value == "-" {
		log.Printf("The pref ''/%q/%q/pagerduty/topic-updater-schedule-id does not seem to be set. Returning without taking action.",
			brokerName, roomId)
		return
	}

	params := map[string]string{
		"include[]":      "users",
		"schedule_ids[]": pref.Value,
	}

	oncalls, err := GetOncalls(token, params)
	if err != nil {
		log.Printf("Failed to fetch oncalls for schedule id %q: %s", pref.Value, err)
		return
	}

	log.Printf("Got %d users for schedule id %q", len(oncalls), pref.Value)

	// there may be more than one entry but if they're both on the same
	// schedule it should be the same primary oncall so ignore all but the first
	if len(oncalls) == 0 {
		log.Printf("no oncall results for id %q", pref.Value)
		return
	}

	// TODO: yet another place some kind of templating support would be handy
	prefix := hal.GetPref("", brokerName, roomId, "pagerduty", "topic-prefix", "")
	suffix := hal.GetPref("", brokerName, roomId, "pagerduty", "topic-suffix", "")

	// e.g. prefix = "", summary = "Al Tobey", suffix = " [team-dl@company.com] !pageus"
	topic := prefix.Value + oncalls[0].User.Summary + suffix.Value

	broker := hal.Router().GetBroker(brokerName)

	oldTopic, err := broker.GetTopic(roomId)
	if err != nil {
		log.Printf("Could not fetch current topic for room %q: %s", roomId, err)
		return
	}

	// only do the update if the topic has changed
	if topic != oldTopic {
		broker.SetTopic(roomId, topic)
	}
}

func cacheFuncName(roomId string) string {
	return fmt.Sprintf("pagerduty-cache-updater-%s", roomId)
}

func topicFuncName(roomId string) string {
	return fmt.Sprintf("pagerduty-topic-updater-%s", roomId)
}

// OncallsByLevel provides sorting by oncall level for []Oncall.
type OncallsByLevel []Oncall

func (a OncallsByLevel) Len() int      { return len(a) }
func (a OncallsByLevel) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
func (a OncallsByLevel) Less(i, j int) bool {
	// sort "always on call" users to the end of the list
	if a[j].Schedule.Summary == "" {
		return true
	}

	return a[i].EscalationLevel < a[j].EscalationLevel
}

func formatOncallReply(wanted string, exactMatchFound bool, oncalls []Oncall) string {
	buf := bytes.NewBuffer([]byte{})

	if exactMatchFound {
		fmt.Fprintf(buf, "exact match found for %q\n", oncalls[0].EscalationPolicy.Summary)
	} else {
		fmt.Fprintf(buf, "%d records matched for query: %q\n", len(oncalls), wanted)
	}

	sort.Sort(OncallsByLevel(oncalls))

	for _, oncall := range oncalls {
		indent := strings.Repeat("    ", oncall.EscalationLevel)
		sched := oncall.Schedule.Summary
		if sched == "" {
			sched = "always on call"
		}

		if exactMatchFound {
			fmt.Fprintf(buf, "%s%s - %s\n", indent, oncall.User.Summary, sched)
		} else {
			fmt.Fprintf(buf, "%s%s - %s - %s\n", indent,
				oncall.EscalationPolicy.Summary, oncall.User.Summary, sched)
		}
	}

	return buf.String()
}
<commit_msg>add a first pass integration of the hal directory<commit_after>package pagerduty

/*
 * Copyright 2016 Albert P. Tobey <atobey@netflix.com>
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

import (
	"bytes"
	"fmt"
	"log"
	"sort"
	"strings"
	"time"

	"github.com/netflix/hal-9001/hal"
)

const OncallUsage = `!oncall <alias>

Find out who is oncall. If only one argument is provided, it must match
a known alias for a Pagerduty service. Otherwise, it is expected to be
a subcommand.

!oncall core
`

const DefaultTopicInterval = "10m"

// TODO: add the service key to the output such that someone trying to contact a team
// can page them from within Slack without having to set up a page alias or go out
// to a web page. !page should be able to take a service key so the output can include something
// like: "To page <team> use the command: !page <servicekey> <message>"
func oncall(msg hal.Evt) {
	parts := msg.BodyAsArgv()

	if len(parts) == 1 {
		msg.Reply(OncallUsage)
		return
	}

	// make sure the pagerduty token is setup in hal.Secrets
	token, err := getSecrets()
	if err != nil || token == "" {
		msg.Replyf("pagerduty: %s is not set up in hal.Secrets. Cannot continue.", PagerdutyTokenKey)
		return
	}

	if parts[1] == "cache-now" {
		msg.Reply("Updating Pagerduty policy cache now.")
		cacheNow(token, msg.RoomId)
		msg.Reply("Pagerduty policy cache update complete.")
		return
	} else if parts[1] == "cache-status" {
		age := int(hal.Cache().Age(CacheKey).Seconds())
		next := time.Time{}
		status := "broken"
		pf := hal.GetPeriodicFunc(cacheFuncName(msg.RoomId))
		if pf != nil {
			next = pf.Last().Add(pf.Interval)
			status = pf.Status()
		}
		msg.Replyf("The cache is %d seconds old. Auto-update is %s and its next update is at %s.",
			age, status, next.Format(time.UnixDate))
		return
	} else if len(parts) > 2 {
		// flatten split words back into position 1, parts isn't used after the ToLower
		parts[1] = strings.Join(parts[1:], " ")
	}

	// TODO: look at the aliases set up for !page and try for an exact match
	// before doing fuzzy search -- move fuzzy search to a "search" subcommand
	// so it's clear that it is not precise
	want := strings.ToLower(parts[1])

	// see if there's an exact match on an alias, e.g. "!oncall core" -> alias.core
	/*
		aliasPref := msg.AsPref().SetUser("").FindKey(aliasKey(want)).One()
		if aliasPref.Success {
			svc, err := GetServiceByKey(token, aliasPref.Value)
			if err == nil {
			}
			// all through to search ...
		}
	*/

	// search over all policies looking for matching policy name, escalation
	// rule name, or service name
	matches := make([]Oncall, 0)
	oncalls := getOncallCache(token, false)
	var exactMatchFound bool

	for _, oncall := range oncalls {
		schedSummary := strings.ToLower(oncall.Schedule.Summary)
		if schedSummary == want {
			matches = append(matches, oncall)
			exactMatchFound = true
			continue
		} else if !exactMatchFound && strings.Contains(schedSummary, want) {
			matches = append(matches, oncall)
			continue
		}

		epSummary := strings.ToLower(oncall.EscalationPolicy.Summary)
		if epSummary == want {
			matches = append(matches, oncall)
			exactMatchFound = true
			continue
		} else if !exactMatchFound && strings.Contains(epSummary, want) {
			matches = append(matches, oncall)
			continue
		}
	}

	// check team names if there were no matches
	// TODO: cache some of these results and always check team names
	if len(matches) == 0 {
		log.Printf("GETTING TEAMS!")
		teams, err := GetTeams(token)
		if err != nil {
			log.Printf("REST call to Pagerduty /teams failed: %s", err)
		} else {
			for _, team := range teams {
				ltname := strings.ToLower(team.Name)
				ltdesc := strings.ToLower(team.Description)

				if strings.Contains(ltname, want) || strings.Contains(ltdesc, want) {
					matches = getTeamOncalls(token, team)
				}
			}
		}
	}

	reply := formatOncallReply(want, exactMatchFound, matches)
	msg.Reply(reply)
}

// getTeamOncalls fetches escalation policies for the team then the oncalls for those
// policies and returns a list.
func getTeamOncalls(token string, team Team) []Oncall {
	out := make([]Oncall, 0)
	log.Printf("Returning empty list but should have fetched oncalls for team %+v", team)

	params := map[string]string{"team_ids[]": team.Id}
	policies, err := GetEscalationPolicies(token, params)
	if err != nil {
		log.Printf("Error while fetching escalation policies for team id %q: %s", team.Id, err)
		return out
	}

	policy_ids := make([]string, 0)
	for _, policy := range policies {
		policy_ids = append(policy_ids, policy.Id)
	}

	params = map[string]string{"escalation_policy_ids[]": strings.Join(policy_ids, ",")}
	oncalls, err := GetOncalls(token, params)
	if err != nil {
		log.Printf("Error while fetching oncalls for team id %q's policies: %s", team.Id, err)
	} else {
		// insert oncalls into the hal directory
		for _, oncall := range oncalls {
			attrs := map[string]string{
				"email":          oncall.User.Email,
				"name":           oncall.User.Name,
				"pd-user-id":     oncall.User.Id,
				"pd-schedule-id": oncall.Schedule.Id,
				"pd-policy-id":   oncall.EscalationPolicy.Id,
			}

			// plug in the contact methods
			// per PD docs, types can be: email_contact_method, phone_contact_method, push_notification_contact_method, or sms_contact_method
			for _, cm := range oncall.User.ContactMethods {
				attrs["contact-method-id"] = cm.Id
				attrs[cm.Type] = cm.Address
			}

			edges := []string{"name", "email", "phone_contact_method", "sms_contact_method"}
			hal.Directory().Put(oncall.User.Id, "pd-oncall", attrs, edges)
		}

		return oncalls
	}

	return out
}

func getOncallCache(token string, forceUpdate bool) []Oncall {
	oncalls := []Oncall{}

	// see if there's a copy cached
	if hal.Cache().Exists(CacheKey) {
		ttl, err := hal.Cache().Get(CacheKey, &oncalls)
		if err != nil {
			log.Printf("Error retreiving oncalls from the Hal TTL cache: %s", err)
			oncalls = []Oncall{}
		} else if ttl == 0 || forceUpdate {
			oncalls = []Oncall{}
		}
	}

	// the cache exists and is still valid, return it now
	if len(oncalls) > 0 {
		return oncalls
	}

	// get all of the defined policies
	var err error
	oncalls, err = GetOncalls(token, nil)
	if err != nil {
		log.Printf("Returning empty list. REST call to Pagerduty failed: %s", err)
		return []Oncall{}
	}

	// always update the cache regardless of ttl
	hal.Cache().Set(CacheKey, &oncalls, cacheExpire)

	return oncalls
}

func oncallInit(i *hal.Instance) {
	cacheFreq := hal.GetPref("", "", i.RoomId, "pagerduty", "cache-update-frequency", DefaultCacheInterval)
	cd, err := time.ParseDuration(cacheFreq.Value)
	if err != nil {
		log.Panicf("BUG: could not parse cache update frequency preference: %q", cacheFreq.Value)
	}

	topicFreq := hal.GetPref("", "", i.RoomId, "pagerduty", "topic-update-frequency", DefaultTopicInterval)
	td, err := time.ParseDuration(topicFreq.Value)
	if err != nil {
		log.Panicf("BUG: could not parse topic update frequency preference: %q", topicFreq.Value)
	}

	token, err := getSecrets()
	if err != nil || token == "" {
		return // getSecrets will log the error
	}

	go func() {
		pf := hal.PeriodicFunc{
			Name:     cacheFuncName(i.RoomId),
			Interval: cd,
			Function: func() { cacheNow(token, i.RoomId) },
		}

		pf.Register()
		pf.Start()
	}()

	go func() {
		pf := hal.PeriodicFunc{
			Name:     topicFuncName(i.RoomId),
			Interval: td,
			Function: func() { topicUpdater(token, i.RoomId, i.Broker.Name()) },
		}

		pf.Register()
		pf.Start()
	}()

	// TODO: add a command to stop, etc.
}

func cacheNow(token, roomId string) {
	getOncallCache(token, true)
}

// topicUpdater runs periodically to update the topic in the room
// it's configured in.
// To fully enable it, you need the oncall schedule id from the pagerduty API.
// !prefs set --room * --broker slack --plugin pagerduty --key topic-updater-schedule-id --value <schedule id>
// !prefs set --room * --broker slack --plugin pagerduty --key topic-prefix --value <text>
// !prefs set --room * --broker slack --plugin pagerduty --key topic-suffix --value <text>
// TODO: see if there's a way to also resolve integration keys instead of using the schedule id
func topicUpdater(token, roomId, brokerName string) {
	log.Printf("ENTER topicUpdater(token, %q, %q)", roomId, brokerName)

	pref := hal.GetPref("", brokerName, roomId, "pagerduty", "topic-updater-schedule-id", "-")
	// probably not configured, nothing to see here...
	if !pref.Success || pref.Value == "-" {
		log.Printf("The pref ''/%q/%q/pagerduty/topic-updater-schedule-id does not seem to be set. Returning without taking action.",
			brokerName, roomId)
		return
	}

	params := map[string]string{
		"include[]":      "users",
		"schedule_ids[]": pref.Value,
	}

	oncalls, err := GetOncalls(token, params)
	if err != nil {
		log.Printf("Failed to fetch oncalls for schedule id %q: %s", pref.Value, err)
		return
	}

	log.Printf("Got %d users for schedule id %q", len(oncalls), pref.Value)

	// there may be more than one entry but if they're both on the same
	// schedule it should be the same primary oncall so ignore all but the first
	if len(oncalls) == 0 {
		log.Printf("no oncall results for id %q", pref.Value)
		return
	}

	// TODO: yet another place some kind of templating support would be handy
	prefix := hal.GetPref("", brokerName, roomId, "pagerduty", "topic-prefix", "")
	suffix := hal.GetPref("", brokerName, roomId, "pagerduty", "topic-suffix", "")

	// e.g. prefix = "", summary = "Al Tobey", suffix = " [team-dl@company.com] !pageus"
	topic := prefix.Value + oncalls[0].User.Summary + suffix.Value

	broker := hal.Router().GetBroker(brokerName)

	oldTopic, err := broker.GetTopic(roomId)
	if err != nil {
		log.Printf("Could not fetch current topic for room %q: %s", roomId, err)
		return
	}

	// only do the update if the topic has changed
	if topic != oldTopic {
		broker.SetTopic(roomId, topic)
	}
}

func cacheFuncName(roomId string) string {
	return fmt.Sprintf("pagerduty-cache-updater-%s", roomId)
}

func topicFuncName(roomId string) string {
	return fmt.Sprintf("pagerduty-topic-updater-%s", roomId)
}

// OncallsByLevel provides sorting by oncall level for []Oncall.
type OncallsByLevel []Oncall

func (a OncallsByLevel) Len() int      { return len(a) }
func (a OncallsByLevel) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
func (a OncallsByLevel) Less(i, j int) bool {
	// sort "always on call" users to the end of the list
	if a[j].Schedule.Summary == "" {
		return true
	}

	return a[i].EscalationLevel < a[j].EscalationLevel
}

func formatOncallReply(wanted string, exactMatchFound bool, oncalls []Oncall) string {
	buf := bytes.NewBuffer([]byte{})

	if exactMatchFound {
		fmt.Fprintf(buf, "exact match found for %q\n", oncalls[0].EscalationPolicy.Summary)
	} else {
		fmt.Fprintf(buf, "%d records matched for query: %q\n", len(oncalls), wanted)
	}

	sort.Sort(OncallsByLevel(oncalls))

	for _, oncall := range oncalls {
		indent := strings.Repeat("    ", oncall.EscalationLevel)
		sched := oncall.Schedule.Summary
		if sched == "" {
			sched = "always on call"
		}

		if exactMatchFound {
			fmt.Fprintf(buf, "%s%s - %s\n", indent, oncall.User.Summary, sched)
		} else {
			fmt.Fprintf(buf, "%s%s - %s - %s\n", indent,
				oncall.EscalationPolicy.Summary, oncall.User.Summary, sched)
		}
	}

	return buf.String()
}
<|endoftext|>


<commit_before>package spotify

import (
	"context"
	v1 "github.com/jirwin/quadlek/pb/quadlek/plugins/spotify/v1"

	"go.uber.org/zap"

	"regexp"

	"github.com/jirwin/quadlek/quadlek"
	"github.com/zmb3/spotify"
	"google.golang.org/protobuf/proto"
)

// FIXME: These shouldn't have to be hardcoded
const (
	sharedPlaylist     = "2oeMx9gAl3fx2qThl11jt1"
	sharedPlaylistUser = "U0RL53ETW"
)

var spotifyTrackRegex = regexp.MustCompile(`spotify:track:(\w+)\b`)

func saveSongsHook(ctx context.Context, hookChannel <-chan *quadlek.HookMsg) {
	for {
		select {
		case hookMsg := <-hookChannel:
			matches := spotifyTrackRegex.FindAllStringSubmatch(hookMsg.Msg.Text, -1)
			if matches == nil {
				continue
			}

			tracks := []spotify.ID{}
			for _, match := range matches {
				tracks = append(tracks, spotify.ID(match[1]))
			}

			err := hookMsg.Store.Get("authtoken-"+sharedPlaylistUser, func(val []byte) error {
				authToken := &v1.AuthToken{}
				err := proto.Unmarshal(val, authToken)
				if err != nil {
					return err
				}

				if authToken.Token == nil {
					zap.L().Error("There wasn't a token for the shared playlist user")
					return nil
				}
				client, needsReauth := getSpotifyClient(authToken)
				if needsReauth {
					zap.L().Info("detected a song, but need to reauth before it can be added.")
				}

				snapshotId, err := client.AddTracksToPlaylist(sharedPlaylist, tracks...)
				if err != nil {
					zap.L().Error("error adding tracks to shared playlist", zap.Error(err))
					return err
				}
				zap.L().Info("Spotify snapshot id", zap.String("snapshotId", snapshotId))

				return nil
			})
			if err != nil {
				continue
			}

		case <-ctx.Done():
			zap.L().Info("Exiting save song hook")
			return
		}
	}
}
<commit_msg>Allow shared playlists to be configured via the env (#33)<commit_after>package spotify

import (
	"context"
	"os"
	"regexp"

	v1 "github.com/jirwin/quadlek/pb/quadlek/plugins/spotify/v1"

	"go.uber.org/zap"

	"github.com/jirwin/quadlek/quadlek"
	"github.com/zmb3/spotify"
	"google.golang.org/protobuf/proto"
)

const (
	sharedPlaylist     = "2oeMx9gAl3fx2qThl11jt1"
	sharedPlaylistUser = "U0RL53ETW"
)

var spotifyTrackRegex = regexp.MustCompile(`spotify:track:(\w+)\b`)

func getSharedPlaylist() string {
	playlist := os.Getenv("SPOTIFY_SHARED_PLAYLIST")
	if playlist != "" {
		return playlist
	}

	return sharedPlaylist
}

func getSharedPlaylistUser() string {
	playlistUser := os.Getenv("SPOTIFY_SHARED_PLAYLIST_USER")
	if playlistUser != "" {
		return playlistUser
	}

	return sharedPlaylistUser
}

func saveSongsHook(ctx context.Context, hookChannel <-chan *quadlek.HookMsg) {
	for {
		select {
		case hookMsg := <-hookChannel:
			matches := spotifyTrackRegex.FindAllStringSubmatch(hookMsg.Msg.Text, -1)
			if matches == nil {
				continue
			}

			tracks := []spotify.ID{}
			for _, match := range matches {
				tracks = append(tracks, spotify.ID(match[1]))
			}

			err := hookMsg.Store.Get("authtoken-"+getSharedPlaylistUser(), func(val []byte) error {
				authToken := &v1.AuthToken{}
				err := proto.Unmarshal(val, authToken)
				if err != nil {
					return err
				}

				if authToken.Token == nil {
					zap.L().Error("There wasn't a token for the shared playlist user")
					return nil
				}
				client, needsReauth := getSpotifyClient(authToken)
				if needsReauth {
					zap.L().Info("detected a song, but need to reauth before it can be added.")
				}

				snapshotId, err := client.AddTracksToPlaylist(spotify.ID(getSharedPlaylist()), tracks...)
				if err != nil {
					zap.L().Error("error adding tracks to shared playlist", zap.Error(err))
					return err
				}
				zap.L().Info("Spotify snapshot id", zap.String("snapshotId", snapshotId))

				return nil
			})
			if err != nil {
				continue
			}

		case <-ctx.Done():
			zap.L().Info("Exiting save song hook")
			return
		}
	}
}
<|endoftext|>


<commit_before>package experiments

import (
	"time"
	"github.com/yaricom/goNEAT/neat/genetics"
)

// The structure to represent one epoch execution results
type Epoch struct {
	// The generation ID for this epoch
	Id          int
	// The time when epoch was evaluated
	Executed    time.Time
	// The best organism of best species
	Best        *genetics.Organism
	// The flag to indicate whether experiment was solved in this epoch
	Solved      bool
	// The list of organisms fitness values per species in population
	Fitness     Floats
	// The age of organisms per species in population
	Age         Floats
	// The list of organisms complexities per species in population
	Compexity   Floats
	// The number of species in population
	Diversity   int

	// The number of evaluations done before winner found
	WinnerEvals int
	// The number of nodes in winner genome or zero if not solved
	WinnerNodes int
	// The numbers of genes (links) in winner genome or zero if not solved
	WinnerGenes int
}

// Collects statistics about given population
func (epoch *Epoch) FillPopulationStatistics(pop *genetics.Population) {
	max_fitness := 0.0
	epoch.Diversity = len(pop.Species)
	epoch.Age = make(Floats, epoch.Diversity)
	epoch.Compexity = make(Floats, epoch.Diversity)
	epoch.Fitness = make(Floats, epoch.Diversity)
	for i, curr_species := range pop.Species {
		epoch.Age[i] = float64(curr_species.Age)
		epoch.Compexity[i] = float64(curr_species.Organisms[0].Phenotype.Complexity())
		epoch.Fitness[i] = curr_species.Organisms[0].Fitness

		// find best organism in epoch if not solved
		if !epoch.Solved && curr_species.Organisms[0].Fitness > max_fitness {
			max_fitness = curr_species.Organisms[0].Fitness
			epoch.Best = curr_species.Organisms[0]
		}
	}
}

// Epochs is a sortable collection of epochs by execution time and Id
type Epochs []Epoch

func (is Epochs) Len() int {
	return len(is)
}
func (is Epochs) Swap(i, j int) {
	is[i], is[j] = is[j], is[i]
}
func (is Epochs) Less(i, j int) bool {
	if is[i].Executed.Equal(is[j].Executed) {
		return is[i].Id < is[j].Id // less is from earlier epochs
	}
	return is[i].Executed.Before(is[j].Executed) // less is from earlier time
}
<commit_msg>Fixed max fitness evaluation<commit_after>package experiments

import (
	"time"
	"github.com/yaricom/goNEAT/neat/genetics"
	"math"
)

// The structure to represent one epoch execution results
type Epoch struct {
	// The generation ID for this epoch
	Id          int
	// The time when epoch was evaluated
	Executed    time.Time
	// The best organism of best species
	Best        *genetics.Organism
	// The flag to indicate whether experiment was solved in this epoch
	Solved      bool
	// The list of organisms fitness values per species in population
	Fitness     Floats
	// The age of organisms per species in population
	Age         Floats
	// The list of organisms complexities per species in population
	Compexity   Floats
	// The number of species in population
	Diversity   int

	// The number of evaluations done before winner found
	WinnerEvals int
	// The number of nodes in winner genome or zero if not solved
	WinnerNodes int
	// The numbers of genes (links) in winner genome or zero if not solved
	WinnerGenes int
}

// Collects statistics about given population
func (epoch *Epoch) FillPopulationStatistics(pop *genetics.Population) {
	max_fitness := float64(math.MinInt64)
	epoch.Diversity = len(pop.Species)
	epoch.Age = make(Floats, epoch.Diversity)
	epoch.Compexity = make(Floats, epoch.Diversity)
	epoch.Fitness = make(Floats, epoch.Diversity)
	for i, curr_species := range pop.Species {
		epoch.Age[i] = float64(curr_species.Age)
		epoch.Compexity[i] = float64(curr_species.Organisms[0].Phenotype.Complexity())
		epoch.Fitness[i] = curr_species.Organisms[0].Fitness

		// find best organism in epoch if not solved
		if !epoch.Solved && curr_species.Organisms[0].Fitness > max_fitness {
			max_fitness = curr_species.Organisms[0].Fitness
			epoch.Best = curr_species.Organisms[0]
		}
	}
}

// Epochs is a sortable collection of epochs by execution time and Id
type Epochs []Epoch

func (is Epochs) Len() int {
	return len(is)
}
func (is Epochs) Swap(i, j int) {
	is[i], is[j] = is[j], is[i]
}
func (is Epochs) Less(i, j int) bool {
	if is[i].Executed.Equal(is[j].Executed) {
		return is[i].Id < is[j].Id // less is from earlier epochs
	}
	return is[i].Executed.Before(is[j].Executed) // less is from earlier time
}
<|endoftext|>


<commit_before>package exec_test

import (
	"context"
	"errors"
	"time"

	"github.com/concourse/concourse/atc"
	"github.com/concourse/concourse/atc/creds/credsfakes"
	"github.com/concourse/concourse/atc/db"
	"github.com/concourse/concourse/atc/exec"
	"github.com/concourse/concourse/atc/exec/build"
	"github.com/concourse/concourse/atc/exec/execfakes"
	"github.com/concourse/concourse/atc/resource/resourcefakes"
	"github.com/concourse/concourse/atc/worker"
	"github.com/concourse/concourse/atc/worker/workerfakes"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("CheckStep", func() {
	var (
		ctx    context.Context
		cancel func()

		fakeWorker          *workerfakes.FakeWorker
		fakePool            *workerfakes.FakePool
		fakeStrategy        *workerfakes.FakeContainerPlacementStrategy
		fakeResourceFactory *resourcefakes.FakeResourceFactory
		fakeSecretManager   *credsfakes.FakeSecrets
		fakeDelegate        *execfakes.FakeCheckDelegate
		checkPlan           *atc.CheckPlan

		interpolatedResourceTypes atc.VersionedResourceTypes

		containerMetadata = db.ContainerMetadata{
			Type:     db.ContainerTypeCheck,
			StepName: "some-step",
		}

		stepMetadata = exec.StepMetadata{
			ResourceConfigID:   1,
			BaseResourceTypeID: 1,
			TeamID:             123,
		}

		owner = db.NewResourceConfigCheckSessionContainerOwner(stepMetadata.ResourceConfigID, stepMetadata.BaseResourceTypeID, db.ContainerOwnerExpiries{
			Min: 5 * time.Minute,
			Max: 1 * time.Hour,
		})

		repo  *artifact.Repository
		state *execfakes.FakeRunState

		checkStep *exec.CheckStep
		stepErr   error

		planID atc.PlanID
	)

	BeforeEach(func() {
		ctx, cancel = context.WithCancel(context.Background())

		planID = atc.PlanID("some-plan-id")

		fakeStrategy = new(workerfakes.FakeContainerPlacementStrategy)
		fakePool = new(workerfakes.FakePool)
		fakeWorker = new(workerfakes.FakeWorker)
		fakeResourceFactory = new(resourcefakes.FakeResourceFactory)

		fakeSecretManager = new(credsfakes.FakeSecrets)
		fakeSecretManager.GetReturnsOnCall(0, "super-secret-source", nil, true, nil)
		fakeSecretManager.GetReturnsOnCall(1, "source", nil, true, nil)

		fakeDelegate = new(execfakes.FakeCheckDelegate)

		repo = artifact.NewRepository()
		state = new(execfakes.FakeRunState)
		state.ArtifactsReturns(repo)

		interpolatedResourceTypes = atc.VersionedResourceTypes{
			{
				ResourceType: atc.ResourceType{
					Name:   "custom-resource",
					Type:   "custom-type",
					Source: atc.Source{"some-custom": "super-secret-source"},
				},
				Version: atc.Version{"some-custom": "version"},
			},
		}

		checkPlan = &atc.CheckPlan{
			Name:                   "some-name",
			Type:                   "some-resource-type",
			Source:                 atc.Source{"some": "super-secret-source"},
			Tags:                   []string{"some", "tags"},
			Timeout:                "10s",
			FromVersion:            atc.Version{"some-custom": "version"},
			VersionedResourceTypes: interpolatedResourceTypes,
		}
	})

	AfterEach(func() {
		cancel()
	})

	JustBeforeEach(func() {
		plan := atc.Plan{
			ID:    atc.PlanID(planID),
			Check: checkPlan,
		}

		checkStep = exec.NewCheckStep(
			plan.ID,
			*plan.Check,
			stepMetadata,
			containerMetadata,
			fakeResourceFactory,
			fakeStrategy,
			fakePool,
			fakeDelegate,
		)

		stepErr = checkStep.Run(ctx, state)
	})

	Context("when find or choosing worker succeeds", func() {
		var (
			fakeResource *resourcefakes.FakeResource
			versions     []atc.Version
		)

		BeforeEach(func() {
			fakeWorker.NameReturns("some-worker")
			fakePool.FindOrChooseWorkerForContainerReturns(fakeWorker, nil)

			fakeResource = new(resourcefakes.FakeResource)
			fakeResourceFactory.NewResourceForContainerReturns(fakeResource)
		})

		It("finds or chooses a worker", func() {
			Expect(fakePool.FindOrChooseWorkerForContainerCallCount()).To(Equal(1))
			_, _, actualOwner, actualContainerSpec, actualWorkerSpec, strategy := fakePool.FindOrChooseWorkerForContainerArgsForCall(0)

			Expect(actualOwner).To(Equal(owner))

			Expect(actualContainerSpec.ImageSpec).To(Equal(worker.ImageSpec{
				ResourceType: "some-resource-type",
			}))
			Expect(actualContainerSpec.Tags).To(Equal([]string{"some", "tags"}))
			Expect(actualContainerSpec.TeamID).To(Equal(123))
			Expect(actualContainerSpec.Env).To(Equal(stepMetadata.Env()))

			Expect(actualWorkerSpec).To(Equal(worker.WorkerSpec{
				ResourceType:  "some-resource-type",
				Tags:          atc.Tags{"some", "tags"},
				TeamID:        stepMetadata.TeamID,
				ResourceTypes: interpolatedResourceTypes,
			}))

			Expect(strategy).To(Equal(fakeStrategy))
		})

		It("creates a container with the correct type and owner", func() {
			_, _, delegate, actualOwner, actualContainerMetadata, actualContainerSpec, actualResourceTypes := fakeWorker.FindOrCreateContainerArgsForCall(0)

			Expect(actualOwner).To(Equal(owner))

			Expect(actualContainerSpec.ImageSpec).To(Equal(worker.ImageSpec{
				ResourceType: "some-resource-type",
			}))
			Expect(actualContainerMetadata).To(Equal(containerMetadata))
			Expect(actualContainerSpec.Tags).To(Equal([]string{"some", "tags"}))
			Expect(actualContainerSpec.TeamID).To(Equal(123))
			Expect(actualContainerSpec.Env).To(Equal(stepMetadata.Env()))

			Expect(actualResourceTypes).To(Equal(interpolatedResourceTypes))
			Expect(delegate).To(Equal(fakeDelegate))
		})

		Context("when the timeout cannot be parsed", func() {
			BeforeEach(func() {
				checkPlan.Timeout = "bad-value"
			})

			It("fails to parse the timeout and returns the error", func() {
				Expect(stepErr).To(HaveOccurred())
				Expect(stepErr).To(MatchError("time: invalid duration bad-value"))
			})
		})

		It("times out after the specified timeout", func() {
			now := time.Now()
			ctx, _, _ := fakeResource.CheckArgsForCall(0)
			deadline, _ := ctx.Deadline()
			Expect(deadline).Should(BeTemporally("~", now.Add(10*time.Second), time.Second))
		})

		It("runs the check resource action", func() {
			Expect(fakeResource.CheckCallCount()).To(Equal(1))
		})

		Context("when resource check succeeds", func() {
			BeforeEach(func() {
				fakeResource.CheckReturns(versions, nil)
			})

			It("saves the versions", func() {
				Expect(fakeDelegate.SaveVersionsCallCount()).To(Equal(1))

				actualVersions := fakeDelegate.SaveVersionsArgsForCall(0)
				Expect(actualVersions).To(Equal(versions))
			})
		})

		Context("when performing the check fails", func() {
			BeforeEach(func() {
				fakeResource.CheckReturns(nil, errors.New("nope"))
			})

			It("returns error", func() {
				Expect(stepErr).To(HaveOccurred())
			})

			It("is not successful", func() {
				Expect(checkStep.Succeeded()).To(BeFalse())
			})
		})

		Context("when find or choosing a worker fails", func() {
			disaster := errors.New("nope")

			BeforeEach(func() {
				fakePool.FindOrChooseWorkerForContainerReturns(nil, disaster)
			})

			It("returns the failure", func() {
				Expect(stepErr).To(Equal(disaster))
			})
		})

		Context("when find or creating a container fails", func() {
			disaster := errors.New("nope")

			BeforeEach(func() {
				fakePool.FindOrChooseWorkerForContainerReturns(fakeWorker, nil)
				fakeWorker.FindOrCreateContainerReturns(nil, disaster)
			})

			It("returns the failure", func() {
				Expect(stepErr).To(Equal(disaster))
			})
		})
	})
})
<commit_msg>fix atc/exec/check_step_test.go<commit_after>package exec_test

import (
	"context"
	"errors"
	"time"

	"github.com/concourse/concourse/atc"
	"github.com/concourse/concourse/atc/creds/credsfakes"
	"github.com/concourse/concourse/atc/db"
	"github.com/concourse/concourse/atc/exec"
	"github.com/concourse/concourse/atc/exec/build"
	"github.com/concourse/concourse/atc/exec/execfakes"
	"github.com/concourse/concourse/atc/resource/resourcefakes"
	"github.com/concourse/concourse/atc/worker"
	"github.com/concourse/concourse/atc/worker/workerfakes"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("CheckStep", func() {
	var (
		ctx    context.Context
		cancel func()

		fakeWorker          *workerfakes.FakeWorker
		fakePool            *workerfakes.FakePool
		fakeStrategy        *workerfakes.FakeContainerPlacementStrategy
		fakeResourceFactory *resourcefakes.FakeResourceFactory
		fakeSecretManager   *credsfakes.FakeSecrets
		fakeDelegate        *execfakes.FakeCheckDelegate
		checkPlan           *atc.CheckPlan

		interpolatedResourceTypes atc.VersionedResourceTypes

		containerMetadata = db.ContainerMetadata{
			Type:     db.ContainerTypeCheck,
			StepName: "some-step",
		}

		stepMetadata = exec.StepMetadata{
			ResourceConfigID:   1,
			BaseResourceTypeID: 1,
			TeamID:             123,
		}

		owner = db.NewResourceConfigCheckSessionContainerOwner(stepMetadata.ResourceConfigID, stepMetadata.BaseResourceTypeID, db.ContainerOwnerExpiries{
			Min: 5 * time.Minute,
			Max: 1 * time.Hour,
		})

		repo  *build.Repository
		state *execfakes.FakeRunState

		checkStep *exec.CheckStep
		stepErr   error

		planID atc.PlanID
	)

	BeforeEach(func() {
		ctx, cancel = context.WithCancel(context.Background())

		planID = atc.PlanID("some-plan-id")

		fakeStrategy = new(workerfakes.FakeContainerPlacementStrategy)
		fakePool = new(workerfakes.FakePool)
		fakeWorker = new(workerfakes.FakeWorker)
		fakeResourceFactory = new(resourcefakes.FakeResourceFactory)

		fakeSecretManager = new(credsfakes.FakeSecrets)
		fakeSecretManager.GetReturnsOnCall(0, "super-secret-source", nil, true, nil)
		fakeSecretManager.GetReturnsOnCall(1, "source", nil, true, nil)

		fakeDelegate = new(execfakes.FakeCheckDelegate)

		repo = build.NewRepository()
		state = new(execfakes.FakeRunState)
		state.ArtifactRepositoryReturns(repo)

		interpolatedResourceTypes = atc.VersionedResourceTypes{
			{
				ResourceType: atc.ResourceType{
					Name:   "custom-resource",
					Type:   "custom-type",
					Source: atc.Source{"some-custom": "super-secret-source"},
				},
				Version: atc.Version{"some-custom": "version"},
			},
		}

		checkPlan = &atc.CheckPlan{
			Name:                   "some-name",
			Type:                   "some-resource-type",
			Source:                 atc.Source{"some": "super-secret-source"},
			Tags:                   []string{"some", "tags"},
			Timeout:                "10s",
			FromVersion:            atc.Version{"some-custom": "version"},
			VersionedResourceTypes: interpolatedResourceTypes,
		}
	})

	AfterEach(func() {
		cancel()
	})

	JustBeforeEach(func() {
		plan := atc.Plan{
			ID:    atc.PlanID(planID),
			Check: checkPlan,
		}

		checkStep = exec.NewCheckStep(
			plan.ID,
			*plan.Check,
			stepMetadata,
			fakeResourceFactory,
			containerMetadata,
			fakeStrategy,
			fakePool,
			fakeDelegate,
		)

		stepErr = checkStep.Run(ctx, state)
	})

	Context("when find or choosing worker succeeds", func() {
		var (
			fakeResource *resourcefakes.FakeResource
			versions     []atc.Version
		)

		BeforeEach(func() {
			fakeWorker.NameReturns("some-worker")
			fakePool.FindOrChooseWorkerForContainerReturns(fakeWorker, nil)

			fakeResource = new(resourcefakes.FakeResource)
			fakeResourceFactory.NewResourceReturns(fakeResource)
		})

		It("finds or chooses a worker", func() {
			Expect(fakePool.FindOrChooseWorkerForContainerCallCount()).To(Equal(1))
			_, _, actualOwner, actualContainerSpec, actualWorkerSpec, strategy := fakePool.FindOrChooseWorkerForContainerArgsForCall(0)

			Expect(actualOwner).To(Equal(owner))

			Expect(actualContainerSpec.ImageSpec).To(Equal(worker.ImageSpec{
				ResourceType: "some-resource-type",
			}))
			Expect(actualContainerSpec.Tags).To(Equal([]string{"some", "tags"}))
			Expect(actualContainerSpec.TeamID).To(Equal(123))
			Expect(actualContainerSpec.Env).To(Equal(stepMetadata.Env()))

			Expect(actualWorkerSpec).To(Equal(worker.WorkerSpec{
				ResourceType:  "some-resource-type",
				Tags:          atc.Tags{"some", "tags"},
				TeamID:        stepMetadata.TeamID,
				ResourceTypes: interpolatedResourceTypes,
			}))

			Expect(strategy).To(Equal(fakeStrategy))
		})

		It("creates a container with the correct type and owner", func() {
			_, _, delegate, actualOwner, actualContainerMetadata, actualContainerSpec, actualResourceTypes := fakeWorker.FindOrCreateContainerArgsForCall(0)

			Expect(actualOwner).To(Equal(owner))

			Expect(actualContainerSpec.ImageSpec).To(Equal(worker.ImageSpec{
				ResourceType: "some-resource-type",
			}))
			Expect(actualContainerMetadata).To(Equal(containerMetadata))
			Expect(actualContainerSpec.Tags).To(Equal([]string{"some", "tags"}))
			Expect(actualContainerSpec.TeamID).To(Equal(123))
			Expect(actualContainerSpec.Env).To(Equal(stepMetadata.Env()))

			Expect(actualResourceTypes).To(Equal(interpolatedResourceTypes))
			Expect(delegate).To(Equal(fakeDelegate))
		})

		Context("when the timeout cannot be parsed", func() {
			BeforeEach(func() {
				checkPlan.Timeout = "bad-value"
			})

			It("fails to parse the timeout and returns the error", func() {
				Expect(stepErr).To(HaveOccurred())
				Expect(stepErr).To(MatchError("time: invalid duration bad-value"))
			})
		})

		It("times out after the specified timeout", func() {
			now := time.Now()
			ctx, _, _ := fakeResource.CheckArgsForCall(0)
			deadline, _ := ctx.Deadline()
			Expect(deadline).Should(BeTemporally("~", now.Add(10*time.Second), time.Second))
		})

		It("invokes resourceFactory -> NewResource with the correct arguments", func() {
			actualSource, actualParams, actualFromVersion := fakeResourceFactory.NewResourceArgsForCall(0)
			Expect(actualSource).To(Equal(checkPlan.Source))
			Expect(actualParams).To(BeNil())
			Expect(actualFromVersion).To(Equal(checkPlan.FromVersion))
		})

		It("runs the check resource action", func() {
			Expect(fakeResource.CheckCallCount()).To(Equal(1))
		})

		Context("when resource check succeeds", func() {
			BeforeEach(func() {
				fakeResource.CheckReturns(versions, nil)
			})

			It("saves the versions", func() {
				Expect(fakeDelegate.SaveVersionsCallCount()).To(Equal(1))

				actualVersions := fakeDelegate.SaveVersionsArgsForCall(0)
				Expect(actualVersions).To(Equal(versions))
			})
		})

		Context("when performing the check fails", func() {
			BeforeEach(func() {
				fakeResource.CheckReturns(nil, errors.New("nope"))
			})

			It("returns error", func() {
				Expect(stepErr).To(HaveOccurred())
			})

			It("is not successful", func() {
				Expect(checkStep.Succeeded()).To(BeFalse())
			})
		})

		Context("when find or choosing a worker fails", func() {
			disaster := errors.New("nope")

			BeforeEach(func() {
				fakePool.FindOrChooseWorkerForContainerReturns(nil, disaster)
			})

			It("returns the failure", func() {
				Expect(stepErr).To(Equal(disaster))
			})
		})

		Context("when find or creating a container fails", func() {
			disaster := errors.New("nope")

			BeforeEach(func() {
				fakePool.FindOrChooseWorkerForContainerReturns(fakeWorker, nil)
				fakeWorker.FindOrCreateContainerReturns(nil, disaster)
			})

			It("returns the failure", func() {
				Expect(stepErr).To(Equal(disaster))
			})
		})
	})
})
<|endoftext|>


<commit_before>// +build !confonly

package tls

import (
	"crypto/tls"
	"crypto/x509"
	"strings"
	"sync"
	"time"

	"v2ray.com/core/common/net"
	"v2ray.com/core/common/protocol/tls/cert"
	"v2ray.com/core/transport/internet"
)

var (
	globalSessionCache = tls.NewLRUClientSessionCache(128)
)

const exp8357 = "experiment:8357"

// ParseCertificate converts a cert.Certificate to Certificate.
func ParseCertificate(c *cert.Certificate) *Certificate {
	certPEM, keyPEM := c.ToPEM()
	return &Certificate{
		Certificate: certPEM,
		Key:         keyPEM,
	}
}

func (c *Config) loadSelfCertPool() (*x509.CertPool, error) {
	root := x509.NewCertPool()
	for _, cert := range c.Certificate {
		if !root.AppendCertsFromPEM(cert.Certificate) {
			return nil, newError("failed to append cert").AtWarning()
		}
	}
	return root, nil
}

// BuildCertificates builds a list of TLS certificates from proto definition.
func (c *Config) BuildCertificates() []tls.Certificate {
	certs := make([]tls.Certificate, 0, len(c.Certificate))
	for _, entry := range c.Certificate {
		if entry.Usage != Certificate_ENCIPHERMENT {
			continue
		}
		keyPair, err := tls.X509KeyPair(entry.Certificate, entry.Key)
		if err != nil {
			newError("ignoring invalid X509 key pair").Base(err).AtWarning().WriteToLog()
			continue
		}
		certs = append(certs, keyPair)
	}
	return certs
}

func isCertificateExpired(c *tls.Certificate) bool {
	if c.Leaf == nil && len(c.Certificate) > 0 {
		if pc, err := x509.ParseCertificate(c.Certificate[0]); err == nil {
			c.Leaf = pc
		}
	}

	// If leaf is not there, the certificate is probably not used yet. We trust user to provide a valid certificate.
	return c.Leaf != nil && c.Leaf.NotAfter.Before(time.Now().Add(-time.Minute))
}

func issueCertificate(rawCA *Certificate, domain string) (*tls.Certificate, error) {
	parent, err := cert.ParseCertificate(rawCA.Certificate, rawCA.Key)
	if err != nil {
		return nil, newError("failed to parse raw certificate").Base(err)
	}
	newCert, err := cert.Generate(parent, cert.CommonName(domain), cert.DNSNames(domain))
	if err != nil {
		return nil, newError("failed to generate new certificate for ", domain).Base(err)
	}
	newCertPEM, newKeyPEM := newCert.ToPEM()
	cert, err := tls.X509KeyPair(newCertPEM, newKeyPEM)
	return &cert, err
}

func (c *Config) getCustomCA() []*Certificate {
	certs := make([]*Certificate, 0, len(c.Certificate))
	for _, certificate := range c.Certificate {
		if certificate.Usage == Certificate_AUTHORITY_ISSUE {
			certs = append(certs, certificate)
		}
	}
	return certs
}

func getGetCertificateFunc(c *tls.Config, ca []*Certificate) func(hello *tls.ClientHelloInfo) (*tls.Certificate, error) {
	var access sync.RWMutex

	return func(hello *tls.ClientHelloInfo) (*tls.Certificate, error) {
		domain := hello.ServerName
		certExpired := false

		access.RLock()
		certificate, found := c.NameToCertificate[domain]
		access.RUnlock()

		if found {
			if !isCertificateExpired(certificate) {
				return certificate, nil
			}
			certExpired = true
		}

		if certExpired {
			newCerts := make([]tls.Certificate, 0, len(c.Certificates))

			access.Lock()
			for _, certificate := range c.Certificates {
				if !isCertificateExpired(&certificate) {
					newCerts = append(newCerts, certificate)
				}
			}

			c.Certificates = newCerts
			access.Unlock()
		}

		var issuedCertificate *tls.Certificate

		// Create a new certificate from existing CA if possible
		for _, rawCert := range ca {
			if rawCert.Usage == Certificate_AUTHORITY_ISSUE {
				newCert, err := issueCertificate(rawCert, domain)
				if err != nil {
					newError("failed to issue new certificate for ", domain).Base(err).WriteToLog()
					continue
				}

				access.Lock()
				c.Certificates = append(c.Certificates, *newCert)
				issuedCertificate = &c.Certificates[len(c.Certificates)-1]
				access.Unlock()
				break
			}
		}

		if issuedCertificate == nil {
			return nil, newError("failed to create a new certificate for ", domain)
		}

		access.Lock()
		c.BuildNameToCertificate()
		access.Unlock()

		return issuedCertificate, nil
	}
}

func (c *Config) IsExperiment8357() bool {
	return strings.HasPrefix(c.ServerName, exp8357)
}

func (c *Config) parseServerName() string {
	if c.IsExperiment8357() {
		return c.ServerName[len(exp8357):]
	}

	return c.ServerName
}

// GetTLSConfig converts this Config into tls.Config.
func (c *Config) GetTLSConfig(opts ...Option) *tls.Config {
	root, err := c.getCertPool()
	if err != nil {
		newError("failed to load system root certificate").AtError().Base(err).WriteToLog()
	}

	config := &tls.Config{
		ClientSessionCache:     globalSessionCache,
		RootCAs:                root,
		SessionTicketsDisabled: c.DisableSessionResumption,
	}
	if c == nil {
		return config
	}

	for _, opt := range opts {
		opt(config)
	}

	if !c.AllowInsecureCiphers && len(config.CipherSuites) == 0 {
		config.CipherSuites = []uint16{
			tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,
			tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,
			tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
			tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
			tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
			tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,
			tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256,
			tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,
			tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256,
			tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,
			tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,
			tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,
		}
	}

	config.InsecureSkipVerify = c.AllowInsecure
	config.Certificates = c.BuildCertificates()
	config.BuildNameToCertificate()

	caCerts := c.getCustomCA()
	if len(caCerts) > 0 {
		config.GetCertificate = getGetCertificateFunc(config, caCerts)
	}

	if sn := c.parseServerName(); len(sn) > 0 {
		config.ServerName = sn
	}

	if len(c.NextProtocol) > 0 {
		config.NextProtos = c.NextProtocol
	}
	if len(config.NextProtos) == 0 {
		config.NextProtos = []string{"http/1.1"}
	}

	return config
}

// Option for building TLS config.
type Option func(*tls.Config)

// WithDestination sets the server name in TLS config.
func WithDestination(dest net.Destination) Option {
	return func(config *tls.Config) {
		if dest.Address.Family().IsDomain() && len(config.ServerName) == 0 {
			config.ServerName = dest.Address.Domain()
		}
	}
}

// WithNextProto sets the ALPN values in TLS config.
func WithNextProto(protocol ...string) Option {
	return func(config *tls.Config) {
		if len(config.NextProtos) == 0 {
			config.NextProtos = protocol
		}
	}
}

// ConfigFromStreamSettings fetches Config from stream settings. Nil if not found.
func ConfigFromStreamSettings(settings *internet.MemoryStreamConfig) *Config {
	if settings == nil {
		return nil
	}
	config, ok := settings.SecuritySettings.(*Config)
	if !ok {
		return nil
	}
	return config
}
<commit_msg>add tls1.3 ciphers<commit_after>// +build !confonly

package tls

import (
	"crypto/tls"
	"crypto/x509"
	"strings"
	"sync"
	"time"

	"v2ray.com/core/common/net"
	"v2ray.com/core/common/protocol/tls/cert"
	"v2ray.com/core/transport/internet"
)

var (
	globalSessionCache = tls.NewLRUClientSessionCache(128)
)

const exp8357 = "experiment:8357"

// ParseCertificate converts a cert.Certificate to Certificate.
func ParseCertificate(c *cert.Certificate) *Certificate {
	certPEM, keyPEM := c.ToPEM()
	return &Certificate{
		Certificate: certPEM,
		Key:         keyPEM,
	}
}

func (c *Config) loadSelfCertPool() (*x509.CertPool, error) {
	root := x509.NewCertPool()
	for _, cert := range c.Certificate {
		if !root.AppendCertsFromPEM(cert.Certificate) {
			return nil, newError("failed to append cert").AtWarning()
		}
	}
	return root, nil
}

// BuildCertificates builds a list of TLS certificates from proto definition.
func (c *Config) BuildCertificates() []tls.Certificate {
	certs := make([]tls.Certificate, 0, len(c.Certificate))
	for _, entry := range c.Certificate {
		if entry.Usage != Certificate_ENCIPHERMENT {
			continue
		}
		keyPair, err := tls.X509KeyPair(entry.Certificate, entry.Key)
		if err != nil {
			newError("ignoring invalid X509 key pair").Base(err).AtWarning().WriteToLog()
			continue
		}
		certs = append(certs, keyPair)
	}
	return certs
}

func isCertificateExpired(c *tls.Certificate) bool {
	if c.Leaf == nil && len(c.Certificate) > 0 {
		if pc, err := x509.ParseCertificate(c.Certificate[0]); err == nil {
			c.Leaf = pc
		}
	}

	// If leaf is not there, the certificate is probably not used yet. We trust user to provide a valid certificate.
	return c.Leaf != nil && c.Leaf.NotAfter.Before(time.Now().Add(-time.Minute))
}

func issueCertificate(rawCA *Certificate, domain string) (*tls.Certificate, error) {
	parent, err := cert.ParseCertificate(rawCA.Certificate, rawCA.Key)
	if err != nil {
		return nil, newError("failed to parse raw certificate").Base(err)
	}
	newCert, err := cert.Generate(parent, cert.CommonName(domain), cert.DNSNames(domain))
	if err != nil {
		return nil, newError("failed to generate new certificate for ", domain).Base(err)
	}
	newCertPEM, newKeyPEM := newCert.ToPEM()
	cert, err := tls.X509KeyPair(newCertPEM, newKeyPEM)
	return &cert, err
}

func (c *Config) getCustomCA() []*Certificate {
	certs := make([]*Certificate, 0, len(c.Certificate))
	for _, certificate := range c.Certificate {
		if certificate.Usage == Certificate_AUTHORITY_ISSUE {
			certs = append(certs, certificate)
		}
	}
	return certs
}

func getGetCertificateFunc(c *tls.Config, ca []*Certificate) func(hello *tls.ClientHelloInfo) (*tls.Certificate, error) {
	var access sync.RWMutex

	return func(hello *tls.ClientHelloInfo) (*tls.Certificate, error) {
		domain := hello.ServerName
		certExpired := false

		access.RLock()
		certificate, found := c.NameToCertificate[domain]
		access.RUnlock()

		if found {
			if !isCertificateExpired(certificate) {
				return certificate, nil
			}
			certExpired = true
		}

		if certExpired {
			newCerts := make([]tls.Certificate, 0, len(c.Certificates))

			access.Lock()
			for _, certificate := range c.Certificates {
				if !isCertificateExpired(&certificate) {
					newCerts = append(newCerts, certificate)
				}
			}

			c.Certificates = newCerts
			access.Unlock()
		}

		var issuedCertificate *tls.Certificate

		// Create a new certificate from existing CA if possible
		for _, rawCert := range ca {
			if rawCert.Usage == Certificate_AUTHORITY_ISSUE {
				newCert, err := issueCertificate(rawCert, domain)
				if err != nil {
					newError("failed to issue new certificate for ", domain).Base(err).WriteToLog()
					continue
				}

				access.Lock()
				c.Certificates = append(c.Certificates, *newCert)
				issuedCertificate = &c.Certificates[len(c.Certificates)-1]
				access.Unlock()
				break
			}
		}

		if issuedCertificate == nil {
			return nil, newError("failed to create a new certificate for ", domain)
		}

		access.Lock()
		c.BuildNameToCertificate()
		access.Unlock()

		return issuedCertificate, nil
	}
}

func (c *Config) IsExperiment8357() bool {
	return strings.HasPrefix(c.ServerName, exp8357)
}

func (c *Config) parseServerName() string {
	if c.IsExperiment8357() {
		return c.ServerName[len(exp8357):]
	}

	return c.ServerName
}

// GetTLSConfig converts this Config into tls.Config.
func (c *Config) GetTLSConfig(opts ...Option) *tls.Config {
	root, err := c.getCertPool()
	if err != nil {
		newError("failed to load system root certificate").AtError().Base(err).WriteToLog()
	}

	config := &tls.Config{
		ClientSessionCache:     globalSessionCache,
		RootCAs:                root,
		SessionTicketsDisabled: c.DisableSessionResumption,
	}
	if c == nil {
		return config
	}

	for _, opt := range opts {
		opt(config)
	}

	if !c.AllowInsecureCiphers && len(config.CipherSuites) == 0 {
		config.CipherSuites = []uint16{
			tls.TLS_AES_128_GCM_SHA256,
			tls.TLS_AES_256_GCM_SHA384,
			tls.TLS_CHACHA20_POLY1305_SHA256,

			tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,
			tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,
			tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
			tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
			tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
			tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,
			tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256,
			tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,
			tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256,
			tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,
			tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,
			tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,
		}
	}

	config.InsecureSkipVerify = c.AllowInsecure
	config.Certificates = c.BuildCertificates()
	config.BuildNameToCertificate()

	caCerts := c.getCustomCA()
	if len(caCerts) > 0 {
		config.GetCertificate = getGetCertificateFunc(config, caCerts)
	}

	if sn := c.parseServerName(); len(sn) > 0 {
		config.ServerName = sn
	}

	if len(c.NextProtocol) > 0 {
		config.NextProtos = c.NextProtocol
	}
	if len(config.NextProtos) == 0 {
		config.NextProtos = []string{"http/1.1"}
	}

	return config
}

// Option for building TLS config.
type Option func(*tls.Config)

// WithDestination sets the server name in TLS config.
func WithDestination(dest net.Destination) Option {
	return func(config *tls.Config) {
		if dest.Address.Family().IsDomain() && len(config.ServerName) == 0 {
			config.ServerName = dest.Address.Domain()
		}
	}
}

// WithNextProto sets the ALPN values in TLS config.
func WithNextProto(protocol ...string) Option {
	return func(config *tls.Config) {
		if len(config.NextProtos) == 0 {
			config.NextProtos = protocol
		}
	}
}

// ConfigFromStreamSettings fetches Config from stream settings. Nil if not found.
func ConfigFromStreamSettings(settings *internet.MemoryStreamConfig) *Config {
	if settings == nil {
		return nil
	}
	config, ok := settings.SecuritySettings.(*Config)
	if !ok {
		return nil
	}
	return config
}
<|endoftext|>


<commit_before>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

// This generates a csv listing of all of our prowjobs to import into a
// spreadsheet so humans could see prowjob info relevant to enforcing
// policies at a glance

// The intent is for actual tests to enforce these policies, but their
// output is not amenable to generating a report that could be broadcast
// to humans

import (
	"flag"
	"fmt"
	"os"
	"regexp"
	"strconv"
	"strings"

	corev1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/resource"

	// pjv1 "k8s.io/test-infra/prow/apis/prowjobs/v1"
	cfg "k8s.io/test-infra/prow/config"
)

// TODO: parse testgrid config to catch
//	- jobs that aren't prowjobs but are on release-informing dashboards
//	- jobs that don't declare testgrid info via annotations
var configPath = flag.String("config", "../../config/prow/config.yaml", "Path to prow config")
var jobConfigPath = flag.String("job-config", "../../config/jobs", "Path to prow job config")

// Loaded at TestMain.
var c *cfg.Config

func main() {
	flag.Parse()
	if *configPath == "" {
		fmt.Println("--config must set")
		os.Exit(1)
	}
	if *jobConfigPath == "" {
		fmt.Println("--job-config must set")
		os.Exit(1)
	}

	conf, err := cfg.Load(*configPath, *jobConfigPath)
	if err != nil {
		fmt.Printf("Could not load config: %v", err)
		os.Exit(1)
	}
	c = conf

	PrintCsvOfJobResources()
}

func allStaticJobs() []cfg.JobBase {
	jobs := []cfg.JobBase{}
	for _, job := range c.AllStaticPresubmits(nil) {
		jobs = append(jobs, job.JobBase)
	}
	for _, job := range c.AllStaticPostsubmits(nil) {
		jobs = append(jobs, job.JobBase)
	}
	for _, job := range c.AllPeriodics() {
		jobs = append(jobs, job.JobBase)
	}
	return jobs
}

func isPodQOSGuaranteed(spec *corev1.PodSpec) bool {
	isGuaranteed := true
	c := spec.Containers[0]
	zero := resource.MustParse("0")
	resources := []corev1.ResourceName{
		corev1.ResourceCPU,
		corev1.ResourceMemory,
	}
	for _, r := range resources {
		limit, ok := c.Resources.Limits[r]
		if !ok || limit.Cmp(zero) == 0 || limit.Cmp(c.Resources.Requests[r]) != 0 {
			isGuaranteed = false
		}
	}
	return isGuaranteed
}

func GetResources(spec *corev1.PodSpec) map[string]string {
	m := make(map[string]string)
	if spec == nil {
		return m
	}
	// TODO: support N containers
	r := spec.Containers[0].Resources
	m = map[string]string{
		"req.cpu": String(r.Requests[corev1.ResourceCPU], resource.Milli),
		"lim.cpu": String(r.Limits[corev1.ResourceCPU], resource.Milli),
		"req.mem": String(r.Requests[corev1.ResourceMemory], resource.Giga),
		"lim.mem": String(r.Limits[corev1.ResourceCPU], resource.Giga),
	}
	return m
}

func String(q resource.Quantity, s resource.Scale) string {
	return strconv.FormatInt(q.ScaledValue(s), 10)
}

func PrimaryDashboard(job cfg.JobBase) string {
	dashboardsAnnotation, ok := job.Annotations["testgrid-dashboards"]
	if !ok {
		// technically it could be specified in a testgrid config, would need testgrid/cmd/configurator code to know for sure
		return "TODO"
	}
	dashboards := []string{}
	for _, db := range strings.Split(dashboardsAnnotation, ",") {
		dashboards = append(dashboards, strings.TrimSpace(db))
	}
	for _, db := range dashboards {
		if strings.HasPrefix(db, "sig-release-") {
			return db
		}
	}
	for _, db := range dashboards {
		if strings.HasPrefix(db, "sig-") {
			return db
		}
	}
	return dashboards[0]
}

func OwnerDashboard(job cfg.JobBase) string {
	dashboardsAnnotation, ok := job.Annotations["testgrid-dashboards"]
	if !ok {
		// technically it could be specified in a testgrid config, would need testgrid/cmd/configurator code to know for sure
		return "TODO"
	}
	dashboards := []string{}
	for _, db := range strings.Split(dashboardsAnnotation, ",") {
		dashboards = append(dashboards, strings.TrimSpace(db))
	}
	for _, db := range dashboards {
		if strings.HasPrefix(db, "sig-") && !strings.HasPrefix(db, "sig-release-") {
			return db
		}
	}
	for _, db := range dashboards {
		if strings.HasPrefix(db, "sig-release-") {
			return db
		}
	}
	return dashboards[0]
}

func PrintCsvOfJobResources() {
	fmt.Printf("primary dashboard, owner dashboard, name, type, repo, cluster, concurrency, req.cpu (m), lim.cpu (m), req.mem (Gi), lim.mem (Gi)\n")
	printJobBaseColumns := func(jobType, repo string, job cfg.JobBase) {
		name := job.Name
		primaryDash := PrimaryDashboard(job)
		ownerDash := OwnerDashboard(job)
		cluster := job.Cluster
		res := GetResources(job.Spec)
		cols := []string{primaryDash, ownerDash, name, jobType, repo, cluster, strconv.Itoa(job.MaxConcurrency), res["req.cpu"], res["lim.cpu"], res["req.mem"], res["lim.mem"]}
		fmt.Println(strings.Join(cols, ", "))
	}
	for _, job := range c.AllPeriodics() {
		// TODO: depending on whether decoration or bootstrap is used repo could be any number of repos
		printJobBaseColumns("periodic", "TODO", job.JobBase)
	}
	for repo, jobs := range c.PostsubmitsStatic {
		for _, job := range jobs {
			printJobBaseColumns("postsubmit", repo, job.JobBase)
		}
	}
	for repo, jobs := range c.PresubmitsStatic {
		for _, job := range jobs {
			printJobBaseColumns("presubmit", repo, job.JobBase)
		}
	}
}

func TestQueryReleaseBlockingJobs() {
	jobs := allStaticJobs()

	filtered := []cfg.JobBase{}
	dashboardRE := regexp.MustCompile(`sig-release-master-blocking`)
	//dashboardRE := regexp.MustCompile(`sig-release-(1.[0-9]{2}|master)-blocking`)
	for _, job := range jobs {
		if dashboards, ok := job.Annotations["testgrid-dashboards"]; ok {
			if dashboardRE.MatchString(dashboards) {
				filtered = append(filtered, job)
			}
		}
	}
	jobs = filtered

	jobNamesByCluster := make(map[string][]string)
	for _, job := range jobs {
		if job.Cluster != "" {
			if jobNames, ok := jobNamesByCluster[job.Cluster]; !ok {
				jobNamesByCluster[job.Cluster] = []string{job.Name}
			} else {
				jobNamesByCluster[job.Cluster] = append(jobNames, job.Name)
			}
		}
	}
	for cluster, jobNames := range jobNamesByCluster {
		fmt.Printf("There are %4d of %4d sig-release-.*-blocking jobs that use cluster: %s\n", len(jobNames), len(jobs), cluster)
		for _, name := range jobNames {
			fmt.Printf("  - %s\n", name)
		}
	}
}
<commit_msg>WIP : Rough work to traverse Job Config and produce JSON output<commit_after>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

// This generates a csv listing of all of our prowjobs to import into a
// spreadsheet so humans could see prowjob info relevant to enforcing
// policies at a glance

// The intent is for actual tests to enforce these policies, but their
// output is not amenable to generating a report that could be broadcast
// to humans

import (
	"flag"
	"fmt"
	"os"
	"regexp"
	"strconv"
	"strings"
	"encoding/json"

	corev1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/resource"

	// pjv1 "k8s.io/test-infra/prow/apis/prowjobs/v1"
	cfg "k8s.io/test-infra/prow/config"
)

// TODO: parse testgrid config to catch
//	- jobs that aren't prowjobs but are on release-informing dashboards
//	- jobs that don't declare testgrid info via annotations
var configPath = flag.String("config", "../../config/prow/config.yaml", "Path to prow config")
var jobConfigPath = flag.String("job-config", "../../config/jobs", "Path to prow job config")
var reportFormat = flag.String("format", "csv", "Output format [csv|json] defaults to csv")

// struct to report on stated resource requests and limits
// a empty strings field indicates that no request was found
type RequestedResources struct {
	cpuRequested string ; // "req.cpu": String(r.Requests[corev1.ResourceCPU], resource.Milli),
	cpuLimitedTo string ; // "lim.cpu": String(r.Limits[corev1.ResourceCPU], resource.Milli),
	memRequested string ; // "req.mem": String(r.Requests[corev1.ResourceMemory], resource.Giga),
	memLimitedTo string ; // "lim.mem": String(r.Limits[corev1.ResourceCPU], resource.Giga),
}

// From a CI runtime perspective a Job has configuration data located in multiple locations
// using multiple means of specification.
// For the purposes of this report, the decoratedJobConfig contains JobBase config data
// along with associated data that we are currently interested in reporting on.
type DecoratedJobConfig struct {
	Job cfg.JobBase;
	Dashboard string;
	Cluster string;
	RequestedContainerResources map[string]map[string]string// Maps cntr name to RequestedResources
}

// Loaded at TestMain.
var c *cfg.Config

func main() {
	flag.Parse()
	if *configPath == "" {
		fmt.Println("--config must set")
		os.Exit(1)
	}
	if *jobConfigPath == "" {
		fmt.Println("--job-config must set")
		os.Exit(1)
	}

	conf, err := cfg.Load(*configPath, *jobConfigPath)
	if err != nil {
		fmt.Printf("configPath: %v\n", *configPath)
		fmt.Printf("jobConfigPath: %v\n", *jobConfigPath)
		fmt.Printf("Could not load config: %v\n", err)
		os.Exit(1)
	}
	c = conf
	switch *reportFormat {
	case "csv":
		PrintCsvOfJobResources()
	case "json":
		PrintJSONJobResources()
	}
}

func allStaticJobs() []cfg.JobBase {
	jobs := []cfg.JobBase{}
	for _, job := range c.AllStaticPresubmits(nil) {
		jobs = append(jobs, job.JobBase)
	}
	for _, job := range c.AllStaticPostsubmits(nil) {
		jobs = append(jobs, job.JobBase)
	}
	for _, job := range c.AllPeriodics() {
		jobs = append(jobs, job.JobBase)
	}
	return jobs
}

func isPodQOSGuaranteed(spec *corev1.PodSpec) bool {
	isGuaranteed := true
	c := spec.Containers[0]
	zero := resource.MustParse("0")
	resources := []corev1.ResourceName{
		corev1.ResourceCPU,
		corev1.ResourceMemory,
	}
	for _, r := range resources {
		limit, ok := c.Resources.Limits[r]
		if !ok || limit.Cmp(zero) == 0 || limit.Cmp(c.Resources.Requests[r]) != 0 {
			isGuaranteed = false
		}
	}
	return isGuaranteed
}
// GetResources returns a map of container names to resources requested by that container
func GetResources(spec *corev1.PodSpec) map[string]map[string]string {
	m := make(map[string]map[string]string)
	if spec == nil {
		return m
	}
	// Range over all Containers present in spec.
	// Q : I have not managed to find any spces with more than one Container

	fmt.Printf("Container count is %d\n", len(spec.Containers));
  	for i,c := range spec.Containers {
		cntrName := c.Name + "-" + strconv.Itoa(i);
		r := c.Resources ;

		m = map[string]map[string]string{
			cntrName : {
			"req.cpu": String(r.Requests[corev1.ResourceCPU], resource.Milli),
			"lim.cpu": String(r.Limits[corev1.ResourceCPU], resource.Milli),
			"req.mem": String(r.Requests[corev1.ResourceMemory], resource.Giga),
				"lim.mem": String(r.Limits[corev1.ResourceCPU], resource.Giga),
			},
		}
	}
	return m
}

func String(q resource.Quantity, s resource.Scale) string {
	return strconv.FormatInt(q.ScaledValue(s), 10)
}

func PrimaryDashboard(job cfg.JobBase) string {
	dashboardsAnnotation, ok := job.Annotations["testgrid-dashboards"]
	if !ok {
		// technically it could be specified in a testgrid config, would need testgrid/cmd/configurator code to know for sure
		return "TODO"
	}
	dashboards := []string{}
	for _, db := range strings.Split(dashboardsAnnotation, ",") {
		dashboards = append(dashboards, strings.TrimSpace(db))
	}
	for _, db := range dashboards {
		if strings.HasPrefix(db, "sig-release-") {
			return db
		}
	}
	for _, db := range dashboards {
		if strings.HasPrefix(db, "sig-") {
			return db
		}
	}
	return dashboards[0]
}

func OwnerDashboard(job cfg.JobBase) string {
	dashboardsAnnotation, ok := job.Annotations["testgrid-dashboards"]
	if !ok {
		// technically it could be specified in a testgrid config, would need testgrid/cmd/configurator code to know for sure
		return "TODO"
	}
	dashboards := []string{}
	for _, db := range strings.Split(dashboardsAnnotation, ",") {
		dashboards = append(dashboards, strings.TrimSpace(db))
	}
	for _, db := range dashboards {
		if strings.HasPrefix(db, "sig-") && !strings.HasPrefix(db, "sig-release-") {
			return db
		}
	}
	for _, db := range dashboards {
		if strings.HasPrefix(db, "sig-release-") {
			return db
		}
	}
	return dashboards[0]
}

func PrintJSONJobResources() {

	var decoratedJobs []DecoratedJobConfig;
	for _, job := range c.JobConfig.AllStaticPresubmits(nil) {
		var cluster string = job.Cluster;
		var dashboard = job.Annotations["testgrid-dashboards"];
		var requestedResources = GetResources(job.Spec);
		decoratedJobs = append(decoratedJobs, DecoratedJobConfig{
			Cluster: cluster,
			Dashboard : dashboard,
			Job : job.JobBase,
			RequestedContainerResources : requestedResources,
		});
	}
	b, err := json.Marshal(decoratedJobs);
	if err != nil {
		fmt.Println("error:", err)
	}
	os.Stdout.Write(b)
}
func PrintCsvOfJobResources() {
	/**
	fmt.Printf("primary dashboard, owner dashboard, name, type, repo, cluster, concurrency, req.cpu (m), lim.cpu (m), req.mem (Gi), lim.mem (Gi)\n")
	printJobBaseColumns := func(jobType, repo string, job cfg.JobBase) {
		name := job.Name
		primaryDash := PrimaryDashboard(job)
		ownerDash := OwnerDashboard(job)
		cluster := job.Cluster
		res := GetResources(job.Spec)
		cols := []string{primaryDash, ownerDash, name, jobType, repo, cluster, strconv.Itoa(job.MaxConcurrency), res["req.cpu"], res["lim.cpu"], res["req.mem"], res["lim.mem"]}
		fmt.Println(strings.Join(cols, ", "))
	}
	for _, job := range c.AllPeriodics() {
		// TODO: depending on whether decoration or bootstrap is used repo could be any number of repos
		printJobBaseColumns("periodic", "TODO", job.JobBase)
	}
	for repo, jobs := range c.PostsubmitsStatic {
		for _, job := range jobs {
			printJobBaseColumns("postsubmit", repo, job.JobBase)
		}
	}
	for repo, jobs := range c.PresubmitsStatic {
		for _, job := range jobs {
			printJobBaseColumns("presubmit", repo, job.JobBase)
		}
	}
*/
}

func TestQueryReleaseBlockingJobs() {
	jobs := allStaticJobs()

	filtered := []cfg.JobBase{}
	dashboardRE := regexp.MustCompile(`sig-release-master-blocking`)
	//dashboardRE := regexp.MustCompile(`sig-release-(1.[0-9]{2}|master)-blocking`)
	for _, job := range jobs {
		if dashboards, ok := job.Annotations["testgrid-dashboards"]; ok {
			if dashboardRE.MatchString(dashboards) {
				filtered = append(filtered, job)
			}
		}
	}
	jobs = filtered

	jobNamesByCluster := make(map[string][]string)
	for _, job := range jobs {
		if job.Cluster != "" {
			if jobNames, ok := jobNamesByCluster[job.Cluster]; !ok {
				jobNamesByCluster[job.Cluster] = []string{job.Name}
			} else {
				jobNamesByCluster[job.Cluster] = append(jobNames, job.Name)
			}
		}
	}
	for cluster, jobNames := range jobNamesByCluster {
		fmt.Printf("There are %4d of %4d sig-release-.*-blocking jobs that use cluster: %s\n", len(jobNames), len(jobs), cluster)
		for _, name := range jobNames {
			fmt.Printf("  - %s\n", name)
		}
	}
}

<|endoftext|>


<commit_before>package autobatch

import (
	"bytes"
	"fmt"
	"testing"

	ds "github.com/ipfs/go-datastore"
)

func TestBasicPuts(t *testing.T) {
	d := NewAutoBatching(ds.NewMapDatastore(), 16)

	k := ds.NewKey("test")
	v := []byte("hello world")

	err := d.Put(k, v)
	if err != nil {
		t.Fatal(err)
	}

	out, err := d.Get(k)
	if err != nil {
		t.Fatal(err)
	}

	if !bytes.Equal(out, v) {
		t.Fatal("wasnt the same! ITS NOT THE SAME")
	}
}

func TestFlushing(t *testing.T) {
	child := ds.NewMapDatastore()
	d := NewAutoBatching(child, 16)

	var keys []ds.Key
	for i := 0; i < 16; i++ {
		keys = append(keys, ds.NewKey(fmt.Sprintf("test%d", i)))
	}
	v := []byte("hello world")

	for _, k := range keys {
		err := d.Put(k, v)
		if err != nil {
			t.Fatal(err)
		}
	}

	_, err := child.Get(keys[0])
	if err != ds.ErrNotFound {
		t.Fatal("shouldnt have found value")
	}

	err = d.Put(ds.NewKey("test16"), v)
	if err != nil {
		t.Fatal(err)
	}

	// should be flushed now, try to get keys from child datastore
	for _, k := range keys {
		val, err := child.Get(k)
		if err != nil {
			t.Fatal(err)
		}

		if !bytes.Equal(val, v) {
			t.Fatal("wrong value")
		}
	}
}
<commit_msg>test autobatch against the test suite<commit_after>package autobatch

import (
	"bytes"
	"fmt"
	"testing"

	ds "github.com/ipfs/go-datastore"
	dstest "github.com/ipfs/go-datastore/test"
)

func TestAutobatch(t *testing.T) {
	dstest.SubtestAll(t, NewAutoBatching(ds.NewMapDatastore(), 16))
}

func TestFlushing(t *testing.T) {
	child := ds.NewMapDatastore()
	d := NewAutoBatching(child, 16)

	var keys []ds.Key
	for i := 0; i < 16; i++ {
		keys = append(keys, ds.NewKey(fmt.Sprintf("test%d", i)))
	}
	v := []byte("hello world")

	for _, k := range keys {
		err := d.Put(k, v)
		if err != nil {
			t.Fatal(err)
		}
	}

	_, err := child.Get(keys[0])
	if err != ds.ErrNotFound {
		t.Fatal("shouldnt have found value")
	}

	err = d.Put(ds.NewKey("test16"), v)
	if err != nil {
		t.Fatal(err)
	}

	// should be flushed now, try to get keys from child datastore
	for _, k := range keys {
		val, err := child.Get(k)
		if err != nil {
			t.Fatal(err)
		}

		if !bytes.Equal(val, v) {
			t.Fatal("wrong value")
		}
	}
}
<|endoftext|>


<commit_before>package facebook

import (
	"os"
	"http"
	"io/ioutil"
	"json"
)

type Response struct {
	Data interface{}

	Url      string
	FinalUrl string
}

func Get(url string) (r *Response, err os.Error) {
	r = &Response{Url: url}

	// Http
	resp, finalUrl, err := http.Get(r.Url)
	if resp != nil {
		defer resp.Body.Close()
	}
	if err != nil {
		return nil, os.NewError("GetResponse(" + url + "): " + err.String())
	}
	r.FinalUrl = finalUrl
	b, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	// Json
	var values interface{}
	err = json.Unmarshal(b, &values)
	if err != nil {
		return nil, err
	}
	r.Data = values
	switch values.(type) {
	case bool:
		err = os.NewError("Call for Response couldn't be handled correctly.")
		// TODO: Extract error message
	case map[string]interface{}:
		// Do nothing
	default:
		err = os.NewError("Unsupported returned JSON-Data.")
	}
	return
}
<commit_msg>Response.Data is a string, now. Get no longer parses the data from facebook into a generic json.<commit_after>package facebook

import (
	"os"
	"http"
	"io/ioutil"
)

type Response struct {
	Data     string
	Url      string
	FinalUrl string
}

func Get(url string) (r *Response, err os.Error) {
	r = &Response{Url: url}

	// Http
	resp, finalUrl, err := http.Get(r.Url)
	if resp != nil {
		defer resp.Body.Close()
	}
	if err != nil {
		return nil, os.NewError("GetResponse(" + url + "): " + err.String())
	}
	r.FinalUrl = finalUrl
	b, err := ioutil.ReadAll(resp.Body)
	r.Data = string(b)
	return
}
<|endoftext|>


<commit_before>package errors

import (
	"crypto/x509"
	"encoding/json"
	"errors"
	"testing"
)

func TestNew(t *testing.T) {
	err := New(CertificateError, Unknown)
	if err == nil {
		t.Fatal("Error creation failed.")
	}
	if err.ErrorCode != int(CertificateError)+int(Unknown) {
		t.Fatal("Error code construction failed.")
	}
	if err.Message != "Unknown certificate error" {
		t.Fatal("Error message construction failed.")
	}

	code := New(OCSPError, ReadFailed).ErrorCode
	if code != 8001 {
		t.Fatal("Improper error code")
	}

	code = New(OCSPError, IssuerMismatch).ErrorCode
	if code != 8100 {
		t.Fatal("Improper error code")
	}

	code = New(OCSPError, InvalidStatus).ErrorCode
	if code != 8200 {
		t.Fatal("Improper error code")
	}

	code = New(CertificateError, Unknown).ErrorCode
	if code != 1000 {
		t.Fatal("Improper error code")
	}
	code = New(CertificateError, ReadFailed).ErrorCode
	if code != 1001 {
		t.Fatal("Improper error code")
	}
	code = New(CertificateError, DecodeFailed).ErrorCode
	if code != 1002 {
		t.Fatal("Improper error code")
	}
	code = New(CertificateError, ParseFailed).ErrorCode
	if code != 1003 {
		t.Fatal("Improper error code")
	}
	code = New(CertificateError, SelfSigned).ErrorCode
	if code != 1100 {
		t.Fatal("Improper error code")
	}
	code = New(CertificateError, VerifyFailed).ErrorCode
	if code != 1200 {
		t.Fatal("Improper error code")
	}
	code = New(CertificateError, BadRequest).ErrorCode
	if code != 1300 {
		t.Fatal("Improper error code")
	}

	code = New(PrivateKeyError, Unknown).ErrorCode
	if code != 2000 {
		t.Fatal("Improper error code")
	}
	code = New(PrivateKeyError, ReadFailed).ErrorCode
	if code != 2001 {
		t.Fatal("Improper error code")
	}
	code = New(PrivateKeyError, DecodeFailed).ErrorCode
	if code != 2002 {
		t.Fatal("Improper error code")
	}
	code = New(PrivateKeyError, ParseFailed).ErrorCode
	if code != 2003 {
		t.Fatal("Improper error code")
	}
	code = New(PrivateKeyError, Encrypted).ErrorCode
	if code != 2100 {
		t.Fatal("Improper error code")
	}
	code = New(PrivateKeyError, NotRSAOrECC).ErrorCode
	if code != 2200 {
		t.Fatal("Improper error code")
	}
	code = New(PrivateKeyError, KeyMismatch).ErrorCode
	if code != 2300 {
		t.Fatal("Improper error code")
	}
	code = New(PrivateKeyError, GenerationFailed).ErrorCode
	if code != 2400 {
		t.Fatal("Improper error code")
	}
	code = New(PrivateKeyError, Unavailable).ErrorCode
	if code != 2500 {
		t.Fatal("Improper error code")
	}

	code = New(IntermediatesError, Unknown).ErrorCode
	if code != 3000 {
		t.Fatal("Improper error code")
	}
	code = New(IntermediatesError, ReadFailed).ErrorCode
	if code != 3001 {
		t.Fatal("Improper error code")
	}
	code = New(IntermediatesError, DecodeFailed).ErrorCode
	if code != 3002 {
		t.Fatal("Improper error code")
	}
	code = New(IntermediatesError, ParseFailed).ErrorCode
	if code != 3003 {
		t.Fatal("Improper error code")
	}

	code = New(RootError, Unknown).ErrorCode
	if code != 4000 {
		t.Fatal("Improper error code")
	}
	code = New(RootError, ReadFailed).ErrorCode
	if code != 4001 {
		t.Fatal("Improper error code")
	}
	code = New(RootError, DecodeFailed).ErrorCode
	if code != 4002 {
		t.Fatal("Improper error code")
	}
	code = New(RootError, ParseFailed).ErrorCode
	if code != 4003 {
		t.Fatal("Improper error code")
	}

	code = New(PolicyError, Unknown).ErrorCode
	if code != 5000 {
		t.Fatal("Improper error code")
	}
	code = New(PolicyError, NoKeyUsages).ErrorCode
	if code != 5100 {
		t.Fatal("Improper error code")
	}
	code = New(PolicyError, InvalidPolicy).ErrorCode
	if code != 5200 {
		t.Fatal("Improper error code")
	}
	code = New(PolicyError, InvalidRequest).ErrorCode
	if code != 5300 {
		t.Fatal("Improper error code")
	}

	code = New(DialError, Unknown).ErrorCode
	if code != 6000 {
		t.Fatal("Improper error code")
	}

	code = New(APIClientError, AuthenticationFailure).ErrorCode
	if code != 7100 {
		t.Fatal("Improper error code")
	}
	code = New(APIClientError, JSONError).ErrorCode
	if code != 7200 {
		t.Fatal("Improper error code")
	}
	code = New(APIClientError, ClientHTTPError).ErrorCode
	if code != 7400 {
		t.Fatal("Improper error code")
	}
	code = New(APIClientError, IOError).ErrorCode
	if code != 7300 {
		t.Fatal("Improper error code")
	}
	code = New(APIClientError, ServerRequestFailed).ErrorCode
	if code != 7500 {
		t.Fatal("Improper error code")
	}

	code = New(CSRError, Unknown).ErrorCode
	if code != 9000 {
		t.Fatal("Improper error code")
	}
	code = New(CSRError, ReadFailed).ErrorCode
	if code != 9001 {
		t.Fatal("Improper error code")
	}
	code = New(CSRError, DecodeFailed).ErrorCode
	if code != 9002 {
		t.Fatal("Improper error code")
	}
	code = New(CSRError, ParseFailed).ErrorCode
	if code != 9003 {
		t.Fatal("Improper error code")
	}
	code = New(CSRError, KeyMismatch).ErrorCode
	if code != 9300 {
		t.Fatal("Improper error code")
	}
	code = New(CSRError, BadRequest).ErrorCode
	if code != 9300 {
		t.Fatal("Improper error code")
	}
}

func TestWrap(t *testing.T) {
	msg := "Arbitrary error message"
	err := Wrap(CertificateError, Unknown, errors.New(msg))
	if err == nil {
		t.Fatal("Error creation failed.")
	}
	if err.ErrorCode != int(CertificateError)+int(Unknown) {
		t.Fatal("Error code construction failed.")
	}
	if err.Message != msg {
		t.Fatal("Error message construction failed.")
	}

	err = Wrap(CertificateError, VerifyFailed, x509.CertificateInvalidError{Reason: x509.Expired})
	if err == nil {
		t.Fatal("Error creation failed.")
	}
	if err.ErrorCode != int(CertificateError)+int(VerifyFailed)+certificateInvalid+int(x509.Expired) {
		t.Fatal("Error code construction failed.")
	}
	if err.Message != "x509: certificate has expired or is not yet valid" {
		t.Fatal("Error message construction failed.")
	}
}

func TestMarshal(t *testing.T) {
	msg := "Arbitrary error message"
	err := Wrap(CertificateError, Unknown, errors.New(msg))
	bytes, _ := json.Marshal(err)
	var received Error
	json.Unmarshal(bytes, &received)
	if received.ErrorCode != int(CertificateError)+int(Unknown) {
		t.Fatal("Error code construction failed.")
	}
	if received.Message != msg {
		t.Fatal("Error message construction failed.")
	}
}

func TestErrorString(t *testing.T) {
	msg := "Arbitrary error message"
	err := Wrap(CertificateError, Unknown, errors.New(msg))
	str := err.Error()
	if str != `{"code":1000,"message":"`+msg+`"}` {
		t.Fatal("Incorrect Error():", str)
	}
}
<commit_msg>Added MissingSerial CertificateError test<commit_after>package errors

import (
	"crypto/x509"
	"encoding/json"
	"errors"
	"testing"
)

func TestNew(t *testing.T) {
	err := New(CertificateError, Unknown)
	if err == nil {
		t.Fatal("Error creation failed.")
	}
	if err.ErrorCode != int(CertificateError)+int(Unknown) {
		t.Fatal("Error code construction failed.")
	}
	if err.Message != "Unknown certificate error" {
		t.Fatal("Error message construction failed.")
	}

	code := New(OCSPError, ReadFailed).ErrorCode
	if code != 8001 {
		t.Fatal("Improper error code")
	}

	code = New(OCSPError, IssuerMismatch).ErrorCode
	if code != 8100 {
		t.Fatal("Improper error code")
	}

	code = New(OCSPError, InvalidStatus).ErrorCode
	if code != 8200 {
		t.Fatal("Improper error code")
	}

	code = New(CertificateError, Unknown).ErrorCode
	if code != 1000 {
		t.Fatal("Improper error code")
	}
	code = New(CertificateError, ReadFailed).ErrorCode
	if code != 1001 {
		t.Fatal("Improper error code")
	}
	code = New(CertificateError, DecodeFailed).ErrorCode
	if code != 1002 {
		t.Fatal("Improper error code")
	}
	code = New(CertificateError, ParseFailed).ErrorCode
	if code != 1003 {
		t.Fatal("Improper error code")
	}
	code = New(CertificateError, SelfSigned).ErrorCode
	if code != 1100 {
		t.Fatal("Improper error code")
	}
	code = New(CertificateError, VerifyFailed).ErrorCode
	if code != 1200 {
		t.Fatal("Improper error code")
	}
	code = New(CertificateError, BadRequest).ErrorCode
	if code != 1300 {
		t.Fatal("Improper error code")
	}
	code = New(CertificateError, MissingSerial).ErrorCode
	if code != 1400 {
		t.Fatal("Improper error code")
	}

	code = New(PrivateKeyError, Unknown).ErrorCode
	if code != 2000 {
		t.Fatal("Improper error code")
	}
	code = New(PrivateKeyError, ReadFailed).ErrorCode
	if code != 2001 {
		t.Fatal("Improper error code")
	}
	code = New(PrivateKeyError, DecodeFailed).ErrorCode
	if code != 2002 {
		t.Fatal("Improper error code")
	}
	code = New(PrivateKeyError, ParseFailed).ErrorCode
	if code != 2003 {
		t.Fatal("Improper error code")
	}
	code = New(PrivateKeyError, Encrypted).ErrorCode
	if code != 2100 {
		t.Fatal("Improper error code")
	}
	code = New(PrivateKeyError, NotRSAOrECC).ErrorCode
	if code != 2200 {
		t.Fatal("Improper error code")
	}
	code = New(PrivateKeyError, KeyMismatch).ErrorCode
	if code != 2300 {
		t.Fatal("Improper error code")
	}
	code = New(PrivateKeyError, GenerationFailed).ErrorCode
	if code != 2400 {
		t.Fatal("Improper error code")
	}
	code = New(PrivateKeyError, Unavailable).ErrorCode
	if code != 2500 {
		t.Fatal("Improper error code")
	}

	code = New(IntermediatesError, Unknown).ErrorCode
	if code != 3000 {
		t.Fatal("Improper error code")
	}
	code = New(IntermediatesError, ReadFailed).ErrorCode
	if code != 3001 {
		t.Fatal("Improper error code")
	}
	code = New(IntermediatesError, DecodeFailed).ErrorCode
	if code != 3002 {
		t.Fatal("Improper error code")
	}
	code = New(IntermediatesError, ParseFailed).ErrorCode
	if code != 3003 {
		t.Fatal("Improper error code")
	}

	code = New(RootError, Unknown).ErrorCode
	if code != 4000 {
		t.Fatal("Improper error code")
	}
	code = New(RootError, ReadFailed).ErrorCode
	if code != 4001 {
		t.Fatal("Improper error code")
	}
	code = New(RootError, DecodeFailed).ErrorCode
	if code != 4002 {
		t.Fatal("Improper error code")
	}
	code = New(RootError, ParseFailed).ErrorCode
	if code != 4003 {
		t.Fatal("Improper error code")
	}

	code = New(PolicyError, Unknown).ErrorCode
	if code != 5000 {
		t.Fatal("Improper error code")
	}
	code = New(PolicyError, NoKeyUsages).ErrorCode
	if code != 5100 {
		t.Fatal("Improper error code")
	}
	code = New(PolicyError, InvalidPolicy).ErrorCode
	if code != 5200 {
		t.Fatal("Improper error code")
	}
	code = New(PolicyError, InvalidRequest).ErrorCode
	if code != 5300 {
		t.Fatal("Improper error code")
	}

	code = New(DialError, Unknown).ErrorCode
	if code != 6000 {
		t.Fatal("Improper error code")
	}

	code = New(APIClientError, AuthenticationFailure).ErrorCode
	if code != 7100 {
		t.Fatal("Improper error code")
	}
	code = New(APIClientError, JSONError).ErrorCode
	if code != 7200 {
		t.Fatal("Improper error code")
	}
	code = New(APIClientError, ClientHTTPError).ErrorCode
	if code != 7400 {
		t.Fatal("Improper error code")
	}
	code = New(APIClientError, IOError).ErrorCode
	if code != 7300 {
		t.Fatal("Improper error code")
	}
	code = New(APIClientError, ServerRequestFailed).ErrorCode
	if code != 7500 {
		t.Fatal("Improper error code")
	}

	code = New(CSRError, Unknown).ErrorCode
	if code != 9000 {
		t.Fatal("Improper error code")
	}
	code = New(CSRError, ReadFailed).ErrorCode
	if code != 9001 {
		t.Fatal("Improper error code")
	}
	code = New(CSRError, DecodeFailed).ErrorCode
	if code != 9002 {
		t.Fatal("Improper error code")
	}
	code = New(CSRError, ParseFailed).ErrorCode
	if code != 9003 {
		t.Fatal("Improper error code")
	}
	code = New(CSRError, KeyMismatch).ErrorCode
	if code != 9300 {
		t.Fatal("Improper error code")
	}
	code = New(CSRError, BadRequest).ErrorCode
	if code != 9300 {
		t.Fatal("Improper error code")
	}
}

func TestWrap(t *testing.T) {
	msg := "Arbitrary error message"
	err := Wrap(CertificateError, Unknown, errors.New(msg))
	if err == nil {
		t.Fatal("Error creation failed.")
	}
	if err.ErrorCode != int(CertificateError)+int(Unknown) {
		t.Fatal("Error code construction failed.")
	}
	if err.Message != msg {
		t.Fatal("Error message construction failed.")
	}

	err = Wrap(CertificateError, VerifyFailed, x509.CertificateInvalidError{Reason: x509.Expired})
	if err == nil {
		t.Fatal("Error creation failed.")
	}
	if err.ErrorCode != int(CertificateError)+int(VerifyFailed)+certificateInvalid+int(x509.Expired) {
		t.Fatal("Error code construction failed.")
	}
	if err.Message != "x509: certificate has expired or is not yet valid" {
		t.Fatal("Error message construction failed.")
	}
}

func TestMarshal(t *testing.T) {
	msg := "Arbitrary error message"
	err := Wrap(CertificateError, Unknown, errors.New(msg))
	bytes, _ := json.Marshal(err)
	var received Error
	json.Unmarshal(bytes, &received)
	if received.ErrorCode != int(CertificateError)+int(Unknown) {
		t.Fatal("Error code construction failed.")
	}
	if received.Message != msg {
		t.Fatal("Error message construction failed.")
	}
}

func TestErrorString(t *testing.T) {
	msg := "Arbitrary error message"
	err := Wrap(CertificateError, Unknown, errors.New(msg))
	str := err.Error()
	if str != `{"code":1000,"message":"`+msg+`"}` {
		t.Fatal("Incorrect Error():", str)
	}
}
<|endoftext|>


<commit_before>package wats

import (
	"encoding/json"
	"strings"

	"github.com/cloudfoundry-incubator/cf-test-helpers/helpers"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("File ACLs", func() {

	var inaccessibleFiles = []string{
		"C:\\bosh",
		"C:\\containerizer",
		"C:\\var",
		"C:\\Windows\\System32\\Sysprep\\Panther\\IE\\setupact.log",
		"C:\\Windows\\System32\\Sysprep\\Panther\\IE\\setuperr.log",
		"C:\\Windows\\System32\\Sysprep\\Panther\\setupact.log",
		"C:\\Windows\\System32\\Sysprep\\Panther\\setuperr.log",
	}

	BeforeEach(func() {
		pushAndStartNora(appName)
		Eventually(helpers.CurlingAppRoot(appName)).Should(ContainSubstring("hello i am nora"))

		// Windows file paths are case-insensitive
		for i, s := range inaccessibleFiles {
			inaccessibleFiles[i] = strings.ToLower(s)
		}
	})

	It("A Container user should not have permission to view sensitive files", func() {

		response := helpers.CurlApp(appName, "/inaccessible_files")
		var files []string
		Expect(json.Unmarshal([]byte(strings.ToLower(response)), &files)).To(Succeed())
		for _, inaccessibleFile := range inaccessibleFiles {
			Expect(files).To(ContainElement(inaccessibleFile))
		}
	})
})
<commit_msg>Bump timeout for walking entire filesystem to 5 minutes<commit_after>package wats

import (
	"encoding/json"
	"strings"
	"time"

	"github.com/cloudfoundry-incubator/cf-test-helpers/helpers"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("File ACLs", func() {

	var inaccessibleFiles = []string{
		"C:\\bosh",
		"C:\\containerizer",
		"C:\\var",
		"C:\\Windows\\System32\\Sysprep\\Panther\\IE\\setupact.log",
		"C:\\Windows\\System32\\Sysprep\\Panther\\IE\\setuperr.log",
		"C:\\Windows\\System32\\Sysprep\\Panther\\setupact.log",
		"C:\\Windows\\System32\\Sysprep\\Panther\\setuperr.log",
	}

	BeforeEach(func() {
		pushAndStartNora(appName)
		Eventually(helpers.CurlingAppRoot(appName)).Should(ContainSubstring("hello i am nora"))

		// Windows file paths are case-insensitive
		for i, s := range inaccessibleFiles {
			inaccessibleFiles[i] = strings.ToLower(s)
		}
	})

	It("A Container user should not have permission to view sensitive files", func() {

		// The 'inaccessible_files' endpoint walks the
		// entire filesystem, which may take a while.
		response := helpers.CurlAppWithTimeout(appName, "/inaccessible_files", 5*time.Minute)
		var files []string
		Expect(json.Unmarshal([]byte(strings.ToLower(response)), &files)).To(Succeed())
		for _, inaccessibleFile := range inaccessibleFiles {
			Expect(files).To(ContainElement(inaccessibleFile))
		}
	})
})
<|endoftext|>