<commit_before>package pruneCmd

import (
	// Stdlib
	"errors"
	"fmt"
	"os"
	"strings"

	// Internal
	"github.com/salsaflow/salsaflow/app"
	"github.com/salsaflow/salsaflow/app/appflags"
	"github.com/salsaflow/salsaflow/errs"
	"github.com/salsaflow/salsaflow/flag"
	"github.com/salsaflow/salsaflow/git"
	"github.com/salsaflow/salsaflow/log"
	"github.com/salsaflow/salsaflow/modules"
	"github.com/salsaflow/salsaflow/modules/common"
	"github.com/salsaflow/salsaflow/prompt"

	// Other
	"gopkg.in/tchap/gocli.v2"
)

const StoryBranchPrefix = "story/"

var stateEnum = []common.StoryState{
	common.StoryStateTested,
	common.StoryStateStaged,
	common.StoryStateAccepted,
	common.StoryStateClosed,
}

var Command = &gocli.Command{
	// UsageLine set below in an init() functions.
	Short: "delete branches that are not needed",
	Long: `
  Delete Git branches that are no longer needed.

  All story branches are checked and the branches that only contain commits
  associated with stories that are in the selected state or further
  are offered to be deleted. Both local and remote branches are affected.

  This commands counts on the fact that all branches starting with story/
  are forked off trunk. In case this is not met, weird things can happen.
	`,
	Action: run,
}

var flagState *flag.StringEnumFlag

func init() {
	states := make([]string, 0, len(stateEnum))
	for _, state := range stateEnum {
		states = append(states, string(state))
	}

	// Finalize Command.
	Command.UsageLine = fmt.Sprintf("prune [-state={%v}]", strings.Join(states, "|"))

	// Register flags.
	flagState = flag.NewStringEnumFlag(states, string(common.StoryStateAccepted))
	Command.Flags.Var(flagState, "state", "set the required story state for branch removal")

	// Register global flags.
	appflags.RegisterGlobalFlags(&Command.Flags)
}

func allowedStoryStates() map[common.StoryState]struct{} {
	var enum []common.StoryState
	v := common.StoryState(flagState.Value())
	switch v {
	case common.StoryStateTested:
		enum = stateEnum
	case common.StoryStateStaged:
		enum = stateEnum[1:]
	case common.StoryStateAccepted:
		enum = stateEnum[2:]
	case common.StoryStateClosed:
		enum = stateEnum[3:]
	default:
		panic("unknown state: " + v)
	}

	m := make(map[common.StoryState]struct{}, len(enum))
	for _, state := range enum {
		m[state] = struct{}{}
	}
	return m
}

func run(cmd *gocli.Command, args []string) {
	if len(args) != 0 {
		cmd.Usage()
		os.Exit(2)
	}

	app.InitOrDie()

	if err := runMain(); err != nil {
		errs.Fatal(err)
	}
}

func runMain() error {
	// Load config.
	config, err := git.LoadConfig()
	if err != nil {
		return err
	}
	var (
		remoteName = config.RemoteName()
		trunkName  = config.TrunkBranchName()
	)

	// Make sure trunk is up to date.
	task := fmt.Sprintf("Make sure branch '%v' is up to date", trunkName)
	log.Run(task)
	if err := git.CheckOrCreateTrackingBranch(trunkName, remoteName); err != nil {
		return errs.NewError(task, err)
	}

	// Collect the story branches.
	task = "Collect the story branches"
	log.Run(task)
	storyBranches, err := collectStoryBranches(remoteName)
	if err != nil {
		return errs.NewError(task, err)
	}

	// Split the branches that are not up to date.
	task = "Split the branches that are not up to date"
	log.Run(task)
	storyBranches, err = splitBranchesNotInSync(storyBranches)
	if err != nil {
		return errs.NewError(task, err)
	}

	// Filter branches according to the story state.
	task = "Filter branches according to the story state"
	log.Run(task)
	storyBranches, err = filterBranches(storyBranches, trunkName)
	if err != nil {
		return errs.NewError(task, err)
	}
	if len(storyBranches) == 0 {
		log.Log("No branches left to be deleted")
		return nil
	}

	// Prompt the user to choose what branches to delete.
	task = "Prompt the user to choose what branches to delete"
	localToDelete, remoteToDelete, err := promptUserToChooseBranches(storyBranches)
	if err != nil {
		return errs.NewError(task, err)
	}

	// Delete chosen local branches.
	if len(localToDelete) != 0 {
		task := "Delete chosen local branches"
		log.Run(task)
		args := make([]string, 1, 1+len(localToDelete))
		args[0] = "-D"
		args = append(args, localToDelete...)
		if ex := git.Branch(args...); ex != nil {
			errs.LogError(task, ex)
			err = errors.New("failed to delete local branches")
		}
	}

	// Delete chosen remote branches.
	if len(remoteToDelete) != 0 {
		task := "Delete chosen remote story branches"
		log.Run(task)
		args := make([]string, 1, 1+len(remoteToDelete))
		args[0] = "--delete"
		args = append(args, remoteToDelete...)
		if ex := git.Push(remoteName, args...); ex != nil {
			errs.LogError(task, ex)
			err = errors.New("failed to delete remote branches")
		}
	}

	return err
}

func collectStoryBranches(remoteName string) ([]*git.GitBranch, error) {
	// Load Git branches.
	branches, err := git.Branches()
	if err != nil {
		return nil, err
	}

	// Filter the branches.
	storyBranches := make([]*git.GitBranch, 0, len(branches))
	for _, branch := range branches {
		// Drop branches not corresponding to the project remote.
		if branch.Remote != "" && branch.Remote != remoteName {
			continue
		}

		// Keep the story branches only.
		isLocalStoryBranch := strings.HasPrefix(branch.BranchName, StoryBranchPrefix)
		isRemoteStoryBranch := strings.HasPrefix(branch.RemoteBranchName, StoryBranchPrefix)
		if isLocalStoryBranch || isRemoteStoryBranch {
			storyBranches = append(storyBranches, branch)
		}
	}

	// Return the result.
	return storyBranches, nil
}

func splitBranchesNotInSync(storyBranches []*git.GitBranch) ([]*git.GitBranch, error) {
	branches := make([]*git.GitBranch, 0, len(storyBranches))
	for _, branch := range storyBranches {
		upToDate, err := branch.IsUpToDate()
		if err != nil {
			return nil, err
		}
		if upToDate {
			branches = append(branches, branch)
			continue
		}

		// In case the branch is not up to date, we split the local and remote
		// reference into their own branch records to treat them separately.
		var (
			branchName       = branch.BranchName
			remoteBranchName = branch.RemoteBranchName
			remote           = branch.Remote
		)
		log.Warn(fmt.Sprintf("Branch '%s' is not up to date", branchName))
		log.NewLine(fmt.Sprintf("Treating '%v' and '%v/%v' as separate branches",
			branchName, remote, remoteBranchName))

		localBranch := &git.GitBranch{
			BranchName: branchName,
		}
		remoteBranch := &git.GitBranch{
			RemoteBranchName: remoteBranchName,
			Remote:           remote,
		}
		branches = append(branches, localBranch, remoteBranch)
	}
	return branches, nil
}

type gitBranch struct {
	tip     *git.GitBranch
	commits []*git.Commit
}

func filterBranches(storyBranches []*git.GitBranch, trunkName string) ([]*git.GitBranch, error) {
	// Pair the branches with commit ranges specified by trunk..story
	task := "Collected commits associated with the story branches"
	branches := make([]*gitBranch, 0, len(storyBranches))
	for _, branch := range storyBranches {
		var revRange string
		if branch.BranchName != "" {
			// Handle branches that exist locally.
			revRange = fmt.Sprintf("%v..%v", trunkName, branch.BranchName)
		} else {
			// Handle branches that exist only in the remote repository.
			// We can use trunkName here since trunk is up to date.
			revRange = fmt.Sprintf("%v..%v/%v", trunkName, branch.Remote, branch.RemoteBranchName)
		}

		commits, err := git.ShowCommitRange(revRange)
		if err != nil {
			return nil, errs.NewError(task, err)
		}
		branches = append(branches, &gitBranch{
			tip:     branch,
			commits: commits,
		})
		continue
	}

	// Collect story tags.
	task = "Collect affected story tags"
	tracker, err := modules.GetIssueTracker()
	if err != nil {
		return nil, errs.NewError(task, err)
	}

	tags := make([]string, 0, len(storyBranches))
BranchLoop:
	for _, branch := range branches {
		for _, commit := range branch.commits {
			commitTag := commit.StoryIdTag

			// Make sure the tag is not in the list already.
			for _, tag := range tags {
				if tag == commitTag {
					continue BranchLoop
				}
			}

			// Drop tags not recognized by the current issue tracker.
			_, err := tracker.StoryTagToReadableStoryId(commitTag)
			if err == nil {
				tags = append(tags, commitTag)
			}
		}
	}

	// Fetch the collected stories.
	task = "Fetch affected stories from the issue tracker"
	log.Run(task)
	stories, err := tracker.ListStoriesByTag(tags)
	if err != nil {
		return nil, errs.NewError(task, err)
	}

	// Filter the branches according to the story state.
	storyByTag := make(map[string]common.Story, len(stories))
	for i, story := range stories {
		// tag[i] corresponds to stories[i]
		storyByTag[tags[i]] = story
	}

	allowedStates := allowedStoryStates()

	// checkCommits returns whether the commits passed in are ok
	// considering the state of the stories found in these commits.
	// In case there is a state that is not allowed detected,
	// the state is returned from the function as well.
	checkCommits := func(commits []*git.Commit) (common.StoryState, bool) {
		for _, commit := range commits {
			// In case the story is not found, the tag is not recognized
			// by the current issue tracker. In that case we just skip the commit.
			story, ok := storyByTag[commit.StoryIdTag]
			if !ok {
				continue
			}

			// When the story state associated with the commit is not ok,
			// we can return false here to reject the branch.
			storyState := story.State()
			if _, ok := allowedStates[storyState]; !ok {
				return storyState, false
			}
		}

		// We went through all the commits and they are fine, check passed.
		return common.StoryStateInvalid, true
	}

	// Go through the branches and only return these that
	// comply with the story state requirements.
	bs := make([]*git.GitBranch, 0, len(branches))
	for _, branch := range branches {
		tip := branch.tip

		// In case the commit check passed, we append the branch.
		state, ok := checkCommits(branch.commits)
		if ok {
			bs = append(bs, tip)
			continue
		}

		// Otherwise we print the skip warning.
		log.V(log.Debug).Log(fmt.Sprintf(
			"Skipping branch '%v', story state is '%v'", tip.CanonicalName(), state))
	}

	return bs, nil
}

func promptUserToChooseBranches(branches []*git.GitBranch) (local, remote []string, err error) {
	// Go through the branches and ask the user for confirmation.
	var (
		localToDelete  = make([]string, 0, len(branches))
		remoteToDelete = make([]string, 0, len(branches))
	)

	fmt.Println()
	defer fmt.Println()

	for _, branch := range branches {
		isLocal := branch.BranchName != ""
		isRemote := branch.RemoteBranchName != ""

		var msg string
		switch {
		case isLocal && isRemote:
			msg = fmt.Sprintf("Delete local branch '%v' and its remote counterpart?", branch.BranchName)
		case isLocal:
			msg = fmt.Sprintf("Delete local branch '%v'?", branch.BranchName)
		case isRemote:
			msg = fmt.Sprintf("Delete remote branch '%v'?", branch.FullRemoteBranchName())
		default:
			panic("bullshit")
		}

		confirmed, err := prompt.Confirm(msg, false)
		if err != nil {
			return nil, nil, err
		}
		if !confirmed {
			continue
		}

		if isLocal {
			localToDelete = append(localToDelete, branch.BranchName)
		}
		if isRemote {
			remoteToDelete = append(remoteToDelete, branch.RemoteBranchName)
		}
	}
	return localToDelete, remoteToDelete, nil
}
<commit_msg>repo prune: Exclude the current branch<commit_after>package pruneCmd

import (
	// Stdlib
	"errors"
	"fmt"
	"os"
	"strings"

	// Internal
	"github.com/salsaflow/salsaflow/app"
	"github.com/salsaflow/salsaflow/app/appflags"
	"github.com/salsaflow/salsaflow/errs"
	"github.com/salsaflow/salsaflow/flag"
	"github.com/salsaflow/salsaflow/git"
	"github.com/salsaflow/salsaflow/log"
	"github.com/salsaflow/salsaflow/modules"
	"github.com/salsaflow/salsaflow/modules/common"
	"github.com/salsaflow/salsaflow/prompt"

	// Other
	"gopkg.in/tchap/gocli.v2"
)

const StoryBranchPrefix = "story/"

var stateEnum = []common.StoryState{
	common.StoryStateTested,
	common.StoryStateStaged,
	common.StoryStateAccepted,
	common.StoryStateClosed,
}

var Command = &gocli.Command{
	// UsageLine set below in an init() functions.
	Short: "delete branches that are not needed",
	Long: `
  Delete Git branches that are no longer needed.

  All story branches are checked and the branches that only contain commits
  associated with stories that are in the selected state or further
  are offered to be deleted. Both local and remote branches are affected.

  This commands counts on the fact that all branches starting with story/
  are forked off trunk. In case this is not met, weird things can happen.
	`,
	Action: run,
}

var flagState *flag.StringEnumFlag

func init() {
	states := make([]string, 0, len(stateEnum))
	for _, state := range stateEnum {
		states = append(states, string(state))
	}

	// Finalize Command.
	Command.UsageLine = fmt.Sprintf("prune [-state={%v}]", strings.Join(states, "|"))

	// Register flags.
	flagState = flag.NewStringEnumFlag(states, string(common.StoryStateAccepted))
	Command.Flags.Var(flagState, "state", "set the required story state for branch removal")

	// Register global flags.
	appflags.RegisterGlobalFlags(&Command.Flags)
}

func allowedStoryStates() map[common.StoryState]struct{} {
	var enum []common.StoryState
	v := common.StoryState(flagState.Value())
	switch v {
	case common.StoryStateTested:
		enum = stateEnum
	case common.StoryStateStaged:
		enum = stateEnum[1:]
	case common.StoryStateAccepted:
		enum = stateEnum[2:]
	case common.StoryStateClosed:
		enum = stateEnum[3:]
	default:
		panic("unknown state: " + v)
	}

	m := make(map[common.StoryState]struct{}, len(enum))
	for _, state := range enum {
		m[state] = struct{}{}
	}
	return m
}

func run(cmd *gocli.Command, args []string) {
	if len(args) != 0 {
		cmd.Usage()
		os.Exit(2)
	}

	app.InitOrDie()

	if err := runMain(); err != nil {
		errs.Fatal(err)
	}
}

func runMain() error {
	// Load config.
	config, err := git.LoadConfig()
	if err != nil {
		return err
	}
	var (
		remoteName = config.RemoteName()
		trunkName  = config.TrunkBranchName()
	)

	// Make sure trunk is up to date.
	task := fmt.Sprintf("Make sure branch '%v' is up to date", trunkName)
	log.Run(task)
	if err := git.CheckOrCreateTrackingBranch(trunkName, remoteName); err != nil {
		return errs.NewError(task, err)
	}

	// Collect the story branches.
	task = "Collect the story branches"
	log.Run(task)
	storyBranches, err := collectStoryBranches(remoteName)
	if err != nil {
		return errs.NewError(task, err)
	}

	// Split the branches that are not up to date.
	task = "Split the branches that are not up to date"
	log.Run(task)
	storyBranches, err = splitBranchesNotInSync(storyBranches)
	if err != nil {
		return errs.NewError(task, err)
	}

	// Filter branches according to the story state.
	task = "Filter branches according to the story state"
	log.Run(task)
	storyBranches, err = filterBranches(storyBranches, trunkName)
	if err != nil {
		return errs.NewError(task, err)
	}
	if len(storyBranches) == 0 {
		log.Log("No branches left to be deleted")
		return nil
	}

	// Prompt the user to choose what branches to delete.
	task = "Prompt the user to choose what branches to delete"
	localToDelete, remoteToDelete, err := promptUserToChooseBranches(storyBranches)
	if err != nil {
		return errs.NewError(task, err)
	}

	// Delete chosen local branches.
	if len(localToDelete) != 0 {
		task := "Delete chosen local branches"
		log.Run(task)
		args := make([]string, 1, 1+len(localToDelete))
		args[0] = "-D"
		args = append(args, localToDelete...)
		if ex := git.Branch(args...); ex != nil {
			errs.LogError(task, ex)
			err = errors.New("failed to delete local branches")
		}
	}

	// Delete chosen remote branches.
	if len(remoteToDelete) != 0 {
		task := "Delete chosen remote story branches"
		log.Run(task)
		args := make([]string, 1, 1+len(remoteToDelete))
		args[0] = "--delete"
		args = append(args, remoteToDelete...)
		if ex := git.Push(remoteName, args...); ex != nil {
			errs.LogError(task, ex)
			err = errors.New("failed to delete remote branches")
		}
	}

	return err
}

func collectStoryBranches(remoteName string) ([]*git.GitBranch, error) {
	// Load Git branches.
	branches, err := git.Branches()
	if err != nil {
		return nil, err
	}

	// Get the current branch name so that it can be excluded.
	currentBranch, err := git.CurrentBranch()
	if err != nil {
		return nil, err
	}

	// Filter the branches.
	storyBranches := make([]*git.GitBranch, 0, len(branches))
	for _, branch := range branches {
		// Drop branches not corresponding to the project remote.
		if branch.Remote != "" && branch.Remote != remoteName {
			continue
		}

		// Exclude the current branch.
		if branch.BranchName == currentBranch {
			log.Warn(fmt.Sprintf("Branch '%v' is checked out, it cannot be deleted", currentBranch))
			continue
		}

		// Keep the story branches only.
		isLocalStoryBranch := strings.HasPrefix(branch.BranchName, StoryBranchPrefix)
		isRemoteStoryBranch := strings.HasPrefix(branch.RemoteBranchName, StoryBranchPrefix)
		if isLocalStoryBranch || isRemoteStoryBranch {
			storyBranches = append(storyBranches, branch)
		}
	}

	// Return the result.
	return storyBranches, nil
}

func splitBranchesNotInSync(storyBranches []*git.GitBranch) ([]*git.GitBranch, error) {
	branches := make([]*git.GitBranch, 0, len(storyBranches))
	for _, branch := range storyBranches {
		upToDate, err := branch.IsUpToDate()
		if err != nil {
			return nil, err
		}
		if upToDate {
			branches = append(branches, branch)
			continue
		}

		// In case the branch is not up to date, we split the local and remote
		// reference into their own branch records to treat them separately.
		var (
			branchName       = branch.BranchName
			remoteBranchName = branch.RemoteBranchName
			remote           = branch.Remote
		)
		log.Warn(fmt.Sprintf("Branch '%s' is not up to date", branchName))
		log.NewLine(fmt.Sprintf("Treating '%v' and '%v/%v' as separate branches",
			branchName, remote, remoteBranchName))

		localBranch := &git.GitBranch{
			BranchName: branchName,
		}
		remoteBranch := &git.GitBranch{
			RemoteBranchName: remoteBranchName,
			Remote:           remote,
		}
		branches = append(branches, localBranch, remoteBranch)
	}
	return branches, nil
}

type gitBranch struct {
	tip     *git.GitBranch
	commits []*git.Commit
}

func filterBranches(storyBranches []*git.GitBranch, trunkName string) ([]*git.GitBranch, error) {
	// Pair the branches with commit ranges specified by trunk..story
	task := "Collected commits associated with the story branches"
	branches := make([]*gitBranch, 0, len(storyBranches))
	for _, branch := range storyBranches {
		var revRange string
		if branch.BranchName != "" {
			// Handle branches that exist locally.
			revRange = fmt.Sprintf("%v..%v", trunkName, branch.BranchName)
		} else {
			// Handle branches that exist only in the remote repository.
			// We can use trunkName here since trunk is up to date.
			revRange = fmt.Sprintf("%v..%v/%v", trunkName, branch.Remote, branch.RemoteBranchName)
		}

		commits, err := git.ShowCommitRange(revRange)
		if err != nil {
			return nil, errs.NewError(task, err)
		}
		branches = append(branches, &gitBranch{
			tip:     branch,
			commits: commits,
		})
		continue
	}

	// Collect story tags.
	task = "Collect affected story tags"
	tracker, err := modules.GetIssueTracker()
	if err != nil {
		return nil, errs.NewError(task, err)
	}

	tags := make([]string, 0, len(storyBranches))
BranchLoop:
	for _, branch := range branches {
		for _, commit := range branch.commits {
			commitTag := commit.StoryIdTag

			// Make sure the tag is not in the list already.
			for _, tag := range tags {
				if tag == commitTag {
					continue BranchLoop
				}
			}

			// Drop tags not recognized by the current issue tracker.
			_, err := tracker.StoryTagToReadableStoryId(commitTag)
			if err == nil {
				tags = append(tags, commitTag)
			}
		}
	}

	// Fetch the collected stories.
	task = "Fetch affected stories from the issue tracker"
	log.Run(task)
	stories, err := tracker.ListStoriesByTag(tags)
	if err != nil {
		return nil, errs.NewError(task, err)
	}

	// Filter the branches according to the story state.
	storyByTag := make(map[string]common.Story, len(stories))
	for i, story := range stories {
		// tag[i] corresponds to stories[i]
		storyByTag[tags[i]] = story
	}

	allowedStates := allowedStoryStates()

	// checkCommits returns whether the commits passed in are ok
	// considering the state of the stories found in these commits.
	// In case there is a state that is not allowed detected,
	// the state is returned from the function as well.
	checkCommits := func(commits []*git.Commit) (common.StoryState, bool) {
		for _, commit := range commits {
			// In case the story is not found, the tag is not recognized
			// by the current issue tracker. In that case we just skip the commit.
			story, ok := storyByTag[commit.StoryIdTag]
			if !ok {
				continue
			}

			// When the story state associated with the commit is not ok,
			// we can return false here to reject the branch.
			storyState := story.State()
			if _, ok := allowedStates[storyState]; !ok {
				return storyState, false
			}
		}

		// We went through all the commits and they are fine, check passed.
		return common.StoryStateInvalid, true
	}

	// Go through the branches and only return these that
	// comply with the story state requirements.
	bs := make([]*git.GitBranch, 0, len(branches))
	for _, branch := range branches {
		tip := branch.tip

		// In case the commit check passed, we append the branch.
		state, ok := checkCommits(branch.commits)
		if ok {
			bs = append(bs, tip)
			continue
		}

		// Otherwise we print the skip warning.
		log.V(log.Debug).Log(fmt.Sprintf(
			"Skipping branch '%v', story state is '%v'", tip.CanonicalName(), state))
	}

	return bs, nil
}

func promptUserToChooseBranches(branches []*git.GitBranch) (local, remote []string, err error) {
	// Go through the branches and ask the user for confirmation.
	var (
		localToDelete  = make([]string, 0, len(branches))
		remoteToDelete = make([]string, 0, len(branches))
	)

	fmt.Println()
	defer fmt.Println()

	for _, branch := range branches {
		isLocal := branch.BranchName != ""
		isRemote := branch.RemoteBranchName != ""

		var msg string
		switch {
		case isLocal && isRemote:
			msg = fmt.Sprintf("Delete local branch '%v' and its remote counterpart?", branch.BranchName)
		case isLocal:
			msg = fmt.Sprintf("Delete local branch '%v'?", branch.BranchName)
		case isRemote:
			msg = fmt.Sprintf("Delete remote branch '%v'?", branch.FullRemoteBranchName())
		default:
			panic("bullshit")
		}

		confirmed, err := prompt.Confirm(msg, false)
		if err != nil {
			return nil, nil, err
		}
		if !confirmed {
			continue
		}

		if isLocal {
			localToDelete = append(localToDelete, branch.BranchName)
		}
		if isRemote {
			remoteToDelete = append(remoteToDelete, branch.RemoteBranchName)
		}
	}
	return localToDelete, remoteToDelete, nil
}
<|endoftext|>


<commit_before>/*
Run Action

Allows to run any available command or script in the filesystem or
in build process host environment: specifically inside the fakemachine created
by Debos.

Yaml syntax:
 - action: run
   chroot: bool
   postprocess: bool
   script: script name
   command: command line
   label: string

Properties 'command' and 'script' are mutually exclusive.

- command -- command with arguments; the command expected to be accessible in
host's or chrooted environment -- depending on 'chroot' property.

- script -- script with arguments; script must be located in recipe directory.

Optional properties:

- chroot -- run script or command in target filesystem if set to true.
Otherwise the command or script is executed within the build process, with
access to the filesystem ($ROOTDIR), the image if any ($IMAGE), the
recipe directory ($RECIPEDIR), the artifact directory ($ARTIFACTDIR) and the
directory where the image is mounted ($IMAGEMNTDIR).
In both cases it is run with root privileges. If unset, chroot is set to false and
the command or script is run in the host environment.

- label -- if non-empty, this string is used to label output. If empty,
a label is derived from the command or script.

- postprocess -- if set script or command is executed after all other commands and
has access to the image file.

Properties 'chroot' and 'postprocess' are mutually exclusive.
*/
package actions

import (
	"errors"
	"github.com/go-debos/fakemachine"
	"path"
	"strings"

	"github.com/go-debos/debos"
)

type RunAction struct {
	debos.BaseAction `yaml:",inline"`
	Chroot           bool
	PostProcess      bool
	Script           string
	Command          string
	Label            string
}

func (run *RunAction) Verify(context *debos.DebosContext) error {
	if run.PostProcess && run.Chroot {
		return errors.New("Cannot run postprocessing in the chroot")
	}

	if run.Script == "" && run.Command == "" {
		return errors.New("Script and Command both cannot be empty")
	}
	return nil
}

func (run *RunAction) PreMachine(context *debos.DebosContext, m *fakemachine.Machine,
	args *[]string) error {

	if run.Script == "" {
		return nil
	}

	run.Script = debos.CleanPathAt(run.Script, context.RecipeDir)
	// Expect we have no blank spaces in path
	scriptpath := strings.Split(run.Script, " ")

	if !run.PostProcess {
		m.AddVolume(path.Dir(scriptpath[0]))
	}

	return nil
}

func (run *RunAction) doRun(context debos.DebosContext) error {
	run.LogStart()
	var cmdline []string
	var label string
	var cmd debos.Command

	if run.Chroot {
		cmd = debos.NewChrootCommandForContext(context)
	} else {
		cmd = debos.Command{}
	}

	if run.Script != "" {
		script := strings.SplitN(run.Script, " ", 2)
		script[0] = debos.CleanPathAt(script[0], context.RecipeDir)
		if run.Chroot {
			scriptpath := path.Dir(script[0])
			cmd.AddBindMount(scriptpath, "/tmp/script")
			script[0] = strings.Replace(script[0], scriptpath, "/tmp/script", 1)
		}
		cmdline = []string{strings.Join(script, " ")}
		label = path.Base(run.Script)
	} else {
		cmdline = []string{run.Command}
		label = run.Command
	}

	if run.Label != "" {
		label = run.Label
	}

	// Command/script with options passed as single string
	cmdline = append([]string{"sh", "-c"}, cmdline...)

	if !run.PostProcess {
		if !run.Chroot {
			cmd.AddEnvKey("ROOTDIR", context.Rootdir)
			cmd.AddEnvKey("RECIPEDIR", context.RecipeDir)
			cmd.AddEnvKey("ARTIFACTDIR", context.Artifactdir)
			if context.ImageMntDir != "" {
				cmd.AddEnvKey("IMAGEMNTDIR", context.ImageMntDir)
			}
		}
		if context.Image != "" {
			cmd.AddEnvKey("IMAGE", context.Image)
		}
	}

	return cmd.Run(label, cmdline...)
}

func (run *RunAction) Run(context *debos.DebosContext) error {
	if run.PostProcess {
		/* This runs in postprocessing instead */
		return nil
	}
	return run.doRun(*context)
}

func (run *RunAction) PostMachine(context *debos.DebosContext) error {
	if !run.PostProcess {
		return nil
	}
	return run.doRun(*context)
}
<commit_msg>actions: run: Expose paths to postprocess commands<commit_after>/*
Run Action

Allows to run any available command or script in the filesystem or
in build process host environment: specifically inside the fakemachine created
by Debos.

Yaml syntax:
 - action: run
   chroot: bool
   postprocess: bool
   script: script name
   command: command line
   label: string

Properties 'command' and 'script' are mutually exclusive.

- command -- command with arguments; the command expected to be accessible in
host's or chrooted environment -- depending on 'chroot' property.

- script -- script with arguments; script must be located in recipe directory.

Optional properties:

- chroot -- run script or command in target filesystem if set to true.
Otherwise the command or script is executed within the build process, with
access to the filesystem ($ROOTDIR), the image if any ($IMAGE), the
recipe directory ($RECIPEDIR), the artifact directory ($ARTIFACTDIR) and the
directory where the image is mounted ($IMAGEMNTDIR).
In both cases it is run with root privileges. If unset, chroot is set to false and
the command or script is run in the host environment.

- label -- if non-empty, this string is used to label output. If empty,
a label is derived from the command or script.

- postprocess -- if set script or command is executed after all other commands and
has access to the recipe directory ($RECIPEDIR) and the artifact directory ($ARTIFACTDIR).
The working directory will be set to the artifact directory.

Properties 'chroot' and 'postprocess' are mutually exclusive.
*/
package actions

import (
	"errors"
	"github.com/go-debos/fakemachine"
	"path"
	"strings"

	"github.com/go-debos/debos"
)

type RunAction struct {
	debos.BaseAction `yaml:",inline"`
	Chroot           bool
	PostProcess      bool
	Script           string
	Command          string
	Label            string
}

func (run *RunAction) Verify(context *debos.DebosContext) error {
	if run.PostProcess && run.Chroot {
		return errors.New("Cannot run postprocessing in the chroot")
	}

	if run.Script == "" && run.Command == "" {
		return errors.New("Script and Command both cannot be empty")
	}
	return nil
}

func (run *RunAction) PreMachine(context *debos.DebosContext, m *fakemachine.Machine,
	args *[]string) error {

	if run.Script == "" {
		return nil
	}

	run.Script = debos.CleanPathAt(run.Script, context.RecipeDir)
	// Expect we have no blank spaces in path
	scriptpath := strings.Split(run.Script, " ")

	if !run.PostProcess {
		m.AddVolume(path.Dir(scriptpath[0]))
	}

	return nil
}

func (run *RunAction) doRun(context debos.DebosContext) error {
	run.LogStart()
	var cmdline []string
	var label string
	var cmd debos.Command

	if run.Chroot {
		cmd = debos.NewChrootCommandForContext(context)
	} else {
		cmd = debos.Command{}
	}

	if run.Script != "" {
		script := strings.SplitN(run.Script, " ", 2)
		script[0] = debos.CleanPathAt(script[0], context.RecipeDir)
		if run.Chroot {
			scriptpath := path.Dir(script[0])
			cmd.AddBindMount(scriptpath, "/tmp/script")
			script[0] = strings.Replace(script[0], scriptpath, "/tmp/script", 1)
		}
		cmdline = []string{strings.Join(script, " ")}
		label = path.Base(run.Script)
	} else {
		cmdline = []string{run.Command}
		label = run.Command
	}

	if run.Label != "" {
		label = run.Label
	}

	// Command/script with options passed as single string
	cmdline = append([]string{"sh", "-c"}, cmdline...)

	if !run.Chroot {
		cmd.AddEnvKey("RECIPEDIR", context.RecipeDir)
		cmd.AddEnvKey("ARTIFACTDIR", context.Artifactdir)
	}

	if !run.PostProcess {
		if !run.Chroot {
			cmd.AddEnvKey("ROOTDIR", context.Rootdir)
			if context.ImageMntDir != "" {
				cmd.AddEnvKey("IMAGEMNTDIR", context.ImageMntDir)
			}
		}
		if context.Image != "" {
			cmd.AddEnvKey("IMAGE", context.Image)
		}
	}

	return cmd.Run(label, cmdline...)
}

func (run *RunAction) Run(context *debos.DebosContext) error {
	if run.PostProcess {
		/* This runs in postprocessing instead */
		return nil
	}
	return run.doRun(*context)
}

func (run *RunAction) PostMachine(context *debos.DebosContext) error {
	if !run.PostProcess {
		return nil
	}
	return run.doRun(*context)
}
<|endoftext|>


<commit_before>// Package router combines routing and middleware handling in a single
// package.
package router

import (
	"net/http"
	"sync"

	"github.com/husobee/vestigo"
)

// Item implements the password hashing system.
type Item struct {
	r         *vestigo.Router
	routeList []string
	listMutex sync.RWMutex
}

// Instance returns the router.
func New() *Item {
	s := new(Item)
	s.r = vestigo.NewRouter()
	return s
}

// Router returns the router.
func (s *Item) Router() *vestigo.Router {
	return s.r
}

// SetNotFound sets the 404 handler.
func (s *Item) SetNotFound(fn http.HandlerFunc) {
	vestigo.CustomNotFoundHandlerFunc(fn)
}

// SetMethodNotAllowed sets the 405 handler.
func (s *Item) SetMethodNotAllowed(fn vestigo.MethodNotAllowedHandlerFunc) {
	vestigo.CustomMethodNotAllowedHandlerFunc(fn)
}

// Param returns the URL parameter.
func (s *Item) Param(r *http.Request, name string) string {
	return vestigo.Param(r, name)
}
<commit_msg>Fix lint issue<commit_after>// Package router combines routing and middleware handling in a single
// package.
package router

import (
	"net/http"
	"sync"

	"github.com/husobee/vestigo"
)

// Item implements the password hashing system.
type Item struct {
	r         *vestigo.Router
	routeList []string
	listMutex sync.RWMutex
}

// New returns a new instance of the router.
func New() *Item {
	s := new(Item)
	s.r = vestigo.NewRouter()
	return s
}

// Router returns the router.
func (s *Item) Router() *vestigo.Router {
	return s.r
}

// SetNotFound sets the 404 handler.
func (s *Item) SetNotFound(fn http.HandlerFunc) {
	vestigo.CustomNotFoundHandlerFunc(fn)
}

// SetMethodNotAllowed sets the 405 handler.
func (s *Item) SetMethodNotAllowed(fn vestigo.MethodNotAllowedHandlerFunc) {
	vestigo.CustomMethodNotAllowedHandlerFunc(fn)
}

// Param returns the URL parameter.
func (s *Item) Param(r *http.Request, name string) string {
	return vestigo.Param(r, name)
}
<|endoftext|>


<commit_before>package configuration

import (
	"encoding/json"
	"flag"
	"io/ioutil"
	"log"

	"github.com/citruspi/milou/projects"
)

type Configuration struct {
	Mode string `json:"mode"`
	Web  struct {
		Address string `json:"address"`
	} `json:"web"`
	TravisCI struct {
		Authenticate bool   `json:"authenticate"`
		Token        string `json:"token"`
	} `json:"travis-ci"`
	Projects []struct {
		Name       string `json:"name"`
		Owner      string `json:"owner"`
		Repository string `json:"repository"`
		Version    string `json:"version"`
		Identifier string `json:"identifier"`
		Domain     string `json:"domain"`
		Subdomain  string `json:"subdomain"`
		Type       string `json:"type"`
	} `json:"projects"`
}

var (
	conf   Configuration
	loaded bool
)

func init() {
	conf := Load()

	for _, project := range conf.Projects {
		projects.List = append(projects.List, projects.Project{
			Name:       project.Name,
			Owner:      project.Owner,
			Repository: project.Version,
			Identifier: project.Identifier,
			Domain:     project.Domain,
			Subdomain:  project.Subdomain,
			Type:       project.Type,
		})
	}
}

func Load() Configuration {
	if loaded {
		return conf
	}

	var conf Configuration

	path := flag.String("config", "/etc/milou.conf", "Configuration file path")
	flag.Parse()

	source, err := ioutil.ReadFile(*path)

	if err != nil {
		log.Fatal(err)
	}

	err = json.Unmarshal(source, &conf)

	if err != nil {
		log.Fatal(err)
	}

	loaded = true

	return conf
}
<commit_msg>Moved flag parsing for the configuration package to init()<commit_after>package configuration

import (
	"encoding/json"
	"flag"
	"io/ioutil"
	"log"

	"github.com/citruspi/milou/projects"
)

type Configuration struct {
	Mode string `json:"mode"`
	Web  struct {
		Address string `json:"address"`
	} `json:"web"`
	TravisCI struct {
		Authenticate bool   `json:"authenticate"`
		Token        string `json:"token"`
	} `json:"travis-ci"`
	Projects []struct {
		Name       string `json:"name"`
		Owner      string `json:"owner"`
		Repository string `json:"repository"`
		Version    string `json:"version"`
		Identifier string `json:"identifier"`
		Domain     string `json:"domain"`
		Subdomain  string `json:"subdomain"`
		Type       string `json:"type"`
	} `json:"projects"`
}

var (
	conf   Configuration
	loaded bool
	path *string
)

func init() {
	path = flag.String("config", "/etc/milou.conf", "Configuration file path")
	flag.Parse()

	conf := Load()

	for _, project := range conf.Projects {
		projects.List = append(projects.List, projects.Project{
			Name:       project.Name,
			Owner:      project.Owner,
			Repository: project.Version,
			Identifier: project.Identifier,
			Domain:     project.Domain,
			Subdomain:  project.Subdomain,
			Type:       project.Type,
		})
	}
}

func Load() Configuration {
	if loaded {
		return conf
	}

	var conf Configuration

	source, err := ioutil.ReadFile(*path)

	if err != nil {
		log.Fatal(err)
	}

	err = json.Unmarshal(source, &conf)

	if err != nil {
		log.Fatal(err)
	}

	loaded = true

	return conf
}
<|endoftext|>


<commit_before>package agent

import (
	"errors"
	"math"
	"sync"
	"sync/atomic"

	"github.com/buildkite/agent/logger"
)

type LogStreamer struct {
	// How many log streamer workers are running at any one time
	Concurrency int

	// The maximum size of chunks
	MaxChunkSizeBytes int

	// A counter of how many chunks failed to upload
	ChunksFailedCount int32

	// The callback called when a chunk is ready for upload
	Callback func(chunk *LogStreamerChunk) error

	// The queue of chunks that are needing to be uploaded
	queue chan *LogStreamerChunk

	// Total size in bytes of the log
	bytes int

	// Each chunk is assigned an order
	order int

	// Every time we add a job to the queue, we increase the wait group
	// queue so when the streamer shuts down, we can block until all work
	// has been added.
	chunkWaitGroup sync.WaitGroup

	// Only allow processing one at a time
	processMutex sync.Mutex
}

type LogStreamerChunk struct {
	// The contents of the chunk
	Data string

	// The sequence number of this chunk
	Order int

	// The byte offset of this chunk
	Offset int

	// The byte size of this chunk
	Size int
}

// Creates a new instance of the log streamer
func (ls LogStreamer) New() *LogStreamer {
	ls.Concurrency = 3
	ls.queue = make(chan *LogStreamerChunk, 1024)

	return &ls
}

// Spins up x number of log streamer workers
func (ls *LogStreamer) Start() error {
	if ls.MaxChunkSizeBytes == 0 {
		return errors.New("Maximum chunk size must be more than 0. No logs will be sent.")
	}

	for i := 0; i < ls.Concurrency; i++ {
		go Worker(i, ls)
	}

	return nil
}

// Takes the full process output, grabs the portion we don't have, and adds it
// to the stream queue
func (ls *LogStreamer) Process(output string) error {
	bytes := len(output)

	// Only allow one streamer process at a time
	ls.processMutex.Lock()

	if ls.bytes != bytes {
		// Grab the part of the log that we haven't seen yet
		blob := output[ls.bytes:bytes]

		// How many chunks do we have that fit within the MaxChunkSizeBytes?
		numberOfChunks := int(math.Ceil(float64(len(blob)) / float64(ls.MaxChunkSizeBytes)))

		// Increase the wait group by the amount of chunks we're going
		// to add
		ls.chunkWaitGroup.Add(numberOfChunks)

		for i := 0; i < numberOfChunks; i++ {
			// Find the upper limit of the blob
			upperLimit := (i + 1) * ls.MaxChunkSizeBytes
			if upperLimit > len(blob) {
				upperLimit = len(blob)
			}

			// Grab the 100kb section of the blob
			partialChunk := blob[i*ls.MaxChunkSizeBytes : upperLimit]

			// Increment the order
			ls.order += 1

			// Create the chunk and append it to our list
			chunk := LogStreamerChunk{
				Data:   partialChunk,
				Order:  ls.order,
				Offset: ls.bytes,
				Size:   bytes - ls.bytes,
			}

			ls.queue <- &chunk
		}

		// Save the new amount of bytes
		ls.bytes = bytes
	}

	ls.processMutex.Unlock()

	return nil
}

// Waits for all the chunks to be uploaded, then shuts down all the workers
func (ls *LogStreamer) Stop() error {
	logger.Debug("[LogStreamer] Waiting for all the chunks to be uploaded")

	ls.chunkWaitGroup.Wait()

	logger.Debug("[LogStreamer] Shutting down all workers")

	for n := 0; n < ls.Concurrency; n++ {
		ls.queue <- nil
	}

	return nil
}

// The actual log streamer worker
func Worker(id int, ls *LogStreamer) {
	logger.Debug("[LogStreamer/Worker#%d] Worker is starting...", id)

	var chunk *LogStreamerChunk
	for {
		// Get the next chunk (pointer) from the queue. This will block
		// until something is returned.
		chunk = <-ls.queue

		// If the next chunk is nil, then there is no more work to do
		if chunk == nil {
			break
		}

		// Upload the chunk
		err := ls.Callback(chunk)
		if err != nil {
			atomic.AddInt32(&ls.ChunksFailedCount, 1)

			logger.Error("Giving up on uploading chunk %d, this will result in only a partial build log on Buildkite", chunk.Order)
		}

		// Signal to the chunkWaitGroup that this one is done
		ls.chunkWaitGroup.Done()
	}

	logger.Debug("[LogStreamer/Worker#%d] Worker has shutdown", id)
}
<commit_msg>Fix bug where a new chunk is created<commit_after>package agent

import (
	"errors"
	"math"
	"sync"
	"sync/atomic"

	"github.com/buildkite/agent/logger"
)

type LogStreamer struct {
	// How many log streamer workers are running at any one time
	Concurrency int

	// The maximum size of chunks
	MaxChunkSizeBytes int

	// A counter of how many chunks failed to upload
	ChunksFailedCount int32

	// The callback called when a chunk is ready for upload
	Callback func(chunk *LogStreamerChunk) error

	// The queue of chunks that are needing to be uploaded
	queue chan *LogStreamerChunk

	// Total size in bytes of the log
	bytes int

	// Each chunk is assigned an order
	order int

	// Every time we add a job to the queue, we increase the wait group
	// queue so when the streamer shuts down, we can block until all work
	// has been added.
	chunkWaitGroup sync.WaitGroup

	// Only allow processing one at a time
	processMutex sync.Mutex
}

type LogStreamerChunk struct {
	// The contents of the chunk
	Data string

	// The sequence number of this chunk
	Order int

	// The byte offset of this chunk
	Offset int

	// The byte size of this chunk
	Size int
}

// Creates a new instance of the log streamer
func (ls LogStreamer) New() *LogStreamer {
	ls.Concurrency = 3
	ls.queue = make(chan *LogStreamerChunk, 1024)

	return &ls
}

// Spins up x number of log streamer workers
func (ls *LogStreamer) Start() error {
	if ls.MaxChunkSizeBytes == 0 {
		return errors.New("Maximum chunk size must be more than 0. No logs will be sent.")
	}

	for i := 0; i < ls.Concurrency; i++ {
		go Worker(i, ls)
	}

	return nil
}

// Takes the full process output, grabs the portion we don't have, and adds it
// to the stream queue
func (ls *LogStreamer) Process(output string) error {
	bytes := len(output)

	// Only allow one streamer process at a time
	ls.processMutex.Lock()

	if ls.bytes != bytes {
		// Grab the part of the log that we haven't seen yet
		blob := output[ls.bytes:bytes]

		// How many chunks do we have that fit within the MaxChunkSizeBytes?
		numberOfChunks := int(math.Ceil(float64(len(blob)) / float64(ls.MaxChunkSizeBytes)))

		// Increase the wait group by the amount of chunks we're going
		// to add
		ls.chunkWaitGroup.Add(numberOfChunks)

		for i := 0; i < numberOfChunks; i++ {
			// Find the upper limit of the blob
			upperLimit := (i + 1) * ls.MaxChunkSizeBytes
			if upperLimit > len(blob) {
				upperLimit = len(blob)
			}

			// Grab the 100kb section of the blob
			partialChunk := blob[i*ls.MaxChunkSizeBytes : upperLimit]

			// Increment the order
			ls.order += 1

			// Create the chunk and append it to our list
			chunk := LogStreamerChunk{
				Data:   partialChunk,
				Order:  ls.order,
				Offset: ls.bytes,
				Size:   len(partialChunk),
			}

			ls.queue <- &chunk

			// Save the new amount of bytes
			ls.bytes += len(partialChunk)
		}
	}

	ls.processMutex.Unlock()

	return nil
}

// Waits for all the chunks to be uploaded, then shuts down all the workers
func (ls *LogStreamer) Stop() error {
	logger.Debug("[LogStreamer] Waiting for all the chunks to be uploaded")

	ls.chunkWaitGroup.Wait()

	logger.Debug("[LogStreamer] Shutting down all workers")

	for n := 0; n < ls.Concurrency; n++ {
		ls.queue <- nil
	}

	return nil
}

// The actual log streamer worker
func Worker(id int, ls *LogStreamer) {
	logger.Debug("[LogStreamer/Worker#%d] Worker is starting...", id)

	var chunk *LogStreamerChunk
	for {
		// Get the next chunk (pointer) from the queue. This will block
		// until something is returned.
		chunk = <-ls.queue

		// If the next chunk is nil, then there is no more work to do
		if chunk == nil {
			break
		}

		// Upload the chunk
		err := ls.Callback(chunk)
		if err != nil {
			atomic.AddInt32(&ls.ChunksFailedCount, 1)

			logger.Error("Giving up on uploading chunk %d, this will result in only a partial build log on Buildkite", chunk.Order)
		}

		// Signal to the chunkWaitGroup that this one is done
		ls.chunkWaitGroup.Done()
	}

	logger.Debug("[LogStreamer/Worker#%d] Worker has shutdown", id)
}
<|endoftext|>


<commit_before>// +build !luminous,!mimic

package rbd

import (
	"testing"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

func TestImagePropertiesNautilus(t *testing.T) {
	conn := radosConnect(t)
	defer conn.Shutdown()

	poolname := GetUUID()
	err := conn.MakePool(poolname)
	require.NoError(t, err)
	defer func() { assert.NoError(t, conn.DeletePool(poolname)) }()

	ioctx, err := conn.OpenIOContext(poolname)
	require.NoError(t, err)
	defer ioctx.Destroy()

	name := GetUUID()
	err = quickCreate(ioctx, name, testImageSize, testImageOrder)
	require.NoError(t, err)

	img, err := OpenImage(ioctx, name, NoSnapshot)
	require.NoError(t, err)
	defer func() { assert.NoError(t, img.Remove()) }()
	defer func() { assert.NoError(t, img.Close()) }()

	_, err = img.GetCreateTimestamp()
	assert.NoError(t, err)

	_, err = img.GetAccessTimestamp()
	assert.NoError(t, err)
}

func TestClosedImageNautilus(t *testing.T) {
	conn := radosConnect(t)
	defer conn.Shutdown()

	poolname := GetUUID()
	err := conn.MakePool(poolname)
	assert.NoError(t, err)
	defer func() { assert.NoError(t, conn.DeletePool(poolname)) }()

	ioctx, err := conn.OpenIOContext(poolname)
	require.NoError(t, err)
	defer ioctx.Destroy()

	name := GetUUID()
	err = quickCreate(ioctx, name, testImageSize, testImageOrder)
	assert.NoError(t, err)

	image, err := OpenImage(ioctx, name, NoSnapshot)
	assert.NoError(t, err)
	defer func() { assert.NoError(t, image.Remove()) }()

	// close the image
	err = image.Close()
	assert.NoError(t, err)

	// functions should now fail with an RBDError

	_, err = image.GetCreateTimestamp()
	assert.Error(t, err)

	_, err = image.GetAccessTimestamp()
	assert.Error(t, err)
}
<commit_msg>rbd: add simple test for Image.GetModifyTimestamp()<commit_after>// +build !luminous,!mimic

package rbd

import (
	"testing"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

func TestImagePropertiesNautilus(t *testing.T) {
	conn := radosConnect(t)
	defer conn.Shutdown()

	poolname := GetUUID()
	err := conn.MakePool(poolname)
	require.NoError(t, err)
	defer func() { assert.NoError(t, conn.DeletePool(poolname)) }()

	ioctx, err := conn.OpenIOContext(poolname)
	require.NoError(t, err)
	defer ioctx.Destroy()

	name := GetUUID()
	err = quickCreate(ioctx, name, testImageSize, testImageOrder)
	require.NoError(t, err)

	img, err := OpenImage(ioctx, name, NoSnapshot)
	require.NoError(t, err)
	defer func() { assert.NoError(t, img.Remove()) }()
	defer func() { assert.NoError(t, img.Close()) }()

	_, err = img.GetCreateTimestamp()
	assert.NoError(t, err)

	_, err = img.GetAccessTimestamp()
	assert.NoError(t, err)

	_, err = img.GetModifyTimestamp()
	assert.NoError(t, err)
}

func TestClosedImageNautilus(t *testing.T) {
	conn := radosConnect(t)
	defer conn.Shutdown()

	poolname := GetUUID()
	err := conn.MakePool(poolname)
	assert.NoError(t, err)
	defer func() { assert.NoError(t, conn.DeletePool(poolname)) }()

	ioctx, err := conn.OpenIOContext(poolname)
	require.NoError(t, err)
	defer ioctx.Destroy()

	name := GetUUID()
	err = quickCreate(ioctx, name, testImageSize, testImageOrder)
	assert.NoError(t, err)

	image, err := OpenImage(ioctx, name, NoSnapshot)
	assert.NoError(t, err)
	defer func() { assert.NoError(t, image.Remove()) }()

	// close the image
	err = image.Close()
	assert.NoError(t, err)

	// functions should now fail with an RBDError

	_, err = image.GetCreateTimestamp()
	assert.Error(t, err)

	_, err = image.GetAccessTimestamp()
	assert.Error(t, err)

	_, err = image.GetModifyTimestamp()
	assert.Error(t, err)
}
<|endoftext|>


<commit_before>package actions

import (
	log "github.com/Sirupsen/logrus"
	"github.com/codegangsta/cli"
	"github.com/loadimpact/speedboat/actions/registry"
	"github.com/loadimpact/speedboat/common"
	"github.com/loadimpact/speedboat/master"
	"github.com/loadimpact/speedboat/message"
	"github.com/loadimpact/speedboat/runner"
	"github.com/loadimpact/speedboat/runner/js"
	"github.com/loadimpact/speedboat/worker"
	"io/ioutil"
)

func init() {
	registry.RegisterCommand(cli.Command{
		Name:   "run",
		Usage:  "Runs a load test",
		Action: actionRun,
		Flags: []cli.Flag{
			cli.StringFlag{
				Name:  "script",
				Usage: "Script file to run",
			},
		},
	})
	registry.RegisterProcessor(func(*worker.Worker) master.Processor {
		return &RunProcessor{}
	})
}

type RunProcessor struct{}

func (p *RunProcessor) Process(msg message.Message) <-chan message.Message {
	ch := make(chan message.Message)

	go func() {
		defer func() {
			ch <- message.NewToClient("run.end", message.Fields{})
			close(ch)
		}()

		switch msg.Type {
		case "run.run":
			filename := msg.Fields["filename"].(string)
			src := msg.Fields["src"].(string)

			log.WithFields(log.Fields{
				"filename": filename,
				"src":      src,
			}).Debug("Source")

			var r runner.Runner = nil

			r, err := js.New()
			if err != nil {
				ch <- message.NewToClient("run.error", message.Fields{"error": err})
				break
			}

			err = r.Load(filename, src)
			if err != nil {
				ch <- message.NewToClient("run.error", message.Fields{"error": err})
				break
			}

			for res := range runner.Run(r, 1) {
				switch res.Type {
				case "log":
					ch <- message.NewToClient("run.log", message.Fields{
						"time": res.LogEntry.Time,
						"text": res.LogEntry.Text,
					})
				}
			}
		}
	}()

	return ch
}

func actionRun(c *cli.Context) {
	client, _ := common.MustGetClient(c)
	in, out, errors := client.Connector.Run()

	if !c.IsSet("script") {
		log.Fatal("No script file specified!")
	}

	filename := c.String("script")
	srcb, err := ioutil.ReadFile(filename)
	src := string(srcb)
	if err != nil {
		log.WithError(err).WithFields(log.Fields{
			"filename": filename,
		}).Fatal("Couldn't read script")
	}

	out <- message.NewToWorker("run.run", message.Fields{
		"filename": c.String("script"),
		"src":      src,
	})

readLoop:
	for {
		select {
		case msg := <-in:
			switch msg.Type {
			case "run.log":
				log.WithFields(log.Fields{
					"time": msg.Fields["time"],
					"text": msg.Fields["text"],
				}).Info("Test Log")
			case "run.error":
				log.WithFields(log.Fields{
					"error": msg.Fields["error"],
				}).Error("Script Error")
			case "run.end":
				log.Info("-- Test End --")
				break readLoop
			}
		case err := <-errors:
			log.WithError(err).Error("Ping failed")
		}
	}
}
<commit_msg>No need to log the script source<commit_after>package actions

import (
	log "github.com/Sirupsen/logrus"
	"github.com/codegangsta/cli"
	"github.com/loadimpact/speedboat/actions/registry"
	"github.com/loadimpact/speedboat/common"
	"github.com/loadimpact/speedboat/master"
	"github.com/loadimpact/speedboat/message"
	"github.com/loadimpact/speedboat/runner"
	"github.com/loadimpact/speedboat/runner/js"
	"github.com/loadimpact/speedboat/worker"
	"io/ioutil"
)

func init() {
	registry.RegisterCommand(cli.Command{
		Name:   "run",
		Usage:  "Runs a load test",
		Action: actionRun,
		Flags: []cli.Flag{
			cli.StringFlag{
				Name:  "script",
				Usage: "Script file to run",
			},
		},
	})
	registry.RegisterProcessor(func(*worker.Worker) master.Processor {
		return &RunProcessor{}
	})
}

type RunProcessor struct{}

func (p *RunProcessor) Process(msg message.Message) <-chan message.Message {
	ch := make(chan message.Message)

	go func() {
		defer func() {
			ch <- message.NewToClient("run.end", message.Fields{})
			close(ch)
		}()

		switch msg.Type {
		case "run.run":
			filename := msg.Fields["filename"].(string)
			src := msg.Fields["src"].(string)

			log.WithFields(log.Fields{
				"filename": filename,
			}).Debug("Running script")

			var r runner.Runner = nil

			r, err := js.New()
			if err != nil {
				ch <- message.NewToClient("run.error", message.Fields{"error": err})
				break
			}

			err = r.Load(filename, src)
			if err != nil {
				ch <- message.NewToClient("run.error", message.Fields{"error": err})
				break
			}

			for res := range runner.Run(r, 1) {
				switch res.Type {
				case "log":
					ch <- message.NewToClient("run.log", message.Fields{
						"time": res.LogEntry.Time,
						"text": res.LogEntry.Text,
					})
				}
			}
		}
	}()

	return ch
}

func actionRun(c *cli.Context) {
	client, _ := common.MustGetClient(c)
	in, out, errors := client.Connector.Run()

	if !c.IsSet("script") {
		log.Fatal("No script file specified!")
	}

	filename := c.String("script")
	srcb, err := ioutil.ReadFile(filename)
	src := string(srcb)
	if err != nil {
		log.WithError(err).WithFields(log.Fields{
			"filename": filename,
		}).Fatal("Couldn't read script")
	}

	out <- message.NewToWorker("run.run", message.Fields{
		"filename": c.String("script"),
		"src":      src,
	})

readLoop:
	for {
		select {
		case msg := <-in:
			switch msg.Type {
			case "run.log":
				log.WithFields(log.Fields{
					"time": msg.Fields["time"],
					"text": msg.Fields["text"],
				}).Info("Test Log")
			case "run.error":
				log.WithFields(log.Fields{
					"error": msg.Fields["error"],
				}).Error("Script Error")
			case "run.end":
				log.Info("-- Test End --")
				break readLoop
			}
		case err := <-errors:
			log.WithError(err).Error("Ping failed")
		}
	}
}
<|endoftext|>


<commit_before>package reddit

// Account defines behaviors only an account can perform on Reddit.
type Account interface {
	// Reply posts a reply to something on reddit. The behavior depends on
	// what is being replied to. For
	//
	//   messages, this sends a private message reply.
	//   posts, this posts a top level comment.
	//   comments, this posts a comment reply.
	//
	// Use .Name on the parent post, message, or comment to find its
	// name.
	Reply(parentName, text string) error

	// SendMessage sends a private message to a user.
	SendMessage(user, subject, text string) error

	// PostSelf makes a text (self) post to a subreddit.
	PostSelf(subreddit, title, text string) error

	// PostLink makes a link post to a subreddit.
	PostLink(subreddit, title, url string) error
}

type account struct {
	// r is used to execute requests to Reddit.
	r reaper
}

// newAccount returns a new Account using the given reaper to make requests
// to Reddit.
func newAccount(r reaper) Account {
	return &account{
		r: r,
	}
}

func (a *account) Reply(parentName, text string) error {
	return a.r.sow(
		"/api/comment", map[string]string{
			"thing_id": parentName,
			"text":     text,
		},
	)
}

func (a *account) SendMessage(user, subject, text string) error {
	return a.r.sow(
		"/api/compose", map[string]string{
			"to":      user,
			"subject": subject,
			"text":    text,
		},
	)
}

func (a *account) PostSelf(subreddit, title, text string) error {
	return a.r.sow(
		"/api/submit", map[string]string{
			"sr":    subreddit,
			"kind":  "self",
			"title": title,
			"text":  text,
		},
	)
}

func (a *account) PostLink(subreddit, title, url string) error {
	return a.r.sow(
		"/api/submit", map[string]string{
			"sr":    subreddit,
			"kind":  "link",
			"title": title,
			"url":   url,
		},
	)
}
<commit_msg>Create methods for getting POSTed content<commit_after>package reddit

// Account defines behaviors only an account can perform on Reddit.
type Account interface {
	// Reply posts a reply to something on reddit. The behavior depends on
	// what is being replied to. For
	//
	//   messages, this sends a private message reply.
	//   posts, this posts a top level comment.
	//   comments, this posts a comment reply.
	//
	// Use .Name on the parent post, message, or comment to find its
	// name.
	Reply(parentName, text string) error
	GetReply(parentName, text string) (Submission, error)

	// SendMessage sends a private message to a user.
	SendMessage(user, subject, text string) error

	// PostSelf makes a text (self) post to a subreddit.
	PostSelf(subreddit, title, text string) error
	GetPostSelf(subreddit, title, text string) (Submission, error)

	// PostLink makes a link post to a subreddit.
	PostLink(subreddit, title, url string) error
	GetPostLink(subreddit, title, url string) (Submission, error)
}

type account struct {
	// r is used to execute requests to Reddit.
	r reaper
}

// newAccount returns a new Account using the given reaper to make requests
// to Reddit.
func newAccount(r reaper) Account {
	return &account{
		r: r,
	}
}

func (a *account) Reply(parentName, text string) error {
	return a.r.sow(
		"/api/comment", map[string]string{
			"thing_id": parentName,
			"text":     text,
		},
	)
}

func (a *account) GetReply(parentName, text string) (Submission, error) {
	return a.r.get_sow(
		"/api/comment", map[string]string{
			"thing_id": parentName,
			"text":     text,
		},
	)
}

func (a *account) SendMessage(user, subject, text string) error {
	return a.r.sow(
		"/api/compose", map[string]string{
			"to":      user,
			"subject": subject,
			"text":    text,
		},
	)
}

func (a *account) PostSelf(subreddit, title, text string) error {
	return a.r.sow(
		"/api/submit", map[string]string{
			"sr":    subreddit,
			"kind":  "self",
			"title": title,
			"text":  text,
		},
	)
}

func (a *account) GetPostSelf(subreddit, title, text string) (Submission, error) {
	return a.r.get_sow(
		"/api/submit", map[string]string{
			"sr":    subreddit,
			"kind":  "self",
			"title": title,
			"text":  text,
		},
	)
}

func (a *account) PostLink(subreddit, title, url string) error {
	return a.r.sow(
		"/api/submit", map[string]string{
			"sr":    subreddit,
			"kind":  "link",
			"title": title,
			"url":   url,
		},
	)
}

func (a *account) GetPostLink(subreddit, title, url string) (Submission, error) {
	return a.r.get_sow(
		"/api/submit", map[string]string{
			"sr":    subreddit,
			"kind":  "link",
			"title": title,
			"url":   url,
		},
	)
}
<|endoftext|>


<commit_before>package agent

import (
	"fmt"
	"strings"
	"time"

	log "github.com/golang/glog"

	"github.com/coreos/coreinit/event"
	"github.com/coreos/coreinit/job"
	"github.com/coreos/coreinit/machine"
	"github.com/coreos/coreinit/registry"
	"github.com/coreos/coreinit/unit"
)

const (
	// TTL to use with all state pushed to Registry
	DefaultTTL = "30s"

	// Refresh TTLs at 1/2 the TTL length
	refreshInterval = 2
)

// The Agent owns all of the coordination between the Registry, the local
// Machine, and the local SystemdManager.
type Agent struct {
	registry   *registry.Registry
	events     *event.EventBus
	machine    *machine.Machine
	ttl        time.Duration
	unitPrefix string

	state   *AgentState
	systemd *unit.SystemdManager

	// channel used to shutdown any open connections/channels the Agent holds
	stop chan bool
}

func New(registry *registry.Registry, events *event.EventBus, machine *machine.Machine, ttl, unitPrefix string) (*Agent, error) {
	ttldur, err := time.ParseDuration(ttl)
	if err != nil {
		return nil, err
	}

	state := NewState()
	mgr := unit.NewSystemdManager(machine, unitPrefix)

	return &Agent{registry, events, machine, ttldur, unitPrefix, state, mgr, nil}, nil
}

// Access Agent's machine field
func (a *Agent) Machine() *machine.Machine {
	return a.machine
}

// Trigger all async processes the Agent intends to run
func (a *Agent) Run() {
	a.stop = make(chan bool)

	handler := NewEventHandler(a)
	a.events.AddListener("agent", a.machine, handler)

	go a.systemd.Publish(a.events, a.stop)
	go a.Heartbeat(a.ttl, a.stop)

	// Block until we receive a stop signal
	<-a.stop

	a.events.RemoveListener("agent", a.machine)
}

// Stop all async processes the Agent is running
func (a *Agent) Stop() {
	log.V(1).Info("Stopping Agent")
	close(a.stop)
}

// Clear any presence data from the Registry
func (a *Agent) Purge() {
	log.V(1).Info("Removing Agent from Registry")
	err := a.registry.RemoveMachineState(a.machine)
	if err != nil {
		log.Errorf("Failed to remove Machine %s from Registry: %s", a.machine.BootId, err.Error())
	}

	for _, j := range a.registry.GetAllJobsByMachine(a.machine) {
		offer := job.NewOfferFromJob(j)
		log.V(2).Infof("Publishing JobOffer(%s)", offer.Job.Name)
		a.registry.CreateJobOffer(offer)
		log.Infof("Published JobOffer(%s)", offer.Job.Name)
	}
}

// Periodically report to the Registry at an interval equal to
// half of the provided ttl. Stop reporting when the provided
// channel is closed.
func (a *Agent) Heartbeat(ttl time.Duration, stop chan bool) {
	interval := ttl / refreshInterval
	for true {
		select {
		case <-stop:
			log.V(2).Info("MachineHeartbeat exiting due to stop signal")
			return
		case <-time.Tick(interval):
			log.V(2).Info("MachineHeartbeat tick")
			a.registry.SetMachineState(a.machine, a.ttl)
		}
	}
}

// Instruct the Agent to start the provided Job
func (a *Agent) StartJob(j *job.Job) {
	log.Infof("Starting Job(%s)", j.Name)
	a.systemd.StartJob(j)
}

// Inform the Registry that a Job must be rescheduled
func (a *Agent) RescheduleJob(j *job.Job) {
	log.V(2).Infof("Cancelling Job(%s)", j.Name)
	a.registry.CancelJob(j.Name)

	offer := job.NewOfferFromJob(*j)
	log.V(2).Infof("Publishing JobOffer(%s)", offer.Job.Name)
	a.registry.CreateJobOffer(offer)
	log.Infof("Published JobOffer(%s)", offer.Job.Name)
}

// Instruct the Agent to stop the provided Job and
// all of its peers
func (a *Agent) StopJob(j *job.Job) {
	log.Infof("Stopping Job(%s)", j.Name)
	a.systemd.StopJob(j)

	a.state.Lock()
	reversePeers := a.state.GetJobsByPeer(j.Name)
	a.state.DropPeersJob(j.Name)
	a.state.Unlock()

	for _, peer := range reversePeers {
		log.Infof("Cancelling Peer(%s) of Job(%s)", peer, j.Name)
		a.registry.CancelJob(peer)
	}
}

func (a *Agent) CancelJob(jobName string) {
	a.registry.CancelJob(jobName)
}

// Persist the state of the given Job into the Registry
func (a *Agent) ReportJobState(j *job.Job) {
	if j.State == nil {
		err := a.registry.RemoveJobState(j)
		if err != nil {
			log.V(1).Infof("Failed to remove JobState from Registry: %s", j.Name, err.Error())
		}
	} else {
		a.registry.SaveJobState(j)
	}
}

// Submit all possible bids for unresolved offers
func (a *Agent) BidForPossibleJobs() {
	a.state.Lock()
	offers := a.state.GetUnbadeOffers()
	a.state.Unlock()

	log.V(2).Infof("Checking %d unbade offers", len(offers))
	for _, offer := range offers {
		log.V(2).Infof("Checking ability to run Job(%s)", offer.Job.Name)
		if a.AbleToRun(&offer.Job) {
			log.V(2).Infof("Able to run Job(%s), submitting bid", offer.Job.Name)
			a.Bid(offer.Job.Name)
		} else {
			log.V(2).Infof("Still unable to run Job(%s)", offer.Job.Name)
		}
	}
}

// Submit a bid for the given Job
func (a *Agent) Bid(jobName string) {
	log.Infof("Submitting JobBid for Job(%s)", jobName)

	jb := job.NewBid(jobName, a.machine.BootId)
	a.registry.SubmitJobBid(jb)

	a.state.Lock()
	defer a.state.Unlock()

	a.state.TrackBid(jb.JobName)
}

// Instruct the Agent that an offer has been created and must
// be tracked until it is resolved
func (a *Agent) TrackOffer(jo job.JobOffer) {
	a.state.Lock()
	defer a.state.Unlock()

	log.V(2).Infof("Tracking JobOffer(%s)", jo.Job.Name)
	a.state.TrackOffer(jo)

	a.state.TrackJobPeers(jo.Job.Name, jo.Job.Payload.Peers)
}

// Instruct the Agent that the given offer has been resolved
// and may be ignored in future conflict calculations
func (a *Agent) OfferResolved(jobName string) {
	a.state.Lock()
	defer a.state.Unlock()

	log.V(2).Infof("Dropping JobOffer(%s)", jobName)
	a.state.DropOffer(jobName)

	a.state.DropBid(jobName)
}

// Pull a Job and its payload from the Registry
func (a *Agent) FetchJob(jobName string) *job.Job {
	log.V(1).Infof("Fetching Job(%s) from Registry", jobName)
	j := a.registry.GetJob(jobName)
	if j == nil {
		log.V(1).Infof("Job not found in Registry")
	}
	return j
}

// Submit all possible bids for known peers of the provided job
func (a *Agent) BidForPossiblePeers(jobName string) {
	a.state.Lock()
	peers := a.state.GetJobsByPeer(jobName)
	a.state.Unlock()

	for _, peer := range peers {
		log.V(1).Infof("Found unresolved offer for Peer(%s) of Job(%s)", peer, jobName)

		peerJob := a.FetchJob(peer)
		if peerJob != nil && a.AbleToRun(peerJob) {
			log.Infof("Submitting bid for Peer(%s) of Job(%s)", peer, jobName)
			a.Bid(peer)
		} else {
			log.V(1).Infof("Unable to bid for Peer(%s) of Job(%s)", peer, jobName)
		}
	}
}

// Determine if the Agent can run the provided Job
func (a *Agent) AbleToRun(j *job.Job) bool {
	metadata := extractMachineMetadata(j.Payload.Requirements)
	if !a.machine.HasMetadata(metadata) {
		log.V(1).Infof("Unable to run Job(%s), local Machine metadata insufficient", j.Name)
		return false
	}

	if a.hasConflict(j) {
		log.V(1).Infof("Unable to run Job(%s), local Job conflict", j.Name)
		return false
	}

	if !a.hasAllLocalPeers(j) {
		log.V(1).Infof("Unable to run Job(%s), necessary peer Jobs are not running locally", j.Name)
		return false
	}

	return true
}

// Determine if all necessary peers of a Job are scheduled to this Agent
func (a *Agent) hasAllLocalPeers(j *job.Job) bool {
	for _, peerName := range j.Payload.Peers {
		log.V(1).Infof("Looking for target of Peer(%s)", peerName)

		//FIXME: ideally the machine would use its own knowledge rather than calling GetJobTarget
		if tgt := a.registry.GetJobTarget(peerName); tgt == nil || tgt.BootId != a.machine.BootId {
			log.V(1).Infof("Peer(%s) of Job(%s) not scheduled here", peerName, j.Name)
			return false
		} else {
			log.V(1).Infof("Peer(%s) of Job(%s) scheduled here", peerName, j.Name)
		}
	}
	return true
}

// Determine whether a given Job conflicts with any other relevant Jobs
func (a *Agent) hasConflict(j *job.Job) bool {
	var reqString string
	for key, slice := range j.Payload.Requirements {
		reqString += fmt.Sprintf("%s = [", key)
		for _, val := range slice {
			reqString += fmt.Sprintf("%s, ", val)
		}
		reqString += fmt.Sprint("] ")
	}

	if len(reqString) > 0 {
		log.V(1).Infof("Job(%s) has requirements %s", j.Name, reqString)
	} else {
		log.V(1).Infof("Job(%s) has no requirements", j.Name)
	}

	isSingleton := func(j *job.Job) bool {
		singleton, ok := j.Payload.Requirements["MachineSingleton"]
		return ok && singleton[0] == "true"
	}

	hasProvides := func(j *job.Job) bool {
		provides, ok := j.Payload.Requirements["Provides"]
		return ok && len(provides) > 0
	}

	if !isSingleton(j) {
		log.V(1).Infof("Job(%s) is not a singleton, therefore no conflict", j.Name)
		return false
	}

	if !hasProvides(j) {
		log.V(1).Infof("Job(%s) does not provide anything, therefore no conflict", j.Name)
		return false
	}

	// Check for conflicts with locally-scheduled jobs
	for _, other := range a.registry.GetAllJobsByMachine(a.machine) {
		if !hasProvides(&other) {
			continue
		}

		// Skip self
		if other.Name == j.Name {
			continue
		}

		for _, provide := range j.Payload.Requirements["Provides"] {
			for _, otherProvide := range other.Payload.Requirements["Provides"] {
				if provide == otherProvide {
					log.V(1).Infof("Local Job(%s) already provides '%s'", other.Name, provide)
					return true
				}
			}
		}
	}

	for _, offer := range a.state.GetBadeOffers() {
		// Skip self
		if offer.Job.Name == j.Name {
			continue
		}

		if !hasProvides(&offer.Job) {
			log.V(1).Infof("Outstanding JobBid(%s) does not provide anything, therefore no conflict", offer.Job.Name)
			continue
		}

		for _, provide := range j.Payload.Requirements["Provides"] {
			for _, offerProvide := range offer.Job.Payload.Requirements["Provides"] {
				if provide == offerProvide {
					log.V(1).Infof("Outstanding JobBid(%s) already provides '%s'", offer.Job.Name, provide)
					return true
				}
			}
		}
	}

	return false
}

// Return all machine-related metadata from a job requirements map
func extractMachineMetadata(requirements map[string][]string) map[string][]string {
	metadata := make(map[string][]string)

	for key, values := range requirements {
		if !strings.HasPrefix(key, "Machine-") {
			log.V(2).Infof("Skipping requirement %s, not machine metadata.", key)
			continue
		}

		// Strip off leading 'Machine-'
		key = key[8:]

		if len(values) == 0 {
			log.V(2).Infof("Metadata(%s) requirement provided no values, ignoring.", key)
			continue
		}

		metadata[key] = values
	}

	return metadata
}
<commit_msg>fix(agent): be careful with the iterator when bidding<commit_after>package agent

import (
	"fmt"
	"strings"
	"time"

	log "github.com/golang/glog"

	"github.com/coreos/coreinit/event"
	"github.com/coreos/coreinit/job"
	"github.com/coreos/coreinit/machine"
	"github.com/coreos/coreinit/registry"
	"github.com/coreos/coreinit/unit"
)

const (
	// TTL to use with all state pushed to Registry
	DefaultTTL = "30s"

	// Refresh TTLs at 1/2 the TTL length
	refreshInterval = 2
)

// The Agent owns all of the coordination between the Registry, the local
// Machine, and the local SystemdManager.
type Agent struct {
	registry   *registry.Registry
	events     *event.EventBus
	machine    *machine.Machine
	ttl        time.Duration
	unitPrefix string

	state   *AgentState
	systemd *unit.SystemdManager

	// channel used to shutdown any open connections/channels the Agent holds
	stop chan bool
}

func New(registry *registry.Registry, events *event.EventBus, machine *machine.Machine, ttl, unitPrefix string) (*Agent, error) {
	ttldur, err := time.ParseDuration(ttl)
	if err != nil {
		return nil, err
	}

	state := NewState()
	mgr := unit.NewSystemdManager(machine, unitPrefix)

	return &Agent{registry, events, machine, ttldur, unitPrefix, state, mgr, nil}, nil
}

// Access Agent's machine field
func (a *Agent) Machine() *machine.Machine {
	return a.machine
}

// Trigger all async processes the Agent intends to run
func (a *Agent) Run() {
	a.stop = make(chan bool)

	handler := NewEventHandler(a)
	a.events.AddListener("agent", a.machine, handler)

	go a.systemd.Publish(a.events, a.stop)
	go a.Heartbeat(a.ttl, a.stop)

	// Block until we receive a stop signal
	<-a.stop

	a.events.RemoveListener("agent", a.machine)
}

// Stop all async processes the Agent is running
func (a *Agent) Stop() {
	log.V(1).Info("Stopping Agent")
	close(a.stop)
}

// Clear any presence data from the Registry
func (a *Agent) Purge() {
	log.V(1).Info("Removing Agent from Registry")
	err := a.registry.RemoveMachineState(a.machine)
	if err != nil {
		log.Errorf("Failed to remove Machine %s from Registry: %s", a.machine.BootId, err.Error())
	}

	for _, j := range a.registry.GetAllJobsByMachine(a.machine) {
		offer := job.NewOfferFromJob(j)
		log.V(2).Infof("Publishing JobOffer(%s)", offer.Job.Name)
		a.registry.CreateJobOffer(offer)
		log.Infof("Published JobOffer(%s)", offer.Job.Name)
	}
}

// Periodically report to the Registry at an interval equal to
// half of the provided ttl. Stop reporting when the provided
// channel is closed.
func (a *Agent) Heartbeat(ttl time.Duration, stop chan bool) {
	interval := ttl / refreshInterval
	for true {
		select {
		case <-stop:
			log.V(2).Info("MachineHeartbeat exiting due to stop signal")
			return
		case <-time.Tick(interval):
			log.V(2).Info("MachineHeartbeat tick")
			a.registry.SetMachineState(a.machine, a.ttl)
		}
	}
}

// Instruct the Agent to start the provided Job
func (a *Agent) StartJob(j *job.Job) {
	log.Infof("Starting Job(%s)", j.Name)
	a.systemd.StartJob(j)
}

// Inform the Registry that a Job must be rescheduled
func (a *Agent) RescheduleJob(j *job.Job) {
	log.V(2).Infof("Cancelling Job(%s)", j.Name)
	a.registry.CancelJob(j.Name)

	offer := job.NewOfferFromJob(*j)
	log.V(2).Infof("Publishing JobOffer(%s)", offer.Job.Name)
	a.registry.CreateJobOffer(offer)
	log.Infof("Published JobOffer(%s)", offer.Job.Name)
}

// Instruct the Agent to stop the provided Job and
// all of its peers
func (a *Agent) StopJob(j *job.Job) {
	log.Infof("Stopping Job(%s)", j.Name)
	a.systemd.StopJob(j)

	a.state.Lock()
	reversePeers := a.state.GetJobsByPeer(j.Name)
	a.state.DropPeersJob(j.Name)
	a.state.Unlock()

	for _, peer := range reversePeers {
		log.Infof("Cancelling Peer(%s) of Job(%s)", peer, j.Name)
		a.registry.CancelJob(peer)
	}
}

func (a *Agent) CancelJob(jobName string) {
	a.registry.CancelJob(jobName)
}

// Persist the state of the given Job into the Registry
func (a *Agent) ReportJobState(j *job.Job) {
	if j.State == nil {
		err := a.registry.RemoveJobState(j)
		if err != nil {
			log.V(1).Infof("Failed to remove JobState from Registry: %s", j.Name, err.Error())
		}
	} else {
		a.registry.SaveJobState(j)
	}
}

// Submit all possible bids for unresolved offers
func (a *Agent) BidForPossibleJobs() {
	a.state.Lock()
	offers := a.state.GetUnbadeOffers()
	a.state.Unlock()

	log.V(2).Infof("Checking %d unbade offers", len(offers))
	for i, _ := range offers {
		offer := offers[i]
		log.V(2).Infof("Checking ability to run Job(%s)", offer.Job.Name)
		if a.AbleToRun(&offer.Job) {
			log.V(2).Infof("Able to run Job(%s), submitting bid", offer.Job.Name)
			a.Bid(offer.Job.Name)
		} else {
			log.V(2).Infof("Still unable to run Job(%s)", offer.Job.Name)
		}
	}
}

// Submit a bid for the given Job
func (a *Agent) Bid(jobName string) {
	log.Infof("Submitting JobBid for Job(%s)", jobName)

	jb := job.NewBid(jobName, a.machine.BootId)
	a.registry.SubmitJobBid(jb)

	a.state.Lock()
	defer a.state.Unlock()

	a.state.TrackBid(jb.JobName)
}

// Instruct the Agent that an offer has been created and must
// be tracked until it is resolved
func (a *Agent) TrackOffer(jo job.JobOffer) {
	a.state.Lock()
	defer a.state.Unlock()

	log.V(2).Infof("Tracking JobOffer(%s)", jo.Job.Name)
	a.state.TrackOffer(jo)

	a.state.TrackJobPeers(jo.Job.Name, jo.Job.Payload.Peers)
}

// Instruct the Agent that the given offer has been resolved
// and may be ignored in future conflict calculations
func (a *Agent) OfferResolved(jobName string) {
	a.state.Lock()
	defer a.state.Unlock()

	log.V(2).Infof("Dropping JobOffer(%s)", jobName)
	a.state.DropOffer(jobName)

	a.state.DropBid(jobName)
}

// Pull a Job and its payload from the Registry
func (a *Agent) FetchJob(jobName string) *job.Job {
	log.V(1).Infof("Fetching Job(%s) from Registry", jobName)
	j := a.registry.GetJob(jobName)
	if j == nil {
		log.V(1).Infof("Job not found in Registry")
	}
	return j
}

// Submit all possible bids for known peers of the provided job
func (a *Agent) BidForPossiblePeers(jobName string) {
	a.state.Lock()
	peers := a.state.GetJobsByPeer(jobName)
	a.state.Unlock()

	for _, peer := range peers {
		log.V(1).Infof("Found unresolved offer for Peer(%s) of Job(%s)", peer, jobName)

		peerJob := a.FetchJob(peer)
		if peerJob != nil && a.AbleToRun(peerJob) {
			log.Infof("Submitting bid for Peer(%s) of Job(%s)", peer, jobName)
			a.Bid(peer)
		} else {
			log.V(1).Infof("Unable to bid for Peer(%s) of Job(%s)", peer, jobName)
		}
	}
}

// Determine if the Agent can run the provided Job
func (a *Agent) AbleToRun(j *job.Job) bool {
	metadata := extractMachineMetadata(j.Payload.Requirements)
	if !a.machine.HasMetadata(metadata) {
		log.V(1).Infof("Unable to run Job(%s), local Machine metadata insufficient", j.Name)
		return false
	}

	if a.hasConflict(j) {
		log.V(1).Infof("Unable to run Job(%s), local Job conflict", j.Name)
		return false
	}

	if !a.hasAllLocalPeers(j) {
		log.V(1).Infof("Unable to run Job(%s), necessary peer Jobs are not running locally", j.Name)
		return false
	}

	return true
}

// Determine if all necessary peers of a Job are scheduled to this Agent
func (a *Agent) hasAllLocalPeers(j *job.Job) bool {
	for _, peerName := range j.Payload.Peers {
		log.V(1).Infof("Looking for target of Peer(%s)", peerName)

		//FIXME: ideally the machine would use its own knowledge rather than calling GetJobTarget
		if tgt := a.registry.GetJobTarget(peerName); tgt == nil || tgt.BootId != a.machine.BootId {
			log.V(1).Infof("Peer(%s) of Job(%s) not scheduled here", peerName, j.Name)
			return false
		} else {
			log.V(1).Infof("Peer(%s) of Job(%s) scheduled here", peerName, j.Name)
		}
	}
	return true
}

// Determine whether a given Job conflicts with any other relevant Jobs
func (a *Agent) hasConflict(j *job.Job) bool {
	var reqString string
	for key, slice := range j.Payload.Requirements {
		reqString += fmt.Sprintf("%s = [", key)
		for _, val := range slice {
			reqString += fmt.Sprintf("%s, ", val)
		}
		reqString += fmt.Sprint("] ")
	}

	if len(reqString) > 0 {
		log.V(1).Infof("Job(%s) has requirements %s", j.Name, reqString)
	} else {
		log.V(1).Infof("Job(%s) has no requirements", j.Name)
	}

	isSingleton := func(j *job.Job) bool {
		singleton, ok := j.Payload.Requirements["MachineSingleton"]
		return ok && singleton[0] == "true"
	}

	hasProvides := func(j *job.Job) bool {
		provides, ok := j.Payload.Requirements["Provides"]
		return ok && len(provides) > 0
	}

	if !isSingleton(j) {
		log.V(1).Infof("Job(%s) is not a singleton, therefore no conflict", j.Name)
		return false
	}

	if !hasProvides(j) {
		log.V(1).Infof("Job(%s) does not provide anything, therefore no conflict", j.Name)
		return false
	}

	// Check for conflicts with locally-scheduled jobs
	for _, other := range a.registry.GetAllJobsByMachine(a.machine) {
		if !hasProvides(&other) {
			continue
		}

		// Skip self
		if other.Name == j.Name {
			continue
		}

		for _, provide := range j.Payload.Requirements["Provides"] {
			for _, otherProvide := range other.Payload.Requirements["Provides"] {
				if provide == otherProvide {
					log.V(1).Infof("Local Job(%s) already provides '%s'", other.Name, provide)
					return true
				}
			}
		}
	}

	for _, offer := range a.state.GetBadeOffers() {
		// Skip self
		if offer.Job.Name == j.Name {
			continue
		}

		if !hasProvides(&offer.Job) {
			log.V(1).Infof("Outstanding JobBid(%s) does not provide anything, therefore no conflict", offer.Job.Name)
			continue
		}

		for _, provide := range j.Payload.Requirements["Provides"] {
			for _, offerProvide := range offer.Job.Payload.Requirements["Provides"] {
				if provide == offerProvide {
					log.V(1).Infof("Outstanding JobBid(%s) already provides '%s'", offer.Job.Name, provide)
					return true
				}
			}
		}
	}

	return false
}

// Return all machine-related metadata from a job requirements map
func extractMachineMetadata(requirements map[string][]string) map[string][]string {
	metadata := make(map[string][]string)

	for key, values := range requirements {
		if !strings.HasPrefix(key, "Machine-") {
			log.V(2).Infof("Skipping requirement %s, not machine metadata.", key)
			continue
		}

		// Strip off leading 'Machine-'
		key = key[8:]

		if len(values) == 0 {
			log.V(2).Infof("Metadata(%s) requirement provided no values, ignoring.", key)
			continue
		}

		metadata[key] = values
	}

	return metadata
}
<|endoftext|>


<commit_before>package api

import (
	"path/filepath"
	"runtime"
	"strings"
	"testing"
	"unicode/utf8"
)

func TestFollowSymlink(t *testing.T) {
	_, path, _, _ := runtime.Caller(0)
	dir := filepath.Join(path, "..", "..", "fixtures", "iteration")

	files := []string{
		filepath.Join(dir, "python", "leap", "symlink.py"),
	}

	iter, err := NewIteration(dir, files)
	if err != nil {
		t.Fatal(err)
	}

	for name, contents := range iter.Solution {
		expected_contents := "# two\n"
		expected_name := "symlink.py"

		if expected_contents != contents {
			t.Errorf("Expected contents to be %s, but got %s", expected_contents, contents)
		}
		if name != expected_name {
			t.Errorf("bad name. expected: %s, got %s", expected_name, name)
		}
	}
}

func TestNewIteration(t *testing.T) {
	_, path, _, _ := runtime.Caller(0)
	dir := filepath.Join(path, "..", "..", "fixtures", "iteration")

	files := []string{
		filepath.Join(dir, "python", "leap", "one.py"),
		filepath.Join(dir, "python", "leap", "two.py"),
		filepath.Join(dir, "python", "leap", "lib", "three.py"),
		filepath.Join(dir, "python", "leap", "utf16le.py"),
		filepath.Join(dir, "python", "leap", "utf16be.py"),
		filepath.Join(dir, "python", "leap", "long-utf8.py"),
	}

	iter, err := NewIteration(dir, files)
	if err != nil {
		t.Fatal(err)
	}

	if iter.TrackID != "python" {
		t.Errorf("Expected language to be python, was %s", iter.TrackID)
	}
	if iter.Problem != "leap" {
		t.Errorf("Expected problem to be leap, was %s", iter.Problem)
	}

	if len(iter.Solution) != 6 {
		t.Fatalf("Expected solution to have 6 files, had %d", len(iter.Solution))
	}

	expected := map[string]struct {
		prefix string
		suffix string
	}{
		"one.py": {prefix: "# one"},
		"two.py": {prefix: "# two"},
		filepath.Join("lib", "three.py"): {prefix: "# three"},
		"utf16le.py":                     {prefix: "# utf16le"},
		"utf16be.py":                     {prefix: "# utf16be"},
		"long-utf8.py":                   {prefix: "# The first 1024", suffix: "👍\n"},
	}

	for filename, code := range expected {
		if !utf8.ValidString(iter.Solution[filename]) {
			t.Errorf("Iteration content is not valid UTF-8 data: %s", iter.Solution[filename])
		}

		if !strings.HasPrefix(iter.Solution[filename], code.prefix) {
			t.Errorf("Expected %s to start with `%s', had `%s'", filename, code.prefix, iter.Solution[filename])
		}
		if !strings.HasSuffix(iter.Solution[filename], code.suffix) {
			t.Errorf("Expected %s to end with `%s', had `%s'", filename, code.suffix, iter.Solution[filename])
		}
	}
}

func TestIterationValidFile(t *testing.T) {
	testCases := []struct {
		file string
		ok   bool
	}{
		{
			file: "/Users/me/exercism/ruby/bob/totally/fine/deep/path/src/bob.rb",
			ok:   true,
		},
		{
			file: "/Users/me/exercism/ruby/bob/bob.rb",
			ok:   true,
		},
		{
			file: "/users/me/exercism/ruby/bob/bob.rb",
			ok:   true,
		},
		{
			file: "/Users/me/bob.rb",
			ok:   false,
		},
		{
			file: "/tmp/bob.rb",
			ok:   false,
		},
	}

	for _, tt := range testCases {
		iter := &Iteration{
			Dir: "/Users/me/exercism",
		}
		ok := iter.isValidFilepath(tt.file)
		if ok && !tt.ok {
			t.Errorf("Expected %s to be invalid.", tt.file)
		}
	}
}
<commit_msg>iteration_test: change variable names to camelCase<commit_after>package api

import (
	"path/filepath"
	"runtime"
	"strings"
	"testing"
	"unicode/utf8"
)

func TestFollowSymlink(t *testing.T) {
	_, path, _, _ := runtime.Caller(0)
	dir := filepath.Join(path, "..", "..", "fixtures", "iteration")

	files := []string{
		filepath.Join(dir, "python", "leap", "symlink.py"),
	}

	iter, err := NewIteration(dir, files)
	if err != nil {
		t.Fatal(err)
	}

	for name, contents := range iter.Solution {
		expectedContents := "# two\n"
		expectedName := "symlink.py"

		if expectedContents != contents {
			t.Errorf("Expected contents to be %s, but got %s", expectedContents, contents)
		}
		if name != expectedName {
			t.Errorf("bad name. expected: %s, got %s", expectedName, name)
		}
	}
}

func TestNewIteration(t *testing.T) {
	_, path, _, _ := runtime.Caller(0)
	dir := filepath.Join(path, "..", "..", "fixtures", "iteration")

	files := []string{
		filepath.Join(dir, "python", "leap", "one.py"),
		filepath.Join(dir, "python", "leap", "two.py"),
		filepath.Join(dir, "python", "leap", "lib", "three.py"),
		filepath.Join(dir, "python", "leap", "utf16le.py"),
		filepath.Join(dir, "python", "leap", "utf16be.py"),
		filepath.Join(dir, "python", "leap", "long-utf8.py"),
	}

	iter, err := NewIteration(dir, files)
	if err != nil {
		t.Fatal(err)
	}

	if iter.TrackID != "python" {
		t.Errorf("Expected language to be python, was %s", iter.TrackID)
	}
	if iter.Problem != "leap" {
		t.Errorf("Expected problem to be leap, was %s", iter.Problem)
	}

	if len(iter.Solution) != 6 {
		t.Fatalf("Expected solution to have 6 files, had %d", len(iter.Solution))
	}

	expected := map[string]struct {
		prefix string
		suffix string
	}{
		"one.py": {prefix: "# one"},
		"two.py": {prefix: "# two"},
		filepath.Join("lib", "three.py"): {prefix: "# three"},
		"utf16le.py":                     {prefix: "# utf16le"},
		"utf16be.py":                     {prefix: "# utf16be"},
		"long-utf8.py":                   {prefix: "# The first 1024", suffix: "👍\n"},
	}

	for filename, code := range expected {
		if !utf8.ValidString(iter.Solution[filename]) {
			t.Errorf("Iteration content is not valid UTF-8 data: %s", iter.Solution[filename])
		}

		if !strings.HasPrefix(iter.Solution[filename], code.prefix) {
			t.Errorf("Expected %s to start with `%s', had `%s'", filename, code.prefix, iter.Solution[filename])
		}
		if !strings.HasSuffix(iter.Solution[filename], code.suffix) {
			t.Errorf("Expected %s to end with `%s', had `%s'", filename, code.suffix, iter.Solution[filename])
		}
	}
}

func TestIterationValidFile(t *testing.T) {
	testCases := []struct {
		file string
		ok   bool
	}{
		{
			file: "/Users/me/exercism/ruby/bob/totally/fine/deep/path/src/bob.rb",
			ok:   true,
		},
		{
			file: "/Users/me/exercism/ruby/bob/bob.rb",
			ok:   true,
		},
		{
			file: "/users/me/exercism/ruby/bob/bob.rb",
			ok:   true,
		},
		{
			file: "/Users/me/bob.rb",
			ok:   false,
		},
		{
			file: "/tmp/bob.rb",
			ok:   false,
		},
	}

	for _, tt := range testCases {
		iter := &Iteration{
			Dir: "/Users/me/exercism",
		}
		ok := iter.isValidFilepath(tt.file)
		if ok && !tt.ok {
			t.Errorf("Expected %s to be invalid.", tt.file)
		}
	}
}
<|endoftext|>


<commit_before>package api

import (
	"io"
	"net/http"
)

// putObjectRequest builds a request for uploading an object
func (c *Client) putObjectRequest(objectID string, content io.ReadCloser) (*http.Request, error) {
	req, err := http.NewRequest("PUT", c.BaseURL+"/blobs/"+objectID,
		content)
	if err != nil {
		return nil, err
	}
	req.Header.Set("Content-Type", "application/octet-stream")
	SignWithKey(req, c.signingPrivateKey)
	return req, nil
}

// PutObject uploads an object to the server
func (c *Client) PutObject(objectID string, content io.ReadCloser) error {
	req, err := c.putObjectRequest(objectID, content)
	if err != nil {
		return err
	}
	_, err = c.doRequest(req, http.StatusCreated, http.StatusOK)
	return err
}

// getObjectRequest builds a request for downloading an object
func (c *Client) getObjectRequest(objectID string) (*http.Request, error) {
	req, err := http.NewRequest("GET", c.BaseURL+"/blobs/"+objectID, nil)
	if err != nil {
		return nil, err
	}
	SignWithKey(req, c.signingPrivateKey)
	return req, nil
}

// GetObject downloads an object from the server
func (c *Client) GetObject(objectID string) (io.Reader, error) {
	req, err := c.getObjectRequest(objectID)
	if err != nil {
		return nil, err
	}
	resp, err := c.doRequest(req, http.StatusCreated, http.StatusOK)
	if resp == nil {
		return nil, err
	}
	return resp.Body, err
}
<commit_msg>api: Changed input interfaces to Client.PutObject from io.ReadCloser to io.Reader. io.ReadCloser is not necessary in this context (the Close() method is not being called).<commit_after>package api

import (
	"io"
	"net/http"
)

// putObjectRequest builds a request for uploading an object
func (c *Client) putObjectRequest(objectID string, content io.Reader) (*http.Request, error) {
	req, err := http.NewRequest("PUT", c.BaseURL+"/blobs/"+objectID,
		content)
	if err != nil {
		return nil, err
	}
	req.Header.Set("Content-Type", "application/octet-stream")
	SignWithKey(req, c.signingPrivateKey)
	return req, nil
}

// PutObject uploads an object to the server
func (c *Client) PutObject(objectID string, content io.Reader) error {
	req, err := c.putObjectRequest(objectID, content)
	if err != nil {
		return err
	}
	_, err = c.doRequest(req, http.StatusCreated, http.StatusOK)
	return err
}

// getObjectRequest builds a request for downloading an object
func (c *Client) getObjectRequest(objectID string) (*http.Request, error) {
	req, err := http.NewRequest("GET", c.BaseURL+"/blobs/"+objectID, nil)
	if err != nil {
		return nil, err
	}
	SignWithKey(req, c.signingPrivateKey)
	return req, nil
}

// GetObject downloads an object from the server
func (c *Client) GetObject(objectID string) (io.Reader, error) {
	req, err := c.getObjectRequest(objectID)
	if err != nil {
		return nil, err
	}
	resp, err := c.doRequest(req, http.StatusCreated, http.StatusOK)
	if resp == nil {
		return nil, err
	}
	return resp.Body, err
}
<|endoftext|>