<commit_before>package v2

import (
	"fmt"
	"math"
	"sort"
	"strings"

	"code.cloudfoundry.org/cli/actors/v2actions"
	"code.cloudfoundry.org/cli/cf"
	"code.cloudfoundry.org/cli/commands"
	"code.cloudfoundry.org/cli/commands/flags"
	"code.cloudfoundry.org/cli/commands/v2/internal"
	"code.cloudfoundry.org/cli/utils/config"
	"code.cloudfoundry.org/cli/utils/sortutils"
)

//go:generate counterfeiter . HelpActor

// HelpActor handles the business logic of the help command
type HelpActor interface {
	// CommandInfoByName returns back a help command information for the given
	// command
	CommandInfoByName(interface{}, string) (v2actions.CommandInfo, error)

	// CommandInfos returns a list of all commands
	CommandInfos(interface{}) map[string]v2actions.CommandInfo
}

type HelpCommand struct {
	UI     commands.UI
	Actor  HelpActor
	Config commands.Config

	OptionalArgs flags.CommandName `positional-args:"yes"`
	AllCommands  bool              `short:"a" description:"All available CLI commands"`
	usage        interface{}       `usage:"CF_NAME help [COMMAND]"`
}

func (cmd *HelpCommand) Setup(config commands.Config, ui commands.UI) error {
	cmd.Actor = v2actions.NewActor()
	cmd.Config = config
	cmd.UI = ui

	return nil
}

func (cmd HelpCommand) Execute(args []string) error {
	var err error
	if cmd.OptionalArgs.CommandName == "" {
		cmd.displayFullHelp()
	} else {
		err = cmd.displayCommand()
	}

	return err
}

func (cmd HelpCommand) displayFullHelp() {
	cmd.displayHelpPreamble()
	if cmd.AllCommands {
		cmd.displayAllCommands()
		cmd.displayHelpFooter()
	} else {
		cmd.displayCommonCommands()
	}
}

func (cmd HelpCommand) displayHelpPreamble() {
	cmd.UI.DisplayHelpHeader("NAME:")
	cmd.UI.DisplayTextWithKeyTranslations("   {{.CommandName}} - {{.CommandDescription}}",
		[]string{"CommandDescription"},
		map[string]interface{}{
			"CommandName":        cmd.Config.BinaryName(),
			"CommandDescription": "A command line tool to interact with Cloud Foundry",
		})
	cmd.UI.DisplayNewline()

	cmd.UI.DisplayHelpHeader("USAGE:")
	cmd.UI.DisplayTextWithKeyTranslations("   {{.CommandName}} {{.CommandUsage}}",
		[]string{"CommandUsage"},
		map[string]interface{}{
			"CommandName":  cmd.Config.BinaryName(),
			"CommandUsage": "[global options] command [arguments...] [command options]",
		})
	cmd.UI.DisplayNewline()

	cmd.UI.DisplayHelpHeader("VERSION:")
	cmd.UI.DisplayText("   {{.Version}}-{{.Time}}", map[string]interface{}{
		"Version": cf.Version,
		"Time":    cf.BuiltOnDate,
	})
	cmd.UI.DisplayNewline()
}

func (cmd HelpCommand) displayCommonCommands() {
	cmdInfo := cmd.Actor.CommandInfos(Commands)

	for _, category := range internal.CommonHelpCategoryList {
		cmd.UI.DisplayHelpHeader(category.CategoryName)
		table := [][]string{}

		for _, row := range category.CommandList {
			finalRow := []string{}

			for _, command := range row {
				separator := ""
				if info, ok := cmdInfo[command]; ok {
					if len(info.Alias) > 0 {
						separator = ","
					}
					finalRow = append(finalRow, fmt.Sprintf("%s%s%s\t", info.Name, separator, info.Alias))
				}
			}

			table = append(table, finalRow)
		}

		cmd.UI.DisplayTable("   ", table)
		cmd.UI.DisplayNewline()
	}

	pluginCommands := cmd.getSortedPluginCommands()
	cmd.UI.DisplayHelpHeader("Commands offered by installed plugins:")

	size := int(math.Ceil(float64(len(pluginCommands)) / 3))
	table := make([][]string, size)
	for i, pluginCommand := range pluginCommands {
		table[i/3] = append(table[i/3], pluginCommand.Name)
	}

	cmd.UI.DisplayTable("   ", table)
	cmd.UI.DisplayNewline()

	cmd.UI.DisplayHelpHeader("Global options:")
	cmd.UI.DisplayTextWithKeyTranslations("   {{.ENVName}}                         {{.Description}}",
		[]string{"Description"},
		map[string]interface{}{
			"ENVName":     "--help, -h",
			"Description": "Show help",
		})
	cmd.UI.DisplayTextWithKeyTranslations("   {{.ENVName}}                                 {{.Description}}",
		[]string{"Description"},
		map[string]interface{}{
			"ENVName":     "-v",
			"Description": "Print API request diagnostics to stdout",
		})
	cmd.UI.DisplayNewline()
	cmd.UI.DisplayText("'cf help -a' lists all commands with short descriptions. See 'cf help <command>' to read about a specific command.")
}

func (cmd HelpCommand) displayAllCommands() {
	pluginCommands := cmd.getSortedPluginCommands()
	cmdInfo := cmd.Actor.CommandInfos(Commands)
	longestCmd := internal.LongestCommandName(cmdInfo, pluginCommands)

	for _, category := range internal.HelpCategoryList {
		cmd.UI.DisplayHelpHeader(category.CategoryName)

		for _, row := range category.CommandList {
			for _, command := range row {
				cmd.UI.DisplayTextWithKeyTranslations("   {{.CommandName}}{{.Gap}}{{.CommandDescription}}",
					[]string{"CommandDescription"},
					map[string]interface{}{
						"CommandName":        cmdInfo[command].Name,
						"CommandDescription": cmdInfo[command].Description,
						"Gap":                strings.Repeat(" ", longestCmd+1-len(command)),
					})
			}

			cmd.UI.DisplayNewline()
		}

		cmd.UI.DisplayNewline()
	}

	cmd.UI.DisplayHelpHeader("INSTALLED PLUGIN COMMANDS:")
	for _, pluginCommand := range pluginCommands {
		cmd.UI.DisplayText("   {{.CommandName}}{{.Gap}}{{.CommandDescription}}", map[string]interface{}{
			"CommandName":        pluginCommand.Name,
			"CommandDescription": pluginCommand.HelpText,
			"Gap":                strings.Repeat(" ", longestCmd+1-len(pluginCommand.Name)),
		})
	}
	cmd.UI.DisplayNewline()
}

func (cmd HelpCommand) displayHelpFooter() {
	cmd.UI.DisplayHelpHeader("ENVIRONMENT VARIABLES:")
	cmd.UI.DisplayTextWithKeyTranslations("   {{.ENVName}}                     {{.Description}}",
		[]string{"Description"},
		map[string]interface{}{
			"ENVName":     "CF_COLOR=false",
			"Description": "Do not colorize output",
		})
	cmd.UI.DisplayTextWithKeyTranslations("   {{.ENVName}}               {{.Description}}",
		[]string{"Description"},
		map[string]interface{}{
			"ENVName":     "CF_HOME=path/to/dir/",
			"Description": "Override path to default config directory",
		})
	cmd.UI.DisplayTextWithKeyTranslations("   {{.ENVName}}        {{.Description}}",
		[]string{"Description"},
		map[string]interface{}{
			"ENVName":     "CF_PLUGIN_HOME=path/to/dir/",
			"Description": "Override path to default plugin config directory",
		})
	cmd.UI.DisplayTextWithKeyTranslations("   {{.ENVName}}              {{.Description}}",
		[]string{"Description"},
		map[string]interface{}{
			"ENVName":     "CF_STAGING_TIMEOUT=15",
			"Description": "Max wait time for buildpack staging, in minutes",
		})
	cmd.UI.DisplayTextWithKeyTranslations("   {{.ENVName}}               {{.Description}}",
		[]string{"Description"},
		map[string]interface{}{
			"ENVName":     "CF_STARTUP_TIMEOUT=5",
			"Description": "Max wait time for app instance startup, in minutes",
		})
	cmd.UI.DisplayTextWithKeyTranslations("   {{.ENVName}}                      {{.Description}}",
		[]string{"Description"},
		map[string]interface{}{
			"ENVName":     "CF_TRACE=true",
			"Description": "Print API request diagnostics to stdout",
		})
	cmd.UI.DisplayTextWithKeyTranslations("   {{.ENVName}}         {{.Description}}",
		[]string{"Description"},
		map[string]interface{}{
			"ENVName":     "CF_TRACE=path/to/trace.log",
			"Description": "Append API request diagnostics to a log file",
		})
	cmd.UI.DisplayTextWithKeyTranslations("   {{.ENVName}} {{.Description}}",
		[]string{"Description"},
		map[string]interface{}{
			"ENVName":     "https_proxy=proxy.example.com:8080",
			"Description": "Enable HTTP proxying for API requests",
		})
	cmd.UI.DisplayNewline()

	cmd.UI.DisplayHelpHeader("GLOBAL OPTIONS:")
	cmd.UI.DisplayTextWithKeyTranslations("   {{.ENVName}}                         {{.Description}}",
		[]string{"Description"},
		map[string]interface{}{
			"ENVName":     "--help, -h",
			"Description": "Show help",
		})
	cmd.UI.DisplayTextWithKeyTranslations("   {{.ENVName}}                                 {{.Description}}",
		[]string{"Description"},
		map[string]interface{}{
			"ENVName":     "-v",
			"Description": "Print API request diagnostics to stdout",
		})
}

func (cmd HelpCommand) displayCommand() error {
	cmdInfo, err := cmd.Actor.CommandInfoByName(Commands, cmd.OptionalArgs.CommandName)
	if err != nil {
		if err, ok := err.(v2actions.ErrorInvalidCommand); ok {
			var found bool
			if cmdInfo, found = cmd.findPlugin(); !found {
				return err
			}
		} else {
			return err
		}
	}

	cmd.UI.DisplayText("NAME:")
	cmd.UI.DisplayTextWithKeyTranslations("   {{.CommandName}} - {{.CommandDescription}}",
		[]string{"CommandDescription"},
		map[string]interface{}{
			"CommandName":        cmdInfo.Name,
			"CommandDescription": cmdInfo.Description,
		})

	cmd.UI.DisplayNewline()
	usageString := strings.Replace(cmdInfo.Usage, "CF_NAME", cmd.Config.BinaryName(), -1)
	cmd.UI.DisplayText("USAGE:")
	cmd.UI.DisplayTextWithKeyTranslations("   {{.CommandUsage}}",
		[]string{"CommandUsage"},
		map[string]interface{}{
			"CommandUsage": usageString,
		})

	if cmdInfo.Alias != "" {
		cmd.UI.DisplayNewline()
		cmd.UI.DisplayText("ALIAS:")
		cmd.UI.DisplayText("   {{.Alias}}",
			map[string]interface{}{
				"Alias": cmdInfo.Alias,
			})
	}

	if len(cmdInfo.Flags) != 0 {
		cmd.UI.DisplayNewline()
		cmd.UI.DisplayText("OPTIONS:")
		nameWidth := internal.LongestFlagWidth(cmdInfo.Flags) + 6
		for _, flag := range cmdInfo.Flags {
			var name string
			if flag.Short != "" && flag.Long != "" {
				name = fmt.Sprintf("--%s, -%s", flag.Long, flag.Short)
			} else if flag.Short != "" {
				name = "-" + flag.Short
			} else {
				name = "--" + flag.Long
			}

			cmd.UI.DisplayTextWithKeyTranslations("   {{.Flags}}{{.Spaces}}{{.Description}}",
				[]string{"Description"},
				map[string]interface{}{
					"Flags":       name,
					"Spaces":      strings.Repeat(" ", nameWidth-len(name)),
					"Description": flag.Description,
				})
		}
	}

	if len(cmdInfo.RelatedCommands) > 0 {
		cmd.UI.DisplayNewline()
		cmd.UI.DisplayText("SEE ALSO:")
		cmd.UI.DisplayText("   {{.RelatedCommands}}",
			map[string]interface{}{
				"RelatedCommands": strings.Join(cmdInfo.RelatedCommands, ", "),
			})
	}

	return nil
}

func (cmd HelpCommand) findPlugin() (v2actions.CommandInfo, bool) {
	for _, pluginConfig := range cmd.Config.PluginConfig() {
		for _, command := range pluginConfig.Commands {
			if command.Name == cmd.OptionalArgs.CommandName {
				return internal.ConvertPluginToCommandInfo(command), true
			}
		}
	}

	return v2actions.CommandInfo{}, false
}

func (cmd HelpCommand) getSortedPluginCommands() config.PluginCommands {
	plugins := cmd.Config.PluginConfig()

	sortedPluginNames := sortutils.Alphabetic{}
	for plugin, _ := range plugins {
		sortedPluginNames = append(sortedPluginNames, plugin)
	}
	sort.Sort(sortedPluginNames)

	pluginCommands := config.PluginCommands{}
	for _, pluginName := range sortedPluginNames {
		sortedCommands := plugins[pluginName].Commands
		sort.Sort(sortedCommands)
		pluginCommands = append(pluginCommands, sortedCommands...)
	}

	return pluginCommands
}
<commit_msg>adjust order of common plugin commands<commit_after>package v2

import (
	"fmt"
	"math"
	"sort"
	"strings"

	"code.cloudfoundry.org/cli/actors/v2actions"
	"code.cloudfoundry.org/cli/cf"
	"code.cloudfoundry.org/cli/commands"
	"code.cloudfoundry.org/cli/commands/flags"
	"code.cloudfoundry.org/cli/commands/v2/internal"
	"code.cloudfoundry.org/cli/utils/config"
	"code.cloudfoundry.org/cli/utils/sortutils"
)

//go:generate counterfeiter . HelpActor

// HelpActor handles the business logic of the help command
type HelpActor interface {
	// CommandInfoByName returns back a help command information for the given
	// command
	CommandInfoByName(interface{}, string) (v2actions.CommandInfo, error)

	// CommandInfos returns a list of all commands
	CommandInfos(interface{}) map[string]v2actions.CommandInfo
}

type HelpCommand struct {
	UI     commands.UI
	Actor  HelpActor
	Config commands.Config

	OptionalArgs flags.CommandName `positional-args:"yes"`
	AllCommands  bool              `short:"a" description:"All available CLI commands"`
	usage        interface{}       `usage:"CF_NAME help [COMMAND]"`
}

func (cmd *HelpCommand) Setup(config commands.Config, ui commands.UI) error {
	cmd.Actor = v2actions.NewActor()
	cmd.Config = config
	cmd.UI = ui

	return nil
}

func (cmd HelpCommand) Execute(args []string) error {
	var err error
	if cmd.OptionalArgs.CommandName == "" {
		cmd.displayFullHelp()
	} else {
		err = cmd.displayCommand()
	}

	return err
}

func (cmd HelpCommand) displayFullHelp() {
	cmd.displayHelpPreamble()
	if cmd.AllCommands {
		cmd.displayAllCommands()
		cmd.displayHelpFooter()
	} else {
		cmd.displayCommonCommands()
	}
}

func (cmd HelpCommand) displayHelpPreamble() {
	cmd.UI.DisplayHelpHeader("NAME:")
	cmd.UI.DisplayTextWithKeyTranslations("   {{.CommandName}} - {{.CommandDescription}}",
		[]string{"CommandDescription"},
		map[string]interface{}{
			"CommandName":        cmd.Config.BinaryName(),
			"CommandDescription": "A command line tool to interact with Cloud Foundry",
		})
	cmd.UI.DisplayNewline()

	cmd.UI.DisplayHelpHeader("USAGE:")
	cmd.UI.DisplayTextWithKeyTranslations("   {{.CommandName}} {{.CommandUsage}}",
		[]string{"CommandUsage"},
		map[string]interface{}{
			"CommandName":  cmd.Config.BinaryName(),
			"CommandUsage": "[global options] command [arguments...] [command options]",
		})
	cmd.UI.DisplayNewline()

	cmd.UI.DisplayHelpHeader("VERSION:")
	cmd.UI.DisplayText("   {{.Version}}-{{.Time}}", map[string]interface{}{
		"Version": cf.Version,
		"Time":    cf.BuiltOnDate,
	})
	cmd.UI.DisplayNewline()
}

func (cmd HelpCommand) displayCommonCommands() {
	cmdInfo := cmd.Actor.CommandInfos(Commands)

	for _, category := range internal.CommonHelpCategoryList {
		cmd.UI.DisplayHelpHeader(category.CategoryName)
		table := [][]string{}

		for _, row := range category.CommandList {
			finalRow := []string{}

			for _, command := range row {
				separator := ""
				if info, ok := cmdInfo[command]; ok {
					if len(info.Alias) > 0 {
						separator = ","
					}
					finalRow = append(finalRow, fmt.Sprintf("%s%s%s\t", info.Name, separator, info.Alias))
				}
			}

			table = append(table, finalRow)
		}

		cmd.UI.DisplayTable("   ", table)
		cmd.UI.DisplayNewline()
	}

	pluginCommands := cmd.getSortedPluginCommands()
	cmd.UI.DisplayHelpHeader("Commands offered by installed plugins:")

	size := int(math.Ceil(float64(len(pluginCommands)) / 3))
	table := make([][]string, size)
	for i := 0; i < size; i++ {
		table[i] = make([]string, 3)
		for j := 0; j < 3; j++ {
			index := i + j*3
			if index < len(pluginCommands) {
				table[i][j] = pluginCommands[index].Name
			}
		}
	}

	cmd.UI.DisplayTable("   ", table)
	cmd.UI.DisplayNewline()

	cmd.UI.DisplayHelpHeader("Global options:")
	cmd.UI.DisplayTextWithKeyTranslations("   {{.ENVName}}                         {{.Description}}",
		[]string{"Description"},
		map[string]interface{}{
			"ENVName":     "--help, -h",
			"Description": "Show help",
		})
	cmd.UI.DisplayTextWithKeyTranslations("   {{.ENVName}}                                 {{.Description}}",
		[]string{"Description"},
		map[string]interface{}{
			"ENVName":     "-v",
			"Description": "Print API request diagnostics to stdout",
		})
	cmd.UI.DisplayNewline()
	cmd.UI.DisplayText("'cf help -a' lists all commands with short descriptions. See 'cf help <command>' to read about a specific command.")
}

func (cmd HelpCommand) displayAllCommands() {
	pluginCommands := cmd.getSortedPluginCommands()
	cmdInfo := cmd.Actor.CommandInfos(Commands)
	longestCmd := internal.LongestCommandName(cmdInfo, pluginCommands)

	for _, category := range internal.HelpCategoryList {
		cmd.UI.DisplayHelpHeader(category.CategoryName)

		for _, row := range category.CommandList {
			for _, command := range row {
				cmd.UI.DisplayTextWithKeyTranslations("   {{.CommandName}}{{.Gap}}{{.CommandDescription}}",
					[]string{"CommandDescription"},
					map[string]interface{}{
						"CommandName":        cmdInfo[command].Name,
						"CommandDescription": cmdInfo[command].Description,
						"Gap":                strings.Repeat(" ", longestCmd+1-len(command)),
					})
			}
		}

		cmd.UI.DisplayNewline()
	}

	cmd.UI.DisplayHelpHeader("INSTALLED PLUGIN COMMANDS:")
	for _, pluginCommand := range pluginCommands {
		cmd.UI.DisplayText("   {{.CommandName}}{{.Gap}}{{.CommandDescription}}", map[string]interface{}{
			"CommandName":        pluginCommand.Name,
			"CommandDescription": pluginCommand.HelpText,
			"Gap":                strings.Repeat(" ", longestCmd+1-len(pluginCommand.Name)),
		})
	}
	cmd.UI.DisplayNewline()
}

func (cmd HelpCommand) displayHelpFooter() {
	cmd.UI.DisplayHelpHeader("ENVIRONMENT VARIABLES:")
	cmd.UI.DisplayTextWithKeyTranslations("   {{.ENVName}}                     {{.Description}}",
		[]string{"Description"},
		map[string]interface{}{
			"ENVName":     "CF_COLOR=false",
			"Description": "Do not colorize output",
		})
	cmd.UI.DisplayTextWithKeyTranslations("   {{.ENVName}}               {{.Description}}",
		[]string{"Description"},
		map[string]interface{}{
			"ENVName":     "CF_HOME=path/to/dir/",
			"Description": "Override path to default config directory",
		})
	cmd.UI.DisplayTextWithKeyTranslations("   {{.ENVName}}        {{.Description}}",
		[]string{"Description"},
		map[string]interface{}{
			"ENVName":     "CF_PLUGIN_HOME=path/to/dir/",
			"Description": "Override path to default plugin config directory",
		})
	cmd.UI.DisplayTextWithKeyTranslations("   {{.ENVName}}              {{.Description}}",
		[]string{"Description"},
		map[string]interface{}{
			"ENVName":     "CF_STAGING_TIMEOUT=15",
			"Description": "Max wait time for buildpack staging, in minutes",
		})
	cmd.UI.DisplayTextWithKeyTranslations("   {{.ENVName}}               {{.Description}}",
		[]string{"Description"},
		map[string]interface{}{
			"ENVName":     "CF_STARTUP_TIMEOUT=5",
			"Description": "Max wait time for app instance startup, in minutes",
		})
	cmd.UI.DisplayTextWithKeyTranslations("   {{.ENVName}}                      {{.Description}}",
		[]string{"Description"},
		map[string]interface{}{
			"ENVName":     "CF_TRACE=true",
			"Description": "Print API request diagnostics to stdout",
		})
	cmd.UI.DisplayTextWithKeyTranslations("   {{.ENVName}}         {{.Description}}",
		[]string{"Description"},
		map[string]interface{}{
			"ENVName":     "CF_TRACE=path/to/trace.log",
			"Description": "Append API request diagnostics to a log file",
		})
	cmd.UI.DisplayTextWithKeyTranslations("   {{.ENVName}} {{.Description}}",
		[]string{"Description"},
		map[string]interface{}{
			"ENVName":     "https_proxy=proxy.example.com:8080",
			"Description": "Enable HTTP proxying for API requests",
		})
	cmd.UI.DisplayNewline()

	cmd.UI.DisplayHelpHeader("GLOBAL OPTIONS:")
	cmd.UI.DisplayTextWithKeyTranslations("   {{.ENVName}}                         {{.Description}}",
		[]string{"Description"},
		map[string]interface{}{
			"ENVName":     "--help, -h",
			"Description": "Show help",
		})
	cmd.UI.DisplayTextWithKeyTranslations("   {{.ENVName}}                                 {{.Description}}",
		[]string{"Description"},
		map[string]interface{}{
			"ENVName":     "-v",
			"Description": "Print API request diagnostics to stdout",
		})
}

func (cmd HelpCommand) displayCommand() error {
	cmdInfo, err := cmd.Actor.CommandInfoByName(Commands, cmd.OptionalArgs.CommandName)
	if err != nil {
		if err, ok := err.(v2actions.ErrorInvalidCommand); ok {
			var found bool
			if cmdInfo, found = cmd.findPlugin(); !found {
				return err
			}
		} else {
			return err
		}
	}

	cmd.UI.DisplayText("NAME:")
	cmd.UI.DisplayTextWithKeyTranslations("   {{.CommandName}} - {{.CommandDescription}}",
		[]string{"CommandDescription"},
		map[string]interface{}{
			"CommandName":        cmdInfo.Name,
			"CommandDescription": cmdInfo.Description,
		})

	cmd.UI.DisplayNewline()
	usageString := strings.Replace(cmdInfo.Usage, "CF_NAME", cmd.Config.BinaryName(), -1)
	cmd.UI.DisplayText("USAGE:")
	cmd.UI.DisplayTextWithKeyTranslations("   {{.CommandUsage}}",
		[]string{"CommandUsage"},
		map[string]interface{}{
			"CommandUsage": usageString,
		})

	if cmdInfo.Alias != "" {
		cmd.UI.DisplayNewline()
		cmd.UI.DisplayText("ALIAS:")
		cmd.UI.DisplayText("   {{.Alias}}",
			map[string]interface{}{
				"Alias": cmdInfo.Alias,
			})
	}

	if len(cmdInfo.Flags) != 0 {
		cmd.UI.DisplayNewline()
		cmd.UI.DisplayText("OPTIONS:")
		nameWidth := internal.LongestFlagWidth(cmdInfo.Flags) + 6
		for _, flag := range cmdInfo.Flags {
			var name string
			if flag.Short != "" && flag.Long != "" {
				name = fmt.Sprintf("--%s, -%s", flag.Long, flag.Short)
			} else if flag.Short != "" {
				name = "-" + flag.Short
			} else {
				name = "--" + flag.Long
			}

			cmd.UI.DisplayTextWithKeyTranslations("   {{.Flags}}{{.Spaces}}{{.Description}}",
				[]string{"Description"},
				map[string]interface{}{
					"Flags":       name,
					"Spaces":      strings.Repeat(" ", nameWidth-len(name)),
					"Description": flag.Description,
				})
		}
	}

	if len(cmdInfo.RelatedCommands) > 0 {
		cmd.UI.DisplayNewline()
		cmd.UI.DisplayText("SEE ALSO:")
		cmd.UI.DisplayText("   {{.RelatedCommands}}",
			map[string]interface{}{
				"RelatedCommands": strings.Join(cmdInfo.RelatedCommands, ", "),
			})
	}

	return nil
}

func (cmd HelpCommand) findPlugin() (v2actions.CommandInfo, bool) {
	for _, pluginConfig := range cmd.Config.PluginConfig() {
		for _, command := range pluginConfig.Commands {
			if command.Name == cmd.OptionalArgs.CommandName {
				return internal.ConvertPluginToCommandInfo(command), true
			}
		}
	}

	return v2actions.CommandInfo{}, false
}

func (cmd HelpCommand) getSortedPluginCommands() config.PluginCommands {
	plugins := cmd.Config.PluginConfig()

	sortedPluginNames := sortutils.Alphabetic{}
	for plugin, _ := range plugins {
		sortedPluginNames = append(sortedPluginNames, plugin)
	}
	sort.Sort(sortedPluginNames)

	pluginCommands := config.PluginCommands{}
	for _, pluginName := range sortedPluginNames {
		sortedCommands := plugins[pluginName].Commands
		sort.Sort(sortedCommands)
		pluginCommands = append(pluginCommands, sortedCommands...)
	}

	return pluginCommands
}
<|endoftext|>


<commit_before>package comments

import (
	"errors"
	"fmt"
	"regexp"
	"strings"

	"github.com/parkr/auto-reply/Godeps/_workspace/src/github.com/google/go-github/github"
)

var (
	mergeCommentRegexp = regexp.MustCompile("@[a-zA-Z-_]+: (merge|:shipit:|:ship:)( \\+([a-zA-Z-_ ]+))?")

	HandlerMergeAndLabel = func(client *github.Client, event github.IssueCommentEvent) error {
		// Is this a pull request?
		if !isPullRequest(event) {
			return errors.New("not a pull request")
		}

		var changeSectionLabel string
		isReq, labelFromComment := parseMergeRequestComment(*event.Comment.Body)

		// Is It a merge request comment?
		if !isReq {
			return errors.New("not a merge request comment")
		}

		// Does the user have merge/label abilities?
		if !isAuthorizedCommenter(event.Comment.User) {
			return errors.New("commenter isn't allowed to merge")
		}

		// Should it be labeled?
		if labelFromComment != "" {
			// Apply label
			changeSectionLabel = sectionForLabel(labelFromComment)
		} else {
			// Get changeSectionLabel from issue labels!
			labels, _, err := client.Issues.ListLabelsForMilestone(
				*event.Repo.Owner.Login,
				*event.Repo.Name,
				*event.Issue.Number,
				nil,
			)
			if err != nil {
				return err
			}
			fmt.Printf("labels from GitHub = %v\n", labels)
			changeSectionLabel = sectionForLabel(selectSectionLabel(labels))
		}
		fmt.Printf("changeSectionLabel = '%s'\n", changeSectionLabel)

		// What is the label for the change section? E.g. "Major Enhancement," "Minor Enhancement," etc

		// Merge
		// Read History.markdown, add line to appropriate change section
		// Commit change to History.markdown

		return nil
	}
)

func isAuthorizedCommenter(user *github.User) bool {
	return *user.Login == "parkr"
}

func parseMergeRequestComment(commentBody string) (bool, string) {
	matches := mergeCommentRegexp.FindAllStringSubmatch(commentBody, -1)
	if matches == nil {
		return false, ""
	}

	var label string
	if labelFromComment := matches[0][3]; labelFromComment != "" {
		label = downcaseAndHyphenize(labelFromComment)
	}

	return true, normalizeLabel(label)
}

func downcaseAndHyphenize(label string) string {
	return strings.Replace(strings.ToLower(label), " ", "-", -1)
}

func normalizeLabel(label string) string {
	if strings.HasPrefix(label, "major") {
		return "major-enhancements"
	}

	if strings.HasPrefix(label, "minor") {
		return "minor-enhancements"
	}

	if strings.HasPrefix(label, "bug") {
		return "bug-fixes"
	}

	if strings.HasPrefix(label, "development") {
		return "development-fixes"
	}

	return label
}

func sectionForLabel(label string) string {
	switch label {
	case "major-enhancements":
		return "Major Enhancements"
	case "minor-enhancements":
		return "Minor Enhancements"
	case "bug-fixes":
		return "Bug Fixes"
	case "development-fixes":
		return "Development Fixes"
	default:
		return label
	}
}

func selectSectionLabel(labels []github.Label) string {
	for _, label := range labels {
		if sectionForLabel(*label.Name) != *label.Name {
			return *label.Name
		}
	}
	return ""
}

func containsChangeLabel(commentBody string) bool {
	_, labelFromComment := parseMergeRequestComment(commentBody)
	return labelFromComment != ""
}
<commit_msg>comments: get history file contents<commit_after>package comments

import (
	"errors"
	"fmt"
	"log"
	"regexp"
	"strings"

	"github.com/parkr/auto-reply/Godeps/_workspace/src/github.com/google/go-github/github"
)

var (
	mergeCommentRegexp = regexp.MustCompile("@[a-zA-Z-_]+: (merge|:shipit:|:ship:)( \\+([a-zA-Z-_ ]+))?")

	HandlerMergeAndLabel = func(client *github.Client, event github.IssueCommentEvent) error {
		// Is this a pull request?
		if !isPullRequest(event) {
			return errors.New("not a pull request")
		}

		var changeSectionLabel string
		isReq, labelFromComment := parseMergeRequestComment(*event.Comment.Body)

		// Is It a merge request comment?
		if !isReq {
			return errors.New("not a merge request comment")
		}

		owner, repo, number := *event.Repo.Owner.Login, *event.Repo.Name, *event.Issue.Number

		// Does the user have merge/label abilities?
		if !isAuthorizedCommenter(event.Comment.User) {
			return errors.New("commenter isn't allowed to merge")
		}

		// Should it be labeled?
		if labelFromComment != "" {
			// Apply label
			changeSectionLabel = sectionForLabel(labelFromComment)
		} else {
			// Get changeSectionLabel from issue labels!
			labels, _, err := client.Issues.ListLabelsForMilestone(owner, repo, number, nil)
			if err != nil {
				return err
			}
			fmt.Printf("labels from GitHub = %v\n", labels)
			changeSectionLabel = sectionForLabel(selectSectionLabel(labels))
		}
		fmt.Printf("changeSectionLabel = '%s'\n", changeSectionLabel)

		// Merge
		// commitMsg := fmt.Sprintf("Merge pull request %v", number)
		// _, _, mergeErr := client.PullRequests.Merge(owner, repo, number, commitMsg)
		// if err != nil {
		//     fmt.Printf("comments: error merging %v\n", err)
		//     return err
		// }

		// Delete branch
		//ref := fmt.Sprintf("heads/%s", branch)
		//res, deleteBranchErr := client.Git.DeleteRef(owner, repo, ref)

		// Read History.markdown, add line to appropriate change section
		historyFileContents := getHistoryContents(client, owner, repo)
		log.Println(historyFileContents)

		// Commit change to History.markdown

		return nil
	}
)

func isAuthorizedCommenter(user *github.User) bool {
	return *user.Login == "parkr"
}

func parseMergeRequestComment(commentBody string) (bool, string) {
	matches := mergeCommentRegexp.FindAllStringSubmatch(commentBody, -1)
	if matches == nil {
		return false, ""
	}

	var label string
	if labelFromComment := matches[0][3]; labelFromComment != "" {
		label = downcaseAndHyphenize(labelFromComment)
	}

	return true, normalizeLabel(label)
}

func downcaseAndHyphenize(label string) string {
	return strings.Replace(strings.ToLower(label), " ", "-", -1)
}

func normalizeLabel(label string) string {
	if strings.HasPrefix(label, "major") {
		return "major-enhancements"
	}

	if strings.HasPrefix(label, "minor") {
		return "minor-enhancements"
	}

	if strings.HasPrefix(label, "bug") {
		return "bug-fixes"
	}

	if strings.HasPrefix(label, "development") {
		return "development-fixes"
	}

	return label
}

func sectionForLabel(label string) string {
	switch label {
	case "major-enhancements":
		return "Major Enhancements"
	case "minor-enhancements":
		return "Minor Enhancements"
	case "bug-fixes":
		return "Bug Fixes"
	case "development-fixes":
		return "Development Fixes"
	default:
		return label
	}
}

func selectSectionLabel(labels []github.Label) string {
	for _, label := range labels {
		if sectionForLabel(*label.Name) != *label.Name {
			return *label.Name
		}
	}
	return ""
}

func containsChangeLabel(commentBody string) bool {
	_, labelFromComment := parseMergeRequestComment(commentBody)
	return labelFromComment != ""
}

func getHistoryContents(client *github.Client, owner, repo string) string {
	content, _, _, err := client.Repositories.GetContents(owner, repo, "History.markdown", nil)
	if err != nil {
		fmt.Printf("comments: error getting History.markdown %v\n", err)
		return ""
	}
	return *content.Content
}
<|endoftext|>


<commit_before>// Copyright 2019 Google Inc. All Rights Reserved.
// This file is available under the Apache license.
// +build integration

package mtail_test

import (
	"fmt"
	"os"
	"path"
	"testing"
	"time"

	"github.com/golang/glog"
	"github.com/google/mtail/internal/mtail"
	"github.com/google/mtail/internal/testutil"
)

func TestGlobRelativeAfterStart(t *testing.T) {
	for _, enableFsnotify := range []bool{false, true} {
		t.Run(fmt.Sprintf("fsnotify=%v", enableFsnotify), func(t *testing.T) {
			tmpDir, rmTmpDir := testutil.TestTempDir(t)
			defer rmTmpDir()

			logDir := path.Join(tmpDir, "logs")
			progDir := path.Join(tmpDir, "progs")
			err := os.Mkdir(logDir, 0700)
			if err != nil {
				t.Fatal(err)
			}
			err = os.Mkdir(progDir, 0700)
			if err != nil {
				t.Fatal(err)
			}
			defer testutil.TestChdir(t, logDir)()

			m, stopM := mtail.TestStartServer(t, 0, enableFsnotify, mtail.ProgramPath(progDir), mtail.LogPathPatterns("log.*"))
			defer stopM()

			startLogCount := mtail.TestGetMetric(t, m.Addr(), "log_count")
			startLineCount := mtail.TestGetMetric(t, m.Addr(), "lines_total")

			logFile := path.Join(logDir, "log.1.txt")
			f := testutil.TestOpenFile(t, logFile)

			n, err := f.WriteString("line 1\n")
			if err != nil {
				t.Fatal(err)
			}
			glog.Infof("Wrote %d bytes", n)
			testutil.FatalIfErr(t, f.Sync())
			time.Sleep(time.Second)

			{
				logCount := mtail.TestGetMetric(t, m.Addr(), "log_count")
				lineCount := mtail.TestGetMetric(t, m.Addr(), "lines_total")

				if logCount.(float64)-startLogCount.(float64) != 1. {
					t.Errorf("Unexpected log count: got %g, want 1", logCount.(float64)-startLogCount.(float64))
				}
				if lineCount.(float64)-startLineCount.(float64) != 1. {
					t.Errorf("Unexpected line count: got %g, want 1", lineCount.(float64)-startLineCount.(float64))
				}
				time.Sleep(time.Second)
			}
			{

				logFile := path.Join(logDir, "log.2.txt")
				f := testutil.TestOpenFile(t, logFile)
				n, err := f.WriteString("line 1\n")
				if err != nil {
					t.Fatal(err)
				}
				glog.Infof("Wrote %d bytes", n)
				time.Sleep(time.Second)

				logCount := mtail.TestGetMetric(t, m.Addr(), "log_count")
				lineCount := mtail.TestGetMetric(t, m.Addr(), "lines_total")

				if logCount.(float64)-startLogCount.(float64) != 2. {
					t.Errorf("Unexpected log count: got %g, want 2", logCount.(float64)-startLogCount.(float64))
				}
				if lineCount.(float64)-startLineCount.(float64) != 2. {
					t.Errorf("Unexpected line count: got %g, want 2", lineCount.(float64)-startLineCount.(float64))
				}
				time.Sleep(time.Second)
			}
			{
				logFile := path.Join(logDir, "log.2.txt")
				f := testutil.TestOpenFile(t, logFile)
				n, err := f.WriteString("line 1\n")
				if err != nil {
					t.Fatal(err)
				}
				glog.Infof("Wrote %d bytes", n)
				time.Sleep(time.Second)

				logCount := mtail.TestGetMetric(t, m.Addr(), "log_count")
				lineCount := mtail.TestGetMetric(t, m.Addr(), "lines_total")

				if logCount.(float64)-startLogCount.(float64) != 2 {
					t.Errorf("Unexpected log count: got %g, want 2", logCount.(float64)-startLogCount.(float64))
				}
				if lineCount.(float64)-startLineCount.(float64) != 3 {
					t.Errorf("Unexpected line count: got %g, want 3", lineCount.(float64)-startLineCount.(float64))
				}
			}

			glog.Infof("end")
		})
	}
}
<commit_msg>Remove more sleeps and move an integration test to the timeout pattern.<commit_after>// Copyright 2019 Google Inc. All Rights Reserved.
// This file is available under the Apache license.
// +build integration

package mtail_test

import (
	"fmt"
	"os"
	"path"
	"sync"
	"testing"
	"time"

	"github.com/golang/glog"
	"github.com/google/mtail/internal/mtail"
	"github.com/google/mtail/internal/testutil"
)

func TestGlobRelativeAfterStart(t *testing.T) {
	for _, enableFsnotify := range []bool{false, true} {
		t.Run(fmt.Sprintf("fsnotify=%v", enableFsnotify), func(t *testing.T) {
			tmpDir, rmTmpDir := testutil.TestTempDir(t)
			defer rmTmpDir()

			logDir := path.Join(tmpDir, "logs")
			progDir := path.Join(tmpDir, "progs")
			err := os.Mkdir(logDir, 0700)
			if err != nil {
				t.Fatal(err)
			}
			err = os.Mkdir(progDir, 0700)
			if err != nil {
				t.Fatal(err)
			}
			defer testutil.TestChdir(t, logDir)()

			m, stopM := mtail.TestStartServer(t, 0, enableFsnotify, mtail.ProgramPath(progDir), mtail.LogPathPatterns("log.*"))
			defer stopM()

			{
				logCountCheck := mtail.ExpectMetricDeltaWithDeadline(t, m.Addr(), "log_count", 1, time.Minute)
				lineCountCheck := mtail.ExpectMetricDeltaWithDeadline(t, m.Addr(), "lines_total", 1, time.Minute)

				logFile := path.Join(logDir, "log.1.txt")
				f := testutil.TestOpenFile(t, logFile)

				n, err := f.WriteString("line 1\n")
				if err != nil {
					t.Fatal(err)
				}
				glog.Infof("Wrote %d bytes", n)
				testutil.FatalIfErr(t, f.Sync())

				var wg sync.WaitGroup
				wg.Add(2)
				go func() {
					defer wg.Done()
					logCountCheck()
				}()
				go func() {
					defer wg.Done()
					lineCountCheck()
				}()
				wg.Wait()
			}

			{

				logCountCheck := mtail.ExpectMetricDeltaWithDeadline(t, m.Addr(), "log_count", 1, time.Minute)
				lineCountCheck := mtail.ExpectMetricDeltaWithDeadline(t, m.Addr(), "lines_total", 1, time.Minute)

				logFile := path.Join(logDir, "log.2.txt")
				f := testutil.TestOpenFile(t, logFile)
				n, err := f.WriteString("line 1\n")
				if err != nil {
					t.Fatal(err)
				}
				glog.Infof("Wrote %d bytes", n)

				var wg sync.WaitGroup
				wg.Add(2)
				go func() {
					defer wg.Done()
					logCountCheck()
				}()
				go func() {
					defer wg.Done()
					lineCountCheck()
				}()
				wg.Wait()
			}
			{
				logCountCheck := mtail.ExpectMetricDeltaWithDeadline(t, m.Addr(), "log_count", 0, time.Minute)
				lineCountCheck := mtail.ExpectMetricDeltaWithDeadline(t, m.Addr(), "lines_total", 1, time.Minute)

				logFile := path.Join(logDir, "log.2.txt")
				f := testutil.TestOpenFile(t, logFile)
				n, err := f.WriteString("line 1\n")
				if err != nil {
					t.Fatal(err)
				}
				glog.Infof("Wrote %d bytes", n)

				var wg sync.WaitGroup
				wg.Add(2)
				go func() {
					defer wg.Done()
					logCountCheck()
				}()
				go func() {
					defer wg.Done()
					lineCountCheck()
				}()
				wg.Wait()
			}

			glog.Infof("end")
		})
	}
}
<|endoftext|>


<commit_before>// Copyright 2019 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package spantest implements creation/destruction of a temporary Spanner
// database.
package spantest

import (
	"context"
	"crypto/rand"
	"encoding/binary"
	"fmt"
	"io/ioutil"
	"os/user"
	"regexp"
	"strings"

	"cloud.google.com/go/spanner"
	spandb "cloud.google.com/go/spanner/admin/database/apiv1"
	"golang.org/x/oauth2"
	"google.golang.org/api/option"
	dbpb "google.golang.org/genproto/googleapis/spanner/admin/database/v1"

	"go.chromium.org/luci/auth"
	"go.chromium.org/luci/common/errors"
	"go.chromium.org/luci/hardcoded/chromeinfra"
)

// TempDBConfig specifies how to create a temporary database.
type TempDBConfig struct {
	// InstanceName is the name of Spannner instance where to create the
	// temporary database.
	// Format: projects/{project}/instances/{instance}.
	// Defaults to chromeinfra.TestSpannerInstance.
	InstanceName string

	// TokenSource will be used to authenticate to Spanner.
	// If nil, auth.Authenticator with SilentLogin and chrome-infra auth options
	// will be used.
	// This means that that the user may have to login with luci-auth tool.
	TokenSource oauth2.TokenSource

	// InitScriptPath is a path to a DDL script to initialize the database.
	//
	// In lieu of a proper DDL parser, it is parsed using regexes.
	// Therefore the script MUST:
	//   - Use `#`` and/or `--`` for comments. No block comments.
	//   - Separate DDL statements with `;\n`.
	//
	// If empty, the database is created with no tables.
	InitScriptPath string
}

func (cfg *TempDBConfig) tokenSource(ctx context.Context) (oauth2.TokenSource, error) {
	if cfg.TokenSource != nil {
		return cfg.TokenSource, nil
	}

	opts := chromeinfra.DefaultAuthOptions()
	opts.Scopes = spandb.DefaultAuthScopes()
	a := auth.NewAuthenticator(ctx, auth.SilentLogin, opts)
	if err := a.CheckLoginRequired(); err != nil {
		return nil, errors.Annotate(err, "please login with `luci-auth login -scopes %q`", strings.Join(opts.Scopes, " ")).Err()
	}
	return a.TokenSource()
}

var ddlStatementSepRe = regexp.MustCompile(`;\s*\n`)
var commentRe = regexp.MustCompile(`(--|#)[^\n]*`)

// readDDLStatements read the file at cfg.InitScriptPath as a sequence of DDL
// statements. If the path is empty, returns (nil, nil).
func (cfg *TempDBConfig) readDDLStatements() ([]string, error) {
	if cfg.InitScriptPath == "" {
		return nil, nil
	}

	contents, err := ioutil.ReadFile(cfg.InitScriptPath)
	if err != nil {
		return nil, err
	}

	statements := ddlStatementSepRe.Split(string(contents), -1)
	ret := statements[:0]
	for _, stmt := range statements {
		stmt = commentRe.ReplaceAllString(stmt, "")
		stmt = strings.TrimSpace(stmt)
		if stmt != "" {
			ret = append(ret, stmt)
		}
	}
	return ret, nil
}

// TempDB is a temporary Spanner database.
type TempDB struct {
	Name string
	ts   oauth2.TokenSource
}

// Client returns a spanner client connected to the database.
func (db *TempDB) Client(ctx context.Context) (*spanner.Client, error) {
	return spanner.NewClient(ctx, db.Name, option.WithTokenSource(db.ts))
}

// Drop deletes the database.
func (db *TempDB) Drop(ctx context.Context) error {
	client, err := spandb.NewDatabaseAdminClient(ctx, option.WithTokenSource(db.ts))
	if err != nil {
		return err
	}
	defer client.Close()

	return client.DropDatabase(ctx, &dbpb.DropDatabaseRequest{
		Database: db.Name,
	})
}

var dbNameAlphabetInversedRe = regexp.MustCompile(`[^\w]+`)

// NewTempDB creates a temporary database with a random name.
// The caller is responsible for calling Drop on the returned TempDB to
// cleanup resources after usage.
func NewTempDB(ctx context.Context, cfg TempDBConfig) (*TempDB, error) {
	instanceName := cfg.InstanceName
	if instanceName == "" {
		instanceName = chromeinfra.TestSpannerInstance
	}

	ts, err := cfg.tokenSource(ctx)
	if err != nil {
		return nil, err
	}

	initStatements, err := cfg.readDDLStatements()
	if err != nil {
		return nil, errors.Annotate(err, "failed to read %q", cfg.InitScriptPath).Err()
	}

	client, err := spandb.NewDatabaseAdminClient(ctx, option.WithTokenSource(ts))
	if err != nil {
		return nil, err
	}
	defer client.Close()

	// Generate a random database name.
	var random uint32
	if err := binary.Read(rand.Reader, binary.LittleEndian, &random); err != nil {
		panic(err)
	}
	dbName := fmt.Sprintf("test%d", random)
	if u, err := user.Current(); err == nil && u.Username != "" {
		dbName += "_by_" + u.Username
	}
	dbName = SanitizeDBName(dbName)

	dbOp, err := client.CreateDatabase(ctx, &dbpb.CreateDatabaseRequest{
		Parent:          instanceName,
		CreateStatement: "CREATE DATABASE " + dbName,
		ExtraStatements: initStatements,
	})
	if err != nil {
		return nil, errors.Annotate(err, "failed to create database").Err()
	}
	db, err := dbOp.Wait(ctx)
	if err != nil {
		return nil, errors.Annotate(err, "failed to create database").Err()
	}

	return &TempDB{
		Name: db.Name,
		ts:   ts,
	}, nil
}

// SanitizeDBName tranforms name to a valid one.
// If name is already valid, returns it without changes.
func SanitizeDBName(name string) string {
	name = strings.ToLower(name)
	name = dbNameAlphabetInversedRe.ReplaceAllLiteralString(name, "_")
	name = strings.TrimRight(name, "_")
	const maxLen = 30
	if len(name) > maxLen {
		name = name[:maxLen]
	}
	return name
}
<commit_msg>[spantest] Trim _ after truncating<commit_after>// Copyright 2019 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package spantest implements creation/destruction of a temporary Spanner
// database.
package spantest

import (
	"context"
	"crypto/rand"
	"encoding/binary"
	"fmt"
	"io/ioutil"
	"os/user"
	"regexp"
	"strings"

	"cloud.google.com/go/spanner"
	spandb "cloud.google.com/go/spanner/admin/database/apiv1"
	"golang.org/x/oauth2"
	"google.golang.org/api/option"
	dbpb "google.golang.org/genproto/googleapis/spanner/admin/database/v1"

	"go.chromium.org/luci/auth"
	"go.chromium.org/luci/common/errors"
	"go.chromium.org/luci/hardcoded/chromeinfra"
)

// TempDBConfig specifies how to create a temporary database.
type TempDBConfig struct {
	// InstanceName is the name of Spannner instance where to create the
	// temporary database.
	// Format: projects/{project}/instances/{instance}.
	// Defaults to chromeinfra.TestSpannerInstance.
	InstanceName string

	// TokenSource will be used to authenticate to Spanner.
	// If nil, auth.Authenticator with SilentLogin and chrome-infra auth options
	// will be used.
	// This means that that the user may have to login with luci-auth tool.
	TokenSource oauth2.TokenSource

	// InitScriptPath is a path to a DDL script to initialize the database.
	//
	// In lieu of a proper DDL parser, it is parsed using regexes.
	// Therefore the script MUST:
	//   - Use `#`` and/or `--`` for comments. No block comments.
	//   - Separate DDL statements with `;\n`.
	//
	// If empty, the database is created with no tables.
	InitScriptPath string
}

func (cfg *TempDBConfig) tokenSource(ctx context.Context) (oauth2.TokenSource, error) {
	if cfg.TokenSource != nil {
		return cfg.TokenSource, nil
	}

	opts := chromeinfra.DefaultAuthOptions()
	opts.Scopes = spandb.DefaultAuthScopes()
	a := auth.NewAuthenticator(ctx, auth.SilentLogin, opts)
	if err := a.CheckLoginRequired(); err != nil {
		return nil, errors.Annotate(err, "please login with `luci-auth login -scopes %q`", strings.Join(opts.Scopes, " ")).Err()
	}
	return a.TokenSource()
}

var ddlStatementSepRe = regexp.MustCompile(`;\s*\n`)
var commentRe = regexp.MustCompile(`(--|#)[^\n]*`)

// readDDLStatements read the file at cfg.InitScriptPath as a sequence of DDL
// statements. If the path is empty, returns (nil, nil).
func (cfg *TempDBConfig) readDDLStatements() ([]string, error) {
	if cfg.InitScriptPath == "" {
		return nil, nil
	}

	contents, err := ioutil.ReadFile(cfg.InitScriptPath)
	if err != nil {
		return nil, err
	}

	statements := ddlStatementSepRe.Split(string(contents), -1)
	ret := statements[:0]
	for _, stmt := range statements {
		stmt = commentRe.ReplaceAllString(stmt, "")
		stmt = strings.TrimSpace(stmt)
		if stmt != "" {
			ret = append(ret, stmt)
		}
	}
	return ret, nil
}

// TempDB is a temporary Spanner database.
type TempDB struct {
	Name string
	ts   oauth2.TokenSource
}

// Client returns a spanner client connected to the database.
func (db *TempDB) Client(ctx context.Context) (*spanner.Client, error) {
	return spanner.NewClient(ctx, db.Name, option.WithTokenSource(db.ts))
}

// Drop deletes the database.
func (db *TempDB) Drop(ctx context.Context) error {
	client, err := spandb.NewDatabaseAdminClient(ctx, option.WithTokenSource(db.ts))
	if err != nil {
		return err
	}
	defer client.Close()

	return client.DropDatabase(ctx, &dbpb.DropDatabaseRequest{
		Database: db.Name,
	})
}

var dbNameAlphabetInversedRe = regexp.MustCompile(`[^\w]+`)

// NewTempDB creates a temporary database with a random name.
// The caller is responsible for calling Drop on the returned TempDB to
// cleanup resources after usage.
func NewTempDB(ctx context.Context, cfg TempDBConfig) (*TempDB, error) {
	instanceName := cfg.InstanceName
	if instanceName == "" {
		instanceName = chromeinfra.TestSpannerInstance
	}

	ts, err := cfg.tokenSource(ctx)
	if err != nil {
		return nil, err
	}

	initStatements, err := cfg.readDDLStatements()
	if err != nil {
		return nil, errors.Annotate(err, "failed to read %q", cfg.InitScriptPath).Err()
	}

	client, err := spandb.NewDatabaseAdminClient(ctx, option.WithTokenSource(ts))
	if err != nil {
		return nil, err
	}
	defer client.Close()

	// Generate a random database name.
	var random uint32
	if err := binary.Read(rand.Reader, binary.LittleEndian, &random); err != nil {
		panic(err)
	}
	dbName := fmt.Sprintf("test%d", random)
	if u, err := user.Current(); err == nil && u.Username != "" {
		dbName += "_by_" + u.Username
	}
	dbName = SanitizeDBName(dbName)

	dbOp, err := client.CreateDatabase(ctx, &dbpb.CreateDatabaseRequest{
		Parent:          instanceName,
		CreateStatement: "CREATE DATABASE " + dbName,
		ExtraStatements: initStatements,
	})
	if err != nil {
		return nil, errors.Annotate(err, "failed to create database").Err()
	}
	db, err := dbOp.Wait(ctx)
	if err != nil {
		return nil, errors.Annotate(err, "failed to create database").Err()
	}

	return &TempDB{
		Name: db.Name,
		ts:   ts,
	}, nil
}

// SanitizeDBName tranforms name to a valid one.
// If name is already valid, returns it without changes.
func SanitizeDBName(name string) string {
	name = strings.ToLower(name)
	name = dbNameAlphabetInversedRe.ReplaceAllLiteralString(name, "_")
	const maxLen = 30
	if len(name) > maxLen {
		name = name[:maxLen]
	}
	name = strings.TrimRight(name, "_")
	return name
}
<|endoftext|>


<commit_before>// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package ssa

// setloc sets the home location of v to loc.
func setloc(home []Location, v *Value, loc Location) []Location {
	for v.ID >= ID(len(home)) {
		home = append(home, nil)
	}
	home[v.ID] = loc
	return home
}

// stackalloc allocates storage in the stack frame for
// all Values that did not get a register.
func stackalloc(f *Func) {
	home := f.RegAlloc

	// Assign stack locations to phis first, because we
	// must also assign the same locations to the phi stores
	// introduced during regalloc.
	for _, b := range f.Blocks {
		for _, v := range b.Values {
			if v.Op != OpPhi {
				continue
			}
			if v.Type.IsMemory() { // TODO: only "regallocable" types
				continue
			}
			if int(v.ID) < len(home) && home[v.ID] != nil {
				continue // register-based phi
			}
			// stack-based phi
			n := f.Config.fe.Auto(v.Type)
			f.Logf("stackalloc: %s: for %v <%v>\n", n, v, v.Type)
			loc := &LocalSlot{n}
			home = setloc(home, v, loc)
			for _, w := range v.Args {
				if w.Op != OpStoreReg {
					f.Fatalf("stack-based phi must have StoreReg args")
				}
				home = setloc(home, w, loc)
			}
		}
	}

	// Now do all other unassigned values.
	for _, b := range f.Blocks {
		for _, v := range b.Values {
			if v.ID < ID(len(home)) && home[v.ID] != nil {
				continue
			}
			if v.Type.IsMemory() { // TODO: only "regallocable" types
				continue
			}
			if len(v.Args) == 0 {
				// v will have been materialized wherever it is needed.
				continue
			}
			if len(v.Args) == 1 && (v.Args[0].Op == OpSP || v.Args[0].Op == OpSB) {
				continue
			}

			n := f.Config.fe.Auto(v.Type)
			f.Logf("stackalloc: %s for %v\n", n, v)
			loc := &LocalSlot{n}
			home = setloc(home, v, loc)
		}
	}

	f.RegAlloc = home

	// TODO: share stack slots among noninterfering (& gc type compatible) values
}

// align increases n to the next multiple of a.  a must be a power of 2.
func align(n int64, a int64) int64 {
	if a == 0 {
		return n
	}
	return (n + a - 1) &^ (a - 1)
}
<commit_msg>[dev.ssa] cmd/compile: Reuse stack slots for spill locations<commit_after>// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// TODO: live at start of block instead?

package ssa

// stackalloc allocates storage in the stack frame for
// all Values that did not get a register.
func stackalloc(f *Func) {
	// Cache value types by ID.
	types := make([]Type, f.NumValues())
	for _, b := range f.Blocks {
		for _, v := range b.Values {
			types[v.ID] = v.Type
		}
	}

	// Build interference graph among StoreReg and stack phi ops.
	live := f.liveSpills()
	interfere := make([][]ID, f.NumValues())
	s := newSparseSet(f.NumValues())
	for _, b := range f.Blocks {
		// Start with known live values at the end of the block.
		s.clear()
		for i := 0; i < len(b.Succs); i++ {
			s.addAll(live[b.ID][i])
		}

		// Propagate backwards to the start of the block.
		// Remember interfering sets.
		for i := len(b.Values) - 1; i >= 0; i-- {
			v := b.Values[i]
			switch {
			case v.Op == OpStoreReg, v.isStackPhi():
				s.remove(v.ID)
				for _, id := range s.contents() {
					if v.Type == types[id] {
						interfere[v.ID] = append(interfere[v.ID], id)
						interfere[id] = append(interfere[id], v.ID)
					}
				}
			case v.Op == OpLoadReg:
				s.add(v.Args[0].ID)
			}
		}
	}

	// Figure out which StoreReg ops are phi args.  We don't pick slots for
	// phi args because a stack phi and its args must all use the same stack slot.
	phiArg := make([]bool, f.NumValues())
	for _, b := range f.Blocks {
		for _, v := range b.Values {
			if !v.isStackPhi() {
				continue
			}
			for _, a := range v.Args {
				phiArg[a.ID] = true
			}
		}
	}

	// For each type, we keep track of all the stack slots we
	// have allocated for that type.
	locations := map[Type][]*LocalSlot{}

	// Each time we assign a stack slot to a value v, we remember
	// the slot we used via an index into locations[v.Type].
	slots := make([]int, f.NumValues())
	for i := f.NumValues() - 1; i >= 0; i-- {
		slots[i] = -1
	}

	// Pick a stack slot for each non-phi-arg StoreReg and each stack phi.
	used := make([]bool, f.NumValues())
	for _, b := range f.Blocks {
		for _, v := range b.Values {
			if v.Op != OpStoreReg && !v.isStackPhi() {
				continue
			}
			if phiArg[v.ID] {
				continue
			}
			// Set of stack slots we could reuse.
			locs := locations[v.Type]
			// Mark all positions in locs used by interfering values.
			for i := 0; i < len(locs); i++ {
				used[i] = false
			}
			for _, xid := range interfere[v.ID] {
				slot := slots[xid]
				if slot >= 0 {
					used[slot] = true
				}
			}
			if v.Op == OpPhi {
				// Stack phi and args must get the same stack slot, so
				// anything they interfere with is something v the phi
				// interferes with.
				for _, a := range v.Args {
					for _, xid := range interfere[a.ID] {
						slot := slots[xid]
						if slot >= 0 {
							used[slot] = true
						}
					}
				}
			}
			// Find an unused stack slot.
			var i int
			for i = 0; i < len(locs); i++ {
				if !used[i] {
					break
				}
			}
			// If there is no unused stack slot, allocate a new one.
			if i == len(locs) {
				locs = append(locs, &LocalSlot{f.Config.fe.Auto(v.Type)})
				locations[v.Type] = locs
			}
			// Use the stack variable at that index for v.
			loc := locs[i]
			f.setHome(v, loc)
			slots[v.ID] = i
			if v.Op == OpPhi {
				for _, a := range v.Args {
					f.setHome(a, loc)
					slots[a.ID] = i
				}
			}
		}
	}
}

// live returns a map from block ID and successor edge index to a list
// of StoreReg/stackphi value IDs live on that edge.
// TODO: this could be quadratic if lots of variables are live across lots of
// basic blocks.  Figure out a way to make this function (or, more precisely, the user
// of this function) require only linear size & time.
func (f *Func) liveSpills() [][][]ID {
	live := make([][][]ID, f.NumBlocks())
	for _, b := range f.Blocks {
		live[b.ID] = make([][]ID, len(b.Succs))
	}
	var phis []*Value

	s := newSparseSet(f.NumValues())
	t := newSparseSet(f.NumValues())

	// Instead of iterating over f.Blocks, iterate over their postordering.
	// Liveness information flows backward, so starting at the end
	// increases the probability that we will stabilize quickly.
	po := postorder(f)
	for {
		changed := false
		for _, b := range po {
			// Start with known live values at the end of the block
			s.clear()
			for i := 0; i < len(b.Succs); i++ {
				s.addAll(live[b.ID][i])
			}

			// Propagate backwards to the start of the block
			phis = phis[:0]
			for i := len(b.Values) - 1; i >= 0; i-- {
				v := b.Values[i]
				switch {
				case v.Op == OpStoreReg:
					s.remove(v.ID)
				case v.Op == OpLoadReg:
					s.add(v.Args[0].ID)
				case v.isStackPhi():
					s.remove(v.ID)
					// save stack phi ops for later
					phis = append(phis, v)
				}
			}

			// for each predecessor of b, expand its list of live-at-end values
			// invariant: s contains the values live at the start of b (excluding phi inputs)
			for i, p := range b.Preds {
				// Find index of b in p's successors.
				var j int
				for j = 0; j < len(p.Succs); j++ {
					if p.Succs[j] == b {
						break
					}
				}
				t.clear()
				t.addAll(live[p.ID][j])
				t.addAll(s.contents())
				for _, v := range phis {
					t.add(v.Args[i].ID)
				}
				if t.size() == len(live[p.ID][j]) {
					continue
				}
				// grow p's live set
				live[p.ID][j] = append(live[p.ID][j][:0], t.contents()...)
				changed = true
			}
		}

		if !changed {
			break
		}
	}
	return live
}

func (f *Func) getHome(v *Value) Location {
	if int(v.ID) >= len(f.RegAlloc) {
		return nil
	}
	return f.RegAlloc[v.ID]
}

func (f *Func) setHome(v *Value, loc Location) {
	for v.ID >= ID(len(f.RegAlloc)) {
		f.RegAlloc = append(f.RegAlloc, nil)
	}
	f.RegAlloc[v.ID] = loc
}

func (v *Value) isStackPhi() bool {
	if v.Op != OpPhi {
		return false
	}
	if v.Type == TypeMem {
		return false
	}
	if int(v.ID) >= len(v.Block.Func.RegAlloc) {
		return true
	}
	return v.Block.Func.RegAlloc[v.ID] == nil
	// TODO: use a separate opcode for StackPhi?
}
<|endoftext|>


<commit_before>package network

import (
	"fmt"
	"net"

	"github.com/lxc/lxd/shared"
)

func Example_parseIPRange() {
	_, allowedv4NetworkA, _ := net.ParseCIDR("192.168.1.0/24")
	_, allowedv4NetworkB, _ := net.ParseCIDR("192.168.0.0/16")
	_, allowedv6NetworkA, _ := net.ParseCIDR("fd22:c952:653e:3df6::/64")
	_, allowedv6NetworkB, _ := net.ParseCIDR("fd22:c952:653e::/48")

	ipRanges := []string{
		// Ranges within allowedv4NetworkA.
		"192.168.1.1-192.168.1.255",
		"0.0.0.1-192.168.1.255",
		"0.0.0.1-0.0.0.255",
		// Ranges outsde of allowedv4NetworkA but within allowedv4NetworkB.
		"192.168.0.1-192.168.0.255",
		"192.168.0.0-192.168.0.0",
		"0.0.2.0-0.0.2.255",
		// Invalid IP ranges.
		"0.0.0.0.1-192.168.1.255",
		"192.0.0.1-192.0.0.255",
		"0.0.0.1-01.0.0.255",
		"0.0.2.1-0.0.0.255",
		// Ranges within allowedv6NetworkA.
		"fd22:c952:653e:3df6::1-fd22:c952:653e:3df6::FFFF",
		"::1-::FFFF",
		// Ranges outsde of allowedv6NetworkA but within allowedv6NetworkB.
		"fd22:c952:653e:FFFF::1-fd22:c952:653e:FFFF::FFFF",
		"::AAAA:FFFF:FFFF:FFFF:1-::AAAA:FFFF:FFFF:FFFF:FFFF",
	}

	fmt.Println("With allowed networks")
	for _, ipRange := range ipRanges {
		parsedRange, err := parseIPRange(ipRange, allowedv4NetworkA, allowedv4NetworkB, allowedv6NetworkA, allowedv6NetworkB)
		if err != nil {
			fmt.Printf("Err: %v\n", err)
			continue
		}

		fmt.Printf("Start: %s, End: %s\n", parsedRange.Start.String(), parsedRange.End.String())
	}

	fmt.Println("Without allowed networks")
	for _, ipRange := range ipRanges {
		parsedRange, err := parseIPRange(ipRange)
		if err != nil {
			fmt.Printf("Err: %v\n", err)
			continue
		}

		fmt.Printf("Start: %s, End: %s\n", parsedRange.Start.String(), parsedRange.End.String())
	}

	// Output: With allowed networks
	// Start: 192.168.1.1, End: 192.168.1.255
	// Start: 192.168.1.1, End: 192.168.1.255
	// Start: 192.168.1.1, End: 192.168.1.255
	// Start: 192.168.0.1, End: 192.168.0.255
	// Start: 192.168.0.0, End: 192.168.0.0
	// Start: 192.168.2.0, End: 192.168.2.255
	// Err: Start IP "0.0.0.0.1" is invalid
	// Err: IP range "192.0.0.1-192.0.0.255" does not fall within any of the allowed networks [192.168.1.0/24 192.168.0.0/16 fd22:c952:653e:3df6::/64 fd22:c952:653e::/48]
	// Err: IP range "0.0.0.1-01.0.0.255" does not fall within any of the allowed networks [192.168.1.0/24 192.168.0.0/16 fd22:c952:653e:3df6::/64 fd22:c952:653e::/48]
	// Err: Start IP "0.0.2.1" must be less than End IP "0.0.0.255"
	// Start: fd22:c952:653e:3df6::1, End: fd22:c952:653e:3df6::ffff
	// Start: fd22:c952:653e:3df6::1, End: fd22:c952:653e:3df6::ffff
	// Start: fd22:c952:653e:ffff::1, End: fd22:c952:653e:ffff::ffff
	// Start: fd22:c952:653e:aaaa:ffff:ffff:ffff:1, End: fd22:c952:653e:aaaa:ffff:ffff:ffff:ffff
	// Without allowed networks
	// Start: 192.168.1.1, End: 192.168.1.255
	// Start: 0.0.0.1, End: 192.168.1.255
	// Start: 0.0.0.1, End: 0.0.0.255
	// Start: 192.168.0.1, End: 192.168.0.255
	// Start: 192.168.0.0, End: 192.168.0.0
	// Start: 0.0.2.0, End: 0.0.2.255
	// Err: Start IP "0.0.0.0.1" is invalid
	// Start: 192.0.0.1, End: 192.0.0.255
	// Start: 0.0.0.1, End: 1.0.0.255
	// Err: Start IP "0.0.2.1" must be less than End IP "0.0.0.255"
	// Start: fd22:c952:653e:3df6::1, End: fd22:c952:653e:3df6::ffff
	// Start: ::1, End: ::ffff
	// Start: fd22:c952:653e:ffff::1, End: fd22:c952:653e:ffff::ffff
	// Start: ::aaaa:ffff:ffff:ffff:1, End: ::aaaa:ffff:ffff:ffff:ffff
}

func Example_ipRangesOverlap() {
	rangePairs := [][2]string{
		{"10.1.1.1-10.1.1.2", "10.1.1.3-10.1.1.4"},
		{"10.1.1.1-10.1.2.1", "10.1.1.254-10.1.1.255"},
		{"10.1.1.1-10.1.1.6", "10.1.1.5-10.1.1.9"},
		{"10.1.1.5-10.1.1.9", "10.1.1.1-10.1.1.6"},
		{"::1-::2", "::3-::4"},
		{"::1-::6", "::5-::9"},
		{"::5-::9", "::1-::6"},
	}

	for _, pair := range rangePairs {
		r0, _ := parseIPRange(pair[0])
		r1, _ := parseIPRange(pair[1])
		result := IPRangesOverlap(r0, r1)
		fmt.Printf("Range1: %v, Range2: %v, overlapped: %t\n", r0, r1, result)
	}

	// also do a couple of tests with ranges that have no end
	singleIPRange := &shared.IPRange{
		Start: net.ParseIP("10.1.1.4"),
	}
	otherRange, _ := parseIPRange("10.1.1.1-10.1.1.6")

	fmt.Printf("Range1: %v, Range2: %v, overlapped: %t\n", singleIPRange, otherRange, IPRangesOverlap(singleIPRange, otherRange))
	fmt.Printf("Range1: %v, Range2: %v, overlapped: %t\n", otherRange, singleIPRange, IPRangesOverlap(otherRange, singleIPRange))
	fmt.Printf("Range1: %v, Range2: %v, overlapped: %t\n", singleIPRange, singleIPRange, IPRangesOverlap(singleIPRange, singleIPRange))

	otherRange, _ = parseIPRange("10.1.1.8-10.1.1.9")

	fmt.Printf("Range1: %v, Range2: %v, overlapped: %t\n", singleIPRange, otherRange, IPRangesOverlap(singleIPRange, otherRange))
	fmt.Printf("Range1: %v, Range2: %v, overlapped: %t\n", otherRange, singleIPRange, IPRangesOverlap(otherRange, singleIPRange))

	// Output:
	// Range1: 10.1.1.1-10.1.1.2, Range2: 10.1.1.3-10.1.1.4, overlapped: false
	// Range1: 10.1.1.1-10.1.2.1, Range2: 10.1.1.254-10.1.1.255, overlapped: true
	// Range1: 10.1.1.1-10.1.1.6, Range2: 10.1.1.5-10.1.1.9, overlapped: true
	// Range1: 10.1.1.5-10.1.1.9, Range2: 10.1.1.1-10.1.1.6, overlapped: true
	// Range1: ::1-::2, Range2: ::3-::4, overlapped: false
	// Range1: ::1-::6, Range2: ::5-::9, overlapped: true
	// Range1: ::5-::9, Range2: ::1-::6, overlapped: true
	// Range1: 10.1.1.4, Range2: 10.1.1.1-10.1.1.6, overlapped: true
	// Range1: 10.1.1.1-10.1.1.6, Range2: 10.1.1.4, overlapped: true
	// Range1: 10.1.1.4, Range2: 10.1.1.4, overlapped: true
	// Range1: 10.1.1.4, Range2: 10.1.1.8-10.1.1.9, overlapped: false
	// Range1: 10.1.1.8-10.1.1.9, Range2: 10.1.1.4, overlapped: false

}
<commit_msg>lxd/network/network/utils/test: Fix tests in Example_parseIPRange to work with Go tip<commit_after>package network

import (
	"fmt"
	"net"

	"github.com/lxc/lxd/shared"
)

func Example_parseIPRange() {
	_, allowedv4NetworkA, _ := net.ParseCIDR("192.168.1.0/24")
	_, allowedv4NetworkB, _ := net.ParseCIDR("192.168.0.0/16")
	_, allowedv6NetworkA, _ := net.ParseCIDR("fd22:c952:653e:3df6::/64")
	_, allowedv6NetworkB, _ := net.ParseCIDR("fd22:c952:653e::/48")

	ipRanges := []string{
		// Ranges within allowedv4NetworkA.
		"192.168.1.1-192.168.1.255",
		"0.0.0.1-192.168.1.255",
		"0.0.0.1-0.0.0.255",
		// Ranges outsde of allowedv4NetworkA but within allowedv4NetworkB.
		"192.168.0.1-192.168.0.255",
		"192.168.0.0-192.168.0.0",
		"0.0.2.0-0.0.2.255",
		// Invalid IP ranges.
		"0.0.0.0.1-192.168.1.255",
		"192.0.0.1-192.0.0.255",
		"0.0.0.1-1.0.0.255",
		"0.0.2.1-0.0.0.255",
		// Ranges within allowedv6NetworkA.
		"fd22:c952:653e:3df6::1-fd22:c952:653e:3df6::FFFF",
		"::1-::FFFF",
		// Ranges outsde of allowedv6NetworkA but within allowedv6NetworkB.
		"fd22:c952:653e:FFFF::1-fd22:c952:653e:FFFF::FFFF",
		"::AAAA:FFFF:FFFF:FFFF:1-::AAAA:FFFF:FFFF:FFFF:FFFF",
	}

	fmt.Println("With allowed networks")
	for _, ipRange := range ipRanges {
		parsedRange, err := parseIPRange(ipRange, allowedv4NetworkA, allowedv4NetworkB, allowedv6NetworkA, allowedv6NetworkB)
		if err != nil {
			fmt.Printf("Err: %v\n", err)
			continue
		}

		fmt.Printf("Start: %s, End: %s\n", parsedRange.Start.String(), parsedRange.End.String())
	}

	fmt.Println("Without allowed networks")
	for _, ipRange := range ipRanges {
		parsedRange, err := parseIPRange(ipRange)
		if err != nil {
			fmt.Printf("Err: %v\n", err)
			continue
		}

		fmt.Printf("Start: %s, End: %s\n", parsedRange.Start.String(), parsedRange.End.String())
	}

	// Output: With allowed networks
	// Start: 192.168.1.1, End: 192.168.1.255
	// Start: 192.168.1.1, End: 192.168.1.255
	// Start: 192.168.1.1, End: 192.168.1.255
	// Start: 192.168.0.1, End: 192.168.0.255
	// Start: 192.168.0.0, End: 192.168.0.0
	// Start: 192.168.2.0, End: 192.168.2.255
	// Err: Start IP "0.0.0.0.1" is invalid
	// Err: IP range "192.0.0.1-192.0.0.255" does not fall within any of the allowed networks [192.168.1.0/24 192.168.0.0/16 fd22:c952:653e:3df6::/64 fd22:c952:653e::/48]
	// Err: IP range "0.0.0.1-1.0.0.255" does not fall within any of the allowed networks [192.168.1.0/24 192.168.0.0/16 fd22:c952:653e:3df6::/64 fd22:c952:653e::/48]
	// Err: Start IP "0.0.2.1" must be less than End IP "0.0.0.255"
	// Start: fd22:c952:653e:3df6::1, End: fd22:c952:653e:3df6::ffff
	// Start: fd22:c952:653e:3df6::1, End: fd22:c952:653e:3df6::ffff
	// Start: fd22:c952:653e:ffff::1, End: fd22:c952:653e:ffff::ffff
	// Start: fd22:c952:653e:aaaa:ffff:ffff:ffff:1, End: fd22:c952:653e:aaaa:ffff:ffff:ffff:ffff
	// Without allowed networks
	// Start: 192.168.1.1, End: 192.168.1.255
	// Start: 0.0.0.1, End: 192.168.1.255
	// Start: 0.0.0.1, End: 0.0.0.255
	// Start: 192.168.0.1, End: 192.168.0.255
	// Start: 192.168.0.0, End: 192.168.0.0
	// Start: 0.0.2.0, End: 0.0.2.255
	// Err: Start IP "0.0.0.0.1" is invalid
	// Start: 192.0.0.1, End: 192.0.0.255
	// Start: 0.0.0.1, End: 1.0.0.255
	// Err: Start IP "0.0.2.1" must be less than End IP "0.0.0.255"
	// Start: fd22:c952:653e:3df6::1, End: fd22:c952:653e:3df6::ffff
	// Start: ::1, End: ::ffff
	// Start: fd22:c952:653e:ffff::1, End: fd22:c952:653e:ffff::ffff
	// Start: ::aaaa:ffff:ffff:ffff:1, End: ::aaaa:ffff:ffff:ffff:ffff
}

func Example_ipRangesOverlap() {
	rangePairs := [][2]string{
		{"10.1.1.1-10.1.1.2", "10.1.1.3-10.1.1.4"},
		{"10.1.1.1-10.1.2.1", "10.1.1.254-10.1.1.255"},
		{"10.1.1.1-10.1.1.6", "10.1.1.5-10.1.1.9"},
		{"10.1.1.5-10.1.1.9", "10.1.1.1-10.1.1.6"},
		{"::1-::2", "::3-::4"},
		{"::1-::6", "::5-::9"},
		{"::5-::9", "::1-::6"},
	}

	for _, pair := range rangePairs {
		r0, _ := parseIPRange(pair[0])
		r1, _ := parseIPRange(pair[1])
		result := IPRangesOverlap(r0, r1)
		fmt.Printf("Range1: %v, Range2: %v, overlapped: %t\n", r0, r1, result)
	}

	// also do a couple of tests with ranges that have no end
	singleIPRange := &shared.IPRange{
		Start: net.ParseIP("10.1.1.4"),
	}
	otherRange, _ := parseIPRange("10.1.1.1-10.1.1.6")

	fmt.Printf("Range1: %v, Range2: %v, overlapped: %t\n", singleIPRange, otherRange, IPRangesOverlap(singleIPRange, otherRange))
	fmt.Printf("Range1: %v, Range2: %v, overlapped: %t\n", otherRange, singleIPRange, IPRangesOverlap(otherRange, singleIPRange))
	fmt.Printf("Range1: %v, Range2: %v, overlapped: %t\n", singleIPRange, singleIPRange, IPRangesOverlap(singleIPRange, singleIPRange))

	otherRange, _ = parseIPRange("10.1.1.8-10.1.1.9")

	fmt.Printf("Range1: %v, Range2: %v, overlapped: %t\n", singleIPRange, otherRange, IPRangesOverlap(singleIPRange, otherRange))
	fmt.Printf("Range1: %v, Range2: %v, overlapped: %t\n", otherRange, singleIPRange, IPRangesOverlap(otherRange, singleIPRange))

	// Output:
	// Range1: 10.1.1.1-10.1.1.2, Range2: 10.1.1.3-10.1.1.4, overlapped: false
	// Range1: 10.1.1.1-10.1.2.1, Range2: 10.1.1.254-10.1.1.255, overlapped: true
	// Range1: 10.1.1.1-10.1.1.6, Range2: 10.1.1.5-10.1.1.9, overlapped: true
	// Range1: 10.1.1.5-10.1.1.9, Range2: 10.1.1.1-10.1.1.6, overlapped: true
	// Range1: ::1-::2, Range2: ::3-::4, overlapped: false
	// Range1: ::1-::6, Range2: ::5-::9, overlapped: true
	// Range1: ::5-::9, Range2: ::1-::6, overlapped: true
	// Range1: 10.1.1.4, Range2: 10.1.1.1-10.1.1.6, overlapped: true
	// Range1: 10.1.1.1-10.1.1.6, Range2: 10.1.1.4, overlapped: true
	// Range1: 10.1.1.4, Range2: 10.1.1.4, overlapped: true
	// Range1: 10.1.1.4, Range2: 10.1.1.8-10.1.1.9, overlapped: false
	// Range1: 10.1.1.8-10.1.1.9, Range2: 10.1.1.4, overlapped: false

}
<|endoftext|>


<commit_before>/*
*  Copyright 2013 CoreOS, Inc
*  Copyright 2013 Docker Authors
*
*  Licensed under the Apache License, Version 2.0 (the "License");
*  you may not use this file except in compliance with the License.
*  You may obtain a copy of the License at
*
*      http://www.apache.org/licenses/LICENSE-2.0
*
*  Unless required by applicable law or agreed to in writing, software
*  distributed under the License is distributed on an "AS IS" BASIS,
*  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*  See the License for the specific language governing permissions and
*  limitations under the License.
 */

package main

import (
	"fmt"
	"github.com/dotcloud/docker"
	"github.com/dotcloud/docker/registry"
	"github.com/gorilla/mux"
	"log"
	"net/http"
	"os"
	"path"
	"regexp"
)

const ContainerDir = "/var/lib/containers/"

type Context struct {
	Path          string
	ContainerPath string
	Registry      *registry.Registry
	Graph         *docker.Graph
	Repositories  *docker.TagStore
}

var context Context

func pullImage(c *Context, imgId, registry string, token []string) error {
	history, err := c.Registry.GetRemoteHistory(imgId, registry, token)
	if err != nil {
		return err
	}

	// FIXME: Try to stream the images?
	// FIXME: Launch the getRemoteImage() in goroutines
	for _, id := range history {
		if !c.Graph.Exists(id) {
			log.Printf("Pulling %s metadata\r\n", id)
			imgJson, err := c.Registry.GetRemoteImageJson(id, registry, token)
			if err != nil {
				// FIXME: Keep goging in case of error?
				return err
			}
			img, err := docker.NewImgJson(imgJson)
			if err != nil {
				return fmt.Errorf("Failed to parse json: %s", err)
			}

			// Get the layer
			log.Printf("Pulling %s fs layer\r\n", img.Id)
			layer, _, err := c.Registry.GetRemoteImageLayer(img.Id, registry, token)
			if err != nil {
				return err
			}
			if err := c.Graph.Register(layer, false, img); err != nil {
				return err
			}
		}
	}
	return nil
}

// TODO: add tag support
func pullHandler(w http.ResponseWriter, r *http.Request, c *Context) {
	vars := mux.Vars(r)
	remote := vars["remote"]

	repoData, err := c.Registry.GetRepositoryData(remote)
	if err != nil {
		log.Fatal(err)
	}

	tagsList, err := c.Registry.GetRemoteTags(repoData.Endpoints, remote, repoData.Tokens)
	if err != nil {
		log.Fatal(err)
	}

	for tag, id := range tagsList {
		repoData.ImgList[id].Tag = tag
	}

	for _, img := range repoData.ImgList {
		log.Printf("Pulling image %s (%s) from %s\n", img.Id, img.Tag, remote)
		success := false

		for _, ep := range repoData.Endpoints {
			if err := pullImage(c, img.Id, "https://"+ep+"/v1", repoData.Tokens); err != nil {
				fmt.Printf("Error while retrieving image for tag: %s; checking next endpoint\n", err)
				continue
			}
			success = true
			break
		}

		if !success {
			log.Fatal("Could not find repository on any of the indexed registries.")
		}
	}

	for tag, id := range tagsList {
		if err := c.Repositories.Set(remote, tag, id, true); err != nil {
			log.Fatal(err)
			return
		}
	}
	if err := c.Repositories.Save(); err != nil {
		log.Fatal(err)
		return
	}

	fmt.Fprintf(w, "%v\n", repoData)
}

func createHandler(w http.ResponseWriter, r *http.Request, c *Context) {
	imageName := r.FormValue("image")

	// TODO: @philips Don't hardcode the tag name here
	image, err := c.Repositories.GetImage(imageName, "latest")
	if err != nil {
		log.Fatal(err)
		return
	}

	if imageName == "" {
		w.WriteHeader(404)
		fmt.Fprintf(w, "Cannot find container image: %s", imageName)
		return
	}

	// Figure out the resting place of the container
	vars := mux.Vars(r)
	container := vars["container"]

	validID := regexp.MustCompile(`^[A-Za-z0-9]+$`)
	if !validID.MatchString(container) {
		w.WriteHeader(400)
		fmt.Fprintf(w, "Invalid container name: %s\n", container)
		return
	}

	container = path.Join(c.ContainerPath, container)

	err = os.Mkdir(container, 0700)
	if os.IsExist(err) {
		w.WriteHeader(400)
		fmt.Fprintf(w, "Existing container: %s\n", container)
		return
	}
	if err != nil {
		w.WriteHeader(400)
		fmt.Fprint(w, "Error")
		log.Fatal(err)
		return
	}

	var images []docker.Image

	createList := func(img *docker.Image) (err error) {
		images = append(images, *img)
		return
	}
	err = image.WalkHistory(createList)

	for i := len(images) - 1; i >= 0; i-- {
		img := images[i]
		log.Printf("Copying %s into %s", img.Id, container)
		tarball, err := img.TarLayer(docker.Uncompressed)
		if err != nil {
			log.Fatal(err)
			return
		}
		if err := docker.Untar(tarball, container); err != nil {
			log.Fatal(err)
			return
		}
	}

	if err != nil {
		log.Fatal(err)
		return
	}

	fmt.Fprint(w, "ok")
	return
}

func setupDocker(r *mux.Router, o Options) {
	// Use the /var/lib/containers directory by default
	context.ContainerPath = path.Join(o.Dir, ContainerDir)
	if err := os.MkdirAll(context.ContainerPath, 0700); err != nil && !os.IsExist(err) {
		log.Fatal(err)
		return
	}
	context.Registry = registry.NewRegistry(context.ContainerPath)

	// Put all docker images into the docker directory
	context.Path = path.Join(o.Dir, StateDir, "docker")

	p := path.Join(context.Path, "graph")
	if err := os.MkdirAll(p, 0700); err != nil && !os.IsExist(err) {
		log.Fatal(err)
		return
	}
	g, _ := docker.NewGraph(p)
	context.Graph = g

	p = path.Join(context.Path, "repositories")
	t, _ := docker.NewTagStore(p, g)
	context.Repositories = t

	makeHandler := func(fn func(http.ResponseWriter, *http.Request, *Context)) http.HandlerFunc {
		return func(w http.ResponseWriter, r *http.Request) {
			fn(w, r, &context)
		}
	}

	r.HandleFunc("/registry/pull/{remote:.*}", makeHandler(pullHandler))
	r.HandleFunc("/container/create/{container:.*}", makeHandler(createHandler))
}
<commit_msg>add(docker): add a symlink to the unit automatically<commit_after>/*
*  Copyright 2013 CoreOS, Inc
*  Copyright 2013 Docker Authors
*
*  Licensed under the Apache License, Version 2.0 (the "License");
*  you may not use this file except in compliance with the License.
*  You may obtain a copy of the License at
*
*      http://www.apache.org/licenses/LICENSE-2.0
*
*  Unless required by applicable law or agreed to in writing, software
*  distributed under the License is distributed on an "AS IS" BASIS,
*  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*  See the License for the specific language governing permissions and
*  limitations under the License.
 */

package main

import (
	"fmt"
	"github.com/dotcloud/docker"
	"github.com/dotcloud/docker/registry"
	"github.com/gorilla/mux"
	"log"
	"net/http"
	"os"
	"path"
	"regexp"
)

const ContainerDir = "/var/lib/containers/"
const UnitTemplate = "/usr/lib/systemd/system/etcd@.service"
const UnitTargetFormat = "/etc/systemd/system/etcd@%s.service"

type Context struct {
	Path          string
	ContainerPath string
	Registry      *registry.Registry
	Graph         *docker.Graph
	Repositories  *docker.TagStore
}

var context Context

func pullImage(c *Context, imgId, registry string, token []string) error {
	history, err := c.Registry.GetRemoteHistory(imgId, registry, token)
	if err != nil {
		return err
	}

	// FIXME: Try to stream the images?
	// FIXME: Launch the getRemoteImage() in goroutines
	for _, id := range history {
		if !c.Graph.Exists(id) {
			log.Printf("Pulling %s metadata\r\n", id)
			imgJson, err := c.Registry.GetRemoteImageJson(id, registry, token)
			if err != nil {
				// FIXME: Keep goging in case of error?
				return err
			}
			img, err := docker.NewImgJson(imgJson)
			if err != nil {
				return fmt.Errorf("Failed to parse json: %s", err)
			}

			// Get the layer
			log.Printf("Pulling %s fs layer\r\n", img.Id)
			layer, _, err := c.Registry.GetRemoteImageLayer(img.Id, registry, token)
			if err != nil {
				return err
			}
			if err := c.Graph.Register(layer, false, img); err != nil {
				return err
			}
		}
	}
	return nil
}

// TODO: add tag support
func pullHandler(w http.ResponseWriter, r *http.Request, c *Context) {
	vars := mux.Vars(r)
	remote := vars["remote"]

	repoData, err := c.Registry.GetRepositoryData(remote)
	if err != nil {
		log.Fatal(err)
	}

	tagsList, err := c.Registry.GetRemoteTags(repoData.Endpoints, remote, repoData.Tokens)
	if err != nil {
		log.Fatal(err)
	}

	for tag, id := range tagsList {
		repoData.ImgList[id].Tag = tag
	}

	for _, img := range repoData.ImgList {
		log.Printf("Pulling image %s (%s) from %s\n", img.Id, img.Tag, remote)
		success := false

		for _, ep := range repoData.Endpoints {
			if err := pullImage(c, img.Id, "https://"+ep+"/v1", repoData.Tokens); err != nil {
				fmt.Printf("Error while retrieving image for tag: %s; checking next endpoint\n", err)
				continue
			}
			success = true
			break
		}

		if !success {
			log.Fatal("Could not find repository on any of the indexed registries.")
		}
	}

	for tag, id := range tagsList {
		if err := c.Repositories.Set(remote, tag, id, true); err != nil {
			log.Fatal(err)
			return
		}
	}
	if err := c.Repositories.Save(); err != nil {
		log.Fatal(err)
		return
	}

	fmt.Fprintf(w, "%v\n", repoData)
}

func createHandler(w http.ResponseWriter, r *http.Request, c *Context) {
	imageName := r.FormValue("image")

	// TODO: @philips Don't hardcode the tag name here
	image, err := c.Repositories.GetImage(imageName, "latest")
	if err != nil {
		log.Fatal(err)
		return
	}

	if imageName == "" {
		w.WriteHeader(404)
		fmt.Fprintf(w, "Cannot find container image: %s", imageName)
		return
	}

	// Figure out the resting place of the container
	vars := mux.Vars(r)
	container := vars["container"]

	validID := regexp.MustCompile(`^[A-Za-z0-9]+$`)
	if !validID.MatchString(container) {
		w.WriteHeader(400)
		fmt.Fprintf(w, "Invalid container name: %s\n", container)
		return
	}

	container = path.Join(c.ContainerPath, container)

	err = os.Mkdir(container, 0700)
	if os.IsExist(err) {
		w.WriteHeader(400)
		fmt.Fprintf(w, "Existing container: %s\n", container)
		return
	}
	if err != nil {
		w.WriteHeader(400)
		fmt.Fprint(w, "Error")
		log.Fatal(err)
		return
	}

	var images []docker.Image

	createList := func(img *docker.Image) (err error) {
		images = append(images, *img)
		return
	}
	err = image.WalkHistory(createList)

	for i := len(images) - 1; i >= 0; i-- {
		img := images[i]
		log.Printf("Copying %s into %s", img.Id, container)
		tarball, err := img.TarLayer(docker.Uncompressed)
		if err != nil {
			log.Fatal(err)
			return
		}
		if err := docker.Untar(tarball, container); err != nil {
			log.Fatal(err)
			return
		}
	}

	if err != nil {
		log.Fatal(err)
		return
	}

	// Symlink in the unit file so it can be started
	target := fmt.Sprintf(UnitTargetFormat, vars["container"])
	err = os.Symlink(UnitTemplate, target)

	if err != nil {
		fmt.Fprintf(w, "%s", err)
		return
	}

	fmt.Fprint(w, "ok")
	return
}

func setupDocker(r *mux.Router, o Options) {
	// Use the /var/lib/containers directory by default
	context.ContainerPath = path.Join(o.Dir, ContainerDir)
	if err := os.MkdirAll(context.ContainerPath, 0700); err != nil && !os.IsExist(err) {
		log.Fatal(err)
		return
	}
	context.Registry = registry.NewRegistry(context.ContainerPath)

	// Put all docker images into the docker directory
	context.Path = path.Join(o.Dir, StateDir, "docker")

	p := path.Join(context.Path, "graph")
	if err := os.MkdirAll(p, 0700); err != nil && !os.IsExist(err) {
		log.Fatal(err)
		return
	}
	g, _ := docker.NewGraph(p)
	context.Graph = g

	p = path.Join(context.Path, "repositories")
	t, _ := docker.NewTagStore(p, g)
	context.Repositories = t

	makeHandler := func(fn func(http.ResponseWriter, *http.Request, *Context)) http.HandlerFunc {
		return func(w http.ResponseWriter, r *http.Request) {
			fn(w, r, &context)
		}
	}

	r.HandleFunc("/registry/pull/{remote:.*}", makeHandler(pullHandler))
	r.HandleFunc("/container/create/{container:.*}", makeHandler(createHandler))
}
<|endoftext|>


<commit_before>// +build linux

package version

import (
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/osarch"
)

func getPlatformVersionStrings() []string {
	versions := []string{}

	// add kernel version
	uname, err := shared.Uname()
	if err == nil {
		versions = append(versions, uname.Release)
	}

	// add distribution info
	lsbRelease, err := osarch.GetLSBRelease()
	if err == nil {
		for _, key := range []string{"NAME", "VERSION_ID"} {
			value, ok := lsbRelease[key]
			if ok {
				versions = append(versions, value)
			}
		}
	}
	return versions
}
<commit_msg>version: Only include kernel version, not build id<commit_after>// +build linux

package version

import (
	"strings"

	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/osarch"
)

func getPlatformVersionStrings() []string {
	versions := []string{}

	// add kernel version
	uname, err := shared.Uname()
	if err == nil {
		versions = append(versions, strings.Split(uname.Release, "-")[0])
	}

	// add distribution info
	lsbRelease, err := osarch.GetLSBRelease()
	if err == nil {
		for _, key := range []string{"NAME", "VERSION_ID"} {
			value, ok := lsbRelease[key]
			if ok {
				versions = append(versions, value)
			}
		}
	}
	return versions
}
<|endoftext|>


<commit_before>// Copyright (C) 2014 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.

package model

import (
	"github.com/syncthing/syncthing/lib/config"
	"github.com/syncthing/syncthing/lib/db"
	"github.com/syncthing/syncthing/lib/events"
	"github.com/syncthing/syncthing/lib/ignore"
	"github.com/syncthing/syncthing/lib/protocol"
	"github.com/syncthing/syncthing/lib/versioner"
)

func init() {
	folderFactories[config.FolderTypeSendOnly] = newSendOnlyFolder
}

type sendOnlyFolder struct {
	folder
}

func newSendOnlyFolder(model *model, fset *db.FileSet, ignores *ignore.Matcher, cfg config.FolderConfiguration, _ versioner.Versioner, evLogger events.Logger, ioLimiter *byteSemaphore) service {
	f := &sendOnlyFolder{
		folder: newFolder(model, fset, ignores, cfg, evLogger, ioLimiter, nil),
	}
	f.folder.puller = f
	return f
}

func (f *sendOnlyFolder) PullErrors() []FileError {
	return nil
}

// pull checks need for files that only differ by metadata (no changes on disk)
func (f *sendOnlyFolder) pull() (bool, error) {
	batch := make([]protocol.FileInfo, 0, maxBatchSizeFiles)
	batchSizeBytes := 0

	snap, err := f.dbSnapshot()
	if err != nil {
		return false, err
	}
	defer snap.Release()
	snap.WithNeed(protocol.LocalDeviceID, func(intf protocol.FileIntf) bool {
		if len(batch) == maxBatchSizeFiles || batchSizeBytes > maxBatchSizeBytes {
			f.updateLocalsFromPulling(batch)
			batch = batch[:0]
			batchSizeBytes = 0
		}

		if f.ignores.ShouldIgnore(intf.FileName()) {
			file := intf.(protocol.FileInfo)
			file.SetIgnored()
			batch = append(batch, file)
			batchSizeBytes += file.ProtoSize()
			l.Debugln(f, "Handling ignored file", file)
			return true
		}

		curFile, ok := snap.Get(protocol.LocalDeviceID, intf.FileName())
		if !ok {
			if intf.IsDeleted() {
				l.Debugln("Should never get a deleted file as needed when we don't have it")
				f.evLogger.Log(events.Failure, "got deleted file that doesn't exist locally as needed when pulling on send-only")
			}
			return true
		}

		file := intf.(protocol.FileInfo)
		if !file.IsEquivalentOptional(curFile, f.modTimeWindow, f.IgnorePerms, false, 0) {
			return true
		}

		batch = append(batch, file)
		batchSizeBytes += file.ProtoSize()
		l.Debugln(f, "Merging versions of identical file", file)

		return true
	})

	if len(batch) > 0 {
		f.updateLocalsFromPulling(batch)
	}

	return true, nil
}

func (f *sendOnlyFolder) Override() {
	f.doInSync(f.override)
}

func (f *sendOnlyFolder) override() error {
	l.Infoln("Overriding global state on folder", f.Description())

	f.setState(FolderScanning)
	defer f.setState(FolderIdle)

	batch := make([]protocol.FileInfo, 0, maxBatchSizeFiles)
	batchSizeBytes := 0
	snap, err := f.dbSnapshot()
	if err != nil {
		return err
	}
	defer snap.Release()
	snap.WithNeed(protocol.LocalDeviceID, func(fi protocol.FileIntf) bool {
		need := fi.(protocol.FileInfo)
		if len(batch) == maxBatchSizeFiles || batchSizeBytes > maxBatchSizeBytes {
			f.updateLocalsFromScanning(batch)
			batch = batch[:0]
			batchSizeBytes = 0
		}

		have, ok := snap.Get(protocol.LocalDeviceID, need.Name)
		// Don't override files that are in a bad state (ignored,
		// unsupported, must rescan, ...).
		if ok && have.IsInvalid() {
			return true
		}
		if !ok || have.Name != need.Name {
			// We are missing the file
			need.SetDeleted(f.shortID)
		} else {
			// We have the file, replace with our version
			have.Version = have.Version.Merge(need.Version).Update(f.shortID)
			need = have
		}
		need.Sequence = 0
		batch = append(batch, need)
		batchSizeBytes += need.ProtoSize()
		return true
	})
	if len(batch) > 0 {
		f.updateLocalsFromScanning(batch)
	}
	return nil
}
<commit_msg>lib/model: Handle invalid needed items on send-only (ref #7476) (#7596)<commit_after>// Copyright (C) 2014 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.

package model

import (
	"github.com/syncthing/syncthing/lib/config"
	"github.com/syncthing/syncthing/lib/db"
	"github.com/syncthing/syncthing/lib/events"
	"github.com/syncthing/syncthing/lib/ignore"
	"github.com/syncthing/syncthing/lib/protocol"
	"github.com/syncthing/syncthing/lib/versioner"
)

func init() {
	folderFactories[config.FolderTypeSendOnly] = newSendOnlyFolder
}

type sendOnlyFolder struct {
	folder
}

func newSendOnlyFolder(model *model, fset *db.FileSet, ignores *ignore.Matcher, cfg config.FolderConfiguration, _ versioner.Versioner, evLogger events.Logger, ioLimiter *byteSemaphore) service {
	f := &sendOnlyFolder{
		folder: newFolder(model, fset, ignores, cfg, evLogger, ioLimiter, nil),
	}
	f.folder.puller = f
	return f
}

func (f *sendOnlyFolder) PullErrors() []FileError {
	return nil
}

// pull checks need for files that only differ by metadata (no changes on disk)
func (f *sendOnlyFolder) pull() (bool, error) {
	batch := newFileInfoBatch(func(files []protocol.FileInfo) error {
		f.updateLocalsFromPulling(files)
		return nil
	})

	snap, err := f.dbSnapshot()
	if err != nil {
		return false, err
	}
	defer snap.Release()
	snap.WithNeed(protocol.LocalDeviceID, func(intf protocol.FileIntf) bool {
		batch.flushIfFull()

		file := intf.(protocol.FileInfo)

		if f.ignores.ShouldIgnore(intf.FileName()) {
			file.SetIgnored()
			batch.append(file)
			l.Debugln(f, "Handling ignored file", file)
			return true
		}

		curFile, ok := snap.Get(protocol.LocalDeviceID, intf.FileName())
		if !ok {
			if intf.IsInvalid() {
				// Global invalid file just exists for need accounting
				batch.append(file)
			} else if intf.IsDeleted() {
				l.Debugln("Should never get a deleted file as needed when we don't have it")
				f.evLogger.Log(events.Failure, "got deleted file that doesn't exist locally as needed when pulling on send-only")
			}
			return true
		}

		if !file.IsEquivalentOptional(curFile, f.modTimeWindow, f.IgnorePerms, false, 0) {
			return true
		}

		batch.append(file)
		l.Debugln(f, "Merging versions of identical file", file)

		return true
	})

	batch.flush()

	return true, nil
}

func (f *sendOnlyFolder) Override() {
	f.doInSync(f.override)
}

func (f *sendOnlyFolder) override() error {
	l.Infoln("Overriding global state on folder", f.Description())

	f.setState(FolderScanning)
	defer f.setState(FolderIdle)

	batch := make([]protocol.FileInfo, 0, maxBatchSizeFiles)
	batchSizeBytes := 0
	snap, err := f.dbSnapshot()
	if err != nil {
		return err
	}
	defer snap.Release()
	snap.WithNeed(protocol.LocalDeviceID, func(fi protocol.FileIntf) bool {
		need := fi.(protocol.FileInfo)
		if len(batch) == maxBatchSizeFiles || batchSizeBytes > maxBatchSizeBytes {
			f.updateLocalsFromScanning(batch)
			batch = batch[:0]
			batchSizeBytes = 0
		}

		have, ok := snap.Get(protocol.LocalDeviceID, need.Name)
		// Don't override files that are in a bad state (ignored,
		// unsupported, must rescan, ...).
		if ok && have.IsInvalid() {
			return true
		}
		if !ok || have.Name != need.Name {
			// We are missing the file
			need.SetDeleted(f.shortID)
		} else {
			// We have the file, replace with our version
			have.Version = have.Version.Merge(need.Version).Update(f.shortID)
			need = have
		}
		need.Sequence = 0
		batch = append(batch, need)
		batchSizeBytes += need.ProtoSize()
		return true
	})
	if len(batch) > 0 {
		f.updateLocalsFromScanning(batch)
	}
	return nil
}
<|endoftext|>


<commit_before>package image

import (
	"encoding/json"
	"fmt"
	"math/rand"
	"reflect"
	"time"

	"github.com/astaxie/beego/orm"

	"bitbucket.org/ikeikeikeike/antenna/lib/cache"
)

/*
	Cache expires: 1 days
*/
func CachedSources(size string) (sources []string) {
	key := fmt.Sprintf("models.image.cachedfallbackimage:%s", size)
	s := reflect.ValueOf(cache.Client.Get(key))

	if !cache.Client.IsExist(key) {
		var q string

		excludes := `
		src not like '%xvideos.%' AND
		src not like '%erovideo.%' AND
		src not like '%redtube.%' AND
		src not like '%xhamster.%' AND
		src not like '%videomega.%' AND
		src not like '%fc2.png' AND
		src not like '%fc2.jpg'
		`

		if size == "small" {
			q = fmt.Sprintf(`SELECT src FROM image WHERE width < 300 AND width > 100 AND %s ORDER BY id DESC LIMIT 3000`, excludes)
			// WHERE width < 300 AND width > 100
		} else if size == "middle" {
			q = fmt.Sprint(`SELECT src FROM image WHERE width < 500 AND width > 300 AND  %s ORDER BY id DESC LIMIT 3000`, excludes)
			// width < 500 AND width > 300
		} else {
			q = fmt.Sprint(`SELECT src FROM image WHERE width > 500 AND				 	 %s ORDER BY id DESC LIMIT 3000`, excludes)
			// width > 500
		}

		var list orm.ParamsList
		orm.NewOrm().Raw(q).ValuesFlat(&list, "src")

		for _, src := range list {
			sources = append(sources, src.(string))
		}

		bytes, _ := json.Marshal(sources)
		cache.Client.Put(key, bytes, 60*60*24*1)
	} else {
		json.Unmarshal(s.Interface().([]uint8), &sources)
	}
	return
}

func CachedRandomSrc(size string) string {
	sources := CachedSources(size)
	return sources[randomNumber(1, len(sources)-1)]
}

func randomNumber(min, max int) int {
	rand.Seed(time.Now().UTC().UnixNano())
	return rand.Intn(max-min) + min
}
<commit_msg>Fixed exclude query in image cache<commit_after>package image

import (
	"encoding/json"
	"fmt"
	"math/rand"
	"reflect"
	"time"

	"github.com/astaxie/beego/orm"

	"bitbucket.org/ikeikeikeike/antenna/lib/cache"
)

/*
	Cache expires: 1 days
*/
func CachedSources(size string) (sources []string) {
	key := fmt.Sprintf("models.image.cachedfallbackimage:%s", size)
	s := reflect.ValueOf(cache.Client.Get(key))

	if !cache.Client.IsExist(key) {
		var q string

		excludes := `
		src not like '%xvideos.%' AND
		src not like '%erovideo.%' AND
		src not like '%redtube.%' AND
		src not like '%xhamster.%' AND
		src not like '%videomega.%' AND
		src not like '%fc2.png' AND
		src not like '%fc2.jpg'
		`

		if size == "small" {
			q = fmt.Sprintf(`SELECT src FROM image WHERE width < 300 AND width > 100 AND %s ORDER BY id DESC LIMIT 3000`, excludes)
			// WHERE width < 300 AND width > 100
		} else if size == "middle" {
			q = fmt.Sprintf(`SELECT src FROM image WHERE width < 500 AND width > 300 AND  %s ORDER BY id DESC LIMIT 3000`, excludes)
			// width < 500 AND width > 300
		} else {
			q = fmt.Sprintf(`SELECT src FROM image WHERE width > 500 AND				 	 %s ORDER BY id DESC LIMIT 3000`, excludes)
			// width > 500
		}

		var list orm.ParamsList
		orm.NewOrm().Raw(q).ValuesFlat(&list, "src")

		for _, src := range list {
			sources = append(sources, src.(string))
		}

		bytes, _ := json.Marshal(sources)
		cache.Client.Put(key, bytes, 60*60*24*1)
	} else {
		json.Unmarshal(s.Interface().([]uint8), &sources)
	}
	return
}

func CachedRandomSrc(size string) string {
	sources := CachedSources(size)
	return sources[randomNumber(1, len(sources)-1)]
}

func randomNumber(min, max int) int {
	rand.Seed(time.Now().UTC().UnixNano())
	return rand.Intn(max-min) + min
}
<|endoftext|>


<commit_before>// Copyright (c) 2016 PaperCut Software International Pty. Ltd.

package osutils

import (
	"syscall"
	"unsafe"
)

var (
	winhttp                               = syscall.NewLazyDLL("winhttp.dll")
	winHttpGetIEProxyConfigForCurrentUser = winhttp.NewProc("WinHttpGetIEProxyConfigForCurrentUser")
)

type winHttpCurrentUserIEProxyConfig struct {
	fAuthDetect       uint32
	lpszAutoConfigUrl *uint16
	lpszProxy         *uint16
	lpszProxyBypass   *uint16
}

func getHTTPProxy() (string, error) {
	proxyConfig := winHttpCurrentUserIEProxyConfig{}
	if r, _, e1 := winHttpGetIEProxyConfigForCurrentUser.Call(uintptr(unsafe.Pointer(&proxyConfig))); r == 0 {
		var err error
		if e1 != nil {
			err = e1
		} else {
			err = syscall.GetLastError()
		}
		return "", err
	}

	return ptrToString(proxyConfig.lpszProxy, 1024), nil
}

func ptrToString(p *uint16, maxLength int) string {
	if p == nil {
		return ""
	}
	// Hack! Cast to a large array pointer, then "pretend" to slice to maxLength
	slice := (*[0xffff]uint16)(unsafe.Pointer(p))[:maxLength]
	ret := syscall.UTF16ToString(slice)
	return ret
}
<commit_msg>Partial implementation of automatic proxy server detection<commit_after>// Copyright (c) 2016 PaperCut Software International Pty. Ltd.

package osutils

import (
	"fmt"
	"net/url"
	"syscall"
	"unsafe"
)

var (
	winhttp                               = syscall.NewLazyDLL("winhttp.dll")
	winHttpGetIEProxyConfigForCurrentUser = winhttp.NewProc("WinHttpGetIEProxyConfigForCurrentUser")
	winHttpGetProxyForUrl                 = winhttp.NewProc("WinHttpGetProxyForUrl")
	winHttpOpen                           = winhttp.NewProc("WinHttpOpen")
	winHttpSetTimeouts                    = winhttp.NewProc("WinHttpSetTimeouts")
)

type winHttpCurrentUserIEProxyConfig struct {
	fAuthDetect       uint32
	lpszAutoConfigUrl *uint16
	lpszProxy         *uint16
	lpszProxyBypass   *uint16
}

/* assumptions:
DWORD = unint32
LPVOID= *uint32
LPCWSTR = *uint32

*/
type winHttpProxyInfo struct {
	dwAccessType    uint32
	lpszProxy       *uint16
	lpszProxyBypass *uint16
}

type winHttpAutoProxyOptions struct {
	dwFlags                uint32
	dwAutoDetectFlags      uint32
	lpszAutoConfigUrl      *uint16
	lpvReserved            *uint32
	dwReserved             uint32
	fAutoLogonIfChallenged uint32
}

func getHTTPProxy() (string, error) {
	fmt.Printf("TEMP: testing\n")
	// TODO: config file proxyurl overrides this default url
	url, err := url.Parse("https://example.com")
	if err != nil {
		return "", err
	}

	proxy, err := getProxyConfigFromURL(url)
	if err != nil {
		return "", err
	} else if proxy != "" {
		fmt.Printf("proxy config from url %v\n", proxy)
		return proxy, nil
	}

	// TODO we should test the connection here.
	proxy, err = getProxyConfigForCurrentUser()
	if err != nil {
		return "", err
	} else if proxy != "" {
		fmt.Printf("proxy config from url %v\n", proxy)
		return proxy, nil
	}
	return proxy, nil
}

func getProxyConfigForCurrentUser() (string, error) {
	proxyConfig := winHttpCurrentUserIEProxyConfig{}
	if r, _, e1 := winHttpGetIEProxyConfigForCurrentUser.Call(uintptr(unsafe.Pointer(&proxyConfig))); r == 0 {
		var err error
		if e1 != nil {
			err = e1
		} else {
			err = syscall.GetLastError()
		}
		return "", err
	}

	return ptrToString(proxyConfig.lpszProxy, 1024), nil
}

func openWinHttpSession() (syscall.Handle, error) {
	const WINHTTP_ACCESS_TYPE_NO_PROXY = 1

	returnHandle, _, e1 := winHttpOpen.Call(uintptr(0), WINHTTP_ACCESS_TYPE_NO_PROXY, uintptr(0), uintptr(0), 0)
	hInternet := syscall.Handle(returnHandle)
	if returnHandle == 0 {
		var err error
		if e1 != nil {
			err = e1
		} else {
			err = syscall.GetLastError()
		}
		return syscall.InvalidHandle, err
	}

	//should we continue on error? only setting the defaults a bit lower from 1 min each stage
	if r, _, e1 := winHttpSetTimeouts.Call(uintptr(hInternet), 10000, 10000, 10000, 10000); r == 0 {

		var err error
		if e1 != nil {
			err = e1
		} else {
			err = syscall.GetLastError()
		}
		return syscall.InvalidHandle, err
	}
	return hInternet, nil
}

// TODO add more logging?

func getProxyConfigFromURL(url *url.URL) (string, error) {
	const WINHTTP_AUTOPROXY_AUTO_DETECT = 1
	const WINHTTP_AUTO_DETECT_TYPE_DHCP = 1
	const WINHTTP_AUTO_DETECT_TYPE_DNS_A = 2

	hSession, err := openWinHttpSession()
	if err != nil {
		return "", err
	}
	fmt.Println("querieng api")
	// FIXME need 32 bit but not there?
	//
	lpcwszUrl, err := syscall.UTF16PtrFromString(url.String())
	if err != nil {
		return "", err
	}

	autoProxyOptions := winHttpAutoProxyOptions{
		dwFlags:                WINHTTP_AUTOPROXY_AUTO_DETECT,
		dwAutoDetectFlags:      WINHTTP_AUTO_DETECT_TYPE_DHCP | WINHTTP_AUTO_DETECT_TYPE_DNS_A,
		fAutoLogonIfChallenged: 0,
	}

	proxyInfo := winHttpProxyInfo{}
	if r, _, e1 := winHttpGetProxyForUrl.Call(uintptr(hSession), uintptr(unsafe.Pointer(lpcwszUrl)), uintptr(unsafe.Pointer(&autoProxyOptions)), uintptr(unsafe.Pointer(&proxyInfo))); r == 0 {
		var err error
		if e1 != nil {
			err = e1
		} else {
			err = syscall.GetLastError()
		}
		return "", err
	}
	return ptrToString(proxyInfo.lpszProxy, 1024), nil
}

func ptrToString(p *uint16, maxLength int) string {
	if p == nil {
		return ""
	}
	// Hack! Cast to a large array pointer, then "pretend" to slice to maxLength
	slice := (*[0xffff]uint16)(unsafe.Pointer(p))[:maxLength]
	ret := syscall.UTF16ToString(slice)
	return ret
}
<|endoftext|>