<commit_before>package main

import (
	"time"

	"github.com/banzaicloud/bank-vaults/pkg/vault"
	"github.com/hashicorp/vault/api"
	"github.com/sirupsen/logrus"
	"github.com/spf13/cobra"
)

const cfgUnsealPeriod = "unseal-period"
const cfgInit = "init"

type unsealCfg struct {
	unsealPeriod time.Duration
	proceedInit  bool
}

var unsealConfig unsealCfg

var unsealCmd = &cobra.Command{
	Use:   "unseal",
	Short: "Unseals Vault with with unseal keys stored in one of the supported Cloud Provider options.",
	Long: `It will continuously attempt to unseal the target Vault instance, by retrieving unseal keys
from one of the followings:
- Google Cloud KMS keyring (backed by GCS)
- AWS KMS keyring (backed by S3)
- Azure Key Vault
- Kubernetes Secrets (should be used only for development purposes)`,
	Run: func(cmd *cobra.Command, args []string) {
		appConfig.BindPFlag(cfgUnsealPeriod, cmd.PersistentFlags().Lookup(cfgUnsealPeriod))
		appConfig.BindPFlag(cfgInit, cmd.PersistentFlags().Lookup(cfgInit))
		appConfig.BindPFlag(cfgInitRootToken, cmd.PersistentFlags().Lookup(cfgInitRootToken))
		appConfig.BindPFlag(cfgStoreRootToken, cmd.PersistentFlags().Lookup(cfgStoreRootToken))
		unsealConfig.unsealPeriod = appConfig.GetDuration(cfgUnsealPeriod)
		unsealConfig.proceedInit = appConfig.GetBool(cfgInit)

		store, err := kvStoreForConfig(appConfig)

		if err != nil {
			logrus.Fatalf("error creating kv store: %s", err.Error())
		}

		cl, err := api.NewClient(nil)

		if err != nil {
			logrus.Fatalf("error connecting to vault: %s", err.Error())
		}

		vaultConfig, err := vaultConfigForConfig(appConfig)

		if err != nil {
			logrus.Fatalf("error building vault config: %s", err.Error())
		}

		v, err := vault.New(store, cl, vaultConfig)

		if err != nil {
			logrus.Fatalf("error creating vault helper: %s", err.Error())
		}

		for {
			func() {
				if unsealConfig.proceedInit {
					logrus.Infof("initializing vault...")
					if err = v.Init(); err != nil {
						logrus.Fatalf("error initializing vault: %s", err.Error())
					} else {
						unsealConfig.proceedInit = false
					}
				}

				logrus.Infof("checking if vault is sealed...")
				sealed, err := v.Sealed()
				if err != nil {
					logrus.Errorf("error checking if vault is sealed: %s", err.Error())
					return
				}

				logrus.Infof("vault sealed: %t", sealed)

				// If vault is not sealed, we stop here and wait another unsealPeriod
				if !sealed {
					return
				}

				if err = v.Unseal(); err != nil {
					logrus.Errorf("error unsealing vault: %s", err.Error())
					return
				}

				logrus.Infof("successfully unsealed vault")
			}()
			// wait unsealPeriod before trying again
			time.Sleep(unsealConfig.unsealPeriod)
		}
	},
}

func init() {
	unsealCmd.PersistentFlags().Duration(cfgUnsealPeriod, time.Second*30, "How often to attempt to unseal the vault instance")
	unsealCmd.PersistentFlags().Bool(cfgInit, false, "Initialize vault instantce if not yet initialized")
	unsealCmd.PersistentFlags().String(cfgInitRootToken, "", "root token for the new vault cluster (only if -init=true)")
	unsealCmd.PersistentFlags().Bool(cfgStoreRootToken, true, "should the root token be stored in the key store (only if -init=true)")

	rootCmd.AddCommand(unsealCmd)
}
<commit_msg>unseal once<commit_after>package main

import (
	"os"
	"time"

	"github.com/banzaicloud/bank-vaults/pkg/vault"
	"github.com/hashicorp/vault/api"
	"github.com/sirupsen/logrus"
	"github.com/spf13/cobra"
)

const cfgUnsealPeriod = "unseal-period"
const cfgInit = "init"
const cfgOnce = "once"

type unsealCfg struct {
	unsealPeriod time.Duration
	proceedInit  bool
	runOnce      bool
}

var unsealConfig unsealCfg

var unsealCmd = &cobra.Command{
	Use:   "unseal",
	Short: "Unseals Vault with with unseal keys stored in one of the supported Cloud Provider options.",
	Long: `It will continuously attempt to unseal the target Vault instance, by retrieving unseal keys
from one of the followings:
- Google Cloud KMS keyring (backed by GCS)
- AWS KMS keyring (backed by S3)
- Azure Key Vault
- Kubernetes Secrets (should be used only for development purposes)`,
	Run: func(cmd *cobra.Command, args []string) {
		appConfig.BindPFlag(cfgUnsealPeriod, cmd.PersistentFlags().Lookup(cfgUnsealPeriod))
		appConfig.BindPFlag(cfgInit, cmd.PersistentFlags().Lookup(cfgInit))
		appConfig.BindPFlag(cfgOnce, cmd.PersistentFlags().Lookup(cfgOnce))
		appConfig.BindPFlag(cfgInitRootToken, cmd.PersistentFlags().Lookup(cfgInitRootToken))
		appConfig.BindPFlag(cfgStoreRootToken, cmd.PersistentFlags().Lookup(cfgStoreRootToken))
		unsealConfig.unsealPeriod = appConfig.GetDuration(cfgUnsealPeriod)
		unsealConfig.proceedInit = appConfig.GetBool(cfgInit)
		unsealConfig.runOnce = appConfig.GetBool(cfgOnce)

		store, err := kvStoreForConfig(appConfig)

		if err != nil {
			logrus.Fatalf("error creating kv store: %s", err.Error())
		}

		cl, err := api.NewClient(nil)

		if err != nil {
			logrus.Fatalf("error connecting to vault: %s", err.Error())
		}

		vaultConfig, err := vaultConfigForConfig(appConfig)

		if err != nil {
			logrus.Fatalf("error building vault config: %s", err.Error())
		}

		v, err := vault.New(store, cl, vaultConfig)

		if err != nil {
			logrus.Fatalf("error creating vault helper: %s", err.Error())
		}

		for {
			func() {
				if unsealConfig.proceedInit {
					logrus.Infof("initializing vault...")
					if err = v.Init(); err != nil {
						logrus.Fatalf("error initializing vault: %s", err.Error())
					} else {
						unsealConfig.proceedInit = false
					}
				}

				logrus.Infof("checking if vault is sealed...")
				sealed, err := v.Sealed()
				if err != nil {
					logrus.Errorf("error checking if vault is sealed: %s", err.Error())
					exitIfNecessary(1)
					return
				}

				logrus.Infof("vault sealed: %t", sealed)

				// If vault is not sealed, we stop here and wait another unsealPeriod
				if !sealed {
					exitIfNecessary(0)
					return
				}

				if err = v.Unseal(); err != nil {
					logrus.Errorf("error unsealing vault: %s", err.Error())
					exitIfNecessary(1)
					return
				}

				logrus.Infof("successfully unsealed vault")
				exitIfNecessary(0)
			}()

			// wait unsealPeriod before trying again
			time.Sleep(unsealConfig.unsealPeriod)
		}
	},
}

func exitIfNecessary(code int) {
	if unsealConfig.runOnce {
		os.Exit(code)
	}
}

func init() {
	unsealCmd.PersistentFlags().Duration(cfgUnsealPeriod, time.Second*30, "How often to attempt to unseal the vault instance")
	unsealCmd.PersistentFlags().Bool(cfgInit, false, "Initialize vault instantce if not yet initialized")
	unsealCmd.PersistentFlags().Bool(cfgOnce, false, "Run unseal only once")
	unsealCmd.PersistentFlags().String(cfgInitRootToken, "", "root token for the new vault cluster (only if -init=true)")
	unsealCmd.PersistentFlags().Bool(cfgStoreRootToken, true, "should the root token be stored in the key store (only if -init=true)")

	rootCmd.AddCommand(unsealCmd)
}
<|endoftext|>


<commit_before>package buse

import (
	"github.com/itchio/butler/cmd/launch/manifest"
	"github.com/itchio/butler/configurator"
	"github.com/itchio/butler/installer/bfs"
	itchio "github.com/itchio/go-itchio"
)

// must be kept in sync with clients, see for example
// https://github.com/itchio/node-butler

//----------------------------------------------------------------------
// Version
//----------------------------------------------------------------------

// Version.Get
type VersionGetParams struct{}

// Result for Version.Get
type VersionGetResult struct {
	// Something short, like `v8.0.0`
	Version string `json:"version"`

	// Something long, like `v8.0.0, built on Aug 27 2017 @ 01:13:55, ref d833cc0aeea81c236c81dffb27bc18b2b8d8b290`
	VersionString string `json:"versionString"`
}

//----------------------------------------------------------------------
// Game
//----------------------------------------------------------------------

// Game.FindUploads
type GameFindUploadsParams struct {
	Game        *itchio.Game     `json:"game"`
	Credentials *GameCredentials `json:"credentials"`
}

type GameFindUploadsResult struct {
	Uploads []*itchio.Upload `json:"uploads"`
}

//----------------------------------------------------------------------
// Operation
//----------------------------------------------------------------------

type Operation string

var (
	OperationInstall   Operation = "install"
	OperationUninstall Operation = "uninstall"
)

// Operation.Start
type OperationStartParams struct {
	ID            string    `json:"id"`
	StagingFolder string    `json:"stagingFolder"`
	Operation     Operation `json:"operation"`

	// this is more or less a union, the relevant field
	// should be set depending on the 'Operation' type
	InstallParams   *InstallParams   `json:"installParams,omitempty"`
	UninstallParams *UninstallParams `json:"uninstallParams,omitempty"`
}

// Operation.Cancel
type OperationCancelParams struct {
	ID string `json:"id"`
}

type OperationCancelResult struct{}

// InstallParams contains all the parameters needed to perform
// an installation for a game
type InstallParams struct {
	Game          *itchio.Game     `json:"game"`
	InstallFolder string           `json:"installFolder"`
	Upload        *itchio.Upload   `json:"upload"`
	Build         *itchio.Build    `json:"build"`
	Credentials   *GameCredentials `json:"credentials"`
}

type UninstallParams struct {
	InstallFolder string `json:"installFolder"`
}

// GameCredentials contains all the credentials required to make API requests
// including the download key if any
type GameCredentials struct {
	Server      string `json:"server"`
	APIKey      string `json:"apiKey"`
	DownloadKey int64  `json:"downloadKey"`
}

type PickUploadParams struct {
	Uploads []*itchio.Upload `json:"uploads"`
}

type PickUploadResult struct {
	Index int64 `json:"index"`
}

type GetReceiptParams struct {
	// muffin
}

type GetReceiptResult struct {
	Receipt *bfs.Receipt `json:"receipt"`
}

// Operation.Progress
// Sent periodically to inform on the current state an operation
type OperationProgressNotification struct {
	Progress float64 `json:"progress"`
	ETA      float64 `json:"eta"`
	BPS      float64 `json:"bps"`
}

type TaskReason string

const (
	TaskReasonInstall   TaskReason = "install"
	TaskReasonUninstall TaskReason = "uninstall"
)

type TaskType string

const (
	TaskTypeDownload  TaskType = "download"
	TaskTypeInstall   TaskType = "install"
	TaskTypeUninstall TaskType = "uninstall"
	TaskTypeUpdate    TaskType = "update"
	TaskTypeHeal      TaskType = "heal"
)

type TaskStartedNotification struct {
	Reason    TaskReason     `json:"reason"`
	Type      TaskType       `json:"type"`
	Game      *itchio.Game   `json:"game"`
	Upload    *itchio.Upload `json:"upload"`
	Build     *itchio.Build  `json:"build,omitempty"`
	TotalSize int64          `json:"totalSize,omitempty"`
}

type TaskSucceededNotification struct {
	Type TaskType `json:"type"`
	// If the task installed something, then this contains
	// info about the game, upload, build that were installed
	InstallResult *InstallResult `json:"installResult,omitempty"`
}

// Result for
//   - Operation.Start
type OperationResult struct{}

type InstallResult struct {
	Game   *itchio.Game   `json:"game"`
	Upload *itchio.Upload `json:"upload"`
	Build  *itchio.Build  `json:"build"`
	// TODO: verdict ?
}

//----------------------------------------------------------------------
// Launch
//----------------------------------------------------------------------

type LaunchParams struct {
	InstallFolder string                `json:"installFolder"`
	Game          *itchio.Game          `json:"game"`
	Upload        *itchio.Upload        `json:"upload"`
	Build         *itchio.Build         `json:"build"`
	Verdict       *configurator.Verdict `json:"verdict"`

	PrereqsDir   string `json:"prereqsDir"`
	ForcePrereqs bool   `json:"forcePrereqs,omitempty"`

	Sandbox bool `json:"sandbox,omitempty"`

	// Used for subkeying
	Credentials *GameCredentials `json:"credentials"`
}

type LaunchResult struct {
}

type LaunchRunningNotification struct{}

type LaunchExitedNotification struct{}

type PickManifestActionParams struct {
	Actions []*manifest.Action `json:"actions"`
}

type PickManifestActionResult struct {
	Name string `json:"name"`
}

type ShellLaunchParams struct {
	ItemPath string `json:"itemPath"`
}

type ShellLaunchResult struct {
}

type HTMLLaunchParams struct {
	RootFolder string `json:"rootFolder"`
	IndexPath  string `json:"indexPath"`

	Args []string          `json:"args"`
	Env  map[string]string `json:"env"`
}

type HTMLLaunchResult struct {
}

type URLLaunchParams struct {
	URL string `json:"url"`
}

type URLLaunchResult struct{}

type PrereqsStartedNotification struct {
	Tasks map[string]*PrereqTask `json:"tasks"`
}

type PrereqTask struct {
	FullName string `json:"fullName"`
	Order    int    `json:"order"`
}

type PrereqStatus string

const (
	PrereqStatusPending     PrereqStatus = "pending"
	PrereqStatusDownloading PrereqStatus = "downloading"
	PrereqStatusReady       PrereqStatus = "ready"
	PrereqStatusInstalling  PrereqStatus = "installing"
	PrereqStatusDone        PrereqStatus = "done"
)

type PrereqsTaskStateNotification struct {
	Name     string       `json:"name"`
	Status   PrereqStatus `json:"status"`
	Progress float64      `json:"progress"`
	ETA      float64      `json:"eta"`
	BPS      float64      `json:"bps"`
}

type PrereqsEndedNotification struct {
}

type PrereqsFailedParams struct {
	Error      string `json:"error"`
	ErrorStack string `json:"errorStack"`
}

type PrereqsFailedResult struct {
	Continue bool `json:"continue"`
}

//----------------------------------------------------------------------
// CleanDownloads
//----------------------------------------------------------------------

// CleanDownloads.Search
type CleanDownloadsSearchParams struct {
	// A list of folders to scan for potential subfolders to clean up
	Roots []string `json:"roots"`
	// A list of subfolders to not consider when cleaning
	// (staging folders for in-progress downloads)
	Whitelist []string `json:"whitelist"`
}

type CleanDownloadsSearchResult struct {
	Entries []*CleanDownloadsEntry `json:"entries"`
}

type CleanDownloadsEntry struct {
	Path string `json:"path"`
	Size int64  `json:"size"`
}

type CleanDownloadsApplyParams struct {
	Entries []*CleanDownloadsEntry `json:"entries"`
}

type CleanDownloadsApplyResult struct{}

//----------------------------------------------------------------------
// Misc.
//----------------------------------------------------------------------

// Log
type LogNotification struct {
	Level   string `json:"level"`
	Message string `json:"message"`
}

// Test.DoubleTwice
type TestDoubleTwiceRequest struct {
	Number int64 `json:"number"`
}

// Result for Test.DoubleTwice
type TestDoubleTwiceResult struct {
	Number int64 `json:"number"`
}

// Test.Double
type TestDoubleRequest struct {
	Number int64 `json:"number"`
}

// Result for Test.Double
type TestDoubleResult struct {
	Number int64 `json:"number"`
}

const (
	CodeOperationCancelled = 499
	CodeOperationAborted   = 410
)
<commit_msg>Add AllowSandboxSetup message<commit_after>package buse

import (
	"github.com/itchio/butler/cmd/launch/manifest"
	"github.com/itchio/butler/configurator"
	"github.com/itchio/butler/installer/bfs"
	itchio "github.com/itchio/go-itchio"
)

// must be kept in sync with clients, see for example
// https://github.com/itchio/node-butler

//----------------------------------------------------------------------
// Version
//----------------------------------------------------------------------

// Version.Get
type VersionGetParams struct{}

// Result for Version.Get
type VersionGetResult struct {
	// Something short, like `v8.0.0`
	Version string `json:"version"`

	// Something long, like `v8.0.0, built on Aug 27 2017 @ 01:13:55, ref d833cc0aeea81c236c81dffb27bc18b2b8d8b290`
	VersionString string `json:"versionString"`
}

//----------------------------------------------------------------------
// Game
//----------------------------------------------------------------------

// Game.FindUploads
type GameFindUploadsParams struct {
	Game        *itchio.Game     `json:"game"`
	Credentials *GameCredentials `json:"credentials"`
}

type GameFindUploadsResult struct {
	Uploads []*itchio.Upload `json:"uploads"`
}

//----------------------------------------------------------------------
// Operation
//----------------------------------------------------------------------

type Operation string

var (
	OperationInstall   Operation = "install"
	OperationUninstall Operation = "uninstall"
)

// Operation.Start
type OperationStartParams struct {
	ID            string    `json:"id"`
	StagingFolder string    `json:"stagingFolder"`
	Operation     Operation `json:"operation"`

	// this is more or less a union, the relevant field
	// should be set depending on the 'Operation' type
	InstallParams   *InstallParams   `json:"installParams,omitempty"`
	UninstallParams *UninstallParams `json:"uninstallParams,omitempty"`
}

// Operation.Cancel
type OperationCancelParams struct {
	ID string `json:"id"`
}

type OperationCancelResult struct{}

// InstallParams contains all the parameters needed to perform
// an installation for a game
type InstallParams struct {
	Game          *itchio.Game     `json:"game"`
	InstallFolder string           `json:"installFolder"`
	Upload        *itchio.Upload   `json:"upload"`
	Build         *itchio.Build    `json:"build"`
	Credentials   *GameCredentials `json:"credentials"`
}

type UninstallParams struct {
	InstallFolder string `json:"installFolder"`
}

// GameCredentials contains all the credentials required to make API requests
// including the download key if any
type GameCredentials struct {
	Server      string `json:"server"`
	APIKey      string `json:"apiKey"`
	DownloadKey int64  `json:"downloadKey"`
}

type PickUploadParams struct {
	Uploads []*itchio.Upload `json:"uploads"`
}

type PickUploadResult struct {
	Index int64 `json:"index"`
}

type GetReceiptParams struct {
	// muffin
}

type GetReceiptResult struct {
	Receipt *bfs.Receipt `json:"receipt"`
}

// Operation.Progress
// Sent periodically to inform on the current state an operation
type OperationProgressNotification struct {
	Progress float64 `json:"progress"`
	ETA      float64 `json:"eta"`
	BPS      float64 `json:"bps"`
}

type TaskReason string

const (
	TaskReasonInstall   TaskReason = "install"
	TaskReasonUninstall TaskReason = "uninstall"
)

type TaskType string

const (
	TaskTypeDownload  TaskType = "download"
	TaskTypeInstall   TaskType = "install"
	TaskTypeUninstall TaskType = "uninstall"
	TaskTypeUpdate    TaskType = "update"
	TaskTypeHeal      TaskType = "heal"
)

type TaskStartedNotification struct {
	Reason    TaskReason     `json:"reason"`
	Type      TaskType       `json:"type"`
	Game      *itchio.Game   `json:"game"`
	Upload    *itchio.Upload `json:"upload"`
	Build     *itchio.Build  `json:"build,omitempty"`
	TotalSize int64          `json:"totalSize,omitempty"`
}

type TaskSucceededNotification struct {
	Type TaskType `json:"type"`
	// If the task installed something, then this contains
	// info about the game, upload, build that were installed
	InstallResult *InstallResult `json:"installResult,omitempty"`
}

// Result for
//   - Operation.Start
type OperationResult struct{}

type InstallResult struct {
	Game   *itchio.Game   `json:"game"`
	Upload *itchio.Upload `json:"upload"`
	Build  *itchio.Build  `json:"build"`
	// TODO: verdict ?
}

//----------------------------------------------------------------------
// Launch
//----------------------------------------------------------------------

type LaunchParams struct {
	InstallFolder string                `json:"installFolder"`
	Game          *itchio.Game          `json:"game"`
	Upload        *itchio.Upload        `json:"upload"`
	Build         *itchio.Build         `json:"build"`
	Verdict       *configurator.Verdict `json:"verdict"`

	PrereqsDir   string `json:"prereqsDir"`
	ForcePrereqs bool   `json:"forcePrereqs,omitempty"`

	Sandbox bool `json:"sandbox,omitempty"`

	// Used for subkeying
	Credentials *GameCredentials `json:"credentials"`
}

type LaunchResult struct {
}

type LaunchRunningNotification struct{}

type LaunchExitedNotification struct{}

type PickManifestActionParams struct {
	Actions []*manifest.Action `json:"actions"`
}

type PickManifestActionResult struct {
	Name string `json:"name"`
}

type ShellLaunchParams struct {
	ItemPath string `json:"itemPath"`
}

type ShellLaunchResult struct {
}

type HTMLLaunchParams struct {
	RootFolder string `json:"rootFolder"`
	IndexPath  string `json:"indexPath"`

	Args []string          `json:"args"`
	Env  map[string]string `json:"env"`
}

type HTMLLaunchResult struct {
}

type URLLaunchParams struct {
	URL string `json:"url"`
}

type URLLaunchResult struct{}

type AllowSandboxSetupParams struct{}

type AllowSandboxSetupResponse struct{}

type PrereqsStartedNotification struct {
	Tasks map[string]*PrereqTask `json:"tasks"`
}

type PrereqTask struct {
	FullName string `json:"fullName"`
	Order    int    `json:"order"`
}

type PrereqStatus string

const (
	PrereqStatusPending     PrereqStatus = "pending"
	PrereqStatusDownloading PrereqStatus = "downloading"
	PrereqStatusReady       PrereqStatus = "ready"
	PrereqStatusInstalling  PrereqStatus = "installing"
	PrereqStatusDone        PrereqStatus = "done"
)

type PrereqsTaskStateNotification struct {
	Name     string       `json:"name"`
	Status   PrereqStatus `json:"status"`
	Progress float64      `json:"progress"`
	ETA      float64      `json:"eta"`
	BPS      float64      `json:"bps"`
}

type PrereqsEndedNotification struct {
}

type PrereqsFailedParams struct {
	Error      string `json:"error"`
	ErrorStack string `json:"errorStack"`
}

type PrereqsFailedResult struct {
	Continue bool `json:"continue"`
}

//----------------------------------------------------------------------
// CleanDownloads
//----------------------------------------------------------------------

// CleanDownloads.Search
type CleanDownloadsSearchParams struct {
	// A list of folders to scan for potential subfolders to clean up
	Roots []string `json:"roots"`
	// A list of subfolders to not consider when cleaning
	// (staging folders for in-progress downloads)
	Whitelist []string `json:"whitelist"`
}

type CleanDownloadsSearchResult struct {
	Entries []*CleanDownloadsEntry `json:"entries"`
}

type CleanDownloadsEntry struct {
	Path string `json:"path"`
	Size int64  `json:"size"`
}

type CleanDownloadsApplyParams struct {
	Entries []*CleanDownloadsEntry `json:"entries"`
}

type CleanDownloadsApplyResult struct{}

//----------------------------------------------------------------------
// Misc.
//----------------------------------------------------------------------

// Log
type LogNotification struct {
	Level   string `json:"level"`
	Message string `json:"message"`
}

// Test.DoubleTwice
type TestDoubleTwiceRequest struct {
	Number int64 `json:"number"`
}

// Result for Test.DoubleTwice
type TestDoubleTwiceResult struct {
	Number int64 `json:"number"`
}

// Test.Double
type TestDoubleRequest struct {
	Number int64 `json:"number"`
}

// Result for Test.Double
type TestDoubleResult struct {
	Number int64 `json:"number"`
}

const (
	CodeOperationCancelled = 499
	CodeOperationAborted   = 410
)
<|endoftext|>


<commit_before>package main

import (
	"compress/gzip"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strings"
	"time"

	"golang.org/x/sync/errgroup"

	"github.com/Debian/debiman/internal/archive"
	"github.com/Debian/debiman/internal/manpage"
	"pault.ag/go/debian/control"
)

// mostPopularArchitecture is used as preferred architecture when we
// need to pick an arbitrary architecture. The rationale is that
// downloading the package for the most popular architecture has the
// least bad influence on the mirror server’s caches.
const mostPopularArchitecture = "amd64"

type stats struct {
	PackagesExtracted uint64
	PackagesDeleted   uint64
	ManpagesRendered  uint64
	ManpageBytes      uint64
	HtmlBytes         uint64
	IndexBytes        uint64
}

type link struct {
	from string
	to   string
}

type globalView struct {
	pkgs          []*pkgEntry
	suites        map[string]bool
	idxSuites     map[string]string
	contentByPath map[string][]*contentEntry
	xref          map[string][]*manpage.Meta
	// alternatives maps from Debian binary package to a slice of
	// links (from→to pairs).
	alternatives map[string][]link
	stats        *stats
	start        time.Time
}

type distributionIdentifier int

const (
	fromCodename = iota
	fromSuite
)

type distribution struct {
	name       string
	identifier distributionIdentifier
}

// distributions returns a list of all distributions (either codenames
// [e.g. wheezy, jessie] or suites [e.g. testing, unstable]) from the
// -sync_codenames and -sync_suites flags.
func distributions(codenames []string, suites []string) []distribution {
	distributions := make([]distribution, 0, len(codenames)+len(suites))
	for _, e := range codenames {
		e = strings.TrimSpace(e)
		if e == "" {
			continue
		}
		distributions = append(distributions, distribution{
			name:       e,
			identifier: fromCodename})
	}
	for _, e := range suites {
		e = strings.TrimSpace(e)
		if e == "" {
			continue
		}
		distributions = append(distributions, distribution{
			name:       e,
			identifier: fromSuite})
	}
	return distributions
}

func parseAlternativesFile(fn, prefix string) (map[string][]link, error) {
	f, err := os.Open(fn)
	if err != nil {
		return nil, err
	}
	defer f.Close()
	r, err := gzip.NewReader(f)
	if err != nil {
		return nil, err
	}
	defer r.Close()
	res := make(map[string][]link)
	dec := json.NewDecoder(r)
	// read open bracket
	if _, err := dec.Token(); err != nil {
		return nil, err
	}
	for dec.More() {
		var m struct {
			Binpackage string
			From       string
			To         string
		}
		if err := dec.Decode(&m); err != nil {
			return nil, err
		}
		log.Printf("adding from %q to %q to pkg %q", m.From, m.To, m.Binpackage)
		key := prefix + "/" + m.Binpackage
		res[key] = append(res[key], link{
			from: m.From,
			to:   m.To,
		})
	}
	return res, nil
}

func parseAlternativesDir(dir string) (map[string][]link, error) {
	if dir == "" {
		return map[string][]link{}, nil
	}
	infos, err := ioutil.ReadDir(dir)
	if err != nil {
		return nil, err
	}
	results := make([]map[string][]link, len(infos))
	var eg errgroup.Group
	for idx, fi := range infos {
		idx, fi := idx, fi // copy
		eg.Go(func() error {
			suite := strings.TrimSuffix(fi.Name(), ".json.gz")
			res, err := parseAlternativesFile(filepath.Join(dir, fi.Name()), suite)
			results[idx] = res
			return err
		})
	}
	if err := eg.Wait(); err != nil {
		return nil, err
	}
	// Merge all subresults into one map. This is non-destructive
	// because the keys are prefixed by the Debian suite, which is
	// derived from the filename and hence unique.
	merged := make(map[string][]link)
	for idx := range infos {
		for key, val := range results[idx] {
			merged[key] = val
		}
	}
	return merged, nil
}

func markPresent(latestVersion map[string]*manpage.PkgMeta, xref map[string][]*manpage.Meta, filename string, key string) error {
	if _, ok := latestVersion[key]; !ok {
		return fmt.Errorf("Could not determine latest version")
	}
	m, err := manpage.FromManPath(strings.TrimPrefix(filename, "usr/share/man/"), latestVersion[key])
	if err != nil {
		return fmt.Errorf("Trying to interpret path %q: %v", filename, err)
	}
	// NOTE(stapelberg): this additional verification step
	// is necessary because manpages such as the French
	// manpage for qelectrotech(1) are present in multiple
	// encodings. manpageFromManPath ignores encodings, so
	// if we didn’t filter, we would end up with what
	// looks like duplicates.
	present := false
	for _, x := range xref[m.Name] {
		if x.ServingPath() == m.ServingPath() {
			present = true
			break
		}
	}
	if !present {
		xref[m.Name] = append(xref[m.Name], m)
	}
	return nil
}

func buildGlobalView(ar *archive.Getter, dists []distribution, alternativesDir string, start time.Time) (globalView, error) {
	var stats stats
	res := globalView{
		suites:        make(map[string]bool, len(dists)),
		idxSuites:     make(map[string]string, len(dists)),
		contentByPath: make(map[string][]*contentEntry),
		xref:          make(map[string][]*manpage.Meta),
		stats:         &stats,
		start:         start,
	}

	var err error
	res.alternatives, err = parseAlternativesDir(alternativesDir)
	if err != nil {
		return res, err
	}

	for _, dist := range dists {
		release, err := ar.GetRelease(dist.name)
		if err != nil {
			return res, err
		}

		var suite string
		if dist.identifier == fromCodename {
			suite = release.Codename
		} else {
			suite = release.Suite
		}

		res.suites[suite] = true
		res.idxSuites[release.Suite] = suite
		res.idxSuites[release.Codename] = suite
		res.idxSuites[dist.name] = suite

		hashByFilename := make(map[string]*control.SHA256FileHash, len(release.SHA256))
		for idx, fh := range release.SHA256 {
			// fh.Filename contains e.g. “non-free/source/Sources”
			hashByFilename[fh.Filename] = &(release.SHA256[idx])
		}

		content, err := getAllContents(ar, suite, release, hashByFilename)
		if err != nil {
			return res, err
		}

		for _, c := range content {
			res.contentByPath[c.filename] = append(res.contentByPath[c.filename], c)
		}

		var latestVersion map[string]*manpage.PkgMeta
		{
			// Collect package download work units
			var pkgs []*pkgEntry
			var err error
			pkgs, latestVersion, err = getAllPackages(ar, suite, release, hashByFilename, buildContainsMains(content, res.alternatives))
			if err != nil {
				return res, err
			}

			log.Printf("Adding %d packages from suite %q", len(pkgs), suite)
			res.pkgs = append(res.pkgs, pkgs...)
		}

		knownIssues := make(map[string][]error)

		// Build a global view of all the manpages (required for cross-referencing).
		// TODO(issue): edge case: packages which got renamed between releases
		for _, c := range content {
			key := c.suite + "/" + c.binarypkg
			if err := markPresent(latestVersion, res.xref, c.filename, key); err != nil {
				knownIssues[key] = append(knownIssues[key], err)
			}
		}

		for key, links := range res.alternatives {
			for _, link := range links {
				log.Printf("key=%q, link=%v, latest = %v", key, link, latestVersion[key])
				if err := markPresent(latestVersion, res.xref, strings.TrimPrefix(link.from, "/"), key); err != nil {
					knownIssues[key] = append(knownIssues[key], err)
				}
			}
		}

		for key, errors := range knownIssues {
			// TODO: write these to a known-issues file, parse bug numbers from an auxilliary file
			log.Printf("package %q has errors: %v", key, errors)
		}
	}
	return res, nil
}
<commit_msg>document globalView fields more extensively<commit_after>package main

import (
	"compress/gzip"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strings"
	"time"

	"golang.org/x/sync/errgroup"

	"github.com/Debian/debiman/internal/archive"
	"github.com/Debian/debiman/internal/manpage"
	"pault.ag/go/debian/control"
)

// mostPopularArchitecture is used as preferred architecture when we
// need to pick an arbitrary architecture. The rationale is that
// downloading the package for the most popular architecture has the
// least bad influence on the mirror server’s caches.
const mostPopularArchitecture = "amd64"

type stats struct {
	PackagesExtracted uint64
	PackagesDeleted   uint64
	ManpagesRendered  uint64
	ManpageBytes      uint64
	HtmlBytes         uint64
	IndexBytes        uint64
}

type link struct {
	from string
	to   string
}

type globalView struct {
	// pkgs contains all binary packages we know of.
	pkgs []*pkgEntry

	// suites contains the Debian suites that we know of. Can either be a codename or a suite,
	// depending on the values of -sync_codenames and -sync_suites.
	// e.g. “stretch” (codename) or “stable” (suite)
	suites map[string]bool

	// idxSuites maps codename, suite and command-line argument to suite (as in
	// suites).
	// e.g. map[oldoldstable:wheezy wheezy:wheezy]
	idxSuites map[string]string

	// contentByPath maps from paths underneath /usr/share/man to a contentEntry.
	contentByPath map[string][]*contentEntry

	// xref maps from manpage.Meta.Name (e.g. “w3m” or “systemd.service”) to
	// the corresponding manpage.Meta.
	xref map[string][]*manpage.Meta

	// alternatives maps from Debian binary package to a slice of
	// links (from→to pairs).
	alternatives map[string][]link

	stats *stats
	start time.Time
}

type distributionIdentifier int

const (
	fromCodename = iota
	fromSuite
)

type distribution struct {
	name       string
	identifier distributionIdentifier
}

// distributions returns a list of all distributions (either codenames
// [e.g. wheezy, jessie] or suites [e.g. testing, unstable]) from the
// -sync_codenames and -sync_suites flags.
func distributions(codenames []string, suites []string) []distribution {
	distributions := make([]distribution, 0, len(codenames)+len(suites))
	for _, e := range codenames {
		e = strings.TrimSpace(e)
		if e == "" {
			continue
		}
		distributions = append(distributions, distribution{
			name:       e,
			identifier: fromCodename})
	}
	for _, e := range suites {
		e = strings.TrimSpace(e)
		if e == "" {
			continue
		}
		distributions = append(distributions, distribution{
			name:       e,
			identifier: fromSuite})
	}
	return distributions
}

func parseAlternativesFile(fn, prefix string) (map[string][]link, error) {
	f, err := os.Open(fn)
	if err != nil {
		return nil, err
	}
	defer f.Close()
	r, err := gzip.NewReader(f)
	if err != nil {
		return nil, err
	}
	defer r.Close()
	res := make(map[string][]link)
	dec := json.NewDecoder(r)
	// read open bracket
	if _, err := dec.Token(); err != nil {
		return nil, err
	}
	for dec.More() {
		var m struct {
			Binpackage string
			From       string
			To         string
		}
		if err := dec.Decode(&m); err != nil {
			return nil, err
		}
		log.Printf("adding from %q to %q to pkg %q", m.From, m.To, m.Binpackage)
		key := prefix + "/" + m.Binpackage
		res[key] = append(res[key], link{
			from: m.From,
			to:   m.To,
		})
	}
	return res, nil
}

func parseAlternativesDir(dir string) (map[string][]link, error) {
	if dir == "" {
		return map[string][]link{}, nil
	}
	infos, err := ioutil.ReadDir(dir)
	if err != nil {
		return nil, err
	}
	results := make([]map[string][]link, len(infos))
	var eg errgroup.Group
	for idx, fi := range infos {
		idx, fi := idx, fi // copy
		eg.Go(func() error {
			suite := strings.TrimSuffix(fi.Name(), ".json.gz")
			res, err := parseAlternativesFile(filepath.Join(dir, fi.Name()), suite)
			results[idx] = res
			return err
		})
	}
	if err := eg.Wait(); err != nil {
		return nil, err
	}
	// Merge all subresults into one map. This is non-destructive
	// because the keys are prefixed by the Debian suite, which is
	// derived from the filename and hence unique.
	merged := make(map[string][]link)
	for idx := range infos {
		for key, val := range results[idx] {
			merged[key] = val
		}
	}
	return merged, nil
}

func markPresent(latestVersion map[string]*manpage.PkgMeta, xref map[string][]*manpage.Meta, filename string, key string) error {
	if _, ok := latestVersion[key]; !ok {
		return fmt.Errorf("Could not determine latest version")
	}
	m, err := manpage.FromManPath(strings.TrimPrefix(filename, "usr/share/man/"), latestVersion[key])
	if err != nil {
		return fmt.Errorf("Trying to interpret path %q: %v", filename, err)
	}
	// NOTE(stapelberg): this additional verification step
	// is necessary because manpages such as the French
	// manpage for qelectrotech(1) are present in multiple
	// encodings. manpageFromManPath ignores encodings, so
	// if we didn’t filter, we would end up with what
	// looks like duplicates.
	present := false
	for _, x := range xref[m.Name] {
		if x.ServingPath() == m.ServingPath() {
			present = true
			break
		}
	}
	if !present {
		xref[m.Name] = append(xref[m.Name], m)
	}
	return nil
}

func buildGlobalView(ar *archive.Getter, dists []distribution, alternativesDir string, start time.Time) (globalView, error) {
	var stats stats
	res := globalView{
		suites:        make(map[string]bool, len(dists)),
		idxSuites:     make(map[string]string, len(dists)),
		contentByPath: make(map[string][]*contentEntry),
		xref:          make(map[string][]*manpage.Meta),
		stats:         &stats,
		start:         start,
	}

	var err error
	res.alternatives, err = parseAlternativesDir(alternativesDir)
	if err != nil {
		return res, err
	}

	for _, dist := range dists {
		release, err := ar.GetRelease(dist.name)
		if err != nil {
			return res, err
		}

		var suite string
		if dist.identifier == fromCodename {
			suite = release.Codename // e.g. “stretch”
		} else {
			suite = release.Suite // e.g. “stable”
		}

		res.suites[suite] = true
		res.idxSuites[release.Suite] = suite
		res.idxSuites[release.Codename] = suite
		res.idxSuites[dist.name] = suite

		hashByFilename := make(map[string]*control.SHA256FileHash, len(release.SHA256))
		for idx, fh := range release.SHA256 {
			// fh.Filename contains e.g. “non-free/source/Sources”
			hashByFilename[fh.Filename] = &(release.SHA256[idx])
		}

		content, err := getAllContents(ar, suite, release, hashByFilename)
		if err != nil {
			return res, err
		}

		for _, c := range content {
			res.contentByPath[c.filename] = append(res.contentByPath[c.filename], c)
		}

		var latestVersion map[string]*manpage.PkgMeta
		{
			// Collect package download work units
			var pkgs []*pkgEntry
			var err error
			pkgs, latestVersion, err = getAllPackages(ar, suite, release, hashByFilename, buildContainsMains(content, res.alternatives))
			if err != nil {
				return res, err
			}

			log.Printf("Adding %d packages from suite %q", len(pkgs), suite)
			res.pkgs = append(res.pkgs, pkgs...)
		}

		knownIssues := make(map[string][]error)

		// Build a global view of all the manpages (required for cross-referencing).
		// TODO(issue): edge case: packages which got renamed between releases
		for _, c := range content {
			key := c.suite + "/" + c.binarypkg
			if err := markPresent(latestVersion, res.xref, c.filename, key); err != nil {
				knownIssues[key] = append(knownIssues[key], err)
			}
		}

		for key, links := range res.alternatives {
			for _, link := range links {
				log.Printf("key=%q, link=%v, latest = %v", key, link, latestVersion[key])
				if err := markPresent(latestVersion, res.xref, strings.TrimPrefix(link.from, "/"), key); err != nil {
					knownIssues[key] = append(knownIssues[key], err)
				}
			}
		}

		for key, errors := range knownIssues {
			// TODO: write these to a known-issues file, parse bug numbers from an auxilliary file
			log.Printf("package %q has errors: %v", key, errors)
		}
	}
	return res, nil
}
<|endoftext|>


<commit_before>package buse

import (
	"github.com/itchio/butler/installer/bfs"
	itchio "github.com/itchio/go-itchio"
)

// must be kept in sync with clients, see for example
// https://github.com/itchio/node-butler

//----------------------------------------------------------------------
// Version
//----------------------------------------------------------------------

// Version.Get
type VersionGetParams struct{}

// Result for Version.Get
type VersionGetResult struct {
	// Something short, like `v8.0.0`
	Version string `json:"version"`

	// Something long, like `v8.0.0, built on Aug 27 2017 @ 01:13:55, ref d833cc0aeea81c236c81dffb27bc18b2b8d8b290`
	VersionString string `json:"versionString"`
}

//----------------------------------------------------------------------
// Game
//----------------------------------------------------------------------

// Game.FindUploads
type GameFindUploadsParams struct {
	Game        *itchio.Game     `json:"game"`
	Credentials *GameCredentials `json:"credentials"`
}

type GameFindUploadsResult struct {
	Uploads []*itchio.Upload `json:"uploads"`
}

//----------------------------------------------------------------------
// Operation
//----------------------------------------------------------------------

type Operation string

var (
	OperationInstall   Operation = "install"
	OperationUninstall Operation = "uninstall"
)

// Operation.Start
type OperationStartParams struct {
	ID            string    `json:"id"`
	StagingFolder string    `json:"stagingFolder"`
	Operation     Operation `json:"operation"`

	// this is more or less a union, the relevant field
	// should be set depending on the 'Operation' type
	InstallParams   *InstallParams   `json:"installParams,omitempty"`
	UninstallParams *UninstallParams `json:"uninstallParams,omitempty"`
}

// Operation.Cancel
type OperationCancelParams struct {
	ID string `json:"id"`
}

type OperationCancelResult struct{}

// InstallParams contains all the parameters needed to perform
// an installation for a game
type InstallParams struct {
	Game          *itchio.Game     `json:"game"`
	InstallFolder string           `json:"installFolder"`
	Upload        *itchio.Upload   `json:"upload"`
	Build         *itchio.Build    `json:"build"`
	Credentials   *GameCredentials `json:"credentials"`
	Fresh         bool             `json:"fresh"`
}

type UninstallParams struct {
	InstallFolder string `json:"installFolder"`
}

// GameCredentials contains all the credentials required to make API requests
// including the download key if any
type GameCredentials struct {
	Server      string `json:"server"`
	APIKey      string `json:"apiKey"`
	DownloadKey int64  `json:"downloadKey"`
}

type PickUploadParams struct {
	Uploads []*itchio.Upload `json:"uploads"`
}

type PickUploadResult struct {
	Index int64 `json:"index"`
}

type GetReceiptParams struct {
	// muffin
}

type GetReceiptResult struct {
	Receipt *bfs.Receipt `json:"receipt"`
}

// Operation.Progress
// Sent periodically to inform on the current state an operation
type OperationProgressNotification struct {
	Progress float64 `json:"progress"`
	ETA      float64 `json:"eta"`
	BPS      float64 `json:"bps"`
}

type TaskReason string

const (
	TaskReasonInstall   TaskReason = "install"
	TaskReasonReinstall TaskReason = "reinstall"
	TaskReasonUpdate    TaskReason = "update"
	TaskReasonRevert    TaskReason = "revert"
	TaskReasonHeal      TaskReason = "heal"
	TaskReasonUninstall TaskReason = "uninstall"
)

type TaskType string

const (
	TaskTypeDownload  TaskType = "download"
	TaskTypeInstall   TaskType = "install"
	TaskTypeUninstall TaskType = "uninstall"
)

type TaskStartedNotification struct {
	Reason    TaskReason     `json:"reason"`
	Type      TaskType       `json:"type"`
	Game      *itchio.Game   `json:"game"`
	Upload    *itchio.Upload `json:"upload"`
	Build     *itchio.Build  `json:"build,omitempty"`
	TotalSize int64          `json:"totalSize,omitempty"`
}

type TaskEndedNotification struct {
	Type TaskType `json:"type"`
	// If the task installed something, then this contains
	// info about the game, upload, build that were installed
	InstallResult *InstallResult `json:"installResult,omitempty"`
}

// Result for
//   - Operation.Start
type OperationResult struct {
	ID           string `json:"id"`
	Success      bool   `json:"success"`
	ErrorMessage string `json:"errorMessage,omitempty"`
	ErrorStack   string `json:"errorStack,omitempty"`
}

type InstallResult struct {
	Game   *itchio.Game   `json:"game"`
	Upload *itchio.Upload `json:"upload"`
	Build  *itchio.Build  `json:"build"`
	// TODO: verdict ?
}

// Log
type LogNotification struct {
	Level   string `json:"level"`
	Message string `json:"message"`
}

// Test.DoubleTwice
type TestDoubleTwiceRequest struct {
	Number int64 `json:"number"`
}

// Result for Test.DoubleTwice
type TestDoubleTwiceResult struct {
	Number int64 `json:"number"`
}

// Test.Double
type TestDoubleRequest struct {
	Number int64 `json:"number"`
}

// Result for Test.Double
type TestDoubleResult struct {
	Number int64 `json:"number"`
}

const (
	CodeOperationCancelled = 499
)
<commit_msg>make 'update' and 'heal' task types, not reasons<commit_after>package buse

import (
	"github.com/itchio/butler/installer/bfs"
	itchio "github.com/itchio/go-itchio"
)

// must be kept in sync with clients, see for example
// https://github.com/itchio/node-butler

//----------------------------------------------------------------------
// Version
//----------------------------------------------------------------------

// Version.Get
type VersionGetParams struct{}

// Result for Version.Get
type VersionGetResult struct {
	// Something short, like `v8.0.0`
	Version string `json:"version"`

	// Something long, like `v8.0.0, built on Aug 27 2017 @ 01:13:55, ref d833cc0aeea81c236c81dffb27bc18b2b8d8b290`
	VersionString string `json:"versionString"`
}

//----------------------------------------------------------------------
// Game
//----------------------------------------------------------------------

// Game.FindUploads
type GameFindUploadsParams struct {
	Game        *itchio.Game     `json:"game"`
	Credentials *GameCredentials `json:"credentials"`
}

type GameFindUploadsResult struct {
	Uploads []*itchio.Upload `json:"uploads"`
}

//----------------------------------------------------------------------
// Operation
//----------------------------------------------------------------------

type Operation string

var (
	OperationInstall   Operation = "install"
	OperationUninstall Operation = "uninstall"
)

// Operation.Start
type OperationStartParams struct {
	ID            string    `json:"id"`
	StagingFolder string    `json:"stagingFolder"`
	Operation     Operation `json:"operation"`

	// this is more or less a union, the relevant field
	// should be set depending on the 'Operation' type
	InstallParams   *InstallParams   `json:"installParams,omitempty"`
	UninstallParams *UninstallParams `json:"uninstallParams,omitempty"`
}

// Operation.Cancel
type OperationCancelParams struct {
	ID string `json:"id"`
}

type OperationCancelResult struct{}

// InstallParams contains all the parameters needed to perform
// an installation for a game
type InstallParams struct {
	Game          *itchio.Game     `json:"game"`
	InstallFolder string           `json:"installFolder"`
	Upload        *itchio.Upload   `json:"upload"`
	Build         *itchio.Build    `json:"build"`
	Credentials   *GameCredentials `json:"credentials"`
	Fresh         bool             `json:"fresh"`
}

type UninstallParams struct {
	InstallFolder string `json:"installFolder"`
}

// GameCredentials contains all the credentials required to make API requests
// including the download key if any
type GameCredentials struct {
	Server      string `json:"server"`
	APIKey      string `json:"apiKey"`
	DownloadKey int64  `json:"downloadKey"`
}

type PickUploadParams struct {
	Uploads []*itchio.Upload `json:"uploads"`
}

type PickUploadResult struct {
	Index int64 `json:"index"`
}

type GetReceiptParams struct {
	// muffin
}

type GetReceiptResult struct {
	Receipt *bfs.Receipt `json:"receipt"`
}

// Operation.Progress
// Sent periodically to inform on the current state an operation
type OperationProgressNotification struct {
	Progress float64 `json:"progress"`
	ETA      float64 `json:"eta"`
	BPS      float64 `json:"bps"`
}

type TaskReason string

const (
	TaskReasonInstall   TaskReason = "install"
	TaskReasonUninstall TaskReason = "uninstall"
)

type TaskType string

const (
	TaskTypeDownload  TaskType = "download"
	TaskTypeInstall   TaskType = "install"
	TaskTypeUninstall TaskType = "uninstall"
	TaskTypeUpdate    TaskType = "update"
	TaskTypeHeal      TaskType = "heal"
)

type TaskStartedNotification struct {
	Reason    TaskReason     `json:"reason"`
	Type      TaskType       `json:"type"`
	Game      *itchio.Game   `json:"game"`
	Upload    *itchio.Upload `json:"upload"`
	Build     *itchio.Build  `json:"build,omitempty"`
	TotalSize int64          `json:"totalSize,omitempty"`
}

type TaskEndedNotification struct {
	Type TaskType `json:"type"`
	// If the task installed something, then this contains
	// info about the game, upload, build that were installed
	InstallResult *InstallResult `json:"installResult,omitempty"`
}

// Result for
//   - Operation.Start
type OperationResult struct {
	ID           string `json:"id"`
	Success      bool   `json:"success"`
	ErrorMessage string `json:"errorMessage,omitempty"`
	ErrorStack   string `json:"errorStack,omitempty"`
}

type InstallResult struct {
	Game   *itchio.Game   `json:"game"`
	Upload *itchio.Upload `json:"upload"`
	Build  *itchio.Build  `json:"build"`
	// TODO: verdict ?
}

// Log
type LogNotification struct {
	Level   string `json:"level"`
	Message string `json:"message"`
}

// Test.DoubleTwice
type TestDoubleTwiceRequest struct {
	Number int64 `json:"number"`
}

// Result for Test.DoubleTwice
type TestDoubleTwiceResult struct {
	Number int64 `json:"number"`
}

// Test.Double
type TestDoubleRequest struct {
	Number int64 `json:"number"`
}

// Result for Test.Double
type TestDoubleResult struct {
	Number int64 `json:"number"`
}

const (
	CodeOperationCancelled = 499
)
<|endoftext|>


<commit_before>package main_test

import (
	"os/exec"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gexec"
)

var _ = Describe("ARP Flusher", func() {
	var (
		binaryPath string
	)

	BeforeEach(func() {
		var err error
		binaryPath, err = gexec.Build("github.com/cloudfoundry-incubator/switchboard/cmd/flusharp")
		Expect(err).NotTo(HaveOccurred())
	})

	AfterEach(func() {
		gexec.CleanupBuildArtifacts()
	})

	It("raises an error if an argument is not provided", func() {
		command := exec.Command(
			binaryPath,
		)

		session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter)
		Expect(err).NotTo(HaveOccurred())

		Eventually(session).Should(gexec.Exit())
		Expect(session.ExitCode()).NotTo(BeZero())

		stderr := string(session.Err.Contents())
		Expect(stderr).To(ContainSubstring("arguments"))
	})

	It("raises an error if the ip is invalid", func() {
		command := exec.Command(
			binaryPath,
			"invalid-ip",
		)

		session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter)
		Expect(err).NotTo(HaveOccurred())

		Eventually(session).Should(gexec.Exit())
		Expect(session.ExitCode()).NotTo(BeZero())

		stderr := string(session.Err.Contents())
		Expect(stderr).To(ContainSubstring("invalid"))
	})

	It("raises an error when ARP flushing fails", func() {
		command := exec.Command(
			binaryPath,
			"192.0.2.1",
		)

		session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter)
		Expect(err).NotTo(HaveOccurred())

		Eventually(session).Should(gexec.Exit())
		Expect(session.ExitCode()).NotTo(BeZero())

		stderr := string(session.Err.Contents())
		Expect(stderr).To(ContainSubstring("exit status 1"))
	})
})
<commit_msg>Fix unit test [#130781143]<commit_after>package main_test

import (
	"os/exec"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gexec"
)

var _ = Describe("ARP Flusher", func() {
	var (
		binaryPath string
	)

	BeforeEach(func() {
		var err error
		binaryPath, err = gexec.Build("github.com/cloudfoundry-incubator/switchboard/cmd/flusharp")
		Expect(err).NotTo(HaveOccurred())
	})

	AfterEach(func() {
		gexec.CleanupBuildArtifacts()
	})

	It("raises an error if an argument is not provided", func() {
		command := exec.Command(
			binaryPath,
		)

		session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter)
		Expect(err).NotTo(HaveOccurred())

		Eventually(session).Should(gexec.Exit())
		Expect(session.ExitCode()).NotTo(BeZero())

		stderr := string(session.Err.Contents())
		Expect(stderr).To(ContainSubstring("arguments"))
	})

	It("raises an error if the ip is invalid", func() {
		command := exec.Command(
			binaryPath,
			"invalid-ip",
		)

		session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter)
		Expect(err).NotTo(HaveOccurred())

		Eventually(session).Should(gexec.Exit())
		Expect(session.ExitCode()).NotTo(BeZero())

		stderr := string(session.Err.Contents())
		Expect(stderr).To(ContainSubstring("invalid"))
	})

	It("raises an error when ARP flushing fails", func() {
		command := exec.Command(
			binaryPath,
			"192.0.2.1",
		)

		session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter)
		Expect(err).NotTo(HaveOccurred())

		Eventually(session).Should(gexec.Exit())
		Expect(session.ExitCode()).NotTo(BeZero())

		stderr := string(session.Err.Contents())
		Expect(stderr).To(ContainSubstring("exit status"))
	})
})
<|endoftext|>


<commit_before>package mmm

import (
	"testing"
	"unsafe"
)

// -----------------------------------------------------------------------------

func TestBytes_BytesOf_bool(t *testing.T) {
	var v bool = true
	size := unsafe.Sizeof(v)
	bytes := make([]byte, size)

	if err := BytesOf(v, bytes); err != nil {
		t.Error(err)
	}

	rv := *((*bool)(unsafe.Pointer(&(bytes[0]))))
	if v != rv {
		t.Error("invalid bytes for bool")
	}
}
<commit_msg>added TestBytes_BytesOf_int<commit_after>package mmm

import (
	"testing"
	"unsafe"
)

// -----------------------------------------------------------------------------

func TestBytes_BytesOf_bool(t *testing.T) {
	var v bool = true
	size := unsafe.Sizeof(v)
	bytes := make([]byte, size)

	if err := BytesOf(v, bytes); err != nil {
		t.Error(err)
	}

	rv := *((*bool)(unsafe.Pointer(&(bytes[0]))))
	if v != rv {
		t.Error("invalid bytes for bool")
	}
}

func TestBytes_BytesOf_int(t *testing.T) {
	var v int = 42
	size := unsafe.Sizeof(v)
	bytes := make([]byte, size)

	if err := BytesOf(v, bytes); err != nil {
		t.Error(err)
	}

	rv := *((*int)(unsafe.Pointer(&(bytes[0]))))
	if v != rv {
		t.Error("invalid bytes for bool")
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"flag"
	"io"
	"math/rand"
	"net"
	"net/http"
	"os"
	"os/exec"
	"path"
	"strings"
	"time"

	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/capabilities"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/client"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/client/record"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/clientauth"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/health"
	_ "github.com/GoogleCloudPlatform/kubernetes/pkg/healthz"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/kubelet"
	kconfig "github.com/GoogleCloudPlatform/kubernetes/pkg/kubelet/config"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/kubelet/dockertools"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/master/ports"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/util"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/version/verflag"
	"github.com/coreos/go-etcd/etcd"
	"github.com/fsouza/go-dockerclient"
	log "github.com/golang/glog"
	"github.com/mesos/mesos-go/mesos"
	"github.com/mesosphere/kubernetes-mesos/pkg/executor"
)

const (
	MESOS_CFG_SOURCE = "mesos" // @see ConfigSourceAnnotationKey
	defaultRootDir   = "/var/lib/kubelet"
)

var (
	syncFrequency           = flag.Duration("sync_frequency", 10*time.Second, "Max period between synchronizing running containers and config")
	address                 = util.IP(net.ParseIP("0.0.0.0"))
	port                    = flag.Uint("port", ports.KubeletPort, "The port for the info server to serve on") // TODO(jdef): use kmmaster.KubeletExecutorPort
	hostnameOverride        = flag.String("hostname_override", "", "If non-empty, will use this string as identification instead of the actual hostname.")
	networkContainerImage   = flag.String("network_container_image", kubelet.NetworkContainerImage, "The image that network containers in each pod will use.")
	dockerEndpoint          = flag.String("docker_endpoint", "", "If non-empty, use this for the docker endpoint to communicate with")
	etcdServerList          util.StringList
	etcdConfigFile          = flag.String("etcd_config", "", "The config file for the etcd client. Mutually exclusive with -etcd_servers")
	rootDirectory           = flag.String("root_dir", defaultRootDir, "Directory path for managing kubelet files (volume mounts,etc).")
	allowPrivileged         = flag.Bool("allow_privileged", false, "If true, allow containers to request privileged mode. [default=false]")
	registryPullQPS         = flag.Float64("registry_qps", 0.0, "If > 0, limit registry pull QPS to this value.  If 0, unlimited. [default=0.0]")
	registryBurst           = flag.Int("registry_burst", 10, "Maximum size of a bursty pulls, temporarily allows pulls to burst to this number, while still not exceeding registry_qps.  Only used if --registry_qps > 0")
	enableDebuggingHandlers = flag.Bool("enable_debugging_handlers", true, "Enables server endpoints for log collection and local running of containers and commands. Default true.")
	minimumGCAge            = flag.Duration("minimum_container_ttl_duration", 0, "Minimum age for a finished container before it is garbage collected.  Examples: '300ms', '10s' or '2h45m'")
	maxContainerCount       = flag.Int("maximum_dead_containers_per_container", 5, "Maximum number of old instances of a container to retain per container.  Each container takes up some disk space.  Default: 5.")
	authPath                = flag.String("auth_path", "", "Path to .kubernetes_auth file, specifying how to authenticate to API server.")
	apiServerList           util.StringList
	clusterDomain           = flag.String("cluster_domain", "", "Domain for this cluster.  If set, kubelet will configure all containers to search this domain in addition to the host's search domains")
	clusterDNS              = util.IP(nil)
)

func init() {
	flag.Var(&etcdServerList, "etcd_servers", "List of etcd servers to watch (http://ip:port), comma separated")
	flag.Var(&address, "address", "The IP address for the info and proxy servers to serve on. Default to 0.0.0.0.")
	flag.Var(&apiServerList, "api_servers", "List of Kubernetes API servers to publish events to. (ip:port), comma separated.")
	flag.Var(&clusterDNS, "cluster_dns", "IP address for a cluster DNS server.  If set, kubelet will configure all containers to use this for DNS resolution in addition to the host's DNS servers")
}

func getDockerEndpoint() string {
	var endpoint string
	if len(*dockerEndpoint) > 0 {
		endpoint = *dockerEndpoint
	} else if len(os.Getenv("DOCKER_HOST")) > 0 {
		endpoint = os.Getenv("DOCKER_HOST")
	} else {
		endpoint = "unix:///var/run/docker.sock"
	}
	log.Infof("Connecting to docker on %s", endpoint)

	return endpoint
}

func getHostname() string {
	hostname := []byte(*hostnameOverride)
	if string(hostname) == "" {
		// Note: We use exec here instead of os.Hostname() because we
		// want the FQDN, and this is the easiest way to get it.
		fqdn, err := exec.Command("hostname", "-f").Output()
		if err != nil {
			log.Fatalf("Couldn't determine hostname: %v", err)
		}
		hostname = fqdn
	}
	return strings.TrimSpace(string(hostname))
}

func getApiserverClient() (*client.Client, error) {
	clientConfig := client.Config{}
	if *authPath != "" {
		authInfo, err := clientauth.LoadFromFile(*authPath)
		if err != nil {
			return nil, err
		}
		clientConfig, err = authInfo.MergeWithConfig(clientConfig)
		if err != nil {
			return nil, err
		}
	}
	// TODO: adapt Kube client to support LB over several servers
	if len(apiServerList) > 1 {
		log.Infof("Mulitple api servers specified.  Picking first one")
	}
	clientConfig.Host = apiServerList[0]
	if c, err := client.New(&clientConfig); err != nil {
		return nil, err
	} else {
		return c, nil
	}
}

func main() {

	flag.Parse()
	util.InitLogs() // TODO(jdef) figure out where this actually sends logs by default
	defer util.FlushLogs()
	rand.Seed(time.Now().UTC().UnixNano())

	verflag.PrintAndExitIfRequested()

	etcd.SetLogger(util.NewLogger("etcd "))

	// Make an API client if possible.
	if len(apiServerList) < 1 {
		log.Info("No api servers specified.")
	} else {
		if apiClient, err := getApiserverClient(); err != nil {
			log.Errorf("Unable to make apiserver client: %v", err)
		} else {
			// Send events to APIserver if there is a client.
			log.Infof("Sending events to APIserver.")
			record.StartRecording(apiClient.Events(""), api.EventSource{Component: "kubelet"})
		}
	}

	// Log the events locally too.
	record.StartLogging(log.Infof)

	capabilities.Initialize(capabilities.Capabilities{
		AllowPrivileged: *allowPrivileged,
	})

	dockerClient, err := docker.NewClient(getDockerEndpoint())
	if err != nil {
		log.Fatal("Couldn't connnect to docker.")
	}

	hostname := getHostname()

	if *rootDirectory == "" {
		log.Fatal("Invalid root directory path.")
	}
	*rootDirectory = path.Clean(*rootDirectory)
	if err := os.MkdirAll(*rootDirectory, 0750); err != nil {
		log.Warningf("Error creating root directory: %v", err)
	}

	// source of all configuration
	cfg := kconfig.NewPodConfig(kconfig.PodConfigNotificationSnapshotAndUpdates)

	// k8sm: no other pod configuration sources supported in this hybrid kubelet-executor

	// define etcd config source and initialize etcd client
	var etcdClient *etcd.Client
	if len(etcdServerList) > 0 {
		etcdClient = etcd.NewClient(etcdServerList)
	} else if *etcdConfigFile != "" {
		var err error
		etcdClient, err = etcd.NewClientFromFile(*etcdConfigFile)
		if err != nil {
			log.Fatalf("Error with etcd config file: %v", err)
		}
	}

	if etcdClient != nil {
		log.Infof("Connected to etcd at %v", etcdClient.GetCluster())
	}

	// TODO(???): Destroy existing k8s containers for now - we don't know how to reconcile yet.
	if containers, err := dockertools.GetKubeletDockerContainers(dockerClient, true); err == nil {
		opts := docker.RemoveContainerOptions{
			RemoveVolumes: true,
			Force:         true,
		}
		for _, container := range containers {
			opts.ID = container.ID
			log.V(2).Infof("Removing container: %v", opts.ID)
			if err := dockerClient.RemoveContainer(opts); err != nil {
				log.Warning(err)
			}
		}
	} else {
		log.Warningf("Failed to list kubelet docker containers: %v", err)
	}

	// TODO(k8s): block until all sources have delivered at least one update to the channel, or break the sync loop
	// up into "per source" synchronizations

	kl := kubelet.NewMainKubelet(
		hostname,
		dockerClient,
		etcdClient,
		*rootDirectory,
		*networkContainerImage,
		*syncFrequency,
		float32(*registryPullQPS),
		*registryBurst,
		*minimumGCAge,
		*maxContainerCount,
		cfg.SeenAllSources,
		*clusterDomain,
		net.IP(clusterDNS))

	kl.BirthCry()

	go func() {
		util.Forever(func() {
			err := kl.GarbageCollectContainers()
			if err != nil {
				log.Errorf("Garbage collect failed: %v", err)
			}
		}, time.Minute*1)
	}()

	/* TODO(jdef): enable cadvisor integration
	go func() {
		defer util.HandleCrash()
		// TODO(k8s): Monitor this connection, reconnect if needed?
		log.V(1).Infof("Trying to create cadvisor client.")
		cadvisorClient, err := cadvisor.NewClient("http://127.0.0.1:4194")
		if err != nil {
			log.Errorf("Error on creating cadvisor client: %v", err)
			return
		}
		log.V(1).Infof("Successfully created cadvisor client.")
		kl.SetCadvisorClient(cadvisorClient)
	}()
	*/

	// TODO(k8s): These should probably become more plugin-ish: register a factory func
	// in each checker's init(), iterate those here.
	health.AddHealthChecker(health.NewExecHealthChecker(kl))
	health.AddHealthChecker(health.NewHTTPHealthChecker(&http.Client{}))
	health.AddHealthChecker(&health.TCPHealthChecker{})

	driver := new(mesos.MesosExecutorDriver)
	kubeletExecutor := executor.New(driver, kl, cfg.Channel(MESOS_CFG_SOURCE), MESOS_CFG_SOURCE)
	driver.Executor = kubeletExecutor

	log.V(2).Infof("Initialize executor driver...")
	driver.Init()
	defer driver.Destroy()

	log.V(2).Infof("Executor driver is running!")
	driver.Start()

	// TODO(who?) Recover running containers from check pointed pod list.

	// start the kubelet
	go util.Forever(func() { kl.Run(cfg.Updates()) }, 0)

	log.Infof("Starting kubelet server...")
	//TODO(jdef) when migrating to release 0.8+ the 10s re-try interval below is specified as
	// zero seconds in pkg/standalone/startKubelet. We don't want 0s because if we can't get
	// the port right away then we generate **tons** of logging and fill up the disk.
	go util.Forever(func() {
		// TODO(nnielsen): Don't hardwire port, but use port from resource offer.
		log.Error(executor.ListenAndServeKubeletServer(kl, net.IP(address), *port, *enableDebuggingHandlers, MESOS_CFG_SOURCE))
	}, 10*time.Second)

	go runProxyService()

	/*
		// TODO(nnielsen): Factor check-pointing into subsystem.
		dat, err := ioutil.ReadFile("/tmp/kubernetes-pods")
		if err == nil {
			var target []api.PodInfo
			err := json.Unmarshal(dat, &target)
			if err == nil {
				log.Infof("Checkpoint: '%v'", target)
			}
		}
	*/

	driver.Join()
}

// this function blocks as long as the proxy service is running; intended to be
// executed asynchronously.
func runProxyService() {
	// TODO(jdef): would be nice if we could run the proxy via an in-memory
	// kubelet config source (in case it crashes, kubelet would restart it);
	// not sure that k8s supports host-networking space for pods
	log.Infof("Starting proxy process...")

	args := []string{"-bind_address=" + address.String(), "-logtostderr=true", "-v=1"}
	if len(etcdServerList) > 0 {
		etcdServerArguments := strings.Join(etcdServerList, ",")
		args = append(args, "-etcd_servers="+etcdServerArguments)
	} else if *etcdConfigFile != "" {
		args = append(args, "-etcd_config="+*etcdConfigFile)
	}
	//TODO(jdef): don't hardcode name of the proxy executable here
	cmd := exec.Command("./kube-proxy", args...)
	_, err := cmd.StdoutPipe()
	if err != nil {
		log.Fatal(err)
	}
	proxylogs, err := cmd.StderrPipe()
	if err != nil {
		log.Fatal(err)
	}
	logfile, err := os.Create("./proxy-log")
	if err != nil {
		log.Fatal(err)
	}
	defer logfile.Close()
	writer := bufio.NewWriter(logfile)
	defer writer.Flush()

	if err := cmd.Start(); err != nil {
		log.Fatal(err)
	}
	defer func() {
		log.V(2).Infof("Cleaning up proxy process...")
		cmd.Process.Kill()
	}()
	io.Copy(writer, proxylogs)
}
<commit_msg>refactored to align with new upstream standalone-based implementation<commit_after>package main

import (
	"bufio"
	"flag"
	"io"
	"math/rand"
	"net"
	"os"
	"os/exec"
	"strings"
	"sync"
	"time"

	_ "github.com/GoogleCloudPlatform/kubernetes/pkg/healthz"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/kubelet"
	kconfig "github.com/GoogleCloudPlatform/kubernetes/pkg/kubelet/config"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/kubelet/dockertools"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/master/ports"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/standalone"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/util"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/version/verflag"
	"github.com/fsouza/go-dockerclient"
	log "github.com/golang/glog"
	"github.com/mesos/mesos-go/mesos"
	"github.com/mesosphere/kubernetes-mesos/pkg/executor"
)

const (
	MESOS_CFG_SOURCE = "mesos" // @see ConfigSourceAnnotationKey
	defaultRootDir   = "/var/lib/kubelet"
)

var (
	syncFrequency           = flag.Duration("sync_frequency", 10*time.Second, "Max period between synchronizing running containers and config")
	address                 = util.IP(net.ParseIP("0.0.0.0"))
	port                    = flag.Uint("port", ports.KubeletPort, "The port for the info server to serve on") // TODO(jdef): use kmmaster.KubeletExecutorPort
	hostnameOverride        = flag.String("hostname_override", "", "If non-empty, will use this string as identification instead of the actual hostname.")
	networkContainerImage   = flag.String("network_container_image", kubelet.NetworkContainerImage, "The image that network containers in each pod will use.")
	dockerEndpoint          = flag.String("docker_endpoint", "", "If non-empty, use this for the docker endpoint to communicate with")
	etcdServerList          util.StringList
	etcdConfigFile          = flag.String("etcd_config", "", "The config file for the etcd client. Mutually exclusive with -etcd_servers")
	rootDirectory           = flag.String("root_dir", defaultRootDir, "Directory path for managing kubelet files (volume mounts,etc).")
	allowPrivileged         = flag.Bool("allow_privileged", false, "If true, allow containers to request privileged mode. [default=false]")
	registryPullQPS         = flag.Float64("registry_qps", 0.0, "If > 0, limit registry pull QPS to this value.  If 0, unlimited. [default=0.0]")
	registryBurst           = flag.Int("registry_burst", 10, "Maximum size of a bursty pulls, temporarily allows pulls to burst to this number, while still not exceeding registry_qps.  Only used if --registry_qps > 0")
	enableDebuggingHandlers = flag.Bool("enable_debugging_handlers", true, "Enables server endpoints for log collection and local running of containers and commands. Default true.")
	minimumGCAge            = flag.Duration("minimum_container_ttl_duration", 0, "Minimum age for a finished container before it is garbage collected.  Examples: '300ms', '10s' or '2h45m'")
	maxContainerCount       = flag.Int("maximum_dead_containers_per_container", 5, "Maximum number of old instances of a container to retain per container.  Each container takes up some disk space.  Default: 5.")
	authPath                = flag.String("auth_path", "", "Path to .kubernetes_auth file, specifying how to authenticate to API server.")
	cAdvisorPort            = flag.Uint("cadvisor_port", 4194, "The port of the localhost cAdvisor endpoint")
	oomScoreAdj             = flag.Int("oom_score_adj", -900, "The oom_score_adj value for kubelet process. Values must be within the range [-1000, 1000]")
	apiServerList           util.StringList
	clusterDomain           = flag.String("cluster_domain", "", "Domain for this cluster.  If set, kubelet will configure all containers to search this domain in addition to the host's search domains")
	clusterDNS              = util.IP(nil)
)

func init() {
	flag.Var(&etcdServerList, "etcd_servers", "List of etcd servers to watch (http://ip:port), comma separated")
	flag.Var(&address, "address", "The IP address for the info and proxy servers to serve on. Default to 0.0.0.0.")
	flag.Var(&apiServerList, "api_servers", "List of Kubernetes API servers to publish events to. (ip:port), comma separated.")
	flag.Var(&clusterDNS, "cluster_dns", "IP address for a cluster DNS server.  If set, kubelet will configure all containers to use this for DNS resolution in addition to the host's DNS servers")
}

func main() {

	flag.Parse()
	util.InitLogs()
	defer util.FlushLogs()
	rand.Seed(time.Now().UTC().UnixNano())

	verflag.PrintAndExitIfRequested()

	if err := util.ApplyOomScoreAdj(*oomScoreAdj); err != nil {
		log.Info(err)
	}

	kcfg := standalone.KubeletConfig{
		Address:                 address,
		AuthPath:                *authPath,
		ApiServerList:           apiServerList,
		AllowPrivileged:         *allowPrivileged,
		HostnameOverride:        *hostnameOverride,
		RootDirectory:           *rootDirectory,
		NetworkContainerImage:   *networkContainerImage,
		SyncFrequency:           *syncFrequency,
		RegistryPullQPS:         *registryPullQPS,
		RegistryBurst:           *registryBurst,
		MinimumGCAge:            *minimumGCAge,
		MaxContainerCount:       *maxContainerCount,
		ClusterDomain:           *clusterDomain,
		ClusterDNS:              clusterDNS,
		Port:                    *port, // TODO(nnielsen): Don't hardwire port, but use port from resource offer.
		CAdvisorPort:            *cAdvisorPort,
		EnableServer:            true,
		EnableDebuggingHandlers: *enableDebuggingHandlers,
		DockerClient:            util.ConnectToDockerOrDie(*dockerEndpoint),
		EtcdClient:              kubelet.EtcdClientOrDie(etcdServerList, *etcdConfigFile),
	}

	driver := new(mesos.MesosExecutorDriver)

	//HACK(jdef) largely hacked from the k8s standalone package, createAndInitKubelet
	builder := standalone.KubeletBuilder(func(kc *standalone.KubeletConfig) (standalone.KubeletBootstrap, *kconfig.PodConfig) {
		pc := kconfig.NewPodConfig(kconfig.PodConfigNotificationSnapshotAndUpdates)
		updates := pc.Channel(MESOS_CFG_SOURCE)

		// TODO(k8s): block until all sources have delivered at least one update to the channel, or break the sync loop
		// up into "per source" synchronizations

		k := &kubeletExecutor{
			Kubelet: kubelet.NewMainKubelet(
				kc.Hostname,
				kc.DockerClient,
				kc.EtcdClient,
				kc.RootDirectory,
				kc.NetworkContainerImage,
				kc.SyncFrequency,
				float32(kc.RegistryPullQPS),
				kc.RegistryBurst,
				kc.MinimumGCAge,
				kc.MaxContainerCount,
				pc.SeenAllSources,
				kc.ClusterDomain,
				net.IP(kc.ClusterDNS)),
			driver: driver,
		}
		driver.Executor = executor.New(driver, k.Kubelet, updates, MESOS_CFG_SOURCE)

		log.V(2).Infof("Initialize executor driver...")
		driver.Init()

		k.BirthCry()
		k.reconcileTasks(kc.DockerClient)

		go k.GarbageCollectLoop()
		// go k.MonitorCAdvisor(kc.CAdvisorPort) // TODO(jdef) support cadvisor at some point

		k.InitHealthChecking()
		return k, pc
	})

	// create, initialize, and run kubelet services
	standalone.RunKubelet(&kcfg, builder)
	defer driver.Destroy()

	// block until driver is shut down
	driver.Join()
}

type kubeletExecutor struct {
	*kubelet.Kubelet
	driver     *mesos.MesosExecutorDriver
	initialize sync.Once
}

func (kl *kubeletExecutor) reconcileTasks(dockerClient dockertools.DockerInterface) {
	// TODO(jdef): Destroy existing k8s containers for now - we don't know how to reconcile yet.
	if containers, err := dockertools.GetKubeletDockerContainers(dockerClient, true); err == nil {
		opts := docker.RemoveContainerOptions{
			RemoveVolumes: true,
			Force:         true,
		}
		for _, container := range containers {
			opts.ID = container.ID
			log.V(2).Infof("Removing container: %v", opts.ID)
			if err := dockerClient.RemoveContainer(opts); err != nil {
				log.Warning(err)
			}
		}
	} else {
		log.Warningf("Failed to list kubelet docker containers: %v", err)
	}
}

func (kl *kubeletExecutor) ListenAndServe(address net.IP, port uint, enableDebuggingHandlers bool) {
	// this func could be called many times, depending how often the HTTP server crashes,
	// so only execute certain initialization procs once
	kl.initialize.Do(func() {
		go util.Forever(runProxyService, 5*time.Second)
		kl.driver.Start()
		log.V(2).Infof("Executor driver is running!")

		// TODO(who?) Recover running containers from check pointed pod list.
		// @see reconcileTasks
	})
	log.Infof("Starting kubelet server...")
	log.Error(executor.ListenAndServeKubeletServer(kl, address, port, enableDebuggingHandlers, MESOS_CFG_SOURCE))
}

// this function blocks as long as the proxy service is running; intended to be
// executed asynchronously.
func runProxyService() {
	// TODO(jdef): would be nice if we could run the proxy via an in-memory
	// kubelet config source (in case it crashes, kubelet would restart it);
	// not sure that k8s supports host-networking space for pods
	log.Infof("Starting proxy process...")

	args := []string{"-bind_address=" + address.String(), "-logtostderr=true", "-v=1"}
	if len(etcdServerList) > 0 {
		etcdServerArguments := strings.Join(etcdServerList, ",")
		args = append(args, "-etcd_servers="+etcdServerArguments)
	} else if *etcdConfigFile != "" {
		args = append(args, "-etcd_config="+*etcdConfigFile)
	}
	//TODO(jdef): don't hardcode name of the proxy executable here
	cmd := exec.Command("./kube-proxy", args...)
	_, err := cmd.StdoutPipe()
	if err != nil {
		log.Fatal(err)
	}
	proxylogs, err := cmd.StderrPipe()
	if err != nil {
		log.Fatal(err)
	}
	logfile, err := os.Create("./proxy-log")
	if err != nil {
		log.Fatal(err)
	}
	defer logfile.Close()
	writer := bufio.NewWriter(logfile)
	defer writer.Flush()

	if err := cmd.Start(); err != nil {
		log.Fatal(err)
	}
	defer func() {
		log.V(2).Infof("Cleaning up proxy process...")
		cmd.Process.Kill()
	}()
	io.Copy(writer, proxylogs)
}
<|endoftext|>


<commit_before>/*
Copyright 2020 Google LLC

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"flag"
	"io/ioutil"
	"os"
	"time"

	"github.com/golang/glog"
	"github.com/grafeas/kritis/pkg/attestlib"
	"github.com/grafeas/kritis/pkg/kritis/apis/kritis/v1beta1"
	"github.com/grafeas/kritis/pkg/kritis/crd/vulnzsigningpolicy"
	"github.com/grafeas/kritis/pkg/kritis/metadata/containeranalysis"
	"github.com/grafeas/kritis/pkg/kritis/signer"
	"github.com/grafeas/kritis/pkg/kritis/util"
	"k8s.io/apimachinery/pkg/util/yaml"
)

type SignerMode string

const (
	CheckAndSign  SignerMode = "check-and-sign"
	CheckOnly     SignerMode = "check-only"
	BypassAndSign SignerMode = "bypass-and-sign"
)

var (
	mode               string
	image              string
	vulnzTimeout       string
	policyPath         string
	attestationProject string
	overwrite          bool
	noteName           string
	// pgp key flags
	pgpPriKeyPath string
	pgpPassphrase string
	// kms flags
	kmsKeyName   string
	kmsDigestAlg string
)

func init() {
	flag.StringVar(&mode, "mode", "check-and-sign", "mode of operation, check-and-sign|check-only|bypass-and-sign")
	flag.StringVar(&image, "image", "", "image url, e.g., gcr.io/foo/bar@sha256:abcd")
	flag.StringVar(&vulnzTimeout, "vulnz_timeout", "5m", "timeout for polling image vulnerability , e.g., 600s, 5m")
	flag.StringVar(&pgpPriKeyPath, "pgp_private_key", "", "pgp private signing key path, e.g., /dev/shm/key.pgp")
	flag.StringVar(&pgpPassphrase, "pgp_passphrase", "", "passphrase for pgp private key, if any")
	flag.StringVar(&policyPath, "policy", "", "vulnerability signing policy file path, e.g., /tmp/vulnz_signing_policy.yaml")
	flag.StringVar(&noteName, "note_name", "", "note name that created attestations are attached to, in the form of projects/[PROVIDER_ID]/notes/[NOTE_ID]")
	flag.StringVar(&attestationProject, "attestation_project", "", "project id for GCP project that stores attestation, default to image project if unspecified")
	flag.BoolVar(&overwrite, "overwrite", false, "overwrite attestation if already existed, default to false")
	flag.StringVar(&kmsKeyName, "kms_key_name", "", "kms key name, in the format of in the format projects/*/locations/*/keyRings/*/cryptoKeys/*/cryptoKeyVersions/*")
	flag.StringVar(&kmsDigestAlg, "kms_digest_alg", "", "kms digest algorithm, must be one of SHA256|SHA384|SHA512, and the same as specified by the key version's algorithm")
}

func main() {
	flag.Parse()

	doCheck, doSign := false, false
	switch SignerMode(mode) {
	case CheckAndSign:
		doCheck, doSign = true, true
	case BypassAndSign:
		doSign = true
	case CheckOnly:
		doCheck = true
	default:
		glog.Fatalf("Unrecognized mode %s.", mode)
	}
	glog.Infof("Signer mode: %s.", mode)

	// Check image url is non-empty
	// TODO: check and format image url to
	//  gcr.io/project-id/rest-of-image-path@sha256:[sha-value]
	if image == "" {
		glog.Fatalf("Image url is empty: %s", image)
	}

	// Create a client
	client, err := containeranalysis.New()
	if err != nil {
		glog.Fatalf("Could not initialize the client %v", err)
	}

	if doCheck {
		// Read the vulnz signing policy
		policy := v1beta1.VulnzSigningPolicy{}
		policyFile, err := os.Open(policyPath)
		if err != nil {
			glog.Fatalf("Fail to load vulnz signing policy: %v", err)
		}
		defer policyFile.Close()
		// err = json.Unmarshal(policyFile, &policy)
		if err := yaml.NewYAMLToJSONDecoder(policyFile).Decode(&policy); err != nil {
			glog.Fatalf("Fail to parse policy file: %v", err)
		} else {
			glog.Infof("Policy req: %v\n", policy.Spec.ImageVulnerabilityRequirements)
		}

		timeout, err := time.ParseDuration(vulnzTimeout)
		if err != nil {
			glog.Fatalf("Fail to parse timeout %v", err)
		}
		err = client.WaitForVulnzAnalysis(image, timeout)
		if err != nil {
			glog.Fatalf("Error waiting for vulnerability analysis %v", err)
		}

		// Read the vulnz scanning events
		vulnz, err := client.Vulnerabilities(image)
		if err != nil {
			glog.Fatalf("Found err %s", err)
		}
		if vulnz == nil {
			glog.Fatalf("Expected some vulnerabilities. Nil found")
		}

		violations, err := vulnzsigningpolicy.ValidateVulnzSigningPolicy(policy, image, vulnz)
		if err != nil {
			glog.Fatalf("Error when evaluating image %q against policy %q", image, policy.Name)
		}
		if violations != nil && len(violations) != 0 {
			glog.Errorf("Image %q does not pass VulnzSigningPolicy %q:", image, policy.Name)
			glog.Errorf("Found %d violations in image %s:", len(violations), image)
			for _, v := range violations {
				glog.Error(v.Reason())
			}
			os.Exit(1)
		}
		glog.Infof("Image %q passes VulnzSigningPolicy %s.", image, policy.Name)
	}

	if doSign {
		// Read the signing credentials
		// Either kmsKeyName or pgpPriKeyPath needs to be set
		if kmsKeyName == "" && pgpPriKeyPath == "" {
			glog.Fatalf("Neither kms_key_name or private_key is specified")
		}
		var cSigner attestlib.Signer
		if kmsKeyName != "" {
			glog.Infof("Using kms key %s for signing.", kmsKeyName)
			if kmsDigestAlg == "" {
				glog.Fatalf("kms_digest_alg is unspecified, must be one of SHA256|SHA384|SHA512, and the same as specified by the key version's algorithm")
			}
			cSigner, err = signer.NewCloudKmsSigner(kmsKeyName, signer.DigestAlgorithm(kmsDigestAlg))
			if err != nil {
				glog.Fatalf("Creating kms signer failed: %v\n", err)
			}
		} else {
			glog.Infof("Using pgp key for signing.")
			signerKey, err := ioutil.ReadFile(pgpPriKeyPath)
			if err != nil {
				glog.Fatalf("Fail to read signer key: %v\n", err)
			}
			// Create a cryptolib signer
			cSigner, err = attestlib.NewPgpSigner(signerKey, pgpPassphrase)
			if err != nil {
				glog.Fatalf("Creating pgp signer failed: %v\n", err)
			}
		}

		// Check note name
		err = util.CheckNoteName(noteName)
		if err != nil {
			glog.Fatalf("Note name is invalid %v", err)
		}

		// Parse attestation project
		if attestationProject == "" {
			attestationProject = util.GetProjectFromContainerImage(image)
			glog.Infof("Using image project as attestation project: %s\n", attestationProject)
		} else {
			glog.Infof("Using specified attestation project: %s\n", attestationProject)
		}

		// Create signer
		r := signer.New(client, cSigner, noteName, attestationProject, overwrite)
		// Sign image
		err := r.SignImage(image)
		if err != nil {
			glog.Fatalf("Signing image failed %v", err)
		}
	}
}
<commit_msg>Add better command line usage.<commit_after>/*
Copyright 2020 Google LLC

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"time"

	"github.com/golang/glog"
	"github.com/grafeas/kritis/pkg/attestlib"
	"github.com/grafeas/kritis/pkg/kritis/apis/kritis/v1beta1"
	"github.com/grafeas/kritis/pkg/kritis/crd/vulnzsigningpolicy"
	"github.com/grafeas/kritis/pkg/kritis/metadata/containeranalysis"
	"github.com/grafeas/kritis/pkg/kritis/signer"
	"github.com/grafeas/kritis/pkg/kritis/util"
	"k8s.io/apimachinery/pkg/util/yaml"
)

type SignerMode string

const (
	CheckAndSign  SignerMode = "check-and-sign"
	CheckOnly     SignerMode = "check-only"
	BypassAndSign SignerMode = "bypass-and-sign"
)

var (
	// input flags
	mode               string
	image              string
	vulnzTimeout       string
	policyPath         string
	attestationProject string
	overwrite          bool
	noteName           string
	// input flags: pgp key flags
	pgpPriKeyPath string
	pgpPassphrase string
	// input flags: kms flags
	kmsKeyName   string
	kmsDigestAlg string

	// helper global variables
	modeFlags *flag.FlagSet
	modeExample string
)

func init() {
	// need to add all flags to avoid "flag not provided error"
	addBasicFlags(flag.CommandLine)
	addCheckFlags(flag.CommandLine)
	addSignFlags(flag.CommandLine)
}

func addBasicFlags(fs *flag.FlagSet) {
	fs.StringVar(&mode, "mode", "check-and-sign", "(required) mode of operation, check-and-sign|check-only|bypass-and-sign")
	fs.StringVar(&image, "image", "", "(required) image url, e.g., gcr.io/foo/bar@sha256:abcd")
}

func addCheckFlags(fs *flag.FlagSet) {
	fs.StringVar(&policyPath, "policy", "", "(required for check) vulnerability signing policy file path, e.g., /tmp/vulnz_signing_policy.yaml")
	fs.StringVar(&vulnzTimeout, "vulnz_timeout", "5m", "timeout for polling image vulnerability , e.g., 600s, 5m")
}

func addSignFlags(fs *flag.FlagSet) {
	fs.StringVar(&noteName, "note_name", "", "(required for sign) note name that created attestations are attached to, in the form of projects/[PROVIDER_ID]/notes/[NOTE_ID]")
	fs.StringVar(&attestationProject, "attestation_project", "", "project id for GCP project that stores attestation, use image project if set to empty")
	fs.BoolVar(&overwrite, "overwrite", false, "overwrite attestation if already existed")
	fs.StringVar(&kmsKeyName, "kms_key_name", "", "kms key name, in the format of in the format projects/*/locations/*/keyRings/*/cryptoKeys/*/cryptoKeyVersions/*")
	fs.StringVar(&kmsDigestAlg, "kms_digest_alg", "", "kms digest algorithm, must be one of SHA256|SHA384|SHA512, and the same as specified by the key version's algorithm")
	fs.StringVar(&pgpPriKeyPath, "pgp_private_key", "", "pgp private signing key path, e.g., /dev/shm/key.pgp")
	fs.StringVar(&pgpPassphrase, "pgp_passphrase", "", "passphrase for pgp private key, if any")
}

// parseSignerMode creates mode-specific flagset and analyze actions (check, sign) for given mode
func parseSignerMode(mode SignerMode) (doCheck bool, doSign bool, err error) {
	modeFlags, doCheck, doSign, err = flag.NewFlagSet("", flag.ExitOnError), false, false, nil
	addBasicFlags(modeFlags)
	switch mode {
	case CheckAndSign:
		addCheckFlags(modeFlags)
		addSignFlags(modeFlags)
		doCheck, doSign = true, true
		modeExample = `	./signer \
	-mode=check-and-sign \
	-image=gcr.io/my-image-repo/image-1@sha256:123 \
	-policy=policy.yaml \
	-note_name=projects/$NOTE_PROJECT/NOTES/$NOTE_ID \
	-kms_key_name=projects/$KMS_PROJECT/locations/$KMS_KEYLOCATION/keyRings/$KMS_KEYRING/cryptoKeys/$KMS_KEYNAME/cryptoKeyVersions/$KMS_KEYVERSION \
	-kms_digest_alg=SHA512`
	case BypassAndSign:
		addSignFlags(modeFlags)
		doSign = true
		modeExample = `	./signer \
	-mode=bypass-and-sign \
	-image=gcr.io/my-image-repo/image-1@sha256:123 \
	-note_name=projects/$NOTE_PROJECT/NOTES/$NOTE_ID \
	-kms_key_name=projects/$KMS_PROJECT/locations/$KMS_KEYLOCATION/keyRings/$KMS_KEYRING/cryptoKeys/$KMS_KEYNAME/cryptoKeyVersions/$KMS_KEYVERSION \
	-kms_digest_alg=SHA512`
	case CheckOnly:
		addCheckFlags(modeFlags)
		doCheck = true
		modeExample = `	./signer \
	-mode=check-only \
	-image=gcr.io/my-image-repo/image-1@sha256:123 \
	-policy=policy.yaml`
	default:
		return false, false, fmt.Errorf("unrecognized mode %s, must be one of check-and-sign|check-only|bypass-and-sign", mode)
	}
	modeFlags.Parse(os.Args[1:])
	return doCheck, doSign, err
}

func exitOnBadFlags(mode SignerMode, err string) {
	fmt.Fprintf(modeFlags.Output(), "Usage of signer's %s mode:\n", mode)
	modeFlags.PrintDefaults()
	fmt.Fprintf(modeFlags.Output(), "Example (%s mode):\n %s\n", mode, modeExample)
	fmt.Fprintf(modeFlags.Output(), "Bad flags for mode %s: %v. \n", mode, err)
	os.Exit(1)
}

func main() {
	flag.Parse()
	glog.Infof("Signer mode: %s.", mode)

	doCheck, doSign, err := parseSignerMode(SignerMode(mode))
	if err != nil {
		glog.Fatalf("Parse mode err %v.", err)
	}

	// Check image url is non-empty
	// TODO: check and format image url to
	//  gcr.io/project-id/rest-of-image-path@sha256:[sha-value]
	if image == "" {
		exitOnBadFlags(SignerMode(mode), "image url is empty")
	}

	// Create a client
	client, err := containeranalysis.New()
	if err != nil {
		glog.Fatalf("Could not initialize the client %v", err)
	}

	if doCheck {
		// Read the vulnz signing policy
		if policyPath == "" {
			exitOnBadFlags(SignerMode(mode), "policy path is empty")
		}
		policy := v1beta1.VulnzSigningPolicy{}
		policyFile, err := os.Open(policyPath)
		if err != nil {
			glog.Fatalf("Fail to load vulnz signing policy: %v", err)
		}
		defer policyFile.Close()
		// err = json.Unmarshal(policyFile, &policy)
		if err := yaml.NewYAMLToJSONDecoder(policyFile).Decode(&policy); err != nil {
			glog.Fatalf("Fail to parse policy file: %v", err)
		} else {
			glog.Infof("Policy req: %v\n", policy.Spec.ImageVulnerabilityRequirements)
		}

		timeout, err := time.ParseDuration(vulnzTimeout)
		if err != nil {
			glog.Fatalf("Fail to parse timeout %v", err)
		}
		err = client.WaitForVulnzAnalysis(image, timeout)
		if err != nil {
			glog.Fatalf("Error waiting for vulnerability analysis %v", err)
		}

		// Read the vulnz scanning events
		vulnz, err := client.Vulnerabilities(image)
		if err != nil {
			glog.Fatalf("Found err %s", err)
		}
		if vulnz == nil {
			glog.Fatalf("Expected some vulnerabilities. Nil found")
		}

		violations, err := vulnzsigningpolicy.ValidateVulnzSigningPolicy(policy, image, vulnz)
		if err != nil {
			glog.Fatalf("Error when evaluating image %q against policy %q", image, policy.Name)
		}
		if violations != nil && len(violations) != 0 {
			glog.Errorf("Image %q does not pass VulnzSigningPolicy %q:", image, policy.Name)
			glog.Errorf("Found %d violations in image %s:", len(violations), image)
			for _, v := range violations {
				glog.Error(v.Reason())
			}
			os.Exit(1)
		}
		glog.Infof("Image %q passes VulnzSigningPolicy %s.", image, policy.Name)
	}

	if doSign {
		// Read the signing credentials
		// Either kmsKeyName or pgpPriKeyPath needs to be set
		if kmsKeyName == "" && pgpPriKeyPath == "" {
			exitOnBadFlags(SignerMode(mode), "neither kms_key_name or private_key is specified")
		}
		var cSigner attestlib.Signer
		if kmsKeyName != "" {
			glog.Infof("Using kms key %s for signing.", kmsKeyName)
			if kmsDigestAlg == "" {
				glog.Fatalf("kms_digest_alg is unspecified, must be one of SHA256|SHA384|SHA512, and the same as specified by the key version's algorithm")
			}
			cSigner, err = signer.NewCloudKmsSigner(kmsKeyName, signer.DigestAlgorithm(kmsDigestAlg))
			if err != nil {
				glog.Fatalf("Creating kms signer failed: %v\n", err)
			}
		} else {
			glog.Infof("Using pgp key for signing.")
			signerKey, err := ioutil.ReadFile(pgpPriKeyPath)
			if err != nil {
				glog.Fatalf("Fail to read signer key: %v\n", err)
			}
			// Create a cryptolib signer
			cSigner, err = attestlib.NewPgpSigner(signerKey, pgpPassphrase)
			if err != nil {
				glog.Fatalf("Creating pgp signer failed: %v\n", err)
			}
		}

		// Check note name
		err = util.CheckNoteName(noteName)
		if err != nil {
			exitOnBadFlags(SignerMode(mode), fmt.Sprintf("note name is invalid %v", err))
		}

		// Parse attestation project
		if attestationProject == "" {
			attestationProject = util.GetProjectFromContainerImage(image)
			glog.Infof("Using image project as attestation project: %s\n", attestationProject)
		} else {
			glog.Infof("Using specified attestation project: %s\n", attestationProject)
		}

		// Create signer
		r := signer.New(client, cSigner, noteName, attestationProject, overwrite)
		// Sign image
		err := r.SignImage(image)
		if err != nil {
			glog.Fatalf("Signing image failed %v", err)
		}
	}
}
<|endoftext|>