<commit_before>package main

import (
	"context"
	"encoding/json"
	"fmt"
	"io"
	"sort"
	"strings"

	"github.com/restic/restic/internal/restic"
	"github.com/restic/restic/internal/ui/table"
	"github.com/spf13/cobra"
)

var cmdSnapshots = &cobra.Command{
	Use:   "snapshots [snapshotID ...]",
	Short: "List all snapshots",
	Long: `
The "snapshots" command lists all snapshots stored in the repository.
`,
	DisableAutoGenTag: true,
	RunE: func(cmd *cobra.Command, args []string) error {
		return runSnapshots(snapshotOptions, globalOptions, args)
	},
}

// SnapshotOptions bundles all options for the snapshots command.
type SnapshotOptions struct {
	Host    string
	Tags    restic.TagLists
	Paths   []string
	Compact bool
	Last    bool
}

var snapshotOptions SnapshotOptions

func init() {
	cmdRoot.AddCommand(cmdSnapshots)

	f := cmdSnapshots.Flags()
	f.StringVarP(&snapshotOptions.Host, "host", "H", "", "only consider snapshots for this `host`")
	f.Var(&snapshotOptions.Tags, "tag", "only consider snapshots which include this `taglist` (can be specified multiple times)")
	f.StringArrayVar(&snapshotOptions.Paths, "path", nil, "only consider snapshots for this `path` (can be specified multiple times)")
	f.BoolVarP(&snapshotOptions.Compact, "compact", "c", false, "use compact format")
	f.BoolVar(&snapshotOptions.Last, "last", false, "only show the last snapshot for each host and path")
}

func runSnapshots(opts SnapshotOptions, gopts GlobalOptions, args []string) error {
	repo, err := OpenRepository(gopts)
	if err != nil {
		return err
	}

	if !gopts.NoLock {
		lock, err := lockRepo(repo)
		defer unlockRepo(lock)
		if err != nil {
			return err
		}
	}

	ctx, cancel := context.WithCancel(gopts.ctx)
	defer cancel()

	var list restic.Snapshots
	for sn := range FindFilteredSnapshots(ctx, repo, opts.Host, opts.Tags, opts.Paths, args) {
		list = append(list, sn)
	}

	if opts.Last {
		list = FilterLastSnapshots(list)
	}

	sort.Sort(sort.Reverse(list))

	if gopts.JSON {
		err := printSnapshotsJSON(gopts.stdout, list)
		if err != nil {
			Warnf("error printing snapshot: %v\n", err)
		}
		return nil
	}
	PrintSnapshots(gopts.stdout, list, nil, opts.Compact)

	return nil
}

// filterLastSnapshotsKey is used by FilterLastSnapshots.
type filterLastSnapshotsKey struct {
	Hostname    string
	JoinedPaths string
}

// newFilterLastSnapshotsKey initializes a filterLastSnapshotsKey from a Snapshot
func newFilterLastSnapshotsKey(sn *restic.Snapshot) filterLastSnapshotsKey {
	// Shallow slice copy
	var paths = make([]string, len(sn.Paths))
	copy(paths, sn.Paths)
	sort.Strings(paths)
	return filterLastSnapshotsKey{sn.Hostname, strings.Join(paths, "|")}
}

// FilterLastSnapshots filters a list of snapshots to only return the last
// entry for each hostname and path. If the snapshot contains multiple paths,
// they will be joined and treated as one item.
func FilterLastSnapshots(list restic.Snapshots) restic.Snapshots {
	// Sort the snapshots so that the newer ones are listed first
	sort.SliceStable(list, func(i, j int) bool {
		return list[i].Time.After(list[j].Time)
	})

	var results restic.Snapshots
	seen := make(map[filterLastSnapshotsKey]bool)
	for _, sn := range list {
		key := newFilterLastSnapshotsKey(sn)
		if !seen[key] {
			seen[key] = true
			results = append(results, sn)
		}
	}
	return results
}

// PrintSnapshots prints a text table of the snapshots in list to stdout.
func PrintSnapshots(stdout io.Writer, list restic.Snapshots, reasons []restic.KeepReason, compact bool) {
	// keep the reasons a snasphot is being kept in a map, so that it doesn't
	// get lost when the list of snapshots is sorted
	keepReasons := make(map[restic.ID]restic.KeepReason, len(reasons))
	if len(reasons) > 0 {
		for i, sn := range list {
			id := sn.ID()
			keepReasons[*id] = reasons[i]
		}
	}

	// always sort the snapshots so that the newer ones are listed last
	sort.SliceStable(list, func(i, j int) bool {
		return list[i].Time.Before(list[j].Time)
	})

	// Determine the max widths for host and tag.
	maxHost, maxTag := 10, 6
	for _, sn := range list {
		if len(sn.Hostname) > maxHost {
			maxHost = len(sn.Hostname)
		}
		for _, tag := range sn.Tags {
			if len(tag) > maxTag {
				maxTag = len(tag)
			}
		}
	}

	tab := table.New()

	if compact {
		tab.AddColumn("ID", "{{ .ID }}")
		tab.AddColumn("Time", "{{ .Timestamp }}")
		tab.AddColumn("Host", "{{ .Hostname }}")
		tab.AddColumn("Tags  ", `{{ join .Tags "\n" }}`)
	} else {
		tab.AddColumn("ID", "{{ .ID }}")
		tab.AddColumn("Time", "{{ .Timestamp }}")
		tab.AddColumn("Host      ", "{{ .Hostname }}")
		tab.AddColumn("Tags      ", `{{ join .Tags "," }}`)
		if len(reasons) > 0 {
			tab.AddColumn("Reasons", `{{ join .Reasons "\n" }}`)
		}
		tab.AddColumn("Paths", `{{ join .Paths "\n" }}`)
	}

	type snapshot struct {
		ID        string
		Timestamp string
		Hostname  string
		Tags      []string
		Reasons   []string
		Paths     []string
	}

	var multiline bool
	for _, sn := range list {
		data := snapshot{
			ID:        sn.ID().Str(),
			Timestamp: sn.Time.Local().Format(TimeFormat),
			Hostname:  sn.Hostname,
			Tags:      sn.Tags,
			Paths:     sn.Paths,
		}

		if len(reasons) > 0 {
			id := sn.ID()
			data.Reasons = keepReasons[*id].Matches
		}

		if len(sn.Paths) > 1 && !compact {
			multiline = true
		}

		tab.AddRow(data)
	}

	tab.AddFooter(fmt.Sprintf("%d snapshots", len(list)))

	if multiline {
		// print an additional blank line between snapshots

		var last int
		tab.PrintData = func(w io.Writer, idx int, s string) error {
			var err error
			if idx == last {
				_, err = fmt.Fprintf(w, "%s\n", s)
			} else {
				_, err = fmt.Fprintf(w, "\n%s\n", s)
			}
			last = idx
			return err
		}
	}

	tab.Write(stdout)
}

// Snapshot helps to print Snaphots as JSON with their ID included.
type Snapshot struct {
	*restic.Snapshot

	ID      *restic.ID `json:"id"`
	ShortID string     `json:"short_id"`
}

// printSnapshotsJSON writes the JSON representation of list to stdout.
func printSnapshotsJSON(stdout io.Writer, list restic.Snapshots) error {

	var snapshots []Snapshot

	for _, sn := range list {

		k := Snapshot{
			Snapshot: sn,
			ID:       sn.ID(),
			ShortID:  sn.ID().Str(),
		}
		snapshots = append(snapshots, k)
	}

	return json.NewEncoder(stdout).Encode(snapshots)
}
<commit_msg>Add GroupBy option to snapshots command<commit_after>package main

import (
	"context"
	"encoding/json"
	"fmt"
	"io"
	"sort"
	"strings"

	"github.com/restic/restic/internal/errors"
	"github.com/restic/restic/internal/restic"
	"github.com/restic/restic/internal/ui/table"
	"github.com/spf13/cobra"
)

var cmdSnapshots = &cobra.Command{
	Use:   "snapshots [snapshotID ...]",
	Short: "List all snapshots",
	Long: `
The "snapshots" command lists all snapshots stored in the repository.
`,
	DisableAutoGenTag: true,
	RunE: func(cmd *cobra.Command, args []string) error {
		return runSnapshots(snapshotOptions, globalOptions, args)
	},
}

// SnapshotOptions bundles all options for the snapshots command.
type SnapshotOptions struct {
	Host    string
	Tags    restic.TagLists
	Paths   []string
	Compact bool
	Last    bool
	GroupBy string
}

var snapshotOptions SnapshotOptions

func init() {
	cmdRoot.AddCommand(cmdSnapshots)

	f := cmdSnapshots.Flags()
	f.StringVarP(&snapshotOptions.Host, "host", "H", "", "only consider snapshots for this `host`")
	f.Var(&snapshotOptions.Tags, "tag", "only consider snapshots which include this `taglist` (can be specified multiple times)")
	f.StringArrayVar(&snapshotOptions.Paths, "path", nil, "only consider snapshots for this `path` (can be specified multiple times)")
	f.BoolVarP(&snapshotOptions.Compact, "compact", "c", false, "use compact format")
	f.BoolVar(&snapshotOptions.Last, "last", false, "only show the last snapshot for each host and path")
	f.StringVarP(&snapshotOptions.GroupBy, "group-by", "g", "", "string for grouping snapshots by host,paths,tags")
}

type groupKey struct {
	Hostname string
	Paths    []string
	Tags     []string
}

func runSnapshots(opts SnapshotOptions, gopts GlobalOptions, args []string) error {
	repo, err := OpenRepository(gopts)
	if err != nil {
		return err
	}

	if !gopts.NoLock {
		lock, err := lockRepo(repo)
		defer unlockRepo(lock)
		if err != nil {
			return err
		}
	}

	// group by hostname and dirs
	snapshotGroups := make(map[string]restic.Snapshots)

	var GroupByTag bool
	var GroupByHost bool
	var GroupByPath bool
	var GroupOptionList []string

	GroupOptionList = strings.Split(opts.GroupBy, ",")

	for _, option := range GroupOptionList {
		switch option {
		case "host":
			GroupByHost = true
		case "paths":
			GroupByPath = true
		case "tags":
			GroupByTag = true
		case "":
		default:
			return errors.Fatal("unknown grouping option: '" + option + "'")
		}
	}

	ctx, cancel := context.WithCancel(gopts.ctx)
	defer cancel()

	for sn := range FindFilteredSnapshots(ctx, repo, opts.Host, opts.Tags, opts.Paths, args) {
		// Determining grouping-keys
		var tags []string
		var hostname string
		var paths []string

		if GroupByTag {
			tags = sn.Tags
			sort.StringSlice(tags).Sort()
		}
		if GroupByHost {
			hostname = sn.Hostname
		}
		if GroupByPath {
			paths = sn.Paths
		}

		sort.StringSlice(sn.Paths).Sort()
		var k []byte
		var err error

		k, err = json.Marshal(groupKey{Tags: tags, Hostname: hostname, Paths: paths})

		if err != nil {
			return err
		}
		snapshotGroups[string(k)] = append(snapshotGroups[string(k)], sn)
	}

	for k, list := range snapshotGroups {
		if opts.Last {
			list = FilterLastSnapshots(list)
		}
		sort.Sort(sort.Reverse(list))
		snapshotGroups[k] = list
	}

	if gopts.JSON {
		err := printSnapshotGroupJSON(gopts.stdout, snapshotGroups, GroupByTag || GroupByHost || GroupByPath)
		if err != nil {
			Warnf("error printing snapshots: %v\n", err)
		}
		return nil
	}

	for k, list := range snapshotGroups {
		err := PrintSnapshotGroupHeader(gopts.stdout, k, GroupByTag, GroupByHost, GroupByPath)
		if err != nil {
			Warnf("error printing snapshots: %v\n", err)
			return nil
		}

		PrintSnapshots(gopts.stdout, list, nil, opts.Compact)
	}

	return nil
}

// filterLastSnapshotsKey is used by FilterLastSnapshots.
type filterLastSnapshotsKey struct {
	Hostname    string
	JoinedPaths string
}

// newFilterLastSnapshotsKey initializes a filterLastSnapshotsKey from a Snapshot
func newFilterLastSnapshotsKey(sn *restic.Snapshot) filterLastSnapshotsKey {
	// Shallow slice copy
	var paths = make([]string, len(sn.Paths))
	copy(paths, sn.Paths)
	sort.Strings(paths)
	return filterLastSnapshotsKey{sn.Hostname, strings.Join(paths, "|")}
}

// FilterLastSnapshots filters a list of snapshots to only return the last
// entry for each hostname and path. If the snapshot contains multiple paths,
// they will be joined and treated as one item.
func FilterLastSnapshots(list restic.Snapshots) restic.Snapshots {
	// Sort the snapshots so that the newer ones are listed first
	sort.SliceStable(list, func(i, j int) bool {
		return list[i].Time.After(list[j].Time)
	})

	var results restic.Snapshots
	seen := make(map[filterLastSnapshotsKey]bool)
	for _, sn := range list {
		key := newFilterLastSnapshotsKey(sn)
		if !seen[key] {
			seen[key] = true
			results = append(results, sn)
		}
	}
	return results
}

// PrintSnapshots prints a text table of the snapshots in list to stdout.
func PrintSnapshots(stdout io.Writer, list restic.Snapshots, reasons []restic.KeepReason, compact bool) {
	// keep the reasons a snasphot is being kept in a map, so that it doesn't
	// get lost when the list of snapshots is sorted
	keepReasons := make(map[restic.ID]restic.KeepReason, len(reasons))
	if len(reasons) > 0 {
		for i, sn := range list {
			id := sn.ID()
			keepReasons[*id] = reasons[i]
		}
	}

	// always sort the snapshots so that the newer ones are listed last
	sort.SliceStable(list, func(i, j int) bool {
		return list[i].Time.Before(list[j].Time)
	})

	// Determine the max widths for host and tag.
	maxHost, maxTag := 10, 6
	for _, sn := range list {
		if len(sn.Hostname) > maxHost {
			maxHost = len(sn.Hostname)
		}
		for _, tag := range sn.Tags {
			if len(tag) > maxTag {
				maxTag = len(tag)
			}
		}
	}

	tab := table.New()

	if compact {
		tab.AddColumn("ID", "{{ .ID }}")
		tab.AddColumn("Time", "{{ .Timestamp }}")
		tab.AddColumn("Host", "{{ .Hostname }}")
		tab.AddColumn("Tags  ", `{{ join .Tags "\n" }}`)
	} else {
		tab.AddColumn("ID", "{{ .ID }}")
		tab.AddColumn("Time", "{{ .Timestamp }}")
		tab.AddColumn("Host      ", "{{ .Hostname }}")
		tab.AddColumn("Tags      ", `{{ join .Tags "," }}`)
		if len(reasons) > 0 {
			tab.AddColumn("Reasons", `{{ join .Reasons "\n" }}`)
		}
		tab.AddColumn("Paths", `{{ join .Paths "\n" }}`)
	}

	type snapshot struct {
		ID        string
		Timestamp string
		Hostname  string
		Tags      []string
		Reasons   []string
		Paths     []string
	}

	var multiline bool
	for _, sn := range list {
		data := snapshot{
			ID:        sn.ID().Str(),
			Timestamp: sn.Time.Local().Format(TimeFormat),
			Hostname:  sn.Hostname,
			Tags:      sn.Tags,
			Paths:     sn.Paths,
		}

		if len(reasons) > 0 {
			id := sn.ID()
			data.Reasons = keepReasons[*id].Matches
		}

		if len(sn.Paths) > 1 && !compact {
			multiline = true
		}

		tab.AddRow(data)
	}

	tab.AddFooter(fmt.Sprintf("%d snapshots", len(list)))

	if multiline {
		// print an additional blank line between snapshots

		var last int
		tab.PrintData = func(w io.Writer, idx int, s string) error {
			var err error
			if idx == last {
				_, err = fmt.Fprintf(w, "%s\n", s)
			} else {
				_, err = fmt.Fprintf(w, "\n%s\n", s)
			}
			last = idx
			return err
		}
	}

	tab.Write(stdout)
}

// PrintSnapshotGroupHeader prints which group of the group-by option the
// following snapshots belong to.
// Prints nothing, if we did not group at all.
func PrintSnapshotGroupHeader(stdout io.Writer, groupKeyJSON string, GroupByTag bool, GroupByHost bool, GroupByPath bool) error {
	if GroupByTag || GroupByHost || GroupByPath {
		var key groupKey
		var err error

		err = json.Unmarshal([]byte(groupKeyJSON), &key)
		if err != nil {
			return err
		}

		// Info
		fmt.Fprintf(stdout, "snapshots")
		var infoStrings []string
		if GroupByTag {
			infoStrings = append(infoStrings, "tags ["+strings.Join(key.Tags, ", ")+"]")
		}
		if GroupByHost {
			infoStrings = append(infoStrings, "host ["+key.Hostname+"]")
		}
		if GroupByPath {
			infoStrings = append(infoStrings, "paths ["+strings.Join(key.Paths, ", ")+"]")
		}
		if infoStrings != nil {
			fmt.Fprintf(stdout, " for (%s)", strings.Join(infoStrings, ", "))
		}
		fmt.Fprintf(stdout, ":\n")
	}

	return nil
}

// Snapshot helps to print Snaphots as JSON with their ID included.
type Snapshot struct {
	*restic.Snapshot

	ID      *restic.ID `json:"id"`
	ShortID string     `json:"short_id"`
}

// SnapshotGroup helps to print SnaphotGroups as JSON with their GroupReasons included.
type SnapshotGroup struct {
	GroupKey  groupKey
	Snapshots []Snapshot
}

// printSnapshotsJSON writes the JSON representation of list to stdout.
func printSnapshotGroupJSON(stdout io.Writer, snGroups map[string]restic.Snapshots, grouped bool) error {

	if grouped {
		var snapshotGroups []SnapshotGroup

		for k, list := range snGroups {
			var key groupKey
			var err error
			var snapshots []Snapshot

			err = json.Unmarshal([]byte(k), &key)
			if err != nil {
				return err
			}

			for _, sn := range list {
				k := Snapshot{
					Snapshot: sn,
					ID:       sn.ID(),
					ShortID:  sn.ID().Str(),
				}
				snapshots = append(snapshots, k)
			}

			group := SnapshotGroup{
				GroupKey:  key,
				Snapshots: snapshots,
			}
			snapshotGroups = append(snapshotGroups, group)
		}

		return json.NewEncoder(stdout).Encode(snapshotGroups)
	} else {
		// Old behavior
		var snapshots []Snapshot

		for _, list := range snGroups {
			for _, sn := range list {
				k := Snapshot{
					Snapshot: sn,
					ID:       sn.ID(),
					ShortID:  sn.ID().Str(),
				}
				snapshots = append(snapshots, k)
			}
		}

		return json.NewEncoder(stdout).Encode(snapshots)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"database/sql"
	_ "expvar"
	"flag"
	"fmt"
	"net/http"
	_ "net/http/pprof"
	"os"
	"os/signal"
	"syscall"

	"github.com/Sirupsen/logrus"
	_ "github.com/docker/distribution/registry/auth/token"
	"github.com/endophage/gotuf/signed"
	_ "github.com/go-sql-driver/mysql"
	"golang.org/x/net/context"

	"github.com/docker/vetinari/config"
	"github.com/docker/vetinari/server"
	"github.com/docker/vetinari/server/version"
	"github.com/docker/vetinari/signer"
)

// DebugAddress is the debug server address to listen on
const DebugAddress = "localhost:8080"

var debug bool
var configFile string

func init() {
	// Set default logging level to Error
	logrus.SetLevel(logrus.ErrorLevel)

	// Setup flags
	flag.StringVar(&configFile, "config", "", "Path to configuration file")
	flag.BoolVar(&debug, "debug", false, "Enable the debugging server on localhost:8080")
}

func main() {
	flag.Usage = usage
	flag.Parse()

	if DebugAddress != "" {
		go debugServer(DebugAddress)
	}

	ctx := context.Background()

	conf, err := parseConfig(configFile)
	if err != nil {
		logrus.Fatal("Error parsing config: ", err.Error())
		return // not strictly needed but let's be explicit
	}
	if conf.Logging.Level > 0 {
		logrus.SetLevel(logrus.Level(conf.Logging.Level))
	}

	sigHup := make(chan os.Signal)
	sigTerm := make(chan os.Signal)

	signal.Notify(sigHup, syscall.SIGHUP)
	signal.Notify(sigTerm, syscall.SIGTERM)

	var trust signed.CryptoService
	if conf.TrustService.Type == "remote" {
		logrus.Info("[Vetinari] : Using remote signing service")
		trust = signer.NewRufusSigner(conf.TrustService.Hostname, conf.TrustService.Port, conf.TrustService.TLSCAFile)
	} else {
		logrus.Info("[Vetinari] : Using local signing service")
		trust = signed.NewEd25519()
	}

	db, err := sql.Open("mysql", "root:@/dockercondemo")
	if err != nil {
		logrus.Fatal("Error starting DB driver: ", err.Error())
		return // not strictly needed but let's be explicit
	}
	ctx = context.WithValue(ctx, "versionStore", version.NewVersionDB(db))
	for {
		logrus.Info("[Vetinari] Starting Server")
		childCtx, cancel := context.WithCancel(ctx)
		go server.Run(childCtx, conf.Server, trust)

		for {
			select {
			// On a sighup we cancel and restart a new server
			// with updated config
			case <-sigHup:
				logrus.Infof("[Vetinari] Server restart requested. Attempting to parse config at %s", configFile)
				conf, err = parseConfig(configFile)
				if err != nil {
					logrus.Infof("[Vetinari] Unable to parse config. Old configuration will keep running. Parse Err: %s", err.Error())
					continue
				} else {
					cancel()
					logrus.Info("[Vetinari] Stopping server for restart")
					break
				}
			// On sigkill we cancel and shutdown
			case <-sigTerm:
				cancel()
				logrus.Info("[Vetinari] Shutting Down Hard")
				os.Exit(0)
			}
		}
	}
}

func usage() {
	fmt.Println("usage:", os.Args[0], "<config>")
	flag.PrintDefaults()
}

// debugServer starts the debug server with pprof, expvar among other
// endpoints. The addr should not be exposed externally. For most of these to
// work, tls cannot be enabled on the endpoint, so it is generally separate.
func debugServer(addr string) {
	logrus.Info("[Vetinari Debug Server] server listening on", addr)
	if err := http.ListenAndServe(addr, nil); err != nil {
		logrus.Fatal("[Vetinari Debug Server] error listening on debug interface: ", err)
	}
}

func parseConfig(path string) (*config.Configuration, error) {
	file, err := os.Open(path)
	defer file.Close()
	if err != nil {
		logrus.Error("Failed to open configuration file located at: ", path)
		return nil, err
	}

	return config.Load(file)
}
<commit_msg>Fixed compilation errors for vetinari-server<commit_after>package main

import (
	"database/sql"
	_ "expvar"
	"flag"
	"fmt"
	"net/http"
	_ "net/http/pprof"
	"os"
	"os/signal"
	"syscall"

	"github.com/Sirupsen/logrus"
	_ "github.com/docker/distribution/registry/auth/token"
	"github.com/endophage/gotuf/signed"
	_ "github.com/go-sql-driver/mysql"
	"golang.org/x/net/context"

	"github.com/docker/vetinari/config"
	"github.com/docker/vetinari/server"
	"github.com/docker/vetinari/server/version"
	"github.com/docker/vetinari/signer"
)

// DebugAddress is the debug server address to listen on
const DebugAddress = "localhost:8080"

var debug bool
var configFile string

func init() {
	// Set default logging level to Error
	logrus.SetLevel(logrus.ErrorLevel)

	// Setup flags
	flag.StringVar(&configFile, "config", "", "Path to configuration file")
	flag.BoolVar(&debug, "debug", false, "Enable the debugging server on localhost:8080")
}

func main() {
	flag.Usage = usage
	flag.Parse()

	if DebugAddress != "" {
		go debugServer(DebugAddress)
	}

	ctx := context.Background()

	conf, err := parseConfig(configFile)
	if err != nil {
		logrus.Fatal("Error parsing config: ", err.Error())
		return // not strictly needed but let's be explicit
	}
	if conf.Logging.Level > 0 {
		logrus.SetLevel(logrus.Level(conf.Logging.Level))
	}

	sigHup := make(chan os.Signal)
	sigTerm := make(chan os.Signal)

	signal.Notify(sigHup, syscall.SIGHUP)
	signal.Notify(sigTerm, syscall.SIGTERM)

	var trust signed.CryptoService
	if conf.TrustService.Type == "remote" {
		logrus.Info("[Vetinari Server] : Using remote signing service")
		trust = server.NewRufusSigner(conf.TrustService.Hostname, conf.TrustService.Port, conf.TrustService.TLSCAFile)
	} else {
		logrus.Info("[Vetinari] : Using local signing service")
		trust = signed.NewEd25519()
	}

	db, err := sql.Open("mysql", "root:@/dockercondemo")
	if err != nil {
		logrus.Fatal("Error starting DB driver: ", err.Error())
		return // not strictly needed but let's be explicit
	}
	ctx = context.WithValue(ctx, "versionStore", version.NewVersionDB(db))
	for {
		logrus.Info("[Vetinari] Starting Server")
		childCtx, cancel := context.WithCancel(ctx)
		go server.Run(childCtx, conf.Server, trust)

		for {
			select {
			// On a sighup we cancel and restart a new server
			// with updated config
			case <-sigHup:
				logrus.Infof("[Vetinari] Server restart requested. Attempting to parse config at %s", configFile)
				conf, err = parseConfig(configFile)
				if err != nil {
					logrus.Infof("[Vetinari] Unable to parse config. Old configuration will keep running. Parse Err: %s", err.Error())
					continue
				} else {
					cancel()
					logrus.Info("[Vetinari] Stopping server for restart")
					break
				}
			// On sigkill we cancel and shutdown
			case <-sigTerm:
				cancel()
				logrus.Info("[Vetinari] Shutting Down Hard")
				os.Exit(0)
			}
		}
	}
}

func usage() {
	fmt.Println("usage:", os.Args[0], "<config>")
	flag.PrintDefaults()
}

// debugServer starts the debug server with pprof, expvar among other
// endpoints. The addr should not be exposed externally. For most of these to
// work, tls cannot be enabled on the endpoint, so it is generally separate.
func debugServer(addr string) {
	logrus.Info("[Vetinari Debug Server] server listening on", addr)
	if err := http.ListenAndServe(addr, nil); err != nil {
		logrus.Fatal("[Vetinari Debug Server] error listening on debug interface: ", err)
	}
}

func parseConfig(path string) (*config.Configuration, error) {
	file, err := os.Open(path)
	defer file.Close()
	if err != nil {
		logrus.Error("Failed to open configuration file located at: ", path)
		return nil, err
	}

	return config.Load(file)
}
<|endoftext|>


<commit_before>package irc

import (
	"bufio"
	"os"
	"net"
	"crypto/tls"
	"fmt"
	"strings"
	"time"
)

// An IRC connection is represented by this struct. Once connected, any errors
// encountered are piped down *Conn.Err; this channel is closed on disconnect.
type Conn struct {
	// Connection Hostname and Nickname
	Host string
	Me   *Nick
	Network string

	// I/O stuff to server
	sock      net.Conn
	io        *bufio.ReadWriter
	in        chan *Line
	out       chan string
	connected bool

    // Are we connecting via SSL?
	SSL bool

	// Error channel to transmit any fail back to the user
	Err chan os.Error

	// Set this to true to disable flood protection and false to re-enable
	Flood bool

	Debug bool

	// Event handler mapping
	events map[string][]func(*Conn, *Line)

	// Map of channels we're on
	chans map[string]*Channel

	// Map of nicks we know about
	nicks map[string]*Nick
}

// We parse an incoming line into this struct. Line.Cmd is used as the trigger
// name for incoming event handlers, see *Conn.recv() for details.
//   Raw =~ ":nick!user@host cmd args[] :text"
//   Src == "nick!user@host"
//   Cmd == e.g. PRIVMSG, 332
type Line struct {
	Nick, Ident, Host, Src string
	Cmd, Text, Raw         string
	Args                   []string
}

// Creates a new IRC connection object, but doesn't connect to anything so
// that you can add event handlers to it. See AddHandler() for details.
func New(nick, user, name string) *Conn {
	conn := new(Conn)
	conn.initialise()
	conn.Me = conn.NewNick(nick, user, name, "")
	conn.setupEvents()
	return conn
}

func (conn *Conn) initialise() {
	// allocate meh some memoraaaahh
	conn.nicks = make(map[string]*Nick)
	conn.chans = make(map[string]*Channel)
	conn.in = make(chan *Line, 32)
	conn.out = make(chan string, 32)
	conn.Err = make(chan os.Error, 4)
	conn.io = nil
	conn.sock = nil

	// if this is being called because we are reconnecting, conn.Me
	// will still have all the old channels referenced -- nuke them!
	if conn.Me != nil {
		conn.Me = conn.NewNick(conn.Me.Nick, conn.Me.Ident, conn.Me.Name, "")
	}
}

// Connect the IRC connection object to "host[:port]" which should be either
// a hostname or an IP address, with an optional port. To enable explicit SSL
// on the connection to the IRC server, set ssl to true. The port will default
// to 6697 if ssl is enabled, and 6667 otherwise. You can also provide an
// optional connect password.
func (conn *Conn) Connect(host string, ssl bool, pass ...string) os.Error {
	if conn.connected {
		return os.NewError(fmt.Sprintf("irc.Connect(): already connected to %s, cannot connect to %s", conn.Host, host))
	}
	if !hasPort(host) {
		if ssl {
			host += ":6697"
		} else {
			host += ":6667"
		}
	}

	var sock net.Conn;
	var err os.Error;
	if ssl {
		sock, err = tls.Dial("tcp", "", host)
	} else {
		sock, err = net.Dial("tcp", "", host)
	}
	if err != nil {
		return err
	}

	conn.Host = host
	conn.SSL = ssl
	conn.sock = sock

	conn.io = bufio.NewReadWriter(
		bufio.NewReader(conn.sock),
		bufio.NewWriter(conn.sock))
	go conn.send()
	go conn.recv()

	if len(pass) > 0 {
		conn.Pass(pass[0])
	}
	conn.Nick(conn.Me.Nick)
	conn.User(conn.Me.Ident, conn.Me.Name)

	go conn.runLoop()
	return nil
}

// dispatch a nicely formatted os.Error to the error channel
func (conn *Conn) error(s string, a ...interface{}) {
	conn.Err <- os.NewError(fmt.Sprintf(s, a...))
}

// copied from http.client for great justice
func hasPort(s string) bool {
	return strings.LastIndex(s, ":") > strings.LastIndex(s, "]")
}

// dispatch input from channel as \r\n terminated line to peer
// flood controlled using hybrid's algorithm if conn.Flood is true
func (conn *Conn) send() {
	lastsent := time.Nanoseconds()
	var badness, linetime, second int64 = 0, 0, 1000000000;
	for line := range conn.out {
		// Hybrid's algorithm allows for 2 seconds per line and an additional
		// 1/120 of a second per character on that line.
		linetime = 2*second + int64(len(line))*second/120
		if !conn.Flood && conn.connected {
			// No point in tallying up flood protection stuff until connected
			if badness += linetime + lastsent - time.Nanoseconds(); badness < 0 {
				// negative badness times are badness...
				badness = int64(0)
			}
		}
		lastsent = time.Nanoseconds()

		// If we've sent more than 10 second's worth of lines according to the
		// calculation above, then we're at risk of "Excess Flood".
		if badness > 10*second && !conn.Flood {
			// so sleep for the current line's time value before sending it
			time.Sleep(linetime)
		}
		if _, err := conn.io.WriteString(line + "\r\n"); err != nil {
			conn.error("irc.send(): %s", err.String())
			conn.shutdown()
			break
		}
		conn.io.Flush()
		if conn.Debug {
			fmt.Println("-> " + line)
		}
	}
}

// receive one \r\n terminated line from peer, parse and dispatch it
func (conn *Conn) recv() {
	for {
		s, err := conn.io.ReadString('\n')
		if err != nil {
			conn.error("irc.recv(): %s", err.String())
			conn.shutdown()
			break
		}
		s = strings.Trim(s, "\r\n")
		if conn.Debug {
			fmt.Println("<- " + s)
		}

		line := &Line{Raw: s}
		if s[0] == ':' {
			// remove a source and parse it
			if idx := strings.Index(s, " "); idx != -1 {
				line.Src, s = s[1:idx], s[idx+1:len(s)]
			} else {
				// pretty sure we shouldn't get here ...
				line.Src = s[1:len(s)]
				conn.in <- line
				continue
			}

			// src can be the hostname of the irc server or a nick!user@host
			line.Host = line.Src
			nidx, uidx := strings.Index(line.Src, "!"), strings.Index(line.Src, "@")
			if uidx != -1 && nidx != -1 {
				line.Nick = line.Src[0:nidx]
				line.Ident = line.Src[nidx+1 : uidx]
				line.Host = line.Src[uidx+1 : len(line.Src)]
			}
		}

		// now we're here, we've parsed a :nick!user@host or :server off
		// s should contain "cmd args[] :text"
		args := strings.Split(s, " :", 2)
		if len(args) > 1 {
			line.Text = args[1]
		}
		args = strings.Fields(args[0])
		line.Cmd = strings.ToUpper(args[0])
		if len(args) > 1 {
			line.Args = args[1:len(args)]
		}
		conn.in <- line
	}
}

func (conn *Conn) runLoop() {
	for line := range conn.in {
			conn.dispatchEvent(line)
	}
}

func (conn *Conn) shutdown() {
	close(conn.in)
	close(conn.out)
	close(conn.Err)
	conn.connected = false
	conn.sock.Close()
	// reinit datastructures ready for next connection
	// do this here rather than after runLoop()'s for due to race
	conn.initialise()
}

// Dumps a load of information about the current state of the connection to a
// string for debugging state tracking and other such things.
func (conn *Conn) String() string {
	str := "GoIRC Connection\n"
	str += "----------------\n\n"
	if conn.connected {
		str += "Connected to " + conn.Host + "\n\n"
	} else {
		str += "Not currently connected!\n\n"
	}
	str += conn.Me.String() + "\n"
	str += "GoIRC Channels\n"
	str += "--------------\n\n"
	for _, ch := range conn.chans {
		str += ch.String() + "\n"
	}
	str += "GoIRC NickNames\n"
	str += "---------------\n\n"
	for _, n := range conn.nicks {
		if n != conn.Me {
			str += n.String() + "\n"
		}
	}
	return str
}
<commit_msg>Somewhat better SSL handling. It'll improve moar if my patches get looked at.<commit_after>package irc

import (
	"bufio"
	"os"
	"net"
	"crypto/tls"
	"fmt"
	"strings"
	"time"
)

// An IRC connection is represented by this struct. Once connected, any errors
// encountered are piped down *Conn.Err; this channel is closed on disconnect.
type Conn struct {
	// Connection Hostname and Nickname
	Host string
	Me   *Nick
	Network string

	// I/O stuff to server
	sock      net.Conn
	io        *bufio.ReadWriter
	in        chan *Line
	out       chan string
	connected bool

	// Are we connecting via SSL? Do we care about certificate validity?
	SSL bool
	SSLConfig *tls.Config

	// Error channel to transmit any fail back to the user
	Err chan os.Error

	// Set this to true to disable flood protection and false to re-enable
	Flood bool

	Debug bool

	// Event handler mapping
	events map[string][]func(*Conn, *Line)

	// Map of channels we're on
	chans map[string]*Channel

	// Map of nicks we know about
	nicks map[string]*Nick
}

// We parse an incoming line into this struct. Line.Cmd is used as the trigger
// name for incoming event handlers, see *Conn.recv() for details.
//   Raw =~ ":nick!user@host cmd args[] :text"
//   Src == "nick!user@host"
//   Cmd == e.g. PRIVMSG, 332
type Line struct {
	Nick, Ident, Host, Src string
	Cmd, Text, Raw         string
	Args                   []string
}

// Creates a new IRC connection object, but doesn't connect to anything so
// that you can add event handlers to it. See AddHandler() for details.
func New(nick, user, name string) *Conn {
	conn := new(Conn)
	conn.initialise()
	conn.Me = conn.NewNick(nick, user, name, "")
	conn.setupEvents()
	return conn
}

func (conn *Conn) initialise() {
	// allocate meh some memoraaaahh
	conn.nicks = make(map[string]*Nick)
	conn.chans = make(map[string]*Channel)
	conn.in = make(chan *Line, 32)
	conn.out = make(chan string, 32)
	conn.Err = make(chan os.Error, 4)
	conn.SSL = false
	conn.SSLConfig = nil
	conn.io = nil
	conn.sock = nil

	// if this is being called because we are reconnecting, conn.Me
	// will still have all the old channels referenced -- nuke them!
	if conn.Me != nil {
		conn.Me = conn.NewNick(conn.Me.Nick, conn.Me.Ident, conn.Me.Name, "")
	}
}

// Connect the IRC connection object to "host[:port]" which should be either
// a hostname or an IP address, with an optional port. To enable explicit SSL
// on the connection to the IRC server, set Conn.SSL to true before calling
// Connect(). The port will default to 6697 if ssl is enabled, and 6667
// otherwise. You can also provide an optional connect password.
func (conn *Conn) Connect(host string, pass ...string) os.Error {
	if conn.connected {
		return os.NewError(fmt.Sprintf(
		  "irc.Connect(): already connected to %s, cannot connect to %s",
		  conn.Host, host))
	}

	if conn.SSL {
		if !hasPort(host) {
			host += ":6697"
		}
		// It's unfortunate that tls.Dial doesn't allow a tls.Config arg,
		// so we simply replicate it here with the correct Config.
		// http://codereview.appspot.com/2883041
		if s, err := net.Dial("tcp", "", host); err == nil {
			// Passing nil config => certs are validated.
			c := tls.Client(s, conn.SSLConfig)
			if err = c.Handshake(); err == nil {
				conn.sock = c
			} else {
				s.Close()
				return err
			}
		} else {
			return err
		}
	} else {
		if !hasPort(host) {
			host += ":6667"
		}
		if s, err := net.Dial("tcp", "", host); err == nil {
			conn.sock = s
		} else {
			return err
		}
	}

	conn.Host = host
	conn.io = bufio.NewReadWriter(
		bufio.NewReader(conn.sock),
		bufio.NewWriter(conn.sock))
	go conn.send()
	go conn.recv()

	if len(pass) > 0 {
		conn.Pass(pass[0])
	}
	conn.Nick(conn.Me.Nick)
	conn.User(conn.Me.Ident, conn.Me.Name)

	go conn.runLoop()
	return nil
}

// dispatch a nicely formatted os.Error to the error channel
func (conn *Conn) error(s string, a ...interface{}) {
	conn.Err <- os.NewError(fmt.Sprintf(s, a...))
}

// copied from http.client for great justice
func hasPort(s string) bool {
	return strings.LastIndex(s, ":") > strings.LastIndex(s, "]")
}

// dispatch input from channel as \r\n terminated line to peer
// flood controlled using hybrid's algorithm if conn.Flood is true
func (conn *Conn) send() {
	lastsent := time.Nanoseconds()
	var badness, linetime, second int64 = 0, 0, 1000000000;
	for line := range conn.out {
		// Hybrid's algorithm allows for 2 seconds per line and an additional
		// 1/120 of a second per character on that line.
		linetime = 2*second + int64(len(line))*second/120
		if !conn.Flood && conn.connected {
			// No point in tallying up flood protection stuff until connected
			if badness += linetime + lastsent - time.Nanoseconds(); badness < 0 {
				// negative badness times are badness...
				badness = int64(0)
			}
		}
		lastsent = time.Nanoseconds()

		// If we've sent more than 10 second's worth of lines according to the
		// calculation above, then we're at risk of "Excess Flood".
		if badness > 10*second && !conn.Flood {
			// so sleep for the current line's time value before sending it
			time.Sleep(linetime)
		}
		if _, err := conn.io.WriteString(line + "\r\n"); err != nil {
			conn.error("irc.send(): %s", err.String())
			conn.shutdown()
			break
		}
		conn.io.Flush()
		if conn.Debug {
			fmt.Println("-> " + line)
		}
	}
}

// receive one \r\n terminated line from peer, parse and dispatch it
func (conn *Conn) recv() {
	for {
		s, err := conn.io.ReadString('\n')
		if err != nil {
			conn.error("irc.recv(): %s", err.String())
			conn.shutdown()
			break
		}
		s = strings.Trim(s, "\r\n")
		if conn.Debug {
			fmt.Println("<- " + s)
		}

		line := &Line{Raw: s}
		if s[0] == ':' {
			// remove a source and parse it
			if idx := strings.Index(s, " "); idx != -1 {
				line.Src, s = s[1:idx], s[idx+1:len(s)]
			} else {
				// pretty sure we shouldn't get here ...
				line.Src = s[1:len(s)]
				conn.in <- line
				continue
			}

			// src can be the hostname of the irc server or a nick!user@host
			line.Host = line.Src
			nidx, uidx := strings.Index(line.Src, "!"), strings.Index(line.Src, "@")
			if uidx != -1 && nidx != -1 {
				line.Nick = line.Src[0:nidx]
				line.Ident = line.Src[nidx+1 : uidx]
				line.Host = line.Src[uidx+1 : len(line.Src)]
			}
		}

		// now we're here, we've parsed a :nick!user@host or :server off
		// s should contain "cmd args[] :text"
		args := strings.Split(s, " :", 2)
		if len(args) > 1 {
			line.Text = args[1]
		}
		args = strings.Fields(args[0])
		line.Cmd = strings.ToUpper(args[0])
		if len(args) > 1 {
			line.Args = args[1:len(args)]
		}
		conn.in <- line
	}
}

func (conn *Conn) runLoop() {
	for line := range conn.in {
			conn.dispatchEvent(line)
	}
}

func (conn *Conn) shutdown() {
	close(conn.in)
	close(conn.out)
	close(conn.Err)
	conn.connected = false
	conn.sock.Close()
	// reinit datastructures ready for next connection
	// do this here rather than after runLoop()'s for due to race
	conn.initialise()
}

// Dumps a load of information about the current state of the connection to a
// string for debugging state tracking and other such things.
func (conn *Conn) String() string {
	str := "GoIRC Connection\n"
	str += "----------------\n\n"
	if conn.connected {
		str += "Connected to " + conn.Host + "\n\n"
	} else {
		str += "Not currently connected!\n\n"
	}
	str += conn.Me.String() + "\n"
	str += "GoIRC Channels\n"
	str += "--------------\n\n"
	for _, ch := range conn.chans {
		str += ch.String() + "\n"
	}
	str += "GoIRC NickNames\n"
	str += "---------------\n\n"
	for _, n := range conn.nicks {
		if n != conn.Me {
			str += n.String() + "\n"
		}
	}
	return str
}
<|endoftext|>


<commit_before>package controllers

import (
	"github.com/UniversityRadioYork/2016-site/models"
	"github.com/UniversityRadioYork/2016-site/structs"
	"github.com/UniversityRadioYork/2016-site/utils"
	"github.com/UniversityRadioYork/myradio-go"
	"github.com/gorilla/mux"
	"log"
	"net/http"
	"strconv"
)

// PeopleController is the controller for the user bio page.
type PeopleController struct {
	Controller
}

// NewPeopleController returns a new PeopleController with the MyRadio session s
// and configuration context c.
func NewPeopleController(s *myradio.Session, c *structs.Config) *PeopleController {
	return &PeopleController{Controller{session: s, config: c}}
}

// Get handles the HTTP GET request r for the user bio page, writing to w.
func (pc *PeopleController) Get(w http.ResponseWriter, r *http.Request) {

	pm := models.NewPeopleModel(pc.session)

	vars := mux.Vars(r)

	id, _ := strconv.Atoi(vars["id"])

	name, bio, officerships, pic, credits, err := pm.Get(id)

	if err != nil {
		//@TODO: Do something proper here, render 404 or something
		log.Println(err)
		return
	}

	data := struct {
		Name           string
		Bio            string
		Officerships   []myradio.Officership
		ProfilePicture myradio.Photo
		ShowCredits    []myradio.ShowMeta
	}{
		Name:           name,
		Bio:            bio,
		Officerships:   officerships,
		ProfilePicture: pic,
		ShowCredits:    credits,
	}

	err = utils.RenderTemplate(w, pc.config.PageContext, data, "people.tmpl")
	if err != nil {
		log.Println(err)
		return
	}

}
<commit_msg>Added ability for {{template current_and_next .}} on people pages.<commit_after>package controllers

import (
	"github.com/UniversityRadioYork/2016-site/models"
	"github.com/UniversityRadioYork/2016-site/structs"
	"github.com/UniversityRadioYork/2016-site/utils"
	"github.com/UniversityRadioYork/myradio-go"
	"github.com/gorilla/mux"
	"log"
	"net/http"
	"strconv"
)

// PeopleController is the controller for the user bio page.
type PeopleController struct {
	Controller
}

// NewPeopleController returns a new PeopleController with the MyRadio session s
// and configuration context c.
func NewPeopleController(s *myradio.Session, c *structs.Config) *PeopleController {
	return &PeopleController{Controller{session: s, config: c}}
}

// Get handles the HTTP GET request r for the user bio page, writing to w.
func (pc *PeopleController) Get(w http.ResponseWriter, r *http.Request) {

	pm := models.NewPeopleModel(pc.session)

	vars := mux.Vars(r)

	id, _ := strconv.Atoi(vars["id"])

	name, bio, officerships, pic, credits, currentAndNext, err := pm.Get(id)

	if err != nil {
		//@TODO: Do something proper here, render 404 or something
		log.Println(err)
		return
	}

	data := struct {
		Name           string
		Bio            string
		Officerships   []myradio.Officership
		ProfilePicture myradio.Photo
		ShowCredits    []myradio.ShowMeta
		CurrentAndNext *myradio.CurrentAndNext
	}{
		Name:           name,
		Bio:            bio,
		Officerships:   officerships,
		ProfilePicture: pic,
		ShowCredits:    credits,
		CurrentAndNext: currentAndNext,
	}

	err = utils.RenderTemplate(w, pc.config.PageContext, data, "people.tmpl", "elements/current_and_next.tmpl")
	if err != nil {
		log.Println(err)
		return
	}

}
<|endoftext|>


<commit_before>package main

import (
	c3to10 "eaciit/wfdemo-git/processapp/threeextractor/convert3secto10min/lib"
	"encoding/json"
	"fmt"
	"io"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strings"
	"sync"

	"github.com/eaciit/database/base"
	"github.com/eaciit/orm"
	"github.com/fsnotify/fsnotify"
	// "github.com/metakeule/fmtdate"
	//dc "eaciit/wfdemo-git/processapp/threeextractor/dataconversion"
	"archive/tar"
	"archive/zip"
	"bytes"
	. "eaciit/wfdemo-git/processapp/watcher/controllers"
	"time"

	tk "github.com/eaciit/toolkit"
)

const (
	NOK = "NOK"
	OK  = "OK"
	// TIME_FORMAT_STR = "DDMMYYhhmmss"
)

type Command struct {
	Action  string
	Command string
	Success string
	Fail    string
}
type Configuration struct {
	Draft    string
	Process  string
	Fail     string
	Success  string
	Errors   string
	Archive  string
	Commands []Command
}

var (
	conn    base.IConnection
	ctx     *orm.DataContext
	conf    Configuration
	mux     = &sync.Mutex{}
	pathSep = string(os.PathSeparator)
)

func main() {

	fmt.Println(".." + pathSep + "conf" + pathSep + "receiver-config.json")

	file, _ := os.Open(".." + pathSep + "conf" + pathSep + "receiver-config.json")
	decoder := json.NewDecoder(file)
	err := decoder.Decode(&conf)

	if err != nil {
		panic(err)
	}

	// now, _ := time.Parse("2006-1-2 15:4:05", "2016-10-22 23:57:30")
	// tenMinInfo := GenTenMinuteInfo(now)

	// log.Printf("%#v \n", tenMinInfo)

	runtime.GOMAXPROCS(runtime.NumCPU())
	log.Println("Starting the app..\n")

	log.Println()
	log.Printf("Draft: %v\n", conf.Draft)
	log.Printf("Process: %v\n", conf.Process)
	log.Printf("Fail: %v\n", conf.Fail)
	log.Printf("Success: %v\n", conf.Success)
	log.Printf("Errors: %v\n", conf.Errors)
	log.Println()

	/*db, _ := PrepareConnection()
	base := new(BaseController)
	base.Ctx = orm.New(db)
	defer base.Ctx.Close()

	log.Println("xxx")

	new(GenTenFromThreeSecond).Generate(base, "sample.csv")*/

	watcher, err := fsnotify.NewWatcher()
	if err != nil {
		log.Fatal(err)
	}
	defer watcher.Close()

	done := make(chan bool)
	go func() {
		for {
			select {
			case event := <-watcher.Events:
				// log.Println("event:", event)
				if event.Op&fsnotify.Create == fsnotify.Create {
					go processFile(event.Name, conf.Commands)
				}
			case err := <-watcher.Errors:
				log.Println("error:", err)
			}
		}
	}()

	// watch draft
	err = watcher.Add(conf.Draft)
	if err != nil {
		log.Fatal(err)
	}

	// watch process
	err = watcher.Add(conf.Process)
	if err != nil {
		log.Fatal(err)
	}

	/*// watch fail
	err = watcher.Add("/Users/frezadev/Documents/watch/fail")
	if err != nil {
		log.Fatal(err)
	}

	// watch success
	err = watcher.Add("/Users/frezadev/Documents/watch/success")
	if err != nil {
		log.Fatal(err)
	}*/
	<-done
}

func processFile(filePath string, com []Command) {
	// dt := time.Now()
	// dtStr := fmtdate.Format(TIME_FORMAT_STR, dt)
	fmt.Println(" >>> Process : ", filePath)
	for true {
		byteOut, err := runCMD("lsof " + filePath)
		if err != nil {
			log.Print("Gagal")
		}
		if len(byteOut) == 0 {
			break
		} else {
			fmt.Println(string(byteOut))
			time.Sleep(200 * time.Millisecond)
		}

	}
	file := strings.Split(filePath, pathSep)
	fileName := file[len(file)-1]

	if strings.Contains(fileName, ".csv") {
		time.Sleep(100 * time.Millisecond)
		log.Printf("Proccess file: %v \n", filePath)
		var action Command

		if strings.Contains(filePath, conf.Draft) {
			action = com[0]
		} else if strings.Contains(filePath, conf.Process) {
			action = com[1]
		}

		// log.Printf("action: %v \n", action)

		next := action.Action
	done:

		for {
			log.Printf("\n\nnext: %v \n", next)

			if next == "DONE" {
				break done
			} else {
				for _, act := range com {
					//fmt.Println("PPPP")
					if act.Action == next {
						next = run(act, fileName)

						break
					}
				}
			}
		}

		log.Printf("DONE for file: %v\n", filePath)
	} else if strings.Contains(fileName, ".tar") {
		time.Sleep(100 * time.Millisecond)
		untar(filePath, conf.Draft)
		_, _ = runCMD(fmt.Sprintf("mv %v %v", filePath, conf.Archive))
	} else if strings.Contains(fileName, ".zip") {
		time.Sleep(100 * time.Millisecond)
		e := unzip(filePath, conf.Draft)
		if e != nil {
			log.Printf(filepath, e.Error())
			_, _ = runCMD(fmt.Sprintf("mv %v %v", filePath, conf.Errors))
		} else {
			fmt.Println("Unzip Done")
			_, _ = runCMD(fmt.Sprintf("mv %v %v", filePath, conf.Archive))
		}
	}
}

func untar(tarball, target string) error {
	reader, err := os.Open(tarball)
	if err != nil {
		return err
	}
	defer reader.Close()
	tarReader := tar.NewReader(reader)

	for {
		header, err := tarReader.Next()

		if err == io.EOF {
			break
		} else if err != nil {
			log.Printf("ERR: %#v \n", err.Error())
			return err
		}

		path := filepath.Join(target, header.Name)
		info := header.FileInfo()
		// log.Printf("path: %#v \n", path)
		// log.Printf("info: %#v \n", info)
		if strings.Contains(info.Name(), ".csv") {
			if info.IsDir() {
				if err = os.MkdirAll(path, info.Mode()); err != nil {
					return err
				}
				continue
			}

			file, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, info.Mode())
			if err != nil {
				return err
			}
			defer file.Close()
			_, err = io.Copy(file, tarReader)
			if err != nil {
				return err
			}
		}
	}

	return nil
}

func unzip(archive, target string) error {
	fmt.Println("Unzip", archive, target)
	reader, err := zip.OpenReader(archive)
	if err != nil {
		return err
	}

	if err := os.MkdirAll(target, 0755); err != nil {
		return err
	}

	for _, file := range reader.File {
		if strings.Contains(file.Name, ".csv") {

			path := filepath.Join(target, file.Name)
			fileReader, err := file.Open()

			if err != nil {
				log.Printf("ERR: %#v \n", err.Error())
				return err
			}
			defer fileReader.Close()

			targetFile, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, file.Mode())
			if err != nil {
				return err
			}
			defer targetFile.Close()

			if _, err := io.Copy(targetFile, fileReader); err != nil {
				return err
			}
		}
	}

	reader.Close()
	return nil
}

func runCMD(cmdStr string) (out []byte, err error) {
	if pathSep == "\\" {
		cmdStr = strings.Replace(cmdStr, "\\", "/", -1)

	}

	if !strings.Contains(cmdStr, "lsof") {
		fmt.Println("sh", "-c", cmdStr)
	}

	var errBuff bytes.Buffer
	cmd := exec.Command("sh", "-c", cmdStr)
	cmd.Stderr = &errBuff
	//cmd.Run()
	//cmd.Path = os.Getenv("Path")
	//fmt.Println(cmd.Path)
	out, err = cmd.Output()
	if err != nil {
		fmt.Println(errBuff.String())
	}
	return
}

func run(action Command, file string) (next string) {
	cmdStr := ""
	runCommand := true
	log.Printf("run: %v | %v \n", action, file)
	// mux.Lock()

	if action.Action == "COPY_TO_PROCESS" {
		cmdStr = fmt.Sprintf(action.Command, conf.Draft+pathSep+file, conf.Process+pathSep+file)
	} else if action.Action == "COPY_TO_SUCCESS" {
		runCommand = doProcess(conf.Process + pathSep + file)

		cmdStr = fmt.Sprintf(action.Command, conf.Process+pathSep+file, conf.Success+pathSep+file)
	} else if action.Action == "COPY_TO_FAIL" {
		cmdStr = fmt.Sprintf(action.Command, conf.Process+pathSep+file, conf.Fail+pathSep+file)
	}
	//log.Printf("cmdstr: %v \n", cmdStr)
	// mux.Unlock()

	if runCommand {
		out, err := runCMD(cmdStr)

		if out != nil {
			log.Printf("%v \n", out)
		}

		if err != nil {
			log.Printf("result: %v %s\n%s", err.Error(), cmdStr, string(out))
			next = action.Success
		} else {
			next = action.Success
		}
	} else {
		log.Println("DONE")
		next = action.Fail
	}

	// log.Printf("next: %v \n", next)

	return
}

func doProcess(file string) (success bool) {
	log.Printf("doProcess: %v \n", file)
	db, e := PrepareConnection()
	if e != nil {
		log.Printf("ERROR on Process: %v\n", e.Error())
		success = false
	} else {
		//start := time.Now()
		base := new(BaseController)
		base.Ctx = orm.New(db)
		defer base.Ctx.Close()

		anFile := strings.Split(file, pathSep)
		fileName := anFile[len(anFile)-1]

		muxDo := &sync.Mutex{}

		muxDo.Lock()
		errorLine := new(ConvScadaTreeSecs).Generate(base, file)
		WriteWatcherErrors(errorLine, fileName, conf.Errors)
		log.Println("ConvScadaTreeSecs: DONE")
		muxDo.Unlock()
		log.Println("Begin Converting ", file)
		err := c3to10.Generate(tk.M{}.Set("selector", "file").Set("file", fileName))
		if err != nil {
			tk.Println(err)
		} else {
			tk.Println(">> DONE <<")
		}

		// errorLine = tk.M{}
		// errorLine = new(GenTenFromThreeSecond).Generate(base, fileName)
		// WriteWatcherErrors(errorLine, fileName+"-ten", conf.Errors)
		// log.Println("GenTenFromThreeSecond: DONE")

		/*
			muxDo.Lock()
			errorLine = tk.M{}
			conv := dc.NewDataConversion(base.Ctx)
			errorLine = conv.Generate(fileName)
			// errorLine = new(GenTenFromThreeSecond).Generate(base, fileName)
			WriteWatcherErrors(errorLine, fileName+"-ten", conf.Errors)
			log.Printf("GenTenFromThreeSecond: %v DONE | in %v seconds \n", file, time.Now().Sub(start).Seconds())
			success = true
			muxDo.Unlock()*/
	}

	return
}
<commit_msg>bug fix<commit_after>package main

import (
	c3to10 "eaciit/wfdemo-git/processapp/threeextractor/convert3secto10min/lib"
	"encoding/json"
	"fmt"
	"io"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strings"
	"sync"

	"github.com/eaciit/database/base"
	"github.com/eaciit/orm"
	"github.com/fsnotify/fsnotify"
	// "github.com/metakeule/fmtdate"
	//dc "eaciit/wfdemo-git/processapp/threeextractor/dataconversion"
	"archive/tar"
	"archive/zip"
	"bytes"
	. "eaciit/wfdemo-git/processapp/watcher/controllers"
	"time"

	tk "github.com/eaciit/toolkit"
)

const (
	NOK = "NOK"
	OK  = "OK"
	// TIME_FORMAT_STR = "DDMMYYhhmmss"
)

type Command struct {
	Action  string
	Command string
	Success string
	Fail    string
}
type Configuration struct {
	Draft    string
	Process  string
	Fail     string
	Success  string
	Errors   string
	Archive  string
	Commands []Command
}

var (
	conn    base.IConnection
	ctx     *orm.DataContext
	conf    Configuration
	mux     = &sync.Mutex{}
	pathSep = string(os.PathSeparator)
)

func main() {

	fmt.Println(".." + pathSep + "conf" + pathSep + "receiver-config.json")

	file, _ := os.Open(".." + pathSep + "conf" + pathSep + "receiver-config.json")
	decoder := json.NewDecoder(file)
	err := decoder.Decode(&conf)

	if err != nil {
		panic(err)
	}

	// now, _ := time.Parse("2006-1-2 15:4:05", "2016-10-22 23:57:30")
	// tenMinInfo := GenTenMinuteInfo(now)

	// log.Printf("%#v \n", tenMinInfo)

	runtime.GOMAXPROCS(runtime.NumCPU())
	log.Println("Starting the app..\n")

	log.Println()
	log.Printf("Draft: %v\n", conf.Draft)
	log.Printf("Process: %v\n", conf.Process)
	log.Printf("Fail: %v\n", conf.Fail)
	log.Printf("Success: %v\n", conf.Success)
	log.Printf("Errors: %v\n", conf.Errors)
	log.Println()

	/*db, _ := PrepareConnection()
	base := new(BaseController)
	base.Ctx = orm.New(db)
	defer base.Ctx.Close()

	log.Println("xxx")

	new(GenTenFromThreeSecond).Generate(base, "sample.csv")*/

	watcher, err := fsnotify.NewWatcher()
	if err != nil {
		log.Fatal(err)
	}
	defer watcher.Close()

	done := make(chan bool)
	go func() {
		for {
			select {
			case event := <-watcher.Events:
				// log.Println("event:", event)
				if event.Op&fsnotify.Create == fsnotify.Create {
					go processFile(event.Name, conf.Commands)
				}
			case err := <-watcher.Errors:
				log.Println("error:", err)
			}
		}
	}()

	// watch draft
	err = watcher.Add(conf.Draft)
	if err != nil {
		log.Fatal(err)
	}

	// watch process
	err = watcher.Add(conf.Process)
	if err != nil {
		log.Fatal(err)
	}

	/*// watch fail
	err = watcher.Add("/Users/frezadev/Documents/watch/fail")
	if err != nil {
		log.Fatal(err)
	}

	// watch success
	err = watcher.Add("/Users/frezadev/Documents/watch/success")
	if err != nil {
		log.Fatal(err)
	}*/
	<-done
}

func processFile(filePath string, com []Command) {
	// dt := time.Now()
	// dtStr := fmtdate.Format(TIME_FORMAT_STR, dt)
	fmt.Println(" >>> Process : ", filePath)
	for true {
		byteOut, err := runCMD("lsof " + filePath)
		if err != nil {
			log.Print("Gagal")
		}
		if len(byteOut) == 0 {
			break
		} else {
			fmt.Println(string(byteOut))
			time.Sleep(200 * time.Millisecond)
		}

	}
	file := strings.Split(filePath, pathSep)
	fileName := file[len(file)-1]

	if strings.Contains(fileName, ".csv") {
		time.Sleep(100 * time.Millisecond)
		log.Printf("Proccess file: %v \n", filePath)
		var action Command

		if strings.Contains(filePath, conf.Draft) {
			action = com[0]
		} else if strings.Contains(filePath, conf.Process) {
			action = com[1]
		}

		// log.Printf("action: %v \n", action)

		next := action.Action
	done:

		for {
			log.Printf("\n\nnext: %v \n", next)

			if next == "DONE" {
				break done
			} else {
				for _, act := range com {
					//fmt.Println("PPPP")
					if act.Action == next {
						next = run(act, fileName)

						break
					}
				}
			}
		}

		log.Printf("DONE for file: %v\n", filePath)
	} else if strings.Contains(fileName, ".tar") {
		time.Sleep(100 * time.Millisecond)
		untar(filePath, conf.Draft)
		_, _ = runCMD(fmt.Sprintf("mv %v %v", filePath, conf.Archive))
	} else if strings.Contains(fileName, ".zip") {
		time.Sleep(100 * time.Millisecond)
		e := unzip(filePath, conf.Draft)
		if e != nil {
			log.Printf("%s - %s", filepath, e.Error())
			_, _ = runCMD(fmt.Sprintf("mv %v %v", filePath, conf.Errors))
		} else {
			fmt.Println("Unzip Done")
			_, _ = runCMD(fmt.Sprintf("mv %v %v", filePath, conf.Archive))
		}
	}
}

func untar(tarball, target string) error {
	reader, err := os.Open(tarball)
	if err != nil {
		return err
	}
	defer reader.Close()
	tarReader := tar.NewReader(reader)

	for {
		header, err := tarReader.Next()

		if err == io.EOF {
			break
		} else if err != nil {
			log.Printf("ERR: %#v \n", err.Error())
			return err
		}

		path := filepath.Join(target, header.Name)
		info := header.FileInfo()
		// log.Printf("path: %#v \n", path)
		// log.Printf("info: %#v \n", info)
		if strings.Contains(info.Name(), ".csv") {
			if info.IsDir() {
				if err = os.MkdirAll(path, info.Mode()); err != nil {
					return err
				}
				continue
			}

			file, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, info.Mode())
			if err != nil {
				return err
			}
			defer file.Close()
			_, err = io.Copy(file, tarReader)
			if err != nil {
				return err
			}
		}
	}

	return nil
}

func unzip(archive, target string) error {
	fmt.Println("Unzip", archive, target)
	reader, err := zip.OpenReader(archive)
	if err != nil {
		return err
	}

	if err := os.MkdirAll(target, 0755); err != nil {
		return err
	}

	for _, file := range reader.File {
		if strings.Contains(file.Name, ".csv") {

			path := filepath.Join(target, file.Name)
			fileReader, err := file.Open()

			if err != nil {
				log.Printf("ERR: %#v \n", err.Error())
				return err
			}
			defer fileReader.Close()

			targetFile, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, file.Mode())
			if err != nil {
				return err
			}
			defer targetFile.Close()

			if _, err := io.Copy(targetFile, fileReader); err != nil {
				return err
			}
		}
	}

	reader.Close()
	return nil
}

func runCMD(cmdStr string) (out []byte, err error) {
	if pathSep == "\\" {
		cmdStr = strings.Replace(cmdStr, "\\", "/", -1)

	}

	if !strings.Contains(cmdStr, "lsof") {
		fmt.Println("sh", "-c", cmdStr)
	}

	var errBuff bytes.Buffer
	cmd := exec.Command("sh", "-c", cmdStr)
	cmd.Stderr = &errBuff
	//cmd.Run()
	//cmd.Path = os.Getenv("Path")
	//fmt.Println(cmd.Path)
	out, err = cmd.Output()
	if err != nil {
		fmt.Println(errBuff.String())
	}
	return
}

func run(action Command, file string) (next string) {
	cmdStr := ""
	runCommand := true
	log.Printf("run: %v | %v \n", action, file)
	// mux.Lock()

	if action.Action == "COPY_TO_PROCESS" {
		cmdStr = fmt.Sprintf(action.Command, conf.Draft+pathSep+file, conf.Process+pathSep+file)
	} else if action.Action == "COPY_TO_SUCCESS" {
		runCommand = doProcess(conf.Process + pathSep + file)

		cmdStr = fmt.Sprintf(action.Command, conf.Process+pathSep+file, conf.Success+pathSep+file)
	} else if action.Action == "COPY_TO_FAIL" {
		cmdStr = fmt.Sprintf(action.Command, conf.Process+pathSep+file, conf.Fail+pathSep+file)
	}
	//log.Printf("cmdstr: %v \n", cmdStr)
	// mux.Unlock()

	if runCommand {
		out, err := runCMD(cmdStr)

		if out != nil {
			log.Printf("%v \n", out)
		}

		if err != nil {
			log.Printf("result: %v %s\n%s", err.Error(), cmdStr, string(out))
			next = action.Success
		} else {
			next = action.Success
		}
	} else {
		log.Println("DONE")
		next = action.Fail
	}

	// log.Printf("next: %v \n", next)

	return
}

func doProcess(file string) (success bool) {
	log.Printf("doProcess: %v \n", file)
	db, e := PrepareConnection()
	if e != nil {
		log.Printf("ERROR on Process: %v\n", e.Error())
		success = false
	} else {
		//start := time.Now()
		base := new(BaseController)
		base.Ctx = orm.New(db)
		defer base.Ctx.Close()

		anFile := strings.Split(file, pathSep)
		fileName := anFile[len(anFile)-1]

		muxDo := &sync.Mutex{}

		muxDo.Lock()
		errorLine := new(ConvScadaTreeSecs).Generate(base, file)
		WriteWatcherErrors(errorLine, fileName, conf.Errors)
		log.Println("ConvScadaTreeSecs: DONE")
		muxDo.Unlock()
		log.Println("Begin Converting ", file)
		err := c3to10.Generate(tk.M{}.Set("selector", "file").Set("file", fileName))
		if err != nil {
			tk.Println(err)
		} else {
			tk.Println(">> DONE <<")
		}

		// errorLine = tk.M{}
		// errorLine = new(GenTenFromThreeSecond).Generate(base, fileName)
		// WriteWatcherErrors(errorLine, fileName+"-ten", conf.Errors)
		// log.Println("GenTenFromThreeSecond: DONE")

		/*
			muxDo.Lock()
			errorLine = tk.M{}
			conv := dc.NewDataConversion(base.Ctx)
			errorLine = conv.Generate(fileName)
			// errorLine = new(GenTenFromThreeSecond).Generate(base, fileName)
			WriteWatcherErrors(errorLine, fileName+"-ten", conf.Errors)
			log.Printf("GenTenFromThreeSecond: %v DONE | in %v seconds \n", file, time.Now().Sub(start).Seconds())
			success = true
			muxDo.Unlock()*/
	}

	return
}
<|endoftext|>


<commit_before>// Copyright 2019 Google LLC All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package v1

import (
	"crypto/x509"
	"net/url"

	agonesv1 "agones.dev/agones/pkg/apis/agones/v1"
	"agones.dev/agones/pkg/util/runtime"
	admregv1 "k8s.io/api/admissionregistration/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/types"
	"k8s.io/apimachinery/pkg/util/intstr"
)

// +genclient
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

// FleetAutoscaler is the data structure for a FleetAutoscaler resource
type FleetAutoscaler struct {
	metav1.TypeMeta   `json:",inline"`
	metav1.ObjectMeta `json:"metadata,omitempty"`

	Spec   FleetAutoscalerSpec   `json:"spec"`
	Status FleetAutoscalerStatus `json:"status"`
}

// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

// FleetAutoscalerList is a list of Fleet Scaler resources
type FleetAutoscalerList struct {
	metav1.TypeMeta `json:",inline"`
	metav1.ListMeta `json:"metadata,omitempty"`

	Items []FleetAutoscaler `json:"items"`
}

// FleetAutoscalerSpec is the spec for a Fleet Scaler
type FleetAutoscalerSpec struct {
	FleetName string `json:"fleetName"`

	// Autoscaling policy
	Policy FleetAutoscalerPolicy `json:"policy"`
	// [Stage:Alpha]
	// [FeatureFlag:CustomFasSyncInterval]
	// Sync defines when FleetAutoscalers runs autoscaling
	// +optional
	Sync *FleetAutoscalerSync `json:"sync,omitempty"`
}

// FleetAutoscalerPolicy describes how to scale a fleet
type FleetAutoscalerPolicy struct {
	// Type of autoscaling policy.
	Type FleetAutoscalerPolicyType `json:"type"`

	// Buffer policy config params. Present only if FleetAutoscalerPolicyType = Buffer.
	// +optional
	Buffer *BufferPolicy `json:"buffer,omitempty"`
	// Webhook policy config params. Present only if FleetAutoscalerPolicyType = Webhook.
	// +optional
	Webhook *WebhookPolicy `json:"webhook,omitempty"`
}

// FleetAutoscalerPolicyType is the policy for autoscaling
// for a given Fleet
type FleetAutoscalerPolicyType string

// FleetAutoscalerSync describes when to sync a fleet
type FleetAutoscalerSync struct {
	// Type of autoscaling sync.
	Type FleetAutoscalerSyncType `json:"type"`

	// FixedInterval config params. Present only if FleetAutoscalerSyncType = FixedInterval.
	// +optional
	FixedInterval FixedIntervalSync `json:"fixedInterval"`
}

// FleetAutoscalerSyncType is the sync strategy for a given Fleet
type FleetAutoscalerSyncType string

const (
	// BufferPolicyType FleetAutoscalerPolicyType is a simple buffering strategy for Ready
	// GameServers
	BufferPolicyType FleetAutoscalerPolicyType = "Buffer"
	// WebhookPolicyType is a simple webhook strategy used for horizontal fleet scaling
	// GameServers
	WebhookPolicyType FleetAutoscalerPolicyType = "Webhook"
	// FixedIntervalSyncType is a simple fixed interval based strategy for trigger autoscaling
	FixedIntervalSyncType FleetAutoscalerSyncType = "FixedInterval"

	defaultIntervalSyncSeconds = 30
)

// BufferPolicy controls the desired behavior of the buffer policy.
type BufferPolicy struct {
	// MaxReplicas is the maximum amount of replicas that the fleet may have.
	// It must be bigger than both MinReplicas and BufferSize
	MaxReplicas int32 `json:"maxReplicas"`

	// MinReplicas is the minimum amount of replicas that the fleet must have
	// If zero, it is ignored.
	// If non zero, it must be smaller than MaxReplicas and bigger than BufferSize
	MinReplicas int32 `json:"minReplicas"`

	// BufferSize defines how many replicas the autoscaler tries to have ready all the time
	// Value can be an absolute number (ex: 5) or a percentage of desired gs instances (ex: 15%)
	// Absolute number is calculated from percentage by rounding up.
	// Example: when this is set to 20%, the autoscaler will make sure that 20%
	//   of the fleet's game server replicas are ready. When this is set to 20,
	//   the autoscaler will make sure that there are 20 available game servers
	// Must be bigger than 0
	// Note: by "ready" we understand in this case "non-allocated"; this is done to ensure robustness
	//       and computation stability in different edge case (fleet just created, not enough
	//       capacity in the cluster etc)
	BufferSize intstr.IntOrString `json:"bufferSize"`
}

// WebhookPolicy controls the desired behavior of the webhook policy.
// It contains the description of the webhook autoscaler service
// used to form url which is accessible inside the cluster
type WebhookPolicy admregv1.WebhookClientConfig

// FixedIntervalSync controls the desired behavior of the fixed interval based sync.
type FixedIntervalSync struct {
	// Seconds defines how often we run fleet autoscaling in seconds
	Seconds int32 `json:"seconds"`
}

// FleetAutoscalerStatus defines the current status of a FleetAutoscaler
type FleetAutoscalerStatus struct {
	// CurrentReplicas is the current number of gameserver replicas
	// of the fleet managed by this autoscaler, as last seen by the autoscaler
	CurrentReplicas int32 `json:"currentReplicas"`

	// DesiredReplicas is the desired number of gameserver replicas
	// of the fleet managed by this autoscaler, as last calculated by the autoscaler
	DesiredReplicas int32 `json:"desiredReplicas"`

	// lastScaleTime is the last time the FleetAutoscaler scaled the attached fleet,
	// +optional
	LastScaleTime *metav1.Time `json:"lastScaleTime"`

	// AbleToScale indicates that we can access the target fleet
	AbleToScale bool `json:"ableToScale"`

	// ScalingLimited indicates that the calculated scale would be above or below the range
	// defined by MinReplicas and MaxReplicas, and has thus been capped.
	ScalingLimited bool `json:"scalingLimited"`
}

// FleetAutoscaleRequest defines the request to webhook autoscaler endpoint
type FleetAutoscaleRequest struct {
	// UID is an identifier for the individual request/response. It allows us to distinguish instances of requests which are
	// otherwise identical (parallel requests, requests when earlier requests did not modify etc)
	// The UID is meant to track the round trip (request/response) between the Autoscaler and the WebHook, not the user request.
	// It is suitable for correlating log entries between the webhook and apiserver, for either auditing or debugging.
	UID types.UID `json:"uid"`
	// Name is the name of the Fleet being scaled
	Name string `json:"name"`
	// Namespace is the namespace associated with the request (if any).
	Namespace string `json:"namespace"`
	// The Fleet's status values
	Status agonesv1.FleetStatus `json:"status"`
}

// FleetAutoscaleResponse defines the response of webhook autoscaler endpoint
type FleetAutoscaleResponse struct {
	// UID is an identifier for the individual request/response.
	// This should be copied over from the corresponding FleetAutoscaleRequest.
	UID types.UID `json:"uid"`
	// Set to false if no scaling should occur to the Fleet
	Scale bool `json:"scale"`
	// The targeted replica count
	Replicas int32 `json:"replicas"`
}

// FleetAutoscaleReview is passed to the webhook with a populated Request value,
// and then returned with a populated Response.
type FleetAutoscaleReview struct {
	Request  *FleetAutoscaleRequest  `json:"request"`
	Response *FleetAutoscaleResponse `json:"response"`
}

// Validate validates the FleetAutoscaler scaling settings
func (fas *FleetAutoscaler) Validate(causes []metav1.StatusCause) []metav1.StatusCause {
	switch fas.Spec.Policy.Type {
	case BufferPolicyType:
		causes = fas.Spec.Policy.Buffer.ValidateBufferPolicy(causes)

	case WebhookPolicyType:
		causes = fas.Spec.Policy.Webhook.ValidateWebhookPolicy(causes)
	}

	if runtime.FeatureEnabled(runtime.FeatureCustomFasSyncInterval) {
		causes = fas.Spec.Sync.FixedInterval.ValidateFixedIntervalSync(causes)
	}
	return causes
}

// ValidateWebhookPolicy validates the FleetAutoscaler Webhook policy settings
func (w *WebhookPolicy) ValidateWebhookPolicy(causes []metav1.StatusCause) []metav1.StatusCause {
	if w == nil {
		return append(causes, metav1.StatusCause{
			Type:    metav1.CauseTypeFieldValueInvalid,
			Field:   "webhook",
			Message: "webhook policy config params are missing",
		})
	}
	if w.Service == nil && w.URL == nil {
		causes = append(causes, metav1.StatusCause{
			Type:    metav1.CauseTypeFieldValueNotFound,
			Field:   "url",
			Message: "url should be provided",
		})
	}
	if w.Service != nil && w.URL != nil {
		causes = append(causes, metav1.StatusCause{
			Type:    metav1.CauseTypeFieldValueNotFound,
			Field:   "url",
			Message: "service and url cannot be used simultaneously",
		})
	}
	if w.CABundle != nil {
		rootCAs := x509.NewCertPool()
		//Check that CABundle provided is correctly encoded certificate
		if ok := rootCAs.AppendCertsFromPEM(w.CABundle); !ok {
			causes = append(causes, metav1.StatusCause{
				Type:    metav1.CauseTypeFieldValueInvalid,
				Field:   "caBundle",
				Message: "CA Bundle is not valid",
			})
		}
	}
	if w.URL != nil {
		u, err := url.Parse(*w.URL)
		if err != nil {
			causes = append(causes, metav1.StatusCause{
				Type:    metav1.CauseTypeFieldValueInvalid,
				Field:   "url",
				Message: "url is not valid",
			})
		} else if u.Scheme == "https" {
			if w.CABundle == nil {
				causes = append(causes, metav1.StatusCause{
					Type:    metav1.CauseTypeFieldValueNotFound,
					Field:   "caBundle",
					Message: "CABundle should be provided if HTTPS webhook is used",
				})
			}
		}

	}
	return causes
}

// ValidateBufferPolicy validates the FleetAutoscaler Buffer policy settings
func (b *BufferPolicy) ValidateBufferPolicy(causes []metav1.StatusCause) []metav1.StatusCause {
	if b == nil {
		return append(causes, metav1.StatusCause{
			Type:    metav1.CauseTypeFieldValueInvalid,
			Field:   "buffer",
			Message: "Buffer policy config params are missing",
		})
	}
	if b.MinReplicas > b.MaxReplicas {
		causes = append(causes, metav1.StatusCause{
			Type:    metav1.CauseTypeFieldValueInvalid,
			Field:   "minReplicas",
			Message: "minReplicas is bigger than maxReplicas",
		})
	}
	if b.BufferSize.Type == intstr.Int {
		if b.BufferSize.IntValue() <= 0 {
			causes = append(causes, metav1.StatusCause{
				Type:    metav1.CauseTypeFieldValueInvalid,
				Field:   "bufferSize",
				Message: "bufferSize must be bigger than 0",
			})
		}
		if b.MaxReplicas < int32(b.BufferSize.IntValue()) {
			causes = append(causes, metav1.StatusCause{
				Type:    metav1.CauseTypeFieldValueInvalid,
				Field:   "maxReplicas",
				Message: "maxReplicas must be bigger than bufferSize",
			})
		}
		if b.MinReplicas != 0 && b.MinReplicas < int32(b.BufferSize.IntValue()) {
			causes = append(causes, metav1.StatusCause{
				Type:    metav1.CauseTypeFieldValueInvalid,
				Field:   "minReplicas",
				Message: "minReplicas is smaller than bufferSize",
			})
		}
	} else {
		r, err := intstr.GetValueFromIntOrPercent(&b.BufferSize, 100, true)
		if err != nil || r < 1 || r > 99 {
			causes = append(causes, metav1.StatusCause{
				Type:    metav1.CauseTypeFieldValueInvalid,
				Field:   "bufferSize",
				Message: "bufferSize does not have a valid percentage value (1%-99%)",
			})
		}
		// When there is no allocated gameservers in a fleet,
		// Fleetautoscaler would reduce size of a fleet to MinReplicas.
		// If we have 0 MinReplicas and 0 Allocated then Fleetautoscaler would set Ready Replicas to 0
		// and we will not be able to raise the number of GS in a Fleet above zero
		if b.MinReplicas < 1 {
			causes = append(causes, metav1.StatusCause{
				Type:    metav1.CauseTypeFieldValueInvalid,
				Field:   "minReplicas",
				Message: "minReplicas should be above 0 when used with percentage value bufferSize",
			})
		}
	}
	return causes
}

// ValidateFixedIntervalSync validates the FixedIntervalSync settings
func (i *FixedIntervalSync) ValidateFixedIntervalSync(causes []metav1.StatusCause) []metav1.StatusCause {
	if i == nil {
		return append(causes, metav1.StatusCause{
			Type:    metav1.CauseTypeFieldValueInvalid,
			Field:   "fixedInterval",
			Message: "fixedInterval config params are missing",
		})
	}
	if i.Seconds <= 0 {
		return append(causes, metav1.StatusCause{
			Type:    metav1.CauseTypeFieldValueInvalid,
			Field:   "seconds",
			Message: "seconds should be bigger than 0",
		})
	}
	return causes
}

// ApplyDefaults applies default values to the FleetAutoscaler
func (fas *FleetAutoscaler) ApplyDefaults() {
	if runtime.FeatureEnabled(runtime.FeatureCustomFasSyncInterval) {
		if fas.Spec.Sync == nil {
			fas.Spec.Sync = &FleetAutoscalerSync{}
		}
		if fas.Spec.Sync.Type == "" {
			fas.Spec.Sync.Type = FixedIntervalSyncType
		}
		if fas.Spec.Sync.FixedInterval.Seconds == 0 {
			fas.Spec.Sync.FixedInterval.Seconds = defaultIntervalSyncSeconds
		}
	}
}
<commit_msg>Add nil check to fleet autoscaler validation for sync field (#2246)<commit_after>// Copyright 2019 Google LLC All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package v1

import (
	"crypto/x509"
	"net/url"

	agonesv1 "agones.dev/agones/pkg/apis/agones/v1"
	"agones.dev/agones/pkg/util/runtime"
	admregv1 "k8s.io/api/admissionregistration/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/types"
	"k8s.io/apimachinery/pkg/util/intstr"
)

// +genclient
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

// FleetAutoscaler is the data structure for a FleetAutoscaler resource
type FleetAutoscaler struct {
	metav1.TypeMeta   `json:",inline"`
	metav1.ObjectMeta `json:"metadata,omitempty"`

	Spec   FleetAutoscalerSpec   `json:"spec"`
	Status FleetAutoscalerStatus `json:"status"`
}

// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

// FleetAutoscalerList is a list of Fleet Scaler resources
type FleetAutoscalerList struct {
	metav1.TypeMeta `json:",inline"`
	metav1.ListMeta `json:"metadata,omitempty"`

	Items []FleetAutoscaler `json:"items"`
}

// FleetAutoscalerSpec is the spec for a Fleet Scaler
type FleetAutoscalerSpec struct {
	FleetName string `json:"fleetName"`

	// Autoscaling policy
	Policy FleetAutoscalerPolicy `json:"policy"`
	// [Stage:Alpha]
	// [FeatureFlag:CustomFasSyncInterval]
	// Sync defines when FleetAutoscalers runs autoscaling
	// +optional
	Sync *FleetAutoscalerSync `json:"sync,omitempty"`
}

// FleetAutoscalerPolicy describes how to scale a fleet
type FleetAutoscalerPolicy struct {
	// Type of autoscaling policy.
	Type FleetAutoscalerPolicyType `json:"type"`

	// Buffer policy config params. Present only if FleetAutoscalerPolicyType = Buffer.
	// +optional
	Buffer *BufferPolicy `json:"buffer,omitempty"`
	// Webhook policy config params. Present only if FleetAutoscalerPolicyType = Webhook.
	// +optional
	Webhook *WebhookPolicy `json:"webhook,omitempty"`
}

// FleetAutoscalerPolicyType is the policy for autoscaling
// for a given Fleet
type FleetAutoscalerPolicyType string

// FleetAutoscalerSync describes when to sync a fleet
type FleetAutoscalerSync struct {
	// Type of autoscaling sync.
	Type FleetAutoscalerSyncType `json:"type"`

	// FixedInterval config params. Present only if FleetAutoscalerSyncType = FixedInterval.
	// +optional
	FixedInterval FixedIntervalSync `json:"fixedInterval"`
}

// FleetAutoscalerSyncType is the sync strategy for a given Fleet
type FleetAutoscalerSyncType string

const (
	// BufferPolicyType FleetAutoscalerPolicyType is a simple buffering strategy for Ready
	// GameServers
	BufferPolicyType FleetAutoscalerPolicyType = "Buffer"
	// WebhookPolicyType is a simple webhook strategy used for horizontal fleet scaling
	// GameServers
	WebhookPolicyType FleetAutoscalerPolicyType = "Webhook"
	// FixedIntervalSyncType is a simple fixed interval based strategy for trigger autoscaling
	FixedIntervalSyncType FleetAutoscalerSyncType = "FixedInterval"

	defaultIntervalSyncSeconds = 30
)

// BufferPolicy controls the desired behavior of the buffer policy.
type BufferPolicy struct {
	// MaxReplicas is the maximum amount of replicas that the fleet may have.
	// It must be bigger than both MinReplicas and BufferSize
	MaxReplicas int32 `json:"maxReplicas"`

	// MinReplicas is the minimum amount of replicas that the fleet must have
	// If zero, it is ignored.
	// If non zero, it must be smaller than MaxReplicas and bigger than BufferSize
	MinReplicas int32 `json:"minReplicas"`

	// BufferSize defines how many replicas the autoscaler tries to have ready all the time
	// Value can be an absolute number (ex: 5) or a percentage of desired gs instances (ex: 15%)
	// Absolute number is calculated from percentage by rounding up.
	// Example: when this is set to 20%, the autoscaler will make sure that 20%
	//   of the fleet's game server replicas are ready. When this is set to 20,
	//   the autoscaler will make sure that there are 20 available game servers
	// Must be bigger than 0
	// Note: by "ready" we understand in this case "non-allocated"; this is done to ensure robustness
	//       and computation stability in different edge case (fleet just created, not enough
	//       capacity in the cluster etc)
	BufferSize intstr.IntOrString `json:"bufferSize"`
}

// WebhookPolicy controls the desired behavior of the webhook policy.
// It contains the description of the webhook autoscaler service
// used to form url which is accessible inside the cluster
type WebhookPolicy admregv1.WebhookClientConfig

// FixedIntervalSync controls the desired behavior of the fixed interval based sync.
type FixedIntervalSync struct {
	// Seconds defines how often we run fleet autoscaling in seconds
	Seconds int32 `json:"seconds"`
}

// FleetAutoscalerStatus defines the current status of a FleetAutoscaler
type FleetAutoscalerStatus struct {
	// CurrentReplicas is the current number of gameserver replicas
	// of the fleet managed by this autoscaler, as last seen by the autoscaler
	CurrentReplicas int32 `json:"currentReplicas"`

	// DesiredReplicas is the desired number of gameserver replicas
	// of the fleet managed by this autoscaler, as last calculated by the autoscaler
	DesiredReplicas int32 `json:"desiredReplicas"`

	// lastScaleTime is the last time the FleetAutoscaler scaled the attached fleet,
	// +optional
	LastScaleTime *metav1.Time `json:"lastScaleTime"`

	// AbleToScale indicates that we can access the target fleet
	AbleToScale bool `json:"ableToScale"`

	// ScalingLimited indicates that the calculated scale would be above or below the range
	// defined by MinReplicas and MaxReplicas, and has thus been capped.
	ScalingLimited bool `json:"scalingLimited"`
}

// FleetAutoscaleRequest defines the request to webhook autoscaler endpoint
type FleetAutoscaleRequest struct {
	// UID is an identifier for the individual request/response. It allows us to distinguish instances of requests which are
	// otherwise identical (parallel requests, requests when earlier requests did not modify etc)
	// The UID is meant to track the round trip (request/response) between the Autoscaler and the WebHook, not the user request.
	// It is suitable for correlating log entries between the webhook and apiserver, for either auditing or debugging.
	UID types.UID `json:"uid"`
	// Name is the name of the Fleet being scaled
	Name string `json:"name"`
	// Namespace is the namespace associated with the request (if any).
	Namespace string `json:"namespace"`
	// The Fleet's status values
	Status agonesv1.FleetStatus `json:"status"`
}

// FleetAutoscaleResponse defines the response of webhook autoscaler endpoint
type FleetAutoscaleResponse struct {
	// UID is an identifier for the individual request/response.
	// This should be copied over from the corresponding FleetAutoscaleRequest.
	UID types.UID `json:"uid"`
	// Set to false if no scaling should occur to the Fleet
	Scale bool `json:"scale"`
	// The targeted replica count
	Replicas int32 `json:"replicas"`
}

// FleetAutoscaleReview is passed to the webhook with a populated Request value,
// and then returned with a populated Response.
type FleetAutoscaleReview struct {
	Request  *FleetAutoscaleRequest  `json:"request"`
	Response *FleetAutoscaleResponse `json:"response"`
}

// Validate validates the FleetAutoscaler scaling settings
func (fas *FleetAutoscaler) Validate(causes []metav1.StatusCause) []metav1.StatusCause {
	switch fas.Spec.Policy.Type {
	case BufferPolicyType:
		causes = fas.Spec.Policy.Buffer.ValidateBufferPolicy(causes)

	case WebhookPolicyType:
		causes = fas.Spec.Policy.Webhook.ValidateWebhookPolicy(causes)
	}

	if runtime.FeatureEnabled(runtime.FeatureCustomFasSyncInterval) && fas.Spec.Sync != nil {
		causes = fas.Spec.Sync.FixedInterval.ValidateFixedIntervalSync(causes)
	}
	return causes
}

// ValidateWebhookPolicy validates the FleetAutoscaler Webhook policy settings
func (w *WebhookPolicy) ValidateWebhookPolicy(causes []metav1.StatusCause) []metav1.StatusCause {
	if w == nil {
		return append(causes, metav1.StatusCause{
			Type:    metav1.CauseTypeFieldValueInvalid,
			Field:   "webhook",
			Message: "webhook policy config params are missing",
		})
	}
	if w.Service == nil && w.URL == nil {
		causes = append(causes, metav1.StatusCause{
			Type:    metav1.CauseTypeFieldValueNotFound,
			Field:   "url",
			Message: "url should be provided",
		})
	}
	if w.Service != nil && w.URL != nil {
		causes = append(causes, metav1.StatusCause{
			Type:    metav1.CauseTypeFieldValueNotFound,
			Field:   "url",
			Message: "service and url cannot be used simultaneously",
		})
	}
	if w.CABundle != nil {
		rootCAs := x509.NewCertPool()
		//Check that CABundle provided is correctly encoded certificate
		if ok := rootCAs.AppendCertsFromPEM(w.CABundle); !ok {
			causes = append(causes, metav1.StatusCause{
				Type:    metav1.CauseTypeFieldValueInvalid,
				Field:   "caBundle",
				Message: "CA Bundle is not valid",
			})
		}
	}
	if w.URL != nil {
		u, err := url.Parse(*w.URL)
		if err != nil {
			causes = append(causes, metav1.StatusCause{
				Type:    metav1.CauseTypeFieldValueInvalid,
				Field:   "url",
				Message: "url is not valid",
			})
		} else if u.Scheme == "https" {
			if w.CABundle == nil {
				causes = append(causes, metav1.StatusCause{
					Type:    metav1.CauseTypeFieldValueNotFound,
					Field:   "caBundle",
					Message: "CABundle should be provided if HTTPS webhook is used",
				})
			}
		}

	}
	return causes
}

// ValidateBufferPolicy validates the FleetAutoscaler Buffer policy settings
func (b *BufferPolicy) ValidateBufferPolicy(causes []metav1.StatusCause) []metav1.StatusCause {
	if b == nil {
		return append(causes, metav1.StatusCause{
			Type:    metav1.CauseTypeFieldValueInvalid,
			Field:   "buffer",
			Message: "Buffer policy config params are missing",
		})
	}
	if b.MinReplicas > b.MaxReplicas {
		causes = append(causes, metav1.StatusCause{
			Type:    metav1.CauseTypeFieldValueInvalid,
			Field:   "minReplicas",
			Message: "minReplicas is bigger than maxReplicas",
		})
	}
	if b.BufferSize.Type == intstr.Int {
		if b.BufferSize.IntValue() <= 0 {
			causes = append(causes, metav1.StatusCause{
				Type:    metav1.CauseTypeFieldValueInvalid,
				Field:   "bufferSize",
				Message: "bufferSize must be bigger than 0",
			})
		}
		if b.MaxReplicas < int32(b.BufferSize.IntValue()) {
			causes = append(causes, metav1.StatusCause{
				Type:    metav1.CauseTypeFieldValueInvalid,
				Field:   "maxReplicas",
				Message: "maxReplicas must be bigger than bufferSize",
			})
		}
		if b.MinReplicas != 0 && b.MinReplicas < int32(b.BufferSize.IntValue()) {
			causes = append(causes, metav1.StatusCause{
				Type:    metav1.CauseTypeFieldValueInvalid,
				Field:   "minReplicas",
				Message: "minReplicas is smaller than bufferSize",
			})
		}
	} else {
		r, err := intstr.GetValueFromIntOrPercent(&b.BufferSize, 100, true)
		if err != nil || r < 1 || r > 99 {
			causes = append(causes, metav1.StatusCause{
				Type:    metav1.CauseTypeFieldValueInvalid,
				Field:   "bufferSize",
				Message: "bufferSize does not have a valid percentage value (1%-99%)",
			})
		}
		// When there is no allocated gameservers in a fleet,
		// Fleetautoscaler would reduce size of a fleet to MinReplicas.
		// If we have 0 MinReplicas and 0 Allocated then Fleetautoscaler would set Ready Replicas to 0
		// and we will not be able to raise the number of GS in a Fleet above zero
		if b.MinReplicas < 1 {
			causes = append(causes, metav1.StatusCause{
				Type:    metav1.CauseTypeFieldValueInvalid,
				Field:   "minReplicas",
				Message: "minReplicas should be above 0 when used with percentage value bufferSize",
			})
		}
	}
	return causes
}

// ValidateFixedIntervalSync validates the FixedIntervalSync settings
func (i *FixedIntervalSync) ValidateFixedIntervalSync(causes []metav1.StatusCause) []metav1.StatusCause {
	if i == nil {
		return append(causes, metav1.StatusCause{
			Type:    metav1.CauseTypeFieldValueInvalid,
			Field:   "fixedInterval",
			Message: "fixedInterval config params are missing",
		})
	}
	if i.Seconds <= 0 {
		return append(causes, metav1.StatusCause{
			Type:    metav1.CauseTypeFieldValueInvalid,
			Field:   "seconds",
			Message: "seconds should be bigger than 0",
		})
	}
	return causes
}

// ApplyDefaults applies default values to the FleetAutoscaler
func (fas *FleetAutoscaler) ApplyDefaults() {
	if runtime.FeatureEnabled(runtime.FeatureCustomFasSyncInterval) {
		if fas.Spec.Sync == nil {
			fas.Spec.Sync = &FleetAutoscalerSync{}
		}
		if fas.Spec.Sync.Type == "" {
			fas.Spec.Sync.Type = FixedIntervalSyncType
		}
		if fas.Spec.Sync.FixedInterval.Seconds == 0 {
			fas.Spec.Sync.FixedInterval.Seconds = defaultIntervalSyncSeconds
		}
	}
}
<|endoftext|>