<commit_before>package backup

import (
	"archive/tar"
	"fmt"
	"io"
	"os"
	"os/exec"
	"strings"
	"time"

	"gopkg.in/yaml.v2"

	"github.com/lxc/lxd/lxd/project"
	"github.com/lxc/lxd/lxd/state"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/api"
)

// Instance represents the backup relevant subset of a LXD instance.
type Instance interface {
	Name() string
	Project() string
}

// Info represents exported backup information.
type Info struct {
	Project          string   `json:"project" yaml:"project"`
	Name             string   `json:"name" yaml:"name"`
	Backend          string   `json:"backend" yaml:"backend"`
	Privileged       bool     `json:"privileged" yaml:"privileged"`
	Pool             string   `json:"pool" yaml:"pool"`
	Snapshots        []string `json:"snapshots,omitempty" yaml:"snapshots,omitempty"`
	OptimizedStorage bool     `json:"-" yaml:"-"`
}

// GetInfo extracts backup information from a given ReadSeeker.
func GetInfo(r io.ReadSeeker) (*Info, error) {
	var tr *tar.Reader
	result := Info{}
	optimizedStorage := false
	hasIndexFile := false

	// Extract
	r.Seek(0, 0)
	_, _, unpacker, err := shared.DetectCompressionFile(r)
	if err != nil {
		return nil, err
	}
	r.Seek(0, 0)

	if unpacker == nil {
		return nil, fmt.Errorf("Unsupported backup compression")
	}

	if len(unpacker) > 0 {
		cmd := exec.Command(unpacker[0], unpacker[1:]...)
		cmd.Stdin = r

		stdout, err := cmd.StdoutPipe()
		if err != nil {
			return nil, err
		}
		defer stdout.Close()

		err = cmd.Start()
		if err != nil {
			return nil, err
		}
		defer cmd.Wait()

		tr = tar.NewReader(stdout)
	} else {
		tr = tar.NewReader(r)
	}

	for {
		hdr, err := tr.Next()
		if err == io.EOF {
			break // End of archive
		}
		if err != nil {
			return nil, err
		}

		if hdr.Name == "backup/index.yaml" {
			err = yaml.NewDecoder(tr).Decode(&result)
			if err != nil {
				return nil, err
			}

			hasIndexFile = true
		}

		if hdr.Name == "backup/container.bin" {
			optimizedStorage = true
		}
	}

	if !hasIndexFile {
		return nil, fmt.Errorf("Backup is missing index.yaml")
	}

	result.OptimizedStorage = optimizedStorage
	return &result, nil
}

// Backup represents a container backup
type Backup struct {
	state    *state.State
	instance Instance

	// Properties
	id                   int
	name                 string
	creationDate         time.Time
	expiryDate           time.Time
	instanceOnly         bool
	optimizedStorage     bool
	compressionAlgorithm string
}

// New instantiates a new Backup struct.
func New(state *state.State, inst Instance, ID int, name string, creationDate, expiryDate time.Time, instanceOnly, optimizedStorage bool) *Backup {
	return &Backup{
		state:            state,
		instance:         inst,
		id:               ID,
		name:             name,
		creationDate:     creationDate,
		expiryDate:       expiryDate,
		instanceOnly:     instanceOnly,
		optimizedStorage: optimizedStorage,
	}
}

// CompressionAlgorithm returns the compression used for the tarball.
func (b *Backup) CompressionAlgorithm() string {
	return b.compressionAlgorithm
}

// SetCompressionAlgorithm sets the tarball compression.
func (b *Backup) SetCompressionAlgorithm(compression string) {
	b.compressionAlgorithm = compression
}

// InstanceOnly returns whether only the instance itself is to be backed up.
func (b *Backup) InstanceOnly() bool {
	return b.instanceOnly
}

// Name returns the name of the backup.
func (b *Backup) Name() string {
	return b.name
}

// OptimizedStorage returns whether the backup is to be performed using
// optimization supported by the storage driver.
func (b *Backup) OptimizedStorage() bool {
	return b.optimizedStorage
}

// Rename renames a container backup
func (b *Backup) Rename(newName string) error {
	oldBackupPath := shared.VarPath("backups", project.Prefix(b.instance.Project(), b.name))
	newBackupPath := shared.VarPath("backups", project.Prefix(b.instance.Project(), newName))

	// Create the new backup path
	backupsPath := shared.VarPath("backups", project.Prefix(b.instance.Project(), b.instance.Name()))
	if !shared.PathExists(backupsPath) {
		err := os.MkdirAll(backupsPath, 0700)
		if err != nil {
			return err
		}
	}

	// Rename the backup directory
	err := os.Rename(oldBackupPath, newBackupPath)
	if err != nil {
		return err
	}

	// Check if we can remove the container directory
	empty, _ := shared.PathIsEmpty(backupsPath)
	if empty {
		err := os.Remove(backupsPath)
		if err != nil {
			return err
		}
	}

	// Rename the database record
	err = b.state.Cluster.ContainerBackupRename(b.name, newName)
	if err != nil {
		return err
	}

	return nil
}

// Delete removes an instance backup
func (b *Backup) Delete() error {
	return DoBackupDelete(b.state, b.instance.Project(), b.name, b.instance.Name())
}

// Render returns an InstanceBackup struct of the backup.
func (b *Backup) Render() *api.InstanceBackup {
	return &api.InstanceBackup{
		Name:             strings.SplitN(b.name, "/", 2)[1],
		CreatedAt:        b.creationDate,
		ExpiresAt:        b.expiryDate,
		InstanceOnly:     b.instanceOnly,
		ContainerOnly:    b.instanceOnly,
		OptimizedStorage: b.optimizedStorage,
	}
}

// DoBackupDelete deletes a backup.
func DoBackupDelete(s *state.State, projectName, backupName, containerName string) error {
	backupPath := shared.VarPath("backups", project.Prefix(projectName, backupName))

	// Delete the on-disk data
	if shared.PathExists(backupPath) {
		err := os.RemoveAll(backupPath)
		if err != nil {
			return err
		}
	}

	// Check if we can remove the container directory
	backupsPath := shared.VarPath("backups", project.Prefix(projectName, containerName))
	empty, _ := shared.PathIsEmpty(backupsPath)
	if empty {
		err := os.Remove(backupsPath)
		if err != nil {
			return err
		}
	}

	// Remove the database record
	err := s.Cluster.ContainerBackupRemove(backupName)
	if err != nil {
		return err
	}

	return nil
}
<commit_msg>lxd/backup/backup: Comment clarifying existence of Instance interface<commit_after>package backup

import (
	"archive/tar"
	"fmt"
	"io"
	"os"
	"os/exec"
	"strings"
	"time"

	"gopkg.in/yaml.v2"

	"github.com/lxc/lxd/lxd/project"
	"github.com/lxc/lxd/lxd/state"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/api"
)

// Instance represents the backup relevant subset of a LXD instance.
// This is used rather than instance.Instance to avoid import loops.
type Instance interface {
	Name() string
	Project() string
}

// Info represents exported backup information.
type Info struct {
	Project          string   `json:"project" yaml:"project"`
	Name             string   `json:"name" yaml:"name"`
	Backend          string   `json:"backend" yaml:"backend"`
	Privileged       bool     `json:"privileged" yaml:"privileged"`
	Pool             string   `json:"pool" yaml:"pool"`
	Snapshots        []string `json:"snapshots,omitempty" yaml:"snapshots,omitempty"`
	OptimizedStorage bool     `json:"-" yaml:"-"`
}

// GetInfo extracts backup information from a given ReadSeeker.
func GetInfo(r io.ReadSeeker) (*Info, error) {
	var tr *tar.Reader
	result := Info{}
	optimizedStorage := false
	hasIndexFile := false

	// Extract
	r.Seek(0, 0)
	_, _, unpacker, err := shared.DetectCompressionFile(r)
	if err != nil {
		return nil, err
	}
	r.Seek(0, 0)

	if unpacker == nil {
		return nil, fmt.Errorf("Unsupported backup compression")
	}

	if len(unpacker) > 0 {
		cmd := exec.Command(unpacker[0], unpacker[1:]...)
		cmd.Stdin = r

		stdout, err := cmd.StdoutPipe()
		if err != nil {
			return nil, err
		}
		defer stdout.Close()

		err = cmd.Start()
		if err != nil {
			return nil, err
		}
		defer cmd.Wait()

		tr = tar.NewReader(stdout)
	} else {
		tr = tar.NewReader(r)
	}

	for {
		hdr, err := tr.Next()
		if err == io.EOF {
			break // End of archive
		}
		if err != nil {
			return nil, err
		}

		if hdr.Name == "backup/index.yaml" {
			err = yaml.NewDecoder(tr).Decode(&result)
			if err != nil {
				return nil, err
			}

			hasIndexFile = true
		}

		if hdr.Name == "backup/container.bin" {
			optimizedStorage = true
		}
	}

	if !hasIndexFile {
		return nil, fmt.Errorf("Backup is missing index.yaml")
	}

	result.OptimizedStorage = optimizedStorage
	return &result, nil
}

// Backup represents a container backup
type Backup struct {
	state    *state.State
	instance Instance

	// Properties
	id                   int
	name                 string
	creationDate         time.Time
	expiryDate           time.Time
	instanceOnly         bool
	optimizedStorage     bool
	compressionAlgorithm string
}

// New instantiates a new Backup struct.
func New(state *state.State, inst Instance, ID int, name string, creationDate, expiryDate time.Time, instanceOnly, optimizedStorage bool) *Backup {
	return &Backup{
		state:            state,
		instance:         inst,
		id:               ID,
		name:             name,
		creationDate:     creationDate,
		expiryDate:       expiryDate,
		instanceOnly:     instanceOnly,
		optimizedStorage: optimizedStorage,
	}
}

// CompressionAlgorithm returns the compression used for the tarball.
func (b *Backup) CompressionAlgorithm() string {
	return b.compressionAlgorithm
}

// SetCompressionAlgorithm sets the tarball compression.
func (b *Backup) SetCompressionAlgorithm(compression string) {
	b.compressionAlgorithm = compression
}

// InstanceOnly returns whether only the instance itself is to be backed up.
func (b *Backup) InstanceOnly() bool {
	return b.instanceOnly
}

// Name returns the name of the backup.
func (b *Backup) Name() string {
	return b.name
}

// OptimizedStorage returns whether the backup is to be performed using
// optimization supported by the storage driver.
func (b *Backup) OptimizedStorage() bool {
	return b.optimizedStorage
}

// Rename renames a container backup
func (b *Backup) Rename(newName string) error {
	oldBackupPath := shared.VarPath("backups", project.Prefix(b.instance.Project(), b.name))
	newBackupPath := shared.VarPath("backups", project.Prefix(b.instance.Project(), newName))

	// Create the new backup path
	backupsPath := shared.VarPath("backups", project.Prefix(b.instance.Project(), b.instance.Name()))
	if !shared.PathExists(backupsPath) {
		err := os.MkdirAll(backupsPath, 0700)
		if err != nil {
			return err
		}
	}

	// Rename the backup directory
	err := os.Rename(oldBackupPath, newBackupPath)
	if err != nil {
		return err
	}

	// Check if we can remove the container directory
	empty, _ := shared.PathIsEmpty(backupsPath)
	if empty {
		err := os.Remove(backupsPath)
		if err != nil {
			return err
		}
	}

	// Rename the database record
	err = b.state.Cluster.ContainerBackupRename(b.name, newName)
	if err != nil {
		return err
	}

	return nil
}

// Delete removes an instance backup
func (b *Backup) Delete() error {
	return DoBackupDelete(b.state, b.instance.Project(), b.name, b.instance.Name())
}

// Render returns an InstanceBackup struct of the backup.
func (b *Backup) Render() *api.InstanceBackup {
	return &api.InstanceBackup{
		Name:             strings.SplitN(b.name, "/", 2)[1],
		CreatedAt:        b.creationDate,
		ExpiresAt:        b.expiryDate,
		InstanceOnly:     b.instanceOnly,
		ContainerOnly:    b.instanceOnly,
		OptimizedStorage: b.optimizedStorage,
	}
}

// DoBackupDelete deletes a backup.
func DoBackupDelete(s *state.State, projectName, backupName, containerName string) error {
	backupPath := shared.VarPath("backups", project.Prefix(projectName, backupName))

	// Delete the on-disk data
	if shared.PathExists(backupPath) {
		err := os.RemoveAll(backupPath)
		if err != nil {
			return err
		}
	}

	// Check if we can remove the container directory
	backupsPath := shared.VarPath("backups", project.Prefix(projectName, containerName))
	empty, _ := shared.PathIsEmpty(backupsPath)
	if empty {
		err := os.Remove(backupsPath)
		if err != nil {
			return err
		}
	}

	// Remove the database record
	err := s.Cluster.ContainerBackupRemove(backupName)
	if err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>package tpl

import (
	"fmt"
	"reflect"
	"strconv"
	"strings"
	"text/template"

	"github.com/golang/protobuf/ptypes"
	"github.com/golang/protobuf/ptypes/duration"
	"github.com/golang/protobuf/ptypes/timestamp"
	pgs "github.com/lyft/protoc-gen-star"
	"github.com/lyft/protoc-gen-validate/templates/shared"
)

func RegisterModule(tpl *template.Template) {
	tpl.Funcs(map[string]interface{}{
		"cmt":           pgs.C80,
		"class":         className,
		"package":       packageName,
		"accessor":      accessor,
		"hasAccessor":   hasAccessor,
		"ctype":         cType,
		"weakCheckMsgs": weakCheckMsgs,
		"err":           err,
		"errCause":      errCause,
		"errIdx":        errIdx,
		"errIdxCause":   errIdxCause,
		"lookup":        lookup,
		"lit":           lit,
		"isBytes":       isBytes,
		"byteStr":       byteStr,
		"oneof":         oneofTypeName,
		"quote":         quote,
		"inType":        inType,
		"inKey":         inKey,
		"durLit":        durLit,
		"durStr":        durStr,
		"durGt":         durGt,
		"tsLit":         tsLit,
		"tsGt":          tsGt,
		"tsStr":         tsStr,
		"unwrap":        unwrap,
		"unimplemented": failUnimplemented,
	})
	template.Must(tpl.Parse(moduleFileTpl))
	template.Must(tpl.New("msg").Parse(msgTpl))
	template.Must(tpl.New("const").Parse(constTpl))
	template.Must(tpl.New("ltgt").Parse(ltgtTpl))
	template.Must(tpl.New("in").Parse(inTpl))
	template.Must(tpl.New("required").Parse(requiredTpl))

	template.Must(tpl.New("none").Parse(noneTpl))
	template.Must(tpl.New("float").Parse(numTpl))
	template.Must(tpl.New("double").Parse(numTpl))
	template.Must(tpl.New("int32").Parse(numTpl))
	template.Must(tpl.New("int64").Parse(numTpl))
	template.Must(tpl.New("uint32").Parse(numTpl))
	template.Must(tpl.New("uint64").Parse(numTpl))
	template.Must(tpl.New("sint32").Parse(numTpl))
	template.Must(tpl.New("sint64").Parse(numTpl))
	template.Must(tpl.New("fixed32").Parse(numTpl))
	template.Must(tpl.New("fixed64").Parse(numTpl))
	template.Must(tpl.New("sfixed32").Parse(numTpl))
	template.Must(tpl.New("sfixed64").Parse(numTpl))

	template.Must(tpl.New("bool").Parse(constTpl))
	template.Must(tpl.New("string").Parse(strTpl))
	template.Must(tpl.New("bytes").Parse(bytesTpl))

	template.Must(tpl.New("email").Parse(emailTpl))
	template.Must(tpl.New("hostname").Parse(hostTpl))

	template.Must(tpl.New("enum").Parse(enumTpl))
	template.Must(tpl.New("message").Parse(messageTpl))
	template.Must(tpl.New("repeated").Parse(repTpl))
	template.Must(tpl.New("map").Parse(mapTpl))

	template.Must(tpl.New("any").Parse(anyTpl))
	template.Must(tpl.New("duration").Parse(durationTpl))
	template.Must(tpl.New("timestamp").Parse(timestampTpl))

	template.Must(tpl.New("wrapper").Parse(wrapperTpl))
}

func RegisterHeader(tpl *template.Template) {
	tpl.Funcs(map[string]interface{}{
		"class":         className,
		"upper":         strings.ToUpper,
	})

	template.Must(tpl.Parse(headerFileTpl))
	template.Must(tpl.New("decl").Parse(declTpl))
}

func methodName(name interface{}) string {
	nameStr := fmt.Sprintf("%s", name)
	switch nameStr {
	case "const":
		return "const_"
	default:
		return nameStr
	}
}

func accessor(ctx shared.RuleContext) string {
	if ctx.AccessorOverride != "" {
		return ctx.AccessorOverride
	}

	return fmt.Sprintf(
		"m.%s()",
		methodName(ctx.Field.Name()))
}

func hasAccessor(ctx shared.RuleContext) string {
	if ctx.AccessorOverride != "" {
		return "true"
	}

	return fmt.Sprintf(
		"m.has_%s()",
		methodName(ctx.Field.Name()))
}

func className(msg pgs.Message) string {
	return packageName(msg) + "::" + string(msg.TypeName())
}

func packageName(msg pgs.Message) string {
	return strings.Join(msg.Package().ProtoName().Split(), "::")
}

func quote(s interface {
	String() string
}) string {
	return strconv.Quote(s.String())
}

func err(ctx shared.RuleContext, reason ...interface{}) string {
	return errIdxCause(ctx, "", "nil", reason...)
}

func errCause(ctx shared.RuleContext, cause string, reason ...interface{}) string {
	return errIdxCause(ctx, "", cause, reason...)
}

func errIdx(ctx shared.RuleContext, idx string, reason ...interface{}) string {
	return errIdxCause(ctx, idx, "nil", reason...)
}

func errIdxCause(ctx shared.RuleContext, idx, cause string, reason ...interface{}) string {
	f := ctx.Field
	errName := fmt.Sprintf("%sValidationError", f.Message().Name())

	output := []string{
		"{",
		`std::ostringstream msg("invalid ");`,
	}

	if ctx.OnKey {
		output = append(output, `msg << "key for ";`)
	}
	output = append(output,
		fmt.Sprintf(`msg << %q << "." << %s;`,
			errName, lit(f.Name().PGGUpperCamelCase().String())))

	if idx != "" {
		output = append(output, fmt.Sprintf(`msg << "[" << %s << "]";`, lit(idx)))
	} else if ctx.Index != "" {
		output = append(output, fmt.Sprintf(`msg << "[" << %s << "]";`, lit(ctx.Index)))
	}

	output = append(output, fmt.Sprintf(`msg << ": " << %s;`, lit(fmt.Sprintf("%q", reason))))

	if cause != "nil" && cause != "" {
		output = append(output, fmt.Sprintf(`msg << " | caused by " << %s;`, cause))
	}

	output = append(output, "*err = msg.str();",
		"return false;",
		"}")
	return strings.Join(output, "\n")
}

func lookup(f pgs.Field, name string) string {
	return fmt.Sprintf(
		"_%s_%s_%s",
		f.Message().Name().PGGUpperCamelCase(),
		f.Name().PGGUpperCamelCase(),
		name,
	)
}

func lit(x interface{}) string {
	val := reflect.ValueOf(x)

	if val.Kind() == reflect.Interface {
		val = val.Elem()
	}

	if val.Kind() == reflect.Ptr {
		val = val.Elem()
	}

	switch val.Kind() {
	case reflect.String:
		return fmt.Sprintf("%q", x)
	case reflect.Uint8:
		return fmt.Sprintf("0x%X", x)
	case reflect.Slice:
		els := make([]string, val.Len())
		switch reflect.TypeOf(x).Elem().Kind() {
		case reflect.Uint8:
			for i, l := 0, val.Len(); i < l; i++ {
				els[i] = fmt.Sprintf("\\x%x", val.Index(i).Interface())
			}
			return fmt.Sprintf("\"%s\"", strings.Join(els, ""))
		default:
			panic(fmt.Sprintf("don't know how to format literals of type %v", val.Kind()))
		}
	case reflect.Float32:
		return fmt.Sprintf("%fF", x)
	default:
		return fmt.Sprint(x)
	}
}

func isBytes(f interface {
	ProtoType() pgs.ProtoType
}) bool {
	return f.ProtoType() == pgs.BytesT
}

func byteStr(x []byte) string {
	elms := make([]string, len(x))
	for i, b := range x {
		elms[i] = fmt.Sprintf(`\x%X`, b)
	}

	return fmt.Sprintf(`"%s"`, strings.Join(elms, ""))
}

func oneofTypeName(f pgs.Field) pgs.TypeName {
	return pgs.TypeName(fmt.Sprintf("%s::%sCase::k%s",
		className(f.Message()),
		f.OneOf().Name().PGGUpperCamelCase(),
		f.Name().PGGUpperCamelCase(),
	))
}

func inType(f pgs.Field, x interface{}) string {
	switch f.Type().ProtoType() {
	case pgs.BytesT:
		return "string"
	case pgs.MessageT:
		switch x.(type) {
		case []string:
			return "string"
		case []*duration.Duration:
			return "pgv::protobuf_wkt::Duration"
		default:
			return className(f.Type().Element().Embed())
		}
	default:
		return cType(f.Type())
	}
}

func cType(t pgs.FieldType) string {
	if t.IsEmbed() {
		return className(t.Embed())
	}
	if t.IsRepeated(){
		if t.ProtoType() == pgs.MessageT {
			return className(t.Element().Embed())
		}
		// Strip the leading []
		return cTypeOfString(t.Name().String()[2:])
	}

	return cTypeOfString(t.Name().String())
}

// Compute unique C++ types that correspond to all message fields in a
// compilation unit that need to be weak (i.e. not already defined). Used to
// generate weak default definitions for CheckMessage.
func weakCheckMsgs(msgs []pgs.Message) []string {
	already_defined := map[string]bool{}
	// First compute the C++ type names for things we're going to provide an explicit
	// CheckMessage() with Validate(..) body in this file. We can't define the
	// same CheckMessage() signature twice in a compilation unit, even if one of
	// them is weak.
	for _, msg := range msgs {
		already_defined[className(msg)] = true
	}
	// Compute the set of C++ type names we need weak definitions for.
	ctype_map := map[string]bool{}
	for _, msg := range msgs {
		if disabled, _ := shared.Disabled(msg); disabled {
			continue
		}
		for _, f := range msg.Fields() {
			ctype := cType(f.Type())
			if already_defined[ctype] {
				continue
			}
			if f.Type().IsEmbed() || (f.Type().IsRepeated() && f.Type().Element().IsEmbed()) {
				ctype_map[ctype] = true
			}
		}
	}
	// Convert to array.
	ctypes := make([]string, 0, len(ctype_map))
	for ctype := range ctype_map {
		ctypes = append(ctypes, ctype)
	}
	return ctypes
}

func cTypeOfString(s string) string {
	switch s {
	case "float32":
		return "float"
	case "float64":
		return "double"
	case "int32":
		return "int32_t"
	case "int64":
		return "int64_t"
	case "uint32":
		return "uint32_t"
	case "uint64":
		return "uint64_t"
	case "[]byte":
		return "string"
	default:
		return s
	}
}

func inKey(f pgs.Field, x interface{}) string {
	switch f.Type().ProtoType() {
	case pgs.BytesT:
		return byteStr(x.([]byte))
	case pgs.MessageT:
		switch x := x.(type) {
		case *duration.Duration:
			return durLit(x)
		default:
			return lit(x)
		}
	case pgs.EnumT:
		return fmt.Sprintf("%s(%d)", cType(f.Type()), x.(int32))
	default:
		return lit(x)
	}
}

func durLit(dur *duration.Duration) string {
	return fmt.Sprintf(
		"pgv::protobuf::util::TimeUtil::SecondsToDuration(%d) + pgv::protobuf::util::TimeUtil::NanosecondsToDuration(%d)",
		dur.GetSeconds(), dur.GetNanos())
}

func durStr(dur *duration.Duration) string {
	d, _ := ptypes.Duration(dur)
	return d.String()
}

func durGt(a, b *duration.Duration) bool {
	ad, _ := ptypes.Duration(a)
	bd, _ := ptypes.Duration(b)

	return ad > bd
}

func tsLit(ts *timestamp.Timestamp) string {
	return fmt.Sprintf(
		"time.Unix(%d, %d)",
		ts.GetSeconds(), ts.GetNanos(),
	)
}

func tsGt(a, b *timestamp.Timestamp) bool {
	at, _ := ptypes.Timestamp(a)
	bt, _ := ptypes.Timestamp(b)

	return !bt.Before(at)
}

func tsStr(ts *timestamp.Timestamp) string {
	t, _ := ptypes.Timestamp(ts)
	return t.String()
}

func unwrap(ctx shared.RuleContext, name string) (shared.RuleContext, error) {
	ctx, err := ctx.Unwrap("wrapper")
	if err != nil {
		return ctx, err
	}

	ctx.AccessorOverride = fmt.Sprintf("%s.%s()", name,
		ctx.Field.Type().Embed().Fields()[0].Name())

	return ctx, nil
}

func failUnimplemented() string {
	return "throw std::string(\"not yet implemented\");"
}
<commit_msg>templates: add "inline" C++ keyword. (#46)<commit_after>package tpl

import (
	"fmt"
	"reflect"
	"strconv"
	"strings"
	"text/template"

	"github.com/golang/protobuf/ptypes"
	"github.com/golang/protobuf/ptypes/duration"
	"github.com/golang/protobuf/ptypes/timestamp"
	pgs "github.com/lyft/protoc-gen-star"
	"github.com/lyft/protoc-gen-validate/templates/shared"
)

func RegisterModule(tpl *template.Template) {
	tpl.Funcs(map[string]interface{}{
		"cmt":           pgs.C80,
		"class":         className,
		"package":       packageName,
		"accessor":      accessor,
		"hasAccessor":   hasAccessor,
		"ctype":         cType,
		"weakCheckMsgs": weakCheckMsgs,
		"err":           err,
		"errCause":      errCause,
		"errIdx":        errIdx,
		"errIdxCause":   errIdxCause,
		"lookup":        lookup,
		"lit":           lit,
		"isBytes":       isBytes,
		"byteStr":       byteStr,
		"oneof":         oneofTypeName,
		"quote":         quote,
		"inType":        inType,
		"inKey":         inKey,
		"durLit":        durLit,
		"durStr":        durStr,
		"durGt":         durGt,
		"tsLit":         tsLit,
		"tsGt":          tsGt,
		"tsStr":         tsStr,
		"unwrap":        unwrap,
		"unimplemented": failUnimplemented,
	})
	template.Must(tpl.Parse(moduleFileTpl))
	template.Must(tpl.New("msg").Parse(msgTpl))
	template.Must(tpl.New("const").Parse(constTpl))
	template.Must(tpl.New("ltgt").Parse(ltgtTpl))
	template.Must(tpl.New("in").Parse(inTpl))
	template.Must(tpl.New("required").Parse(requiredTpl))

	template.Must(tpl.New("none").Parse(noneTpl))
	template.Must(tpl.New("float").Parse(numTpl))
	template.Must(tpl.New("double").Parse(numTpl))
	template.Must(tpl.New("int32").Parse(numTpl))
	template.Must(tpl.New("int64").Parse(numTpl))
	template.Must(tpl.New("uint32").Parse(numTpl))
	template.Must(tpl.New("uint64").Parse(numTpl))
	template.Must(tpl.New("sint32").Parse(numTpl))
	template.Must(tpl.New("sint64").Parse(numTpl))
	template.Must(tpl.New("fixed32").Parse(numTpl))
	template.Must(tpl.New("fixed64").Parse(numTpl))
	template.Must(tpl.New("sfixed32").Parse(numTpl))
	template.Must(tpl.New("sfixed64").Parse(numTpl))

	template.Must(tpl.New("bool").Parse(constTpl))
	template.Must(tpl.New("string").Parse(strTpl))
	template.Must(tpl.New("bytes").Parse(bytesTpl))

	template.Must(tpl.New("email").Parse(emailTpl))
	template.Must(tpl.New("hostname").Parse(hostTpl))

	template.Must(tpl.New("enum").Parse(enumTpl))
	template.Must(tpl.New("message").Parse(messageTpl))
	template.Must(tpl.New("repeated").Parse(repTpl))
	template.Must(tpl.New("map").Parse(mapTpl))

	template.Must(tpl.New("any").Parse(anyTpl))
	template.Must(tpl.New("duration").Parse(durationTpl))
	template.Must(tpl.New("timestamp").Parse(timestampTpl))

	template.Must(tpl.New("wrapper").Parse(wrapperTpl))
}

func RegisterHeader(tpl *template.Template) {
	tpl.Funcs(map[string]interface{}{
		"class":         className,
		"upper":         strings.ToUpper,
	})

	template.Must(tpl.Parse(headerFileTpl))
	template.Must(tpl.New("decl").Parse(declTpl))
}

func methodName(name interface{}) string {
	nameStr := fmt.Sprintf("%s", name)
	switch nameStr {
	case "const":
		return "const_"
	case "inline":
		return "inline_"
	default:
		return nameStr
	}
}

func accessor(ctx shared.RuleContext) string {
	if ctx.AccessorOverride != "" {
		return ctx.AccessorOverride
	}

	return fmt.Sprintf(
		"m.%s()",
		methodName(ctx.Field.Name()))
}

func hasAccessor(ctx shared.RuleContext) string {
	if ctx.AccessorOverride != "" {
		return "true"
	}

	return fmt.Sprintf(
		"m.has_%s()",
		methodName(ctx.Field.Name()))
}

func className(msg pgs.Message) string {
	return packageName(msg) + "::" + string(msg.TypeName())
}

func packageName(msg pgs.Message) string {
	return strings.Join(msg.Package().ProtoName().Split(), "::")
}

func quote(s interface {
	String() string
}) string {
	return strconv.Quote(s.String())
}

func err(ctx shared.RuleContext, reason ...interface{}) string {
	return errIdxCause(ctx, "", "nil", reason...)
}

func errCause(ctx shared.RuleContext, cause string, reason ...interface{}) string {
	return errIdxCause(ctx, "", cause, reason...)
}

func errIdx(ctx shared.RuleContext, idx string, reason ...interface{}) string {
	return errIdxCause(ctx, idx, "nil", reason...)
}

func errIdxCause(ctx shared.RuleContext, idx, cause string, reason ...interface{}) string {
	f := ctx.Field
	errName := fmt.Sprintf("%sValidationError", f.Message().Name())

	output := []string{
		"{",
		`std::ostringstream msg("invalid ");`,
	}

	if ctx.OnKey {
		output = append(output, `msg << "key for ";`)
	}
	output = append(output,
		fmt.Sprintf(`msg << %q << "." << %s;`,
			errName, lit(f.Name().PGGUpperCamelCase().String())))

	if idx != "" {
		output = append(output, fmt.Sprintf(`msg << "[" << %s << "]";`, lit(idx)))
	} else if ctx.Index != "" {
		output = append(output, fmt.Sprintf(`msg << "[" << %s << "]";`, lit(ctx.Index)))
	}

	output = append(output, fmt.Sprintf(`msg << ": " << %s;`, lit(fmt.Sprintf("%q", reason))))

	if cause != "nil" && cause != "" {
		output = append(output, fmt.Sprintf(`msg << " | caused by " << %s;`, cause))
	}

	output = append(output, "*err = msg.str();",
		"return false;",
		"}")
	return strings.Join(output, "\n")
}

func lookup(f pgs.Field, name string) string {
	return fmt.Sprintf(
		"_%s_%s_%s",
		f.Message().Name().PGGUpperCamelCase(),
		f.Name().PGGUpperCamelCase(),
		name,
	)
}

func lit(x interface{}) string {
	val := reflect.ValueOf(x)

	if val.Kind() == reflect.Interface {
		val = val.Elem()
	}

	if val.Kind() == reflect.Ptr {
		val = val.Elem()
	}

	switch val.Kind() {
	case reflect.String:
		return fmt.Sprintf("%q", x)
	case reflect.Uint8:
		return fmt.Sprintf("0x%X", x)
	case reflect.Slice:
		els := make([]string, val.Len())
		switch reflect.TypeOf(x).Elem().Kind() {
		case reflect.Uint8:
			for i, l := 0, val.Len(); i < l; i++ {
				els[i] = fmt.Sprintf("\\x%x", val.Index(i).Interface())
			}
			return fmt.Sprintf("\"%s\"", strings.Join(els, ""))
		default:
			panic(fmt.Sprintf("don't know how to format literals of type %v", val.Kind()))
		}
	case reflect.Float32:
		return fmt.Sprintf("%fF", x)
	default:
		return fmt.Sprint(x)
	}
}

func isBytes(f interface {
	ProtoType() pgs.ProtoType
}) bool {
	return f.ProtoType() == pgs.BytesT
}

func byteStr(x []byte) string {
	elms := make([]string, len(x))
	for i, b := range x {
		elms[i] = fmt.Sprintf(`\x%X`, b)
	}

	return fmt.Sprintf(`"%s"`, strings.Join(elms, ""))
}

func oneofTypeName(f pgs.Field) pgs.TypeName {
	return pgs.TypeName(fmt.Sprintf("%s::%sCase::k%s",
		className(f.Message()),
		f.OneOf().Name().PGGUpperCamelCase(),
		f.Name().PGGUpperCamelCase(),
	))
}

func inType(f pgs.Field, x interface{}) string {
	switch f.Type().ProtoType() {
	case pgs.BytesT:
		return "string"
	case pgs.MessageT:
		switch x.(type) {
		case []string:
			return "string"
		case []*duration.Duration:
			return "pgv::protobuf_wkt::Duration"
		default:
			return className(f.Type().Element().Embed())
		}
	default:
		return cType(f.Type())
	}
}

func cType(t pgs.FieldType) string {
	if t.IsEmbed() {
		return className(t.Embed())
	}
	if t.IsRepeated(){
		if t.ProtoType() == pgs.MessageT {
			return className(t.Element().Embed())
		}
		// Strip the leading []
		return cTypeOfString(t.Name().String()[2:])
	}

	return cTypeOfString(t.Name().String())
}

// Compute unique C++ types that correspond to all message fields in a
// compilation unit that need to be weak (i.e. not already defined). Used to
// generate weak default definitions for CheckMessage.
func weakCheckMsgs(msgs []pgs.Message) []string {
	already_defined := map[string]bool{}
	// First compute the C++ type names for things we're going to provide an explicit
	// CheckMessage() with Validate(..) body in this file. We can't define the
	// same CheckMessage() signature twice in a compilation unit, even if one of
	// them is weak.
	for _, msg := range msgs {
		already_defined[className(msg)] = true
	}
	// Compute the set of C++ type names we need weak definitions for.
	ctype_map := map[string]bool{}
	for _, msg := range msgs {
		if disabled, _ := shared.Disabled(msg); disabled {
			continue
		}
		for _, f := range msg.Fields() {
			ctype := cType(f.Type())
			if already_defined[ctype] {
				continue
			}
			if f.Type().IsEmbed() || (f.Type().IsRepeated() && f.Type().Element().IsEmbed()) {
				ctype_map[ctype] = true
			}
		}
	}
	// Convert to array.
	ctypes := make([]string, 0, len(ctype_map))
	for ctype := range ctype_map {
		ctypes = append(ctypes, ctype)
	}
	return ctypes
}

func cTypeOfString(s string) string {
	switch s {
	case "float32":
		return "float"
	case "float64":
		return "double"
	case "int32":
		return "int32_t"
	case "int64":
		return "int64_t"
	case "uint32":
		return "uint32_t"
	case "uint64":
		return "uint64_t"
	case "[]byte":
		return "string"
	default:
		return s
	}
}

func inKey(f pgs.Field, x interface{}) string {
	switch f.Type().ProtoType() {
	case pgs.BytesT:
		return byteStr(x.([]byte))
	case pgs.MessageT:
		switch x := x.(type) {
		case *duration.Duration:
			return durLit(x)
		default:
			return lit(x)
		}
	case pgs.EnumT:
		return fmt.Sprintf("%s(%d)", cType(f.Type()), x.(int32))
	default:
		return lit(x)
	}
}

func durLit(dur *duration.Duration) string {
	return fmt.Sprintf(
		"pgv::protobuf::util::TimeUtil::SecondsToDuration(%d) + pgv::protobuf::util::TimeUtil::NanosecondsToDuration(%d)",
		dur.GetSeconds(), dur.GetNanos())
}

func durStr(dur *duration.Duration) string {
	d, _ := ptypes.Duration(dur)
	return d.String()
}

func durGt(a, b *duration.Duration) bool {
	ad, _ := ptypes.Duration(a)
	bd, _ := ptypes.Duration(b)

	return ad > bd
}

func tsLit(ts *timestamp.Timestamp) string {
	return fmt.Sprintf(
		"time.Unix(%d, %d)",
		ts.GetSeconds(), ts.GetNanos(),
	)
}

func tsGt(a, b *timestamp.Timestamp) bool {
	at, _ := ptypes.Timestamp(a)
	bt, _ := ptypes.Timestamp(b)

	return !bt.Before(at)
}

func tsStr(ts *timestamp.Timestamp) string {
	t, _ := ptypes.Timestamp(ts)
	return t.String()
}

func unwrap(ctx shared.RuleContext, name string) (shared.RuleContext, error) {
	ctx, err := ctx.Unwrap("wrapper")
	if err != nil {
		return ctx, err
	}

	ctx.AccessorOverride = fmt.Sprintf("%s.%s()", name,
		ctx.Field.Type().Embed().Fields()[0].Name())

	return ctx, nil
}

func failUnimplemented() string {
	return "throw std::string(\"not yet implemented\");"
}
<|endoftext|>


<commit_before>package core

import (
	"errors"
	"io"
	"strings"
	"unicode"
)

var (
	errInvalidCodePoint = errors.New("invalid code point")
)

type TokenType int

const (
	Invalid TokenType = iota
	Annotation
	LineString
	Indent
	Unindent
	EOF
	_SOF
)

type Token struct {
	Type  TokenType
	Value string
}

type Scanner struct {
	reader
	indenter
	tokenQueue
	err error
}

func NewScanner(r io.RuneScanner) *Scanner {
	s := &Scanner{
		reader: reader{r: r},
		indenter: indenter{
			indents: []string{""},
		},
		tokenQueue: tokenQueue{
			toks: []Token{Token{Type: _SOF}},
		},
	}
	return s
}

func (s *Scanner) Scan() bool {
	s.popTok()
	if s.tokCount() > 0 {
		return true
	}
	if s.err != nil {
		return false
	}
	s.scanLine()
	if s.err == io.EOF {
		for i := 0; i < s.eofIndentLevel(); i++ {
			s.pushTok(Token{Type: Unindent})
		}
		s.pushTok(Token{Type: EOF})
	}
	return s.tokCount() > 0
}

func (s *Scanner) scanLine() {
	indent, err := s.readIndent()
	if err != nil {
		s.err = err
		return
	}
	indentType, n, err := s.indentLevel(indent)
	if err != nil {
		s.err = err
		return
	}
	for i := 0; i < n; i++ {
		s.pushTok(Token{Type: indentType})
	}
	var line string
	line, s.err = s.readLine()
	if line[0] == '#' {
		s.pushTok(Token{Type: Annotation, Value: line[1:]})
	} else {
		s.pushTok(Token{Type: LineString, Value: line})
	}
}

func (s *Scanner) Err() error {
	if s.err == io.EOF {
		return nil
	}
	return s.err
}

type reader struct {
	r   io.RuneScanner
	ch  rune
	err error
}

func (s *reader) readLine() (string, error) {
	rs := []rune{}
	for s.next() {
		switch s.ch {
		case '\r', '\n':
			s.prev()
			goto ret
		}
		rs = append(rs, s.ch)
	}
ret:
	return string(rs), s.err
}

func (s *reader) readIndent() (indent string, err error) {
	for {
		var ok bool
		indent, ok = s.indentSpaces()
		if !ok {
			err = s.err
			return
		}
		var hasNewline bool
		hasNewline, ok = s.skipLineBreaks()
		if !ok {
			err = s.err
			return
		} else if !hasNewline {
			return
		}
	}
}
func (s *reader) skipLineBreaks() (hasNewline bool, ok bool) {
	for s.next() {
		switch s.ch {
		case '\r', '\n':
			hasNewline = true
		default:
			s.prev()
			return hasNewline, true
		}
	}
	return hasNewline, false
}
func (s *reader) indentSpaces() (indent string, ok bool) {
	rs := []rune{}
	for s.next() {
		switch s.ch {
		case ' ', '\t':
		default:
			s.prev()
			return string(rs), true
		}
		rs = append(rs, s.ch)
	}
	return "", false
}

func (s *reader) next() bool {
	var err error
	s.ch, _, err = s.r.ReadRune()
	if err != nil {
		s.err = err
		return false
	}
	switch s.ch {
	case '\t', ' ', '\r', '\n':
	case unicode.ReplacementChar:
		s.err = errInvalidCodePoint
		return false
	default:
		if '\x00' <= s.ch && s.ch <= '\x19' {
			s.err = errInvalidCodePoint
			return false
		}
	}
	return true
}

func (s *reader) prev() bool {
	s.err = s.r.UnreadRune()
	return s.err == nil
}

type indenter struct {
	indents []string
}

func (s *indenter) indentLevel(indent string) (TokenType, int, error) {
	last := s.indents[len(s.indents)-1]
	if indent == last {
		return 0, 0, nil
	} else if strings.HasPrefix(indent, last) {
		s.indents = append(s.indents, indent)
		return Indent, 1, nil
	}
	for i := 1; i < len(s.indents); i++ {
		if indent == s.indents[len(s.indents)-i-1] {
			s.indents = s.indents[:len(s.indents)-i]
			return Unindent, i, nil
		}
	}
	return 0, 0, errors.New("mismatch indent")
}

func (s *indenter) eofIndentLevel() int {
	if len(s.indents) > 1 {
		n := len(s.indents) - 1
		s.indents = s.indents[:1]
		return n
	}
	return 0
}

type tokenQueue struct {
	toks []Token
}

func (s *tokenQueue) Token() Token {
	return s.toks[0]
}

func (s *tokenQueue) pushTok(tok Token) {
	s.toks = append(s.toks, tok)
}

func (s *tokenQueue) popTok() {
	s.toks = s.toks[1:]
}

func (s *tokenQueue) tokCount() int {
	return len(s.toks)
}
<commit_msg>refactor.<commit_after>package core

import (
	"errors"
	"io"
	"strings"
	"unicode"
)

var (
	errInvalidCodePoint = errors.New("invalid code point")
)

type TokenType int

const (
	Invalid TokenType = iota
	Annotation
	LineString
	Indent
	Unindent
	EOF
	_SOF
)

type Token struct {
	Type  TokenType
	Value string
}

type Scanner struct {
	reader
	indenter
	tokenQueue
	err error
}

func NewScanner(r io.RuneScanner) *Scanner {
	return &Scanner{
		reader: reader{r: r},
		indenter: indenter{
			indents: []string{""},
		},
		tokenQueue: tokenQueue{
			toks: []Token{Token{Type: _SOF}},
		},
	}
}

func (s *Scanner) Scan() bool {
	s.popTok()
	if s.tokCount() > 0 {
		return true
	}
	if s.err != nil {
		return false
	}
	s.scanLine()
	if s.err == io.EOF {
		for i := 0; i < s.eofUnindentLevel(); i++ {
			s.pushTok(Token{Type: Unindent})
		}
		s.pushTok(Token{Type: EOF})
	}
	return s.tokCount() > 0
}

func (s *Scanner) scanLine() {
	indent, err := s.readIndent()
	if err != nil {
		s.err = err
		return
	}
	indentType, n, err := s.indentLevel(indent)
	if err != nil {
		s.err = err
		return
	}
	for i := 0; i < n; i++ {
		s.pushTok(Token{Type: indentType})
	}
	var line string
	line, s.err = s.readLine()
	if line[0] == '#' {
		s.pushTok(Token{Type: Annotation, Value: line[1:]})
	} else {
		s.pushTok(Token{Type: LineString, Value: line})
	}
}

func (s *Scanner) Err() error {
	if s.err == io.EOF {
		return nil
	}
	return s.err
}

type reader struct {
	r   io.RuneScanner
	ch  rune
	err error
}

func (s *reader) readLine() (string, error) {
	rs := []rune{}
	for s.next() {
		switch s.ch {
		case '\r', '\n':
			s.prev()
			return string(rs), nil
		}
		rs = append(rs, s.ch)
	}
	return string(rs), s.err
}

func (s *reader) readIndent() (string, error) {
	for {
		indent, ok := s.indentSpaces()
		if !ok {
			return indent, s.err
		}
		if ok := s.skipLineBreaks(); !ok {
			return indent, s.err
		}
	}
}
func (s *reader) skipLineBreaks() (hasNewline bool) {
	for s.next() {
		switch s.ch {
		case '\r', '\n':
			hasNewline = true
		default:
			s.prev()
			return hasNewline
		}
	}
	return false
}
func (s *reader) indentSpaces() (indent string, ok bool) {
	rs := []rune{}
	for s.next() {
		switch s.ch {
		case ' ', '\t':
		default:
			s.prev()
			return string(rs), true
		}
		rs = append(rs, s.ch)
	}
	return "", false
}

func (s *reader) next() bool {
	var err error
	s.ch, _, err = s.r.ReadRune()
	if err != nil {
		s.err = err
		return false
	}
	switch s.ch {
	case '\t', ' ', '\r', '\n':
	case unicode.ReplacementChar:
		s.err = errInvalidCodePoint
		return false
	default:
		if '\x00' <= s.ch && s.ch <= '\x19' {
			s.err = errInvalidCodePoint
			return false
		}
	}
	return true
}

func (s *reader) prev() bool {
	s.err = s.r.UnreadRune()
	return s.err == nil
}

type indenter struct {
	indents []string
}

func (s *indenter) indentLevel(indent string) (TokenType, int, error) {
	last := s.indents[len(s.indents)-1]
	if indent == last {
		return 0, 0, nil
	} else if strings.HasPrefix(indent, last) {
		s.indents = append(s.indents, indent)
		return Indent, 1, nil
	}
	for i := 1; i < len(s.indents); i++ {
		if indent == s.indents[len(s.indents)-i-1] {
			s.indents = s.indents[:len(s.indents)-i]
			return Unindent, i, nil
		}
	}
	return 0, 0, errors.New("mismatch indent")
}

func (s *indenter) eofUnindentLevel() int {
	if len(s.indents) > 1 {
		n := len(s.indents) - 1
		s.indents = s.indents[:1]
		return n
	}
	return 0
}

type tokenQueue struct {
	toks []Token
}

func (s *tokenQueue) Token() Token {
	return s.toks[0]
}

func (s *tokenQueue) pushTok(tok Token) {
	s.toks = append(s.toks, tok)
}

func (s *tokenQueue) popTok() {
	s.toks = s.toks[1:]
}

func (s *tokenQueue) tokCount() int {
	return len(s.toks)
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"io"
	"mime"
	"mime/multipart"
	"os"
	"path/filepath"

	"github.com/lxc/lxd/shared"

	log "gopkg.in/inconshreveable/log15.v2"
)

type Progress struct {
	io.Reader
	total  int64
	length int64
	op     *operation
}

func (pt *Progress) Read(p []byte) (int, error) {
	n, err := pt.Reader.Read(p)
	if n > 0 {
		pt.total += int64(n)
		percentage := float64(pt.total) / float64(pt.length) * float64(100)

		if pt.op != nil {
			meta := pt.op.metadata
			if meta == nil {
				meta = make(map[string]interface{})
			}

			progressInt := 10 - (int(percentage) % 10) + int(percentage)
			if progressInt > 100 {
				progressInt = 100
			}
			progress := fmt.Sprintf("%d%%", progressInt)

			if meta["download_progress"] != progress {
				meta["download_progress"] = progress
				pt.op.UpdateMetadata(meta)
			}
		}
	}

	return n, err
}

// ImageDownload checks if we have that Image Fingerprint else
// downloads the image from a remote server.
func (d *Daemon) ImageDownload(op *operation,
	server, fp string, secret string, forContainer bool) error {

	if _, err := dbImageGet(d.db, fp, false, false); err == nil {
		shared.Log.Debug("Image already exists in the db", log.Ctx{"image": fp})
		// already have it
		return nil
	}

	shared.Log.Info(
		"Image not in the db, downloading it",
		log.Ctx{"image": fp, "server": server})

	// Now check if we already downloading the image
	d.imagesDownloadingLock.RLock()
	if waitChannel, ok := d.imagesDownloading[fp]; ok {
		// We already download the image
		d.imagesDownloadingLock.RUnlock()

		shared.Log.Info(
			"Already downloading the image, waiting for it to succeed",
			log.Ctx{"image": fp})

		// Wait until the download finishes (channel closes)
		if _, ok := <-waitChannel; ok {
			shared.Log.Warn("Value transmitted over image lock semaphore?")
		}

		if _, err := dbImageGet(d.db, fp, false, true); err != nil {
			shared.Log.Error(
				"Previous download didn't succeed",
				log.Ctx{"image": fp})

			return fmt.Errorf("Previous download didn't succeed")
		}

		shared.Log.Info(
			"Previous download succeeded",
			log.Ctx{"image": fp})

		return nil
	}

	d.imagesDownloadingLock.RUnlock()

	shared.Log.Info(
		"Downloading the image",
		log.Ctx{"image": fp})

	// Add the download to the queue
	d.imagesDownloadingLock.Lock()
	d.imagesDownloading[fp] = make(chan bool)
	d.imagesDownloadingLock.Unlock()

	// Unlock once this func ends.
	defer func() {
		d.imagesDownloadingLock.Lock()
		if waitChannel, ok := d.imagesDownloading[fp]; ok {
			close(waitChannel)
			delete(d.imagesDownloading, fp)
		}
		d.imagesDownloadingLock.Unlock()
	}()

	/* grab the metadata from /1.0/images/%s */
	var url string
	if secret != "" {
		url = fmt.Sprintf(
			"%s/%s/images/%s?secret=%s",
			server, shared.APIVersion, fp, secret)

	} else {
		url = fmt.Sprintf("%s/%s/images/%s", server, shared.APIVersion, fp)
	}

	resp, err := d.httpGetSync(url)
	if err != nil {
		shared.Log.Error(
			"Failed to download image metadata",
			log.Ctx{"image": fp, "err": err})

		return nil
	}

	info := shared.ImageInfo{}
	if err := json.Unmarshal(resp.Metadata, &info); err != nil {
		return err
	}

	/* now grab the actual file from /1.0/images/%s/export */
	var exporturl string
	if secret != "" {
		exporturl = fmt.Sprintf(
			"%s/%s/images/%s/export?secret=%s",
			server, shared.APIVersion, fp, secret)

	} else {
		exporturl = fmt.Sprintf(
			"%s/%s/images/%s/export",
			server, shared.APIVersion, fp)
	}

	raw, err := d.httpGetFile(exporturl)
	if err != nil {
		shared.Log.Error(
			"Failed to download image",
			log.Ctx{"image": fp, "err": err})
		return err
	}

	destDir := shared.VarPath("images")
	destName := filepath.Join(destDir, fp)
	if shared.PathExists(destName) {
		d.Storage.ImageDelete(fp)
	}

	ctype, ctypeParams, err := mime.ParseMediaType(raw.Header.Get("Content-Type"))
	if err != nil {
		ctype = "application/octet-stream"
	}

	body := &Progress{Reader: raw.Body, length: raw.ContentLength, op: op}

	if ctype == "multipart/form-data" {
		// Parse the POST data
		mr := multipart.NewReader(body, ctypeParams["boundary"])

		// Get the metadata tarball
		part, err := mr.NextPart()
		if err != nil {
			shared.Log.Error(
				"Invalid multipart image",
				log.Ctx{"image": fp, "err": err})

			return err
		}

		if part.FormName() != "metadata" {
			shared.Log.Error(
				"Invalid multipart image",
				log.Ctx{"image": fp, "err": err})

			return fmt.Errorf("Invalid multipart image")
		}

		destName := filepath.Join(destDir, info.Fingerprint)
		f, err := os.Create(destName)
		if err != nil {
			shared.Log.Error(
				"Failed to save image",
				log.Ctx{"image": fp, "err": err})

			return err
		}

		_, err = io.Copy(f, part)
		f.Close()

		if err != nil {
			shared.Log.Error(
				"Failed to save image",
				log.Ctx{"image": fp, "err": err})

			return err
		}

		// Get the rootfs tarball
		part, err = mr.NextPart()
		if err != nil {
			shared.Log.Error(
				"Invalid multipart image",
				log.Ctx{"image": fp, "err": err})

			return err
		}

		if part.FormName() != "rootfs" {
			shared.Log.Error(
				"Invalid multipart image",
				log.Ctx{"image": fp})
			return fmt.Errorf("Invalid multipart image")
		}

		destName = filepath.Join(destDir, info.Fingerprint+".rootfs")
		f, err = os.Create(destName)
		if err != nil {
			shared.Log.Error(
				"Failed to save image",
				log.Ctx{"image": fp, "err": err})
			return err
		}

		_, err = io.Copy(f, part)
		f.Close()

		if err != nil {
			shared.Log.Error(
				"Failed to save image",
				log.Ctx{"image": fp, "err": err})
			return err
		}
	} else {
		destName := filepath.Join(destDir, info.Fingerprint)

		f, err := os.Create(destName)
		if err != nil {
			shared.Log.Error(
				"Failed to save image",
				log.Ctx{"image": fp, "err": err})

			return err
		}

		_, err = io.Copy(f, body)
		f.Close()

		if err != nil {
			shared.Log.Error(
				"Failed to save image",
				log.Ctx{"image": fp, "err": err})
			return err
		}
	}

	// By default, make all downloaded images private
	info.Public = false

	_, err = imageBuildFromInfo(d, info)
	if err != nil {
		shared.Log.Error(
			"Failed to create image",
			log.Ctx{"image": fp, "err": err})

		return err
	}

	shared.Log.Info(
		"Download succeeded",
		log.Ctx{"image": fp})

	if forContainer {
		return dbImageLastAccessInit(d.db, fp)
	}

	return nil
}
<commit_msg>Optimize progress reporting<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"io"
	"mime"
	"mime/multipart"
	"os"
	"path/filepath"

	"github.com/lxc/lxd/shared"

	log "gopkg.in/inconshreveable/log15.v2"
)

type Progress struct {
	io.Reader
	total      int64
	length     int64
	percentage float64
	op         *operation
}

func (pt *Progress) Read(p []byte) (int, error) {
	n, err := pt.Reader.Read(p)
	if n > 0 {
		pt.total += int64(n)
		percentage := float64(pt.total) / float64(pt.length) * float64(100)

		if percentage-pt.percentage > 9 && pt.op != nil {
			meta := pt.op.metadata
			if meta == nil {
				meta = make(map[string]interface{})
			}

			progressInt := 10 - (int(percentage) % 10) + int(percentage)
			if progressInt > 100 {
				progressInt = 100
			}
			progress := fmt.Sprintf("%d%%", progressInt)

			if meta["download_progress"] != progress {
				meta["download_progress"] = progress
				pt.op.UpdateMetadata(meta)
			}

			pt.percentage = percentage
		}
	}

	return n, err
}

// ImageDownload checks if we have that Image Fingerprint else
// downloads the image from a remote server.
func (d *Daemon) ImageDownload(op *operation,
	server, fp string, secret string, forContainer bool) error {

	if _, err := dbImageGet(d.db, fp, false, false); err == nil {
		shared.Log.Debug("Image already exists in the db", log.Ctx{"image": fp})
		// already have it
		return nil
	}

	shared.Log.Info(
		"Image not in the db, downloading it",
		log.Ctx{"image": fp, "server": server})

	// Now check if we already downloading the image
	d.imagesDownloadingLock.RLock()
	if waitChannel, ok := d.imagesDownloading[fp]; ok {
		// We already download the image
		d.imagesDownloadingLock.RUnlock()

		shared.Log.Info(
			"Already downloading the image, waiting for it to succeed",
			log.Ctx{"image": fp})

		// Wait until the download finishes (channel closes)
		if _, ok := <-waitChannel; ok {
			shared.Log.Warn("Value transmitted over image lock semaphore?")
		}

		if _, err := dbImageGet(d.db, fp, false, true); err != nil {
			shared.Log.Error(
				"Previous download didn't succeed",
				log.Ctx{"image": fp})

			return fmt.Errorf("Previous download didn't succeed")
		}

		shared.Log.Info(
			"Previous download succeeded",
			log.Ctx{"image": fp})

		return nil
	}

	d.imagesDownloadingLock.RUnlock()

	shared.Log.Info(
		"Downloading the image",
		log.Ctx{"image": fp})

	// Add the download to the queue
	d.imagesDownloadingLock.Lock()
	d.imagesDownloading[fp] = make(chan bool)
	d.imagesDownloadingLock.Unlock()

	// Unlock once this func ends.
	defer func() {
		d.imagesDownloadingLock.Lock()
		if waitChannel, ok := d.imagesDownloading[fp]; ok {
			close(waitChannel)
			delete(d.imagesDownloading, fp)
		}
		d.imagesDownloadingLock.Unlock()
	}()

	/* grab the metadata from /1.0/images/%s */
	var url string
	if secret != "" {
		url = fmt.Sprintf(
			"%s/%s/images/%s?secret=%s",
			server, shared.APIVersion, fp, secret)

	} else {
		url = fmt.Sprintf("%s/%s/images/%s", server, shared.APIVersion, fp)
	}

	resp, err := d.httpGetSync(url)
	if err != nil {
		shared.Log.Error(
			"Failed to download image metadata",
			log.Ctx{"image": fp, "err": err})

		return nil
	}

	info := shared.ImageInfo{}
	if err := json.Unmarshal(resp.Metadata, &info); err != nil {
		return err
	}

	/* now grab the actual file from /1.0/images/%s/export */
	var exporturl string
	if secret != "" {
		exporturl = fmt.Sprintf(
			"%s/%s/images/%s/export?secret=%s",
			server, shared.APIVersion, fp, secret)

	} else {
		exporturl = fmt.Sprintf(
			"%s/%s/images/%s/export",
			server, shared.APIVersion, fp)
	}

	raw, err := d.httpGetFile(exporturl)
	if err != nil {
		shared.Log.Error(
			"Failed to download image",
			log.Ctx{"image": fp, "err": err})
		return err
	}

	destDir := shared.VarPath("images")
	destName := filepath.Join(destDir, fp)
	if shared.PathExists(destName) {
		d.Storage.ImageDelete(fp)
	}

	ctype, ctypeParams, err := mime.ParseMediaType(raw.Header.Get("Content-Type"))
	if err != nil {
		ctype = "application/octet-stream"
	}

	body := &Progress{Reader: raw.Body, length: raw.ContentLength, op: op}

	if ctype == "multipart/form-data" {
		// Parse the POST data
		mr := multipart.NewReader(body, ctypeParams["boundary"])

		// Get the metadata tarball
		part, err := mr.NextPart()
		if err != nil {
			shared.Log.Error(
				"Invalid multipart image",
				log.Ctx{"image": fp, "err": err})

			return err
		}

		if part.FormName() != "metadata" {
			shared.Log.Error(
				"Invalid multipart image",
				log.Ctx{"image": fp, "err": err})

			return fmt.Errorf("Invalid multipart image")
		}

		destName := filepath.Join(destDir, info.Fingerprint)
		f, err := os.Create(destName)
		if err != nil {
			shared.Log.Error(
				"Failed to save image",
				log.Ctx{"image": fp, "err": err})

			return err
		}

		_, err = io.Copy(f, part)
		f.Close()

		if err != nil {
			shared.Log.Error(
				"Failed to save image",
				log.Ctx{"image": fp, "err": err})

			return err
		}

		// Get the rootfs tarball
		part, err = mr.NextPart()
		if err != nil {
			shared.Log.Error(
				"Invalid multipart image",
				log.Ctx{"image": fp, "err": err})

			return err
		}

		if part.FormName() != "rootfs" {
			shared.Log.Error(
				"Invalid multipart image",
				log.Ctx{"image": fp})
			return fmt.Errorf("Invalid multipart image")
		}

		destName = filepath.Join(destDir, info.Fingerprint+".rootfs")
		f, err = os.Create(destName)
		if err != nil {
			shared.Log.Error(
				"Failed to save image",
				log.Ctx{"image": fp, "err": err})
			return err
		}

		_, err = io.Copy(f, part)
		f.Close()

		if err != nil {
			shared.Log.Error(
				"Failed to save image",
				log.Ctx{"image": fp, "err": err})
			return err
		}
	} else {
		destName := filepath.Join(destDir, info.Fingerprint)

		f, err := os.Create(destName)
		if err != nil {
			shared.Log.Error(
				"Failed to save image",
				log.Ctx{"image": fp, "err": err})

			return err
		}

		_, err = io.Copy(f, body)
		f.Close()

		if err != nil {
			shared.Log.Error(
				"Failed to save image",
				log.Ctx{"image": fp, "err": err})
			return err
		}
	}

	// By default, make all downloaded images private
	info.Public = false

	_, err = imageBuildFromInfo(d, info)
	if err != nil {
		shared.Log.Error(
			"Failed to create image",
			log.Ctx{"image": fp, "err": err})

		return err
	}

	shared.Log.Info(
		"Download succeeded",
		log.Ctx{"image": fp})

	if forContainer {
		return dbImageLastAccessInit(d.db, fp)
	}

	return nil
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The TensorFlow Authors. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package tensorflow

// #include <stdlib.h>
// #include "tensorflow/c/c_api.h"
import "C"

import (
	"errors"
	"fmt"
	"runtime"
	"sync"
	"unsafe"
)

// Session drives a TensorFlow graph computation.
//
// When a Session is created with a given target, a new Session object is bound
// to the universe of resources specified by that target. Those resources are
// available to this session to perform computation described in the GraphDef.
// After creating the session with a graph, the caller uses the Run() API to
// perform the computation and potentially fetch outputs as Tensors.
// A Session allows concurrent calls to Run().
type Session struct {
	c *C.TF_Session

	// For ensuring that:
	// - Close() blocks on all Run() calls to complete.
	// - Close() can be called multiple times.
	wg sync.WaitGroup
	mu sync.Mutex
}

// NewSession creates a new execution session with the associated graph.
// options may be nil to use the default options.
func NewSession(graph *Graph, options *SessionOptions) (*Session, error) {
	status := newStatus()
	cOpt, doneOpt, err := options.c()
	defer doneOpt()
	if err != nil {
		return nil, err
	}
	cSess := C.TF_NewSession(graph.c, cOpt, status.c)
	if err := status.Err(); err != nil {
		return nil, err
	}

	s := &Session{c: cSess}
	runtime.SetFinalizer(s, func(s *Session) { s.Close() })
	return s, nil
}

// Device structure contains information about a device associated with a session, as returned by ListDevices()
type Device struct {
	Name, Type       string
	MemoryLimitBytes int64
}

// String describes d and implements fmt.Stringer.
func (d Device) String() string {
	memStr := "no memory limit"
	if d.MemoryLimitBytes >= 0 {
		memStr = fmt.Sprintf("memory limit %d bytes", d.MemoryLimitBytes)
	}
	return fmt.Sprintf("(Device: name \"%s\", type %s, %s)", d.Name, d.Type, memStr)
}

// Return list of devices associated with a Session
func (s *Session) ListDevices() ([]Device, error) {
	var devices []Device

	status := newStatus()
	devices_list := C.TF_SessionListDevices(s.c, status.c)
	if err := status.Err(); err != nil {
		return nil, fmt.Errorf("SessionListDevices() failed: %v", err)
	}
	defer C.TF_DeleteDeviceList(devices_list)

	for i := 0; i < int(C.TF_DeviceListCount(devices_list)); i++ {
		device_name := C.TF_DeviceListName(devices_list, C.int(i), status.c)
		if err := status.Err(); err != nil {
			return nil, fmt.Errorf("DeviceListName(index=%d) failed: %v", i, err)
		}

		device_type := C.TF_DeviceListType(devices_list, C.int(i), status.c)
		if err := status.Err(); err != nil {
			return nil, fmt.Errorf("DeviceListType(index=%d) failed: %v", i, err)
		}

		memory_limit_bytes := C.TF_DeviceListMemoryBytes(devices_list, C.int(i), status.c)
		if err := status.Err(); err != nil {
			return nil, fmt.Errorf("DeviceListMemoryBytes(index=%d) failed: %v", i, err)
		}

		device := Device{
			Name:             C.GoString(device_name),
			Type:             C.GoString(device_type),
			MemoryLimitBytes: int64(memory_limit_bytes),
		}

		devices = append(devices, device)
	}

	return devices, nil
}

// Run the graph with the associated session starting with the supplied feeds
// to compute the value of the requested fetches. Runs, but does not return
// Tensors for operations specified in targets.
//
// On success, returns the fetched Tensors in the same order as supplied in
// the fetches argument. If fetches is set to nil, the returned Tensor fetches
// is empty.
func (s *Session) Run(feeds map[Output]*Tensor, fetches []Output, targets []*Operation) ([]*Tensor, error) {
	s.mu.Lock()
	if s.c == nil {
		s.mu.Unlock()
		return nil, errors.New("session is closed")
	}
	s.wg.Add(1)
	s.mu.Unlock()
	defer s.wg.Done()

	c := newCRunArgs(feeds, fetches, targets)
	status := newStatus()
	C.TF_SessionRun(s.c, nil,
		ptrOutput(c.feeds), ptrTensor(c.feedTensors), C.int(len(feeds)),
		ptrOutput(c.fetches), ptrTensor(c.fetchTensors), C.int(len(fetches)),
		ptrOperation(c.targets), C.int(len(targets)),
		nil, status.c)

	// Make sure GC won't harvest input tensors until SessionRun() is finished
	runtime.KeepAlive(feeds)

	if err := status.Err(); err != nil {
		return nil, err
	}
	return c.toGo(), nil
}

// PartialRun enables incremental evaluation of graphs.
//
// PartialRun allows the caller to pause the evaluation of a graph, run
// arbitrary code that depends on the intermediate computation of the graph,
// and then resume graph execution. The results of the arbitrary code can be
// fed into the graph when resuming execution.  In contrast, Session.Run
// executes the graph to compute the requested fetches using the provided feeds
// and discards all intermediate state (e.g., value of intermediate tensors)
// when it returns.
//
// For example, consider a graph for unsupervised training of a neural network
// model. PartialRun can be used to pause execution after the forward pass of
// the network, let the caller actuate the output (e.g., play a game, actuate a
// robot etc.), determine the error/loss and then feed this calculated loss
// when resuming the backward pass of the graph.
type PartialRun struct {
	session *Session
	handle  *C.char
}

// Run resumes execution of the graph to compute the requested fetches and
// targets with the provided feeds.
func (pr *PartialRun) Run(feeds map[Output]*Tensor, fetches []Output, targets []*Operation) ([]*Tensor, error) {
	var (
		c      = newCRunArgs(feeds, fetches, targets)
		status = newStatus()
		s      = pr.session
	)
	s.mu.Lock()
	if s.c == nil {
		s.mu.Unlock()
		return nil, errors.New("session is closed")
	}
	s.wg.Add(1)
	s.mu.Unlock()
	defer s.wg.Done()

	C.TF_SessionPRun(s.c, pr.handle,
		ptrOutput(c.feeds), ptrTensor(c.feedTensors), C.int(len(feeds)),
		ptrOutput(c.fetches), ptrTensor(c.fetchTensors), C.int(len(fetches)),
		ptrOperation(c.targets), C.int(len(targets)),
		status.c)
	if err := status.Err(); err != nil {
		return nil, err
	}
	return c.toGo(), nil
}

// NewPartialRun sets up the graph for incremental evaluation.
//
// All values of feeds, fetches and targets that may be provided to Run calls
// on the returned PartialRun need to be provided to NewPartialRun.
//
// See documentation for the PartialRun type.
func (s *Session) NewPartialRun(feeds, fetches []Output, targets []*Operation) (*PartialRun, error) {
	var (
		cfeeds   = make([]C.TF_Output, len(feeds))
		cfetches = make([]C.TF_Output, len(fetches))
		ctargets = make([]*C.TF_Operation, len(targets))

		pcfeeds   *C.TF_Output
		pcfetches *C.TF_Output
		pctargets **C.TF_Operation

		status = newStatus()
	)
	if len(feeds) > 0 {
		pcfeeds = &cfeeds[0]
		for i, o := range feeds {
			cfeeds[i] = o.c()
		}
	}
	if len(fetches) > 0 {
		pcfetches = &cfetches[0]
		for i, o := range fetches {
			cfetches[i] = o.c()
		}
	}
	if len(targets) > 0 {
		pctargets = &ctargets[0]
		for i, o := range targets {
			ctargets[i] = o.c
		}
	}

	s.mu.Lock()
	if s.c == nil {
		s.mu.Unlock()
		return nil, errors.New("session is closed")
	}
	s.wg.Add(1)
	s.mu.Unlock()
	defer s.wg.Done()

	pr := &PartialRun{session: s}
	C.TF_SessionPRunSetup(s.c,
		pcfeeds, C.int(len(feeds)),
		pcfetches, C.int(len(fetches)),
		pctargets, C.int(len(targets)),
		&pr.handle, status.c)
	if err := status.Err(); err != nil {
		return nil, err
	}
	runtime.SetFinalizer(pr, func(pr *PartialRun) {
		C.TF_DeletePRunHandle(pr.handle)
	})
	return pr, nil
}

// Close a session. This contacts any other processes associated with this
// session, if applicable. Blocks until all previous calls to Run have returned.
func (s *Session) Close() error {
	s.mu.Lock()
	defer s.mu.Unlock()
	s.wg.Wait()
	if s.c == nil {
		return nil
	}
	status := newStatus()
	C.TF_CloseSession(s.c, status.c)
	if err := status.Err(); err != nil {
		return err
	}
	C.TF_DeleteSession(s.c, status.c)
	s.c = nil
	return status.Err()
}

// SessionOptions contains configuration information for a session.
type SessionOptions struct {
	// Target indicates the TensorFlow runtime to connect to.
	//
	// If 'target' is empty or unspecified, the local TensorFlow runtime
	// implementation will be used.  Otherwise, the TensorFlow engine
	// defined by 'target' will be used to perform all computations.
	//
	// "target" can be either a single entry or a comma separated list
	// of entries. Each entry is a resolvable address of one of the
	// following formats:
	//   local
	//   ip:port
	//   host:port
	//   ... other system-specific formats to identify tasks and jobs ...
	//
	// NOTE: at the moment 'local' maps to an in-process service-based
	// runtime.
	//
	// Upon creation, a single session affines itself to one of the
	// remote processes, with possible load balancing choices when the
	// "target" resolves to a list of possible processes.
	//
	// If the session disconnects from the remote process during its
	// lifetime, session calls may fail immediately.
	Target string

	// Config is a binary-serialized representation of the
	// tensorflow.ConfigProto protocol message
	// (https://www.tensorflow.org/code/tensorflow/core/protobuf/config.proto).
	Config []byte
}

// c converts the SessionOptions to the C API's TF_SessionOptions. Callers must
// deallocate by calling the returned done() closure.
func (o *SessionOptions) c() (ret *C.TF_SessionOptions, done func(), err error) {
	opt := C.TF_NewSessionOptions()
	if o == nil {
		return opt, func() { C.TF_DeleteSessionOptions(opt) }, nil
	}
	t := C.CString(o.Target)
	C.TF_SetTarget(opt, t)
	C.free(unsafe.Pointer(t))

	var cConfig unsafe.Pointer
	if sz := len(o.Config); sz > 0 {
		status := newStatus()
		// Copying into C-memory is the simplest thing to do in terms
		// of memory safety and cgo rules ("C code may not keep a copy
		// of a Go pointer after the call returns" from
		// https://golang.org/cmd/cgo/#hdr-Passing_pointers).
		cConfig = C.CBytes(o.Config)
		C.TF_SetConfig(opt, cConfig, C.size_t(sz), status.c)
		if err := status.Err(); err != nil {
			C.TF_DeleteSessionOptions(opt)
			return nil, func() {}, fmt.Errorf("invalid SessionOptions.Config: %v", err)
		}
	}
	return opt, func() {
		C.TF_DeleteSessionOptions(opt)
		C.free(cConfig)
	}, nil
}

// cRunArgs translates the arguments to Session.Run and PartialRun.Run into
// values suitable for C library calls.
type cRunArgs struct {
	feeds        []C.TF_Output
	feedTensors  []*C.TF_Tensor
	fetches      []C.TF_Output
	fetchTensors []*C.TF_Tensor
	targets      []*C.TF_Operation
}

func newCRunArgs(feeds map[Output]*Tensor, fetches []Output, targets []*Operation) *cRunArgs {
	c := &cRunArgs{
		fetches:      make([]C.TF_Output, len(fetches)),
		fetchTensors: make([]*C.TF_Tensor, len(fetches)),
		targets:      make([]*C.TF_Operation, len(targets)),
	}
	for o, t := range feeds {
		c.feeds = append(c.feeds, o.c())
		c.feedTensors = append(c.feedTensors, t.c)
	}
	for i, o := range fetches {
		c.fetches[i] = o.c()
	}
	for i, t := range targets {
		c.targets[i] = t.c
	}
	return c
}

func (c *cRunArgs) toGo() []*Tensor {
	ret := make([]*Tensor, len(c.fetchTensors))
	for i, ct := range c.fetchTensors {
		ret[i] = newTensorFromC(ct)
	}
	return ret
}

func ptrOutput(l []C.TF_Output) *C.TF_Output {
	if len(l) == 0 {
		return nil
	}
	return &l[0]
}

func ptrTensor(l []*C.TF_Tensor) **C.TF_Tensor {
	if len(l) == 0 {
		return nil
	}
	return &l[0]
}

func ptrOperation(l []*C.TF_Operation) **C.TF_Operation {
	if len(l) == 0 {
		return nil
	}
	return &l[0]
}
<commit_msg>Refactor ListDevices in TF Go API<commit_after>/*
Copyright 2016 The TensorFlow Authors. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package tensorflow

// #include <stdlib.h>
// #include "tensorflow/c/c_api.h"
import "C"

import (
	"errors"
	"fmt"
	"runtime"
	"sync"
	"unsafe"
)

// Session drives a TensorFlow graph computation.
//
// When a Session is created with a given target, a new Session object is bound
// to the universe of resources specified by that target. Those resources are
// available to this session to perform computation described in the GraphDef.
// After creating the session with a graph, the caller uses the Run() API to
// perform the computation and potentially fetch outputs as Tensors.
// A Session allows concurrent calls to Run().
type Session struct {
	c *C.TF_Session

	// For ensuring that:
	// - Close() blocks on all Run() calls to complete.
	// - Close() can be called multiple times.
	wg sync.WaitGroup
	mu sync.Mutex
}

// NewSession creates a new execution session with the associated graph.
// options may be nil to use the default options.
func NewSession(graph *Graph, options *SessionOptions) (*Session, error) {
	status := newStatus()
	cOpt, doneOpt, err := options.c()
	defer doneOpt()
	if err != nil {
		return nil, err
	}
	cSess := C.TF_NewSession(graph.c, cOpt, status.c)
	if err := status.Err(); err != nil {
		return nil, err
	}

	s := &Session{c: cSess}
	runtime.SetFinalizer(s, func(s *Session) { s.Close() })
	return s, nil
}

// Device structure contains information about a device associated with a session, as returned by ListDevices()
type Device struct {
	Name, Type       string
	MemoryLimitBytes int64
}

// String describes d and implements fmt.Stringer.
func (d Device) String() string {
	memStr := "no memory limit"
	if d.MemoryLimitBytes >= 0 {
		memStr = fmt.Sprintf("memory limit %d bytes", d.MemoryLimitBytes)
	}
	return fmt.Sprintf("(Device: name \"%s\", type %s, %s)", d.Name, d.Type, memStr)
}

func deviceSliceFromDeviceList(list *C.TF_DeviceList) ([]Device, error) {
	var devices []Device
	status := newStatus()

	for i := 0; i < int(C.TF_DeviceListCount(list)); i++ {
		name := C.TF_DeviceListName(list, C.int(i), status.c)
		if err := status.Err(); err != nil {
			return nil, fmt.Errorf("DeviceListName(index=%d) failed: %v", i, err)
		}

		deviceType := C.TF_DeviceListType(list, C.int(i), status.c)
		if err := status.Err(); err != nil {
			return nil, fmt.Errorf("DeviceListType(index=%d) failed: %v", i, err)
		}

		memoryLimitBytes := C.TF_DeviceListMemoryBytes(list, C.int(i), status.c)
		if err := status.Err(); err != nil {
			return nil, fmt.Errorf("DeviceListMemoryBytes(index=%d) failed: %v", i, err)
		}

		device := Device{
			Name:             C.GoString(name),
			Type:             C.GoString(deviceType),
			MemoryLimitBytes: int64(memoryLimitBytes),
		}

		devices = append(devices, device)
	}

	return devices, nil
}

// ListDevices returns the list of devices associated with a Session.
func (s *Session) ListDevices() ([]Device, error) {
	status := newStatus()
	devicesList := C.TF_SessionListDevices(s.c, status.c)
	if err := status.Err(); err != nil {
		return nil, fmt.Errorf("SessionListDevices() failed: %v", err)
	}
	defer C.TF_DeleteDeviceList(devicesList)
	return deviceSliceFromDeviceList(devicesList)
}

// Run the graph with the associated session starting with the supplied feeds
// to compute the value of the requested fetches. Runs, but does not return
// Tensors for operations specified in targets.
//
// On success, returns the fetched Tensors in the same order as supplied in
// the fetches argument. If fetches is set to nil, the returned Tensor fetches
// is empty.
func (s *Session) Run(feeds map[Output]*Tensor, fetches []Output, targets []*Operation) ([]*Tensor, error) {
	s.mu.Lock()
	if s.c == nil {
		s.mu.Unlock()
		return nil, errors.New("session is closed")
	}
	s.wg.Add(1)
	s.mu.Unlock()
	defer s.wg.Done()

	c := newCRunArgs(feeds, fetches, targets)
	status := newStatus()
	C.TF_SessionRun(s.c, nil,
		ptrOutput(c.feeds), ptrTensor(c.feedTensors), C.int(len(feeds)),
		ptrOutput(c.fetches), ptrTensor(c.fetchTensors), C.int(len(fetches)),
		ptrOperation(c.targets), C.int(len(targets)),
		nil, status.c)

	// Make sure GC won't harvest input tensors until SessionRun() is finished
	runtime.KeepAlive(feeds)

	if err := status.Err(); err != nil {
		return nil, err
	}
	return c.toGo(), nil
}

// PartialRun enables incremental evaluation of graphs.
//
// PartialRun allows the caller to pause the evaluation of a graph, run
// arbitrary code that depends on the intermediate computation of the graph,
// and then resume graph execution. The results of the arbitrary code can be
// fed into the graph when resuming execution.  In contrast, Session.Run
// executes the graph to compute the requested fetches using the provided feeds
// and discards all intermediate state (e.g., value of intermediate tensors)
// when it returns.
//
// For example, consider a graph for unsupervised training of a neural network
// model. PartialRun can be used to pause execution after the forward pass of
// the network, let the caller actuate the output (e.g., play a game, actuate a
// robot etc.), determine the error/loss and then feed this calculated loss
// when resuming the backward pass of the graph.
type PartialRun struct {
	session *Session
	handle  *C.char
}

// Run resumes execution of the graph to compute the requested fetches and
// targets with the provided feeds.
func (pr *PartialRun) Run(feeds map[Output]*Tensor, fetches []Output, targets []*Operation) ([]*Tensor, error) {
	var (
		c      = newCRunArgs(feeds, fetches, targets)
		status = newStatus()
		s      = pr.session
	)
	s.mu.Lock()
	if s.c == nil {
		s.mu.Unlock()
		return nil, errors.New("session is closed")
	}
	s.wg.Add(1)
	s.mu.Unlock()
	defer s.wg.Done()

	C.TF_SessionPRun(s.c, pr.handle,
		ptrOutput(c.feeds), ptrTensor(c.feedTensors), C.int(len(feeds)),
		ptrOutput(c.fetches), ptrTensor(c.fetchTensors), C.int(len(fetches)),
		ptrOperation(c.targets), C.int(len(targets)),
		status.c)
	if err := status.Err(); err != nil {
		return nil, err
	}
	return c.toGo(), nil
}

// NewPartialRun sets up the graph for incremental evaluation.
//
// All values of feeds, fetches and targets that may be provided to Run calls
// on the returned PartialRun need to be provided to NewPartialRun.
//
// See documentation for the PartialRun type.
func (s *Session) NewPartialRun(feeds, fetches []Output, targets []*Operation) (*PartialRun, error) {
	var (
		cfeeds   = make([]C.TF_Output, len(feeds))
		cfetches = make([]C.TF_Output, len(fetches))
		ctargets = make([]*C.TF_Operation, len(targets))

		pcfeeds   *C.TF_Output
		pcfetches *C.TF_Output
		pctargets **C.TF_Operation

		status = newStatus()
	)
	if len(feeds) > 0 {
		pcfeeds = &cfeeds[0]
		for i, o := range feeds {
			cfeeds[i] = o.c()
		}
	}
	if len(fetches) > 0 {
		pcfetches = &cfetches[0]
		for i, o := range fetches {
			cfetches[i] = o.c()
		}
	}
	if len(targets) > 0 {
		pctargets = &ctargets[0]
		for i, o := range targets {
			ctargets[i] = o.c
		}
	}

	s.mu.Lock()
	if s.c == nil {
		s.mu.Unlock()
		return nil, errors.New("session is closed")
	}
	s.wg.Add(1)
	s.mu.Unlock()
	defer s.wg.Done()

	pr := &PartialRun{session: s}
	C.TF_SessionPRunSetup(s.c,
		pcfeeds, C.int(len(feeds)),
		pcfetches, C.int(len(fetches)),
		pctargets, C.int(len(targets)),
		&pr.handle, status.c)
	if err := status.Err(); err != nil {
		return nil, err
	}
	runtime.SetFinalizer(pr, func(pr *PartialRun) {
		C.TF_DeletePRunHandle(pr.handle)
	})
	return pr, nil
}

// Close a session. This contacts any other processes associated with this
// session, if applicable. Blocks until all previous calls to Run have returned.
func (s *Session) Close() error {
	s.mu.Lock()
	defer s.mu.Unlock()
	s.wg.Wait()
	if s.c == nil {
		return nil
	}
	status := newStatus()
	C.TF_CloseSession(s.c, status.c)
	if err := status.Err(); err != nil {
		return err
	}
	C.TF_DeleteSession(s.c, status.c)
	s.c = nil
	return status.Err()
}

// SessionOptions contains configuration information for a session.
type SessionOptions struct {
	// Target indicates the TensorFlow runtime to connect to.
	//
	// If 'target' is empty or unspecified, the local TensorFlow runtime
	// implementation will be used.  Otherwise, the TensorFlow engine
	// defined by 'target' will be used to perform all computations.
	//
	// "target" can be either a single entry or a comma separated list
	// of entries. Each entry is a resolvable address of one of the
	// following formats:
	//   local
	//   ip:port
	//   host:port
	//   ... other system-specific formats to identify tasks and jobs ...
	//
	// NOTE: at the moment 'local' maps to an in-process service-based
	// runtime.
	//
	// Upon creation, a single session affines itself to one of the
	// remote processes, with possible load balancing choices when the
	// "target" resolves to a list of possible processes.
	//
	// If the session disconnects from the remote process during its
	// lifetime, session calls may fail immediately.
	Target string

	// Config is a binary-serialized representation of the
	// tensorflow.ConfigProto protocol message
	// (https://www.tensorflow.org/code/tensorflow/core/protobuf/config.proto).
	Config []byte
}

// c converts the SessionOptions to the C API's TF_SessionOptions. Callers must
// deallocate by calling the returned done() closure.
func (o *SessionOptions) c() (ret *C.TF_SessionOptions, done func(), err error) {
	opt := C.TF_NewSessionOptions()
	if o == nil {
		return opt, func() { C.TF_DeleteSessionOptions(opt) }, nil
	}
	t := C.CString(o.Target)
	C.TF_SetTarget(opt, t)
	C.free(unsafe.Pointer(t))

	var cConfig unsafe.Pointer
	if sz := len(o.Config); sz > 0 {
		status := newStatus()
		// Copying into C-memory is the simplest thing to do in terms
		// of memory safety and cgo rules ("C code may not keep a copy
		// of a Go pointer after the call returns" from
		// https://golang.org/cmd/cgo/#hdr-Passing_pointers).
		cConfig = C.CBytes(o.Config)
		C.TF_SetConfig(opt, cConfig, C.size_t(sz), status.c)
		if err := status.Err(); err != nil {
			C.TF_DeleteSessionOptions(opt)
			return nil, func() {}, fmt.Errorf("invalid SessionOptions.Config: %v", err)
		}
	}
	return opt, func() {
		C.TF_DeleteSessionOptions(opt)
		C.free(cConfig)
	}, nil
}

// cRunArgs translates the arguments to Session.Run and PartialRun.Run into
// values suitable for C library calls.
type cRunArgs struct {
	feeds        []C.TF_Output
	feedTensors  []*C.TF_Tensor
	fetches      []C.TF_Output
	fetchTensors []*C.TF_Tensor
	targets      []*C.TF_Operation
}

func newCRunArgs(feeds map[Output]*Tensor, fetches []Output, targets []*Operation) *cRunArgs {
	c := &cRunArgs{
		fetches:      make([]C.TF_Output, len(fetches)),
		fetchTensors: make([]*C.TF_Tensor, len(fetches)),
		targets:      make([]*C.TF_Operation, len(targets)),
	}
	for o, t := range feeds {
		c.feeds = append(c.feeds, o.c())
		c.feedTensors = append(c.feedTensors, t.c)
	}
	for i, o := range fetches {
		c.fetches[i] = o.c()
	}
	for i, t := range targets {
		c.targets[i] = t.c
	}
	return c
}

func (c *cRunArgs) toGo() []*Tensor {
	ret := make([]*Tensor, len(c.fetchTensors))
	for i, ct := range c.fetchTensors {
		ret[i] = newTensorFromC(ct)
	}
	return ret
}

func ptrOutput(l []C.TF_Output) *C.TF_Output {
	if len(l) == 0 {
		return nil
	}
	return &l[0]
}

func ptrTensor(l []*C.TF_Tensor) **C.TF_Tensor {
	if len(l) == 0 {
		return nil
	}
	return &l[0]
}

func ptrOperation(l []*C.TF_Operation) **C.TF_Operation {
	if len(l) == 0 {
		return nil
	}
	return &l[0]
}
<|endoftext|>


<commit_before>/*
Copyright 2015 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package v1alpha1

import (
	"time"

	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	kruntime "k8s.io/apimachinery/pkg/runtime"
	apiserverconfigv1alpha1 "k8s.io/apiserver/pkg/apis/config/v1alpha1"
	utilpointer "k8s.io/utils/pointer"
)

func addDefaultingFuncs(scheme *kruntime.Scheme) error {
	return RegisterDefaults(scheme)
}

func SetDefaults_CloudControllerManagerConfiguration(obj *CloudControllerManagerConfiguration) {
	zero := metav1.Duration{}
	if obj.ServiceController.ConcurrentServiceSyncs == 0 {
		obj.ServiceController.ConcurrentServiceSyncs = 1
	}
	if obj.NodeStatusUpdateFrequency == zero {
		obj.NodeStatusUpdateFrequency = metav1.Duration{Duration: 5 * time.Minute}
	}
}

func SetDefaults_KubeControllerManagerConfiguration(obj *KubeControllerManagerConfiguration) {
	zero := metav1.Duration{}
	if len(obj.Controllers) == 0 {
		obj.Controllers = []string{"*"}
	}
	if obj.EndPointController.ConcurrentEndpointSyncs == 0 {
		obj.EndPointController.ConcurrentEndpointSyncs = 5
	}
	if obj.ServiceController.ConcurrentServiceSyncs == 0 {
		obj.ServiceController.ConcurrentServiceSyncs = 1
	}
	if obj.ReplicationController.ConcurrentRCSyncs == 0 {
		obj.ReplicationController.ConcurrentRCSyncs = 5
	}
	if obj.ReplicaSetController.ConcurrentRSSyncs == 0 {
		obj.ReplicaSetController.ConcurrentRSSyncs = 5
	}
	if obj.DaemonSetController.ConcurrentDaemonSetSyncs == 0 {
		obj.DaemonSetController.ConcurrentDaemonSetSyncs = 2
	}
	if obj.JobController.ConcurrentJobSyncs == 0 {
		obj.JobController.ConcurrentJobSyncs = 5
	}
	if obj.ResourceQuotaController.ConcurrentResourceQuotaSyncs == 0 {
		obj.ResourceQuotaController.ConcurrentResourceQuotaSyncs = 5
	}
	if obj.DeploymentController.ConcurrentDeploymentSyncs == 0 {
		obj.DeploymentController.ConcurrentDeploymentSyncs = 5
	}
	if obj.NamespaceController.ConcurrentNamespaceSyncs == 0 {
		obj.NamespaceController.ConcurrentNamespaceSyncs = 10
	}
	if obj.SAController.ConcurrentSATokenSyncs == 0 {
		obj.SAController.ConcurrentSATokenSyncs = 5
	}
	if obj.ResourceQuotaController.ResourceQuotaSyncPeriod == zero {
		obj.ResourceQuotaController.ResourceQuotaSyncPeriod = metav1.Duration{Duration: 5 * time.Minute}
	}
	if obj.NamespaceController.NamespaceSyncPeriod == zero {
		obj.NamespaceController.NamespaceSyncPeriod = metav1.Duration{Duration: 5 * time.Minute}
	}
	if obj.PersistentVolumeBinderController.PVClaimBinderSyncPeriod == zero {
		obj.PersistentVolumeBinderController.PVClaimBinderSyncPeriod = metav1.Duration{Duration: 15 * time.Second}
	}
	if obj.HPAController.HorizontalPodAutoscalerSyncPeriod == zero {
		obj.HPAController.HorizontalPodAutoscalerSyncPeriod = metav1.Duration{Duration: 30 * time.Second}
	}
	if obj.HPAController.HorizontalPodAutoscalerUpscaleForbiddenWindow == zero {
		obj.HPAController.HorizontalPodAutoscalerUpscaleForbiddenWindow = metav1.Duration{Duration: 3 * time.Minute}
	}
	if obj.HPAController.HorizontalPodAutoscalerCPUTaintPeriod == zero {
		// Assuming CPU is collected every minute and initialization takes another minute HPA should
		// disregard samples from first two minutes as contaminated by initialization.
		obj.HPAController.HorizontalPodAutoscalerCPUTaintPeriod = metav1.Duration{Duration: time.Minute}
	}
	if obj.HPAController.HorizontalPodAutoscalerInitialReadinessDelay == zero {
		obj.HPAController.HorizontalPodAutoscalerInitialReadinessDelay = metav1.Duration{Duration: 30 * time.Second}
	}
	if obj.HPAController.HorizontalPodAutoscalerDownscaleForbiddenWindow == zero {
		obj.HPAController.HorizontalPodAutoscalerDownscaleForbiddenWindow = metav1.Duration{Duration: 5 * time.Minute}
	}
	if obj.HPAController.HorizontalPodAutoscalerTolerance == 0 {
		obj.HPAController.HorizontalPodAutoscalerTolerance = 0.1
	}
	if obj.DeploymentController.DeploymentControllerSyncPeriod == zero {
		obj.DeploymentController.DeploymentControllerSyncPeriod = metav1.Duration{Duration: 30 * time.Second}
	}
	if obj.DeprecatedController.RegisterRetryCount == 0 {
		obj.DeprecatedController.RegisterRetryCount = 10
	}
	if obj.NodeLifecycleController.PodEvictionTimeout == zero {
		obj.NodeLifecycleController.PodEvictionTimeout = metav1.Duration{Duration: 5 * time.Minute}
	}
	if obj.NodeLifecycleController.NodeMonitorGracePeriod == zero {
		obj.NodeLifecycleController.NodeMonitorGracePeriod = metav1.Duration{Duration: 40 * time.Second}
	}
	if obj.NodeLifecycleController.NodeStartupGracePeriod == zero {
		obj.NodeLifecycleController.NodeStartupGracePeriod = metav1.Duration{Duration: 60 * time.Second}
	}
	if obj.NodeIpamController.NodeCIDRMaskSize == 0 {
		obj.NodeIpamController.NodeCIDRMaskSize = 24
	}
	if obj.PodGCController.TerminatedPodGCThreshold == 0 {
		obj.PodGCController.TerminatedPodGCThreshold = 12500
	}
	if obj.GarbageCollectorController.EnableGarbageCollector == nil {
		obj.GarbageCollectorController.EnableGarbageCollector = utilpointer.BoolPtr(true)
	}
	if obj.GarbageCollectorController.ConcurrentGCSyncs == 0 {
		obj.GarbageCollectorController.ConcurrentGCSyncs = 20
	}
	if obj.CSRSigningController.ClusterSigningCertFile == "" {
		obj.CSRSigningController.ClusterSigningCertFile = "/etc/kubernetes/ca/ca.pem"
	}
	if obj.CSRSigningController.ClusterSigningKeyFile == "" {
		obj.CSRSigningController.ClusterSigningKeyFile = "/etc/kubernetes/ca/ca.key"
	}
	if obj.CSRSigningController.ClusterSigningDuration == zero {
		obj.CSRSigningController.ClusterSigningDuration = metav1.Duration{Duration: 365 * 24 * time.Hour}
	}
	if obj.AttachDetachController.ReconcilerSyncLoopPeriod == zero {
		obj.AttachDetachController.ReconcilerSyncLoopPeriod = metav1.Duration{Duration: 60 * time.Second}
	}
	if obj.NodeLifecycleController.EnableTaintManager == nil {
		obj.NodeLifecycleController.EnableTaintManager = utilpointer.BoolPtr(true)
	}
	if obj.HPAController.HorizontalPodAutoscalerUseRESTClients == nil {
		obj.HPAController.HorizontalPodAutoscalerUseRESTClients = utilpointer.BoolPtr(true)
	}
}

func SetDefaults_GenericComponentConfiguration(obj *GenericComponentConfiguration) {
	zero := metav1.Duration{}
	if obj.MinResyncPeriod == zero {
		obj.MinResyncPeriod = metav1.Duration{Duration: 12 * time.Hour}
	}
	if obj.ContentType == "" {
		obj.ContentType = "application/vnd.kubernetes.protobuf"
	}
	if obj.KubeAPIQPS == 0 {
		obj.KubeAPIQPS = 20.0
	}
	if obj.KubeAPIBurst == 0 {
		obj.KubeAPIBurst = 30
	}
	if obj.ControllerStartInterval == zero {
		obj.ControllerStartInterval = metav1.Duration{Duration: 0 * time.Second}
	}

	// Use the default LeaderElectionConfiguration options
	apiserverconfigv1alpha1.RecommendedDefaultLeaderElectionConfiguration(&obj.LeaderElection)
}

func SetDefaults_KubeCloudSharedConfiguration(obj *KubeCloudSharedConfiguration) {
	zero := metav1.Duration{}
	// Port
	if obj.Address == "" {
		obj.Address = "0.0.0.0"
	}
	if obj.RouteReconciliationPeriod == zero {
		obj.RouteReconciliationPeriod = metav1.Duration{Duration: 10 * time.Second}
	}
	if obj.NodeMonitorPeriod == zero {
		obj.NodeMonitorPeriod = metav1.Duration{Duration: 5 * time.Second}
	}
	if obj.ClusterName == "" {
		obj.ClusterName = "kubernetes"
	}
	if obj.ConfigureCloudRoutes == nil {
		obj.ConfigureCloudRoutes = utilpointer.BoolPtr(true)
	}
}

func SetDefaults_PersistentVolumeRecyclerConfiguration(obj *PersistentVolumeRecyclerConfiguration) {
	if obj.MaximumRetry == 0 {
		obj.MaximumRetry = 3
	}
	if obj.MinimumTimeoutNFS == 0 {
		obj.MinimumTimeoutNFS = 300
	}
	if obj.IncrementTimeoutNFS == 0 {
		obj.IncrementTimeoutNFS = 30
	}
	if obj.MinimumTimeoutHostPath == 0 {
		obj.MinimumTimeoutHostPath = 60
	}
	if obj.IncrementTimeoutHostPath == 0 {
		obj.IncrementTimeoutHostPath = 30
	}
}

func SetDefaults_VolumeConfiguration(obj *VolumeConfiguration) {
	if obj.EnableHostPathProvisioning == nil {
		obj.EnableHostPathProvisioning = utilpointer.BoolPtr(false)
	}
	if obj.EnableDynamicProvisioning == nil {
		obj.EnableDynamicProvisioning = utilpointer.BoolPtr(true)
	}
	if obj.FlexVolumePluginDir == "" {
		obj.FlexVolumePluginDir = "/usr/libexec/kubernetes/kubelet-plugins/volume/exec/"
	}
}
<commit_msg>Increase Horizontal Pod Autoscaler update frequency (every 30s -> every 15s).<commit_after>/*
Copyright 2015 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package v1alpha1

import (
	"time"

	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	kruntime "k8s.io/apimachinery/pkg/runtime"
	apiserverconfigv1alpha1 "k8s.io/apiserver/pkg/apis/config/v1alpha1"
	utilpointer "k8s.io/utils/pointer"
)

func addDefaultingFuncs(scheme *kruntime.Scheme) error {
	return RegisterDefaults(scheme)
}

func SetDefaults_CloudControllerManagerConfiguration(obj *CloudControllerManagerConfiguration) {
	zero := metav1.Duration{}
	if obj.ServiceController.ConcurrentServiceSyncs == 0 {
		obj.ServiceController.ConcurrentServiceSyncs = 1
	}
	if obj.NodeStatusUpdateFrequency == zero {
		obj.NodeStatusUpdateFrequency = metav1.Duration{Duration: 5 * time.Minute}
	}
}

func SetDefaults_KubeControllerManagerConfiguration(obj *KubeControllerManagerConfiguration) {
	zero := metav1.Duration{}
	if len(obj.Controllers) == 0 {
		obj.Controllers = []string{"*"}
	}
	if obj.EndPointController.ConcurrentEndpointSyncs == 0 {
		obj.EndPointController.ConcurrentEndpointSyncs = 5
	}
	if obj.ServiceController.ConcurrentServiceSyncs == 0 {
		obj.ServiceController.ConcurrentServiceSyncs = 1
	}
	if obj.ReplicationController.ConcurrentRCSyncs == 0 {
		obj.ReplicationController.ConcurrentRCSyncs = 5
	}
	if obj.ReplicaSetController.ConcurrentRSSyncs == 0 {
		obj.ReplicaSetController.ConcurrentRSSyncs = 5
	}
	if obj.DaemonSetController.ConcurrentDaemonSetSyncs == 0 {
		obj.DaemonSetController.ConcurrentDaemonSetSyncs = 2
	}
	if obj.JobController.ConcurrentJobSyncs == 0 {
		obj.JobController.ConcurrentJobSyncs = 5
	}
	if obj.ResourceQuotaController.ConcurrentResourceQuotaSyncs == 0 {
		obj.ResourceQuotaController.ConcurrentResourceQuotaSyncs = 5
	}
	if obj.DeploymentController.ConcurrentDeploymentSyncs == 0 {
		obj.DeploymentController.ConcurrentDeploymentSyncs = 5
	}
	if obj.NamespaceController.ConcurrentNamespaceSyncs == 0 {
		obj.NamespaceController.ConcurrentNamespaceSyncs = 10
	}
	if obj.SAController.ConcurrentSATokenSyncs == 0 {
		obj.SAController.ConcurrentSATokenSyncs = 5
	}
	if obj.ResourceQuotaController.ResourceQuotaSyncPeriod == zero {
		obj.ResourceQuotaController.ResourceQuotaSyncPeriod = metav1.Duration{Duration: 5 * time.Minute}
	}
	if obj.NamespaceController.NamespaceSyncPeriod == zero {
		obj.NamespaceController.NamespaceSyncPeriod = metav1.Duration{Duration: 5 * time.Minute}
	}
	if obj.PersistentVolumeBinderController.PVClaimBinderSyncPeriod == zero {
		obj.PersistentVolumeBinderController.PVClaimBinderSyncPeriod = metav1.Duration{Duration: 15 * time.Second}
	}
	if obj.HPAController.HorizontalPodAutoscalerSyncPeriod == zero {
		obj.HPAController.HorizontalPodAutoscalerSyncPeriod = metav1.Duration{Duration: 15 * time.Second}
	}
	if obj.HPAController.HorizontalPodAutoscalerUpscaleForbiddenWindow == zero {
		obj.HPAController.HorizontalPodAutoscalerUpscaleForbiddenWindow = metav1.Duration{Duration: 3 * time.Minute}
	}
	if obj.HPAController.HorizontalPodAutoscalerCPUTaintPeriod == zero {
		// Assuming CPU is collected every minute and initialization takes another minute HPA should
		// disregard samples from first two minutes as contaminated by initialization.
		obj.HPAController.HorizontalPodAutoscalerCPUTaintPeriod = metav1.Duration{Duration: time.Minute}
	}
	if obj.HPAController.HorizontalPodAutoscalerInitialReadinessDelay == zero {
		obj.HPAController.HorizontalPodAutoscalerInitialReadinessDelay = metav1.Duration{Duration: 30 * time.Second}
	}
	if obj.HPAController.HorizontalPodAutoscalerDownscaleForbiddenWindow == zero {
		obj.HPAController.HorizontalPodAutoscalerDownscaleForbiddenWindow = metav1.Duration{Duration: 5 * time.Minute}
	}
	if obj.HPAController.HorizontalPodAutoscalerTolerance == 0 {
		obj.HPAController.HorizontalPodAutoscalerTolerance = 0.1
	}
	if obj.DeploymentController.DeploymentControllerSyncPeriod == zero {
		obj.DeploymentController.DeploymentControllerSyncPeriod = metav1.Duration{Duration: 30 * time.Second}
	}
	if obj.DeprecatedController.RegisterRetryCount == 0 {
		obj.DeprecatedController.RegisterRetryCount = 10
	}
	if obj.NodeLifecycleController.PodEvictionTimeout == zero {
		obj.NodeLifecycleController.PodEvictionTimeout = metav1.Duration{Duration: 5 * time.Minute}
	}
	if obj.NodeLifecycleController.NodeMonitorGracePeriod == zero {
		obj.NodeLifecycleController.NodeMonitorGracePeriod = metav1.Duration{Duration: 40 * time.Second}
	}
	if obj.NodeLifecycleController.NodeStartupGracePeriod == zero {
		obj.NodeLifecycleController.NodeStartupGracePeriod = metav1.Duration{Duration: 60 * time.Second}
	}
	if obj.NodeIpamController.NodeCIDRMaskSize == 0 {
		obj.NodeIpamController.NodeCIDRMaskSize = 24
	}
	if obj.PodGCController.TerminatedPodGCThreshold == 0 {
		obj.PodGCController.TerminatedPodGCThreshold = 12500
	}
	if obj.GarbageCollectorController.EnableGarbageCollector == nil {
		obj.GarbageCollectorController.EnableGarbageCollector = utilpointer.BoolPtr(true)
	}
	if obj.GarbageCollectorController.ConcurrentGCSyncs == 0 {
		obj.GarbageCollectorController.ConcurrentGCSyncs = 20
	}
	if obj.CSRSigningController.ClusterSigningCertFile == "" {
		obj.CSRSigningController.ClusterSigningCertFile = "/etc/kubernetes/ca/ca.pem"
	}
	if obj.CSRSigningController.ClusterSigningKeyFile == "" {
		obj.CSRSigningController.ClusterSigningKeyFile = "/etc/kubernetes/ca/ca.key"
	}
	if obj.CSRSigningController.ClusterSigningDuration == zero {
		obj.CSRSigningController.ClusterSigningDuration = metav1.Duration{Duration: 365 * 24 * time.Hour}
	}
	if obj.AttachDetachController.ReconcilerSyncLoopPeriod == zero {
		obj.AttachDetachController.ReconcilerSyncLoopPeriod = metav1.Duration{Duration: 60 * time.Second}
	}
	if obj.NodeLifecycleController.EnableTaintManager == nil {
		obj.NodeLifecycleController.EnableTaintManager = utilpointer.BoolPtr(true)
	}
	if obj.HPAController.HorizontalPodAutoscalerUseRESTClients == nil {
		obj.HPAController.HorizontalPodAutoscalerUseRESTClients = utilpointer.BoolPtr(true)
	}
}

func SetDefaults_GenericComponentConfiguration(obj *GenericComponentConfiguration) {
	zero := metav1.Duration{}
	if obj.MinResyncPeriod == zero {
		obj.MinResyncPeriod = metav1.Duration{Duration: 12 * time.Hour}
	}
	if obj.ContentType == "" {
		obj.ContentType = "application/vnd.kubernetes.protobuf"
	}
	if obj.KubeAPIQPS == 0 {
		obj.KubeAPIQPS = 20.0
	}
	if obj.KubeAPIBurst == 0 {
		obj.KubeAPIBurst = 30
	}
	if obj.ControllerStartInterval == zero {
		obj.ControllerStartInterval = metav1.Duration{Duration: 0 * time.Second}
	}

	// Use the default LeaderElectionConfiguration options
	apiserverconfigv1alpha1.RecommendedDefaultLeaderElectionConfiguration(&obj.LeaderElection)
}

func SetDefaults_KubeCloudSharedConfiguration(obj *KubeCloudSharedConfiguration) {
	zero := metav1.Duration{}
	// Port
	if obj.Address == "" {
		obj.Address = "0.0.0.0"
	}
	if obj.RouteReconciliationPeriod == zero {
		obj.RouteReconciliationPeriod = metav1.Duration{Duration: 10 * time.Second}
	}
	if obj.NodeMonitorPeriod == zero {
		obj.NodeMonitorPeriod = metav1.Duration{Duration: 5 * time.Second}
	}
	if obj.ClusterName == "" {
		obj.ClusterName = "kubernetes"
	}
	if obj.ConfigureCloudRoutes == nil {
		obj.ConfigureCloudRoutes = utilpointer.BoolPtr(true)
	}
}

func SetDefaults_PersistentVolumeRecyclerConfiguration(obj *PersistentVolumeRecyclerConfiguration) {
	if obj.MaximumRetry == 0 {
		obj.MaximumRetry = 3
	}
	if obj.MinimumTimeoutNFS == 0 {
		obj.MinimumTimeoutNFS = 300
	}
	if obj.IncrementTimeoutNFS == 0 {
		obj.IncrementTimeoutNFS = 30
	}
	if obj.MinimumTimeoutHostPath == 0 {
		obj.MinimumTimeoutHostPath = 60
	}
	if obj.IncrementTimeoutHostPath == 0 {
		obj.IncrementTimeoutHostPath = 30
	}
}

func SetDefaults_VolumeConfiguration(obj *VolumeConfiguration) {
	if obj.EnableHostPathProvisioning == nil {
		obj.EnableHostPathProvisioning = utilpointer.BoolPtr(false)
	}
	if obj.EnableDynamicProvisioning == nil {
		obj.EnableDynamicProvisioning = utilpointer.BoolPtr(true)
	}
	if obj.FlexVolumePluginDir == "" {
		obj.FlexVolumePluginDir = "/usr/libexec/kubernetes/kubelet-plugins/volume/exec/"
	}
}
<|endoftext|>