<commit_before>package user

import (
	"errors"
	"golang.org/x/crypto/bcrypt"
	"regexp"
	"strings"

	"github.com/jmcvetta/neoism"
	"github.com/nathandao/vantaa/core/vantaadb"
	"github.com/nathandao/vantaa/settings"
)

type User struct {
	Id             int    `json:"id"`
	Name           string `json:"name"`
	Email          string `json:"email"`
	PasswordDigest []byte `json:"-"`
	Password       string `json:"-"`
}

type UserAdapter struct {
	Id             int    `json:"id(u)"`
	Name           string `json:"u.name"`
	Email          string `json:"u.email"`
	PasswordDigest []byte `json:"u.password_digest"`
}

// Transform creates a User struct from the UserAdapter
func (ua *UserAdapter) Transform() *User {
	u := User{
		ua.Id,
		ua.Name,
		ua.Email,
		ua.PasswordDigest,
		"",
	}
	return &u
}

// New initiate a new User struct
func New() *User {
	return &User{}
}

// Save or u.Save calls on CreateUser to write a newu User node in the database.
func (u *User) Save() (*User, error) {
	newu, err := CreateUser(u)
	if err != nil {
		return nil, err
	}
	return newu, nil
}

// Delete removes the use
// func (u *User) Delete() error {
// 	if err := DeleteUser(u); err != nil {
// 		return err
// 	}
// 	return nil
// }

// CreateUser validates the User's information. Upon successful validations,
// it creates a new User node in the database,
func CreateUser(u *User) (*User, error) {
	// user sanitization
	u.sanitize()
	if _, err := ValidateUser(u); err != nil {
		return nil, err
	}
	passwordDigest, err := bcrypt.GenerateFromPassword([]byte(u.Password), settings.Get().HashCost)
	if err != nil {
		return nil, err
	}
	// remove passsword string right after
	u.Password = ""
	u.PasswordDigest = passwordDigest
	res := []UserAdapter{}
	db := vantaadb.Connect()
	cq := neoism.CypherQuery{
		Statement: `CREATE (u:User {name:{name}, email:{email}, password_digest:{password_digest}})
                RETURN id(u), u.name, u.email`,
		Parameters: neoism.Props{"name": u.Name, "email": u.Email, "password_digest": u.PasswordDigest},
		Result:     &res,
	}
	if err := db.Cypher(&cq); err != nil {
		return nil, err
	}
	ua := &res[0]
	return ua.Transform(), nil
}

// DeleteUser removes a User node from the database.
// error is return even when user is not found
// func DeleteUser(u *User) error {
// 	db := vantaadb.Connect()
// 	cq := neoism.CypherQuery{
// 		Statement: `MATCH (u:User)
//                 WHERE ` + vantaadb.PropString("u", props) + `DELETE u, s, r`,
// 		Parameters: props,
// 	}
// 	if err := db.Cypher(&cq); err != nil {
// 		return err
// 	}
// 	return nil
// }

// FindUser finds a single User by calling on FindUsers and returns the
// first element of the []*User slice.
func FindUser(props neoism.Props) (*User, error) {
	u, err := FindUsers(props)
	if err != nil {
		return nil, err
	}
	return u[0], nil
}

// FindUsers contructs and executes a cypher query with the provided parametrs
// and return all User nodes that satisfy the props conditions.
func FindUsers(props map[string]interface{}) ([]*User, error) {
	db := vantaadb.Connect()
	res := []UserAdapter{}
	// generate condition string to be used in the cypher statement
	condstr := vantaadb.PropString("u", props)
	cq := neoism.CypherQuery{
		Statement: `MATCH (u:User)
                WHERE ` + condstr + `
                RETURN id(u), u.name, u.email, u.password_digest`,
		Parameters: props,
		Result:     &res,
	}
	err := db.Cypher(&cq)
	if err != nil {
		return []*User{}, err
	}
	users := []*User{}
	for _, ua := range res {
		users = append(users, ua.Transform())
	}
	if len(users) < 1 {
		return nil, errors.New("not found")
	}
	return users, nil
}

// ValidateUser makes sure all user's information is satisfied
func ValidateUser(u *User) (bool, error) {
	// sanitization
	u.sanitize()
	// validate user name
	if u.Name == "" {
		return false, errors.New("missing user's name")
	}
	if matched, err := regexp.MatchString("^[a-z0-9_]+$", u.Name); err != nil {
		panic(err)
	} else if matched == false {
		return false, errors.New("invalid user's name format")
	}
	// validate user email
	if u.Email == "" {
		return false, errors.New("missing user's email")
	}
	if matched, err := regexp.MatchString(`^\w+([-+.']\w+)*@\w+([-.]\w+).\w+([-.]\w+)*$`, u.Email); err != nil || !matched {
		return false, errors.New("invalid email")
	}
	// validate uniqueness
	for _, props := range []map[string]string{
		{"name": u.Name},
		{"email": u.Email},
	} {
		for k, v := range props {
			if u, _ := FindUser(neoism.Props{k: v}); u != nil {
				return false, errors.New("user with " + k + " \"" + v + "\" already exists.")
			}
		}
	}
	// validate user password
	if u.Password == "" {
		return false, errors.New("missing user's password")
	}
	return true, nil
}

// sanitize convert user's name and email to lower case
func (u *User) sanitize() {
	u.Name = strings.ToLower(strings.TrimSpace(u.Name))
	u.Email = strings.ToLower(strings.TrimSpace(u.Email))
}
<commit_msg>reformat user Cypher queries<commit_after>package user

import (
	"errors"
	"golang.org/x/crypto/bcrypt"
	"regexp"
	"strings"

	"github.com/jmcvetta/neoism"
	"github.com/nathandao/vantaa/core/vantaadb"
	"github.com/nathandao/vantaa/settings"
)

type User struct {
	Id             int    `json:"id"`
	Name           string `json:"name"`
	Email          string `json:"email"`
	PasswordDigest []byte `json:"-"`
	Password       string `json:"-"`
}

type UserAdapter struct {
	Id             int    `json:"id(u)"`
	Name           string `json:"u.name"`
	Email          string `json:"u.email"`
	PasswordDigest []byte `json:"u.password_digest"`
}

// Transform creates a User struct from the UserAdapter
func (ua *UserAdapter) Transform() *User {
	u := User{
		ua.Id,
		ua.Name,
		ua.Email,
		ua.PasswordDigest,
		"",
	}
	return &u
}

// New initiate a new User struct
func New() *User {
	return &User{}
}

// Save or u.Save calls on CreateUser to write a newu User node in the database.
func (u *User) Save() (*User, error) {
	newu, err := CreateUser(u)
	if err != nil {
		return nil, err
	}
	return newu, nil
}

// Delete removes the use
// func (u *User) Delete() error {
// 	if err := DeleteUser(u); err != nil {
// 		return err
// 	}
// 	return nil
// }

// CreateUser validates the User's information. Upon successful validations,
// it creates a new User node in the database,
func CreateUser(u *User) (*User, error) {
	// user sanitization
	u.sanitize()
	if _, err := ValidateUser(u); err != nil {
		return nil, err
	}
	passwordDigest, err := bcrypt.GenerateFromPassword([]byte(u.Password), settings.Get().HashCost)
	if err != nil {
		return nil, err
	}
	// remove passsword string right after
	u.Password = ""
	u.PasswordDigest = passwordDigest
	res := []UserAdapter{}
	db := vantaadb.Connect()
	cq := neoism.CypherQuery{
		Statement: `CREATE (u:User {
                  name:{name},
                  email:{email},
                  password_digest:{password_digest}
                })
                RETURN id(u), u.name, u.email`,
		Parameters: neoism.Props{"name": u.Name, "email": u.Email, "password_digest": u.PasswordDigest},
		Result:     &res,
	}
	if err := db.Cypher(&cq); err != nil {
		return nil, err
	}
	ua := &res[0]
	return ua.Transform(), nil
}

// DeleteUser removes a User node from the database.
// error is return even when user is not found
// func DeleteUser(u *User) error {
// 	db := vantaadb.Connect()
// 	cq := neoism.CypherQuery{
// 		Statement: `MATCH (u:User)
//                 WHERE ` + vantaadb.PropString("u", props) + `DELETE u, s, r`,
// 		Parameters: props,
// 	}
// 	if err := db.Cypher(&cq); err != nil {
// 		return err
// 	}
// 	return nil
// }

// FindUser finds a single User by calling on FindUsers and returns the
// first element of the []*User slice.
func FindUser(props neoism.Props) (*User, error) {
	u, err := FindUsers(props)
	if err != nil {
		return nil, err
	}
	return u[0], nil
}

// FindUsers contructs and executes a cypher query with the provided parametrs
// and return all User nodes that satisfy the props conditions.
func FindUsers(props map[string]interface{}) ([]*User, error) {
	db := vantaadb.Connect()
	res := []UserAdapter{}
	// generate condition string to be used in the cypher statement
	condstr := vantaadb.PropString("u", props)
	cq := neoism.CypherQuery{
		Statement: `MATCH (u:User)
                WHERE ` + condstr + `
                RETURN
                  id(u),
                  u.name,
                  u.email,
                  u.password_digest`,
		Parameters: props,
		Result:     &res,
	}
	err := db.Cypher(&cq)
	if err != nil {
		return []*User{}, err
	}
	users := []*User{}
	for _, ua := range res {
		users = append(users, ua.Transform())
	}
	if len(users) < 1 {
		return nil, errors.New("not found")
	}
	return users, nil
}

// ValidateUser makes sure all user's information is satisfied
func ValidateUser(u *User) (bool, error) {
	// sanitization
	u.sanitize()
	// validate user name
	if u.Name == "" {
		return false, errors.New("missing user's name")
	}
	if matched, err := regexp.MatchString("^[a-z0-9_]+$", u.Name); err != nil {
		panic(err)
	} else if matched == false {
		return false, errors.New("invalid user's name format")
	}
	// validate user email
	if u.Email == "" {
		return false, errors.New("missing user's email")
	}
	if matched, err := regexp.MatchString(`^\w+([-+.']\w+)*@\w+([-.]\w+).\w+([-.]\w+)*$`, u.Email); err != nil || !matched {
		return false, errors.New("invalid email")
	}
	// validate uniqueness
	for _, props := range []map[string]string{
		{"name": u.Name},
		{"email": u.Email},
	} {
		for k, v := range props {
			if u, _ := FindUser(neoism.Props{k: v}); u != nil {
				return false, errors.New("user with " + k + " \"" + v + "\" already exists.")
			}
		}
	}
	// validate user password
	if u.Password == "" {
		return false, errors.New("missing user's password")
	}
	return true, nil
}

// sanitize convert user's name and email to lower case
func (u *User) sanitize() {
	u.Name = strings.ToLower(strings.TrimSpace(u.Name))
	u.Email = strings.ToLower(strings.TrimSpace(u.Email))
}
<|endoftext|>


<commit_before>package trace

import (
	"fmt"
	"github.com/gin-gonic/gin"
	"golang.org/x/net/trace"
	"net/http"
)

type traceError gin.Error

// add custom stringer method to gin error type
func (e traceError) String() string {
	return fmt.Sprintf("Error: %s\nMeta: %s", e.Err, e.Meta)
}

// Trace will gather information from the request and also add the trace methods to handlers
func Trace() gin.HandlerFunc {
	return func(c *gin.Context) {

		if gin.IsDebugging() && (c.HandlerName() != c.HandlerName()) {
			// a new trace
			tr := trace.New(c.HandlerName(), c.Request.URL.Path)

			c.Next()

			if len(c.Errors) != 0 {
				// loop through errors
				for _, err := range c.Errors {
					tr.LazyLog(traceError(*err), false)
				}
				tr.SetError()
			}

			// finish the trace
			tr.Finish()

			return
		}

		c.Next()

	}
}

// TraceController returns the default trace requests page
// example handler: r.GET("/debug/requests", trace.TraceController)
func TraceController(c *gin.Context) {

	if gin.IsDebugging() {
		// render the requests page
		trace.Render(c.Writer, c.Request, false)
		return
	}

	c.String(http.StatusNotFound, "Not found")
	return
}
<commit_msg>add errors and gin debug flag check<commit_after>package trace

import (
	"fmt"
	"github.com/gin-gonic/gin"
	"golang.org/x/net/trace"
	"net/http"
)

type traceError gin.Error

// add custom stringer method to gin error type
func (e traceError) String() string {
	return fmt.Sprintf("Error: %s\nMeta: %s", e.Err, e.Meta)
}

// Trace will gather information from the request and also add the trace methods to handlers
func Trace() gin.HandlerFunc {
	return func(c *gin.Context) {

		if gin.IsDebugging() && c.HandlerName() != c.HandlerName() {
			// a new trace
			tr := trace.New(c.HandlerName(), c.Request.URL.Path)

			c.Next()

			if len(c.Errors) != 0 {
				// loop through errors
				for _, err := range c.Errors {
					tr.LazyLog(traceError(*err), false)
				}
				tr.SetError()
			}

			// finish the trace
			tr.Finish()

			return
		}

		c.Next()

	}
}

// TraceController returns the default trace requests page
// example handler: r.GET("/debug/requests", trace.TraceController)
func TraceController(c *gin.Context) {

	if gin.IsDebugging() {
		// render the requests page
		trace.Render(c.Writer, c.Request, false)
		return
	}

	c.String(http.StatusNotFound, "Not found")
	return
}
<|endoftext|>


<commit_before>package libkb

import (
	"fmt"
	"github.com/keybase/go-jsonw"
	"sync"
	"time"
)

// Can either be a RemoteProofChainLink or one of the identities
// listed in a tracking statement
type TrackIdComponent interface {
	ToIdString() string
	ToKeyValuePair() (string, string)
	GetProofState() int
}

type TrackSet map[string]TrackIdComponent

func (ts TrackSet) Add(t TrackIdComponent) {
	ts[t.ToIdString()] = t
}

func (a TrackSet) GetProofState(tic TrackIdComponent) int {
	ret := PROOF_STATE_NONE
	if obj := a[tic.ToIdString()]; obj != nil {
		ret = obj.GetProofState()
	}
	return ret
}

func (A TrackSet) Subtract(B TrackSet) (out []TrackIdComponent) {
	for _, c := range A {
		if !B.HasMember(c) {
			out = append(out, c)
		}
	}
	return
}

func (a TrackSet) HasMember(t TrackIdComponent) bool {
	_, found := a[t.ToIdString()]
	return found
}

func (a TrackSet) LenEq(b TrackSet) bool {
	return len(a) == len(b)
}

//=====================================================================

type TrackInstructions struct {
	Local  bool
	Remote bool
}

//=====================================================================

type TrackLookup struct {
	link  *TrackChainLink     // The original chain link that I signed
	set   TrackSet            // The total set of tracked identities
	ids   map[string][]string // A http -> [foo.com, boo.com] lookup
	mutex *sync.Mutex         // in case we're accessing in mutliple threads
}

func (l TrackLookup) GetProofState(tic TrackIdComponent) int {
	return l.set.GetProofState(tic)
}

func (tl *TrackLookup) ComputeKeyDiff(curr PgpFingerprint) TrackDiff {
	prev, err := tl.link.GetTrackedPgpFingerprint()
	if err != nil {
		return TrackDiffError{err}
	} else if prev.Eq(curr) {
		return TrackDiffNone{}
	} else {
		return TrackDiffClash{curr.ToQuads(), prev.ToQuads()}
	}
}

func (tl TrackLookup) IsRemote() bool {
	return tl.link.IsRemote()
}

type TrackDiff interface {
	BreaksTracking() bool
	ToDisplayString() string
	IsSameAsTracked() bool
}

type TrackDiffError struct {
	err error
}

func (t TrackDiffError) BreaksTracking() bool {
	return true
}
func (t TrackDiffError) ToDisplayString() string {
	return "error"
}
func (t TrackDiffError) IsSameAsTracked() bool {
	return false
}

type TrackDiffUpgraded struct {
	prev, curr string
}

func (t TrackDiffUpgraded) IsSameAsTracked() bool {
	return false
}

func (t TrackDiffUpgraded) BreaksTracking() bool {
	return false
}
func (t TrackDiffUpgraded) ToDisplayString() string {
	return "Upgraded from " + t.prev + " to " + t.curr
}
func (t TrackDiffUpgraded) GetPrev() string { return t.prev }
func (t TrackDiffUpgraded) GetCurr() string { return t.curr }

type TrackDiffNone struct{}

func (t TrackDiffNone) BreaksTracking() bool {
	return false
}
func (t TrackDiffNone) IsSameAsTracked() bool {
	return true
}

func (t TrackDiffNone) ToDisplayString() string {
	return "OK"
}

type TrackDiffNew struct{}

func (t TrackDiffNew) BreaksTracking() bool {
	return false
}
func (t TrackDiffNew) IsSameAsTracked() bool {
	return false
}

type TrackDiffClash struct {
	observed, expected string
}

func (t TrackDiffNew) ToDisplayString() string {
	return "new"
}

func (t TrackDiffClash) BreaksTracking() bool {
	return true
}

func (t TrackDiffClash) ToDisplayString() string {
	return "CHANGED from " + t.expected
}
func (t TrackDiffClash) IsSameAsTracked() bool {
	return false
}

type TrackDiffDeleted struct {
	idc TrackIdComponent
}

func (t TrackDiffDeleted) BreaksTracking() bool {
	return true
}
func (t TrackDiffDeleted) ToDisplayString() string {
	return "Deleted proof: " + t.idc.ToIdString()
}
func (t TrackDiffDeleted) IsSameAsTracked() bool {
	return false
}

type TrackDiffRemoteFail struct {
	observed int
}

func (t TrackDiffRemoteFail) BreaksTracking() bool {
	return true
}
func (t TrackDiffRemoteFail) ToDisplayString() string {
	return "remote failed"
}
func (t TrackDiffRemoteFail) IsSameAsTracked() bool {
	return false
}

type TrackDiffRemoteWorking struct {
	tracked int
}

func (t TrackDiffRemoteWorking) BreaksTracking() bool {
	return false
}
func (t TrackDiffRemoteWorking) ToDisplayString() string {
	return "working"
}
func (t TrackDiffRemoteWorking) IsSameAsTracked() bool {
	return false
}

type TrackDiffRemoteChanged struct {
	tracked, observed int
}

func (t TrackDiffRemoteChanged) BreaksTracking() bool {
	return false
}
func (t TrackDiffRemoteChanged) ToDisplayString() string {
	return "changed"
}
func (t TrackDiffRemoteChanged) IsSameAsTracked() bool {
	return false
}

func NewTrackLookup(link *TrackChainLink) *TrackLookup {
	sbs := link.ToServiceBlocks()
	set := make(TrackSet)
	ids := make(map[string][]string)
	for _, sb := range sbs {
		set.Add(sb)
		k, v := sb.ToKeyValuePair()
		list, found := ids[k]
		if !found {
			list = make([]string, 0, 1)
		}
		ids[k] = append(list, v)
	}
	ret := &TrackLookup{link: link, set: set, ids: ids, mutex: new(sync.Mutex)}
	return ret
}

func (l *TrackLookup) Lock() {
	l.mutex.Lock()
}

func (l *TrackLookup) Unlock() {
	l.mutex.Unlock()
}

func (e *TrackLookup) GetCTime() time.Time {
	return e.link.GetCTime()
}

//=====================================================================

type TrackEngine struct {
	TheirName    string
	Them         *User
	Me           *User
	Interactive  bool
	NoSelf       bool
	StrictProofs bool
	MeRequired   bool

	trackStatementBytes []byte
	trackStatement      *jsonw.Wrapper
	signingKey          *PgpKeyBundle
	sig                 string
	sigid               *SigId
}

func (e *TrackEngine) LoadThem() error {

	if e.Them == nil && len(e.TheirName) == 0 {
		return fmt.Errorf("No 'them' passed to TrackEngine")
	}
	if e.Them == nil {
		if u, err := LoadUser(LoadUserArg{
			Name:        e.TheirName,
			Self:        false,
			LoadSecrets: false,
			ForceReload: false,
		}); err != nil {
			return err
		} else {
			e.Them = u
		}
	}
	return nil
}

func (e *TrackEngine) LoadMe() error {
	if e.Me == nil {
		if me, err := LoadMe(LoadUserArg{LoadSecrets: true}); err != nil && e.MeRequired {
			return err
		} else {
			e.Me = me
		}
	}
	return nil
}

func (e *TrackEngine) Run() (err error) {

	if err = e.LoadThem(); err != nil {
		return
	} else if err = e.LoadMe(); err != nil {
		return
	} else if e.NoSelf && e.Me.Equal(*e.Them) {
		err = fmt.Errorf("Cannot track yourself")
	}

	var ti TrackInstructions
	ti, err = e.Them.Identify(IdentifyArg{
		Me: e.Me,
		Ui: G.UI.GetIdentifyTrackUI(e.Them, e.StrictProofs),
	})

	if err != nil {
		return
	}

	e.trackStatement, err = e.Me.TrackingProofFor(e.Them)
	if err != nil {
		return err
	}

	if e.trackStatementBytes, err = e.trackStatement.Marshal(); err != nil {
		return
	}

	G.Log.Debug("| Tracking statement: %s", string(e.trackStatementBytes))

	if ti.Remote {
		err = e.StoreRemoteTrack()
	} else if ti.Local {
		err = e.StoreLocalTrack()
	}
	return
}

func GetLocalTrack(i UID) (ret *TrackChainLink, err error) {
	uid_s := i.ToString()
	G.Log.Debug("+ GetLocalTrack(%s)", uid_s)
	defer G.Log.Debug("- GetLocalTrack(%s) -> %s", uid_s, ErrToOk(err))

	var obj *jsonw.Wrapper
	obj, err = G.LocalDb.Get(
		DbKey{Typ: DB_LOCAL_TRACK, Key: i.ToString()},
	)
	if err != nil {
		G.Log.Debug("| DB lookup failed")
		return
	}

	cl := &ChainLink{payloadJson: obj, unsigned: true}
	if err = cl.UnpackLocal(); err != nil {
		G.Log.Debug("| unpack failed -> %s", err.Error())
		return
	}
	base := GenericChainLink{cl}
	ret, err = ParseTrackChainLink(base)
	if ret != nil && err == nil {
		ret.local = true
	}

	return
}

func (e *TrackEngine) StoreLocalTrack() error {
	return StoreLocalTrack(e.Them.GetUid(), e.trackStatement)
}

func StoreLocalTrack(id UID, statement *jsonw.Wrapper) error {
	G.Log.Debug("| StoreLocalTrack")
	return G.LocalDb.Put(
		DbKey{Typ: DB_LOCAL_TRACK, Key: id.ToString()},
		nil,
		statement,
	)
}

func (e *TrackEngine) StoreRemoteTrack() (err error) {
	G.Log.Debug("+ StoreRemoteTrack")
	defer G.Log.Debug("- StoreRemoteTrack -> %s", ErrToOk(err))

	if e.signingKey, err = G.Keyrings.GetSecretKey("tracking signature"); err != nil {
		return
	} else if e.signingKey == nil {
		err = NoSecretKeyError{}
		return
	}

	if e.sig, e.sigid, err = SimpleSign(e.trackStatementBytes, *e.signingKey); err != nil {
		return
	}

	_, err = G.API.Post(ApiArg{
		Endpoint:    "follow",
		NeedSession: true,
		Args: HttpArgs{
			"sig_id_base":  S{e.sigid.ToString(false)},
			"sig_id_short": S{e.sigid.ToShortId()},
			"sig":          S{e.sig},
			"uid":          S{e.Them.GetUid().ToString()},
			"type":         S{"track"},
		},
	})

	return
}

//=====================================================================
<commit_msg>Close #61<commit_after>package libkb

import (
	"fmt"
	"github.com/keybase/go-jsonw"
	"sync"
	"time"
)

// Can either be a RemoteProofChainLink or one of the identities
// listed in a tracking statement
type TrackIdComponent interface {
	ToIdString() string
	ToKeyValuePair() (string, string)
	GetProofState() int
}

type TrackSet map[string]TrackIdComponent

func (ts TrackSet) Add(t TrackIdComponent) {
	ts[t.ToIdString()] = t
}

func (a TrackSet) GetProofState(tic TrackIdComponent) int {
	ret := PROOF_STATE_NONE
	if obj := a[tic.ToIdString()]; obj != nil {
		ret = obj.GetProofState()
	}
	return ret
}

func (A TrackSet) Subtract(B TrackSet) (out []TrackIdComponent) {
	for _, c := range A {
		if !B.HasMember(c) {
			out = append(out, c)
		}
	}
	return
}

func (a TrackSet) HasMember(t TrackIdComponent) bool {
	_, found := a[t.ToIdString()]
	return found
}

func (a TrackSet) LenEq(b TrackSet) bool {
	return len(a) == len(b)
}

//=====================================================================

type TrackInstructions struct {
	Local  bool
	Remote bool
}

//=====================================================================

type TrackLookup struct {
	link  *TrackChainLink     // The original chain link that I signed
	set   TrackSet            // The total set of tracked identities
	ids   map[string][]string // A http -> [foo.com, boo.com] lookup
	mutex *sync.Mutex         // in case we're accessing in mutliple threads
}

func (l TrackLookup) GetProofState(tic TrackIdComponent) int {
	return l.set.GetProofState(tic)
}

func (tl *TrackLookup) ComputeKeyDiff(curr PgpFingerprint) TrackDiff {
	prev, err := tl.link.GetTrackedPgpFingerprint()
	if err != nil {
		return TrackDiffError{err}
	} else if prev.Eq(curr) {
		return TrackDiffNone{}
	} else {
		return TrackDiffClash{curr.ToQuads(), prev.ToQuads()}
	}
}

func (tl TrackLookup) IsRemote() bool {
	return tl.link.IsRemote()
}

type TrackDiff interface {
	BreaksTracking() bool
	ToDisplayString() string
	IsSameAsTracked() bool
}

type TrackDiffError struct {
	err error
}

func (t TrackDiffError) BreaksTracking() bool {
	return true
}
func (t TrackDiffError) ToDisplayString() string {
	return "error"
}
func (t TrackDiffError) IsSameAsTracked() bool {
	return false
}

type TrackDiffUpgraded struct {
	prev, curr string
}

func (t TrackDiffUpgraded) IsSameAsTracked() bool {
	return false
}

func (t TrackDiffUpgraded) BreaksTracking() bool {
	return false
}
func (t TrackDiffUpgraded) ToDisplayString() string {
	return "Upgraded from " + t.prev + " to " + t.curr
}
func (t TrackDiffUpgraded) GetPrev() string { return t.prev }
func (t TrackDiffUpgraded) GetCurr() string { return t.curr }

type TrackDiffNone struct{}

func (t TrackDiffNone) BreaksTracking() bool {
	return false
}
func (t TrackDiffNone) IsSameAsTracked() bool {
	return true
}

func (t TrackDiffNone) ToDisplayString() string {
	return "tracked"
}

type TrackDiffNew struct{}

func (t TrackDiffNew) BreaksTracking() bool {
	return false
}
func (t TrackDiffNew) IsSameAsTracked() bool {
	return false
}

type TrackDiffClash struct {
	observed, expected string
}

func (t TrackDiffNew) ToDisplayString() string {
	return "new"
}

func (t TrackDiffClash) BreaksTracking() bool {
	return true
}

func (t TrackDiffClash) ToDisplayString() string {
	return "CHANGED from " + t.expected
}
func (t TrackDiffClash) IsSameAsTracked() bool {
	return false
}

type TrackDiffDeleted struct {
	idc TrackIdComponent
}

func (t TrackDiffDeleted) BreaksTracking() bool {
	return true
}
func (t TrackDiffDeleted) ToDisplayString() string {
	return "Deleted proof: " + t.idc.ToIdString()
}
func (t TrackDiffDeleted) IsSameAsTracked() bool {
	return false
}

type TrackDiffRemoteFail struct {
	observed int
}

func (t TrackDiffRemoteFail) BreaksTracking() bool {
	return true
}
func (t TrackDiffRemoteFail) ToDisplayString() string {
	return "remote failed"
}
func (t TrackDiffRemoteFail) IsSameAsTracked() bool {
	return false
}

type TrackDiffRemoteWorking struct {
	tracked int
}

func (t TrackDiffRemoteWorking) BreaksTracking() bool {
	return false
}
func (t TrackDiffRemoteWorking) ToDisplayString() string {
	return "working"
}
func (t TrackDiffRemoteWorking) IsSameAsTracked() bool {
	return false
}

type TrackDiffRemoteChanged struct {
	tracked, observed int
}

func (t TrackDiffRemoteChanged) BreaksTracking() bool {
	return false
}
func (t TrackDiffRemoteChanged) ToDisplayString() string {
	return "changed"
}
func (t TrackDiffRemoteChanged) IsSameAsTracked() bool {
	return false
}

func NewTrackLookup(link *TrackChainLink) *TrackLookup {
	sbs := link.ToServiceBlocks()
	set := make(TrackSet)
	ids := make(map[string][]string)
	for _, sb := range sbs {
		set.Add(sb)
		k, v := sb.ToKeyValuePair()
		list, found := ids[k]
		if !found {
			list = make([]string, 0, 1)
		}
		ids[k] = append(list, v)
	}
	ret := &TrackLookup{link: link, set: set, ids: ids, mutex: new(sync.Mutex)}
	return ret
}

func (l *TrackLookup) Lock() {
	l.mutex.Lock()
}

func (l *TrackLookup) Unlock() {
	l.mutex.Unlock()
}

func (e *TrackLookup) GetCTime() time.Time {
	return e.link.GetCTime()
}

//=====================================================================

type TrackEngine struct {
	TheirName    string
	Them         *User
	Me           *User
	Interactive  bool
	NoSelf       bool
	StrictProofs bool
	MeRequired   bool

	trackStatementBytes []byte
	trackStatement      *jsonw.Wrapper
	signingKey          *PgpKeyBundle
	sig                 string
	sigid               *SigId
}

func (e *TrackEngine) LoadThem() error {

	if e.Them == nil && len(e.TheirName) == 0 {
		return fmt.Errorf("No 'them' passed to TrackEngine")
	}
	if e.Them == nil {
		if u, err := LoadUser(LoadUserArg{
			Name:        e.TheirName,
			Self:        false,
			LoadSecrets: false,
			ForceReload: false,
		}); err != nil {
			return err
		} else {
			e.Them = u
		}
	}
	return nil
}

func (e *TrackEngine) LoadMe() error {
	if e.Me == nil {
		if me, err := LoadMe(LoadUserArg{LoadSecrets: true}); err != nil && e.MeRequired {
			return err
		} else {
			e.Me = me
		}
	}
	return nil
}

func (e *TrackEngine) Run() (err error) {

	if err = e.LoadThem(); err != nil {
		return
	} else if err = e.LoadMe(); err != nil {
		return
	} else if e.NoSelf && e.Me.Equal(*e.Them) {
		err = fmt.Errorf("Cannot track yourself")
	}

	var ti TrackInstructions
	ti, err = e.Them.Identify(IdentifyArg{
		Me: e.Me,
		Ui: G.UI.GetIdentifyTrackUI(e.Them, e.StrictProofs),
	})

	if err != nil {
		return
	}

	e.trackStatement, err = e.Me.TrackingProofFor(e.Them)
	if err != nil {
		return err
	}

	if e.trackStatementBytes, err = e.trackStatement.Marshal(); err != nil {
		return
	}

	G.Log.Debug("| Tracking statement: %s", string(e.trackStatementBytes))

	if ti.Remote {
		err = e.StoreRemoteTrack()
	} else if ti.Local {
		err = e.StoreLocalTrack()
	}
	return
}

func GetLocalTrack(i UID) (ret *TrackChainLink, err error) {
	uid_s := i.ToString()
	G.Log.Debug("+ GetLocalTrack(%s)", uid_s)
	defer G.Log.Debug("- GetLocalTrack(%s) -> %s", uid_s, ErrToOk(err))

	var obj *jsonw.Wrapper
	obj, err = G.LocalDb.Get(
		DbKey{Typ: DB_LOCAL_TRACK, Key: i.ToString()},
	)
	if err != nil {
		G.Log.Debug("| DB lookup failed")
		return
	}

	cl := &ChainLink{payloadJson: obj, unsigned: true}
	if err = cl.UnpackLocal(); err != nil {
		G.Log.Debug("| unpack failed -> %s", err.Error())
		return
	}
	base := GenericChainLink{cl}
	ret, err = ParseTrackChainLink(base)
	if ret != nil && err == nil {
		ret.local = true
	}

	return
}

func (e *TrackEngine) StoreLocalTrack() error {
	return StoreLocalTrack(e.Them.GetUid(), e.trackStatement)
}

func StoreLocalTrack(id UID, statement *jsonw.Wrapper) error {
	G.Log.Debug("| StoreLocalTrack")
	return G.LocalDb.Put(
		DbKey{Typ: DB_LOCAL_TRACK, Key: id.ToString()},
		nil,
		statement,
	)
}

func (e *TrackEngine) StoreRemoteTrack() (err error) {
	G.Log.Debug("+ StoreRemoteTrack")
	defer G.Log.Debug("- StoreRemoteTrack -> %s", ErrToOk(err))

	if e.signingKey, err = G.Keyrings.GetSecretKey("tracking signature"); err != nil {
		return
	} else if e.signingKey == nil {
		err = NoSecretKeyError{}
		return
	}

	if e.sig, e.sigid, err = SimpleSign(e.trackStatementBytes, *e.signingKey); err != nil {
		return
	}

	_, err = G.API.Post(ApiArg{
		Endpoint:    "follow",
		NeedSession: true,
		Args: HttpArgs{
			"sig_id_base":  S{e.sigid.ToString(false)},
			"sig_id_short": S{e.sigid.ToShortId()},
			"sig":          S{e.sig},
			"uid":          S{e.Them.GetUid().ToString()},
			"type":         S{"track"},
		},
	})

	return
}

//=====================================================================
<|endoftext|>


<commit_before>package neural

import (
	"fmt"
	"math/rand"
	"time"

	"github.com/mrfuxi/neural/mat"
)

// WeightUpdates is per Layer representation of how to adjust weights of the network
type WeightUpdates struct {
	Biases  [][]float64
	Weights [][][]float64
}

// Trainer implements calculations of weights adjustments (WeightUpdates) in the network
// It operates on a single training example to prepare fractional result
type Trainer interface {
	Process(sample TrainExample, weightUpdates *WeightUpdates)
}

// TrainerFactory build Trainers. Multiple trainers will be created at the beginning of the training.
type TrainerFactory func(network Evaluator, cost Cost) Trainer

// NewWeightUpdates creates WeightUpdates according to structure of the network (neurons in each layer)
func NewWeightUpdates(network Evaluator) WeightUpdates {
	layers := network.Layers()
	layersCount := len(layers)
	deltaBias := make([][]float64, layersCount, layersCount)
	deltaWeights := make([][][]float64, layersCount, layersCount)

	for l, layer := range layers {
		weightsRow, weightsCol, biasesCol := layer.Shapes()
		deltaBias[l] = make([]float64, biasesCol, biasesCol)
		deltaWeights[l] = make([][]float64, weightsRow, weightsRow)

		for r := range deltaWeights[l] {
			deltaWeights[l][r] = make([]float64, weightsCol, weightsCol)
		}
	}

	return WeightUpdates{
		Biases:  deltaBias,
		Weights: deltaWeights,
	}
}

// Zero sets all weights values to 0
func (w *WeightUpdates) Zero() {
	mat.ZeroMatrix(w.Biases)
	mat.ZeroVectorOfMatrixes(w.Weights)
}

type batchRange struct {
	from, to int
}

// Train executes training algorithm using provided Trainers (build with TrainerFactory)
// Training happens in randomized batches where samples are processed concurrently
func Train(network Evaluator, trainExamples []TrainExample, epochs int, miniBatchSize int, learningRate float64, cost Cost, trainerFactory TrainerFactory) {
	batchRanges := getBatchRanges(len(trainExamples), miniBatchSize)
	ready := make(chan int, miniBatchSize)

	layers := network.Layers()

	trainers := make([]Trainer, miniBatchSize, miniBatchSize)
	for i := range trainers {
		trainers[i] = trainerFactory(network, cost)
	}

	weightUpdates := make([]WeightUpdates, miniBatchSize, miniBatchSize)
	for i := range weightUpdates {
		weightUpdates[i] = NewWeightUpdates(network)
	}

	sumWeights := NewWeightUpdates(network)

	for epoch := 1; epoch <= epochs; epoch++ {
		shuffleTrainExamples(trainExamples)

		for b, batch := range batchRanges {
			t0 := time.Now()
			samples := trainExamples[batch.from:batch.to]
			for i := range samples {
				go func(i int) {
					trainers[i].Process(samples[i], &weightUpdates[i])
					ready <- i
				}(i)
			}

			sumWeights.Zero()
			batchSize := batch.to - batch.from
			processed := 0
			for i := range ready {
				weightUpdate := weightUpdates[i]
				for l := range layers {
					mat.SumMatrix(sumWeights.Weights[l], weightUpdate.Weights[l])
					mat.SumVector(sumWeights.Biases[l], weightUpdate.Biases[l])
				}
				processed++
				if processed == batchSize {
					break
				}
			}

			rate := learningRate / float64(batchSize)
			for l, layer := range layers {
				mat.MulVectorByScalar(sumWeights.Biases[l], rate)
				mat.MulMatrixByScalar(sumWeights.Weights[l], rate)
				layer.UpdateWeights(sumWeights.Weights[l], sumWeights.Biases[l])
			}

			if b%1000 == 0 {
				dt := time.Since(t0)
				fmt.Printf("%v/%v %v/%v    %v     \r", epoch, epochs, b, len(batchRanges), dt)
			}
		}
	}
}

func shuffleTrainExamples(trainExamples []TrainExample) {
	for i := range trainExamples {
		j := rand.Intn(i + 1)
		trainExamples[i], trainExamples[j] = trainExamples[j], trainExamples[i]
	}
}

func getBatchRanges(samples, miniBatchSize int) []batchRange {
	batches := samples / miniBatchSize
	if samples%miniBatchSize != 0 {
		batches++
	}

	batchRanges := make([]batchRange, batches, batches)
	for b := range batchRanges {
		min := b * miniBatchSize
		max := min + miniBatchSize
		if max > samples {
			max = samples
		}
		batchRanges[b] = batchRange{min, max}
	}

	return batchRanges
}

type backwardPropagationTrainer struct {
	network Evaluator
	layers  []Layer
	cost    Cost

	acticationPerLayer [][]float64
	potentialsPerLayer [][]float64
}

// NewBackwardPropagationTrainer builds new trainer that uses backward propagation algorithm
func NewBackwardPropagationTrainer(network Evaluator, cost Cost) Trainer {
	trainer := backwardPropagationTrainer{
		network: network,
		layers:  network.Layers(),
		cost:    cost,
	}

	layersCount := len(trainer.layers)
	trainer.acticationPerLayer = make([][]float64, layersCount+1, layersCount+1)
	trainer.potentialsPerLayer = make([][]float64, layersCount, layersCount)

	for l, layer := range trainer.layers {
		_, weightsCol, biasesCol := layer.Shapes()
		if l == 0 {
			trainer.acticationPerLayer[0] = make([]float64, weightsCol, weightsCol)
		}

		trainer.acticationPerLayer[l+1] = make([]float64, biasesCol, biasesCol)
		trainer.potentialsPerLayer[l] = make([]float64, biasesCol, biasesCol)
	}
	return &trainer
}

// Process executes backward propagation algorithm to get weight updates
func (b *backwardPropagationTrainer) Process(sample TrainExample, weightUpdates *WeightUpdates) {
	layersCount := len(b.layers)

	copy(b.acticationPerLayer[0], sample.Input)
	for l, layer := range b.layers {
		layer.Forward(b.potentialsPerLayer[l], b.acticationPerLayer[l])
		layer.Activator().Activation(b.acticationPerLayer[l+1], b.potentialsPerLayer[l])
	}

	spOut := make([]float64, len(sample.Output), len(sample.Output))
	outError := make([]float64, len(sample.Output), len(sample.Output))

	b.cost.CostDerivative(outError, b.acticationPerLayer[len(b.acticationPerLayer)-1], sample.Output)

	// Propagate output error to weights of output layer
	b.network.Layers()[layersCount-1].Activator().Derivative(spOut, b.potentialsPerLayer[len(b.potentialsPerLayer)-1])
	delta := mat.MulVectorElementWise(weightUpdates.Biases[layersCount-1], spOut, outError)

	mat.MulTransposeVector(weightUpdates.Weights[layersCount-1], delta, b.acticationPerLayer[len(b.acticationPerLayer)-2])

	for l := 2; l <= layersCount; l++ {
		lNo := layersCount - l
		potentials := b.potentialsPerLayer[len(b.potentialsPerLayer)-l]
		sp := make([]float64, len(potentials), len(potentials))
		b.network.Layers()[lNo].Activator().Derivative(sp, potentials)

		delta = mat.MulVectorElementWise(weightUpdates.Biases[lNo], b.layers[lNo+1].Backward(delta), sp)
		mat.MulTransposeVector(weightUpdates.Weights[lNo], delta, b.acticationPerLayer[len(b.acticationPerLayer)-l-1])
	}
}
<commit_msg>Preallocate data buffers used for training<commit_after>package neural

import (
	"fmt"
	"math/rand"
	"time"

	"github.com/mrfuxi/neural/mat"
)

// WeightUpdates is per Layer representation of how to adjust weights of the network
type WeightUpdates struct {
	Biases  [][]float64
	Weights [][][]float64
}

// Trainer implements calculations of weights adjustments (WeightUpdates) in the network
// It operates on a single training example to prepare fractional result
type Trainer interface {
	Process(sample TrainExample, weightUpdates *WeightUpdates)
}

// TrainerFactory build Trainers. Multiple trainers will be created at the beginning of the training.
type TrainerFactory func(network Evaluator, cost Cost) Trainer

// NewWeightUpdates creates WeightUpdates according to structure of the network (neurons in each layer)
func NewWeightUpdates(network Evaluator) WeightUpdates {
	layers := network.Layers()
	layersCount := len(layers)
	deltaBias := make([][]float64, layersCount, layersCount)
	deltaWeights := make([][][]float64, layersCount, layersCount)

	for l, layer := range layers {
		weightsRow, weightsCol, biasesCol := layer.Shapes()
		deltaBias[l] = make([]float64, biasesCol, biasesCol)
		deltaWeights[l] = make([][]float64, weightsRow, weightsRow)

		for r := range deltaWeights[l] {
			deltaWeights[l][r] = make([]float64, weightsCol, weightsCol)
		}
	}

	return WeightUpdates{
		Biases:  deltaBias,
		Weights: deltaWeights,
	}
}

// Zero sets all weights values to 0
func (w *WeightUpdates) Zero() {
	mat.ZeroMatrix(w.Biases)
	mat.ZeroVectorOfMatrixes(w.Weights)
}

type batchRange struct {
	from, to int
}

// Train executes training algorithm using provided Trainers (build with TrainerFactory)
// Training happens in randomized batches where samples are processed concurrently
func Train(network Evaluator, trainExamples []TrainExample, epochs int, miniBatchSize int, learningRate float64, cost Cost, trainerFactory TrainerFactory) {
	batchRanges := getBatchRanges(len(trainExamples), miniBatchSize)
	ready := make(chan int, miniBatchSize)

	layers := network.Layers()

	trainers := make([]Trainer, miniBatchSize, miniBatchSize)
	for i := range trainers {
		trainers[i] = trainerFactory(network, cost)
	}

	weightUpdates := make([]WeightUpdates, miniBatchSize, miniBatchSize)
	for i := range weightUpdates {
		weightUpdates[i] = NewWeightUpdates(network)
	}

	sumWeights := NewWeightUpdates(network)

	for epoch := 1; epoch <= epochs; epoch++ {
		shuffleTrainExamples(trainExamples)

		for b, batch := range batchRanges {
			t0 := time.Now()
			samples := trainExamples[batch.from:batch.to]
			for i := range samples {
				go func(i int) {
					trainers[i].Process(samples[i], &weightUpdates[i])
					ready <- i
				}(i)
			}

			sumWeights.Zero()
			batchSize := batch.to - batch.from
			processed := 0
			for i := range ready {
				weightUpdate := weightUpdates[i]
				for l := range layers {
					mat.SumMatrix(sumWeights.Weights[l], weightUpdate.Weights[l])
					mat.SumVector(sumWeights.Biases[l], weightUpdate.Biases[l])
				}
				processed++
				if processed == batchSize {
					break
				}
			}

			rate := learningRate / float64(batchSize)
			for l, layer := range layers {
				mat.MulVectorByScalar(sumWeights.Biases[l], rate)
				mat.MulMatrixByScalar(sumWeights.Weights[l], rate)
				layer.UpdateWeights(sumWeights.Weights[l], sumWeights.Biases[l])
			}

			if b%1000 == 0 {
				dt := time.Since(t0)
				fmt.Printf("%v/%v %v/%v    %v     \r", epoch, epochs, b, len(batchRanges), dt)
			}
		}
	}
}

func shuffleTrainExamples(trainExamples []TrainExample) {
	for i := range trainExamples {
		j := rand.Intn(i + 1)
		trainExamples[i], trainExamples[j] = trainExamples[j], trainExamples[i]
	}
}

func getBatchRanges(samples, miniBatchSize int) []batchRange {
	batches := samples / miniBatchSize
	if samples%miniBatchSize != 0 {
		batches++
	}

	batchRanges := make([]batchRange, batches, batches)
	for b := range batchRanges {
		min := b * miniBatchSize
		max := min + miniBatchSize
		if max > samples {
			max = samples
		}
		batchRanges[b] = batchRange{min, max}
	}

	return batchRanges
}

type backwardPropagationTrainer struct {
	network Evaluator
	layers  []Layer
	cost    Cost

	acticationPerLayer [][]float64
	potentialsPerLayer [][]float64
	outError           []float64
	sp                 [][]float64
}

// NewBackwardPropagationTrainer builds new trainer that uses backward propagation algorithm
func NewBackwardPropagationTrainer(network Evaluator, cost Cost) Trainer {
	trainer := backwardPropagationTrainer{
		network: network,
		layers:  network.Layers(),
		cost:    cost,
	}

	layersCount := len(trainer.layers)
	trainer.acticationPerLayer = make([][]float64, layersCount+1, layersCount+1)
	trainer.potentialsPerLayer = make([][]float64, layersCount, layersCount)
	trainer.sp = make([][]float64, layersCount, layersCount)

	for l, layer := range trainer.layers {
		_, weightsCol, biasesCol := layer.Shapes()
		if l == 0 {
			trainer.acticationPerLayer[0] = make([]float64, weightsCol, weightsCol)
		}
		if l == len(trainer.layers)-1 {
			trainer.outError = make([]float64, biasesCol, biasesCol)
		}

		trainer.acticationPerLayer[l+1] = make([]float64, biasesCol, biasesCol)
		trainer.potentialsPerLayer[l] = make([]float64, biasesCol, biasesCol)
		trainer.sp[l] = make([]float64, biasesCol, biasesCol)
	}
	return &trainer
}

// Process executes backward propagation algorithm to get weight updates
func (b *backwardPropagationTrainer) Process(sample TrainExample, weightUpdates *WeightUpdates) {
	layersCount := len(b.layers)

	copy(b.acticationPerLayer[0], sample.Input)
	for l, layer := range b.layers {
		layer.Forward(b.potentialsPerLayer[l], b.acticationPerLayer[l])
		layer.Activator().Activation(b.acticationPerLayer[l+1], b.potentialsPerLayer[l])
	}

	b.cost.CostDerivative(b.outError, b.acticationPerLayer[len(b.acticationPerLayer)-1], sample.Output)

	// Propagate output error to weights of output layer
	b.network.Layers()[layersCount-1].Activator().Derivative(b.sp[layersCount-1], b.potentialsPerLayer[len(b.potentialsPerLayer)-1])
	delta := mat.MulVectorElementWise(weightUpdates.Biases[layersCount-1], b.sp[layersCount-1], b.outError)

	mat.MulTransposeVector(weightUpdates.Weights[layersCount-1], delta, b.acticationPerLayer[len(b.acticationPerLayer)-2])

	for l := 2; l <= layersCount; l++ {
		lNo := layersCount - l
		potentials := b.potentialsPerLayer[len(b.potentialsPerLayer)-l]
		b.network.Layers()[lNo].Activator().Derivative(b.sp[lNo], potentials)

		delta = mat.MulVectorElementWise(weightUpdates.Biases[lNo], b.layers[lNo+1].Backward(delta), b.sp[lNo])
		mat.MulTransposeVector(weightUpdates.Weights[lNo], delta, b.acticationPerLayer[len(b.acticationPerLayer)-l-1])
	}
}
<|endoftext|>


<commit_before>// +build windows

package i18n_test

import (
	"os"
	"path/filepath"

	"github.com/XenoPhex/jibber_jabber"
	"github.com/cloudfoundry/cli/cf/configuration"
	"github.com/cloudfoundry/cli/cf/i18n"
	testconfig "github.com/cloudfoundry/cli/testhelpers/configuration"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("i18n.Init() function", func() {
	var (
		configRepo configuration.ReadWriter
	)

	BeforeEach(func() {
		i18n.Resources_path = filepath.Join("cf", "i18n", "test_fixtures")
		configRepo = testconfig.NewRepositoryWithDefaults()
	})

	Describe("When a user has a locale configuration set", func() {
		Context("creates a valid T function", func() {
			BeforeEach(func() {
				configRepo.SetLocale("en_US")
			})

			It("returns a usable T function for simple strings", func() {
				T := i18n.Init(configRepo)
				Ω(T).ShouldNot(BeNil())

				translation := T("Hello world!")
				Ω("Hello world!").Should(Equal(translation))
			})

			It("returns a usable T function for complex strings (interpolated)", func() {
				T := i18n.Init(configRepo)
				Ω(T).ShouldNot(BeNil())

				translation := T("Deleting domain {{.DomainName}} as {{.Username}}...", map[string]interface{}{"DomainName": "foo.com", "Username": "Anand"})
				Ω("Deleting domain foo.com as Anand...").Should(Equal(translation))
			})
		})
	})

	Describe("When a user does not have a locale configuration set", func() {
		BeforeEach(func() {
			//All these tests require the system language to be English
			Ω(jibber_jabber.DetectIETF()).Should(Equal("en-US"))
		})

		Context("creates a valid T function", func() {
			BeforeEach(func() {
				os.Setenv("LC_ALL", "en_US.UTF-8")
			})

			It("returns a usable T function for simple strings", func() {
				T := i18n.Init(configRepo)
				Ω(T).ShouldNot(BeNil())

				translation := T("Change user password")
				Ω("Change user password").Should(Equal(translation))
			})

			It("returns a usable T function for complex strings (interpolated)", func() {
				T := i18n.Init(configRepo)
				Ω(T).ShouldNot(BeNil())

				translation := T("Deleting domain {{.DomainName}} as {{.Username}}...", map[string]interface{}{"DomainName": "foo", "Username": "Anand"})
				Ω("Deleting domain foo as Anand...").Should(Equal(translation))
			})
		})
	})
})
<commit_msg>Import jibber-jabber from correct package<commit_after>// +build windows

package i18n_test

import (
	"os"
	"path/filepath"

	"github.com/cloudfoundry/cli/cf/configuration"
	"github.com/cloudfoundry/cli/cf/i18n"
	testconfig "github.com/cloudfoundry/cli/testhelpers/configuration"
	"github.com/pivotal-cf-experimental/jibber_jabber"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("i18n.Init() function", func() {
	var (
		configRepo configuration.ReadWriter
	)

	BeforeEach(func() {
		i18n.Resources_path = filepath.Join("cf", "i18n", "test_fixtures")
		configRepo = testconfig.NewRepositoryWithDefaults()
	})

	Describe("When a user has a locale configuration set", func() {
		Context("creates a valid T function", func() {
			BeforeEach(func() {
				configRepo.SetLocale("en_US")
			})

			It("returns a usable T function for simple strings", func() {
				T := i18n.Init(configRepo)
				Ω(T).ShouldNot(BeNil())

				translation := T("Hello world!")
				Ω("Hello world!").Should(Equal(translation))
			})

			It("returns a usable T function for complex strings (interpolated)", func() {
				T := i18n.Init(configRepo)
				Ω(T).ShouldNot(BeNil())

				translation := T("Deleting domain {{.DomainName}} as {{.Username}}...", map[string]interface{}{"DomainName": "foo.com", "Username": "Anand"})
				Ω("Deleting domain foo.com as Anand...").Should(Equal(translation))
			})
		})
	})

	Describe("When a user does not have a locale configuration set", func() {
		BeforeEach(func() {
			//All these tests require the system language to be English
			Ω(jibber_jabber.DetectIETF()).Should(Equal("en-US"))
		})

		Context("creates a valid T function", func() {
			BeforeEach(func() {
				os.Setenv("LC_ALL", "en_US.UTF-8")
			})

			It("returns a usable T function for simple strings", func() {
				T := i18n.Init(configRepo)
				Ω(T).ShouldNot(BeNil())

				translation := T("Change user password")
				Ω("Change user password").Should(Equal(translation))
			})

			It("returns a usable T function for complex strings (interpolated)", func() {
				T := i18n.Init(configRepo)
				Ω(T).ShouldNot(BeNil())

				translation := T("Deleting domain {{.DomainName}} as {{.Username}}...", map[string]interface{}{"DomainName": "foo", "Username": "Anand"})
				Ω("Deleting domain foo as Anand...").Should(Equal(translation))
			})
		})
	})
})
<|endoftext|>


<commit_before>// Copyright 2016 VMware, Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

//
// VMDK Docker driver sanity tests.
//

package main

import (
	"flag"
	"fmt"
	"strings"
	"testing"

	log "github.com/Sirupsen/logrus"
	"github.com/docker/engine-api/client"
	"github.com/docker/engine-api/types"
	"github.com/docker/engine-api/types/container"
	"github.com/docker/engine-api/types/filters"
	"github.com/docker/engine-api/types/strslice"
	"github.com/vmware/docker-volume-vsphere/vmdk_plugin/utils"
	"github.com/vmware/docker-volume-vsphere/vmdk_plugin/utils/config"
	"github.com/vmware/docker-volume-vsphere/vmdk_plugin/utils/refcount"
	"golang.org/x/net/context"
)

const (
	defaultMountLocation = "/mnt/testvol"
	// tests are often run under regular account and have no access to /var/log
	defaultTestLogPath = "/tmp/test-docker-volume-vsphere.log"
	// Number of volumes per client for parallel tests
)

var (
	// flag vars - see init() for help
	endPoint1        string
	endPoint2        string
	volumeName       string
	driverName       string
	defaultHeaders   map[string]string
	removeContainers bool
	parallelVolumes  int
	parallelClones   int
)

type testClient struct {
	endPoint string
	client   *client.Client
}

// prepares the environment. Kind of "main"-ish code for tests.
// Parses flags and inits logs and mount ref counters (the latter waits on Docker
// actually replying). As any other init(), it is called somewhere during init phase
// so do not expect ALL inits from other tests (if any) to compete by now.
func init() {
	logLevel := flag.String("log_level", "debug", "Logging Level")
	logFile := flag.String("log_file", config.DefaultLogPath, "Log file path")
	configFile := flag.String("config", config.DefaultConfigPath, "Configuration file path")

	flag.BoolVar(&removeContainers, "rm", true, "rm container after run")
	flag.StringVar(&driverName, "d", "vsphere", "Driver name. We refcount volumes on this driver")
	flag.IntVar(&parallelVolumes, "parallel_volumes", 5, "Volumes per docker daemon for create/delete concurrent tests")
	flag.IntVar(&parallelClones, "parallel_clones", 3, "Volumes per docker daemon for clone concurrent tests")
	flag.Parse()
	usingConfigFileDefaults := logInit(logLevel, logFile, configFile)

	defaultHeaders = map[string]string{"User-Agent": "engine-api-client-1.0"}

	log.WithFields(log.Fields{
		"driver":                   driverName,
		"log_level":                *logLevel,
		"log_file":                 *logFile,
		"conf_file":                *configFile,
		"using_conf_file_defaults": usingConfigFileDefaults,
	}).Info("VMDK plugin tests started ")
}

// returns in-container mount point for a volume
func getMountpoint(vol string) string {
	return defaultMountLocation + "/" + vol
}

// runs a command in a container , with volume mounted
// returns completion code.
// exits (t.Fatal() or create/start/wait errors
func runContainerCmd(t *testing.T, client *client.Client, volumeName string,
	image string, cmd *strslice.StrSlice, addr string) int {

	mountPoint := getMountpoint(volumeName)
	bind := volumeName + ":" + mountPoint
	t.Logf("Running cmd=%v with vol=%s on client %s", cmd, volumeName, addr)

	r, err := client.ContainerCreate(context.Background(),
		&container.Config{Image: image, Cmd: *cmd,
			Volumes: map[string]struct{}{mountPoint: {}}},
		&container.HostConfig{Binds: []string{bind}}, nil, "")
	if err != nil {
		t.Fatalf("\tContainer create failed: %v", err)
	}

	err = client.ContainerStart(context.Background(), r.ID,
		types.ContainerStartOptions{})
	if err != nil {
		t.Fatalf("\tContainer start failed: id=%s, err %v", r.ID, err)
	}

	code, err := client.ContainerWait(context.Background(), r.ID)
	if err != nil {
		t.Fatalf("\tContainer wait failed: id=%s, err %v", r.ID, err)
	}

	if removeContainers == false {
		t.Logf("\tSkipping container removal, id=%s (removeContainers == false)",
			r.ID)
		return code
	}

	err = client.ContainerRemove(context.Background(), r.ID,
		types.ContainerRemoveOptions{
			RemoveVolumes: true,
			Force:         true,
		})
	if err != nil {
		t.Fatalf("\nContainer removal failed: %v", err)
	}

	return code
}

// Checks that we can touch a file in one container and then stat it
// in another container, using the same (vmdk-based) volume
//
// goes over 'cases' and runs commands, then checks expected return code
func checkTouch(t *testing.T, c *client.Client, vol string,
	file string, addr string) {

	cases := []struct {
		image    string             // Container image to use
		cmd      *strslice.StrSlice // Command to run under busybox
		expected int                // expected results
	}{
		{"busybox", &strslice.StrSlice{"touch", getMountpoint(vol) + "/" + file}, 0},
		{"busybox", &strslice.StrSlice{"stat", getMountpoint(vol) + "/" + file}, 0},
	}

	for _, i := range cases {
		code := runContainerCmd(t, c, vol, i.image, i.cmd, addr)
		if code != i.expected {
			t.Errorf("Expected  %d, got %d (cmd: %v)", i.expected, code, i.cmd)
		}
	}
}

// returns nil for NOT_FOUND and  if volume exists
// still fails the test if driver for this volume is not vmdk
func volumeVmdkExists(t *testing.T, c *client.Client, vol string) *types.Volume {
	reply, err := c.VolumeList(context.Background(), filters.Args{})
	if err != nil {
		t.Fatalf("Failed to enumerate  volumes: %v", err)
	}

	for _, v := range reply.Volumes {
		//	t.Log(v.Name, v.Driver, v.Mountpoint)
		// "docker volume ls" return volume with format vol_name@datastore
		// variable "vol" is specified with short notation (just the volume name,
		// not including the @datastore part)
		if strings.Contains(v.Name, vol) {
			return v
		}
	}
	return nil
}

// Initialize the clients and connect with the docker daemons
func getClients(t *testing.T) []testClient {

	clients := []testClient{
		{TestInputParamsUtil.GetEndPoint1(), new(client.Client)},
		{TestInputParamsUtil.GetEndPoint2(), new(client.Client)},
	}

	for idx, elem := range clients {
		c, err := client.NewClient(elem.endPoint, refcount.ApiVersion, nil, defaultHeaders)
		if err != nil {
			t.Fatalf("Failed to connect to %s, err: %v", elem.endPoint, err)
		}
		t.Logf("Successfully connected to %s", elem.endPoint)
		clients[idx].client = c
	}
	return clients
}

// Sanity test for VMDK volumes
// - check we can attach/detach correct volume (we use 'touch' and 'stat' to validate
// - check volumes are correctly created and deleted.
// - check we see it properly from another docker VM (-H2 flag)
func TestSanity(t *testing.T) {

	fmt.Printf("Running tests on  %s (may take a while)...\n", TestInputParamsUtil.GetEndPoint1())

	clients := getClients(t)
	c := clients[0].client // this is the endpoint we use as master
	volumeName := TestInputParamsUtil.GetVolumeName()
	t.Logf("Creating vol=%s on client %s.", volumeName, clients[0].endPoint)
	_, err := c.VolumeCreate(context.Background(),
		types.VolumeCreateRequest{
			Name:   volumeName,
			Driver: driverName,
			DriverOpts: map[string]string{
				"size": "1gb",
			},
		})
	if err != nil {
		t.Fatal(err)
	}

	checkTouch(t, c, volumeName, "file_to_touch", clients[0].endPoint)

	for _, elem := range clients {
		v := volumeVmdkExists(t, elem.client, volumeName)
		if v == nil {
			t.Fatalf("Volume=%s is missing on %s after create",
				volumeName, elem.endPoint)
		}
		if v.Driver != driverName {
			t.Fatalf("wrong driver (%s) for volume %s", v.Driver, v.Name)
		}
	}

	err = c.VolumeRemove(context.Background(), volumeName)
	if err != nil {
		t.Fatalf("Failed to delete volume, err: %v", err)
	}

	for _, elem := range clients {
		if volumeVmdkExists(t, elem.client, volumeName) != nil {
			t.Errorf("Volume=%s is still present on %s after removal",
				volumeName, elem.endPoint)
		}
	}
}

// Test concurrent volume operations
// - concurrent create/delete between different docker hosts
// - concurrent create/delete on the same docker host
// - concurrent clone/delete between different docker hosts
func TestConcurrency(t *testing.T) {

	clients := getClients(t)
	volumeName := "volTestP"

	fmt.Printf("Running concurrent tests on %s and %s (may take a while)...\n",
		clients[0].endPoint, clients[1].endPoint)

	// Buffered channel to read results from
	results := make(chan error, parallelVolumes)

	createRequest := types.VolumeCreateRequest{
		Name:   volumeName,
		Driver: driverName,
		DriverOpts: map[string]string{
			"size": "1gb",
		},
	}

	// Only run this if testing against different clients
	if clients[0].endPoint != clients[1].endPoint {
		fmt.Printf("Running create/delete concurrent test...\n")
		// Create/delete goroutine
		for idx, elem := range clients {
			go func(idx int, c *client.Client) {
				for i := 0; i < parallelVolumes; i++ {
					volName := fmt.Sprintf("%s%d%d", volumeName, idx, i)
					createRequest.Name = volName
					_, err := c.VolumeCreate(context.Background(), createRequest)
					results <- err
					err = c.VolumeRemove(context.Background(), volName)
					results <- err
				}
			}(idx, elem.client)
		}
		// Read the results from the channel
		for i := 0; i < len(clients)*parallelVolumes*2; i++ {
			err := <-results
			if err != nil {
				t.Errorf("Create/delete concurrent test failed, err: %v", err)
			}
		}
	} else {
		fmt.Printf("Skipping create/delete concurrent test, same docker host. Will be tested next.\n")
	}

	fmt.Printf("Running same docker host concurrent create/delete test on %s...\n", clients[0].endPoint)
	parallelVolumes1 := parallelVolumes / 2
	for idx := 0; idx < 3; idx++ {
		go func(idx int, c *client.Client) {
			for i := 0; i < parallelVolumes1; i++ {
				volName := fmt.Sprintf("%s-same%d%d", volumeName, idx, i)
				createRequest.Name = volName
				_, err := c.VolumeCreate(context.Background(), createRequest)
				results <- err
				err = c.VolumeRemove(context.Background(), volName)
				results <- err
			}
		}(idx, clients[0].client)
	}
	// Read the results from the channel
	for i := 0; i < 3*parallelVolumes1*2; i++ {
		err := <-results
		if err != nil {
			t.Errorf("Same docker host concurrent create/delete test failed, err: %v", err)
		}
	}

	fmt.Printf("Running clone concurrent test...\n")
	masterVolName := volumeName + "Clone"

	// Create master volume for cloning
	createRequest.Name = masterVolName
	createRequest.DriverOpts["size"] = "100mb"

	_, err := clients[0].client.VolumeCreate(context.Background(), createRequest)
	if err != nil {
		t.Fatal(err)
	}

	// Remove size option for clone
	delete(createRequest.DriverOpts, "size")

	// Change diskformat for the 1st clones to produce waits
	createRequest.DriverOpts["diskformat"] = "eagerzeroedthick"

	// Clone goroutine
	for idx, elem := range clients {
		go func(idx int, c *client.Client) {
			for i := 0; i < parallelClones; i++ {
				volName := fmt.Sprintf("%s-clone%d%d", masterVolName, idx, i)
				createRequest.Name = volName
				createRequest.DriverOpts["clone-from"] = masterVolName

				// After 1st clone use thin diskformat
				if i > 0 {
					createRequest.DriverOpts["diskformat"] = "thin"
				}
				_, err := c.VolumeCreate(context.Background(), createRequest)
				results <- err
				err = c.VolumeRemove(context.Background(), volName)
				results <- err
			}
		}(idx, elem.client)
	}

	// Read the results from the channel
	for i := 0; i < len(clients)*parallelClones*2; i++ {
		err := <-results
		if err != nil {
			t.Errorf("Running clone concurrent test failed, err: %v", err)
		}
	}

	// Remove the master volume
	clients[0].client.VolumeRemove(context.Background(), masterVolName)
	if err != nil {
		t.Fatal(err)
	}
}
<commit_msg>Disabling concurrency test on same host (#1066)<commit_after>// Copyright 2016 VMware, Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

//
// VMDK Docker driver sanity tests.
//

package main

import (
	"flag"
	"fmt"
	"strings"
	"testing"

	log "github.com/Sirupsen/logrus"
	"github.com/docker/engine-api/client"
	"github.com/docker/engine-api/types"
	"github.com/docker/engine-api/types/container"
	"github.com/docker/engine-api/types/filters"
	"github.com/docker/engine-api/types/strslice"
	"github.com/vmware/docker-volume-vsphere/vmdk_plugin/utils"
	"github.com/vmware/docker-volume-vsphere/vmdk_plugin/utils/config"
	"github.com/vmware/docker-volume-vsphere/vmdk_plugin/utils/refcount"
	"golang.org/x/net/context"
)

const (
	defaultMountLocation = "/mnt/testvol"
	// tests are often run under regular account and have no access to /var/log
	defaultTestLogPath = "/tmp/test-docker-volume-vsphere.log"
	// Number of volumes per client for parallel tests
)

var (
	// flag vars - see init() for help
	endPoint1        string
	endPoint2        string
	volumeName       string
	driverName       string
	defaultHeaders   map[string]string
	removeContainers bool
	parallelVolumes  int
	parallelClones   int
)

type testClient struct {
	endPoint string
	client   *client.Client
}

// prepares the environment. Kind of "main"-ish code for tests.
// Parses flags and inits logs and mount ref counters (the latter waits on Docker
// actually replying). As any other init(), it is called somewhere during init phase
// so do not expect ALL inits from other tests (if any) to compete by now.
func init() {
	logLevel := flag.String("log_level", "debug", "Logging Level")
	logFile := flag.String("log_file", config.DefaultLogPath, "Log file path")
	configFile := flag.String("config", config.DefaultConfigPath, "Configuration file path")

	flag.BoolVar(&removeContainers, "rm", true, "rm container after run")
	flag.StringVar(&driverName, "d", "vsphere", "Driver name. We refcount volumes on this driver")
	flag.IntVar(&parallelVolumes, "parallel_volumes", 5, "Volumes per docker daemon for create/delete concurrent tests")
	flag.IntVar(&parallelClones, "parallel_clones", 3, "Volumes per docker daemon for clone concurrent tests")
	flag.Parse()
	usingConfigFileDefaults := logInit(logLevel, logFile, configFile)

	defaultHeaders = map[string]string{"User-Agent": "engine-api-client-1.0"}

	log.WithFields(log.Fields{
		"driver":                   driverName,
		"log_level":                *logLevel,
		"log_file":                 *logFile,
		"conf_file":                *configFile,
		"using_conf_file_defaults": usingConfigFileDefaults,
	}).Info("VMDK plugin tests started ")
}

// returns in-container mount point for a volume
func getMountpoint(vol string) string {
	return defaultMountLocation + "/" + vol
}

// runs a command in a container , with volume mounted
// returns completion code.
// exits (t.Fatal() or create/start/wait errors
func runContainerCmd(t *testing.T, client *client.Client, volumeName string,
	image string, cmd *strslice.StrSlice, addr string) int {

	mountPoint := getMountpoint(volumeName)
	bind := volumeName + ":" + mountPoint
	t.Logf("Running cmd=%v with vol=%s on client %s", cmd, volumeName, addr)

	r, err := client.ContainerCreate(context.Background(),
		&container.Config{Image: image, Cmd: *cmd,
			Volumes: map[string]struct{}{mountPoint: {}}},
		&container.HostConfig{Binds: []string{bind}}, nil, "")
	if err != nil {
		t.Fatalf("\tContainer create failed: %v", err)
	}

	err = client.ContainerStart(context.Background(), r.ID,
		types.ContainerStartOptions{})
	if err != nil {
		t.Fatalf("\tContainer start failed: id=%s, err %v", r.ID, err)
	}

	code, err := client.ContainerWait(context.Background(), r.ID)
	if err != nil {
		t.Fatalf("\tContainer wait failed: id=%s, err %v", r.ID, err)
	}

	if removeContainers == false {
		t.Logf("\tSkipping container removal, id=%s (removeContainers == false)",
			r.ID)
		return code
	}

	err = client.ContainerRemove(context.Background(), r.ID,
		types.ContainerRemoveOptions{
			RemoveVolumes: true,
			Force:         true,
		})
	if err != nil {
		t.Fatalf("\nContainer removal failed: %v", err)
	}

	return code
}

// Checks that we can touch a file in one container and then stat it
// in another container, using the same (vmdk-based) volume
//
// goes over 'cases' and runs commands, then checks expected return code
func checkTouch(t *testing.T, c *client.Client, vol string,
	file string, addr string) {

	cases := []struct {
		image    string             // Container image to use
		cmd      *strslice.StrSlice // Command to run under busybox
		expected int                // expected results
	}{
		{"busybox", &strslice.StrSlice{"touch", getMountpoint(vol) + "/" + file}, 0},
		{"busybox", &strslice.StrSlice{"stat", getMountpoint(vol) + "/" + file}, 0},
	}

	for _, i := range cases {
		code := runContainerCmd(t, c, vol, i.image, i.cmd, addr)
		if code != i.expected {
			t.Errorf("Expected  %d, got %d (cmd: %v)", i.expected, code, i.cmd)
		}
	}
}

// returns nil for NOT_FOUND and  if volume exists
// still fails the test if driver for this volume is not vmdk
func volumeVmdkExists(t *testing.T, c *client.Client, vol string) *types.Volume {
	reply, err := c.VolumeList(context.Background(), filters.Args{})
	if err != nil {
		t.Fatalf("Failed to enumerate  volumes: %v", err)
	}

	for _, v := range reply.Volumes {
		//	t.Log(v.Name, v.Driver, v.Mountpoint)
		// "docker volume ls" return volume with format vol_name@datastore
		// variable "vol" is specified with short notation (just the volume name,
		// not including the @datastore part)
		if strings.Contains(v.Name, vol) {
			return v
		}
	}
	return nil
}

// Initialize the clients and connect with the docker daemons
func getClients(t *testing.T) []testClient {

	clients := []testClient{
		{TestInputParamsUtil.GetEndPoint1(), new(client.Client)},
		{TestInputParamsUtil.GetEndPoint2(), new(client.Client)},
	}

	for idx, elem := range clients {
		c, err := client.NewClient(elem.endPoint, refcount.ApiVersion, nil, defaultHeaders)
		if err != nil {
			t.Fatalf("Failed to connect to %s, err: %v", elem.endPoint, err)
		}
		t.Logf("Successfully connected to %s", elem.endPoint)
		clients[idx].client = c
	}
	return clients
}

// Sanity test for VMDK volumes
// - check we can attach/detach correct volume (we use 'touch' and 'stat' to validate
// - check volumes are correctly created and deleted.
// - check we see it properly from another docker VM (-H2 flag)
func TestSanity(t *testing.T) {

	fmt.Printf("Running tests on  %s (may take a while)...\n", TestInputParamsUtil.GetEndPoint1())

	clients := getClients(t)
	c := clients[0].client // this is the endpoint we use as master
	volumeName := TestInputParamsUtil.GetVolumeName()
	t.Logf("Creating vol=%s on client %s.", volumeName, clients[0].endPoint)
	_, err := c.VolumeCreate(context.Background(),
		types.VolumeCreateRequest{
			Name:   volumeName,
			Driver: driverName,
			DriverOpts: map[string]string{
				"size": "1gb",
			},
		})
	if err != nil {
		t.Fatal(err)
	}

	checkTouch(t, c, volumeName, "file_to_touch", clients[0].endPoint)

	for _, elem := range clients {
		v := volumeVmdkExists(t, elem.client, volumeName)
		if v == nil {
			t.Fatalf("Volume=%s is missing on %s after create",
				volumeName, elem.endPoint)
		}
		if v.Driver != driverName {
			t.Fatalf("wrong driver (%s) for volume %s", v.Driver, v.Name)
		}
	}

	err = c.VolumeRemove(context.Background(), volumeName)
	if err != nil {
		t.Fatalf("Failed to delete volume, err: %v", err)
	}

	for _, elem := range clients {
		if volumeVmdkExists(t, elem.client, volumeName) != nil {
			t.Errorf("Volume=%s is still present on %s after removal",
				volumeName, elem.endPoint)
		}
	}
}

// Test concurrent volume operations
// - concurrent create/delete between different docker hosts
// - concurrent create/delete on the same docker host
// - concurrent clone/delete between different docker hosts
func TestConcurrency(t *testing.T) {

	clients := getClients(t)
	volumeName := "volTestP"

	fmt.Printf("Running concurrent tests on %s and %s (may take a while)...\n",
		clients[0].endPoint, clients[1].endPoint)

	// Buffered channel to read results from
	results := make(chan error, parallelVolumes)

	createRequest := types.VolumeCreateRequest{
		Name:   volumeName,
		Driver: driverName,
		DriverOpts: map[string]string{
			"size": "1gb",
		},
	}

	// Only run this if testing against different clients
	if clients[0].endPoint != clients[1].endPoint {
		fmt.Printf("Running create/delete concurrent test...\n")
		// Create/delete goroutine
		for idx, elem := range clients {
			go func(idx int, c *client.Client) {
				for i := 0; i < parallelVolumes; i++ {
					volName := fmt.Sprintf("%s%d%d", volumeName, idx, i)
					createRequest.Name = volName
					_, err := c.VolumeCreate(context.Background(), createRequest)
					results <- err
					err = c.VolumeRemove(context.Background(), volName)
					results <- err
				}
			}(idx, elem.client)
		}
		// Read the results from the channel
		for i := 0; i < len(clients)*parallelVolumes*2; i++ {
			err := <-results
			if err != nil {
				t.Errorf("Create/delete concurrent test failed, err: %v", err)
			}
		}
	} else {
		fmt.Printf("Skipping create/delete concurrent test, same docker host. Will be tested next.\n")
	}

	// TODO: Temporarily disable test until #1062 is root caused & fixed.
	// fmt.Printf("Running same docker host concurrent create/delete test on %s...\n", clients[0].endPoint)
	// parallelVolumes1 := parallelVolumes / 2
	// for idx := 0; idx < 3; idx++ {
	// 	go func(idx int, c *client.Client) {
	// 		for i := 0; i < parallelVolumes1; i++ {
	// 			volName := fmt.Sprintf("%s-same%d%d", volumeName, idx, i)
	// 			createRequest.Name = volName
	// 			_, err := c.VolumeCreate(context.Background(), createRequest)
	// 			results <- err
	// 			err = c.VolumeRemove(context.Background(), volName)
	// 			results <- err
	// 		}
	// 	}(idx, clients[0].client)
	// }
	// Read the results from the channel
	// for i := 0; i < 3*parallelVolumes1*2; i++ {
	// 	err := <-results
	// 	if err != nil {
	// 		t.Errorf("Same docker host concurrent create/delete test failed, err: %v", err)
	// 	}
	// }

	fmt.Printf("Running clone concurrent test...\n")
	masterVolName := volumeName + "Clone"

	// Create master volume for cloning
	createRequest.Name = masterVolName
	createRequest.DriverOpts["size"] = "100mb"

	_, err := clients[0].client.VolumeCreate(context.Background(), createRequest)
	if err != nil {
		t.Fatal(err)
	}

	// Remove size option for clone
	delete(createRequest.DriverOpts, "size")

	// Change diskformat for the 1st clones to produce waits
	createRequest.DriverOpts["diskformat"] = "eagerzeroedthick"

	// Clone goroutine
	for idx, elem := range clients {
		go func(idx int, c *client.Client) {
			for i := 0; i < parallelClones; i++ {
				volName := fmt.Sprintf("%s-clone%d%d", masterVolName, idx, i)
				createRequest.Name = volName
				createRequest.DriverOpts["clone-from"] = masterVolName

				// After 1st clone use thin diskformat
				if i > 0 {
					createRequest.DriverOpts["diskformat"] = "thin"
				}
				_, err := c.VolumeCreate(context.Background(), createRequest)
				results <- err
				err = c.VolumeRemove(context.Background(), volName)
				results <- err
			}
		}(idx, elem.client)
	}

	// Read the results from the channel
	for i := 0; i < len(clients)*parallelClones*2; i++ {
		err := <-results
		if err != nil {
			t.Errorf("Running clone concurrent test failed, err: %v", err)
		}
	}

	// Remove the master volume
	clients[0].client.VolumeRemove(context.Background(), masterVolName)
	if err != nil {
		t.Fatal(err)
	}
}
<|endoftext|>


<commit_before>package metric

import (
	"crypto/sha1"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"sort"
	"strconv"
	"strings"

	"edgeg.io/gtm/env"
	"edgeg.io/gtm/epoch"
	"edgeg.io/gtm/event"
	"edgeg.io/gtm/scm"
	"github.com/dickeyxxx/golock"
)

func Process(dryRun bool) error {
	_, gtmPath, err := env.Paths()
	if err != nil {
		return err
	}

	lockFile := filepath.Join(gtmPath, "gtm.lock")
	if err := golock.Lock(lockFile); err != nil {
		return err
	}
	defer func() {
		if err := golock.Unlock(lockFile); err != nil {
			log.Printf("Error releasing lock file, %s", err)
		}
	}()

	epochEventMap, err := event.Sweep(gtmPath, dryRun)
	if err != nil {
		return err
	}

	metricMap, err := loadMetrics(gtmPath)
	if err != nil {
		return err
	}

	for epoch := range epochEventMap {
		err := allocateTime(metricMap, epochEventMap[epoch])
		if err != nil {
			return err
		}
	}

	m, err := scm.GitCommitMsg()
	if err != nil {
		return err
	}
	_, _, commitFiles := scm.GitParseMessage(m)

	commitMap := map[string]metricFile{}
	if !dryRun {
		//for only files in the last commit
		for _, f := range commitFiles {
			fileID := getFileID(f)
			if _, ok := metricMap[fileID]; !ok {
				continue
			}
			commitMap[fileID] = metricMap[fileID]
		}
	}

	if err := writeNote(gtmPath, metricMap, commitMap, dryRun); err != nil {
		return err
	}
	if err := saveMetrics(gtmPath, metricMap, commitMap, dryRun); err != nil {
		return err
	}

	log.Printf("epochEventMap -> %+v", epochEventMap)
	log.Printf("metricMap -> %+v", metricMap)
	log.Printf("commitMap -> %+v", commitMap)
	log.Printf("dryRun -> %+v", dryRun)

	return nil
}

func getFileID(filePath string) string {
	return fmt.Sprintf("%x", sha1.Sum([]byte(filePath)))
}

func allocateTime(metricMap map[string]metricFile, eventMap map[string]int) error {
	total := 0
	for file := range eventMap {
		total += eventMap[file]
	}

	lastFile := ""
	timeAllocated := 0
	for file := range eventMap {
		t := int(float64(eventMap[file]) / float64(total) * float64(epoch.WindowSize))
		fileID := getFileID(file)

		var (
			mf  metricFile
			ok  bool
			err error
		)
		mf, ok = metricMap[fileID]
		if !ok {
			mf, err = newMetricFile(file, 0, true)
			if err != nil {
				return err
			}
		}
		mf.AddTime(t)

		//NOTE - Go has some gotchas when it comes to structs contained within maps
		// a copy is returned and not the reference to the struct
		// https://groups.google.com/forum/#!topic/golang-nuts/4_pabWnsMp0
		// assigning the new & updated metricFile instance to the map
		metricMap[fileID] = mf

		timeAllocated += t
		lastFile = file
	}
	// let's make sure all of the EpochWindowSize seconds are allocated
	// we put the remaining on the last list of events
	if lastFile != "" && timeAllocated < epoch.WindowSize {
		mf := metricMap[getFileID(lastFile)]
		mf.AddTime(epoch.WindowSize - timeAllocated)
	}
	return nil
}

type metricFile struct {
	Updated bool // Updated signifies if we need to save metric file
	GitFile string
	Time    int
}

func (m *metricFile) AddTime(t int) {
	m.Updated = true
	m.Time += t
}

func (m *metricFile) GitTracked() bool {
	tracked, err := scm.GitTracked(m.GitFile)
	if err != nil {
		// for ease of use, let's squash errors
		log.Printf("%s", err)
		return false
	}

	return tracked
}

func (m *metricFile) GitModified() bool {
	modified, err := scm.GitModified(m.GitFile)
	if err != nil {
		// for ease of use, let's squash errors
		log.Printf("%s", err)
		return false
	}

	return modified
}

func newMetricFile(f string, t int, updated bool) (metricFile, error) {
	return metricFile{GitFile: f, Time: t, Updated: updated}, nil
}

func loadMetrics(gtmPath string) (map[string]metricFile, error) {
	files, err := ioutil.ReadDir(gtmPath)
	if err != nil {
		return nil, err
	}

	metrics := map[string]metricFile{}
	for _, file := range files {

		if !strings.HasSuffix(file.Name(), ".metric") {
			continue
		}

		metricFilePath := filepath.Join(gtmPath, file.Name())

		metricFile, err := readMetricFile(metricFilePath)
		if err != nil {
			continue
		}
		metrics[strings.Replace(file.Name(), ".metric", "", 1)] = metricFile
	}

	return metrics, nil
}

func saveMetrics(gtmPath string, metricMap map[string]metricFile, commitMap map[string]metricFile, dryRun bool) error {
	if !dryRun {
		for fileID, mf := range metricMap {
			_, inCommit := commitMap[fileID]
			if mf.Updated && !inCommit {
				if err := writeMetricFile(gtmPath, mf); err != nil {
					return err
				}
			}
			// remove files in commit or
			// remove git tracked and not modified files not in commit
			if inCommit || (!inCommit && mf.GitTracked() && !mf.GitModified()) {
				if err := removeMetricFile(gtmPath, fileID); err != nil {
					return err
				}
			}
		}
	}
	return nil
}

func readMetricFile(filePath string) (metricFile, error) {
	log.Printf("readMetricFile -> %+v", filePath)
	b, err := ioutil.ReadFile(filePath)
	if err != nil {
		return metricFile{}, err
	}

	parts := strings.Split(string(b), ",")
	if len(parts) < 2 {
		return metricFile{}, fmt.Errorf("Unable to parse metric file %s, invalid contents", filePath)
	}

	t, err := strconv.Atoi(string(parts[1]))
	if err != nil {
		return metricFile{}, fmt.Errorf("Unable to parse metric file %s, invalid time -> %s", filePath, err)
	}

	mf, err := newMetricFile(parts[0], t, false)
	if err != nil {
		return metricFile{}, err
	}

	return mf, nil
}

func writeMetricFile(gtmPath string, mf metricFile) error {
	log.Printf("writeMetricFile -> %+v", mf)
	if err := ioutil.WriteFile(
		filepath.Join(gtmPath, fmt.Sprintf("%s.metric", getFileID(mf.GitFile))),
		[]byte(fmt.Sprintf("%s,%d", mf.GitFile, mf.Time)), 0644); err != nil {
		return err
	}

	return nil
}

func removeMetricFile(gtmPath, fileID string) error {
	log.Printf("removeMetricFile -> %+v", fileID)
	if err := os.Remove(
		filepath.Join(
			gtmPath, fmt.Sprintf("%s.metric", fileID))); err != nil {
		return err
	}

	return nil
}

type metricFilePair struct {
	Key   string
	Value metricFile
}

type metricFileList []metricFilePair

func newMetricFileList(m map[string]metricFile) metricFileList {
	mfs := make(metricFileList, len(m))
	i := 0
	for k, v := range m {
		mfs[i] = metricFilePair{k, v}
		i++
	}
	return mfs
}

func (p metricFileList) Len() int           { return len(p) }
func (p metricFileList) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
func (p metricFileList) Less(i, j int) bool { return p[i].Value.Time < p[j].Value.Time }

func writeNote(gtmPath string, metricMap map[string]metricFile, commitMap map[string]metricFile, dryRun bool) error {
	if dryRun {
		commitMap = map[string]metricFile{}
		for fileID, mf := range metricMap {
			//include modified and git tracked files in commit map
			if mf.GitTracked() && mf.GitModified() {
				commitMap[fileID] = mf
			}
		}
	}

	var (
		total int
		note  string
	)

	commitList := newMetricFileList(commitMap)
	sort.Sort(sort.Reverse(commitList))
	for _, mf := range commitList {
		total += mf.Value.Time
		note += fmt.Sprintf("%s: %d [m]\n", mf.Value.GitFile, mf.Value.Time)
	}

	metricList := newMetricFileList(metricMap)
	sort.Sort(sort.Reverse(metricList))
	for _, mf := range metricList {
		// include git tracked and not modified files not in commit
		if _, ok := commitMap[mf.Key]; !ok && mf.Value.GitTracked() && !mf.Value.GitModified() {
			total += mf.Value.Time
			note += fmt.Sprintf("%s: %d [r]\n", mf.Value.GitFile, mf.Value.Time)
		}
	}
	note = fmt.Sprintf("total: %d\n", total) + note

	if dryRun {
		fmt.Print(note)
	} else {
		err := scm.GitAddNote(note)
		if err != nil {
			return err
		}
	}

	return nil
}
<commit_msg>Unexport metricFile struct methods<commit_after>package metric

import (
	"crypto/sha1"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"sort"
	"strconv"
	"strings"

	"edgeg.io/gtm/env"
	"edgeg.io/gtm/epoch"
	"edgeg.io/gtm/event"
	"edgeg.io/gtm/scm"
	"github.com/dickeyxxx/golock"
)

func Process(dryRun bool) error {
	_, gtmPath, err := env.Paths()
	if err != nil {
		return err
	}

	lockFile := filepath.Join(gtmPath, "gtm.lock")
	if err := golock.Lock(lockFile); err != nil {
		return err
	}
	defer func() {
		if err := golock.Unlock(lockFile); err != nil {
			log.Printf("Error releasing lock file, %s", err)
		}
	}()

	epochEventMap, err := event.Sweep(gtmPath, dryRun)
	if err != nil {
		return err
	}

	metricMap, err := loadMetrics(gtmPath)
	if err != nil {
		return err
	}

	for epoch := range epochEventMap {
		err := allocateTime(metricMap, epochEventMap[epoch])
		if err != nil {
			return err
		}
	}

	m, err := scm.GitCommitMsg()
	if err != nil {
		return err
	}
	_, _, commitFiles := scm.GitParseMessage(m)

	commitMap := map[string]metricFile{}
	if !dryRun {
		//for only files in the last commit
		for _, f := range commitFiles {
			fileID := getFileID(f)
			if _, ok := metricMap[fileID]; !ok {
				continue
			}
			commitMap[fileID] = metricMap[fileID]
		}
	}

	if err := writeNote(gtmPath, metricMap, commitMap, dryRun); err != nil {
		return err
	}
	if err := saveMetrics(gtmPath, metricMap, commitMap, dryRun); err != nil {
		return err
	}

	log.Printf("epochEventMap -> %+v", epochEventMap)
	log.Printf("metricMap -> %+v", metricMap)
	log.Printf("commitMap -> %+v", commitMap)
	log.Printf("dryRun -> %+v", dryRun)

	return nil
}

func getFileID(filePath string) string {
	return fmt.Sprintf("%x", sha1.Sum([]byte(filePath)))
}

func allocateTime(metricMap map[string]metricFile, eventMap map[string]int) error {
	total := 0
	for file := range eventMap {
		total += eventMap[file]
	}

	lastFile := ""
	timeAllocated := 0
	for file := range eventMap {
		t := int(float64(eventMap[file]) / float64(total) * float64(epoch.WindowSize))
		fileID := getFileID(file)

		var (
			mf  metricFile
			ok  bool
			err error
		)
		mf, ok = metricMap[fileID]
		if !ok {
			mf, err = newMetricFile(file, 0, true)
			if err != nil {
				return err
			}
		}
		mf.addTime(t)

		//NOTE - Go has some gotchas when it comes to structs contained within maps
		// a copy is returned and not the reference to the struct
		// https://groups.google.com/forum/#!topic/golang-nuts/4_pabWnsMp0
		// assigning the new & updated metricFile instance to the map
		metricMap[fileID] = mf

		timeAllocated += t
		lastFile = file
	}
	// let's make sure all of the EpochWindowSize seconds are allocated
	// we put the remaining on the last list of events
	if lastFile != "" && timeAllocated < epoch.WindowSize {
		mf := metricMap[getFileID(lastFile)]
		mf.addTime(epoch.WindowSize - timeAllocated)
	}
	return nil
}

type metricFile struct {
	Updated bool // Updated signifies if we need to save metric file
	GitFile string
	Time    int
}

func (m *metricFile) addTime(t int) {
	m.Updated = true
	m.Time += t
}

func (m *metricFile) gitTracked() bool {
	tracked, err := scm.GitTracked(m.GitFile)
	if err != nil {
		// for ease of use, let's squash errors
		log.Printf("%s", err)
		return false
	}

	return tracked
}

func (m *metricFile) gitModified() bool {
	modified, err := scm.GitModified(m.GitFile)
	if err != nil {
		// for ease of use, let's squash errors
		log.Printf("%s", err)
		return false
	}

	return modified
}

func newMetricFile(f string, t int, updated bool) (metricFile, error) {
	return metricFile{GitFile: f, Time: t, Updated: updated}, nil
}

func loadMetrics(gtmPath string) (map[string]metricFile, error) {
	files, err := ioutil.ReadDir(gtmPath)
	if err != nil {
		return nil, err
	}

	metrics := map[string]metricFile{}
	for _, file := range files {

		if !strings.HasSuffix(file.Name(), ".metric") {
			continue
		}

		metricFilePath := filepath.Join(gtmPath, file.Name())

		metricFile, err := readMetricFile(metricFilePath)
		if err != nil {
			continue
		}
		metrics[strings.Replace(file.Name(), ".metric", "", 1)] = metricFile
	}

	return metrics, nil
}

func saveMetrics(gtmPath string, metricMap map[string]metricFile, commitMap map[string]metricFile, dryRun bool) error {
	if !dryRun {
		for fileID, mf := range metricMap {
			_, inCommit := commitMap[fileID]
			if mf.Updated && !inCommit {
				if err := writeMetricFile(gtmPath, mf); err != nil {
					return err
				}
			}
			// remove files in commit or
			// remove git tracked and not modified files not in commit
			if inCommit || (!inCommit && mf.gitTracked() && !mf.gitModified()) {
				if err := removeMetricFile(gtmPath, fileID); err != nil {
					return err
				}
			}
		}
	}
	return nil
}

func readMetricFile(filePath string) (metricFile, error) {
	log.Printf("readMetricFile -> %+v", filePath)
	b, err := ioutil.ReadFile(filePath)
	if err != nil {
		return metricFile{}, err
	}

	parts := strings.Split(string(b), ",")
	if len(parts) < 2 {
		return metricFile{}, fmt.Errorf("Unable to parse metric file %s, invalid contents", filePath)
	}

	t, err := strconv.Atoi(string(parts[1]))
	if err != nil {
		return metricFile{}, fmt.Errorf("Unable to parse metric file %s, invalid time -> %s", filePath, err)
	}

	mf, err := newMetricFile(parts[0], t, false)
	if err != nil {
		return metricFile{}, err
	}

	return mf, nil
}

func writeMetricFile(gtmPath string, mf metricFile) error {
	log.Printf("writeMetricFile -> %+v", mf)
	if err := ioutil.WriteFile(
		filepath.Join(gtmPath, fmt.Sprintf("%s.metric", getFileID(mf.GitFile))),
		[]byte(fmt.Sprintf("%s,%d", mf.GitFile, mf.Time)), 0644); err != nil {
		return err
	}

	return nil
}

func removeMetricFile(gtmPath, fileID string) error {
	log.Printf("removeMetricFile -> %+v", fileID)
	if err := os.Remove(
		filepath.Join(
			gtmPath, fmt.Sprintf("%s.metric", fileID))); err != nil {
		return err
	}

	return nil
}

type metricFilePair struct {
	Key   string
	Value metricFile
}

type metricFileList []metricFilePair

func newMetricFileList(m map[string]metricFile) metricFileList {
	mfs := make(metricFileList, len(m))
	i := 0
	for k, v := range m {
		mfs[i] = metricFilePair{k, v}
		i++
	}
	return mfs
}

func (p metricFileList) Len() int           { return len(p) }
func (p metricFileList) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
func (p metricFileList) Less(i, j int) bool { return p[i].Value.Time < p[j].Value.Time }

func writeNote(gtmPath string, metricMap map[string]metricFile, commitMap map[string]metricFile, dryRun bool) error {
	if dryRun {
		commitMap = map[string]metricFile{}
		for fileID, mf := range metricMap {
			//include modified and git tracked files in commit map
			if mf.gitTracked() && mf.gitModified() {
				commitMap[fileID] = mf
			}
		}
	}

	var (
		total int
		note  string
	)

	commitList := newMetricFileList(commitMap)
	sort.Sort(sort.Reverse(commitList))
	for _, mf := range commitList {
		total += mf.Value.Time
		note += fmt.Sprintf("%s: %d [m]\n", mf.Value.GitFile, mf.Value.Time)
	}

	metricList := newMetricFileList(metricMap)
	sort.Sort(sort.Reverse(metricList))
	for _, mf := range metricList {
		// include git tracked and not modified files not in commit
		if _, ok := commitMap[mf.Key]; !ok && mf.Value.gitTracked() && !mf.Value.gitModified() {
			total += mf.Value.Time
			note += fmt.Sprintf("%s: %d [r]\n", mf.Value.GitFile, mf.Value.Time)
		}
	}
	note = fmt.Sprintf("total: %d\n", total) + note

	if dryRun {
		fmt.Print(note)
	} else {
		err := scm.GitAddNote(note)
		if err != nil {
			return err
		}
	}

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"database/sql"
	"errors"
	"fmt"
	"log"

	"github.com/satori/go.uuid"
)

type forestCustodian struct {
	input     chan somaRepositoryRequest
	shutdown  chan bool
	conn      *sql.DB
	add_stmt  *sql.Stmt
	load_stmt *sql.Stmt
}

func (f *forestCustodian) run() {
	var err error

	f.add_stmt, err = f.conn.Prepare(`
INSERT INTO soma.repositories (
	repository_id,
	repository_name,
	repository_active,
	repository_deleted,
	organizational_team_id)
SELECT $1::uuid, $2::varchar, $3::boolean, $4::boolean, $5::uuid
WHERE NOT EXISTS (
	SELECT repository_id
	FROM   soma.repositories
	WHERE  repository_id = $1::uuid
	OR     repository_name = $2::varchar);`)
	if err != nil {
		log.Fatal("repository/add: ", err)
	}
	defer f.add_stmt.Close()

	f.load_stmt, err = f.conn.Prepare(`
SELECT repository_id,
	   repository_name,
	   repository_deleted,
	   repository_active,
	   organizational_team_id
FROM   soma.repositories;`)
	if err != nil {
		log.Fatal("repository/load: ", err)
	}
	defer f.load_stmt.Close()

	f.initialLoad()

runloop:
	for {
		select {
		case <-f.shutdown:
			break runloop
		case req := <-f.input:
			f.process(&req)
		}
	}
}

func (f *forestCustodian) process(q *somaRepositoryRequest) {
	var (
		res        sql.Result
		err        error
		sTree      *somatree.SomaTree
		actionChan chan *somatree.Action
		errChan    chan *somatree.Error
		team       string
	)
	result := somaResult{}

	switch q.action {
	case "add":
		log.Printf("R: repository/add for %s", q.Repository.Name)
		actionChan = make(chan *somatree.Action, 1024000)
		errChan = make(chan *somatree.Error, 1024000)

		id := uuid.NewV4()
		q.Repository.Id = id.String()

		sTree = somatree.New(somatree.TreeSpec{
			Id:     uuid.NewV4().String(),
			Name:   fmt.Sprintf("root_%s", q.Repository.Name),
			Action: actionChan,
		})
		somatree.NewRepository(somatree.RepositorySpec{
			Id:      q.Repository.Id,
			Name:    q.Repository.Name,
			Team:    q.Repository.TeamId,
			Deleted: false,
			Active:  q.Repository.IsActive,
		}).Attach(somatree.AttachRequest{
			Root:       sTree,
			ParentType: "root",
			ParentId:   sTree.GetID(),
		})
		sTree.SetError(errChan)

		for i := len(actionChan); i > 0; i-- {
			action := <-actionChan
			switch action.Action {
			case "create":
				if action.Type == "fault" {
					continue
				}
				if action.Type == "repository" {
					res, err = f.add_stmt.Exec(
						action.Repository.Id,
						action.Repository.Name,
						action.Repository.IsActive,
						false,
						action.Repository.TeamId,
					)
					team = action.Repository.TeamId
				}
			case "attached":
			}
		}
	default:
		log.Printf("R: unimplemented repository/%s", q.action)
		result.SetNotImplemented()
		q.reply <- result
		return
	}
	if result.SetRequestError(err) {
		q.reply <- result
		return
	}

	rowCnt, _ := res.RowsAffected()
	switch {
	case rowCnt == 0:
		result.Append(errors.New("No rows affected"), &somaRepositoryResult{})
	case rowCnt > 1:
		result.Append(fmt.Errorf("Too many rows affected: %d", rowCnt),
			&somaRepositoryResult{})
	default:
		result.Append(nil, &somaRepositoryResult{
			Repository: q.Repository,
		})
		if q.action == "add" {
			f.spawnTreeKeeper(q, sTree, errChan, actionChan, team)
		}
	}
	q.reply <- result
}

func (f *forestCustodian) initialLoad() {
	var (
		rows                     *sql.Rows
		err                      error
		repoId, repoName, teamId string
		repoActive, repoDeleted  bool
	)
	log.Printf("Loading existing repositories")
	rows, err = f.load_stmt.Query()
	if err != nil {
		log.Fatal("Error loading repositories: ", err)
	}
	defer rows.Close()

	for rows.Next() {
		err := rows.Scan(
			&repoId,
			&repoName,
			&repoDeleted,
			&repoActive,
			&teamId,
		)
		if err != nil {
			log.Printf("Error: %s", err.Error())
		}
		f.loadSomaTree(&somaRepositoryRequest{
			Repository: proto.Repository{
				Id:        repoId,
				Name:      repoName,
				TeamId:    teamId,
				IsDeleted: repoDeleted,
				IsActive:  repoActive,
			},
		})
	}
}

func (f *forestCustodian) loadSomaTree(q *somaRepositoryRequest) {
	actionChan := make(chan *somatree.Action, 1024000)
	errChan := make(chan *somatree.Error, 1024000)

	sTree := somatree.New(somatree.TreeSpec{
		Id:     uuid.NewV4().String(),
		Name:   fmt.Sprintf("root_%s", q.Repository.Name),
		Action: actionChan,
	})
	somatree.NewRepository(somatree.RepositorySpec{
		Id:      q.Repository.Id,
		Name:    q.Repository.Name,
		Team:    q.Repository.TeamId,
		Deleted: q.Repository.IsDeleted,
		Active:  q.Repository.IsActive,
	}).Attach(somatree.AttachRequest{
		Root:       sTree,
		ParentType: "root",
		ParentId:   sTree.GetID(),
	})
	sTree.SetError(errChan)
	for i := len(actionChan); i > 0; i-- {
		// discard actions on initial load
		<-actionChan
	}
	for i := len(errChan); i > 0; i-- {
		// discard actions on initial load
		<-errChan
	}
	f.spawnTreeKeeper(q, sTree, errChan, actionChan, q.Repository.TeamId)
}

func (f *forestCustodian) spawnTreeKeeper(q *somaRepositoryRequest, s *somatree.SomaTree,
	ec chan *somatree.Error, ac chan *somatree.Action, team string) {
	tK := new(treeKeeper)
	tK.input = make(chan treeRequest, 1024)
	tK.shutdown = make(chan bool)
	tK.conn = f.conn
	tK.tree = s
	tK.errChan = ec
	tK.actionChan = ac
	tK.broken = false
	tK.ready = false
	tK.frozen = false
	tK.repoId = q.Repository.Id
	tK.repoName = q.Repository.Name
	tK.team = team
	keeperName := fmt.Sprintf("repository_%s", q.Repository.Name)
	handlerMap[keeperName] = tK
	go tK.run()
}

// vim: ts=4 sw=4 sts=4 noet fenc=utf-8 ffs=unix
<commit_msg>Assert teamid is set for a new tree<commit_after>package main

import (
	"database/sql"
	"errors"
	"fmt"
	"log"

	"github.com/satori/go.uuid"
)

type forestCustodian struct {
	input     chan somaRepositoryRequest
	shutdown  chan bool
	conn      *sql.DB
	add_stmt  *sql.Stmt
	load_stmt *sql.Stmt
}

func (f *forestCustodian) run() {
	var err error

	f.add_stmt, err = f.conn.Prepare(`
INSERT INTO soma.repositories (
	repository_id,
	repository_name,
	repository_active,
	repository_deleted,
	organizational_team_id)
SELECT $1::uuid, $2::varchar, $3::boolean, $4::boolean, $5::uuid
WHERE NOT EXISTS (
	SELECT repository_id
	FROM   soma.repositories
	WHERE  repository_id = $1::uuid
	OR     repository_name = $2::varchar);`)
	if err != nil {
		log.Fatal("repository/add: ", err)
	}
	defer f.add_stmt.Close()

	f.load_stmt, err = f.conn.Prepare(`
SELECT repository_id,
	   repository_name,
	   repository_deleted,
	   repository_active,
	   organizational_team_id
FROM   soma.repositories;`)
	if err != nil {
		log.Fatal("repository/load: ", err)
	}
	defer f.load_stmt.Close()

	f.initialLoad()

runloop:
	for {
		select {
		case <-f.shutdown:
			break runloop
		case req := <-f.input:
			f.process(&req)
		}
	}
}

func (f *forestCustodian) process(q *somaRepositoryRequest) {
	var (
		res        sql.Result
		err        error
		sTree      *somatree.SomaTree
		actionChan chan *somatree.Action
		errChan    chan *somatree.Error
		team       string
	)
	result := somaResult{}

	switch q.action {
	case "add":
		log.Printf("R: repository/add for %s", q.Repository.Name)
		actionChan = make(chan *somatree.Action, 1024000)
		errChan = make(chan *somatree.Error, 1024000)

		id := uuid.NewV4()
		q.Repository.Id = id.String()

		sTree = somatree.New(somatree.TreeSpec{
			Id:     uuid.NewV4().String(),
			Name:   fmt.Sprintf("root_%s", q.Repository.Name),
			Action: actionChan,
		})
		somatree.NewRepository(somatree.RepositorySpec{
			Id:      q.Repository.Id,
			Name:    q.Repository.Name,
			Team:    q.Repository.TeamId,
			Deleted: false,
			Active:  q.Repository.IsActive,
		}).Attach(somatree.AttachRequest{
			Root:       sTree,
			ParentType: "root",
			ParentId:   sTree.GetID(),
		})
		sTree.SetError(errChan)

		for i := len(actionChan); i > 0; i-- {
			action := <-actionChan
			switch action.Action {
			case "create":
				if action.Type == "fault" {
					continue
				}
				if action.Type == "repository" {
					res, err = f.add_stmt.Exec(
						action.Repository.Id,
						action.Repository.Name,
						action.Repository.IsActive,
						false,
						action.Repository.TeamId,
					)
					team = action.Repository.TeamId
				}
			case "attached":
			default:
				log.Printf("R: Unhandled action during tree creation: %s", q.action)
				result.SetNotImplemented()
				q.reply <- result
				return
			}
		}
	default:
		log.Printf("R: unimplemented repository/%s", q.action)
		result.SetNotImplemented()
		q.reply <- result
		return
	}
	if result.SetRequestError(err) {
		q.reply <- result
		return
	}

	rowCnt, _ := res.RowsAffected()
	switch {
	case rowCnt == 0:
		result.Append(errors.New("No rows affected"), &somaRepositoryResult{})
	case rowCnt > 1:
		result.Append(fmt.Errorf("Too many rows affected: %d", rowCnt),
			&somaRepositoryResult{})
	default:
		if team == "" {
			panic(fmt.Sprintf("Team has not been set prior to spawning TreeKeeper for repo: %s", q.Repository.Name))
		}
		result.Append(nil, &somaRepositoryResult{
			Repository: q.Repository,
		})
		if q.action == "add" {
			f.spawnTreeKeeper(q, sTree, errChan, actionChan, team)
		}
	}
	q.reply <- result
}

func (f *forestCustodian) initialLoad() {
	var (
		rows                     *sql.Rows
		err                      error
		repoId, repoName, teamId string
		repoActive, repoDeleted  bool
	)
	log.Printf("Loading existing repositories")
	rows, err = f.load_stmt.Query()
	if err != nil {
		log.Fatal("Error loading repositories: ", err)
	}
	defer rows.Close()

	for rows.Next() {
		err := rows.Scan(
			&repoId,
			&repoName,
			&repoDeleted,
			&repoActive,
			&teamId,
		)
		if err != nil {
			log.Printf("Error: %s", err.Error())
		}
		f.loadSomaTree(&somaRepositoryRequest{
			Repository: proto.Repository{
				Id:        repoId,
				Name:      repoName,
				TeamId:    teamId,
				IsDeleted: repoDeleted,
				IsActive:  repoActive,
			},
		})
	}
}

func (f *forestCustodian) loadSomaTree(q *somaRepositoryRequest) {
	actionChan := make(chan *somatree.Action, 1024000)
	errChan := make(chan *somatree.Error, 1024000)

	sTree := somatree.New(somatree.TreeSpec{
		Id:     uuid.NewV4().String(),
		Name:   fmt.Sprintf("root_%s", q.Repository.Name),
		Action: actionChan,
	})
	somatree.NewRepository(somatree.RepositorySpec{
		Id:      q.Repository.Id,
		Name:    q.Repository.Name,
		Team:    q.Repository.TeamId,
		Deleted: q.Repository.IsDeleted,
		Active:  q.Repository.IsActive,
	}).Attach(somatree.AttachRequest{
		Root:       sTree,
		ParentType: "root",
		ParentId:   sTree.GetID(),
	})
	sTree.SetError(errChan)
	for i := len(actionChan); i > 0; i-- {
		// discard actions on initial load
		<-actionChan
	}
	for i := len(errChan); i > 0; i-- {
		// discard actions on initial load
		<-errChan
	}
	f.spawnTreeKeeper(q, sTree, errChan, actionChan, q.Repository.TeamId)
}

func (f *forestCustodian) spawnTreeKeeper(q *somaRepositoryRequest, s *somatree.SomaTree,
	ec chan *somatree.Error, ac chan *somatree.Action, team string) {
	tK := new(treeKeeper)
	tK.input = make(chan treeRequest, 1024)
	tK.shutdown = make(chan bool)
	tK.conn = f.conn
	tK.tree = s
	tK.errChan = ec
	tK.actionChan = ac
	tK.broken = false
	tK.ready = false
	tK.frozen = false
	tK.repoId = q.Repository.Id
	tK.repoName = q.Repository.Name
	tK.team = team
	keeperName := fmt.Sprintf("repository_%s", q.Repository.Name)
	handlerMap[keeperName] = tK
	go tK.run()
}

// vim: ts=4 sw=4 sts=4 noet fenc=utf-8 ffs=unix
<|endoftext|>