<commit_before>package Mongo

import (
	"GoRepositories/Mongo"
	"time"

	"github.com/chadit/GoShare"

	"gopkg.in/mgo.v2/bson"
)

// Tenant model
type Tenant struct {
	// (ReadOnly) Id of the document, created by system utilizing Mongo Bson Id
	ID string `json:"Id"  bson:"_id" binding:"required"`
	// (ReadOnly) Date the document was created (UTC)
	DateCreated time.Time `json:"DateCreated" bson:"DateCreated" binding:"required"`
	// (ReadOnly) Date the document was modified (UTC)
	DateModified time.Time `json:"DateModified" bson:"DateModified"`
	// (ReadOnly) What user modified the document
	UserModified string `json:"UserModified" bson:"UserModified,omitempty"`
	// (ReadOnly) Has the record been deleted
	IsDeleted bool `json:"IsDeleted" bson:"IsDeleted,omitempty"`
	// (Migration/Sync only)ExternalId is used to sync with external systems (Not used for anything internally)
	ExternalID string `json:"ExternalId" bson:"ExternalId,omitempty"`
	// (Migration/Sync only)ExternalNumber is used to sync with external systems (Not used for anything internally)
	ExternalNumber string `json:"ExternalNumber" bson:"ExternalNumber,omitempty"`
	// TenantID pertains to the tenant the document belongs to.  this is not serialized out
	TenantID string `json:"-" bson:"TenantId"`

	APIKey           string      `json:"ApiKey" bson:"ApiKey" binding:"required"`
	Name             string      `json:"Name"  bson:"Name" binding:"required"`
	FullName         string      `json:"FullName"  bson:"FullName" binding:"required"`
	SubDomain        string      `json:"SubDomain"  bson:"SubDomain" binding:"required"`
	Comments         string      `json:"Comments"  bson:"Comments"`
	DatabaseNode     string      `json:"DatabaseNode"  bson:"DatabaseNode"`
	DatabaseName     string      `json:"DatabaseName"  bson:"DatabaseName"`
	IsEnabled        bool        `json:"IsEnabled"  bson:"IsEnabled"`
	PrimaryContactID string      `json:"PrimaryContactId"  bson:"PrimaryContactId"`
	Status           string      `json:"Status"  bson:"Status"`
	CustomerCode     string      `json:"CustomerCode"  bson:"CustomerCode"`
	Address          interface{} `json:"Address"  bson:"Address"`
	PhoneNumber      interface{} `json:"PhoneNumber"  bson:"PhoneNumber"`
	DispatchSystem   string      `json:"DispatchSystem"  bson:"DispatchSystem"`
}

// NewTenant gets a new object
func NewTenant(tenantID string) *Tenant {
	newItem := new(Tenant)
	newItem.Init(tenantID)
	return newItem
}

// Init sets the defaults
func (p *Tenant) Init(tenantID string) {
	eventTime := time.Now().UTC()
	p.ID = Mongo.GetNewBsonIDString()
	if p.DateCreated.IsZero() {
		p.DateCreated = eventTime
	}
	p.DateModified = eventTime
	p.IsDeleted = false
	p.TenantID = tenantID
}

// SetupSave updates the object
func (p *Tenant) SetupSave(tenantID string) {
	eventTime := time.Now().UTC()
	if p.ID == "" {
		p.ID = bson.NewObjectId().Hex()
	}

	if p.DateCreated.IsZero() {
		p.DateCreated = eventTime
	}

	p.DateModified = eventTime
	p.TenantID = tenantID
}

// TenantConnectionInfo model
type TenantConnectionInfo struct {
	Tenant           Tenant `json:"-" bson:"-"`
	ConnectionString string `json:"-"  bson:"-"`
	DatabaseNode     string `json:"-"  bson:"-"`
	DatabaseName     string `json:"-"  bson:"-"`
	UserModified     string `json:"-" bson:"-"`
	CollectionName   string `json:"-" bson:"-"`
}

// InitBaseTenantConnectionInfo sets the defaults for a new userlogin
func (p *TenantConnectionInfo) InitBaseTenantConnectionInfo() {
	p.ConnectionString = Shares.MongoConnection
}

// Node model used to hold tenant connection information
type Node struct {
	// (ReadOnly) Id of the document, created by system utilizing Mongo Bson Id
	ID string `json:"Id"  bson:"_id" binding:"required"`
	// (ReadOnly) Date the document was created (UTC)
	DateCreated time.Time `json:"DateCreated" bson:"DateCreated" binding:"required"`
	// (ReadOnly) Date the document was modified (UTC)
	DateModified time.Time `json:"DateModified" bson:"DateModified"`
	// (ReadOnly) What user modified the document
	UserModified string `json:"UserModified" bson:"UserModified,omitempty"`
	// (ReadOnly) Has the record been deleted
	IsDeleted bool `json:"IsDeleted" bson:"IsDeleted,omitempty"`
	// (Migration/Sync only)ExternalId is used to sync with external systems (Not used for anything internally)
	ExternalID string `json:"ExternalId" bson:"ExternalId,omitempty"`
	// (Migration/Sync only)ExternalNumber is used to sync with external systems (Not used for anything internally)
	ExternalNumber string `json:"ExternalNumber" bson:"ExternalNumber,omitempty"`
	// TenantID pertains to the tenant the document belongs to.  this is not serialized out
	TenantID   string `json:"-" bson:"TenantId"`
	Name       string `json:"Name" bson:"Name" binding:"required"`
	Connection string `json:"Connection"  bson:"Connection" binding:"required"`
}

// InitNode sets the defaults for a new node
func (p *Node) InitNode() {
	eventTime := time.Now().UTC()
	p.ID = bson.NewObjectId().Hex()
	p.DateCreated = eventTime
	p.DateModified = eventTime
	p.IsDeleted = false
}

// GetBaseConnectionInformation will attempt to locate the tenant by provided informaiton
func GetBaseConnectionInformation(collectionName string) (*TenantConnectionInfo, error) {
	item := new(Tenant)
	item.ID = "global"
	item.Name = "global"
	baseConnectionInfo := new(TenantConnectionInfo)
	baseConnectionInfo.InitBaseTenantConnectionInfo()
	baseConnectionInfo.Tenant = *item
	baseConnectionInfo.CollectionName = collectionName

	return baseConnectionInfo, nil
}
<commit_msg>fixed a reference error<commit_after>package Mongo

import (
	"time"

	"github.com/chadit/GoShare"

	"gopkg.in/mgo.v2/bson"
)

// Tenant model
type Tenant struct {
	// (ReadOnly) Id of the document, created by system utilizing Mongo Bson Id
	ID string `json:"Id"  bson:"_id" binding:"required"`
	// (ReadOnly) Date the document was created (UTC)
	DateCreated time.Time `json:"DateCreated" bson:"DateCreated" binding:"required"`
	// (ReadOnly) Date the document was modified (UTC)
	DateModified time.Time `json:"DateModified" bson:"DateModified"`
	// (ReadOnly) What user modified the document
	UserModified string `json:"UserModified" bson:"UserModified,omitempty"`
	// (ReadOnly) Has the record been deleted
	IsDeleted bool `json:"IsDeleted" bson:"IsDeleted,omitempty"`
	// (Migration/Sync only)ExternalId is used to sync with external systems (Not used for anything internally)
	ExternalID string `json:"ExternalId" bson:"ExternalId,omitempty"`
	// (Migration/Sync only)ExternalNumber is used to sync with external systems (Not used for anything internally)
	ExternalNumber string `json:"ExternalNumber" bson:"ExternalNumber,omitempty"`
	// TenantID pertains to the tenant the document belongs to.  this is not serialized out
	TenantID string `json:"-" bson:"TenantId"`

	APIKey           string      `json:"ApiKey" bson:"ApiKey" binding:"required"`
	Name             string      `json:"Name"  bson:"Name" binding:"required"`
	FullName         string      `json:"FullName"  bson:"FullName" binding:"required"`
	SubDomain        string      `json:"SubDomain"  bson:"SubDomain" binding:"required"`
	Comments         string      `json:"Comments"  bson:"Comments"`
	DatabaseNode     string      `json:"DatabaseNode"  bson:"DatabaseNode"`
	DatabaseName     string      `json:"DatabaseName"  bson:"DatabaseName"`
	IsEnabled        bool        `json:"IsEnabled"  bson:"IsEnabled"`
	PrimaryContactID string      `json:"PrimaryContactId"  bson:"PrimaryContactId"`
	Status           string      `json:"Status"  bson:"Status"`
	CustomerCode     string      `json:"CustomerCode"  bson:"CustomerCode"`
	Address          interface{} `json:"Address"  bson:"Address"`
	PhoneNumber      interface{} `json:"PhoneNumber"  bson:"PhoneNumber"`
	DispatchSystem   string      `json:"DispatchSystem"  bson:"DispatchSystem"`
}

// NewTenant gets a new object
func NewTenant(tenantID string) *Tenant {
	newItem := new(Tenant)
	newItem.Init(tenantID)
	return newItem
}

// Init sets the defaults
func (p *Tenant) Init(tenantID string) {
	eventTime := time.Now().UTC()
	p.ID = GetNewBsonIDString()
	if p.DateCreated.IsZero() {
		p.DateCreated = eventTime
	}
	p.DateModified = eventTime
	p.IsDeleted = false
	p.TenantID = tenantID
}

// SetupSave updates the object
func (p *Tenant) SetupSave(tenantID string) {
	eventTime := time.Now().UTC()
	if p.ID == "" {
		p.ID = bson.NewObjectId().Hex()
	}

	if p.DateCreated.IsZero() {
		p.DateCreated = eventTime
	}

	p.DateModified = eventTime
	p.TenantID = tenantID
}

// TenantConnectionInfo model
type TenantConnectionInfo struct {
	Tenant           Tenant `json:"-" bson:"-"`
	ConnectionString string `json:"-"  bson:"-"`
	DatabaseNode     string `json:"-"  bson:"-"`
	DatabaseName     string `json:"-"  bson:"-"`
	UserModified     string `json:"-" bson:"-"`
	CollectionName   string `json:"-" bson:"-"`
}

// InitBaseTenantConnectionInfo sets the defaults for a new userlogin
func (p *TenantConnectionInfo) InitBaseTenantConnectionInfo() {
	p.ConnectionString = Shares.MongoConnection
}

// Node model used to hold tenant connection information
type Node struct {
	// (ReadOnly) Id of the document, created by system utilizing Mongo Bson Id
	ID string `json:"Id"  bson:"_id" binding:"required"`
	// (ReadOnly) Date the document was created (UTC)
	DateCreated time.Time `json:"DateCreated" bson:"DateCreated" binding:"required"`
	// (ReadOnly) Date the document was modified (UTC)
	DateModified time.Time `json:"DateModified" bson:"DateModified"`
	// (ReadOnly) What user modified the document
	UserModified string `json:"UserModified" bson:"UserModified,omitempty"`
	// (ReadOnly) Has the record been deleted
	IsDeleted bool `json:"IsDeleted" bson:"IsDeleted,omitempty"`
	// (Migration/Sync only)ExternalId is used to sync with external systems (Not used for anything internally)
	ExternalID string `json:"ExternalId" bson:"ExternalId,omitempty"`
	// (Migration/Sync only)ExternalNumber is used to sync with external systems (Not used for anything internally)
	ExternalNumber string `json:"ExternalNumber" bson:"ExternalNumber,omitempty"`
	// TenantID pertains to the tenant the document belongs to.  this is not serialized out
	TenantID   string `json:"-" bson:"TenantId"`
	Name       string `json:"Name" bson:"Name" binding:"required"`
	Connection string `json:"Connection"  bson:"Connection" binding:"required"`
}

// InitNode sets the defaults for a new node
func (p *Node) InitNode() {
	eventTime := time.Now().UTC()
	p.ID = bson.NewObjectId().Hex()
	p.DateCreated = eventTime
	p.DateModified = eventTime
	p.IsDeleted = false
}

// GetBaseConnectionInformation will attempt to locate the tenant by provided informaiton
func GetBaseConnectionInformation(collectionName string) (*TenantConnectionInfo, error) {
	item := new(Tenant)
	item.ID = "global"
	item.Name = "global"
	baseConnectionInfo := new(TenantConnectionInfo)
	baseConnectionInfo.InitBaseTenantConnectionInfo()
	baseConnectionInfo.Tenant = *item
	baseConnectionInfo.CollectionName = collectionName

	return baseConnectionInfo, nil
}
<|endoftext|>


<commit_before>package ed

func minDistance(word1, word2 string) int {
	return 0
}
<commit_msg>fix: solve 72 using dp with comments<commit_after>package ed

import "github.com/catorpilor/LeetCode/utils"

func minDistance(word1, word2 string) int {
	n1, n2 := len(word1), len(word2)
	if n1 == 0 {
		return n2
	}
	if n2 == 0 {
		return n1
	}
	// edge cases when word1 and word2 are equal
	if word1 == word2 {
		return 0
	}
	dp := make([][]int, n1+1)
	for i := range dp {
		dp[i] = make([]int, n2+1)
	}
	// init row 0
	// for example word1="", iterator word2, update dp[0][j] to j
	for j := 1; j <= n2; j++ {
		dp[0][j] = j
	}
	// init column 0
	// for example word2="", iterator word1, update dp[i][0] to i
	for i := 1; i <= n1; i++ {
		dp[i][0] = i
	}
	for i := 1; i <= n1; i++ {
		for j := 1; j <= n2; j++ {
			if word1[i-1] == word2[j-1] {
				// at current pos i, j word1[i-1] == word2[j-1]
				// no need to change
				dp[i][j] = dp[i-1][j-1]
			} else {
				//dp[i-1][j-1]+1 ----> the replace op
				// dp[i][j-1]+1, dp[i-1][j]+1  ---> delete & insert op
				// udpate dp[i][j] to the minimal of  three steps
				dp[i][j] = utils.Min(dp[i-1][j-1]+1, utils.Min(dp[i-1][j]+1, dp[i][j-1]+1))
			}
		}
	}

	return dp[n1][n2]
}
<|endoftext|>


<commit_before>package instancewriter

import (
	"archive/tar"
	"io"
	"os"
	"strings"

	"github.com/pkg/errors"

	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/idmap"
)

// InstanceTarWriter provides a TarWriter implementation that handles ID shifting and hardlink tracking.
type InstanceTarWriter struct {
	tarWriter *tar.Writer
	idmapSet  *idmap.IdmapSet
	linkMap   map[uint64]string
}

// NewInstanceTarWriter returns a ContainerTarWriter for the provided target Writer and id map.
func NewInstanceTarWriter(writer io.Writer, idmapSet *idmap.IdmapSet) *InstanceTarWriter {
	ctw := new(InstanceTarWriter)
	ctw.tarWriter = tar.NewWriter(writer)
	ctw.idmapSet = idmapSet
	ctw.linkMap = map[uint64]string{}
	return ctw
}

// ResetHardLinkMap resets the hard link map. Use when copying multiple instances (or snapshots) into a tarball.
// So that the hard link map doesn't work across different instances/snapshots.
func (ctw *InstanceTarWriter) ResetHardLinkMap() {
	ctw.linkMap = map[uint64]string{}
}

// WriteFile adds a file to the tarball with the specified name using the srcPath file as the contents of the file.
func (ctw *InstanceTarWriter) WriteFile(name string, srcPath string, fi os.FileInfo) error {
	var err error
	var major, minor uint32
	var nlink int
	var ino uint64

	link := ""
	if fi.Mode()&os.ModeSymlink == os.ModeSymlink {
		link, err = os.Readlink(srcPath)
		if err != nil {
			return errors.Wrapf(err, "Failed to resolve symlink for %q", srcPath)
		}
	}

	// Sockets cannot be stored in tarballs, just skip them (consistent with tar).
	if fi.Mode()&os.ModeSocket == os.ModeSocket {
		return nil
	}

	hdr, err := tar.FileInfoHeader(fi, link)
	if err != nil {
		return errors.Wrap(err, "Failed to create tar info header")
	}

	hdr.Name = name
	if fi.IsDir() || fi.Mode()&os.ModeSymlink == os.ModeSymlink {
		hdr.Size = 0
	} else {
		hdr.Size = fi.Size()
	}

	hdr.Uid, hdr.Gid, major, minor, ino, nlink, err = shared.GetFileStat(srcPath)
	if err != nil {
		return errors.Wrapf(err, "Failed to get file stat %q", srcPath)
	}

	// Unshift the id under rootfs/ for unpriv containers.
	if strings.HasPrefix(hdr.Name, "rootfs") && ctw.idmapSet != nil {
		hUID, hGID := ctw.idmapSet.ShiftFromNs(int64(hdr.Uid), int64(hdr.Gid))
		hdr.Uid = int(hUID)
		hdr.Gid = int(hGID)
		if hdr.Uid == -1 || hdr.Gid == -1 {
			return nil
		}
	}

	hdr.Devmajor = int64(major)
	hdr.Devminor = int64(minor)

	// If it's a hardlink we've already seen use the old name.
	if fi.Mode().IsRegular() && nlink > 1 {
		if firstPath, found := ctw.linkMap[ino]; found {
			hdr.Typeflag = tar.TypeLink
			hdr.Linkname = firstPath
			hdr.Size = 0
		} else {
			ctw.linkMap[ino] = hdr.Name
		}
	}

	// Handle xattrs (for real files only).
	if link == "" {
		hdr.Xattrs, err = shared.GetAllXattr(srcPath)
		if err != nil {
			return errors.Wrapf(err, "Failed to read xattr for %q", srcPath)
		}
	}

	err = ctw.tarWriter.WriteHeader(hdr)
	if err != nil {
		return errors.Wrap(err, "Failed to write tar header")
	}

	if hdr.Typeflag == tar.TypeReg {
		f, err := os.Open(srcPath)
		if err != nil {
			return errors.Wrapf(err, "Failed to open file %q", srcPath)
		}
		defer f.Close()

		_, err = io.Copy(ctw.tarWriter, f)
		if err != nil {
			return errors.Wrapf(err, "Failed to copy file content %q", srcPath)
		}
	}

	return nil
}

// WriteFileFromReader streams a file into the tarball using the src reader.
// A manually generated os.FileInfo should be supplied so that the tar header can be added before streaming starts.
func (ctw *InstanceTarWriter) WriteFileFromReader(src io.Reader, fi os.FileInfo) error {
	hdr, err := tar.FileInfoHeader(fi, "")
	if err != nil {
		return errors.Wrap(err, "Failed to create tar info header")
	}

	err = ctw.tarWriter.WriteHeader(hdr)
	if err != nil {
		return errors.Wrap(err, "Failed to write tar header")
	}

	_, err = io.Copy(ctw.tarWriter, src)
	return err
}

// Close finishes writing the tarball.
func (ctw *InstanceTarWriter) Close() error {
	err := ctw.tarWriter.Close()
	if err != nil {
		return errors.Wrap(err, "Failed to close tar writer")
	}
	return nil
}
<commit_msg>shared/instancewriter/instance/tar/writer: Adds ignoreGrowth arg to WriteFile<commit_after>package instancewriter

import (
	"archive/tar"
	"io"
	"os"
	"strings"

	"github.com/pkg/errors"

	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/idmap"
)

// InstanceTarWriter provides a TarWriter implementation that handles ID shifting and hardlink tracking.
type InstanceTarWriter struct {
	tarWriter *tar.Writer
	idmapSet  *idmap.IdmapSet
	linkMap   map[uint64]string
}

// NewInstanceTarWriter returns a ContainerTarWriter for the provided target Writer and id map.
func NewInstanceTarWriter(writer io.Writer, idmapSet *idmap.IdmapSet) *InstanceTarWriter {
	ctw := new(InstanceTarWriter)
	ctw.tarWriter = tar.NewWriter(writer)
	ctw.idmapSet = idmapSet
	ctw.linkMap = map[uint64]string{}
	return ctw
}

// ResetHardLinkMap resets the hard link map. Use when copying multiple instances (or snapshots) into a tarball.
// So that the hard link map doesn't work across different instances/snapshots.
func (ctw *InstanceTarWriter) ResetHardLinkMap() {
	ctw.linkMap = map[uint64]string{}
}

// WriteFile adds a file to the tarball with the specified name using the srcPath file as the contents of the file.
// The ignoreGrowth argument indicates whether to error if the srcPath file increases in size beyond the size in fi
// during the write. If false the write will return an error. If true, no error is returned, instead only the size
// specified in fi is written to the tarball. This can be used when you don't need a consistent copy of the file.
func (ctw *InstanceTarWriter) WriteFile(name string, srcPath string, fi os.FileInfo, ignoreGrowth bool) error {
	var err error
	var major, minor uint32
	var nlink int
	var ino uint64

	link := ""
	if fi.Mode()&os.ModeSymlink == os.ModeSymlink {
		link, err = os.Readlink(srcPath)
		if err != nil {
			return errors.Wrapf(err, "Failed to resolve symlink for %q", srcPath)
		}
	}

	// Sockets cannot be stored in tarballs, just skip them (consistent with tar).
	if fi.Mode()&os.ModeSocket == os.ModeSocket {
		return nil
	}

	hdr, err := tar.FileInfoHeader(fi, link)
	if err != nil {
		return errors.Wrap(err, "Failed to create tar info header")
	}

	hdr.Name = name
	if fi.IsDir() || fi.Mode()&os.ModeSymlink == os.ModeSymlink {
		hdr.Size = 0
	} else {
		hdr.Size = fi.Size()
	}

	hdr.Uid, hdr.Gid, major, minor, ino, nlink, err = shared.GetFileStat(srcPath)
	if err != nil {
		return errors.Wrapf(err, "Failed to get file stat %q", srcPath)
	}

	// Unshift the id under rootfs/ for unpriv containers.
	if strings.HasPrefix(hdr.Name, "rootfs") && ctw.idmapSet != nil {
		hUID, hGID := ctw.idmapSet.ShiftFromNs(int64(hdr.Uid), int64(hdr.Gid))
		hdr.Uid = int(hUID)
		hdr.Gid = int(hGID)
		if hdr.Uid == -1 || hdr.Gid == -1 {
			return nil
		}
	}

	hdr.Devmajor = int64(major)
	hdr.Devminor = int64(minor)

	// If it's a hardlink we've already seen use the old name.
	if fi.Mode().IsRegular() && nlink > 1 {
		if firstPath, found := ctw.linkMap[ino]; found {
			hdr.Typeflag = tar.TypeLink
			hdr.Linkname = firstPath
			hdr.Size = 0
		} else {
			ctw.linkMap[ino] = hdr.Name
		}
	}

	// Handle xattrs (for real files only).
	if link == "" {
		hdr.Xattrs, err = shared.GetAllXattr(srcPath)
		if err != nil {
			return errors.Wrapf(err, "Failed to read xattr for %q", srcPath)
		}
	}

	err = ctw.tarWriter.WriteHeader(hdr)
	if err != nil {
		return errors.Wrap(err, "Failed to write tar header")
	}

	if hdr.Typeflag == tar.TypeReg {
		f, err := os.Open(srcPath)
		if err != nil {
			return errors.Wrapf(err, "Failed to open file %q", srcPath)
		}
		defer f.Close()

		r := io.Reader(f)
		if ignoreGrowth {
			r = io.LimitReader(r, fi.Size())
		}

		_, err = io.Copy(ctw.tarWriter, r)
		if err != nil {
			return errors.Wrapf(err, "Failed to copy file content %q", srcPath)
		}
	}

	return nil
}

// WriteFileFromReader streams a file into the tarball using the src reader.
// A manually generated os.FileInfo should be supplied so that the tar header can be added before streaming starts.
func (ctw *InstanceTarWriter) WriteFileFromReader(src io.Reader, fi os.FileInfo) error {
	hdr, err := tar.FileInfoHeader(fi, "")
	if err != nil {
		return errors.Wrap(err, "Failed to create tar info header")
	}

	err = ctw.tarWriter.WriteHeader(hdr)
	if err != nil {
		return errors.Wrap(err, "Failed to write tar header")
	}

	_, err = io.Copy(ctw.tarWriter, src)
	return err
}

// Close finishes writing the tarball.
func (ctw *InstanceTarWriter) Close() error {
	err := ctw.tarWriter.Close()
	if err != nil {
		return errors.Wrap(err, "Failed to close tar writer")
	}
	return nil
}
<|endoftext|>


<commit_before>package shell

import (
	"context"
	"errors"
	"fmt"
	"io"
	"os"
	"reflect"
	"strings"
	"sync"

	"github.com/zetamatta/go-findfile"

	"github.com/zetamatta/nyagos/defined"
	"github.com/zetamatta/nyagos/nodos"
)

var WildCardExpansionAlways = false

type CommandNotFound struct {
	Name string
	Err  error
}

func (this CommandNotFound) Stringer() string {
	return fmt.Sprintf("'%s' is not recognized as an internal or external command,\noperable program or batch file", this.Name)
}

func (this CommandNotFound) Error() string {
	return this.Stringer()
}

type session struct {
	unreadline []string
}

type CloneCloser interface {
	Clone(context.Context) (context.Context, CloneCloser, error)
	Close() error
}

type Shell struct {
	*session
	Stdout       *os.File
	Stderr       *os.File
	Stdin        *os.File
	Console      io.Writer
	tag          CloneCloser
	IsBackGround bool
}

func (sh *Shell) In() io.Reader          { return sh.Stdin }
func (sh *Shell) Out() io.Writer         { return sh.Stdout }
func (sh *Shell) Err() io.Writer         { return sh.Stderr }
func (sh *Shell) Term() io.Writer        { return sh.Console }
func (sh *Shell) Tag() CloneCloser       { return sh.tag }
func (sh *Shell) SetTag(tag CloneCloser) { sh.tag = tag }

type Cmd struct {
	Shell
	args            []string
	rawArgs         []string
	fullPath        string
	UseShellExecute bool
	Closers         []io.Closer
	env             map[string]string
}

func (cmd *Cmd) Arg(n int) string      { return cmd.args[n] }
func (cmd *Cmd) Args() []string        { return cmd.args }
func (cmd *Cmd) SetArgs(s []string)    { cmd.args = s }
func (cmd *Cmd) RawArg(n int) string   { return cmd.rawArgs[n] }
func (cmd *Cmd) RawArgs() []string     { return cmd.rawArgs }
func (cmd *Cmd) SetRawArgs(s []string) { cmd.rawArgs = s }

func (cmd *Cmd) Getenv(key string) string {
	if cmd.env != nil {
		if val, ok := cmd.env[strings.ToUpper(key)]; ok {
			return val
		}
	}
	return os.Getenv(key)
}

func (cmd *Cmd) Setenv(key, val string) {
	if cmd.env == nil {
		cmd.env = make(map[string]string)
	}
	cmd.env[strings.ToUpper(key)] = val
}

func (cmd *Cmd) DumpEnv() []string {
	if cmd.env == nil {
		return nil
	}
	osEnv := os.Environ()
	result := make([]string, 0, len(cmd.env)+len(osEnv))
	for _, equation := range osEnv {
		eqIndex := strings.IndexRune(equation, '=')
		if _, ok := cmd.env[strings.ToUpper(equation[:eqIndex])]; !ok {
			result = append(result, equation)
		}
	}
	for key, val := range cmd.env {
		result = append(result, key+"="+val)
	}
	return result
}

var LookCurdirOrder = nodos.LookCurdirFirst

func (cmd *Cmd) FullPath() string {
	if cmd.args == nil || len(cmd.args) <= 0 {
		return ""
	}
	if cmd.fullPath == "" {
		cmd.fullPath = cmd.lookpath()
	}
	return cmd.fullPath
}

func (cmd *Cmd) Close() {
	if cmd.Closers != nil {
		for _, c := range cmd.Closers {
			c.Close()
		}
		cmd.Closers = nil
	}
}

func (sh *Shell) Close() {}

func New() *Shell {
	return &Shell{
		Stdin:   os.Stdin,
		Stdout:  os.Stdout,
		Stderr:  os.Stderr,
		session: &session{},
	}
}

func (sh *Shell) Command() *Cmd {
	cmd := &Cmd{
		Shell: Shell{
			Stdin:   sh.Stdin,
			Stdout:  sh.Stdout,
			Stderr:  sh.Stderr,
			Console: sh.Console,
			tag:     sh.tag,
		},
	}
	if sh.session != nil {
		cmd.session = sh.session
	} else {
		cmd.session = &session{}
	}
	return cmd
}

type ArgsHookT func(ctx context.Context, sh *Shell, args []string) ([]string, error)

var argsHook = func(ctx context.Context, sh *Shell, args []string) ([]string, error) {
	return args, nil
}

func SetArgsHook(argsHook_ ArgsHookT) (rv ArgsHookT) {
	rv, argsHook = argsHook, argsHook_
	return
}

type HookT func(context.Context, *Cmd) (int, bool, error)

var hook = func(context.Context, *Cmd) (int, bool, error) {
	return 0, false, nil
}

func SetHook(hook_ HookT) (rv HookT) {
	rv, hook = hook, hook_
	return
}

var OnCommandNotFound = func(ctx context.Context, cmd *Cmd, err error) error {
	err = &CommandNotFound{cmd.args[0], err}
	return err
}

var LastErrorLevel int

func makeCmdline(args, rawargs []string) string {
	var buffer strings.Builder
	for i, s := range args {
		if i > 0 {
			buffer.WriteByte(' ')
		}
		if (len(rawargs) > i && len(rawargs[i]) > 0 && rawargs[i][0] == '"') || strings.ContainsAny(s, " &|<>\t\"") {
			fmt.Fprintf(&buffer, `"%s"`, strings.Replace(s, `"`, `\"`, -1))
		} else {
			buffer.WriteString(s)
		}
	}
	return buffer.String()
}

var UseSourceRunBatch = true

func encloseWithQuote(fullpath string) string {
	if strings.ContainsRune(fullpath, ' ') {
		var f strings.Builder
		f.WriteByte('"')
		f.WriteString(fullpath)
		f.WriteByte('"')
		return f.String()
	} else {
		return fullpath
	}
}

func (cmd *Cmd) spawnvpSilent(ctx context.Context) (int, error) {
	for {
		// command is empty.
		if len(cmd.args) <= 0 {
			return 0, nil
		}
		eq := strings.IndexRune(cmd.args[0], '=')
		if eq <= 0 {
			break
		}
		envName := cmd.args[0][:eq]
		envNewValue := cmd.args[0][eq+1:]
		cmd.Setenv(envName, envNewValue)
		cmd.args = cmd.args[1:]
	}

	if defined.DBG {
		print("spawnvpSilent('", cmd.args[0], "')\n")
	}

	// aliases and lua-commands
	if errorlevel, done, err := hook(ctx, cmd); done || err != nil {
		return errorlevel, err
	}

	// command not found hook
	fullpath := cmd.FullPath()
	if fullpath == "" {
		return 255, OnCommandNotFound(ctx, cmd, os.ErrNotExist)
	}
	cmd.args[0] = fullpath

	if defined.DBG {
		print("exec.LookPath(", cmd.args[0], ")==", fullpath, "\n")
	}

	if WildCardExpansionAlways {
		cmd.args = findfile.Globs(cmd.args)
	}
	return cmd.startProcess(ctx)
}

func startAndWaitProcess(ctx context.Context, name string, args []string, procAttr *os.ProcAttr) (int, error) {
	if ctx != nil {
		select {
		case <-ctx.Done():
			return 252, ctx.Err()
		default:
		}
	}

	process, err := os.StartProcess(name, args, procAttr)
	if err != nil {
		return 255, err
	}

	if ctx != nil {
		done := make(chan struct{})
		go func() {
			select {
			case <-ctx.Done():
				os.Stderr.WriteString("^C\n")
				process.Kill()
			case <-done:
			}
		}()
		defer func() {
			close(done)
		}()
	}
	processState, err := process.Wait()
	return processState.ExitCode(), err
}

type AlreadyReportedError struct {
	Err error
}

func (_ AlreadyReportedError) Error() string {
	return ""
}

func IsAlreadyReported(err error) bool {
	_, ok := err.(AlreadyReportedError)
	return ok
}

func (cmd *Cmd) Spawnvp(ctx context.Context) (int, error) {
	errorlevel, err := cmd.spawnvpSilent(ctx)
	if err != nil && err != io.EOF && !IsAlreadyReported(err) {
		if defined.DBG {
			val := reflect.ValueOf(err)
			fmt.Fprintf(cmd.Stderr, "error-type=%s\n", val.Type())
		}
		fmt.Fprintln(cmd.Stderr, err.Error())
		err = AlreadyReportedError{err}
	}
	return errorlevel, err
}

func (sh *Shell) Spawnlpe(ctx context.Context, args, rawargs []string, env map[string]string) (int, error) {
	cmd := sh.Command()
	defer cmd.Close()
	cmd.SetArgs(args)
	cmd.SetRawArgs(rawargs)
	cmd.env = env
	return cmd.Spawnvp(ctx)
}

func (sh *Shell) Spawnlp(ctx context.Context, args, rawargs []string) (int, error) {

	return sh.Spawnlpe(ctx, args, rawargs, nil)
}

func (sh *Shell) Interpret(ctx context.Context, text string) (errorlevel int, finalerr error) {
	if defined.DBG {
		print("Interpret('", text, "')\n")
	}
	if sh == nil {
		return 255, errors.New("Fatal Error: Interpret: instance is nil")
	}
	errorlevel = 0
	finalerr = nil

	statements, statementsErr := Parse(text)
	if statementsErr != nil {
		if defined.DBG {
			print("Parse Error:", statementsErr.Error(), "\n")
		}
		return 0, statementsErr
	}
	if argsHook != nil {
		if defined.DBG {
			print("call argsHook\n")
		}
		for _, pipeline := range statements {
			for _, state := range pipeline {
				var err error
				state.Args, err = argsHook(ctx, sh, state.Args)
				if err != nil {
					return 255, err
				}
			}
		}
		if defined.DBG {
			print("done argsHook\n")
		}
	}
	for _, pipeline := range statements {
		for i, state := range pipeline {
			if state.Term == "|" && (i+1 >= len(pipeline) || len(pipeline[i+1].Args) <= 0) {
				return 255, errors.New("The syntax of the command is incorrect.")
			}
		}
	}

	for _, pipeline := range statements {

		var pipeIn *os.File = nil
		isBackGround := sh.IsBackGround
		for _, state := range pipeline {
			if state.Term == "&" {
				isBackGround = true
				break
			}
		}
		var wg sync.WaitGroup
		shutdown_immediately := false
		for i, state := range pipeline {
			if defined.DBG {
				print(i, ": pipeline loop(", state.Args[0], ")\n")
			}
			cmd := sh.Command()
			cmd.IsBackGround = isBackGround

			if pipeIn != nil {
				cmd.Stdin = pipeIn
				cmd.Closers = append(cmd.Closers, pipeIn)
				pipeIn = nil
			}

			var err error
			if state.Term[0] == '|' {
				var pipeOut *os.File
				pipeIn, pipeOut, err = os.Pipe()
				cmd.Stdout = pipeOut
				if state.Term == "|&" {
					cmd.Stderr = pipeOut
				}
				cmd.Closers = append(cmd.Closers, pipeOut)
			}

			for _, red := range state.Redirect {
				var closer io.Closer
				closer, err = red.OpenOn(cmd)
				if err != nil {
					return 0, err
				}
				cmd.Closers = append(cmd.Closers, closer)
			}

			cmd.args = state.Args
			cmd.rawArgs = state.RawArgs
			if i > 0 {
				cmd.IsBackGround = true
			}
			if len(pipeline) == 1 && isGui(cmd.FullPath()) {
				cmd.UseShellExecute = true
			}
			if i == len(pipeline)-1 && state.Term != "&" {
				// foreground execution.
				errorlevel, finalerr = cmd.Spawnvp(ctx)
				LastErrorLevel = errorlevel
				cmd.Close()
			} else {
				// background
				var newctx context.Context
				if isBackGround {
					// let Context not terminate background-work (#313's 2nd)
					// for the problem gvim starts with empty buffer
					// executing `git blame FILE | type | gvim - &`.
					newctx = context.Background()
				} else {
					wg.Add(1)
					newctx = ctx
				}
				if tag := cmd.Tag(); tag != nil {
					var newtag CloneCloser
					if newctx, newtag, err = tag.Clone(newctx); err != nil {
						fmt.Fprintln(os.Stderr, err.Error())
						return -1, err
					} else {
						cmd.SetTag(newtag)
					}
				}
				go func(ctx1 context.Context, cmd1 *Cmd) {
					if !isBackGround {
						defer wg.Done()
					}
					cmd1.Spawnvp(ctx1)
					if tag := cmd1.Tag(); tag != nil {
						if err := tag.Close(); err != nil {
							fmt.Fprintln(os.Stderr, err.Error())
						}
					}
					cmd1.Close()
				}(newctx, cmd)
			}
		}
		if !isBackGround {
			wg.Wait()
			if shutdown_immediately {
				return errorlevel, nil
			}
			if len(pipeline) > 0 {
				switch pipeline[len(pipeline)-1].Term {
				case "&&":
					if errorlevel != 0 {
						return errorlevel, nil
					}
				case "||":
					if errorlevel == 0 {
						return errorlevel, nil
					}
				}
			}
		}
	}
	return
}
<commit_msg>Fix: `NAME=VAR BATCHNAME ARG1..` => %1 was set BATCHNAME rather than ARG1<commit_after>package shell

import (
	"context"
	"errors"
	"fmt"
	"io"
	"os"
	"reflect"
	"strings"
	"sync"

	"github.com/zetamatta/go-findfile"

	"github.com/zetamatta/nyagos/defined"
	"github.com/zetamatta/nyagos/nodos"
)

var WildCardExpansionAlways = false

type CommandNotFound struct {
	Name string
	Err  error
}

func (this CommandNotFound) Stringer() string {
	return fmt.Sprintf("'%s' is not recognized as an internal or external command,\noperable program or batch file", this.Name)
}

func (this CommandNotFound) Error() string {
	return this.Stringer()
}

type session struct {
	unreadline []string
}

type CloneCloser interface {
	Clone(context.Context) (context.Context, CloneCloser, error)
	Close() error
}

type Shell struct {
	*session
	Stdout       *os.File
	Stderr       *os.File
	Stdin        *os.File
	Console      io.Writer
	tag          CloneCloser
	IsBackGround bool
}

func (sh *Shell) In() io.Reader          { return sh.Stdin }
func (sh *Shell) Out() io.Writer         { return sh.Stdout }
func (sh *Shell) Err() io.Writer         { return sh.Stderr }
func (sh *Shell) Term() io.Writer        { return sh.Console }
func (sh *Shell) Tag() CloneCloser       { return sh.tag }
func (sh *Shell) SetTag(tag CloneCloser) { sh.tag = tag }

type Cmd struct {
	Shell
	args            []string
	rawArgs         []string
	fullPath        string
	UseShellExecute bool
	Closers         []io.Closer
	env             map[string]string
}

func (cmd *Cmd) Arg(n int) string      { return cmd.args[n] }
func (cmd *Cmd) Args() []string        { return cmd.args }
func (cmd *Cmd) SetArgs(s []string)    { cmd.args = s }
func (cmd *Cmd) RawArg(n int) string   { return cmd.rawArgs[n] }
func (cmd *Cmd) RawArgs() []string     { return cmd.rawArgs }
func (cmd *Cmd) SetRawArgs(s []string) { cmd.rawArgs = s }

func (cmd *Cmd) Getenv(key string) string {
	if cmd.env != nil {
		if val, ok := cmd.env[strings.ToUpper(key)]; ok {
			return val
		}
	}
	return os.Getenv(key)
}

func (cmd *Cmd) Setenv(key, val string) {
	if cmd.env == nil {
		cmd.env = make(map[string]string)
	}
	cmd.env[strings.ToUpper(key)] = val
}

func (cmd *Cmd) DumpEnv() []string {
	if cmd.env == nil {
		return nil
	}
	osEnv := os.Environ()
	result := make([]string, 0, len(cmd.env)+len(osEnv))
	for _, equation := range osEnv {
		eqIndex := strings.IndexRune(equation, '=')
		if _, ok := cmd.env[strings.ToUpper(equation[:eqIndex])]; !ok {
			result = append(result, equation)
		}
	}
	for key, val := range cmd.env {
		result = append(result, key+"="+val)
	}
	return result
}

var LookCurdirOrder = nodos.LookCurdirFirst

func (cmd *Cmd) FullPath() string {
	if cmd.args == nil || len(cmd.args) <= 0 {
		return ""
	}
	if cmd.fullPath == "" {
		cmd.fullPath = cmd.lookpath()
	}
	return cmd.fullPath
}

func (cmd *Cmd) Close() {
	if cmd.Closers != nil {
		for _, c := range cmd.Closers {
			c.Close()
		}
		cmd.Closers = nil
	}
}

func (sh *Shell) Close() {}

func New() *Shell {
	return &Shell{
		Stdin:   os.Stdin,
		Stdout:  os.Stdout,
		Stderr:  os.Stderr,
		session: &session{},
	}
}

func (sh *Shell) Command() *Cmd {
	cmd := &Cmd{
		Shell: Shell{
			Stdin:   sh.Stdin,
			Stdout:  sh.Stdout,
			Stderr:  sh.Stderr,
			Console: sh.Console,
			tag:     sh.tag,
		},
	}
	if sh.session != nil {
		cmd.session = sh.session
	} else {
		cmd.session = &session{}
	}
	return cmd
}

type ArgsHookT func(ctx context.Context, sh *Shell, args []string) ([]string, error)

var argsHook = func(ctx context.Context, sh *Shell, args []string) ([]string, error) {
	return args, nil
}

func SetArgsHook(argsHook_ ArgsHookT) (rv ArgsHookT) {
	rv, argsHook = argsHook, argsHook_
	return
}

type HookT func(context.Context, *Cmd) (int, bool, error)

var hook = func(context.Context, *Cmd) (int, bool, error) {
	return 0, false, nil
}

func SetHook(hook_ HookT) (rv HookT) {
	rv, hook = hook, hook_
	return
}

var OnCommandNotFound = func(ctx context.Context, cmd *Cmd, err error) error {
	err = &CommandNotFound{cmd.args[0], err}
	return err
}

var LastErrorLevel int

func makeCmdline(args, rawargs []string) string {
	var buffer strings.Builder
	for i, s := range args {
		if i > 0 {
			buffer.WriteByte(' ')
		}
		if (len(rawargs) > i && len(rawargs[i]) > 0 && rawargs[i][0] == '"') || strings.ContainsAny(s, " &|<>\t\"") {
			fmt.Fprintf(&buffer, `"%s"`, strings.Replace(s, `"`, `\"`, -1))
		} else {
			buffer.WriteString(s)
		}
	}
	return buffer.String()
}

var UseSourceRunBatch = true

func encloseWithQuote(fullpath string) string {
	if strings.ContainsRune(fullpath, ' ') {
		var f strings.Builder
		f.WriteByte('"')
		f.WriteString(fullpath)
		f.WriteByte('"')
		return f.String()
	} else {
		return fullpath
	}
}

func (cmd *Cmd) spawnvpSilent(ctx context.Context) (int, error) {
	for {
		// command is empty.
		if len(cmd.args) <= 0 {
			return 0, nil
		}
		eq := strings.IndexRune(cmd.args[0], '=')
		if eq <= 0 {
			break
		}
		envName := cmd.args[0][:eq]
		envNewValue := cmd.args[0][eq+1:]
		cmd.Setenv(envName, envNewValue)
		cmd.args = cmd.args[1:]
		cmd.rawArgs = cmd.rawArgs[1:]
	}

	if defined.DBG {
		print("spawnvpSilent('", cmd.args[0], "')\n")
	}

	// aliases and lua-commands
	if errorlevel, done, err := hook(ctx, cmd); done || err != nil {
		return errorlevel, err
	}

	// command not found hook
	fullpath := cmd.FullPath()
	if fullpath == "" {
		return 255, OnCommandNotFound(ctx, cmd, os.ErrNotExist)
	}
	cmd.args[0] = fullpath

	if defined.DBG {
		print("exec.LookPath(", cmd.args[0], ")==", fullpath, "\n")
	}

	if WildCardExpansionAlways {
		cmd.args = findfile.Globs(cmd.args)
	}
	return cmd.startProcess(ctx)
}

func startAndWaitProcess(ctx context.Context, name string, args []string, procAttr *os.ProcAttr) (int, error) {
	if ctx != nil {
		select {
		case <-ctx.Done():
			return 252, ctx.Err()
		default:
		}
	}

	process, err := os.StartProcess(name, args, procAttr)
	if err != nil {
		return 255, err
	}

	if ctx != nil {
		done := make(chan struct{})
		go func() {
			select {
			case <-ctx.Done():
				os.Stderr.WriteString("^C\n")
				process.Kill()
			case <-done:
			}
		}()
		defer func() {
			close(done)
		}()
	}
	processState, err := process.Wait()
	return processState.ExitCode(), err
}

type AlreadyReportedError struct {
	Err error
}

func (_ AlreadyReportedError) Error() string {
	return ""
}

func IsAlreadyReported(err error) bool {
	_, ok := err.(AlreadyReportedError)
	return ok
}

func (cmd *Cmd) Spawnvp(ctx context.Context) (int, error) {
	errorlevel, err := cmd.spawnvpSilent(ctx)
	if err != nil && err != io.EOF && !IsAlreadyReported(err) {
		if defined.DBG {
			val := reflect.ValueOf(err)
			fmt.Fprintf(cmd.Stderr, "error-type=%s\n", val.Type())
		}
		fmt.Fprintln(cmd.Stderr, err.Error())
		err = AlreadyReportedError{err}
	}
	return errorlevel, err
}

func (sh *Shell) Spawnlpe(ctx context.Context, args, rawargs []string, env map[string]string) (int, error) {
	cmd := sh.Command()
	defer cmd.Close()
	cmd.SetArgs(args)
	cmd.SetRawArgs(rawargs)
	cmd.env = env
	return cmd.Spawnvp(ctx)
}

func (sh *Shell) Spawnlp(ctx context.Context, args, rawargs []string) (int, error) {

	return sh.Spawnlpe(ctx, args, rawargs, nil)
}

func (sh *Shell) Interpret(ctx context.Context, text string) (errorlevel int, finalerr error) {
	if defined.DBG {
		print("Interpret('", text, "')\n")
	}
	if sh == nil {
		return 255, errors.New("Fatal Error: Interpret: instance is nil")
	}
	errorlevel = 0
	finalerr = nil

	statements, statementsErr := Parse(text)
	if statementsErr != nil {
		if defined.DBG {
			print("Parse Error:", statementsErr.Error(), "\n")
		}
		return 0, statementsErr
	}
	if argsHook != nil {
		if defined.DBG {
			print("call argsHook\n")
		}
		for _, pipeline := range statements {
			for _, state := range pipeline {
				var err error
				state.Args, err = argsHook(ctx, sh, state.Args)
				if err != nil {
					return 255, err
				}
			}
		}
		if defined.DBG {
			print("done argsHook\n")
		}
	}
	for _, pipeline := range statements {
		for i, state := range pipeline {
			if state.Term == "|" && (i+1 >= len(pipeline) || len(pipeline[i+1].Args) <= 0) {
				return 255, errors.New("The syntax of the command is incorrect.")
			}
		}
	}

	for _, pipeline := range statements {

		var pipeIn *os.File = nil
		isBackGround := sh.IsBackGround
		for _, state := range pipeline {
			if state.Term == "&" {
				isBackGround = true
				break
			}
		}
		var wg sync.WaitGroup
		shutdown_immediately := false
		for i, state := range pipeline {
			if defined.DBG {
				print(i, ": pipeline loop(", state.Args[0], ")\n")
			}
			cmd := sh.Command()
			cmd.IsBackGround = isBackGround

			if pipeIn != nil {
				cmd.Stdin = pipeIn
				cmd.Closers = append(cmd.Closers, pipeIn)
				pipeIn = nil
			}

			var err error
			if state.Term[0] == '|' {
				var pipeOut *os.File
				pipeIn, pipeOut, err = os.Pipe()
				cmd.Stdout = pipeOut
				if state.Term == "|&" {
					cmd.Stderr = pipeOut
				}
				cmd.Closers = append(cmd.Closers, pipeOut)
			}

			for _, red := range state.Redirect {
				var closer io.Closer
				closer, err = red.OpenOn(cmd)
				if err != nil {
					return 0, err
				}
				cmd.Closers = append(cmd.Closers, closer)
			}

			cmd.args = state.Args
			cmd.rawArgs = state.RawArgs
			if i > 0 {
				cmd.IsBackGround = true
			}
			if len(pipeline) == 1 && isGui(cmd.FullPath()) {
				cmd.UseShellExecute = true
			}
			if i == len(pipeline)-1 && state.Term != "&" {
				// foreground execution.
				errorlevel, finalerr = cmd.Spawnvp(ctx)
				LastErrorLevel = errorlevel
				cmd.Close()
			} else {
				// background
				var newctx context.Context
				if isBackGround {
					// let Context not terminate background-work (#313's 2nd)
					// for the problem gvim starts with empty buffer
					// executing `git blame FILE | type | gvim - &`.
					newctx = context.Background()
				} else {
					wg.Add(1)
					newctx = ctx
				}
				if tag := cmd.Tag(); tag != nil {
					var newtag CloneCloser
					if newctx, newtag, err = tag.Clone(newctx); err != nil {
						fmt.Fprintln(os.Stderr, err.Error())
						return -1, err
					} else {
						cmd.SetTag(newtag)
					}
				}
				go func(ctx1 context.Context, cmd1 *Cmd) {
					if !isBackGround {
						defer wg.Done()
					}
					cmd1.Spawnvp(ctx1)
					if tag := cmd1.Tag(); tag != nil {
						if err := tag.Close(); err != nil {
							fmt.Fprintln(os.Stderr, err.Error())
						}
					}
					cmd1.Close()
				}(newctx, cmd)
			}
		}
		if !isBackGround {
			wg.Wait()
			if shutdown_immediately {
				return errorlevel, nil
			}
			if len(pipeline) > 0 {
				switch pipeline[len(pipeline)-1].Term {
				case "&&":
					if errorlevel != 0 {
						return errorlevel, nil
					}
				case "||":
					if errorlevel == 0 {
						return errorlevel, nil
					}
				}
			}
		}
	}
	return
}
<|endoftext|>


<commit_before>package adaptor

import (
	"fmt"
	"strings"
	"sync"
	"time"

	"github.com/compose/transporter/pkg/message"
	"github.com/compose/transporter/pkg/pipe"
	"gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
)

const (
	MONGO_BUFFER_SIZE int = 1e6
	MONGO_BUFFER_LEN  int = 1e5
)

// Mongodb is an adaptor to read / write to mongodb.
// it works as a source by copying files, and then optionally tailing the oplog
type Mongodb struct {
	// pull these in from the node
	uri   string
	tail  bool // run the tail oplog
	debug bool

	// save time by setting these once
	collection string
	database   string

	oplogTime bson.MongoTimestamp

	//
	pipe *pipe.Pipe
	path string

	// mongo connection and options
	mongoSession *mgo.Session
	oplogTimeout time.Duration

	// a buffer to hold documents
	buffLock         sync.Mutex
	opsBuffer        []interface{}
	opsBufferSize    int
	bulkWriteChannel chan interface{}
	bulkQuitChannel  chan chan bool
	bulk             bool

	restartable bool // this refers to being able to refresh the iterator, not to the restart based on session op
}

// NewMongodb creates a new Mongodb adaptor
func NewMongodb(p *pipe.Pipe, path string, extra Config) (StopStartListener, error) {
	var (
		conf MongodbConfig
		err  error
	)
	if err = extra.Construct(&conf); err != nil {
		return nil, err
	}

	if conf.URI == "" || conf.Namespace == "" {
		return nil, fmt.Errorf("both uri and namespace required, but missing ")
	}

	if conf.Debug {
		fmt.Printf("Mongo Config %+v\n", conf)
	}

	m := &Mongodb{
		restartable:      true,            // assume for that we're able to restart the process
		oplogTimeout:     5 * time.Second, // timeout the oplog iterator
		pipe:             p,
		uri:              conf.URI,
		tail:             conf.Tail,
		debug:            conf.Debug,
		path:             path,
		opsBuffer:        make([]interface{}, 0, MONGO_BUFFER_SIZE),
		bulkWriteChannel: make(chan interface{}),
		bulkQuitChannel:  make(chan chan bool),
		bulk:             conf.Bulk,
	}

	m.database, m.collection, err = m.splitNamespace(conf.Namespace)
	if err != nil {
		return m, err
	}

	m.mongoSession, err = mgo.Dial(m.uri)

	if err != nil {
		return m, err
	}

	// set some options on the session
	m.mongoSession.EnsureSafe(&mgo.Safe{W: conf.Wc, FSync: conf.FSync})
	m.mongoSession.SetBatch(1000)
	m.mongoSession.SetPrefetch(0.5)

	return m, nil
}

// Start the adaptor as a source
func (m *Mongodb) Start() (err error) {
	defer func() {
		m.pipe.Stop()
	}()

	m.oplogTime = nowAsMongoTimestamp()
	if m.debug {
		fmt.Printf("setting start timestamp: %d\n", m.oplogTime)
	}

	err = m.catData()
	if err != nil {
		m.pipe.Err <- err
		return err
	}
	if m.tail {
		// replay the oplog
		err = m.tailData()
		if err != nil {
			m.pipe.Err <- err
			return err
		}
	}

	return
}

// Listen starts the pipe's listener
func (m *Mongodb) Listen() (err error) {
	defer func() {
		m.pipe.Stop()
	}()

	if m.bulk {
		go m.bulkWriter()
	}
	return m.pipe.Listen(m.writeMessage)
}

// Stop the adaptor
func (m *Mongodb) Stop() error {
	m.pipe.Stop()

	// if we're bulk writing, ask our writer to exit here
	if m.bulk {
		q := make(chan bool)
		m.bulkQuitChannel <- q
		<-q
	}

	return nil
}

// writeMessage writes one message to the destination mongo, or sends an error down the pipe
// TODO this can be cleaned up.  I'm not sure whether this should pipe the error, or whether the
//   caller should pipe the error
func (m *Mongodb) writeMessage(msg *message.Msg) (*message.Msg, error) {
	collection := m.mongoSession.DB(m.database).C(m.collection)

	if !msg.IsMap() {
		m.pipe.Err <- NewError(ERROR, m.path, fmt.Sprintf("mongodb error (document must be a bson document, got %T instead)", msg.Data), msg.Data)
		return msg, nil
	}

	doc := msg.Map()
	if m.bulk {
		m.bulkWriteChannel <- doc
	} else {
		err := collection.Insert(doc)
		if mgo.IsDup(err) {
			err = collection.Update(bson.M{"_id": doc["_id"]}, doc)
		}
		if err != nil {
			m.pipe.Err <- NewError(ERROR, m.path, fmt.Sprintf("mongodb error (%s)", err.Error()), msg.Data)
		}
	}

	return msg, nil
}

func (m *Mongodb) bulkWriter() {

	for {
		select {
		case doc := <-m.bulkWriteChannel:
			sz, err := docSize(doc)
			if err != nil {
				m.pipe.Err <- NewError(ERROR, m.path, fmt.Sprintf("mongodb error (%s)", err.Error()), doc)
				break
			}

			if ((sz + m.opsBufferSize) > MONGO_BUFFER_SIZE) || (len(m.opsBuffer) == MONGO_BUFFER_LEN) {
				m.writeBuffer() // send it off to be inserted
			}

			m.buffLock.Lock()
			m.opsBuffer = append(m.opsBuffer, doc)
			m.opsBufferSize += sz
			m.buffLock.Unlock()
		case <-time.After(2 * time.Second):
			m.writeBuffer()
		case q := <-m.bulkQuitChannel:
			m.writeBuffer()
			q <- true
		}
	}
}

func (m *Mongodb) writeBuffer() {
	m.buffLock.Lock()
	defer m.buffLock.Unlock()
	collection := m.mongoSession.DB(m.database).C(m.collection)

	err := collection.Insert(m.opsBuffer...)

	if err != nil {
		if mgo.IsDup(err) {
			err = nil
			for _, op := range m.opsBuffer {
				e := collection.Insert(op)
				if mgo.IsDup(e) {
					doc, ok := op.(map[string]interface{})
					if !ok {
						m.pipe.Err <- NewError(ERROR, m.path, "mongodb error (Cannot cast document to bson)", op)
					}

					e = collection.Update(bson.M{"_id": doc["_id"]}, doc)
				}
				if e != nil {
					m.pipe.Err <- NewError(ERROR, m.path, fmt.Sprintf("mongodb error (%s)", e.Error()), op)
				}
			}
		} else {
			m.pipe.Err <- NewError(ERROR, m.path, fmt.Sprintf("mongodb error (%s)", err.Error()), m.opsBuffer[0])
		}
	}

	m.opsBuffer = make([]interface{}, 0, MONGO_BUFFER_SIZE)
	m.opsBufferSize = 0
}

// catdata pulls down the original collection
func (m *Mongodb) catData() (err error) {
	var (
		collection = m.mongoSession.DB(m.database).C(m.collection)
		query      = bson.M{}
		result     bson.M // hold the document
	)

	iter := collection.Find(query).Sort("_id").Iter()

	for {
		for iter.Next(&result) {
			if stop := m.pipe.Stopped; stop {
				return
			}

			// set up the message
			msg := message.NewMsg(message.Insert, result)

			m.pipe.Send(msg)
			result = bson.M{}
		}

		// we've exited the mongo read loop, lets figure out why
		// check here again if we've been asked to quit
		if stop := m.pipe.Stopped; stop {
			return
		}

		if iter.Err() != nil && m.restartable {
			fmt.Printf("got err reading collection. reissuing query %v\n", iter.Err())
			time.Sleep(1 * time.Second)
			iter = collection.Find(query).Sort("_id").Iter()
			continue
		}

		return
	}
}

/*
 * tail the oplog
 */
func (m *Mongodb) tailData() (err error) {

	var (
		collection = m.mongoSession.DB("local").C("oplog.rs")
		result     oplogDoc // hold the document
		query      = bson.M{
			"ts": bson.M{"$gte": m.oplogTime},
			"ns": m.getNamespace(),
		}

		iter = collection.Find(query).LogReplay().Sort("$natural").Tail(m.oplogTimeout)
	)

	for {
		for iter.Next(&result) {
			if stop := m.pipe.Stopped; stop {
				return
			}
			if result.validOp() {

				var doc bson.M
				switch result.Op {
				case "i":
					doc = result.O
				case "d":
					doc = result.O
				case "u":
					doc, err = m.getOriginalDoc(result.O2)
					if err != nil { // errors aren't fatal here, but we need to send it down the pipe
						m.pipe.Err <- NewError(ERROR, m.path, fmt.Sprintf("Mongodb error (%s)", err.Error()), nil)
						continue
					}
				default:
					m.pipe.Err <- NewError(ERROR, m.path, "Mongodb error (unknown op type)", nil)
					continue
				}

				msg := message.NewMsg(message.OpTypeFromString(result.Op), doc)
				msg.Timestamp = int64(result.Ts) >> 32

				m.oplogTime = result.Ts
				m.pipe.Send(msg)
			}
			result = oplogDoc{}
		}

		// we've exited the mongo read loop, lets figure out why
		// check here again if we've been asked to quit
		if stop := m.pipe.Stopped; stop {
			return
		}
		if iter.Timeout() {
			continue
		}
		if iter.Err() != nil {
			return NewError(CRITICAL, m.path, fmt.Sprintf("Mongodb error (error reading collection %s)", iter.Err()), nil)
		}

		// query will change,
		query = bson.M{
			"ts": bson.M{"$gte": m.oplogTime},
			"ns": m.getNamespace(),
		}
		iter = collection.Find(query).LogReplay().Tail(m.oplogTimeout)
	}
}

// getOriginalDoc retrieves the original document from the database.  transport has no knowledge of update operations, all updates
// work as wholesale document replaces
func (m *Mongodb) getOriginalDoc(doc bson.M) (result bson.M, err error) {
	id, exists := doc["_id"]
	if !exists {
		return result, fmt.Errorf("can't get _id from document")
	}

	err = m.mongoSession.DB(m.database).C(m.collection).FindId(id).One(&result)
	if err != nil {
		err = fmt.Errorf("%s %v %v", m.getNamespace(), id, err)
	}
	return
}

func (m *Mongodb) getNamespace() string {
	return strings.Join([]string{m.database, m.collection}, ".")
}

// splitNamespace split's a mongo namespace by the first '.' into a database and a collection
func (m *Mongodb) splitNamespace(namespace string) (string, string, error) {
	fields := strings.SplitN(namespace, ".", 2)

	if len(fields) != 2 {
		return "", "", fmt.Errorf("malformed mongo namespace")
	}
	return fields[0], fields[1], nil
}

// oplogDoc are representations of the mongodb oplog document
// detailed here, among other places.  http://www.kchodorow.com/blog/2010/10/12/replication-internals/
type oplogDoc struct {
	Ts bson.MongoTimestamp `bson:"ts"`
	H  int64               `bson:"h"`
	V  int                 `bson:"v"`
	Op string              `bson:"op"`
	Ns string              `bson:"ns"`
	O  bson.M              `bson:"o"`
	O2 bson.M              `bson:"o2"`
}

// validOp checks to see if we're an insert, delete, or update, otherwise the
// document is skilled.
// TODO: skip system collections
func (o *oplogDoc) validOp() bool {
	return o.Op == "i" || o.Op == "d" || o.Op == "u"
}

// MongodbConfig provides configuration options for a mongodb adaptor
// the notable difference between this and dbConfig is the presence of the Tail option
type MongodbConfig struct {
	URI       string `json:"uri" doc:"the uri to connect to, in the form mongodb://user:password@host.com:27017/auth_database"`
	Namespace string `json:"namespace" doc:"mongo namespace to read/write"`
	Debug     bool   `json:"debug" doc:"display debug information"`
	Tail      bool   `json:"tail" doc:"if tail is true, then the mongodb source will tail the oplog after copying the namespace"`
	Wc        int    `json:"wc" doc:"The write concern to use for writes, Int, indicating the minimum number of servers to write to before returning success/failure"`
	FSync     bool   `json:"fsync" doc:"When writing, should we flush to disk before returning success"`
	Bulk      bool   `json:"bulk" doc:"use a buffer to bulk insert documents"`
}

func nowAsMongoTimestamp() bson.MongoTimestamp {
	return bson.MongoTimestamp(time.Now().Unix() << 32)
}

func newMongoTimestamp(s, i int) bson.MongoTimestamp {
	return bson.MongoTimestamp(int64(s)<<32 + int64(i))
}

// find the size of a document in bytes
func docSize(ops interface{}) (int, error) {
	b, err := bson.Marshal(ops)
	if err != nil {
		return 0, err
	}
	return len(b), nil
}
<commit_msg>fixes #50, check for empty array (and increase buffer size)<commit_after>package adaptor

import (
	"fmt"
	"strings"
	"sync"
	"time"

	"github.com/compose/transporter/pkg/message"
	"github.com/compose/transporter/pkg/pipe"
	"gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
)

const (
	MONGO_BUFFER_SIZE int = 5e6
	MONGO_BUFFER_LEN  int = 5e5
)

// Mongodb is an adaptor to read / write to mongodb.
// it works as a source by copying files, and then optionally tailing the oplog
type Mongodb struct {
	// pull these in from the node
	uri   string
	tail  bool // run the tail oplog
	debug bool

	// save time by setting these once
	collection string
	database   string

	oplogTime bson.MongoTimestamp

	//
	pipe *pipe.Pipe
	path string

	// mongo connection and options
	mongoSession *mgo.Session
	oplogTimeout time.Duration

	// a buffer to hold documents
	buffLock         sync.Mutex
	opsBuffer        []interface{}
	opsBufferSize    int
	bulkWriteChannel chan interface{}
	bulkQuitChannel  chan chan bool
	bulk             bool

	restartable bool // this refers to being able to refresh the iterator, not to the restart based on session op
}

// NewMongodb creates a new Mongodb adaptor
func NewMongodb(p *pipe.Pipe, path string, extra Config) (StopStartListener, error) {
	var (
		conf MongodbConfig
		err  error
	)
	if err = extra.Construct(&conf); err != nil {
		return nil, err
	}

	if conf.URI == "" || conf.Namespace == "" {
		return nil, fmt.Errorf("both uri and namespace required, but missing ")
	}

	if conf.Debug {
		fmt.Printf("Mongo Config %+v\n", conf)
	}

	m := &Mongodb{
		restartable:      true,            // assume for that we're able to restart the process
		oplogTimeout:     5 * time.Second, // timeout the oplog iterator
		pipe:             p,
		uri:              conf.URI,
		tail:             conf.Tail,
		debug:            conf.Debug,
		path:             path,
		opsBuffer:        make([]interface{}, 0, MONGO_BUFFER_SIZE),
		bulkWriteChannel: make(chan interface{}),
		bulkQuitChannel:  make(chan chan bool),
		bulk:             conf.Bulk,
	}

	m.database, m.collection, err = m.splitNamespace(conf.Namespace)
	if err != nil {
		return m, err
	}

	m.mongoSession, err = mgo.Dial(m.uri)

	if err != nil {
		return m, err
	}

	// set some options on the session
	m.mongoSession.EnsureSafe(&mgo.Safe{W: conf.Wc, FSync: conf.FSync})
	m.mongoSession.SetBatch(1000)
	m.mongoSession.SetPrefetch(0.5)

	return m, nil
}

// Start the adaptor as a source
func (m *Mongodb) Start() (err error) {
	defer func() {
		m.pipe.Stop()
	}()

	m.oplogTime = nowAsMongoTimestamp()
	if m.debug {
		fmt.Printf("setting start timestamp: %d\n", m.oplogTime)
	}

	err = m.catData()
	if err != nil {
		m.pipe.Err <- err
		return err
	}
	if m.tail {
		// replay the oplog
		err = m.tailData()
		if err != nil {
			m.pipe.Err <- err
			return err
		}
	}

	return
}

// Listen starts the pipe's listener
func (m *Mongodb) Listen() (err error) {
	defer func() {
		m.pipe.Stop()
	}()

	if m.bulk {
		go m.bulkWriter()
	}
	return m.pipe.Listen(m.writeMessage)
}

// Stop the adaptor
func (m *Mongodb) Stop() error {
	m.pipe.Stop()

	// if we're bulk writing, ask our writer to exit here
	if m.bulk {
		q := make(chan bool)
		m.bulkQuitChannel <- q
		<-q
	}

	return nil
}

// writeMessage writes one message to the destination mongo, or sends an error down the pipe
// TODO this can be cleaned up.  I'm not sure whether this should pipe the error, or whether the
//   caller should pipe the error
func (m *Mongodb) writeMessage(msg *message.Msg) (*message.Msg, error) {
	collection := m.mongoSession.DB(m.database).C(m.collection)

	if !msg.IsMap() {
		m.pipe.Err <- NewError(ERROR, m.path, fmt.Sprintf("mongodb error (document must be a bson document, got %T instead)", msg.Data), msg.Data)
		return msg, nil
	}

	doc := msg.Map()
	if m.bulk {
		m.bulkWriteChannel <- doc
	} else {
		err := collection.Insert(doc)
		if mgo.IsDup(err) {
			err = collection.Update(bson.M{"_id": doc["_id"]}, doc)
		}
		if err != nil {
			m.pipe.Err <- NewError(ERROR, m.path, fmt.Sprintf("mongodb error (%s)", err.Error()), msg.Data)
		}
	}

	return msg, nil
}

func (m *Mongodb) bulkWriter() {

	for {
		select {
		case doc := <-m.bulkWriteChannel:
			sz, err := docSize(doc)
			if err != nil {
				m.pipe.Err <- NewError(ERROR, m.path, fmt.Sprintf("mongodb error (%s)", err.Error()), doc)
				break
			}

			if ((sz + m.opsBufferSize) > MONGO_BUFFER_SIZE) || (len(m.opsBuffer) == MONGO_BUFFER_LEN) {
				m.writeBuffer() // send it off to be inserted
			}

			m.buffLock.Lock()
			m.opsBuffer = append(m.opsBuffer, doc)
			m.opsBufferSize += sz
			m.buffLock.Unlock()
		case <-time.After(2 * time.Second):
			m.writeBuffer()
		case q := <-m.bulkQuitChannel:
			m.writeBuffer()
			q <- true
		}
	}
}

func (m *Mongodb) writeBuffer() {
	m.buffLock.Lock()
	defer m.buffLock.Unlock()
	collection := m.mongoSession.DB(m.database).C(m.collection)
	if len(m.opsBuffer) == 0 {
		return
	}

	err := collection.Insert(m.opsBuffer...)

	if err != nil {
		if mgo.IsDup(err) {
			err = nil
			for _, op := range m.opsBuffer {
				e := collection.Insert(op)
				if mgo.IsDup(e) {
					doc, ok := op.(map[string]interface{})
					if !ok {
						m.pipe.Err <- NewError(ERROR, m.path, "mongodb error (Cannot cast document to bson)", op)
					}

					e = collection.Update(bson.M{"_id": doc["_id"]}, doc)
				}
				if e != nil {
					m.pipe.Err <- NewError(ERROR, m.path, fmt.Sprintf("mongodb error (%s)", e.Error()), op)
				}
			}
		} else {
			m.pipe.Err <- NewError(ERROR, m.path, fmt.Sprintf("mongodb error (%s)", err.Error()), m.opsBuffer[0])
		}
	}

	m.opsBuffer = make([]interface{}, 0, MONGO_BUFFER_SIZE)
	m.opsBufferSize = 0
}

// catdata pulls down the original collection
func (m *Mongodb) catData() (err error) {
	var (
		collection = m.mongoSession.DB(m.database).C(m.collection)
		query      = bson.M{}
		result     bson.M // hold the document
	)

	iter := collection.Find(query).Sort("_id").Iter()

	for {
		for iter.Next(&result) {
			if stop := m.pipe.Stopped; stop {
				return
			}

			// set up the message
			msg := message.NewMsg(message.Insert, result)

			m.pipe.Send(msg)
			result = bson.M{}
		}

		// we've exited the mongo read loop, lets figure out why
		// check here again if we've been asked to quit
		if stop := m.pipe.Stopped; stop {
			return
		}

		if iter.Err() != nil && m.restartable {
			fmt.Printf("got err reading collection. reissuing query %v\n", iter.Err())
			time.Sleep(1 * time.Second)
			iter = collection.Find(query).Sort("_id").Iter()
			continue
		}

		return
	}
}

/*
 * tail the oplog
 */
func (m *Mongodb) tailData() (err error) {

	var (
		collection = m.mongoSession.DB("local").C("oplog.rs")
		result     oplogDoc // hold the document
		query      = bson.M{
			"ts": bson.M{"$gte": m.oplogTime},
			"ns": m.getNamespace(),
		}

		iter = collection.Find(query).LogReplay().Sort("$natural").Tail(m.oplogTimeout)
	)

	for {
		for iter.Next(&result) {
			if stop := m.pipe.Stopped; stop {
				return
			}
			if result.validOp() {

				var doc bson.M
				switch result.Op {
				case "i":
					doc = result.O
				case "d":
					doc = result.O
				case "u":
					doc, err = m.getOriginalDoc(result.O2)
					if err != nil { // errors aren't fatal here, but we need to send it down the pipe
						m.pipe.Err <- NewError(ERROR, m.path, fmt.Sprintf("Mongodb error (%s)", err.Error()), nil)
						continue
					}
				default:
					m.pipe.Err <- NewError(ERROR, m.path, "Mongodb error (unknown op type)", nil)
					continue
				}

				msg := message.NewMsg(message.OpTypeFromString(result.Op), doc)
				msg.Timestamp = int64(result.Ts) >> 32

				m.oplogTime = result.Ts
				m.pipe.Send(msg)
			}
			result = oplogDoc{}
		}

		// we've exited the mongo read loop, lets figure out why
		// check here again if we've been asked to quit
		if stop := m.pipe.Stopped; stop {
			return
		}
		if iter.Timeout() {
			continue
		}
		if iter.Err() != nil {
			return NewError(CRITICAL, m.path, fmt.Sprintf("Mongodb error (error reading collection %s)", iter.Err()), nil)
		}

		// query will change,
		query = bson.M{
			"ts": bson.M{"$gte": m.oplogTime},
			"ns": m.getNamespace(),
		}
		iter = collection.Find(query).LogReplay().Tail(m.oplogTimeout)
	}
}

// getOriginalDoc retrieves the original document from the database.  transport has no knowledge of update operations, all updates
// work as wholesale document replaces
func (m *Mongodb) getOriginalDoc(doc bson.M) (result bson.M, err error) {
	id, exists := doc["_id"]
	if !exists {
		return result, fmt.Errorf("can't get _id from document")
	}

	err = m.mongoSession.DB(m.database).C(m.collection).FindId(id).One(&result)
	if err != nil {
		err = fmt.Errorf("%s %v %v", m.getNamespace(), id, err)
	}
	return
}

func (m *Mongodb) getNamespace() string {
	return strings.Join([]string{m.database, m.collection}, ".")
}

// splitNamespace split's a mongo namespace by the first '.' into a database and a collection
func (m *Mongodb) splitNamespace(namespace string) (string, string, error) {
	fields := strings.SplitN(namespace, ".", 2)

	if len(fields) != 2 {
		return "", "", fmt.Errorf("malformed mongo namespace")
	}
	return fields[0], fields[1], nil
}

// oplogDoc are representations of the mongodb oplog document
// detailed here, among other places.  http://www.kchodorow.com/blog/2010/10/12/replication-internals/
type oplogDoc struct {
	Ts bson.MongoTimestamp `bson:"ts"`
	H  int64               `bson:"h"`
	V  int                 `bson:"v"`
	Op string              `bson:"op"`
	Ns string              `bson:"ns"`
	O  bson.M              `bson:"o"`
	O2 bson.M              `bson:"o2"`
}

// validOp checks to see if we're an insert, delete, or update, otherwise the
// document is skilled.
// TODO: skip system collections
func (o *oplogDoc) validOp() bool {
	return o.Op == "i" || o.Op == "d" || o.Op == "u"
}

// MongodbConfig provides configuration options for a mongodb adaptor
// the notable difference between this and dbConfig is the presence of the Tail option
type MongodbConfig struct {
	URI       string `json:"uri" doc:"the uri to connect to, in the form mongodb://user:password@host.com:27017/auth_database"`
	Namespace string `json:"namespace" doc:"mongo namespace to read/write"`
	Debug     bool   `json:"debug" doc:"display debug information"`
	Tail      bool   `json:"tail" doc:"if tail is true, then the mongodb source will tail the oplog after copying the namespace"`
	Wc        int    `json:"wc" doc:"The write concern to use for writes, Int, indicating the minimum number of servers to write to before returning success/failure"`
	FSync     bool   `json:"fsync" doc:"When writing, should we flush to disk before returning success"`
	Bulk      bool   `json:"bulk" doc:"use a buffer to bulk insert documents"`
}

func nowAsMongoTimestamp() bson.MongoTimestamp {
	return bson.MongoTimestamp(time.Now().Unix() << 32)
}

func newMongoTimestamp(s, i int) bson.MongoTimestamp {
	return bson.MongoTimestamp(int64(s)<<32 + int64(i))
}

// find the size of a document in bytes
func docSize(ops interface{}) (int, error) {
	b, err := bson.Marshal(ops)
	if err != nil {
		return 0, err
	}
	return len(b), nil
}
<|endoftext|>


<commit_before>/*
Copyright 2014 Google Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package api

import (
	"encoding/json"
	"flag"
	"fmt"
	"reflect"
	"testing"

	"github.com/GoogleCloudPlatform/kubernetes/pkg/util"
	"github.com/fsouza/go-dockerclient"
)

var fuzzIters = flag.Int("fuzz_iters", 3, "How many fuzzing iterations to do.")

// apiObjectFuzzer can randomly populate api objects.
var apiObjectFuzzer = util.NewFuzzer(
	func(j *JSONBase) {
		// We have to customize the randomization of JSONBases because their
		// APIVersion and Kind must remain blank in memory.
		j.APIVersion = ""
		j.Kind = ""
		j.ID = util.RandString()
		// TODO: Fix JSON/YAML packages and/or write custom encoding
		// for uint64's. Somehow the LS *byte* of this is lost, but
		// only when all 8 bytes are set.
		j.ResourceVersion = util.RandUint64() >> 8
		j.SelfLink = util.RandString()
		j.CreationTimestamp = util.RandString()
	},
	func(intstr *util.IntOrString) {
		// util.IntOrString will panic if its kind is set wrong.
		if util.RandBool() {
			intstr.Kind = util.IntstrInt
			intstr.IntVal = int(util.RandUint64())
			intstr.StrVal = ""
		} else {
			intstr.Kind = util.IntstrString
			intstr.IntVal = 0
			intstr.StrVal = util.RandString()
		}
	},
	func(u64 *uint64) {
		// TODO: uint64's are NOT handled right.
		*u64 = util.RandUint64() >> 8
	},
	func(pb map[docker.Port][]docker.PortBinding) {
		// This is necessary because keys with nil values get omitted.
		// TODO: Is this a bug?
		pb[docker.Port(util.RandString())] = []docker.PortBinding{
			{util.RandString(), util.RandString()},
			{util.RandString(), util.RandString()},
		}
	},
	func(pm map[string]docker.PortMapping) {
		// This is necessary because keys with nil values get omitted.
		// TODO: Is this a bug?
		pm[util.RandString()] = docker.PortMapping{
			util.RandString(): util.RandString(),
		}
	},
)

func objDiff(a, b interface{}) string {
	ab, err := json.Marshal(a)
	if err != nil {
		panic("a")
	}
	bb, err := json.Marshal(b)
	if err != nil {
		panic("b")
	}
	return util.StringDiff(string(ab), string(bb))

	// An alternate diff attempt, in case json isn't showing you
	// the difference. (reflect.DeepEqual makes a distinction between
	// nil and empty slices, for example.)
	return util.StringDiff(
		fmt.Sprintf("%#v", a),
		fmt.Sprintf("%#v", b),
	)
}

func runTest(t *testing.T, source interface{}) {
	name := reflect.TypeOf(source).Elem().Name()
	apiObjectFuzzer.Fuzz(source)
	j, err := FindJSONBase(source)
	if err != nil {
		t.Fatalf("Unexpected error %v for %#v", err, source)
	}
	j.SetKind("")
	j.SetAPIVersion("")

	data, err := Encode(source)
	if err != nil {
		t.Errorf("%v: %v (%#v)", name, err, source)
		return
	}
	obj2, err := Decode(data)
	if err != nil {
		t.Errorf("%v: %v", name, err)
		return
	} else {
		if !reflect.DeepEqual(source, obj2) {
			t.Errorf("1: %v: diff: %v", name, objDiff(source, obj2))
			return
		}
	}
	obj3 := reflect.New(reflect.TypeOf(source).Elem()).Interface()
	err = DecodeInto(data, obj3)
	if err != nil {
		t.Errorf("2: %v: %v", name, err)
		return
	} else {
		if !reflect.DeepEqual(source, obj3) {
			t.Errorf("3: %v: diff: %v", name, objDiff(source, obj3))
			return
		}
	}
}

func TestTypes(t *testing.T) {
	// TODO: auto-fill all fields.
	table := []interface{}{
		&PodList{},
		&Pod{},
		&ServiceList{},
		&Service{},
		&ReplicationControllerList{},
		&ReplicationController{},
		&MinionList{},
		&Minion{},
		&Status{},
		&ServerOpList{},
		&ServerOp{},
		&ContainerManifestList{},
		&Endpoints{},
	}
	for _, item := range table {
		// Try a few times, since runTest uses random values.
		for i := 0; i < *fuzzIters; i++ {
			runTest(t, item)
		}
	}
}

func TestEncode_NonPtr(t *testing.T) {
	pod := Pod{
		Labels: map[string]string{"name": "foo"},
	}
	obj := interface{}(pod)
	data, err := Encode(obj)
	obj2, err2 := Decode(data)
	if err != nil || err2 != nil {
		t.Fatalf("Failure: '%v' '%v'", err, err2)
	}
	if _, ok := obj2.(*Pod); !ok {
		t.Fatalf("Got wrong type")
	}
	if !reflect.DeepEqual(obj2, &pod) {
		t.Errorf("Expected:\n %#v,\n Got:\n %#v", &pod, obj2)
	}
}

func TestEncode_Ptr(t *testing.T) {
	pod := &Pod{
		Labels: map[string]string{"name": "foo"},
	}
	obj := interface{}(pod)
	data, err := Encode(obj)
	obj2, err2 := Decode(data)
	if err != nil || err2 != nil {
		t.Fatalf("Failure: '%v' '%v'", err, err2)
	}
	if _, ok := obj2.(*Pod); !ok {
		t.Fatalf("Got wrong type")
	}
	if !reflect.DeepEqual(obj2, pod) {
		t.Errorf("Expected:\n %#v,\n Got:\n %#v", &pod, obj2)
	}
}

func TestBadJSONRejection(t *testing.T) {
	badJSONMissingKind := []byte(`{ }`)
	if _, err := Decode(badJSONMissingKind); err == nil {
		t.Errorf("Did not reject despite lack of kind field: %s", badJSONMissingKind)
	}
	badJSONUnknownType := []byte(`{"kind": "bar"}`)
	if _, err1 := Decode(badJSONUnknownType); err1 == nil {
		t.Errorf("Did not reject despite use of unknown type: %s", badJSONUnknownType)
	}
	/*badJSONKindMismatch := []byte(`{"kind": "Pod"}`)
	if err2 := DecodeInto(badJSONKindMismatch, &Minion{}); err2 == nil {
		t.Errorf("Kind is set but doesn't match the object type: %s", badJSONKindMismatch)
	}*/
}
<commit_msg>Switch to new external fuzz package<commit_after>/*
Copyright 2014 Google Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package api

import (
	"encoding/json"
	"flag"
	"fmt"
	"reflect"
	"testing"

	"github.com/GoogleCloudPlatform/kubernetes/pkg/util"
	"github.com/fsouza/go-dockerclient"
	"github.com/google/gofuzz"
)

var fuzzIters = flag.Int("fuzz_iters", 50, "How many fuzzing iterations to do.")

// apiObjectFuzzer can randomly populate api objects.
var apiObjectFuzzer = fuzz.New().NilChance(.5).NumElements(1, 1).Funcs(
	func(j *JSONBase, c fuzz.Continue) {
		// We have to customize the randomization of JSONBases because their
		// APIVersion and Kind must remain blank in memory.
		j.APIVersion = ""
		j.Kind = ""
		j.ID = c.RandString()
		// TODO: Fix JSON/YAML packages and/or write custom encoding
		// for uint64's. Somehow the LS *byte* of this is lost, but
		// only when all 8 bytes are set.
		j.ResourceVersion = c.RandUint64() >> 8
		j.SelfLink = c.RandString()
		j.CreationTimestamp = c.RandString()
	},
	func(intstr *util.IntOrString, c fuzz.Continue) {
		// util.IntOrString will panic if its kind is set wrong.
		if c.RandBool() {
			intstr.Kind = util.IntstrInt
			intstr.IntVal = int(c.RandUint64())
			intstr.StrVal = ""
		} else {
			intstr.Kind = util.IntstrString
			intstr.IntVal = 0
			intstr.StrVal = c.RandString()
		}
	},
	func(u64 *uint64, c fuzz.Continue) {
		// TODO: uint64's are NOT handled right.
		*u64 = c.RandUint64() >> 8
	},
	func(pb map[docker.Port][]docker.PortBinding, c fuzz.Continue) {
		// This is necessary because keys with nil values get omitted.
		// TODO: Is this a bug?
		pb[docker.Port(c.RandString())] = []docker.PortBinding{
			{c.RandString(), c.RandString()},
			{c.RandString(), c.RandString()},
		}
	},
	func(pm map[string]docker.PortMapping, c fuzz.Continue) {
		// This is necessary because keys with nil values get omitted.
		// TODO: Is this a bug?
		pm[c.RandString()] = docker.PortMapping{
			c.RandString(): c.RandString(),
		}
	},
)

func objDiff(a, b interface{}) string {
	ab, err := json.Marshal(a)
	if err != nil {
		panic("a")
	}
	bb, err := json.Marshal(b)
	if err != nil {
		panic("b")
	}
	return util.StringDiff(string(ab), string(bb))

	// An alternate diff attempt, in case json isn't showing you
	// the difference. (reflect.DeepEqual makes a distinction between
	// nil and empty slices, for example.)
	return util.StringDiff(
		fmt.Sprintf("%#v", a),
		fmt.Sprintf("%#v", b),
	)
}

func runTest(t *testing.T, source interface{}) {
	name := reflect.TypeOf(source).Elem().Name()
	apiObjectFuzzer.Fuzz(source)
	j, err := FindJSONBase(source)
	if err != nil {
		t.Fatalf("Unexpected error %v for %#v", err, source)
	}
	j.SetKind("")
	j.SetAPIVersion("")

	data, err := Encode(source)
	if err != nil {
		t.Errorf("%v: %v (%#v)", name, err, source)
		return
	}
	obj2, err := Decode(data)
	if err != nil {
		t.Errorf("%v: %v", name, err)
		return
	} else {
		if !reflect.DeepEqual(source, obj2) {
			t.Errorf("1: %v: diff: %v", name, objDiff(source, obj2))
			return
		}
	}
	obj3 := reflect.New(reflect.TypeOf(source).Elem()).Interface()
	err = DecodeInto(data, obj3)
	if err != nil {
		t.Errorf("2: %v: %v", name, err)
		return
	} else {
		if !reflect.DeepEqual(source, obj3) {
			t.Errorf("3: %v: diff: %v", name, objDiff(source, obj3))
			return
		}
	}
}

func TestTypes(t *testing.T) {
	// TODO: auto-fill all fields.
	table := []interface{}{
		&PodList{},
		&Pod{},
		&ServiceList{},
		&Service{},
		&ReplicationControllerList{},
		&ReplicationController{},
		&MinionList{},
		&Minion{},
		&Status{},
		&ServerOpList{},
		&ServerOp{},
		&ContainerManifestList{},
		&Endpoints{},
	}
	for _, item := range table {
		// Try a few times, since runTest uses random values.
		for i := 0; i < *fuzzIters; i++ {
			runTest(t, item)
		}
	}
}

func TestEncode_NonPtr(t *testing.T) {
	pod := Pod{
		Labels: map[string]string{"name": "foo"},
	}
	obj := interface{}(pod)
	data, err := Encode(obj)
	obj2, err2 := Decode(data)
	if err != nil || err2 != nil {
		t.Fatalf("Failure: '%v' '%v'", err, err2)
	}
	if _, ok := obj2.(*Pod); !ok {
		t.Fatalf("Got wrong type")
	}
	if !reflect.DeepEqual(obj2, &pod) {
		t.Errorf("Expected:\n %#v,\n Got:\n %#v", &pod, obj2)
	}
}

func TestEncode_Ptr(t *testing.T) {
	pod := &Pod{
		Labels: map[string]string{"name": "foo"},
	}
	obj := interface{}(pod)
	data, err := Encode(obj)
	obj2, err2 := Decode(data)
	if err != nil || err2 != nil {
		t.Fatalf("Failure: '%v' '%v'", err, err2)
	}
	if _, ok := obj2.(*Pod); !ok {
		t.Fatalf("Got wrong type")
	}
	if !reflect.DeepEqual(obj2, pod) {
		t.Errorf("Expected:\n %#v,\n Got:\n %#v", &pod, obj2)
	}
}

func TestBadJSONRejection(t *testing.T) {
	badJSONMissingKind := []byte(`{ }`)
	if _, err := Decode(badJSONMissingKind); err == nil {
		t.Errorf("Did not reject despite lack of kind field: %s", badJSONMissingKind)
	}
	badJSONUnknownType := []byte(`{"kind": "bar"}`)
	if _, err1 := Decode(badJSONUnknownType); err1 == nil {
		t.Errorf("Did not reject despite use of unknown type: %s", badJSONUnknownType)
	}
	/*badJSONKindMismatch := []byte(`{"kind": "Pod"}`)
	if err2 := DecodeInto(badJSONKindMismatch, &Minion{}); err2 == nil {
		t.Errorf("Kind is set but doesn't match the object type: %s", badJSONKindMismatch)
	}*/
}
<|endoftext|>


<commit_before>// Copyright 2015-present Oursky Ltd.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package model

import (
	"time"

	"github.com/skygeario/skygear-server/pkg/auth/dependency/userprofile"
	"github.com/skygeario/skygear-server/pkg/core/auth/authinfo"
)

// User is the unify way of returning a AuthInfo with LoginID to SDK
type User struct {
	ID          string           `json:"id,omitempty"`
	CreatedAt   time.Time        `json:"created_at"`
	LastLoginAt *time.Time       `json:"last_login_at,omitempty"`
	Verified    bool             `json:"is_verified"`
	Disabled    bool             `json:"is_disabled"`
	VerifyInfo  map[string]bool  `json:"verify_info"`
	Metadata    userprofile.Data `json:"metadata"`
}

func NewUser(
	authInfo authinfo.AuthInfo,
	userProfile userprofile.UserProfile,
) User {
	// copy data to prevent sharing
	if authInfo.LastLoginAt != nil {
		lastLoginAt := *authInfo.LastLoginAt
		authInfo.LastLoginAt = &lastLoginAt
	}
	verifyInfo := map[string]bool{}
	for k, v := range authInfo.VerifyInfo {
		verifyInfo[k] = v
	}
	metadata := userprofile.Data{}
	for k, v := range userProfile.Data {
		metadata[k] = v
	}

	return User{
		ID:          authInfo.ID,
		CreatedAt:   userProfile.CreatedAt,
		LastLoginAt: authInfo.LastLoginAt,
		Verified:    authInfo.Verified,
		Disabled:    authInfo.Disabled,
		VerifyInfo:  verifyInfo,
		Metadata:    metadata,
	}
}

// @JSONSchema
const UserSchema = `
{
	"$id": "#User",
	"type": "object",
	"properties": {
		"id": { "type": "string" },
		"created_at": { "type": "string" },
		"last_login_at": { "type": "string" },
		"is_verified": { "type": "boolean" },
		"is_disabled": { "type": "boolean" },
		"verify_info": { "type": "object" },
		"metadata": { "type": "object" }
	}
}
`
<commit_msg>Expose manual verification flag in API<commit_after>// Copyright 2015-present Oursky Ltd.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package model

import (
	"time"

	"github.com/skygeario/skygear-server/pkg/auth/dependency/userprofile"
	"github.com/skygeario/skygear-server/pkg/core/auth/authinfo"
)

// User is the unify way of returning a AuthInfo with LoginID to SDK
type User struct {
	ID               string           `json:"id,omitempty"`
	CreatedAt        time.Time        `json:"created_at"`
	LastLoginAt      *time.Time       `json:"last_login_at,omitempty"`
	Verified         bool             `json:"is_verified"`
	ManuallyVerified bool             `json:"is_manually_verified"`
	Disabled         bool             `json:"is_disabled"`
	VerifyInfo       map[string]bool  `json:"verify_info"`
	Metadata         userprofile.Data `json:"metadata"`
}

func NewUser(
	authInfo authinfo.AuthInfo,
	userProfile userprofile.UserProfile,
) User {
	// copy data to prevent sharing
	if authInfo.LastLoginAt != nil {
		lastLoginAt := *authInfo.LastLoginAt
		authInfo.LastLoginAt = &lastLoginAt
	}
	verifyInfo := map[string]bool{}
	for k, v := range authInfo.VerifyInfo {
		verifyInfo[k] = v
	}
	metadata := userprofile.Data{}
	for k, v := range userProfile.Data {
		metadata[k] = v
	}

	return User{
		ID:               authInfo.ID,
		CreatedAt:        userProfile.CreatedAt,
		LastLoginAt:      authInfo.LastLoginAt,
		Verified:         authInfo.IsVerified(),
		ManuallyVerified: authInfo.ManuallyVerified,
		Disabled:         authInfo.Disabled,
		VerifyInfo:       verifyInfo,
		Metadata:         metadata,
	}
}

// @JSONSchema
const UserSchema = `
{
	"$id": "#User",
	"type": "object",
	"properties": {
		"id": { "type": "string" },
		"created_at": { "type": "string" },
		"last_login_at": { "type": "string" },
		"is_verified": { "type": "boolean" },
		"is_manually_verified": { "type": "boolean" },
		"is_disabled": { "type": "boolean" },
		"verify_info": { "type": "object" },
		"metadata": { "type": "object" }
	}
}
`
<|endoftext|>


<commit_before>// Package batcher provides retriable batch buffer: all succeed or rollback for all.
package batcher

import (
	"sync/atomic"
	"time"
)

const backoff = time.Millisecond

type Batcher struct {
	size       uint32
	w, r       uint32
	okN, failN uint32
	stopped    uint32
	contents   []interface{}
}

func NewBatcher(size int) *Batcher {
	return &Batcher{
		size:     uint32(size),
		stopped:  0,
		contents: make([]interface{}, size),
	}
}

func (b *Batcher) Close() {
	atomic.StoreUint32(&b.stopped, 1)
}

func (b *Batcher) Write(v interface{}) error {
	for atomic.LoadUint32(&b.w) == b.size {
		// wait for the batch commit
		if atomic.LoadUint32(&b.stopped) == 1 {
			return ErrStopping
		}

		time.Sleep(backoff)
	}

	idx := atomic.AddUint32(&b.w, 1) - 1 // lock step
	b.contents[idx] = v
	return nil
}

func (b *Batcher) ReadOne() (interface{}, error) {
	for {
		if atomic.LoadUint32(&b.stopped) == 1 {
			return nil, ErrStopping
		}

		r, w := atomic.LoadUint32(&b.r), atomic.LoadUint32(&b.w)
		if r == b.size ||
			// batch tail reached, but not committed
			r >= w {
			// reader overrun writer
			time.Sleep(backoff)
		} else {
			break
		}
	}

	idx := atomic.AddUint32(&b.r, 1) - 1
	return b.contents[int(idx)], nil
}

func (b *Batcher) Advance() {
	ok := atomic.AddUint32(&b.okN, 1)
	if ok == b.size {
		// batch commit
		atomic.StoreUint32(&b.okN, 0)
		atomic.StoreUint32(&b.failN, 0)
		atomic.StoreUint32(&b.r, 0)
		atomic.StoreUint32(&b.w, 0)
	} else if ok+atomic.LoadUint32(&b.failN) == b.size {
		// batch rollback, reset reader cursor, retry the batch, hold writer
		atomic.StoreUint32(&b.okN, 0)
		atomic.StoreUint32(&b.failN, 0)
		atomic.StoreUint32(&b.r, 0)
	}
}

func (b *Batcher) Rollback() {
	fail := atomic.AddUint32(&b.failN, 1)
	if fail+atomic.LoadUint32(&b.okN) == b.size {
		// batch rollback
		atomic.StoreUint32(&b.okN, 0)
		atomic.StoreUint32(&b.failN, 0)
		atomic.StoreUint32(&b.r, 0)
	}
}
<commit_msg>wording<commit_after>// Package batcher provides retriable batch buffer: all succeed or rollback for all.
package batcher

import (
	"sync/atomic"
	"time"
)

const backoff = time.Millisecond

type Batcher struct {
	size       uint32
	w, r       uint32
	okN, failN uint32
	stopped    uint32
	contents   []interface{}
}

func NewBatcher(size int) *Batcher {
	return &Batcher{
		size:     uint32(size),
		stopped:  0,
		contents: make([]interface{}, size),
	}
}

func (b *Batcher) Close() {
	atomic.StoreUint32(&b.stopped, 1)
}

func (b *Batcher) Write(v interface{}) error {
	for atomic.LoadUint32(&b.w) == b.size {
		// wait for the batch commit
		if atomic.LoadUint32(&b.stopped) == 1 {
			return ErrStopping
		}

		time.Sleep(backoff)
	}

	idx := atomic.AddUint32(&b.w, 1) - 1 // lock step
	b.contents[idx] = v
	return nil
}

func (b *Batcher) ReadOne() (interface{}, error) {
	for {
		if atomic.LoadUint32(&b.stopped) == 1 {
			return nil, ErrStopping
		}

		r, w := atomic.LoadUint32(&b.r), atomic.LoadUint32(&b.w)
		if r == b.size ||
			// batch tail reached, but not committed
			r >= w {
			// reader out-run writer
			time.Sleep(backoff)
		} else {
			break
		}
	}

	idx := atomic.AddUint32(&b.r, 1) - 1
	return b.contents[int(idx)], nil
}

func (b *Batcher) Advance() {
	ok := atomic.AddUint32(&b.okN, 1)
	if ok == b.size {
		// batch commit
		atomic.StoreUint32(&b.okN, 0)
		atomic.StoreUint32(&b.failN, 0)
		atomic.StoreUint32(&b.r, 0)
		atomic.StoreUint32(&b.w, 0)
	} else if ok+atomic.LoadUint32(&b.failN) == b.size {
		// batch rollback, reset reader cursor, retry the batch, hold writer
		atomic.StoreUint32(&b.okN, 0)
		atomic.StoreUint32(&b.failN, 0)
		atomic.StoreUint32(&b.r, 0)
	}
}

func (b *Batcher) Rollback() {
	fail := atomic.AddUint32(&b.failN, 1)
	if fail+atomic.LoadUint32(&b.okN) == b.size {
		// batch rollback
		atomic.StoreUint32(&b.okN, 0)
		atomic.StoreUint32(&b.failN, 0)
		atomic.StoreUint32(&b.r, 0)
	}
}
<|endoftext|>


<commit_before>// Package batcher provides retriable batch queue: all succeed or rollback for all.
// In kafka it is called RecordAccumulator.
package batcher

import (
	"sync/atomic"

	"github.com/funkygao/dbus/pkg/sys"
)

// Batcher is a batched lock free queue that borrows design from disruptor.
// It maintains a queue with the sematics of all succeed and advance or any fails and retry.
type Batcher struct {
	_padding0 [sys.CacheLineSize / 8]uint64
	capacity  uint32

	_padding1 [sys.CacheLineSize / 8]uint64
	stopped   uint32

	_padding2 [sys.CacheLineSize / 8]uint64
	w         uint32

	_padding3 [sys.CacheLineSize / 8]uint64
	r         uint32

	_padding4 [sys.CacheLineSize / 8]uint64
	c         uint32

	_padding5 [sys.CacheLineSize / 8]uint64
	okN       uint32

	_padding6 [sys.CacheLineSize / 8]uint64
	failN     uint32

	// [nil, item1, item2, ..., itemN]
	_padding7 [sys.CacheLineSize / 8]uint64
	contents  []interface{}
}

// NewBatcher create a new smart batcher instance.
func NewBatcher(capacity int) *Batcher {
	return &Batcher{
		capacity: uint32(capacity),
		stopped:  0,
		w:        1, // 0 is reserved for c, w ranges in [1, n+1]
		r:        1, // 0 is reserved for c, r ranges in [1, n+1]
		c:        0, // [0, n]
		contents: make([]interface{}, capacity+1),
	}
}

// Close closes the batcher from R/W.
func (b *Batcher) Close() {
	atomic.StoreUint32(&b.stopped, 1)
}

// Put writes an item to the batcher. If queue is full, it will block till
// all inflight items marked success.
func (b *Batcher) Put(item interface{}) error {
	for atomic.LoadUint32(&b.w) == b.capacity+1 {
		if atomic.LoadUint32(&b.stopped) == 1 {
			return ErrStopping
		}

		// tail reached, wait for the current batch successful completion
		yield()
	}

	myIndex := atomic.AddUint32(&b.w, 1) - 1
	b.contents[myIndex] = item

	// mark the item available for reading
	atomic.StoreUint32(&b.c, myIndex)
	return nil
}

// Get reads an item from the batcher.
func (b *Batcher) Get() (interface{}, error) {
	for {
		r, w := atomic.LoadUint32(&b.r), atomic.LoadUint32(&b.w)
		if r == b.capacity+1 ||
			// tail reached, but not committed
			r >= w {
			// reader out-run writer

			if atomic.LoadUint32(&b.stopped) == 1 {
				return nil, ErrStopping
			}

			yield()
		} else {
			break
		}
	}

	myIndex := atomic.AddUint32(&b.r, 1) - 1
	for myIndex > atomic.LoadUint32(&b.c) {
		if atomic.LoadUint32(&b.stopped) == 1 {
			return nil, ErrStopping
		}

		// reader out-run commit sequence
		yield()
	}

	return b.contents[int(myIndex)], nil
}

// Succeed marks an item handling success.
func (b *Batcher) Succeed() {
	if okN := atomic.AddUint32(&b.okN, 1); okN == b.capacity {
		// batch commit
		atomic.StoreUint32(&b.okN, 0)
		atomic.StoreUint32(&b.failN, 0)
		atomic.StoreUint32(&b.c, 0)
		atomic.StoreUint32(&b.w, 1)
		atomic.StoreUint32(&b.r, 1)
	} else if okN+atomic.LoadUint32(&b.failN) == b.capacity {
		b.rewind()
	}
}

// Fail marks an item handling failure.
func (b *Batcher) Fail() (rewind bool) {
	if failN := atomic.AddUint32(&b.failN, 1); failN+atomic.LoadUint32(&b.okN) == b.capacity {
		b.rewind()
		rewind = true
	}

	return
}

// batch rollback, reset reader cursor, retry the batch, hold writer
func (b *Batcher) rewind() {
	atomic.StoreUint32(&b.okN, 0)
	atomic.StoreUint32(&b.failN, 0)
	atomic.StoreUint32(&b.r, 1)
}
<commit_msg>wip: add timer for batcher<commit_after>// Package batcher provides retriable batch queue: all succeed or rollback for all.
// In kafka it is called RecordAccumulator.
package batcher

import (
	"sync/atomic"
	"time"

	"github.com/funkygao/dbus/pkg/sys"
)

// Batcher is a batched lock free queue that borrows design from disruptor.
// It maintains a queue with the sematics of all succeed and advance or any fails and retry.
type Batcher struct {
	_padding0 [sys.CacheLineSize / 8]uint64
	capacity  uint32

	_padding1 [sys.CacheLineSize / 8]uint64
	stopped   uint32

	_padding2 [sys.CacheLineSize / 8]uint64
	w         uint32

	_padding3 [sys.CacheLineSize / 8]uint64
	r         uint32

	_padding4 [sys.CacheLineSize / 8]uint64
	c         uint32

	_padding5 [sys.CacheLineSize / 8]uint64
	okN       uint32

	_padding6 [sys.CacheLineSize / 8]uint64
	failN     uint32

	_padding7 [sys.CacheLineSize / 8]uint64
	kickoff   time.Time

	// [nil, item1, item2, ..., itemN]
	_padding8 [sys.CacheLineSize / 8]uint64
	contents  []interface{}
}

// NewBatcher create a new smart batcher instance.
func NewBatcher(capacity int) *Batcher {
	return &Batcher{
		capacity: uint32(capacity),
		stopped:  0,
		w:        1, // 0 is reserved for c, w ranges in [1, n+1]
		r:        1, // 0 is reserved for c, r ranges in [1, n+1]
		c:        0, // [0, n]
		contents: make([]interface{}, capacity+1),
	}
}

// Close closes the batcher from R/W.
func (b *Batcher) Close() {
	atomic.StoreUint32(&b.stopped, 1)
}

// Put writes an item to the batcher. If queue is full, it will block till
// all inflight items marked success.
func (b *Batcher) Put(item interface{}) error {
	for atomic.LoadUint32(&b.w) == b.capacity+1 {
		if atomic.LoadUint32(&b.stopped) == 1 {
			return ErrStopping
		}

		// tail reached, wait for the current batch successful completion
		yield()
	}

	myIndex := atomic.AddUint32(&b.w, 1) - 1
	b.contents[myIndex] = item

	// mark the item available for reading
	atomic.StoreUint32(&b.c, myIndex)
	return nil
}

// Get reads an item from the batcher.
func (b *Batcher) Get() (interface{}, error) {
	for {
		r, w := atomic.LoadUint32(&b.r), atomic.LoadUint32(&b.w)
		if r == b.capacity+1 ||
			// tail reached, but not committed
			r >= w {
			// reader out-run writer

			if atomic.LoadUint32(&b.stopped) == 1 {
				return nil, ErrStopping
			}

			yield()
		} else {
			break
		}
	}

	myIndex := atomic.AddUint32(&b.r, 1) - 1
	for myIndex > atomic.LoadUint32(&b.c) {
		if atomic.LoadUint32(&b.stopped) == 1 {
			return nil, ErrStopping
		}

		// reader out-run commit sequence
		yield()
	}

	return b.contents[int(myIndex)], nil
}

// Succeed marks an item handling success.
func (b *Batcher) Succeed() {
	if okN := atomic.AddUint32(&b.okN, 1); okN == b.capacity {
		// batch commit
		atomic.StoreUint32(&b.okN, 0)
		atomic.StoreUint32(&b.failN, 0)
		atomic.StoreUint32(&b.c, 0)
		atomic.StoreUint32(&b.w, 1)
		atomic.StoreUint32(&b.r, 1)
		b.kickoff = time.Now()
	} else if okN+atomic.LoadUint32(&b.failN) == b.capacity {
		b.rewind()
	}
}

// Fail marks an item handling failure.
func (b *Batcher) Fail() (rewind bool) {
	if failN := atomic.AddUint32(&b.failN, 1); failN+atomic.LoadUint32(&b.okN) == b.capacity {
		b.rewind()
		rewind = true
	}

	return
}

// batch rollback, reset reader cursor, retry the batch, hold writer
func (b *Batcher) rewind() {
	atomic.StoreUint32(&b.okN, 0)
	atomic.StoreUint32(&b.failN, 0)
	atomic.StoreUint32(&b.r, 1)

	b.kickoff = time.Now()
}
<|endoftext|>