<commit_before>// @author Couchbase <info@couchbase.com>
// @copyright 2014 Couchbase, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package repository

import (
	"github.com/couchbase/gometa/common"
	"github.com/couchbase/gometa/log"
	// fdb "github.com/couchbase/goforestdb"
	fdb "github.com/couchbase/indexing/secondary/fdb"
	"sync"
)

/////////////////////////////////////////////////////////////////////////////
// Repository
/////////////////////////////////////////////////////////////////////////////

type RepoKind int

const (
	MAIN RepoKind = iota
	COMMIT_LOG
	SERVER_CONFIG
	LOCAL
)

type Repository struct {
	dbfile    *fdb.File
	stores    map[RepoKind]*fdb.KVStore
	snapshots map[RepoKind][]*Snapshot
	mutex     sync.Mutex
}

type RepoIterator struct {
	iter  *fdb.Iterator
	store *fdb.KVStore
}

type Snapshot struct {
	snapshot *fdb.KVStore
	count    int
	txnid    common.Txnid
	mutex    sync.Mutex
}

/////////////////////////////////////////////////////////////////////////////
// Repository Public Function
/////////////////////////////////////////////////////////////////////////////

//
// Open a repository
//
func OpenRepository() (*Repository, error) {
	return OpenRepositoryWithName(common.REPOSITORY_NAME, uint64(0))
}

func OpenRepositoryWithName(name string, memory_quota uint64) (repo *Repository, err error) {

	if memory_quota < common.MIN_FOREST_DB_CACHE_SIZE {
		memory_quota = common.MIN_FOREST_DB_CACHE_SIZE
	}

	log.Current.Debugf("Repo.OpenRepositoryWithName(): open repo with name %s, buffer cache size %d", name, memory_quota)

	config := fdb.DefaultConfig()
	config.SetBufferCacheSize(memory_quota)
	dbfile, err := fdb.Open(name, config)
	if err != nil {
		return nil, err
	}

	cleanup := common.NewCleanup(func() {
		dbfile.Close()
	})
	defer cleanup.Run()

	stores := make(map[RepoKind]*fdb.KVStore)

	if stores[MAIN], err = dbfile.OpenKVStore("MAIN", nil); err != nil {
		return nil, err
	}
	if stores[COMMIT_LOG], err = dbfile.OpenKVStore("COMMIT_LOG", nil); err != nil {
		return nil, err
	}
	if stores[SERVER_CONFIG], err = dbfile.OpenKVStore("SERVER_CONFIG", nil); err != nil {
		return nil, err
	}
	if stores[LOCAL], err = dbfile.OpenKVStore("LOCAL", nil); err != nil {
		return nil, err
	}
	cleanup.Cancel()

	snapshots := make(map[RepoKind][]*Snapshot)
	snapshots[MAIN] = nil
	snapshots[COMMIT_LOG] = nil
	snapshots[SERVER_CONFIG] = nil
	snapshots[LOCAL] = nil

	repo = &Repository{dbfile: dbfile,
		stores:    stores,
		snapshots: snapshots}

	return repo, nil
}

//
// Update/Insert into the repository
//
func (r *Repository) Set(kind RepoKind, key string, content []byte) error {

	r.mutex.Lock()
	defer r.mutex.Unlock()

	log.Current.Debugf("Repo.Set(): key %s, len(content) %d", key, len(content))

	//convert key to its collatejson encoded byte representation
	k, err := CollateString(key)
	if err != nil {
		return err
	}

	// set value
	err = r.stores[kind].SetKV(k, content)
	if err != nil {
		return err
	}

	return r.dbfile.Commit(fdb.COMMIT_NORMAL)
}

func (r *Repository) CreateSnapshot(kind RepoKind, txnid common.Txnid) error {

	r.mutex.Lock()
	defer r.mutex.Unlock()

	info, err := r.stores[kind].Info()
	if err != nil {
		return err
	}

	fdbSnapshot, err := r.stores[kind].SnapshotOpen(info.LastSeqNum())
	if err != nil {
		return err
	}

	snapshot := &Snapshot{snapshot: fdbSnapshot,
		txnid: txnid,
		count: 0}

	r.pruneSnapshot(kind)

	r.snapshots[kind] = append(r.snapshots[kind], snapshot)

	log.Current.Debugf("Repo.CreateSnapshot(): txnid %v, forestdb seqnum %v", txnid, info.LastSeqNum())
	return nil
}

func (r *Repository) AcquireSnapshot(kind RepoKind) (common.Txnid, *RepoIterator, error) {

	r.mutex.Lock()
	defer r.mutex.Unlock()

	if len(r.snapshots[kind]) == 0 {
		return common.Txnid(0), nil, nil
	}

	snapshot := r.snapshots[kind][len(r.snapshots[kind])-1]
	snapshot.count++

	iter, err := snapshot.snapshot.IteratorInit(nil, nil, fdb.ITR_NO_DELETES)
	if err != nil {
		return common.Txnid(0), nil, err
	}
	return snapshot.txnid, &RepoIterator{iter: iter}, nil
}

func (r *Repository) ReleaseSnapshot(kind RepoKind, txnid common.Txnid) {

	r.mutex.Lock()
	defer r.mutex.Unlock()

	for _, snapshot := range r.snapshots[kind] {
		if snapshot.txnid == txnid && snapshot.count > 0 {
			snapshot.count--
		}
	}
}

func (r *Repository) pruneSnapshot(kind RepoKind) {

	var newList []*Snapshot = nil
	for _, snapshot := range r.snapshots[kind] {
		if snapshot.count > 0 {
			newList = append(newList, snapshot)
		} else {
			// closing snapshot
			snapshot.snapshot.Close()
		}
	}

	r.snapshots[kind] = newList
}

//
// Update/Insert into the repository
//
func (r *Repository) SetNoCommit(kind RepoKind, key string, content []byte) error {

	r.mutex.Lock()
	defer r.mutex.Unlock()

	log.Current.Debugf("Repo.SetNoCommit(): key %s, len(content) %d", key, len(content))

	//convert key to its collatejson encoded byte representation
	k, err := CollateString(key)
	if err != nil {
		return err
	}

	// set value
	return r.stores[kind].SetKV(k, content)
}

//
// Retrieve from repository
//
func (r *Repository) Get(kind RepoKind, key string) ([]byte, error) {

	r.mutex.Lock()
	defer r.mutex.Unlock()

	//convert key to its collatejson encoded byte representation
	k, err := CollateString(key)
	if err != nil {
		return nil, err
	}

	value, err := r.stores[kind].GetKV(k)
	log.Current.Debugf("Repo.Get(): key %s, found=%v", key, err == nil)
	return value, err
}

//
// Delete from repository
//
func (r *Repository) Delete(kind RepoKind, key string) error {

	r.mutex.Lock()
	defer r.mutex.Unlock()

	//convert key to its collatejson encoded byte representation
	k, err := CollateString(key)
	if err != nil {
		return err
	}

	err = r.stores[kind].DeleteKV(k)
	if err != nil {
		return err
	}

	return r.dbfile.Commit(fdb.COMMIT_NORMAL)
}

//
// Delete from repository
//
func (r *Repository) DeleteNoCommit(kind RepoKind, key string) error {

	r.mutex.Lock()
	defer r.mutex.Unlock()

	//convert key to its collatejson encoded byte representation
	k, err := CollateString(key)
	if err != nil {
		return err
	}

	return r.stores[kind].DeleteKV(k)
}

//
// Delete from repository
//
func (r *Repository) Commit() error {

	r.mutex.Lock()
	defer r.mutex.Unlock()

	return r.dbfile.Commit(fdb.COMMIT_NORMAL)
}

//
// Close repository.
//
func (r *Repository) Close() {
	r.mutex.Lock()
	defer r.mutex.Unlock()

	if r.dbfile != nil {
		for _, snapshot := range r.snapshots[MAIN] {
			snapshot.snapshot.Close()
		}
		for _, snapshot := range r.snapshots[COMMIT_LOG] {
			snapshot.snapshot.Close()
		}
		for _, snapshot := range r.snapshots[SERVER_CONFIG] {
			snapshot.snapshot.Close()
		}
		for _, snapshot := range r.snapshots[LOCAL] {
			snapshot.snapshot.Close()
		}
		r.snapshots = nil

		for _, store := range r.stores {
			store.Close()
		}
		r.stores = nil

		r.dbfile.Close()
		r.dbfile = nil
	}
}

/////////////////////////////////////////////////////////////////////////////
// RepoIterator Public Function
/////////////////////////////////////////////////////////////////////////////

//
// Create a new iterator.  EndKey is inclusive.
//
func (r *Repository) NewIterator(kind RepoKind, startKey, endKey string) (*RepoIterator, error) {
	r.mutex.Lock()
	defer r.mutex.Unlock()

	// TODO: Check if fdb is closed.

	k1, err := CollateString(startKey)
	if err != nil {
		return nil, err
	}

	k2, err := CollateString(endKey)
	if err != nil {
		return nil, err
	}

	iter, err := r.stores[kind].IteratorInit(k1, k2, fdb.ITR_NO_DELETES)
	if err != nil {
		return nil, err
	}
	result := &RepoIterator{iter: iter, store: r.stores[kind]}
	return result, nil
}

// Get value from iterator
func (i *RepoIterator) Next() (key string, content []byte, err error) {

	if i.iter == nil {
		return "", nil, fdb.RESULT_ITERATOR_FAIL
	}

	doc, err := i.iter.Get()
	if err != nil {
		return "", nil, err
	}

	err = i.iter.Next()
	if err != nil && err != fdb.RESULT_ITERATOR_FAIL {
		return "", nil, err
	}

	//i.db.Get(doc)
	key = DecodeString(doc.Key())
	body := doc.Body()

	if err == fdb.RESULT_ITERATOR_FAIL {
		i.iter = nil
	}

	return key, body, nil
}

// close iterator
func (i *RepoIterator) Close() {
	// TODO: Check if fdb iterator is closed
	if i.iter != nil {
		i.iter.Close()
		i.iter = nil
	}
}

// This only support ascii.
func CollateString(key string) ([]byte, error) {
	if key == "" {
		return nil, nil
	}

	return ([]byte)(key), nil
}

func DecodeString(data []byte) string {
	return string(data)
}
<commit_msg>Undo mutex lock -- use advisory lock from 2i fdb clone<commit_after>// @author Couchbase <info@couchbase.com>
// @copyright 2014 Couchbase, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package repository

import (
	"github.com/couchbase/gometa/common"
	"github.com/couchbase/gometa/log"
	// fdb "github.com/couchbase/goforestdb"
	fdb "github.com/couchbase/indexing/secondary/fdb"
	"sync"
)

/////////////////////////////////////////////////////////////////////////////
// Repository
/////////////////////////////////////////////////////////////////////////////

type RepoKind int

const (
	MAIN RepoKind = iota
	COMMIT_LOG
	SERVER_CONFIG
	LOCAL
)

type Repository struct {
	dbfile    *fdb.File
	stores    map[RepoKind]*fdb.KVStore
	snapshots map[RepoKind][]*Snapshot
	mutex     sync.Mutex
}

type RepoIterator struct {
	iter  *fdb.Iterator
	store *fdb.KVStore
}

type Snapshot struct {
	snapshot *fdb.KVStore
	count    int
	txnid    common.Txnid
	mutex    sync.Mutex
}

/////////////////////////////////////////////////////////////////////////////
// Repository Public Function
/////////////////////////////////////////////////////////////////////////////

//
// Open a repository
//
func OpenRepository() (*Repository, error) {
	return OpenRepositoryWithName(common.REPOSITORY_NAME, uint64(0))
}

func OpenRepositoryWithName(name string, memory_quota uint64) (repo *Repository, err error) {

	if memory_quota < common.MIN_FOREST_DB_CACHE_SIZE {
		memory_quota = common.MIN_FOREST_DB_CACHE_SIZE
	}

	log.Current.Debugf("Repo.OpenRepositoryWithName(): open repo with name %s, buffer cache size %d", name, memory_quota)

	config := fdb.DefaultConfig()
	config.SetBufferCacheSize(memory_quota)
	dbfile, err := fdb.Open(name, config)
	if err != nil {
		return nil, err
	}

	cleanup := common.NewCleanup(func() {
		dbfile.Close()
	})
	defer cleanup.Run()

	stores := make(map[RepoKind]*fdb.KVStore)

	if stores[MAIN], err = dbfile.OpenKVStore("MAIN", nil); err != nil {
		return nil, err
	}
	if stores[COMMIT_LOG], err = dbfile.OpenKVStore("COMMIT_LOG", nil); err != nil {
		return nil, err
	}
	if stores[SERVER_CONFIG], err = dbfile.OpenKVStore("SERVER_CONFIG", nil); err != nil {
		return nil, err
	}
	if stores[LOCAL], err = dbfile.OpenKVStore("LOCAL", nil); err != nil {
		return nil, err
	}
	cleanup.Cancel()

	snapshots := make(map[RepoKind][]*Snapshot)
	snapshots[MAIN] = nil
	snapshots[COMMIT_LOG] = nil
	snapshots[SERVER_CONFIG] = nil
	snapshots[LOCAL] = nil

	repo = &Repository{dbfile: dbfile,
		stores:    stores,
		snapshots: snapshots}

	return repo, nil
}

//
// Update/Insert into the repository
//
func (r *Repository) Set(kind RepoKind, key string, content []byte) error {

	r.mutex.Lock()
	defer r.mutex.Unlock()

	log.Current.Debugf("Repo.Set(): key %s, len(content) %d", key, len(content))

	//convert key to its collatejson encoded byte representation
	k, err := CollateString(key)
	if err != nil {
		return err
	}

	// set value
	err = r.stores[kind].SetKV(k, content)
	if err != nil {
		return err
	}

	return r.dbfile.Commit(fdb.COMMIT_NORMAL)
}

func (r *Repository) CreateSnapshot(kind RepoKind, txnid common.Txnid) error {

	r.mutex.Lock()
	defer r.mutex.Unlock()

	info, err := r.stores[kind].Info()
	if err != nil {
		return err
	}

	fdbSnapshot, err := r.stores[kind].SnapshotOpen(info.LastSeqNum())
	if err != nil {
		return err
	}

	snapshot := &Snapshot{snapshot: fdbSnapshot,
		txnid: txnid,
		count: 0}

	r.pruneSnapshot(kind)

	r.snapshots[kind] = append(r.snapshots[kind], snapshot)

	log.Current.Debugf("Repo.CreateSnapshot(): txnid %v, forestdb seqnum %v", txnid, info.LastSeqNum())
	return nil
}

func (r *Repository) AcquireSnapshot(kind RepoKind) (common.Txnid, *RepoIterator, error) {

	r.mutex.Lock()
	defer r.mutex.Unlock()

	if len(r.snapshots[kind]) == 0 {
		return common.Txnid(0), nil, nil
	}

	snapshot := r.snapshots[kind][len(r.snapshots[kind])-1]
	snapshot.count++

	iter, err := snapshot.snapshot.IteratorInit(nil, nil, fdb.ITR_NO_DELETES)
	if err != nil {
		return common.Txnid(0), nil, err
	}
	return snapshot.txnid, &RepoIterator{iter: iter}, nil
}

func (r *Repository) ReleaseSnapshot(kind RepoKind, txnid common.Txnid) {

	r.mutex.Lock()
	defer r.mutex.Unlock()

	for _, snapshot := range r.snapshots[kind] {
		if snapshot.txnid == txnid && snapshot.count > 0 {
			snapshot.count--
		}
	}
}

func (r *Repository) pruneSnapshot(kind RepoKind) {

	var newList []*Snapshot = nil
	for _, snapshot := range r.snapshots[kind] {
		if snapshot.count > 0 {
			newList = append(newList, snapshot)
		} else {
			// closing snapshot
			snapshot.snapshot.Close()
		}
	}

	r.snapshots[kind] = newList
}

//
// Update/Insert into the repository
//
func (r *Repository) SetNoCommit(kind RepoKind, key string, content []byte) error {

	r.mutex.Lock()
	defer r.mutex.Unlock()

	log.Current.Debugf("Repo.SetNoCommit(): key %s, len(content) %d", key, len(content))

	//convert key to its collatejson encoded byte representation
	k, err := CollateString(key)
	if err != nil {
		return err
	}

	// set value
	return r.stores[kind].SetKV(k, content)
}

//
// Retrieve from repository
//
func (r *Repository) Get(kind RepoKind, key string) ([]byte, error) {

	//convert key to its collatejson encoded byte representation
	k, err := CollateString(key)
	if err != nil {
		return nil, err
	}

	value, err := r.stores[kind].GetKV(k)
	log.Current.Debugf("Repo.Get(): key %s, found=%v", key, err == nil)
	return value, err
}

//
// Delete from repository
//
func (r *Repository) Delete(kind RepoKind, key string) error {

	r.mutex.Lock()
	defer r.mutex.Unlock()

	//convert key to its collatejson encoded byte representation
	k, err := CollateString(key)
	if err != nil {
		return err
	}

	err = r.stores[kind].DeleteKV(k)
	if err != nil {
		return err
	}

	return r.dbfile.Commit(fdb.COMMIT_NORMAL)
}

//
// Delete from repository
//
func (r *Repository) DeleteNoCommit(kind RepoKind, key string) error {

	r.mutex.Lock()
	defer r.mutex.Unlock()

	//convert key to its collatejson encoded byte representation
	k, err := CollateString(key)
	if err != nil {
		return err
	}

	return r.stores[kind].DeleteKV(k)
}

//
// Delete from repository
//
func (r *Repository) Commit() error {

	r.mutex.Lock()
	defer r.mutex.Unlock()

	return r.dbfile.Commit(fdb.COMMIT_NORMAL)
}

//
// Close repository.
//
func (r *Repository) Close() {
	r.mutex.Lock()
	defer r.mutex.Unlock()

	if r.dbfile != nil {
		for _, snapshot := range r.snapshots[MAIN] {
			snapshot.snapshot.Close()
		}
		for _, snapshot := range r.snapshots[COMMIT_LOG] {
			snapshot.snapshot.Close()
		}
		for _, snapshot := range r.snapshots[SERVER_CONFIG] {
			snapshot.snapshot.Close()
		}
		for _, snapshot := range r.snapshots[LOCAL] {
			snapshot.snapshot.Close()
		}
		r.snapshots = nil

		for _, store := range r.stores {
			store.Close()
		}
		r.stores = nil

		r.dbfile.Close()
		r.dbfile = nil
	}
}

/////////////////////////////////////////////////////////////////////////////
// RepoIterator Public Function
/////////////////////////////////////////////////////////////////////////////

//
// Create a new iterator.  EndKey is inclusive.
//
func (r *Repository) NewIterator(kind RepoKind, startKey, endKey string) (*RepoIterator, error) {

	// TODO: Check if fdb is closed.

	k1, err := CollateString(startKey)
	if err != nil {
		return nil, err
	}

	k2, err := CollateString(endKey)
	if err != nil {
		return nil, err
	}

	iter, err := r.stores[kind].IteratorInit(k1, k2, fdb.ITR_NO_DELETES)
	if err != nil {
		return nil, err
	}
	result := &RepoIterator{iter: iter, store: r.stores[kind]}
	return result, nil
}

// Get value from iterator
func (i *RepoIterator) Next() (key string, content []byte, err error) {

	if i.iter == nil {
		return "", nil, fdb.RESULT_ITERATOR_FAIL
	}

	doc, err := i.iter.Get()
	if err != nil {
		return "", nil, err
	}

	err = i.iter.Next()
	if err != nil && err != fdb.RESULT_ITERATOR_FAIL {
		return "", nil, err
	}

	//i.db.Get(doc)
	key = DecodeString(doc.Key())
	body := doc.Body()

	if err == fdb.RESULT_ITERATOR_FAIL {
		i.iter = nil
	}

	return key, body, nil
}

// close iterator
func (i *RepoIterator) Close() {
	// TODO: Check if fdb iterator is closed
	if i.iter != nil {
		i.iter.Close()
		i.iter = nil
	}
}

// This only support ascii.
func CollateString(key string) ([]byte, error) {
	if key == "" {
		return nil, nil
	}

	return ([]byte)(key), nil
}

func DecodeString(data []byte) string {
	return string(data)
}
<|endoftext|>


<commit_before>package ftp

import (
	"path"
)

//Walker traverses the directory tree of a remote FTP server
type Walker struct {
	serverConn *ServerConn
	root       string
	cur        *item
	stack      []*item
	descend    bool
}

type item struct {
	path  string
	entry *Entry
	err   error
}

// Next advances the Walker to the next file or directory,
// which will then be available through the Path, Stat, and Err methods.
// It returns false when the walk stops at the end of the tree.
func (w *Walker) Next() bool {
	// check if we need to init cur, maybe this should be inside Walk
	if w.cur == nil {
		w.cur = &item{
			path: w.root,
			entry: &Entry{
				Type: EntryTypeFolder,
			},
		}
	}

	if w.descend && w.cur.entry.Type == EntryTypeFolder {
		entries, err := w.serverConn.List(w.cur.path)

		// an error occured, drop out and stop walking
		if err != nil {
			w.cur.err = err
			return false
		}

		for _, entry := range entries {
			if entry.Name == "." || entry.Name == ".." {
				continue
			}

			item := &item{
				path:  path.Join(w.cur.path, entry.Name),
				entry: entry,
			}

			w.stack = append(w.stack, item)
		}
	}

	if len(w.stack) == 0 {
		return false
	}

	// update cur
	i := len(w.stack) - 1
	w.cur = w.stack[i]
	w.stack = w.stack[:i]

	// reset SkipDir
	w.descend = true

	return true
}

//SkipDir tells the Next function to skip the currently processed directory
func (w *Walker) SkipDir() {
	w.descend = false
}

//Err returns the error, if any, for the most recent attempt by Next to
//visit a file or a directory. If a directory has an error, the walker
//will not descend in that directory
func (w *Walker) Err() error {
	return w.cur.err
}

// Stat returns info for the most recent file or directory
// visited by a call to Step.
func (w *Walker) Stat() *Entry {
	return w.cur.entry
}

// Path returns the path to the most recent file or directory
// visited by a call to Next. It contains the argument to Walk
// as a prefix; that is, if Walk is called with "dir", which is
// a directory containing the file "a", Path will return "dir/a".
func (w *Walker) Path() string {
	return w.cur.path
}
<commit_msg>Fix spelling error reported by goreportcard.com<commit_after>package ftp

import (
	"path"
)

//Walker traverses the directory tree of a remote FTP server
type Walker struct {
	serverConn *ServerConn
	root       string
	cur        *item
	stack      []*item
	descend    bool
}

type item struct {
	path  string
	entry *Entry
	err   error
}

// Next advances the Walker to the next file or directory,
// which will then be available through the Path, Stat, and Err methods.
// It returns false when the walk stops at the end of the tree.
func (w *Walker) Next() bool {
	// check if we need to init cur, maybe this should be inside Walk
	if w.cur == nil {
		w.cur = &item{
			path: w.root,
			entry: &Entry{
				Type: EntryTypeFolder,
			},
		}
	}

	if w.descend && w.cur.entry.Type == EntryTypeFolder {
		entries, err := w.serverConn.List(w.cur.path)

		// an error occurred, drop out and stop walking
		if err != nil {
			w.cur.err = err
			return false
		}

		for _, entry := range entries {
			if entry.Name == "." || entry.Name == ".." {
				continue
			}

			item := &item{
				path:  path.Join(w.cur.path, entry.Name),
				entry: entry,
			}

			w.stack = append(w.stack, item)
		}
	}

	if len(w.stack) == 0 {
		return false
	}

	// update cur
	i := len(w.stack) - 1
	w.cur = w.stack[i]
	w.stack = w.stack[:i]

	// reset SkipDir
	w.descend = true

	return true
}

//SkipDir tells the Next function to skip the currently processed directory
func (w *Walker) SkipDir() {
	w.descend = false
}

//Err returns the error, if any, for the most recent attempt by Next to
//visit a file or a directory. If a directory has an error, the walker
//will not descend in that directory
func (w *Walker) Err() error {
	return w.cur.err
}

// Stat returns info for the most recent file or directory
// visited by a call to Step.
func (w *Walker) Stat() *Entry {
	return w.cur.entry
}

// Path returns the path to the most recent file or directory
// visited by a call to Next. It contains the argument to Walk
// as a prefix; that is, if Walk is called with "dir", which is
// a directory containing the file "a", Path will return "dir/a".
func (w *Walker) Path() string {
	return w.cur.path
}
<|endoftext|>


<commit_before><commit_msg>Combine log lines into single line<commit_after><|endoftext|>


<commit_before><commit_msg>Manually send open message on server sent events channel<commit_after><|endoftext|>


<commit_before>package token

import "fmt"

type TokenID int

const (
	LEFT_PAREN TokenID = iota
	RIGHT_PAREN
	LEFT_BRACE
	RIGHT_BRACE
	COMMA
	DOT
	MINUS
	PLUS
	SEMICOLON
	SLASH
	STAR
	BANG
	BANG_EQUAL
	EQUAL
	EQUAL_EQUAL
	GREATER
	GREATER_EQUAL
	LESS
	LESS_EQUAL
	IDENTIFIER
	STRING
	NUMBER
	AND
	ELSE
	FALSE
	FOR
	IF
	NIL
	OR
	RETURN
	TRUE
	VAR
	EOF
	COMMENT
	WHITESPACE
)

var TokenIDString = map[TokenID]string{
	LEFT_PAREN:    "LEFT_PAREN",
	RIGHT_PAREN:   "RIGHT_PAREN",
	LEFT_BRACE:    "LEFT_BRACE",
	RIGHT_BRACE:   "RIGHT_BRACE",
	COMMA:         "COMMA",
	DOT:           "DOT",
	MINUS:         "MINUS",
	PLUS:          "PLUS",
	SEMICOLON:     "SEMICOLON",
	SLASH:         "SLASH",
	STAR:          "STAR",
	BANG:          "BANG",
	BANG_EQUAL:    "BANG_EQUAL",
	EQUAL:         "EQUAL",
	EQUAL_EQUAL:   "EQUAL_EQUAL",
	GREATER:       "GREATER",
	GREATER_EQUAL: "GREATER_EQUAL",
	LESS:          "LESS",
	LESS_EQUAL:    "LESS_EQUAL",
	IDENTIFIER:    "IDENTIFIER",
	STRING:        "STRING",
	NUMBER:        "NUMBER",
	AND:           "AND",
	ELSE:          "ELSE",
	FALSE:         "FALSE",
	FOR:           "FOR",
	IF:            "IF",
	NIL:           "NIL",
	OR:            "OR",
	RETURN:        "RETURN",
	TRUE:          "TRUE",
	VAR:           "VAR",
	EOF:           "EOF",
	COMMENT:       "COMMENT",
	WHITESPACE:    "WHITESPACE",
}

var IdentifiersList = map[string]TokenID{
	"and":    AND,
	"else":   ELSE,
	"false":  FALSE,
	"for":    FOR,
	"if":     IF,
	"nil":    NIL,
	"or":     OR,
	"return": RETURN,
	"true":   TRUE,
	"var":    VAR,
}

type Token struct {
	ID    TokenID
	Value string
}

func NewToken(id TokenID, value string) *Token {
	return &Token{ID: id, Value: value}
}

func (t *Token) String() string {
	return fmt.Sprintf("[%s] %s", TokenIDString[t.ID], t.Value)
}
<commit_msg>token: reorder tokens for readability<commit_after>package token

import "fmt"

type TokenID int

const (
	LEFT_PAREN TokenID = iota
	RIGHT_PAREN
	LEFT_BRACE
	RIGHT_BRACE
	COMMA
	DOT
	MINUS
	PLUS
	SEMICOLON
	SLASH
	STAR
	BANG
	BANG_EQUAL
	EQUAL
	EQUAL_EQUAL
	GREATER
	GREATER_EQUAL
	LESS
	LESS_EQUAL
	IDENTIFIER
	AND
	IF
	ELSE
	TRUE
	FALSE
	FOR
	OR
	RETURN
	VAR
	EOF
	COMMENT
	WHITESPACE
	STRING
	NUMBER
	NIL
)

var TokenIDString = map[TokenID]string{
	LEFT_PAREN:    "LEFT_PAREN",
	RIGHT_PAREN:   "RIGHT_PAREN",
	LEFT_BRACE:    "LEFT_BRACE",
	RIGHT_BRACE:   "RIGHT_BRACE",
	COMMA:         "COMMA",
	DOT:           "DOT",
	MINUS:         "MINUS",
	PLUS:          "PLUS",
	SEMICOLON:     "SEMICOLON",
	SLASH:         "SLASH",
	STAR:          "STAR",
	BANG:          "BANG",
	BANG_EQUAL:    "BANG_EQUAL",
	EQUAL:         "EQUAL",
	EQUAL_EQUAL:   "EQUAL_EQUAL",
	GREATER:       "GREATER",
	GREATER_EQUAL: "GREATER_EQUAL",
	LESS:          "LESS",
	LESS_EQUAL:    "LESS_EQUAL",
	IDENTIFIER:    "IDENTIFIER",
	AND:           "AND",
	IF:            "IF",
	ELSE:          "ELSE",
	TRUE:          "TRUE",
	FALSE:         "FALSE",
	FOR:           "FOR",
	OR:            "OR",
	RETURN:        "RETURN",
	VAR:           "VAR",
	EOF:           "EOF",
	COMMENT:       "COMMENT",
	WHITESPACE:    "WHITESPACE",
	STRING:        "STRING",
	NUMBER:        "NUMBER",
	NIL:           "NIL",
}

var IdentifiersList = map[string]TokenID{
	"and":    AND,
	"if":     IF,
	"else":   ELSE,
	"true":   TRUE,
	"false":  FALSE,
	"for":    FOR,
	"nil":    NIL,
	"or":     OR,
	"return": RETURN,
	"var":    VAR,
}

type Token struct {
	ID    TokenID
	Value string
}

func NewToken(id TokenID, value string) *Token {
	return &Token{ID: id, Value: value}
}

func (t *Token) String() string {
	return fmt.Sprintf("[%s] %s", TokenIDString[t.ID], t.Value)
}
<|endoftext|>


<commit_before>// tonic lets you write simpler gin handlers.
// The way it works is that it generates wrapping gin-compatible handlers,
// that do all the repetitive work and wrap the call to your simple tonic handler.
//
// tonic handles path/query/body parameter binding in a single consolidated input object
// which allows you to remove all the boilerplate code that retrieves and tests the presence
// of various parameters.
//
// Example input object:
// type MyInput struct {
//     Foo int `path:"foo, required"`
//     Bar float `query:"bar, default=foobar"`
//     Baz string `json:"baz" binding:"required"`
// }
//
// Output objects can be of any type, and will be marshaled to JSON.
//
// The handler can return an error, which will be returned to the caller.
package tonic

import (
	"encoding"
	"fmt"
	"reflect"
	"runtime"
	"strconv"
	"strings"

	"github.com/gin-gonic/gin"
)

const (
	query_tag = "query"
	path_tag  = "path"
)

// An ErrorHook lets you interpret errors returned by your handlers.
// After analysis, the hook should return a suitable http status code and
// error payload.
// This lets you deeply inspect custom error types.
// See sub-package 'jujuerrhook' for a ready-to-use implementation
// that relies on juju/errors (https://github.com/juju/errors).
type ErrorHook func(error) (int, interface{})

type ExecHook func(*gin.Context, gin.HandlerFunc, string)

var (
	errorHook ErrorHook = DefaultErrorHook
	execHook  ExecHook  = DefaultExecHook
	routes              = make(map[string]*Route)
)

func GetRoutes() map[string]*Route {
	return routes
}

// SetErrorHook lets you set your own error hook.
func SetErrorHook(eh ErrorHook) {
	if eh != nil {
		errorHook = eh
	}
}

func GetErrorHook() ErrorHook {
	return errorHook
}

func SetExecHook(eh ExecHook) {
	if eh != nil {
		execHook = eh
	}
}

func GetExecHook() ExecHook {
	return execHook
}

func DefaultExecHook(c *gin.Context, h gin.HandlerFunc, fname string) { h(c) }
func DefaultErrorHook(e error) (int, interface{})                     { return 400, e.Error() }

// Handler returns a wrapping gin-compatible handler that calls the tonic handler
// passed in parameter.
// The tonic handler may use the following signature:
// func(*gin.Context, [input object ptr]) ([output object], error)
// Input and output objects are both optional (tonic analyzes the handler signature reflexively).
// As such, the minimal accepted signature is:
// func(*gin.Context) error
// The wrapping gin-handler will handle the binding code (JSON + path/query)
// and the error handling.
// This will PANIC if the tonic-handler is of incompatible type.
func Handler(f interface{}, retcode int) gin.HandlerFunc {

	fval := reflect.ValueOf(f)
	ftype := fval.Type()
	fname := runtime.FuncForPC(fval.Pointer()).Name()

	if fval.Kind() != reflect.Func {
		panic(fmt.Sprintf("Handler parameter not a function: %T", f))
	}

	var typeIn reflect.Type
	var typeOut reflect.Type

	// Check tonic-handler inputs
	numin := ftype.NumIn()
	if numin < 1 || numin > 2 {
		panic(fmt.Sprintf("Incorrect number of handler input params: %d", numin))
	}
	hasIn := (numin == 2)
	if !ftype.In(0).ConvertibleTo(reflect.TypeOf(&gin.Context{})) {
		panic(fmt.Sprintf("Unsupported type for handler input parameter: %v. Should be gin.Context.", ftype.In(0)))
	}
	if hasIn {
		if ftype.In(1).Kind() != reflect.Ptr && ftype.In(1).Elem().Kind() != reflect.Struct {
			panic(fmt.Sprintf("Unsupported type for handler input parameter: %v. Should be struct ptr.", ftype.In(1)))
		} else {
			typeIn = ftype.In(1)
		}
	}

	// Check tonic handler outputs
	numout := ftype.NumOut()
	if numout < 1 || numout > 2 {
		panic(fmt.Sprintf("Incorrect number of handler output params: %d", numout))
	}
	hasOut := (numout == 2)
	errIdx := 0
	if hasOut {
		errIdx += 1
		typeOut = ftype.Out(0)
	}
	typeOfError := reflect.TypeOf((*error)(nil)).Elem()
	if !ftype.Out(errIdx).Implements(typeOfError) {
		panic(fmt.Sprintf("Unsupported type for handler output parameter: %v. Should be error.", ftype.Out(errIdx)))
	}

	// Wrapping gin-handler
	retfunc := func(c *gin.Context) {

		funcIn := []reflect.Value{reflect.ValueOf(c)}

		if hasIn {
			// tonic-handler has custom input object, handle binding
			input := reflect.New(typeIn.Elem())
			err := c.Bind(input.Interface())
			if err != nil {
				c.JSON(400, gin.H{`error`: err.Error()})
				return
			}
			err = bindQueryPath(c, input, query_tag, extractQuery)
			if err != nil {
				c.JSON(400, gin.H{`error`: err.Error()})
				return
			}
			err = bindQueryPath(c, input, path_tag, extractPath)
			if err != nil {
				c.JSON(400, gin.H{`error`: err.Error()})
				return
			}
			funcIn = append(funcIn, input)
		}

		// Call tonic-handler
		ret := fval.Call(funcIn)
		var errOut interface{}
		var outval interface{}
		if hasOut {
			outval = ret[0].Interface()
			errOut = ret[1].Interface()
		} else {
			errOut = ret[0].Interface()
		}
		// Raised error, handle it
		if errOut != nil {
			reterr := errOut.(error)
			c.Error(reterr)
			errcode, errpl := errorHook(reterr)
			if errpl != nil {
				c.JSON(errcode, gin.H{`error`: errpl})
			} else {
				c.String(errcode, "")
			}
			return
		}
		// Normal output, either serialize custom output object or send empty body
		if hasOut {
			c.JSON(retcode, outval)
		} else {
			c.String(retcode, "")
		}
	}

	routes[fname] = &Route{
		defaultStatusCode: retcode,
		handler:           fval,
		handlerType:       ftype,
		inputType:         typeIn,
		outputType:        typeOut,
	}

	return func(c *gin.Context) { execHook(c, retfunc, fname) }
}

func bindQueryPath(c *gin.Context, in reflect.Value, targetTag string, extractor func(*gin.Context, string) (string, []string, error)) error {

	inType := in.Type().Elem()

	for i := 0; i < in.Elem().NumField(); i++ {
		fieldType := inType.Field(i)
		tag := fieldType.Tag.Get(targetTag)
		if tag == "" {
			continue
		}
		name, values, err := extractor(c, tag)
		if err != nil {
			return err
		}
		if len(values) == 0 {
			continue
		}
		field := in.Elem().Field(i)
		if field.Kind() == reflect.Ptr {
			f := reflect.New(field.Type().Elem())
			field.Set(f)
			field = field.Elem()
		}
		if field.Kind() == reflect.Slice {
			for _, v := range values {
				newV := reflect.New(field.Type().Elem()).Elem()
				err := bindValue(v, newV)
				if err != nil {
					return err
				}
				field.Set(reflect.Append(field, newV))
			}
			return nil
		} else if len(values) > 1 {
			return fmt.Errorf("Query parameter '%s' does not support multiple values", name)
		} else {
			err = bindValue(values[0], field)
			if err != nil {
				return err
			}
		}
	}

	return nil
}

func extractQuery(c *gin.Context, tag string) (string, []string, error) {

	name, required, defval, err := ExtractTag(tag, true)
	if err != nil {
		return "", nil, err
	}

	q := c.Request.URL.Query()[name]

	if defval != "" && len(q) == 0 {
		q = []string{defval}
	}

	if required && len(q) == 0 {
		return "", nil, fmt.Errorf("Field %s is missing: required.", name)
	}

	return name, q, nil
}

func extractPath(c *gin.Context, tag string) (string, []string, error) {

	name, required, _, err := ExtractTag(tag, false)
	if err != nil {
		return "", nil, err
	}

	out := c.Param(name)
	if required && out == "" {
		return "", nil, fmt.Errorf("Field %s is missing: required.", name)
	}
	return name, []string{out}, nil
}

func ExtractTag(tag string, defaultValue bool) (string, bool, string, error) {

	parts := strings.SplitN(tag, ",", -1)
	name := parts[0]
	options := parts[1:]

	var defval string
	var required bool
	for _, o := range options {
		o = strings.TrimSpace(o)
		if o == "required" {
			required = true
		} else if defaultValue && strings.HasPrefix(o, "default=") {
			o = strings.TrimPrefix(o, "default=")
			defval = o
		} else {
			return "", false, "", fmt.Errorf("Malformed tag for param '%s': unknown opt '%s'", name, o)
		}
	}
	return name, required, defval, nil
}

func bindValue(s string, v reflect.Value) error {

	vIntf := reflect.New(v.Type()).Interface()
	unmarshaler, ok := vIntf.(encoding.TextUnmarshaler)
	if ok {
		err := unmarshaler.UnmarshalText([]byte(s))
		if err != nil {
			return err
		}
		v.Set(reflect.Indirect(reflect.ValueOf(unmarshaler)))
		return nil
	}

	switch v.Kind() {
	case reflect.String:
		v.SetString(s)
	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
		i, err := strconv.ParseInt(s, 10, v.Type().Bits())
		if err != nil {
			return err
		}
		v.SetInt(i)
	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
		i, err := strconv.ParseUint(s, 10, v.Type().Bits())
		if err != nil {
			return err
		}
		v.SetUint(i)
	case reflect.Bool:
		b, err := strconv.ParseBool(s)
		if err != nil {
			return err
		}
		v.SetBool(b)
	default:
		return fmt.Errorf("Unsupported type for query param bind: %v", v.Kind())
	}
	return nil
}
<commit_msg>use json.numbers when decoding json (avoid float64 -> scientific notation for large integers)<commit_after>// tonic lets you write simpler gin handlers.
// The way it works is that it generates wrapping gin-compatible handlers,
// that do all the repetitive work and wrap the call to your simple tonic handler.
//
// tonic handles path/query/body parameter binding in a single consolidated input object
// which allows you to remove all the boilerplate code that retrieves and tests the presence
// of various parameters.
//
// Example input object:
// type MyInput struct {
//     Foo int `path:"foo, required"`
//     Bar float `query:"bar, default=foobar"`
//     Baz string `json:"baz" binding:"required"`
// }
//
// Output objects can be of any type, and will be marshaled to JSON.
//
// The handler can return an error, which will be returned to the caller.
package tonic

import (
	"encoding"
	"fmt"
	"reflect"
	"runtime"
	"strconv"
	"strings"

	"github.com/gin-gonic/gin"
)

const (
	query_tag = "query"
	path_tag  = "path"
)

// An ErrorHook lets you interpret errors returned by your handlers.
// After analysis, the hook should return a suitable http status code and
// error payload.
// This lets you deeply inspect custom error types.
// See sub-package 'jujuerrhook' for a ready-to-use implementation
// that relies on juju/errors (https://github.com/juju/errors).
type ErrorHook func(error) (int, interface{})

type ExecHook func(*gin.Context, gin.HandlerFunc, string)

type BindHook func(*gin.Context, interface{}) error

var (
	DefaultBindingHook           = (*gin.Context).Bind
	errorHook          ErrorHook = DefaultErrorHook
	execHook           ExecHook  = DefaultExecHook
	bindHook           BindHook  = DefaultBindingHook
	routes                       = make(map[string]*Route)
)

func GetRoutes() map[string]*Route {
	return routes
}

// SetErrorHook lets you set your own error hook.
func SetErrorHook(eh ErrorHook) {
	if eh != nil {
		errorHook = eh
	}
}

func GetErrorHook() ErrorHook {
	return errorHook
}

func SetExecHook(eh ExecHook) {
	if eh != nil {
		execHook = eh
	}
}

func GetExecHook() ExecHook {
	return execHook
}

func SetBindHook(bh BindHook) {
	if bh != nil {
		bindHook = bh
	}
}

func GetBindHook() BindHook {
	return bindHook
}

func DefaultExecHook(c *gin.Context, h gin.HandlerFunc, fname string) { h(c) }
func DefaultErrorHook(e error) (int, interface{})                     { return 400, e.Error() }

// Handler returns a wrapping gin-compatible handler that calls the tonic handler
// passed in parameter.
// The tonic handler may use the following signature:
// func(*gin.Context, [input object ptr]) ([output object], error)
// Input and output objects are both optional (tonic analyzes the handler signature reflexively).
// As such, the minimal accepted signature is:
// func(*gin.Context) error
// The wrapping gin-handler will handle the binding code (JSON + path/query)
// and the error handling.
// This will PANIC if the tonic-handler is of incompatible type.
func Handler(f interface{}, retcode int) gin.HandlerFunc {

	fval := reflect.ValueOf(f)
	ftype := fval.Type()
	fname := runtime.FuncForPC(fval.Pointer()).Name()

	if fval.Kind() != reflect.Func {
		panic(fmt.Sprintf("Handler parameter not a function: %T", f))
	}

	var typeIn reflect.Type
	var typeOut reflect.Type

	// Check tonic-handler inputs
	numin := ftype.NumIn()
	if numin < 1 || numin > 2 {
		panic(fmt.Sprintf("Incorrect number of handler input params: %d", numin))
	}
	hasIn := (numin == 2)
	if !ftype.In(0).ConvertibleTo(reflect.TypeOf(&gin.Context{})) {
		panic(fmt.Sprintf("Unsupported type for handler input parameter: %v. Should be gin.Context.", ftype.In(0)))
	}
	if hasIn {
		if ftype.In(1).Kind() != reflect.Ptr && ftype.In(1).Elem().Kind() != reflect.Struct {
			panic(fmt.Sprintf("Unsupported type for handler input parameter: %v. Should be struct ptr.", ftype.In(1)))
		} else {
			typeIn = ftype.In(1)
		}
	}

	// Check tonic handler outputs
	numout := ftype.NumOut()
	if numout < 1 || numout > 2 {
		panic(fmt.Sprintf("Incorrect number of handler output params: %d", numout))
	}
	hasOut := (numout == 2)
	errIdx := 0
	if hasOut {
		errIdx += 1
		typeOut = ftype.Out(0)
	}
	typeOfError := reflect.TypeOf((*error)(nil)).Elem()
	if !ftype.Out(errIdx).Implements(typeOfError) {
		panic(fmt.Sprintf("Unsupported type for handler output parameter: %v. Should be error.", ftype.Out(errIdx)))
	}

	// Wrapping gin-handler
	retfunc := func(c *gin.Context) {

		funcIn := []reflect.Value{reflect.ValueOf(c)}

		if hasIn {
			// tonic-handler has custom input object, handle binding
			input := reflect.New(typeIn.Elem())
			err := bindHook(c, input.Interface())
			if err != nil {
				c.JSON(400, gin.H{`error`: err.Error()})
				return
			}
			err = bindQueryPath(c, input, query_tag, extractQuery)
			if err != nil {
				c.JSON(400, gin.H{`error`: err.Error()})
				return
			}
			err = bindQueryPath(c, input, path_tag, extractPath)
			if err != nil {
				c.JSON(400, gin.H{`error`: err.Error()})
				return
			}
			funcIn = append(funcIn, input)
		}

		// Call tonic-handler
		ret := fval.Call(funcIn)
		var errOut interface{}
		var outval interface{}
		if hasOut {
			outval = ret[0].Interface()
			errOut = ret[1].Interface()
		} else {
			errOut = ret[0].Interface()
		}
		// Raised error, handle it
		if errOut != nil {
			reterr := errOut.(error)
			c.Error(reterr)
			errcode, errpl := errorHook(reterr)
			if errpl != nil {
				c.JSON(errcode, gin.H{`error`: errpl})
			} else {
				c.String(errcode, "")
			}
			return
		}
		// Normal output, either serialize custom output object or send empty body
		if hasOut {
			c.JSON(retcode, outval)
		} else {
			c.String(retcode, "")
		}
	}

	routes[fname] = &Route{
		defaultStatusCode: retcode,
		handler:           fval,
		handlerType:       ftype,
		inputType:         typeIn,
		outputType:        typeOut,
	}

	return func(c *gin.Context) { execHook(c, retfunc, fname) }
}

func bindQueryPath(c *gin.Context, in reflect.Value, targetTag string, extractor func(*gin.Context, string) (string, []string, error)) error {

	inType := in.Type().Elem()

	for i := 0; i < in.Elem().NumField(); i++ {
		fieldType := inType.Field(i)
		tag := fieldType.Tag.Get(targetTag)
		if tag == "" {
			continue
		}
		name, values, err := extractor(c, tag)
		if err != nil {
			return err
		}
		if len(values) == 0 {
			continue
		}
		field := in.Elem().Field(i)
		if field.Kind() == reflect.Ptr {
			f := reflect.New(field.Type().Elem())
			field.Set(f)
			field = field.Elem()
		}
		if field.Kind() == reflect.Slice {
			for _, v := range values {
				newV := reflect.New(field.Type().Elem()).Elem()
				err := bindValue(v, newV)
				if err != nil {
					return err
				}
				field.Set(reflect.Append(field, newV))
			}
			return nil
		} else if len(values) > 1 {
			return fmt.Errorf("Query parameter '%s' does not support multiple values", name)
		} else {
			err = bindValue(values[0], field)
			if err != nil {
				return err
			}
		}
	}

	return nil
}

func extractQuery(c *gin.Context, tag string) (string, []string, error) {

	name, required, defval, err := ExtractTag(tag, true)
	if err != nil {
		return "", nil, err
	}

	q := c.Request.URL.Query()[name]

	if defval != "" && len(q) == 0 {
		q = []string{defval}
	}

	if required && len(q) == 0 {
		return "", nil, fmt.Errorf("Field %s is missing: required.", name)
	}

	return name, q, nil
}

func extractPath(c *gin.Context, tag string) (string, []string, error) {

	name, required, _, err := ExtractTag(tag, false)
	if err != nil {
		return "", nil, err
	}

	out := c.Param(name)
	if required && out == "" {
		return "", nil, fmt.Errorf("Field %s is missing: required.", name)
	}
	return name, []string{out}, nil
}

func ExtractTag(tag string, defaultValue bool) (string, bool, string, error) {

	parts := strings.SplitN(tag, ",", -1)
	name := parts[0]
	options := parts[1:]

	var defval string
	var required bool
	for _, o := range options {
		o = strings.TrimSpace(o)
		if o == "required" {
			required = true
		} else if defaultValue && strings.HasPrefix(o, "default=") {
			o = strings.TrimPrefix(o, "default=")
			defval = o
		} else {
			return "", false, "", fmt.Errorf("Malformed tag for param '%s': unknown opt '%s'", name, o)
		}
	}
	return name, required, defval, nil
}

func bindValue(s string, v reflect.Value) error {

	vIntf := reflect.New(v.Type()).Interface()
	unmarshaler, ok := vIntf.(encoding.TextUnmarshaler)
	if ok {
		err := unmarshaler.UnmarshalText([]byte(s))
		if err != nil {
			return err
		}
		v.Set(reflect.Indirect(reflect.ValueOf(unmarshaler)))
		return nil
	}

	switch v.Kind() {
	case reflect.String:
		v.SetString(s)
	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
		i, err := strconv.ParseInt(s, 10, v.Type().Bits())
		if err != nil {
			return err
		}
		v.SetInt(i)
	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
		i, err := strconv.ParseUint(s, 10, v.Type().Bits())
		if err != nil {
			return err
		}
		v.SetUint(i)
	case reflect.Bool:
		b, err := strconv.ParseBool(s)
		if err != nil {
			return err
		}
		v.SetBool(b)
	default:
		return fmt.Errorf("Unsupported type for query param bind: %v", v.Kind())
	}
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package static provides a modifier that allow Martian to reurn static files
// local to Martian. The static modifier does not support setting explicit path
// mappings via the JSON API.
package static

import (
	"encoding/json"
	"mime"
	"net/http"
	"os"
	"path"
	"path/filepath"

	"github.com/google/martian"
	"github.com/google/martian/parse"
)

type Modifier struct {
	rootPath      string
	explicitPaths map[string]string
}

type staticJSON struct {
	RootPath string               `json:"rootPath"`
	Scope    []parse.ModifierType `json:"scope"`
}

func init() {
	parse.Register("static.Modifier", modifierFromJSON)
}

// NewModifier constructs a static.Modifier that takes a path to serve files from, as well as an optional mapping of request paths to local
// file paths (still rooted at rootPath).
func NewModifier(rootPath string) *Modifier {
	return &Modifier{
		rootPath:      path.Clean(rootPath),
		explicitPaths: make(map[string]string),
	}
}

// ModifyRequest marks the context to skip the roundtrip.
func (s *Modifier) ModifyRequest(req *http.Request) error {
	ctx := martian.NewContext(req)
	ctx.SkipRoundTrip()
	return nil
}

// ModifyResponse reads the file rooted at rootPath joined with the request URL
// path. In the case that the the request path is a key in s.explicitPaths, ModifyRequest
// will attempt to open the file located at s.rootPath joined by the value in s.explicitPaths
// (keyed by res.Request.URL.Path).  In the case that the file cannot be found, the response
// will be a 404. ModifyResponse will return a 404 for any path that is defined in s.explictPaths
// and that does not exist locally, even if that file does exist in s.rootPath.
func (s *Modifier) ModifyResponse(res *http.Response) error {
	reqpth := filepath.Clean(res.Request.URL.Path)
	fpth := filepath.Join(s.rootPath, reqpth)

	if _, ok := s.explicitPaths[reqpth]; ok {
		fpth = filepath.Join(s.rootPath, s.explicitPaths[reqpth])
	}

	f, err := os.Open(fpth)
	switch {
	case os.IsNotExist(err):
		res.StatusCode = http.StatusNotFound
		return nil
	case os.IsPermission(err):
		// This is returning a StatusUnauthorized to reflect that the Martian does
		// not have the appropriate permissions on the local file system.  This is a
		// deviation from the standard assumption around an HTTP 401 response.
		res.StatusCode = http.StatusUnauthorized
		return err
	case err != nil:
		res.StatusCode = http.StatusInternalServerError
		return err
	}

	res.Body.Close()
	res.Body = f

	res.Header.Set("Content-Type", mime.TypeByExtension(filepath.Ext(fpth)))

	return nil
}

// SetExplicitPathMappings sets an optional mapping of request paths to local
// file paths rooted at s.rootPath.
func (s *Modifier) SetExplicitPathMappings(ep map[string]string) {
	s.explicitPaths = ep
}

func modifierFromJSON(b []byte) (*parse.Result, error) {
	msg := &staticJSON{}
	if err := json.Unmarshal(b, msg); err != nil {
		return nil, err
	}

	return parse.NewResult(NewModifier(msg.RootPath), msg.Scope)
}
<commit_msg>add comment to static.Modifier struct (#152)<commit_after>// Copyright 2015 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package static provides a modifier that allow Martian to reurn static files
// local to Martian. The static modifier does not support setting explicit path
// mappings via the JSON API.
package static

import (
	"encoding/json"
	"mime"
	"net/http"
	"os"
	"path"
	"path/filepath"

	"github.com/google/martian"
	"github.com/google/martian/parse"
)

// Modifier is a martian.RequestResponseModifier that routes reqeusts to rootPath
// and serves the assets there, while skipping the HTTP roundtrip.
type Modifier struct {
	rootPath      string
	explicitPaths map[string]string
}

type staticJSON struct {
	RootPath string               `json:"rootPath"`
	Scope    []parse.ModifierType `json:"scope"`
}

func init() {
	parse.Register("static.Modifier", modifierFromJSON)
}

// NewModifier constructs a static.Modifier that takes a path to serve files from, as well as an optional mapping of request paths to local
// file paths (still rooted at rootPath).
func NewModifier(rootPath string) *Modifier {
	return &Modifier{
		rootPath:      path.Clean(rootPath),
		explicitPaths: make(map[string]string),
	}
}

// ModifyRequest marks the context to skip the roundtrip.
func (s *Modifier) ModifyRequest(req *http.Request) error {
	ctx := martian.NewContext(req)
	ctx.SkipRoundTrip()
	return nil
}

// ModifyResponse reads the file rooted at rootPath joined with the request URL
// path. In the case that the the request path is a key in s.explicitPaths, ModifyRequest
// will attempt to open the file located at s.rootPath joined by the value in s.explicitPaths
// (keyed by res.Request.URL.Path).  In the case that the file cannot be found, the response
// will be a 404. ModifyResponse will return a 404 for any path that is defined in s.explictPaths
// and that does not exist locally, even if that file does exist in s.rootPath.
func (s *Modifier) ModifyResponse(res *http.Response) error {
	reqpth := filepath.Clean(res.Request.URL.Path)
	fpth := filepath.Join(s.rootPath, reqpth)

	if _, ok := s.explicitPaths[reqpth]; ok {
		fpth = filepath.Join(s.rootPath, s.explicitPaths[reqpth])
	}

	f, err := os.Open(fpth)
	switch {
	case os.IsNotExist(err):
		res.StatusCode = http.StatusNotFound
		return nil
	case os.IsPermission(err):
		// This is returning a StatusUnauthorized to reflect that the Martian does
		// not have the appropriate permissions on the local file system.  This is a
		// deviation from the standard assumption around an HTTP 401 response.
		res.StatusCode = http.StatusUnauthorized
		return err
	case err != nil:
		res.StatusCode = http.StatusInternalServerError
		return err
	}

	res.Body.Close()
	res.Body = f

	res.Header.Set("Content-Type", mime.TypeByExtension(filepath.Ext(fpth)))

	return nil
}

// SetExplicitPathMappings sets an optional mapping of request paths to local
// file paths rooted at s.rootPath.
func (s *Modifier) SetExplicitPathMappings(ep map[string]string) {
	s.explicitPaths = ep
}

func modifierFromJSON(b []byte) (*parse.Result, error) {
	msg := &staticJSON{}
	if err := json.Unmarshal(b, msg); err != nil {
		return nil, err
	}

	return parse.NewResult(NewModifier(msg.RootPath), msg.Scope)
}
<|endoftext|>


<commit_before>package migrations

import (
	"fmt"
	"runtime"
	"time"

	"github.com/cozy/cozy-stack/model/instance"
	"github.com/cozy/cozy-stack/model/job"
	"github.com/cozy/cozy-stack/model/vfs"
	"github.com/cozy/cozy-stack/model/vfs/vfsswift"
	"github.com/cozy/cozy-stack/pkg/config/config"
	"github.com/cozy/cozy-stack/pkg/consts"
	"github.com/cozy/cozy-stack/pkg/couchdb"
	"github.com/cozy/cozy-stack/pkg/lock"
	"github.com/cozy/cozy-stack/pkg/logger"
	multierror "github.com/hashicorp/go-multierror"
	"github.com/ncw/swift"
)

const (
	swiftV1ToV2 = "swift-v1-to-v2"
	toSwiftV3   = "to-swift-v3"

	swiftV1ContainerPrefixCozy = "cozy-"
	swiftV1ContainerPrefixData = "data-"
	swiftV2ContainerPrefixCozy = "cozy-v2-"
	swiftV2ContainerPrefixData = "data-v2-"
	swiftV3ContainerPrefix     = "cozy-v3-"
)

// maxSimultaneousCalls is the maximal number of simultaneous calls to Swift
// made by a single migration.
const maxSimultaneousCalls = 8

func init() {
	job.AddWorker(&job.WorkerConfig{
		WorkerType:   "migrations",
		Concurrency:  runtime.NumCPU(),
		MaxExecCount: 1,
		Reserved:     true,
		WorkerFunc:   worker,
		WorkerCommit: commit,
		Timeout:      1 * time.Hour,
	})
}

type message struct {
	Type string `json:"type"`
}

func worker(ctx *job.WorkerContext) error {
	var msg message
	if err := ctx.UnmarshalMessage(&msg); err != nil {
		return err
	}

	logger.WithDomain(ctx.Instance.Domain).WithField("nspace", "migration").
		Infof("Start the migration %s", msg.Type)

	switch msg.Type {
	case toSwiftV3:
		return migrateToSwiftV3(ctx.Instance.Domain)
	case swiftV1ToV2:
		return fmt.Errorf("this migration type is no longer supported")
	default:
		return fmt.Errorf("unknown migration type %q", msg.Type)
	}
}

func commit(ctx *job.WorkerContext, err error) error {
	log := logger.WithDomain(ctx.Instance.Domain).WithField("nspace", "migration")
	if err == nil {
		log.Infof("Migration success")
	} else {
		log.Errorf("Migration error: %s", err)
	}
	return err
}

func migrateToSwiftV3(domain string) error {
	c := config.GetSwiftConnection()
	inst, err := instance.GetFromCouch(domain)
	if err != nil {
		return err
	}

	var srcContainer, migratedFrom string
	switch inst.SwiftLayout {
	case 0: // layout v1
		srcContainer = swiftV1ContainerPrefixCozy + domain
		migratedFrom = "v1"
	case 1: // layout v2
		srcContainer = swiftV2ContainerPrefixCozy + inst.DBPrefix()
		migratedFrom = "v2"
	case 2: // layout v3
		return nil // Nothing to do!
	default:
		return instance.ErrInvalidSwiftLayout
	}

	vfs := inst.VFS()
	root, err := vfs.DirByID(consts.RootDirID)
	if err != nil {
		return err
	}

	mutex := lock.ReadWrite(inst, "vfs")
	if err = mutex.Lock(); err != nil {
		return err
	}
	defer mutex.Unlock()

	dstContainer := swiftV3ContainerPrefix + inst.DBPrefix()
	if err = c.ContainerCreate(dstContainer, nil); err != nil {
		return err
	}
	defer func() {
		if err != nil {
			if err := vfsswift.DeleteContainer(c, dstContainer); err != nil {
				log := logger.WithDomain(inst.Domain).WithField("nspace", "migration")
				log.Errorf("Failed to delete v3 container %s: %s", dstContainer, err)
			}
		}
	}()

	if err = copyTheFilesToSwiftV3(inst, c, root, srcContainer, dstContainer); err != nil {
		return err
	}

	meta := &swift.Metadata{"cozy-migrated-from": migratedFrom}
	_ = c.ContainerUpdate(dstContainer, meta.ContainerHeaders())
	if in, err := instance.GetFromCouch(domain); err == nil {
		inst = in
	}
	inst.SwiftLayout = 2
	if err = couchdb.UpdateDoc(couchdb.GlobalDB, inst); err != nil {
		return err
	}
	_ = vfs.Delete()
	return nil
}

func copyTheFilesToSwiftV3(inst *instance.Instance, c *swift.Connection, root *vfs.DirDoc, src, dst string) error {
	nb := 0
	ch := make(chan error)
	log := logger.WithDomain(inst.Domain).
		WithField("nspace", "migration")

	var thumbsContainer string
	switch inst.SwiftLayout {
	case 0: // layout v1
		thumbsContainer = swiftV1ContainerPrefixData + inst.Domain
	case 1: // layout v2
		thumbsContainer = swiftV2ContainerPrefixData + inst.DBPrefix()
	default:
		return instance.ErrInvalidSwiftLayout
	}

	// Use a system of tokens to limit the number of simultaneous calls to
	// Swift: only a goroutine that has a token can make a call.
	tokens := make(chan int, maxSimultaneousCalls)
	for k := 0; k < maxSimultaneousCalls; k++ {
		tokens <- k
	}

	fs := inst.VFS()
	errm := vfs.WalkAlreadyLocked(fs, root, func(_ string, d *vfs.DirDoc, f *vfs.FileDoc, err error) error {
		if err != nil {
			return err
		}
		if f == nil {
			return nil
		}

		srcName := getSrcName(inst, f)
		dstName := getDstName(inst, f)
		if srcName == "" || dstName == "" {
			return fmt.Errorf("Unexpected copy: %q -> %q", srcName, dstName)
		}

		nb++
		go func() {
			k := <-tokens
			_, err := c.ObjectCopy(src, srcName, dst, dstName, nil)
			if err != nil {
				log.Warningf("Cannot copy file from %s %s to %s %s: %s",
					src, srcName, dst, dstName, err)
			}
			ch <- err
			tokens <- k
		}()

		// Copy the thumbnails
		if f.Class == "image" {
			srcSmall, srcMedium, srcLarge := getThumbsSrcNames(inst, f)
			dstSmall, dstMedium, dstLarge := getThumbsDstNames(inst, f)
			nb += 3
			go func() {
				k := <-tokens
				_, err := c.ObjectCopy(thumbsContainer, srcSmall, dst, dstSmall, nil)
				if err != nil {
					log.Infof("Cannot copy thumbnail small from %s %s to %s %s: %s",
						thumbsContainer, srcSmall, dst, dstSmall, err)
				}
				ch <- nil
				_, err = c.ObjectCopy(thumbsContainer, srcMedium, dst, dstMedium, nil)
				if err != nil {
					log.Infof("Cannot copy thumbnail medium from %s %s to %s %s: %s",
						thumbsContainer, srcMedium, dst, dstMedium, err)
				}
				ch <- nil
				_, err = c.ObjectCopy(thumbsContainer, srcLarge, dst, dstLarge, nil)
				if err != nil {
					log.Infof("Cannot copy thumbnail large from %s %s to %s %s: %s",
						thumbsContainer, srcLarge, dst, dstLarge, err)
				}
				ch <- nil
				tokens <- k
			}()
		}
		return nil
	})

	for i := 0; i < nb; i++ {
		if err := <-ch; err != nil {
			errm = multierror.Append(errm, err)
		}
	}
	// Get back the tokens to ensure that each goroutine can finish.
	for k := 0; k < maxSimultaneousCalls; k++ {
		<-tokens
	}
	return errm
}

func getSrcName(inst *instance.Instance, f *vfs.FileDoc) string {
	srcName := ""
	switch inst.SwiftLayout {
	case 0: // layout v1
		srcName = f.DirID + "/" + f.DocName
	case 1: // layout v2
		srcName = vfsswift.MakeObjectName(f.DocID)
	}
	return srcName
}

func getDstName(inst *instance.Instance, f *vfs.FileDoc) string {
	if f.InternalID == "" {
		f.InternalID = vfsswift.NewInternalID()
		if err := couchdb.UpdateDoc(inst, f); err != nil {
			return ""
		}
	}
	return vfsswift.MakeObjectNameV3(f.DocID, f.InternalID)
}

func getThumbsSrcNames(inst *instance.Instance, f *vfs.FileDoc) (string, string, string) {
	var small, medium, large string
	switch inst.SwiftLayout {
	case 0: // layout v1
		small = fmt.Sprintf("thumbs/%s-small", f.DocID)
		medium = fmt.Sprintf("thumbs/%s-medium", f.DocID)
		large = fmt.Sprintf("thumbs/%s-large", f.DocID)
	case 1: // layout v2
		obj := vfsswift.MakeObjectName(f.DocID)
		small = fmt.Sprintf("thumbs/%s-small", obj)
		medium = fmt.Sprintf("thumbs/%s-medium", obj)
		large = fmt.Sprintf("thumbs/%s-large", obj)
	}
	return small, medium, large
}

func getThumbsDstNames(inst *instance.Instance, f *vfs.FileDoc) (string, string, string) {
	obj := vfsswift.MakeObjectName(f.DocID)
	small := fmt.Sprintf("thumbs/%s-small", obj)
	medium := fmt.Sprintf("thumbs/%s-medium", obj)
	large := fmt.Sprintf("thumbs/%s-large", obj)
	return small, medium, large
}
<commit_msg>More specific v2 version in cozy-migrated-from<commit_after>package migrations

import (
	"fmt"
	"runtime"
	"time"

	"github.com/cozy/cozy-stack/model/instance"
	"github.com/cozy/cozy-stack/model/job"
	"github.com/cozy/cozy-stack/model/vfs"
	"github.com/cozy/cozy-stack/model/vfs/vfsswift"
	"github.com/cozy/cozy-stack/pkg/config/config"
	"github.com/cozy/cozy-stack/pkg/consts"
	"github.com/cozy/cozy-stack/pkg/couchdb"
	"github.com/cozy/cozy-stack/pkg/lock"
	"github.com/cozy/cozy-stack/pkg/logger"
	multierror "github.com/hashicorp/go-multierror"
	"github.com/ncw/swift"
)

const (
	swiftV1ToV2 = "swift-v1-to-v2"
	toSwiftV3   = "to-swift-v3"

	swiftV1ContainerPrefixCozy = "cozy-"
	swiftV1ContainerPrefixData = "data-"
	swiftV2ContainerPrefixCozy = "cozy-v2-"
	swiftV2ContainerPrefixData = "data-v2-"
	swiftV3ContainerPrefix     = "cozy-v3-"
)

// maxSimultaneousCalls is the maximal number of simultaneous calls to Swift
// made by a single migration.
const maxSimultaneousCalls = 8

func init() {
	job.AddWorker(&job.WorkerConfig{
		WorkerType:   "migrations",
		Concurrency:  runtime.NumCPU(),
		MaxExecCount: 1,
		Reserved:     true,
		WorkerFunc:   worker,
		WorkerCommit: commit,
		Timeout:      1 * time.Hour,
	})
}

type message struct {
	Type string `json:"type"`
}

func worker(ctx *job.WorkerContext) error {
	var msg message
	if err := ctx.UnmarshalMessage(&msg); err != nil {
		return err
	}

	logger.WithDomain(ctx.Instance.Domain).WithField("nspace", "migration").
		Infof("Start the migration %s", msg.Type)

	switch msg.Type {
	case toSwiftV3:
		return migrateToSwiftV3(ctx.Instance.Domain)
	case swiftV1ToV2:
		return fmt.Errorf("this migration type is no longer supported")
	default:
		return fmt.Errorf("unknown migration type %q", msg.Type)
	}
}

func commit(ctx *job.WorkerContext, err error) error {
	log := logger.WithDomain(ctx.Instance.Domain).WithField("nspace", "migration")
	if err == nil {
		log.Infof("Migration success")
	} else {
		log.Errorf("Migration error: %s", err)
	}
	return err
}

func migrateToSwiftV3(domain string) error {
	c := config.GetSwiftConnection()
	inst, err := instance.GetFromCouch(domain)
	if err != nil {
		return err
	}

	var srcContainer, migratedFrom string
	switch inst.SwiftLayout {
	case 0: // layout v1
		srcContainer = swiftV1ContainerPrefixCozy + domain
		migratedFrom = "v1"
	case 1: // layout v2
		srcContainer = swiftV2ContainerPrefixCozy + inst.DBPrefix()
		switch inst.DBPrefix() {
		case inst.Domain:
			migratedFrom = "v2a"
		case inst.Prefix:
			migratedFrom = "v2b"
		default:
			return instance.ErrInvalidSwiftLayout
		}
	case 2: // layout v3
		return nil // Nothing to do!
	default:
		return instance.ErrInvalidSwiftLayout
	}

	vfs := inst.VFS()
	root, err := vfs.DirByID(consts.RootDirID)
	if err != nil {
		return err
	}

	mutex := lock.ReadWrite(inst, "vfs")
	if err = mutex.Lock(); err != nil {
		return err
	}
	defer mutex.Unlock()

	dstContainer := swiftV3ContainerPrefix + inst.DBPrefix()
	if err = c.ContainerCreate(dstContainer, nil); err != nil {
		return err
	}
	defer func() {
		if err != nil {
			if err := vfsswift.DeleteContainer(c, dstContainer); err != nil {
				log := logger.WithDomain(inst.Domain).WithField("nspace", "migration")
				log.Errorf("Failed to delete v3 container %s: %s", dstContainer, err)
			}
		}
	}()

	if err = copyTheFilesToSwiftV3(inst, c, root, srcContainer, dstContainer); err != nil {
		return err
	}

	meta := &swift.Metadata{"cozy-migrated-from": migratedFrom}
	_ = c.ContainerUpdate(dstContainer, meta.ContainerHeaders())
	if in, err := instance.GetFromCouch(domain); err == nil {
		inst = in
	}
	inst.SwiftLayout = 2
	if err = couchdb.UpdateDoc(couchdb.GlobalDB, inst); err != nil {
		return err
	}
	_ = vfs.Delete()
	return nil
}

func copyTheFilesToSwiftV3(inst *instance.Instance, c *swift.Connection, root *vfs.DirDoc, src, dst string) error {
	nb := 0
	ch := make(chan error)
	log := logger.WithDomain(inst.Domain).
		WithField("nspace", "migration")

	var thumbsContainer string
	switch inst.SwiftLayout {
	case 0: // layout v1
		thumbsContainer = swiftV1ContainerPrefixData + inst.Domain
	case 1: // layout v2
		thumbsContainer = swiftV2ContainerPrefixData + inst.DBPrefix()
	default:
		return instance.ErrInvalidSwiftLayout
	}

	// Use a system of tokens to limit the number of simultaneous calls to
	// Swift: only a goroutine that has a token can make a call.
	tokens := make(chan int, maxSimultaneousCalls)
	for k := 0; k < maxSimultaneousCalls; k++ {
		tokens <- k
	}

	fs := inst.VFS()
	errm := vfs.WalkAlreadyLocked(fs, root, func(_ string, d *vfs.DirDoc, f *vfs.FileDoc, err error) error {
		if err != nil {
			return err
		}
		if f == nil {
			return nil
		}

		srcName := getSrcName(inst, f)
		dstName := getDstName(inst, f)
		if srcName == "" || dstName == "" {
			return fmt.Errorf("Unexpected copy: %q -> %q", srcName, dstName)
		}

		nb++
		go func() {
			k := <-tokens
			_, err := c.ObjectCopy(src, srcName, dst, dstName, nil)
			if err != nil {
				log.Warningf("Cannot copy file from %s %s to %s %s: %s",
					src, srcName, dst, dstName, err)
			}
			ch <- err
			tokens <- k
		}()

		// Copy the thumbnails
		if f.Class == "image" {
			srcSmall, srcMedium, srcLarge := getThumbsSrcNames(inst, f)
			dstSmall, dstMedium, dstLarge := getThumbsDstNames(inst, f)
			nb += 3
			go func() {
				k := <-tokens
				_, err := c.ObjectCopy(thumbsContainer, srcSmall, dst, dstSmall, nil)
				if err != nil {
					log.Infof("Cannot copy thumbnail small from %s %s to %s %s: %s",
						thumbsContainer, srcSmall, dst, dstSmall, err)
				}
				ch <- nil
				_, err = c.ObjectCopy(thumbsContainer, srcMedium, dst, dstMedium, nil)
				if err != nil {
					log.Infof("Cannot copy thumbnail medium from %s %s to %s %s: %s",
						thumbsContainer, srcMedium, dst, dstMedium, err)
				}
				ch <- nil
				_, err = c.ObjectCopy(thumbsContainer, srcLarge, dst, dstLarge, nil)
				if err != nil {
					log.Infof("Cannot copy thumbnail large from %s %s to %s %s: %s",
						thumbsContainer, srcLarge, dst, dstLarge, err)
				}
				ch <- nil
				tokens <- k
			}()
		}
		return nil
	})

	for i := 0; i < nb; i++ {
		if err := <-ch; err != nil {
			errm = multierror.Append(errm, err)
		}
	}
	// Get back the tokens to ensure that each goroutine can finish.
	for k := 0; k < maxSimultaneousCalls; k++ {
		<-tokens
	}
	return errm
}

func getSrcName(inst *instance.Instance, f *vfs.FileDoc) string {
	srcName := ""
	switch inst.SwiftLayout {
	case 0: // layout v1
		srcName = f.DirID + "/" + f.DocName
	case 1: // layout v2
		srcName = vfsswift.MakeObjectName(f.DocID)
	}
	return srcName
}

func getDstName(inst *instance.Instance, f *vfs.FileDoc) string {
	if f.InternalID == "" {
		f.InternalID = vfsswift.NewInternalID()
		if err := couchdb.UpdateDoc(inst, f); err != nil {
			return ""
		}
	}
	return vfsswift.MakeObjectNameV3(f.DocID, f.InternalID)
}

func getThumbsSrcNames(inst *instance.Instance, f *vfs.FileDoc) (string, string, string) {
	var small, medium, large string
	switch inst.SwiftLayout {
	case 0: // layout v1
		small = fmt.Sprintf("thumbs/%s-small", f.DocID)
		medium = fmt.Sprintf("thumbs/%s-medium", f.DocID)
		large = fmt.Sprintf("thumbs/%s-large", f.DocID)
	case 1: // layout v2
		obj := vfsswift.MakeObjectName(f.DocID)
		small = fmt.Sprintf("thumbs/%s-small", obj)
		medium = fmt.Sprintf("thumbs/%s-medium", obj)
		large = fmt.Sprintf("thumbs/%s-large", obj)
	}
	return small, medium, large
}

func getThumbsDstNames(inst *instance.Instance, f *vfs.FileDoc) (string, string, string) {
	obj := vfsswift.MakeObjectName(f.DocID)
	small := fmt.Sprintf("thumbs/%s-small", obj)
	medium := fmt.Sprintf("thumbs/%s-medium", obj)
	large := fmt.Sprintf("thumbs/%s-large", obj)
	return small, medium, large
}
<|endoftext|>


<commit_before>package ytapigo

import (
    "fmt"
    "log"
    "github.com/spf13/viper"
    "net/http"
    "net/url"
    "io/ioutil"
    "encoding/json"
    "os"
    "strings"
    // "io"
)
func CheckYT() string {
    return TestMsg
}

const (
    TestMsg string = "YtapiGo"
    ConfDir string = "$HOME"
    ConfName string = ".ytapigo"
)
var (
    // 0-Spelling, 1-Translation
    YtJsonURLs = [2]string{
        "http://speller.yandex.net/services/spellservice.json/checkText?",
        "https://translate.yandex.net/api/v1.5/tr.json/translate?"}
    LoggerError *log.Logger
    LoggerDebug *log.Logger
)

// Initiate Logger handlers
func LoggerInit(ytconfig *YtConfig) {
    errorHandle, debugHandle := os.Stdout, ioutil.Discard
    if ytconfig.Debug {
        debugHandle = os.Stdout
    }

    LoggerDebug = log.New(debugHandle,
        "DEBUG: ",
        log.Ldate|log.Lmicroseconds|log.Lshortfile)

    LoggerError = log.New(errorHandle,
        "ERROR: ",
        log.Ldate|log.Ltime|log.Lshortfile)
}
// Interface to prepare JSON response
type Builder interface {
    Build(m map[string]interface{})
}
// configuration data
type YtConfig struct {
    APItr string
    APIdict string
    Debug bool
}
func (yt YtConfig) String() string {
    return fmt.Sprintf("\nConfig:\n APItr=%v\n APIdict=%v\n Debug=%v", yt.APItr, yt.APIdict, yt.Debug)
}
type JsonSpelResp struct {
    word string
    s []string
    code float64
    pos float64
    row float64
    col float64
    len float64
}
type JsonTrResp struct {
    code float64
    lang string
    text []string
}
type JsonSpelResps []JsonSpelResp

func (jspell *JsonSpelResp) Exists() bool {
    if len(jspell.s) > 0 {
        return true
    }
    return false
}
func (jspells *JsonSpelResps) Exists() bool {
    if len(*jspells) > 0 {
        return true
    }
    return false
}
func (jspell *JsonSpelResp) String() string {
    return fmt.Sprintf("%v -> %v", jspell.word, jspell.s)
}
func (jstr *JsonTrResp) String() string {
    if len(jstr.text) == 0 {
        return ""
    }
    return jstr.text[0]
}
func (jspells JsonSpelResps) String() string {
    spellstr := make([]string, len(jspells))
    for i, v := range jspells {
        if v.Exists() {
            spellstr[i] = v.String()
        }
    }
    return fmt.Sprintf("Spelling: \n\t%v\n", strings.Join(spellstr, "\n\t"))
}
func (jspell *JsonSpelResp) Build(m map[string]interface{}) {
    s_array := m["s"].([]interface{})
    jspell.s = make([]string, len(s_array))
    for i, v := range s_array {
        jspell.s[i] = v.(string)
    }
    jspell.word = m["word"].(string)
    jspell.code = m["code"].(float64)
    jspell.pos = m["pos"].(float64)
    jspell.row = m["row"].(float64)
    jspell.col = m["col"].(float64)
    jspell.len = m["len"].(float64)
}
func (jstr *JsonTrResp) Build(m map[string]interface{}) {
    s_array := m["text"].([]interface{})
    jstr.text = make([]string, len(s_array))
    for i, v := range s_array {
        jstr.text[i] = v.(string)
    }
    jstr.code = m["code"].(float64)
    jstr.lang = m["lang"].(string)
}

// read configuration file
func ReadConfig() (YtConfig, error) {
    viper.SetConfigName(ConfName)
    viper.SetConfigType("json")
    viper.AddConfigPath(ConfDir)
    viper.ReadInConfig()
    ytconfig := YtConfig{viper.GetString("APItr"), viper.GetString("APIdict"), viper.GetBool("Debug")}
    if (ytconfig.APItr == "") || (ytconfig.APIdict == "") {
        return ytconfig, fmt.Errorf("Can not read API keys value from config file: %v/%v", ConfDir, ConfName)
    }
    return ytconfig, nil
}

func GetYtResponse(url string, params *url.Values) ([]byte, error) {
    // TODO: http://golang.org/pkg/net/http/#Transport
    result := []byte("")
    resp, err := http.Get(url + params.Encode())
    if err != nil {
        LoggerError.Println(err)
        return result, err
    }
    defer resp.Body.Close()

    if resp.StatusCode != 200 {
        return result, fmt.Errorf("Wrong response code=%v", resp.StatusCode)
    }
    body, err := ioutil.ReadAll(resp.Body)
    if err != nil {
        LoggerError.Println(err)
        return result, err
    }
    // LoggerDebug.Println(string(body))
    return body, nil
}

// get spelling
func GetSpelling(lang string, txt string) (JsonSpelResps, error) {
    LoggerDebug.Println("Call GetSpelling")
    result := make(JsonSpelResps, 0)
    params := url.Values{
        "lang": {lang},
        "text": {txt},
        "format": {"plain"},
        "options": {"518"}}
    body, err := GetYtResponse(YtJsonURLs[0], &params)
    if err != nil {
        return result, err
    }
    var jsr []map[string]interface{}
    if err := json.Unmarshal(body, &jsr); err != nil {
        LoggerError.Println(err)
        return result, err
    }
    result = make(JsonSpelResps, len(jsr))
    for i, val := range jsr {
        result[i].Build(val)
    }
    return result, nil
}
// get Translation
func GetTranslation(lang string, txt string, key string) (JsonTrResp, error) {
    LoggerDebug.Println("Call GetTranslation")
    result := JsonTrResp{}
    params := url.Values{
        "lang": {lang},
        "text": {txt},
        "key": {key},
        "format": {"plain"}}
    body, err := GetYtResponse(YtJsonURLs[1], &params)
    if err != nil {
        return result, err
    }
    var jsr map[string]interface{}
    if err := json.Unmarshal(body, &jsr); err != nil {
        LoggerError.Println(err)
        return result, err
    }
    result.Build(jsr)
    return result, nil
}

// main YtapiGo function
func GetTr() {
    var lang, langs, txt string
    lenparams := len(os.Args)
    if lenparams < 2 {
        log.Fatalln("Too few parameters")
    } else if lenparams == 2 {
        lang, langs, txt = "en", "en-ru", os.Args[1]
    } else {
        switch os.Args[1] {
            case "ru", "ру":
                lang, langs = "ru", "ru-en"
                txt = strings.Join(os.Args[2:], " ")
            case "en", "анг":
                lang, langs = "en", "en-ru"
                txt = strings.Join(os.Args[2:], " ")
            default:
                lang, langs = "en", "en-ru"
                txt = strings.Join(os.Args[1:], " ")
        }
    }
    cfg, err := ReadConfig()
    if err != nil {
        log.Fatalln(err)
    }
    LoggerInit(&cfg)
    LoggerDebug.Printf(cfg.String())
    // spellig
    spells, err := GetSpelling(lang, txt)
    if err != nil {
        LoggerError.Fatalln(err)
    }
    if spells.Exists() {
        fmt.Println(spells)
    }
    // translation
    trs, err := GetTranslation(langs, txt, cfg.APItr)
    if err != nil {
        LoggerError.Fatalln(err)
    }
    fmt.Println(trs.String())
}<commit_msg>added env proxy support<commit_after>package ytapigo

import (
    "fmt"
    "log"
    "github.com/spf13/viper"
    "net/http"
    "net/url"
    "io/ioutil"
    "encoding/json"
    "os"
    "strings"
    "time"
)
func CheckYT() string {
    return TestMsg
}

const (
    TestMsg string = "YtapiGo"
    ConfDir string = "$HOME"
    ConfName string = ".ytapigo"
)
var (
    // 0-Spelling, 1-Translation, 2-Dict
    YtJsonURLs = [3]string{
        "http://speller.yandex.net/services/spellservice.json/checkText?",
        "https://translate.yandex.net/api/v1.5/tr.json/translate?",
        "https://dictionary.yandex.net/api/v1/dicservice.json/lookup?"
    }
    LoggerError *log.Logger
    LoggerDebug *log.Logger
)

// Initiate Logger handlers
func LoggerInit(ytconfig *YtConfig) {
    errorHandle, debugHandle := os.Stdout, ioutil.Discard
    if ytconfig.Debug {
        debugHandle = os.Stdout
    }

    LoggerDebug = log.New(debugHandle,
        "DEBUG: ",
        log.Ldate|log.Lmicroseconds|log.Lshortfile)

    LoggerError = log.New(errorHandle,
        "ERROR: ",
        log.Ldate|log.Ltime|log.Lshortfile)
}
// Interface to prepare JSON response
type Builder interface {
    Build(m map[string]interface{})
}
// configuration data
type YtConfig struct {
    APItr string
    APIdict string
    Debug bool
}
func (yt YtConfig) String() string {
    return fmt.Sprintf("\nConfig:\n APItr=%v\n APIdict=%v\n Debug=%v", yt.APItr, yt.APIdict, yt.Debug)
}
type JsonSpelResp struct {
    word string
    s []string
    code float64
    pos float64
    row float64
    col float64
    len float64
}
type JsonTrResp struct {
    code float64
    lang string
    text []string
}
type JsonSpelResps []JsonSpelResp

func (jspell *JsonSpelResp) Exists() bool {
    if len(jspell.s) > 0 {
        return true
    }
    return false
}
func (jspells *JsonSpelResps) Exists() bool {
    if len(*jspells) > 0 {
        return true
    }
    return false
}
func (jspell *JsonSpelResp) String() string {
    return fmt.Sprintf("%v -> %v", jspell.word, jspell.s)
}
func (jstr *JsonTrResp) String() string {
    if len(jstr.text) == 0 {
        return ""
    }
    return jstr.text[0]
}
func (jspells JsonSpelResps) String() string {
    spellstr := make([]string, len(jspells))
    for i, v := range jspells {
        if v.Exists() {
            spellstr[i] = v.String()
        }
    }
    return fmt.Sprintf("Spelling: \n\t%v\n", strings.Join(spellstr, "\n\t"))
}
func (jspell *JsonSpelResp) Build(m map[string]interface{}) {
    s_array := m["s"].([]interface{})
    jspell.s = make([]string, len(s_array))
    for i, v := range s_array {
        jspell.s[i] = v.(string)
    }
    jspell.word = m["word"].(string)
    jspell.code = m["code"].(float64)
    jspell.pos = m["pos"].(float64)
    jspell.row = m["row"].(float64)
    jspell.col = m["col"].(float64)
    jspell.len = m["len"].(float64)
}
func (jstr *JsonTrResp) Build(m map[string]interface{}) {
    s_array := m["text"].([]interface{})
    jstr.text = make([]string, len(s_array))
    for i, v := range s_array {
        jstr.text[i] = v.(string)
    }
    jstr.code = m["code"].(float64)
    jstr.lang = m["lang"].(string)
}

// read configuration file
func ReadConfig() (YtConfig, error) {
    viper.SetConfigName(ConfName)
    viper.SetConfigType("json")
    viper.AddConfigPath(ConfDir)
    viper.ReadInConfig()
    ytconfig := YtConfig{viper.GetString("APItr"), viper.GetString("APIdict"), viper.GetBool("Debug")}
    if (ytconfig.APItr == "") || (ytconfig.APIdict == "") {
        return ytconfig, fmt.Errorf("Can not read API keys value from config file: %v/%v", ConfDir, ConfName)
    }
    return ytconfig, nil
}

func GetYtResponse(url string, params *url.Values) ([]byte, error) {
    // TODO: http://golang.org/pkg/net/http/#Transport
    result := []byte("")
    tr := &http.Transport{
        Proxy: http.ProxyFromEnvironment,
        TLSHandshakeTimeout: 10 * time.Second,
    }
    client := &http.Client{Transport: tr}
    resp, err := client.Get(url + params.Encode())
    if err != nil {
        LoggerError.Println(err)
        return result, err
    }
    defer resp.Body.Close()

    if resp.StatusCode != 200 {
        return result, fmt.Errorf("Wrong response code=%v", resp.StatusCode)
    }
    body, err := ioutil.ReadAll(resp.Body)
    if err != nil {
        LoggerError.Println(err)
        return result, err
    }
    // LoggerDebug.Println(string(body))
    return body, nil
}

// get spelling
func GetSpelling(lang string, txt string) (JsonSpelResps, error) {
    LoggerDebug.Println("Call GetSpelling")
    result := make(JsonSpelResps, 0)
    params := url.Values{
        "lang": {lang},
        "text": {txt},
        "format": {"plain"},
        "options": {"518"}}
    body, err := GetYtResponse(YtJsonURLs[0], &params)
    if err != nil {
        return result, err
    }
    var jsr []map[string]interface{}
    if err := json.Unmarshal(body, &jsr); err != nil {
        LoggerError.Println(err)
        return result, err
    }
    result = make(JsonSpelResps, len(jsr))
    for i, val := range jsr {
        result[i].Build(val)
    }
    return result, nil
}
// get Translation
func GetTranslation(lang string, txt string, key string) (JsonTrResp, error) {
    LoggerDebug.Println("Call GetTranslation")
    result := JsonTrResp{}
    params := url.Values{
        "lang": {lang},
        "text": {txt},
        "key": {key},
        "format": {"plain"}}
    body, err := GetYtResponse(YtJsonURLs[1], &params)
    if err != nil {
        return result, err
    }
    var jsr map[string]interface{}
    if err := json.Unmarshal(body, &jsr); err != nil {
        LoggerError.Println(err)
        return result, err
    }
    result.Build(jsr)
    return result, nil
}

// main YtapiGo function
func GetTr() {
    var lang, langs, txt string
    lenparams := len(os.Args)
    if lenparams < 2 {
        log.Fatalln("Too few parameters")
    } else if lenparams == 2 {
        lang, langs, txt = "en", "en-ru", os.Args[1]
    } else {
        switch os.Args[1] {
            case "ru", "ру":
                lang, langs = "ru", "ru-en"
                txt = strings.Join(os.Args[2:], " ")
            case "en", "анг":
                lang, langs = "en", "en-ru"
                txt = strings.Join(os.Args[2:], " ")
            default:
                lang, langs = "en", "en-ru"
                txt = strings.Join(os.Args[1:], " ")
        }
    }
    cfg, err := ReadConfig()
    if err != nil {
        log.Fatalln(err)
    }
    LoggerInit(&cfg)
    LoggerDebug.Printf(cfg.String())
    // spellig
    spells, err := GetSpelling(lang, txt)
    if err != nil {
        LoggerError.Fatalln(err)
    }
    if spells.Exists() {
        fmt.Println(spells)
    }
    // translation
    trs, err := GetTranslation(langs, txt, cfg.APItr)
    if err != nil {
        LoggerError.Fatalln(err)
    }
    fmt.Println(trs.String())
}<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// This package provides basic interfaces to I/O primitives.
// Its primary job is to wrap existing implementations of such primitives,
// such as those in package os, into shared public interfaces that
// abstract the functionality.
// It also provides buffering primitives and some other basic operations.
package io

import (
	"bytes";
	"os";
)

// Error represents an unexpected I/O behavior.
type Error struct {
	os.ErrorString
}

// ErrShortWrite means that a write accepted fewer bytes than requested
// but failed to return an explicit error.
var ErrShortWrite os.Error = &Error{"short write"}

// ErrUnexpectedEOF means that os.EOF was encountered in the
// middle of reading a fixed-size block or data structure.
var ErrUnexpectedEOF os.Error = &Error{"unexpected EOF"}

// Reader is the interface that wraps the basic Read method.
//
// Read reads up to len(p) bytes into p.  It returns the number of bytes
// read and any error encountered.  Even if Read returns n < len(p),
// it may use all of p as scratch space during the call.
// If some data is available but not len(p) bytes, Read conventionally
// returns what is available rather than block waiting for more.
//
// At the end of the input stream, Read returns 0, os.EOF.
// Read may return a non-zero number of bytes with a non-nil err.
// In particular, a Read that exhausts the input may return n > 0, os.EOF.
type Reader interface {
	Read(p []byte) (n int, err os.Error);
}

// Writer is the interface that wraps the basic Write method.
type Writer interface {
	Write(p []byte) (n int, err os.Error);
}

// Closer is the interface that wraps the basic Close method.
type Closer interface {
	Close() os.Error;
}

// ReadWrite is the interface that groups the basic Read and Write methods.
type ReadWriter interface {
	Reader;
	Writer;
}

// ReadCloser is the interface that groups the basic Read and Close methods.
type ReadCloser interface {
	Reader;
	Closer;
}

// WriteCloser is the interface that groups the basic Write and Close methods.
type WriteCloser interface {
	Writer;
	Closer;
}

// ReadWriteCloser is the interface that groups the basic Read, Write and Close methods.
type ReadWriteCloser interface {
	Reader;
	Writer;
	Closer;
}

// Convert a string to an array of bytes for easy marshaling.
func StringBytes(s string) []byte {
	b := make([]byte, len(s));
	for i := 0; i < len(s); i++ {
		b[i] = s[i];
	}
	return b;
}

// WriteString writes the contents of the string s to w, which accepts an array of bytes.
func WriteString(w Writer, s string) (n int, err os.Error) {
	return w.Write(StringBytes(s))
}

// ReadAtLeast reads from r into buf until it has read at least min bytes.
// It returns the number of bytes copied and an error if fewer bytes were read.
// The error is os.EOF only if no bytes were read.
// If an EOF happens after reading fewer than min bytes,
// ReadAtLeast returns ErrUnexpectedEOF.
func ReadAtLeast(r Reader, buf []byte, min int) (n int, err os.Error) {
	n = 0;
	for n < min {
		nn, e := r.Read(buf[n:len(buf)]);
		if nn > 0 {
			n += nn
		}
		if e != nil {
			if e == os.EOF && n > 0 {
				e = ErrUnexpectedEOF;
			}
			return n, e
		}
	}
	return n, nil
}

// ReadFull reads exactly len(buf) bytes from r into buf.
// It returns the number of bytes copied and an error if fewer bytes were read.
// The error is os.EOF only if no bytes were read.
// If an EOF happens after reading some but not all the bytes,
// ReadFull returns ErrUnexpectedEOF.
func ReadFull(r Reader, buf []byte) (n int, err os.Error) {
	// TODO(rsc): 6g bug keeps us from writing the obvious 1-liner
	n, err = ReadAtLeast(r, buf, len(buf));
	return;
}

// Copyn copies n bytes (or until an error) from src to dst.
// It returns the number of bytes copied and the error, if any.
func Copyn(src Reader, dst Writer, n int64) (written int64, err os.Error) {
	buf := make([]byte, 32*1024);
	for written < n {
		l := len(buf);
		if d := n - written; d < int64(l) {
			l = int(d);
		}
		nr, er := src.Read(buf[0 : l]);
		if nr > 0 {
			nw, ew := dst.Write(buf[0 : nr]);
			if nw > 0 {
				written += int64(nw);
			}
			if ew != nil {
				err = ew;
				break;
			}
			if nr != nw {
				err = ErrShortWrite;
				break;
			}
		}
		if er != nil {
			err = er;
			break;
		}
	}
	return written, err
}

// Copy copies from src to dst until either EOF is reached
// on src or an error occurs.  It returns the number of bytes
// copied and the error, if any.
func Copy(src Reader, dst Writer) (written int64, err os.Error) {
	buf := make([]byte, 32*1024);
	for {
		nr, er := src.Read(buf);
		if nr > 0 {
			nw, ew := dst.Write(buf[0:nr]);
			if nw > 0 {
				written += int64(nw);
			}
			if ew != nil {
				err = ew;
				break;
			}
			if nr != nw {
				err = ErrShortWrite;
				break;
			}
		}
		if er == os.EOF {
			break;
		}
		if er != nil {
			err = er;
			break;
		}
	}
	return written, err
}

// A ByteReader satisfies Reads by consuming data from a slice of bytes.
// Clients can call NewByteReader to create one or wrap pointers
// to their own slices: r := ByteReader{&data}.
type ByteReader struct {
	Data *[]byte
}

func (r ByteReader) Read(p []byte) (int, os.Error) {
	n := len(p);
	b := *r.Data;
	if len(b) == 0 {
		return 0, os.EOF;
	}
	if n > len(b) {
		n = len(b);
	}
	bytes.Copy(p, b[0:n]);
	*r.Data = b[n:len(b)];
	return n, nil;
}

// NewByteReader returns a new ByteReader reading from data.
func NewByteReader(data []byte) ByteReader {
	return ByteReader{ &data };
}

<commit_msg>document requirements on Write method<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// This package provides basic interfaces to I/O primitives.
// Its primary job is to wrap existing implementations of such primitives,
// such as those in package os, into shared public interfaces that
// abstract the functionality.
// It also provides buffering primitives and some other basic operations.
package io

import (
	"bytes";
	"os";
)

// Error represents an unexpected I/O behavior.
type Error struct {
	os.ErrorString
}

// ErrShortWrite means that a write accepted fewer bytes than requested
// but failed to return an explicit error.
var ErrShortWrite os.Error = &Error{"short write"}

// ErrUnexpectedEOF means that os.EOF was encountered in the
// middle of reading a fixed-size block or data structure.
var ErrUnexpectedEOF os.Error = &Error{"unexpected EOF"}

// Reader is the interface that wraps the basic Read method.
//
// Read reads up to len(p) bytes into p.  It returns the number of bytes
// read (0 <= n <= len(p)) and any error encountered.
// Even if Read returns n < len(p),
// it may use all of p as scratch space during the call.
// If some data is available but not len(p) bytes, Read conventionally
// returns what is available rather than block waiting for more.
//
// At the end of the input stream, Read returns 0, os.EOF.
// Read may return a non-zero number of bytes with a non-nil err.
// In particular, a Read that exhausts the input may return n > 0, os.EOF.
type Reader interface {
	Read(p []byte) (n int, err os.Error);
}

// Writer is the interface that wraps the basic Write method.
//
// Write writes len(p) bytes from p to the underlying data stream.
// It returns the number of bytes written from p (0 <= n <= len(p))
// and any error encountered that caused the write to stop early.
// Write must return a non-nil error if it returns n < len(p).
type Writer interface {
	Write(p []byte) (n int, err os.Error);
}

// Closer is the interface that wraps the basic Close method.
type Closer interface {
	Close() os.Error;
}

// ReadWrite is the interface that groups the basic Read and Write methods.
type ReadWriter interface {
	Reader;
	Writer;
}

// ReadCloser is the interface that groups the basic Read and Close methods.
type ReadCloser interface {
	Reader;
	Closer;
}

// WriteCloser is the interface that groups the basic Write and Close methods.
type WriteCloser interface {
	Writer;
	Closer;
}

// ReadWriteCloser is the interface that groups the basic Read, Write and Close methods.
type ReadWriteCloser interface {
	Reader;
	Writer;
	Closer;
}

// Convert a string to an array of bytes for easy marshaling.
func StringBytes(s string) []byte {
	b := make([]byte, len(s));
	for i := 0; i < len(s); i++ {
		b[i] = s[i];
	}
	return b;
}

// WriteString writes the contents of the string s to w, which accepts an array of bytes.
func WriteString(w Writer, s string) (n int, err os.Error) {
	return w.Write(StringBytes(s))
}

// ReadAtLeast reads from r into buf until it has read at least min bytes.
// It returns the number of bytes copied and an error if fewer bytes were read.
// The error is os.EOF only if no bytes were read.
// If an EOF happens after reading fewer than min bytes,
// ReadAtLeast returns ErrUnexpectedEOF.
func ReadAtLeast(r Reader, buf []byte, min int) (n int, err os.Error) {
	n = 0;
	for n < min {
		nn, e := r.Read(buf[n:len(buf)]);
		if nn > 0 {
			n += nn
		}
		if e != nil {
			if e == os.EOF && n > 0 {
				e = ErrUnexpectedEOF;
			}
			return n, e
		}
	}
	return n, nil
}

// ReadFull reads exactly len(buf) bytes from r into buf.
// It returns the number of bytes copied and an error if fewer bytes were read.
// The error is os.EOF only if no bytes were read.
// If an EOF happens after reading some but not all the bytes,
// ReadFull returns ErrUnexpectedEOF.
func ReadFull(r Reader, buf []byte) (n int, err os.Error) {
	// TODO(rsc): 6g bug keeps us from writing the obvious 1-liner
	n, err = ReadAtLeast(r, buf, len(buf));
	return;
}

// Copyn copies n bytes (or until an error) from src to dst.
// It returns the number of bytes copied and the error, if any.
func Copyn(src Reader, dst Writer, n int64) (written int64, err os.Error) {
	buf := make([]byte, 32*1024);
	for written < n {
		l := len(buf);
		if d := n - written; d < int64(l) {
			l = int(d);
		}
		nr, er := src.Read(buf[0 : l]);
		if nr > 0 {
			nw, ew := dst.Write(buf[0 : nr]);
			if nw > 0 {
				written += int64(nw);
			}
			if ew != nil {
				err = ew;
				break;
			}
			if nr != nw {
				err = ErrShortWrite;
				break;
			}
		}
		if er != nil {
			err = er;
			break;
		}
	}
	return written, err
}

// Copy copies from src to dst until either EOF is reached
// on src or an error occurs.  It returns the number of bytes
// copied and the error, if any.
func Copy(src Reader, dst Writer) (written int64, err os.Error) {
	buf := make([]byte, 32*1024);
	for {
		nr, er := src.Read(buf);
		if nr > 0 {
			nw, ew := dst.Write(buf[0:nr]);
			if nw > 0 {
				written += int64(nw);
			}
			if ew != nil {
				err = ew;
				break;
			}
			if nr != nw {
				err = ErrShortWrite;
				break;
			}
		}
		if er == os.EOF {
			break;
		}
		if er != nil {
			err = er;
			break;
		}
	}
	return written, err
}

// A ByteReader satisfies Reads by consuming data from a slice of bytes.
// Clients can call NewByteReader to create one or wrap pointers
// to their own slices: r := ByteReader{&data}.
type ByteReader struct {
	Data *[]byte
}

func (r ByteReader) Read(p []byte) (int, os.Error) {
	n := len(p);
	b := *r.Data;
	if len(b) == 0 {
		return 0, os.EOF;
	}
	if n > len(b) {
		n = len(b);
	}
	bytes.Copy(p, b[0:n]);
	*r.Data = b[n:len(b)];
	return n, nil;
}

// NewByteReader returns a new ByteReader reading from data.
func NewByteReader(data []byte) ByteReader {
	return ByteReader{ &data };
}

<|endoftext|>


<commit_before>package domain

import "time"

// Tweet is the domain object representing a tweet
type Tweet struct {
	ID   string
	Text string
	Time time.Time
}

// TweetWords represents the words contained in a tweet
type TweetWords struct {
	Words     []string
	TweetID   string
	TweetTime time.Time
}

// Tweets are a stoppable stream of tweets
type Tweets struct {
	Data <-chan Tweet
	Stop chan<- bool
}
<commit_msg>Add TweetsWords type<commit_after>package domain

import "time"

// Tweet is the domain object representing a tweet
type Tweet struct {
	ID   string
	Text string
	Time time.Time
}

// TweetWords represents the words contained in a tweet
type TweetWords struct {
	Words     []string
	TweetID   string
	TweetTime time.Time
}

// Tweets are a stoppable stream of tweets
type Tweets struct {
	Data <-chan Tweet
	Stop chan<- bool
}

// TweetsWords is a stoppable stream of TweetWords
type TweetsWords struct {
	Data <-chan TweetWords
	Stop chan<- bool
}
<|endoftext|>