<commit_before>package main

import (
	"encoding/gob"
	"errors"
	"fmt"
	"github.com/hanwen/go-fuse/fuse"
	"github.com/hanwen/go-fuse/fuse/nodefs"
	"io"
	"io/ioutil"
)

const (
	kMaxEntryNumberInFile = 1024
)

type Entry struct {
	Id       int
	ParentId int
	Dir      bool
	Key      string
	Name     string
	FsName   string
	fs       *FileSystem
	nodefs.Node
	size uint64
}

func NewDir(id int, name string, parent *Entry) (*Entry, error) {
	return newEntry(id, name, "", "", parent, true)
}

func NewFile(id int, name string, fsName string, key string, parent *Entry) (*Entry, error) {
	return newEntry(id, name, fsName, key, parent, false)
}

func newEntry(id int, name string, fsName string, key string, parent *Entry, dir bool) (*Entry, error) {
	if parent != nil && parent.Dir == false {
		return nil, errors.New("parent entry can't be file type")
	}
	entry := Entry{id, 0, dir, key, name, fsName, nil, nil, 0}

	entry.Id = id
	if parent == nil {
		entry.ParentId = 0
	} else {
		entry.ParentId = parent.Id
	}
	entry.Key = key
	entry.Dir = dir
	entry.Name = name
	entry.FsName = fsName
	return &entry, nil
}

func ReadEntries(reader io.Reader, buff []*Entry) error {
	dec := gob.NewDecoder(reader)
	return dec.Decode(&buff)
}

func WriteEntries(writer io.Writer, buff []*Entry) error {
	enc := gob.NewEncoder(writer)
	return enc.Encode(buff)
}

func (e *Entry) Print() {
	if e.Dir {
		fmt.Printf("[Dir]: Id: %d, ParentId: %d, Name: %s, FsName: %s\n",
			e.Id, e.ParentId, e.Name, e.FsName)
	} else {
		fmt.Printf("Id: %d, ParentId: %d, Name: %s, FsName: %s, Key %s\n",
			e.Id, e.ParentId, e.Name, e.FsName, e.Key)
	}
}

func (e *Entry) Stat(out *fuse.Attr) {
	out.Mode = fuse.S_IFREG | 0444
	if e.size != 0 {
		out.Size = e.fs.oss.Size(e.FsName)
		e.size = out.Size
	} else {
		out.Size = e.size
	}
}

func (e *Entry) Data() (data []byte) {
	fp, err := e.fs.oss.Open(e.FsName)
	if err != nil {
		return nil
	}
	fpd := MakeDecryptor(e.fs.config.Enc, fp)
	data, _ = ioutil.ReadAll(fpd)

	out := fuse.Attr{}
	e.Stat(&out)
	size := int(out.Size)
	if len(data) < size {
		data = append(data, make([]byte, size-len(data))...)
	}

	fpd.Close()
	fp.Close()
	return
}

func (n *Entry) OnMount(c *nodefs.FileSystemConnector) {
	n.fs.onMount()
}

func (n *Entry) OpenDir(context *fuse.Context) (stream []fuse.DirEntry, code fuse.Status) {
	children := n.Inode().Children()
	stream = make([]fuse.DirEntry, 0, len(children))
	for k, v := range children {
		mode := fuse.S_IFREG | 0666
		if v.IsDir() {
			mode = fuse.S_IFDIR | 0777
		}
		stream = append(stream, fuse.DirEntry{
			Name: k,
			Mode: uint32(mode),
		})
	}
	return stream, fuse.OK
}

func (n *Entry) Open(flags uint32, context *fuse.Context) (fuseFile nodefs.File, code fuse.Status) {
	if flags&fuse.O_ANYWRITE != 0 {
		return nil, fuse.EPERM
	}

	return nodefs.NewDataFile(n.Data()), fuse.OK
}

func (n *Entry) Deletable() bool {
	return false
}

func (n *Entry) GetAttr(out *fuse.Attr, file nodefs.File, context *fuse.Context) fuse.Status {
	if n.Inode().IsDir() {
		out.Mode = fuse.S_IFDIR | 0777
		return fuse.OK
	}
	n.Stat(out)
	return fuse.OK
}
<commit_msg>fix data size issue<commit_after>package main

import (
	"encoding/gob"
	"fmt"
	"github.com/hanwen/go-fuse/fuse"
	"github.com/hanwen/go-fuse/fuse/nodefs"
	"io"
	"time"
)

const kDefaultFsNameLength = 16
const kDefaultFsKeyLength = 16

type FileSystem struct {
	config  *Config
	oss     Oss
	entryId int
	Name    string
	entries []*Entry
}

func NewFileSystem(config *Config) *FileSystem {
	fs := &FileSystem{}
	fs.config = config
	fs.oss = MakeOss(config.Oss)
	return fs
}

func (fs *FileSystem) WriteFileEntries() {
	var fp io.WriteCloser
	var fpe Encryptor
	var enc *gob.Encoder
	for i := 0; i < len(fs.entries); i++ {
		if i%fs.config.Fs.BlockSize == 0 {
			fName := fmt.Sprintf("%s_%d", fs.config.Fs.Prefix, i/fs.config.Fs.BlockSize)
			println(fName)
			if fpe != nil {
				fpe.Close()
			}
			if fp != nil {
				fp.Close()
			}
			fp, _ = fs.oss.Create(fName)
			fpe = MakeEncryptor(fs.config.Enc, fp)
			enc = gob.NewEncoder(fpe)
		}
		entry := fs.entries[i]
		enc.Encode(entry)
	}
	if fpe != nil {
		fpe.Close()
		fp.Close()
	}
}

func (fs *FileSystem) readFileEntries() {
	index := 0
	for {
		fName := fmt.Sprintf("%s_%d", fs.config.Fs.Prefix, index)
		index++
		if err := fs.oss.Stat(fName); err != nil {
			break
		} else {
			fp, _ := fs.oss.Open(fName)
			fpd := MakeDecryptor(fs.config.Enc, fp)
			dec := gob.NewDecoder(fpd)
			for {
				entry := &Entry{}
				if err := dec.Decode(entry); err != nil {
					fpd.Close()
					fp.Close()
					break
				} else {
					if entry.Id != 0 {
						entry.fs = fs
						entry.Node = nodefs.NewDefaultNode()
						fs.entries = append(fs.entries, entry)
						if fs.entryId+1 != entry.Id {
							panic("entry ID error")
						}
						entry.Print()
						fs.entryId = entry.Id
						entry.fs = fs
					} else {
						panic("root should not in database")
					}
				}
			}
		}
	}
}

func (fs *FileSystem) String() string {
	return fs.Name
}

func (fs *FileSystem) Root() nodefs.Node {
	return fs.entries[0]
}

func (fs *FileSystem) onMount() {
	fs.readFileEntries()
	for _, entry := range fs.entries {
		if entry.Id == 0 {
			continue
		}
		pentry := fs.entries[entry.ParentId]
		println(entry.Name)
		pentry.Inode().NewChild(entry.Name, entry.Dir, entry)
	}
}

func (fs *FileSystem) Mount() (*fuse.Server, error) {
	root := &Entry{0, 0, true, "", "", "", fs, nodefs.NewDefaultNode(), 0}
	fs.entries = append(fs.entries, root)
	opts := &nodefs.Options{
		AttrTimeout:  time.Duration(float64(time.Second)),
		EntryTimeout: time.Duration(float64(time.Second)),
		Debug:        false,
	}
	println(fs.config.Fs.MountPoint)
	server, _, err := nodefs.MountRoot(fs.config.Fs.MountPoint, root, opts)
	return server, err
}
<|endoftext|>


<commit_before>package log

import (
	"fmt"
	"sync"
	"time"
)

type Fields map[string]interface{}

type Entry struct {
	wg        *sync.WaitGroup
	logger    *Logger
	calldepth int

	AppID     string
	Host      string
	Level     Level  `json:"level"`
	Message   string `json:"message"`
	File      string
	Line      int       `json:"line"`
	Timestamp time.Time `json:"timestamp"`
	Fields    Fields
}

// Debug level message.
func (e *Entry) Debug(v ...interface{}) {
	e.Level = DebugLevel
	e.Message = fmt.Sprint(v...)

	e.logger.handleEntry(e)
}

// Info level message.
func (e *Entry) Info(v ...interface{}) {
	e.Level = InfoLevel
	e.Message = fmt.Sprint(v...)

	e.logger.handleEntry(e)
}

// Warning level message.
func (e *Entry) Warn(v ...interface{}) {
	e.Level = WarnLevel
	e.Message = fmt.Sprint(v...)

	e.logger.handleEntry(e)
}

// Error level message.
func (e *Entry) Error(v ...interface{}) {
	e.Level = ErrorLevel
	e.Message = fmt.Sprint(v...)

	e.logger.handleEntry(e)
}

// Fatal level message.
func (e *Entry) Fatal(v ...interface{}) {
	e.Level = FatalLevel
	e.Message = fmt.Sprint(v...)

	e.logger.handleEntry(e)
}

// Consumed lets the Entry and subsequently the Logger
// instance know that it has been used by a handler
func (e *Entry) Consumed() {
	if e.wg != nil {
		e.wg.Done()
	}
}
<commit_msg>add panic function to entry<commit_after>package log

import (
	"fmt"
	"sync"
	"time"
)

type Fields map[string]interface{}

type Entry struct {
	wg        *sync.WaitGroup
	logger    *Logger
	calldepth int

	AppID     string
	Host      string
	Level     Level  `json:"level"`
	Message   string `json:"message"`
	File      string
	Line      int       `json:"line"`
	Timestamp time.Time `json:"timestamp"`
	Fields    Fields
}

// Debug level message.
func (e *Entry) Debug(v ...interface{}) {
	e.Level = DebugLevel
	e.Message = fmt.Sprint(v...)

	e.logger.handleEntry(e)
}

// Info level message.
func (e *Entry) Info(v ...interface{}) {
	e.Level = InfoLevel
	e.Message = fmt.Sprint(v...)

	e.logger.handleEntry(e)
}

// Warning level message.
func (e *Entry) Warn(v ...interface{}) {
	e.Level = WarnLevel
	e.Message = fmt.Sprint(v...)
	e.logger.handleEntry(e)
}

// Error level message.
func (e *Entry) Error(v ...interface{}) {
	e.Level = ErrorLevel
	e.Message = fmt.Sprint(v...)
	e.logger.handleEntry(e)
}

// Panic level message.
func (e *Entry) Panic(v ...interface{}) {
	e.Level = PanicLevel
	e.Message = fmt.Sprint(v...)
	e.logger.handleEntry(e)
	panic(e.Message)
}

// Fatal level message.
func (e *Entry) Fatal(v ...interface{}) {
	e.Level = FatalLevel
	e.Message = fmt.Sprint(v...)
	e.logger.handleEntry(e)
	exitFunc(1)
}

// Consumed lets the Entry and subsequently the Logger
// instance know that it has been used by a handler
func (e *Entry) Consumed() {
	if e.wg != nil {
		e.wg.Done()
	}
}
<|endoftext|>


<commit_before>package reuseport

import (
	"context"
	"net"
	"syscall"

	"golang.org/x/sys/unix"
)

var listenConfig = net.ListenConfig{
	Control: func(network, address string, c syscall.RawConn) (err error) {
		return c.Control(func(fd uintptr) {
			err = unix.SetsockoptInt(int(fd), unix.SOL_SOCKET, unix.SO_REUSEADDR, 1)
			if err == nil {
				err = unix.SetsockoptInt(int(fd), unix.SOL_SOCKET, unix.SO_REUSEPORT, 1)
			}
		})
	},
}

// Listen returns TCP listener with SO_REUSEADDR and SO_REUSEPORT option set, so it uses
func Listen(network, addr string) (net.Listener, error) {
	return listenConfig.Listen(context.Background(), network, addr)
}
<commit_msg>Fix apparent documentation typo (#1330)<commit_after>package reuseport

import (
	"context"
	"net"
	"syscall"

	"golang.org/x/sys/unix"
)

var listenConfig = net.ListenConfig{
	Control: func(network, address string, c syscall.RawConn) (err error) {
		return c.Control(func(fd uintptr) {
			err = unix.SetsockoptInt(int(fd), unix.SOL_SOCKET, unix.SO_REUSEADDR, 1)
			if err == nil {
				err = unix.SetsockoptInt(int(fd), unix.SOL_SOCKET, unix.SO_REUSEPORT, 1)
			}
		})
	},
}

// Listen returns a TCP listener with the SO_REUSEADDR and SO_REUSEPORT options set.
func Listen(network, addr string) (net.Listener, error) {
	return listenConfig.Listen(context.Background(), network, addr)
}
<|endoftext|>


<commit_before>package err

import "errors"
import "strconv"

// E 组装 json 格式错误信息：
// {"code": 105,"error": "invalid field name: bl!ng"}
func E(code int, msg string) error {
	text := `{"code": ` + strconv.Itoa(code) + `,"error": "` + msg + `"}`
	return errors.New(text)
}
<commit_msg>添加错误码<commit_after>package err

import "errors"
import "strconv"

// E 组装 json 格式错误信息：
// {"code": 105,"error": "invalid field name: bl!ng"}
func E(code int, msg string) error {
	text := `{"code": ` + strconv.Itoa(code) + `,"error": "` + msg + `"}`
	return errors.New(text)
}

// OtherCause ...
// Error code indicating some error other than those enumerated here.
var OtherCause = -1

// InternalServerError ...
// Error code indicating that something has gone wrong with the server.
// If you get this error code, it is Parse's fault. Contact us at
// https://parse.com/help
var InternalServerError = 1

// ConnectionFailed ...
// Error code indicating the connection to the Parse servers failed.
var ConnectionFailed = 100

// ObjectNotFound ...
// Error code indicating the specified object doesn't exist.
var ObjectNotFound = 101

// InvalidQuery ...
// Error code indicating you tried to query with a datatype that doesn't
// support it, like exact matching an array or object.
var InvalidQuery = 102

// InvalidClassName ...
// Error code indicating a missing or invalid classname. Classnames are
// case-sensitive. They must start with a letter, and a-zA-Z0-9 are the
// only valid characters.
var InvalidClassName = 103

// MissingObjectID ...
// Error code indicating an unspecified object id.
var MissingObjectID = 104

// InvalidKeyName ...
// Error code indicating an invalid key name. Keys are case-sensitive. They
// must start with a letter, and a-zA-Z0-9 are the only valid characters.
var InvalidKeyName = 105

// InvalidPointer ...
// Error code indicating a malformed pointer. You should not see this unless
// you have been mucking about changing internal Parse code.
var InvalidPointer = 106

// InvalidJSON ...
// Error code indicating that badly formed JSON was received upstream. This
// either indicates you have done something unusual with modifying how
// things encode to JSON, or the network is failing badly.
var InvalidJSON = 107

// CommandUnavailable ...
// Error code indicating that the feature you tried to access is only
// available internally for testing purposes.
var CommandUnavailable = 108

// NotInitialized ...
// You must call Parse.initialize before using the Parse library.
var NotInitialized = 109

// IncorrectType ...
// Error code indicating that a field was set to an inconsistent type.
var IncorrectType = 111

// InvalidChannelName ...
// Error code indicating an invalid channel name. A channel name is either
// an empty string (the broadcast channel) or contains only a-zA-Z0-9
// characters and starts with a letter.
var InvalidChannelName = 112

// PushMisconfigured ...
// Error code indicating that push is misconfigured.
var PushMisconfigured = 115

// ObjectTooLarge ...
// Error code indicating that the object is too large.
var ObjectTooLarge = 116

// OperationForbidden ...
// Error code indicating that the operation isn't allowed for clients.
var OperationForbidden = 119

// CacheMiss ...
// Error code indicating the result was not found in the cache.
var CacheMiss = 120

// InvalidNestedKey ...
// Error code indicating that an invalid key was used in a nested
// JSONObject.
var InvalidNestedKey = 121

// InvalidFileName ...
// Error code indicating that an invalid filename was used for ParseFile.
// A valid file name contains only a-zA-Z0-9. characters and is between 1
// and 128 characters.
var InvalidFileName = 122

// InvalidAcl ...
// Error code indicating an invalid ACL was provided.
var InvalidAcl = 123

// Timeout ...
// Error code indicating that the request timed out on the server. Typically
// this indicates that the request is too expensive to run.
var Timeout = 124

// InvalidEmailAddress ...
// Error code indicating that the email address was invalid.
var InvalidEmailAddress = 125

// MissingContentType ...
// Error code indicating a missing content type.
var MissingContentType = 126

// MissingContentLength ...
// Error code indicating a missing content length.
var MissingContentLength = 127

// InvalidContentLength ...
// Error code indicating an invalid content length.
var InvalidContentLength = 128

// FileTooLarge ...
// Error code indicating a file that was too large.
var FileTooLarge = 129

// FileSaveError ...
// Error code indicating an error saving a file.
var FileSaveError = 130

// DuplicateValue ...
// Error code indicating that a unique field was given a value that is
// already taken.
var DuplicateValue = 137

// InvalidRoleName ...
// Error code indicating that a role's name is invalid.
var InvalidRoleName = 139

// ExceededQuota ...
// Error code indicating that an application quota was exceeded.  Upgrade to
// resolve.
var ExceededQuota = 140

// ScriptFailed ...
// Error code indicating that a Cloud Code script failed.
var ScriptFailed = 141

// ValidationError ...
// Error code indicating that a Cloud Code validation failed.
var ValidationError = 142

// InvalidImageData ...
// Error code indicating that invalid image data was provided.
var InvalidImageData = 143

// UnsavedFileError ...
// Error code indicating an unsaved file.
var UnsavedFileError = 151

// InvalidPushTimeError ...
// Error code indicating an invalid push time.
var InvalidPushTimeError = 152

// FileDeleteError ...
// Error code indicating an error deleting a file.
var FileDeleteError = 153

// RequestLimitExceeded ...
// Error code indicating that the application has exceeded its request
// limit.
var RequestLimitExceeded = 155

// InvalidEventName ...
// Error code indicating an invalid event name.
var InvalidEventName = 160

// UsernameMissing ...
// Error code indicating that the username is missing or empty.
var UsernameMissing = 200

// PasswordMissing ...
// Error code indicating that the password is missing or empty.
var PasswordMissing = 201

// UsernameTaken ...
// Error code indicating that the username has already been taken.
var UsernameTaken = 202

// EmailTaken ...
// Error code indicating that the email has already been taken.
var EmailTaken = 203

// EmailMissing ...
// Error code indicating that the email is missing, but must be specified.
var EmailMissing = 204

// EmailNotFound ...
// Error code indicating that a user with the specified email was not found.
var EmailNotFound = 205

// SessionMissing ...
// Error code indicating that a user object without a valid session could
// not be altered.
var SessionMissing = 206

// MustCreateUserThroughSignup ...
// Error code indicating that a user can only be created through signup.
var MustCreateUserThroughSignup = 207

// AccountAlreadyLinked ...
// Error code indicating that an an account being linked is already linked
// to another user.
var AccountAlreadyLinked = 208

// InvalidSessionToken ...
// Error code indicating that the current session token is invalid.
var InvalidSessionToken = 209

// LinkedIDMissing ...
// Error code indicating that a user cannot be linked to an account because
// that account's id could not be found.
var LinkedIDMissing = 250

// InvalidLinkedSession ...
// Error code indicating that a user with a linked (e.g. Facebook) account
// has an invalid session.
var InvalidLinkedSession = 251

// UnsupportedService ...
// Error code indicating that a service being linked (e.g. Facebook or
// Twitter) is unsupported.
var UnsupportedService = 252

// AggregateError ...
// Error code indicating that there were multiple errors. Aggregate errors
// have an "errors" property, which is an array of error objects with more
// detail about each error that occurred.
var AggregateError = 600

// FileReadError ...
// Error code indicating the client was unable to read an input file.
var FileReadError = 601

// XDomainRequest ...
// Error code indicating a real error code is unavailable because
// we had to use an XDomainRequest object to allow CORS requests in
// Internet Explorer, which strips the body from HTTP responses that have
// a non-2XX status code.
var XDomainRequest = 602
<|endoftext|>


<commit_before>package flags

type ErrorType uint

const (
	ErrUnknown ErrorType = iota

	// Expected an argument but got none
	ErrExpectedArgument

	// Unknown flag
	ErrUnknownFlag

	// Failed to marshal value
	ErrMarshal

	// The builtin help message was printed
	ErrHelpShown

	// An argument for a boolean value was specified
	ErrNoArgumentForBool
)

type Error struct {
	Type ErrorType
	Message string
}

func (e *Error) Error() string {
	return e.Message
}

func newError(tp ErrorType, message string) error {
	return &Error {
		Type: tp,
		Message: message,
	}
}

var ErrHelp = newError(ErrHelpShown, "help shown")
<commit_msg>Added documentation for the error type<commit_after>package flags

// ErrorType represents the type of error.
type ErrorType uint

const (
	// Unknown or generic error
	ErrUnknown ErrorType = iota

	// Expected an argument but got none
	ErrExpectedArgument

	// Unknown flag
	ErrUnknownFlag

	// Failed to marshal value
	ErrMarshal

	// The builtin help message was printed
	ErrHelpShown

	// An argument for a boolean value was specified
	ErrNoArgumentForBool
)

// Error represents a parser error. The error returned from Parse is of this
// type. The error contains both a Type and Message.
type Error struct {
	// The type of error
	Type ErrorType

	// The error message
	Message string
}

// Get the errors error message.
func (e *Error) Error() string {
	return e.Message
}

func newError(tp ErrorType, message string) error {
	return &Error {
		Type: tp,
		Message: message,
	}
}

// The builtin help message was printed. This value will be returned from
// Parse when the builtin help message was shown (i.e. when the ShowHelp
// option is set and either -h or --help was specified on the command line).
var ErrHelp = newError(ErrHelpShown, "help shown")
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package pgzip implements reading and writing of gzip format compressed files,
// as specified in RFC 1952.
//
// This is a drop in replacement for "compress/gzip".
// This will split compression into blocks that are compressed in parallel.
// This can be useful for compressing big amounts of data.
// The gzip decompression has not been modified, but remains in the package,
// so you can use it as a complete replacement for "compress/gzip".
//
// See more at https://github.com/klauspost/pgzip
package pgzip

import (
	"bufio"
	"compress/flate"
	"errors"
	"hash"
	"io"
	"sync"
	"time"

	"github.com/klauspost/crc32"
)

const (
	gzipID1     = 0x1f
	gzipID2     = 0x8b
	gzipDeflate = 8
	flagText    = 1 << 0
	flagHdrCrc  = 1 << 1
	flagExtra   = 1 << 2
	flagName    = 1 << 3
	flagComment = 1 << 4
)

func makeReader(r io.Reader) flate.Reader {
	if rr, ok := r.(flate.Reader); ok {
		return rr
	}
	return bufio.NewReader(r)
}

var (
	// ErrChecksum is returned when reading GZIP data that has an invalid checksum.
	ErrChecksum = errors.New("gzip: invalid checksum")
	// ErrHeader is returned when reading GZIP data that has an invalid header.
	ErrHeader = errors.New("gzip: invalid header")
)

// The gzip file stores a header giving metadata about the compressed file.
// That header is exposed as the fields of the Writer and Reader structs.
type Header struct {
	Comment string    // comment
	Extra   []byte    // "extra data"
	ModTime time.Time // modification time
	Name    string    // file name
	OS      byte      // operating system type
}

// A Reader is an io.Reader that can be read to retrieve
// uncompressed data from a gzip-format compressed file.
//
// In general, a gzip file can be a concatenation of gzip files,
// each with its own header.  Reads from the Reader
// return the concatenation of the uncompressed data of each.
// Only the first header is recorded in the Reader fields.
//
// Gzip files store a length and checksum of the uncompressed data.
// The Reader will return a ErrChecksum when Read
// reaches the end of the uncompressed data if it does not
// have the expected length or checksum.  Clients should treat data
// returned by Read as tentative until they receive the io.EOF
// marking the end of the data.
type Reader struct {
	Header
	r            flate.Reader
	decompressor io.ReadCloser
	digest       hash.Hash32
	size         uint32
	flg          byte
	buf          [512]byte
	err          error
	closeErr     chan error
	multistream  bool

	readAhead   chan interface{}
	current     []byte
	closeReader chan struct{}
	lastBlock   bool
	blockSize   int
	blocks      int

	activeRA bool       // Indication if readahead is active
	mu       sync.Mutex // Lock for above
}

// NewReader creates a new Reader reading the given reader.
// The implementation buffers input and may read more data than necessary from r.
// It is the caller's responsibility to call Close on the Reader when done.
func NewReader(r io.Reader) (*Reader, error) {
	z := new(Reader)
	z.blocks = defaultBlocks
	z.blockSize = defaultBlockSize
	z.r = makeReader(r)
	z.digest = crc32.NewIEEE()
	z.multistream = true
	if err := z.readHeader(true); err != nil {
		return nil, err
	}
	return z, nil
}

// NewReaderN creates a new Reader reading the given reader.
// The implementation buffers input and may read more data than necessary from r.
// It is the caller's responsibility to call Close on the Reader when done.
//
// With this you can control the approximate size of your blocks,
// as well as how many blocks you want to have prefetched.
//
// Default values for this is blockSize = 250000, blocks = 16,
// meaning up to 16 blocks of maximum 250000 bytes will be
// prefetched.
func NewReaderN(r io.Reader, blockSize, blocks int) (*Reader, error) {
	z := new(Reader)
	z.blocks = blocks
	z.blockSize = blockSize
	z.r = makeReader(r)
	z.digest = crc32.NewIEEE()
	z.multistream = true
	if err := z.readHeader(true); err != nil {
		return nil, err
	}
	return z, nil
}

// Reset discards the Reader z's state and makes it equivalent to the
// result of its original state from NewReader, but reading from r instead.
// This permits reusing a Reader rather than allocating a new one.
func (z *Reader) Reset(r io.Reader) error {
	z.r = makeReader(r)
	z.digest = crc32.NewIEEE()
	z.size = 0
	z.err = nil
	z.multistream = true
	return z.readHeader(true)
}

// Multistream controls whether the reader supports multistream files.
//
// If enabled (the default), the Reader expects the input to be a sequence
// of individually gzipped data streams, each with its own header and
// trailer, ending at EOF. The effect is that the concatenation of a sequence
// of gzipped files is treated as equivalent to the gzip of the concatenation
// of the sequence. This is standard behavior for gzip readers.
//
// Calling Multistream(false) disables this behavior; disabling the behavior
// can be useful when reading file formats that distinguish individual gzip
// data streams or mix gzip data streams with other data streams.
// In this mode, when the Reader reaches the end of the data stream,
// Read returns io.EOF. If the underlying reader implements io.ByteReader,
// it will be left positioned just after the gzip stream.
// To start the next stream, call z.Reset(r) followed by z.Multistream(false).
// If there is no next stream, z.Reset(r) will return io.EOF.
func (z *Reader) Multistream(ok bool) {
	z.multistream = ok
}

// GZIP (RFC 1952) is little-endian, unlike ZLIB (RFC 1950).
func get4(p []byte) uint32 {
	return uint32(p[0]) | uint32(p[1])<<8 | uint32(p[2])<<16 | uint32(p[3])<<24
}

func (z *Reader) readString() (string, error) {
	var err error
	needconv := false
	for i := 0; ; i++ {
		if i >= len(z.buf) {
			return "", ErrHeader
		}
		z.buf[i], err = z.r.ReadByte()
		if err != nil {
			return "", err
		}
		if z.buf[i] > 0x7f {
			needconv = true
		}
		if z.buf[i] == 0 {
			// GZIP (RFC 1952) specifies that strings are NUL-terminated ISO 8859-1 (Latin-1).
			if needconv {
				s := make([]rune, 0, i)
				for _, v := range z.buf[0:i] {
					s = append(s, rune(v))
				}
				return string(s), nil
			}
			return string(z.buf[0:i]), nil
		}
	}
}

func (z *Reader) read2() (uint32, error) {
	_, err := io.ReadFull(z.r, z.buf[0:2])
	if err != nil {
		return 0, err
	}
	return uint32(z.buf[0]) | uint32(z.buf[1])<<8, nil
}

func (z *Reader) readHeader(save bool) error {
	z.killReadAhead()
	_, err := io.ReadFull(z.r, z.buf[0:10])
	if err != nil {
		return err
	}
	if z.buf[0] != gzipID1 || z.buf[1] != gzipID2 || z.buf[2] != gzipDeflate {
		return ErrHeader
	}
	z.flg = z.buf[3]
	if save {
		z.ModTime = time.Unix(int64(get4(z.buf[4:8])), 0)
		// z.buf[8] is xfl, ignored
		z.OS = z.buf[9]
	}
	z.digest.Reset()
	z.digest.Write(z.buf[0:10])

	if z.flg&flagExtra != 0 {
		n, err := z.read2()
		if err != nil {
			return err
		}
		data := make([]byte, n)
		if _, err = io.ReadFull(z.r, data); err != nil {
			return err
		}
		if save {
			z.Extra = data
		}
	}

	var s string
	if z.flg&flagName != 0 {
		if s, err = z.readString(); err != nil {
			return err
		}
		if save {
			z.Name = s
		}
	}

	if z.flg&flagComment != 0 {
		if s, err = z.readString(); err != nil {
			return err
		}
		if save {
			z.Comment = s
		}
	}

	if z.flg&flagHdrCrc != 0 {
		n, err := z.read2()
		if err != nil {
			return err
		}
		sum := z.digest.Sum32() & 0xFFFF
		if n != sum {
			return ErrHeader
		}
	}

	z.digest.Reset()
	z.decompressor = flate.NewReader(z.r)
	z.doReadAhead()
	return nil
}

func (z *Reader) killReadAhead() error {
	z.mu.Lock()
	defer z.mu.Unlock()
	if z.activeRA {
		if z.closeReader != nil {
			close(z.closeReader)
		}

		// Wait for decompressor to be closed and return error, if any.
		e, ok := <-z.closeErr
		z.activeRA = false
		if !ok {
			// Channel is closed, so if there was any error it has already been returned.
			return nil
		}
		return e
	}
	return nil
}

// Starts readahead.
// Will return on error (including io.EOF)
// or when z.closeReader is closed.
func (z *Reader) doReadAhead() {
	z.mu.Lock()
	defer z.mu.Unlock()
	z.activeRA = true

	if z.blocks <= 0 {
		z.blocks = defaultBlocks
	}
	if z.blockSize <= 512 {
		z.blockSize = defaultBlockSize
	}
	z.readAhead = make(chan interface{}, z.blocks*2)
	closeReader := make(chan struct{}, 0)
	z.closeReader = closeReader
	z.lastBlock = false
	closeErr := make(chan error, 1)
	z.closeErr = closeErr
	z.size = 0

	go func() {
		defer close(z.readAhead)
		defer func() {
			closeErr <- z.decompressor.Close()
			close(closeErr)
		}()

		// We hold a local reference to digest, since
		// it way be changed by reset.
		digest := z.digest
		// Lock for our digest.
		dLock := sync.Mutex{}
		for {
			buf := make([]byte, z.blockSize)
			n, err := z.decompressor.Read(buf)
			if n < len(buf) {
				buf = buf[0:n]
			}

			dLock.Lock()
			go func() {
				digest.Write(buf)
				dLock.Unlock()
			}()
			z.size += uint32(n)

			select {
			case z.readAhead <- buf:
				// Also send the error
				if err != nil {
					// When we send an error, digest must be finished.
					dLock.Lock()
					dLock.Unlock()
				}
				z.readAhead <- err

			case <-closeReader:
				// Sent on close, we don't care about the next results
				return
			}
			if err != nil {
				return
			}
		}
	}()
}

func (z *Reader) Read(p []byte) (n int, err error) {
	if z.err != nil {
		return 0, z.err
	}
	if len(p) == 0 {
		return 0, nil
	}

	for {
		if len(z.current) == 0 && !z.lastBlock {
			bufi := <-z.readAhead
			erri := <-z.readAhead

			if erri != nil {
				// If not nil, the reader will have exited
				z.closeReader = nil

				err = erri.(error)
				if err != io.EOF {
					z.err = err
					return
				}
				if err == io.EOF {
					z.lastBlock = true
					err = nil
				}
			}
			buf := bufi.([]byte)
			z.current = buf
		}
		if len(p) >= len(z.current) {
			// If len(p) >= len(current), return all content of current
			copy(p, z.current)
			n = len(z.current)
			z.current = nil
			if z.lastBlock {
				err = io.EOF
				break
			}
		} else {
			// We copy as much as there is space for, and reslice current
			copy(p, z.current[0:len(p)])
			n = len(p)
			z.current = z.current[n:]
		}
		return
	}

	// Finished file; check checksum + size.
	if _, err := io.ReadFull(z.r, z.buf[0:8]); err != nil {
		z.err = err
		return 0, err
	}
	crc32, isize := get4(z.buf[0:4]), get4(z.buf[4:8])
	sum := z.digest.Sum32()
	if sum != crc32 || isize != z.size {
		z.err = ErrChecksum
		return 0, z.err
	}

	// File is ok; should we attempt reading one more?
	if !z.multistream {
		return 0, io.EOF
	}

	// Is there another?
	if err = z.readHeader(false); err != nil {
		z.err = err
		return
	}

	// Yes.  Reset and read from it.
	return z.Read(p)
}

// Close closes the Reader. It does not close the underlying io.Reader.
func (z *Reader) Close() error {
	return z.killReadAhead()
}
<commit_msg>Use combined struct for result and error.<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package pgzip implements reading and writing of gzip format compressed files,
// as specified in RFC 1952.
//
// This is a drop in replacement for "compress/gzip".
// This will split compression into blocks that are compressed in parallel.
// This can be useful for compressing big amounts of data.
// The gzip decompression has not been modified, but remains in the package,
// so you can use it as a complete replacement for "compress/gzip".
//
// See more at https://github.com/klauspost/pgzip
package pgzip

import (
	"bufio"
	"compress/flate"
	"errors"
	"hash"
	"io"
	"sync"
	"time"

	"github.com/klauspost/crc32"
)

const (
	gzipID1     = 0x1f
	gzipID2     = 0x8b
	gzipDeflate = 8
	flagText    = 1 << 0
	flagHdrCrc  = 1 << 1
	flagExtra   = 1 << 2
	flagName    = 1 << 3
	flagComment = 1 << 4
)

func makeReader(r io.Reader) flate.Reader {
	if rr, ok := r.(flate.Reader); ok {
		return rr
	}
	return bufio.NewReader(r)
}

var (
	// ErrChecksum is returned when reading GZIP data that has an invalid checksum.
	ErrChecksum = errors.New("gzip: invalid checksum")
	// ErrHeader is returned when reading GZIP data that has an invalid header.
	ErrHeader = errors.New("gzip: invalid header")
)

// The gzip file stores a header giving metadata about the compressed file.
// That header is exposed as the fields of the Writer and Reader structs.
type Header struct {
	Comment string    // comment
	Extra   []byte    // "extra data"
	ModTime time.Time // modification time
	Name    string    // file name
	OS      byte      // operating system type
}

// A Reader is an io.Reader that can be read to retrieve
// uncompressed data from a gzip-format compressed file.
//
// In general, a gzip file can be a concatenation of gzip files,
// each with its own header.  Reads from the Reader
// return the concatenation of the uncompressed data of each.
// Only the first header is recorded in the Reader fields.
//
// Gzip files store a length and checksum of the uncompressed data.
// The Reader will return a ErrChecksum when Read
// reaches the end of the uncompressed data if it does not
// have the expected length or checksum.  Clients should treat data
// returned by Read as tentative until they receive the io.EOF
// marking the end of the data.
type Reader struct {
	Header
	r            flate.Reader
	decompressor io.ReadCloser
	digest       hash.Hash32
	size         uint32
	flg          byte
	buf          [512]byte
	err          error
	closeErr     chan error
	multistream  bool

	readAhead   chan read
	current     []byte
	closeReader chan struct{}
	lastBlock   bool
	blockSize   int
	blocks      int

	activeRA bool       // Indication if readahead is active
	mu       sync.Mutex // Lock for above
}

type read struct {
	b []byte
	err error
}

// NewReader creates a new Reader reading the given reader.
// The implementation buffers input and may read more data than necessary from r.
// It is the caller's responsibility to call Close on the Reader when done.
func NewReader(r io.Reader) (*Reader, error) {
	z := new(Reader)
	z.blocks = defaultBlocks
	z.blockSize = defaultBlockSize
	z.r = makeReader(r)
	z.digest = crc32.NewIEEE()
	z.multistream = true
	if err := z.readHeader(true); err != nil {
		return nil, err
	}
	return z, nil
}

// NewReaderN creates a new Reader reading the given reader.
// The implementation buffers input and may read more data than necessary from r.
// It is the caller's responsibility to call Close on the Reader when done.
//
// With this you can control the approximate size of your blocks,
// as well as how many blocks you want to have prefetched.
//
// Default values for this is blockSize = 250000, blocks = 16,
// meaning up to 16 blocks of maximum 250000 bytes will be
// prefetched.
func NewReaderN(r io.Reader, blockSize, blocks int) (*Reader, error) {
	z := new(Reader)
	z.blocks = blocks
	z.blockSize = blockSize
	z.r = makeReader(r)
	z.digest = crc32.NewIEEE()
	z.multistream = true
	if err := z.readHeader(true); err != nil {
		return nil, err
	}
	return z, nil
}

// Reset discards the Reader z's state and makes it equivalent to the
// result of its original state from NewReader, but reading from r instead.
// This permits reusing a Reader rather than allocating a new one.
func (z *Reader) Reset(r io.Reader) error {
	z.r = makeReader(r)
	z.digest = crc32.NewIEEE()
	z.size = 0
	z.err = nil
	z.multistream = true
	return z.readHeader(true)
}

// Multistream controls whether the reader supports multistream files.
//
// If enabled (the default), the Reader expects the input to be a sequence
// of individually gzipped data streams, each with its own header and
// trailer, ending at EOF. The effect is that the concatenation of a sequence
// of gzipped files is treated as equivalent to the gzip of the concatenation
// of the sequence. This is standard behavior for gzip readers.
//
// Calling Multistream(false) disables this behavior; disabling the behavior
// can be useful when reading file formats that distinguish individual gzip
// data streams or mix gzip data streams with other data streams.
// In this mode, when the Reader reaches the end of the data stream,
// Read returns io.EOF. If the underlying reader implements io.ByteReader,
// it will be left positioned just after the gzip stream.
// To start the next stream, call z.Reset(r) followed by z.Multistream(false).
// If there is no next stream, z.Reset(r) will return io.EOF.
func (z *Reader) Multistream(ok bool) {
	z.multistream = ok
}

// GZIP (RFC 1952) is little-endian, unlike ZLIB (RFC 1950).
func get4(p []byte) uint32 {
	return uint32(p[0]) | uint32(p[1])<<8 | uint32(p[2])<<16 | uint32(p[3])<<24
}

func (z *Reader) readString() (string, error) {
	var err error
	needconv := false
	for i := 0; ; i++ {
		if i >= len(z.buf) {
			return "", ErrHeader
		}
		z.buf[i], err = z.r.ReadByte()
		if err != nil {
			return "", err
		}
		if z.buf[i] > 0x7f {
			needconv = true
		}
		if z.buf[i] == 0 {
			// GZIP (RFC 1952) specifies that strings are NUL-terminated ISO 8859-1 (Latin-1).
			if needconv {
				s := make([]rune, 0, i)
				for _, v := range z.buf[0:i] {
					s = append(s, rune(v))
				}
				return string(s), nil
			}
			return string(z.buf[0:i]), nil
		}
	}
}

func (z *Reader) read2() (uint32, error) {
	_, err := io.ReadFull(z.r, z.buf[0:2])
	if err != nil {
		return 0, err
	}
	return uint32(z.buf[0]) | uint32(z.buf[1])<<8, nil
}

func (z *Reader) readHeader(save bool) error {
	z.killReadAhead()
	_, err := io.ReadFull(z.r, z.buf[0:10])
	if err != nil {
		return err
	}
	if z.buf[0] != gzipID1 || z.buf[1] != gzipID2 || z.buf[2] != gzipDeflate {
		return ErrHeader
	}
	z.flg = z.buf[3]
	if save {
		z.ModTime = time.Unix(int64(get4(z.buf[4:8])), 0)
		// z.buf[8] is xfl, ignored
		z.OS = z.buf[9]
	}
	z.digest.Reset()
	z.digest.Write(z.buf[0:10])

	if z.flg&flagExtra != 0 {
		n, err := z.read2()
		if err != nil {
			return err
		}
		data := make([]byte, n)
		if _, err = io.ReadFull(z.r, data); err != nil {
			return err
		}
		if save {
			z.Extra = data
		}
	}

	var s string
	if z.flg&flagName != 0 {
		if s, err = z.readString(); err != nil {
			return err
		}
		if save {
			z.Name = s
		}
	}

	if z.flg&flagComment != 0 {
		if s, err = z.readString(); err != nil {
			return err
		}
		if save {
			z.Comment = s
		}
	}

	if z.flg&flagHdrCrc != 0 {
		n, err := z.read2()
		if err != nil {
			return err
		}
		sum := z.digest.Sum32() & 0xFFFF
		if n != sum {
			return ErrHeader
		}
	}

	z.digest.Reset()
	z.decompressor = flate.NewReader(z.r)
	z.doReadAhead()
	return nil
}

func (z *Reader) killReadAhead() error {
	z.mu.Lock()
	defer z.mu.Unlock()
	if z.activeRA {
		if z.closeReader != nil {
			close(z.closeReader)
		}

		// Wait for decompressor to be closed and return error, if any.
		e, ok := <-z.closeErr
		z.activeRA = false
		if !ok {
			// Channel is closed, so if there was any error it has already been returned.
			return nil
		}
		return e
	}
	return nil
}

// Starts readahead.
// Will return on error (including io.EOF)
// or when z.closeReader is closed.
func (z *Reader) doReadAhead() {
	z.mu.Lock()
	defer z.mu.Unlock()
	z.activeRA = true

	if z.blocks <= 0 {
		z.blocks = defaultBlocks
	}
	if z.blockSize <= 512 {
		z.blockSize = defaultBlockSize
	}
	z.readAhead = make(chan read, z.blocks)
	closeReader := make(chan struct{}, 0)
	z.closeReader = closeReader
	z.lastBlock = false
	closeErr := make(chan error, 1)
	z.closeErr = closeErr
	z.size = 0

	go func() {
		defer close(z.readAhead)
		defer func() {
			closeErr <- z.decompressor.Close()
			close(closeErr)
		}()

		// We hold a local reference to digest, since
		// it way be changed by reset.
		digest := z.digest
		var wg sync.WaitGroup
		for {
			buf := make([]byte, z.blockSize)
			n, err := z.decompressor.Read(buf)
			if n < len(buf) {
				buf = buf[0:n]
			}
			wg.Wait()
			wg.Add(1)
			go func() {
				digest.Write(buf)
				wg.Done()
			}()
			z.size += uint32(n)

			if err != nil {
				wg.Wait()
			}
			select {
			case z.readAhead <- read{b: buf, err: err}:
			case <-closeReader:
				// Sent on close, we don't care about the next results
				return
			}
			if err != nil {
				return
			}
		}
	}()
}

func (z *Reader) Read(p []byte) (n int, err error) {
	if z.err != nil {
		return 0, z.err
	}
	if len(p) == 0 {
		return 0, nil
	}

	for {
		if len(z.current) == 0 && !z.lastBlock {
			read := <-z.readAhead

			if read.err != nil {
				// If not nil, the reader will have exited
				z.closeReader = nil

				if read.err != io.EOF {
					z.err = read.err
					return
				}
				if read.err == io.EOF {
					z.lastBlock = true
					err = nil
				}
			}
			z.current = read.b
		}
		if len(p) >= len(z.current) {
			// If len(p) >= len(current), return all content of current
			copy(p, z.current)
			n = len(z.current)
			z.current = nil
			if z.lastBlock {
				err = io.EOF
				break
			}
		} else {
			// We copy as much as there is space for, and reslice current
			copy(p, z.current[0:len(p)])
			n = len(p)
			z.current = z.current[n:]
		}
		return
	}

	// Finished file; check checksum + size.
	if _, err := io.ReadFull(z.r, z.buf[0:8]); err != nil {
		z.err = err
		return 0, err
	}
	crc32, isize := get4(z.buf[0:4]), get4(z.buf[4:8])
	sum := z.digest.Sum32()
	if sum != crc32 || isize != z.size {
		z.err = ErrChecksum
		return 0, z.err
	}

	// File is ok; should we attempt reading one more?
	if !z.multistream {
		return 0, io.EOF
	}

	// Is there another?
	if err = z.readHeader(false); err != nil {
		z.err = err
		return
	}

	// Yes.  Reset and read from it.
	return z.Read(p)
}

// Close closes the Reader. It does not close the underlying io.Reader.
func (z *Reader) Close() error {
	return z.killReadAhead()
}
<|endoftext|>


<commit_before>/*
 * Copyright 2014 Google Inc. All rights reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

// Package docker contains simple utilities for pulling a docker image, starting
// a container, and stoping a container. It assumes that docker is installed. If
// it is not, it will simply throw an error.
package docker

import (
	"bytes"
	"errors"
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	glog "github.com/google/shipshape/third_party/go-glog"
)

const (
	shipshapeWork = "/shipshape-workspace"
	shipshapeLogs = "/shipshape-output"
)

// TODO(ciera): Consider making these all use channels.
type CommandResult struct {
	Stdout string
	Stderr string
	Err    error
}

func trimResult(stdout, stderr *bytes.Buffer, err error) CommandResult {
	return CommandResult{strings.TrimSpace(stdout.String()), strings.TrimSpace(stderr.String()), err}
}

// ContainerExists checks if a container with the given name is in the list of running, or old, containers.
func ContainerExists(container string) (bool, error) {
	// Setup and run command
	// TODO(ciera): When Travis and other places we run this at support docker
	// 1.8, we can drastically reduce this code by using the flag --format={{.Names}}
	// below and removing the logic to parse out the name ourselves.
	stdout := bytes.NewBuffer(nil)
	cmd := exec.Command("docker", "ps", "-a")
	cmd.Stdout = stdout
	if err := cmd.Run(); err != nil {
		fmt.Printf("Problem running command, err: %v", err)
		return false, err
	}
	// Process output in search for container name match.
	// The 'docker ps' output lists the container names in the last column.
	// Prefixing the container name with a space to avoid substring matching.
	spacedName := fmt.Sprintf(" %s", container)
	for itr, line := range strings.Split(stdout.String(), "\n") {
		// Skip the title row
		if itr == 0 {
			continue
		}
		// Docker adds spaces to the end of each row
		trimmedLine := strings.Trim(line, " ")
		if strings.HasSuffix(trimmedLine, spacedName) {
			return true, nil
		}
	}
	return false, nil
}

// HasDocker determines whether docker is installed
// and included in PATH.
func HasDocker() bool {
	cmd := exec.Command("which", "docker")
	return cmd.Run() == nil
}

// FullImageName creates a full image name from a repository URI, an image name, and a tag.
func FullImageName(repo, image, tag string) string {
	fullImage := repo
	if fullImage != "" && !strings.HasSuffix(fullImage, "/") {
		fullImage += "/"
	}
	fullImage += image

	if tag != "" {
		fullImage += ":" + tag
	}
	return fullImage
}

// Pull makes a command line call to docker to pull the specified container.
// docker pull repository/name:tag.
// It returns stdout, stderr, and any errors from running.
// This is a blocking call, and should be wrapped in a go routine for asynchonous use.
func Pull(image string) CommandResult {
	cmd := exec.Command("docker", "pull", image)
	stdout := bytes.NewBuffer(nil)
	stderr := bytes.NewBuffer(nil)
	cmd.Stdout = stdout
	cmd.Stderr = stderr
	err := cmd.Run()
	return CommandResult{stdout.String(), stderr.String(), err}
}

func setupArgs(container string, portMap map[int]int, volumeMap map[string]string, linkContainers []string, environment map[string]string) []string {
	var linkArgs []string
	for _, container := range linkContainers {
		linkArgs = append(linkArgs, fmt.Sprintf("--link=%s:%s", container, container))
	}

	var environmentVars []string
	for ev, val := range environment {
		environmentVars = append(environmentVars, "-e="+strconv.Quote(ev+"="+val))
	}

	var volumeList []string
	for hostVolume, containerVolume := range volumeMap {
		volumeList = append(volumeList, fmt.Sprintf("-v=%s:%s", hostVolume, containerVolume))
	}

	var exposePorts []string
	for hostPort, containerPort := range portMap {
		exposePorts = append(exposePorts, fmt.Sprintf("-p=127.0.0.1:%d:%d", hostPort, containerPort))
	}

	args := exposePorts
	args = append(args, linkArgs...)
	args = append(args, volumeList...)
	args = append(args, environmentVars...)
	args = append(args, fmt.Sprintf("--name=%s", container))
	return args
}

// RunAnalyzer runs the analyzer image with container analyzerContainer. It runs it at port (mapped
// to internal port 10005), binds the volumes for the workspacePath and logsPath, and gives the privileged
// if dind (docker-in-docker) is true.
func RunAnalyzer(image, analyzerContainer, workspacePath, logsPath string, port int, dind bool) CommandResult {
	stdout := bytes.NewBuffer(nil)
	stderr := bytes.NewBuffer(nil)
	if len(analyzerContainer) == 0 {
		return CommandResult{"", "", errors.New("need to provide a name for the container")}
	}

	volumeMap := map[string]string{
		workspacePath: shipshapeWork,
		logsPath:      shipshapeLogs,
	}
	args := []string{"run"}
	if dind {
		args = append(args, "--privileged")
	}
	args = append(args, setupArgs(analyzerContainer, map[int]int{port: 10005}, volumeMap, nil, nil)...)
	args = append(args, "-d", image)

	glog.Infof("Running 'docker %v'\n", args)

	cmd := exec.Command("docker", args...)
	cmd.Stdout = stdout
	cmd.Stderr = stderr
	err := cmd.Run()
	return CommandResult{stdout.String(), stderr.String(), err}
}

// RunService runs the shipshape service at image, as the container named container. It binds the
// shipshape workspace and logs appropriately. It starts with the third-party analyzers already
// running at analyzerContainers. The service is started with the privileged flag if dind (docker-in-docker)
// is true.
func RunService(image, container, workspacePath, logsPath string, analyzerContainers []string, dind bool) CommandResult {
	stdout := bytes.NewBuffer(nil)
	stderr := bytes.NewBuffer(nil)
	if len(container) == 0 {
		return CommandResult{"", "", errors.New("need to provide a name for the container")}
	}

	volumeMap := map[string]string{workspacePath: shipshapeWork, logsPath: shipshapeLogs}

	var locations []string
	for _, container := range analyzerContainers {
		locations = append(locations, fmt.Sprintf(`$%s_PORT_10005_TCP_ADDR:$%s_PORT_10005_TCP_PORT`, strings.ToUpper(container), strings.ToUpper(container)))
	}
	locations = append(locations, "localhost:10005", "localhost:10006", "localhost:10008")

	args := []string{"run"}
	if dind {
		args = append(args, "--privileged")
	}
	args = append(args, setupArgs(container, map[int]int{10007: 10007}, volumeMap, analyzerContainers, map[string]string{"START_SERVICE": "true", "ANALYZERS": strings.Join(locations, ",")})...)
	args = append(args, "-d", image)

	glog.Infof("Running 'docker %v'\n", args)

	cmd := exec.Command("docker", args...)
	cmd.Stdout = stdout
	cmd.Stderr = stderr
	err := cmd.Run()
	return CommandResult{stdout.String(), stderr.String(), err}
}

// RunKythe runs the specified kythe docker image at the named container. It uses the
// source root and extractor specified, and gives the privileged flag if dind (docker-in-docker) is true.
// It returns stdout, stderr, and any errors from running.
// This is a blocking call, and should be wrapped in a go routine for asynchonous use.
func RunKythe(image, container, sourcePath, extractor string, dind bool) CommandResult {
	stdout := bytes.NewBuffer(nil)
	stderr := bytes.NewBuffer(nil)
	if len(container) == 0 {
		return CommandResult{"", "", errors.New("need to provide a name for the container")}
	}

	volumeMap := map[string]string{
		filepath.Join(sourcePath, "compilations"): "/compilations",
		sourcePath: "/repo",
	}
	home := os.Getenv("HOME")
	if len(home) > 0 {
		volumeMap[filepath.Join(home, ".m2")] = "/root/.m2"
	}

	// TODO(ciera): Can we exclude files in the .shipshape ignore path?
	// TODO(ciera/emso): Can we use the same command for blaze extraction?
	args := []string{"run"}
	if dind {
		args = append(args, "--privileged")
	}
	args = append(args, setupArgs(container, nil, volumeMap, nil, nil)...)
	args = append(args, "-i", "-a", "stdin", "-a", "stderr", "-a", "stdout", image)
	args = append(args, "--extract", extractor)

	cmd := exec.Command("docker", args...)
	cmd.Stdout = stdout
	cmd.Stderr = stderr
	err := cmd.Run()
	return CommandResult{stdout.String(), stderr.String(), err}
}

// Stop stops a running container.
// It returns stdout, stderr, and any errors from running.
// This is a blocking call, and should be wrapped in a go routine for asynchonous use.
// If requested, also remove the container.
func Stop(container string, waitTime time.Duration, remove bool) CommandResult {
	stdout := bytes.NewBuffer(nil)
	stderr := bytes.NewBuffer(nil)
	if container == "" {
		return CommandResult{"", "", errors.New("need to provide a name for the container")}
	}

	cmd := exec.Command("docker", "stop", fmt.Sprintf("-t=%d", int(waitTime.Seconds())), container)
	cmd.Stdout = stdout
	cmd.Stderr = stderr
	err := cmd.Run()

	if err == nil && remove {
		cmd := exec.Command("docker", "rm", container)
		cmd.Stdout = stdout
		cmd.Stderr = stderr
		err = cmd.Run()
	}

	return CommandResult{stdout.String(), stderr.String(), err}
}

// OutOfDate returns true if the image specified
// has not been pulled recently.
func OutOfDate(image string) bool {
	// TODO(ciera): Rather than always return true,
	// check a file that contains the last time this
	// image was updated, and only return true if it
	// is at least N days old.
	return true
}

// ImageMatches returns whether the container is running
// the current version of image.
func ImageMatches(image, container string) bool {
	imageHash, err := inspect(image, "{{.Id}}")
	if err != nil {
		return false
	}
	containerHash, err := inspect(container, "{{.Image}}")
	if err != nil {
		return false
	}
	return bytes.Equal(imageHash, containerHash)
}

// ContainerId returns the id of the requested container
func ContainerId(container string) (string, error) {
	res, err := inspect(container, "{{.Id}}")
	return string(res), err
}

// MappedVolume returns whether path is already mapped into the workspace
// of the shipshape service running at container. If it is, it returns the relative path
// of path within the mapped volume.
func MappedVolume(path, container string) (bool, string) {
	// Sigh.
	// In docker < 1.8.0, docker inspect has a Volumes entry that is a map of strings to strings. We can't
	// directly index into the map though, because go requires indicies to have alphanumberics only,
	// and ours have punctuation since they are a path name.
	// In docker 1.8+, docker CHANGED THE FORMAT. docker inspect now has a Mounts that is an array of
	// objects. We have to go through the array and look for the item of interest.
	// into the array that contains the Destination of interest.

	// mappedVolumeFormat is a go format string that will retrieve the path that is mapped to
	// /shipshape-workspace
	mappedVolumeFormat := `{{range $k, $v := .Mounts}} {{if eq $v.Destination "/shipshape-workspace"}} {{$v.Source}} {{end}} {{end}}`
	versionArray, err := version()
	// Oh, yeah. In docker <1.7, getting the version doesn't even work because it doesn't have a
	// --format flag. So let's just assume that if we get an error here, it's old.
	if err != nil || (versionArray[0] == 1 && versionArray[1] < 8) {
		mappedVolumeFormat = `{{range $k, $v := .Volumes}} {{if eq $k "/shipshape-workspace"}} {{$v}} {{end}} {{end}}`
	}

	v, err := inspect(container, mappedVolumeFormat)
	if err != nil {
		return false, ""
	}
	volume := strings.TrimSpace(string(v))
	// Handle the equal case
	if path == volume {
		return true, ""
	}
	// Handle the subdirectory case by adding a trailing '/'
	// Want to rule out the case: volume='/a/b2' and path='/a/b'
	path += "/"
	// We want to return true if the path we need is a subpath
	// of the directory we have mounted. That is, the start of path
	// is our volume.
	return strings.HasPrefix(path, volume), strings.TrimPrefix(path, volume)
}

// ContainsLinks returns whether the given container has links to the given
// list of containers.
func ContainsLinks(container string, linkedContainers []string) bool {
	l, err := inspect(container, `{{.HostConfig.Links}}`)
	if err != nil {
		return false
	}
	links := strings.TrimSpace(string(l))
	for _, linkedContainer := range linkedContainers {
		if !strings.Contains(links, linkedContainer) {
			return false
		}
	}
	return true
}

// inspect runs docker inspect on name, which must be either an image or a container.
// If non-empty, it uses the specified format string.
// Returns the combined stdout/stderr from running docker inspect
func inspect(name, format string) ([]byte, error) {
	var formatter string
	if len(format) != 0 {
		formatter = fmt.Sprintf("--format='%s'", format)
	}
	cmd := exec.Command("docker", "inspect", formatter, name)
	return cmd.CombinedOutput()
}

// version returns the docker version, as a []int. So docker version 1.8.3
// will be [1, 8, 3]
func version() ([]int, error) {
	cmd := exec.Command("docker", "version", "--format='{{.Server.Version}}'")
	raw, err := cmd.CombinedOutput()
	if err != nil {
		return nil, err
	}
	strParts := strings.Split(strings.TrimSpace(string(raw)), ".")
	parts := make([]int, len(strParts))
	for i, p := range strParts {
		parts[i], err = strconv.Atoi(p)
		if err != nil {
			return nil, err
		}
	}
	return parts, nil
}
<commit_msg>fixed typo<commit_after>/*
 * Copyright 2014 Google Inc. All rights reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

// Package docker contains simple utilities for pulling a docker image, starting
// a container, and stoping a container. It assumes that docker is installed. If
// it is not, it will simply throw an error.
package docker

import (
	"bytes"
	"errors"
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	glog "github.com/google/shipshape/third_party/go-glog"
)

const (
	shipshapeWork = "/shipshape-workspace"
	shipshapeLogs = "/shipshape-output"
)

// TODO(ciera): Consider making these all use channels.
type CommandResult struct {
	Stdout string
	Stderr string
	Err    error
}

func trimResult(stdout, stderr *bytes.Buffer, err error) CommandResult {
	return CommandResult{strings.TrimSpace(stdout.String()), strings.TrimSpace(stderr.String()), err}
}

// ContainerExists checks if a container with the given name is in the list of running, or old, containers.
func ContainerExists(container string) (bool, error) {
	// Setup and run command
	// TODO(ciera): When Travis and other places we run this at support docker
	// 1.8, we can drastically reduce this code by using the flag --format={{.Names}}
	// below and removing the logic to parse out the name ourselves.
	stdout := bytes.NewBuffer(nil)
	cmd := exec.Command("docker", "ps", "-a")
	cmd.Stdout = stdout
	if err := cmd.Run(); err != nil {
		fmt.Printf("Problem running command, err: %v", err)
		return false, err
	}
	// Process output in search for container name match.
	// The 'docker ps' output lists the container names in the last column.
	// Prefixing the container name with a space to avoid substring matching.
	spacedName := fmt.Sprintf(" %s", container)
	for itr, line := range strings.Split(stdout.String(), "\n") {
		// Skip the title row
		if itr == 0 {
			continue
		}
		// Docker adds spaces to the end of each row
		trimmedLine := strings.Trim(line, " ")
		if strings.HasSuffix(trimmedLine, spacedName) {
			return true, nil
		}
	}
	return false, nil
}

// HasDocker determines whether docker is installed
// and included in PATH.
func HasDocker() bool {
	cmd := exec.Command("which", "docker")
	return cmd.Run() == nil
}

// FullImageName creates a full image name from a repository URI, an image name, and a tag.
func FullImageName(repo, image, tag string) string {
	fullImage := repo
	if fullImage != "" && !strings.HasSuffix(fullImage, "/") {
		fullImage += "/"
	}
	fullImage += image

	if tag != "" {
		fullImage += ":" + tag
	}
	return fullImage
}

// Pull makes a command line call to docker to pull the specified container.
// docker pull repository/name:tag.
// It returns stdout, stderr, and any errors from running.
// This is a blocking call, and should be wrapped in a go routine for asynchonous use.
func Pull(image string) CommandResult {
	cmd := exec.Command("docker", "pull", image)
	stdout := bytes.NewBuffer(nil)
	stderr := bytes.NewBuffer(nil)
	cmd.Stdout = stdout
	cmd.Stderr = stderr
	err := cmd.Run()
	return CommandResult{stdout.String(), stderr.String(), err}
}

func setupArgs(container string, portMap map[int]int, volumeMap map[string]string, linkContainers []string, environment map[string]string) []string {
	var linkArgs []string
	for _, container := range linkContainers {
		linkArgs = append(linkArgs, fmt.Sprintf("--link=%s:%s", container, container))
	}

	var environmentVars []string
	for ev, val := range environment {
		environmentVars = append(environmentVars, "-e="+strconv.Quote(ev+"="+val))
	}

	var volumeList []string
	for hostVolume, containerVolume := range volumeMap {
		volumeList = append(volumeList, fmt.Sprintf("-v=%s:%s", hostVolume, containerVolume))
	}

	var exposePorts []string
	for hostPort, containerPort := range portMap {
		exposePorts = append(exposePorts, fmt.Sprintf("-p=127.0.0.1:%d:%d", hostPort, containerPort))
	}

	args := exposePorts
	args = append(args, linkArgs...)
	args = append(args, volumeList...)
	args = append(args, environmentVars...)
	args = append(args, fmt.Sprintf("--name=%s", container))
	return args
}

// RunAnalyzer runs the analyzer image with container analyzerContainer. It runs it at port (mapped
// to internal port 10005), binds the volumes for the workspacePath and logsPath, and gives the privileged
// if dind (docker-in-docker) is true.
func RunAnalyzer(image, analyzerContainer, workspacePath, logsPath string, port int, dind bool) CommandResult {
	stdout := bytes.NewBuffer(nil)
	stderr := bytes.NewBuffer(nil)
	if len(analyzerContainer) == 0 {
		return CommandResult{"", "", errors.New("need to provide a name for the container")}
	}

	volumeMap := map[string]string{
		workspacePath: shipshapeWork,
		logsPath:      shipshapeLogs,
	}
	args := []string{"run"}
	if dind {
		args = append(args, "--privileged")
	}
	args = append(args, setupArgs(analyzerContainer, map[int]int{port: 10005}, volumeMap, nil, nil)...)
	args = append(args, "-d", image)

	glog.Infof("Running 'docker %v'\n", args)

	cmd := exec.Command("docker", args...)
	cmd.Stdout = stdout
	cmd.Stderr = stderr
	err := cmd.Run()
	return CommandResult{stdout.String(), stderr.String(), err}
}

// RunService runs the shipshape service at image, as the container named container. It binds the
// shipshape workspace and logs appropriately. It starts with the third-party analyzers already
// running at analyzerContainers. The service is started with the privileged flag if dind (docker-in-docker)
// is true.
func RunService(image, container, workspacePath, logsPath string, analyzerContainers []string, dind bool) CommandResult {
	stdout := bytes.NewBuffer(nil)
	stderr := bytes.NewBuffer(nil)
	if len(container) == 0 {
		return CommandResult{"", "", errors.New("need to provide a name for the container")}
	}

	volumeMap := map[string]string{workspacePath: shipshapeWork, logsPath: shipshapeLogs}

	var locations []string
	for _, container := range analyzerContainers {
		locations = append(locations, fmt.Sprintf(`$%s_PORT_10005_TCP_ADDR:$%s_PORT_10005_TCP_PORT`, strings.ToUpper(container), strings.ToUpper(container)))
	}
	locations = append(locations, "localhost:10005", "localhost:10006", "localhost:10008")

	args := []string{"run"}
	if dind {
		args = append(args, "--privileged")
	}
	args = append(args, setupArgs(container, map[int]int{10007: 10007}, volumeMap, analyzerContainers, map[string]string{"START_SERVICE": "true", "ANALYZERS": strings.Join(locations, ",")})...)
	args = append(args, "-d", image)

	glog.Infof("Running 'docker %v'\n", args)

	cmd := exec.Command("docker", args...)
	cmd.Stdout = stdout
	cmd.Stderr = stderr
	err := cmd.Run()
	return CommandResult{stdout.String(), stderr.String(), err}
}

// RunKythe runs the specified kythe docker image at the named container. It uses the
// source root and extractor specified, and gives the privileged flag if dind (docker-in-docker) is true.
// It returns stdout, stderr, and any errors from running.
// This is a blocking call, and should be wrapped in a go routine for asynchonous use.
func RunKythe(image, container, sourcePath, extractor string, dind bool) CommandResult {
	stdout := bytes.NewBuffer(nil)
	stderr := bytes.NewBuffer(nil)
	if len(container) == 0 {
		return CommandResult{"", "", errors.New("need to provide a name for the container")}
	}

	volumeMap := map[string]string{
		filepath.Join(sourcePath, "compilations"): "/compilations",
		sourcePath: "/repo",
	}
	home := os.Getenv("HOME")
	if len(home) > 0 {
		volumeMap[filepath.Join(home, ".m2")] = "/root/.m2"
	}

	// TODO(ciera): Can we exclude files in the .shipshape ignore path?
	// TODO(ciera/emso): Can we use the same command for blaze extraction?
	args := []string{"run"}
	if dind {
		args = append(args, "--privileged")
	}
	args = append(args, setupArgs(container, nil, volumeMap, nil, nil)...)
	args = append(args, "-i", "-a", "stdin", "-a", "stderr", "-a", "stdout", image)
	args = append(args, "--extract", extractor)

	cmd := exec.Command("docker", args...)
	cmd.Stdout = stdout
	cmd.Stderr = stderr
	err := cmd.Run()
	return CommandResult{stdout.String(), stderr.String(), err}
}

// Stop stops a running container.
// It returns stdout, stderr, and any errors from running.
// This is a blocking call, and should be wrapped in a go routine for asynchonous use.
// If requested, also remove the container.
func Stop(container string, waitTime time.Duration, remove bool) CommandResult {
	stdout := bytes.NewBuffer(nil)
	stderr := bytes.NewBuffer(nil)
	if container == "" {
		return CommandResult{"", "", errors.New("need to provide a name for the container")}
	}

	cmd := exec.Command("docker", "stop", fmt.Sprintf("-t=%d", int(waitTime.Seconds())), container)
	cmd.Stdout = stdout
	cmd.Stderr = stderr
	err := cmd.Run()

	if err == nil && remove {
		cmd := exec.Command("docker", "rm", container)
		cmd.Stdout = stdout
		cmd.Stderr = stderr
		err = cmd.Run()
	}

	return CommandResult{stdout.String(), stderr.String(), err}
}

// OutOfDate returns true if the image specified
// has not been pulled recently.
func OutOfDate(image string) bool {
	// TODO(ciera): Rather than always return true,
	// check a file that contains the last time this
	// image was updated, and only return true if it
	// is at least N days old.
	return true
}

// ImageMatches returns whether the container is running
// the current version of image.
func ImageMatches(image, container string) bool {
	imageHash, err := inspect(image, "{{.Id}}")
	if err != nil {
		return false
	}
	containerHash, err := inspect(container, "{{.Image}}")
	if err != nil {
		return false
	}
	return bytes.Equal(imageHash, containerHash)
}

// ContainerId returns the id of the requested container
func ContainerId(container string) (string, error) {
	res, err := inspect(container, "{{.Id}}")
	return string(res), err
}

// MappedVolume returns whether path is already mapped into the workspace
// of the shipshape service running at container. If it is, it returns the relative path
// of path within the mapped volume.
func MappedVolume(path, container string) (bool, string) {
	// Sigh.
	// In docker < 1.8.0, docker inspect has a Volumes entry that is a map of strings to strings. We can't
	// directly index into the map though, because go requires indicies to have alphanumerics only,
	// and ours have punctuation since they are a path name.
	// In docker 1.8+, docker CHANGED THE FORMAT. docker inspect now has a Mounts that is an array of
	// objects. We have to go through the array and look for the item of interest.
	// into the array that contains the Destination of interest.

	// mappedVolumeFormat is a go format string that will retrieve the path that is mapped to
	// /shipshape-workspace
	mappedVolumeFormat := `{{range $k, $v := .Mounts}} {{if eq $v.Destination "/shipshape-workspace"}} {{$v.Source}} {{end}} {{end}}`
	versionArray, err := version()
	// Oh, yeah. In docker <1.7, getting the version doesn't even work because it doesn't have a
	// --format flag. So let's just assume that if we get an error here, it's old.
	if err != nil || (versionArray[0] == 1 && versionArray[1] < 8) {
		mappedVolumeFormat = `{{range $k, $v := .Volumes}} {{if eq $k "/shipshape-workspace"}} {{$v}} {{end}} {{end}}`
	}

	v, err := inspect(container, mappedVolumeFormat)
	if err != nil {
		return false, ""
	}
	volume := strings.TrimSpace(string(v))
	// Handle the equal case
	if path == volume {
		return true, ""
	}
	// Handle the subdirectory case by adding a trailing '/'
	// Want to rule out the case: volume='/a/b2' and path='/a/b'
	path += "/"
	// We want to return true if the path we need is a subpath
	// of the directory we have mounted. That is, the start of path
	// is our volume.
	return strings.HasPrefix(path, volume), strings.TrimPrefix(path, volume)
}

// ContainsLinks returns whether the given container has links to the given
// list of containers.
func ContainsLinks(container string, linkedContainers []string) bool {
	l, err := inspect(container, `{{.HostConfig.Links}}`)
	if err != nil {
		return false
	}
	links := strings.TrimSpace(string(l))
	for _, linkedContainer := range linkedContainers {
		if !strings.Contains(links, linkedContainer) {
			return false
		}
	}
	return true
}

// inspect runs docker inspect on name, which must be either an image or a container.
// If non-empty, it uses the specified format string.
// Returns the combined stdout/stderr from running docker inspect
func inspect(name, format string) ([]byte, error) {
	var formatter string
	if len(format) != 0 {
		formatter = fmt.Sprintf("--format='%s'", format)
	}
	cmd := exec.Command("docker", "inspect", formatter, name)
	return cmd.CombinedOutput()
}

// version returns the docker version, as a []int. So docker version 1.8.3
// will be [1, 8, 3]
func version() ([]int, error) {
	cmd := exec.Command("docker", "version", "--format='{{.Server.Version}}'")
	raw, err := cmd.CombinedOutput()
	if err != nil {
		return nil, err
	}
	strParts := strings.Split(strings.TrimSpace(string(raw)), ".")
	parts := make([]int, len(strParts))
	for i, p := range strParts {
		parts[i], err = strconv.Atoi(p)
		if err != nil {
			return nil, err
		}
	}
	return parts, nil
}
<|endoftext|>


<commit_before>// Package etree provides XML services through an Element Tree abstraction.
package etree

import (
    "bufio"
    "container/list"
    "fmt"
    "io"
)

const sp string = "\n                                                            "

// A Token is an empty interface that represents an Element,
// Comment, CharData or ProcInst.
type Token interface {
    writeTo(w *bufio.Writer)
}

// An Element represents an XML element.  The Children list contains
// Tokens.
type Element struct {
    Name     string
    Attr     []Attr
    Children *list.List
}

// A Comment represents an XML comment
type Comment []byte

// CharData represents character data within XML.
type CharData []byte

// An Attr represents a key-value attribute of an XML element.
type Attr struct {
    Key   string
    Value string
}

// NewElement creates a root-level XML element with the specified name.
func NewElement(name string) *Element {
    return &Element{
        Name:     name,
        Attr:     make([]Attr, 0),
        Children: list.New(),       // list of Tokens
    }
}

// CreateElement creates a child element of the the receiving element and
// gives it the specified name.
func (e *Element) CreateElement(name string) *Element {
    c := NewElement(name)
    e.addChild(c)
    return c
}

// WriteTo serializes the element and its children as XML into
// the writer w.
func (e *Element) WriteTo(w io.Writer) error {
    b := bufio.NewWriter(w)
    e.writeTo(b)
    return b.Flush()
}

// Indent modifies the element tree by inserting CharData entities
// that introduce indentation.  The amount of indenting per depth
// level is equal to spaces.
func (e *Element) Indent(spaces int) {
    e.indent(1, spaces)
}

// indent recursively inserts proper indentation into between an
// XML element's child tokens.
func (e *Element) indent(depth, spaces int) {
    for c := e.Children.Front(); c != nil; {
        n := c.Next()
        e.Children.InsertBefore(indentCharData(depth, spaces), c)
        if ce, ok := c.Value.(*Element); ok {
            ce.indent(depth + 1, spaces)
        }
        c = n
    }
    if b := e.Children.Back(); b != nil {
        e.Children.InsertAfter(indentCharData(depth - 1, spaces), b)
    }
}

// addChild adds a child token to the receiving element.
func (e *Element) addChild(t Token) {
    e.Children.PushBack(t)
}

// writeTo serializes the element to the writer w.
func (e *Element) writeTo(w *bufio.Writer) {
    w.WriteString(fmt.Sprintf("<%s", e.Name))
    for _, a := range e.Attr {
        w.WriteByte(' ')
        a.writeTo(w)
    }
    if e.Children.Len() > 0 {
        w.WriteString(">")
        for c := e.Children.Front(); c != nil; c = c.Next() {
            c.Value.(Token).writeTo(w)
        }
        w.WriteString(fmt.Sprintf("</%s>", e.Name))
    } else {
        w.WriteString("/>")
    }
}

// AddAttr adds an attribute to the element.
func (e *Element) AddAttr(a Attr) {
    e.Attr = append(e.Attr, a)
}

// CreateAttr creates an attribute and adds it to the receiving element.
func (e *Element) CreateAttr(key, value string) Attr {
    a := Attr{key, value}
    e.Attr = append(e.Attr, a)
    return a
}

// writeTo serializes the attribute to the writer.
func (a *Attr) writeTo(w *bufio.Writer) {
    w.WriteString(fmt.Sprintf(`%s="%s"`, a.Key, escape(a.Value)))
}

// newCharData creates an XML character data entity.
func newCharData(charData string) *CharData {
    c := new(CharData)
    *c = CharData(charData)
    return c
}

// CreateCharData creates an XML character data entity and adds it
// as a child of the receiving element.
func (e *Element) CreateCharData(charData string) *CharData {
    c := newCharData(charData)
    e.addChild(c)
    return c
}

// writeTo serializes the character data entity to the writer.
func (c *CharData) writeTo(w *bufio.Writer) {
    w.WriteString(string(*c))
}

// NewComment creates an XML comment.
func newComment(comment string) *Comment {
    c := new(Comment)
    *c = Comment(comment)
    return c
}

// CreateComment creates an XML comment and adds it as a child of the
// receiving element.
func (e *Element) CreateComment(comment string) *Comment {
    c := newComment(comment)
    e.addChild(c)
    return c
}

// writeTo serialies the comment to the writer.
func (c *Comment) writeTo(w *bufio.Writer) {
    w.WriteString(fmt.Sprintf("<!-- %s -->", string(*c)))
}

// indentCharData returns the indentation CharData token for the given
// depth level with the given number of spaces per level.
func indentCharData(depth, spaces int) *CharData {
    c := 1 + depth * spaces
    if c > len(sp) {
        return newCharData(sp)
    } else {
        return newCharData(sp[:c])
    }
}

// escape generates an escaped XML string.
func escape(s string) string {
    buf := make([]byte, 0, len(s))
    for i := 0; i < len(s); i++ {
        switch {
        case s[i] == '&':
            buf = append(buf, []byte{'&', 'a', 'm', 'p', ';'}...)
        case s[i] == '\'':
            buf = append(buf, []byte{'&', 'a', 'p', 'o', 's', ';'}...)
        case s[i] == '<':
            buf = append(buf, []byte{'&', 'l', 't', ';'}...)
        case s[i] == '>':
            buf = append(buf, []byte{'&', 'g', 't', ';'}...)
        default:
            buf = append(buf, s[i])
        }
    }
    return string(buf)
}
<commit_msg>optimizations<commit_after>// Package etree provides XML services through an Element Tree abstraction.
package etree

import (
    "bufio"
    "container/list"
    "io"
)

const sp string = "\n                                                            "

// A Token is an empty interface that represents an Element,
// Comment, CharData or ProcInst.
type Token interface {
    writeTo(w *bufio.Writer)
}

// An Element represents an XML element.  The Children list contains
// Tokens.
type Element struct {
    Name     []byte
    Attr     []Attr
    Children *list.List
}

// A Comment represents an XML comment
type Comment []byte

// CharData represents character data within XML.
type CharData []byte

// An Attr represents a key-value attribute of an XML element.
type Attr struct {
    Key   []byte
    Value []byte
}

// NewElement creates a root-level XML element with the specified name.
func NewElement(name string) *Element {
    return &Element{
        Name:     []byte(name),
        Attr:     make([]Attr, 0),
        Children: list.New(), // list of Tokens
    }
}

// CreateElement creates a child element of the receiving element and
// gives it the specified name.
func (e *Element) CreateElement(name string) *Element {
    c := NewElement(name)
    e.addChild(c)
    return c
}

// WriteTo serializes the element and its children as XML into
// the writer w.
func (e *Element) WriteTo(w io.Writer) error {
    b := bufio.NewWriter(w)
    e.writeTo(b)
    return b.Flush()
}

// Indent modifies the element tree by inserting CharData entities
// that introduce indentation.  The amount of indenting per depth
// level is equal to spaces.
func (e *Element) Indent(spaces int) {
    e.indent(1, spaces)
}

// indent recursively inserts proper indentation between an
// XML element's child tokens.
func (e *Element) indent(depth, spaces int) {
    for c := e.Children.Front(); c != nil; {
        n := c.Next()
        e.Children.InsertBefore(indentCharData(depth, spaces), c)
        if ce, ok := c.Value.(*Element); ok {
            ce.indent(depth+1, spaces)
        }
        c = n
    }
    if b := e.Children.Back(); b != nil {
        e.Children.InsertAfter(indentCharData(depth-1, spaces), b)
    }
}

// addChild adds a child token to the receiving element.
func (e *Element) addChild(t Token) {
    e.Children.PushBack(t)
}

// writeTo serializes the element to the writer w.
func (e *Element) writeTo(w *bufio.Writer) {
    w.WriteByte('<')
    w.Write(e.Name)
    for _, a := range e.Attr {
        w.WriteByte(' ')
        a.writeTo(w)
    }
    if e.Children.Len() > 0 {
        w.WriteString(">")
        for c := e.Children.Front(); c != nil; c = c.Next() {
            c.Value.(Token).writeTo(w)
        }
        w.Write([]byte{'<', '/'})
        w.Write(e.Name)
        w.WriteByte('>')
    } else {
        w.Write([]byte{'/', '>'})
    }
}

// CreateAttr creates an attribute and adds it to the receiving element.
func (e *Element) CreateAttr(key, value string) Attr {
    a := Attr{[]byte(key), []byte(value)}
    e.Attr = append(e.Attr, a)
    return a
}

// writeTo serializes the attribute to the writer.
func (a *Attr) writeTo(w *bufio.Writer) {
    w.Write(a.Key)
    w.Write([]byte{'=', '"'})
    w.Write(a.Value)
    w.WriteByte('"')
}

// newCharData creates an XML character data entity.
func newCharData(charData string) *CharData {
    c := new(CharData)
    *c = CharData(charData)
    return c
}

// CreateCharData creates an XML character data entity and adds it
// as a child of the receiving element.
func (e *Element) CreateCharData(charData string) *CharData {
    c := newCharData(charData)
    e.addChild(c)
    return c
}

// writeTo serializes the character data entity to the writer.
func (c *CharData) writeTo(w *bufio.Writer) {
    w.Write(escape(*c))
}

// NewComment creates an XML comment.
func newComment(comment string) *Comment {
    c := new(Comment)
    *c = Comment(comment)
    return c
}

// CreateComment creates an XML comment and adds it as a child of the
// receiving element.
func (e *Element) CreateComment(comment string) *Comment {
    c := newComment(comment)
    e.addChild(c)
    return c
}

// writeTo serialies the comment to the writer.
func (c *Comment) writeTo(w *bufio.Writer) {
    w.Write([]byte{'<', '!', '-', '-', ' '})
    w.Write(*c)
    w.Write([]byte{' ', '-', '-', '>'})
}

// indentCharData returns the indentation CharData token for the given
// depth level with the given number of spaces per level.
func indentCharData(depth, spaces int) *CharData {
    c := 1 + depth*spaces
    if c > len(sp) {
        return newCharData(sp)
    } else {
        return newCharData(sp[:c])
    }
}

var escapeTable = [...]byte{
    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
    0, 0, 1, 0, 0, 0, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0,
    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 5, 0,
    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
}

var substTable = [...][]byte{
    {'&', 'q', 'u', 'o', 't', ';'}, // 1
    {'&', 'a', 'm', 'p', ';'},      // 2
    {'&', 'a', 'p', 'o', 's', ';'}, // 3
    {'&', 'l', 't', ';'},           // 4
    {'&', 'g', 't', ';'},           // 5
}

// escape generates an escaped XML string.
func escape(b []byte) []byte {
    buf := make([]byte, 0, len(b))
    for _, c := range b {
        subst := escapeTable[c]
        if subst > 0 {
            buf = append(buf, substTable[subst-1]...)
        } else {
            buf = append(buf, c)
        }
    }
    return buf
}
<|endoftext|>


<commit_before>// Copyright 2016 Factom Foundation
// Use of this source code is governed by the MIT
// license that can be found in the LICENSE file.

package wallet

import (
	"encoding/hex"
	"fmt"

	"github.com/FactomProject/factom"
	"github.com/FactomProject/factomd/common/directoryBlock"
	"github.com/FactomProject/factomd/common/factoid"
	"github.com/FactomProject/factomd/common/interfaces"
	"github.com/FactomProject/factomd/common/primitives"
	"github.com/FactomProject/factomd/database/databaseOverlay"
	"github.com/FactomProject/factomd/database/hybridDB"
	"github.com/FactomProject/factomd/database/mapdb"
	"os"
)

// Database keys and key prefixes
var (
	fblockDBPrefix = []byte("FBlock")
)

type TXDatabaseOverlay struct {
	DBO databaseOverlay.Overlay
}

func NewTXOverlay(db interfaces.IDatabase) *TXDatabaseOverlay {
	answer := new(TXDatabaseOverlay)
	answer.DBO.DB = db
	return answer
}

func NewTXMapDB() *TXDatabaseOverlay {
	return NewTXOverlay(new(mapdb.MapDB))
}

func NewTXLevelDB(ldbpath string) (*TXDatabaseOverlay, error) {
	db, err := hybridDB.NewLevelMapHybridDB(ldbpath, false)
	if err != nil {
		fmt.Printf("err opening transaction db: %v\n", err)
	}

	if db == nil {
		fmt.Println("Creating new transaction db ...")
		db, err = hybridDB.NewLevelMapHybridDB(ldbpath, true)

		if err != nil {
			return nil, err
		}
	}
	fmt.Println("Transaction database started from: " + ldbpath)
	return NewTXOverlay(db), nil
}

func NewTXBoltDB(boltPath string) (*TXDatabaseOverlay, error) {
	fileInfo, err := os.Stat(boltPath)
	if err == nil { //if it exists
		if fileInfo.IsDir() { //if it is a folder though
			return nil, fmt.Errorf("The path %s is a directory.  Please specify a file name.", boltPath)
		}
	}
	if err != nil && !os.IsNotExist(err) { //some other error, besides the file not existing
		fmt.Printf("database error %s\n", err)
		return nil, err
	}

	defer func() {
		if r := recover(); r != nil {
			fmt.Printf("Could not use wallet cache database file \"%s\"\n%v\n", boltPath, r)
			os.Exit(1)
		}
	}()
	db := hybridDB.NewBoltMapHybridDB(nil, boltPath)

	fmt.Println("Database started from: " + boltPath)
	return NewTXOverlay(db), nil
}

func (db *TXDatabaseOverlay) Close() error {
	return db.DBO.Close()
}

// GetAllTXs returns a list of all transactions in the history of Factom. A
// local database is used to cache the factoid blocks.
func (db *TXDatabaseOverlay) GetAllTXs() ([]interfaces.ITransaction, error) {
	// update the database and get the newest fblock
	_, err := db.update()
	if err != nil {
		return nil, err
	}
	fblock, err := db.DBO.FetchFBlockHead()
	if err != nil {
		return nil, err
	}
	if fblock == nil {
		return nil, fmt.Errorf("FBlock Chain has not finished syncing")
	}
	txs := make([]interfaces.ITransaction, 0)

	for {
		// get all of the txs from the block
		height := fblock.GetDatabaseHeight()
		for _, tx := range fblock.GetTransactions() {
			ins, err := tx.TotalInputs()
			if err != nil {
				return nil, err
			}
			outs, err := tx.TotalOutputs()
			if err != nil {
				return nil, err
			}

			if ins != 0 || outs != 0 {
				tx.SetBlockHeight(height)
				txs = append(txs, tx)
			}
		}

		if pre := fblock.GetPrevKeyMR().String(); pre != factom.ZeroHash {
			// get the previous block
			fblock, err = db.GetFBlock(pre)
			if err != nil {
				return nil, err
			} else if fblock == nil {
				return nil, fmt.Errorf("Missing fblock in database: %s", pre)
			}
		} else {
			break
		}
	}

	return txs, nil
}

// GetTX gets a transaction by the transaction id
func (db *TXDatabaseOverlay) GetTX(txid string) (
	interfaces.ITransaction, error) {
	txs, err := db.GetAllTXs()
	if err != nil {
		return nil, err
	}

	for _, tx := range txs {
		if tx.GetSigHash().String() == txid {
			return tx, nil
		}
	}

	return nil, fmt.Errorf("Transaction not found")
}

// GetTXAddress returns a list of all transactions in the history of Factom that
// include a specific address.
func (db *TXDatabaseOverlay) GetTXAddress(adr string) (
	[]interfaces.ITransaction, error) {
	filtered := make([]interfaces.ITransaction, 0)

	txs, err := db.GetAllTXs()
	if err != nil {
		return nil, err
	}

	if factom.AddressStringType(adr) == factom.FactoidPub {
		for _, tx := range txs {
			for _, in := range tx.GetInputs() {
				if primitives.ConvertFctAddressToUserStr(in.GetAddress()) == adr {
					filtered = append(filtered, tx)
				}
			}
			for _, out := range tx.GetOutputs() {
				if primitives.ConvertFctAddressToUserStr(out.GetAddress()) == adr {
					filtered = append(filtered, tx)
				}
			}
		}
	} else if factom.AddressStringType(adr) == factom.ECPub {
		for _, tx := range txs {
			for _, out := range tx.GetECOutputs() {
				if primitives.ConvertECAddressToUserStr(out.GetAddress()) == adr {
					filtered = append(filtered, tx)
				}
			}
		}
	} else {
		return nil, fmt.Errorf("not a valid address")
	}

	return filtered, nil
}

func (db *TXDatabaseOverlay) GetTXRange(start, end int) (
	[]interfaces.ITransaction, error) {
	if start < 0 || end < 0 {
		return nil, fmt.Errorf("Range cannot have negative numbers")
	}
	s, e := uint32(start), uint32(end)

	filtered := make([]interfaces.ITransaction, 0)

	txs, err := db.GetAllTXs()
	if err != nil {
		return nil, err
	}

	for _, tx := range txs {
		if s <= tx.GetBlockHeight() && tx.GetBlockHeight() <= e {
			filtered = append(filtered, tx)
		}
	}

	return filtered, nil
}

// GetFBlock retrives a Factoid Block from Factom
func (db *TXDatabaseOverlay) GetFBlock(keymr string) (interfaces.IFBlock, error) {
	h, err := primitives.NewShaHashFromStr(keymr)
	if err != nil {
		return nil, err
	}

	fBlock, err := db.DBO.FetchFBlock(h)
	if err != nil {
		return nil, err
	}
	return fBlock, nil
}

func (db *TXDatabaseOverlay) FetchNextFBlockHeight() (uint32, error) {
	block, err := db.DBO.FetchFBlockHead()
	if err != nil {
		return 0, err
	}
	if block == nil {
		return 0, nil
	}
	return block.GetDBHeight() + 1, nil
}

func (db *TXDatabaseOverlay) InsertFBlockHead(fblock interfaces.IFBlock) error {
	return db.DBO.SaveFactoidBlockHead(fblock)
}

// update gets all fblocks written since the database was last updated, and
// returns the most recent fblock keymr.
func (db *TXDatabaseOverlay) update() (string, error) {
	newestFBlock, err := fblockHead()
	if err != nil {
		return "", err
	}

	start, err := db.FetchNextFBlockHeight()
	if err != nil {
		return "", err
	}

	//Making sure we didn't switch networks
	genesis, err := db.DBO.FetchFBlockByHeight(0)
	if err != nil {
		return "", err
	}
	if genesis != nil {
		genesis2, err := getdblockbyheight(0)
		var gensisFBlockKeyMr interfaces.IHash
		for _, e := range genesis2.GetDBEntries() {
			if e.GetChainID().String() == "000000000000000000000000000000000000000000000000000000000000000f" {
				gensisFBlockKeyMr = e.GetKeyMR()
				break
			}
		}
		if err != nil {
			return "", err
		}
		if gensisFBlockKeyMr != nil && !gensisFBlockKeyMr.IsSameAs(genesis.GetKeyMR()) {
			start = 0
		}
	}

	newestHeight := newestFBlock.GetDatabaseHeight()

	if start >= newestHeight {
		return newestFBlock.GetKeyMR().String(), nil
	}

	db.DBO.StartMultiBatch()
	for i := start; i <= newestHeight; i++ {
		if i%1000 == 0 {
			if newestHeight-start > 1000 {
				fmt.Printf("Fetching block %v / %v\n", i, newestHeight)
			}
		}
		fblock, err := getfblockbyheight(i)
		if err != nil {
			db.DBO.ExecuteMultiBatch()
			return "", err
		}
		db.DBO.ProcessFBlockMultiBatch(fblock)

		// Save to DB every 500 blocks
		if i%500 == 0 {
			db.DBO.ExecuteMultiBatch()
			db.DBO.StartMultiBatch()
		}
	}
	fmt.Printf("Fetching block %v / %v\n", newestHeight, newestHeight)
	err = db.DBO.ExecuteMultiBatch() // Save remaining blocks
	if err != nil {
		return "", err
	}

	return newestFBlock.GetKeyMR().String(), nil
}

// fblockHead gets the most recent fblock.
func fblockHead() (interfaces.IFBlock, error) {
	fblockID := "000000000000000000000000000000000000000000000000000000000000000f"

	dbhead, err := factom.GetDBlockHead()
	if err != nil {
		return nil, err
	}
	dblock, err := factom.GetDBlock(dbhead)
	if err != nil {
		return nil, err
	}

	var fblockmr string
	for _, eblock := range dblock.EntryBlockList {
		if eblock.ChainID == fblockID {
			fblockmr = eblock.KeyMR
		}
	}
	if fblockmr == "" {
		return nil, err
	}

	return getfblock(fblockmr)
}

func getfblock(keymr string) (interfaces.IFBlock, error) {
	p, err := factom.GetRaw(keymr)
	if err != nil {
		return nil, err
	}
	return factoid.UnmarshalFBlock(p)
}

func getfblockbyheight(height uint32) (interfaces.IFBlock, error) {
	p, err := factom.GetFBlockByHeight(int64(height))
	if err != nil {
		return nil, err
	}
	h, err := hex.DecodeString(p.RawData)
	if err != nil {
		return nil, err
	}
	return factoid.UnmarshalFBlock(h)
}

func getdblockbyheight(height uint32) (interfaces.IDirectoryBlock, error) {
	p, err := factom.GetDBlockByHeight(int64(height))
	if err != nil {
		return nil, err
	}
	h, err := hex.DecodeString(p.RawData)
	if err != nil {
		return nil, err
	}
	return directoryBlock.UnmarshalDBlock(h)
}
<commit_msg>Better error handling<commit_after>// Copyright 2016 Factom Foundation
// Use of this source code is governed by the MIT
// license that can be found in the LICENSE file.

package wallet

import (
	"encoding/hex"
	"fmt"

	"github.com/FactomProject/factom"
	"github.com/FactomProject/factomd/common/directoryBlock"
	"github.com/FactomProject/factomd/common/factoid"
	"github.com/FactomProject/factomd/common/interfaces"
	"github.com/FactomProject/factomd/common/primitives"
	"github.com/FactomProject/factomd/database/databaseOverlay"
	"github.com/FactomProject/factomd/database/hybridDB"
	"github.com/FactomProject/factomd/database/mapdb"
	"os"
)

// Database keys and key prefixes
var (
	fblockDBPrefix = []byte("FBlock")
)

type TXDatabaseOverlay struct {
	DBO databaseOverlay.Overlay
}

func NewTXOverlay(db interfaces.IDatabase) *TXDatabaseOverlay {
	answer := new(TXDatabaseOverlay)
	answer.DBO.DB = db
	return answer
}

func NewTXMapDB() *TXDatabaseOverlay {
	return NewTXOverlay(new(mapdb.MapDB))
}

func NewTXLevelDB(ldbpath string) (*TXDatabaseOverlay, error) {
	db, err := hybridDB.NewLevelMapHybridDB(ldbpath, false)
	if err != nil {
		fmt.Printf("err opening transaction db: %v\n", err)
	}

	if db == nil {
		fmt.Println("Creating new transaction db ...")
		db, err = hybridDB.NewLevelMapHybridDB(ldbpath, true)

		if err != nil {
			return nil, err
		}
	}
	fmt.Println("Transaction database started from: " + ldbpath)
	return NewTXOverlay(db), nil
}

func NewTXBoltDB(boltPath string) (*TXDatabaseOverlay, error) {
	fileInfo, err := os.Stat(boltPath)
	if err == nil { //if it exists
		if fileInfo.IsDir() { //if it is a folder though
			return nil, fmt.Errorf("The path %s is a directory.  Please specify a file name.", boltPath)
		}
	}
	if err != nil && !os.IsNotExist(err) { //some other error, besides the file not existing
		fmt.Printf("database error %s\n", err)
		return nil, err
	}

	defer func() {
		if r := recover(); r != nil {
			fmt.Printf("Could not use wallet cache database file \"%s\"\n%v\n", boltPath, r)
			os.Exit(1)
		}
	}()
	db := hybridDB.NewBoltMapHybridDB(nil, boltPath)

	fmt.Println("Database started from: " + boltPath)
	return NewTXOverlay(db), nil
}

func (db *TXDatabaseOverlay) Close() error {
	return db.DBO.Close()
}

// GetAllTXs returns a list of all transactions in the history of Factom. A
// local database is used to cache the factoid blocks.
func (db *TXDatabaseOverlay) GetAllTXs() ([]interfaces.ITransaction, error) {
	// update the database and get the newest fblock
	_, err := db.update()
	if err != nil {
		return nil, err
	}
	fblock, err := db.DBO.FetchFBlockHead()
	if err != nil {
		return nil, err
	}
	if fblock == nil {
		return nil, fmt.Errorf("FBlock Chain has not finished syncing")
	}
	txs := make([]interfaces.ITransaction, 0)

	for {
		// get all of the txs from the block
		height := fblock.GetDatabaseHeight()
		for _, tx := range fblock.GetTransactions() {
			ins, err := tx.TotalInputs()
			if err != nil {
				return nil, err
			}
			outs, err := tx.TotalOutputs()
			if err != nil {
				return nil, err
			}

			if ins != 0 || outs != 0 {
				tx.SetBlockHeight(height)
				txs = append(txs, tx)
			}
		}

		if pre := fblock.GetPrevKeyMR().String(); pre != factom.ZeroHash {
			// get the previous block
			fblock, err = db.GetFBlock(pre)
			if err != nil {
				return nil, err
			} else if fblock == nil {
				return nil, fmt.Errorf("Missing fblock in database: %s", pre)
			}
		} else {
			break
		}
	}

	return txs, nil
}

// GetTX gets a transaction by the transaction id
func (db *TXDatabaseOverlay) GetTX(txid string) (
	interfaces.ITransaction, error) {
	txs, err := db.GetAllTXs()
	if err != nil {
		return nil, err
	}

	for _, tx := range txs {
		if tx.GetSigHash().String() == txid {
			return tx, nil
		}
	}

	return nil, fmt.Errorf("Transaction not found")
}

// GetTXAddress returns a list of all transactions in the history of Factom that
// include a specific address.
func (db *TXDatabaseOverlay) GetTXAddress(adr string) (
	[]interfaces.ITransaction, error) {
	filtered := make([]interfaces.ITransaction, 0)

	txs, err := db.GetAllTXs()
	if err != nil {
		return nil, err
	}

	if factom.AddressStringType(adr) == factom.FactoidPub {
		for _, tx := range txs {
			for _, in := range tx.GetInputs() {
				if primitives.ConvertFctAddressToUserStr(in.GetAddress()) == adr {
					filtered = append(filtered, tx)
				}
			}
			for _, out := range tx.GetOutputs() {
				if primitives.ConvertFctAddressToUserStr(out.GetAddress()) == adr {
					filtered = append(filtered, tx)
				}
			}
		}
	} else if factom.AddressStringType(adr) == factom.ECPub {
		for _, tx := range txs {
			for _, out := range tx.GetECOutputs() {
				if primitives.ConvertECAddressToUserStr(out.GetAddress()) == adr {
					filtered = append(filtered, tx)
				}
			}
		}
	} else {
		return nil, fmt.Errorf("not a valid address")
	}

	return filtered, nil
}

func (db *TXDatabaseOverlay) GetTXRange(start, end int) (
	[]interfaces.ITransaction, error) {
	if start < 0 || end < 0 {
		return nil, fmt.Errorf("Range cannot have negative numbers")
	}
	s, e := uint32(start), uint32(end)

	filtered := make([]interfaces.ITransaction, 0)

	txs, err := db.GetAllTXs()
	if err != nil {
		return nil, err
	}

	for _, tx := range txs {
		if s <= tx.GetBlockHeight() && tx.GetBlockHeight() <= e {
			filtered = append(filtered, tx)
		}
	}

	return filtered, nil
}

// GetFBlock retrives a Factoid Block from Factom
func (db *TXDatabaseOverlay) GetFBlock(keymr string) (interfaces.IFBlock, error) {
	h, err := primitives.NewShaHashFromStr(keymr)
	if err != nil {
		return nil, err
	}

	fBlock, err := db.DBO.FetchFBlock(h)
	if err != nil {
		return nil, err
	}
	return fBlock, nil
}

func (db *TXDatabaseOverlay) FetchNextFBlockHeight() (uint32, error) {
	block, err := db.DBO.FetchFBlockHead()
	if err != nil {
		return 0, err
	}
	if block == nil {
		return 0, nil
	}
	return block.GetDBHeight() + 1, nil
}

func (db *TXDatabaseOverlay) InsertFBlockHead(fblock interfaces.IFBlock) error {
	return db.DBO.SaveFactoidBlockHead(fblock)
}

// update gets all fblocks written since the database was last updated, and
// returns the most recent fblock keymr.
func (db *TXDatabaseOverlay) update() (string, error) {
	newestFBlock, err := fblockHead()
	if err != nil {
		return "", err
	}

	start, err := db.FetchNextFBlockHeight()
	if err != nil {
		return "", err
	}

	//Making sure we didn't switch networks
	genesis, err := db.DBO.FetchFBlockByHeight(0)
	if err != nil {
		return "", err
	}
	if genesis != nil {
		genesis2, err := getdblockbyheight(0)
		if err != nil {
			return "", err
		}

		var gensisFBlockKeyMr interfaces.IHash
		for _, e := range genesis2.GetDBEntries() {
			if e.GetChainID().String() == "000000000000000000000000000000000000000000000000000000000000000f" {
				gensisFBlockKeyMr = e.GetKeyMR()
				break
			}
		}

		if gensisFBlockKeyMr == nil {
			return "", fmt.Errorf("there was an error fetching the genesis block via the api.")
		}

		if !gensisFBlockKeyMr.IsSameAs(genesis.GetKeyMR()) {
			start = 0
		}
	}

	newestHeight := newestFBlock.GetDatabaseHeight()

	if start >= newestHeight {
		return newestFBlock.GetKeyMR().String(), nil
	}

	db.DBO.StartMultiBatch()
	for i := start; i <= newestHeight; i++ {
		if i%1000 == 0 {
			if newestHeight-start > 1000 {
				fmt.Printf("Fetching block %v / %v\n", i, newestHeight)
			}
		}
		fblock, err := getfblockbyheight(i)
		if err != nil {
			db.DBO.ExecuteMultiBatch()
			return "", err
		}
		db.DBO.ProcessFBlockMultiBatch(fblock)

		// Save to DB every 500 blocks
		if i%500 == 0 {
			db.DBO.ExecuteMultiBatch()
			db.DBO.StartMultiBatch()
		}
	}
	fmt.Printf("Fetching block %v / %v\n", newestHeight, newestHeight)
	err = db.DBO.ExecuteMultiBatch() // Save remaining blocks
	if err != nil {
		return "", err
	}

	return newestFBlock.GetKeyMR().String(), nil
}

// fblockHead gets the most recent fblock.
func fblockHead() (interfaces.IFBlock, error) {
	fblockID := "000000000000000000000000000000000000000000000000000000000000000f"

	dbhead, err := factom.GetDBlockHead()
	if err != nil {
		return nil, err
	}
	dblock, err := factom.GetDBlock(dbhead)
	if err != nil {
		return nil, err
	}

	var fblockmr string
	for _, eblock := range dblock.EntryBlockList {
		if eblock.ChainID == fblockID {
			fblockmr = eblock.KeyMR
		}
	}
	if fblockmr == "" {
		return nil, err
	}

	return getfblock(fblockmr)
}

func getfblock(keymr string) (interfaces.IFBlock, error) {
	p, err := factom.GetRaw(keymr)
	if err != nil {
		return nil, err
	}
	return factoid.UnmarshalFBlock(p)
}

func getfblockbyheight(height uint32) (interfaces.IFBlock, error) {
	p, err := factom.GetFBlockByHeight(int64(height))
	if err != nil {
		return nil, err
	}
	h, err := hex.DecodeString(p.RawData)
	if err != nil {
		return nil, err
	}
	return factoid.UnmarshalFBlock(h)
}

func getdblockbyheight(height uint32) (interfaces.IDirectoryBlock, error) {
	p, err := factom.GetDBlockByHeight(int64(height))
	if err != nil {
		return nil, err
	}
	h, err := hex.DecodeString(p.RawData)
	if err != nil {
		return nil, err
	}
	return directoryBlock.UnmarshalDBlock(h)
}
<|endoftext|>