<commit_before>package storage

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"log"
	"os"
	"path"
	"sync"
)

const (
	SuperBlockSize = 8
)

type SuperBlock struct {
	Version     Version
	ReplicaType ReplicationType
}

func (s *SuperBlock) Bytes() []byte {
	header := make([]byte, SuperBlockSize)
	header[0] = byte(s.Version)
	header[1] = s.ReplicaType.Byte()
	return header
}

type Volume struct {
	Id       VolumeId
	dir      string
	dataFile *os.File
	nm       NeedleMapper
	readOnly bool

	SuperBlock

	accessLock sync.Mutex
}

func NewVolume(dirname string, id VolumeId, replicationType ReplicationType) (v *Volume, e error) {
	v = &Volume{dir: dirname, Id: id}
	v.SuperBlock = SuperBlock{ReplicaType: replicationType}
	e = v.load(true)
	return
}
func LoadVolumeOnly(dirname string, id VolumeId) (v *Volume, e error) {
	v = &Volume{dir: dirname, Id: id}
	v.SuperBlock = SuperBlock{ReplicaType: CopyNil}
	e = v.load(false)
	return
}
func (v *Volume) load(alsoLoadIndex bool) error {
	var e error
	fileName := path.Join(v.dir, v.Id.String())
	v.dataFile, e = os.OpenFile(fileName+".dat", os.O_RDWR|os.O_CREATE, 0644)
	if e != nil {
		if !os.IsPermission(e) {
			return fmt.Errorf("cannot create Volume Data %s.dat: %s", fileName, e)
		}
		if v.dataFile, e = os.Open(fileName + ".dat"); e != nil {
			return fmt.Errorf("cannot open Volume Data %s.dat: %s", fileName, e)
		}
		log.Printf("opening " + fileName + ".dat in READONLY mode")
		v.readOnly = true
	}
	if v.ReplicaType == CopyNil {
		e = v.readSuperBlock()
	} else {
		e = v.maybeWriteSuperBlock()
	}
	if e == nil && alsoLoadIndex {
		var indexFile *os.File
		if v.readOnly {
			if indexFile, e = os.Open(fileName + ".idx"); e != nil && !os.IsNotExist(e) {
				return fmt.Errorf("cannot open index file %s.idx: %s", fileName, e)
			}
			if indexFile != nil {
				log.Printf("converting %s.idx to %s.cdb", fileName, fileName)
				if e = ConvertIndexToCdb(fileName+".cdb", indexFile); e != nil {
					log.Printf("error converting %s.idx to %s.cdb: %s", fileName, fileName)
				} else {
					indexFile.Close()
					os.Remove(indexFile.Name())
					indexFile = nil
				}
			}
			v.nm, e = OpenCdbMap(fileName + ".cdb")
			return e
		} else {
			indexFile, e = os.OpenFile(fileName+".idx", os.O_RDWR|os.O_CREATE, 0644)
			if e != nil {
				return fmt.Errorf("cannot create Volume Data %s.dat: %s", fileName, e)
			}
		}
		v.nm, e = LoadNeedleMap(indexFile)
	}
	return e
}
func (v *Volume) Version() Version {
	return v.SuperBlock.Version
}
func (v *Volume) Size() int64 {
	v.accessLock.Lock()
	defer v.accessLock.Unlock()
	stat, e := v.dataFile.Stat()
	if e == nil {
		return stat.Size()
	}
	log.Printf("Failed to read file size %s %s\n", v.dataFile.Name(), e.Error())
	return -1
}
func (v *Volume) Close() {
	v.accessLock.Lock()
	defer v.accessLock.Unlock()
	v.nm.Close()
	_ = v.dataFile.Close()
}
func (v *Volume) maybeWriteSuperBlock() error {
	stat, e := v.dataFile.Stat()
	if e != nil {
		log.Printf("failed to stat datafile %s: %s", v.dataFile, e)
		return e
	}
	if stat.Size() == 0 {
		v.SuperBlock.Version = CurrentVersion
		_, e = v.dataFile.Write(v.SuperBlock.Bytes())
		if e != nil && os.IsPermission(e) {
			//read-only, but zero length - recreate it!
			if v.dataFile, e = os.Create(v.dataFile.Name()); e == nil {
				if _, e = v.dataFile.Write(v.SuperBlock.Bytes()); e == nil {
					v.readOnly = false
				}
			}
		}
	}
	return e
}
func (v *Volume) readSuperBlock() (err error) {
	if _, err = v.dataFile.Seek(0, 0); err != nil {
		return fmt.Errorf("cannot seek to the beginning of %s: %s", v.dataFile, err)
	}
	header := make([]byte, SuperBlockSize)
	if _, e := v.dataFile.Read(header); e != nil {
		return fmt.Errorf("cannot read superblock: %s", e)
	}
	v.SuperBlock, err = ParseSuperBlock(header)
	return err
}
func ParseSuperBlock(header []byte) (superBlock SuperBlock, err error) {
	superBlock.Version = Version(header[0])
	if superBlock.ReplicaType, err = NewReplicationTypeFromByte(header[1]); err != nil {
		err = fmt.Errorf("cannot read replica type: %s", err)
	}
	return
}
func (v *Volume) NeedToReplicate() bool {
	return v.ReplicaType.GetCopyCount() > 1
}

func (v *Volume) isFileUnchanged(n *Needle) bool {
	nv, ok := v.nm.Get(n.Id)
	if ok && nv.Offset > 0 {
		if _, err := v.dataFile.Seek(int64(nv.Offset)*NeedlePaddingSize, 0); err != nil {
			return false
		}
		oldNeedle := new(Needle)
		oldNeedle.Read(v.dataFile, nv.Size, v.Version())
		if oldNeedle.Checksum == n.Checksum && bytes.Equal(oldNeedle.Data, n.Data) {
			n.Size = oldNeedle.Size
			return true
		}
	}
	return false
}
func (v *Volume) write(n *Needle) (size uint32, err error) {
	if v.readOnly {
		err = fmt.Errorf("%s is read-only", v.dataFile)
		return
	}
	v.accessLock.Lock()
	defer v.accessLock.Unlock()
	if v.isFileUnchanged(n) {
		size = n.Size
		return
	}
	var offset int64
	if offset, err = v.dataFile.Seek(0, 2); err != nil {
		return
	}

	//ensure file writing starting from aligned positions
	if offset%NeedlePaddingSize != 0 {
		offset = offset + (NeedlePaddingSize - offset%NeedlePaddingSize)
		if offset, err = v.dataFile.Seek(offset, 0); err != nil {
			return
		}
	}

	if size, err = n.Append(v.dataFile, v.Version()); err != nil {
		if e := v.dataFile.Truncate(offset); e != nil {
			err = fmt.Errorf("%s\ncannot truncate %s: %s", err, v.dataFile, e)
		}
		return
	}
	nv, ok := v.nm.Get(n.Id)
	if !ok || int64(nv.Offset)*NeedlePaddingSize < offset {
		_, err = v.nm.Put(n.Id, uint32(offset/NeedlePaddingSize), n.Size)
	}
	return
}

func (v *Volume) delete(n *Needle) (uint32, error) {
	if v.readOnly {
		return 0, fmt.Errorf("%s is read-only", v.dataFile)
	}
	v.accessLock.Lock()
	defer v.accessLock.Unlock()
	nv, ok := v.nm.Get(n.Id)
	//fmt.Println("key", n.Id, "volume offset", nv.Offset, "data_size", n.Size, "cached size", nv.Size)
	if ok {
		var err error
		if err = v.nm.Delete(n.Id); err != nil {
			return nv.Size, err
		}
		if _, err = v.dataFile.Seek(0, 2); err != nil {
			return nv.Size, err
		}
		n.Data = make([]byte, 0)
		_, err = n.Append(v.dataFile, v.Version())
		return nv.Size, err
	}
	return 0, nil
}

func (v *Volume) read(n *Needle) (int, error) {
	v.accessLock.Lock()
	defer v.accessLock.Unlock()
	nv, ok := v.nm.Get(n.Id)
	if ok && nv.Offset > 0 {
		if _, err := v.dataFile.Seek(int64(nv.Offset)*NeedlePaddingSize, 0); err != nil {
			return -1, err
		}
		return n.Read(v.dataFile, nv.Size, v.Version())
	}
	return -1, errors.New("Not Found")
}

func (v *Volume) garbageLevel() float64 {
	return float64(v.nm.DeletedSize()) / float64(v.ContentSize())
}

func (v *Volume) compact() error {
	v.accessLock.Lock()
	defer v.accessLock.Unlock()

	filePath := path.Join(v.dir, v.Id.String())
	return v.copyDataAndGenerateIndexFile(filePath+".cpd", filePath+".cpx")
}
func (v *Volume) commitCompact() error {
	v.accessLock.Lock()
	defer v.accessLock.Unlock()
	_ = v.dataFile.Close()
	var e error
	if e = os.Rename(path.Join(v.dir, v.Id.String()+".cpd"), path.Join(v.dir, v.Id.String()+".dat")); e != nil {
		return e
	}
	if e = os.Rename(path.Join(v.dir, v.Id.String()+".cpx"), path.Join(v.dir, v.Id.String()+".idx")); e != nil {
		return e
	}
	if e = v.load(true); e != nil {
		return e
	}
	return nil
}
func (v *Volume) freeze() error {
	if v.readOnly {
		return nil
	}
	nm, ok := v.nm.(*NeedleMap)
	if !ok {
		return nil
	}
	v.accessLock.Lock()
	defer v.accessLock.Unlock()
	bn, _ := nakeFilename(v.dataFile.Name())
	cdbFn := bn + ".cdb"
	log.Printf("converting %s to %s", nm.indexFile.Name(), cdbFn)
	err := DumpNeedleMapToCdb(cdbFn, nm)
	if err != nil {
		return err
	}
	if v.nm, err = OpenCdbMap(cdbFn); err != nil {
		return err
	}
	nm.indexFile.Close()
	os.Remove(nm.indexFile.Name())
	v.readOnly = true
	return nil
}

func ScanVolumeFile(dirname string, id VolumeId,
	visitSuperBlock func(SuperBlock) error,
	visitNeedle func(n *Needle, offset uint32) error) (err error) {
	var v *Volume
	if v, err = LoadVolumeOnly(dirname, id); err != nil {
		return
	}
	if err = visitSuperBlock(v.SuperBlock); err != nil {
		return
	}

	version := v.Version()

	offset := uint32(SuperBlockSize)
	n, rest, e := ReadNeedleHeader(v.dataFile, version)
	if e != nil {
		err = fmt.Errorf("cannot read needle header: %s", e)
		return
	}
	for n != nil {
		if err = n.ReadNeedleBody(v.dataFile, version, rest); err != nil {
			err = fmt.Errorf("cannot read needle body: %s", err)
			return
		}
		if err = visitNeedle(n, offset); err != nil {
			return
		}
		offset += NeedleHeaderSize + rest
		if n, rest, err = ReadNeedleHeader(v.dataFile, version); err != nil {
			if err == io.EOF {
				return nil
			}
			return fmt.Errorf("cannot read needle header: %s", err)
		}
	}

	return
}

func (v *Volume) copyDataAndGenerateIndexFile(dstName, idxName string) (err error) {
	var (
		dst, idx *os.File
	)
	if dst, err = os.OpenFile(dstName, os.O_WRONLY|os.O_CREATE, 0644); err != nil {
		return
	}
	defer dst.Close()

	if idx, err = os.OpenFile(idxName, os.O_WRONLY|os.O_CREATE, 0644); err != nil {
		return
	}
	defer idx.Close()

	nm := NewNeedleMap(idx)
	new_offset := uint32(SuperBlockSize)

	err = ScanVolumeFile(v.dir, v.Id, func(superBlock SuperBlock) error {
		_, err = dst.Write(superBlock.Bytes())
		return err
	}, func(n *Needle, offset uint32) error {
		nv, ok := v.nm.Get(n.Id)
		//log.Println("file size is", n.Size, "rest", rest)
		if ok && nv.Offset*NeedlePaddingSize == offset {
			if nv.Size > 0 {
				if _, err = nm.Put(n.Id, new_offset/NeedlePaddingSize, n.Size); err != nil {
					return fmt.Errorf("cannot put needle: %s", err)
				}
				if _, err = n.Append(dst, v.Version()); err != nil {
					return fmt.Errorf("cannot append needle: %s", err)
				}
				new_offset += n.DiskSize()
				//log.Println("saving key", n.Id, "volume offset", old_offset, "=>", new_offset, "data_size", n.Size, "rest", rest)
			}
		}
		return nil
	})

	return
}
func (v *Volume) ContentSize() uint64 {
	return v.nm.ContentSize()
}
<commit_msg>Issue 37:	Replicate delete<commit_after>package storage

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"log"
	"os"
	"path"
	"sync"
)

const (
	SuperBlockSize = 8
)

type SuperBlock struct {
	Version     Version
	ReplicaType ReplicationType
}

func (s *SuperBlock) Bytes() []byte {
	header := make([]byte, SuperBlockSize)
	header[0] = byte(s.Version)
	header[1] = s.ReplicaType.Byte()
	return header
}

type Volume struct {
	Id       VolumeId
	dir      string
	dataFile *os.File
	nm       NeedleMapper
	readOnly bool

	SuperBlock

	accessLock sync.Mutex
}

func NewVolume(dirname string, id VolumeId, replicationType ReplicationType) (v *Volume, e error) {
	v = &Volume{dir: dirname, Id: id}
	v.SuperBlock = SuperBlock{ReplicaType: replicationType}
	e = v.load(true)
	return
}
func LoadVolumeOnly(dirname string, id VolumeId) (v *Volume, e error) {
	v = &Volume{dir: dirname, Id: id}
	v.SuperBlock = SuperBlock{ReplicaType: CopyNil}
	e = v.load(false)
	return
}
func (v *Volume) load(alsoLoadIndex bool) error {
	var e error
	fileName := path.Join(v.dir, v.Id.String())
	v.dataFile, e = os.OpenFile(fileName+".dat", os.O_RDWR|os.O_CREATE, 0644)
	if e != nil {
		if !os.IsPermission(e) {
			return fmt.Errorf("cannot create Volume Data %s.dat: %s", fileName, e)
		}
		if v.dataFile, e = os.Open(fileName + ".dat"); e != nil {
			return fmt.Errorf("cannot open Volume Data %s.dat: %s", fileName, e)
		}
		log.Printf("opening " + fileName + ".dat in READONLY mode")
		v.readOnly = true
	}
	if v.ReplicaType == CopyNil {
		e = v.readSuperBlock()
	} else {
		e = v.maybeWriteSuperBlock()
	}
	if e == nil && alsoLoadIndex {
		var indexFile *os.File
		if v.readOnly {
			if indexFile, e = os.Open(fileName + ".idx"); e != nil && !os.IsNotExist(e) {
				return fmt.Errorf("cannot open index file %s.idx: %s", fileName, e)
			}
			if indexFile != nil {
				log.Printf("converting %s.idx to %s.cdb", fileName, fileName)
				if e = ConvertIndexToCdb(fileName+".cdb", indexFile); e != nil {
					log.Printf("error converting %s.idx to %s.cdb: %s", fileName, fileName)
				} else {
					indexFile.Close()
					os.Remove(indexFile.Name())
					indexFile = nil
				}
			}
			v.nm, e = OpenCdbMap(fileName + ".cdb")
			return e
		} else {
			indexFile, e = os.OpenFile(fileName+".idx", os.O_RDWR|os.O_CREATE, 0644)
			if e != nil {
				return fmt.Errorf("cannot create Volume Data %s.dat: %s", fileName, e)
			}
		}
		v.nm, e = LoadNeedleMap(indexFile)
	}
	return e
}
func (v *Volume) Version() Version {
	return v.SuperBlock.Version
}
func (v *Volume) Size() int64 {
	v.accessLock.Lock()
	defer v.accessLock.Unlock()
	stat, e := v.dataFile.Stat()
	if e == nil {
		return stat.Size()
	}
	log.Printf("Failed to read file size %s %s\n", v.dataFile.Name(), e.Error())
	return -1
}
func (v *Volume) Close() {
	v.accessLock.Lock()
	defer v.accessLock.Unlock()
	v.nm.Close()
	_ = v.dataFile.Close()
}
func (v *Volume) maybeWriteSuperBlock() error {
	stat, e := v.dataFile.Stat()
	if e != nil {
		log.Printf("failed to stat datafile %s: %s", v.dataFile, e)
		return e
	}
	if stat.Size() == 0 {
		v.SuperBlock.Version = CurrentVersion
		_, e = v.dataFile.Write(v.SuperBlock.Bytes())
		if e != nil && os.IsPermission(e) {
			//read-only, but zero length - recreate it!
			if v.dataFile, e = os.Create(v.dataFile.Name()); e == nil {
				if _, e = v.dataFile.Write(v.SuperBlock.Bytes()); e == nil {
					v.readOnly = false
				}
			}
		}
	}
	return e
}
func (v *Volume) readSuperBlock() (err error) {
	if _, err = v.dataFile.Seek(0, 0); err != nil {
		return fmt.Errorf("cannot seek to the beginning of %s: %s", v.dataFile, err)
	}
	header := make([]byte, SuperBlockSize)
	if _, e := v.dataFile.Read(header); e != nil {
		return fmt.Errorf("cannot read superblock: %s", e)
	}
	v.SuperBlock, err = ParseSuperBlock(header)
	return err
}
func ParseSuperBlock(header []byte) (superBlock SuperBlock, err error) {
	superBlock.Version = Version(header[0])
	if superBlock.ReplicaType, err = NewReplicationTypeFromByte(header[1]); err != nil {
		err = fmt.Errorf("cannot read replica type: %s", err)
	}
	return
}
func (v *Volume) NeedToReplicate() bool {
	return v.ReplicaType.GetCopyCount() > 1
}

func (v *Volume) isFileUnchanged(n *Needle) bool {
	nv, ok := v.nm.Get(n.Id)
	if ok && nv.Offset > 0 {
		if _, err := v.dataFile.Seek(int64(nv.Offset)*NeedlePaddingSize, 0); err != nil {
			return false
		}
		oldNeedle := new(Needle)
		oldNeedle.Read(v.dataFile, nv.Size, v.Version())
		if oldNeedle.Checksum == n.Checksum && bytes.Equal(oldNeedle.Data, n.Data) {
			n.Size = oldNeedle.Size
			return true
		}
	}
	return false
}
func (v *Volume) write(n *Needle) (size uint32, err error) {
	if v.readOnly {
		err = fmt.Errorf("%s is read-only", v.dataFile)
		return
	}
	v.accessLock.Lock()
	defer v.accessLock.Unlock()
	if v.isFileUnchanged(n) {
		size = n.Size
		return
	}
	var offset int64
	if offset, err = v.dataFile.Seek(0, 2); err != nil {
		return
	}

	//ensure file writing starting from aligned positions
	if offset%NeedlePaddingSize != 0 {
		offset = offset + (NeedlePaddingSize - offset%NeedlePaddingSize)
		if offset, err = v.dataFile.Seek(offset, 0); err != nil {
			return
		}
	}

	if size, err = n.Append(v.dataFile, v.Version()); err != nil {
		if e := v.dataFile.Truncate(offset); e != nil {
			err = fmt.Errorf("%s\ncannot truncate %s: %s", err, v.dataFile, e)
		}
		return
	}
	nv, ok := v.nm.Get(n.Id)
	if !ok || int64(nv.Offset)*NeedlePaddingSize < offset {
		_, err = v.nm.Put(n.Id, uint32(offset/NeedlePaddingSize), n.Size)
	}
	return
}

func (v *Volume) delete(n *Needle) (uint32, error) {
	if v.readOnly {
		return 0, fmt.Errorf("%s is read-only", v.dataFile)
	}
	v.accessLock.Lock()
	defer v.accessLock.Unlock()
	nv, ok := v.nm.Get(n.Id)
	//fmt.Println("key", n.Id, "volume offset", nv.Offset, "data_size", n.Size, "cached size", nv.Size)
	if ok {
		var err error
		size := nv.Size
		if err = v.nm.Delete(n.Id); err != nil {
			return size, err
		}
		if _, err = v.dataFile.Seek(0, 2); err != nil {
			return size, err
		}
		n.Data = make([]byte, 0)
		_, err = n.Append(v.dataFile, v.Version())
		return size, err
	}
	return 0, nil
}

func (v *Volume) read(n *Needle) (int, error) {
	v.accessLock.Lock()
	defer v.accessLock.Unlock()
	nv, ok := v.nm.Get(n.Id)
	if ok && nv.Offset > 0 {
		if _, err := v.dataFile.Seek(int64(nv.Offset)*NeedlePaddingSize, 0); err != nil {
			return -1, err
		}
		return n.Read(v.dataFile, nv.Size, v.Version())
	}
	return -1, errors.New("Not Found")
}

func (v *Volume) garbageLevel() float64 {
	return float64(v.nm.DeletedSize()) / float64(v.ContentSize())
}

func (v *Volume) compact() error {
	v.accessLock.Lock()
	defer v.accessLock.Unlock()

	filePath := path.Join(v.dir, v.Id.String())
	return v.copyDataAndGenerateIndexFile(filePath+".cpd", filePath+".cpx")
}
func (v *Volume) commitCompact() error {
	v.accessLock.Lock()
	defer v.accessLock.Unlock()
	_ = v.dataFile.Close()
	var e error
	if e = os.Rename(path.Join(v.dir, v.Id.String()+".cpd"), path.Join(v.dir, v.Id.String()+".dat")); e != nil {
		return e
	}
	if e = os.Rename(path.Join(v.dir, v.Id.String()+".cpx"), path.Join(v.dir, v.Id.String()+".idx")); e != nil {
		return e
	}
	if e = v.load(true); e != nil {
		return e
	}
	return nil
}
func (v *Volume) freeze() error {
	if v.readOnly {
		return nil
	}
	nm, ok := v.nm.(*NeedleMap)
	if !ok {
		return nil
	}
	v.accessLock.Lock()
	defer v.accessLock.Unlock()
	bn, _ := nakeFilename(v.dataFile.Name())
	cdbFn := bn + ".cdb"
	log.Printf("converting %s to %s", nm.indexFile.Name(), cdbFn)
	err := DumpNeedleMapToCdb(cdbFn, nm)
	if err != nil {
		return err
	}
	if v.nm, err = OpenCdbMap(cdbFn); err != nil {
		return err
	}
	nm.indexFile.Close()
	os.Remove(nm.indexFile.Name())
	v.readOnly = true
	return nil
}

func ScanVolumeFile(dirname string, id VolumeId,
	visitSuperBlock func(SuperBlock) error,
	visitNeedle func(n *Needle, offset uint32) error) (err error) {
	var v *Volume
	if v, err = LoadVolumeOnly(dirname, id); err != nil {
		return
	}
	if err = visitSuperBlock(v.SuperBlock); err != nil {
		return
	}

	version := v.Version()

	offset := uint32(SuperBlockSize)
	n, rest, e := ReadNeedleHeader(v.dataFile, version)
	if e != nil {
		err = fmt.Errorf("cannot read needle header: %s", e)
		return
	}
	for n != nil {
		if err = n.ReadNeedleBody(v.dataFile, version, rest); err != nil {
			err = fmt.Errorf("cannot read needle body: %s", err)
			return
		}
		if err = visitNeedle(n, offset); err != nil {
			return
		}
		offset += NeedleHeaderSize + rest
		if n, rest, err = ReadNeedleHeader(v.dataFile, version); err != nil {
			if err == io.EOF {
				return nil
			}
			return fmt.Errorf("cannot read needle header: %s", err)
		}
	}

	return
}

func (v *Volume) copyDataAndGenerateIndexFile(dstName, idxName string) (err error) {
	var (
		dst, idx *os.File
	)
	if dst, err = os.OpenFile(dstName, os.O_WRONLY|os.O_CREATE, 0644); err != nil {
		return
	}
	defer dst.Close()

	if idx, err = os.OpenFile(idxName, os.O_WRONLY|os.O_CREATE, 0644); err != nil {
		return
	}
	defer idx.Close()

	nm := NewNeedleMap(idx)
	new_offset := uint32(SuperBlockSize)

	err = ScanVolumeFile(v.dir, v.Id, func(superBlock SuperBlock) error {
		_, err = dst.Write(superBlock.Bytes())
		return err
	}, func(n *Needle, offset uint32) error {
		nv, ok := v.nm.Get(n.Id)
		//log.Println("file size is", n.Size, "rest", rest)
		if ok && nv.Offset*NeedlePaddingSize == offset {
			if nv.Size > 0 {
				if _, err = nm.Put(n.Id, new_offset/NeedlePaddingSize, n.Size); err != nil {
					return fmt.Errorf("cannot put needle: %s", err)
				}
				if _, err = n.Append(dst, v.Version()); err != nil {
					return fmt.Errorf("cannot append needle: %s", err)
				}
				new_offset += n.DiskSize()
				//log.Println("saving key", n.Id, "volume offset", old_offset, "=>", new_offset, "data_size", n.Size, "rest", rest)
			}
		}
		return nil
	})

	return
}
func (v *Volume) ContentSize() uint64 {
	return v.nm.ContentSize()
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io/ioutil"
	"os/exec"
	"reflect"
	"time"

	pbb "github.com/brotherlogic/buildserver/proto"
	pb "github.com/brotherlogic/gobuildslave/proto"
	"github.com/brotherlogic/goserver/utils"
	pbt "github.com/brotherlogic/tracer/proto"
	"github.com/golang/protobuf/proto"
	"golang.org/x/net/context"
)

const (
	pendWait = time.Minute
)

func (s *Server) runTransition(ctx context.Context, job *pb.JobAssignment) {
	utils.SendTrace(ctx, fmt.Sprintf("run_transition_%v", job.State), time.Now(), pbt.Milestone_MARKER, job.Job.Name)
	stState := job.State
	switch job.State {
	case pb.State_ACKNOWLEDGED:
		key := s.scheduleBuild(ctx, job.Job)
		s.stateMutex.Lock()
		s.stateMap[job.Job.Name] = fmt.Sprintf("SCHED: %v @ %v", key, time.Now())
		s.stateMutex.Unlock()
		if !job.Job.Bootstrap {
			if key != "" {
				job.Server = s.Registry.Identifier
				job.State = pb.State_BUILT
				job.RunningVersion = key
			}
		} else {
			job.CommandKey = key
			job.State = pb.State_BUILDING
			job.Server = s.Registry.Identifier
		}
	case pb.State_BUILDING:
		s.stateMutex.Lock()
		s.stateMap[job.Job.Name] = fmt.Sprintf("BUILD(%v): %v", job.CommandKey, s.scheduler.getState(job.CommandKey))
		s.stateMutex.Unlock()
		if s.taskComplete(job.CommandKey) {
			job.State = pb.State_BUILT
		}
	case pb.State_BUILT:
		output := s.scheduler.getOutput(job.CommandKey)
		s.stateMutex.Lock()
		s.stateMap[job.Job.Name] = fmt.Sprintf("BUILT(%v): (%v): %v", job.CommandKey, len(output), output)
		s.stateMutex.Unlock()
		if job.Job.Bootstrap && len(output) > 0 {
			if job.BuildFail == 5 {
				s.deliverCrashReport(ctx, job, output)
				job.BuildFail = 0
			}
			job.BuildFail++
			job.State = pb.State_DIED
		} else {
			job.BuildFail = 0
			key := s.scheduleRun(job.Job)
			job.CommandKey = key
			job.StartTime = time.Now().Unix()
			job.State = pb.State_PENDING
			if _, ok := s.pendingMap[time.Now().Weekday()]; !ok {
				s.pendingMap[time.Now().Weekday()] = make(map[string]int)
			}
			s.pendingMap[time.Now().Weekday()][job.Job.Name]++
		}
	case pb.State_PENDING:
		s.stateMutex.Lock()
		s.stateMap[job.Job.Name] = fmt.Sprintf("OUTPUT = %v", s.scheduler.getOutput(job.CommandKey))
		s.stateMutex.Unlock()
		if time.Now().Add(-time.Minute).Unix() > job.StartTime {
			job.State = pb.State_RUNNING
		}
	case pb.State_RUNNING:
		output := s.scheduler.getOutput(job.CommandKey)
		s.stateMutex.Lock()
		s.stateMap[job.Job.Name] = fmt.Sprintf("ROUTPUT = %v, %v", s.scheduler.getOutput(job.CommandKey), s.scheduler.getStatus(job.CommandKey))
		job.Status = s.scheduler.getStatus(job.CommandKey)
		s.stateMutex.Unlock()
		if s.taskComplete(job.CommandKey) {
			s.stateMutex.Lock()
			s.stateMap[job.Job.Name] = fmt.Sprintf("COMPLETE = %v", output)
			s.stateMutex.Unlock()
			s.deliverCrashReport(ctx, job, output)
			job.State = pb.State_DIED
		}

		err := s.discover.discover(job.Job.Name, s.Registry.Identifier)
		if err != nil {
			if job.DiscoverCount > 30 {
				s.RaiseIssue(ctx, "Cannot Discover Running Server", fmt.Sprintf("%v on %v is not discoverable, despite running (%v) the output says %v", job.Job.Name, s.Registry.Identifier, err, output), false)
			}
			job.DiscoverCount++
		} else {
			job.DiscoverCount = 0
		}

		// Restart this job if we need to
		if !job.Job.Bootstrap {
			version, err := s.getVersion(ctx, job.Job)
			if err == nil && version.Version != job.RunningVersion {
				s.versions[job.Job.Name] = version
				s.stateMutex.Lock()
				s.stateMap[job.Job.Name] = fmt.Sprintf("VERSION_MISMATCH = %v,%v", version, job.RunningVersion)
				s.stateMutex.Unlock()
				s.scheduler.killJob(job.CommandKey)
				job.State = pb.State_ACKNOWLEDGED
			}
		}
	case pb.State_DIED:
		s.stateMutex.Lock()
		s.stateMap[job.Job.Name] = fmt.Sprintf("DIED %v", job.CommandKey)
		s.stateMutex.Unlock()
		s.scheduler.removeJob(job.CommandKey)
		job.State = pb.State_ACKNOWLEDGED
	}

	if job.State != stState {
		s.stateTime[job.Job.Name] = time.Now()
	}

	if job.State == pb.State_DIED {
	}

	utils.SendTrace(ctx, fmt.Sprintf("end_transition_%v_%v", job.State, stState), time.Now(), pbt.Milestone_MARKER, job.Job.Name)
	utils.SendTrace(ctx, fmt.Sprintf("end_transition_func_%v", job.State), time.Now(), pbt.Milestone_MARKER, job.Job.Name)
}

type translator interface {
	build(job *pb.Job) *exec.Cmd
	run(job *pb.Job) *exec.Cmd
}

type checker interface {
	isAlive(ctx context.Context, job *pb.JobAssignment) bool
}

func (s *Server) getVersion(ctx context.Context, job *pb.Job) (*pbb.Version, error) {
	versions, err := s.builder.build(ctx, job)
	if err != nil {
		return &pbb.Version{}, err
	}

	if len(versions) == 0 {
		return &pbb.Version{}, nil
	}

	return versions[0], nil

}

// scheduleBuild builds out the job, returning the current version
func (s *Server) scheduleBuild(ctx context.Context, job *pb.Job) string {
	utils.SendTrace(ctx, fmt.Sprintf("schedule_build_%v", job.Bootstrap), time.Now(), pbt.Milestone_MARKER, job.Name)
	if job.Bootstrap {
		c := s.translator.build(job)
		return s.scheduler.Schedule(&rCommand{command: c})
	}

	versions, err := s.builder.build(ctx, job)

	s.lastCopyStatus = fmt.Sprintf("%v", err)
	if len(versions) == 0 {
		s.Log(fmt.Sprintf("No versions for %v because %v", job.Name, err))
		s.stateMutex.Lock()
		s.stateMap[job.Name] = fmt.Sprintf("No Versions: %v", err)
		s.stateMutex.Unlock()
		return ""
	}

	t := time.Now()

	//Only copy if the latest version is different to the local version
	v, ok := s.versions[job.Name]
	if !ok || v.Version != versions[0].Version {
		s.Log(fmt.Sprintf("We're okay %v (%v) given %v", ok, job.Name, reflect.ValueOf(s.versions).MapKeys()))
		s.Log(fmt.Sprintf("Requesting copy %v and %v", v, versions[0].Version))
		s.Log(fmt.Sprintf("Got %v and %v", v, versions[0]))
		s.copies++

		err = s.builder.copy(ctx, versions[0])
		s.lastCopyTime = time.Now().Sub(t)
		s.lastCopyStatus = fmt.Sprintf("%v", err)
		if err != nil {
			s.stateMutex.Lock()
			s.stateMap[job.Name] = fmt.Sprintf("Copy fail (%v) -> %v", time.Now().Sub(t), err)
			s.stateMutex.Unlock()
			return ""
		}
		s.stateMutex.Lock()
		s.stateMap[job.Name] = fmt.Sprintf("Copied version %v", versions[0].Version)
		s.stateMutex.Unlock()

		//Save the version file alongside the binary
		data, _ := proto.Marshal(versions[0])
		ioutil.WriteFile("/home/simon/gobuild/bin/"+job.Name+".version", data, 0644)
	} else {
		s.skippedCopies++
	}

	return versions[0].Version
}

func (s *Server) scheduleRun(job *pb.Job) string {
	c := s.translator.run(job)
	return s.scheduler.Schedule(&rCommand{command: c})
}

func (s *Server) taskComplete(key string) bool {
	return s.scheduler.schedulerComplete(key)
}
<commit_msg>Closes off copy work<commit_after>package main

import (
	"fmt"
	"io/ioutil"
	"os/exec"
	"time"

	pbb "github.com/brotherlogic/buildserver/proto"
	pb "github.com/brotherlogic/gobuildslave/proto"
	"github.com/brotherlogic/goserver/utils"
	pbt "github.com/brotherlogic/tracer/proto"
	"github.com/golang/protobuf/proto"
	"golang.org/x/net/context"
)

const (
	pendWait = time.Minute
)

func (s *Server) runTransition(ctx context.Context, job *pb.JobAssignment) {
	utils.SendTrace(ctx, fmt.Sprintf("run_transition_%v", job.State), time.Now(), pbt.Milestone_MARKER, job.Job.Name)
	stState := job.State
	switch job.State {
	case pb.State_ACKNOWLEDGED:
		key := s.scheduleBuild(ctx, job.Job)
		s.stateMutex.Lock()
		s.stateMap[job.Job.Name] = fmt.Sprintf("SCHED: %v @ %v", key, time.Now())
		s.stateMutex.Unlock()
		if !job.Job.Bootstrap {
			if key != "" {
				job.Server = s.Registry.Identifier
				job.State = pb.State_BUILT
				job.RunningVersion = key
			}
		} else {
			job.CommandKey = key
			job.State = pb.State_BUILDING
			job.Server = s.Registry.Identifier
		}
	case pb.State_BUILDING:
		s.stateMutex.Lock()
		s.stateMap[job.Job.Name] = fmt.Sprintf("BUILD(%v): %v", job.CommandKey, s.scheduler.getState(job.CommandKey))
		s.stateMutex.Unlock()
		if s.taskComplete(job.CommandKey) {
			job.State = pb.State_BUILT
		}
	case pb.State_BUILT:
		output := s.scheduler.getOutput(job.CommandKey)
		s.stateMutex.Lock()
		s.stateMap[job.Job.Name] = fmt.Sprintf("BUILT(%v): (%v): %v", job.CommandKey, len(output), output)
		s.stateMutex.Unlock()
		if job.Job.Bootstrap && len(output) > 0 {
			if job.BuildFail == 5 {
				s.deliverCrashReport(ctx, job, output)
				job.BuildFail = 0
			}
			job.BuildFail++
			job.State = pb.State_DIED
		} else {
			job.BuildFail = 0
			key := s.scheduleRun(job.Job)
			job.CommandKey = key
			job.StartTime = time.Now().Unix()
			job.State = pb.State_PENDING
			if _, ok := s.pendingMap[time.Now().Weekday()]; !ok {
				s.pendingMap[time.Now().Weekday()] = make(map[string]int)
			}
			s.pendingMap[time.Now().Weekday()][job.Job.Name]++
		}
	case pb.State_PENDING:
		s.stateMutex.Lock()
		s.stateMap[job.Job.Name] = fmt.Sprintf("OUTPUT = %v", s.scheduler.getOutput(job.CommandKey))
		s.stateMutex.Unlock()
		if time.Now().Add(-time.Minute).Unix() > job.StartTime {
			job.State = pb.State_RUNNING
		}
	case pb.State_RUNNING:
		output := s.scheduler.getOutput(job.CommandKey)
		s.stateMutex.Lock()
		s.stateMap[job.Job.Name] = fmt.Sprintf("ROUTPUT = %v, %v", s.scheduler.getOutput(job.CommandKey), s.scheduler.getStatus(job.CommandKey))
		job.Status = s.scheduler.getStatus(job.CommandKey)
		s.stateMutex.Unlock()
		if s.taskComplete(job.CommandKey) {
			s.stateMutex.Lock()
			s.stateMap[job.Job.Name] = fmt.Sprintf("COMPLETE = %v", output)
			s.stateMutex.Unlock()
			s.deliverCrashReport(ctx, job, output)
			job.State = pb.State_DIED
		}

		err := s.discover.discover(job.Job.Name, s.Registry.Identifier)
		if err != nil {
			if job.DiscoverCount > 30 {
				s.RaiseIssue(ctx, "Cannot Discover Running Server", fmt.Sprintf("%v on %v is not discoverable, despite running (%v) the output says %v", job.Job.Name, s.Registry.Identifier, err, output), false)
			}
			job.DiscoverCount++
		} else {
			job.DiscoverCount = 0
		}

		// Restart this job if we need to
		if !job.Job.Bootstrap {
			version, err := s.getVersion(ctx, job.Job)
			if err == nil && version.Version != job.RunningVersion {
				s.versions[job.Job.Name] = version
				s.stateMutex.Lock()
				s.stateMap[job.Job.Name] = fmt.Sprintf("VERSION_MISMATCH = %v,%v", version, job.RunningVersion)
				s.stateMutex.Unlock()
				s.scheduler.killJob(job.CommandKey)
				job.State = pb.State_ACKNOWLEDGED
			}
		}
	case pb.State_DIED:
		s.stateMutex.Lock()
		s.stateMap[job.Job.Name] = fmt.Sprintf("DIED %v", job.CommandKey)
		s.stateMutex.Unlock()
		s.scheduler.removeJob(job.CommandKey)
		job.State = pb.State_ACKNOWLEDGED
	}

	if job.State != stState {
		s.stateTime[job.Job.Name] = time.Now()
	}

	if job.State == pb.State_DIED {
	}

	utils.SendTrace(ctx, fmt.Sprintf("end_transition_%v_%v", job.State, stState), time.Now(), pbt.Milestone_MARKER, job.Job.Name)
	utils.SendTrace(ctx, fmt.Sprintf("end_transition_func_%v", job.State), time.Now(), pbt.Milestone_MARKER, job.Job.Name)
}

type translator interface {
	build(job *pb.Job) *exec.Cmd
	run(job *pb.Job) *exec.Cmd
}

type checker interface {
	isAlive(ctx context.Context, job *pb.JobAssignment) bool
}

func (s *Server) getVersion(ctx context.Context, job *pb.Job) (*pbb.Version, error) {
	versions, err := s.builder.build(ctx, job)
	if err != nil {
		return &pbb.Version{}, err
	}

	if len(versions) == 0 {
		return &pbb.Version{}, nil
	}

	return versions[0], nil

}

// scheduleBuild builds out the job, returning the current version
func (s *Server) scheduleBuild(ctx context.Context, job *pb.Job) string {
	utils.SendTrace(ctx, fmt.Sprintf("schedule_build_%v", job.Bootstrap), time.Now(), pbt.Milestone_MARKER, job.Name)
	if job.Bootstrap {
		c := s.translator.build(job)
		return s.scheduler.Schedule(&rCommand{command: c})
	}

	versions, err := s.builder.build(ctx, job)

	s.lastCopyStatus = fmt.Sprintf("%v", err)
	if len(versions) == 0 {
		s.Log(fmt.Sprintf("No versions for %v because %v", job.Name, err))
		s.stateMutex.Lock()
		s.stateMap[job.Name] = fmt.Sprintf("No Versions: %v", err)
		s.stateMutex.Unlock()
		return ""
	}

	t := time.Now()

	//Only copy if the latest version is different to the local version
	v, ok := s.versions[job.Name]
	if !ok || v.Version != versions[0].Version {
		s.copies++

		err = s.builder.copy(ctx, versions[0])
		s.lastCopyTime = time.Now().Sub(t)
		s.lastCopyStatus = fmt.Sprintf("%v", err)
		if err != nil {
			s.stateMutex.Lock()
			s.stateMap[job.Name] = fmt.Sprintf("Copy fail (%v) -> %v", time.Now().Sub(t), err)
			s.stateMutex.Unlock()
			return ""
		}
		s.stateMutex.Lock()
		s.stateMap[job.Name] = fmt.Sprintf("Copied version %v", versions[0].Version)
		s.stateMutex.Unlock()

		//Save the version file alongside the binary
		data, _ := proto.Marshal(versions[0])
		ioutil.WriteFile("/home/simon/gobuild/bin/"+job.Name+".version", data, 0644)
	} else {
		s.skippedCopies++
	}

	return versions[0].Version
}

func (s *Server) scheduleRun(job *pb.Job) string {
	c := s.translator.run(job)
	return s.scheduler.Schedule(&rCommand{command: c})
}

func (s *Server) taskComplete(key string) bool {
	return s.scheduler.schedulerComplete(key)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os/exec"
	"time"

	pb "github.com/brotherlogic/gobuildslave/proto"
	"github.com/brotherlogic/goserver/utils"
	pbt "github.com/brotherlogic/tracer/proto"
	"golang.org/x/net/context"
)

const (
	pendWait = time.Minute
)

func (s *Server) runTransition(ctx context.Context, job *pb.JobAssignment) {
	utils.SendTrace(ctx, fmt.Sprintf("run_transition_%v", job.State), time.Now(), pbt.Milestone_MARKER, job.Job.Name)
	stState := job.State
	switch job.State {
	case pb.State_ACKNOWLEDGED:
		key := s.scheduleBuild(ctx, job.Job)
		s.stateMap[job.Job.Name] = fmt.Sprintf("SCHED: %v", key)
		if job.Job.NonBootstrap {
			if key != "" {
				job.Server = s.Registry.Identifier
				job.State = pb.State_BUILT
				job.RunningVersion = key
			}
		} else {
			job.CommandKey = key
			job.State = pb.State_BUILDING
			job.Server = s.Registry.Identifier
		}
	case pb.State_BUILDING:
		if s.taskComplete(job.CommandKey) {
			job.State = pb.State_BUILT
		}
	case pb.State_BUILT:
		output := s.scheduler.getOutput(job.CommandKey)
		if len(output) > 0 {
			if job.BuildFail == 5 {
				s.deliverCrashReport(ctx, job, output)
			}
			job.BuildFail++
			job.State = pb.State_DIED
		} else {
			job.BuildFail = 0
			key := s.scheduleRun(job.Job)
			job.CommandKey = key
			job.StartTime = time.Now().Unix()
			job.State = pb.State_PENDING
			if _, ok := s.pendingMap[time.Now().Weekday()]; !ok {
				s.pendingMap[time.Now().Weekday()] = make(map[string]int)
			}
			s.pendingMap[time.Now().Weekday()][job.Job.Name]++
		}
	case pb.State_PENDING:
		s.stateMap[job.Job.Name] = fmt.Sprintf("OUTPUT = %v", s.scheduler.getOutput(job.CommandKey))
		if time.Now().Add(-time.Minute).Unix() > job.StartTime {
			job.State = pb.State_RUNNING
		}
	case pb.State_RUNNING:
		if s.taskComplete(job.CommandKey) {
			output := s.scheduler.getOutput(job.CommandKey)
			s.stateMap[job.Job.Name] = fmt.Sprintf("COMPLETE = %v", output)
			s.deliverCrashReport(ctx, job, output)
			job.State = pb.State_DIED
		}

		if s.discover.discover(job.Job.Name, s.Registry.Identifier) != nil {
			s.RaiseIssue(ctx, "Cannot Discover Running Server", fmt.Sprintf("%v on %v is not discoverable, despite running", job.Job.Name, s.Registry.Identifier), false)
		}

		// Restart this job if we need to
		if job.Job.NonBootstrap {
			version := s.getVersion(ctx, job.Job)
			if version != job.RunningVersion {
				s.stateMap[job.Job.Name] = fmt.Sprintf("VERSION_MISMATCH = %v,%v", version, job.RunningVersion)
				s.scheduler.killJob(job.CommandKey)
				job.State = pb.State_ACKNOWLEDGED
			}
		}
	case pb.State_DIED:
		job.State = pb.State_ACKNOWLEDGED
	}

	if job.State != stState {
		s.stateTime[job.Job.Name] = time.Now()
	}

	if job.State == pb.State_DIED {
		s.RaiseIssue(ctx, "Job Died", fmt.Sprintf("Job %v has died", job.Job.Name), false)
	}

	utils.SendTrace(ctx, fmt.Sprintf("end_transition_%v_%v", job.State, stState), time.Now(), pbt.Milestone_MARKER, job.Job.Name)
	utils.SendTrace(ctx, fmt.Sprintf("end_transition_func_%v", job.State), time.Now(), pbt.Milestone_MARKER, job.Job.Name)
}

type translator interface {
	build(job *pb.Job) *exec.Cmd
	run(job *pb.Job) *exec.Cmd
}

type checker interface {
	isAlive(ctx context.Context, job *pb.JobAssignment) bool
}

func (s *Server) getVersion(ctx context.Context, job *pb.Job) string {
	versions, _ := s.builder.build(ctx, job)

	if len(versions) == 0 {
		return ""
	}

	return versions[0].Version

}

// scheduleBuild builds out the job, returning the current version
func (s *Server) scheduleBuild(ctx context.Context, job *pb.Job) string {
	utils.SendTrace(ctx, fmt.Sprintf("schedule_build_%v", job.NonBootstrap), time.Now(), pbt.Milestone_MARKER, job.Name)
	if !job.NonBootstrap {
		c := s.translator.build(job)
		return s.scheduler.Schedule(&rCommand{command: c})
	}

	versions, err := s.builder.build(ctx, job)

	if len(versions) == 0 {
		s.stateMap[job.Name] = fmt.Sprintf("No Versions: %v", err)
		return ""
	}

	t := time.Now()
	err = s.builder.copy(ctx, versions[0])
	if err != nil {
		s.stateMap[job.Name] = fmt.Sprintf("Copy fail (%v) -> %v", time.Now().Sub(t), err)
		return ""
	}
	s.stateMap[job.Name] = fmt.Sprintf("Found version %v", versions[0].Version)
	return versions[0].Version
}

func (s *Server) scheduleRun(job *pb.Job) string {
	c := s.translator.run(job)
	return s.scheduler.Schedule(&rCommand{command: c})
}

func (s *Server) taskComplete(key string) bool {
	return s.scheduler.schedulerComplete(key)
}
<commit_msg>Better monitoring<commit_after>package main

import (
	"fmt"
	"os/exec"
	"time"

	pb "github.com/brotherlogic/gobuildslave/proto"
	"github.com/brotherlogic/goserver/utils"
	pbt "github.com/brotherlogic/tracer/proto"
	"golang.org/x/net/context"
)

const (
	pendWait = time.Minute
)

func (s *Server) runTransition(ctx context.Context, job *pb.JobAssignment) {
	utils.SendTrace(ctx, fmt.Sprintf("run_transition_%v", job.State), time.Now(), pbt.Milestone_MARKER, job.Job.Name)
	stState := job.State
	switch job.State {
	case pb.State_ACKNOWLEDGED:
		key := s.scheduleBuild(ctx, job.Job)
		s.stateMap[job.Job.Name] = fmt.Sprintf("SCHED: %v", key)
		if job.Job.NonBootstrap {
			if key != "" {
				job.Server = s.Registry.Identifier
				job.State = pb.State_BUILT
				job.RunningVersion = key
			}
		} else {
			job.CommandKey = key
			job.State = pb.State_BUILDING
			job.Server = s.Registry.Identifier
		}
	case pb.State_BUILDING:
		if s.taskComplete(job.CommandKey) {
			job.State = pb.State_BUILT
		}
	case pb.State_BUILT:
		output := s.scheduler.getOutput(job.CommandKey)
		if len(output) > 0 {
			if job.BuildFail == 5 {
				s.deliverCrashReport(ctx, job, output)
			}
			job.BuildFail++
			job.State = pb.State_DIED
		} else {
			job.BuildFail = 0
			key := s.scheduleRun(job.Job)
			job.CommandKey = key
			job.StartTime = time.Now().Unix()
			job.State = pb.State_PENDING
			if _, ok := s.pendingMap[time.Now().Weekday()]; !ok {
				s.pendingMap[time.Now().Weekday()] = make(map[string]int)
			}
			s.pendingMap[time.Now().Weekday()][job.Job.Name]++
		}
	case pb.State_PENDING:
		s.stateMap[job.Job.Name] = fmt.Sprintf("OUTPUT = %v", s.scheduler.getOutput(job.CommandKey))
		if time.Now().Add(-time.Minute).Unix() > job.StartTime {
			job.State = pb.State_RUNNING
		}
	case pb.State_RUNNING:
		if s.taskComplete(job.CommandKey) {
			output := s.scheduler.getOutput(job.CommandKey)
			s.stateMap[job.Job.Name] = fmt.Sprintf("COMPLETE = %v", output)
			s.deliverCrashReport(ctx, job, output)
			job.State = pb.State_DIED
		}

		if s.discover.discover(job.Job.Name, s.Registry.Identifier) != nil {
			s.RaiseIssue(ctx, "Cannot Discover Running Server", fmt.Sprintf("%v on %v is not discoverable, despite running", job.Job.Name, s.Registry.Identifier), false)
		}

		// Restart this job if we need to
		if job.Job.NonBootstrap {
			version := s.getVersion(ctx, job.Job)
			if version != job.RunningVersion {
				s.stateMap[job.Job.Name] = fmt.Sprintf("VERSION_MISMATCH = %v,%v", version, job.RunningVersion)
				s.scheduler.killJob(job.CommandKey)
				job.State = pb.State_ACKNOWLEDGED
			}
		}
	case pb.State_DIED:
		job.State = pb.State_ACKNOWLEDGED
	}

	if job.State != stState {
		s.stateTime[job.Job.Name] = time.Now()
	}

	if job.State == pb.State_DIED {
		s.RaiseIssue(ctx, "Job Died", fmt.Sprintf("Job %v has died %v", job.Job.Name, stState), false)
	}

	utils.SendTrace(ctx, fmt.Sprintf("end_transition_%v_%v", job.State, stState), time.Now(), pbt.Milestone_MARKER, job.Job.Name)
	utils.SendTrace(ctx, fmt.Sprintf("end_transition_func_%v", job.State), time.Now(), pbt.Milestone_MARKER, job.Job.Name)
}

type translator interface {
	build(job *pb.Job) *exec.Cmd
	run(job *pb.Job) *exec.Cmd
}

type checker interface {
	isAlive(ctx context.Context, job *pb.JobAssignment) bool
}

func (s *Server) getVersion(ctx context.Context, job *pb.Job) string {
	versions, _ := s.builder.build(ctx, job)

	if len(versions) == 0 {
		return ""
	}

	return versions[0].Version

}

// scheduleBuild builds out the job, returning the current version
func (s *Server) scheduleBuild(ctx context.Context, job *pb.Job) string {
	utils.SendTrace(ctx, fmt.Sprintf("schedule_build_%v", job.NonBootstrap), time.Now(), pbt.Milestone_MARKER, job.Name)
	if !job.NonBootstrap {
		c := s.translator.build(job)
		return s.scheduler.Schedule(&rCommand{command: c})
	}

	versions, err := s.builder.build(ctx, job)

	if len(versions) == 0 {
		s.stateMap[job.Name] = fmt.Sprintf("No Versions: %v", err)
		return ""
	}

	t := time.Now()
	err = s.builder.copy(ctx, versions[0])
	if err != nil {
		s.stateMap[job.Name] = fmt.Sprintf("Copy fail (%v) -> %v", time.Now().Sub(t), err)
		return ""
	}
	s.stateMap[job.Name] = fmt.Sprintf("Found version %v", versions[0].Version)
	return versions[0].Version
}

func (s *Server) scheduleRun(job *pb.Job) string {
	c := s.translator.run(job)
	return s.scheduler.Schedule(&rCommand{command: c})
}

func (s *Server) taskComplete(key string) bool {
	return s.scheduler.schedulerComplete(key)
}
<|endoftext|>


<commit_before>package main

import(
	"bufio"
	"fmt"
  "log"
	"os"
  "strconv"
  "strings"
  "time"
)

type FlightInfo struct {
  airline string
  dep_time string
  arr_delay int
}

func (fi FlightInfo) String() string {
  return fi.airline + " " + fi.dep_time + " " + strconv.FormatInt(int64(fi.arr_delay), 10)
}

// key: src_dst_date_morning
var best_flights_of_day map[string]FlightInfo

func g3_q2_reducer_main() {
  best_flights_of_day = make(map[string]FlightInfo)
  cass := setup_cassandra()
  scanner := bufio.NewScanner(os.Stdin)
  for scanner.Scan() {
    line := scanner.Text()
    toks := strings.Split(line, "\t")
    if len(toks) != 2 {
      log.Fatal(line)
    }
    src_dst_date_morning := strings.TrimSpace(toks[0])
    flight_info := strings.TrimSpace(toks[1])
    val_toks := strings.Fields(flight_info)
    if len(val_toks) != 3 {
      log.Fatal("unexpected val format: ", line)
      return
    }
    delay_str := val_toks[2]

    delay, err := strconv.Atoi(delay_str)
    if err != nil {
      log.Fatal("unexpected delay format: ", line)
      return
    }
    prev_best, pres := best_flights_of_day[src_dst_date_morning]
    if !pres || prev_best.arr_delay > delay {
      best_flights_of_day[src_dst_date_morning] = FlightInfo {airline: val_toks[0], dep_time: val_toks[1], arr_delay: delay}
    }
  }
  for src_dst_date_morning, f_info := range best_flights_of_day {
    key_toks := strings.Split(src_dst_date_morning, "_")
    if len(key_toks) != 4 {
      log.Fatal("unexpected key format: ", src_dst_date_morning)
      return
    }
    src := key_toks[0]
    dst := key_toks[1]
    date := key_toks[2]
    morning_str := key_toks[3]
    morning := morning_str == "true"
    if !morning && morning_str != "false" {
      log.Fatal("unexpected bool format: ", src_dst_date_morning)
      return
    }
    if cass == nil {
      fmt.Println(src, dst, date, morning, f_info)
      continue
    }
    date_t, err := time.Parse("2006-01-02", date)
    if err != nil {
      log.Fatal(err, date)
    }
    query := "INSERT INTO src_dst_best_flights_of_day (src, dst, date, morning, UniqueCarrier, CRSDepTime, ArrDelay) VALUES (?, ?, ?, ?, ?, ?, ?)"
    if err := cass.Query(query,
      src, dst, date_t, morning, f_info.airline, f_info.dep_time, f_info.arr_delay).Exec(); err != nil {
      log.Fatal(err, date)
    }
  }
}
<commit_msg>g3 q2 reducer: optimize for memory<commit_after>package main

import(
	"bufio"
	"fmt"
  "github.com/gocql/gocql"
  "log"
	"os"
  "strconv"
  "strings"
  "time"
)

type FlightInfo struct {
  airline string
  dep_time string
  arr_delay int
}

func (fi FlightInfo) String() string {
  return fi.airline + " " + fi.dep_time + " " + strconv.FormatInt(int64(fi.arr_delay), 10)
}

// key: src_dst_date_morning
var cur_src_dst_date_morning string
var best_flight FlightInfo

func g3_q2_reducer_main() {
  cass := setup_cassandra()
  scanner := bufio.NewScanner(os.Stdin)
  for scanner.Scan() {
    line := scanner.Text()
    toks := strings.Split(line, "\t")
    if len(toks) != 2 {
      log.Fatal(line)
    }
    src_dst_date_morning := strings.TrimSpace(toks[0])
    flight_info := strings.TrimSpace(toks[1])
    val_toks := strings.Fields(flight_info)
    if len(val_toks) != 3 {
      log.Fatal("unexpected val format: ", line)
      return
    }
    delay_str := val_toks[2]
    delay, err := strconv.Atoi(delay_str)
    if err != nil {
      log.Fatal("unexpected delay format: ", line)
      return
    }

    if src_dst_date_morning != cur_src_dst_date_morning && len(cur_src_dst_date_morning) > 0 {
      best_flights_of_day_publish(cass)
      best_flight = FlightInfo {airline: "", dep_time: "", arr_delay: 1<<30}
    }
    cur_src_dst_date_morning = src_dst_date_morning

    if best_flight.airline == "" || best_flight.arr_delay > delay {
      best_flight = FlightInfo {airline: val_toks[0], dep_time: val_toks[1], arr_delay: delay}
    }
  }
  best_flights_of_day_publish(cass)
}

func best_flights_of_day_publish(cass *gocql.Session) {
  key_toks := strings.Split(cur_src_dst_date_morning, "_")
  if len(key_toks) != 4 {
    log.Fatal("unexpected key format: ", cur_src_dst_date_morning)
    return
  }
  src := key_toks[0]
  dst := key_toks[1]
  date := key_toks[2]
  morning_str := key_toks[3]
  morning := morning_str == "true"
  if !morning && morning_str != "false" {
    log.Fatal("unexpected bool format: ", cur_src_dst_date_morning)
    return
  }
  if cass == nil {
    fmt.Println(src, dst, date, morning, best_flight)
    return
  }
  date_t, err := time.Parse("2006-01-02", date)
  if err != nil {
    log.Fatal(err, date)
  }
  query := "INSERT INTO src_dst_best_flights_of_day (src, dst, date, morning, UniqueCarrier, CRSDepTime, ArrDelay) VALUES (?, ?, ?, ?, ?, ?, ?)"
  if err := cass.Query(query,
    src, dst, date_t, morning, best_flight.airline, best_flight.dep_time, best_flight.arr_delay).Exec(); err != nil {
    log.Fatal(err)
  }
}
<|endoftext|>


<commit_before>// Copyright 2017 The GoReporter Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package processbar

import (
	"log"
	"os"
	"runtime"
	"sync/atomic"
	"time"

	"github.com/reconquest/loreley"
	"github.com/wgliang/barely"
)

// LinterProcessBar provides a function to display the current progress of
// generating your project, you can predict the time to generate your report.
// And it will communicate with engine through chans(lintersProcessChans and
// lintersFinishedSignal).
func LinterProcessBar(lintersProcessChans chan int64, lintersFinishedSignal chan string) {
	// Set process bar corlor,for a better experience.
	format, err := loreley.CompileWithReset(
		` {bg 2}{fg 15}{bold} {.Mode} `+
			`{bg 253}{fg 0} `+
			`{if .Updated}{fg 70}{end}{.Done}{fg 0}/{.Total} `,
		nil,
	)
	if err != nil {
		log.Fatal(err)
	}
	// Initialize some of the parameters, set some of the properties of our progress bar.
	var (
		bar = barely.NewStatusBar(format.Template)

		status = &struct {
			Mode  string
			Total int
			Done  int64

			Updated int64
		}{
			Mode:  "GOREPORTER",
			Total: 100,
		}
	)

	bar.SetStatus(status)
	processCount := int64(100)

	if runtime.GOOS == "windows" {
	PROCESSNORENDER:
		for {
			select {
			case process := <-lintersProcessChans:
				atomic.AddInt64(&status.Done, process)
				atomic.AddInt64(&status.Updated, process)
				processCount = processCount - process
				if processCount <= int64(0) {
					break PROCESSNORENDER
				}
			case signal := <-lintersFinishedSignal:
				log.Println(signal)
			}
		}
	} else {
		bar.Render(os.Stderr)
	PROCESSRENDER:
		for {
			select {
			case process := <-lintersProcessChans:
				atomic.AddInt64(&status.Done, process)
				atomic.AddInt64(&status.Updated, process)
				bar.Render(os.Stderr)
				processCount = processCount - process
				if processCount <= int64(0) {
					break PROCESSRENDER
				}
			case signal := <-lintersFinishedSignal:
				log.Println(signal)
				bar.Render(os.Stderr)
			case <-time.After(1 * time.Second):
				bar.Render(os.Stderr)
			}
		}
	}

	bar.Clear(os.Stderr)
	return
}
<commit_msg>-Replace github.com/wgliang/barely with github.com/reconquest/barely<commit_after>// Copyright 2017 The GoReporter Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package processbar

import (
	"log"
	"os"
	"runtime"
	"sync/atomic"
	"time"

	"github.com/reconquest/loreley"
	"github.com/reconquest/barely"
)

// LinterProcessBar provides a function to display the current progress of
// generating your project, you can predict the time to generate your report.
// And it will communicate with engine through chans(lintersProcessChans and
// lintersFinishedSignal).
func LinterProcessBar(lintersProcessChans chan int64, lintersFinishedSignal chan string) {
	// Set process bar corlor,for a better experience.
	format, err := loreley.CompileWithReset(
		` {bg 2}{fg 15}{bold} {.Mode} `+
			`{bg 253}{fg 0} `+
			`{if .Updated}{fg 70}{end}{.Done}{fg 0}/{.Total} `,
		nil,
	)
	if err != nil {
		log.Fatal(err)
	}
	// Initialize some of the parameters, set some of the properties of our progress bar.
	var (
		bar = barely.NewStatusBar(format.Template)

		status = &struct {
			Mode  string
			Total int
			Done  int64

			Updated int64
		}{
			Mode:  "GOREPORTER",
			Total: 100,
		}
	)

	bar.SetStatus(status)
	processCount := int64(100)

	if runtime.GOOS == "windows" {
	PROCESSNORENDER:
		for {
			select {
			case process := <-lintersProcessChans:
				atomic.AddInt64(&status.Done, process)
				atomic.AddInt64(&status.Updated, process)
				processCount = processCount - process
				if processCount <= int64(0) {
					break PROCESSNORENDER
				}
			case signal := <-lintersFinishedSignal:
				log.Println(signal)
			}
		}
	} else {
		bar.Render(os.Stderr)
	PROCESSRENDER:
		for {
			select {
			case process := <-lintersProcessChans:
				atomic.AddInt64(&status.Done, process)
				atomic.AddInt64(&status.Updated, process)
				bar.Render(os.Stderr)
				processCount = processCount - process
				if processCount <= int64(0) {
					break PROCESSRENDER
				}
			case signal := <-lintersFinishedSignal:
				log.Println(signal)
				bar.Render(os.Stderr)
			case <-time.After(1 * time.Second):
				bar.Render(os.Stderr)
			}
		}
	}

	bar.Clear(os.Stderr)
	return
}
<|endoftext|>


<commit_before>//  Copyright 2016 The goscope Authors
//
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the License.
//  You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
//  Unless required by applicable law or agreed to in writing, software
//  distributed under the License is distributed on an "AS IS" BASIS,
//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//  See the License for the specific language governing permissions and
//  limitations under the License.

package gui

import (
	"fmt"

	"github.com/mjibson/go-dsp/fft"
	"github.com/zagrodzki/goscope/scope"
)

type interpMethod interface {
	interpolate(samples []scope.Sample, size int) ([]scope.Sample, error)
}

type constInterpolation struct{}

func (interp constInterpolation) interpolate(samples []scope.Sample, size int) ([]scope.Sample, error) {
	if err := checkSizes(len(samples), size); err != nil {
		return nil, err
	}
	if len(samples) == 1 {
		return constFunction(size, samples[0]), nil
	}

	interpSamples := make([]scope.Sample, size)
	interval := float64(size-1) / float64(len(samples)-1)
	lastIndex := 0
	lastInterp := 0.0
	nextInterp := interval
	for i, _ := range interpSamples {
		fi := float64(i)
		if fi-lastInterp > nextInterp-fi {
			lastIndex++
			lastInterp = float64(lastIndex) * interval
			nextInterp = float64(lastIndex+1) * interval
		}
		interpSamples[i] = samples[lastIndex]
	}
	return interpSamples, nil
}

type linearInterpolation struct {
}

func (interp linearInterpolation) interpolate(samples []scope.Sample, size int) ([]scope.Sample, error) {
	if err := checkSizes(len(samples), size); err != nil {
		return nil, err
	}
	if len(samples) == 1 {
		return constFunction(size, samples[0]), nil
	}

	interpSamples := make([]scope.Sample, size)
	interval := float64(size-1) / float64(len(samples)-1)
	lastIndex := 0
	lastInterp := 0.0
	nextInterp := interval
	a := float64(samples[lastIndex+1]-samples[lastIndex]) / (nextInterp - lastInterp)
	b := float64(samples[lastIndex]) - a*lastInterp
	for i, _ := range interpSamples {
		if float64(i) > nextInterp {
			lastIndex++
			lastInterp = float64(lastIndex) * interval
			nextInterp = float64(lastIndex+1) * interval
			a = float64(samples[lastIndex+1]-samples[lastIndex]) / (nextInterp - lastInterp)
			b = float64(samples[lastIndex]) - a*lastInterp
		}
		interpSamples[i] = scope.Sample(a*float64(i) + b)
	}
	return interpSamples, nil
}

type sincInterpolation struct {
}

func (interp sincInterpolation) interpolate(samples []scope.Sample, size int) ([]scope.Sample, error) {
	if err := checkSizes(len(samples), size); err != nil {
		return nil, err
	}

	floatSamples := make([]float64, len(samples))
	for i, s := range samples {
		floatSamples[i] = float64(s)
	}
	frequencies := fft.FFTReal(floatSamples)
	freqInterp := make([]complex128, size)
	freqSize := len(frequencies)
	cmplxFreqSize := complex(float64(freqSize), 0)
	for i := 0; i < freqSize/2; i++ {
		freqInterp[i] = frequencies[i] / cmplxFreqSize
		freqInterp[len(freqInterp)-1-i] = frequencies[freqSize-1-i] / cmplxFreqSize
	}
	cmplxInterpSamples := fft.IFFT(freqInterp)

	interpSamples := make([]scope.Sample, size)
	floatSize := float64(size)
	for i, s := range cmplxInterpSamples {
		interpSamples[i] = scope.Sample(floatSize * real(s))
	}
	return interpSamples, nil
}

func checkSizes(samplesSize, requestedSize int) error {
	if samplesSize == 0 {
		return fmt.Errorf("input samples should contain at least one element")
	}
	if samplesSize >= requestedSize {
		return fmt.Errorf("requested samples size (%v) is less or equal than input samples size (%v)", requestedSize, samplesSize)
	}
	return nil
}

func constFunction(size int, value scope.Sample) []scope.Sample {
	samples := make([]scope.Sample, size)
	for i, _ := range samples {
		samples[i] = value
	}
	return samples
}
<commit_msg>2nd value from range removed<commit_after>//  Copyright 2016 The goscope Authors
//
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the License.
//  You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
//  Unless required by applicable law or agreed to in writing, software
//  distributed under the License is distributed on an "AS IS" BASIS,
//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//  See the License for the specific language governing permissions and
//  limitations under the License.

package gui

import (
	"fmt"

	"github.com/mjibson/go-dsp/fft"
	"github.com/zagrodzki/goscope/scope"
)

type interpMethod interface {
	interpolate(samples []scope.Sample, size int) ([]scope.Sample, error)
}

type constInterpolation struct{}

func (interp constInterpolation) interpolate(samples []scope.Sample, size int) ([]scope.Sample, error) {
	if err := checkSizes(len(samples), size); err != nil {
		return nil, err
	}
	if len(samples) == 1 {
		return constFunction(size, samples[0]), nil
	}

	interpSamples := make([]scope.Sample, size)
	interval := float64(size-1) / float64(len(samples)-1)
	lastIndex := 0
	lastInterp := 0.0
	nextInterp := interval
	for i := range interpSamples {
		fi := float64(i)
		if fi-lastInterp > nextInterp-fi {
			lastIndex++
			lastInterp = float64(lastIndex) * interval
			nextInterp = float64(lastIndex+1) * interval
		}
		interpSamples[i] = samples[lastIndex]
	}
	return interpSamples, nil
}

type linearInterpolation struct {
}

func (interp linearInterpolation) interpolate(samples []scope.Sample, size int) ([]scope.Sample, error) {
	if err := checkSizes(len(samples), size); err != nil {
		return nil, err
	}
	if len(samples) == 1 {
		return constFunction(size, samples[0]), nil
	}

	interpSamples := make([]scope.Sample, size)
	interval := float64(size-1) / float64(len(samples)-1)
	lastIndex := 0
	lastInterp := 0.0
	nextInterp := interval
	a := float64(samples[lastIndex+1]-samples[lastIndex]) / (nextInterp - lastInterp)
	b := float64(samples[lastIndex]) - a*lastInterp
	for i := range interpSamples {
		if float64(i) > nextInterp {
			lastIndex++
			lastInterp = float64(lastIndex) * interval
			nextInterp = float64(lastIndex+1) * interval
			a = float64(samples[lastIndex+1]-samples[lastIndex]) / (nextInterp - lastInterp)
			b = float64(samples[lastIndex]) - a*lastInterp
		}
		interpSamples[i] = scope.Sample(a*float64(i) + b)
	}
	return interpSamples, nil
}

type sincInterpolation struct {
}

func (interp sincInterpolation) interpolate(samples []scope.Sample, size int) ([]scope.Sample, error) {
	if err := checkSizes(len(samples), size); err != nil {
		return nil, err
	}

	floatSamples := make([]float64, len(samples))
	for i, s := range samples {
		floatSamples[i] = float64(s)
	}
	frequencies := fft.FFTReal(floatSamples)
	freqInterp := make([]complex128, size)
	freqSize := len(frequencies)
	cmplxFreqSize := complex(float64(freqSize), 0)
	for i := 0; i < freqSize/2; i++ {
		freqInterp[i] = frequencies[i] / cmplxFreqSize
		freqInterp[len(freqInterp)-1-i] = frequencies[freqSize-1-i] / cmplxFreqSize
	}
	cmplxInterpSamples := fft.IFFT(freqInterp)

	interpSamples := make([]scope.Sample, size)
	floatSize := float64(size)
	for i, s := range cmplxInterpSamples {
		interpSamples[i] = scope.Sample(floatSize * real(s))
	}
	return interpSamples, nil
}

func checkSizes(samplesSize, requestedSize int) error {
	if samplesSize == 0 {
		return fmt.Errorf("input samples should contain at least one element")
	}
	if samplesSize >= requestedSize {
		return fmt.Errorf("requested samples size (%v) is less or equal than input samples size (%v)", requestedSize, samplesSize)
	}
	return nil
}

func constFunction(size int, value scope.Sample) []scope.Sample {
	samples := make([]scope.Sample, size)
	for i := range samples {
		samples[i] = value
	}
	return samples
}
<|endoftext|>


<commit_before>// Copyright (c) 2015-2016, NVIDIA CORPORATION. All rights reserved.

package main

import (
	"errors"
	"fmt"
	"log"
	"net/url"
	"os"
	"regexp"
	"strings"

	"github.com/NVIDIA/nvidia-docker/tools/src/docker"
)

const (
	envDockerHost = "DOCKER_HOST"
	envNVDocker   = "NV_DOCKER"
	envNVHost     = "NV_HOST"
	envNVGPU      = "NV_GPU"
)

var ErrInvalidURI = errors.New("invalid remote host URI")

func LoadEnvironment() (err error) {
	Host, err = getHost()
	if err != nil {
		return
	}

	GPU = getGPU()
	cmd := getDocker()
	docker.SetCommand(cmd...)
	return
}

func parseAddr(addr string) (host, sport, hport string) {
	re := regexp.MustCompile(`^(\[[0-9a-f.:]+\]|[0-9A-Za-z.\-_]+)?(:\d+)?:(\d+)?$`)

	host, sport, hport = "localhost", "22", "3476"
	if addr == "" {
		return
	}
	m := re.FindStringSubmatch(addr)
	if m == nil {
		return "", "", ""
	}
	if m[1] != "" {
		host = m[1]
	}
	if m[2] != "" {
		sport = m[2][1:]
	}
	if m[3] != "" {
		hport = m[3]
	}
	return
}

func getHost() (*url.URL, error) {
	var env string

	nvhost := os.Getenv(envNVHost)
	dhost := os.Getenv(envDockerHost)

	if nvhost != "" {
		env = nvhost
	} else if dhost != "" {
		env = dhost
	} else {
		return nil, nil
	}

	if nvhost != "" && dhost == "" {
		log.Printf("Warning: %s is set but %s is not\n", envNVHost, envDockerHost)
	}

	if ok, _ := regexp.MatchString("^[a-z0-9+.-]+://", env); !ok {
		env = "tcp://" + env
	}
	uri, err := url.Parse(env)
	if err != nil {
		return nil, ErrInvalidURI
	}
	host, sport, hport := parseAddr(uri.Host)
	if host == "" {
		return nil, ErrInvalidURI
	}

	switch uri.Scheme {
	case "tcp":
		uri.Scheme = "http"
		fallthrough
	case "http":
		if nvhost == "" && dhost != "" {
			hport = "3476"
		}
		uri.Host = fmt.Sprintf("%s:%s", host, hport)
		return uri, nil
	case "ssh":
		uri.Host = fmt.Sprintf("%s:%s", host, sport)
		uri.Opaque = fmt.Sprintf("localhost:%s", hport)
		if uri.User == nil {
			uri.User = url.UserPassword(os.Getenv("USER"), "")
		}
		return uri, nil
	}

	return nil, ErrInvalidURI
}

func getGPU() []string {
	return strings.FieldsFunc(os.Getenv(envNVGPU), func(c rune) bool {
		return c == ' ' || c == ','
	})
}

func getDocker() []string {
	return strings.Fields(os.Getenv(envNVDocker))
}
<commit_msg>Support unix sockets for DOCKER_HOST<commit_after>// Copyright (c) 2015-2016, NVIDIA CORPORATION. All rights reserved.

package main

import (
	"errors"
	"fmt"
	"log"
	"net/url"
	"os"
	"regexp"
	"strings"

	"github.com/NVIDIA/nvidia-docker/tools/src/docker"
)

const (
	envDockerHost = "DOCKER_HOST"
	envNVDocker   = "NV_DOCKER"
	envNVHost     = "NV_HOST"
	envNVGPU      = "NV_GPU"
)

var ErrInvalidURI = errors.New("invalid remote host URI")

func LoadEnvironment() (err error) {
	Host, err = getHost()
	if err != nil {
		return
	}

	GPU = getGPU()
	cmd := getDocker()
	docker.SetCommand(cmd...)
	return
}

func parseAddr(addr string) (host, sport, hport string) {
	re := regexp.MustCompile(`^(\[[0-9a-f.:]+\]|[0-9A-Za-z.\-_]+)?(:\d+)?:(\d+)?$`)

	host, sport, hport = "localhost", "22", "3476"
	if addr == "" {
		return
	}
	m := re.FindStringSubmatch(addr)
	if m == nil {
		return "", "", ""
	}
	if m[1] != "" {
		host = m[1]
	}
	if m[2] != "" {
		sport = m[2][1:]
	}
	if m[3] != "" {
		hport = m[3]
	}
	return
}

func getHost() (*url.URL, error) {
	var env string

	nvhost := os.Getenv(envNVHost)
	dhost := os.Getenv(envDockerHost)

	if nvhost != "" {
		env = nvhost
	} else if dhost != "" {
		env = dhost
	} else {
		return nil, nil
	}

	if nvhost != "" && dhost == "" {
		log.Printf("Warning: %s is set but %s is not\n", envNVHost, envDockerHost)
	}

	if ok, _ := regexp.MatchString("^[a-z0-9+.-]+://", env); !ok {
		env = "tcp://" + env
	}
	uri, err := url.Parse(env)
	if err != nil {
		return nil, ErrInvalidURI
	}
	if uri.Scheme == "unix" {
		return nil, nil
	}

	host, sport, hport := parseAddr(uri.Host)
	if host == "" {
		return nil, ErrInvalidURI
	}

	switch uri.Scheme {
	case "tcp":
		uri.Scheme = "http"
		fallthrough
	case "http":
		if nvhost == "" && dhost != "" {
			hport = "3476"
		}
		uri.Host = fmt.Sprintf("%s:%s", host, hport)
		return uri, nil
	case "ssh":
		uri.Host = fmt.Sprintf("%s:%s", host, sport)
		uri.Opaque = fmt.Sprintf("localhost:%s", hport)
		if uri.User == nil {
			uri.User = url.UserPassword(os.Getenv("USER"), "")
		}
		return uri, nil
	}

	return nil, ErrInvalidURI
}

func getGPU() []string {
	return strings.FieldsFunc(os.Getenv(envNVGPU), func(c rune) bool {
		return c == ' ' || c == ','
	})
}

func getDocker() []string {
	return strings.Fields(os.Getenv(envNVDocker))
}
<|endoftext|>


<commit_before>// Copyright 2012 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package sdb

import (
	"errors"
	. "github.com/jacobsa/oglematchers"
	. "github.com/jacobsa/ogletest"
	"strings"
)

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

// A common helper class.
type simpleDBTest struct {
	c *fakeConn
	db SimpleDB
}

func (t *simpleDBTest) SetUp(i *TestInfo) {
	var err error

	t.c = &fakeConn{}

	t.db, err = newSimpleDB(t.c)
	AssertEq(nil, err)
}

type fakeDomain struct {
	name string
}

func (d *fakeDomain) Name() string {
	return d.name
}

func (d *fakeDomain) PutAttributes(
		item ItemName,
		updates []PutUpdate,
		preconditions []Precondition) error {
	panic("Unsupported")
}

func (d *fakeDomain) BatchPutAttributes(updateMap map[ItemName][]PutUpdate) error {
	panic("Unsupported")
}

func (d *fakeDomain) DeleteAttributes(
		item ItemName,
		deletes []DeleteUpdate,
		preconditions []Precondition) error {
	panic("Unsupported")
}

func (d *fakeDomain) BatchDeleteAttributes(deleteMap map[ItemName][]DeleteUpdate) error {
	panic("Unsupported")
}

func (d *fakeDomain) GetAttributes(
		item ItemName,
		constistentRead bool,
		attrNames []string) (attrs []Attribute, err error) {
	panic("Unsupported")
}

////////////////////////////////////////////////////////////////////////
// OpenDomain
////////////////////////////////////////////////////////////////////////

type OpenDomainTest struct {
	simpleDBTest

	name string

	domain Domain
	err error
}

func init() { RegisterTestSuite(&OpenDomainTest{}) }

func (t *OpenDomainTest) SetUp(i *TestInfo) {
	// Call common setup code.
	t.simpleDBTest.SetUp(i)

	// Make the request legal by default.
	t.name = "foo"
}

func (t *OpenDomainTest) callDB() {
	t.domain, t.err = t.db.OpenDomain(t.name)
}

func (t *OpenDomainTest) NameIsTooShort() {
	t.name = "aa"

	// Call
	t.callDB()

	ExpectThat(t.err, Error(HasSubstr("domain")))
	ExpectThat(t.err, Error(HasSubstr("name")))
}

func (t *OpenDomainTest) NameIsTooLong() {
	t.name = strings.Repeat("x", 256)

	// Call
	t.callDB()

	ExpectThat(t.err, Error(HasSubstr("domain")))
	ExpectThat(t.err, Error(HasSubstr("name")))
}

func (t *OpenDomainTest) NameContainsUnusableCharacter() {
	t.name = "foo%bar"

	// Call
	t.callDB()

	ExpectThat(t.err, Error(HasSubstr("domain")))
	ExpectThat(t.err, Error(HasSubstr("name")))
	ExpectThat(t.err, Error(HasSubstr("foo%bar")))
}

func (t *OpenDomainTest) CallsConn() {
	t.name = "taco"

	// Call
	t.callDB()
	AssertNe(nil, t.c.req)

	AssertThat(
		getSortedKeys(t.c.req),
		ElementsAre(
			"DomainName",
		),
	)

	ExpectEq("taco", t.c.req["DomainName"])
}

func (t *OpenDomainTest) ConnReturnsError() {
	// Conn
	t.c.err = errors.New("taco")

	// Call
	t.callDB()

	ExpectThat(t.err, Error(HasSubstr("SendRequest")))
	ExpectThat(t.err, Error(HasSubstr("taco")))
}

func (t *OpenDomainTest) CallsFactoryFuncAndReturnsResult() {
	ExpectFalse(true, "TODO")
}

////////////////////////////////////////////////////////////////////////
// DeleteDomain
////////////////////////////////////////////////////////////////////////

type DeleteDomainTest struct {
	simpleDBTest

	domain Domain
	err error
}

func init() { RegisterTestSuite(&DeleteDomainTest{}) }

func (t *DeleteDomainTest) SetUp(i *TestInfo) {
	// Call common setup code.
	t.simpleDBTest.SetUp(i)

	// Set up a fake named domain.
	t.domain = &fakeDomain{"some_domain"}
}

func (t *DeleteDomainTest) callDB() {
	t.err = t.db.DeleteDomain(t.domain)
}

func (t *DeleteDomainTest) DoesFoo() {
	ExpectFalse(true, "TODO")
}
<commit_msg>OpenDomainTest.ConnSaysOkay<commit_after>// Copyright 2012 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package sdb

import (
	"errors"
	. "github.com/jacobsa/oglematchers"
	. "github.com/jacobsa/ogletest"
	"strings"
)

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

// A common helper class.
type simpleDBTest struct {
	c *fakeConn
	db SimpleDB
}

func (t *simpleDBTest) SetUp(i *TestInfo) {
	var err error

	t.c = &fakeConn{}

	t.db, err = newSimpleDB(t.c)
	AssertEq(nil, err)
}

type fakeDomain struct {
	name string
}

func (d *fakeDomain) Name() string {
	return d.name
}

func (d *fakeDomain) PutAttributes(
		item ItemName,
		updates []PutUpdate,
		preconditions []Precondition) error {
	panic("Unsupported")
}

func (d *fakeDomain) BatchPutAttributes(updateMap map[ItemName][]PutUpdate) error {
	panic("Unsupported")
}

func (d *fakeDomain) DeleteAttributes(
		item ItemName,
		deletes []DeleteUpdate,
		preconditions []Precondition) error {
	panic("Unsupported")
}

func (d *fakeDomain) BatchDeleteAttributes(deleteMap map[ItemName][]DeleteUpdate) error {
	panic("Unsupported")
}

func (d *fakeDomain) GetAttributes(
		item ItemName,
		constistentRead bool,
		attrNames []string) (attrs []Attribute, err error) {
	panic("Unsupported")
}

////////////////////////////////////////////////////////////////////////
// OpenDomain
////////////////////////////////////////////////////////////////////////

type OpenDomainTest struct {
	simpleDBTest

	name string

	domain Domain
	err error
}

func init() { RegisterTestSuite(&OpenDomainTest{}) }

func (t *OpenDomainTest) SetUp(i *TestInfo) {
	// Call common setup code.
	t.simpleDBTest.SetUp(i)

	// Make the request legal by default.
	t.name = "foo"
}

func (t *OpenDomainTest) callDB() {
	t.domain, t.err = t.db.OpenDomain(t.name)
}

func (t *OpenDomainTest) NameIsTooShort() {
	t.name = "aa"

	// Call
	t.callDB()

	ExpectThat(t.err, Error(HasSubstr("domain")))
	ExpectThat(t.err, Error(HasSubstr("name")))
}

func (t *OpenDomainTest) NameIsTooLong() {
	t.name = strings.Repeat("x", 256)

	// Call
	t.callDB()

	ExpectThat(t.err, Error(HasSubstr("domain")))
	ExpectThat(t.err, Error(HasSubstr("name")))
}

func (t *OpenDomainTest) NameContainsUnusableCharacter() {
	t.name = "foo%bar"

	// Call
	t.callDB()

	ExpectThat(t.err, Error(HasSubstr("domain")))
	ExpectThat(t.err, Error(HasSubstr("name")))
	ExpectThat(t.err, Error(HasSubstr("foo%bar")))
}

func (t *OpenDomainTest) CallsConn() {
	t.name = "taco"

	// Call
	t.callDB()
	AssertNe(nil, t.c.req)

	AssertThat(
		getSortedKeys(t.c.req),
		ElementsAre(
			"DomainName",
		),
	)

	ExpectEq("taco", t.c.req["DomainName"])
}

func (t *OpenDomainTest) ConnReturnsError() {
	// Conn
	t.c.err = errors.New("taco")

	// Call
	t.callDB()

	ExpectThat(t.err, Error(HasSubstr("SendRequest")))
	ExpectThat(t.err, Error(HasSubstr("taco")))
}

func (t *OpenDomainTest) ConnSaysOkay() {
	t.name = "taco"

	// Conn
	t.c.resp = []byte{}

	// Call
	t.callDB()
	AssertEq(nil, t.err)

	castedDomain, ok := t.domain.(*domain)
	AssertTrue(ok)
	ExpectEq("taco", castedDomain.name)
	ExpectEq(t.c, castedDomain.c)
}

////////////////////////////////////////////////////////////////////////
// DeleteDomain
////////////////////////////////////////////////////////////////////////

type DeleteDomainTest struct {
	simpleDBTest

	domain Domain
	err error
}

func init() { RegisterTestSuite(&DeleteDomainTest{}) }

func (t *DeleteDomainTest) SetUp(i *TestInfo) {
	// Call common setup code.
	t.simpleDBTest.SetUp(i)

	// Set up a fake named domain.
	t.domain = &fakeDomain{"some_domain"}
}

func (t *DeleteDomainTest) callDB() {
	t.err = t.db.DeleteDomain(t.domain)
}

func (t *DeleteDomainTest) DoesFoo() {
	ExpectFalse(true, "TODO")
}
<|endoftext|>


<commit_before>// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package vision_test

import (
	"context"
	"fmt"
	"os"

	vision "cloud.google.com/go/vision/apiv1"
	pb "google.golang.org/genproto/googleapis/cloud/vision/v1"
)

func Example_NewImageFromReader() {
	f, err := os.Open("path/to/image.jpg")
	if err != nil {
		// TODO: handle error.
	}
	img, err := vision.NewImageFromReader(f)
	if err != nil {
		// TODO: handle error.
	}
	fmt.Println(img)
}

func Example_NewImageFromURI() {
	img := vision.NewImageFromURI("gs://my-bucket/my-image.png")
	fmt.Println(img)
}

func Example_FaceFromLandmarks() {
	ctx := context.Background()
	c, err := vision.NewImageAnnotatorClient(ctx)
	if err != nil {
		// TODO: Handle error.
	}
	resp, err := c.BatchAnnotateImages(ctx, &pb.BatchAnnotateImagesRequest{
		Requests: []*pb.AnnotateImageRequest{
			{
				Image: vision.NewImageFromURI("gs://bucket/image.jpg"),
				Features: []*pb.Feature{{
					Type:       pb.Feature_FACE_DETECTION,
					MaxResults: 5,
				}},
			},
		},
	})
	if err != nil {
		// TODO: Handle error.
	}
	res := resp.Responses[0]
	if res.Error != nil {
		// TODO: Handle error.
	}
	for _, a := range res.FaceAnnotations {
		face := vision.FaceFromLandmarks(a.Landmarks)
		fmt.Println(face.Nose.Tip)
		fmt.Println(face.Eyes.Left.Pupil)
	}
}
<commit_msg>vision/apiv1: fix context package<commit_after>// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package vision_test

import (
	"fmt"
	"os"

	vision "cloud.google.com/go/vision/apiv1"
	"golang.org/x/net/context"
	pb "google.golang.org/genproto/googleapis/cloud/vision/v1"
)

func Example_NewImageFromReader() {
	f, err := os.Open("path/to/image.jpg")
	if err != nil {
		// TODO: handle error.
	}
	img, err := vision.NewImageFromReader(f)
	if err != nil {
		// TODO: handle error.
	}
	fmt.Println(img)
}

func Example_NewImageFromURI() {
	img := vision.NewImageFromURI("gs://my-bucket/my-image.png")
	fmt.Println(img)
}

func Example_FaceFromLandmarks() {
	ctx := context.Background()
	c, err := vision.NewImageAnnotatorClient(ctx)
	if err != nil {
		// TODO: Handle error.
	}
	resp, err := c.BatchAnnotateImages(ctx, &pb.BatchAnnotateImagesRequest{
		Requests: []*pb.AnnotateImageRequest{
			{
				Image: vision.NewImageFromURI("gs://bucket/image.jpg"),
				Features: []*pb.Feature{{
					Type:       pb.Feature_FACE_DETECTION,
					MaxResults: 5,
				}},
			},
		},
	})
	if err != nil {
		// TODO: Handle error.
	}
	res := resp.Responses[0]
	if res.Error != nil {
		// TODO: Handle error.
	}
	for _, a := range res.FaceAnnotations {
		face := vision.FaceFromLandmarks(a.Landmarks)
		fmt.Println(face.Nose.Tip)
		fmt.Println(face.Eyes.Left.Pupil)
	}
}
<|endoftext|>


<commit_before>package aws

import (
	"errors"
	"fmt"
	"log"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/ec2"
	"github.com/hashicorp/terraform-plugin-sdk/helper/schema"
)

func dataSourceAwsEc2TransitGatewayVpnAttachment() *schema.Resource {
	return &schema.Resource{
		Read: dataSourceAwsEc2TransitGatewayVpnAttachmentRead,

		Schema: map[string]*schema.Schema{
			"tags": tagsSchemaComputed(),
			"transit_gateway_id": {
				Type:     schema.TypeString,
				Required: true,
			},
			"vpn_connection_id": {
				Type:     schema.TypeString,
				Required: true,
			},
		},
	}
}

func dataSourceAwsEc2TransitGatewayVpnAttachmentRead(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).ec2conn

	input := &ec2.DescribeTransitGatewayAttachmentsInput{
		Filters: []*ec2.Filter{
			{
				Name:   aws.String("resource-id"),
				Values: []*string{aws.String(d.Get("vpn_connection_id").(string))},
			},
			{
				Name:   aws.String("resource-type"),
				Values: []*string{aws.String(ec2.TransitGatewayAttachmentResourceTypeVpn)},
			},
			{
				Name:   aws.String("transit-gateway-id"),
				Values: []*string{aws.String(d.Get("transit_gateway_id").(string))},
			},
		},
	}

	log.Printf("[DEBUG] Reading EC2 Transit Gateway VPN Attachments: %s", input)
	output, err := conn.DescribeTransitGatewayAttachments(input)

	if err != nil {
		return fmt.Errorf("error reading EC2 Transit Gateway VPN Attachment: %s", err)
	}

	if output == nil || len(output.TransitGatewayAttachments) == 0 || output.TransitGatewayAttachments[0] == nil {
		return errors.New("error reading EC2 Transit Gateway VPN Attachment: no results found")
	}

	if len(output.TransitGatewayAttachments) > 1 {
		return errors.New("error reading EC2 Transit Gateway VPN Attachment: multiple results found, try adjusting search criteria")
	}

	transitGatewayAttachment := output.TransitGatewayAttachments[0]

	if err := d.Set("tags", tagsToMap(transitGatewayAttachment.Tags)); err != nil {
		return fmt.Errorf("error setting tags: %s", err)
	}

	d.Set("transit_gateway_id", aws.StringValue(transitGatewayAttachment.TransitGatewayId))
	d.Set("vpn_connection_id", aws.StringValue(transitGatewayAttachment.ResourceId))

	d.SetId(aws.StringValue(transitGatewayAttachment.TransitGatewayAttachmentId))

	return nil
}
<commit_msg>d/aws_ec2_transit_gateway_vpn_attachment: Refactor tagging logic to keyvaluetags package.<commit_after>package aws

import (
	"errors"
	"fmt"
	"log"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/ec2"
	"github.com/hashicorp/terraform-plugin-sdk/helper/schema"
	"github.com/terraform-providers/terraform-provider-aws/aws/internal/keyvaluetags"
)

func dataSourceAwsEc2TransitGatewayVpnAttachment() *schema.Resource {
	return &schema.Resource{
		Read: dataSourceAwsEc2TransitGatewayVpnAttachmentRead,

		Schema: map[string]*schema.Schema{
			"tags": tagsSchemaComputed(),
			"transit_gateway_id": {
				Type:     schema.TypeString,
				Required: true,
			},
			"vpn_connection_id": {
				Type:     schema.TypeString,
				Required: true,
			},
		},
	}
}

func dataSourceAwsEc2TransitGatewayVpnAttachmentRead(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).ec2conn

	input := &ec2.DescribeTransitGatewayAttachmentsInput{
		Filters: []*ec2.Filter{
			{
				Name:   aws.String("resource-id"),
				Values: []*string{aws.String(d.Get("vpn_connection_id").(string))},
			},
			{
				Name:   aws.String("resource-type"),
				Values: []*string{aws.String(ec2.TransitGatewayAttachmentResourceTypeVpn)},
			},
			{
				Name:   aws.String("transit-gateway-id"),
				Values: []*string{aws.String(d.Get("transit_gateway_id").(string))},
			},
		},
	}

	log.Printf("[DEBUG] Reading EC2 Transit Gateway VPN Attachments: %s", input)
	output, err := conn.DescribeTransitGatewayAttachments(input)

	if err != nil {
		return fmt.Errorf("error reading EC2 Transit Gateway VPN Attachment: %s", err)
	}

	if output == nil || len(output.TransitGatewayAttachments) == 0 || output.TransitGatewayAttachments[0] == nil {
		return errors.New("error reading EC2 Transit Gateway VPN Attachment: no results found")
	}

	if len(output.TransitGatewayAttachments) > 1 {
		return errors.New("error reading EC2 Transit Gateway VPN Attachment: multiple results found, try adjusting search criteria")
	}

	transitGatewayAttachment := output.TransitGatewayAttachments[0]

	if err := d.Set("tags", keyvaluetags.Ec2KeyValueTags(transitGatewayAttachment.Tags).IgnoreAws().Map()); err != nil {
		return fmt.Errorf("error setting tags: %s", err)
	}

	d.Set("transit_gateway_id", aws.StringValue(transitGatewayAttachment.TransitGatewayId))
	d.Set("vpn_connection_id", aws.StringValue(transitGatewayAttachment.ResourceId))

	d.SetId(aws.StringValue(transitGatewayAttachment.TransitGatewayAttachmentId))

	return nil
}
<|endoftext|>


<commit_before>// Copyright (c) 2015 FactomProject/FactomCode Systems LLC.
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

// factomlog is based on github.com/alexcesaro/log and
// github.com/alexcesaro/log/golog (MIT License)
package factomlog

import (
	"fmt"
	"io"
	"os"
	"time"
)

// Level specifies a level of verbosity. The available levels are the eight
// severities described in RFC 5424 and none.
type Level int8

const (
	None Level = iota - 1
	Emergency
	Alert
	Critical
	Error
	Warning
	Notice
	Info
	Debug
)

// A FLogger represents an active logging object that generates lines of output
// to an io.Writer.
type FLogger struct {
	out    io.Writer
	level  Level
	prefix string
}

func New(w io.Writer, level, prefix string) *FLogger {
	return &FLogger{
		out:    w,
		level:  levelFromString(level),
		prefix: prefix,
	}
}

// Emergency logs with an emergency level.
func (logger *FLogger) Emergency(args ...interface{}) {
	logger.write(Emergency, args...)
}

// Emergencyf logs with an emergency level.
// Arguments are handled in the manner of fmt.Printf.
func (logger *FLogger) Emergencyf(format string, args ...interface{}) {
	logger.write(Emergency, fmt.Sprintf(format, args...))
}

// Alert logs with an alert level.
func (logger *FLogger) Alert(args ...interface{}) {
	logger.write(Alert, args...)
}

// Alertf logs with an alert level.
// Arguments are handled in the manner of fmt.Printf.
func (logger *FLogger) Alertf(format string, args ...interface{}) {
	logger.write(Alert, fmt.Sprintf(format, args...))
}

// Critical logs with a critical level.
func (logger *FLogger) Critical(args ...interface{}) {
	logger.write(Critical, args...)
}

// Criticalf logs with a critical level.
// Arguments are handled in the manner of fmt.Printf.
func (logger *FLogger) Criticalf(format string, args ...interface{}) {
	logger.write(Critical, fmt.Sprintf(format, args...))
}

// Error logs with an error level.
func (logger *FLogger) Error(args ...interface{}) {
	logger.write(Error, args...)
}

// Errorf logs with an error level.
// Arguments are handled in the manner of fmt.Printf.
func (logger *FLogger) Errorf(format string, args ...interface{}) {
	logger.write(Error, fmt.Sprintf(format, args...))
}

// Warning logs with a warning level.
func (logger *FLogger) Warning(args ...interface{}) {
	logger.write(Warning, args...)
}

// Warningf logs with a warning level.
// Arguments are handled in the manner of fmt.Printf.
func (logger *FLogger) Warningf(format string, args ...interface{}) {
	logger.write(Warning, fmt.Sprintf(format, args...))
}

// Notice logs with a notice level.
func (logger *FLogger) Notice(args ...interface{}) {
	logger.write(Notice, args...)
}

// Noticef logs with a notice level.
// Arguments are handled in the manner of fmt.Printf.
func (logger *FLogger) Noticef(format string, args ...interface{}) {
	logger.write(Notice, fmt.Sprintf(format, args...))
}

// Info logs with an info level.
func (logger *FLogger) Info(args ...interface{}) {
	logger.write(Info, args...)
}

// Infof logs with an info level.
// Arguments are handled in the manner of fmt.Printf.
func (logger *FLogger) Infof(format string, args ...interface{}) {
	logger.write(Info, fmt.Sprintf(format, args...))
}

// Debug logs with a debug level.
func (logger *FLogger) Debug(args ...interface{}) {
	logger.write(Debug, args...)
}

// Debugf logs with a debug level.
// Arguments are handled in the manner of fmt.Printf.
func (logger *FLogger) Debugf(format string, args ...interface{}) {
	logger.write(Debug, fmt.Sprintf(format, args...))
}

// write outputs to the FLogger.out based on the FLogger.level and calls os.Exit
// if the level is <= Error
func (logger *FLogger) write(level Level, args ...interface{}) {
	if level > logger.level {
		return
	}

	fmt.Fprint(logger.out, time.Now(), logger.prefix, levelPrefix[level], args)
	if level <= Error {
		os.Exit(1)
	}
}

var levelPrefix = map[Level]string{
	Emergency: " EMERGENCY ",
	Alert:     " ALERT ",
	Critical:  " CRITICAL ",
	Error:     " ERROR ",
	Warning:   " WARNING ",
	Notice:    " NOTICE ",
	Info:      " INFO ",
	Debug:     " DEBUG ",
}

func levelFromString(levelName string) (level Level) {
	switch levelName {
	case "debug":
		level = Debug
	case "info":
		level = Info
	case "notice":
		level = Notice
	case "warning":
		level = Warning
	case "error":
		level = Error
	case "critical":
		level = Critical
	case "alert":
		level = Alert
	case "emergency":
		level = Emergency
	case "none":
		level = None
	default:
		fmt.Fprintf(os.Stderr, "Invalid level value %q, allowed values are: debug, info, notice, warning, error, critical, alert, emergency and none\n", levelName)
		os.Exit(2)
	}
	return
}
<commit_msg>added simple test and updated time format of output<commit_after>// Copyright (c) 2015 FactomProject/FactomCode Systems LLC.
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

// factomlog is based on github.com/alexcesaro/log and
// github.com/alexcesaro/log/golog (MIT License)
package factomlog

import (
	"fmt"
	"io"
	"os"
	"time"
)

// Level specifies a level of verbosity. The available levels are the eight
// severities described in RFC 5424 and none.
type Level int8

const (
	None Level = iota - 1
	Emergency
	Alert
	Critical
	Error
	Warning
	Notice
	Info
	Debug
)

// A FLogger represents an active logging object that generates lines of output
// to an io.Writer.
type FLogger struct {
	out    io.Writer
	level  Level
	prefix string
}

func New(w io.Writer, level, prefix string) *FLogger {
	return &FLogger{
		out:    w,
		level:  levelFromString(level),
		prefix: prefix,
	}
}

// Emergency logs with an emergency level.
func (logger *FLogger) Emergency(args ...interface{}) {
	logger.write(Emergency, args...)
}

// Emergencyf logs with an emergency level.
// Arguments are handled in the manner of fmt.Printf.
func (logger *FLogger) Emergencyf(format string, args ...interface{}) {
	logger.write(Emergency, fmt.Sprintf(format, args...))
}

// Alert logs with an alert level.
func (logger *FLogger) Alert(args ...interface{}) {
	logger.write(Alert, args...)
}

// Alertf logs with an alert level.
// Arguments are handled in the manner of fmt.Printf.
func (logger *FLogger) Alertf(format string, args ...interface{}) {
	logger.write(Alert, fmt.Sprintf(format, args...))
}

// Critical logs with a critical level.
func (logger *FLogger) Critical(args ...interface{}) {
	logger.write(Critical, args...)
}

// Criticalf logs with a critical level.
// Arguments are handled in the manner of fmt.Printf.
func (logger *FLogger) Criticalf(format string, args ...interface{}) {
	logger.write(Critical, fmt.Sprintf(format, args...))
}

// Error logs with an error level.
func (logger *FLogger) Error(args ...interface{}) {
	logger.write(Error, args...)
}

// Errorf logs with an error level.
// Arguments are handled in the manner of fmt.Printf.
func (logger *FLogger) Errorf(format string, args ...interface{}) {
	logger.write(Error, fmt.Sprintf(format, args...))
}

// Warning logs with a warning level.
func (logger *FLogger) Warning(args ...interface{}) {
	logger.write(Warning, args...)
}

// Warningf logs with a warning level.
// Arguments are handled in the manner of fmt.Printf.
func (logger *FLogger) Warningf(format string, args ...interface{}) {
	logger.write(Warning, fmt.Sprintf(format, args...))
}

// Notice logs with a notice level.
func (logger *FLogger) Notice(args ...interface{}) {
	logger.write(Notice, args...)
}

// Noticef logs with a notice level.
// Arguments are handled in the manner of fmt.Printf.
func (logger *FLogger) Noticef(format string, args ...interface{}) {
	logger.write(Notice, fmt.Sprintf(format, args...))
}

// Info logs with an info level.
func (logger *FLogger) Info(args ...interface{}) {
	logger.write(Info, args...)
}

// Infof logs with an info level.
// Arguments are handled in the manner of fmt.Printf.
func (logger *FLogger) Infof(format string, args ...interface{}) {
	logger.write(Info, fmt.Sprintf(format, args...))
}

// Debug logs with a debug level.
func (logger *FLogger) Debug(args ...interface{}) {
	logger.write(Debug, args...)
}

// Debugf logs with a debug level.
// Arguments are handled in the manner of fmt.Printf.
func (logger *FLogger) Debugf(format string, args ...interface{}) {
	logger.write(Debug, fmt.Sprintf(format, args...))
}

// write outputs to the FLogger.out based on the FLogger.level and calls os.Exit
// if the level is <= Error
func (logger *FLogger) write(level Level, args ...interface{}) {
	if level > logger.level {
		return
	}

	fmt.Fprintln(logger.out, time.Now().Round(time.Second), " ", logger.prefix, levelPrefix[level], args)
	if level <= Error {
		os.Exit(1)
	}
}

var levelPrefix = map[Level]string{
	Emergency: " EMERGENCY ",
	Alert:     " ALERT ",
	Critical:  " CRITICAL ",
	Error:     " ERROR ",
	Warning:   " WARNING ",
	Notice:    " NOTICE ",
	Info:      " INFO ",
	Debug:     " DEBUG ",
}

func levelFromString(levelName string) (level Level) {
	switch levelName {
	case "debug":
		level = Debug
	case "info":
		level = Info
	case "notice":
		level = Notice
	case "warning":
		level = Warning
	case "error":
		level = Error
	case "critical":
		level = Critical
	case "alert":
		level = Alert
	case "emergency":
		level = Emergency
	case "none":
		level = None
	default:
		fmt.Fprintf(os.Stderr, "Invalid level value %q, allowed values are: debug, info, notice, warning, error, critical, alert, emergency and none\n", levelName)
		os.Exit(2)
	}
	return
}
<|endoftext|>


<commit_before>package fetcher

import (
	"sort"

	"github.com/cloudfoundry-incubator/bbs/models"
)

type LRPs map[string]*LRP

func (l LRPs) SortedByProcessGuid() []*LRP {
	var lrps []*LRP

	for _, lrp := range l {
		lrps = append(lrps, lrp)
	}

	sort.Sort(ByProcessGuid(lrps))
	return lrps
}

func ByProcessGuid(lrps []*LRP) LRPsByProcessGuid {
	return lrps
}

type LRPsByProcessGuid []*LRP

func (l LRPsByProcessGuid) Len() int      { return len(l) }
func (l LRPsByProcessGuid) Swap(i, j int) { l[i], l[j] = l[j], l[i] }
func (l LRPsByProcessGuid) Less(i, j int) bool {
	return l[i].Desired.ProcessGuid < l[j].Desired.ProcessGuid
}

type LRP struct {
	Desired *models.DesiredLRP
	Actuals []*Actual
}
<commit_msg>Add start command to lrps<commit_after>package fetcher

import (
	"fmt"
	"sort"
	"strings"

	"github.com/cloudfoundry-incubator/bbs/models"
)

type LRPs map[string]*LRP

func (l LRPs) SortedByProcessGuid() []*LRP {
	var lrps []*LRP

	for _, lrp := range l {
		lrps = append(lrps, lrp)
	}

	sort.Sort(ByProcessGuid(lrps))
	return lrps
}

func ByProcessGuid(lrps []*LRP) LRPsByProcessGuid {
	return lrps
}

type LRPsByProcessGuid []*LRP

func (l LRPsByProcessGuid) Len() int      { return len(l) }
func (l LRPsByProcessGuid) Swap(i, j int) { l[i], l[j] = l[j], l[i] }
func (l LRPsByProcessGuid) Less(i, j int) bool {
	return l[i].Desired.ProcessGuid < l[j].Desired.ProcessGuid
}

type LRP struct {
	Desired *models.DesiredLRP
	Actuals []*Actual
}

func (l *LRP) StartCommand() string {
	return printAction(l.Desired.Action)
}

func printAction(action *models.Action) string {
	unwrappedAction := models.UnwrapAction(action)
	switch unwrappedAction.ActionType() {
	case models.ActionTypeDownload:
		download := unwrappedAction.(*models.DownloadAction)
		return fmt.Sprintf("Download: %s to %s as %s", download.From, download.To, download.User)
	case models.ActionTypeEmitProgress:
		emit := unwrappedAction.(*models.EmitProgressAction)
		return printAction(emit.Action)
	case models.ActionTypeRun:
		run := unwrappedAction.(*models.RunAction)
		return fmt.Sprintf("Run: %s %s", run.Path, strings.Join(run.Args, " "))
	case models.ActionTypeUpload:
		upload := unwrappedAction.(*models.UploadAction)
		return fmt.Sprintf("Upload: %s to %s", upload.From, upload.To)
	case models.ActionTypeTimeout:
		timeout := unwrappedAction.(*models.TimeoutAction)
		return printAction(timeout.Action)
	case models.ActionTypeTry:
		try := unwrappedAction.(*models.TryAction)
		return printAction(try.Action)
	case models.ActionTypeParallel:
		paralell := unwrappedAction.(*models.ParallelAction)
		actionText := "Parallel:\n"
		for _, action := range paralell.Actions {
			actionText += fmt.Sprintf("\t%s\n", printAction(action))
		}
		return actionText
	case models.ActionTypeSerial:
		serial := unwrappedAction.(*models.SerialAction)
		actionText := "Serial:\n"
		for _, action := range serial.Actions {
			actionText += fmt.Sprintf("\t%s\n", printAction(action))
		}
		return actionText
	case models.ActionTypeCodependent:
		codependent := unwrappedAction.(*models.CodependentAction)
		actionText := "Codependent:\n"
		for _, action := range codependent.Actions {
			actionText += fmt.Sprintf("\t%s\n", printAction(action))
		}
		return actionText
	default:
		return "Error: Unknown start command"
	}
}
<|endoftext|>