<commit_before>package brimutil

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"hash"
	"io"
)

// ChecksummedReader reads content written by ChecksummedWriter, verifying
// checksums when requested. Implements the io.ReadSeeker and io.Closer
// interfaces.
//
// Any errors from Read or Verify should make no assumptions about any
// resulting position and should Seek before continuing to use the
// ChecksummedReader.
type ChecksummedReader interface {
	// Read implements the io.Reader interface.
	//
	// Any error should make no assumption about any resulting position and
	// should Seek before continuing to use the ChecksummedReader.
	Read(v []byte) (n int, err error)
	// Seek implements the io.Seeker interface.
	Seek(offset int64, whence int) (n int64, err error)
	// Verify verifies the checksum for the section of the content containing
	// the current read position.
	//
	// If there is an error, whether the section is checksum valid is
	// indeterminate by this routine and the caller should decide what to do
	// based on the error. For example, if the underlying i/o causes a timeout
	// error, the content may be fine and just temporarily unreachable.
	//
	// Any error should also make no assumption about any resulting position
	// and should Seek before continuing to use the ChecksummedReader.
	//
	// With no error, the bool indicates whether the content is checksum valid
	// and the position within the ChecksummedReader will not have changed.
	Verify() (bool, error)
	// Close implements the io.Closer interface.
	Close() error
}

// NewChecksummedReader returns a ChecksummedReader that delegates requests to
// an underlying io.ReadSeeker expecting checksums of the content at given
// intervals using the hashing function given.
func NewChecksummedReader(delegate io.ReadSeeker, interval int, newHash func() hash.Hash32) ChecksummedReader {
	return newChecksummedReaderImpl(delegate, interval, newHash)
}

// ChecksummedWriter writes content with additional checksums embedded in the
// underlying content.
//
// Implements the io.WriteCloser interface.
//
// Note that this generally only works for brand new writers starting at offset
// 0. Appending to existing files or starting at offsets other than 0 requires
// special care when working with ChecksummedReader later and is beyond the
// basic usage described here.
//
// Also, do not forget to Close the ChecksummedWriter to ensure the final
// checksum is emitted.
type ChecksummedWriter interface {
	// Write implements the io.Writer interface.
	Write(v []byte) (n int, err error)
	// Close implements the io.Closer interface.
	Close() error
}

// NewChecksummedWriter returns a ChecksummedWriter that delegates requests to
// an underlying io.Writer and embeds checksums of the content at given
// intervals using the hashing function given.
func NewChecksummedWriter(delegate io.Writer, checksumInterval int, newHash func() hash.Hash32) ChecksummedWriter {
	return newChecksummedWriterImpl(delegate, checksumInterval, newHash)
}

type checksummedReaderImpl struct {
	delegate         io.ReadSeeker
	checksumInterval int
	checksumOffset   int
	newHash          func() hash.Hash32
	hash             hash.Hash32
	checksum         []byte
}

func newChecksummedReaderImpl(delegate io.ReadSeeker, interval int, newHash func() hash.Hash32) ChecksummedReader {
	return &checksummedReaderImpl{
		delegate:         delegate,
		checksumInterval: interval,
		newHash:          newHash,
		checksum:         make([]byte, 4),
	}
}

func (cri *checksummedReaderImpl) Read(v []byte) (int, error) {
	if cri.checksumOffset+len(v) > cri.checksumInterval {
		v = v[:cri.checksumInterval-cri.checksumOffset]
	}
	n, err := cri.delegate.Read(v)
	cri.checksumOffset += n
	if err == io.EOF {
		n -= 4
		if n < 0 {
			n = 0
		}
	} else if err == nil {
		if cri.checksumOffset == cri.checksumInterval {
			n2, err := io.ReadFull(cri.delegate, cri.checksum)
			if err == io.EOF || err == io.ErrUnexpectedEOF {
				n -= 4 - n2
				err = io.EOF
			}
			cri.checksumOffset = 0
		} else {
			n2, err := io.ReadFull(cri.delegate, cri.checksum)
			if err == io.EOF || err == io.ErrUnexpectedEOF {
				n -= 4 - n2
				err = io.EOF
			} else {
				_, err = cri.delegate.Seek(-int64(n2), 1)
			}
		}
	}
	return n, err
}

func (cri *checksummedReaderImpl) Seek(offset int64, whence int) (int64, error) {
	switch whence {
	case 0:
	case 1:
		o, err := cri.delegate.Seek(0, 1)
		cri.checksumOffset = int(o % (int64(cri.checksumInterval) + 4))
		if err != nil {
			return o - (o / int64(cri.checksumInterval) * 4), err
		}
		offset = o - (o / int64(cri.checksumInterval) * 4) + offset
	case 2:
		o, err := cri.delegate.Seek(0, 2)
		cri.checksumOffset = int(o % (int64(cri.checksumInterval) + 4))
		if err != nil {
			return o - (o / int64(cri.checksumInterval) * 4), err
		}
		offset = o - (o / int64(cri.checksumInterval) * 4) + offset
	default:
		o, _ := cri.delegate.Seek(0, 1)
		return o, fmt.Errorf("invalid whence %d", whence)
	}
	o, err := cri.delegate.Seek(offset+(offset/int64(cri.checksumInterval)*4), 0)
	cri.checksumOffset = int(o % (int64(cri.checksumInterval) + 4))
	return o - (o / int64(cri.checksumInterval) * 4), err
}

func (cri *checksummedReaderImpl) Verify() (bool, error) {
	originalOffset, err := cri.delegate.Seek(0, 1)
	if err != nil {
		return false, err
	}
	if cri.checksumOffset > 0 {
		_, err = cri.delegate.Seek(-int64(cri.checksumOffset), 1)
		if err != nil {
			return false, err
		}
	}
	block := make([]byte, cri.checksumInterval+4)
	checksum := block[cri.checksumInterval:]
	n, err := io.ReadFull(cri.delegate, block)
	if err == io.ErrUnexpectedEOF {
		checksum = block[n-4 : n]
		block = block[:n-4]
	} else if err != nil {
		return false, err
	} else {
		block = block[:cri.checksumInterval]
	}
	hash := cri.newHash()
	hash.Write(block)
	verified := bytes.Equal(checksum, hash.Sum(cri.checksum[:0]))
	_, err = cri.delegate.Seek(originalOffset, 0)
	if err != nil {
		return verified, err
	}
	return verified, nil
}

func (cri *checksummedReaderImpl) Close() error {
	var err error
	if c, ok := cri.delegate.(io.Closer); ok {
		err = c.Close()
	}
	cri.delegate = _ERR_DELEGATE
	return err
}

type checksummedWriterImpl struct {
	delegate         io.Writer
	checksumInterval int
	checksumOffset   int
	newHash          func() hash.Hash32
	hash             hash.Hash32
	checksum         []byte
}

func newChecksummedWriterImpl(delegate io.Writer, checksumInterval int, newHash func() hash.Hash32) *checksummedWriterImpl {
	return &checksummedWriterImpl{
		delegate:         delegate,
		checksumInterval: checksumInterval,
		newHash:          newHash,
		hash:             newHash(),
		checksum:         make([]byte, 4),
	}
}

func (cwi *checksummedWriterImpl) Write(v []byte) (int, error) {
	var n int
	var n2 int
	var err error
	for cwi.checksumOffset+len(v) >= cwi.checksumInterval {
		n2, err = cwi.delegate.Write(v[:cwi.checksumInterval-cwi.checksumOffset])
		n += n2
		if err != nil {
			cwi.delegate = _ERR_DELEGATE
			return n, err
		}
		cwi.hash.Write(v[:cwi.checksumInterval-cwi.checksumOffset])
		v = v[cwi.checksumInterval-cwi.checksumOffset:]
		binary.BigEndian.PutUint32(cwi.checksum, cwi.hash.Sum32())
		_, err = cwi.delegate.Write(cwi.checksum)
		if err != nil {
			cwi.delegate = _ERR_DELEGATE
			return n, err
		}
		cwi.hash = cwi.newHash()
		cwi.checksumOffset = 0
	}
	if len(v) > 0 {
		n2, err = cwi.delegate.Write(v)
		n += n2
		if err != nil {
			cwi.delegate = _ERR_DELEGATE
			return n, err
		}
		cwi.hash.Write(v)
		cwi.checksumOffset += n2
	}
	return n, err
}

func (cwi *checksummedWriterImpl) Close() error {
	var err error
	if cwi.checksumOffset > 0 {
		binary.BigEndian.PutUint32(cwi.checksum, cwi.hash.Sum32())
		_, err = cwi.delegate.Write(cwi.checksum)
		if err != nil {
			cwi.delegate = _ERR_DELEGATE
			return err
		}
	}
	if c, ok := cwi.delegate.(io.Closer); ok {
		err = c.Close()
	}
	cwi.delegate = _ERR_DELEGATE
	return err
}

type multiCoreChecksummedWriter struct {
	delegate         io.Writer
	checksumInterval int
	newHash          func() hash.Hash32
	hash             hash.Hash32
	buffer           *multiCoreChecksummedWriterBuffer
	freeChan         chan *multiCoreChecksummedWriterBuffer
	checksumChan     chan *multiCoreChecksummedWriterBuffer
	writeChan        chan *multiCoreChecksummedWriterBuffer
	doneChan         chan struct{}
	err              error
	closed           bool
}

type multiCoreChecksummedWriterBuffer struct {
	seq int64
	buf []byte
}

// NewMultiCoreChecksummedWriter returns a ChecksummedWriter that delegates
// requests to an underlying io.Writer and embeds checksums of the content at
// given intervals using the hashing function given; it will use multiple cores
// for computing the checksums.
//
// Note that this is generally only faster for large files and reasonably sized
// checksum intervals (e.g. 65532). It can be quite a bit slower on single core
// systems or when using tiny checksum intervals.
func NewMultiCoreChecksummedWriter(delegate io.Writer, checksumInterval int, newHash func() hash.Hash32, cores int) ChecksummedWriter {
	cwi := &multiCoreChecksummedWriter{
		delegate:         delegate,
		checksumInterval: checksumInterval,
		newHash:          newHash,
		freeChan:         make(chan *multiCoreChecksummedWriterBuffer, cores),
		checksumChan:     make(chan *multiCoreChecksummedWriterBuffer, cores),
		writeChan:        make(chan *multiCoreChecksummedWriterBuffer, cores),
		doneChan:         make(chan struct{}),
	}
	for i := 0; i < cores; i++ {
		cwi.freeChan <- &multiCoreChecksummedWriterBuffer{0, make([]byte, 0, checksumInterval+4)}
	}
	cwi.buffer = <-cwi.freeChan
	go cwi.writer()
	for i := 0; i < cores; i++ {
		go cwi.checksummer()
	}
	return cwi
}

func (cwi *multiCoreChecksummedWriter) Write(v []byte) (int, error) {
	var n int
	for len(cwi.buffer.buf)+len(v) >= cwi.checksumInterval {
		n2 := cwi.checksumInterval - len(cwi.buffer.buf)
		cwi.buffer.buf = append(cwi.buffer.buf, v[:n2]...)
		s := cwi.buffer.seq + 1
		cwi.checksumChan <- cwi.buffer
		n += n2
		v = v[n2:]
		cwi.buffer = <-cwi.freeChan
		cwi.buffer.seq = s
	}
	if len(v) > 0 {
		cwi.buffer.buf = append(cwi.buffer.buf, v...)
		n += len(v)
	}
	return n, cwi.err
}

func (cwi *multiCoreChecksummedWriter) Close() error {
	if cwi.closed {
		return cwi.err
	}
	cwi.closed = true
	if len(cwi.buffer.buf) > 0 {
		cwi.checksumChan <- cwi.buffer
	}
	close(cwi.checksumChan)
	for i := 0; i < cap(cwi.checksumChan); i++ {
		<-cwi.doneChan
	}
	cwi.writeChan <- nil
	<-cwi.doneChan
	return cwi.err
}

func (cwi *multiCoreChecksummedWriter) checksummer() {
	for {
		b := <-cwi.checksumChan
		if b == nil {
			break
		}
		h := cwi.newHash()
		h.Write(b.buf)
		b.buf = b.buf[:len(b.buf)+4]
		binary.BigEndian.PutUint32(b.buf[len(b.buf)-4:], h.Sum32())
		cwi.writeChan <- b
	}
	cwi.doneChan <- struct{}{}
}

func (cwi *multiCoreChecksummedWriter) writer() {
	var seq int64
	var lastWasNil bool
	for {
		b := <-cwi.writeChan
		if b == nil {
			if lastWasNil {
				break
			}
			lastWasNil = true
			cwi.writeChan <- nil
			continue
		}
		lastWasNil = false
		if b.seq != seq {
			cwi.writeChan <- b
			continue
		}
		_, cwi.err = cwi.delegate.Write(b.buf)
		b.buf = b.buf[:0]
		cwi.freeChan <- b
		seq++
	}
	if c, ok := cwi.delegate.(io.Closer); ok {
		cwi.err = c.Close()
	}
	cwi.doneChan <- struct{}{}
}

type errDelegate struct {
}

var _ERR_DELEGATE = &errDelegate{}

func (ed *errDelegate) Read(v []byte) (int, error) {
	return 0, fmt.Errorf("closed")
}

func (ed *errDelegate) Write(v []byte) (int, error) {
	return 0, fmt.Errorf("closed")
}

func (ed *errDelegate) Seek(offset int64, whence int) (int64, error) {
	return 0, fmt.Errorf("closed")
}

func (ed *errDelegate) Close() error {
	return fmt.Errorf("already closed")
}
<commit_msg>Fixed race<commit_after>package brimutil

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"hash"
	"io"
	"sync"
)

// ChecksummedReader reads content written by ChecksummedWriter, verifying
// checksums when requested. Implements the io.ReadSeeker and io.Closer
// interfaces.
//
// Any errors from Read or Verify should make no assumptions about any
// resulting position and should Seek before continuing to use the
// ChecksummedReader.
type ChecksummedReader interface {
	// Read implements the io.Reader interface.
	//
	// Any error should make no assumption about any resulting position and
	// should Seek before continuing to use the ChecksummedReader.
	Read(v []byte) (n int, err error)
	// Seek implements the io.Seeker interface.
	Seek(offset int64, whence int) (n int64, err error)
	// Verify verifies the checksum for the section of the content containing
	// the current read position.
	//
	// If there is an error, whether the section is checksum valid is
	// indeterminate by this routine and the caller should decide what to do
	// based on the error. For example, if the underlying i/o causes a timeout
	// error, the content may be fine and just temporarily unreachable.
	//
	// Any error should also make no assumption about any resulting position
	// and should Seek before continuing to use the ChecksummedReader.
	//
	// With no error, the bool indicates whether the content is checksum valid
	// and the position within the ChecksummedReader will not have changed.
	Verify() (bool, error)
	// Close implements the io.Closer interface.
	Close() error
}

// NewChecksummedReader returns a ChecksummedReader that delegates requests to
// an underlying io.ReadSeeker expecting checksums of the content at given
// intervals using the hashing function given.
func NewChecksummedReader(delegate io.ReadSeeker, interval int, newHash func() hash.Hash32) ChecksummedReader {
	return newChecksummedReaderImpl(delegate, interval, newHash)
}

// ChecksummedWriter writes content with additional checksums embedded in the
// underlying content.
//
// Implements the io.WriteCloser interface.
//
// Note that this generally only works for brand new writers starting at offset
// 0. Appending to existing files or starting at offsets other than 0 requires
// special care when working with ChecksummedReader later and is beyond the
// basic usage described here.
//
// Also, do not forget to Close the ChecksummedWriter to ensure the final
// checksum is emitted.
type ChecksummedWriter interface {
	// Write implements the io.Writer interface.
	Write(v []byte) (n int, err error)
	// Close implements the io.Closer interface.
	Close() error
}

// NewChecksummedWriter returns a ChecksummedWriter that delegates requests to
// an underlying io.Writer and embeds checksums of the content at given
// intervals using the hashing function given.
func NewChecksummedWriter(delegate io.Writer, checksumInterval int, newHash func() hash.Hash32) ChecksummedWriter {
	return newChecksummedWriterImpl(delegate, checksumInterval, newHash)
}

type checksummedReaderImpl struct {
	delegate         io.ReadSeeker
	checksumInterval int
	checksumOffset   int
	newHash          func() hash.Hash32
	hash             hash.Hash32
	checksum         []byte
}

func newChecksummedReaderImpl(delegate io.ReadSeeker, interval int, newHash func() hash.Hash32) ChecksummedReader {
	return &checksummedReaderImpl{
		delegate:         delegate,
		checksumInterval: interval,
		newHash:          newHash,
		checksum:         make([]byte, 4),
	}
}

func (cri *checksummedReaderImpl) Read(v []byte) (int, error) {
	if cri.checksumOffset+len(v) > cri.checksumInterval {
		v = v[:cri.checksumInterval-cri.checksumOffset]
	}
	n, err := cri.delegate.Read(v)
	cri.checksumOffset += n
	if err == io.EOF {
		n -= 4
		if n < 0 {
			n = 0
		}
	} else if err == nil {
		if cri.checksumOffset == cri.checksumInterval {
			n2, err := io.ReadFull(cri.delegate, cri.checksum)
			if err == io.EOF || err == io.ErrUnexpectedEOF {
				n -= 4 - n2
				err = io.EOF
			}
			cri.checksumOffset = 0
		} else {
			n2, err := io.ReadFull(cri.delegate, cri.checksum)
			if err == io.EOF || err == io.ErrUnexpectedEOF {
				n -= 4 - n2
				err = io.EOF
			} else {
				_, err = cri.delegate.Seek(-int64(n2), 1)
			}
		}
	}
	return n, err
}

func (cri *checksummedReaderImpl) Seek(offset int64, whence int) (int64, error) {
	switch whence {
	case 0:
	case 1:
		o, err := cri.delegate.Seek(0, 1)
		cri.checksumOffset = int(o % (int64(cri.checksumInterval) + 4))
		if err != nil {
			return o - (o / int64(cri.checksumInterval) * 4), err
		}
		offset = o - (o / int64(cri.checksumInterval) * 4) + offset
	case 2:
		o, err := cri.delegate.Seek(0, 2)
		cri.checksumOffset = int(o % (int64(cri.checksumInterval) + 4))
		if err != nil {
			return o - (o / int64(cri.checksumInterval) * 4), err
		}
		offset = o - (o / int64(cri.checksumInterval) * 4) + offset
	default:
		o, _ := cri.delegate.Seek(0, 1)
		return o, fmt.Errorf("invalid whence %d", whence)
	}
	o, err := cri.delegate.Seek(offset+(offset/int64(cri.checksumInterval)*4), 0)
	cri.checksumOffset = int(o % (int64(cri.checksumInterval) + 4))
	return o - (o / int64(cri.checksumInterval) * 4), err
}

func (cri *checksummedReaderImpl) Verify() (bool, error) {
	originalOffset, err := cri.delegate.Seek(0, 1)
	if err != nil {
		return false, err
	}
	if cri.checksumOffset > 0 {
		_, err = cri.delegate.Seek(-int64(cri.checksumOffset), 1)
		if err != nil {
			return false, err
		}
	}
	block := make([]byte, cri.checksumInterval+4)
	checksum := block[cri.checksumInterval:]
	n, err := io.ReadFull(cri.delegate, block)
	if err == io.ErrUnexpectedEOF {
		checksum = block[n-4 : n]
		block = block[:n-4]
	} else if err != nil {
		return false, err
	} else {
		block = block[:cri.checksumInterval]
	}
	hash := cri.newHash()
	hash.Write(block)
	verified := bytes.Equal(checksum, hash.Sum(cri.checksum[:0]))
	_, err = cri.delegate.Seek(originalOffset, 0)
	if err != nil {
		return verified, err
	}
	return verified, nil
}

func (cri *checksummedReaderImpl) Close() error {
	var err error
	if c, ok := cri.delegate.(io.Closer); ok {
		err = c.Close()
	}
	cri.delegate = _ERR_DELEGATE
	return err
}

type checksummedWriterImpl struct {
	delegate         io.Writer
	checksumInterval int
	checksumOffset   int
	newHash          func() hash.Hash32
	hash             hash.Hash32
	checksum         []byte
}

func newChecksummedWriterImpl(delegate io.Writer, checksumInterval int, newHash func() hash.Hash32) *checksummedWriterImpl {
	return &checksummedWriterImpl{
		delegate:         delegate,
		checksumInterval: checksumInterval,
		newHash:          newHash,
		hash:             newHash(),
		checksum:         make([]byte, 4),
	}
}

func (cwi *checksummedWriterImpl) Write(v []byte) (int, error) {
	var n int
	var n2 int
	var err error
	for cwi.checksumOffset+len(v) >= cwi.checksumInterval {
		n2, err = cwi.delegate.Write(v[:cwi.checksumInterval-cwi.checksumOffset])
		n += n2
		if err != nil {
			cwi.delegate = _ERR_DELEGATE
			return n, err
		}
		cwi.hash.Write(v[:cwi.checksumInterval-cwi.checksumOffset])
		v = v[cwi.checksumInterval-cwi.checksumOffset:]
		binary.BigEndian.PutUint32(cwi.checksum, cwi.hash.Sum32())
		_, err = cwi.delegate.Write(cwi.checksum)
		if err != nil {
			cwi.delegate = _ERR_DELEGATE
			return n, err
		}
		cwi.hash = cwi.newHash()
		cwi.checksumOffset = 0
	}
	if len(v) > 0 {
		n2, err = cwi.delegate.Write(v)
		n += n2
		if err != nil {
			cwi.delegate = _ERR_DELEGATE
			return n, err
		}
		cwi.hash.Write(v)
		cwi.checksumOffset += n2
	}
	return n, err
}

func (cwi *checksummedWriterImpl) Close() error {
	var err error
	if cwi.checksumOffset > 0 {
		binary.BigEndian.PutUint32(cwi.checksum, cwi.hash.Sum32())
		_, err = cwi.delegate.Write(cwi.checksum)
		if err != nil {
			cwi.delegate = _ERR_DELEGATE
			return err
		}
	}
	if c, ok := cwi.delegate.(io.Closer); ok {
		err = c.Close()
	}
	cwi.delegate = _ERR_DELEGATE
	return err
}

type multiCoreChecksummedWriter struct {
	delegate         io.Writer
	checksumInterval int
	newHash          func() hash.Hash32
	hash             hash.Hash32
	buffer           *multiCoreChecksummedWriterBuffer
	freeChan         chan *multiCoreChecksummedWriterBuffer
	checksumChan     chan *multiCoreChecksummedWriterBuffer
	writeChan        chan *multiCoreChecksummedWriterBuffer
	doneChan         chan struct{}
	lock             sync.Mutex
	err              error
	closed           bool
}

type multiCoreChecksummedWriterBuffer struct {
	seq int64
	buf []byte
}

// NewMultiCoreChecksummedWriter returns a ChecksummedWriter that delegates
// requests to an underlying io.Writer and embeds checksums of the content at
// given intervals using the hashing function given; it will use multiple cores
// for computing the checksums.
//
// Note that this is generally only faster for large files and reasonably sized
// checksum intervals (e.g. 65532). It can be quite a bit slower on single core
// systems or when using tiny checksum intervals.
func NewMultiCoreChecksummedWriter(delegate io.Writer, checksumInterval int, newHash func() hash.Hash32, cores int) ChecksummedWriter {
	cwi := &multiCoreChecksummedWriter{
		delegate:         delegate,
		checksumInterval: checksumInterval,
		newHash:          newHash,
		freeChan:         make(chan *multiCoreChecksummedWriterBuffer, cores),
		checksumChan:     make(chan *multiCoreChecksummedWriterBuffer, cores),
		writeChan:        make(chan *multiCoreChecksummedWriterBuffer, cores),
		doneChan:         make(chan struct{}),
	}
	for i := 0; i < cores; i++ {
		cwi.freeChan <- &multiCoreChecksummedWriterBuffer{0, make([]byte, 0, checksumInterval+4)}
	}
	cwi.buffer = <-cwi.freeChan
	go cwi.writer()
	for i := 0; i < cores; i++ {
		go cwi.checksummer()
	}
	return cwi
}

func (cwi *multiCoreChecksummedWriter) Write(v []byte) (int, error) {
	var n int
	for len(cwi.buffer.buf)+len(v) >= cwi.checksumInterval {
		n2 := cwi.checksumInterval - len(cwi.buffer.buf)
		cwi.buffer.buf = append(cwi.buffer.buf, v[:n2]...)
		s := cwi.buffer.seq + 1
		cwi.checksumChan <- cwi.buffer
		n += n2
		v = v[n2:]
		cwi.buffer = <-cwi.freeChan
		cwi.buffer.seq = s
	}
	if len(v) > 0 {
		cwi.buffer.buf = append(cwi.buffer.buf, v...)
		n += len(v)
	}
	cwi.lock.Lock()
	err := cwi.err
	cwi.lock.Unlock()
	return n, err
}

func (cwi *multiCoreChecksummedWriter) Close() error {
	cwi.lock.Lock()
	if cwi.closed {
		err := cwi.err
		cwi.lock.Unlock()
		return err
	}
	cwi.closed = true
	cwi.lock.Unlock()
	if len(cwi.buffer.buf) > 0 {
		cwi.checksumChan <- cwi.buffer
	}
	close(cwi.checksumChan)
	for i := 0; i < cap(cwi.checksumChan); i++ {
		<-cwi.doneChan
	}
	cwi.writeChan <- nil
	<-cwi.doneChan
	cwi.lock.Lock()
	err := cwi.err
	cwi.lock.Unlock()
	return err
}

func (cwi *multiCoreChecksummedWriter) checksummer() {
	for {
		b := <-cwi.checksumChan
		if b == nil {
			break
		}
		h := cwi.newHash()
		h.Write(b.buf)
		b.buf = b.buf[:len(b.buf)+4]
		binary.BigEndian.PutUint32(b.buf[len(b.buf)-4:], h.Sum32())
		cwi.writeChan <- b
	}
	cwi.doneChan <- struct{}{}
}

func (cwi *multiCoreChecksummedWriter) writer() {
	var seq int64
	var lastWasNil bool
	for {
		b := <-cwi.writeChan
		if b == nil {
			if lastWasNil {
				break
			}
			lastWasNil = true
			cwi.writeChan <- nil
			continue
		}
		lastWasNil = false
		if b.seq != seq {
			cwi.writeChan <- b
			continue
		}
		_, err := cwi.delegate.Write(b.buf)
		cwi.lock.Lock()
		cwi.err = err
		cwi.lock.Unlock()
		b.buf = b.buf[:0]
		cwi.freeChan <- b
		seq++
	}
	if c, ok := cwi.delegate.(io.Closer); ok {
		err := c.Close()
		cwi.lock.Lock()
		cwi.err = err
		cwi.lock.Unlock()
	}
	cwi.doneChan <- struct{}{}
}

type errDelegate struct {
}

var _ERR_DELEGATE = &errDelegate{}

func (ed *errDelegate) Read(v []byte) (int, error) {
	return 0, fmt.Errorf("closed")
}

func (ed *errDelegate) Write(v []byte) (int, error) {
	return 0, fmt.Errorf("closed")
}

func (ed *errDelegate) Seek(offset int64, whence int) (int64, error) {
	return 0, fmt.Errorf("closed")
}

func (ed *errDelegate) Close() error {
	return fmt.Errorf("already closed")
}
<|endoftext|>


<commit_before>// Package lumberjack provides a rolling logger.
//
// Note that this is v2.0 of lumberjack, and should be imported using gopkg.in
// thusly:
//
//   import "gopkg.in/natefinch/lumberjack.v2"
//
// The package name remains simply lumberjack, and the code resides at
// https://github.com/natefinch/lumberjack under the v2.0 branch.
//
// Lumberjack is intended to be one part of a logging infrastructure.
// It is not an all-in-one solution, but instead is a pluggable
// component at the bottom of the logging stack that simply controls the files
// to which logs are written.
//
// Lumberjack plays well with any logging package that can write to an
// io.Writer, including the standard library's log package.
//
// Lumberjack assumes that only one process is writing to the output files.
// Using the same lumberjack configuration from multiple processes on the same
// machine will result in improper behavior.
package lumberjack

import (
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"sort"
	"strings"
	"sync"
	"time"
)

const (
	backupTimeFormat = "2006-01-02T15-04-05.000"
	defaultMaxSize   = 100
)

// ensure we always implement io.WriteCloser
var _ io.WriteCloser = (*Logger)(nil)

// Logger is an io.WriteCloser that writes to the specified filename.
//
// Logger opens or creates the logfile on first Write.  If the file exists and
// is less than MaxSize megabytes, lumberjack will open and append to that file.
// If the file exists and its size is >= MaxSize megabytes, the file is renamed
// by putting the current time in a timestamp in the name immediately before the
// file's extension (or the end of the filename if there's no extension). A new
// log file is then created using original filename.
//
// Whenever a write would cause the current log file exceed MaxSize megabytes,
// the current file is closed, renamed, and a new log file created with the
// original name. Thus, the filename you give Logger is always the "current" log
// file.
//
// Cleaning Up Old Log Files
//
// Whenever a new logfile gets created, old log files may be deleted.  The most
// recent files according to the encoded timestamp will be retained, up to a
// number equal to MaxBackups (or all of them if MaxBackups is 0).  Any files
// with an encoded timestamp older than MaxAge days are deleted, regardless of
// MaxBackups.  Note that the time encoded in the timestamp is the rotation
// time, which may differ from the last time that file was written to.
//
// If MaxBackups and MaxAge are both 0, no old log files will be deleted.
type Logger struct {
	// Filename is the file to write logs to.  Backup log files will be retained
	// in the same directory.  It uses <processname>-lumberjack.log in
	// os.TempDir() if empty.
	Filename string `json:"filename" yaml:"filename"`

	// MaxSize is the maximum size in megabytes of the log file before it gets
	// rotated. It defaults to 100 megabytes.
	MaxSize int `json:"maxsize" yaml:"maxsize"`

	// MaxAge is the maximum number of days to retain old log files based on the
	// timestamp encoded in their filename.  Note that a day is defined as 24
	// hours and may not exactly correspond to calendar days due to daylight
	// savings, leap seconds, etc. The default is not to remove old log files
	// based on age.
	MaxAge int `json:"maxage" yaml:"maxage"`

	// MaxBackups is the maximum number of old log files to retain.  The default
	// is to retain all old log files (though MaxAge may still cause them to get
	// deleted.)
	MaxBackups int `json:"maxbackups" yaml:"maxbackups"`

	// LocalTime determines if the time used for formatting the timestamps in
	// backup files is the computer's local time.  The default is to use UTC
	// time.
	LocalTime bool `json:"localtime" yaml:"localtime"`

	size int64
	file *os.File
	mu   sync.Mutex
}

var (
	// currentTime exists so it can be mocked out by tests.
	currentTime = time.Now

	// os_Stat exists so it can be mocked out by tests.
	os_Stat = os.Stat

	// megabyte is the conversion factor between MaxSize and bytes.  It is a
	// variable so tests can mock it out and not need to write megabytes of data
	// to disk.
	megabyte = 1024 * 1024
)

// Write implements io.Writer.  If a write would cause the log file to be larger
// than MaxSize, the file is closed, renamed to include a timestamp of the
// current time, and a new log file is created using the original log file name.
// If the length of the write is greater than MaxSize, an error is returned.
func (l *Logger) Write(p []byte) (n int, err error) {
	l.mu.Lock()
	defer l.mu.Unlock()

	writeLen := int64(len(p))
	if writeLen > l.max() {
		return 0, fmt.Errorf(
			"write length %d exceeds maximum file size %d", writeLen, l.max(),
		)
	}

	if l.file == nil {
		if err = l.openExistingOrNew(len(p)); err != nil {
			return 0, err
		}
	}

	if l.size+writeLen > l.max() {
		if err := l.rotate(); err != nil {
			return 0, err
		}
	}

	n, err = l.file.Write(p)
	l.size += int64(n)

	return n, err
}

// Close implements io.Closer, and closes the current logfile.
func (l *Logger) Close() error {
	l.mu.Lock()
	defer l.mu.Unlock()
	return l.close()
}

// close closes the file if it is open.
func (l *Logger) close() error {
	if l.file == nil {
		return nil
	}
	err := l.file.Close()
	l.file = nil
	return err
}

// Rotate causes Logger to close the existing log file and immediately create a
// new one.  This is a helper function for applications that want to initiate
// rotations outside of the normal rotation rules, such as in response to
// SIGHUP.  After rotating, this initiates a cleanup of old log files according
// to the normal rules.
func (l *Logger) Rotate() error {
	l.mu.Lock()
	defer l.mu.Unlock()
	return l.rotate()
}

// rotate closes the current file, moves it aside with a timestamp in the name,
// (if it exists), opens a new file with the original filename, and then runs
// cleanup.
func (l *Logger) rotate() error {
	if err := l.close(); err != nil {
		return err
	}

	if err := l.openNew(); err != nil {
		return err
	}
	return l.cleanup()
}

// openNew opens a new log file for writing, moving any old log file out of the
// way.  This methods assumes the file has already been closed.
func (l *Logger) openNew() error {
	err := os.MkdirAll(l.dir(), 0744)
	if err != nil {
		return fmt.Errorf("can't make directories for new logfile: %s", err)
	}

	name := l.filename()
	mode := os.FileMode(0644)
	info, err := os_Stat(name)
	if err == nil {
		// Copy the mode off the old logfile.
		mode = info.Mode()
		// move the existing file
		newname := backupName(name, l.LocalTime)
		if err := os.Rename(name, newname); err != nil {
			return fmt.Errorf("can't rename log file: %s", err)
		}

		// this is a no-op anywhere but linux
		if err := chown(name, info); err != nil {
			return err
		}
	}

	// we use truncate here because this should only get called when we've moved
	// the file ourselves. if someone else creates the file in the meantime,
	// just wipe out the contents.
	f, err := os.OpenFile(name, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, mode)
	if err != nil {
		return fmt.Errorf("can't open new logfile: %s", err)
	}
	l.file = f
	l.size = 0
	return nil
}

// backupName creates a new filename from the given name, inserting a timestamp
// between the filename and the extension, using the local time if requested
// (otherwise UTC).
func backupName(name string, local bool) string {
	dir := filepath.Dir(name)
	filename := filepath.Base(name)
	ext := filepath.Ext(filename)
	prefix := filename[:len(filename)-len(ext)]
	t := currentTime()
	if !local {
		t = t.UTC()
	}

	timestamp := t.Format(backupTimeFormat)
	return filepath.Join(dir, fmt.Sprintf("%s-%s%s", prefix, timestamp, ext))
}

// openExistingOrNew opens the logfile if it exists and if the current write
// would not put it over MaxSize.  If there is no such file or the write would
// put it over the MaxSize, a new file is created.
func (l *Logger) openExistingOrNew(writeLen int) error {
	filename := l.filename()
	info, err := os_Stat(filename)
	if os.IsNotExist(err) {
		return l.openNew()
	}
	if err != nil {
		return fmt.Errorf("error getting log file info: %s", err)
	}

	if info.Size()+int64(writeLen) >= l.max() {
		return l.rotate()
	}

	file, err := os.OpenFile(filename, os.O_APPEND|os.O_WRONLY, 0644)
	if err != nil {
		// if we fail to open the old log file for some reason, just ignore
		// it and open a new log file.
		return l.openNew()
	}
	l.file = file
	l.size = info.Size()
	return nil
}

// genFilename generates the name of the logfile from the current time.
func (l *Logger) filename() string {
	if l.Filename != "" {
		return l.Filename
	}
	name := filepath.Base(os.Args[0]) + "-lumberjack.log"
	return filepath.Join(os.TempDir(), name)
}

// cleanup deletes old log files, keeping at most l.MaxBackups files, as long as
// none of them are older than MaxAge.
func (l *Logger) cleanup() error {
	if l.MaxBackups == 0 && l.MaxAge == 0 {
		return nil
	}

	files, err := l.oldLogFiles()
	if err != nil {
		return err
	}

	var deletes []logInfo

	if l.MaxBackups > 0 && l.MaxBackups < len(files) {
		deletes = files[l.MaxBackups:]
		files = files[:l.MaxBackups]
	}
	if l.MaxAge > 0 {
		diff := time.Duration(int64(24*time.Hour) * int64(l.MaxAge))

		cutoff := currentTime().Add(-1 * diff)

		for _, f := range files {
			if f.timestamp.Before(cutoff) {
				deletes = append(deletes, f)
			}
		}
	}

	if len(deletes) == 0 {
		return nil
	}

	go deleteAll(l.dir(), deletes)

	return nil
}

func deleteAll(dir string, files []logInfo) {
	// remove files on a separate goroutine
	for _, f := range files {
		// what am I going to do, log this?
		_ = os.Remove(filepath.Join(dir, f.Name()))
	}
}

// oldLogFiles returns the list of backup log files stored in the same
// directory as the current log file, sorted by ModTime
func (l *Logger) oldLogFiles() ([]logInfo, error) {
	files, err := ioutil.ReadDir(l.dir())
	if err != nil {
		return nil, fmt.Errorf("can't read log file directory: %s", err)
	}
	logFiles := []logInfo{}

	prefix, ext := l.prefixAndExt()

	for _, f := range files {
		if f.IsDir() {
			continue
		}
		name := l.timeFromName(f.Name(), prefix, ext)
		if name == "" {
			continue
		}
		t, err := time.Parse(backupTimeFormat, name)
		if err == nil {
			logFiles = append(logFiles, logInfo{t, f})
		}
		// error parsing means that the suffix at the end was not generated
		// by lumberjack, and therefore it's not a backup file.
	}

	sort.Sort(byFormatTime(logFiles))

	return logFiles, nil
}

// timeFromName extracts the formatted time from the filename by stripping off
// the filename's prefix and extension. This prevents someone's filename from
// confusing time.parse.
func (l *Logger) timeFromName(filename, prefix, ext string) string {
	if !strings.HasPrefix(filename, prefix) {
		return ""
	}
	filename = filename[len(prefix):]

	if !strings.HasSuffix(filename, ext) {
		return ""
	}
	filename = filename[:len(filename)-len(ext)]
	return filename
}

// max returns the maximum size in bytes of log files before rolling.
func (l *Logger) max() int64 {
	if l.MaxSize == 0 {
		return int64(defaultMaxSize * megabyte)
	}
	return int64(l.MaxSize) * int64(megabyte)
}

// dir returns the directory for the current filename.
func (l *Logger) dir() string {
	return filepath.Dir(l.filename())
}

// prefixAndExt returns the filename part and extension part from the Logger's
// filename.
func (l *Logger) prefixAndExt() (prefix, ext string) {
	filename := filepath.Base(l.filename())
	ext = filepath.Ext(filename)
	prefix = filename[:len(filename)-len(ext)] + "-"
	return prefix, ext
}

// logInfo is a convenience struct to return the filename and its embedded
// timestamp.
type logInfo struct {
	timestamp time.Time
	os.FileInfo
}

// byFormatTime sorts by newest time formatted in the name.
type byFormatTime []logInfo

func (b byFormatTime) Less(i, j int) bool {
	return b[i].timestamp.After(b[j].timestamp)
}

func (b byFormatTime) Swap(i, j int) {
	b[i], b[j] = b[j], b[i]
}

func (b byFormatTime) Len() int {
	return len(b)
}
<commit_msg>add Callback when a log file is opened<commit_after>// Package lumberjack provides a rolling logger.
//
// Note that this is v2.0 of lumberjack, and should be imported using gopkg.in
// thusly:
//
//   import "gopkg.in/natefinch/lumberjack.v2"
//
// The package name remains simply lumberjack, and the code resides at
// https://github.com/natefinch/lumberjack under the v2.0 branch.
//
// Lumberjack is intended to be one part of a logging infrastructure.
// It is not an all-in-one solution, but instead is a pluggable
// component at the bottom of the logging stack that simply controls the files
// to which logs are written.
//
// Lumberjack plays well with any logging package that can write to an
// io.Writer, including the standard library's log package.
//
// Lumberjack assumes that only one process is writing to the output files.
// Using the same lumberjack configuration from multiple processes on the same
// machine will result in improper behavior.
package lumberjack

import (
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"sort"
	"strings"
	"sync"
	"time"
)

const (
	backupTimeFormat = "2006-01-02T15-04-05.000"
	defaultMaxSize   = 100
)

// ensure we always implement io.WriteCloser
var _ io.WriteCloser = (*Logger)(nil)

// Logger is an io.WriteCloser that writes to the specified filename.
//
// Logger opens or creates the logfile on first Write.  If the file exists and
// is less than MaxSize megabytes, lumberjack will open and append to that file.
// If the file exists and its size is >= MaxSize megabytes, the file is renamed
// by putting the current time in a timestamp in the name immediately before the
// file's extension (or the end of the filename if there's no extension). A new
// log file is then created using original filename.
//
// Whenever a write would cause the current log file exceed MaxSize megabytes,
// the current file is closed, renamed, and a new log file created with the
// original name. Thus, the filename you give Logger is always the "current" log
// file.
//
// Cleaning Up Old Log Files
//
// Whenever a new logfile gets created, old log files may be deleted.  The most
// recent files according to the encoded timestamp will be retained, up to a
// number equal to MaxBackups (or all of them if MaxBackups is 0).  Any files
// with an encoded timestamp older than MaxAge days are deleted, regardless of
// MaxBackups.  Note that the time encoded in the timestamp is the rotation
// time, which may differ from the last time that file was written to.
//
// If MaxBackups and MaxAge are both 0, no old log files will be deleted.
type Logger struct {
	// Filename is the file to write logs to.  Backup log files will be retained
	// in the same directory.  It uses <processname>-lumberjack.log in
	// os.TempDir() if empty.
	Filename string `json:"filename" yaml:"filename"`

	// MaxSize is the maximum size in megabytes of the log file before it gets
	// rotated. It defaults to 100 megabytes.
	MaxSize int `json:"maxsize" yaml:"maxsize"`

	// MaxAge is the maximum number of days to retain old log files based on the
	// timestamp encoded in their filename.  Note that a day is defined as 24
	// hours and may not exactly correspond to calendar days due to daylight
	// savings, leap seconds, etc. The default is not to remove old log files
	// based on age.
	MaxAge int `json:"maxage" yaml:"maxage"`

	// MaxBackups is the maximum number of old log files to retain.  The default
	// is to retain all old log files (though MaxAge may still cause them to get
	// deleted.)
	MaxBackups int `json:"maxbackups" yaml:"maxbackups"`

	// LocalTime determines if the time used for formatting the timestamps in
	// backup files is the computer's local time.  The default is to use UTC
	// time.
	LocalTime bool `json:"localtime" yaml:"localtime"`

	// Callback is called when the log file is opened. openNew indicates if open
	// a new file or an exist one, the return data will be written in the log
	// file
	OpenCallback func(openNew bool) []byte

	size int64
	file *os.File
	mu   sync.Mutex
}

var (
	// currentTime exists so it can be mocked out by tests.
	currentTime = time.Now

	// os_Stat exists so it can be mocked out by tests.
	os_Stat = os.Stat

	// megabyte is the conversion factor between MaxSize and bytes.  It is a
	// variable so tests can mock it out and not need to write megabytes of data
	// to disk.
	megabyte = 1024 * 1024
)

// Write implements io.Writer.  If a write would cause the log file to be larger
// than MaxSize, the file is closed, renamed to include a timestamp of the
// current time, and a new log file is created using the original log file name.
// If the length of the write is greater than MaxSize, an error is returned.
func (l *Logger) Write(p []byte) (n int, err error) {
	l.mu.Lock()
	defer l.mu.Unlock()

	writeLen := int64(len(p))
	if writeLen > l.max() {
		return 0, fmt.Errorf(
			"write length %d exceeds maximum file size %d", writeLen, l.max(),
		)
	}

	if l.file == nil {
		if err = l.openExistingOrNew(len(p)); err != nil {
			return 0, err
		}
	}

	if l.size+writeLen > l.max() {
		if err := l.rotate(); err != nil {
			return 0, err
		}
	}

	n, err = l.file.Write(p)
	l.size += int64(n)

	return n, err
}

// Close implements io.Closer, and closes the current logfile.
func (l *Logger) Close() error {
	l.mu.Lock()
	defer l.mu.Unlock()
	return l.close()
}

// close closes the file if it is open.
func (l *Logger) close() error {
	if l.file == nil {
		return nil
	}
	err := l.file.Close()
	l.file = nil
	return err
}

// Rotate causes Logger to close the existing log file and immediately create a
// new one.  This is a helper function for applications that want to initiate
// rotations outside of the normal rotation rules, such as in response to
// SIGHUP.  After rotating, this initiates a cleanup of old log files according
// to the normal rules.
func (l *Logger) Rotate() error {
	l.mu.Lock()
	defer l.mu.Unlock()
	return l.rotate()
}

// rotate closes the current file, moves it aside with a timestamp in the name,
// (if it exists), opens a new file with the original filename, and then runs
// cleanup.
func (l *Logger) rotate() error {
	if err := l.close(); err != nil {
		return err
	}

	if err := l.openNew(); err != nil {
		return err
	}
	return l.cleanup()
}

// openNew opens a new log file for writing, moving any old log file out of the
// way.  This methods assumes the file has already been closed.
func (l *Logger) openNew() error {
	err := os.MkdirAll(l.dir(), 0744)
	if err != nil {
		return fmt.Errorf("can't make directories for new logfile: %s", err)
	}

	name := l.filename()
	mode := os.FileMode(0644)
	info, err := os_Stat(name)
	if err == nil {
		// Copy the mode off the old logfile.
		mode = info.Mode()
		// move the existing file
		newname := backupName(name, l.LocalTime)
		if err := os.Rename(name, newname); err != nil {
			return fmt.Errorf("can't rename log file: %s", err)
		}

		// this is a no-op anywhere but linux
		if err := chown(name, info); err != nil {
			return err
		}
	}

	// we use truncate here because this should only get called when we've moved
	// the file ourselves. if someone else creates the file in the meantime,
	// just wipe out the contents.
	f, err := os.OpenFile(name, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, mode)
	if err != nil {
		return fmt.Errorf("can't open new logfile: %s", err)
	}
	l.file = f
	l.size = 0

	if l.OpenCallback != nil {
		data := l.OpenCallback(true)
		if data != nil {
			n, _ := l.file.Write(data)
			l.size += int64(n)
		}
	}
	return nil
}

// backupName creates a new filename from the given name, inserting a timestamp
// between the filename and the extension, using the local time if requested
// (otherwise UTC).
func backupName(name string, local bool) string {
	dir := filepath.Dir(name)
	filename := filepath.Base(name)
	ext := filepath.Ext(filename)
	prefix := filename[:len(filename)-len(ext)]
	t := currentTime()
	if !local {
		t = t.UTC()
	}

	timestamp := t.Format(backupTimeFormat)
	return filepath.Join(dir, fmt.Sprintf("%s-%s%s", prefix, timestamp, ext))
}

// openExistingOrNew opens the logfile if it exists and if the current write
// would not put it over MaxSize.  If there is no such file or the write would
// put it over the MaxSize, a new file is created.
func (l *Logger) openExistingOrNew(writeLen int) error {
	filename := l.filename()
	info, err := os_Stat(filename)
	if os.IsNotExist(err) {
		return l.openNew()
	}
	if err != nil {
		return fmt.Errorf("error getting log file info: %s", err)
	}

	if info.Size()+int64(writeLen) >= l.max() {
		return l.rotate()
	}

	file, err := os.OpenFile(filename, os.O_APPEND|os.O_WRONLY, 0644)
	if err != nil {
		// if we fail to open the old log file for some reason, just ignore
		// it and open a new log file.
		return l.openNew()
	}
	l.file = file
	l.size = info.Size()

	if l.OpenCallback != nil {
		data := l.OpenCallback(false)
		if data != nil {
			n, _ := l.file.Write(data)
			l.size += int64(n)
		}
	}
	return nil
}

// genFilename generates the name of the logfile from the current time.
func (l *Logger) filename() string {
	if l.Filename != "" {
		return l.Filename
	}
	name := filepath.Base(os.Args[0]) + "-lumberjack.log"
	return filepath.Join(os.TempDir(), name)
}

// cleanup deletes old log files, keeping at most l.MaxBackups files, as long as
// none of them are older than MaxAge.
func (l *Logger) cleanup() error {
	if l.MaxBackups == 0 && l.MaxAge == 0 {
		return nil
	}

	files, err := l.oldLogFiles()
	if err != nil {
		return err
	}

	var deletes []logInfo

	if l.MaxBackups > 0 && l.MaxBackups < len(files) {
		deletes = files[l.MaxBackups:]
		files = files[:l.MaxBackups]
	}
	if l.MaxAge > 0 {
		diff := time.Duration(int64(24*time.Hour) * int64(l.MaxAge))

		cutoff := currentTime().Add(-1 * diff)

		for _, f := range files {
			if f.timestamp.Before(cutoff) {
				deletes = append(deletes, f)
			}
		}
	}

	if len(deletes) == 0 {
		return nil
	}

	go deleteAll(l.dir(), deletes)

	return nil
}

func deleteAll(dir string, files []logInfo) {
	// remove files on a separate goroutine
	for _, f := range files {
		// what am I going to do, log this?
		_ = os.Remove(filepath.Join(dir, f.Name()))
	}
}

// oldLogFiles returns the list of backup log files stored in the same
// directory as the current log file, sorted by ModTime
func (l *Logger) oldLogFiles() ([]logInfo, error) {
	files, err := ioutil.ReadDir(l.dir())
	if err != nil {
		return nil, fmt.Errorf("can't read log file directory: %s", err)
	}
	logFiles := []logInfo{}

	prefix, ext := l.prefixAndExt()

	for _, f := range files {
		if f.IsDir() {
			continue
		}
		name := l.timeFromName(f.Name(), prefix, ext)
		if name == "" {
			continue
		}
		t, err := time.Parse(backupTimeFormat, name)
		if err == nil {
			logFiles = append(logFiles, logInfo{t, f})
		}
		// error parsing means that the suffix at the end was not generated
		// by lumberjack, and therefore it's not a backup file.
	}

	sort.Sort(byFormatTime(logFiles))

	return logFiles, nil
}

// timeFromName extracts the formatted time from the filename by stripping off
// the filename's prefix and extension. This prevents someone's filename from
// confusing time.parse.
func (l *Logger) timeFromName(filename, prefix, ext string) string {
	if !strings.HasPrefix(filename, prefix) {
		return ""
	}
	filename = filename[len(prefix):]

	if !strings.HasSuffix(filename, ext) {
		return ""
	}
	filename = filename[:len(filename)-len(ext)]
	return filename
}

// max returns the maximum size in bytes of log files before rolling.
func (l *Logger) max() int64 {
	if l.MaxSize == 0 {
		return int64(defaultMaxSize * megabyte)
	}
	return int64(l.MaxSize) * int64(megabyte)
}

// dir returns the directory for the current filename.
func (l *Logger) dir() string {
	return filepath.Dir(l.filename())
}

// prefixAndExt returns the filename part and extension part from the Logger's
// filename.
func (l *Logger) prefixAndExt() (prefix, ext string) {
	filename := filepath.Base(l.filename())
	ext = filepath.Ext(filename)
	prefix = filename[:len(filename)-len(ext)] + "-"
	return prefix, ext
}

// logInfo is a convenience struct to return the filename and its embedded
// timestamp.
type logInfo struct {
	timestamp time.Time
	os.FileInfo
}

// byFormatTime sorts by newest time formatted in the name.
type byFormatTime []logInfo

func (b byFormatTime) Less(i, j int) bool {
	return b[i].timestamp.After(b[j].timestamp)
}

func (b byFormatTime) Swap(i, j int) {
	b[i], b[j] = b[j], b[i]
}

func (b byFormatTime) Len() int {
	return len(b)
}
<|endoftext|>


<commit_before>/*
 * Minio S3verify Library for Amazon S3 Compatible Cloud Storage (C) 2016 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package main

import (
	"bytes"
	"encoding/hex"
	"encoding/xml"
	"fmt"
	"io/ioutil"
	"net/http"

	"github.com/minio/s3verify/signv4"
)

var (
	testBuckets = [][]BucketInfo{
		validBuckets,
		invalidBuckets,
	}
	validBuckets = []BucketInfo{
		BucketInfo{
			Name: "s3verify-put-bucket-test",
		},
		BucketInfo{
			Name: "s3verify-put-bucket-copy-test",
		},
	}
	// See http://docs.aws.amazon.com/AmazonS3/latest/dev/BucketRestrictions.html for all bucket naming restrictions.
	invalidBuckets = []BucketInfo{
		BucketInfo{
			Name: "s3", // Bucket names must be at least 3 chars long.
		},
		BucketInfo{
			Name: "babcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwzyz", // Bucket names must be less than 63 chars long. This is only on regions other than us-east-1.
		},
		BucketInfo{
			Name: "S3verify", // Bucket names must start with a lowercase letter or a number.
		},
		BucketInfo{
			Name: "192.168.5.4", // Bucket names must not be formatted as an IP address.
		},
		BucketInfo{
			Name: "s3..verify", // Bucket names can not have adjacent periods in them.
		},
		BucketInfo{
			Name: ".s3verify", // Bucket names can not start with periods.
		},
		BucketInfo{
			Name: "s3verify.", // Bucket names can not end with periods.
		},
	}
)

// newPutBucketReq - Create a new Make bucket request.
func newPutBucketReq(config ServerConfig, bucketName string) (*http.Request, error) {
	// putBucketReq - hardcode the static portions of a new Make Bucket request.
	var putBucketReq = &http.Request{
		Header: map[string][]string{
			"X-Amz-Content-Sha256": {hex.EncodeToString(signv4.Sum256([]byte{}))},
		},
		Method: "PUT",
		// Body: will be set if the region is different from us-east-1.
	}

	targetURL, err := makeTargetURL(config.Endpoint, bucketName, "", config.Region, nil)
	if err != nil {
		return nil, err
	}
	putBucketReq.URL = targetURL
	if config.Region != globalDefaultRegion { // Must set the request elements for non us-east-1 regions.
		bucketConfig := createBucketConfiguration{}
		bucketConfig.Location = config.Region
		bucketConfigBytes, err := xml.Marshal(bucketConfig)
		if err != nil {
			return nil, err
		}
		bucketConfigBuffer := bytes.NewReader(bucketConfigBytes)
		_, sha256Sum, contentLength, err := computeHash(bucketConfigBuffer)
		if err != nil {
			return nil, err
		}
		// Set the body.
		putBucketReq.Body = ioutil.NopCloser(bucketConfigBuffer)
		putBucketReq.ContentLength = contentLength
		// Fill request headers and URL.
		putBucketReq.Header.Set("X-Amz-Content-Sha256", hex.EncodeToString(sha256Sum))
	}
	putBucketReq = signv4.SignV4(*putBucketReq, config.Access, config.Secret, config.Region)
	return putBucketReq, nil
}

// putBucketVerify - Check the response Body, Header, Status for AWS S3 compliance.
func putBucketVerify(res *http.Response, bucketName, expectedStatus string, expectedError ErrorResponse) error {
	if err := verifyStatusPutBucket(res, expectedStatus); err != nil {
		return err
	}
	if err := verifyHeaderPutBucket(res, bucketName, expectedStatus); err != nil {
		return err
	}
	if err := verifyBodyPutBucket(res, expectedError); err != nil {
		return err
	}
	return nil
}

// verifyStatusPutBucket - Check the response status for AWS S3 compliance.
func verifyStatusPutBucket(res *http.Response, expectedStatus string) error {
	if res.Status != expectedStatus {
		err := fmt.Errorf("Unexpected Response Status Code: wanted %v, got %v", expectedStatus, res.StatusCode)
		return err
	}
	return nil
}

// verifyBodyPutBucket - Check the response body for AWS S3 compliance.
func verifyBodyPutBucket(res *http.Response, expectedError ErrorResponse) error {
	if expectedError.Message != "" {
		resError := ErrorResponse{}
		err := xmlDecoder(res.Body, &resError)
		if err != nil {
			return err
		}
		if resError.Message != expectedError.Message {
			err := fmt.Errorf("Unexpected Error Message: wanted %v, got %v", expectedError.Message, resError.Message)
			return err
		}
		return nil
	} else {
		body, err := ioutil.ReadAll(res.Body)
		if err != nil {
			return err
		}
		// There is no body returned by a Put Bucket request.
		if string(body) != "" {
			err := fmt.Errorf("Unexpected Body: %v", string(body))
			return err
		}
	}
	return nil
}

// verifyHeaderPutBucket - Check the response header for AWS S3 compliance.
func verifyHeaderPutBucket(res *http.Response, bucketName, expectedStatus string) error {
	if expectedStatus == "200 OK" {
		location := res.Header["Location"][0]
		if location != "http://"+bucketName+".s3.amazonaws.com/" && location != "/"+bucketName {
			// TODO: wait for Minio server to fix endpoint detection.
			err := fmt.Errorf("Unexpected Location: got %v", location)
			return err
		}
	}
	if err := verifyStandardHeaders(res); err != nil {
		return err
	}
	return nil
}

// Test the PutBucket API when no extra headers are set. This creates three new buckets and leaves them for the next tests to use.
func mainPutBucket(config ServerConfig, curTest int) bool {
	message := fmt.Sprintf("[%02d/%d] PutBucket:", curTest, globalTotalNumTest)
	// Spin the scanBar
	scanBar(message)
	// Test that valid names are PUT correctly.
	for _, bucket := range validBuckets {
		// Spin the scanBar
		scanBar(message)
		// Create a new Make bucket request.
		req, err := newPutBucketReq(config, bucket.Name)
		if err != nil {
			printMessage(message, err)
			return false
		}
		// Spin the scanBar
		scanBar(message)
		// Execute the request.
		res, err := execRequest(req, config.Client)
		if err != nil {
			printMessage(message, err)
			return false
		}
		// Spin the scanBar
		scanBar(message)
		// Check the responses Body, Status, Header.
		if err := putBucketVerify(res, bucket.Name, "200 OK", ErrorResponse{}); err != nil {
			printMessage(message, err)
			return false
		}
		// Spin the scanBar
		scanBar(message)
	}
	expectedError := ErrorResponse{
		Message: "The specified bucket is not valid.",
	}
	// Test that all invalid names fail correctly.
	for _, bucket := range invalidBuckets {
		// Spin scanBar
		scanBar(message)
		// Create a new PUT bucket request.
		req, err := newPutBucketReq(config, bucket.Name)
		if err != nil {
			printMessage(message, err)
			return false
		}
		// Spin scanBar
		scanBar(message)
		// Execute the request.
		res, err := execRequest(req, config.Client)
		if err != nil {
			printMessage(message, err)
			return false
		}
		// Spin scanBar
		scanBar(message)
		// Verify that the request failed as predicted.
		if err := putBucketVerify(res, bucket.Name, "400 Bad Request", expectedError); err != nil {
			printMessage(message, err)
			return false
		}
		// Spin scanBar
		scanBar(message)
	}
	printMessage(message, nil)
	return true
}
<commit_msg>PutBucket: use random bucket names throughout s3verify tests in order to allow multiple users at once (#116)<commit_after>/*
 * Minio S3verify Library for Amazon S3 Compatible Cloud Storage (C) 2016 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package main

import (
	"bytes"
	"encoding/hex"
	"encoding/xml"
	"fmt"
	"io/ioutil"
	"math/rand"
	"net/http"
	"time"

	"github.com/minio/s3verify/signv4"
)

var (
	// Make two random buckets below in the test.
	validBuckets = make([]BucketInfo, 2)
	// See http://docs.aws.amazon.com/AmazonS3/latest/dev/BucketRestrictions.html for all bucket naming restrictions.
	invalidBuckets = []BucketInfo{
		BucketInfo{
			Name: "s3", // Bucket names must be at least 3 chars long.
		},
		BucketInfo{
			Name: "babcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwzyz", // Bucket names must be less than 63 chars long. This is only on regions other than us-east-1.
		},
		BucketInfo{
			Name: "S3verify", // Bucket names must start with a lowercase letter or a number.
		},
		BucketInfo{
			Name: "192.168.5.4", // Bucket names must not be formatted as an IP address.
		},
		BucketInfo{
			Name: "s3..verify", // Bucket names can not have adjacent periods in them.
		},
		BucketInfo{
			Name: ".s3verify", // Bucket names can not start with periods.
		},
		BucketInfo{
			Name: "s3verify.", // Bucket names can not end with periods.
		},
	}
)

// newPutBucketReq - Create a new Make bucket request.
func newPutBucketReq(config ServerConfig, bucketName string) (*http.Request, error) {
	// putBucketReq - hardcode the static portions of a new Make Bucket request.
	var putBucketReq = &http.Request{
		Header: map[string][]string{
			"X-Amz-Content-Sha256": {hex.EncodeToString(signv4.Sum256([]byte{}))},
		},
		Method: "PUT",
		// Body: will be set if the region is different from us-east-1.
	}

	targetURL, err := makeTargetURL(config.Endpoint, bucketName, "", config.Region, nil)
	if err != nil {
		return nil, err
	}
	putBucketReq.URL = targetURL
	if config.Region != globalDefaultRegion { // Must set the request elements for non us-east-1 regions.
		bucketConfig := createBucketConfiguration{}
		bucketConfig.Location = config.Region
		bucketConfigBytes, err := xml.Marshal(bucketConfig)
		if err != nil {
			return nil, err
		}
		bucketConfigBuffer := bytes.NewReader(bucketConfigBytes)
		_, sha256Sum, contentLength, err := computeHash(bucketConfigBuffer)
		if err != nil {
			return nil, err
		}
		// Set the body.
		putBucketReq.Body = ioutil.NopCloser(bucketConfigBuffer)
		putBucketReq.ContentLength = contentLength
		// Fill request headers and URL.
		putBucketReq.Header.Set("X-Amz-Content-Sha256", hex.EncodeToString(sha256Sum))
	}
	putBucketReq = signv4.SignV4(*putBucketReq, config.Access, config.Secret, config.Region)
	return putBucketReq, nil
}

// putBucketVerify - Check the response Body, Header, Status for AWS S3 compliance.
func putBucketVerify(res *http.Response, bucketName, expectedStatus string, expectedError ErrorResponse) error {
	if err := verifyStatusPutBucket(res, expectedStatus); err != nil {
		return err
	}
	if err := verifyHeaderPutBucket(res, bucketName, expectedStatus); err != nil {
		return err
	}
	if err := verifyBodyPutBucket(res, expectedError); err != nil {
		return err
	}
	return nil
}

// verifyStatusPutBucket - Check the response status for AWS S3 compliance.
func verifyStatusPutBucket(res *http.Response, expectedStatus string) error {
	if res.Status != expectedStatus {
		err := fmt.Errorf("Unexpected Response Status Code: wanted %v, got %v", expectedStatus, res.StatusCode)
		return err
	}
	return nil
}

// verifyBodyPutBucket - Check the response body for AWS S3 compliance.
func verifyBodyPutBucket(res *http.Response, expectedError ErrorResponse) error {
	if expectedError.Message != "" {
		resError := ErrorResponse{}
		err := xmlDecoder(res.Body, &resError)
		if err != nil {
			return err
		}
		if resError.Message != expectedError.Message {
			err := fmt.Errorf("Unexpected Error Message: wanted %v, got %v", expectedError.Message, resError.Message)
			return err
		}
		return nil
	} else {
		body, err := ioutil.ReadAll(res.Body)
		if err != nil {
			return err
		}
		// There is no body returned by a Put Bucket request.
		if string(body) != "" {
			err := fmt.Errorf("Unexpected Body: %v", string(body))
			return err
		}
	}
	return nil
}

// verifyHeaderPutBucket - Check the response header for AWS S3 compliance.
func verifyHeaderPutBucket(res *http.Response, bucketName, expectedStatus string) error {
	if expectedStatus == "200 OK" {
		location := res.Header["Location"][0]
		if location != "http://"+bucketName+".s3.amazonaws.com/" && location != "/"+bucketName {
			// TODO: wait for Minio server to fix endpoint detection.
			err := fmt.Errorf("Unexpected Location: got %v", location)
			return err
		}
	}
	if err := verifyStandardHeaders(res); err != nil {
		return err
	}
	return nil
}

// Test the PutBucket API when no extra headers are set. This creates two new buckets and leaves them for the next tests to use.
func mainPutBucket(config ServerConfig, curTest int) bool {
	message := fmt.Sprintf("[%02d/%d] PutBucket:", curTest, globalTotalNumTest)
	// Spin the scanBar
	scanBar(message)
	// Create two new buckets with valid names.
	for i := 0; i < 2; i++ {
		validBucket := BucketInfo{
			Name: randString(60, rand.NewSource(time.Now().UnixNano()), "s3verify"),
		}
		validBuckets[i] = validBucket
		// Spin the scanBar
		scanBar(message)
		// Create a new Make bucket request.
		req, err := newPutBucketReq(config, validBucket.Name)
		if err != nil {
			printMessage(message, err)
			return false
		}
		// Spin the scanBar
		scanBar(message)
		// Execute the request.
		res, err := execRequest(req, config.Client)
		if err != nil {
			printMessage(message, err)
			return false
		}
		// Spin the scanBar
		scanBar(message)
		// Check the responses Body, Status, Header.
		if err := putBucketVerify(res, validBucket.Name, "200 OK", ErrorResponse{}); err != nil {
			printMessage(message, err)
			return false
		}
		// Spin the scanBar
		scanBar(message)
	}
	expectedError := ErrorResponse{
		Message: "The specified bucket is not valid.",
	}
	// Test that all invalid names fail correctly.
	for _, bucket := range invalidBuckets {
		// Spin scanBar
		scanBar(message)
		// Create a new PUT bucket request.
		req, err := newPutBucketReq(config, bucket.Name)
		if err != nil {
			printMessage(message, err)
			return false
		}
		// Spin scanBar
		scanBar(message)
		// Execute the request.
		res, err := execRequest(req, config.Client)
		if err != nil {
			printMessage(message, err)
			return false
		}
		// Spin scanBar
		scanBar(message)
		// Verify that the request failed as predicted.
		if err := putBucketVerify(res, bucket.Name, "400 Bad Request", expectedError); err != nil {
			printMessage(message, err)
			return false
		}
		// Spin scanBar
		scanBar(message)
	}
	// Spin scanBar
	scanBar(message)
	// Test passed.
	printMessage(message, nil)
	return true
}
<|endoftext|>


<commit_before>// Copyright © 2016 The Things Network
// Use of this source code is governed by the MIT license that can be found in the LICENSE file.

package account

import (
	"fmt"

	"github.com/TheThingsNetwork/ttn/core/account/util"
	"golang.org/x/oauth2"
)

// Account is a proxy to an account on the account server
type Account struct {
	// server is the server where the account lives
	server string

	// login is the login strategy used by the account to log in
	tokenSource oauth2.TokenSource
}

// New creates a new Account for the given server and accessToken
func New(server string, source oauth2.TokenSource) *Account {
	return &Account{
		server:      server,
		tokenSource: source,
	}
}

// Token returns the last valid accessToken/refreshToken pair
// and should be used to store the latest token after doing things
// with an Account. If you fail to do this, the token might have refreshed
// and restoring your last copy will return an invalid token
func (a *Account) Token() (*oauth2.Token, error) {
	if a.tokenSource == nil {
		return nil, fmt.Errorf("Could not get credentials for account")
	}
	return a.tokenSource.Token()
}

// AccessToken returns a valid access token for the account,
// refreshing it if necessary
func (a *Account) AccessToken() (accessToken string, err error) {
	token, err := a.Token()
	if err != nil {
		return accessToken, err
	}
	return token.AccessToken, nil
}

func (a *Account) get(URI string, res interface{}) error {
	accessToken, err := a.AccessToken()
	if err != nil {
		return err
	}

	return util.GET(a.server, accessToken, URI, res)
}

func (a *Account) put(URI string, body, res interface{}) error {
	accessToken, err := a.AccessToken()
	if err != nil {
		return err
	}

	return util.PUT(a.server, accessToken, URI, body, res)
}

func (a *Account) post(URI string, body, res interface{}) error {
	accessToken, err := a.AccessToken()
	if err != nil {
		return err
	}

	return util.POST(a.server, accessToken, URI, body, res)
}

func (a *Account) patch(URI string, body, res interface{}) error {
	accessToken, err := a.AccessToken()
	if err != nil {
		return err
	}

	return util.PATCH(a.server, accessToken, URI, body, res)
}

func (a *Account) del(URI string) error {
	accessToken, err := a.AccessToken()
	if err != nil {
		return err
	}

	return util.DELETE(a.server, accessToken, URI)
}
<commit_msg>add constructor for static tokens<commit_after>// Copyright © 2016 The Things Network
// Use of this source code is governed by the MIT license that can be found in the LICENSE file.

package account

import (
	"fmt"

	"github.com/TheThingsNetwork/ttn/core/account/util"
	"golang.org/x/oauth2"
)

// Account is a proxy to an account on the account server
type Account struct {
	// server is the server where the account lives
	server string

	// login is the login strategy used by the account to log in
	tokenSource oauth2.TokenSource
}

// New creates a new Account for the given server and accessToken
func New(server string, source oauth2.TokenSource) *Account {
	return &Account{
		server:      server,
		tokenSource: source,
	}
}

// WithAccessToken creates a new Account that just has an accessToken,
// which it cannot refresh itself. This is useful if you need to manage
// the token refreshes outside of the Account.
func WithAccessToken(server, accessToken string) *Account {
	source := oauth2.StaticTokenSource(&oauth2.Token{
		AccessToken: accessToken,
	})
	return &Account{
		server:      server,
		tokenSource: source,
	}
}

// Token returns the last valid accessToken/refreshToken pair
// and should be used to store the latest token after doing things
// with an Account. If you fail to do this, the token might have refreshed
// and restoring your last copy will return an invalid token
func (a *Account) Token() (*oauth2.Token, error) {
	if a.tokenSource == nil {
		return nil, fmt.Errorf("Could not get credentials for account")
	}
	return a.tokenSource.Token()
}

// AccessToken returns a valid access token for the account,
// refreshing it if necessary
func (a *Account) AccessToken() (accessToken string, err error) {
	token, err := a.Token()
	if err != nil {
		return accessToken, err
	}
	return token.AccessToken, nil
}

func (a *Account) get(URI string, res interface{}) error {
	accessToken, err := a.AccessToken()
	if err != nil {
		return err
	}

	return util.GET(a.server, accessToken, URI, res)
}

func (a *Account) put(URI string, body, res interface{}) error {
	accessToken, err := a.AccessToken()
	if err != nil {
		return err
	}

	return util.PUT(a.server, accessToken, URI, body, res)
}

func (a *Account) post(URI string, body, res interface{}) error {
	accessToken, err := a.AccessToken()
	if err != nil {
		return err
	}

	return util.POST(a.server, accessToken, URI, body, res)
}

func (a *Account) patch(URI string, body, res interface{}) error {
	accessToken, err := a.AccessToken()
	if err != nil {
		return err
	}

	return util.PATCH(a.server, accessToken, URI, body, res)
}

func (a *Account) del(URI string) error {
	accessToken, err := a.AccessToken()
	if err != nil {
		return err
	}

	return util.DELETE(a.server, accessToken, URI)
}
<|endoftext|>


<commit_before>// Copyright 2010 The GoGo Authors. All rights reserved.
// Use of this source code is governed by the MIT
// license that can be found in the LICENSE file.

//
// GoGo stack functions
//

package libgogo

//
// List data structure
//
type List struct {
    baseAddress uint64; //Where the list starts
    itemCount uint64; //How many items there currently are in the list
    capacity uint64; //How many items there can be max. (not to be changed from outside the library)
};

//
// Initializes the given list
//
func InitializeList(uninitializedList *List) {
    uninitializedList.baseAddress = Alloc(16 * 8); //Allocate 16 items by default
    uninitializedList.capacity = 16;
    uninitializedList.itemCount = 0; //Reset item count (to zero)
}

//
// Adds an item to the given list, increasing its capacity if required
//
func AddItem(list *List, value uint64) {
    var newAddress uint64;
    if (list.capacity == list.itemCount) { //Grow list if its capacity doesn't suffice to add another item
        newAddress = Alloc(list.capacity * 2 * 8); //Double the capacity
        CopyMem(list.baseAddress, newAddress, list.capacity * 8); //Copy old list items
        list.baseAddress = newAddress; //Set new address as base address
        list.capacity = list.capacity * 2; //Update (increase) capacity
    }
    CopyMem(ToUint64FromUint64Ptr(&value), list.baseAddress + 8 * list.itemCount, 8); //Append the new value by copying its value into the memory of the corresponding list item
    list.itemCount = list.itemCount + 1; //Update item count
}

//
// Removes the i-th item of the given list and returns it
// Note that this function call fails if there are less than i items in the list or if the list is empty
//
func RemoveItemAt(list *List, index uint64) uint64 {
    var returnValue uint64 = GetItemAt(list, index); //Get the correspondig value value from the list
    var i uint64;
    for i = index; i < list.itemCount - 1; i = i + 1 { //Remove item by moving the following ones "backwards" in order to fill the gap caused by the deleted item
        CopyMem(list.baseAddress + (i + 1) * 8, list.baseAddress + 8 * i, 8); //Move item at position i + 1 to position i
    }
    list.itemCount = list.itemCount - 1; //Update (decrease) item count
    return returnValue; //Return value removed from list
}

//
// Returns the i-th item of a given list without removing it
// Note that this function call fails if there are less than i items in the list or if the list is empty
//
func GetItemAt(list *List, index uint64) uint64 {
    var returnValue uint64;
    if (list.itemCount <= index) { //Check if there are enough items in the list
        PrintString("Tried to GetItemAt(");
        PrintNumber(index);
        PrintString(") from a list with only ");
        PrintNumber(list.itemCount);
        ExitError(" items - out of bounds error", 125);
    }
    CopyMem(list.baseAddress + index * 8, ToUint64FromUint64Ptr(&returnValue), 8); //Copy the value on position i of the list (e.g. its corresponding memory) into the return variable
    return returnValue;
}

//
// Returns the number of items in the given list
//
func GetListItemCount(list *List) uint64 {
    return list.itemCount;
}

//
// Returns the current capacity of the given list
//
func GetListCapacity(list *List) uint64 {
    return list.capacity;
}
<commit_msg>list.go: Fix typo<commit_after>// Copyright 2010 The GoGo Authors. All rights reserved.
// Use of this source code is governed by the MIT
// license that can be found in the LICENSE file.

//
// GoGo list functions
//

package libgogo

//
// List data structure
//
type List struct {
    baseAddress uint64; //Where the list starts
    itemCount uint64; //How many items there currently are in the list
    capacity uint64; //How many items there can be max. (not to be changed from outside the library)
};

//
// Initializes the given list
//
func InitializeList(uninitializedList *List) {
    uninitializedList.baseAddress = Alloc(16 * 8); //Allocate 16 items by default
    uninitializedList.capacity = 16;
    uninitializedList.itemCount = 0; //Reset item count (to zero)
}

//
// Adds an item to the given list, increasing its capacity if required
//
func AddItem(list *List, value uint64) {
    var newAddress uint64;
    if (list.capacity == list.itemCount) { //Grow list if its capacity doesn't suffice to add another item
        newAddress = Alloc(list.capacity * 2 * 8); //Double the capacity
        CopyMem(list.baseAddress, newAddress, list.capacity * 8); //Copy old list items
        list.baseAddress = newAddress; //Set new address as base address
        list.capacity = list.capacity * 2; //Update (increase) capacity
    }
    CopyMem(ToUint64FromUint64Ptr(&value), list.baseAddress + 8 * list.itemCount, 8); //Append the new value by copying its value into the memory of the corresponding list item
    list.itemCount = list.itemCount + 1; //Update item count
}

//
// Removes the i-th item of the given list and returns it
// Note that this function call fails if there are less than i items in the list or if the list is empty
//
func RemoveItemAt(list *List, index uint64) uint64 {
    var returnValue uint64 = GetItemAt(list, index); //Get the correspondig value value from the list
    var i uint64;
    for i = index; i < list.itemCount - 1; i = i + 1 { //Remove item by moving the following ones "backwards" in order to fill the gap caused by the deleted item
        CopyMem(list.baseAddress + (i + 1) * 8, list.baseAddress + 8 * i, 8); //Move item at position i + 1 to position i
    }
    list.itemCount = list.itemCount - 1; //Update (decrease) item count
    return returnValue; //Return value removed from list
}

//
// Returns the i-th item of a given list without removing it
// Note that this function call fails if there are less than i items in the list or if the list is empty
//
func GetItemAt(list *List, index uint64) uint64 {
    var returnValue uint64;
    if (list.itemCount <= index) { //Check if there are enough items in the list
        PrintString("Tried to GetItemAt(");
        PrintNumber(index);
        PrintString(") from a list with only ");
        PrintNumber(list.itemCount);
        ExitError(" items - out of bounds error", 125);
    }
    CopyMem(list.baseAddress + index * 8, ToUint64FromUint64Ptr(&returnValue), 8); //Copy the value on position i of the list (e.g. its corresponding memory) into the return variable
    return returnValue;
}

//
// Returns the number of items in the given list
//
func GetListItemCount(list *List) uint64 {
    return list.itemCount;
}

//
// Returns the current capacity of the given list
//
func GetListCapacity(list *List) uint64 {
    return list.capacity;
}
<|endoftext|>


<commit_before>// octopus project octopus.go
package octopus

import (
	"math"
	"runtime"
	"errors"
	"time"
	"sync/atomic"
)

const(
	JOBUNSTART uint32 = 0
	JOBDOING uint32 = 1
	JOBDONE uint32 = 2
)

const(
	UNINTERRUPT uint32 = 3
	INTERRUPT uint32 = 4
)

const(
	WORKERFREE uint32 = 5
	WORKERRUNNING uint32 = 6
	WORKERSTOP uint32 = 7
	WORKERPREPARE uint32 = 8
)

const(
	POOLOPEN uint32 = 8
	POOLCLOSE uint32 = 9
)

type Runnable func()

type Callable func() (interface{})

type Future interface {
	Cancel() error
	Get() (interface{},error)
	GetTimed(time.Duration) (interface{},error)
	IsCancelled() bool//default 0 ,interrupt 1
	Status() uint32
}

type futureTask struct {
	job interface{}
	resultChannel chan interface{}
	interrupt uint32
	status  uint32
}
func newfutureTask(fn interface{}) *futureTask {
	return &futureTask{
			job: fn,
			resultChannel: make(chan interface{}),
			interrupt: UNINTERRUPT,
			status: JOBUNSTART,
	}
}
func (f *futureTask) execute() {
	if f.IsCancelled() {
		return
	}
	switch j := f.job.(type) {
		case Runnable :
			atomic.SwapUint32(&f.status, JOBDOING)
			j()
			atomic.SwapUint32(&f.status, JOBDONE)
		case Callable :
			atomic.SwapUint32(&f.status,JOBDOING)
			result := j()
			f.resultChannel  <-  result
			atomic.SwapUint32(&f.status, JOBDONE)
		default :
	}
}
func (f *futureTask) Cancel() error {
	if f.Status() == JOBDONE {
		return errors.New("has done")
	}
	atomic.SwapUint32(&f.interrupt,INTERRUPT)
	return nil
}

func (f *futureTask) Get() (result interface{}, err error) {
	switch f.job.(type) {
		case Runnable :
			return
		case Callable :
			var ok bool
			result,ok  = <- f.resultChannel
			if !ok {
				return nil, errors.New("channel close")
			}
			return
	}
	return
}
func (f *futureTask) GetTimed(timeout time.Duration) (result interface{}, err error) {
	switch f.job.(type) {
		case Runnable :
			return
		case Callable :
			var ok bool
			select {
				case result,ok = <- f.resultChannel :
					if !ok {
						return nil, errors.New("channel close")
					}
					return
				case <- time.After(timeout*time.Millisecond) :
					return nil, errors.New("timeout")
			}
	}
	return
}
func (f *futureTask) IsCancelled() bool {
	return (atomic.LoadUint32(&f.interrupt) == INTERRUPT)
}
func (f *futureTask) Status() uint32 {
	return atomic.LoadUint32(&f.status)
}

type worker struct {
	jobChannel chan *futureTask
	stop chan bool
	status uint32
	willClose bool
}
func newWorker( s chan bool) *worker {
	return &worker {
		jobChannel : make(chan *futureTask) ,
		stop : s ,
		status : WORKERPREPARE ,
		willClose: false,
	}
}
func newWorkerForCachedPool() *worker {
	return &worker {
		jobChannel : make(chan *futureTask) ,
		stop : make(chan bool),
		status : WORKERPREPARE ,
		willClose : false,
	}
}
func (w *worker) getStatus() uint32 {
	return atomic.LoadUint32(&w.status)
}
func (w *worker) start(pool WorkPool) {
	atomic.SwapUint32(&w.status, WORKERFREE)
	go func() {
		var job *futureTask
		for {
			// worker free, add it to pool
			//pool.workerChannel <- w
			pool.getWorkerChannel() <- w
			select {
				case job = <- w.jobChannel :
					atomic.SwapUint32(&w.status, WORKERRUNNING)
					job.execute()
					atomic.SwapUint32(&w.status,WORKERFREE)
				case stop := <- w.stop:
					if stop {
						//w.stop <- true
						atomic.SwapUint32(&w.status, WORKERSTOP)
						return
					}

			}
		}
	}()
}

type WorkPool interface {
	IsPoolOpen() bool
	Close()
	Release()
	SubmitRunnable(Runnable) (Future, error)
	SubmitCallable(job Callable) (Future, error)
	InvokeAllRunnable([]Runnable) ([]Future,error)
	InvokeAllCallable([]Callable) ([]Future,error)
	getWorkerChannel() chan *worker
}

type cachedWorkerPool struct {
	*fixWorkerPool
	workers []*worker
	msignal chan bool
}
func NewCachedWorkerPool() WorkPool {
	num := runtime.NumCPU()*20
	wc := make(chan *worker,num )
	jc := make(chan *futureTask, num)
	sc := make(chan bool)
	ms := make(chan bool)
	var pool cachedWorkerPool
	pool.workerChannel = wc
	pool.jobChannel = jc
	pool.stop = sc
	pool.poolOpen = POOLOPEN
	pool.msignal = ms
	ws := make([]*worker, num)
	pool.workers = ws
	for i:=0; i<num; i++ {
		pool.workers[i] = newWorkerForCachedPool()
		pool.workers[i].start(pool)
	}

	go pool.dispatch()
	go pool.monitor()
	return &pool
}

func newWorkerForChannel(pool *cachedWorkerPool, workerChannel chan *worker) chan *worker {

	if len(pool.workers) < math.MaxInt16 {
		w := newWorkerForCachedPool()
	          workerChannel <- w
	}

	return workerChannel
}

func (pool *cachedWorkerPool) monitor() {
	for {
		select {
			case signal := <- pool.msignal :
				if signal {
					return
				}
			default :
				for i:=0; i<len(pool.workers); i++ {
					w := pool.workers[i]
					if w.getStatus() == WORKERFREE && w.willClose != true {
						w.willClose = true
						continue
					}

					if w.getStatus() == WORKERSTOP && w.willClose != true {
						w.willClose = true
						continue
					}

					if w.getStatus() ==WORKERFREE && w.willClose == true {
						w.stop <- true
						close(w.stop)
						close(w.jobChannel)
						pool.workers = append(pool.workers[ : i], pool.workers[i+1 : ]...)
					}

					if w.getStatus() == WORKERSTOP && w.willClose == true {
						w.stop <- true
						close(w.stop)
						close(w.jobChannel)
						pool.workers = append(pool.workers[ : i], pool.workers[i+1 : ]...)
					}
				}
		}

	}
}

func (pool *cachedWorkerPool) dispatch() {
	wc := make(chan *worker)
	for {
		select {
		case w := <- pool.workerChannel :
			job := <- pool.jobChannel
			w.jobChannel <- job
		case w := <- newWorkerForChannel(pool, wc) :
			pool.workers = append(pool.workers, w)
			pool.workerChannel <- w
		case stop := <- pool.stop :
			if stop {
				// close all workers
				for i := 0; i < len(pool.workers); i++ {
					pool.workers[i].stop <- true
				}
				//close monitor
				pool.msignal <- true

				pool.stop <- true
				return
			}
		}
	}
}

type fixWorkerPool struct {
	workerChannel chan *worker
	jobChannel chan *futureTask
	stop chan bool
	poolOpen  uint32 // 1 represent pool open to receive jobs,0 to close
}

func NewFixedWorkerPool(numWorkers int) WorkPool {
	wc := make(chan *worker, numWorkers)
	jc := make(chan *futureTask, numWorkers)
	sc := make(chan bool)
	pool := &fixWorkerPool{
		workerChannel : wc ,
		jobChannel : jc ,
		stop : sc,
		poolOpen : POOLOPEN ,
	}

	for i := 0; i < cap(pool.workerChannel); i++ {
		w := newWorker(sc)
		w.start(pool)
	}

	go pool.dispatch()

	return pool
}

func (pool *fixWorkerPool) getWorkerChannel() chan *worker {
	return pool.workerChannel
}

func (pool *fixWorkerPool) dispatch() {
	for {
		select {
		case job := <- pool.jobChannel :
			w  := <- pool.workerChannel
			w.jobChannel <- job
		case stop := <- pool.stop :
			if stop {
				for i := 0; i < cap(pool.workerChannel); i++ {
					w := <- pool.workerChannel

					w.stop <- true
					<-w.stop
				}

				pool.stop <- true
				return
			}
		}
	}
}
func (pool *fixWorkerPool) IsPoolOpen() bool {
	return (atomic.LoadUint32(&pool.poolOpen) == POOLOPEN)
}
func (pool *fixWorkerPool) Close() {
	atomic.SwapUint32(&pool.poolOpen, POOLCLOSE)
}
func (pool *fixWorkerPool) Release() {
	if pool.IsPoolOpen() {
		pool.Close()
	}
	pool.stop <- true
	<- pool.stop
	close(pool.jobChannel)
	close(pool.workerChannel)
	close(pool.stop)
}
func (pool *fixWorkerPool) SubmitRunnable(job Runnable) (future Future, err error) {
	if !pool.IsPoolOpen() {
		return nil,errors.New("pool has closed")
	}
          f := newfutureTask(job)
	future = f
	pool.jobChannel <- f
	return
}
func (pool *fixWorkerPool) SubmitCallable(job Callable) (future Future, err error) {
	if !pool.IsPoolOpen() {
		return nil,errors.New("pool has closed")
	}
          f := newfutureTask(job)
	future = f
	pool.jobChannel <- f
	return
}
func (pool *fixWorkerPool) InvokeAllRunnable(jobs []Runnable) (futures []Future,err error) {
	if !pool.IsPoolOpen() {
		err = errors.New("pool has closed")
		return
	}
	futures = make([]Future,len(jobs))
	for i,j := range jobs {
		f := newfutureTask(j)
		pool.jobChannel <- f
		futures[i] = f
	}
	return
}
func (pool *fixWorkerPool) InvokeAllCallable(jobs []Callable) (futures []Future,err error) {
	if !pool.IsPoolOpen() {
		err = errors.New("pool has closed")
		return
	}
	futures = make([]Future,len(jobs))
	for i,j := range jobs {
		f := newfutureTask(j)
		pool.jobChannel <- f
		futures[i] = f
	}
	return
}
<commit_msg>add comments<commit_after>// octopus project octopus.go
package octopus

import (
	"runtime"
	"errors"
	"time"
	"sync/atomic"
)

const(
	JOBUNSTART uint32 = 0
	JOBDOING uint32 = 1
	JOBDONE uint32 = 2
)

const(
	UNINTERRUPT uint32 = 3
	INTERRUPT uint32 = 4
)

const(
	WORKERFREE uint32 = 5
	WORKERRUNNING uint32 = 6
	WORKERSTOP uint32 = 7
)

const(
	POOLOPEN uint32 = 8
	POOLCLOSE uint32 = 9
)

type Runnable func()

type Callable func() (interface{})

type Future interface {
	// Cancel method will set a cancel tag attempt to cancel execute job before starting this job represented by Future.
	Cancel() error
	// Get method can get value from Callable , if not ready it will block.
	Get() (interface{},error)
	// GetTimed method can get value from Callable with setting timeout.
	GetTimed(time.Duration) (interface{},error)
	// IsCancelled will return whether the job was setting a cancel tag, but it does not mean that the job has been terminated.
	IsCancelled() bool//default 0 ,interrupt 1
	// IsDone will return whether the job was done.
	IsDone() bool
}

type futureTask struct {
	job interface{}
	resultChannel chan interface{}
	interrupt uint32
	status  uint32
}
func newfutureTask(fn interface{}) *futureTask {
	return &futureTask{
			job: fn,
			resultChannel: make(chan interface{}),
			interrupt: UNINTERRUPT,
			status: JOBUNSTART,
	}
}

// free worker will call execute method
func (f *futureTask) execute() {
	if f.IsCancelled() {
		return
	}
	switch j := f.job.(type) {
		case Runnable :
			atomic.SwapUint32(&f.status, JOBDOING)
			j()
			atomic.SwapUint32(&f.status, JOBDONE)
		case Callable :
			atomic.SwapUint32(&f.status,JOBDOING)
			result := j()
			f.resultChannel  <-  result
			atomic.SwapUint32(&f.status, JOBDONE)
		default :
	}
}
func (f *futureTask) Cancel() error {
	if f.Status() == JOBDONE {
		return errors.New("has done")
	}
	atomic.SwapUint32(&f.interrupt,INTERRUPT)
	return nil
}

func (f *futureTask) Get() (result interface{}, err error) {
	switch f.job.(type) {
		case Runnable :
			return
		case Callable :
			var ok bool
			result,ok  = <- f.resultChannel
			if !ok {
				return nil, errors.New("channel close")
			}
			return
	}
	return
}
func (f *futureTask) GetTimed(timeout time.Duration) (result interface{}, err error) {
	switch f.job.(type) {
		case Runnable :
			return
		case Callable :
			var ok bool
			select {
				case result,ok = <- f.resultChannel :
					if !ok {
						return nil, errors.New("channel close")
					}
					return
				case <- time.After(timeout*time.Millisecond) :
					return nil, errors.New("timeout")
			}
	}
	return
}
func (f *futureTask) IsCancelled() bool {
	return (atomic.LoadUint32(&f.interrupt) == INTERRUPT)
}
func (f *futureTask) Status() uint32 {
	return atomic.LoadUint32(&f.status)
}

type worker struct {
	jobChannel chan *futureTask
	stop chan bool
	status uint32
}
func newWorker( s chan bool) *worker {
	return &worker {
		jobChannel : make(chan *futureTask) ,
		stop : s ,
		status : WORKERSTOP ,
	}
}
func newWorkerForCachedPool() *worker {
	return &worker {
		jobChannel : make(chan *futureTask) ,
		stop : make(chan bool),
		status : WORKERSTOP ,
	}
}
func (w *worker) status() {
	return atomic.LoadUint32(&w.status)
}
func (w *worker) start(pool WorkPool) {
	atomic.SwapUint32(&w.status, WORKERFREE)
	go func() {
		var job *futureTask
		for {
			// worker free, add it to pool
			//pool.workerChannel <- w
			pool.getWorkerChannel() <- w
			select {
				case job = <- w.jobChannel :
					atomic.SwapUint32(&w.status, WORKERRUNNING)
					job.execute()
					atomic.SwapUint32(&w.status,WORKERFREE)
				case stop := <- w.stop:
					if stop {
						//w.stop <- true
						atomic.SwapUint32(&w.status, WORKERSTOP)
						return
					}

			}
		}
	}()
}

type WorkPool interface {
	IsPoolOpen() bool
	Close()
	Release()
	SubmitRunnable(Runnable) (Future, error)
	SubmitCallable(job Callable) (Future, error)
	InvokeAllRunnable([]Runnable) ([]Future,error)
	InvokeAllCallable([]Callable) ([]Future,error)
	getWorkerChannel() chan *worker
}

type cachedWorkerPool struct {
	*fixWorkerPool
	workers []*worker
}
func NewCachedWorkerPool() WorkPool {
	num := runtime.NumCPU()*20
	wc := make(chan *worker,num )
	jc := make(chan *futureTask, num)
	sc := make(chan bool)
	var pool cachedWorkerPool
	pool.workerChannel = wc
	pool.jobChannel = jc
	pool.stop = sc
	pool.poolOpen = POOLOPEN
	ws := make([]*worker, num)
	pool.workers = ws
	for i:=0; i<num; i++ {
		pool.workers[i] = newWorkerForCachedPool()
		pool.workers[i].start(pool)
	}

	go pool.dispatch()
	return &pool
}

func newWorkerForChannel() chan *worker {
	w := newWorkerForCachedPool()
	wc := make(chan *worker)
	wc <- w
	return wc
}

func (pool *cachedWorkerPool) dispatch() {
	for {
		select {
		case w := <- pool.workerChannel :
			job := <- pool.jobChannel
			w.jobChannel <- job
		case w := <- newWorkerForChannel() :
			pool.workers = append(pool.workers, w)
			pool.workerChannel <- w
//		case stop := <- pool.stop :
//			if stop {
//				for i := 0; i < cap(pool.workerChannel); i++ {
//					w := <- pool.workerChannel

//					w.stop <- true
//					<-w.stop
//				}

//				pool.stop <- true
//				return
//			}
		}
	}
}

type fixWorkerPool struct {
	workerChannel chan *worker
	jobChannel chan *futureTask
	stop chan bool
	poolOpen  uint32 // 1 represent pool open to receive jobs,0 to close
}

// Create a goroutine pool that can be reused for a number of fixed goroutines
func newFixedWorkerPool(numWorkers int) WorkPool {
	wc := make(chan *worker, numWorkers)
	jc := make(chan *futureTask, numWorkers)
	sc := make(chan bool)
	pool := &fixWorkerPool{
		workerChannel : wc ,
		jobChannel : jc ,
		stop : sc,
		poolOpen : POOLOPEN ,
	}

	for i := 0; i < cap(pool.workerChannel); i++ {
		w := newWorker(jc, sc)
		w.start(pool)
	}

	go pool.dispatch()

	return pool
}

func (pool *fixWorkerPool) getWorkerChannel() chan *worker {
	return pool.workerChannel
}

func (pool *fixWorkerPool) dispatch() {
	for {
		select {
		case job := <- pool.jobChannel :
			w  := <- pool.workerChannel
			w.jobChannel <- job
		case stop := <- pool.stop :
			if stop {
				for i := 0; i < cap(pool.workerChannel); i++ {
					w := <- pool.workerChannel

					w.stop <- true
					<-w.stop
				}

				pool.stop <- true
				return
			}
		}
	}
}
func (pool *fixWorkerPool) IsPoolOpen() bool {
	return (atomic.LoadUint32(&pool.poolOpen) == POOLOPEN)
}
func (pool *fixWorkerPool) Close() {
	atomic.SwapUint32(&pool.poolOpen, POOLCLOSE)
}
func (pool *fixWorkerPool) Release() {
	if pool.IsPoolOpen() {
		pool.Close()
	}
	pool.stop <- true
	<- pool.stop
	close(pool.jobChannel)
	close(pool.workerChannel)
	close(pool.stop)
}
func (pool *fixWorkerPool) SubmitRunnable(job Runnable) (future Future, err error) {
	if !pool.IsPoolOpen() {
		return nil,errors.New("pool has closed")
	}
          f := newfutureTask(job)
	future = f
	pool.jobChannel <- f
	return
}
func (pool *fixWorkerPool) SubmitCallable(job Callable) (future Future, err error) {
	if !pool.IsPoolOpen() {
		return nil,errors.New("pool has closed")
	}
          f := newfutureTask(job)
	future = f
	pool.jobChannel <- f
	return
}
func (pool *fixWorkerPool) InvokeAllRunnable(jobs []Runnable) (futures []Future,err error) {
	if !pool.IsPoolOpen() {
		err = errors.New("pool has closed")
		return
	}
	futures = make([]Future,len(jobs))
	for i,j := range jobs {
		f := newfutureTask(j)
		pool.jobChannel <- f
		futures[i] = f
	}
	return
}
func (pool *fixWorkerPool) InvokeAllCallable(jobs []Callable) (futures []Future,err error) {
	if !pool.IsPoolOpen() {
		err = errors.New("pool has closed")
		return
	}
	futures = make([]Future,len(jobs))
	for i,j := range jobs {
		f := newfutureTask(j)
		pool.jobChannel <- f
		futures[i] = f
	}
	return
}
<|endoftext|>