<commit_before>package openjpeg

// #cgo LDFLAGS: -lopenjp2
// #include <openjpeg-2.1/openjpeg.h>
import "C"

import (
	"errors"
	"fmt"
	"image"
	"reflect"
	"runtime"
	"unsafe"

	"github.com/nfnt/resize"
)

// Container for our simple JP2 operations
type JP2Image struct {
	filename        string
	stream          *C.opj_stream_t
	codec           *C.opj_codec_t
	image           *C.opj_image_t
	decodeWidth     int
	decodeHeight    int
	scaleFactor     float64
	decodeArea      image.Rectangle
	crop            bool
	resizeByPercent bool
	resizeByPixels  bool
}

func NewJP2Image(filename string) (*JP2Image, error) {
	i := &JP2Image{filename: filename}
	runtime.SetFinalizer(i, finalizer)

	if err := i.initializeStream(); err != nil {
		return nil, err
	}

	return i, nil
}

// SetScale sets the image to scale by the given multiplier, typically a
// percentage from 0 to 1.  This is mutually exclusive with resizing by a set
// width/height value.
func (i *JP2Image) SetScale(m float64) {
	i.scaleFactor = m
	i.resizeByPercent = true
	i.resizeByPixels = false
}

// SetResizeWH sets the image to scale to the given width and height.  If one
// dimension is 0, the decoded image will preserve the aspect ratio while
// scaling to the non-zero dimension.
func (i *JP2Image) SetResizeWH(width, height int) {
	i.decodeWidth = width
	i.decodeHeight = height
	i.resizeByPixels = true
	i.resizeByPercent = false
}

func (i *JP2Image) SetCrop(r image.Rectangle) {
	i.decodeArea = r
	i.crop = true
}

// DecodeImage returns an image.Image that holds the decoded image data,
// resized and cropped if resizing or cropping was requested.  Both cropping
// and resizing happen here due to the nature of openjpeg, so SetScale,
// SetResizeWH, and SetCrop must be called before this function.
func (i *JP2Image) DecodeImage() (image.Image, error) {
	// We need the codec to be ready for all operations below
	if err := i.initializeCodec(); err != nil {
		goLog(3, "Error initializing codec - aborting")
		return nil, err
	}

	// If we want to resize, but not crop, we have to set the decode area to the
	// full image - which means reading in the image header and then cleaning up
	// all previously-initialized data
	if (i.resizeByPixels || i.resizeByPercent) && !i.crop {
		if err := i.ReadHeader(); err != nil {
			goLog(3, "Error getting dimensions - aborting")
			return nil, err
		}
		i.decodeArea = i.Dimensions()
		i.CleanupResources()
	}

	// If resize is by percent, we now have the decode area, and can use that to
	// get pixel dimensions
	if i.resizeByPercent {
		i.decodeWidth = int(float64(i.decodeArea.Dx()) * i.scaleFactor)
		i.decodeHeight = int(float64(i.decodeArea.Dy()) * i.scaleFactor)
		i.resizeByPixels = true
	}

	// Get progression level if we're resizing to specific dimensions (it's zero
	// if there isn't any scaling of the output)
	if i.resizeByPixels {
		level := desiredProgressionLevel(i.decodeArea, i.decodeWidth, i.decodeHeight)
		if err := i.SetDynamicProgressionLevel(level); err != nil {
			goLog(3, "Unable to set dynamic progression level - aborting")
			return nil, err
		}
	}

	if err := i.ReadHeader(); err != nil {
		goLog(3, "Error reading header before decode - aborting")
		return nil, err
	}

	goLog(6, fmt.Sprintf("num comps: %d", i.image.numcomps))
	goLog(6, fmt.Sprintf("x0: %d, x1: %d, y0: %d, y1: %d", i.image.x0, i.image.x1, i.image.y0, i.image.y1))

	// Setting decode area has to happen *after* reading the header / image data
	if i.crop {
		r := i.decodeArea
		if C.opj_set_decode_area(i.codec, i.image, C.OPJ_INT32(r.Min.X), C.OPJ_INT32(r.Min.Y), C.OPJ_INT32(r.Max.X), C.OPJ_INT32(r.Max.Y)) == C.OPJ_FALSE {
			return nil, errors.New("failed to set the decoded area")
		}
	}

	// Decode the JP2 into the image stream
	if C.opj_decode(i.codec, i.stream, i.image) == C.OPJ_FALSE {
		return nil, errors.New("failed to decode image")
	}
	if C.opj_end_decompress(i.codec, i.stream) == C.OPJ_FALSE {
		return nil, errors.New("failed to close decompression")
	}

	var comps []C.opj_image_comp_t
	compsSlice := (*reflect.SliceHeader)((unsafe.Pointer(&comps)))
	compsSlice.Cap = int(i.image.numcomps)
	compsSlice.Len = int(i.image.numcomps)
	compsSlice.Data = uintptr(unsafe.Pointer(i.image.comps))

	bounds := image.Rect(0, 0, int(comps[0].w), int(comps[0].h))
	var img image.Image

	// We assume grayscale if we don't have at least 3 components, because it's
	// probably the safest default
	if len(comps) < 3 {
		img = &image.Gray{Pix: JP2ComponentData(comps[0]), Stride: bounds.Dx(), Rect: bounds}
	} else {
		// If we have 3+ components, we only care about the first three - I have no
		// idea what else we might have other than alpha, and as a tile server, we
		// don't care about alpha.  It's worth noting that this will almost certainly
		// blow up on any JP2 that isn't using RGB.
		realData := make([]uint8, (bounds.Dx()*bounds.Dy())<<2)
		for x, comp := range comps[0:3] {
			compData := JP2ComponentData(comp)
			for y, point := range compData {
				realData[y*4+x] = point
			}
		}

		img = &image.RGBA{Pix: realData, Stride: bounds.Dx() << 2, Rect: bounds}
	}

	if i.resizeByPixels {
		img = resize.Resize(uint(i.decodeWidth), uint(i.decodeHeight), img, resize.Bilinear)
	}

	return img, nil
}

func (i *JP2Image) ReadHeader() error {
	if i.image != nil {
		return nil
	}

	if err := i.initializeCodec(); err != nil {
		return err
	}

	if err := i.initializeStream(); err != nil {
		return err
	}

	if C.opj_read_header(i.stream, i.codec, &i.image) == C.OPJ_FALSE {
		return errors.New("failed to read the header")
	}

	return nil
}

// GetDimensions reads the JP2 headers and pulls dimensions in order to satisfy
// the IIIFImage interface
func (i *JP2Image) GetDimensions() (image.Rectangle, error) {
	if err := i.ReadHeader(); err != nil {
		return image.Rectangle{}, err
	}

	return i.Dimensions(), nil
}

func (i *JP2Image) Dimensions() image.Rectangle {
	return image.Rect(int(i.image.x0), int(i.image.y0), int(i.image.x1), int(i.image.y1))
}

// Attempts to set the progression level to the given value, then re-read the
// header.  If reading the header fails, attempts to set the level to one level
// below the initial level.  If reading the header fails at level 0, an error
// is returned.
func (i *JP2Image) SetDynamicProgressionLevel(level int) error {
	onErr := func(err error) error {
		if level > 0 {
			goLog(6, fmt.Sprintf("Unable to set progression level to %d; trying again (%s)", level, err))
			i.CleanupResources()
			return i.SetDynamicProgressionLevel(level - 1)
		}

		return err
	}

	goLog(6, fmt.Sprintf("Setting progression level to %d", level))

	if err := i.initializeCodec(); err != nil {
		return onErr(err)
	}

	if C.opj_set_decoded_resolution_factor(i.codec, C.OPJ_UINT32(level)) == C.OPJ_FALSE {
		return onErr(errors.New("Error trying to set decoded resolution factor"))
	}

	if err := i.ReadHeader(); err != nil {
		return onErr(err)
	}

	return nil
}

// JP2ComponentData returns a slice of Image-usable uint8s from the JP2 raw
// data in the given component struct
func JP2ComponentData(comp C.struct_opj_image_comp) []uint8 {
	var data []int32
	dataSlice := (*reflect.SliceHeader)((unsafe.Pointer(&data)))
	size := int(comp.w) * int(comp.h)
	dataSlice.Cap = size
	dataSlice.Len = size
	dataSlice.Data = uintptr(unsafe.Pointer(comp.data))

	realData := make([]uint8, len(data))
	for index, point := range data {
		realData[index] = uint8(point)
	}

	return realData
}
<commit_msg>Store source width/height to skip some computation<commit_after>package openjpeg

// #cgo LDFLAGS: -lopenjp2
// #include <openjpeg-2.1/openjpeg.h>
import "C"

import (
	"errors"
	"fmt"
	"image"
	"reflect"
	"runtime"
	"unsafe"

	"github.com/nfnt/resize"
)

// Container for our simple JP2 operations
type JP2Image struct {
	filename        string
	stream          *C.opj_stream_t
	codec           *C.opj_codec_t
	image           *C.opj_image_t
	decodeWidth     int
	decodeHeight    int
	scaleFactor     float64
	decodeArea      image.Rectangle
	crop            bool
	resizeByPercent bool
	resizeByPixels  bool
}

func NewJP2Image(filename string) (*JP2Image, error) {
	i := &JP2Image{filename: filename}
	runtime.SetFinalizer(i, finalizer)

	if err := i.initializeStream(); err != nil {
		return nil, err
	}

	return i, nil
}

// SetScale sets the image to scale by the given multiplier, typically a
// percentage from 0 to 1.  This is mutually exclusive with resizing by a set
// width/height value.
func (i *JP2Image) SetScale(m float64) {
	i.scaleFactor = m
	i.resizeByPercent = true
	i.resizeByPixels = false
}

// SetResizeWH sets the image to scale to the given width and height.  If one
// dimension is 0, the decoded image will preserve the aspect ratio while
// scaling to the non-zero dimension.
func (i *JP2Image) SetResizeWH(width, height int) {
	i.decodeWidth = width
	i.decodeHeight = height
	i.resizeByPixels = true
	i.resizeByPercent = false
}

func (i *JP2Image) SetCrop(r image.Rectangle) {
	i.decodeArea = r
	i.crop = true
}

// DecodeImage returns an image.Image that holds the decoded image data,
// resized and cropped if resizing or cropping was requested.  Both cropping
// and resizing happen here due to the nature of openjpeg, so SetScale,
// SetResizeWH, and SetCrop must be called before this function.
func (i *JP2Image) DecodeImage() (image.Image, error) {
	// We need the codec to be ready for all operations below
	if err := i.initializeCodec(); err != nil {
		goLog(3, "Error initializing codec - aborting")
		return nil, err
	}

	// If we want to resize, but not crop, we have to set the decode area to the
	// full image - which means reading in the image header and then cleaning up
	// all previously-initialized data
	if (i.resizeByPixels || i.resizeByPercent) && !i.crop {
		if err := i.ReadHeader(); err != nil {
			goLog(3, "Error getting dimensions - aborting")
			return nil, err
		}
		i.decodeArea = i.Dimensions()
		i.CleanupResources()
	}

	// If resize is by percent, we now have the decode area, and can use that to
	// get pixel dimensions
	if i.resizeByPercent {
		i.decodeWidth = int(float64(i.decodeArea.Dx()) * i.scaleFactor)
		i.decodeHeight = int(float64(i.decodeArea.Dy()) * i.scaleFactor)
		i.resizeByPixels = true
	}

	// Get progression level if we're resizing to specific dimensions (it's zero
	// if there isn't any scaling of the output)
	if i.resizeByPixels {
		level := desiredProgressionLevel(i.decodeArea, i.decodeWidth, i.decodeHeight)
		if err := i.SetDynamicProgressionLevel(level); err != nil {
			goLog(3, "Unable to set dynamic progression level - aborting")
			return nil, err
		}
	}

	if err := i.ReadHeader(); err != nil {
		goLog(3, "Error reading header before decode - aborting")
		return nil, err
	}

	goLog(6, fmt.Sprintf("num comps: %d", i.image.numcomps))
	goLog(6, fmt.Sprintf("x0: %d, x1: %d, y0: %d, y1: %d", i.image.x0, i.image.x1, i.image.y0, i.image.y1))

	// Setting decode area has to happen *after* reading the header / image data
	if i.crop {
		r := i.decodeArea
		if C.opj_set_decode_area(i.codec, i.image, C.OPJ_INT32(r.Min.X), C.OPJ_INT32(r.Min.Y), C.OPJ_INT32(r.Max.X), C.OPJ_INT32(r.Max.Y)) == C.OPJ_FALSE {
			return nil, errors.New("failed to set the decoded area")
		}
	}

	// Decode the JP2 into the image stream
	if C.opj_decode(i.codec, i.stream, i.image) == C.OPJ_FALSE {
		return nil, errors.New("failed to decode image")
	}
	if C.opj_end_decompress(i.codec, i.stream) == C.OPJ_FALSE {
		return nil, errors.New("failed to close decompression")
	}

	var comps []C.opj_image_comp_t
	compsSlice := (*reflect.SliceHeader)((unsafe.Pointer(&comps)))
	compsSlice.Cap = int(i.image.numcomps)
	compsSlice.Len = int(i.image.numcomps)
	compsSlice.Data = uintptr(unsafe.Pointer(i.image.comps))

	width := int(comps[0].w)
	height := int(comps[0].h)
	bounds := image.Rect(0, 0, width, height)
	var img image.Image

	// We assume grayscale if we don't have at least 3 components, because it's
	// probably the safest default
	if len(comps) < 3 {
		img = &image.Gray{Pix: JP2ComponentData(comps[0]), Stride: width, Rect: bounds}
	} else {
		// If we have 3+ components, we only care about the first three - I have no
		// idea what else we might have other than alpha, and as a tile server, we
		// don't care about alpha.  It's worth noting that this will almost certainly
		// blow up on any JP2 that isn't using RGB.
		area := width*height
		bytes := area<<2
		realData := make([]uint8, bytes)
		for x, comp := range comps[0:3] {
			compData := JP2ComponentData(comp)
			for y, point := range compData {
				realData[y*4+x] = point
			}
		}

		img = &image.RGBA{Pix: realData, Stride: width<<2, Rect: bounds}
	}

	if i.resizeByPixels {
		img = resize.Resize(uint(i.decodeWidth), uint(i.decodeHeight), img, resize.Bilinear)
	}

	return img, nil
}

func (i *JP2Image) ReadHeader() error {
	if i.image != nil {
		return nil
	}

	if err := i.initializeCodec(); err != nil {
		return err
	}

	if err := i.initializeStream(); err != nil {
		return err
	}

	if C.opj_read_header(i.stream, i.codec, &i.image) == C.OPJ_FALSE {
		return errors.New("failed to read the header")
	}

	return nil
}

// GetDimensions reads the JP2 headers and pulls dimensions in order to satisfy
// the IIIFImage interface
func (i *JP2Image) GetDimensions() (image.Rectangle, error) {
	if err := i.ReadHeader(); err != nil {
		return image.Rectangle{}, err
	}

	return i.Dimensions(), nil
}

func (i *JP2Image) Dimensions() image.Rectangle {
	return image.Rect(int(i.image.x0), int(i.image.y0), int(i.image.x1), int(i.image.y1))
}

// Attempts to set the progression level to the given value, then re-read the
// header.  If reading the header fails, attempts to set the level to one level
// below the initial level.  If reading the header fails at level 0, an error
// is returned.
func (i *JP2Image) SetDynamicProgressionLevel(level int) error {
	onErr := func(err error) error {
		if level > 0 {
			goLog(6, fmt.Sprintf("Unable to set progression level to %d; trying again (%s)", level, err))
			i.CleanupResources()
			return i.SetDynamicProgressionLevel(level - 1)
		}

		return err
	}

	goLog(6, fmt.Sprintf("Setting progression level to %d", level))

	if err := i.initializeCodec(); err != nil {
		return onErr(err)
	}

	if C.opj_set_decoded_resolution_factor(i.codec, C.OPJ_UINT32(level)) == C.OPJ_FALSE {
		return onErr(errors.New("Error trying to set decoded resolution factor"))
	}

	if err := i.ReadHeader(); err != nil {
		return onErr(err)
	}

	return nil
}

// JP2ComponentData returns a slice of Image-usable uint8s from the JP2 raw
// data in the given component struct
func JP2ComponentData(comp C.struct_opj_image_comp) []uint8 {
	var data []int32
	dataSlice := (*reflect.SliceHeader)((unsafe.Pointer(&data)))
	size := int(comp.w) * int(comp.h)
	dataSlice.Cap = size
	dataSlice.Len = size
	dataSlice.Data = uintptr(unsafe.Pointer(comp.data))

	realData := make([]uint8, len(data))
	for index, point := range data {
		realData[index] = uint8(point)
	}

	return realData
}
<|endoftext|>


<commit_before>package db

import (
	"bytes"
	"database/sql"
	"errors"
	"os"
	"strconv"
	"sync"

	"github.com/lestrrat/go-pdebug"
	"github.com/lestrrat/go-tx-guard"
)

type DB struct {
	*guard.DB
}

type Tx struct {
	*guard.Tx
}

var _db *DB // global database connection
var ErrNoTLSRequested = errors.New("TLS environment variables not set")
var Trace bool

var stmtBufPool = sync.Pool{
	New: func() interface{} {
		return &bytes.Buffer{}
	},
}

func getStmtBuf() *bytes.Buffer {
	return stmtBufPool.Get().(*bytes.Buffer)
}
func releaseStmtBuf(buf *bytes.Buffer) {
	buf.Reset()
	stmtBufPool.Put(buf)
}

func init() {
	if f := os.Getenv("OCTAV_TRACE_DB"); f != "" {
		if b, err := strconv.ParseBool(f); b && err == nil {
			Trace = true
		}
	}
}

func Init(dsn string) (err error) {
	if pdebug.Enabled {
		g := pdebug.Marker("db.Init").BindError(&err)
		defer g.End()
	}

	if dsn == "" {
		dsn, err = ConfigureDSN()
		if err != nil {
			return err
		}
	}

	dn := driverName()
	conn, err := sql.Open(dn, dsn)
	if err != nil {
		return err
	}

	if err := onConnect(conn); err != nil {
		return err
	}

	_db = &DB{&guard.DB{conn}}

	return nil
}

// Begin creates a new transaction (`Tx`) from the current
// global database connection
func Begin() (*Tx, error) {
	if _db == nil {
		return nil, errors.New("database has not been initialized")
	}

	tx, err := _db.Begin()
	if err != nil {
		return nil, err
	}
	return &Tx{tx}, nil
}
<commit_msg>add Grow(0)<commit_after>package db

import (
	"bytes"
	"database/sql"
	"errors"
	"os"
	"strconv"
	"sync"

	"github.com/lestrrat/go-pdebug"
	"github.com/lestrrat/go-tx-guard"
)

type DB struct {
	*guard.DB
}

type Tx struct {
	*guard.Tx
}

var _db *DB // global database connection
var ErrNoTLSRequested = errors.New("TLS environment variables not set")
var Trace bool

var stmtBufPool = sync.Pool{
	New: func() interface{} {
		return &bytes.Buffer{}
	},
}

func getStmtBuf() *bytes.Buffer {
	return stmtBufPool.Get().(*bytes.Buffer)
}
func releaseStmtBuf(buf *bytes.Buffer) {
	buf.Reset()
	buf.Grow(0)
	stmtBufPool.Put(buf)
}

func init() {
	if f := os.Getenv("OCTAV_TRACE_DB"); f != "" {
		if b, err := strconv.ParseBool(f); b && err == nil {
			Trace = true
		}
	}
}

func Init(dsn string) (err error) {
	if pdebug.Enabled {
		g := pdebug.Marker("db.Init").BindError(&err)
		defer g.End()
	}

	if dsn == "" {
		dsn, err = ConfigureDSN()
		if err != nil {
			return err
		}
	}

	dn := driverName()
	conn, err := sql.Open(dn, dsn)
	if err != nil {
		return err
	}

	if err := onConnect(conn); err != nil {
		return err
	}

	_db = &DB{&guard.DB{conn}}

	return nil
}

// Begin creates a new transaction (`Tx`) from the current
// global database connection
func Begin() (*Tx, error) {
	if _db == nil {
		return nil, errors.New("database has not been initialized")
	}

	tx, err := _db.Begin()
	if err != nil {
		return nil, err
	}
	return &Tx{tx}, nil
}
<|endoftext|>


<commit_before>package filer

import (
	"bytes"
	"fmt"
	"github.com/chrislusf/seaweedfs/weed/wdclient"
	"math"
	"sort"
	"sync"

	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/chrislusf/seaweedfs/weed/util"
)

func TotalSize(chunks []*filer_pb.FileChunk) (size uint64) {
	for _, c := range chunks {
		t := uint64(c.Offset + int64(c.Size))
		if size < t {
			size = t
		}
	}
	return
}

func FileSize(entry *filer_pb.Entry) (size uint64) {
	return maxUint64(TotalSize(entry.Chunks), entry.Attributes.FileSize)
}

func ETag(entry *filer_pb.Entry) (etag string) {
	if entry.Attributes == nil || entry.Attributes.Md5 == nil {
		return ETagChunks(entry.Chunks)
	}
	return fmt.Sprintf("%x", entry.Attributes.Md5)
}

func ETagEntry(entry *Entry) (etag string) {
	if entry.Attr.Md5 == nil {
		return ETagChunks(entry.Chunks)
	}
	return fmt.Sprintf("%x", entry.Attr.Md5)
}

func ETagChunks(chunks []*filer_pb.FileChunk) (etag string) {
	if len(chunks) == 1 {
		return fmt.Sprintf("%x", util.Base64Md5ToBytes(chunks[0].ETag))
	}
	md5_digests := [][]byte{}
	for _, c := range chunks {
		md5_digests = append(md5_digests, util.Base64Md5ToBytes(c.ETag))
	}
	return fmt.Sprintf("%x-%d", util.Md5(bytes.Join(md5_digests, nil)), len(chunks))
}

func CompactFileChunks(lookupFileIdFn wdclient.LookupFileIdFunctionType, chunks []*filer_pb.FileChunk) (compacted, garbage []*filer_pb.FileChunk) {

	visibles, _ := NonOverlappingVisibleIntervals(lookupFileIdFn, chunks, 0, math.MaxInt64)

	fileIds := make(map[string]bool)
	for _, interval := range visibles {
		fileIds[interval.fileId] = true
	}
	for _, chunk := range chunks {
		if _, found := fileIds[chunk.GetFileIdString()]; found {
			compacted = append(compacted, chunk)
		} else {
			garbage = append(garbage, chunk)
		}
	}

	return
}

func MinusChunks(lookupFileIdFn wdclient.LookupFileIdFunctionType, as, bs []*filer_pb.FileChunk) (delta []*filer_pb.FileChunk, err error) {

	aData, aMeta, aErr := ResolveChunkManifest(lookupFileIdFn, as, 0, math.MaxInt64)
	if aErr != nil {
		return nil, aErr
	}
	bData, bMeta, bErr := ResolveChunkManifest(lookupFileIdFn, bs, 0, math.MaxInt64)
	if bErr != nil {
		return nil, bErr
	}

	delta = append(delta, DoMinusChunks(aData, bData)...)
	delta = append(delta, DoMinusChunks(aMeta, bMeta)...)
	return
}

func DoMinusChunks(as, bs []*filer_pb.FileChunk) (delta []*filer_pb.FileChunk) {

	fileIds := make(map[string]bool)
	for _, interval := range bs {
		fileIds[interval.GetFileIdString()] = true
	}
	for _, chunk := range as {
		if _, found := fileIds[chunk.GetFileIdString()]; !found {
			delta = append(delta, chunk)
		}
	}

	return
}

type ChunkView struct {
	FileId      string
	Offset      int64
	Size        uint64
	LogicOffset int64 // actual offset in the file, for the data specified via [offset, offset+size) in current chunk
	ChunkSize   uint64
	CipherKey   []byte
	IsGzipped   bool
}

func (cv *ChunkView) IsFullChunk() bool {
	return cv.Size == cv.ChunkSize
}

func ViewFromChunks(lookupFileIdFn wdclient.LookupFileIdFunctionType, chunks []*filer_pb.FileChunk, offset int64, size int64) (views []*ChunkView) {

	visibles, _ := NonOverlappingVisibleIntervals(lookupFileIdFn, chunks, offset, offset+size)

	return ViewFromVisibleIntervals(visibles, offset, size)

}

func ViewFromVisibleIntervals(visibles []VisibleInterval, offset int64, size int64) (views []*ChunkView) {

	stop := offset + size
	if size == math.MaxInt64 {
		stop = math.MaxInt64
	}
	if stop < offset {
		stop = math.MaxInt64
	}

	for _, chunk := range visibles {

		chunkStart, chunkStop := max(offset, chunk.start), min(stop, chunk.stop)

		if chunkStart < chunkStop {
			views = append(views, &ChunkView{
				FileId:      chunk.fileId,
				Offset:      chunkStart - chunk.start + chunk.chunkOffset,
				Size:        uint64(chunkStop - chunkStart),
				LogicOffset: chunkStart,
				ChunkSize:   chunk.chunkSize,
				CipherKey:   chunk.cipherKey,
				IsGzipped:   chunk.isGzipped,
			})
		}
	}

	return views

}

func logPrintf(name string, visibles []VisibleInterval) {

	/*
		glog.V(0).Infof("%s len %d", name, len(visibles))
		for _, v := range visibles {
			glog.V(0).Infof("%s:  [%d,%d) %s %d", name, v.start, v.stop, v.fileId, v.chunkOffset)
		}
	*/
}

var bufPool = sync.Pool{
	New: func() interface{} {
		return new(VisibleInterval)
	},
}

func MergeIntoVisibles(visibles []VisibleInterval, chunk *filer_pb.FileChunk) (newVisibles []VisibleInterval) {

	newV := newVisibleInterval(chunk.Offset, chunk.Offset+int64(chunk.Size), chunk.GetFileIdString(), chunk.Mtime, 0, chunk.Size, chunk.CipherKey, chunk.IsCompressed)

	length := len(visibles)
	if length == 0 {
		return append(visibles, newV)
	}
	last := visibles[length-1]
	if last.stop <= chunk.Offset {
		return append(visibles, newV)
	}

	logPrintf("  before", visibles)
	// glog.V(0).Infof("newVisibles %d adding chunk [%d,%d) %s size:%d", len(newVisibles), chunk.Offset, chunk.Offset+int64(chunk.Size), chunk.GetFileIdString(), chunk.Size)
	chunkStop := chunk.Offset + int64(chunk.Size)
	for _, v := range visibles {
		if v.start < chunk.Offset && chunk.Offset < v.stop {
			t := newVisibleInterval(v.start, chunk.Offset, v.fileId, v.modifiedTime, v.chunkOffset, v.chunkSize, v.cipherKey, v.isGzipped)
			newVisibles = append(newVisibles, t)
			// glog.V(0).Infof("visible %d [%d,%d) =1> [%d,%d)", i, v.start, v.stop, t.start, t.stop)
		}
		if v.start < chunkStop && chunkStop < v.stop {
			t := newVisibleInterval(chunkStop, v.stop, v.fileId, v.modifiedTime, v.chunkOffset+(chunkStop-v.start), v.chunkSize, v.cipherKey, v.isGzipped)
			newVisibles = append(newVisibles, t)
			// glog.V(0).Infof("visible %d [%d,%d) =2> [%d,%d)", i, v.start, v.stop, t.start, t.stop)
		}
		if chunkStop <= v.start || v.stop <= chunk.Offset {
			newVisibles = append(newVisibles, v)
			// glog.V(0).Infof("visible %d [%d,%d) =3> [%d,%d)", i, v.start, v.stop, v.start, v.stop)
		}
	}
	newVisibles = append(newVisibles, newV)

	logPrintf("  append", newVisibles)

	for i := len(newVisibles) - 1; i >= 0; i-- {
		if i > 0 && newV.start < newVisibles[i-1].start {
			newVisibles[i] = newVisibles[i-1]
		} else {
			newVisibles[i] = newV
			break
		}
	}
	logPrintf("  sorted", newVisibles)

	return newVisibles
}

// NonOverlappingVisibleIntervals translates the file chunk into VisibleInterval in memory
// If the file chunk content is a chunk manifest
func NonOverlappingVisibleIntervals(lookupFileIdFn wdclient.LookupFileIdFunctionType, chunks []*filer_pb.FileChunk, startOffset int64, stopOffset int64) (visibles []VisibleInterval, err error) {

	chunks, _, err = ResolveChunkManifest(lookupFileIdFn, chunks, startOffset, stopOffset)

	visibles2 := readResolvedChunks(chunks)

	if true {
		return visibles2, err
	}

	sort.Slice(chunks, func(i, j int) bool {
		if chunks[i].Mtime == chunks[j].Mtime {
			filer_pb.EnsureFid(chunks[i])
			filer_pb.EnsureFid(chunks[j])
			if chunks[i].Fid == nil || chunks[j].Fid == nil {
				return true
			}
			return chunks[i].Fid.FileKey < chunks[j].Fid.FileKey
		}
		return chunks[i].Mtime < chunks[j].Mtime // keep this to make tests run
	})

	for _, chunk := range chunks {

		// glog.V(0).Infof("merge [%d,%d)", chunk.Offset, chunk.Offset+int64(chunk.Size))
		visibles = MergeIntoVisibles(visibles, chunk)

		logPrintf("add", visibles)

	}

	if len(visibles) != len(visibles2) {
		fmt.Printf("different visibles size %d : %d\n", len(visibles), len(visibles2))
	} else {
		for i := 0; i < len(visibles); i++ {
			checkDifference(visibles[i], visibles2[i])
		}
	}

	return
}

func checkDifference(x, y VisibleInterval) {
	if x.start != y.start ||
		x.stop != y.stop ||
		x.fileId != y.fileId ||
		x.modifiedTime != y.modifiedTime {
		fmt.Printf("different visible %+v : %+v\n", x, y)
	}
}

// find non-overlapping visible intervals
// visible interval map to one file chunk

type VisibleInterval struct {
	start        int64
	stop         int64
	modifiedTime int64
	fileId       string
	chunkOffset  int64
	chunkSize    uint64
	cipherKey    []byte
	isGzipped    bool
}

func newVisibleInterval(start, stop int64, fileId string, modifiedTime int64, chunkOffset int64, chunkSize uint64, cipherKey []byte, isGzipped bool) VisibleInterval {
	return VisibleInterval{
		start:        start,
		stop:         stop,
		fileId:       fileId,
		modifiedTime: modifiedTime,
		chunkOffset:  chunkOffset, // the starting position in the chunk
		chunkSize:    chunkSize,
		cipherKey:    cipherKey,
		isGzipped:    isGzipped,
	}
}

func min(x, y int64) int64 {
	if x <= y {
		return x
	}
	return y
}
func max(x, y int64) int64 {
	if x <= y {
		return y
	}
	return x
}
<commit_msg>remove deprecated code<commit_after>package filer

import (
	"bytes"
	"fmt"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/chrislusf/seaweedfs/weed/util"
	"github.com/chrislusf/seaweedfs/weed/wdclient"
	"math"
)

func TotalSize(chunks []*filer_pb.FileChunk) (size uint64) {
	for _, c := range chunks {
		t := uint64(c.Offset + int64(c.Size))
		if size < t {
			size = t
		}
	}
	return
}

func FileSize(entry *filer_pb.Entry) (size uint64) {
	return maxUint64(TotalSize(entry.Chunks), entry.Attributes.FileSize)
}

func ETag(entry *filer_pb.Entry) (etag string) {
	if entry.Attributes == nil || entry.Attributes.Md5 == nil {
		return ETagChunks(entry.Chunks)
	}
	return fmt.Sprintf("%x", entry.Attributes.Md5)
}

func ETagEntry(entry *Entry) (etag string) {
	if entry.Attr.Md5 == nil {
		return ETagChunks(entry.Chunks)
	}
	return fmt.Sprintf("%x", entry.Attr.Md5)
}

func ETagChunks(chunks []*filer_pb.FileChunk) (etag string) {
	if len(chunks) == 1 {
		return fmt.Sprintf("%x", util.Base64Md5ToBytes(chunks[0].ETag))
	}
	md5_digests := [][]byte{}
	for _, c := range chunks {
		md5_digests = append(md5_digests, util.Base64Md5ToBytes(c.ETag))
	}
	return fmt.Sprintf("%x-%d", util.Md5(bytes.Join(md5_digests, nil)), len(chunks))
}

func CompactFileChunks(lookupFileIdFn wdclient.LookupFileIdFunctionType, chunks []*filer_pb.FileChunk) (compacted, garbage []*filer_pb.FileChunk) {

	visibles, _ := NonOverlappingVisibleIntervals(lookupFileIdFn, chunks, 0, math.MaxInt64)

	fileIds := make(map[string]bool)
	for _, interval := range visibles {
		fileIds[interval.fileId] = true
	}
	for _, chunk := range chunks {
		if _, found := fileIds[chunk.GetFileIdString()]; found {
			compacted = append(compacted, chunk)
		} else {
			garbage = append(garbage, chunk)
		}
	}

	return
}

func MinusChunks(lookupFileIdFn wdclient.LookupFileIdFunctionType, as, bs []*filer_pb.FileChunk) (delta []*filer_pb.FileChunk, err error) {

	aData, aMeta, aErr := ResolveChunkManifest(lookupFileIdFn, as, 0, math.MaxInt64)
	if aErr != nil {
		return nil, aErr
	}
	bData, bMeta, bErr := ResolveChunkManifest(lookupFileIdFn, bs, 0, math.MaxInt64)
	if bErr != nil {
		return nil, bErr
	}

	delta = append(delta, DoMinusChunks(aData, bData)...)
	delta = append(delta, DoMinusChunks(aMeta, bMeta)...)
	return
}

func DoMinusChunks(as, bs []*filer_pb.FileChunk) (delta []*filer_pb.FileChunk) {

	fileIds := make(map[string]bool)
	for _, interval := range bs {
		fileIds[interval.GetFileIdString()] = true
	}
	for _, chunk := range as {
		if _, found := fileIds[chunk.GetFileIdString()]; !found {
			delta = append(delta, chunk)
		}
	}

	return
}

type ChunkView struct {
	FileId      string
	Offset      int64
	Size        uint64
	LogicOffset int64 // actual offset in the file, for the data specified via [offset, offset+size) in current chunk
	ChunkSize   uint64
	CipherKey   []byte
	IsGzipped   bool
}

func (cv *ChunkView) IsFullChunk() bool {
	return cv.Size == cv.ChunkSize
}

func ViewFromChunks(lookupFileIdFn wdclient.LookupFileIdFunctionType, chunks []*filer_pb.FileChunk, offset int64, size int64) (views []*ChunkView) {

	visibles, _ := NonOverlappingVisibleIntervals(lookupFileIdFn, chunks, offset, offset+size)

	return ViewFromVisibleIntervals(visibles, offset, size)

}

func ViewFromVisibleIntervals(visibles []VisibleInterval, offset int64, size int64) (views []*ChunkView) {

	stop := offset + size
	if size == math.MaxInt64 {
		stop = math.MaxInt64
	}
	if stop < offset {
		stop = math.MaxInt64
	}

	for _, chunk := range visibles {

		chunkStart, chunkStop := max(offset, chunk.start), min(stop, chunk.stop)

		if chunkStart < chunkStop {
			views = append(views, &ChunkView{
				FileId:      chunk.fileId,
				Offset:      chunkStart - chunk.start + chunk.chunkOffset,
				Size:        uint64(chunkStop - chunkStart),
				LogicOffset: chunkStart,
				ChunkSize:   chunk.chunkSize,
				CipherKey:   chunk.cipherKey,
				IsGzipped:   chunk.isGzipped,
			})
		}
	}

	return views

}

func logPrintf(name string, visibles []VisibleInterval) {

	/*
		glog.V(0).Infof("%s len %d", name, len(visibles))
		for _, v := range visibles {
			glog.V(0).Infof("%s:  [%d,%d) %s %d", name, v.start, v.stop, v.fileId, v.chunkOffset)
		}
	*/
}

// NonOverlappingVisibleIntervals translates the file chunk into VisibleInterval in memory
// If the file chunk content is a chunk manifest
func NonOverlappingVisibleIntervals(lookupFileIdFn wdclient.LookupFileIdFunctionType, chunks []*filer_pb.FileChunk, startOffset int64, stopOffset int64) (visibles []VisibleInterval, err error) {

	chunks, _, err = ResolveChunkManifest(lookupFileIdFn, chunks, startOffset, stopOffset)

	visibles = readResolvedChunks(chunks)

	return

}

// find non-overlapping visible intervals
// visible interval map to one file chunk

type VisibleInterval struct {
	start        int64
	stop         int64
	modifiedTime int64
	fileId       string
	chunkOffset  int64
	chunkSize    uint64
	cipherKey    []byte
	isGzipped    bool
}

func min(x, y int64) int64 {
	if x <= y {
		return x
	}
	return y
}
func max(x, y int64) int64 {
	if x <= y {
		return y
	}
	return x
}
<|endoftext|>


<commit_before>// Go wrapper for the Brotli C decoder implementation
package dec

/*
#include "./decode.h"

typedef uint8_t dict[122784];
dict* decodeBrotliDictionary;
*/
import "C"

import (
	"errors"
	"io"
	"runtime"
	"unsafe"

	"gopkg.in/kothar/brotli-go.v0/shared"
)

func init() {
	// Set up the default dictionary from the data in the shared package
	C.decodeBrotliDictionary = (*C.dict)(shared.GetDictionary())
}

// Decompress a Brotli-encoded buffer. Uses decodedBuffer as the destination buffer unless it is too small,
// in which case a new buffer is allocated.
// Returns the slice of the decodedBuffer containing the output, or an error.
func DecompressBuffer(encodedBuffer []byte, decodedBuffer []byte) ([]byte, error) {
	encodedLength := len(encodedBuffer)
	var decodedSize C.size_t

	// If the user has provided a sensibly size buffer, assume they know how long the output should be
	// Otherwise try to determine the correct length from the input
	if len(decodedBuffer) < len(encodedBuffer) {
		success := C.BrotliDecompressedSize(C.size_t(encodedLength), toC(encodedBuffer), &decodedSize)
		if success != 1 {
			// We can't know in advance how much buffer to allocate, so we will just have to guess
			decodedSize = C.size_t(len(encodedBuffer) * 6)
		}

		if len(decodedBuffer) < int(decodedSize) {
			decodedBuffer = make([]byte, decodedSize)
		}
	}

	// The size of the ouput buffer available
	decodedLength := C.size_t(len(decodedBuffer))
	result := C.BrotliDecompressBuffer(C.size_t(encodedLength), toC(encodedBuffer), &decodedLength, toC(decodedBuffer))
	switch result {
	case C.BROTLI_RESULT_SUCCESS:
		// We're finished
		return decodedBuffer[0:decodedLength], nil
	case C.BROTLI_RESULT_NEEDS_MORE_OUTPUT:
		// We needed more output buffer
		decodedBuffer = make([]byte, len(decodedBuffer)*2)
		return DecompressBuffer(encodedBuffer, decodedBuffer)
	case C.BROTLI_RESULT_ERROR:
		return nil, errors.New("Brotli decompression error")
	case C.BROTLI_RESULT_NEEDS_MORE_INPUT:
		// We can't handle streaming more input results here
		return nil, errors.New("Brotli decompression error: needs more input")
	default:
		return nil, errors.New("Unrecognised Brotli decompression error")
	}
}

func toC(array []byte) *C.uint8_t {
	return (*C.uint8_t)(unsafe.Pointer(&array[0]))
}

// Decompresses a Brotli-encoded stream using the io.Reader interface
type BrotliReader struct {
	reader io.Reader
	state  *C.BrotliState

	needOutput bool  // State bounces between needing input and output
	err        error // Persistent error

	// Internal buffer for compressed data
	buffer []byte

	availableIn C.size_t
	nextIn      *C.uint8_t
	totalOut    C.size_t
}

// Fill a buffer, p, with the decompressed contents of the stream.
// Returns the number of bytes read, or an error
func (r *BrotliReader) Read(p []byte) (cnt int, err error) {
	if len(p) == 0 || r.err != nil {
		return 0, r.err
	}

	// Prepare arguments
	maxOutput := len(p)
	availableOut := C.size_t(maxOutput)
	nextOut := (*C.uint8_t)(unsafe.Pointer(&p[0]))

	for availableOut > 0 && r.err == nil {
		// Read more compressed data
		if r.availableIn == 0 && !r.needOutput {
			read, err := r.reader.Read(r.buffer)
			if read > 0 && err == io.EOF {
				err = nil // Let next Read call return (0, io.EOF)
			}
			if err != nil {
				if err == io.EOF {
					err = io.ErrUnexpectedEOF
				}
				r.err = err
			}
			r.availableIn = C.size_t(read)
			r.nextIn = (*C.uint8_t)(unsafe.Pointer(&r.buffer[0]))
		}

		if r.availableIn > 0 || r.needOutput {
			// Decompress
			result := C.BrotliDecompressStream(
				&r.availableIn,
				&r.nextIn,
				&availableOut,
				&nextOut,
				&r.totalOut,
				r.state,
			)

			cnt = maxOutput - int(availableOut)
			switch result {
			case C.BROTLI_RESULT_SUCCESS:
				r.err = io.EOF
			case C.BROTLI_RESULT_NEEDS_MORE_OUTPUT:
				r.needOutput = true
				if cnt > 0 {
					return cnt, r.err
				}
				r.err = errors.New("Brotli decompression error: needs more output buffer")
			case C.BROTLI_RESULT_ERROR:
				r.err = errors.New("Brotli decompression error")
			case C.BROTLI_RESULT_NEEDS_MORE_INPUT:
				r.needOutput = false
			default:
				r.err = errors.New("Unrecognized Brotli decompression error")
			}
		}
	}
	return cnt, r.err
}

// Close the reader and clean up any decompressor state.
func (r *BrotliReader) Close() error {
	if r.state == nil {
		return r.err
	}
	C.BrotliStateCleanup(r.state)
	r.state = nil
	if r.err == nil || r.err == io.EOF {
		r.err = io.ErrClosedPipe // Make sure future operations fail
		return nil
	}
	return r.err
}

// Returns a Reader that decompresses the stream from another reader.
//
// Ensure that you Close the stream when you are finished in order to clean up the
// Brotli decompression state.
//
// The internal decompression buffer defaults to 128kb
func NewBrotliReader(stream io.Reader) *BrotliReader {
	return NewBrotliReaderSize(stream, 128*1024)
}

// The same as NewBrotliReader, but allows the internal buffer size to be set.
//
// The size of the internal buffer may be specified which will hold compressed data
// before being read by the decompressor
func NewBrotliReaderSize(stream io.Reader, size int) *BrotliReader {
	r := &BrotliReader{
		reader: stream,
		state:  new(C.BrotliState),
		buffer: make([]byte, size),
	}

	C.BrotliStateInit(r.state)

	runtime.SetFinalizer(r, func(c io.Closer) { c.Close() })

	return r
}
<commit_msg>Closes #14<commit_after>// Go wrapper for the Brotli C decoder implementation
package dec

/*
#include "./decode.h"

typedef uint8_t dict[122784];
dict* decodeBrotliDictionary;
*/
import "C"

import (
	"errors"
	"io"
	"runtime"
	"unsafe"

	"gopkg.in/kothar/brotli-go.v0/shared"
)

func init() {
	// Set up the default dictionary from the data in the shared package
	C.decodeBrotliDictionary = (*C.dict)(shared.GetDictionary())
}

// Decompress a Brotli-encoded buffer. Uses decodedBuffer as the destination buffer unless it is too small,
// in which case a new buffer is allocated.
// Returns the slice of the decodedBuffer containing the output, or an error.
func DecompressBuffer(encodedBuffer []byte, decodedBuffer []byte) ([]byte, error) {
	encodedLength := len(encodedBuffer)
	var decodedSize C.size_t

	// If the user has provided a sensibly size buffer, assume they know how long the output should be
	// Otherwise try to determine the correct length from the input
	if len(decodedBuffer) < len(encodedBuffer) {
		success := C.BrotliDecompressedSize(C.size_t(encodedLength), toC(encodedBuffer), &decodedSize)
		if success != 1 {
			// We can't know in advance how much buffer to allocate, so we will just have to guess
			decodedSize = C.size_t(len(encodedBuffer) * 6)
		}

		if len(decodedBuffer) < int(decodedSize) {
			decodedBuffer = make([]byte, decodedSize)
		}
	}

	// The size of the ouput buffer available
	decodedLength := C.size_t(len(decodedBuffer))
	result := C.BrotliDecompressBuffer(C.size_t(encodedLength), toC(encodedBuffer), &decodedLength, toC(decodedBuffer))
	switch result {
	case C.BROTLI_RESULT_SUCCESS:
		// We're finished
		return decodedBuffer[0:decodedLength], nil
	case C.BROTLI_RESULT_NEEDS_MORE_OUTPUT:
		// We needed more output buffer
		decodedBuffer = make([]byte, len(decodedBuffer)*2)
		return DecompressBuffer(encodedBuffer, decodedBuffer)
	case C.BROTLI_RESULT_ERROR:
		return nil, errors.New("Brotli decompression error")
	case C.BROTLI_RESULT_NEEDS_MORE_INPUT:
		// We can't handle streaming more input results here
		return nil, errors.New("Brotli decompression error: needs more input")
	default:
		return nil, errors.New("Unrecognised Brotli decompression error")
	}
}

func toC(array []byte) *C.uint8_t {
	return (*C.uint8_t)(unsafe.Pointer(&array[0]))
}

// don't let the go 1.5.1 GC put its dirty paws in our beautiful C struct (#14)
// cf. https://github.com/youtube/vitess/blob/071d0e649f22034ad4285c7431ac0a2c9c20090d/go/cgzip/zstream.go#L86-L89
type CBrotliState [unsafe.Sizeof(C.BrotliState{})]C.char

// Decompresses a Brotli-encoded stream using the io.Reader interface
type BrotliReader struct {
	reader io.Reader
	state  CBrotliState
	closed bool

	needOutput bool  // State bounces between needing input and output
	err        error // Persistent error

	// Internal buffer for compressed data
	buffer []byte

	availableIn C.size_t
	nextIn      *C.uint8_t
	totalOut    C.size_t
}

// Fill a buffer, p, with the decompressed contents of the stream.
// Returns the number of bytes read, or an error
func (r *BrotliReader) Read(p []byte) (cnt int, err error) {
	if len(p) == 0 || r.err != nil {
		return 0, r.err
	}

	// Prepare arguments
	maxOutput := len(p)
	availableOut := C.size_t(maxOutput)
	nextOut := (*C.uint8_t)(unsafe.Pointer(&p[0]))

	for availableOut > 0 && r.err == nil {
		// Read more compressed data
		if r.availableIn == 0 && !r.needOutput {
			read, err := r.reader.Read(r.buffer)
			if read > 0 && err == io.EOF {
				err = nil // Let next Read call return (0, io.EOF)
			}
			if err != nil {
				if err == io.EOF {
					err = io.ErrUnexpectedEOF
				}
				r.err = err
			}
			r.availableIn = C.size_t(read)
			r.nextIn = (*C.uint8_t)(unsafe.Pointer(&r.buffer[0]))
		}

		if r.availableIn > 0 || r.needOutput {
			// Decompress
			result := C.BrotliDecompressStream(
				&r.availableIn,
				&r.nextIn,
				&availableOut,
				&nextOut,
				&r.totalOut,
				(*C.BrotliState)(unsafe.Pointer(&r.state[0])),
			)

			cnt = maxOutput - int(availableOut)
			switch result {
			case C.BROTLI_RESULT_SUCCESS:
				r.err = io.EOF
			case C.BROTLI_RESULT_NEEDS_MORE_OUTPUT:
				r.needOutput = true
				if cnt > 0 {
					return cnt, r.err
				}
				r.err = errors.New("Brotli decompression error: needs more output buffer")
			case C.BROTLI_RESULT_ERROR:
				r.err = errors.New("Brotli decompression error")
			case C.BROTLI_RESULT_NEEDS_MORE_INPUT:
				r.needOutput = false
			default:
				r.err = errors.New("Unrecognized Brotli decompression error")
			}
		}
	}
	return cnt, r.err
}

// Close the reader and clean up any decompressor state.
func (r *BrotliReader) Close() error {
	if r.closed {
		return r.err
	}
	C.BrotliStateCleanup((*C.BrotliState)(unsafe.Pointer(&r.state[0])))
	r.closed = true
	if r.err == nil || r.err == io.EOF {
		r.err = io.ErrClosedPipe // Make sure future operations fail
		return nil
	}
	return r.err
}

// Returns a Reader that decompresses the stream from another reader.
//
// Ensure that you Close the stream when you are finished in order to clean up the
// Brotli decompression state.
//
// The internal decompression buffer defaults to 128kb
func NewBrotliReader(stream io.Reader) *BrotliReader {
	return NewBrotliReaderSize(stream, 128*1024)
}

// The same as NewBrotliReader, but allows the internal buffer size to be set.
//
// The size of the internal buffer may be specified which will hold compressed data
// before being read by the decompressor
func NewBrotliReaderSize(stream io.Reader, size int) *BrotliReader {
	r := &BrotliReader{
		reader: stream,
		buffer: make([]byte, size),
	}

	C.BrotliStateInit((*C.BrotliState)(unsafe.Pointer(&r.state[0])))

	runtime.SetFinalizer(r, func(c io.Closer) { c.Close() })

	return r
}
<|endoftext|>


<commit_before>package data

import (
	"bytes"
	"crypto/sha512"
	"fmt"
	"math"
)

type NodeIndex uint64

func (i *NodeIndex) Previous() *NodeIndex {
	if i == nil || *i == 0 {
		return nil
	}
	prev := *i - 1
	return &prev
}

func (i *NodeIndex) Next() *NodeIndex {
	if i == nil || *i == math.MaxUint64 {
		return nil
	}
	next := *i + 1
	return &next
}

func LedgerIndex(le LedgerEntry) (*Hash256, error) {
	switch v := le.(type) {
	case *AccountRoot:
		return GetAccountRootIndex(v.Account)
	case *RippleState:
		return GetRippleStateIndex(v.LowLimit.Issuer, v.HighLimit.Issuer, v.Balance.Currency)
	case *Offer:
		return GetOfferIndex(v.Account, v.Sequence)
	case *LedgerHashes:
		return GetLedgerHashIndex()
	case *Directory:
		return GetDirectoryNodeIndex(v.RootIndex, v.IndexPrevious.Next())
	default:
		return nil, fmt.Errorf("Unknown LedgerEntry")
	}
}

func GetAccountRootIndex(account Account) (*Hash256, error) {
	return buildIndex([]interface{}{NS_ACCOUNT, account.Bytes()})
}

func GetOfferIndex(account Account, sequence uint32) (*Hash256, error) {
	return buildIndex([]interface{}{NS_OFFER, account.Bytes(), sequence})
}

func GetRippleStateIndex(a, b Account, c Currency) (*Hash256, error) {
	if bytes.Compare(a.Bytes(), b.Bytes()) < 0 {
		return buildIndex([]interface{}{NS_RIPPLE_STATE, a.Bytes(), b.Bytes(), c.Bytes()})
	}
	return buildIndex([]interface{}{NS_RIPPLE_STATE, b.Bytes(), a.Bytes(), c.Bytes()})
}

func GetDirectoryNodeIndex(root Hash256, index *NodeIndex) (*Hash256, error) {
	if index == nil {
		return &root, nil
	}
	return buildIndex([]interface{}{NS_DIRECTORY_NODE, root, *index})
}

func GetOwnerDirectoryIndex(account Account) (*Hash256, error) {
	return buildIndex([]interface{}{NS_OWNER_DIRECTORY, account.Bytes()})
}

func GetBookIndex(paysCurrency, getsCurrency Hash160, paysIssuer, getsIssuer Hash160) (*Hash256, error) {
	//TODO: change types to Currency and Account
	index, err := buildIndex([]interface{}{NS_BOOK_DIRECTORY, paysCurrency.Bytes(), getsCurrency.Bytes(), paysCurrency.Bytes(), getsCurrency.Bytes()})
	if err != nil {
		return nil, err
	}
	var zero [8]byte
	copy(index[24:], zero[:])
	return index, nil
}

func GetFeeIndex() (*Hash256, error) {
	return buildIndex([]interface{}{NS_FEE})
}

func GetAmendmentsIndex() (*Hash256, error) {
	return buildIndex([]interface{}{NS_AMENDMENT})
}

func GetLedgerHashIndex() (*Hash256, error) {
	return buildIndex([]interface{}{NS_SKIP_LIST})
}

func GetPreviousLedgerHashIndex(sequence uint32) (*Hash256, error) {
	return buildIndex([]interface{}{NS_SKIP_LIST, sequence >> 16})
}

func buildIndex(items []interface{}) (*Hash256, error) {
	index := sha512.New()
	for _, item := range items {
		if err := write(index, item); err != nil {
			return nil, err
		}
	}
	return NewHash256(index.Sum(nil)[:32])
}
<commit_msg>Add Indexing for FeeSettings and Amendments<commit_after>package data

import (
	"bytes"
	"crypto/sha512"
	"fmt"
	"math"
)

type NodeIndex uint64

func (i *NodeIndex) Previous() *NodeIndex {
	if i == nil || *i == 0 {
		return nil
	}
	prev := *i - 1
	return &prev
}

func (i *NodeIndex) Next() *NodeIndex {
	if i == nil || *i == math.MaxUint64 {
		return nil
	}
	next := *i + 1
	return &next
}

func LedgerIndex(le LedgerEntry) (*Hash256, error) {
	switch v := le.(type) {
	case *AccountRoot:
		return GetAccountRootIndex(v.Account)
	case *RippleState:
		return GetRippleStateIndex(v.LowLimit.Issuer, v.HighLimit.Issuer, v.Balance.Currency)
	case *Offer:
		return GetOfferIndex(v.Account, v.Sequence)
	case *LedgerHashes:
		return GetLedgerHashIndex()
	case *Directory:
		return GetDirectoryNodeIndex(v.RootIndex, v.IndexPrevious.Next())
	case *FeeSettings:
		return buildIndex([]interface{}{NS_FEE})
	case *Amendments:
		return buildIndex([]interface{}{NS_AMENDMENT})
	default:
		return nil, fmt.Errorf("Unknown LedgerEntry")
	}
}

func GetAccountRootIndex(account Account) (*Hash256, error) {
	return buildIndex([]interface{}{NS_ACCOUNT, account.Bytes()})
}

func GetOfferIndex(account Account, sequence uint32) (*Hash256, error) {
	return buildIndex([]interface{}{NS_OFFER, account.Bytes(), sequence})
}

func GetRippleStateIndex(a, b Account, c Currency) (*Hash256, error) {
	if bytes.Compare(a.Bytes(), b.Bytes()) < 0 {
		return buildIndex([]interface{}{NS_RIPPLE_STATE, a.Bytes(), b.Bytes(), c.Bytes()})
	}
	return buildIndex([]interface{}{NS_RIPPLE_STATE, b.Bytes(), a.Bytes(), c.Bytes()})
}

func GetDirectoryNodeIndex(root Hash256, index *NodeIndex) (*Hash256, error) {
	if index == nil {
		return &root, nil
	}
	return buildIndex([]interface{}{NS_DIRECTORY_NODE, root, *index})
}

func GetOwnerDirectoryIndex(account Account) (*Hash256, error) {
	return buildIndex([]interface{}{NS_OWNER_DIRECTORY, account.Bytes()})
}

func GetBookIndex(paysCurrency, getsCurrency Hash160, paysIssuer, getsIssuer Hash160) (*Hash256, error) {
	//TODO: change types to Currency and Account
	index, err := buildIndex([]interface{}{NS_BOOK_DIRECTORY, paysCurrency.Bytes(), getsCurrency.Bytes(), paysCurrency.Bytes(), getsCurrency.Bytes()})
	if err != nil {
		return nil, err
	}
	var zero [8]byte
	copy(index[24:], zero[:])
	return index, nil
}

func GetFeeIndex() (*Hash256, error) {
	return buildIndex([]interface{}{NS_FEE})
}

func GetAmendmentsIndex() (*Hash256, error) {
	return buildIndex([]interface{}{NS_AMENDMENT})
}

func GetLedgerHashIndex() (*Hash256, error) {
	return buildIndex([]interface{}{NS_SKIP_LIST})
}

func GetPreviousLedgerHashIndex(sequence uint32) (*Hash256, error) {
	return buildIndex([]interface{}{NS_SKIP_LIST, sequence >> 16})
}

func buildIndex(items []interface{}) (*Hash256, error) {
	index := sha512.New()
	for _, item := range items {
		if err := write(index, item); err != nil {
			return nil, err
		}
	}
	return NewHash256(index.Sum(nil)[:32])
}
<|endoftext|>


<commit_before>package mpb

import (
	"fmt"
	"strconv"
	"time"
)

type decoratorFuncType uint

const (
	decoratorAppend decoratorFuncType = iota
	decoratorPrepend
)

// DecoratorFunc is a function that can be prepended and appended to the progress bar
type DecoratorFunc func(s *Statistics) string

type decorator struct {
	kind decoratorFuncType
	f    DecoratorFunc
}

func (b *Bar) PrependName(name string, padding int) *Bar {
	layout := "%" + strconv.Itoa(padding) + "s"
	b.PrependFunc(func(s *Statistics) string {
		return fmt.Sprintf(layout, name)
	})
	return b
}

func (b *Bar) PrependCounters(unit Units, padding int) *Bar {
	layout := "%" + strconv.Itoa(padding) + "s"
	b.PrependFunc(func(s *Statistics) string {
		current := Format(s.Current).To(unit)
		total := Format(s.Total).To(unit)
		str := fmt.Sprintf("%s / %s", current, total)
		return fmt.Sprintf(layout, str)
	})
	return b
}

func (b *Bar) PrependETA(padding int) *Bar {
	layout := "ETA%" + strconv.Itoa(padding) + "s"
	b.PrependFunc(func(s *Statistics) string {
		return fmt.Sprintf(layout, time.Duration(s.Eta().Seconds())*time.Second)
	})
	return b
}

func (b *Bar) AppendETA(padding int) *Bar {
	layout := "ETA %" + strconv.Itoa(padding) + "s"
	b.AppendFunc(func(s *Statistics) string {
		return fmt.Sprintf(layout, time.Duration(s.Eta().Seconds())*time.Second)
	})
	return b
}

func (b *Bar) PrependElapsed(padding int) *Bar {
	layout := "%" + strconv.Itoa(padding) + "s"
	b.PrependFunc(func(s *Statistics) string {
		return fmt.Sprintf(layout, time.Duration(s.TimeElapsed.Seconds())*time.Second)
	})
	return b
}

func (b *Bar) AppendElapsed() *Bar {
	b.AppendFunc(func(s *Statistics) string {
		return fmt.Sprint(time.Duration(s.TimeElapsed.Seconds()) * time.Second)
	})
	return b
}

func (b *Bar) AppendPercentage() *Bar {
	b.AppendFunc(func(s *Statistics) string {
		completed := int(100 * float64(s.Current) / float64(s.Total))
		return fmt.Sprintf("%3d %%", completed)
	})
	return b
}

func (b *Bar) PrependPercentage(padding int) *Bar {
	layout := "%" + strconv.Itoa(padding) + "d %%"
	b.PrependFunc(func(s *Statistics) string {
		completed := int(100 * float64(s.Current) / float64(s.Total))
		return fmt.Sprintf(layout, completed)
	})
	return b
}
<commit_msg>use percentage func<commit_after>package mpb

import (
	"fmt"
	"strconv"
	"time"
)

type decoratorFuncType uint

const (
	decoratorAppend decoratorFuncType = iota
	decoratorPrepend
)

// DecoratorFunc is a function that can be prepended and appended to the progress bar
type DecoratorFunc func(s *Statistics) string

type decorator struct {
	kind decoratorFuncType
	f    DecoratorFunc
}

func (b *Bar) PrependName(name string, padding int) *Bar {
	layout := "%" + strconv.Itoa(padding) + "s"
	b.PrependFunc(func(s *Statistics) string {
		return fmt.Sprintf(layout, name)
	})
	return b
}

func (b *Bar) PrependCounters(unit Units, padding int) *Bar {
	layout := "%" + strconv.Itoa(padding) + "s"
	b.PrependFunc(func(s *Statistics) string {
		current := Format(s.Current).To(unit)
		total := Format(s.Total).To(unit)
		str := fmt.Sprintf("%s / %s", current, total)
		return fmt.Sprintf(layout, str)
	})
	return b
}

func (b *Bar) PrependETA(padding int) *Bar {
	layout := "ETA%" + strconv.Itoa(padding) + "s"
	b.PrependFunc(func(s *Statistics) string {
		return fmt.Sprintf(layout, time.Duration(s.Eta().Seconds())*time.Second)
	})
	return b
}

func (b *Bar) AppendETA(padding int) *Bar {
	layout := "ETA %" + strconv.Itoa(padding) + "s"
	b.AppendFunc(func(s *Statistics) string {
		return fmt.Sprintf(layout, time.Duration(s.Eta().Seconds())*time.Second)
	})
	return b
}

func (b *Bar) PrependElapsed(padding int) *Bar {
	layout := "%" + strconv.Itoa(padding) + "s"
	b.PrependFunc(func(s *Statistics) string {
		return fmt.Sprintf(layout, time.Duration(s.TimeElapsed.Seconds())*time.Second)
	})
	return b
}

func (b *Bar) AppendElapsed() *Bar {
	b.AppendFunc(func(s *Statistics) string {
		return fmt.Sprint(time.Duration(s.TimeElapsed.Seconds()) * time.Second)
	})
	return b
}

func (b *Bar) AppendPercentage() *Bar {
	b.AppendFunc(func(s *Statistics) string {
		completed := percentage(s.Total, s.Current, 100)
		return fmt.Sprintf("%3d %%", completed)
	})
	return b
}

func (b *Bar) PrependPercentage(padding int) *Bar {
	layout := "%" + strconv.Itoa(padding) + "d %%"
	b.PrependFunc(func(s *Statistics) string {
		completed := percentage(s.Total, s.Current, 100)
		return fmt.Sprintf(layout, completed)
	})
	return b
}
<|endoftext|>


<commit_before>package tilemap

import (
	"math"

	"github.com/veandco/go-sdl2/sdl"
	"github.com/yukiisbored/Kaori/texture"
)

// Tileset is a set of sprites or textures for the tiled map
type Tileset struct {
	Name string `xml:"name,attr"`

	TileWidth  int `xml:"tilewidth,attr"`
	TileHeight int `xml:"tileheight,attr"`

	TileCount int `xml:"tilecount,attr"`

	Columns int `xml:"columns,attr"`

	Spacing int `xml:"spacing,attr"`
	Margin  int `xml:"margin,attr"`

	FirstID int `xml:"firstgid,attr"`

	Image Image `xml:"image"`
}

// Image is the actual image for the spritesheet or texture
type Image struct {
	Source string `xml:"source,attr"`

	Width  int `xml:"width,attr"`
	Height int `xml:"height,attr"`
}

// Load loads the image needed as an SDL Texture via Kaori's texture package
func (t *Tileset) Load(renderer *sdl.Renderer, folder string) error {
	fileName := t.Image.Source
	path := folder + "/" + fileName

	return texture.Load(renderer, path, t.Name)
}

// Free free the loaded image texture
func (t *Tileset) Free() {
	texture.Free(t.Name)
}

// Draw Tile draw a specific tile to the renderer at a specific location
// The tile number is a continuous number starting from the tile set's first id
func (t *Tileset) DrawTile(renderer *sdl.Renderer, x, y int32, tile int) {
	// Subtract the value by the tileset's first id
	tile = tile - t.FirstID + 1

	// Just ignore when it's lower than 0
	if tile < 0 {
		return
	}

	row := int32(math.Ceil(float64(tile / t.Columns)))
	frame := tile % t.Columns

	texture.DrawFrame(renderer, t.Name, x, y,
		int32(t.TileWidth), int32(t.TileHeight),
		int32(row), int32(frame),
		int32(t.Spacing), int32(t.Margin), 0, sdl.FLIP_NONE)
}
<commit_msg>tilemap: fixed golint problems on tileset<commit_after>package tilemap

import (
	"math"

	"github.com/veandco/go-sdl2/sdl"
	"github.com/yukiisbored/Kaori/texture"
)

// Tileset is a set of sprites or textures for the tiled map
type Tileset struct {
	Name string `xml:"name,attr"`

	TileWidth  int `xml:"tilewidth,attr"`
	TileHeight int `xml:"tileheight,attr"`

	TileCount int `xml:"tilecount,attr"`

	Columns int `xml:"columns,attr"`

	Spacing int `xml:"spacing,attr"`
	Margin  int `xml:"margin,attr"`

	FirstID int `xml:"firstgid,attr"`

	Image Image `xml:"image"`
}

// Image is the actual image for the spritesheet or texture
type Image struct {
	Source string `xml:"source,attr"`

	Width  int `xml:"width,attr"`
	Height int `xml:"height,attr"`
}

// Load loads the image needed as an SDL Texture via Kaori's texture package
func (t *Tileset) Load(renderer *sdl.Renderer, folder string) error {
	fileName := t.Image.Source
	path := folder + "/" + fileName

	return texture.Load(renderer, path, t.Name)
}

// Free free the loaded image texture
func (t *Tileset) Free() {
	texture.Free(t.Name)
}

// DrawTile draw a specific tile to the renderer at a specific location
// The tile number is a continuous number starting from the tile set's first id
func (t *Tileset) DrawTile(renderer *sdl.Renderer, x, y int32, tile int) {
	// Subtract the value by the tileset's first id
	tile = tile - t.FirstID + 1

	// Just ignore when it's lower than 0
	if tile < 0 {
		return
	}

	row := int32(math.Ceil(float64(tile / t.Columns)))
	frame := tile % t.Columns

	texture.DrawFrame(renderer, t.Name, x, y,
		int32(t.TileWidth), int32(t.TileHeight),
		int32(row), int32(frame),
		int32(t.Spacing), int32(t.Margin), 0, sdl.FLIP_NONE)
}
<|endoftext|>


<commit_before>package stretcher

import (
	"bufio"
	"bytes"
	"context"
	crand "crypto/rand"
	"encoding/binary"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	mrand "math/rand"
	"net/http"
	"net/url"
	"os"
	"strings"
	"time"

	"cloud.google.com/go/storage"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3"
)

var (
	LogBuffer bytes.Buffer
	Version   string
)

type Config struct {
	MaxBandWidth uint64
	Timeout      time.Duration
	InitSleep    time.Duration
	Retry        int
	RetryWait    time.Duration
}

const Nanoseconds = 1000 * 1000 * 1000

func init() {
	log.SetOutput(io.MultiWriter(os.Stderr, &LogBuffer))
}

func RandomTime(delay float64) time.Duration {
	if delay <= 0 {
		return time.Duration(0)
	}
	// http://stackoverflow.com/questions/6181260/generating-random-numbers-in-go
	var s int64
	if err := binary.Read(crand.Reader, binary.LittleEndian, &s); err != nil {
		s = time.Now().UnixNano()
	}
	mrand.Seed(s)
	n := mrand.Int63n(int64(delay * Nanoseconds))
	return time.Duration(n)
}

func Run(conf Config) error {
	var err error
	log.Println("Starting up stretcher agent", Version)
	if conf.InitSleep > 0 {
		log.Printf("Sleeping %s", conf.InitSleep)
		time.Sleep(conf.InitSleep)
	}

	manifestURL, err := parseEvents()
	if err != nil {
		return fmt.Errorf("Could not parse event: %s", err)
	}

	log.Println("Loading manifest:", manifestURL)
	m, err := getManifest(manifestURL)
	if err != nil {
		return fmt.Errorf("Load manifest failed: %s", err)
	}
	log.Printf("Executing manifest %#v", m)

	err = m.Deploy(conf)
	if err != nil {
		log.Println("Deploy manifest failed:", err)
		if ferr := m.Commands.Failure.InvokePipe(&LogBuffer); ferr != nil {
			log.Println(ferr)
		}
		return fmt.Errorf("Deploy manifest failed: %s", err)
	}
	log.Println("Deploy manifest succeeded.")
	err = m.Commands.Success.InvokePipe(&LogBuffer)
	if err != nil {
		log.Println(err)
	}
	log.Println("Done")
	return nil
}

func getS3(u *url.URL) (io.ReadCloser, error) {
	sess, err := session.NewSession()
	if err != nil {
		return nil, err
	}
	log.Println("region:", *sess.Config.Region)

	svc := s3.New(sess)
	key := strings.TrimLeft(u.Path, "/")
	result, err := svc.GetObject(&s3.GetObjectInput{
		Bucket: aws.String(u.Host),
		Key:    aws.String(key),
	})
	if err != nil {
		return nil, err
	}
	return result.Body, nil
}

func getGS(u *url.URL) (io.ReadCloser, error) {
	var err error
	trimPath := strings.Trim(u.Path, "/")

	ctx := context.Background()
	client, err := storage.NewClient(ctx)
	if err != nil {
		return nil, err
	}
	bucket, err := client.Bucket(u.Host).Object(trimPath).NewReader(ctx)
	if err != nil {
		return nil, err
	}
	return bucket, nil
}

func getFile(u *url.URL) (io.ReadCloser, error) {
	return os.Open(u.Path)
}

func getHTTP(u *url.URL) (io.ReadCloser, error) {
	req, err := http.NewRequest("GET", u.String(), nil)
	if err != nil {
		return nil, err
	}
	req.Header.Add("User-Agent", "Stretcher/"+Version)

	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return nil, err
	}
	return resp.Body, nil
}

func getURL(urlStr string) (io.ReadCloser, error) {
	log.Println("Loading URL", urlStr)
	u, err := url.Parse(urlStr)
	if err != nil {
		return nil, err
	}
	switch u.Scheme {
	case "s3":
		return getS3(u)
	case "gs":
		return getGS(u)
	case "http", "https":
		return getHTTP(u)
	case "file":
		return getFile(u)
	default:
		return nil, fmt.Errorf("manifest URL scheme must be s3, gs, http(s) or file: %s", urlStr)
	}
}

func getManifest(manifestURL string) (*Manifest, error) {
	rc, err := getURL(manifestURL)
	if err != nil {
		return nil, err
	}
	data, _ := ioutil.ReadAll(rc)
	return ParseManifest(data)
}

func parseEvents() (string, error) {
	log.Println("Waiting for events from STDIN...")
	reader := bufio.NewReader(os.Stdin)
	b, err := reader.Peek(1)
	if err != nil {
		return "", err
	}
	if os.Getenv("CONSUL_INDEX") != "" || string(b) == "[" {
		log.Println("Reading Consul event")
		ev, err := ParseConsulEvents(reader)
		if err != nil {
			return "", err
		}
		return string(ev.Payload), nil
	} else {
		if userEvent := os.Getenv("SERF_USER_EVENT"); userEvent != "" {
			log.Println("Reading Serf user event:", userEvent)
		}
		// event passed by stdin (raw string)
		line, err := reader.ReadString('\n')
		return strings.Trim(line, "\n"), err
	}
}
<commit_msg>create a s3 client once.<commit_after>package stretcher

import (
	"bufio"
	"bytes"
	"context"
	crand "crypto/rand"
	"encoding/binary"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	mrand "math/rand"
	"net/http"
	"net/url"
	"os"
	"strings"
	"time"

	"cloud.google.com/go/storage"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3"
)

var (
	LogBuffer bytes.Buffer
	Version   string
	s3svc     *s3.S3
)

type Config struct {
	MaxBandWidth uint64
	Timeout      time.Duration
	InitSleep    time.Duration
	Retry        int
	RetryWait    time.Duration
}

const Nanoseconds = 1000 * 1000 * 1000

func init() {
	log.SetOutput(io.MultiWriter(os.Stderr, &LogBuffer))
}

func RandomTime(delay float64) time.Duration {
	if delay <= 0 {
		return time.Duration(0)
	}
	// http://stackoverflow.com/questions/6181260/generating-random-numbers-in-go
	var s int64
	if err := binary.Read(crand.Reader, binary.LittleEndian, &s); err != nil {
		s = time.Now().UnixNano()
	}
	mrand.Seed(s)
	n := mrand.Int63n(int64(delay * Nanoseconds))
	return time.Duration(n)
}

func Run(conf Config) error {
	var err error
	log.Println("Starting up stretcher agent", Version)
	if conf.InitSleep > 0 {
		log.Printf("Sleeping %s", conf.InitSleep)
		time.Sleep(conf.InitSleep)
	}

	manifestURL, err := parseEvents()
	if err != nil {
		return fmt.Errorf("Could not parse event: %s", err)
	}

	log.Println("Loading manifest:", manifestURL)
	m, err := getManifest(manifestURL)
	if err != nil {
		return fmt.Errorf("Load manifest failed: %s", err)
	}
	log.Printf("Executing manifest %#v", m)

	err = m.Deploy(conf)
	if err != nil {
		log.Println("Deploy manifest failed:", err)
		if ferr := m.Commands.Failure.InvokePipe(&LogBuffer); ferr != nil {
			log.Println(ferr)
		}
		return fmt.Errorf("Deploy manifest failed: %s", err)
	}
	log.Println("Deploy manifest succeeded.")
	err = m.Commands.Success.InvokePipe(&LogBuffer)
	if err != nil {
		log.Println(err)
	}
	log.Println("Done")
	return nil
}

func getS3(u *url.URL) (io.ReadCloser, error) {
	svc, err := initS3()
	if err != nil {
		return nil, err
	}
	key := strings.TrimLeft(u.Path, "/")
	result, err := svc.GetObject(&s3.GetObjectInput{
		Bucket: aws.String(u.Host),
		Key:    aws.String(key),
	})
	if err != nil {
		return nil, err
	}
	return result.Body, nil
}

func getGS(u *url.URL) (io.ReadCloser, error) {
	var err error
	trimPath := strings.Trim(u.Path, "/")

	ctx := context.Background()
	client, err := storage.NewClient(ctx)
	if err != nil {
		return nil, err
	}
	bucket, err := client.Bucket(u.Host).Object(trimPath).NewReader(ctx)
	if err != nil {
		return nil, err
	}
	return bucket, nil
}

func getFile(u *url.URL) (io.ReadCloser, error) {
	return os.Open(u.Path)
}

func getHTTP(u *url.URL) (io.ReadCloser, error) {
	req, err := http.NewRequest("GET", u.String(), nil)
	if err != nil {
		return nil, err
	}
	req.Header.Add("User-Agent", "Stretcher/"+Version)

	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return nil, err
	}
	return resp.Body, nil
}

func getURL(urlStr string) (io.ReadCloser, error) {
	log.Println("Loading URL", urlStr)
	u, err := url.Parse(urlStr)
	if err != nil {
		return nil, err
	}
	switch u.Scheme {
	case "s3":
		return getS3(u)
	case "gs":
		return getGS(u)
	case "http", "https":
		return getHTTP(u)
	case "file":
		return getFile(u)
	default:
		return nil, fmt.Errorf("manifest URL scheme must be s3, gs, http(s) or file: %s", urlStr)
	}
}

func getManifest(manifestURL string) (*Manifest, error) {
	rc, err := getURL(manifestURL)
	if err != nil {
		return nil, err
	}
	data, _ := ioutil.ReadAll(rc)
	return ParseManifest(data)
}

func parseEvents() (string, error) {
	log.Println("Waiting for events from STDIN...")
	reader := bufio.NewReader(os.Stdin)
	b, err := reader.Peek(1)
	if err != nil {
		return "", err
	}
	if os.Getenv("CONSUL_INDEX") != "" || string(b) == "[" {
		log.Println("Reading Consul event")
		ev, err := ParseConsulEvents(reader)
		if err != nil {
			return "", err
		}
		return string(ev.Payload), nil
	} else {
		if userEvent := os.Getenv("SERF_USER_EVENT"); userEvent != "" {
			log.Println("Reading Serf user event:", userEvent)
		}
		// event passed by stdin (raw string)
		line, err := reader.ReadString('\n')
		return strings.Trim(line, "\n"), err
	}
}

func initS3() (*s3.S3, error) {
	if s3svc != nil {
		return s3svc, nil
	}
	sess, err := session.NewSession()
	if err != nil {
		return nil, err
	}
	log.Println("region:", *sess.Config.Region)
	return s3.New(sess), nil
}
<|endoftext|>


<commit_before>// Copyright 2014 The gocui Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"log"

	"github.com/jroimartin/gocui"
)

func main() {
	g := gocui.NewGui()
	if err := g.Init(); err != nil {
		log.Panicln(err)
	}
	defer g.Close()

	g.SetLayout(layout)
	if err := g.SetKeybinding("", gocui.KeyCtrlC, gocui.ModNone, quit); err != nil {
		log.Panicln(err)
	}

	if err := g.MainLoop(); err != nil && err != gocui.ErrQuit {
		log.Panicln(err)
	}
}

func quit(g *gocui.Gui, v *gocui.View) error {
	return gocui.ErrQuit
}

func layout(g *gocui.Gui) error {
	maxX, maxY := g.Size()

	// Overlap (front)
	if v, err := g.SetView("v1", 10, 2, 30, 6); err != nil {
		if err != gocui.ErrUnknownView {
			return err
		}
		v.Title = "Regular title"
	}
	if v, err := g.SetView("v2", 20, 4, 40, 8); err != nil {
		if err != gocui.ErrUnknownView {
			return err
		}
		v.Title = "Regular title"
	}

	// Overlap (back)
	if v, err := g.SetView("v3", 60, 4, 80, 8); err != nil {
		if err != gocui.ErrUnknownView {
			return err
		}
		v.Title = "Regular title"
	}
	if v, err := g.SetView("v4", 50, 2, 70, 6); err != nil {
		if err != gocui.ErrUnknownView {
			return err
		}
		v.Title = "Regular title"
	}

	// Long title
	if v, err := g.SetView("v5", 10, 12, 30, 16); err != nil {
		if err != gocui.ErrUnknownView {
			return err
		}
		v.Title = "Long long long long title"
	}

	// No title
	if v, err := g.SetView("v6", 35, 12, 55, 16); err != nil {
		if err != gocui.ErrUnknownView {
			return err
		}
		v.Title = ""
	}
	if _, err := g.SetView("v7", 60, 12, 80, 16); err != nil {
		if err != gocui.ErrUnknownView {
			return err
		}
	}

	// Small view
	if v, err := g.SetView("v8", 85, 12, 88, 16); err != nil {
		if err != gocui.ErrUnknownView {
			return err
		}
		v.Title = "Regular title"
	}

	// Screen borders
	if v, err := g.SetView("v9", -10, 20, 10, 24); err != nil {
		if err != gocui.ErrUnknownView {
			return err
		}
		v.Title = "Regular title"
	}
	if v, err := g.SetView("v10", maxX-10, 20, maxX+10, 24); err != nil {
		if err != gocui.ErrUnknownView {
			return err
		}
		v.Title = "Regular title"
	}

	// Out of screen
	if v, err := g.SetView("v11", -21, 28, -1, 32); err != nil {
		if err != gocui.ErrUnknownView {
			return err
		}
		v.Title = "Regular title"
	}
	if v, err := g.SetView("v12", maxX, 28, maxX+20, 32); err != nil {
		if err != gocui.ErrUnknownView {
			return err
		}
		v.Title = "Regular title"
	}
	if v, err := g.SetView("v13", 10, -7, 30, -1); err != nil {
		if err != gocui.ErrUnknownView {
			return err
		}
		v.Title = "Regular title"
	}
	if v, err := g.SetView("v14", 10, maxY, 30, maxY+6); err != nil {
		if err != gocui.ErrUnknownView {
			return err
		}
		v.Title = "Regular title"
	}

	return nil
}
<commit_msg>Add more test cases to _examples/title.go<commit_after>// Copyright 2014 The gocui Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"log"

	"github.com/jroimartin/gocui"
)

func main() {
	g := gocui.NewGui()
	if err := g.Init(); err != nil {
		log.Panicln(err)
	}
	defer g.Close()

	g.SetLayout(layout)
	if err := g.SetKeybinding("", gocui.KeyCtrlC, gocui.ModNone, quit); err != nil {
		log.Panicln(err)
	}

	if err := g.MainLoop(); err != nil && err != gocui.ErrQuit {
		log.Panicln(err)
	}
}

func quit(g *gocui.Gui, v *gocui.View) error {
	return gocui.ErrQuit
}

func layout(g *gocui.Gui) error {
	maxX, maxY := g.Size()

	// Overlap (front)
	if v, err := g.SetView("v1", 10, 2, 30, 6); err != nil {
		if err != gocui.ErrUnknownView {
			return err
		}
		v.Title = "Regular title"
	}
	if v, err := g.SetView("v2", 20, 4, 40, 8); err != nil {
		if err != gocui.ErrUnknownView {
			return err
		}
		v.Title = "Regular title"
	}

	// Overlap (back)
	if v, err := g.SetView("v3", 60, 4, 80, 8); err != nil {
		if err != gocui.ErrUnknownView {
			return err
		}
		v.Title = "Regular title"
	}
	if v, err := g.SetView("v4", 50, 2, 70, 6); err != nil {
		if err != gocui.ErrUnknownView {
			return err
		}
		v.Title = "Regular title"
	}

	// Overlap (frame)
	if v, err := g.SetView("v15", 90, 2, 110, 5); err != nil {
		if err != gocui.ErrUnknownView {
			return err
		}
		v.Title = "Regular title"
	}
	if v, err := g.SetView("v16", 100, 5, 120, 8); err != nil {
		if err != gocui.ErrUnknownView {
			return err
		}
		v.Title = "Regular title"
	}
	if v, err := g.SetView("v17", 140, 5, 160, 8); err != nil {
		if err != gocui.ErrUnknownView {
			return err
		}
		v.Title = "Regular title"
	}
	if v, err := g.SetView("v18", 130, 2, 150, 5); err != nil {
		if err != gocui.ErrUnknownView {
			return err
		}
		v.Title = "Regular title"
	}

	// Long title
	if v, err := g.SetView("v5", 10, 12, 30, 16); err != nil {
		if err != gocui.ErrUnknownView {
			return err
		}
		v.Title = "Long long long long title"
	}

	// No title
	if v, err := g.SetView("v6", 35, 12, 55, 16); err != nil {
		if err != gocui.ErrUnknownView {
			return err
		}
		v.Title = ""
	}
	if _, err := g.SetView("v7", 60, 12, 80, 16); err != nil {
		if err != gocui.ErrUnknownView {
			return err
		}
	}

	// Small view
	if v, err := g.SetView("v8", 85, 12, 88, 16); err != nil {
		if err != gocui.ErrUnknownView {
			return err
		}
		v.Title = "Regular title"
	}

	// Screen borders
	if v, err := g.SetView("v9", -10, 20, 10, 24); err != nil {
		if err != gocui.ErrUnknownView {
			return err
		}
		v.Title = "Regular title"
	}
	if v, err := g.SetView("v10", maxX-10, 20, maxX+10, 24); err != nil {
		if err != gocui.ErrUnknownView {
			return err
		}
		v.Title = "Regular title"
	}

	// Out of screen
	if v, err := g.SetView("v11", -21, 28, -1, 32); err != nil {
		if err != gocui.ErrUnknownView {
			return err
		}
		v.Title = "Regular title"
	}
	if v, err := g.SetView("v12", maxX, 28, maxX+20, 32); err != nil {
		if err != gocui.ErrUnknownView {
			return err
		}
		v.Title = "Regular title"
	}
	if v, err := g.SetView("v13", 10, -7, 30, -1); err != nil {
		if err != gocui.ErrUnknownView {
			return err
		}
		v.Title = "Regular title"
	}
	if v, err := g.SetView("v14", 10, maxY, 30, maxY+6); err != nil {
		if err != gocui.ErrUnknownView {
			return err
		}
		v.Title = "Regular title"
	}

	return nil
}
<|endoftext|>


<commit_before>// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package pe

import (
	"encoding/binary"
	"fmt"
	"io"
	"unsafe"
)

const COFFSymbolSize = 18

// COFFSymbol represents single COFF symbol table record.
type COFFSymbol struct {
	Name               [8]uint8
	Value              uint32
	SectionNumber      int16
	Type               uint16
	StorageClass       uint8
	NumberOfAuxSymbols uint8
}

// readCOFFSymbols reads in the symbol table for a PE file, returning
// a slice of COFFSymbol objects. The PE format includes both primary
// symbols (whose fields are described by COFFSymbol above) and
// auxiliary symbols; all symbols are 18 bytes in size. The auxiliary
// symbols for a given primary symbol are placed following it in the
// array, e.g.
//
//	...
//	k+0:  regular sym k
//	k+1:    1st aux symbol for k
//	k+2:    2nd aux symbol for k
//	k+3:  regular sym k+3
//	k+4:    1st aux symbol for k+3
//	k+5:  regular sym k+5
//	k+6:  regular sym k+6
//
// The PE format allows for several possible aux symbol formats. For
// more info see:
//
//	https://docs.microsoft.com/en-us/windows/win32/debug/pe-format#auxiliary-symbol-records
//
// At the moment this package only provides APIs for looking at
// aux symbols of format 5 (associated with section definition symbols).
func readCOFFSymbols(fh *FileHeader, r io.ReadSeeker) ([]COFFSymbol, error) {
	if fh.PointerToSymbolTable == 0 {
		return nil, nil
	}
	if fh.NumberOfSymbols <= 0 {
		return nil, nil
	}
	_, err := r.Seek(int64(fh.PointerToSymbolTable), seekStart)
	if err != nil {
		return nil, fmt.Errorf("fail to seek to symbol table: %v", err)
	}
	syms := make([]COFFSymbol, fh.NumberOfSymbols)
	naux := 0
	for k := range syms {
		if naux == 0 {
			// Read a primary symbol.
			err = binary.Read(r, binary.LittleEndian, &syms[k])
			if err != nil {
				return nil, fmt.Errorf("fail to read symbol table: %v", err)
			}
			// Record how many auxiliary symbols it has.
			naux = int(syms[k].NumberOfAuxSymbols)
		} else {
			// Read an aux symbol. At the moment we assume all
			// aux symbols are format 5 (obviously this doesn't always
			// hold; more cases will be needed below if more aux formats
			// are supported in the future).
			naux--
			aux := (*COFFSymbolAuxFormat5)(unsafe.Pointer(&syms[k]))
			err = binary.Read(r, binary.LittleEndian, aux)
			if err != nil {
				return nil, fmt.Errorf("fail to read symbol table: %v", err)
			}
		}
	}
	if naux != 0 {
		return nil, fmt.Errorf("fail to read symbol table: %d aux symbols unread", naux)
	}
	return syms, nil
}

// isSymNameOffset checks symbol name if it is encoded as offset into string table.
func isSymNameOffset(name [8]byte) (bool, uint32) {
	if name[0] == 0 && name[1] == 0 && name[2] == 0 && name[3] == 0 {
		return true, binary.LittleEndian.Uint32(name[4:])
	}
	return false, 0
}

// FullName finds real name of symbol sym. Normally name is stored
// in sym.Name, but if it is longer then 8 characters, it is stored
// in COFF string table st instead.
func (sym *COFFSymbol) FullName(st StringTable) (string, error) {
	if ok, offset := isSymNameOffset(sym.Name); ok {
		return st.String(offset)
	}
	return cstring(sym.Name[:]), nil
}

func removeAuxSymbols(allsyms []COFFSymbol, st StringTable) ([]*Symbol, error) {
	if len(allsyms) == 0 {
		return nil, nil
	}
	syms := make([]*Symbol, 0)
	aux := uint8(0)
	for _, sym := range allsyms {
		if aux > 0 {
			aux--
			continue
		}
		name, err := sym.FullName(st)
		if err != nil {
			return nil, err
		}
		aux = sym.NumberOfAuxSymbols
		s := &Symbol{
			Name:          name,
			Value:         sym.Value,
			SectionNumber: sym.SectionNumber,
			Type:          sym.Type,
			StorageClass:  sym.StorageClass,
		}
		syms = append(syms, s)
	}
	return syms, nil
}

// Symbol is similar to COFFSymbol with Name field replaced
// by Go string. Symbol also does not have NumberOfAuxSymbols.
type Symbol struct {
	Name          string
	Value         uint32
	SectionNumber int16
	Type          uint16
	StorageClass  uint8
}

// COFFSymbolAuxFormat5 describes the expected form of an aux symbol
// attached to a section definition symbol. The PE format defines a
// number of different aux symbol formats: format 1 for function
// definitions, format 2 for .be and .ef symbols, and so on. Format 5
// holds extra info associated with a section definition, including
// number of relocations + line numbers, as well as COMDAT info. See
// https://docs.microsoft.com/en-us/windows/win32/debug/pe-format#auxiliary-format-5-section-definitions
// for more on what's going on here.
type COFFSymbolAuxFormat5 struct {
	Size           uint32
	NumRelocs      uint16
	NumLineNumbers uint16
	Checksum       uint32
	SecNum         uint16
	Selection      uint8
	_              [3]uint8 // padding
}

// These constants make up the possible values for the 'Selection'
// field in an AuxFormat5.
const (
	IMAGE_COMDAT_SELECT_NODUPLICATES = 1
	IMAGE_COMDAT_SELECT_ANY          = 2
	IMAGE_COMDAT_SELECT_SAME_SIZE    = 3
	IMAGE_COMDAT_SELECT_EXACT_MATCH  = 4
	IMAGE_COMDAT_SELECT_ASSOCIATIVE  = 5
	IMAGE_COMDAT_SELECT_LARGEST      = 6
)

// COFFSymbolReadSectionDefAux returns a blob of axiliary information
// (including COMDAT info) for a section definition symbol. Here 'idx'
// is the index of a section symbol in the main COFFSymbol array for
// the File. Return value is a pointer to the appropriate aux symbol
// struct. For more info, see:
//
// auxiliary symbols: https://docs.microsoft.com/en-us/windows/win32/debug/pe-format#auxiliary-symbol-records
// COMDAT sections: https://docs.microsoft.com/en-us/windows/win32/debug/pe-format#comdat-sections-object-only
// auxiliary info for section definitions: https://docs.microsoft.com/en-us/windows/win32/debug/pe-format#auxiliary-format-5-section-definitions
func (f *File) COFFSymbolReadSectionDefAux(idx int) (*COFFSymbolAuxFormat5, error) {
	var rv *COFFSymbolAuxFormat5
	if idx < 0 || idx >= len(f.COFFSymbols) {
		return rv, fmt.Errorf("invalid symbol index")
	}
	pesym := &f.COFFSymbols[idx]
	const IMAGE_SYM_CLASS_STATIC = 3
	if pesym.StorageClass != uint8(IMAGE_SYM_CLASS_STATIC) {
		return rv, fmt.Errorf("incorrect symbol storage class")
	}
	if pesym.NumberOfAuxSymbols == 0 || idx+1 >= len(f.COFFSymbols) {
		return rv, fmt.Errorf("aux symbol unavailable")
	}
	// Locate and return a pointer to the successor aux symbol.
	pesymn := &f.COFFSymbols[idx+1]
	rv = (*COFFSymbolAuxFormat5)(unsafe.Pointer(pesymn))
	return rv, nil
}
<commit_msg>debug/pe: use saferio to set symbol slice capacity<commit_after>// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package pe

import (
	"encoding/binary"
	"errors"
	"fmt"
	"internal/saferio"
	"io"
	"unsafe"
)

const COFFSymbolSize = 18

// COFFSymbol represents single COFF symbol table record.
type COFFSymbol struct {
	Name               [8]uint8
	Value              uint32
	SectionNumber      int16
	Type               uint16
	StorageClass       uint8
	NumberOfAuxSymbols uint8
}

// readCOFFSymbols reads in the symbol table for a PE file, returning
// a slice of COFFSymbol objects. The PE format includes both primary
// symbols (whose fields are described by COFFSymbol above) and
// auxiliary symbols; all symbols are 18 bytes in size. The auxiliary
// symbols for a given primary symbol are placed following it in the
// array, e.g.
//
//	...
//	k+0:  regular sym k
//	k+1:    1st aux symbol for k
//	k+2:    2nd aux symbol for k
//	k+3:  regular sym k+3
//	k+4:    1st aux symbol for k+3
//	k+5:  regular sym k+5
//	k+6:  regular sym k+6
//
// The PE format allows for several possible aux symbol formats. For
// more info see:
//
//	https://docs.microsoft.com/en-us/windows/win32/debug/pe-format#auxiliary-symbol-records
//
// At the moment this package only provides APIs for looking at
// aux symbols of format 5 (associated with section definition symbols).
func readCOFFSymbols(fh *FileHeader, r io.ReadSeeker) ([]COFFSymbol, error) {
	if fh.PointerToSymbolTable == 0 {
		return nil, nil
	}
	if fh.NumberOfSymbols <= 0 {
		return nil, nil
	}
	_, err := r.Seek(int64(fh.PointerToSymbolTable), seekStart)
	if err != nil {
		return nil, fmt.Errorf("fail to seek to symbol table: %v", err)
	}
	c := saferio.SliceCap(COFFSymbol{}, uint64(fh.NumberOfSymbols))
	if c < 0 {
		return nil, errors.New("too many symbols; file may be corrupt")
	}
	syms := make([]COFFSymbol, 0, c)
	naux := 0
	for k := uint32(0); k < fh.NumberOfSymbols; k++ {
		var sym COFFSymbol
		if naux == 0 {
			// Read a primary symbol.
			err = binary.Read(r, binary.LittleEndian, &sym)
			if err != nil {
				return nil, fmt.Errorf("fail to read symbol table: %v", err)
			}
			// Record how many auxiliary symbols it has.
			naux = int(sym.NumberOfAuxSymbols)
		} else {
			// Read an aux symbol. At the moment we assume all
			// aux symbols are format 5 (obviously this doesn't always
			// hold; more cases will be needed below if more aux formats
			// are supported in the future).
			naux--
			aux := (*COFFSymbolAuxFormat5)(unsafe.Pointer(&sym))
			err = binary.Read(r, binary.LittleEndian, aux)
			if err != nil {
				return nil, fmt.Errorf("fail to read symbol table: %v", err)
			}
		}
		syms = append(syms, sym)
	}
	if naux != 0 {
		return nil, fmt.Errorf("fail to read symbol table: %d aux symbols unread", naux)
	}
	return syms, nil
}

// isSymNameOffset checks symbol name if it is encoded as offset into string table.
func isSymNameOffset(name [8]byte) (bool, uint32) {
	if name[0] == 0 && name[1] == 0 && name[2] == 0 && name[3] == 0 {
		return true, binary.LittleEndian.Uint32(name[4:])
	}
	return false, 0
}

// FullName finds real name of symbol sym. Normally name is stored
// in sym.Name, but if it is longer then 8 characters, it is stored
// in COFF string table st instead.
func (sym *COFFSymbol) FullName(st StringTable) (string, error) {
	if ok, offset := isSymNameOffset(sym.Name); ok {
		return st.String(offset)
	}
	return cstring(sym.Name[:]), nil
}

func removeAuxSymbols(allsyms []COFFSymbol, st StringTable) ([]*Symbol, error) {
	if len(allsyms) == 0 {
		return nil, nil
	}
	syms := make([]*Symbol, 0)
	aux := uint8(0)
	for _, sym := range allsyms {
		if aux > 0 {
			aux--
			continue
		}
		name, err := sym.FullName(st)
		if err != nil {
			return nil, err
		}
		aux = sym.NumberOfAuxSymbols
		s := &Symbol{
			Name:          name,
			Value:         sym.Value,
			SectionNumber: sym.SectionNumber,
			Type:          sym.Type,
			StorageClass:  sym.StorageClass,
		}
		syms = append(syms, s)
	}
	return syms, nil
}

// Symbol is similar to COFFSymbol with Name field replaced
// by Go string. Symbol also does not have NumberOfAuxSymbols.
type Symbol struct {
	Name          string
	Value         uint32
	SectionNumber int16
	Type          uint16
	StorageClass  uint8
}

// COFFSymbolAuxFormat5 describes the expected form of an aux symbol
// attached to a section definition symbol. The PE format defines a
// number of different aux symbol formats: format 1 for function
// definitions, format 2 for .be and .ef symbols, and so on. Format 5
// holds extra info associated with a section definition, including
// number of relocations + line numbers, as well as COMDAT info. See
// https://docs.microsoft.com/en-us/windows/win32/debug/pe-format#auxiliary-format-5-section-definitions
// for more on what's going on here.
type COFFSymbolAuxFormat5 struct {
	Size           uint32
	NumRelocs      uint16
	NumLineNumbers uint16
	Checksum       uint32
	SecNum         uint16
	Selection      uint8
	_              [3]uint8 // padding
}

// These constants make up the possible values for the 'Selection'
// field in an AuxFormat5.
const (
	IMAGE_COMDAT_SELECT_NODUPLICATES = 1
	IMAGE_COMDAT_SELECT_ANY          = 2
	IMAGE_COMDAT_SELECT_SAME_SIZE    = 3
	IMAGE_COMDAT_SELECT_EXACT_MATCH  = 4
	IMAGE_COMDAT_SELECT_ASSOCIATIVE  = 5
	IMAGE_COMDAT_SELECT_LARGEST      = 6
)

// COFFSymbolReadSectionDefAux returns a blob of axiliary information
// (including COMDAT info) for a section definition symbol. Here 'idx'
// is the index of a section symbol in the main COFFSymbol array for
// the File. Return value is a pointer to the appropriate aux symbol
// struct. For more info, see:
//
// auxiliary symbols: https://docs.microsoft.com/en-us/windows/win32/debug/pe-format#auxiliary-symbol-records
// COMDAT sections: https://docs.microsoft.com/en-us/windows/win32/debug/pe-format#comdat-sections-object-only
// auxiliary info for section definitions: https://docs.microsoft.com/en-us/windows/win32/debug/pe-format#auxiliary-format-5-section-definitions
func (f *File) COFFSymbolReadSectionDefAux(idx int) (*COFFSymbolAuxFormat5, error) {
	var rv *COFFSymbolAuxFormat5
	if idx < 0 || idx >= len(f.COFFSymbols) {
		return rv, fmt.Errorf("invalid symbol index")
	}
	pesym := &f.COFFSymbols[idx]
	const IMAGE_SYM_CLASS_STATIC = 3
	if pesym.StorageClass != uint8(IMAGE_SYM_CLASS_STATIC) {
		return rv, fmt.Errorf("incorrect symbol storage class")
	}
	if pesym.NumberOfAuxSymbols == 0 || idx+1 >= len(f.COFFSymbols) {
		return rv, fmt.Errorf("aux symbol unavailable")
	}
	// Locate and return a pointer to the successor aux symbol.
	pesymn := &f.COFFSymbols[idx+1]
	rv = (*COFFSymbolAuxFormat5)(unsafe.Pointer(pesymn))
	return rv, nil
}
<|endoftext|>


<commit_before><commit_msg>[game] add vanity space<commit_after><|endoftext|>


<commit_before>// Copyright 2015 ikawaha
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// 	You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package tokenizer

import (
	"fmt"
	"strings"

	"github.com/ikawaha/kagome/internal/dic"
	"github.com/ikawaha/kagome/internal/lattice"
)

// TokenClass represents the token type.
type TokenClass lattice.NodeClass

const (
	// DUMMY represents the dummy token.
	DUMMY = TokenClass(lattice.DUMMY)
	// KNOWN represents the token in the dictionary.
	KNOWN = TokenClass(lattice.KNOWN)
	// UNKNOWN represents the token which is not in the dictionary.
	UNKNOWN = TokenClass(lattice.UNKNOWN)
	// USER represents the token in the user dictionary.
	USER = TokenClass(lattice.USER)
)

func (c TokenClass) String() string {
	ret := ""
	switch c {
	case DUMMY:
		ret = "DUMMY"
	case KNOWN:
		ret = "KNOWN"
	case UNKNOWN:
		ret = "UNKNOWN"
	case USER:
		ret = "USER"
	}
	return ret
}

// Token represents a morph of a sentence.
type Token struct {
	ID      int
	Class   TokenClass
	Start   int
	End     int
	Surface string
	dic     *dic.Dic
	udic    *dic.UserDic
}

// Features returns contents of a token.
func (t Token) Features() (features []string) {
	switch lattice.NodeClass(t.Class) {
	case lattice.DUMMY:
		return
	case lattice.KNOWN:
		features = t.dic.Contents[t.ID]
	case lattice.UNKNOWN:
		features = t.dic.UnkContents[t.ID]
	case lattice.USER:
		pos := t.udic.Contents[t.ID].Pos
		tokens := strings.Join(t.udic.Contents[t.ID].Tokens, "/")
		yomi := strings.Join(t.udic.Contents[t.ID].Yomi, "/")
		features = append(features, pos, tokens, yomi)
	}
	return
}

// String returns a string representation of a token.
func (t Token) String() string {
	return fmt.Sprintf("%v(%v, %v)%v[%v]", t.Surface, t.Start, t.End, t.Class, t.ID)
}
<commit_msg>Add a function to get a part-of-speech tag<commit_after>// Copyright 2015 ikawaha
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// 	You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package tokenizer

import (
	"fmt"
	"strings"

	"github.com/ikawaha/kagome/internal/dic"
	"github.com/ikawaha/kagome/internal/lattice"
)

// TokenClass represents the token type.
type TokenClass lattice.NodeClass

const (
	// DUMMY represents the dummy token.
	DUMMY = TokenClass(lattice.DUMMY)
	// KNOWN represents the token in the dictionary.
	KNOWN = TokenClass(lattice.KNOWN)
	// UNKNOWN represents the token which is not in the dictionary.
	UNKNOWN = TokenClass(lattice.UNKNOWN)
	// USER represents the token in the user dictionary.
	USER = TokenClass(lattice.USER)
)

func (c TokenClass) String() string {
	ret := ""
	switch c {
	case DUMMY:
		ret = "DUMMY"
	case KNOWN:
		ret = "KNOWN"
	case UNKNOWN:
		ret = "UNKNOWN"
	case USER:
		ret = "USER"
	}
	return ret
}

// Token represents a morph of a sentence.
type Token struct {
	ID      int
	Class   TokenClass
	Start   int
	End     int
	Surface string
	dic     *dic.Dic
	udic    *dic.UserDic
}

// Features returns contents of a token.
func (t Token) Features() (features []string) {
	switch lattice.NodeClass(t.Class) {
	case lattice.DUMMY:
		return
	case lattice.KNOWN:
		features = t.dic.Contents[t.ID]
	case lattice.UNKNOWN:
		features = t.dic.UnkContents[t.ID]
	case lattice.USER:
		pos := t.udic.Contents[t.ID].Pos
		tokens := strings.Join(t.udic.Contents[t.ID].Tokens, "/")
		yomi := strings.Join(t.udic.Contents[t.ID].Yomi, "/")
		features = append(features, pos, tokens, yomi)
	}
	return
}

// Pos returns the first element of features.
func (t Token) Pos() string {
	f := t.Features()
	if len(f) < 1 {
		return ""
	}
	return f[0]
}

// String returns a string representation of a token.
func (t Token) String() string {
	return fmt.Sprintf("%v(%v, %v)%v[%v]", t.Surface, t.Start, t.End, t.Class, t.ID)
}
<|endoftext|>