<commit_before>package writer

import (
	"context"
	"encoding/binary"
	"reflect"
	"sync"
	"errors"

	"github.com/apache/thrift/lib/go/thrift"
	"github.com/xitongsys/parquet-go/common"
	"github.com/xitongsys/parquet-go/layout"
	"github.com/xitongsys/parquet-go/marshal"
	"github.com/xitongsys/parquet-go/source"
	"github.com/xitongsys/parquet-go/schema"
	"github.com/xitongsys/parquet-go/parquet"
)

//ParquetWriter is a writer  parquet file
type ParquetWriter struct {
	SchemaHandler *schema.SchemaHandler
	NP            int64 //parallel number
	Footer        *parquet.FileMetaData
	PFile         source.ParquetFile

	PageSize        int64
	RowGroupSize    int64
	CompressionType parquet.CompressionCodec
	Offset          int64

	Objs              []interface{}
	ObjsSize          int64
	ObjSize           int64
	CheckSizeCritical int64

	PagesMapBuf map[string][]*layout.Page
	Size        int64
	NumRows     int64

	DictRecs map[string]*layout.DictRecType

	MarshalFunc func(src []interface{}, bgn int, end int, sh *schema.SchemaHandler) (*map[string]*layout.Table, error)
}

//Create a parquet handler
func NewParquetWriter(pFile source.ParquetFile, obj interface{}, np int64) (*ParquetWriter, error) {
	var err error

	res := new(ParquetWriter)
	res.NP = np
	res.PageSize = 8 * 1024              //8K
	res.RowGroupSize = 128 * 1024 * 1024 //128M
	res.CompressionType = parquet.CompressionCodec_SNAPPY
	res.ObjsSize = 0
	res.CheckSizeCritical = 0
	res.Size = 0
	res.NumRows = 0
	res.Offset = 4
	res.PFile = pFile
	res.PagesMapBuf = make(map[string][]*layout.Page)
	res.DictRecs = make(map[string]*layout.DictRecType)
	res.Footer = parquet.NewFileMetaData()
	res.Footer.Version = 1
	_, err = res.PFile.Write([]byte("PAR1"))
	res.MarshalFunc = marshal.Marshal

	if obj != nil {
		if res.SchemaHandler, err = schema.NewSchemaHandlerFromStruct(obj); err != nil {
			return res, err
		}
		res.Footer.Schema = append(res.Footer.Schema, res.SchemaHandler.SchemaElements...)
	}

	return res, err
}

func (self *ParquetWriter) SetSchemaHandlerFromJSON(jsonSchema string) error {
	var err error
	if self.SchemaHandler, err = schema.NewSchemaHandlerFromJSON(jsonSchema); err != nil {
		return err
	}
	self.Footer.Schema = self.Footer.Schema[:0]
	self.Footer.Schema = append(self.Footer.Schema, self.SchemaHandler.SchemaElements...)
	return nil
}

//Rename schema name to exname in tags
func (self *ParquetWriter) RenameSchema() {
	for i := 0; i < len(self.Footer.Schema); i++ {
		self.Footer.Schema[i].Name = self.SchemaHandler.Infos[i].ExName
	}
	for _, rowGroup := range self.Footer.RowGroups {
		for _, chunk := range rowGroup.Columns {
			inPathStr := common.PathToStr(chunk.MetaData.PathInSchema)
			exPathStr := self.SchemaHandler.InPathToExPath[inPathStr]
			exPath := common.StrToPath(exPathStr)[1:]
			chunk.MetaData.PathInSchema = exPath
		}
	}
}

//Write the footer and stop writing
func (self *ParquetWriter) WriteStop() error {
	var err error

	if err = self.Flush(true); err != nil {
		return err
	}
	ts := thrift.NewTSerializer()
	ts.Protocol = thrift.NewTCompactProtocolFactory().GetProtocol(ts.Transport)
	self.RenameSchema()
	footerBuf, err := ts.Write(context.TODO(), self.Footer)
	if err != nil {
		return err
	}

	if _, err = self.PFile.Write(footerBuf); err != nil {
		return err
	}
	footerSizeBuf := make([]byte, 4)
	binary.LittleEndian.PutUint32(footerSizeBuf, uint32(len(footerBuf)))

	if _, err = self.PFile.Write(footerSizeBuf); err != nil {
		return err
	}
	if _, err = self.PFile.Write([]byte("PAR1")); err != nil {
		return err
	}
	return nil

}

//Write one object to parquet file
func (self *ParquetWriter) Write(src interface{}) error {
	var err error
	ln := int64(len(self.Objs))

	val := reflect.ValueOf(src)
	if val.Kind() == reflect.Ptr {
		val = val.Elem()
		src = val.Interface()
	}

	if self.CheckSizeCritical <= ln {
		self.ObjSize = common.SizeOf(val) + 1
	}
	self.ObjsSize += self.ObjSize
	self.Objs = append(self.Objs, src)

	criSize := self.NP * self.PageSize * self.SchemaHandler.GetColumnNum()

	if self.ObjsSize >= criSize {
		err = self.Flush(false)

	} else {
		dln := (criSize - self.ObjsSize + self.ObjSize - 1) / self.ObjSize / 2
		self.CheckSizeCritical = dln + ln
	}
	return err

}

func (self *ParquetWriter) flushObjs() error {
	var err error
	l := int64(len(self.Objs))
	if l <= 0 {
		return nil
	}
	pagesMapList := make([]map[string][]*layout.Page, self.NP)
	for i := 0; i < int(self.NP); i++ {
		pagesMapList[i] = make(map[string][]*layout.Page)
	}
	doneChan := make(chan int)

	var c int64 = 0
	delta := (l + self.NP - 1) / self.NP
	lock := new(sync.Mutex)
	for c = 0; c < self.NP; c++ {
		bgn := c * delta
		end := bgn + delta
		if end > l {
			end = l
		}
		if bgn >= l {
			bgn, end = l, l
		}

		go func(b, e int, index int64) {
			defer func() {
				if r := recover(); r != nil {
					switch x := r.(type) {
					case string:
						err = errors.New(x)
					case error:
						err = x
					default:
						err = errors.New("unknown error")
					}
				}
			}()

			if e <= b {
				doneChan <- 0
				return
			}

			tableMap, err2 := self.MarshalFunc(self.Objs, b, e, self.SchemaHandler)

			if err2 == nil {
				for name, table := range *tableMap {
					if table.Info.Encoding == parquet.Encoding_PLAIN_DICTIONARY {
						lock.Lock()
						if _, ok := self.DictRecs[name]; !ok {
							self.DictRecs[name] = layout.NewDictRec(table.Type)
						}
						pagesMapList[index][name], _ = layout.TableToDictDataPages(self.DictRecs[name],
							table, int32(self.PageSize), 32, self.CompressionType)
						lock.Unlock()

					} else {
						pagesMapList[index][name], _ = layout.TableToDataPages(table, int32(self.PageSize),
							self.CompressionType)
					}
				}
			} else {
				err = err2
			}

			doneChan <- 0
		}(int(bgn), int(end), c)
	}

	for c = 0; c < self.NP; c++ {
		<-doneChan
	}

	for _, pagesMap := range pagesMapList {
		for name, pages := range pagesMap {
			if _, ok := self.PagesMapBuf[name]; !ok {
				self.PagesMapBuf[name] = pages
			} else {
				self.PagesMapBuf[name] = append(self.PagesMapBuf[name], pages...)
			}
			for _, page := range pages {
				self.Size += int64(len(page.RawData))
				page.DataTable = nil //release memory
			}
		}
	}

	self.NumRows += int64(len(self.Objs))
	return err
}

//Flush the write buffer to parquet file
func (self *ParquetWriter) Flush(flag bool) error {
	var err error

	if err = self.flushObjs(); err != nil {
		return err
	}

	if (self.Size+self.ObjsSize >= self.RowGroupSize || flag) && len(self.PagesMapBuf) > 0 {
		//pages -> chunk
		chunkMap := make(map[string]*layout.Chunk)
		for name, pages := range self.PagesMapBuf {
			if len(pages) > 0 && pages[0].Info.Encoding == parquet.Encoding_PLAIN_DICTIONARY {
				dictPage, _ := layout.DictRecToDictPage(self.DictRecs[name], int32(self.PageSize), self.CompressionType)
				tmp := append([]*layout.Page{dictPage}, pages...)
				chunkMap[name] = layout.PagesToDictChunk(tmp)
			} else {
				chunkMap[name] = layout.PagesToChunk(pages)

			}
		}

		self.DictRecs = make(map[string]*layout.DictRecType) //clean records for next chunks

		//chunks -> rowGroup
		rowGroup := layout.NewRowGroup()
		rowGroup.RowGroupHeader.Columns = make([]*parquet.ColumnChunk, 0)

		for k := 0; k < len(self.SchemaHandler.SchemaElements); k++ {
			//for _, chunk := range chunkMap {
			schema := self.SchemaHandler.SchemaElements[k]
			if schema.GetNumChildren() > 0 {
				continue
			}
			chunk := chunkMap[self.SchemaHandler.IndexMap[int32(k)]]
			if chunk == nil {
				continue
			}
			rowGroup.Chunks = append(rowGroup.Chunks, chunk)
			//rowGroup.RowGroupHeader.TotalByteSize += chunk.ChunkHeader.MetaData.TotalCompressedSize
			rowGroup.RowGroupHeader.TotalByteSize += chunk.ChunkHeader.MetaData.TotalUncompressedSize
			rowGroup.RowGroupHeader.Columns = append(rowGroup.RowGroupHeader.Columns, chunk.ChunkHeader)
		}
		rowGroup.RowGroupHeader.NumRows = self.NumRows
		self.NumRows = 0

		for k := 0; k < len(rowGroup.Chunks); k++ {
			rowGroup.Chunks[k].ChunkHeader.MetaData.DataPageOffset = -1
			rowGroup.Chunks[k].ChunkHeader.FileOffset = self.Offset

			for l := 0; l < len(rowGroup.Chunks[k].Pages); l++ {
				if rowGroup.Chunks[k].Pages[l].Header.Type == parquet.PageType_DICTIONARY_PAGE {
					tmp := self.Offset
					rowGroup.Chunks[k].ChunkHeader.MetaData.DictionaryPageOffset = &tmp
				} else if rowGroup.Chunks[k].ChunkHeader.MetaData.DataPageOffset <= 0 {
					rowGroup.Chunks[k].ChunkHeader.MetaData.DataPageOffset = self.Offset

				}
				data := rowGroup.Chunks[k].Pages[l].RawData
				if _, err = self.PFile.Write(data); err != nil {
					return err
				}
				self.Offset += int64(len(data))
			}
		}
		self.Footer.RowGroups = append(self.Footer.RowGroups, rowGroup.RowGroupHeader)
		self.Size = 0
		self.PagesMapBuf = make(map[string][]*layout.Page)
	}
	self.Footer.NumRows += int64(len(self.Objs))
	self.Objs = self.Objs[:0]
	self.ObjsSize = 0
	return nil

}
<commit_msg>fix obj size func<commit_after>package writer

import (
	"context"
	"encoding/binary"
	"reflect"
	"sync"
	"errors"

	"github.com/apache/thrift/lib/go/thrift"
	"github.com/xitongsys/parquet-go/common"
	"github.com/xitongsys/parquet-go/layout"
	"github.com/xitongsys/parquet-go/marshal"
	"github.com/xitongsys/parquet-go/source"
	"github.com/xitongsys/parquet-go/schema"
	"github.com/xitongsys/parquet-go/parquet"
)

//ParquetWriter is a writer  parquet file
type ParquetWriter struct {
	SchemaHandler *schema.SchemaHandler
	NP            int64 //parallel number
	Footer        *parquet.FileMetaData
	PFile         source.ParquetFile

	PageSize        int64
	RowGroupSize    int64
	CompressionType parquet.CompressionCodec
	Offset          int64

	Objs              []interface{}
	ObjsSize          int64
	ObjSize           int64
	CheckSizeCritical int64

	PagesMapBuf map[string][]*layout.Page
	Size        int64
	NumRows     int64

	DictRecs map[string]*layout.DictRecType

	MarshalFunc func(src []interface{}, bgn int, end int, sh *schema.SchemaHandler) (*map[string]*layout.Table, error)
}

//Create a parquet handler
func NewParquetWriter(pFile source.ParquetFile, obj interface{}, np int64) (*ParquetWriter, error) {
	var err error

	res := new(ParquetWriter)
	res.NP = np
	res.PageSize = 8 * 1024              //8K
	res.RowGroupSize = 128 * 1024 * 1024 //128M
	res.CompressionType = parquet.CompressionCodec_SNAPPY
	res.ObjsSize = 0
	res.CheckSizeCritical = 0
	res.Size = 0
	res.NumRows = 0
	res.Offset = 4
	res.PFile = pFile
	res.PagesMapBuf = make(map[string][]*layout.Page)
	res.DictRecs = make(map[string]*layout.DictRecType)
	res.Footer = parquet.NewFileMetaData()
	res.Footer.Version = 1
	_, err = res.PFile.Write([]byte("PAR1"))
	res.MarshalFunc = marshal.Marshal

	if obj != nil {
		if res.SchemaHandler, err = schema.NewSchemaHandlerFromStruct(obj); err != nil {
			return res, err
		}
		res.Footer.Schema = append(res.Footer.Schema, res.SchemaHandler.SchemaElements...)
	}

	return res, err
}

func (self *ParquetWriter) SetSchemaHandlerFromJSON(jsonSchema string) error {
	var err error
	if self.SchemaHandler, err = schema.NewSchemaHandlerFromJSON(jsonSchema); err != nil {
		return err
	}
	self.Footer.Schema = self.Footer.Schema[:0]
	self.Footer.Schema = append(self.Footer.Schema, self.SchemaHandler.SchemaElements...)
	return nil
}

//Rename schema name to exname in tags
func (self *ParquetWriter) RenameSchema() {
	for i := 0; i < len(self.Footer.Schema); i++ {
		self.Footer.Schema[i].Name = self.SchemaHandler.Infos[i].ExName
	}
	for _, rowGroup := range self.Footer.RowGroups {
		for _, chunk := range rowGroup.Columns {
			inPathStr := common.PathToStr(chunk.MetaData.PathInSchema)
			exPathStr := self.SchemaHandler.InPathToExPath[inPathStr]
			exPath := common.StrToPath(exPathStr)[1:]
			chunk.MetaData.PathInSchema = exPath
		}
	}
}

//Write the footer and stop writing
func (self *ParquetWriter) WriteStop() error {
	var err error

	if err = self.Flush(true); err != nil {
		return err
	}
	ts := thrift.NewTSerializer()
	ts.Protocol = thrift.NewTCompactProtocolFactory().GetProtocol(ts.Transport)
	self.RenameSchema()
	footerBuf, err := ts.Write(context.TODO(), self.Footer)
	if err != nil {
		return err
	}

	if _, err = self.PFile.Write(footerBuf); err != nil {
		return err
	}
	footerSizeBuf := make([]byte, 4)
	binary.LittleEndian.PutUint32(footerSizeBuf, uint32(len(footerBuf)))

	if _, err = self.PFile.Write(footerSizeBuf); err != nil {
		return err
	}
	if _, err = self.PFile.Write([]byte("PAR1")); err != nil {
		return err
	}
	return nil

}

//Write one object to parquet file
func (self *ParquetWriter) Write(src interface{}) error {
	var err error
	ln := int64(len(self.Objs))

	val := reflect.ValueOf(src)
	if val.Kind() == reflect.Ptr {
		val = val.Elem()
		src = val.Interface()
	}

	if self.CheckSizeCritical <= ln {
		self.ObjSize = (self.ObjSize + common.SizeOf(val))/2 + 1
	}
	self.ObjsSize += self.ObjSize
	self.Objs = append(self.Objs, src)

	criSize := self.NP * self.PageSize * self.SchemaHandler.GetColumnNum()

	if self.ObjsSize >= criSize {
		err = self.Flush(false)

	} else {
		dln := (criSize - self.ObjsSize + self.ObjSize - 1) / self.ObjSize / 2
		self.CheckSizeCritical = dln + ln
	}
	return err

}

func (self *ParquetWriter) flushObjs() error {
	var err error
	l := int64(len(self.Objs))
	if l <= 0 {
		return nil
	}
	pagesMapList := make([]map[string][]*layout.Page, self.NP)
	for i := 0; i < int(self.NP); i++ {
		pagesMapList[i] = make(map[string][]*layout.Page)
	}
	doneChan := make(chan int)

	var c int64 = 0
	delta := (l + self.NP - 1) / self.NP
	lock := new(sync.Mutex)
	for c = 0; c < self.NP; c++ {
		bgn := c * delta
		end := bgn + delta
		if end > l {
			end = l
		}
		if bgn >= l {
			bgn, end = l, l
		}

		go func(b, e int, index int64) {
			defer func() {
				if r := recover(); r != nil {
					switch x := r.(type) {
					case string:
						err = errors.New(x)
					case error:
						err = x
					default:
						err = errors.New("unknown error")
					}
				}
			}()

			if e <= b {
				doneChan <- 0
				return
			}

			tableMap, err2 := self.MarshalFunc(self.Objs, b, e, self.SchemaHandler)

			if err2 == nil {
				for name, table := range *tableMap {
					if table.Info.Encoding == parquet.Encoding_PLAIN_DICTIONARY {
						lock.Lock()
						if _, ok := self.DictRecs[name]; !ok {
							self.DictRecs[name] = layout.NewDictRec(table.Type)
						}
						pagesMapList[index][name], _ = layout.TableToDictDataPages(self.DictRecs[name],
							table, int32(self.PageSize), 32, self.CompressionType)
						lock.Unlock()

					} else {
						pagesMapList[index][name], _ = layout.TableToDataPages(table, int32(self.PageSize),
							self.CompressionType)
					}
				}
			} else {
				err = err2
			}

			doneChan <- 0
		}(int(bgn), int(end), c)
	}

	for c = 0; c < self.NP; c++ {
		<-doneChan
	}

	for _, pagesMap := range pagesMapList {
		for name, pages := range pagesMap {
			if _, ok := self.PagesMapBuf[name]; !ok {
				self.PagesMapBuf[name] = pages
			} else {
				self.PagesMapBuf[name] = append(self.PagesMapBuf[name], pages...)
			}
			for _, page := range pages {
				self.Size += int64(len(page.RawData))
				page.DataTable = nil //release memory
			}
		}
	}

	self.NumRows += int64(len(self.Objs))
	return err
}

//Flush the write buffer to parquet file
func (self *ParquetWriter) Flush(flag bool) error {
	var err error

	if err = self.flushObjs(); err != nil {
		return err
	}

	if (self.Size+self.ObjsSize >= self.RowGroupSize || flag) && len(self.PagesMapBuf) > 0 {
		//pages -> chunk
		chunkMap := make(map[string]*layout.Chunk)
		for name, pages := range self.PagesMapBuf {
			if len(pages) > 0 && pages[0].Info.Encoding == parquet.Encoding_PLAIN_DICTIONARY {
				dictPage, _ := layout.DictRecToDictPage(self.DictRecs[name], int32(self.PageSize), self.CompressionType)
				tmp := append([]*layout.Page{dictPage}, pages...)
				chunkMap[name] = layout.PagesToDictChunk(tmp)
			} else {
				chunkMap[name] = layout.PagesToChunk(pages)

			}
		}

		self.DictRecs = make(map[string]*layout.DictRecType) //clean records for next chunks

		//chunks -> rowGroup
		rowGroup := layout.NewRowGroup()
		rowGroup.RowGroupHeader.Columns = make([]*parquet.ColumnChunk, 0)

		for k := 0; k < len(self.SchemaHandler.SchemaElements); k++ {
			//for _, chunk := range chunkMap {
			schema := self.SchemaHandler.SchemaElements[k]
			if schema.GetNumChildren() > 0 {
				continue
			}
			chunk := chunkMap[self.SchemaHandler.IndexMap[int32(k)]]
			if chunk == nil {
				continue
			}
			rowGroup.Chunks = append(rowGroup.Chunks, chunk)
			//rowGroup.RowGroupHeader.TotalByteSize += chunk.ChunkHeader.MetaData.TotalCompressedSize
			rowGroup.RowGroupHeader.TotalByteSize += chunk.ChunkHeader.MetaData.TotalUncompressedSize
			rowGroup.RowGroupHeader.Columns = append(rowGroup.RowGroupHeader.Columns, chunk.ChunkHeader)
		}
		rowGroup.RowGroupHeader.NumRows = self.NumRows
		self.NumRows = 0

		for k := 0; k < len(rowGroup.Chunks); k++ {
			rowGroup.Chunks[k].ChunkHeader.MetaData.DataPageOffset = -1
			rowGroup.Chunks[k].ChunkHeader.FileOffset = self.Offset

			for l := 0; l < len(rowGroup.Chunks[k].Pages); l++ {
				if rowGroup.Chunks[k].Pages[l].Header.Type == parquet.PageType_DICTIONARY_PAGE {
					tmp := self.Offset
					rowGroup.Chunks[k].ChunkHeader.MetaData.DictionaryPageOffset = &tmp
				} else if rowGroup.Chunks[k].ChunkHeader.MetaData.DataPageOffset <= 0 {
					rowGroup.Chunks[k].ChunkHeader.MetaData.DataPageOffset = self.Offset

				}
				data := rowGroup.Chunks[k].Pages[l].RawData
				if _, err = self.PFile.Write(data); err != nil {
					return err
				}
				self.Offset += int64(len(data))
			}
		}
		self.Footer.RowGroups = append(self.Footer.RowGroups, rowGroup.RowGroupHeader)
		self.Size = 0
		self.PagesMapBuf = make(map[string][]*layout.Page)
	}
	self.Footer.NumRows += int64(len(self.Objs))
	self.Objs = self.Objs[:0]
	self.ObjsSize = 0
	return nil

}
<|endoftext|>


<commit_before>package clui

import (
	"github.com/atotto/clipboard"
	xs "github.com/huandu/xstrings"
	term "github.com/nsf/termbox-go"
)

/*
EditField is a single-line text edit contol. Edit field consumes some keyboard
events when it is active: all printable charaters; Delete, BackSpace, Home,
End, left and right arrows; Ctrl+R to clear EditField.
Edit text can be limited. By default a user can enter text of any length.
Use SetMaxWidth to limit the maximum text length. If the text is longer than
maximun then the text is automatically truncated.
EditField calls onChage in case of its text is changed. Event field Msg contains the new text
*/
type EditField struct {
	BaseControl
	// cursor position in edit text
	cursorPos int
	// the number of the first displayed text character - it is used in case of text is longer than edit width
	offset   int
	readonly bool
	maxWidth int

	onChange   func(Event)
	onKeyPress func(term.Key) bool
}

// NewEditField creates a new EditField control
// view - is a View that manages the control
// parent - is container that keeps the control. The same View can be a view and a parent at the same time.
// width - is minimal width of the control.
// text - text to edit.
// scale - the way of scaling the control when the parent is resized. Use DoNotScale constant if the
//  control should keep its original size.
func CreateEditField(parent Control, width int, text string, scale int) *EditField {
	e := new(EditField)
	e.onChange = nil
	e.SetTitle(text)
	e.SetEnabled(true)

	if width == AutoSize {
		width = xs.Len(text) + 1
	}

	e.SetSize(width, 1)
	e.cursorPos = xs.Len(text)
	e.offset = 0
	e.parent = parent
	e.parent = parent
	e.readonly = false
	e.SetScale(scale)

	e.SetConstraints(width, 1)

	e.end()

	if parent != nil {
		parent.AddChild(e)
	}

	return e
}

// OnChange sets the callback that is called when EditField content is changed
func (e *EditField) OnChange(fn func(Event)) {
	e.onChange = fn
}

// OnKeyPress sets the callback that is called when a user presses a Key while
// the controls is active. If a handler processes the key it should return
// true. If handler returns false it means that the default handler will
// process the key
func (e *EditField) OnKeyPress(fn func(term.Key) bool) {
	e.onKeyPress = fn
}

// SetTitle changes the EditField content and emits OnChage eventif the new value does not equal to old one
func (e *EditField) SetTitle(title string) {
	if e.title != title {
		e.title = title
		if e.onChange != nil {
			ev := Event{Msg: title}
			go e.onChange(ev)
		}
	}
}

// Repaint draws the control on its View surface
func (e *EditField) Draw() {
	PushAttributes()
	defer PopAttributes()

	x, y := e.Pos()
	w, _ := e.Size()

	parts := []rune(SysObject(ObjEdit))
	chLeft, chRight := string(parts[0]), string(parts[1])

	var textOut string
	curOff := 0
	if e.offset == 0 && xs.Len(e.title) < e.width {
		textOut = e.title
	} else {
		fromIdx := 0
		toIdx := 0
		if e.offset == 0 {
			toIdx = e.width - 1
			textOut = xs.Slice(e.title, 0, toIdx) + chRight
			curOff = -e.offset
		} else {
			curOff = 1 - e.offset
			fromIdx = e.offset
			if e.width-1 <= xs.Len(e.title)-e.offset {
				toIdx = e.offset + e.width - 2
				textOut = chLeft + xs.Slice(e.title, fromIdx, toIdx) + chRight
			} else {
				textOut = chLeft + xs.Slice(e.title, fromIdx, -1)
			}
		}
	}

	fg, bg := RealColor(e.fg, ColorEditText), RealColor(e.bg, ColorEditBack)
	if !e.Enabled() {
		fg, bg = RealColor(e.fg, ColorDisabledText), RealColor(e.fg, ColorDisabledBack)
	} else if e.Active() {
		fg, bg = RealColor(e.fg, ColorEditActiveText), RealColor(e.bg, ColorEditActiveBack)
	}

	SetTextColor(fg)
	SetBackColor(bg)
	FillRect(x, y, w, 1, ' ')
	DrawRawText(x, y, textOut)
	if e.Active() {
		SetCursorPos(e.cursorPos+e.x+curOff, e.y)
	}
}

func (e *EditField) insertRune(ch rune) {
	if e.readonly {
		return
	}

	if e.maxWidth > 0 && xs.Len(e.title) >= e.maxWidth {
		return
	}

	idx := e.cursorPos

	if idx == 0 {
		e.SetTitle(string(ch) + e.title)
	} else if idx >= xs.Len(e.title) {
		e.SetTitle(e.title + string(ch))
	} else {
		e.SetTitle(xs.Slice(e.title, 0, idx) + string(ch) + xs.Slice(e.title, idx, -1))
	}

	e.cursorPos++

	if e.cursorPos >= e.width {
		if e.offset == 0 {
			e.offset = 2
		} else {
			e.offset++
		}
	}
}

func (e *EditField) backspace() {
	if e.title == "" || e.cursorPos == 0 || e.readonly {
		return
	}

	length := xs.Len(e.title)
	if e.cursorPos >= length {
		e.cursorPos--
		e.SetTitle(xs.Slice(e.title, 0, length-1))
	} else if e.cursorPos == 1 {
		e.cursorPos = 0
		e.SetTitle(xs.Slice(e.title, 1, -1))
		e.offset = 0
	} else {
		e.cursorPos--
		e.SetTitle(xs.Slice(e.title, 0, e.cursorPos) + xs.Slice(e.title, e.cursorPos+1, -1))
	}

	if length-1 < e.width {
		e.offset = 0
	}
}

func (e *EditField) del() {
	length := xs.Len(e.title)

	if e.title == "" || e.cursorPos == length || e.readonly {
		return
	}

	if e.cursorPos == length-1 {
		e.SetTitle(xs.Slice(e.title, 0, length-1))
	} else {
		e.SetTitle(xs.Slice(e.title, 0, e.cursorPos) + xs.Slice(e.title, e.cursorPos+1, -1))
	}

	if length-1 < e.width {
		e.offset = 0
	}
}

func (e *EditField) charLeft() {
	if e.cursorPos == 0 || e.title == "" {
		return
	}

	if e.cursorPos == e.offset {
		e.offset--
	}

	e.cursorPos--
}

func (e *EditField) charRight() {
	length := xs.Len(e.title)
	if e.cursorPos == length || e.title == "" {
		return
	}

	e.cursorPos++
	if e.cursorPos != length && e.cursorPos >= e.offset+e.width-2 {
		e.offset++
	}
}

func (e *EditField) home() {
	e.offset = 0
	e.cursorPos = 0
}

func (e *EditField) end() {
	length := xs.Len(e.title)
	e.cursorPos = length

	if length < e.width {
		return
	}

	e.offset = length - (e.width - 2)
}

// Clear empties the EditField and emits OnChange event
func (e *EditField) Clear() {
	e.home()
	e.SetTitle("")
}

/*
ProcessEvent processes all events come from the control parent. If a control
processes an event it should return true. If the method returns false it means
that the control do not want or cannot process the event and the caller sends
the event to the control parent
*/
func (e *EditField) ProcessEvent(event Event) bool {
	if !e.Active() || !e.Enabled() {
		return false
	}

	if event.Type == EventActivate && event.X == 0 {
		term.HideCursor()
	}

	if event.Type == EventKey && event.Key != term.KeyTab {
		if e.onKeyPress != nil {
			res := e.onKeyPress(event.Key)
			if res {
				return true
			}
		}

		switch event.Key {
		case term.KeyEnter:
			return false
		case term.KeySpace:
			e.insertRune(' ')
			return true
		case term.KeyBackspace:
			e.backspace()
			return true
		case term.KeyDelete:
			e.del()
			return true
		case term.KeyArrowLeft:
			e.charLeft()
			return true
		case term.KeyHome:
			e.home()
			return true
		case term.KeyEnd:
			e.end()
			return true
		case term.KeyCtrlR:
			if !e.readonly {
				e.Clear()
			}
			return true
		case term.KeyArrowRight:
			e.charRight()
			return true
		case term.KeyCtrlC:
			clipboard.WriteAll(e.Title())
			return true
		case term.KeyCtrlV:
			if !e.readonly {
				s, _ := clipboard.ReadAll()
				e.SetTitle(s)
				e.end()
			}
			return true
		default:
			if event.Ch != 0 {
				e.insertRune(event.Ch)
				return true
			}
		}
		return false
	}

	return false
}

// SetMaxWidth sets the maximum lenght of the EditField text. If the current text is longer it is truncated
func (e *EditField) SetMaxWidth(w int) {
	e.maxWidth = w
	if w > 0 && xs.Len(e.title) > w {
		e.title = xs.Slice(e.title, 0, w)
		e.end()
	}
}

// MaxWidth returns the current maximum text length. Zero means no limit
func (e *EditField) MaxWidth() int {
	return e.maxWidth
}

// SetSize changes control size. Constant DoNotChange can be
// used as placeholder to indicate that the control attrubute
// should be unchanged.
// Method does nothing if new size is less than minimal size
// EditField height cannot be changed - it equals 1 always
func (e *EditField) SetSize(width, height int) {
	if width != KeepValue && (width > 1000 || width < e.minW) {
		return
	}
	if height != KeepValue && (height > 200 || height < e.minH) {
		return
	}

	if width != KeepValue {
		e.width = width
	}

	e.height = 1
}
<commit_msg>Remove duplicate assignment<commit_after>package clui

import (
	"github.com/atotto/clipboard"
	xs "github.com/huandu/xstrings"
	term "github.com/nsf/termbox-go"
)

/*
EditField is a single-line text edit contol. Edit field consumes some keyboard
events when it is active: all printable charaters; Delete, BackSpace, Home,
End, left and right arrows; Ctrl+R to clear EditField.
Edit text can be limited. By default a user can enter text of any length.
Use SetMaxWidth to limit the maximum text length. If the text is longer than
maximun then the text is automatically truncated.
EditField calls onChage in case of its text is changed. Event field Msg contains the new text
*/
type EditField struct {
	BaseControl
	// cursor position in edit text
	cursorPos int
	// the number of the first displayed text character - it is used in case of text is longer than edit width
	offset   int
	readonly bool
	maxWidth int

	onChange   func(Event)
	onKeyPress func(term.Key) bool
}

// NewEditField creates a new EditField control
// view - is a View that manages the control
// parent - is container that keeps the control. The same View can be a view and a parent at the same time.
// width - is minimal width of the control.
// text - text to edit.
// scale - the way of scaling the control when the parent is resized. Use DoNotScale constant if the
//  control should keep its original size.
func CreateEditField(parent Control, width int, text string, scale int) *EditField {
	e := new(EditField)
	e.onChange = nil
	e.SetTitle(text)
	e.SetEnabled(true)

	if width == AutoSize {
		width = xs.Len(text) + 1
	}

	e.SetSize(width, 1)
	e.cursorPos = xs.Len(text)
	e.offset = 0
	e.parent = parent
	e.readonly = false
	e.SetScale(scale)

	e.SetConstraints(width, 1)

	e.end()

	if parent != nil {
		parent.AddChild(e)
	}

	return e
}

// OnChange sets the callback that is called when EditField content is changed
func (e *EditField) OnChange(fn func(Event)) {
	e.onChange = fn
}

// OnKeyPress sets the callback that is called when a user presses a Key while
// the controls is active. If a handler processes the key it should return
// true. If handler returns false it means that the default handler will
// process the key
func (e *EditField) OnKeyPress(fn func(term.Key) bool) {
	e.onKeyPress = fn
}

// SetTitle changes the EditField content and emits OnChage eventif the new value does not equal to old one
func (e *EditField) SetTitle(title string) {
	if e.title != title {
		e.title = title
		if e.onChange != nil {
			ev := Event{Msg: title}
			go e.onChange(ev)
		}
	}
}

// Repaint draws the control on its View surface
func (e *EditField) Draw() {
	PushAttributes()
	defer PopAttributes()

	x, y := e.Pos()
	w, _ := e.Size()

	parts := []rune(SysObject(ObjEdit))
	chLeft, chRight := string(parts[0]), string(parts[1])

	var textOut string
	curOff := 0
	if e.offset == 0 && xs.Len(e.title) < e.width {
		textOut = e.title
	} else {
		fromIdx := 0
		toIdx := 0
		if e.offset == 0 {
			toIdx = e.width - 1
			textOut = xs.Slice(e.title, 0, toIdx) + chRight
			curOff = -e.offset
		} else {
			curOff = 1 - e.offset
			fromIdx = e.offset
			if e.width-1 <= xs.Len(e.title)-e.offset {
				toIdx = e.offset + e.width - 2
				textOut = chLeft + xs.Slice(e.title, fromIdx, toIdx) + chRight
			} else {
				textOut = chLeft + xs.Slice(e.title, fromIdx, -1)
			}
		}
	}

	fg, bg := RealColor(e.fg, ColorEditText), RealColor(e.bg, ColorEditBack)
	if !e.Enabled() {
		fg, bg = RealColor(e.fg, ColorDisabledText), RealColor(e.fg, ColorDisabledBack)
	} else if e.Active() {
		fg, bg = RealColor(e.fg, ColorEditActiveText), RealColor(e.bg, ColorEditActiveBack)
	}

	SetTextColor(fg)
	SetBackColor(bg)
	FillRect(x, y, w, 1, ' ')
	DrawRawText(x, y, textOut)
	if e.Active() {
		SetCursorPos(e.cursorPos+e.x+curOff, e.y)
	}
}

func (e *EditField) insertRune(ch rune) {
	if e.readonly {
		return
	}

	if e.maxWidth > 0 && xs.Len(e.title) >= e.maxWidth {
		return
	}

	idx := e.cursorPos

	if idx == 0 {
		e.SetTitle(string(ch) + e.title)
	} else if idx >= xs.Len(e.title) {
		e.SetTitle(e.title + string(ch))
	} else {
		e.SetTitle(xs.Slice(e.title, 0, idx) + string(ch) + xs.Slice(e.title, idx, -1))
	}

	e.cursorPos++

	if e.cursorPos >= e.width {
		if e.offset == 0 {
			e.offset = 2
		} else {
			e.offset++
		}
	}
}

func (e *EditField) backspace() {
	if e.title == "" || e.cursorPos == 0 || e.readonly {
		return
	}

	length := xs.Len(e.title)
	if e.cursorPos >= length {
		e.cursorPos--
		e.SetTitle(xs.Slice(e.title, 0, length-1))
	} else if e.cursorPos == 1 {
		e.cursorPos = 0
		e.SetTitle(xs.Slice(e.title, 1, -1))
		e.offset = 0
	} else {
		e.cursorPos--
		e.SetTitle(xs.Slice(e.title, 0, e.cursorPos) + xs.Slice(e.title, e.cursorPos+1, -1))
	}

	if length-1 < e.width {
		e.offset = 0
	}
}

func (e *EditField) del() {
	length := xs.Len(e.title)

	if e.title == "" || e.cursorPos == length || e.readonly {
		return
	}

	if e.cursorPos == length-1 {
		e.SetTitle(xs.Slice(e.title, 0, length-1))
	} else {
		e.SetTitle(xs.Slice(e.title, 0, e.cursorPos) + xs.Slice(e.title, e.cursorPos+1, -1))
	}

	if length-1 < e.width {
		e.offset = 0
	}
}

func (e *EditField) charLeft() {
	if e.cursorPos == 0 || e.title == "" {
		return
	}

	if e.cursorPos == e.offset {
		e.offset--
	}

	e.cursorPos--
}

func (e *EditField) charRight() {
	length := xs.Len(e.title)
	if e.cursorPos == length || e.title == "" {
		return
	}

	e.cursorPos++
	if e.cursorPos != length && e.cursorPos >= e.offset+e.width-2 {
		e.offset++
	}
}

func (e *EditField) home() {
	e.offset = 0
	e.cursorPos = 0
}

func (e *EditField) end() {
	length := xs.Len(e.title)
	e.cursorPos = length

	if length < e.width {
		return
	}

	e.offset = length - (e.width - 2)
}

// Clear empties the EditField and emits OnChange event
func (e *EditField) Clear() {
	e.home()
	e.SetTitle("")
}

/*
ProcessEvent processes all events come from the control parent. If a control
processes an event it should return true. If the method returns false it means
that the control do not want or cannot process the event and the caller sends
the event to the control parent
*/
func (e *EditField) ProcessEvent(event Event) bool {
	if !e.Active() || !e.Enabled() {
		return false
	}

	if event.Type == EventActivate && event.X == 0 {
		term.HideCursor()
	}

	if event.Type == EventKey && event.Key != term.KeyTab {
		if e.onKeyPress != nil {
			res := e.onKeyPress(event.Key)
			if res {
				return true
			}
		}

		switch event.Key {
		case term.KeyEnter:
			return false
		case term.KeySpace:
			e.insertRune(' ')
			return true
		case term.KeyBackspace:
			e.backspace()
			return true
		case term.KeyDelete:
			e.del()
			return true
		case term.KeyArrowLeft:
			e.charLeft()
			return true
		case term.KeyHome:
			e.home()
			return true
		case term.KeyEnd:
			e.end()
			return true
		case term.KeyCtrlR:
			if !e.readonly {
				e.Clear()
			}
			return true
		case term.KeyArrowRight:
			e.charRight()
			return true
		case term.KeyCtrlC:
			clipboard.WriteAll(e.Title())
			return true
		case term.KeyCtrlV:
			if !e.readonly {
				s, _ := clipboard.ReadAll()
				e.SetTitle(s)
				e.end()
			}
			return true
		default:
			if event.Ch != 0 {
				e.insertRune(event.Ch)
				return true
			}
		}
		return false
	}

	return false
}

// SetMaxWidth sets the maximum lenght of the EditField text. If the current text is longer it is truncated
func (e *EditField) SetMaxWidth(w int) {
	e.maxWidth = w
	if w > 0 && xs.Len(e.title) > w {
		e.title = xs.Slice(e.title, 0, w)
		e.end()
	}
}

// MaxWidth returns the current maximum text length. Zero means no limit
func (e *EditField) MaxWidth() int {
	return e.maxWidth
}

// SetSize changes control size. Constant DoNotChange can be
// used as placeholder to indicate that the control attrubute
// should be unchanged.
// Method does nothing if new size is less than minimal size
// EditField height cannot be changed - it equals 1 always
func (e *EditField) SetSize(width, height int) {
	if width != KeepValue && (width > 1000 || width < e.minW) {
		return
	}
	if height != KeepValue && (height > 200 || height < e.minH) {
		return
	}

	if width != KeepValue {
		e.width = width
	}

	e.height = 1
}
<|endoftext|>


<commit_before>package main

import (
	"testing"
	"time"

	"github.com/stretchr/testify/assert"
)

func TestTimerCallsSendFunc(t *testing.T) {
	ch := make(chan bool)
	testFunc := func(callTime time.Time) (time.Time, bool) {
		ch <- true
		return time.Now(), false
	}

	timer := NewTimer(testFunc)
	defer timer.Stop()

	timeout := time.After(time.Duration(3) * time.Millisecond)
	timer.Reset(time.Now().Add(time.Duration(1)*time.Millisecond), true)

	select {
	case _ = <-ch:
		return
	case _ = <-timeout:
		assert.Fail(t, "test timed out")
	}
}
<commit_msg>Give the timer test more wiggle room<commit_after>package main

import (
	"testing"
	"time"

	"github.com/stretchr/testify/assert"
)

func TestTimerCallsSendFunc(t *testing.T) {
	ch := make(chan bool, 1)
	testFunc := func(callTime time.Time) (time.Time, bool) {
		close(ch)
		return time.Now(), false
	}

	timer := NewTimer(testFunc)
	defer timer.Stop()

	timeout := time.After(time.Duration(500) * time.Millisecond)
	timer.Reset(time.Now().Add(time.Duration(2)*time.Millisecond), true)

	select {
	case <-ch:
		break
	case <-timeout:
		assert.Fail(t, "test timed out")
	}
}
<|endoftext|>


<commit_before>package ParquetReader

import (
	"fmt"
	"git.apache.org/thrift.git/lib/go/thrift"
	"github.com/xitongsys/parquet-go/Common"
	"github.com/xitongsys/parquet-go/Layout"
	"github.com/xitongsys/parquet-go/ParquetFile"
	"github.com/xitongsys/parquet-go/SchemaHandler"
	"github.com/xitongsys/parquet-go/parquet"
)

type ColumnBufferType struct {
	PFile        ParquetFile.ParquetFile
	ThriftReader *thrift.TBufferedTransport

	Footer        *parquet.FileMetaData
	SchemaHandler *SchemaHandler.SchemaHandler

	PathStr       string
	RowGroupIndex int64
	ChunkHeader   *parquet.ColumnChunk

	ChunkReadValues int64

	DictPage *Layout.Page

	DataTable        *Layout.Table
	DataTableNumRows int64
}

func NewColumnBuffer(pFile ParquetFile.ParquetFile, footer *parquet.FileMetaData, schemaHandler *SchemaHandler.SchemaHandler, pathStr string) (*ColumnBufferType, error) {
	newPFile, err := pFile.Open("")
	if err != nil {
		return nil, err
	}
	res := &ColumnBufferType{
		PFile:            newPFile,
		Footer:           footer,
		SchemaHandler:    schemaHandler,
		PathStr:          pathStr,
		DataTableNumRows: -1,
	}
	err = res.NextRowGroup()
	return res, err
}

func (self *ColumnBufferType) NextRowGroup() error {
	rowGroups := self.Footer.GetRowGroups()
	ln := int64(len(rowGroups))
	if self.RowGroupIndex >= ln {
		return fmt.Errorf("End of row groups")
	}
	self.RowGroupIndex++

	columnChunks := rowGroups[self.RowGroupIndex-1].GetColumns()
	i := int64(0)
	ln = int64(len(columnChunks))
	for i = 0; i < ln; i++ {
		path := make([]string, 0)
		path = append(path, self.SchemaHandler.GetRootName())
		path = append(path, columnChunks[i].MetaData.GetPathInSchema()...)
		if self.PathStr == Common.PathToStr(path) {
			break
		}
	}
	if i >= ln {
		return fmt.Errorf("Column not found", self.PathStr)
	}

	self.ChunkHeader = columnChunks[i]
	if columnChunks[i].FilePath != nil {
		self.PFile.Close()
		self.PFile, _ = self.PFile.Open(*columnChunks[i].FilePath)
	}
	offset := columnChunks[i].FileOffset
	size := columnChunks[i].MetaData.GetTotalCompressedSize()
	if self.ThriftReader != nil {
		self.ThriftReader.Close()
	}
	self.ThriftReader = ParquetFile.ConvertToThriftReader(self.PFile, offset, size)
	self.ChunkReadValues = 0
	self.DictPage = nil
	return nil
}

func (self *ColumnBufferType) ReadPage() error {
	if self.ChunkReadValues < self.ChunkHeader.MetaData.NumValues {
		page, numValues, numRows := Layout.ReadPage(self.ThriftReader, self.SchemaHandler, self.ChunkHeader.MetaData)
		if page.Header.GetType() == parquet.PageType_DICTIONARY_PAGE {
			self.DictPage = page
			return nil
		}
		page.Decode(self.DictPage)

		if self.DataTable == nil {
			self.DataTable = Layout.NewTableFromTable(page.DataTable)
		}
		self.DataTable.Merge(page.DataTable)
		self.ChunkReadValues += numValues
		if self.ChunkReadValues >= self.ChunkHeader.MetaData.NumValues {
			numRows++
		}
		self.DataTableNumRows += numRows
	} else {
		err := self.NextRowGroup()
		if err != nil {
			return err
		}
		self.ReadPage()
	}

	return nil
}

func (self *ColumnBufferType) ReadRows(num int64) (*Layout.Table, int64) {
	var err error

	for self.DataTableNumRows < num && err == nil {
		err = self.ReadPage()

	}
	if num > self.DataTableNumRows {
		num = self.DataTableNumRows
	}
	res := self.DataTable.Pop(num)
	self.DataTableNumRows -= num
	return res, num
}
<commit_msg>fix errorf bug<commit_after>package ParquetReader

import (
	"fmt"
	"git.apache.org/thrift.git/lib/go/thrift"
	"github.com/xitongsys/parquet-go/Common"
	"github.com/xitongsys/parquet-go/Layout"
	"github.com/xitongsys/parquet-go/ParquetFile"
	"github.com/xitongsys/parquet-go/SchemaHandler"
	"github.com/xitongsys/parquet-go/parquet"
)

type ColumnBufferType struct {
	PFile        ParquetFile.ParquetFile
	ThriftReader *thrift.TBufferedTransport

	Footer        *parquet.FileMetaData
	SchemaHandler *SchemaHandler.SchemaHandler

	PathStr       string
	RowGroupIndex int64
	ChunkHeader   *parquet.ColumnChunk

	ChunkReadValues int64

	DictPage *Layout.Page

	DataTable        *Layout.Table
	DataTableNumRows int64
}

func NewColumnBuffer(pFile ParquetFile.ParquetFile, footer *parquet.FileMetaData, schemaHandler *SchemaHandler.SchemaHandler, pathStr string) (*ColumnBufferType, error) {
	newPFile, err := pFile.Open("")
	if err != nil {
		return nil, err
	}
	res := &ColumnBufferType{
		PFile:            newPFile,
		Footer:           footer,
		SchemaHandler:    schemaHandler,
		PathStr:          pathStr,
		DataTableNumRows: -1,
	}
	err = res.NextRowGroup()
	return res, err
}

func (self *ColumnBufferType) NextRowGroup() error {
	rowGroups := self.Footer.GetRowGroups()
	ln := int64(len(rowGroups))
	if self.RowGroupIndex >= ln {
		return fmt.Errorf("End of row groups")
	}
	self.RowGroupIndex++

	columnChunks := rowGroups[self.RowGroupIndex-1].GetColumns()
	i := int64(0)
	ln = int64(len(columnChunks))
	for i = 0; i < ln; i++ {
		path := make([]string, 0)
		path = append(path, self.SchemaHandler.GetRootName())
		path = append(path, columnChunks[i].MetaData.GetPathInSchema()...)
		if self.PathStr == Common.PathToStr(path) {
			break
		}
	}
	if i >= ln {
		return fmt.Errorf("Column not found: %v", self.PathStr)
	}

	self.ChunkHeader = columnChunks[i]
	if columnChunks[i].FilePath != nil {
		self.PFile.Close()
		self.PFile, _ = self.PFile.Open(*columnChunks[i].FilePath)
	}
	offset := columnChunks[i].FileOffset
	size := columnChunks[i].MetaData.GetTotalCompressedSize()
	if self.ThriftReader != nil {
		self.ThriftReader.Close()
	}
	self.ThriftReader = ParquetFile.ConvertToThriftReader(self.PFile, offset, size)
	self.ChunkReadValues = 0
	self.DictPage = nil
	return nil
}

func (self *ColumnBufferType) ReadPage() error {
	if self.ChunkReadValues < self.ChunkHeader.MetaData.NumValues {
		page, numValues, numRows := Layout.ReadPage(self.ThriftReader, self.SchemaHandler, self.ChunkHeader.MetaData)
		if page.Header.GetType() == parquet.PageType_DICTIONARY_PAGE {
			self.DictPage = page
			return nil
		}
		page.Decode(self.DictPage)

		if self.DataTable == nil {
			self.DataTable = Layout.NewTableFromTable(page.DataTable)
		}
		self.DataTable.Merge(page.DataTable)
		self.ChunkReadValues += numValues
		if self.ChunkReadValues >= self.ChunkHeader.MetaData.NumValues {
			numRows++
		}
		self.DataTableNumRows += numRows
	} else {
		err := self.NextRowGroup()
		if err != nil {
			return err
		}
		self.ReadPage()
	}

	return nil
}

func (self *ColumnBufferType) ReadRows(num int64) (*Layout.Table, int64) {
	var err error

	for self.DataTableNumRows < num && err == nil {
		err = self.ReadPage()

	}
	if num > self.DataTableNumRows {
		num = self.DataTableNumRows
	}
	res := self.DataTable.Pop(num)
	self.DataTableNumRows -= num
	return res, num
}
<|endoftext|>


<commit_before>package durafmt

import (
	"reflect"
	"strconv"
	"testing"
)

func TestParseUnits(t *testing.T) {

}

func TestUnitsCoder_Encode(t *testing.T) {
	tests := []struct {
		coder UnitsCoder
		want  string
	}{
		{DefaultUnitsCoder, "year:years,week:weeks,day:days,hour:hours,minute:minutes,second:seconds,millisecond:milliseconds,microsecond:microseconds"},
		{UnitsCoder{"+", "|"}, "year+years|week+weeks|day+days|hour+hours|minute+minutes|second+seconds|millisecond+milliseconds|microsecond+microseconds"},
	}
	for i, tt := range tests {
		t.Run(strconv.Itoa(i), func(t *testing.T) {
			if got := tt.coder.Encode(units); got != tt.want {
				t.Errorf("Encode() = %v, want %v", got, tt.want)
			}
		})
	}
}

func TestUnitsCoder_Decode(t *testing.T) {
	tests := []struct {
		test    string
		wantU   Units
		wantErr bool
	}{
		{"y,w,d,h,M,s,m,mi", Units{
			Unit{"y", "ys"},
			Unit{"w", "ws"},
			Unit{"d", "ds"},
			Unit{"h", "hs"},
			Unit{"M", "Ms"},
			Unit{"s", "ss"},
			Unit{"m", "ms"},
			Unit{"mi", "mis"},
		}, false},
		{"ano,semana,dia,hora,minuto,segundo,milissegundo,microssegundo", Units{
			Unit{"ano", "anos"},
			Unit{"semana", "semanas"},
			Unit{"dia", "dias"},
			Unit{"hora", "horas"},
			Unit{"minuto", "minutos"},
			Unit{"segundo", "segundos"},
			Unit{"milissegundo", "milissegundos"},
			Unit{"microssegundo", "microssegundos"},
		}, false},
		{"y:YS,w:WS,d:DS,h:HS,M:MS,s:SS,m:mS,mi:MiS", Units{
			Unit{"y", "YS"},
			Unit{"w", "WS"},
			Unit{"d", "DS"},
			Unit{"h", "HS"},
			Unit{"M", "MS"},
			Unit{"s", "SS"},
			Unit{"m", "mS"},
			Unit{"mi", "MiS"},
		}, false},
		{"y:Y:Y_,w,d,h,M,s,m,mi", Units{}, true},
		{"", Units{}, true},
	}
	for i, tt := range tests {
		t.Run(strconv.Itoa(i), func(t *testing.T) {
			gotU, err := DefaultUnitsCoder.Decode(tt.test)
			if (err != nil) != tt.wantErr {
				t.Errorf("ParseUnits() error = %v, wantErr %v", err, tt.wantErr)
				return
			}
			if !reflect.DeepEqual(gotU, tt.wantU) {
				t.Errorf("ParseUnits() gotU = %v, want %v", gotU, tt.wantU)
			}
		})
	}
}
<commit_msg>Remove reflect package from unit_test.<commit_after>package durafmt

import (
	"strconv"
	"testing"
)

func TestParseUnits(t *testing.T) {

}

func TestUnitsCoder_Encode(t *testing.T) {
	tests := []struct {
		coder UnitsCoder
		want  string
	}{
		{DefaultUnitsCoder, "year:years,week:weeks,day:days,hour:hours,minute:minutes,second:seconds,millisecond:milliseconds,microsecond:microseconds"},
		{UnitsCoder{"+", "|"}, "year+years|week+weeks|day+days|hour+hours|minute+minutes|second+seconds|millisecond+milliseconds|microsecond+microseconds"},
	}
	for i, tt := range tests {
		t.Run(strconv.Itoa(i), func(t *testing.T) {
			if got := tt.coder.Encode(units); got != tt.want {
				t.Errorf("Encode() = %v, want %v", got, tt.want)
			}
		})
	}
}

func TestUnitsCoder_Decode(t *testing.T) {
	tests := []struct {
		test    string
		wantU   Units
		wantErr bool
	}{
		{"y,w,d,h,M,s,m,mi", Units{
			Unit{"y", "ys"},
			Unit{"w", "ws"},
			Unit{"d", "ds"},
			Unit{"h", "hs"},
			Unit{"M", "Ms"},
			Unit{"s", "ss"},
			Unit{"m", "ms"},
			Unit{"mi", "mis"},
		}, false},
		{"ano,semana,dia,hora,minuto,segundo,milissegundo,microssegundo", Units{
			Unit{"ano", "anos"},
			Unit{"semana", "semanas"},
			Unit{"dia", "dias"},
			Unit{"hora", "horas"},
			Unit{"minuto", "minutos"},
			Unit{"segundo", "segundos"},
			Unit{"milissegundo", "milissegundos"},
			Unit{"microssegundo", "microssegundos"},
		}, false},
		{"y:YS,w:WS,d:DS,h:HS,M:MS,s:SS,m:mS,mi:MiS", Units{
			Unit{"y", "YS"},
			Unit{"w", "WS"},
			Unit{"d", "DS"},
			Unit{"h", "HS"},
			Unit{"M", "MS"},
			Unit{"s", "SS"},
			Unit{"m", "mS"},
			Unit{"mi", "MiS"},
		}, false},
		{"y:Y:Y_,w,d,h,M,s,m,mi", Units{}, true},
		{"", Units{}, true},
	}
	for i, tt := range tests {
		t.Run(strconv.Itoa(i), func(t *testing.T) {
			gotU, err := DefaultUnitsCoder.Decode(tt.test)
			if (err != nil) != tt.wantErr {
				t.Errorf("ParseUnits() error = %v, wantErr %v", err, tt.wantErr)
				return
			}
			if !unitsEqual(gotU, tt.wantU) {
				t.Errorf("ParseUnits() gotU = %v, want %v", gotU, tt.wantU)
			}
		})
	}
}

func unitEqual(u1, u2 Unit) bool {
	if u1.Singular != u2.Singular || u1.Plural != u2.Plural {
		return false
	}
	return true
}

func unitsEqual(u1, u2 Units) bool {
	if !unitEqual(u1.Year, u2.Year) {
		return false
	}
	if !unitEqual(u1.Week, u2.Week) {
		return false
	}
	if !unitEqual(u1.Day, u2.Day) {
		return false
	}
	if !unitEqual(u1.Hour, u2.Hour) {
		return false
	}
	if !unitEqual(u1.Minute, u2.Minute) {
		return false
	}
	if !unitEqual(u1.Second, u2.Second) {
		return false
	}
	if !unitEqual(u1.Millisecond, u2.Millisecond) {
		return false
	}
	if !unitEqual(u1.Microsecond, u2.Microsecond) {
		return false
	}
	return true
}
<|endoftext|>


<commit_before>// Copyright 2015, David Howden
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package index defines functionality for creating and manipulating a Tchaik music index.
package index

import (
	"compress/gzip"
	"encoding/json"
	"fmt"
	"io"
	"time"
)

// Library is an interface which defines methods for listing tracks.
type Library interface {
	// Tracks returns a slice of all the tracks in the library.
	Tracks() []Track

	// Track returns the track from the given identifier, second return value true
	// if successful.
	Track(identifier string) (Track, bool)
}

// Track is an interface which defines methods for retrieving track metadata.  Methods return
// zero values if attributes are unset and panic on undefined attributes and type mismatches.
type Track interface {
	// GetString returns a string value for the given attribute name.  Panics
	// if no such string attribute exists.
	GetString(string) string

	// GetStrings returns a list of strings for the given attribute name.
	// Panics if no such attribute exists.
	GetStrings(string) []string

	// GetInt returns an int value for the given attribute name.  Panics if no such
	// attribute exists.
	GetInt(string) int

	// GetTime returns a time.Time value for the given attribute name. Panics if no
	// such attribute exists.
	GetTime(string) time.Time
}

// Convert reads all the data exported by the Library and writes into the standard
// tchaik Library implementation.
// NB: The identifier field is set to be the value of ID on every track, regardless
// of whether this value has already been set in the input Library.
func Convert(l Library, id string) *library {
	allTracks := l.Tracks()
	tracks := make(map[string]*track, len(allTracks))

	for _, t := range allTracks {
		identifier := t.GetString(id)
		tracks[identifier] = &track{
			// string fields
			ID:          identifier,
			Name:        t.GetString("Name"),
			Album:       t.GetString("Album"),
			AlbumArtist: t.GetString("AlbumArtist"),
			Artist:      t.GetString("Artist"),
			Composer:    t.GetString("Composer"),
			Genre:       t.GetString("Genre"),
			Location:    t.GetString("Location"),

			// integer fields
			TotalTime:   t.GetInt("TotalTime"),
			Year:        t.GetInt("Year"),
			DiscNumber:  t.GetInt("DiscNumber"),
			TrackNumber: t.GetInt("TrackNumber"),
			TrackCount:  t.GetInt("TrackCount"),
			DiscCount:   t.GetInt("DiscCount"),
			BitRate:     t.GetInt("BitRate"),

			// date fields
			DateAdded:    t.GetTime("DateAdded"),
			DateModified: t.GetTime("DateModified"),
		}
	}
	return &library{
		tracks,
	}
}

// library is the default internal implementation Library which acts as the data
// source for all media tracks.
type library struct {
	trks map[string]*track
}

// Tracks implements Library.
func (l *library) Tracks() []Track {
	tracks := make([]Track, 0, len(l.trks))
	for _, t := range l.trks {
		tracks = append(tracks, t)
	}
	return tracks
}

// Track implements Library.
func (l *library) Track(id string) (Track, bool) {
	t, ok := l.trks[id]
	return t, ok
}

func (l *library) MarshalJSON() ([]byte, error) {
	return json.Marshal(l.trks)
}

func (l *library) UnmarshalJSON(b []byte) error {
	l.trks = make(map[string]*track)
	return json.Unmarshal(b, &l.trks)
}

// WriteTo writes the Library data to the writer, currently using gzipped-JSON.
func WriteTo(l Library, w io.Writer) error {
	gzw, err := gzip.NewWriterLevel(w, gzip.BestSpeed)
	if err != nil {
		return err
	}
	defer gzw.Close()
	enc := json.NewEncoder(gzw)
	return enc.Encode(l)
}

// ReadFrom reads the gzipped-JSON representation of a Library.
func ReadFrom(r io.Reader) (Library, error) {
	gzr, err := gzip.NewReader(r)
	if err != nil {
		return nil, err
	}
	defer gzr.Close()
	dec := json.NewDecoder(gzr)
	l := &library{}
	err = dec.Decode(l)
	return l, err
}

// track is the default implementation of the Track interface.
type track struct {
	ID          string `json:"id,omitempty"`
	Name        string `json:"name,omitempty"`
	Album       string `json:"album,omitempty"`
	AlbumArtist string `json:"albumArtist,omitempty"`
	Artist      string `json:"artist,omitempty"`
	Composer    string `json:"composer,omitempty"`
	Genre       string `json:"genre,omitempty"`
	Location    string `json:"location,omitempty"`

	TotalTime   int `json:"totalTime,omitempty"`
	Year        int `json:"year,omitempty"`
	DiscNumber  int `json:"discNumber,omitempty"`
	TrackNumber int `json:"trackNumber,omitempty"`
	TrackCount  int `json:"trackCount,omitempty"`
	DiscCount   int `json:"discCount,omitempty"`
	BitRate     int `json:"bitRate,omitempty"`

	DateAdded    time.Time `json:"dateAdded,omitempty"`
	DateModified time.Time `json:"dateModified,omitempty"`
}

// GetString implements Track.
func (t *track) GetString(name string) string {
	switch name {
	case "ID":
		return t.ID
	case "Name":
		return t.Name
	case "Album":
		return t.Album
	case "AlbumArtist":
		return t.AlbumArtist
	case "Artist":
		return t.Artist
	case "Composer":
		return t.Composer
	case "Genre":
		return t.Genre
	case "Location":
		return t.Location
	}
	panic(fmt.Sprintf("unknown string field '%v'", name))
}

// DefaultGetStrings is a function which returns the default value for a GetStrings
// attribute which is based on an existing GetString attribute.  In particular, we handle
// the case where an empty 'GetString' attribute would be "", whereas the corresponding
// 'GetStrings' method should return 'nil' and not '[]string{""}'.
func DefaultGetStrings(t Track, f string) []string {
	v := t.GetString(f)
	if v != "" {
		return []string{v}
	}
	return nil
}

// GetStrings implements Track.
func (t *track) GetStrings(name string) []string {
	switch name {
	case "Artist", "AlbumArtist", "Composer":
		return DefaultGetStrings(t, name)
	}
	panic(fmt.Sprintf("unknown strings field '%v", name))
}

// GetInt implements Track.
func (t *track) GetInt(name string) int {
	switch name {
	case "TotalTime":
		return t.TotalTime
	case "Year":
		return t.Year
	case "DiscNumber":
		return t.DiscNumber
	case "TrackNumber":
		return t.TrackNumber
	case "TrackCount":
		return t.TrackCount
	case "DiscCount":
		return t.DiscCount
	case "BitRate":
		return t.BitRate
	}
	panic(fmt.Sprintf("unknown int field '%v'", name))
}

// GetTime implements Track.
func (t *track) GetTime(name string) time.Time {
	switch name {
	case "DateAdded":
		return t.DateAdded
	case "DateModified":
		return t.DateModified
	}
	panic(fmt.Sprintf("unknown time field '%v'", name))
}
<commit_msg>Improve Library Track method godoc.<commit_after>// Copyright 2015, David Howden
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package index defines functionality for creating and manipulating a Tchaik music index.
package index

import (
	"compress/gzip"
	"encoding/json"
	"fmt"
	"io"
	"time"
)

// Library is an interface which defines methods for listing tracks.
type Library interface {
	// Tracks returns a slice of all the tracks in the library.
	Tracks() []Track

	// Track returns the track from the given identifier and true if successful,
	// false otherwise.
	Track(identifier string) (Track, bool)
}

// Track is an interface which defines methods for retrieving track metadata.  Methods return
// zero values if attributes are unset and panic on undefined attributes and type mismatches.
type Track interface {
	// GetString returns a string value for the given attribute name.  Panics
	// if no such string attribute exists.
	GetString(string) string

	// GetStrings returns a list of strings for the given attribute name.
	// Panics if no such attribute exists.
	GetStrings(string) []string

	// GetInt returns an int value for the given attribute name.  Panics if no such
	// attribute exists.
	GetInt(string) int

	// GetTime returns a time.Time value for the given attribute name. Panics if no
	// such attribute exists.
	GetTime(string) time.Time
}

// Convert reads all the data exported by the Library and writes into the standard
// tchaik Library implementation.
// NB: The identifier field is set to be the value of ID on every track, regardless
// of whether this value has already been set in the input Library.
func Convert(l Library, id string) *library {
	allTracks := l.Tracks()
	tracks := make(map[string]*track, len(allTracks))

	for _, t := range allTracks {
		identifier := t.GetString(id)
		tracks[identifier] = &track{
			// string fields
			ID:          identifier,
			Name:        t.GetString("Name"),
			Album:       t.GetString("Album"),
			AlbumArtist: t.GetString("AlbumArtist"),
			Artist:      t.GetString("Artist"),
			Composer:    t.GetString("Composer"),
			Genre:       t.GetString("Genre"),
			Location:    t.GetString("Location"),

			// integer fields
			TotalTime:   t.GetInt("TotalTime"),
			Year:        t.GetInt("Year"),
			DiscNumber:  t.GetInt("DiscNumber"),
			TrackNumber: t.GetInt("TrackNumber"),
			TrackCount:  t.GetInt("TrackCount"),
			DiscCount:   t.GetInt("DiscCount"),
			BitRate:     t.GetInt("BitRate"),

			// date fields
			DateAdded:    t.GetTime("DateAdded"),
			DateModified: t.GetTime("DateModified"),
		}
	}
	return &library{
		tracks,
	}
}

// library is the default internal implementation Library which acts as the data
// source for all media tracks.
type library struct {
	trks map[string]*track
}

// Tracks implements Library.
func (l *library) Tracks() []Track {
	tracks := make([]Track, 0, len(l.trks))
	for _, t := range l.trks {
		tracks = append(tracks, t)
	}
	return tracks
}

// Track implements Library.
func (l *library) Track(id string) (Track, bool) {
	t, ok := l.trks[id]
	return t, ok
}

func (l *library) MarshalJSON() ([]byte, error) {
	return json.Marshal(l.trks)
}

func (l *library) UnmarshalJSON(b []byte) error {
	l.trks = make(map[string]*track)
	return json.Unmarshal(b, &l.trks)
}

// WriteTo writes the Library data to the writer, currently using gzipped-JSON.
func WriteTo(l Library, w io.Writer) error {
	gzw, err := gzip.NewWriterLevel(w, gzip.BestSpeed)
	if err != nil {
		return err
	}
	defer gzw.Close()
	enc := json.NewEncoder(gzw)
	return enc.Encode(l)
}

// ReadFrom reads the gzipped-JSON representation of a Library.
func ReadFrom(r io.Reader) (Library, error) {
	gzr, err := gzip.NewReader(r)
	if err != nil {
		return nil, err
	}
	defer gzr.Close()
	dec := json.NewDecoder(gzr)
	l := &library{}
	err = dec.Decode(l)
	return l, err
}

// track is the default implementation of the Track interface.
type track struct {
	ID          string `json:"id,omitempty"`
	Name        string `json:"name,omitempty"`
	Album       string `json:"album,omitempty"`
	AlbumArtist string `json:"albumArtist,omitempty"`
	Artist      string `json:"artist,omitempty"`
	Composer    string `json:"composer,omitempty"`
	Genre       string `json:"genre,omitempty"`
	Location    string `json:"location,omitempty"`

	TotalTime   int `json:"totalTime,omitempty"`
	Year        int `json:"year,omitempty"`
	DiscNumber  int `json:"discNumber,omitempty"`
	TrackNumber int `json:"trackNumber,omitempty"`
	TrackCount  int `json:"trackCount,omitempty"`
	DiscCount   int `json:"discCount,omitempty"`
	BitRate     int `json:"bitRate,omitempty"`

	DateAdded    time.Time `json:"dateAdded,omitempty"`
	DateModified time.Time `json:"dateModified,omitempty"`
}

// GetString implements Track.
func (t *track) GetString(name string) string {
	switch name {
	case "ID":
		return t.ID
	case "Name":
		return t.Name
	case "Album":
		return t.Album
	case "AlbumArtist":
		return t.AlbumArtist
	case "Artist":
		return t.Artist
	case "Composer":
		return t.Composer
	case "Genre":
		return t.Genre
	case "Location":
		return t.Location
	}
	panic(fmt.Sprintf("unknown string field '%v'", name))
}

// DefaultGetStrings is a function which returns the default value for a GetStrings
// attribute which is based on an existing GetString attribute.  In particular, we handle
// the case where an empty 'GetString' attribute would be "", whereas the corresponding
// 'GetStrings' method should return 'nil' and not '[]string{""}'.
func DefaultGetStrings(t Track, f string) []string {
	v := t.GetString(f)
	if v != "" {
		return []string{v}
	}
	return nil
}

// GetStrings implements Track.
func (t *track) GetStrings(name string) []string {
	switch name {
	case "Artist", "AlbumArtist", "Composer":
		return DefaultGetStrings(t, name)
	}
	panic(fmt.Sprintf("unknown strings field '%v", name))
}

// GetInt implements Track.
func (t *track) GetInt(name string) int {
	switch name {
	case "TotalTime":
		return t.TotalTime
	case "Year":
		return t.Year
	case "DiscNumber":
		return t.DiscNumber
	case "TrackNumber":
		return t.TrackNumber
	case "TrackCount":
		return t.TrackCount
	case "DiscCount":
		return t.DiscCount
	case "BitRate":
		return t.BitRate
	}
	panic(fmt.Sprintf("unknown int field '%v'", name))
}

// GetTime implements Track.
func (t *track) GetTime(name string) time.Time {
	switch name {
	case "DateAdded":
		return t.DateAdded
	case "DateModified":
		return t.DateModified
	}
	panic(fmt.Sprintf("unknown time field '%v'", name))
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/APTrust/bagman/bagman"
	"github.com/APTrust/bagman/dpn"
	"github.com/APTrust/bagman/workers"
	"path/filepath"
)

const NONCE = "12345"

// dpn_package_test runs a single APTrust bag through the DPN packager.
// This is for ad-hoc dev testing.
func main() {
	// The workers below will share one ProcUtil, which means
	// some of ProcUtil's internal variables, like success count
	// and failure count will be updated by each worker.
	procUtil := workers.CreateProcUtil()

	pathToConfigFile := "dpn/dpn_config.json"
	dpnConfig, err := dpn.LoadConfig(pathToConfigFile, "test")
	if err != nil {
		procUtil.MessageLog.Fatal(err.Error())
	}
	fmt.Println("Creating packager...")
	packager := dpn.NewPackager(procUtil, dpnConfig)
	dpnResult := packager.RunTest("test.edu/ncsu.1840.16-1004")
	if dpnResult.ErrorMessage == "" {
		fmt.Println("Packager succeeded. Moving to storage.")
		storer, err := dpn.NewStorer(procUtil, dpnConfig)
		if err != nil {
			procUtil.MessageLog.Fatal(err.Error())
		}
		storer.RunTest(dpnResult)
		if dpnResult.DPNBag == nil {
			procUtil.MessageLog.Fatal("DPNBag is nil! " +
				"DPNResult should have non-nil DPNBag after storage!")
		}
		verifySizeAndChecksums(dpnResult, procUtil)
	} else {
		fmt.Println("Packager failed. Skipping storage step.")
		fmt.Println(dpnResult.ErrorMessage)
	}

	// The bag that the packager created should still be on disk.
	// Let's validate it. Make sure we have the path to the tar
	// file, so the validator knows where to look.
	dpnResult.LocalPath = dpnResult.PackageResult.TarFilePath
	fmt.Println("Creating validator...")
	validator, err := dpn.NewValidator(procUtil, dpnConfig)
	if err != nil {
		procUtil.MessageLog.Fatal(err.Error())
	}

	// This will print success or error messages to the console & log.
	// Since this is a local bag, the validator won't try to update
	// a replication request.
	validator.RunTest(dpnResult)

	if dpnResult.ValidationResult == nil {
		procUtil.MessageLog.Error("After validation, bag is missing validation request.")
	} else if dpnResult.ValidationResult.IsValid() == false {
		procUtil.MessageLog.Error("Bag failed validation.")
	}

	// Record the result of the DPN ingest in the local DPN REST
	// service and in Fluctus.
	recorder, err := dpn.NewRecorder(procUtil, dpnConfig)
	if err != nil {
		procUtil.MessageLog.Fatal(err.Error())
	}
	recorder.RunTest(dpnResult)

	// Check record result
	if dpnResult.RecordResult == nil {
		procUtil.MessageLog.Error("Record result is nil")
	} else {
		if dpnResult.RecordResult.DPNBagCreatedAt.IsZero() {
			procUtil.MessageLog.Error("DPNBagCreatedAt was not set")
		}
		if len(dpnResult.RecordResult.DPNReplicationRequests) != 2 {
			procUtil.MessageLog.Error("Expected 2 replication requests, found %d",
				len(dpnResult.RecordResult.DPNReplicationRequests))
		}
		if dpnResult.RecordResult.PremisIngestEventId == "" {
			procUtil.MessageLog.Error("PremisIngestEventId was not set")
		}
		if dpnResult.RecordResult.PremisIdentifierEventId == "" {
			procUtil.MessageLog.Error("PremisIdentifierEventId was not set")
		}
		if dpnResult.RecordResult.ErrorMessage != "" {
			procUtil.MessageLog.Error(dpnResult.RecordResult.ErrorMessage)
		}
	}

	dpnResult.ErrorMessage += "  Nothing wrong. Just testing the trouble processor."
	troubleProcessor := dpn.NewTroubleProcessor(procUtil)
	troubleProcessor.RunTest(dpnResult)

	// Make sure the trouble worker wrote its file
	troubleFile := filepath.Join(dpnConfig.LogDirectory, "dpn_trouble", dpnResult.DPNBag.UUID)
	if !bagman.FileExists(troubleFile) {
		procUtil.MessageLog.Error("Trouble worker did not write JSON file to %s", troubleFile)
	} else {
		procUtil.MessageLog.Debug("Trouble worker successfully wrote file to %s", troubleFile)
	}
}

func getClient(config *dpn.DPNConfig, procUtil *bagman.ProcessUtil) (*dpn.DPNRestClient, error) {
	restClient, err := dpn.NewDPNRestClient(
		config.RestClient.LocalServiceURL,
		config.RestClient.LocalAPIRoot,
		config.RestClient.LocalAuthToken,
		config.LocalNode,
		config,
		procUtil.MessageLog)
	if err != nil {
		return nil, err
	}
	return restClient, nil
}

func createXferRequest(client *dpn.DPNRestClient, uuid, checksum string) (*dpn.DPNReplicationTransfer, error) {
	nonce := "12345"
	xfer := &dpn.DPNReplicationTransfer{
		FromNode: "aptrust",
		ToNode: "aptrust",
		UUID: uuid,
		FixityAlgorithm: "sha256",
		FixityNonce: &nonce,
		FixityValue: &checksum,
		Status: "Requested",
		Protocol: "R",
		Link: "rsync://our/sink.tar",
	}
	savedXfer, err := client.ReplicationTransferCreate(xfer)
	if err != nil {
		return nil, err
	}
	return savedXfer, nil
}

func verifySizeAndChecksums(dpnResult *dpn.DPNResult, procUtil *bagman.ProcessUtil) {
	if dpnResult.BagMd5Digest == "" {
		procUtil.MessageLog.Fatal("Result is missing md5 checksum")
	}
	if dpnResult.BagSha256Digest == "" {
		procUtil.MessageLog.Fatal("Result is missing sha256 checksum")
	}
	if dpnResult.BagSize == 0 {
		procUtil.MessageLog.Fatal("Result is missing bag size")
	}
}
<commit_msg>Fixed build issue in test harness<commit_after>package main

import (
	"fmt"
	"github.com/APTrust/bagman/bagman"
	"github.com/APTrust/bagman/dpn"
	"github.com/APTrust/bagman/workers"
	"path/filepath"
)

const NONCE = "12345"

// dpn_package_test runs a single APTrust bag through the DPN packager.
// This is for ad-hoc dev testing.
func main() {
	// The workers below will share one ProcUtil, which means
	// some of ProcUtil's internal variables, like success count
	// and failure count will be updated by each worker.
	procUtil := workers.CreateProcUtil()

	pathToConfigFile := "dpn/dpn_config.json"
	dpnConfig, err := dpn.LoadConfig(pathToConfigFile, "test")
	if err != nil {
		procUtil.MessageLog.Fatal(err.Error())
	}
	fmt.Println("Creating packager...")
	packager := dpn.NewPackager(procUtil, dpnConfig)
	dpnResult := packager.RunTest("test.edu/ncsu.1840.16-1004")
	if dpnResult.ErrorMessage == "" {
		fmt.Println("Packager succeeded. Moving to storage.")
		storer, err := dpn.NewStorer(procUtil, dpnConfig)
		if err != nil {
			procUtil.MessageLog.Fatal(err.Error())
		}
		storer.RunTest(dpnResult)
		if dpnResult.DPNBag == nil {
			procUtil.MessageLog.Fatal("DPNBag is nil! " +
				"DPNResult should have non-nil DPNBag after storage!")
		}
		verifySizeAndChecksums(dpnResult, procUtil)
	} else {
		fmt.Println("Packager failed. Skipping storage step.")
		fmt.Println(dpnResult.ErrorMessage)
	}

	// The bag that the packager created should still be on disk.
	// Let's validate it. Make sure we have the path to the tar
	// file, so the validator knows where to look.
	dpnResult.LocalPath = dpnResult.PackageResult.TarFilePath
	fmt.Println("Creating validator...")
	validator, err := dpn.NewValidator(procUtil, dpnConfig)
	if err != nil {
		procUtil.MessageLog.Fatal(err.Error())
	}

	// This will print success or error messages to the console & log.
	// Since this is a local bag, the validator won't try to update
	// a replication request.
	validator.RunTest(dpnResult)

	if dpnResult.ValidationResult == nil {
		procUtil.MessageLog.Error("After validation, bag is missing validation request.")
	} else if dpnResult.ValidationResult.IsValid() == false {
		procUtil.MessageLog.Error("Bag failed validation.")
	}

	// Record the result of the DPN ingest in the local DPN REST
	// service and in Fluctus.
	recorder, err := dpn.NewRecorder(procUtil, dpnConfig)
	if err != nil {
		procUtil.MessageLog.Fatal(err.Error())
	}
	recorder.RunTest(dpnResult)

	// Check record result
	if dpnResult.RecordResult == nil {
		procUtil.MessageLog.Error("Record result is nil")
	} else {
		if dpnResult.RecordResult.DPNBagCreatedAt.IsZero() {
			procUtil.MessageLog.Error("DPNBagCreatedAt was not set")
		}
		if len(dpnResult.RecordResult.DPNReplicationRequests) != 2 {
			procUtil.MessageLog.Error("Expected 2 replication requests, found %d",
				len(dpnResult.RecordResult.DPNReplicationRequests))
		}
		if dpnResult.RecordResult.PremisIngestEventId == "" {
			procUtil.MessageLog.Error("PremisIngestEventId was not set")
		}
		if dpnResult.RecordResult.PremisIdentifierEventId == "" {
			procUtil.MessageLog.Error("PremisIdentifierEventId was not set")
		}
		if dpnResult.RecordResult.ErrorMessage != "" {
			procUtil.MessageLog.Error(dpnResult.RecordResult.ErrorMessage)
		}
	}

	dpnResult.ErrorMessage += "  Nothing wrong. Just testing the trouble processor."
	troubleProcessor := dpn.NewTroubleProcessor(procUtil)
	troubleProcessor.RunTest(dpnResult)

	// Make sure the trouble worker wrote its file
	troubleFile := filepath.Join(dpnConfig.LogDirectory, "dpn_trouble", dpnResult.DPNBag.UUID)
	if !bagman.FileExists(troubleFile) {
		procUtil.MessageLog.Error("Trouble worker did not write JSON file to %s", troubleFile)
	} else {
		procUtil.MessageLog.Debug("Trouble worker successfully wrote file to %s", troubleFile)
	}
}

func getClient(config *dpn.DPNConfig, procUtil *bagman.ProcessUtil) (*dpn.DPNRestClient, error) {
	restClient, err := dpn.NewDPNRestClient(
		config.RestClient.LocalServiceURL,
		config.RestClient.LocalAPIRoot,
		config.RestClient.LocalAuthToken,
		config.LocalNode,
		config,
		procUtil.MessageLog)
	if err != nil {
		return nil, err
	}
	return restClient, nil
}

func createXferRequest(client *dpn.DPNRestClient, uuid, checksum string) (*dpn.DPNReplicationTransfer, error) {
	nonce := "12345"
	xfer := &dpn.DPNReplicationTransfer{
		FromNode: "aptrust",
		ToNode: "aptrust",
		BagId: uuid,
		FixityAlgorithm: "sha256",
		FixityNonce: &nonce,
		FixityValue: &checksum,
		Status: "Requested",
		Protocol: "R",
		Link: "rsync://our/sink.tar",
	}
	savedXfer, err := client.ReplicationTransferCreate(xfer)
	if err != nil {
		return nil, err
	}
	return savedXfer, nil
}

func verifySizeAndChecksums(dpnResult *dpn.DPNResult, procUtil *bagman.ProcessUtil) {
	if dpnResult.BagMd5Digest == "" {
		procUtil.MessageLog.Fatal("Result is missing md5 checksum")
	}
	if dpnResult.BagSha256Digest == "" {
		procUtil.MessageLog.Fatal("Result is missing sha256 checksum")
	}
	if dpnResult.BagSize == 0 {
		procUtil.MessageLog.Fatal("Result is missing bag size")
	}
}
<|endoftext|>


<commit_before>package user

import (
	"encoding/json"
	"net/http"

	"github.com/pressly/chi"
)

func NewHandler(s Service) *chi.Mux {
	r := chi.NewRouter()

	r.Get("/", list(s))
	r.Get("/:username", get(s))

	return r
}

func list(s Service) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		users, err := s.FindAll()
		if err != nil {
			return // TODO
		}

		data, err := json.Marshal(users)
		if err != nil {
			return // TODO
		}

		w.Write(data)
	}
}

func get(s Service) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		username := chi.URLParam(r, "username")

		user, err := s.FindByUsername(Username(username))
		if err != nil {
			return // TODO
		}

		data, err := json.Marshal(user)
		if err != nil {
			return // TODO
		}
		w.Write(data)
	}
}
<commit_msg>Add a update handler to http endpoint for a user<commit_after>package user

import (
	"encoding/json"
	"net/http"

	"github.com/pressly/chi"
)

func NewHandler(s Service) *chi.Mux {
	r := chi.NewRouter()

	r.Get("/", list(s))
	r.Get("/:username", get(s))
	r.Put("/:username", update(s))

	return r
}

func list(s Service) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		users, err := s.FindAll()
		if err != nil {
			return // TODO
		}

		data, err := json.Marshal(users)
		if err != nil {
			return // TODO
		}

		w.Write(data)
	}
}

func get(s Service) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		username := chi.URLParam(r, "username")

		user, err := s.FindByUsername(username)
		if err != nil {
			return // TODO
		}

		data, err := json.Marshal(user)
		if err != nil {
			return // TODO
		}
		w.Write(data)
	}
}

func update(s Service) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		username := chi.URLParam(r, "username")

		var user *User
		if err := json.NewDecoder(r.Body).Decode(&user); err != nil {
			return // TODO
		}

		user, err := s.Update(username, user)
		if err != nil {
			return // TODO
		}

		data, err := json.Marshal(user)
		if err != nil {
			return // TODO
		}

		w.Write(data)
	}
}
<|endoftext|>