<commit_before>package workers

import (
	"encoding/json"
	"fmt"
	"github.com/APTrust/exchange/constants"
	"github.com/APTrust/exchange/context"
	"github.com/APTrust/exchange/dpn/models"
	"github.com/APTrust/exchange/dpn/network"
	apt_models "github.com/APTrust/exchange/models"
	"net/url"
	"strconv"
	"time"
)

type DPNQueue struct {
	// LocalClient is the DPN REST client that talks to our own
	// local DPN REST server.
	LocalClient    *network.DPNRestClient
	// RemoteNodes is a map of remote nodes. Key is the namespace
	// and value is the node.
	RemoteNodes    map[string]*models.Node
	// RemoteClients is a collection of clients that talk to the
	// DPN REST servers on other nodes. The key is the namespace
	// of the remote node, and the value is the client that talks
	// to that node.
	RemoteClients   map[string]*network.DPNRestClient
	// Context provides access to information about our environment
	// and config settings, and access to basic services like
	// logging and a Pharos client.
	Context         *context.Context
	// ExamineItemsSince is a timestamp. We will examine any items
	// updated since this timestamp to see if they need to be queued.
	ExamineItemsSince time.Time
	// QueueResult contains information about which items were
	// queued during this run of the program.
	QueueResult      *models.QueueResult
}

// NewDPNQueue creates a new DPNQueue object. Param _context is a Context
// object, and param hours tells the code to examine all Replication,
// Restore and DPN Ingest requests from the past N hours.
func NewDPNQueue(_context *context.Context, hours int) (*DPNQueue, error) {
	if _context == nil {
		return nil, fmt.Errorf("Param _context cannot be nil.")
	}
	localClient, err := network.NewDPNRestClient(
		_context.Config.DPN.RestClient.LocalServiceURL,
		_context.Config.DPN.DPNAPIVersion,
		_context.Config.DPN.RestClient.LocalAuthToken,
		_context.Config.DPN.LocalNode,
		_context.Config.DPN)
	if err != nil {
		return nil, fmt.Errorf("Error creating local DPN REST client: %v", err)
	}
	remoteClients, err := localClient.GetRemoteClients()
	if err != nil {
		return nil, fmt.Errorf("Error creating remote DPN REST client: %v", err)
	}
	sinceWhen := time.Now().UTC().Add(time.Duration(-1 * hours) * time.Hour)
	_context.MessageLog.Info("Checking records since %d hours ago (%s)",
		hours, sinceWhen.Format(time.RFC3339))
	dpnQueue := DPNQueue{
		LocalClient: localClient,
		RemoteNodes: make(map[string]*models.Node),
		RemoteClients: remoteClients,
		Context: _context,
		ExamineItemsSince: sinceWhen,
		QueueResult: models.NewQueueResult(),
	}
	return &dpnQueue, nil
}

// Run checks for ReplicationTransfers, RestoreTransfers and IngestRequests
// that need to be queued. It creates DPNWorkItems and NSQ entries for each
// request that needs to be queued.
func (dpnQueue *DPNQueue) Run() {
	dpnQueue.QueueResult.StartTime = time.Now().UTC()
	dpnQueue.queueReplicationRequests()
	dpnQueue.queueRestoreRequests()
	dpnQueue.queueIngestRequests()
	dpnQueue.QueueResult.EndTime = time.Now().UTC()
	dpnQueue.logJsonResults()
}

/***************************************************************************
 ReplicationTransfer Methods
***************************************************************************/

// queueReplicationRequests collects ReplicationTransfer requests from
// the local DPN server and if necessary 1) creates a DPNWorkItem record
// in our Pharos server for the replication request, and 2) creates an
// entry in NSQ telling our replication workers to copy the bag.
//
// We query our local DPN node after synching data from other nodes, and
// we're looking for ReplicationTransfers where the to_node is our node.
// We want to skip transfers that are cancelled or already stored. We
// also want to skip transfers where store_requested is true, because
// those transfers are already in progress. (The remote node sets
// store_requested to true only after it sees that the replicating node
// has returned a valid fixity value for the bag's tag manifest.)
func (dpnQueue *DPNQueue) queueReplicationRequests() {
	pageNumber := 1
	params := dpnQueue.replicationParams(pageNumber)
	for {
		dpnResp := dpnQueue.LocalClient.ReplicationTransferList(params)
		if dpnResp.Error != nil {
			dpnQueue.err("Error getting ReplicationTransfers from local node: %v", dpnResp.Error)
			break
		}
		xfers := dpnResp.ReplicationTransfers()
		for _, xfer := range xfers {
			queueItem := models.NewQueueItem(xfer.ReplicationId)
			dpnWorkItem := dpnQueue.getOrCreateWorkItem(xfer.ReplicationId, constants.DPNTaskReplication)
			queueItem.ItemId = dpnWorkItem.Id
			if dpnWorkItem.QueuedAt.IsZero() {
				dpnQueue.queueReplication(dpnWorkItem, xfer)
			}
			queueItem.QueuedAt = *dpnWorkItem.QueuedAt
			dpnQueue.QueueResult.AddReplication(queueItem)
		}
		if dpnResp.Next == nil {
			break
		} else {
			pageNumber += 1
			params = dpnQueue.replicationParams(pageNumber)
		}
	}
}

// replicationParams returns the URL parameters we need to query our local
// DPN REST server for ReplicationTransfer requests that we will need to
// service.
func (dpnQueue *DPNQueue) replicationParams(pageNumber int) (url.Values) {
	params := url.Values{}
	params.Set("after", dpnQueue.ExamineItemsSince.Format(time.RFC3339))
	params.Set("to_node", dpnQueue.Context.Config.DPN.LocalNode)
	params.Set("cancelled", "false")
	params.Set("stored", "false")
	params.Set("store_requested", "false")
	params.Set("order_by", "updated_at")
	params.Set("page_size", "100")
	params.Set("page", strconv.Itoa(pageNumber))
	return params
}

// queueReplication adds a ReplicationTransfer to NSQ and records info about
// when the item was queued in DPNWorkItem.QueuedAt, which is saved to Pharos.
func (dpnQueue *DPNQueue) queueReplication(dpnWorkItem *apt_models.DPNWorkItem, xfer *models.ReplicationTransfer) {
	err := dpnQueue.Context.NSQClient.Enqueue(
		dpnQueue.Context.Config.DPN.DPNCopyWorker.NsqTopic,
		dpnWorkItem.Id)
	if err != nil {
		dpnQueue.err("Error getting DPNWorkItemList from Pharos: %v", err)
	} else {
		*dpnWorkItem.QueuedAt = time.Now().UTC()
		resp := dpnQueue.Context.PharosClient.DPNWorkItemSave(dpnWorkItem)
		if resp.Error != nil {
			dpnQueue.err("Error updating DPNWorkItem for ReplicationXfer %s from %s: %v",
				xfer.ReplicationId, xfer.FromNode, resp.Error)
			return
		}
		dpnWorkItem = resp.DPNWorkItem()
	}
}

/***************************************************************************
 RestoreTransfer Methods
***************************************************************************/

// queueRestoreRequests collects RestoreTransfer requests from
// the local DPN server and if necessary 1) creates a DPNWorkItem record
// in our Pharos server for the replication request, and 2) creates an
// entry in NSQ telling our replication workers to copy the bag.
//
// We query our local DPN node after synching data from other nodes, and
// we're looking for RestoreTransfers where the from_node is our node.
// We want to skip transfers that are cancelled or already finished. We
// also want to skip transfers where accepted is true, because
// those transfers are already in progress.
func (dpnQueue *DPNQueue) queueRestoreRequests() {
	pageNumber := 1
	params := dpnQueue.restoreParams(pageNumber)
	for {
		dpnResp := dpnQueue.LocalClient.RestoreTransferList(params)
		if dpnResp.Error != nil {
			dpnQueue.err("Error getting RestoreTransfers from local node: %v", dpnResp.Error)
			break
		}
		xfers := dpnResp.RestoreTransfers()
		for _, xfer := range xfers {
			queueItem := models.NewQueueItem(xfer.RestoreId)
			dpnWorkItem := dpnQueue.getOrCreateWorkItem(xfer.RestoreId, constants.DPNTaskRestore)
			queueItem.ItemId = dpnWorkItem.Id
			if dpnWorkItem.QueuedAt.IsZero() {
				dpnQueue.queueRestore(dpnWorkItem, xfer)
			}
			queueItem.QueuedAt = *dpnWorkItem.QueuedAt
			dpnQueue.QueueResult.AddRestore(queueItem)
		}
		if dpnResp.Next == nil {
			break
		} else {
			pageNumber += 1
			params = dpnQueue.restoreParams(pageNumber)
		}
	}
}

// restoreParams returns the URL parameters we need to query our local
// DPN REST server for RestoreTransfer requests that we will need to
// service.
func (dpnQueue *DPNQueue) restoreParams(pageNumber int) (url.Values) {
	params := url.Values{}
	params.Set("after", dpnQueue.ExamineItemsSince.Format(time.RFC3339))
	params.Set("from_node", dpnQueue.Context.Config.DPN.LocalNode)
	params.Set("cancelled", "false")
	params.Set("finished", "false")
	params.Set("accepted", "false")
	params.Set("order_by", "updated_at")
	params.Set("page_size", "100")
	params.Set("page", strconv.Itoa(pageNumber))
	return params
}

// queueRestore adds a RestoreTransfer to NSQ and records info about
// when the item was queued in DPNWorkItem.QueuedAt, which is saved to Pharos.
func (dpnQueue *DPNQueue) queueRestore(dpnWorkItem *apt_models.DPNWorkItem, xfer *models.RestoreTransfer) {
	err := dpnQueue.Context.NSQClient.Enqueue(
		dpnQueue.Context.Config.DPN.DPNRestoreWorker.NsqTopic,
		dpnWorkItem.Id)
	if err != nil {
		dpnQueue.err("Error getting DPNWorkItemList from Pharos: %v", err)
	} else {
		*dpnWorkItem.QueuedAt = time.Now().UTC()
		resp := dpnQueue.Context.PharosClient.DPNWorkItemSave(dpnWorkItem)
		if resp.Error != nil {
			dpnQueue.err("Error updating DPNWorkItem for RestoreXfer %s to %s: %v",
				xfer.RestoreId, xfer.ToNode, resp.Error)
			return
		}
		dpnWorkItem = resp.DPNWorkItem()
	}
}

/***************************************************************************
 Ingest Methods
***************************************************************************/

// queueIngestRequests queues ingest requests that are sitting in Pharos'
// WorkItems table. Users request DPN ingest directly in Pharos, and that's
// the only place those requests exist. (You won't find them in the DPN REST
// server.) We want to queue DPN ingest requests that have no QueuedAt timestamp.
// The DPN REST server will have no record of these items until we finish
// ingesting them into DPN.
func (dpnQueue *DPNQueue) queueIngestRequests() {

}

// ingestParams returns the URL params we need to query Pharos about
// unserviced DPN Ingest requests. These are WorkItems where the action
// is "DPN" and the queued_at timestamp is empty.
func (dpnQueue *DPNQueue) ingestParams(pageNumber int) (url.Values) {
	return nil
}

// queueIngest adds the id of a DPN Ingest WorkItem to NSQ and records info about
// when the item was queued in WorkItem.QueuedAt, which is saved to Pharos.
func (dpnQueue *DPNQueue) queueIngest(workItem *apt_models.WorkItem) {

}

/***************************************************************************
 Misc Utility Methods
***************************************************************************/

// getOrCreateWorkItem returns the DPNWorkItem for the specified
// replication/restore transfer from Pharos. If no DPNWorkItem for the specified
// transfer exists, this creates it in Pharos and returns a copy of it.
func (dpnQueue *DPNQueue) getOrCreateWorkItem(identifier, taskType string) (*apt_models.DPNWorkItem) {
	params := url.Values{}
	params.Set("identifier", identifier)
	params.Set("task", taskType)
	getResp := dpnQueue.Context.PharosClient.DPNWorkItemList(params)
	if getResp.Error != nil {
		dpnQueue.err("Error getting DPNWorkItemList from Pharos: %v", getResp.Error)
		return nil
	}
	existingItem := getResp.DPNWorkItem()
	if existingItem != nil {
		return existingItem
	} else {
		dpnWorkItem := &apt_models.DPNWorkItem{
			Task: taskType,
			Identifier: identifier,
		}
		createResp := dpnQueue.Context.PharosClient.DPNWorkItemSave(dpnWorkItem)
		if createResp.Error != nil {
			dpnQueue.err("Error creating DPNWorkItem for %s Xfer %s: %v",
				taskType, identifier, getResp.Error)
			return nil
		}
		newItem := createResp.DPNWorkItem()
		if newItem == nil {
			dpnQueue.err("DPNWorkItemSave returned nil for %s Xfer %s: %v",
				taskType, identifier, getResp.Error)
		}
		return newItem
	}
}

// err logs an error and adds it to the QueueResult.Errors list.
func (dpnQueue *DPNQueue) err(format string, a ...interface{}) {
	errMsg := fmt.Sprintf(format, a...)
	dpnQueue.Context.MessageLog.Error(errMsg)
	dpnQueue.QueueResult.AddError(errMsg)
}

// logJsonResults dumps the results of this queue run to a machine-readable
// JSON file. This is used in integration post tests to verify that certain
// items were processed, and can be used in production to run automated
// audits and spot checks.
func (dpnQueue *DPNQueue) logJsonResults() {
	jsonData, err := json.MarshalIndent(dpnQueue.QueueResult, "", "  ")
	if err != nil {
		dpnQueue.Context.MessageLog.Warning("Could not dump QueueResult to JSON log: %v", err)
	} else {
		dpnQueue.Context.JsonLog.Println(jsonData)
	}
}
<commit_msg>More work on dpn_queue<commit_after>package workers

import (
	"encoding/json"
	"fmt"
	"github.com/APTrust/exchange/constants"
	"github.com/APTrust/exchange/context"
	"github.com/APTrust/exchange/dpn/models"
	"github.com/APTrust/exchange/dpn/network"
	apt_models "github.com/APTrust/exchange/models"
	"net/url"
	"strconv"
	"time"
)

type DPNQueue struct {
	// LocalClient is the DPN REST client that talks to our own
	// local DPN REST server.
	LocalClient    *network.DPNRestClient
	// RemoteNodes is a map of remote nodes. Key is the namespace
	// and value is the node.
	RemoteNodes    map[string]*models.Node
	// RemoteClients is a collection of clients that talk to the
	// DPN REST servers on other nodes. The key is the namespace
	// of the remote node, and the value is the client that talks
	// to that node.
	RemoteClients   map[string]*network.DPNRestClient
	// Context provides access to information about our environment
	// and config settings, and access to basic services like
	// logging and a Pharos client.
	Context         *context.Context
	// ExamineItemsSince is a timestamp. We will examine any items
	// updated since this timestamp to see if they need to be queued.
	ExamineItemsSince time.Time
	// QueueResult contains information about which items were
	// queued during this run of the program.
	QueueResult      *models.QueueResult
}

// NewDPNQueue creates a new DPNQueue object. Param _context is a Context
// object, and param hours tells the code to examine all Replication,
// Restore and DPN Ingest requests from the past N hours.
func NewDPNQueue(_context *context.Context, hours int) (*DPNQueue, error) {
	if _context == nil {
		return nil, fmt.Errorf("Param _context cannot be nil.")
	}
	localClient, err := network.NewDPNRestClient(
		_context.Config.DPN.RestClient.LocalServiceURL,
		_context.Config.DPN.DPNAPIVersion,
		_context.Config.DPN.RestClient.LocalAuthToken,
		_context.Config.DPN.LocalNode,
		_context.Config.DPN)
	if err != nil {
		return nil, fmt.Errorf("Error creating local DPN REST client: %v", err)
	}
	remoteClients, err := localClient.GetRemoteClients()
	if err != nil {
		return nil, fmt.Errorf("Error creating remote DPN REST client: %v", err)
	}
	sinceWhen := time.Now().UTC().Add(time.Duration(-1 * hours) * time.Hour)
	_context.MessageLog.Info("Checking records since %d hours ago (%s)",
		hours, sinceWhen.Format(time.RFC3339))
	dpnQueue := DPNQueue{
		LocalClient: localClient,
		RemoteNodes: make(map[string]*models.Node),
		RemoteClients: remoteClients,
		Context: _context,
		ExamineItemsSince: sinceWhen,
		QueueResult: models.NewQueueResult(),
	}
	return &dpnQueue, nil
}

// Run checks for ReplicationTransfers, RestoreTransfers and IngestRequests
// that need to be queued. It creates DPNWorkItems and NSQ entries for each
// request that needs to be queued.
func (dpnQueue *DPNQueue) Run() {
	dpnQueue.QueueResult.StartTime = time.Now().UTC()
	dpnQueue.queueReplicationRequests()
	dpnQueue.queueRestoreRequests()
	dpnQueue.queueIngestRequests()
	dpnQueue.QueueResult.EndTime = time.Now().UTC()
	dpnQueue.logJsonResults()
}

/***************************************************************************
 ReplicationTransfer Methods
***************************************************************************/

// queueReplicationRequests collects ReplicationTransfer requests from
// the local DPN server and if necessary 1) creates a DPNWorkItem record
// in our Pharos server for the replication request, and 2) creates an
// entry in NSQ telling our replication workers to copy the bag.
//
// We query our local DPN node after synching data from other nodes, and
// we're looking for ReplicationTransfers where the to_node is our node.
// We want to skip transfers that are cancelled or already stored. We
// also want to skip transfers where store_requested is true, because
// those transfers are already in progress. (The remote node sets
// store_requested to true only after it sees that the replicating node
// has returned a valid fixity value for the bag's tag manifest.)
func (dpnQueue *DPNQueue) queueReplicationRequests() {
	pageNumber := 1
	params := dpnQueue.replicationParams(pageNumber)
	for {
		dpnResp := dpnQueue.LocalClient.ReplicationTransferList(params)
		if dpnResp.Error != nil {
			dpnQueue.err("Error getting ReplicationTransfers from local node: %v", dpnResp.Error)
			break
		}
		xfers := dpnResp.ReplicationTransfers()
		for _, xfer := range xfers {
			queueItem := models.NewQueueItem(xfer.ReplicationId)
			dpnWorkItem := dpnQueue.getOrCreateWorkItem(xfer.ReplicationId, constants.DPNTaskReplication)
			queueItem.ItemId = dpnWorkItem.Id
			if dpnWorkItem.QueuedAt.IsZero() {
				dpnQueue.queueTransfer(dpnWorkItem, constants.DPNTaskReplication)
			}
			queueItem.QueuedAt = *dpnWorkItem.QueuedAt
			dpnQueue.QueueResult.AddReplication(queueItem)
		}
		if dpnResp.Next == nil {
			break
		} else {
			pageNumber += 1
			params = dpnQueue.replicationParams(pageNumber)
		}
	}
}

// replicationParams returns the URL parameters we need to query our local
// DPN REST server for ReplicationTransfer requests that we will need to
// service.
func (dpnQueue *DPNQueue) replicationParams(pageNumber int) (url.Values) {
	params := url.Values{}
	params.Set("after", dpnQueue.ExamineItemsSince.Format(time.RFC3339))
	params.Set("to_node", dpnQueue.Context.Config.DPN.LocalNode)
	params.Set("cancelled", "false")
	params.Set("stored", "false")
	params.Set("store_requested", "false")
	params.Set("order_by", "updated_at")
	params.Set("page_size", "100")
	params.Set("page", strconv.Itoa(pageNumber))
	return params
}

/***************************************************************************
 RestoreTransfer Methods
***************************************************************************/

// queueRestoreRequests collects RestoreTransfer requests from
// the local DPN server and if necessary 1) creates a DPNWorkItem record
// in our Pharos server for the replication request, and 2) creates an
// entry in NSQ telling our replication workers to copy the bag.
//
// We query our local DPN node after synching data from other nodes, and
// we're looking for RestoreTransfers where the from_node is our node.
// We want to skip transfers that are cancelled or already finished. We
// also want to skip transfers where accepted is true, because
// those transfers are already in progress.
func (dpnQueue *DPNQueue) queueRestoreRequests() {
	pageNumber := 1
	params := dpnQueue.restoreParams(pageNumber)
	for {
		dpnResp := dpnQueue.LocalClient.RestoreTransferList(params)
		if dpnResp.Error != nil {
			dpnQueue.err("Error getting RestoreTransfers from local node: %v", dpnResp.Error)
			break
		}
		xfers := dpnResp.RestoreTransfers()
		for _, xfer := range xfers {
			queueItem := models.NewQueueItem(xfer.RestoreId)
			dpnWorkItem := dpnQueue.getOrCreateWorkItem(xfer.RestoreId, constants.DPNTaskRestore)
			queueItem.ItemId = dpnWorkItem.Id
			if dpnWorkItem.QueuedAt.IsZero() {
				dpnQueue.queueTransfer(dpnWorkItem, constants.DPNTaskRestore)
			}
			queueItem.QueuedAt = *dpnWorkItem.QueuedAt
			dpnQueue.QueueResult.AddRestore(queueItem)
		}
		if dpnResp.Next == nil {
			break
		} else {
			pageNumber += 1
			params = dpnQueue.restoreParams(pageNumber)
		}
	}
}

// restoreParams returns the URL parameters we need to query our local
// DPN REST server for RestoreTransfer requests that we will need to
// service.
func (dpnQueue *DPNQueue) restoreParams(pageNumber int) (url.Values) {
	params := url.Values{}
	params.Set("after", dpnQueue.ExamineItemsSince.Format(time.RFC3339))
	params.Set("from_node", dpnQueue.Context.Config.DPN.LocalNode)
	params.Set("cancelled", "false")
	params.Set("finished", "false")
	params.Set("accepted", "false")
	params.Set("order_by", "updated_at")
	params.Set("page_size", "100")
	params.Set("page", strconv.Itoa(pageNumber))
	return params
}

/***************************************************************************
 Ingest Methods
***************************************************************************/

// queueIngestRequests queues ingest requests that are sitting in Pharos'
// WorkItems table. Users request DPN ingest directly in Pharos, and that's
// the only place those requests exist. (You won't find them in the DPN REST
// server.) We want to queue DPN ingest requests that have no QueuedAt timestamp.
// The DPN REST server will have no record of these items until we finish
// ingesting them into DPN.
func (dpnQueue *DPNQueue) queueIngestRequests() {

}

// ingestParams returns the URL params we need to query Pharos about
// unserviced DPN Ingest requests. These are WorkItems where the action
// is "DPN" and the queued_at timestamp is empty.
func (dpnQueue *DPNQueue) ingestParams(pageNumber int) (url.Values) {
	return nil
}

// queueIngest adds the id of a DPN Ingest WorkItem to NSQ and records info about
// when the item was queued in WorkItem.QueuedAt, which is saved to Pharos.
func (dpnQueue *DPNQueue) queueIngest(workItem *apt_models.WorkItem) {

}

/***************************************************************************
 Misc Utility Methods
***************************************************************************/

// queueTransfer adds a transfer task to NSQ and records info about
// when the item was queued in DPNWorkItem.QueuedAt, which is saved to Pharos.
func (dpnQueue *DPNQueue) queueTransfer(dpnWorkItem *apt_models.DPNWorkItem, taskType string) {
	queueTopic := ""
	if taskType == constants.DPNTaskReplication {
		// Copy is first step of replication
		queueTopic = dpnQueue.Context.Config.DPN.DPNCopyWorker.NsqTopic
	} else if taskType == constants.DPNTaskRestore {
		queueTopic = dpnQueue.Context.Config.DPN.DPNRestoreWorker.NsqTopic
	} else {
		dpnQueue.Context.MessageLog.Error("Illegal taskType '%s'", taskType)
		return
	}
	// Put the item into NSQ
	err := dpnQueue.Context.NSQClient.Enqueue(queueTopic, dpnWorkItem.Id)
	if err != nil {
		dpnQueue.err("Error queueing DPNWorkItem %d, %s %s: %v",
			dpnWorkItem.Id, taskType, dpnWorkItem.Identifier, err)
	} else {
		// Let Pharos know this item has been queued
		*dpnWorkItem.QueuedAt = time.Now().UTC()
		resp := dpnQueue.Context.PharosClient.DPNWorkItemSave(dpnWorkItem)
		if resp.Error != nil {
			dpnQueue.err("Error updating DPNWorkItem %d for %s %s to %s: %v",
				dpnWorkItem.Id, taskType, dpnWorkItem.Identifier, resp.Error)
			return
		}
		dpnWorkItem = resp.DPNWorkItem()
	}
}

// getOrCreateWorkItem returns the DPNWorkItem for the specified
// replication/restore transfer from Pharos. If no DPNWorkItem for the specified
// transfer exists, this creates it in Pharos and returns a copy of it.
func (dpnQueue *DPNQueue) getOrCreateWorkItem(identifier, taskType string) (*apt_models.DPNWorkItem) {
	params := url.Values{}
	params.Set("identifier", identifier)
	params.Set("task", taskType)
	getResp := dpnQueue.Context.PharosClient.DPNWorkItemList(params)
	if getResp.Error != nil {
		dpnQueue.err("Error getting DPNWorkItemList from Pharos: %v", getResp.Error)
		return nil
	}
	existingItem := getResp.DPNWorkItem()
	if existingItem != nil {
		return existingItem
	} else {
		dpnWorkItem := &apt_models.DPNWorkItem{
			Task: taskType,
			Identifier: identifier,
		}
		createResp := dpnQueue.Context.PharosClient.DPNWorkItemSave(dpnWorkItem)
		if createResp.Error != nil {
			dpnQueue.err("Error creating DPNWorkItem for %s Xfer %s: %v",
				taskType, identifier, getResp.Error)
			return nil
		}
		newItem := createResp.DPNWorkItem()
		if newItem == nil {
			dpnQueue.err("DPNWorkItemSave returned nil for %s Xfer %s: %v",
				taskType, identifier, getResp.Error)
		}
		return newItem
	}
}

// err logs an error and adds it to the QueueResult.Errors list.
func (dpnQueue *DPNQueue) err(format string, a ...interface{}) {
	errMsg := fmt.Sprintf(format, a...)
	dpnQueue.Context.MessageLog.Error(errMsg)
	dpnQueue.QueueResult.AddError(errMsg)
}

// logJsonResults dumps the results of this queue run to a machine-readable
// JSON file. This is used in integration post tests to verify that certain
// items were processed, and can be used in production to run automated
// audits and spot checks.
func (dpnQueue *DPNQueue) logJsonResults() {
	jsonData, err := json.MarshalIndent(dpnQueue.QueueResult, "", "  ")
	if err != nil {
		dpnQueue.Context.MessageLog.Warning("Could not dump QueueResult to JSON log: %v", err)
	} else {
		dpnQueue.Context.JsonLog.Println(jsonData)
	}
}
<|endoftext|>


<commit_before>package dlog

import (
	"bufio"
	"bytes"
	"fmt"
	"os"
	"runtime"
	"strconv"
	"strings"
	"time"
)

type logger struct {
	byt         *bytes.Buffer
	debugLevel  Level
	debugFilter string
	writer      *bufio.Writer
}

// NewLogger returns an instance of the default logger with no filter,
// no file, and level set to ERROR
func NewLogger() Logger {
	return &logger{
		byt:        bytes.NewBuffer(make([]byte, 0)),
		debugLevel: ERROR,
	}
}

// GetLogLevel returns the current log level, i.e WARN or INFO...
func (l *logger) GetLogLevel() Level {
	return l.debugLevel
}

// dLog, the primary function of the package,
// prints out and writes to file a string
// containing the logged data separated by spaces,
// prepended with file and line information.
// It only includes logs which pass the current filters.
// Todo: use io.Multiwriter to simplify the writing to
// both logfiles and stdout
func (l *logger) dLog(console, override bool, in ...interface{}) {
	//(pc uintptr, file string, line int, ok bool)
	_, f, line, ok := runtime.Caller(2)
	if strings.Contains(f, "dlog") {
		_, f, line, ok = runtime.Caller(3)
	}
	if ok {
		f = truncateFileName(f)
		if !l.checkFilter(f, in) && !override {
			return
		}

		// Note on errors: these functions all return
		// errors, but they are always nil.
		l.byt.WriteRune('[')
		l.byt.WriteString(f)
		l.byt.WriteRune(':')
		l.byt.WriteString(strconv.Itoa(line))
		l.byt.WriteString("]  ")
		l.byt.WriteString(logLevels[l.GetLogLevel()])
		l.byt.WriteRune(':')
		for _, elem := range in {
			l.byt.WriteString(fmt.Sprintf("%v ", elem))
		}
		l.byt.WriteRune('\n')

		if console {
			fmt.Print(l.byt.String())
		}

		if l.writer != nil {
			_, err := l.writer.WriteString(l.byt.String())
			if err != nil {
				// We can't log errors while we are in the error
				// logging function.
				fmt.Println("Logging error", err)
			}
			err = l.writer.Flush()
			if err != nil {
				fmt.Println("Logging error", err)
			}
		}

		l.byt.Reset()
	}
}

// FileWrite runs dLog, but JUST writes to file instead
// of also to stdout.
func (l *logger) FileWrite(in ...interface{}) {
	l.dLog(false, true, in...)
}

func truncateFileName(f string) string {
	index := strings.LastIndex(f, "/")
	lIndex := strings.LastIndex(f, ".")
	return f[index+1 : lIndex]
}

func (l *logger) checkFilter(f string, in ...interface{}) bool {
	ret := false
	for _, elem := range in {
		ret = ret || strings.Contains(fmt.Sprintf("%s", elem), l.debugFilter)
	}
	return ret || strings.Contains(f, l.debugFilter)
}

// SetDebugFilter sets the string which determines
// what debug messages get printed. Only messages
// which contain the filer as a pseudo-regex
func (l *logger) SetDebugFilter(filter string) {
	l.debugFilter = filter
}

// SetDebugLevel sets what message levels of debug
// will be printed.
func (l *logger) SetDebugLevel(dL Level) {
	if dL < NONE || dL > VERBOSE {
		Warn("Unknown debug level: ", dL)
		l.debugLevel = NONE
	} else {
		l.debugLevel = dL
	}
}

// CreateLogFile creates a file in the 'logs' directory
// of the starting point of this program to write logs to
func (l *logger) CreateLogFile() {
	file := "logs/dlog"
	file += time.Now().Format("_Jan_2_15-04-05_2006")
	file += ".txt"
	fHandle, err := os.Create(file)
	if err != nil {
		// We can't log an error that comes from
		// our error logging functions
		//panic(err)
		// But this is also not an error we want to panic on!
		fmt.Println("[oak]-------- No logs directory found. No logs will be written to file.")
		return
	}
	l.writer = bufio.NewWriter(fHandle)
}

// Error will write a dlog if the debug level is not NONE
func (l *logger) Error(in ...interface{}) {
	if l.debugLevel > NONE {
		l.dLog(true, true, in)
	}
}

// Warn will write a dLog if the debug level is higher than ERROR
func (l *logger) Warn(in ...interface{}) {
	if l.debugLevel > ERROR {
		l.dLog(true, true, in)
	}
}

// Info will write a dLog if the debug level is higher than WARN
func (l *logger) Info(in ...interface{}) {
	if l.debugLevel > WARN {
		l.dLog(true, false, in)
	}
}

// Verb will write a dLog if the debug level is higher than INFO
func (l *logger) Verb(in ...interface{}) {
	if l.debugLevel > INFO {
		l.dLog(true, false, in)
	}
}
<commit_msg>Remove logging error prints<commit_after>package dlog

import (
	"bufio"
	"bytes"
	"fmt"
	"os"
	"runtime"
	"strconv"
	"strings"
	"time"
)

type logger struct {
	byt         *bytes.Buffer
	debugLevel  Level
	debugFilter string
	writer      *bufio.Writer
}

// NewLogger returns an instance of the default logger with no filter,
// no file, and level set to ERROR
func NewLogger() Logger {
	return &logger{
		byt:        bytes.NewBuffer(make([]byte, 0)),
		debugLevel: ERROR,
	}
}

// GetLogLevel returns the current log level, i.e WARN or INFO...
func (l *logger) GetLogLevel() Level {
	return l.debugLevel
}

// dLog, the primary function of the package,
// prints out and writes to file a string
// containing the logged data separated by spaces,
// prepended with file and line information.
// It only includes logs which pass the current filters.
// Todo: use io.Multiwriter to simplify the writing to
// both logfiles and stdout
func (l *logger) dLog(console, override bool, in ...interface{}) {
	//(pc uintptr, file string, line int, ok bool)
	_, f, line, ok := runtime.Caller(2)
	if strings.Contains(f, "dlog") {
		_, f, line, ok = runtime.Caller(3)
	}
	if ok {
		f = truncateFileName(f)
		if !l.checkFilter(f, in) && !override {
			return
		}

		// Note on errors: these functions all return
		// errors, but they are always nil.
		l.byt.WriteRune('[')
		l.byt.WriteString(f)
		l.byt.WriteRune(':')
		l.byt.WriteString(strconv.Itoa(line))
		l.byt.WriteString("]  ")
		l.byt.WriteString(logLevels[l.GetLogLevel()])
		l.byt.WriteRune(':')
		for _, elem := range in {
			l.byt.WriteString(fmt.Sprintf("%v ", elem))
		}
		l.byt.WriteRune('\n')

		if console {
			fmt.Print(l.byt.String())
		}

		if l.writer != nil {
			l.writer.WriteString(l.byt.String())
			l.writer.Flush()
		}

		l.byt.Reset()
	}
}

// FileWrite runs dLog, but JUST writes to file instead
// of also to stdout.
func (l *logger) FileWrite(in ...interface{}) {
	l.dLog(false, true, in...)
}

func truncateFileName(f string) string {
	index := strings.LastIndex(f, "/")
	lIndex := strings.LastIndex(f, ".")
	return f[index+1 : lIndex]
}

func (l *logger) checkFilter(f string, in ...interface{}) bool {
	ret := false
	for _, elem := range in {
		ret = ret || strings.Contains(fmt.Sprintf("%s", elem), l.debugFilter)
	}
	return ret || strings.Contains(f, l.debugFilter)
}

// SetDebugFilter sets the string which determines
// what debug messages get printed. Only messages
// which contain the filer as a pseudo-regex
func (l *logger) SetDebugFilter(filter string) {
	l.debugFilter = filter
}

// SetDebugLevel sets what message levels of debug
// will be printed.
func (l *logger) SetDebugLevel(dL Level) {
	if dL < NONE || dL > VERBOSE {
		Warn("Unknown debug level: ", dL)
		l.debugLevel = NONE
	} else {
		l.debugLevel = dL
	}
}

// CreateLogFile creates a file in the 'logs' directory
// of the starting point of this program to write logs to
func (l *logger) CreateLogFile() {
	file := "logs/dlog"
	file += time.Now().Format("_Jan_2_15-04-05_2006")
	file += ".txt"
	fHandle, err := os.Create(file)
	if err != nil {
		// We can't log an error that comes from
		// our error logging functions
		//panic(err)
		// But this is also not an error we want to panic on!
		fmt.Println("[oak]-------- No logs directory found. No logs will be written to file.")
		return
	}
	l.writer = bufio.NewWriter(fHandle)
}

// Error will write a dlog if the debug level is not NONE
func (l *logger) Error(in ...interface{}) {
	if l.debugLevel > NONE {
		l.dLog(true, true, in)
	}
}

// Warn will write a dLog if the debug level is higher than ERROR
func (l *logger) Warn(in ...interface{}) {
	if l.debugLevel > ERROR {
		l.dLog(true, true, in)
	}
}

// Info will write a dLog if the debug level is higher than WARN
func (l *logger) Info(in ...interface{}) {
	if l.debugLevel > WARN {
		l.dLog(true, false, in)
	}
}

// Verb will write a dLog if the debug level is higher than INFO
func (l *logger) Verb(in ...interface{}) {
	if l.debugLevel > INFO {
		l.dLog(true, false, in)
	}
}
<|endoftext|>


<commit_before>// Copyright 2016 Michael Schenk. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package note

import (
	"fmt"
	"path/filepath"
	"regexp"
	"sort"
	"strconv"
	"strings"

	"github.com/git-time-metric/gtm/util"
)

// CommitNote contains the time metrics for a commit
type CommitNote struct {
	Files []FileDetail
}

// FilterOutTerminal filters out terminal time from commit note
func (n CommitNote) FilterOutTerminal() CommitNote {
	fds := []FileDetail{}
	for _, f := range n.Files {
		if !f.IsTerminal() {
			fds = append(fds, f)
		}
	}
	return CommitNote{Files: fds}
}

// Total returns the total time for a commit note
func (n CommitNote) Total() int {
	total := 0
	for _, fm := range n.Files {
		total += fm.TimeSpent
	}
	return total
}

// Marshal converts a commit note to a serialized string
func Marshal(n CommitNote) string {
	s := fmt.Sprintf("[ver:%s,total:%d]\n", "1", n.Total())
	for _, fl := range n.Files {
		// nomralize file paths to unix convention
		s += fmt.Sprintf("%s:%d,", filepath.ToSlash(fl.SourceFile), fl.TimeSpent)
		for _, e := range fl.SortEpochs() {
			s += fmt.Sprintf("%d:%d,", e, fl.Timeline[e])
		}
		s += fmt.Sprintf("%s\n", fl.Status)
	}
	return s
}

// UnMarshal unserializes a git note string into a commit note
func UnMarshal(s string) (CommitNote, error) {
	var (
		version string
		files   = []FileDetail{}
	)

	reHeader := regexp.MustCompile(`\[ver:\d+,total:\d+]`)
	reHeaderVals := regexp.MustCompile(`\d+`)

	lines := strings.Split(s, "\n")
	for lineIdx := 0; lineIdx < len(lines); lineIdx++ {
		switch {
		case strings.TrimSpace(lines[lineIdx]) == "":
			version = ""
		case reHeader.MatchString(lines[lineIdx]):
			if matches := reHeaderVals.FindAllString(lines[lineIdx], 2); matches != nil && len(matches) == 2 {
				version = matches[0]
			} else {
				return CommitNote{}, fmt.Errorf("Unable to unmarshal time logged, header format invalid, %s", lines[lineIdx])
			}
		case version == "1":
			fieldGroups := strings.Split(lines[lineIdx], ",")
			if len(fieldGroups) < 3 {
				return CommitNote{}, fmt.Errorf("Unable to unmarshal time logged, format invalid, %s", lines[lineIdx])
			}

			var (
				filePath     string
				fileTotal    int
				fileStatus   string
				fileTimeline = map[int64]int{}
			)

			for groupIdx := range fieldGroups {
				fieldVals := strings.Split(fieldGroups[groupIdx], ":")
				switch {
				case groupIdx == 0 && len(fieldVals) == 2:
					// file name and total, filename:total
					filePath = fieldVals[0]
					t, err := strconv.Atoi(fieldVals[1])
					if err != nil {
						return CommitNote{}, fmt.Errorf("Unable to unmarshal time logged, format invalid, %s", err)
					}
					fileTotal = t
				case groupIdx == len(fieldGroups)-1 && len(fieldVals) == 1:
					// file status of m or r
					fileStatus = fieldVals[0]
				case len(fieldVals) == 2:
					// epoch timeline, epoch:total
					e, err := strconv.ParseInt(fieldVals[0], 10, 64)
					if err != nil {
						return CommitNote{}, fmt.Errorf("Unable to unmarshal time logged, format invalid, %s", err)
					}
					t, err := strconv.Atoi(fieldVals[1])
					if err != nil {
						return CommitNote{}, fmt.Errorf("Unable to unmarshal time logged, format invalid, %s", err)
					}
					fileTimeline[e] = t
				default:
					// error
					return CommitNote{}, fmt.Errorf("Unable to unmarshal time logged, format invalid")
				}
			}

			// check for existing file path and merge if found
			// for example, this can happen when rewriting commits with git commit --amend
			found := false
			for idx := range files {
				if files[idx].SourceFile == filePath {
					for epoch, secs := range fileTimeline {
						files[idx].TimeSpent += secs
						files[idx].Timeline[epoch] += secs
					}
					// only change file status if modified or deleted
					if fileStatus == "m" || fileStatus == "d" {
						files[idx].Status = fileStatus
					}
					found = true
					break
				}
			}

			if !found {
				files = append(files,
					FileDetail{
						SourceFile: filePath,
						TimeSpent:  fileTotal,
						Timeline:   fileTimeline,
						Status:     fileStatus})
			}

		default:
			return CommitNote{}, fmt.Errorf("Unable to unmarshal time logged, unknown version %s", version)
		}
	}
	sort.Sort(sort.Reverse(FileByTime(files)))
	return CommitNote{Files: files}, nil
}

// FileDetail contains a source file's time metrics
type FileDetail struct {
	SourceFile string
	TimeSpent  int
	Timeline   map[int64]int
	Status     string
}

func (f *FileDetail) ShortenSourceFile(n int) string {
	x := len(f.SourceFile) - n - 1
	if x <= 0 {
		return f.SourceFile
	}

	idx := strings.Index(f.SourceFile, string(filepath.Separator))
	if idx >= 0 {
		x = idx
	}
	return fmt.Sprintf("...%s", f.SourceFile[x:])
}

// SortEpochs returns timeline keys sorted by epoch
func (f *FileDetail) SortEpochs() []int64 {
	keys := []int64{}
	for k := range f.Timeline {
		keys = append(keys, k)
	}
	sort.Sort(util.ByInt64(keys))
	return keys
}

// IsTerminal returns true if file is terminal
func (f *FileDetail) IsTerminal() bool {
	return f.SourceFile == ".gtm/terminal.app"
}

// FileByTime is list of FileDetails
type FileByTime []FileDetail

func (a FileByTime) Len() int           { return len(a) }
func (a FileByTime) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a FileByTime) Less(i, j int) bool { return a[i].TimeSpent < a[j].TimeSpent }
<commit_msg>file shortening - start at max size<commit_after>// Copyright 2016 Michael Schenk. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package note

import (
	"fmt"
	"path/filepath"
	"regexp"
	"sort"
	"strconv"
	"strings"

	"github.com/git-time-metric/gtm/util"
)

// CommitNote contains the time metrics for a commit
type CommitNote struct {
	Files []FileDetail
}

// FilterOutTerminal filters out terminal time from commit note
func (n CommitNote) FilterOutTerminal() CommitNote {
	fds := []FileDetail{}
	for _, f := range n.Files {
		if !f.IsTerminal() {
			fds = append(fds, f)
		}
	}
	return CommitNote{Files: fds}
}

// Total returns the total time for a commit note
func (n CommitNote) Total() int {
	total := 0
	for _, fm := range n.Files {
		total += fm.TimeSpent
	}
	return total
}

// Marshal converts a commit note to a serialized string
func Marshal(n CommitNote) string {
	s := fmt.Sprintf("[ver:%s,total:%d]\n", "1", n.Total())
	for _, fl := range n.Files {
		// nomralize file paths to unix convention
		s += fmt.Sprintf("%s:%d,", filepath.ToSlash(fl.SourceFile), fl.TimeSpent)
		for _, e := range fl.SortEpochs() {
			s += fmt.Sprintf("%d:%d,", e, fl.Timeline[e])
		}
		s += fmt.Sprintf("%s\n", fl.Status)
	}
	return s
}

// UnMarshal unserializes a git note string into a commit note
func UnMarshal(s string) (CommitNote, error) {
	var (
		version string
		files   = []FileDetail{}
	)

	reHeader := regexp.MustCompile(`\[ver:\d+,total:\d+]`)
	reHeaderVals := regexp.MustCompile(`\d+`)

	lines := strings.Split(s, "\n")
	for lineIdx := 0; lineIdx < len(lines); lineIdx++ {
		switch {
		case strings.TrimSpace(lines[lineIdx]) == "":
			version = ""
		case reHeader.MatchString(lines[lineIdx]):
			if matches := reHeaderVals.FindAllString(lines[lineIdx], 2); matches != nil && len(matches) == 2 {
				version = matches[0]
			} else {
				return CommitNote{}, fmt.Errorf("Unable to unmarshal time logged, header format invalid, %s", lines[lineIdx])
			}
		case version == "1":
			fieldGroups := strings.Split(lines[lineIdx], ",")
			if len(fieldGroups) < 3 {
				return CommitNote{}, fmt.Errorf("Unable to unmarshal time logged, format invalid, %s", lines[lineIdx])
			}

			var (
				filePath     string
				fileTotal    int
				fileStatus   string
				fileTimeline = map[int64]int{}
			)

			for groupIdx := range fieldGroups {
				fieldVals := strings.Split(fieldGroups[groupIdx], ":")
				switch {
				case groupIdx == 0 && len(fieldVals) == 2:
					// file name and total, filename:total
					filePath = fieldVals[0]
					t, err := strconv.Atoi(fieldVals[1])
					if err != nil {
						return CommitNote{}, fmt.Errorf("Unable to unmarshal time logged, format invalid, %s", err)
					}
					fileTotal = t
				case groupIdx == len(fieldGroups)-1 && len(fieldVals) == 1:
					// file status of m or r
					fileStatus = fieldVals[0]
				case len(fieldVals) == 2:
					// epoch timeline, epoch:total
					e, err := strconv.ParseInt(fieldVals[0], 10, 64)
					if err != nil {
						return CommitNote{}, fmt.Errorf("Unable to unmarshal time logged, format invalid, %s", err)
					}
					t, err := strconv.Atoi(fieldVals[1])
					if err != nil {
						return CommitNote{}, fmt.Errorf("Unable to unmarshal time logged, format invalid, %s", err)
					}
					fileTimeline[e] = t
				default:
					// error
					return CommitNote{}, fmt.Errorf("Unable to unmarshal time logged, format invalid")
				}
			}

			// check for existing file path and merge if found
			// for example, this can happen when rewriting commits with git commit --amend
			found := false
			for idx := range files {
				if files[idx].SourceFile == filePath {
					for epoch, secs := range fileTimeline {
						files[idx].TimeSpent += secs
						files[idx].Timeline[epoch] += secs
					}
					// only change file status if modified or deleted
					if fileStatus == "m" || fileStatus == "d" {
						files[idx].Status = fileStatus
					}
					found = true
					break
				}
			}

			if !found {
				files = append(files,
					FileDetail{
						SourceFile: filePath,
						TimeSpent:  fileTotal,
						Timeline:   fileTimeline,
						Status:     fileStatus})
			}

		default:
			return CommitNote{}, fmt.Errorf("Unable to unmarshal time logged, unknown version %s", version)
		}
	}
	sort.Sort(sort.Reverse(FileByTime(files)))
	return CommitNote{Files: files}, nil
}

// FileDetail contains a source file's time metrics
type FileDetail struct {
	SourceFile string
	TimeSpent  int
	Timeline   map[int64]int
	Status     string
}

func (f *FileDetail) ShortenSourceFile(n int) string {
	x := len(f.SourceFile) - n - 1
	if x <= 0 {
		return f.SourceFile
	}

	idx := strings.Index(f.SourceFile[x:], string(filepath.Separator))
	if idx >= 0 {
		x = x + idx
	}
	return fmt.Sprintf("...%s", f.SourceFile[x:])
}

// SortEpochs returns timeline keys sorted by epoch
func (f *FileDetail) SortEpochs() []int64 {
	keys := []int64{}
	for k := range f.Timeline {
		keys = append(keys, k)
	}
	sort.Sort(util.ByInt64(keys))
	return keys
}

// IsTerminal returns true if file is terminal
func (f *FileDetail) IsTerminal() bool {
	return f.SourceFile == ".gtm/terminal.app"
}

// FileByTime is list of FileDetails
type FileByTime []FileDetail

func (a FileByTime) Len() int           { return len(a) }
func (a FileByTime) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a FileByTime) Less(i, j int) bool { return a[i].TimeSpent < a[j].TimeSpent }
<|endoftext|>


<commit_before>// Copyright © 2015, The T Authors.

package edit

import (
	"bytes"
	"errors"
	"strings"
	"testing"

	"github.com/eaburns/T/edit/runes"
)

// String returns a string containing the entire editor contents.
func (ed *Editor) String() string {
	rs, err := runes.ReadAll(ed.buf.runes.Reader(0))
	if err != nil {
		panic(err)
	}
	return string(rs)
}

func (ed *Editor) change(a Address, s string) error {
	return ed.Do(Change(All, s), bytes.NewBuffer(nil))
}

func TestRetry(t *testing.T) {
	ed := NewEditor(NewBuffer())
	defer ed.buf.Close()

	const str = "Hello, 世界!"
	ch := func() (addr, error) {
		if ed.buf.seq < 10 {
			// Simulate concurrent changes, necessitating retries.
			ed.buf.seq++
		}
		return change{a: All, op: 'c', str: str}.do(ed, nil)
	}
	if err := ed.do(ch); err != nil {
		t.Fatalf("ed.do(ch)=%v, want nil", err)
	}
	if s := ed.String(); s != str {
		t.Errorf("ed.String()=%q, want %q,nil\n", s, str)
	}
}

func TestWhere(t *testing.T) {
	tests := []struct {
		init string
		a    Address
		at   addr
	}{
		{init: "", a: All, at: addr{0, 0}},
		{init: "H\ne\nl\nl\no\n 世\n界\n!", a: All, at: addr{0, 16}},
		{init: "Hello\n 世界!", a: All, at: addr{0, 10}},
		{init: "Hello\n 世界!", a: End, at: addr{10, 10}},
		{init: "Hello\n 世界!", a: Line(1), at: addr{0, 6}},
		{init: "Hello\n 世界!", a: Line(2), at: addr{6, 10}},
		{init: "Hello\n 世界!", a: Regexp("/Hello"), at: addr{0, 5}},
		{init: "Hello\n 世界!", a: Regexp("/世界"), at: addr{7, 9}},
	}
	for _, test := range tests {
		ed := NewEditor(NewBuffer())
		defer ed.buf.Close()
		if err := ed.change(All, test.init); err != nil {
			t.Errorf("failed to init %#v: %v", test, err)
			continue
		}
		at, err := ed.Where(test.a)
		if at != test.at || err != nil {
			t.Errorf("ed.Where(%q)=%v,%v, want %v,<nil>", test.a, at, err, test.at)
		}
	}
}

func TestWriterTo(t *testing.T) {
	tests := []struct {
		init, want string
		a          Address
		dot        addr
	}{
		{init: "", a: All, want: "", dot: addr{}},
		{init: "", a: End, want: "", dot: addr{}},
		{init: "", a: Rune(0), want: "", dot: addr{}},
		{init: "Hello, 世界", a: All, want: "Hello, 世界", dot: addr{0, 9}},
		{init: "Hello, 世界", a: Regexp("/Hello/"), want: "Hello", dot: addr{0, 5}},
		{init: "Hello, 世界", a: End, want: "", dot: addr{9, 9}},
		{init: "Hello, 世界", a: Rune(0), want: "", dot: addr{}},
		{init: "a\nb\nc\n", a: Line(0), want: "", dot: addr{}},
		{init: "a\nb\nc\n", a: Line(1), want: "a\n", dot: addr{0, 2}},
		{init: "a\nb\nc\n", a: Line(2), want: "b\n", dot: addr{2, 4}},
		{init: "a\nb\nc\n", a: Line(3), want: "c\n", dot: addr{4, 6}},
	}
	for _, test := range tests {
		ed := NewEditor(NewBuffer())
		defer ed.buf.Close()
		if err := ed.change(All, test.init); err != nil {
			t.Errorf("failed to init %#v: %v", test, err)
			continue
		}

		b := bytes.NewBuffer(nil)
		n, err := ed.WriterTo(test.a).WriteTo(b)
		str := b.String()
		if n != int64(len(str)) || err != nil {
			t.Errorf("ed.WriterTo(%q).WriteTo(b)=%d,%v, want %d,<nil>", test.a, n, err, len(str))
			continue
		}
		if str != test.want {
			t.Errorf("ed.WriterTo(%q).WriteTo(b); b.String()=%q, want %q", test.a, str, test.want)
		}
		if dot := ed.marks['.']; dot != test.dot {
			t.Errorf("ed.WriterTo(%q).WriteTo(b); ed.marks['.']=%v, want %v", test.a, dot, test.dot)
		}
	}
}

func TestReaderFrom(t *testing.T) {
	tests := []struct {
		init, read, want string
		a                Address
		dot              addr
	}{
		{init: "", a: All, read: "", want: "", dot: addr{}},
		{init: "", a: All, read: "αβξ", want: "αβξ", dot: addr{0, 3}},
		{init: "Hello, 世界", a: Line(0), read: "", want: "Hello, 世界", dot: addr{}},
		{init: "Hello, 世界", a: End, read: "", want: "Hello, 世界", dot: addr{9, 9}},
		{init: "Hello, 世界", a: All, read: "", want: "", dot: addr{0, 0}},
		{init: "Hello, 世界", a: All, read: "αβξ", want: "αβξ", dot: addr{0, 3}},
		{init: "Hello, 世界", a: Regexp("/世界/"), read: "World", want: "Hello, World", dot: addr{7, 12}},
		{init: "a\nb\nc\n", a: Line(0), read: "z\n", want: "z\na\nb\nc\n", dot: addr{0, 2}},
		{init: "a\nb\nc\n", a: Line(1), read: "z\n", want: "z\nb\nc\n", dot: addr{0, 2}},
		{init: "a\nb\nc\n", a: Line(2), read: "z\n", want: "a\nz\nc\n", dot: addr{2, 4}},
		{init: "a\nb\nc\n", a: Line(3), read: "z\n", want: "a\nb\nz\n", dot: addr{4, 6}},
	}
	for _, test := range tests {
		ed := NewEditor(NewBuffer())
		defer ed.buf.Close()
		if err := ed.change(All, test.init); err != nil {
			t.Errorf("failed to init %#v: %v", test, err)
			continue
		}

		n, err := ed.ReaderFrom(test.a).ReadFrom(strings.NewReader(test.read))
		if n != int64(len(test.read)) || err != nil {
			t.Errorf("ed.ReaderFrom(%q).ReadFrom(%q)=%d,%v, want %d,<nil>", test.a, test.read, n, err, len(test.read))
			continue
		}
		if str := ed.String(); str != test.want {
			t.Errorf("ed.ReaderFrom(%q).ReadFrom(%q); b.String()=%q, want %q", test.a, test.read, str, test.want)
		}
		if dot := ed.marks['.']; dot != test.dot {
			t.Errorf("ed.ReaderFrom(%q).ReadFrom(%q); ed.marks['.']=%v, want %v", test.a, test.read, dot, test.dot)
		}
	}
}

func TestEditorDoPendingError(t *testing.T) {
	ed := NewEditor(NewBuffer())
	defer ed.buf.Close()
	maddr := addr{5, 10}
	ed.marks['m'] = maddr

	testErr := errors.New("test error")
	err := ed.do(func() (addr, error) {
		// Change a mark, it should be restored to its origin.
		m := ed.marks['m']
		m.to += 10
		m.from += 20
		ed.marks['m'] = m
		return addr{}, testErr
	})
	if err != testErr {
		t.Errorf("ed.do(…)=%v, want %v", err, testErr)
	}
	if ed.marks['m'] != maddr {
		t.Errorf("ed.marks['m']=%v, want %v", ed.marks['m'], maddr)
	}
}

func TestEditorDoOutOfSequence(t *testing.T) {
	ed := NewEditor(NewBuffer())
	defer ed.buf.Close()
	const init = "Hello, 世界"
	if err := ed.change(All, init); err != nil {
		t.Fatalf("failed to init: %v", err)
	}
	maddr := addr{5, 10}
	ed.marks['m'] = maddr

	err := ed.do(func() (addr, error) {
		if err := pend(ed, addr{10, 20}, runes.EmptyReader()); err != nil {
			panic(err)
		}
		if err := pend(ed, addr{0, 10}, runes.EmptyReader()); err != nil {
			panic(err)
		}
		return addr{0, 20}, nil
	})
	if err == nil {
		t.Error("ed.do({out-of-sequence})=<nil>, want error")
	}
	if ed.marks['m'] != maddr {
		t.Errorf("ed.marks['m']=%v, want %v", ed.marks['m'], maddr)
	}
}
<commit_msg>Test Editor.Close.<commit_after>// Copyright © 2015, The T Authors.

package edit

import (
	"bytes"
	"errors"
	"strings"
	"testing"

	"github.com/eaburns/T/edit/runes"
)

// String returns a string containing the entire editor contents.
func (ed *Editor) String() string {
	rs, err := runes.ReadAll(ed.buf.runes.Reader(0))
	if err != nil {
		panic(err)
	}
	return string(rs)
}

func (ed *Editor) change(a Address, s string) error {
	return ed.Do(Change(All, s), bytes.NewBuffer(nil))
}

func TestEditorClose(t *testing.T) {
	buf := NewBuffer()
	ed := NewEditor(buf)
	if err := ed.Close(); err != nil {
		t.Fatalf("failed to close the editor: %v", err)
	}
	if err := ed.Close(); err == nil {
		t.Fatal("ed.Close()=nil, want error")
	}
	if err := buf.Close(); err != nil {
		t.Fatalf("failed to close the buffer: %v", err)
	}
}

func TestRetry(t *testing.T) {
	ed := NewEditor(NewBuffer())
	defer ed.buf.Close()

	const str = "Hello, 世界!"
	ch := func() (addr, error) {
		if ed.buf.seq < 10 {
			// Simulate concurrent changes, necessitating retries.
			ed.buf.seq++
		}
		return change{a: All, op: 'c', str: str}.do(ed, nil)
	}
	if err := ed.do(ch); err != nil {
		t.Fatalf("ed.do(ch)=%v, want nil", err)
	}
	if s := ed.String(); s != str {
		t.Errorf("ed.String()=%q, want %q,nil\n", s, str)
	}
}

func TestWhere(t *testing.T) {
	tests := []struct {
		init string
		a    Address
		at   addr
	}{
		{init: "", a: All, at: addr{0, 0}},
		{init: "H\ne\nl\nl\no\n 世\n界\n!", a: All, at: addr{0, 16}},
		{init: "Hello\n 世界!", a: All, at: addr{0, 10}},
		{init: "Hello\n 世界!", a: End, at: addr{10, 10}},
		{init: "Hello\n 世界!", a: Line(1), at: addr{0, 6}},
		{init: "Hello\n 世界!", a: Line(2), at: addr{6, 10}},
		{init: "Hello\n 世界!", a: Regexp("/Hello"), at: addr{0, 5}},
		{init: "Hello\n 世界!", a: Regexp("/世界"), at: addr{7, 9}},
	}
	for _, test := range tests {
		ed := NewEditor(NewBuffer())
		defer ed.buf.Close()
		if err := ed.change(All, test.init); err != nil {
			t.Errorf("failed to init %#v: %v", test, err)
			continue
		}
		at, err := ed.Where(test.a)
		if at != test.at || err != nil {
			t.Errorf("ed.Where(%q)=%v,%v, want %v,<nil>", test.a, at, err, test.at)
		}
	}
}

func TestWriterTo(t *testing.T) {
	tests := []struct {
		init, want string
		a          Address
		dot        addr
	}{
		{init: "", a: All, want: "", dot: addr{}},
		{init: "", a: End, want: "", dot: addr{}},
		{init: "", a: Rune(0), want: "", dot: addr{}},
		{init: "Hello, 世界", a: All, want: "Hello, 世界", dot: addr{0, 9}},
		{init: "Hello, 世界", a: Regexp("/Hello/"), want: "Hello", dot: addr{0, 5}},
		{init: "Hello, 世界", a: End, want: "", dot: addr{9, 9}},
		{init: "Hello, 世界", a: Rune(0), want: "", dot: addr{}},
		{init: "a\nb\nc\n", a: Line(0), want: "", dot: addr{}},
		{init: "a\nb\nc\n", a: Line(1), want: "a\n", dot: addr{0, 2}},
		{init: "a\nb\nc\n", a: Line(2), want: "b\n", dot: addr{2, 4}},
		{init: "a\nb\nc\n", a: Line(3), want: "c\n", dot: addr{4, 6}},
	}
	for _, test := range tests {
		ed := NewEditor(NewBuffer())
		defer ed.buf.Close()
		if err := ed.change(All, test.init); err != nil {
			t.Errorf("failed to init %#v: %v", test, err)
			continue
		}

		b := bytes.NewBuffer(nil)
		n, err := ed.WriterTo(test.a).WriteTo(b)
		str := b.String()
		if n != int64(len(str)) || err != nil {
			t.Errorf("ed.WriterTo(%q).WriteTo(b)=%d,%v, want %d,<nil>", test.a, n, err, len(str))
			continue
		}
		if str != test.want {
			t.Errorf("ed.WriterTo(%q).WriteTo(b); b.String()=%q, want %q", test.a, str, test.want)
		}
		if dot := ed.marks['.']; dot != test.dot {
			t.Errorf("ed.WriterTo(%q).WriteTo(b); ed.marks['.']=%v, want %v", test.a, dot, test.dot)
		}
	}
}

func TestReaderFrom(t *testing.T) {
	tests := []struct {
		init, read, want string
		a                Address
		dot              addr
	}{
		{init: "", a: All, read: "", want: "", dot: addr{}},
		{init: "", a: All, read: "αβξ", want: "αβξ", dot: addr{0, 3}},
		{init: "Hello, 世界", a: Line(0), read: "", want: "Hello, 世界", dot: addr{}},
		{init: "Hello, 世界", a: End, read: "", want: "Hello, 世界", dot: addr{9, 9}},
		{init: "Hello, 世界", a: All, read: "", want: "", dot: addr{0, 0}},
		{init: "Hello, 世界", a: All, read: "αβξ", want: "αβξ", dot: addr{0, 3}},
		{init: "Hello, 世界", a: Regexp("/世界/"), read: "World", want: "Hello, World", dot: addr{7, 12}},
		{init: "a\nb\nc\n", a: Line(0), read: "z\n", want: "z\na\nb\nc\n", dot: addr{0, 2}},
		{init: "a\nb\nc\n", a: Line(1), read: "z\n", want: "z\nb\nc\n", dot: addr{0, 2}},
		{init: "a\nb\nc\n", a: Line(2), read: "z\n", want: "a\nz\nc\n", dot: addr{2, 4}},
		{init: "a\nb\nc\n", a: Line(3), read: "z\n", want: "a\nb\nz\n", dot: addr{4, 6}},
	}
	for _, test := range tests {
		ed := NewEditor(NewBuffer())
		defer ed.buf.Close()
		if err := ed.change(All, test.init); err != nil {
			t.Errorf("failed to init %#v: %v", test, err)
			continue
		}

		n, err := ed.ReaderFrom(test.a).ReadFrom(strings.NewReader(test.read))
		if n != int64(len(test.read)) || err != nil {
			t.Errorf("ed.ReaderFrom(%q).ReadFrom(%q)=%d,%v, want %d,<nil>", test.a, test.read, n, err, len(test.read))
			continue
		}
		if str := ed.String(); str != test.want {
			t.Errorf("ed.ReaderFrom(%q).ReadFrom(%q); b.String()=%q, want %q", test.a, test.read, str, test.want)
		}
		if dot := ed.marks['.']; dot != test.dot {
			t.Errorf("ed.ReaderFrom(%q).ReadFrom(%q); ed.marks['.']=%v, want %v", test.a, test.read, dot, test.dot)
		}
	}
}

func TestEditorDoPendingError(t *testing.T) {
	ed := NewEditor(NewBuffer())
	defer ed.buf.Close()
	maddr := addr{5, 10}
	ed.marks['m'] = maddr

	testErr := errors.New("test error")
	err := ed.do(func() (addr, error) {
		// Change a mark, it should be restored to its origin.
		m := ed.marks['m']
		m.to += 10
		m.from += 20
		ed.marks['m'] = m
		return addr{}, testErr
	})
	if err != testErr {
		t.Errorf("ed.do(…)=%v, want %v", err, testErr)
	}
	if ed.marks['m'] != maddr {
		t.Errorf("ed.marks['m']=%v, want %v", ed.marks['m'], maddr)
	}
}

func TestEditorDoOutOfSequence(t *testing.T) {
	ed := NewEditor(NewBuffer())
	defer ed.buf.Close()
	const init = "Hello, 世界"
	if err := ed.change(All, init); err != nil {
		t.Fatalf("failed to init: %v", err)
	}
	maddr := addr{5, 10}
	ed.marks['m'] = maddr

	err := ed.do(func() (addr, error) {
		if err := pend(ed, addr{10, 20}, runes.EmptyReader()); err != nil {
			panic(err)
		}
		if err := pend(ed, addr{0, 10}, runes.EmptyReader()); err != nil {
			panic(err)
		}
		return addr{0, 20}, nil
	})
	if err == nil {
		t.Error("ed.do({out-of-sequence})=<nil>, want error")
	}
	if ed.marks['m'] != maddr {
		t.Errorf("ed.marks['m']=%v, want %v", ed.marks['m'], maddr)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/md5"
	"flag"
	"fmt"
	"github.com/bitly/nsq/nsq"
	"github.com/bitly/nsq/util"
	"hash/crc32"
	"io"
	"log"
	"net"
	"os"
	"os/signal"
	"regexp"
	"strconv"
	"strings"
	"syscall"
	"time"
)

var (
	showVersion          = flag.Bool("version", false, "print version string")
	httpAddress          = flag.String("http-address", "0.0.0.0:4151", "<addr>:<port> to listen on for HTTP clients")
	tcpAddress           = flag.String("tcp-address", "0.0.0.0:4150", "<addr>:<port> to listen on for TCP clients")
	memQueueSize         = flag.Int64("mem-queue-size", 10000, "number of messages to keep in memory (per topic/channel)")
	maxBytesPerFile      = flag.Int64("max-bytes-per-file", 104857600, "number of bytes per diskqueue file before rolling")
	syncEvery            = flag.Int64("sync-every", 2500, "number of messages between diskqueue syncs")
	msgTimeout           = flag.String("msg-timeout", "60s", "duration to wait before auto-requeing a message")
	maxMessageSize       = flag.Int64("max-message-size", 1024768, "maximum size of a single message in bytes")
	maxBodySize          = flag.Int64("max-body-size", 5*1024768, "maximum size of a single command body")
	maxMsgTimeout        = flag.Duration("max-msg-timeout", 15*time.Minute, "maximum duration before a message will timeout")
	maxHeartbeatInterval = flag.Duration("max-heartbeat-interval", 60*time.Second, "maximum duration of time between heartbeats that a client can configure")
	maxRdyCount          = flag.Int64("max-rdy-count", 2500, "maximum RDY count for a single client")
	dataPath             = flag.String("data-path", "", "path to store disk-backed messages")
	workerId             = flag.Int64("worker-id", 0, "unique identifier (int) for this worker (will default to a hash of hostname)")
	verbose              = flag.Bool("verbose", false, "enable verbose logging")
	statsdAddress        = flag.String("statsd-address", "", "UDP <addr>:<port> of a statsd daemon for writing stats")
	statsdInterval       = flag.Int("statsd-interval", 30, "seconds between pushing to statsd")
	broadcastAddress     = flag.String("broadcast-address", "", "address that will be registered with lookupd, (default to the OS hostname)")
	lookupdTCPAddrs      = util.StringArray{}
)

func init() {
	flag.Var(&lookupdTCPAddrs, "lookupd-tcp-address", "lookupd TCP address (may be given multiple times)")
}

var nsqd *NSQd
var protocols = map[string]nsq.Protocol{}

func main() {
	flag.Parse()

	hostname, err := os.Hostname()
	if err != nil {
		log.Fatal(err)
	}

	if *showVersion {
		fmt.Println(util.Version("nsqd"))
		return
	}

	if *workerId == 0 {
		h := md5.New()
		io.WriteString(h, hostname)
		*workerId = int64(crc32.ChecksumIEEE(h.Sum(nil)) % 1024)
	}

	tcpAddr, err := net.ResolveTCPAddr("tcp", *tcpAddress)
	if err != nil {
		log.Fatal(err)
	}

	httpAddr, err := net.ResolveTCPAddr("tcp", *httpAddress)
	if err != nil {
		log.Fatal(err)
	}

	if *broadcastAddress == "" {
		*broadcastAddress = hostname
	}

	log.Println(util.Version("nsqd"))
	log.Printf("worker id %d", *workerId)

	exitChan := make(chan int)
	signalChan := make(chan os.Signal, 1)
	go func() {
		<-signalChan
		exitChan <- 1
	}()
	signal.Notify(signalChan, syscall.SIGINT, syscall.SIGTERM)

	if *statsdAddress != "" {
		underHostname := fmt.Sprintf("%s_%d", strings.Replace(hostname, ".", "_", -1), httpAddr.Port)
		prefix := fmt.Sprintf("nsq.%s.", underHostname)
		go statsdLoop(*statsdAddress, prefix, *statsdInterval)
	}

	// for backwards compatibility if --msg-timeout only
	// contains numbers then default to ms
	var msgTimeoutDuration time.Duration
	if regexp.MustCompile(`^[0-9]+$`).MatchString(*msgTimeout) {
		intMsgTimeout, err := strconv.Atoi(*msgTimeout)
		if err != nil {
			log.Fatalf("ERROR: failed to Atoi --msg-timeout %s - %s", *msgTimeout, err.Error())
		}
		msgTimeoutDuration = time.Duration(intMsgTimeout) * time.Millisecond
	} else {
		msgTimeoutDuration, err = time.ParseDuration(*msgTimeout)
		if err != nil {
			log.Fatalf("ERROR: failed to ParseDuration --msg-timeout %s - %s", *msgTimeout, err.Error())
		}
	}

	options := NewNsqdOptions()
	options.maxRdyCount = *maxRdyCount
	options.maxMessageSize = *maxMessageSize
	options.maxBodySize = *maxBodySize
	options.memQueueSize = *memQueueSize
	options.dataPath = *dataPath
	options.maxBytesPerFile = *maxBytesPerFile
	options.syncEvery = *syncEvery
	options.msgTimeout = msgTimeoutDuration
	options.maxMsgTimeout = *maxMsgTimeout
	options.broadcastAddress = *broadcastAddress
	options.maxHeartbeatInterval = *maxHeartbeatInterval

	nsqd = NewNSQd(*workerId, options)
	nsqd.tcpAddr = tcpAddr
	nsqd.httpAddr = httpAddr
	nsqd.lookupdTCPAddrs = lookupdTCPAddrs

	nsqd.LoadMetadata()
	err = nsqd.PersistMetadata()
	if err != nil {
		log.Fatalf("ERROR: failed to persist metadata - %s", err.Error())
	}
	nsqd.Main()
	<-exitChan
	nsqd.Exit()
}
<commit_msg>nsqd: use broadcast address (default of hostname) for statsd prefix<commit_after>package main

import (
	"crypto/md5"
	"flag"
	"fmt"
	"github.com/bitly/nsq/nsq"
	"github.com/bitly/nsq/util"
	"hash/crc32"
	"io"
	"log"
	"net"
	"os"
	"os/signal"
	"regexp"
	"strconv"
	"strings"
	"syscall"
	"time"
)

var (
	showVersion          = flag.Bool("version", false, "print version string")
	httpAddress          = flag.String("http-address", "0.0.0.0:4151", "<addr>:<port> to listen on for HTTP clients")
	tcpAddress           = flag.String("tcp-address", "0.0.0.0:4150", "<addr>:<port> to listen on for TCP clients")
	memQueueSize         = flag.Int64("mem-queue-size", 10000, "number of messages to keep in memory (per topic/channel)")
	maxBytesPerFile      = flag.Int64("max-bytes-per-file", 104857600, "number of bytes per diskqueue file before rolling")
	syncEvery            = flag.Int64("sync-every", 2500, "number of messages between diskqueue syncs")
	msgTimeout           = flag.String("msg-timeout", "60s", "duration to wait before auto-requeing a message")
	maxMessageSize       = flag.Int64("max-message-size", 1024768, "maximum size of a single message in bytes")
	maxBodySize          = flag.Int64("max-body-size", 5*1024768, "maximum size of a single command body")
	maxMsgTimeout        = flag.Duration("max-msg-timeout", 15*time.Minute, "maximum duration before a message will timeout")
	maxHeartbeatInterval = flag.Duration("max-heartbeat-interval", 60*time.Second, "maximum duration of time between heartbeats that a client can configure")
	maxRdyCount          = flag.Int64("max-rdy-count", 2500, "maximum RDY count for a single client")
	dataPath             = flag.String("data-path", "", "path to store disk-backed messages")
	workerId             = flag.Int64("worker-id", 0, "unique identifier (int) for this worker (will default to a hash of hostname)")
	verbose              = flag.Bool("verbose", false, "enable verbose logging")
	statsdAddress        = flag.String("statsd-address", "", "UDP <addr>:<port> of a statsd daemon for writing stats")
	statsdInterval       = flag.Int("statsd-interval", 30, "seconds between pushing to statsd")
	broadcastAddress     = flag.String("broadcast-address", "", "address that will be registered with lookupd, (default to the OS hostname)")
	lookupdTCPAddrs      = util.StringArray{}
)

func init() {
	flag.Var(&lookupdTCPAddrs, "lookupd-tcp-address", "lookupd TCP address (may be given multiple times)")
}

var nsqd *NSQd
var protocols = map[string]nsq.Protocol{}

func main() {
	flag.Parse()

	hostname, err := os.Hostname()
	if err != nil {
		log.Fatal(err)
	}

	if *showVersion {
		fmt.Println(util.Version("nsqd"))
		return
	}

	if *workerId == 0 {
		h := md5.New()
		io.WriteString(h, hostname)
		*workerId = int64(crc32.ChecksumIEEE(h.Sum(nil)) % 1024)
	}

	tcpAddr, err := net.ResolveTCPAddr("tcp", *tcpAddress)
	if err != nil {
		log.Fatal(err)
	}

	httpAddr, err := net.ResolveTCPAddr("tcp", *httpAddress)
	if err != nil {
		log.Fatal(err)
	}

	if *broadcastAddress == "" {
		*broadcastAddress = hostname
	}

	log.Println(util.Version("nsqd"))
	log.Printf("worker id %d", *workerId)

	exitChan := make(chan int)
	signalChan := make(chan os.Signal, 1)
	go func() {
		<-signalChan
		exitChan <- 1
	}()
	signal.Notify(signalChan, syscall.SIGINT, syscall.SIGTERM)

	if *statsdAddress != "" {
		undered := fmt.Sprintf("%s_%d", strings.Replace(*broadcastAddress, ".", "_", -1), httpAddr.Port)
		prefix := fmt.Sprintf("nsq.%s.", undered)
		go statsdLoop(*statsdAddress, prefix, *statsdInterval)
	}

	// for backwards compatibility if --msg-timeout only
	// contains numbers then default to ms
	var msgTimeoutDuration time.Duration
	if regexp.MustCompile(`^[0-9]+$`).MatchString(*msgTimeout) {
		intMsgTimeout, err := strconv.Atoi(*msgTimeout)
		if err != nil {
			log.Fatalf("ERROR: failed to Atoi --msg-timeout %s - %s", *msgTimeout, err.Error())
		}
		msgTimeoutDuration = time.Duration(intMsgTimeout) * time.Millisecond
	} else {
		msgTimeoutDuration, err = time.ParseDuration(*msgTimeout)
		if err != nil {
			log.Fatalf("ERROR: failed to ParseDuration --msg-timeout %s - %s", *msgTimeout, err.Error())
		}
	}

	options := NewNsqdOptions()
	options.maxRdyCount = *maxRdyCount
	options.maxMessageSize = *maxMessageSize
	options.maxBodySize = *maxBodySize
	options.memQueueSize = *memQueueSize
	options.dataPath = *dataPath
	options.maxBytesPerFile = *maxBytesPerFile
	options.syncEvery = *syncEvery
	options.msgTimeout = msgTimeoutDuration
	options.maxMsgTimeout = *maxMsgTimeout
	options.broadcastAddress = *broadcastAddress
	options.maxHeartbeatInterval = *maxHeartbeatInterval

	nsqd = NewNSQd(*workerId, options)
	nsqd.tcpAddr = tcpAddr
	nsqd.httpAddr = httpAddr
	nsqd.lookupdTCPAddrs = lookupdTCPAddrs

	nsqd.LoadMetadata()
	err = nsqd.PersistMetadata()
	if err != nil {
		log.Fatalf("ERROR: failed to persist metadata - %s", err.Error())
	}
	nsqd.Main()
	<-exitChan
	nsqd.Exit()
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"strings"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/rackspace/gophercloud"
	"github.com/rackspace/gophercloud/openstack"
	"github.com/rackspace/gophercloud/openstack/compute/v2/servers"
	"github.com/rackspace/gophercloud/openstack/identity/v2/tokens"
	"github.com/rackspace/gophercloud/pagination"
)

const (
	CREDENTIAL_FILE      = "credential.dat"
	CREDENTIAL_FILE_MODE = 0600
)

type machine struct {
	Name   string
	Ipaddr string
}

func newMechine(s servers.Server) (*machine, error) {
	m := &machine{
		Name: s.Name,
	}

	// For ConoHa
	for key, value := range s.Metadata {
		if key == "instance_name_tag" {
			m.Name = value.(string)
		}
	}

	// Detecting public IP address for connectiong SSH
	for name, addressSet := range s.Addresses {
		if name == "public" {
			// Response of rackspace API has the key either "public" or "private"

			// TODO: Rackspace API and other OpenStack Systems

		} else if strings.HasPrefix(name, "ext-") {
			// Response of ConoHa API has the prefix either "ext-" or "local-"
			as, ok := addressSet.([]interface{})
			if !ok {
				return nil, fmt.Errorf("Invalid address set(type assertion failed).")
			}

			for _, v := range as {
				addr, ok := v.(map[string]interface{})
				if !ok {
					return nil, fmt.Errorf("Invalid address set(type assertion failed).")
				}

				// TODO: support IPv6 address
				if addr["version"] == 4.0 {
					m.Ipaddr = addr["addr"].(string)
				}
			}
		}
	}

	return m, nil
}

type nova struct {
	servers  []servers.Server
	provider *gophercloud.ServiceClient
}

type Credential struct {
	*tokens.Token
}

func NewNova() *nova {
	nova := &nova{
		servers: nil,
	}
	return nova
}

func (n *nova) Init() (err error) {
	// Credentials from env
	opts, err := openstack.AuthOptionsFromEnv()
	if err != nil {
		return err
	}

	// Endpoint options
	eo := gophercloud.EndpointOpts{
		Type:   "compute",
		Region: os.Getenv("OS_REGION_NAME"),
	}

	// Try to use cached credential if file exists.
	_, err = os.Stat(CREDENTIAL_FILE)
	if err == nil {
		// Use cache file
		strdata, err := ioutil.ReadFile(CREDENTIAL_FILE)
		if err != nil {
			log.Warnf("Failed to load cache file: %v", err)
			goto AUTH
		}

		cred := &Credential{Token: &tokens.Token{}}
		if err = json.Unmarshal(strdata, cred); err != nil {
			log.Warnf("Failed to unmarchal the cache data: %v", err)
			goto AUTH
		}

		if time.Now().Before(cred.ExpiresAt) {
			log.Debugf("Token has expired. Try to reauth.")
			goto AUTH
		}

		client, err := openstack.NewClient(opts.IdentityEndpoint)
		client.EndpointLocator = func(o gophercloud.EndpointOpts) (string, error) {
			return "https://compute.tyo1.conoha.io/v2/6150e7c42bab40c59db53d415629841f/", nil
		}
		client.TokenID = cred.ID

		// Set service client
		n.provider, err = openstack.NewComputeV2(client, eo)
		return nil
	}

AUTH:

	client, err := openstack.AuthenticatedClient(opts)
	if err != nil {
		return err
	}

	// Store credential to cache file
	cred := &Credential{
		Token: &tokens.Token{
			ID:        client.TokenID,
			ExpiresAt: time.Now(),
		},
	}
	strdata, err := json.Marshal(cred)
	if err != nil {
		return err
	}

	if err = ioutil.WriteFile(CREDENTIAL_FILE, strdata, CREDENTIAL_FILE_MODE); err != nil {
		log.Warnf("Can not write the credential cache file: %s", CREDENTIAL_FILE)
	}

	// Set service client
	n.provider, err = openstack.NewComputeV2(client, eo)
	return nil
}

func (n *nova) Find(name string) (m *machine, err error) {
	if n.servers == nil {
		n.servers, err = n.listServers()
		if err != nil {
			return nil, err
		}
	}

	// For Rackspace or other OpenStack systems
	for _, server := range n.servers {
		if server.Name == name {
			return newMechine(server)
		}
	}

	//  For ConoHa
	for _, server := range n.servers {
		instanceName, ok := server.Metadata["instance_name_tag"].(string)
		if ok && instanceName == name {
			return newMechine(server)
		}
	}
	return nil, nil
}

func (n *nova) listServers() ([]servers.Server, error) {
	pager := servers.List(n.provider, servers.ListOpts{})

	ss := []servers.Server{}
	pager.EachPage(func(page pagination.Page) (bool, error) {
		sss, err := servers.ExtractServers(page)
		if err != nil {
			return false, err
		}
		ss = append(ss, sss...)
		return true, nil
	})

	return ss, nil
}
<commit_msg>設定ファイルの保存先を変更<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/mitchellh/go-homedir"
	"github.com/rackspace/gophercloud"
	"github.com/rackspace/gophercloud/openstack"
	"github.com/rackspace/gophercloud/openstack/compute/v2/servers"
	"github.com/rackspace/gophercloud/openstack/identity/v2/tokens"
	"github.com/rackspace/gophercloud/pagination"
)

const (
	CREDENTIAL_FILE      = ".novassh"
	CREDENTIAL_FILE_MODE = 0600
)

type machine struct {
	Name   string
	Ipaddr string
}

func newMechine(s servers.Server) (*machine, error) {
	m := &machine{
		Name: s.Name,
	}

	// For ConoHa
	for key, value := range s.Metadata {
		if key == "instance_name_tag" {
			m.Name = value.(string)
		}
	}

	// Detecting public IP address for connectiong SSH
	for name, addressSet := range s.Addresses {
		if name == "public" {
			// Response of rackspace API has the key either "public" or "private"

			// TODO: Rackspace API and other OpenStack Systems

		} else if strings.HasPrefix(name, "ext-") {
			// Response of ConoHa API has the prefix either "ext-" or "local-"
			as, ok := addressSet.([]interface{})
			if !ok {
				return nil, fmt.Errorf("Invalid address set(type assertion failed).")
			}

			for _, v := range as {
				addr, ok := v.(map[string]interface{})
				if !ok {
					return nil, fmt.Errorf("Invalid address set(type assertion failed).")
				}

				// TODO: support IPv6 address
				if addr["version"] == 4.0 {
					m.Ipaddr = addr["addr"].(string)
				}
			}
		}
	}

	return m, nil
}

type nova struct {
	servers  []servers.Server
	provider *gophercloud.ServiceClient
}

type Credential struct {
	*tokens.Token
}

func NewNova() *nova {
	nova := &nova{
		servers: nil,
	}
	return nova
}

func (n *nova) Init() (err error) {
	// Credentials from env
	opts, err := openstack.AuthOptionsFromEnv()
	if err != nil {
		return err
	}

	// Endpoint options
	eo := gophercloud.EndpointOpts{
		Type:   "compute",
		Region: os.Getenv("OS_REGION_NAME"),
	}

	// Try to use cached credential if file exists.
	_, err = os.Stat(n.credentialCachePath())
	if err == nil {
		// Use cache file
		strdata, err := ioutil.ReadFile(n.credentialCachePath())
		if err != nil {
			log.Warnf("Failed to load cache file: %v", err)
			goto AUTH
		}

		cred := &Credential{Token: &tokens.Token{}}
		if err = json.Unmarshal(strdata, cred); err != nil {
			log.Warnf("Failed to unmarchal the cache data: %v", err)
			goto AUTH
		}

		if time.Now().Before(cred.ExpiresAt) {
			log.Debugf("Token has expired. Try to reauth.")
			goto AUTH
		}

		client, err := openstack.NewClient(opts.IdentityEndpoint)
		client.EndpointLocator = func(o gophercloud.EndpointOpts) (string, error) {
			return e, nil
		}
		client.TokenID = cred.ID

		// Set service client
		n.provider, err = openstack.NewComputeV2(client, eo)
		return nil
	}

AUTH:

	client, err := openstack.AuthenticatedClient(opts)
	if err != nil {
		return err
	}

	// Store credential to cache file
	cred := &Credential{
		Token: &tokens.Token{
			ID:        client.TokenID,
			ExpiresAt: time.Now(),
		},
	}
	strdata, err := json.Marshal(cred)
	if err != nil {
		return err
	}

	if err = ioutil.WriteFile(n.credentialCachePath(), strdata, CREDENTIAL_FILE_MODE); err != nil {
		log.Warnf("Can not write the credential cache file: %s", n.credentialCachePath())
	}

	// Set service client
	n.provider, err = openstack.NewComputeV2(client, eo)
	return nil
}

func (n *nova) Find(name string) (m *machine, err error) {
	if n.servers == nil {
		n.servers, err = n.listServers()
		if err != nil {
			return nil, err
		}
	}

	// For Rackspace or other OpenStack systems
	for _, server := range n.servers {
		if server.Name == name {
			return newMechine(server)
		}
	}

	//  For ConoHa
	for _, server := range n.servers {
		instanceName, ok := server.Metadata["instance_name_tag"].(string)
		if ok && instanceName == name {
			return newMechine(server)
		}
	}
	return nil, nil
}

func (n *nova) listServers() ([]servers.Server, error) {
	pager := servers.List(n.provider, servers.ListOpts{})

	ss := []servers.Server{}
	pager.EachPage(func(page pagination.Page) (bool, error) {
		sss, err := servers.ExtractServers(page)
		if err != nil {
			return false, err
		}
		ss = append(ss, sss...)
		return true, nil
	})

	return ss, nil
}

func (n *nova) credentialCachePath() string {
	d, err := homedir.Dir()
	if err == nil {
		return fmt.Sprintf("%s%c%s", d, filepath.Separator, CREDENTIAL_FILE)
	} else {
		return CREDENTIAL_FILE
	}
}
<|endoftext|>


<commit_before>// Copyright 2013, Homin Lee. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package cp949

import (
	"io"
)

// convert utf-8 stream to cp949 stream
func From(in []byte) ([]byte, error) {
	tr, err := fromCp949()
	if err != nil {
		return nil, err
	}

	_, out, err := tr.Translate(in)
	if err != nil {
		return nil, err
	}

	return out, nil
}

// convert cp949 stream to utf-8 stream
func To(in []byte) ([]byte, error) {
	tr, err := toCp949()
	if err != nil {
		return nil, err
	}

	_, out, err := tr.Translate(in)
	if err != nil {
		return nil, err
	}

	return out, nil
}

type cp949Reader struct {
	r     io.Reader
	tr    translator
	cdata []byte // unconsumed data from converter.
	rdata []byte // unconverted data from reader.
	err   error  // final error from reader.
}

// create io.Reader which read cp949 source to utf-8
func NewReader(r io.Reader) (io.Reader, error) {
	tr, err := fromCp949()
	if err != nil {
		return nil, err
	}

	return &cp949Reader{r: r, tr: tr}, nil
}

func (r *cp949Reader) Read(buf []byte) (int, error) {
	for {
		if len(r.cdata) > 0 {
			n := copy(buf, r.cdata)
			r.cdata = r.cdata[n:]
			return n, nil
		}
		if r.err == nil {
			r.rdata = ensureCap(r.rdata, len(r.rdata)+len(buf))
			n, err := r.r.Read(r.rdata[len(r.rdata):cap(r.rdata)])
			// Guard against non-compliant Readers.
			if n == 0 && err == nil {
				err = io.EOF
			}
			r.rdata = r.rdata[0 : len(r.rdata)+n]
			r.err = err
		} else if len(r.rdata) == 0 {
			break
		}
		nc, cdata, cvterr := r.tr.Translate(r.rdata)
		if cvterr != nil {
			// TODO
		}
		r.cdata = cdata

		// Ensure that we consume all bytes at eof
		// if the converter refuses them.
		if nc == 0 && r.err != nil {
			nc = len(r.rdata)
		}

		// Copy unconsumed data to the start of the rdata buffer.
		r.rdata = r.rdata[0:copy(r.rdata, r.rdata[nc:])]
	}
	return 0, r.err
}

type cp949Writer struct {
	w   io.Writer
	tr  translator
	buf []byte // unconsumed data from writer.
}

// creater io.Writer which write utf8 input src to cp949
func NewWriter(w io.Writer) (io.Writer, error) {
	tr, err := toCp949()
	if err != nil {
		return nil, err
	}

	return &cp949Writer{w: w, tr: tr}, nil
}

func (w *cp949Writer) Write(data []byte) (int, error) {
	wdata := data
	if len(w.buf) > 0 {
		w.buf = append(w.buf, data...)
		wdata = w.buf
	}
	n, cdata, err := w.tr.Translate(wdata)
	if err != nil {
		// TODO
	}
	if n > 0 {
		_, err = w.w.Write(cdata)
		if err != nil {
			return 0, err
		}
	}
	w.buf = w.buf[:0]
	if n < len(wdata) {
		w.buf = append(w.buf, wdata[n:]...)
	}
	return len(data), nil
}
<commit_msg>Fix godoc documents for From() and To().<commit_after>// Copyright 2013, Homin Lee. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package cp949

import (
	"io"
)

// convert cp949 stream to utf-8 stream
func From(in []byte) ([]byte, error) {
	tr, err := fromCp949()
	if err != nil {
		return nil, err
	}

	_, out, err := tr.Translate(in)
	if err != nil {
		return nil, err
	}

	return out, nil
}

// convert utf-8 stream to cp949 stream
func To(in []byte) ([]byte, error) {
	tr, err := toCp949()
	if err != nil {
		return nil, err
	}

	_, out, err := tr.Translate(in)
	if err != nil {
		return nil, err
	}

	return out, nil
}

type cp949Reader struct {
	r     io.Reader
	tr    translator
	cdata []byte // unconsumed data from converter.
	rdata []byte // unconverted data from reader.
	err   error  // final error from reader.
}

// create io.Reader which read cp949 source to utf-8
func NewReader(r io.Reader) (io.Reader, error) {
	tr, err := fromCp949()
	if err != nil {
		return nil, err
	}

	return &cp949Reader{r: r, tr: tr}, nil
}

func (r *cp949Reader) Read(buf []byte) (int, error) {
	for {
		if len(r.cdata) > 0 {
			n := copy(buf, r.cdata)
			r.cdata = r.cdata[n:]
			return n, nil
		}
		if r.err == nil {
			r.rdata = ensureCap(r.rdata, len(r.rdata)+len(buf))
			n, err := r.r.Read(r.rdata[len(r.rdata):cap(r.rdata)])
			// Guard against non-compliant Readers.
			if n == 0 && err == nil {
				err = io.EOF
			}
			r.rdata = r.rdata[0 : len(r.rdata)+n]
			r.err = err
		} else if len(r.rdata) == 0 {
			break
		}
		nc, cdata, cvterr := r.tr.Translate(r.rdata)
		if cvterr != nil {
			// TODO
		}
		r.cdata = cdata

		// Ensure that we consume all bytes at eof
		// if the converter refuses them.
		if nc == 0 && r.err != nil {
			nc = len(r.rdata)
		}

		// Copy unconsumed data to the start of the rdata buffer.
		r.rdata = r.rdata[0:copy(r.rdata, r.rdata[nc:])]
	}
	return 0, r.err
}

type cp949Writer struct {
	w   io.Writer
	tr  translator
	buf []byte // unconsumed data from writer.
}

// creater io.Writer which write utf8 input src to cp949
func NewWriter(w io.Writer) (io.Writer, error) {
	tr, err := toCp949()
	if err != nil {
		return nil, err
	}

	return &cp949Writer{w: w, tr: tr}, nil
}

func (w *cp949Writer) Write(data []byte) (int, error) {
	wdata := data
	if len(w.buf) > 0 {
		w.buf = append(w.buf, data...)
		wdata = w.buf
	}
	n, cdata, err := w.tr.Translate(wdata)
	if err != nil {
		// TODO
	}
	if n > 0 {
		_, err = w.w.Write(cdata)
		if err != nil {
			return 0, err
		}
	}
	w.buf = w.buf[:0]
	if n < len(wdata) {
		w.buf = append(w.buf, wdata[n:]...)
	}
	return len(data), nil
}
<|endoftext|>


<commit_before>// Package text supports marshalling Cap'n Proto as text based on a schema.
package text

import (
	"bytes"
	"strconv"

	"zombiezen.com/go/capnproto2"
	"zombiezen.com/go/capnproto2/internal/schema"
)

// AppendStruct appends the text representation of s to dst.
func AppendStruct(dst []byte, s capnp.Struct, typeID uint64, schemaData []byte) ([]byte, error) {
	msg, err := capnp.Unmarshal(schemaData)
	if err != nil {
		return nil, err
	}
	req, err := schema.ReadRootCodeGeneratorRequest(msg)
	if err != nil {
		return nil, err
	}
	nodes, err := req.Nodes()
	if err != nil {
		return nil, err
	}
	nodeMap := make(map[uint64]schema.Node, nodes.Len())
	for i := 0; i < nodes.Len(); i++ {
		n := nodes.At(i)
		nodeMap[n.Id()] = n
	}
	m := &marshaller{
		buf:   bytes.NewBuffer(dst),
		nodes: nodeMap,
	}
	err = m.marshalStruct(typeID, s)
	return m.buf.Bytes(), err
}

type marshaller struct {
	buf   *bytes.Buffer
	tmp   []byte
	nodes map[uint64]schema.Node
}

func (m *marshaller) marshalBool(v bool) {
	if v {
		m.buf.WriteString("true")
	} else {
		m.buf.WriteString("false")
	}
}

func (m *marshaller) marshalInt(i int64) {
	m.tmp = strconv.AppendInt(m.tmp[:0], i, 10)
	m.buf.Write(m.tmp)
}

func (m *marshaller) marshalUint(i uint64) {
	m.tmp = strconv.AppendUint(m.tmp[:0], i, 10)
	m.buf.Write(m.tmp)
}

func (m *marshaller) marshalStruct(typeID uint64, s capnp.Struct) error {
	return nil
}
<commit_msg>encoding/text: implement struct marshalling<commit_after>// Package text supports marshalling Cap'n Proto as text based on a schema.
package text

import (
	"bytes"
	"fmt"
	"strconv"

	"zombiezen.com/go/capnproto2"
	"zombiezen.com/go/capnproto2/internal/schema"
)

// AppendStruct appends the text representation of s to dst.
func AppendStruct(dst []byte, s capnp.Struct, typeID uint64, schemaData []byte) ([]byte, error) {
	msg, err := capnp.Unmarshal(schemaData)
	if err != nil {
		return nil, err
	}
	req, err := schema.ReadRootCodeGeneratorRequest(msg)
	if err != nil {
		return nil, err
	}
	nodes, err := req.Nodes()
	if err != nil {
		return nil, err
	}
	nodeMap := make(map[uint64]schema.Node, nodes.Len())
	for i := 0; i < nodes.Len(); i++ {
		n := nodes.At(i)
		nodeMap[n.Id()] = n
	}
	m := &marshaller{
		buf:   bytes.NewBuffer(dst),
		nodes: nodeMap,
	}
	err = m.marshalStruct(typeID, s)
	return m.buf.Bytes(), err
}

type marshaller struct {
	buf   *bytes.Buffer
	tmp   []byte
	nodes map[uint64]schema.Node
}

func (m *marshaller) marshalBool(v bool) {
	if v {
		m.buf.WriteString("true")
	} else {
		m.buf.WriteString("false")
	}
}

func (m *marshaller) marshalInt(i int64) {
	m.tmp = strconv.AppendInt(m.tmp[:0], i, 10)
	m.buf.Write(m.tmp)
}

func (m *marshaller) marshalUint(i uint64) {
	m.tmp = strconv.AppendUint(m.tmp[:0], i, 10)
	m.buf.Write(m.tmp)
}

func (m *marshaller) marshalText(t []byte) {
	m.buf.WriteByte('"')
	last := 0
	for i, b := range t {
		if !needsEscape(b) {
			continue
		}
		m.buf.Write(t[last:i])
		switch b {
		case '\a':
			m.buf.WriteString("\\a")
		case '\b':
			m.buf.WriteString("\\b")
		case '\f':
			m.buf.WriteString("\\f")
		case '\n':
			m.buf.WriteString("\\n")
		case '\r':
			m.buf.WriteString("\\r")
		case '\t':
			m.buf.WriteString("\\t")
		case '\v':
			m.buf.WriteString("\\v")
		case '\'':
			m.buf.WriteString("\\'")
		case '"':
			m.buf.WriteString("\\\"")
		case '\\':
			m.buf.WriteString("\\\\")
		default:
			m.buf.WriteString("\\x")
			m.buf.WriteByte(hexDigit(b / 16))
			m.buf.WriteByte(hexDigit(b % 16))
		}
		last = i + 1
	}
	m.buf.Write(t[last:])
	m.buf.WriteByte('"')
}

func needsEscape(b byte) bool {
	return b < 0x20 || b >= 0x7f
}

func hexDigit(b byte) byte {
	const digits = "0123456789abcdef"
	return digits[b]
}

func (m *marshaller) marshalStruct(typeID uint64, s capnp.Struct) error {
	n := m.nodes[typeID]
	if !n.IsValid() || n.Which() != schema.Node_Which_structNode {
		return fmt.Errorf("cannot find struct type %#x", typeID)
	}
	var discriminant uint16
	if n.StructNode().DiscriminantCount() > 0 {
		discriminant = s.Uint16(capnp.DataOffset(n.StructNode().DiscriminantOffset() * 2))
	}
	m.buf.WriteByte('(')
	fields := codeOrderFields(n.StructNode())
	first := true
	for _, f := range fields {
		if !(f.Which() == schema.Field_Which_slot || f.Which() == schema.Field_Which_group) {
			continue
		}
		if dv := f.DiscriminantValue(); !(dv == schema.Field_noDiscriminant || dv == discriminant) {
			continue
		}
		if !first {
			m.buf.WriteString(", ")
		}
		first = false
		name, err := f.NameBytes()
		if err != nil {
			return err
		}
		m.buf.Write(name)
		m.buf.WriteString(" = ")
		switch f.Which() {
		case schema.Field_Which_slot:
			if err := m.marshalFieldValue(s, f); err != nil {
				return err
			}
		case schema.Field_Which_group:
			if err := m.marshalStruct(f.Group().TypeId(), s); err != nil {
				return err
			}
		}
	}
	m.buf.WriteByte(')')
	return nil
}

func (m *marshaller) marshalFieldValue(s capnp.Struct, f schema.Field) error {
	typ, err := f.Slot().Type()
	if err != nil {
		return err
	}
	dv, err := f.Slot().DefaultValue()
	if err != nil {
		return err
	}
	if dv.IsValid() && int(typ.Which()) != int(dv.Which()) {
		name, _ := f.Name()
		return fmt.Errorf("marshal field %s: default value is a %v, want %v", name, dv.Which(), typ.Which())
	}
	switch typ.Which() {
	case schema.Type_Which_void:
		m.buf.WriteString("void")
	case schema.Type_Which_bool:
		v := s.Bit(capnp.BitOffset(f.Slot().Offset()))
		d := dv.Bool()
		m.marshalBool(!d && v || d && !v)
	case schema.Type_Which_int8:
		v := s.Uint8(capnp.DataOffset(f.Slot().Offset()))
		d := uint8(dv.Int8())
		m.marshalInt(int64(int8(v ^ d)))
	case schema.Type_Which_int16:
		v := s.Uint16(capnp.DataOffset(f.Slot().Offset() * 2))
		d := uint16(dv.Int16())
		m.marshalInt(int64(int16(v ^ d)))
	case schema.Type_Which_int32:
		v := s.Uint32(capnp.DataOffset(f.Slot().Offset() * 4))
		d := uint32(dv.Int32())
		m.marshalInt(int64(int32(v ^ d)))
	case schema.Type_Which_int64:
		v := s.Uint64(capnp.DataOffset(f.Slot().Offset() * 8))
		d := uint64(dv.Int64())
		m.marshalInt(int64(v ^ d))
	case schema.Type_Which_structType:
		p, err := s.Ptr(uint16(f.Slot().Offset()))
		if err != nil {
			return err
		}
		if !p.IsValid() {
			p, _ = dv.StructValuePtr()
		}
		return m.marshalStruct(typ.StructType().TypeId(), p.Struct())
	case schema.Type_Which_data:
		p, err := s.Ptr(uint16(f.Slot().Offset()))
		if err != nil {
			return err
		}
		if !p.IsValid() {
			b, _ := dv.Data()
			m.marshalText(b)
			return nil
		}
		m.marshalText(p.Data())
	case schema.Type_Which_text:
		p, err := s.Ptr(uint16(f.Slot().Offset()))
		if err != nil {
			return err
		}
		if !p.IsValid() {
			b, _ := dv.TextBytes()
			m.marshalText(b)
			return nil
		}
		b := p.Data()
		if len(b) > 0 {
			// Trim NUL byte
			b = b[:len(b)-1]
		}
		m.marshalText(b)
	}
	return nil
}

func codeOrderFields(s schema.Node_structNode) []schema.Field {
	list, _ := s.Fields()
	n := list.Len()
	fields := make([]schema.Field, n)
	for i := 0; i < n; i++ {
		f := list.At(i)
		fields[f.CodeOrder()] = f
	}
	return fields
}
<|endoftext|>


<commit_before>package errors

type EmptyResourceError struct{}

func (e *EmptyResourceError) Error() string {
	return "EmptyResourceError"
}
<commit_msg>change constructor for EmptyResource error<commit_after>package errors

func NewEmptyResourceError() *EmptyResourceError {
	return &EmptyResourceError{}
}

type EmptyResourceError struct{}

func (e *EmptyResourceError) Error() string {
	return "EmptyResourceError"
}
<|endoftext|>


<commit_before>// Prg1: NPRG030
// ADS2: NTIN061
// Haskell: NAIL097
package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"path"
	"regexp"
	"strconv"
	"strings"

	"github.com/iamwave/samorozvrh/backend/cache"
	"github.com/iamwave/samorozvrh/backend/sisparse"
)

const FRONTEND_DIR = "frontend/dist"
const TIME_PER_QUERY = 3000

var rootDir string

func sisQueryHandler(w http.ResponseWriter, r *http.Request) {
	query := r.URL.Path[len("/sisquery/"):]
	query = strings.Trim(query, " ")

	if len(query) > 32 {
		fmt.Fprintf(w, `{"error":"The query is too long"}`)
		return
	}

	re := regexp.MustCompile(`\w*$`)

	if !re.MatchString(query) {
		fmt.Fprintf(w, `{"error":"Query must contain only alphanumeric characters"}`)
		return
	}
	var res string
	var err error

	query = strings.ToUpper(query)
	log.Printf("Sisquery: %s", ellipsis(query, 10))
	cacheName := []string{"courses", query}

	if cache.Has(cacheName) {
		log.Println("  (using cache)")
		res, err = cache.Get(cacheName)
	} else {
		log.Println("  (querying)")
		var events [][]sisparse.Event
		events, err = sisparse.GetCourseEvents(query)
		if err == nil {
			var s []byte
			s, err = json.Marshal(events)
			if err == nil {
				res = fmt.Sprintf(`{"data":%s}`, string(s))
				err = cache.Set(cacheName, res)
			}
		}
	}

	if err != nil {
		log.Printf("Sisquery error: %s", err)
		fmt.Fprintf(w, `{"error":"%s"}`, err)
	} else {
		log.Printf(`Sisquery answer: %s`, ellipsis(res, 30))
		fmt.Fprint(w, res)
	}
}

func solverQueryHandler(w http.ResponseWriter, r *http.Request) {
	body, err := ioutil.ReadAll(r.Body)
	r.Body.Close()
	if err != nil {
		log.Printf("Solverquery error: %s", err)
		fmt.Fprintf(w, `{"error":"%s"}`, err)
		return
	}
	if len(body) == 0 {
		log.Printf("Solverquery error: %s", err)
		fmt.Fprint(w, `{"error":"Request body must be non-empty"}`)
		return
	}

	log.Printf("Solverquery: %s\n", ellipsis(string(body), 30))
	res, err := Solve(body, TIME_PER_QUERY)
	if err != nil {
		log.Printf("Solverquery error: %s", err)
		fmt.Fprintf(w, `{"error":"%s"}`, err)
	} else {
		log.Printf("Solverquery answer: %s", ellipsis(string(res), 30))
		fmt.Fprint(w, string(res))
	}
}

func main() {
	rdir := flag.String("rootdir", ".", "path to Samorozvrh root directory")
	port := flag.Int("port", 8080, "port on which to start the server")
	flag.Parse()
	rootDir = *rdir
	cache.SetRootDir(rootDir)

	http.HandleFunc("/sisquery/", sisQueryHandler)
	http.HandleFunc("/solverquery/", solverQueryHandler)

	fs := http.FileServer(http.Dir(path.Join(rootDir, FRONTEND_DIR)))
	http.Handle("/", fs)

	log.Printf("Listening on: %d", *port)
	err := http.ListenAndServe(":"+strconv.Itoa(*port), nil)
	if err != nil {
		log.Fatalf("Could not start server: %s\n", err)
	}
}

func ellipsis(s string, n int) string {
	if len(s) < n {
		return s
	} else {
		return s[:n] + "..."
	}
}
<commit_msg>Log IP of requests<commit_after>// Prg1: NPRG030
// ADS2: NTIN061
// Haskell: NAIL097
package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"path"
	"regexp"
	"strconv"
	"strings"

	"github.com/iamwave/samorozvrh/backend/cache"
	"github.com/iamwave/samorozvrh/backend/sisparse"
)

const FRONTEND_DIR = "frontend/dist"
const TIME_PER_QUERY = 3000

var rootDir string

func sisQueryHandler(w http.ResponseWriter, r *http.Request) {
	query := r.URL.Path[len("/sisquery/"):]
	query = strings.Trim(query, " ")

	if len(query) > 32 {
		fmt.Fprintf(w, `{"error":"The query is too long"}`)
		return
	}

	re := regexp.MustCompile(`\w*$`)

	if !re.MatchString(query) {
		fmt.Fprintf(w, `{"error":"Query must contain only alphanumeric characters"}`)
		return
	}
	var res string
	var err error

	query = strings.ToUpper(query)
	log.Printf("Sisquery (from %s): %s", r.RemoteAddr, ellipsis(query, 10))
	cacheName := []string{"courses", query}

	if cache.Has(cacheName) {
		log.Println("  (using cache)")
		res, err = cache.Get(cacheName)
	} else {
		log.Println("  (querying)")
		var events [][]sisparse.Event
		events, err = sisparse.GetCourseEvents(query)
		if err == nil {
			var s []byte
			s, err = json.Marshal(events)
			if err == nil {
				res = fmt.Sprintf(`{"data":%s}`, string(s))
				err = cache.Set(cacheName, res)
			}
		}
	}

	if err != nil {
		log.Printf("Sisquery error: %s", err)
		fmt.Fprintf(w, `{"error":"%s"}`, err)
	} else {
		log.Printf(`Sisquery answer: %s`, ellipsis(res, 30))
		fmt.Fprint(w, res)
	}
}

func solverQueryHandler(w http.ResponseWriter, r *http.Request) {
	body, err := ioutil.ReadAll(r.Body)
	r.Body.Close()
	if err != nil {
		log.Printf("Solverquery error: %s", err)
		fmt.Fprintf(w, `{"error":"%s"}`, err)
		return
	}
	if len(body) == 0 {
		log.Printf("Solverquery error: %s", err)
		fmt.Fprint(w, `{"error":"Request body must be non-empty"}`)
		return
	}

	log.Printf("Solverquery (from %s): %s\n", r.RemoteAddr, ellipsis(string(body), 30))
	res, err := Solve(body, TIME_PER_QUERY)
	if err != nil {
		log.Printf("Solverquery error: %s", err)
		fmt.Fprintf(w, `{"error":"%s"}`, err)
	} else {
		log.Printf("Solverquery answer: %s", ellipsis(string(res), 30))
		fmt.Fprint(w, string(res))
	}
}

func main() {
	rdir := flag.String("rootdir", ".", "path to Samorozvrh root directory")
	port := flag.Int("port", 8080, "port on which to start the server")
	flag.Parse()
	rootDir = *rdir
	cache.SetRootDir(rootDir)

	http.HandleFunc("/sisquery/", sisQueryHandler)
	http.HandleFunc("/solverquery/", solverQueryHandler)

	fs := http.FileServer(http.Dir(path.Join(rootDir, FRONTEND_DIR)))
	http.Handle("/", fs)

	log.Printf("Listening on: %d", *port)
	err := http.ListenAndServe(":"+strconv.Itoa(*port), nil)
	if err != nil {
		log.Fatalf("Could not start server: %s\n", err)
	}
}

func ellipsis(s string, n int) string {
	if len(s) < n {
		return s
	} else {
		return s[:n] + "..."
	}
}
<|endoftext|>


<commit_before>package email

import (
	"github.com/nathan-osman/go-cannon/queue"
	"github.com/nathan-osman/go-cannon/util"

	"testing"
)

var (
	addr1A  = "1@a.com"
	addr2A  = "2@a.com"
	addr1B  = "1@b.com"
	subject = "Test"
	text    = "test"
	html    = "test"
)

func TestEmailMessageCount(t *testing.T) {
	if d, err := util.NewTempDir(); err == nil {
		defer d.Delete()
		if q, err := queue.NewQueue(d.Path); err == nil {
			e := &Email{
				To: []string{addr1A, addr2A},
				Cc: []string{addr1B},
			}
			if messages, err := e.Messages(q); err == nil {
				if len(messages) != 2 {
					t.Fatalf("%d != 2", len(messages))
				}
			} else {
				t.Fatal(err)
			}
		} else {
			t.Fatal(err)
		}
	} else {
		t.Fatal(err)
	}
}
<commit_msg>Updated email test.<commit_after>package email

import (
	"github.com/nathan-osman/go-cannon/queue"
	"github.com/nathan-osman/go-cannon/util"

	"testing"
)

var (
	addr1A  = "1@a.com"
	addr2A  = "2@a.com"
	addr1B  = "1@b.com"
	subject = "Test"
	text    = "test"
	html    = "test"
)

func emailToMessages(e *Email) ([]*queue.Message, error) {
	if d, err := util.NewTempDir(); err == nil {
		defer d.Delete()
		if s, err := queue.NewStorage(d.Path); err == nil {
			if messages, err := e.Messages(s); err == nil {
				return messages, nil
			} else {
				return nil, err
			}
		} else {
			return nil, err
		}
	} else {
		return nil, err
	}
}

func TestEmailCount(t *testing.T) {
	if messages, err := emailToMessages(&Email{
		To:      []string{addr1A, addr1B},
		Cc:      []string{addr2A},
		Subject: subject,
		Text:    text,
		Html:    html,
	}); err == nil {
		if len(messages) != 2 {
			t.Fatal("%d != 2", len(messages))
		}
	} else {
		t.Fatal(err)
	}
}
<|endoftext|>