<commit_before>// BucketReader gets a list of files awaiting processing in the S3
// intake buckets and adds each item to the queue for processing.
package main

import (
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"github.com/APTrust/bagman"
	"github.com/APTrust/bagman/fluctus/client"
	"github.com/crowdmob/goamz/aws"
	"log"
	"net/http"
	"os"
	"strings"
	"time"
)

// Send S3 files to queue in batches of 500.
// Wait X milliseconds between batches. The wait time is really
// only necessary when the queue is running on an AWS small EC2
// instance, where the number of open network connections is
// severely restricted.
const (
	batchSize        = 500
	waitMilliseconds = 5000
)

var (
	config        bagman.Config
	jsonLog       *log.Logger
	messageLog    *log.Logger
	fluctusClient *client.Client
	statusCache   map[string]*bagman.ProcessStatus
)

func main() {
	err := initialize()
	if err != nil {
		fmt.Fprintf(os.Stderr, "Initialization failed for bucket_reader: %v", err)
		os.Exit(1)
	}
	run()
}

func initialize() (err error) {
	// Load the config or die.
	requestedConfig := flag.String("config", "", "configuration to run")
	flag.Parse()
	config = bagman.LoadRequestedConfig(requestedConfig)
	jsonLog, messageLog = bagman.InitLoggers(config.LogDirectory, "bucket_reader")
	fluctusClient, err = client.New(
		config.FluctusURL,
		os.Getenv("FLUCTUS_API_USER"),
		os.Getenv("FLUCTUS_API_KEY"),
		messageLog)
	return err
}

func run() {
	s3Client, err := bagman.NewS3Client(aws.USEast)
	if err != nil {
		messageLog.Println("[ERROR]", err)
		return
	}
	bucketSummaries, err := s3Client.CheckAllBuckets(config.Buckets)
	if err != nil {
		messageLog.Println("[ERROR]", err)
		return
	}
	loadStatusCache()
	url := fmt.Sprintf("%s/mput?topic=%s", config.NsqdHttpAddress,
		config.BagProcessorTopic)
	messageLog.Printf("[INFO] Sending S3 file info to %s \n", url)
	s3Files := filterLargeFiles(bucketSummaries)
	messageLog.Printf("[INFO] %d S3 Files are within our size limit\n",
		len(s3Files))
	filesToProcess := s3Files
	// SkipAlreadyProcessed will almost always be true.
	// The exception is when we want to reprocess items to test new code.
	if config.SkipAlreadyProcessed == true {
		messageLog.Println("[INFO] Skipping already processed files, because config says so")
		filesToProcess = filterProcessedFiles(s3Files)
	} else {
		messageLog.Println("[INFO] Reprocessing already processed files, because config says so")
	}
	start := 0
	end := min(len(filesToProcess), batchSize)
	messageLog.Printf("[INFO] %d Unprocessed files\n", len(filesToProcess))
	for start <= end {
		batch := filesToProcess[start:end]
		messageLog.Printf("[INFO] Queuing batch of %d items\n", len(batch))
		enqueue(url, batch)
		start = end + 1
		if start < len(filesToProcess) {
			end = min(len(filesToProcess), start+batchSize)
		}
		// Sleep so we don't max out connections on EC2 small.
		// The utility server is running a lot of other network I/O
		// in addition to our queue.
		time.Sleep(time.Millisecond * waitMilliseconds)
	}
}

// min returns the minimum of x or y. The Math package has this function
// but you have to cast to floats.
func min(x, y int) int {
	if x < y {
		return x
	} else {
		return y
	}
}

// filterLargeFiles returns only those S3 files that are not larger
// than config.MaxFileSize. This is useful when running tests and
// demos on your local machine, so that you can limit your test
// runs to files under 100k (or whatever you set in config.json).
// You don't want to pull down lots of multi-gig files when you're
// just running local tests. In production, set maxFileSize to
// zero, or to some huge value to get all files.
func filterLargeFiles(bucketSummaries []*bagman.BucketSummary) (s3Files []*bagman.S3File) {
	for _, bucketSummary := range bucketSummaries {
		for _, key := range bucketSummary.Keys {
			if config.MaxFileSize == 0 || key.Size < config.MaxFileSize {
				s3Files = append(s3Files, &bagman.S3File{bucketSummary.BucketName, key})
			}
		}
	}
	return s3Files
}

// Remove S3 files that have been processed successfully.
// No need to reprocess those!
func filterProcessedFiles(s3Files []*bagman.S3File) (filesToProcess []*bagman.S3File) {
	for _, s3File := range s3Files {
		bagDate, err := time.Parse(bagman.S3DateFormat, s3File.Key.LastModified)
		if err != nil {
			messageLog.Printf("[ERROR] Cannot parse S3File mod date '%s'. "+
				"File %s will be re-processed.",
				s3File.Key.LastModified, s3File.Key.Key)
			filesToProcess = append(filesToProcess, s3File)
			continue
		}
		etag := strings.Replace(s3File.Key.ETag, "\"", "", 2)
		status := findInStatusCache(etag, s3File.Key.Key, bagDate)
		if status == nil {
			status, err = fluctusClient.GetBagStatus(etag, s3File.Key.Key, bagDate)
		}
		if err != nil {
			messageLog.Printf("[ERROR] Cannot get Fluctus bag status for %s. "+
				"Will re-process bag. Error was %v", s3File.Key.Key, err)
			filesToProcess = append(filesToProcess, s3File)
		} else if  status == nil || status.Stage == "Receive" ||
			(status.Status == "Processing" && status.Retry == true) {
			reason := "Bag has not yet been successfully processed."
			if status == nil {
				err = createFluctusRecord(s3File)
				if err != nil {
					// TODO: Notify someone?
					messageLog.Printf("[ERROR] Could not create Fluctus ProcessedItem "+
						"for %s: %v", s3File.Key.Key, err)
				}
			}
			messageLog.Printf("[INFO] Will process bag %s: %s", s3File.Key.Key, reason)
			filesToProcess = append(filesToProcess, s3File)
		} else if status.Status != "Failed" {
			messageLog.Printf("[INFO] Skipping %s: already processed successfully.", s3File.Key.Key)
		} else if status.Retry == false {
			messageLog.Printf("[INFO] Skipping %s: retry flag is set to false.", s3File.Key.Key)
		}
	}
	return filesToProcess
}

// Loads status of all bags received in the past two hours from fluctus
// in a single call.
func loadStatusCache() {
	twoHoursAgo := time.Now().Add(time.Hour * -2)
	statusRecords, err := fluctusClient.BulkStatusGet(twoHoursAgo)
	if err != nil {
		messageLog.Println("[WARNING] Could not get bulk status records")
	} else {
		messageLog.Printf("[INFO] Got %d status records from the fluctopus\n", len(statusRecords))
		statusCache = make(map[string]*bagman.ProcessStatus, len(statusRecords))
		for i := range(statusRecords) {
			record := statusRecords[i]
			key := fmt.Sprintf("%s%s%s", record.ETag, record.Name, record.BagDate)
			statusCache[key] = record
		}
	}
}

// Finds the status of the specified tar bag in the cache that
// we retrieved from Fluctus. The cache can save us hundreds or
// thousands of HTTP calls each time the bucket reader runs.
func findInStatusCache(etag, name string, bagDate time.Time) (*bagman.ProcessStatus) {
	key := fmt.Sprintf("%s%s%s", etag, name, bagDate)
	item, exists := statusCache[key]
	if exists {
		messageLog.Printf("[INFO] Found item in cache: %s\n", name)
		return item
	}
	messageLog.Printf("[INFO] Item not in cache. Will have to ask the fluctopus for %s\n", name)
	return nil
}

func createFluctusRecord(s3File *bagman.S3File) (err error) {
	status := &bagman.ProcessStatus{}
	status.Date = time.Now().UTC()
	status.Action = "Ingest"
	status.Name = s3File.Key.Key
	bagDate, _ := time.Parse(bagman.S3DateFormat, s3File.Key.LastModified)
	status.BagDate = bagDate
	status.Bucket = s3File.BucketName
	// Strip the quotes off the ETag
	status.ETag = strings.Replace(s3File.Key.ETag, "\"", "", 2)
	status.Stage = "Receive"
	status.Status = "Succeeded"
	status.Note = "Item is in receiving bucket. Processing has not started."
	status.Institution = bagman.OwnerOf(s3File.BucketName)
	status.Outcome = status.Status

	// Retry should be true until we know for sure that there
	// is something wrong with the bag.
	status.Retry = true

	err = fluctusClient.UpdateBagStatus(status)
	if err != nil {
		return err
	}
	messageLog.Printf("[INFO] Created Fluctus ProcessedItem for %s\n",
		s3File.Key.Key)
	return nil
}

// enqueue adds a batch of items to the nsqd work queue
func enqueue(url string, s3Files []*bagman.S3File) {
	jsonData := make([]string, len(s3Files))
	for i, s3File := range s3Files {
		json, err := json.Marshal(s3File)
		if err != nil {
			messageLog.Printf("[ERROR] Error marshalling s3 file to JSON: %v", err)
		} else {
			jsonData[i] = string(json)
			messageLog.Println("[INFO]", "Put", s3File.Key.Key, "into fetch queue")
		}
	}
	batch := strings.Join(jsonData, "\n")
	resp, err := http.Post(url, "application/json", bytes.NewBuffer([]byte(batch)))
	if err != nil {
		messageLog.Printf("[ERROR] nsqd returned an error: %v", err)
	}
	if resp == nil {
		msg := "[ERROR] No response from nsqd. Is it running? bucket_reader is quitting."
		messageLog.Printf(msg)
		fmt.Println(msg)
		os.Exit(1)
	} else if resp.StatusCode != 200 {
		messageLog.Printf("[ERROR] nsqd returned status code %d on last mput", resp.StatusCode)
	}
}
<commit_msg>Don't requeue items in Receive state, since queue is now stable<commit_after>// BucketReader gets a list of files awaiting processing in the S3
// intake buckets and adds each item to the queue for processing.
package main

import (
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"github.com/APTrust/bagman"
	"github.com/APTrust/bagman/fluctus/client"
	"github.com/crowdmob/goamz/aws"
	"log"
	"net/http"
	"os"
	"strings"
	"time"
)

// Send S3 files to queue in batches of 500.
// Wait X milliseconds between batches. The wait time is really
// only necessary when the queue is running on an AWS small EC2
// instance, where the number of open network connections is
// severely restricted.
const (
	batchSize        = 500
	waitMilliseconds = 5000
)

var (
	config        bagman.Config
	jsonLog       *log.Logger
	messageLog    *log.Logger
	fluctusClient *client.Client
	statusCache   map[string]*bagman.ProcessStatus
)

func main() {
	err := initialize()
	if err != nil {
		fmt.Fprintf(os.Stderr, "Initialization failed for bucket_reader: %v", err)
		os.Exit(1)
	}
	run()
}

func initialize() (err error) {
	// Load the config or die.
	requestedConfig := flag.String("config", "", "configuration to run")
	flag.Parse()
	config = bagman.LoadRequestedConfig(requestedConfig)
	jsonLog, messageLog = bagman.InitLoggers(config.LogDirectory, "bucket_reader")
	fluctusClient, err = client.New(
		config.FluctusURL,
		os.Getenv("FLUCTUS_API_USER"),
		os.Getenv("FLUCTUS_API_KEY"),
		messageLog)
	return err
}

func run() {
	s3Client, err := bagman.NewS3Client(aws.USEast)
	if err != nil {
		messageLog.Println("[ERROR]", err)
		return
	}
	bucketSummaries, err := s3Client.CheckAllBuckets(config.Buckets)
	if err != nil {
		messageLog.Println("[ERROR]", err)
		return
	}
	loadStatusCache()
	url := fmt.Sprintf("%s/mput?topic=%s", config.NsqdHttpAddress,
		config.BagProcessorTopic)
	messageLog.Printf("[INFO] Sending S3 file info to %s \n", url)
	s3Files := filterLargeFiles(bucketSummaries)
	messageLog.Printf("[INFO] %d S3 Files are within our size limit\n",
		len(s3Files))
	filesToProcess := s3Files
	// SkipAlreadyProcessed will almost always be true.
	// The exception is when we want to reprocess items to test new code.
	if config.SkipAlreadyProcessed == true {
		messageLog.Println("[INFO] Skipping already processed files, because config says so")
		filesToProcess = filterProcessedFiles(s3Files)
	} else {
		messageLog.Println("[INFO] Reprocessing already processed files, because config says so")
	}
	start := 0
	end := min(len(filesToProcess), batchSize)
	messageLog.Printf("[INFO] %d Unprocessed files\n", len(filesToProcess))
	for start <= end {
		batch := filesToProcess[start:end]
		messageLog.Printf("[INFO] Queuing batch of %d items\n", len(batch))
		enqueue(url, batch)
		start = end + 1
		if start < len(filesToProcess) {
			end = min(len(filesToProcess), start+batchSize)
		}
		// Sleep so we don't max out connections on EC2 small.
		// The utility server is running a lot of other network I/O
		// in addition to our queue.
		time.Sleep(time.Millisecond * waitMilliseconds)
	}
}

// min returns the minimum of x or y. The Math package has this function
// but you have to cast to floats.
func min(x, y int) int {
	if x < y {
		return x
	} else {
		return y
	}
}

// filterLargeFiles returns only those S3 files that are not larger
// than config.MaxFileSize. This is useful when running tests and
// demos on your local machine, so that you can limit your test
// runs to files under 100k (or whatever you set in config.json).
// You don't want to pull down lots of multi-gig files when you're
// just running local tests. In production, set maxFileSize to
// zero, or to some huge value to get all files.
func filterLargeFiles(bucketSummaries []*bagman.BucketSummary) (s3Files []*bagman.S3File) {
	for _, bucketSummary := range bucketSummaries {
		for _, key := range bucketSummary.Keys {
			if config.MaxFileSize == 0 || key.Size < config.MaxFileSize {
				s3Files = append(s3Files, &bagman.S3File{bucketSummary.BucketName, key})
			}
		}
	}
	return s3Files
}

// Remove S3 files that have been processed successfully.
// No need to reprocess those!
func filterProcessedFiles(s3Files []*bagman.S3File) (filesToProcess []*bagman.S3File) {
	for _, s3File := range s3Files {
		bagDate, err := time.Parse(bagman.S3DateFormat, s3File.Key.LastModified)
		if err != nil {
			messageLog.Printf("[ERROR] Cannot parse S3File mod date '%s'. "+
				"File %s will be re-processed.",
				s3File.Key.LastModified, s3File.Key.Key)
			filesToProcess = append(filesToProcess, s3File)
			continue
		}
		etag := strings.Replace(s3File.Key.ETag, "\"", "", 2)
		status := findInStatusCache(etag, s3File.Key.Key, bagDate)
		if status == nil {
			status, err = fluctusClient.GetBagStatus(etag, s3File.Key.Key, bagDate)
		}
		if err != nil {
			messageLog.Printf("[ERROR] Cannot get Fluctus bag status for %s. "+
				"Will re-process bag. Error was %v", s3File.Key.Key, err)
			filesToProcess = append(filesToProcess, s3File)
		} else if  status == nil || (status.Status == "Processing" && status.Retry == true) {
			reason := "Bag has not yet been successfully processed."
			if status == nil {
				err = createFluctusRecord(s3File)
				if err != nil {
					// TODO: Notify someone?
					messageLog.Printf("[ERROR] Could not create Fluctus ProcessedItem "+
						"for %s: %v", s3File.Key.Key, err)
				}
			}
			messageLog.Printf("[INFO] Will process bag %s: %s", s3File.Key.Key, reason)
			filesToProcess = append(filesToProcess, s3File)
		} else if status.Status != "Failed" {
			messageLog.Printf("[INFO] Skipping %s: already processed successfully.", s3File.Key.Key)
		} else if status.Retry == false {
			messageLog.Printf("[INFO] Skipping %s: retry flag is set to false.", s3File.Key.Key)
		}
	}
	return filesToProcess
}

// Loads status of all bags received in the past two hours from fluctus
// in a single call.
func loadStatusCache() {
	twoHoursAgo := time.Now().Add(time.Hour * -2)
	statusRecords, err := fluctusClient.BulkStatusGet(twoHoursAgo)
	if err != nil {
		messageLog.Println("[WARNING] Could not get bulk status records")
	} else {
		messageLog.Printf("[INFO] Got %d status records from the fluctopus\n", len(statusRecords))
		statusCache = make(map[string]*bagman.ProcessStatus, len(statusRecords))
		for i := range(statusRecords) {
			record := statusRecords[i]
			key := fmt.Sprintf("%s%s%s", record.ETag, record.Name, record.BagDate)
			statusCache[key] = record
		}
	}
}

// Finds the status of the specified tar bag in the cache that
// we retrieved from Fluctus. The cache can save us hundreds or
// thousands of HTTP calls each time the bucket reader runs.
func findInStatusCache(etag, name string, bagDate time.Time) (*bagman.ProcessStatus) {
	key := fmt.Sprintf("%s%s%s", etag, name, bagDate)
	item, exists := statusCache[key]
	if exists {
		messageLog.Printf("[INFO] Found item in cache: %s\n", name)
		return item
	}
	messageLog.Printf("[INFO] Item not in cache. Will have to ask the fluctopus for %s\n", name)
	return nil
}

func createFluctusRecord(s3File *bagman.S3File) (err error) {
	status := &bagman.ProcessStatus{}
	status.Date = time.Now().UTC()
	status.Action = "Ingest"
	status.Name = s3File.Key.Key
	bagDate, _ := time.Parse(bagman.S3DateFormat, s3File.Key.LastModified)
	status.BagDate = bagDate
	status.Bucket = s3File.BucketName
	// Strip the quotes off the ETag
	status.ETag = strings.Replace(s3File.Key.ETag, "\"", "", 2)
	status.Stage = "Receive"
	status.Status = "Succeeded"
	status.Note = "Item is in receiving bucket. Processing has not started."
	status.Institution = bagman.OwnerOf(s3File.BucketName)
	status.Outcome = status.Status

	// Retry should be true until we know for sure that there
	// is something wrong with the bag.
	status.Retry = true

	err = fluctusClient.UpdateBagStatus(status)
	if err != nil {
		return err
	}
	messageLog.Printf("[INFO] Created Fluctus ProcessedItem for %s\n",
		s3File.Key.Key)
	return nil
}

// enqueue adds a batch of items to the nsqd work queue
func enqueue(url string, s3Files []*bagman.S3File) {
	jsonData := make([]string, len(s3Files))
	for i, s3File := range s3Files {
		json, err := json.Marshal(s3File)
		if err != nil {
			messageLog.Printf("[ERROR] Error marshalling s3 file to JSON: %v", err)
		} else {
			jsonData[i] = string(json)
			messageLog.Println("[INFO]", "Put", s3File.Key.Key, "into fetch queue")
		}
	}
	batch := strings.Join(jsonData, "\n")
	resp, err := http.Post(url, "application/json", bytes.NewBuffer([]byte(batch)))
	if err != nil {
		messageLog.Printf("[ERROR] nsqd returned an error: %v", err)
	}
	if resp == nil {
		msg := "[ERROR] No response from nsqd. Is it running? bucket_reader is quitting."
		messageLog.Printf(msg)
		fmt.Println(msg)
		os.Exit(1)
	} else if resp.StatusCode != 200 {
		messageLog.Printf("[ERROR] nsqd returned status code %d on last mput", resp.StatusCode)
	}
}
<|endoftext|>


<commit_before>package tcp

import (
	"context"
	"net"
	"time"

	logging "github.com/ipfs/go-log"
	peer "github.com/libp2p/go-libp2p-peer"
	tpt "github.com/libp2p/go-libp2p-transport"
	tptu "github.com/libp2p/go-libp2p-transport-upgrader"
	rtpt "github.com/libp2p/go-reuseport-transport"
	ma "github.com/multiformats/go-multiaddr"
	manet "github.com/multiformats/go-multiaddr-net"
	mafmt "github.com/whyrusleeping/mafmt"
)

// DefaultConnectTimeout is the (default) maximum amount of time the TCP
// transport will spend on the initial TCP connect before giving up.
var DefaultConnectTimeout = 5 * time.Second

var log = logging.Logger("tcp-tpt")

// try to set linger on the connection, if possible.
func tryLinger(conn net.Conn, sec int) {
	if lingerConn, ok := conn.(interface {
		SetLinger(int) error
	}); ok {
		_ = lingerConn.SetLinger(sec)
	}
}

type lingerListener struct {
	manet.Listener
	sec int
}

func (ll *lingerListener) Accept() (manet.Conn, error) {
	c, err := ll.Listener.Accept()
	if err != nil {
		return nil, err
	}
	tryLinger(c, ll.sec)
	return c, nil
}

// TcpTransport is the TCP transport.
type TcpTransport struct {
	// Connection upgrader for upgrading insecure stream connections to
	// secure multiplex connections.
	Upgrader *tptu.Upgrader

	// Explicitly disable reuseport.
	DisableReuseport bool

	// TCP connect timeout
	ConnectTimeout time.Duration

	reuse rtpt.Transport
}

var _ tpt.Transport = &TcpTransport{}

// NewTCPTransport creates a tcp transport object that tracks dialers and listeners
// created. It represents an entire tcp stack (though it might not necessarily be)
func NewTCPTransport(upgrader *tptu.Upgrader) *TcpTransport {
	return &TcpTransport{Upgrader: upgrader, ConnectTimeout: DefaultConnectTimeout}
}

// CanDial returns true if this transport believes it can dial the given
// multiaddr.
func (t *TcpTransport) CanDial(addr ma.Multiaddr) bool {
	return mafmt.TCP.Matches(addr)
}

func (t *TcpTransport) maDial(ctx context.Context, raddr ma.Multiaddr) (manet.Conn, error) {
	// Apply the deadline iff applicable
	if t.ConnectTimeout > 0 {
		deadline := time.Now().Add(t.ConnectTimeout)
		if d, ok := ctx.Deadline(); !ok || deadline.Before(d) {
			var cancel func()
			ctx, cancel = context.WithDeadline(ctx, deadline)
			defer cancel()
		}
	}

	if t.UseReuseport() {
		return t.reuse.DialContext(ctx, raddr)
	}
	var d manet.Dialer
	return d.DialContext(ctx, raddr)
}

// Dial dials the peer at the remote address.
func (t *TcpTransport) Dial(ctx context.Context, raddr ma.Multiaddr, p peer.ID) (tpt.Conn, error) {
	conn, err := t.maDial(ctx, raddr)
	if err != nil {
		return nil, err
	}
	// Set linger to 0 so we never get stuck in the TIME-WAIT state. When
	// linger is 0, connections are _reset_ instead of closed with a FIN.
	// This means we can immediately reuse the 5-tuple and reconnect.
	tryLinger(conn, 0)
	return t.Upgrader.UpgradeOutbound(ctx, t, conn, p)
}

// UseReuseport returns true if reuseport is enabled and available.
func (t *TcpTransport) UseReuseport() bool {
	return !t.DisableReuseport && ReuseportIsAvailable()
}

func (t *TcpTransport) maListen(laddr ma.Multiaddr) (manet.Listener, error) {
	if t.UseReuseport() {
		return t.reuse.Listen(laddr)
	}
	return manet.Listen(laddr)
}

// Listen listens on the given multiaddr.
func (t *TcpTransport) Listen(laddr ma.Multiaddr) (tpt.Listener, error) {
	list, err := t.maListen(laddr)
	if err != nil {
		return nil, err
	}
	list = &lingerListener{list, 0}
	return t.Upgrader.UpgradeListener(t, list), nil
}

// Protocols returns the list of terminal protocols this transport can dial.
func (t *TcpTransport) Protocols() []int {
	return []int{ma.P_TCP}
}

// Proxy always returns false for the TCP transport.
func (t *TcpTransport) Proxy() bool {
	return false
}

func (t *TcpTransport) String() string {
	return "TCP"
}
<commit_msg>chore: remove inline interface<commit_after>package tcp

import (
	"context"
	"net"
	"time"

	logging "github.com/ipfs/go-log"
	peer "github.com/libp2p/go-libp2p-peer"
	tpt "github.com/libp2p/go-libp2p-transport"
	tptu "github.com/libp2p/go-libp2p-transport-upgrader"
	rtpt "github.com/libp2p/go-reuseport-transport"
	ma "github.com/multiformats/go-multiaddr"
	manet "github.com/multiformats/go-multiaddr-net"
	mafmt "github.com/whyrusleeping/mafmt"
)

// DefaultConnectTimeout is the (default) maximum amount of time the TCP
// transport will spend on the initial TCP connect before giving up.
var DefaultConnectTimeout = 5 * time.Second

var log = logging.Logger("tcp-tpt")

// try to set linger on the connection, if possible.
func tryLinger(conn net.Conn, sec int) {
	type canLinger interface {
		SetLinger(int) error
	}

	if lingerConn, ok := conn.(canLinger); ok {
		_ = lingerConn.SetLinger(sec)
	}
}

type lingerListener struct {
	manet.Listener
	sec int
}

func (ll *lingerListener) Accept() (manet.Conn, error) {
	c, err := ll.Listener.Accept()
	if err != nil {
		return nil, err
	}
	tryLinger(c, ll.sec)
	return c, nil
}

// TcpTransport is the TCP transport.
type TcpTransport struct {
	// Connection upgrader for upgrading insecure stream connections to
	// secure multiplex connections.
	Upgrader *tptu.Upgrader

	// Explicitly disable reuseport.
	DisableReuseport bool

	// TCP connect timeout
	ConnectTimeout time.Duration

	reuse rtpt.Transport
}

var _ tpt.Transport = &TcpTransport{}

// NewTCPTransport creates a tcp transport object that tracks dialers and listeners
// created. It represents an entire tcp stack (though it might not necessarily be)
func NewTCPTransport(upgrader *tptu.Upgrader) *TcpTransport {
	return &TcpTransport{Upgrader: upgrader, ConnectTimeout: DefaultConnectTimeout}
}

// CanDial returns true if this transport believes it can dial the given
// multiaddr.
func (t *TcpTransport) CanDial(addr ma.Multiaddr) bool {
	return mafmt.TCP.Matches(addr)
}

func (t *TcpTransport) maDial(ctx context.Context, raddr ma.Multiaddr) (manet.Conn, error) {
	// Apply the deadline iff applicable
	if t.ConnectTimeout > 0 {
		deadline := time.Now().Add(t.ConnectTimeout)
		if d, ok := ctx.Deadline(); !ok || deadline.Before(d) {
			var cancel func()
			ctx, cancel = context.WithDeadline(ctx, deadline)
			defer cancel()
		}
	}

	if t.UseReuseport() {
		return t.reuse.DialContext(ctx, raddr)
	}
	var d manet.Dialer
	return d.DialContext(ctx, raddr)
}

// Dial dials the peer at the remote address.
func (t *TcpTransport) Dial(ctx context.Context, raddr ma.Multiaddr, p peer.ID) (tpt.Conn, error) {
	conn, err := t.maDial(ctx, raddr)
	if err != nil {
		return nil, err
	}
	// Set linger to 0 so we never get stuck in the TIME-WAIT state. When
	// linger is 0, connections are _reset_ instead of closed with a FIN.
	// This means we can immediately reuse the 5-tuple and reconnect.
	tryLinger(conn, 0)
	return t.Upgrader.UpgradeOutbound(ctx, t, conn, p)
}

// UseReuseport returns true if reuseport is enabled and available.
func (t *TcpTransport) UseReuseport() bool {
	return !t.DisableReuseport && ReuseportIsAvailable()
}

func (t *TcpTransport) maListen(laddr ma.Multiaddr) (manet.Listener, error) {
	if t.UseReuseport() {
		return t.reuse.Listen(laddr)
	}
	return manet.Listen(laddr)
}

// Listen listens on the given multiaddr.
func (t *TcpTransport) Listen(laddr ma.Multiaddr) (tpt.Listener, error) {
	list, err := t.maListen(laddr)
	if err != nil {
		return nil, err
	}
	list = &lingerListener{list, 0}
	return t.Upgrader.UpgradeListener(t, list), nil
}

// Protocols returns the list of terminal protocols this transport can dial.
func (t *TcpTransport) Protocols() []int {
	return []int{ma.P_TCP}
}

// Proxy always returns false for the TCP transport.
func (t *TcpTransport) Proxy() bool {
	return false
}

func (t *TcpTransport) String() string {
	return "TCP"
}
<|endoftext|>


<commit_before>/**
 * (c) 2014, Tonnerre Lombard <tonnerre@ancient-solutions.com>,
 *	     Ancient Solutions. All rights reserved.
 *
 * Redistribution and use in source  and binary forms, with or without
 * modification, are permitted  provided that the following conditions
 * are met:
 *
 * * Redistributions of  source code  must retain the  above copyright
 *   notice, this list of conditions and the following disclaimer.
 * * Redistributions in binary form must reproduce the above copyright
 *   notice, this  list of conditions and the  following disclaimer in
 *   the  documentation  and/or  other  materials  provided  with  the
 *   distribution.
 * * Neither  the  name  of  Ancient Solutions  nor  the  name  of its
 *   contributors may  be used to endorse or  promote products derived
 *   from this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * "AS IS"  AND ANY EXPRESS  OR IMPLIED WARRANTIES  OF MERCHANTABILITY
 * AND FITNESS  FOR A PARTICULAR  PURPOSE ARE DISCLAIMED. IN  NO EVENT
 * SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
 * INDIRECT, INCIDENTAL, SPECIAL,  EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED  TO, PROCUREMENT OF SUBSTITUTE GOODS OR
 * SERVICES; LOSS OF USE,  DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
 * STRICT  LIABILITY,  OR  TORT  (INCLUDING NEGLIGENCE  OR  OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
 * OF THE POSSIBILITY OF SUCH DAMAGE.
 */

// Client library for blubberstore end users.
package blubberstore

import (
	"bytes"
	"code.google.com/p/goprotobuf/proto"
	"errors"
	"io"
	"io/ioutil"
)

// A number of hosts have been contacted but storing the block
// failed on all of them (e.g. because they could not be reached).
// This means that no data has been written whatsoever.
var Err_NoHostsReached = errors.New("Unable to copy the block to even one server - please try again")

// Less than a quorum of the requested nodes have been written.
// This means that the data may disappear again in case of
// outages.
var Err_NoQuorumReached = errors.New("No quorum could be reached")

// The write failed to reach a fully replicated set, but a quorum has
// been achieved. This may be safe to ignore, but the file should
// be distributed better.
var Err_IncompleteWrite = errors.New("Write did not complete to a sufficiently large quorum")

// BlubberStore client which handles replication, quorums and server
// lookups transparently.
type BlubberStoreClient struct {
	directoryClient   *BlubberDirectoryClient
	cert, key, cacert string
	insecure          bool
	errorLog          chan error
}

/*
Create a new Blubber Store client which will talk independently to
the directory service, discover and talk to blubber backends, and
handle timeouts etc.

uri is an URI pointing to the blubber directory.
cert and key are the path to a PEM encoded X.509 certificate and
private key.
cacert is the path to a PEM encoded X.509 CA certificate.
Setting insecure to true disables the use of certificates.
errlog is a channel for reporting non-fatal errors which occur during
operation back to the clients.
*/
func NewBlubberStoreClient(uri, cert, key, cacert string,
	insecure bool, errorlog chan error) (*BlubberStoreClient, error) {
	var dirclient *BlubberDirectoryClient
	var err error

	dirclient, err = NewBlubberDirectoryClient(
		uri, cert, key, cacert, insecure)
	if err != nil {
		return nil, err
	}

	return &BlubberStoreClient{
		directoryClient: dirclient,
		cert:            cert,
		key:             key,
		cacert:          cacert,
		insecure:        insecure,
		errorLog:        errorlog,
	}, nil
}

/*
Write the given data to a blob with the given ID, overwriting it if
specified. This will ensure that there are more than replication/2
copies of the block.
overwrite=false will disable lookups for current block holders.
*/
func (b *BlubberStoreClient) StoreBlock(id []byte, data io.Reader,
	overwrite bool, replication int) error {
	var client *BlubberRPCClient
	var block BlockWithData
	var serverlist []string
	var bs BlockSource
	var attempt int
	var servers []string
	var server string
	var err error

	if overwrite {
		var bid BlockId
		var list *BlockHolderList

		bid.BlockId = make([]byte, len(id))
		copy(bid.BlockId, id)
		list, err = b.directoryClient.LookupBlob(bid)
		if err != nil {
			return err
		}

		for _, server = range list.GetHostPort() {
			servers = append(servers, server)
		}
	}

	if len(servers) < replication {
		var freeHostReq FreeHostsRequest
		var list *BlockHolderList

		freeHostReq.NumHosts = proto.Int32(int32(replication - len(servers)))
		list, err = b.directoryClient.GetFreeHosts(freeHostReq)
		if err != nil {
			return err
		}

		for _, server = range list.GetHostPort() {
			servers = append(servers, server)
		}
	}

	// TODO(tonnerre): use the HTTP interface so we can handle large blobs.
	block.BlockId = make([]byte, len(id))
	copy(block.BlockId, id)
	serverlist = make([]string, 0)

	block.BlockData, err = ioutil.ReadAll(data)
	if err != nil {
		return err
	}

	if len(block.BlockData) == 0 {
		return errors.New("Zero-length input was detected")
	}

	// TODO(tonnerre): write this to be asynchronous and get the result from
	// a channel.
	for _, server = range servers {
		client, err = NewBlubberRPCClient(
			"tcp://"+server, b.cert, b.key, b.cacert, b.insecure)
		if err != nil {
			b.errorLog <- err
			continue
		}

		err = client.StoreBlob(block)
		if err != nil {
			b.errorLog <- err
			continue
		}

		serverlist = append(serverlist, server)
	}

	if len(serverlist) == 0 {
		return Err_NoHostsReached
	}

	// Try to copy from the existing set to a bunch of other hosts.
	for len(serverlist) < replication && attempt < replication {
		var freeHostReq FreeHostsRequest
		var list *BlockHolderList
		var sid = attempt % len(serverlist)

		// Try twice as many servers as we had left, since we may just get
		// the ones back which we tried before.
		freeHostReq.NumHosts = proto.Int32(int32(
			2 * (replication - len(serverlist))))
		list, err = b.directoryClient.GetFreeHosts(freeHostReq)
		if err != nil {
			return err
		}

		// Start the server list from scratch.
		servers = make([]string, 0)
		for _, server = range list.GetHostPort() {
			servers = append(servers, server)
		}

		bs.BlockId = make([]byte, len(id))
		copy(bs.BlockId, id)

		// TODO(tonnerre): write this to be asynchronous and get the result
		// from a channel.
		for _, server = range servers {
			client, err = NewBlubberRPCClient(
				"tcp://"+server, b.cert, b.key, b.cacert, b.insecure)
			if err != nil {
				b.errorLog <- err
				continue
			}

			bs.SourceHost = proto.String(serverlist[sid])
			err = client.CopyBlob(bs)
			if err != nil {
				b.errorLog <- err
				continue
			}

			serverlist = append(serverlist, server)
		}

		attempt += 1
	}

	// We give up (or we're done).
	if len(serverlist) < (replication / 2) {
		return Err_NoQuorumReached
	}

	if len(serverlist) < replication {
		return Err_IncompleteWrite
	}

	return nil
}

/*
Read the blob with the given ID.
*/
func (b *BlubberStoreClient) RetrieveBlob(id []byte) (io.Reader, error) {
	var holders *BlockHolderList
	var bid BlockId
	var server string
	var err error

	bid.BlockId = make([]byte, len(id))
	copy(bid.BlockId, id)
	holders, err = b.directoryClient.LookupBlob(bid)
	if err != nil {
		return nil, err
	}

	for _, server = range holders.HostPort {
		var client *BlubberRPCClient
		var bwd BlockWithData

		// TODO(tonnerre): use the HTTP interface so we can handle large
		// blobs.
		client, err = NewBlubberRPCClient(
			"tcp://"+server, b.cert, b.key, b.cacert, b.insecure)
		if err != nil {
			b.errorLog <- err
			continue
		}

		bwd, err = client.RetrieveBlob(bid)
		if err != nil {
			b.errorLog <- err
			continue
		}

		return bytes.NewReader(bwd.GetBlockData()), nil
	}

	return nil, Err_NoHostsReached
}
<commit_msg>Improve comments of the client API.<commit_after>/**
 * (c) 2014, Tonnerre Lombard <tonnerre@ancient-solutions.com>,
 *	     Ancient Solutions. All rights reserved.
 *
 * Redistribution and use in source  and binary forms, with or without
 * modification, are permitted  provided that the following conditions
 * are met:
 *
 * * Redistributions of  source code  must retain the  above copyright
 *   notice, this list of conditions and the following disclaimer.
 * * Redistributions in binary form must reproduce the above copyright
 *   notice, this  list of conditions and the  following disclaimer in
 *   the  documentation  and/or  other  materials  provided  with  the
 *   distribution.
 * * Neither  the  name  of  Ancient Solutions  nor  the  name  of its
 *   contributors may  be used to endorse or  promote products derived
 *   from this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * "AS IS"  AND ANY EXPRESS  OR IMPLIED WARRANTIES  OF MERCHANTABILITY
 * AND FITNESS  FOR A PARTICULAR  PURPOSE ARE DISCLAIMED. IN  NO EVENT
 * SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
 * INDIRECT, INCIDENTAL, SPECIAL,  EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED  TO, PROCUREMENT OF SUBSTITUTE GOODS OR
 * SERVICES; LOSS OF USE,  DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
 * STRICT  LIABILITY,  OR  TORT  (INCLUDING NEGLIGENCE  OR  OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
 * OF THE POSSIBILITY OF SUCH DAMAGE.
 */

// Client library for blubberstore end users.
//
// By design, you will most likely want to use the BlubberStoreClient
// class, which is part of this package and makes use of all the other
// packages to seamlessly provide access to blobs stored on remote hosts
// which are looked up for you in the blob directory.
package blubberstore

import (
	"bytes"
	"code.google.com/p/goprotobuf/proto"
	"errors"
	"io"
	"io/ioutil"
)

// A number of hosts have been contacted but storing the block
// failed on all of them (e.g. because they could not be reached).
// This means that no data has been written whatsoever.
var Err_NoHostsReached = errors.New("Unable to copy the block to even one server - please try again")

// Less than a quorum of the requested nodes have been written.
// This means that the data may disappear again in case of
// outages.
var Err_NoQuorumReached = errors.New("No quorum could be reached")

// The write failed to reach a fully replicated set, but a quorum has
// been achieved. This may be safe to ignore, but the file should
// be distributed better.
var Err_IncompleteWrite = errors.New("Write did not complete to a sufficiently large quorum")

// BlubberStore client which handles replication, quorums and server
// lookups transparently.
type BlubberStoreClient struct {
	directoryClient   *BlubberDirectoryClient
	cert, key, cacert string
	insecure          bool
	errorLog          chan error
}

/*
Create a new Blubber Store client which will talk independently to
the directory service, discover and talk to blubber backends, and
handle timeouts etc.

uri is an URI pointing to the blubber directory.
cert and key are the path to a PEM encoded X.509 certificate and
private key.
cacert is the path to a PEM encoded X.509 CA certificate.
Setting insecure to true disables the use of certificates.
errlog is a channel for reporting non-fatal errors which occur during
operation back to the clients.
*/
func NewBlubberStoreClient(uri, cert, key, cacert string,
	insecure bool, errorlog chan error) (*BlubberStoreClient, error) {
	var dirclient *BlubberDirectoryClient
	var err error

	dirclient, err = NewBlubberDirectoryClient(
		uri, cert, key, cacert, insecure)
	if err != nil {
		return nil, err
	}

	return &BlubberStoreClient{
		directoryClient: dirclient,
		cert:            cert,
		key:             key,
		cacert:          cacert,
		insecure:        insecure,
		errorLog:        errorlog,
	}, nil
}

/*
Write the given data to a blob with the given ID, overwriting it if
specified. This will ensure that there are more than replication/2
copies of the block.
overwrite=false will disable lookups for current block holders.
*/
func (b *BlubberStoreClient) StoreBlock(id []byte, data io.Reader,
	overwrite bool, replication int) error {
	var client *BlubberRPCClient
	var block BlockWithData
	var serverlist []string
	var bs BlockSource
	var attempt int
	var servers []string
	var server string
	var err error

	if overwrite {
		var bid BlockId
		var list *BlockHolderList

		bid.BlockId = make([]byte, len(id))
		copy(bid.BlockId, id)
		list, err = b.directoryClient.LookupBlob(bid)
		if err != nil {
			return err
		}

		for _, server = range list.GetHostPort() {
			servers = append(servers, server)
		}
	}

	if len(servers) < replication {
		var freeHostReq FreeHostsRequest
		var list *BlockHolderList

		freeHostReq.NumHosts = proto.Int32(int32(replication - len(servers)))
		list, err = b.directoryClient.GetFreeHosts(freeHostReq)
		if err != nil {
			return err
		}

		for _, server = range list.GetHostPort() {
			servers = append(servers, server)
		}
	}

	// TODO(tonnerre): use the HTTP interface so we can handle large blobs.
	block.BlockId = make([]byte, len(id))
	copy(block.BlockId, id)
	serverlist = make([]string, 0)

	block.BlockData, err = ioutil.ReadAll(data)
	if err != nil {
		return err
	}

	if len(block.BlockData) == 0 {
		return errors.New("Zero-length input was detected")
	}

	// TODO(tonnerre): write this to be asynchronous and get the result from
	// a channel.
	for _, server = range servers {
		client, err = NewBlubberRPCClient(
			"tcp://"+server, b.cert, b.key, b.cacert, b.insecure)
		if err != nil {
			b.errorLog <- err
			continue
		}

		err = client.StoreBlob(block)
		if err != nil {
			b.errorLog <- err
			continue
		}

		serverlist = append(serverlist, server)
	}

	if len(serverlist) == 0 {
		return Err_NoHostsReached
	}

	// Try to copy from the existing set to a bunch of other hosts.
	for len(serverlist) < replication && attempt < replication {
		var freeHostReq FreeHostsRequest
		var list *BlockHolderList
		var sid = attempt % len(serverlist)

		// Try twice as many servers as we had left, since we may just get
		// the ones back which we tried before.
		freeHostReq.NumHosts = proto.Int32(int32(
			2 * (replication - len(serverlist))))
		list, err = b.directoryClient.GetFreeHosts(freeHostReq)
		if err != nil {
			return err
		}

		// Start the server list from scratch.
		servers = make([]string, 0)
		for _, server = range list.GetHostPort() {
			servers = append(servers, server)
		}

		bs.BlockId = make([]byte, len(id))
		copy(bs.BlockId, id)

		// TODO(tonnerre): write this to be asynchronous and get the result
		// from a channel.
		for _, server = range servers {
			client, err = NewBlubberRPCClient(
				"tcp://"+server, b.cert, b.key, b.cacert, b.insecure)
			if err != nil {
				b.errorLog <- err
				continue
			}

			bs.SourceHost = proto.String(serverlist[sid])
			err = client.CopyBlob(bs)
			if err != nil {
				b.errorLog <- err
				continue
			}

			serverlist = append(serverlist, server)
		}

		attempt += 1
	}

	// We give up (or we're done).
	if len(serverlist) < (replication / 2) {
		return Err_NoQuorumReached
	}

	if len(serverlist) < replication {
		return Err_IncompleteWrite
	}

	return nil
}

/*
Read the blob with the given ID. The blob will be looked up for you in the
block directory and a read request will be made to an appropriate server.
*/
func (b *BlubberStoreClient) RetrieveBlob(id []byte) (io.Reader, error) {
	var holders *BlockHolderList
	var bid BlockId
	var server string
	var err error

	bid.BlockId = make([]byte, len(id))
	copy(bid.BlockId, id)
	holders, err = b.directoryClient.LookupBlob(bid)
	if err != nil {
		return nil, err
	}

	// TODO(tonnerre): Don't just go by random, choose the host which
	// seems most reasonable (most frequent agreed-upon version / majority
	// vote, distance vector, etc).
	for _, server = range holders.HostPort {
		var client *BlubberRPCClient
		var bwd BlockWithData

		// TODO(tonnerre): use the HTTP interface so we can handle large
		// blobs.
		client, err = NewBlubberRPCClient(
			"tcp://"+server, b.cert, b.key, b.cacert, b.insecure)
		if err != nil {
			b.errorLog <- err
			continue
		}

		bwd, err = client.RetrieveBlob(bid)
		if err != nil {
			b.errorLog <- err
			continue
		}

		return bytes.NewReader(bwd.GetBlockData()), nil
	}

	return nil, Err_NoHostsReached
}
<|endoftext|>


<commit_before>package app

import (
	"fmt"
	"os"
	_ "strconv"
	"strings"

	car "github.com/mohae/carchivum"
	"github.com/mohae/contour"
	jww "github.com/spf13/jwalterweatherman"
)

func Create(destination string, sources ...string) (string, error) {
	var err error
	var message string

	fmt.Printf("\nCreate %q from %v\n", destination, sources)

	switch contour.GetString(Format) {
	case "tar":
		message, err = createTar(destination, sources...)
	case "zip":
		message, err = createZip(destination, sources...)
	default:
		err = fmt.Errorf("%s not supported", contour.GetString(Format))
	}

	if err != nil {
		jww.ERROR.Print(err)
		return "", err
	}

	return message, nil
}

func createZip(destination string, sources ...string) (string, error) {
	var err error

	jww.INFO.Printf("Creating zip: %s from %s", destination, sources)
	zipper := car.NewZip()
	zipper.Name = destination
	zipper.UseFullpath = contour.GetBool("usefullpath")
	_, err = zipper.Create(destination, sources...)
	if err != nil {
		jww.ERROR.Print(err)
		return "", err
	}

	return zipper.Message(), nil
}

func createTar(destination string, sources ...string) (string, error) {
	var err error

	jww.INFO.Printf("Creating tar: %s from %s", destination, sources)
	tballer := car.NewTar()
	tballer.Name = destination
	tballer.Owner = contour.GetInt("owner")
	tballer.Group = contour.GetInt("group")
	tballer.Mode = os.FileMode(contour.GetInt64("mode"))

	//	tabller.Exclude = contour.GetString("exclude")
	tballer.ExcludeAnchored = contour.GetString("exclude-anchored")
	temp := contour.GetString("exclude-ext")
	if temp != "" {
		tballer.ExcludeExt = strings.Split(temp, ",")
		tballer.ExcludeExtCount = len(tballer.ExcludeExt)
	}

	tballer.IncludeAnchored = contour.GetString("include-anchored")
	temp = contour.GetString("include-ext")
	if temp != "" {
		tballer.IncludeExt = strings.Split(temp, ",")
		tballer.IncludeExtCount = len(tballer.IncludeExt)
	}

	// TODO figure out how to convert the incoming time info to time.Time
	// tballer.NewerMTime = contour.GetTime("newer-mtime")
	//	tballer.UseFullpath = contour.GetBool("usefullpath")
	_, err = tballer.Create(destination, sources...)
	if err != nil {
		jww.ERROR.Print(err)
		return "", err
	}

	return tballer.Message(), nil
}
<commit_msg>add proper handling of type flag<commit_after>package app

import (
	"fmt"
	"os"
	_ "strconv"
	"strings"

	car "github.com/mohae/carchivum"
	"github.com/mohae/contour"
	jww "github.com/spf13/jwalterweatherman"
)

func Create(destination string, sources ...string) (string, error) {
	var err error
	var message string

	fmt.Printf("\nCreate %q from %v\n", destination, sources)

	switch contour.GetString(Format) {
	case "tar":
		message, err = createTar(destination, sources...)
	case "zip":
		message, err = createZip(destination, sources...)
	default:
		err = fmt.Errorf("%s not supported", contour.GetString(Format))
	}

	if err != nil {
		jww.ERROR.Print(err)
		return "", err
	}

	return message, nil
}

func createZip(destination string, sources ...string) (string, error) {
	var err error

	jww.INFO.Printf("Creating zip: %s from %s", destination, sources)
	zipper := car.NewZip()
	zipper.Name = destination
	zipper.UseFullpath = contour.GetBool("usefullpath")
	_, err = zipper.Create(destination, sources...)
	if err != nil {
		jww.ERROR.Print(err)
		return "", err
	}

	return zipper.Message(), nil
}

func createTar(destination string, sources ...string) (string, error) {
	var err error

	jww.INFO.Printf("Creating tar: %s from %s", destination, sources)
	tballer := car.NewTar()
	t := contour.GetString("type")
	if t != "" {
		f := car.FormatFromString(t)
		if f == car.UnsupportedFmt {
			fmt.Errorf("Unuspported format: %s", t)
			jww.ERROR.Print(err)
			return "", err
		}
		tballer.Format = f
	}
	fmt.Println(t)
	tballer.Name = destination
	tballer.Owner = contour.GetInt("owner")
	tballer.Group = contour.GetInt("group")
	tballer.Mode = os.FileMode(contour.GetInt64("mode"))

	//	tabller.Exclude = contour.GetString("exclude")
	tballer.ExcludeAnchored = contour.GetString("exclude-anchored")
	temp := contour.GetString("exclude-ext")
	if temp != "" {
		tballer.ExcludeExt = strings.Split(temp, ",")
		tballer.ExcludeExtCount = len(tballer.ExcludeExt)
	}

	tballer.IncludeAnchored = contour.GetString("include-anchored")
	temp = contour.GetString("include-ext")
	if temp != "" {
		tballer.IncludeExt = strings.Split(temp, ",")
		tballer.IncludeExtCount = len(tballer.IncludeExt)
	}

	// TODO figure out how to convert the incoming time info to time.Time
	// tballer.NewerMTime = contour.GetTime("newer-mtime")
	//	tballer.UseFullpath = contour.GetBool("usefullpath")
	_, err = tballer.Create(destination, sources...)
	if err != nil {
		jww.ERROR.Print(err)
		return "", err
	}

	return tballer.Message(), nil
}
<|endoftext|>


<commit_before>package nsq

import (
	"fmt"
	"strconv"
	"testing"
	"time"
)

func TestProducer(t *testing.T) {
	for _, n := range []int{10 /*, 10, 100, 1000*/} {
		count := n
		topic := fmt.Sprintf("test-publisher-%d", n)
		t.Run(topic, func(t *testing.T) {
			t.Parallel()

			c, _ := StartConsumer(ConsumerConfig{
				Topic:   topic,
				Channel: "channel",
				Address: "localhost:4150",
			})
			defer c.Stop()

			// Give some time for the consumer to connect.
			time.Sleep(100 * time.Millisecond)

			p, _ := StartProducer(ProducerConfig{
				Address:        "localhost:4150",
				MaxConcurrency: 3,
			})
			defer p.Stop()

			for i := 0; i != count; i++ {
				if err := p.Publish(topic, []byte(strconv.Itoa(i))); err != nil {
					t.Error(err)
					return
				}
			}

			buckets := make([]int, count)

			deadline := time.NewTimer(10 * time.Second)
			defer deadline.Stop()

			for i := 0; i != count; i++ {
				select {
				case msg := <-c.Messages():
					b, err := strconv.Atoi(string(msg.Body))
					if err != nil {
						t.Error(err)
					}
					buckets[b]++
					msg.Finish()
				case <-deadline.C:
					t.Error("timeout")
					return
				}
			}

			for i, b := range buckets {
				if b != 1 {
					t.Errorf("bucket at index %d has value %d", i, b)
				}
			}
		})
	}
}
<commit_msg>add more tests<commit_after>package nsq

import (
	"fmt"
	"strconv"
	"testing"
	"time"
)

func TestProducer(t *testing.T) {
	for _, n := range []int{1, 10, 100, 1000} {
		count := n
		topic := fmt.Sprintf("test-publisher-%d", n)
		t.Run(topic, func(t *testing.T) {
			t.Parallel()

			c, _ := StartConsumer(ConsumerConfig{
				Topic:   topic,
				Channel: "channel",
				Address: "localhost:4150",
			})
			defer c.Stop()

			// Give some time for the consumer to connect.
			time.Sleep(100 * time.Millisecond)

			p, _ := StartProducer(ProducerConfig{
				Address:        "localhost:4150",
				MaxConcurrency: 3,
			})
			defer p.Stop()

			for i := 0; i != count; i++ {
				if err := p.Publish(topic, []byte(strconv.Itoa(i))); err != nil {
					t.Error(err)
					return
				}
			}

			buckets := make([]int, count)

			deadline := time.NewTimer(10 * time.Second)
			defer deadline.Stop()

			for i := 0; i != count; i++ {
				select {
				case msg := <-c.Messages():
					b, err := strconv.Atoi(string(msg.Body))
					if err != nil {
						t.Error(err)
					}
					buckets[b]++
					msg.Finish()
				case <-deadline.C:
					t.Error("timeout")
					return
				}
			}

			for i, b := range buckets {
				if b != 1 {
					t.Errorf("bucket at index %d has value %d", i, b)
				}
			}
		})
	}
}
<|endoftext|>


<commit_before>package resolves

import (
	"encoding/json"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"strconv"
	"strings"
	"time"

	. "github.com/wanliu/flow/builtin/luis"
	. "github.com/wanliu/flow/context"
)

// 处理开单的逻辑结构, 不需要是组件
// 作为context的一个部分，或者存在一个Value中
type OpenOrderResolve struct {
	// goflow.Component
	// Ctx        <-chan Context
	// Out        chan<- ReplyData
	// Address    string
	// Time       time.Time
	// Products   []ProductResolve
	LuisParams ResultParams
	Products   ItemsResolve
	// Products   ProductsResolve
	// Address    AddressResolve
	// Time       OrderTimeResolve
	Address string
	Time    time.Time
	DefTime string
	Current Resolve
}

func NewOpenOrderResolve(ctx Context) *OpenOrderResolve {
	resolve := new(OpenOrderResolve)

	luis := ctx.Value("Result").(ResultParams)

	luis.Entities = DistinctEntites(luis.Entities)
	luis.Entities = DeduplicateEntities(luis.Entities)

	resolve.LuisParams = luis
	resolve.ExtractFromLuis()

	return resolve
}

func (r *OpenOrderResolve) Solve(luis ResultParams) (bool, string, string) {
	solved, finishNotition, nextNotition := r.Current.Solve(luis)

	if solved {
		if r.Fullfilled() {
			return true, finishNotition + "\n" + r.Answer(), ""
		} else {
			r.Current = r.Next()
			hint := r.Current.Hint()

			return false, finishNotition, finishNotition + "\n" + hint
		}
	} else {
		return solved, finishNotition, nextNotition
	}

}

func (r OpenOrderResolve) Hint() string {
	return r.Current.Hint()
}

// 从ｌｕｉｓ数据构造结构数据
func (r *OpenOrderResolve) ExtractFromLuis() {
	// log.Printf("====:: %v", r.LuisParams.Entities)

	// r.ExtractProducts()
	r.ExtractItems()
	r.ExtractAddress()
	r.ExtractTime()
	// r.ExtractQuantity()

	// log.Printf("----> %v", r.Products)
}

// TODO 无法识别全角数字
func (r *OpenOrderResolve) ExtractItems() {
	r.ExtractProducts()
	quantities := r.ExtractQuantity()

	for i, q := range quantities {
		if len(r.Products.Products) >= i+1 {
			pr := r.Products.Products[i]
			pr.Quantity = q
		}
	}

	for _, p := range r.Products.Products {
		p.CheckResolved()
	}
}

func (r *OpenOrderResolve) ExtractQuantity() []int {
	result := make([]int, 0, 10)

	for _, item := range r.LuisParams.Entities {
		if item.Type == "builtin.number" {
			number := strings.Trim(item.Resolution.Value, " ")
			q, _ := strconv.ParseInt(number, 10, 64)
			result = append(result, int(q))
		}
	}

	return result
}

func (r *OpenOrderResolve) ExtractProducts() {
	for _, item := range r.LuisParams.Entities {
		if item.Type == "products" {
			product := ItemResolve{
				Resolved:   false,
				Name:       item.Entity,
				Price:      0,
				Quantity:   0, // 默认值
				Product:    "",
				Resolution: item.Resolution,
			}

			r.Products.Add(product)
		} else {
			log.Printf("type: %v", item.Type)
		}
	}
}

func (r *OpenOrderResolve) ExtractAddress() {
	for _, item := range r.LuisParams.Entities {

		if item.Type == "address" || item.Type == "地点" {
			log.Printf("Address ... %v", item.Entity)
			r.Address = item.Entity
		}
	}
}

func (r *OpenOrderResolve) ExtractTime() {
	for _, item := range r.LuisParams.Entities {
		if item.Type == "builtin.datetime.date" {
			luisTime, err := time.Parse("2006-01-02", item.Resolution.Date)

			if err != nil {
				log.Printf("::::::::ERROR: %v", err)
			} else {
				// dTime = luisTime
				r.Time = luisTime
			}
		}
	}
}

func (r *OpenOrderResolve) SetDefTime(t string) {
	r.DefTime = t

	if r.Time.IsZero() && r.DefTime != "" {
		r.SetTimeByDef()
	}
}

func (r *OpenOrderResolve) SetTimeByDef() {
	if r.DefTime == "今天" {
		r.Time = time.Now()
	} else if r.DefTime == "明天" {
		r.Time = time.Now().Add(24 * time.Hour)
	}
}

func (r OpenOrderResolve) ProductsFullfilled() bool {
	return r.Products.Fullfilled()
}

func (r OpenOrderResolve) TimeFullfilled() bool {
	// return r.Time.Fullfilled()
	return !r.Time.IsZero()
}

func (r OpenOrderResolve) AddressFullfilled() bool {
	// return r.Address.Fullfilled()
	return r.Address != ""
}

// 是否条件全部满足
func (r OpenOrderResolve) Fullfilled() bool {
	return r.ProductsFullfilled() &&
		r.TimeFullfilled() &&
		r.AddressFullfilled()
}

// 下一个为满足项目
func (r *OpenOrderResolve) Next() Resolve {
	if !r.ProductsFullfilled() {
		unsolved := r.NextProduct()
		r.Current = unsolved
		return unsolved
	} else if !r.AddressFullfilled() {
		unsolved := AddressResolve{Parent: r}
		r.Current = unsolved
		return unsolved
	} else if !r.TimeFullfilled() {
		unsolved := OrderTimeResolve{Parent: r}
		r.Current = unsolved
		return unsolved
	} else {
		return nil
	}
}

func (r OpenOrderResolve) EmptyProducts() bool {
	return len(r.Products.Products) == 0
}

func (r OpenOrderResolve) NextNotify() string {
	unsolved := r.Next()
	return unsolved.Hint()
}

//
func (r OpenOrderResolve) PostService() string {
	return ""
}

func (r OpenOrderResolve) NextProduct() Resolve {
	return r.Products.NextProduct()
}

func (r OpenOrderResolve) Answer() string {
	desc := ""

	desc = desc + "=== 订单输入完成 ===\n"
	desc = desc + "本订单包含如下商品：" + "\n"

	params := url.Values{
		"auth_token":   {"5f567b5efc3e4d0aa0d9c40922ae07aa"},
		"street":       {r.Address},
		"deliver_time": {r.Time.Format("2006年01月02日")},
	}

	for i, p := range r.Products.Products {
		desc = desc + p.Product + " " + strconv.Itoa(p.Quantity) + "件\n"

		nk := "items[" + strconv.Itoa(i) + "][name]"
		nv := p.Product
		params.Add(nk, nv)

		qk := "items[" + strconv.Itoa(i) + "][quantity]"
		qv := strconv.Itoa(p.Quantity)
		params.Add(qk, qv)
	}

	desc = desc + "地址:" + r.Address + "\n"
	desc = desc + "送货时间" + r.Time.Format("2006年01月02日") + "\n"
	desc = desc + "=== 结束 ===\n"

	res, err := http.PostForm("http://192.168.0.155:3000/api/v1/temp_orders", params)

	if err != nil {
		// return err.Error()
		return "服务暂时不可用，请稍后再试"
	} else {
		defer res.Body.Close()
		body, e := ioutil.ReadAll(res.Body)

		if e != nil {
			return e.Error()
		} else {
			var result Res
			json.Unmarshal(body, &result)

			if res.StatusCode == 422 {
				return result.Error
			} else {
				return desc + "请通过以下地址完成订单操作：" + result.Confirm_path
			}

		}
	}
}

type Res struct {
	Id           int
	Confirm_path string
	Error        string
}
<commit_msg>fix address<commit_after>package resolves

import (
	"encoding/json"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"strconv"
	"strings"
	"time"

	. "github.com/wanliu/flow/builtin/luis"
	. "github.com/wanliu/flow/context"
)

// 处理开单的逻辑结构, 不需要是组件
// 作为context的一个部分，或者存在一个Value中
type OpenOrderResolve struct {
	// goflow.Component
	// Ctx        <-chan Context
	// Out        chan<- ReplyData
	// Address    string
	// Time       time.Time
	// Products   []ProductResolve
	LuisParams ResultParams
	Products   ItemsResolve
	// Products   ProductsResolve
	// Address    AddressResolve
	// Time       OrderTimeResolve
	Address string
	Time    time.Time
	DefTime string
	Current Resolve
}

func NewOpenOrderResolve(ctx Context) *OpenOrderResolve {
	resolve := new(OpenOrderResolve)

	luis := ctx.Value("Result").(ResultParams)

	luis.Entities = DistinctEntites(luis.Entities)
	luis.Entities = DeduplicateEntities(luis.Entities)

	resolve.LuisParams = luis
	resolve.ExtractFromLuis()

	return resolve
}

func (r *OpenOrderResolve) Solve(luis ResultParams) (bool, string, string) {
	solved, finishNotition, nextNotition := r.Current.Solve(luis)

	if solved {
		if r.Fullfilled() {
			return true, finishNotition + "\n" + r.Answer(), ""
		} else {
			r.Current = r.Next()
			hint := r.Current.Hint()

			return false, finishNotition, finishNotition + "\n" + hint
		}
	} else {
		return solved, finishNotition, nextNotition
	}

}

func (r OpenOrderResolve) Hint() string {
	return r.Current.Hint()
}

// 从ｌｕｉｓ数据构造结构数据
func (r *OpenOrderResolve) ExtractFromLuis() {
	// log.Printf("====:: %v", r.LuisParams.Entities)

	// r.ExtractProducts()
	r.ExtractItems()
	r.ExtractAddress()
	r.ExtractTime()
	// r.ExtractQuantity()

	// log.Printf("----> %v", r.Products)
}

// TODO 无法识别全角数字
func (r *OpenOrderResolve) ExtractItems() {
	r.ExtractProducts()
	quantities := r.ExtractQuantity()

	for i, q := range quantities {
		if len(r.Products.Products) >= i+1 {
			pr := r.Products.Products[i]
			pr.Quantity = q
		}
	}

	for _, p := range r.Products.Products {
		p.CheckResolved()
	}
}

func (r *OpenOrderResolve) ExtractQuantity() []int {
	result := make([]int, 0, 10)

	for _, item := range r.LuisParams.Entities {
		if item.Type == "builtin.number" {
			number := strings.Trim(item.Resolution.Value, " ")
			q, _ := strconv.ParseInt(number, 10, 64)
			result = append(result, int(q))
		}
	}

	return result
}

func (r *OpenOrderResolve) ExtractProducts() {
	for _, item := range r.LuisParams.Entities {
		if item.Type == "products" {
			product := ItemResolve{
				Resolved:   false,
				Name:       item.Entity,
				Price:      0,
				Quantity:   0, // 默认值
				Product:    "",
				Resolution: item.Resolution,
			}

			r.Products.Add(product)
		} else {
			log.Printf("type: %v", item.Type)
		}
	}
}

func (r *OpenOrderResolve) ExtractAddress() {
	for _, item := range r.LuisParams.Entities {

		if item.Type == "地址" {
			r.Address = item.Entity
		}
	}
}

func (r *OpenOrderResolve) ExtractTime() {
	for _, item := range r.LuisParams.Entities {
		if item.Type == "builtin.datetime.date" {
			luisTime, err := time.Parse("2006-01-02", item.Resolution.Date)

			if err != nil {
				log.Printf("::::::::ERROR: %v", err)
			} else {
				// dTime = luisTime
				r.Time = luisTime
			}
		}
	}
}

func (r *OpenOrderResolve) SetDefTime(t string) {
	r.DefTime = t

	if r.Time.IsZero() && r.DefTime != "" {
		r.SetTimeByDef()
	}
}

func (r *OpenOrderResolve) SetTimeByDef() {
	if r.DefTime == "今天" {
		r.Time = time.Now()
	} else if r.DefTime == "明天" {
		r.Time = time.Now().Add(24 * time.Hour)
	}
}

func (r OpenOrderResolve) ProductsFullfilled() bool {
	return r.Products.Fullfilled()
}

func (r OpenOrderResolve) TimeFullfilled() bool {
	// return r.Time.Fullfilled()
	return !r.Time.IsZero()
}

func (r OpenOrderResolve) AddressFullfilled() bool {
	// return r.Address.Fullfilled()
	return r.Address != ""
}

// 是否条件全部满足
func (r OpenOrderResolve) Fullfilled() bool {
	return r.ProductsFullfilled() &&
		r.TimeFullfilled() &&
		r.AddressFullfilled()
}

// 下一个为满足项目
func (r *OpenOrderResolve) Next() Resolve {
	if !r.ProductsFullfilled() {
		unsolved := r.NextProduct()
		r.Current = unsolved
		return unsolved
	} else if !r.AddressFullfilled() {
		unsolved := AddressResolve{Parent: r}
		r.Current = unsolved
		return unsolved
	} else if !r.TimeFullfilled() {
		unsolved := OrderTimeResolve{Parent: r}
		r.Current = unsolved
		return unsolved
	} else {
		return nil
	}
}

func (r OpenOrderResolve) EmptyProducts() bool {
	return len(r.Products.Products) == 0
}

func (r OpenOrderResolve) NextNotify() string {
	unsolved := r.Next()
	return unsolved.Hint()
}

//
func (r OpenOrderResolve) PostService() string {
	return ""
}

func (r OpenOrderResolve) NextProduct() Resolve {
	return r.Products.NextProduct()
}

func (r OpenOrderResolve) Answer() string {
	desc := ""

	desc = desc + "=== 订单输入完成 ===\n"
	desc = desc + "本订单包含如下商品：" + "\n"

	params := url.Values{
		"auth_token":   {"5f567b5efc3e4d0aa0d9c40922ae07aa"},
		"street":       {r.Address},
		"deliver_time": {r.Time.Format("2006年01月02日")},
	}

	for i, p := range r.Products.Products {
		desc = desc + p.Product + " " + strconv.Itoa(p.Quantity) + "件\n"

		nk := "items[" + strconv.Itoa(i) + "][name]"
		nv := p.Product
		params.Add(nk, nv)

		qk := "items[" + strconv.Itoa(i) + "][quantity]"
		qv := strconv.Itoa(p.Quantity)
		params.Add(qk, qv)
	}

	desc = desc + "地址:" + r.Address + "\n"
	desc = desc + "送货时间" + r.Time.Format("2006年01月02日") + "\n"
	desc = desc + "=== 结束 ===\n"

	res, err := http.PostForm("http://192.168.0.155:3000/api/v1/temp_orders", params)

	if err != nil {
		// return err.Error()
		return "服务暂时不可用，请稍后再试"
	} else {
		defer res.Body.Close()
		body, e := ioutil.ReadAll(res.Body)

		if e != nil {
			return e.Error()
		} else {
			var result Res
			json.Unmarshal(body, &result)

			if res.StatusCode == 422 {
				return result.Error
			} else {
				return desc + "请通过以下地址完成订单操作：" + result.Confirm_path
			}

		}
	}
}

type Res struct {
	Id           int
	Confirm_path string
	Error        string
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
)

func main() {

	x := []int{
		48, 96, 86, 68,
		57, 82, 63, 70,
		37, 34, 83, 27,
		19, 97, 9, 17,
	}
	small:= x[0];

	for  i:=1;i<len(x); i++ {
		if x[i] < small {
			small = x[i]
		}
	}
	fmt.Println("Smallest Number is ",small);

	for _, value:= range x {
		if value < small {
			small = value
		}
	}

	fmt.Println("Smallest Number is ",small);
}
<commit_msg>Made the smallest logic a function<commit_after>package main

import (
	"fmt"
)

func smallest(x[] int) int {
	small:= x[0]
	for _, value := range x {
		if value < small {
			small = value
		}
	}
	return small;
}

func main() {

	x := []int{
		48, 96, 86, 68,
		57, 82, 63, 70,
		37, 34, 83, 27,
		19, 97, 9, 17,
	}
	small := smallest(x)

	fmt.Println("Smallest Number is ", small)

}
<|endoftext|>


<commit_before>// Copyright 2016 The lime Authors.
// Use of this source code is governed by a 2-clause
// BSD-style license that can be found in the LICENSE file.

package packages

import (
	"os"
	"path/filepath"
	"testing"
	"time"
)

func TestWatchDir(t *testing.T) {
	// The backend libraries expect absolute paths
	testPath, testPathErr := filepath.Abs("testdata/file")

	if testPathErr != nil {
		t.Fatalf("Couldn't get absolute path for testdata/file: %s", testPathErr)
	}

	pkg := &dummyPackage{path: testPath}
	rec := &Record{func(s string) bool { return s == testPath },
		func(s string) Package { return pkg }}

	Register(rec)
	defer Unregister(rec)
	watchDir(filepath.Dir(testPath))

	if _, err := os.Create(testPath); err != nil {
		t.Fatalf("Error creating '%s' file: %s", testPath, err)
	}
	defer os.Remove(testPath)
	time.Sleep(100 * time.Millisecond)
	if !pkg.IsLoaded() {
		t.Error("Expected package loaded")
	}
}
<commit_msg>Increased delay in TestWatchDir test<commit_after>// Copyright 2016 The lime Authors.
// Use of this source code is governed by a 2-clause
// BSD-style license that can be found in the LICENSE file.

package packages

import (
	"os"
	"path/filepath"
	"testing"
	"time"
)

func TestWatchDir(t *testing.T) {
	// The backend libraries expect absolute paths
	testPath, testPathErr := filepath.Abs("testdata/file")

	if testPathErr != nil {
		t.Fatalf("Couldn't get absolute path for testdata/file: %s", testPathErr)
	}

	pkg := &dummyPackage{path: testPath}
	rec := &Record{func(s string) bool { return s == testPath },
		func(s string) Package { return pkg }}

	Register(rec)
	defer Unregister(rec)
	watchDir(filepath.Dir(testPath))

	if _, err := os.Create(testPath); err != nil {
		t.Fatalf("Error creating '%s' file: %s", testPath, err)
	}
	defer os.Remove(testPath)
	time.Sleep(500 * time.Millisecond)
	if !pkg.IsLoaded() {
		t.Error("Expected package loaded")
	}
}
<|endoftext|>


<commit_before>package zipkintracer

import (
	"encoding/binary"
	"io"
	"strconv"
	"strings"

	"github.com/gogo/protobuf/proto"
	opentracing "github.com/opentracing/opentracing-go"

	"github.com/openzipkin/zipkin-go-opentracing/flag"
	"github.com/openzipkin/zipkin-go-opentracing/wire"
)

type textMapPropagator struct {
	tracer *tracerImpl
}
type binaryPropagator struct {
	tracer *tracerImpl
}

const (
	prefixBaggage = "ot-baggage-"

	tracerStateFieldCount = 3 // not 5, X-B3-ParentSpanId is optional and we allow optional Sampled header

	zipkinTraceID      = "X-B3-TraceId"
	zipkinSpanID       = "X-B3-SpanId"
	zipkinParentSpanID = "X-B3-ParentSpanId"
	zipkinSampled      = "X-B3-Sampled"
	zipkinFlags        = "X-B3-Flags"

	zipkinTraceIDLower      = "x-b3-traceid"
	zipkinSpanIDLower       = "x-b3-spanid"
	zipkinParentSpanIDLower = "x-b3-parentspanid"
	zipkinSampledLower      = "x-b3-sampled"
	zipkinFlagsLower        = "x-b3-flags"
)

func (p *textMapPropagator) Inject(
	spanContext opentracing.SpanContext,
	opaqueCarrier interface{},
) error {
	sc, ok := spanContext.(SpanContext)
	if !ok {
		return opentracing.ErrInvalidSpanContext
	}
	carrier, ok := opaqueCarrier.(opentracing.TextMapWriter)
	if !ok {
		return opentracing.ErrInvalidCarrier
	}
	carrier.Set(zipkinTraceID, strconv.FormatUint(sc.TraceID, 16))
	carrier.Set(zipkinSpanID, strconv.FormatUint(sc.SpanID, 16))
	carrier.Set(zipkinSampled, strconv.FormatBool(sc.Sampled))

	if sc.ParentSpanID != nil {
		// we only set ParentSpanID header if there is a parent span
		carrier.Set(zipkinParentSpanID, strconv.FormatUint(*sc.ParentSpanID, 16))
	}
	// we only need to inject the debug flag if set. see flag package for details.
	flags := sc.Flags & flag.Debug
	carrier.Set(zipkinFlags, strconv.FormatUint(uint64(flags), 10))

	for k, v := range sc.Baggage {
		carrier.Set(prefixBaggage+k, v)
	}
	return nil
}

func (p *textMapPropagator) Extract(
	opaqueCarrier interface{},
) (opentracing.SpanContext, error) {
	carrier, ok := opaqueCarrier.(opentracing.TextMapReader)
	if !ok {
		return nil, opentracing.ErrInvalidCarrier
	}
	requiredFieldCount := 0
	var (
		traceID      uint64
		spanID       uint64
		sampled      bool
		parentSpanID *uint64
		flags        flag.Flags
		err          error
	)
	decodedBaggage := make(map[string]string)
	err = carrier.ForeachKey(func(k, v string) error {
		switch strings.ToLower(k) {
		case zipkinTraceIDLower:
			traceID, err = strconv.ParseUint(v, 16, 64)
			if err != nil {
				return opentracing.ErrSpanContextCorrupted
			}
		case zipkinSpanIDLower:
			spanID, err = strconv.ParseUint(v, 16, 64)
			if err != nil {
				return opentracing.ErrSpanContextCorrupted
			}
		case zipkinParentSpanIDLower:
			var id uint64
			id, err = strconv.ParseUint(v, 16, 64)
			if err != nil {
				return opentracing.ErrSpanContextCorrupted
			}
			parentSpanID = &id
		case zipkinSampledLower:
			sampled, err = strconv.ParseBool(v)
			if err != nil {
				return opentracing.ErrSpanContextCorrupted
			}
			// Sampled header was explicitly set
			flags |= flag.SamplingSet
		case zipkinFlagsLower:
			var f uint64
			f, err = strconv.ParseUint(v, 10, 64)
			if err != nil {
				return opentracing.ErrSpanContextCorrupted
			}
			if flag.Flags(f)&flag.Debug == flag.Debug {
				flags |= flag.Debug
			}
		default:
			lowercaseK := strings.ToLower(k)
			if strings.HasPrefix(lowercaseK, prefixBaggage) {
				decodedBaggage[strings.TrimPrefix(lowercaseK, prefixBaggage)] = v
			}
			// Balance off the requiredFieldCount++ just below...
			requiredFieldCount--
		}
		requiredFieldCount++
		return nil
	})
	if err != nil {
		return nil, err
	}
	if requiredFieldCount < tracerStateFieldCount {
		if requiredFieldCount == 0 {
			return nil, opentracing.ErrSpanContextNotFound
		}
		return nil, opentracing.ErrSpanContextCorrupted
	}

	// check if Sample state was communicated through the Flags bitset
	if !sampled && flags&flag.Sampled == flag.Sampled {
		sampled = true
	}

	return SpanContext{
		TraceID:      traceID,
		SpanID:       spanID,
		Sampled:      sampled,
		Baggage:      decodedBaggage,
		ParentSpanID: parentSpanID,
		Flags:        flags,
	}, nil
}

func (p *binaryPropagator) Inject(
	spanContext opentracing.SpanContext,
	opaqueCarrier interface{},
) error {
	sc, ok := spanContext.(SpanContext)
	if !ok {
		return opentracing.ErrInvalidSpanContext
	}
	carrier, ok := opaqueCarrier.(io.Writer)
	if !ok {
		return opentracing.ErrInvalidCarrier
	}

	state := wire.TracerState{}
	state.TraceId = sc.TraceID
	state.SpanId = sc.SpanID
	state.Sampled = sc.Sampled
	state.BaggageItems = sc.Baggage

	// encode the debug bit
	flags := sc.Flags & flag.Debug
	if sc.ParentSpanID != nil {
		state.ParentSpanId = *sc.ParentSpanID
	} else {
		// root span...
		state.ParentSpanId = 0
		flags |= flag.IsRoot
	}

	// we explicitly inform our sampling state downstream
	flags |= flag.SamplingSet
	if sc.Sampled {
		flags |= flag.Sampled
	}
	state.Flags = uint64(flags)

	b, err := proto.Marshal(&state)
	if err != nil {
		return err
	}

	// Write the length of the marshalled binary to the writer.
	length := uint32(len(b))
	if err = binary.Write(carrier, binary.BigEndian, &length); err != nil {
		return err
	}

	_, err = carrier.Write(b)
	return err
}

func (p *binaryPropagator) Extract(
	opaqueCarrier interface{},
) (opentracing.SpanContext, error) {
	carrier, ok := opaqueCarrier.(io.Reader)
	if !ok {
		return nil, opentracing.ErrInvalidCarrier
	}

	// Read the length of marshalled binary. io.ReadAll isn't that performant
	// since it keeps resizing the underlying buffer as it encounters more bytes
	// to read. By reading the length, we can allocate a fixed sized buf and read
	// the exact amount of bytes into it.
	var length uint32
	if err := binary.Read(carrier, binary.BigEndian, &length); err != nil {
		return nil, opentracing.ErrSpanContextCorrupted
	}
	buf := make([]byte, length)
	if n, err := carrier.Read(buf); err != nil {
		if n > 0 {
			return nil, opentracing.ErrSpanContextCorrupted
		}
		return nil, opentracing.ErrSpanContextNotFound
	}

	ctx := wire.TracerState{}
	if err := proto.Unmarshal(buf, &ctx); err != nil {
		return nil, opentracing.ErrSpanContextCorrupted
	}

	flags := flag.Flags(ctx.Flags)
	if flags&flag.Sampled == flag.Sampled {
		ctx.Sampled = true
	}
	// this propagator expects sampling state to be explicitly propagated by the
	// upstream service. so set this flag to indentify to tracer it should not
	// run its sampler in case it is not the root of the trace.
	flags |= flag.SamplingSet

	return SpanContext{
		TraceID:      ctx.TraceId,
		SpanID:       ctx.SpanId,
		Sampled:      ctx.Sampled,
		Baggage:      ctx.BaggageItems,
		ParentSpanID: &ctx.ParentSpanId,
		Flags:        flags,
	}, nil
}
<commit_msg>Tolerate 128bit ids by throwing out high bits<commit_after>package zipkintracer

import (
	"encoding/binary"
	"io"
	"strconv"
	"strings"

	"github.com/gogo/protobuf/proto"
	opentracing "github.com/opentracing/opentracing-go"

	"github.com/openzipkin/zipkin-go-opentracing/flag"
	"github.com/openzipkin/zipkin-go-opentracing/wire"
)

type textMapPropagator struct {
	tracer *tracerImpl
}
type binaryPropagator struct {
	tracer *tracerImpl
}

const (
	prefixBaggage = "ot-baggage-"

	tracerStateFieldCount = 3 // not 5, X-B3-ParentSpanId is optional and we allow optional Sampled header

	zipkinTraceID      = "X-B3-TraceId"
	zipkinSpanID       = "X-B3-SpanId"
	zipkinParentSpanID = "X-B3-ParentSpanId"
	zipkinSampled      = "X-B3-Sampled"
	zipkinFlags        = "X-B3-Flags"

	zipkinTraceIDLower      = "x-b3-traceid"
	zipkinSpanIDLower       = "x-b3-spanid"
	zipkinParentSpanIDLower = "x-b3-parentspanid"
	zipkinSampledLower      = "x-b3-sampled"
	zipkinFlagsLower        = "x-b3-flags"
)

func (p *textMapPropagator) Inject(
	spanContext opentracing.SpanContext,
	opaqueCarrier interface{},
) error {
	sc, ok := spanContext.(SpanContext)
	if !ok {
		return opentracing.ErrInvalidSpanContext
	}
	carrier, ok := opaqueCarrier.(opentracing.TextMapWriter)
	if !ok {
		return opentracing.ErrInvalidCarrier
	}
	carrier.Set(zipkinTraceID, strconv.FormatUint(sc.TraceID, 16))
	carrier.Set(zipkinSpanID, strconv.FormatUint(sc.SpanID, 16))
	carrier.Set(zipkinSampled, strconv.FormatBool(sc.Sampled))

	if sc.ParentSpanID != nil {
		// we only set ParentSpanID header if there is a parent span
		carrier.Set(zipkinParentSpanID, strconv.FormatUint(*sc.ParentSpanID, 16))
	}
	// we only need to inject the debug flag if set. see flag package for details.
	flags := sc.Flags & flag.Debug
	carrier.Set(zipkinFlags, strconv.FormatUint(uint64(flags), 10))

	for k, v := range sc.Baggage {
		carrier.Set(prefixBaggage+k, v)
	}
	return nil
}

func (p *textMapPropagator) Extract(
	opaqueCarrier interface{},
) (opentracing.SpanContext, error) {
	carrier, ok := opaqueCarrier.(opentracing.TextMapReader)
	if !ok {
		return nil, opentracing.ErrInvalidCarrier
	}
	requiredFieldCount := 0
	var (
		traceID      uint64
		spanID       uint64
		sampled      bool
		parentSpanID *uint64
		flags        flag.Flags
		err          error
	)
	decodedBaggage := make(map[string]string)
	err = carrier.ForeachKey(func(k, v string) error {
		switch strings.ToLower(k) {
		case zipkinTraceIDLower:
			// TODO: add logic for most significant 64 bits when 128bit traceID's are
			// supported.
			// SEE: https://github.com/openzipkin/b3-propagation/issues/6
			if len(v) > 16 {
				v = v[len(v)-16:]
			}
			traceID, err = strconv.ParseUint(v, 16, 64)
			if err != nil {
				return opentracing.ErrSpanContextCorrupted
			}
		case zipkinSpanIDLower:
			spanID, err = strconv.ParseUint(v, 16, 64)
			if err != nil {
				return opentracing.ErrSpanContextCorrupted
			}
		case zipkinParentSpanIDLower:
			var id uint64
			id, err = strconv.ParseUint(v, 16, 64)
			if err != nil {
				return opentracing.ErrSpanContextCorrupted
			}
			parentSpanID = &id
		case zipkinSampledLower:
			sampled, err = strconv.ParseBool(v)
			if err != nil {
				return opentracing.ErrSpanContextCorrupted
			}
			// Sampled header was explicitly set
			flags |= flag.SamplingSet
		case zipkinFlagsLower:
			var f uint64
			f, err = strconv.ParseUint(v, 10, 64)
			if err != nil {
				return opentracing.ErrSpanContextCorrupted
			}
			if flag.Flags(f)&flag.Debug == flag.Debug {
				flags |= flag.Debug
			}
		default:
			lowercaseK := strings.ToLower(k)
			if strings.HasPrefix(lowercaseK, prefixBaggage) {
				decodedBaggage[strings.TrimPrefix(lowercaseK, prefixBaggage)] = v
			}
			// Balance off the requiredFieldCount++ just below...
			requiredFieldCount--
		}
		requiredFieldCount++
		return nil
	})
	if err != nil {
		return nil, err
	}
	if requiredFieldCount < tracerStateFieldCount {
		if requiredFieldCount == 0 {
			return nil, opentracing.ErrSpanContextNotFound
		}
		return nil, opentracing.ErrSpanContextCorrupted
	}

	// check if Sample state was communicated through the Flags bitset
	if !sampled && flags&flag.Sampled == flag.Sampled {
		sampled = true
	}

	return SpanContext{
		TraceID:      traceID,
		SpanID:       spanID,
		Sampled:      sampled,
		Baggage:      decodedBaggage,
		ParentSpanID: parentSpanID,
		Flags:        flags,
	}, nil
}

func (p *binaryPropagator) Inject(
	spanContext opentracing.SpanContext,
	opaqueCarrier interface{},
) error {
	sc, ok := spanContext.(SpanContext)
	if !ok {
		return opentracing.ErrInvalidSpanContext
	}
	carrier, ok := opaqueCarrier.(io.Writer)
	if !ok {
		return opentracing.ErrInvalidCarrier
	}

	state := wire.TracerState{}
	state.TraceId = sc.TraceID
	state.SpanId = sc.SpanID
	state.Sampled = sc.Sampled
	state.BaggageItems = sc.Baggage

	// encode the debug bit
	flags := sc.Flags & flag.Debug
	if sc.ParentSpanID != nil {
		state.ParentSpanId = *sc.ParentSpanID
	} else {
		// root span...
		state.ParentSpanId = 0
		flags |= flag.IsRoot
	}

	// we explicitly inform our sampling state downstream
	flags |= flag.SamplingSet
	if sc.Sampled {
		flags |= flag.Sampled
	}
	state.Flags = uint64(flags)

	b, err := proto.Marshal(&state)
	if err != nil {
		return err
	}

	// Write the length of the marshalled binary to the writer.
	length := uint32(len(b))
	if err = binary.Write(carrier, binary.BigEndian, &length); err != nil {
		return err
	}

	_, err = carrier.Write(b)
	return err
}

func (p *binaryPropagator) Extract(
	opaqueCarrier interface{},
) (opentracing.SpanContext, error) {
	carrier, ok := opaqueCarrier.(io.Reader)
	if !ok {
		return nil, opentracing.ErrInvalidCarrier
	}

	// Read the length of marshalled binary. io.ReadAll isn't that performant
	// since it keeps resizing the underlying buffer as it encounters more bytes
	// to read. By reading the length, we can allocate a fixed sized buf and read
	// the exact amount of bytes into it.
	var length uint32
	if err := binary.Read(carrier, binary.BigEndian, &length); err != nil {
		return nil, opentracing.ErrSpanContextCorrupted
	}
	buf := make([]byte, length)
	if n, err := carrier.Read(buf); err != nil {
		if n > 0 {
			return nil, opentracing.ErrSpanContextCorrupted
		}
		return nil, opentracing.ErrSpanContextNotFound
	}

	ctx := wire.TracerState{}
	if err := proto.Unmarshal(buf, &ctx); err != nil {
		return nil, opentracing.ErrSpanContextCorrupted
	}

	flags := flag.Flags(ctx.Flags)
	if flags&flag.Sampled == flag.Sampled {
		ctx.Sampled = true
	}
	// this propagator expects sampling state to be explicitly propagated by the
	// upstream service. so set this flag to indentify to tracer it should not
	// run its sampler in case it is not the root of the trace.
	flags |= flag.SamplingSet

	return SpanContext{
		TraceID:      ctx.TraceId,
		SpanID:       ctx.SpanId,
		Sampled:      ctx.Sampled,
		Baggage:      ctx.BaggageItems,
		ParentSpanID: &ctx.ParentSpanId,
		Flags:        flags,
	}, nil
}
<|endoftext|>


<commit_before><commit_msg>Added Owner() to IRodsObj interface<commit_after><|endoftext|>


<commit_before><commit_msg>Panic on chunk write errors<commit_after><|endoftext|>


<commit_before><commit_msg>remove oracle native<commit_after><|endoftext|>


<commit_before>package api

import (
	"fmt"
	"net/http"
	"io/ioutil"
	"time"

	"github.com/jcloudpub/speedy/chunkmaster/util"
	"github.com/jcloudpub/speedy/chunkmaster/util/log"
	"github.com/jcloudpub/speedy/chunkmaster/metadata"
)

const (
	MAX_RANGE = 10000
	ALLOCATE_RANGE = MAX_RANGE / 100

	INIT_STATUS = 0
	RW_STATUS = 1
	RO_STATUS = 2
	ERR_STATUS = 3

	GLOBAL_NORMAL_STATUS = 0
	GLOBAL_TRANSFER_STAUS = 8

	RO_STATUS_COUNT_MAX = 5
)

func MonitorTicker(intervalSecond int, timeoutSecond int) {
	timer := time.NewTicker(time.Duration(intervalSecond) * time.Second)
	for {
		select {
			case <- timer.C:
				chunkserverMonitor(timeoutSecond)
		}
	}
}

func chunkserverMonitor(timeoutSecond int) {
	serverInfoAbnormal := make(map[string]*metadata.Chunkserver)
	now := time.Now()
	timeout := time.Duration(timeoutSecond)*time.Second

	lock.RLock()

	for key, chunkserver := range serverInfo {
		if RW_STATUS == chunkserver.Status {
			if now.Sub(chunkserver.UpdateTime) > timeout {
				serverInfoAbnormal[key] = chunkserver
			}
		}
	}
	lock.RUnlock()

	if len(serverInfoAbnormal) == 0 {
		return
	}

	for key, chunkserver := range serverInfoAbnormal {
		err := mdDriver.UpdateChunkserverStatus(chunkserver, RW_STATUS, RO_STATUS)
		if err != nil {
			log.Errorf("[ChunkServerMonitor] update chunkserver failed: %v, %v", chunkserver, err)
		} else {
			updateServerInfo(key, RO_STATUS)
		}
	}
}

func updateServerInfo(key string, status int) {
	lock.Lock()
	chunkserver, ok := serverInfo[key]
	if !ok {
		log.Errorf("[updateServerInfo] key: %v do not exist", key)
		lock.Unlock()
		return
	}

	chunkserver.Status = status
	lock.Unlock()
	log.Errorf("[updateServerInfo] update RW_STATUS to RO_STATUS key: %v", key)
}

func reportChunkserverInfoHandler(resp http.ResponseWriter, req *http.Request) {
	reqData, err := ioutil.ReadAll(req.Body)
	if err != nil {
		util.HandleError(resp, "", err, http.StatusBadRequest)
		return
	}
	defer req.Body.Close()

	log.Infof("[reportChunkserverInfoHandler] reqData: %v", string(reqData))
	jsonMap, err := util.DecodeJson(reqData)
	if err != nil {
		util.HandleError(resp, "", err, http.StatusBadRequest)
		return
	}

	chunkserver, err := metadata.GenChunkserver(jsonMap)
	if err != nil {
		util.HandleError(resp, "", err, http.StatusBadRequest)
		return
	}

	key := fmt.Sprintf("%s:%d", chunkserver.Ip, chunkserver.Port)
	log.Infof("key: %v", key)
	log.Infof("serverInfo: %v", serverInfo)

	lock.RLock()
	oldChunkserver, ok := serverInfo[key]
	if !ok {
		lock.RUnlock()
		util.HandleError(resp, "", fmt.Errorf("not exist: %v ", chunkserver), http.StatusBadRequest)
		return
	}
	lock.RUnlock()

	err = reportChunkserverInfo(key, chunkserver, oldChunkserver)
	if err != nil {
		util.HandleError(resp, "", err, http.StatusInternalServerError)
	}

	log.Debugf("[reportInfoHandler] update chunkserver success: %v", chunkserver)
	util.Response(nil, http.StatusOK, resp)
}

func reportChunkserverInfo(key string, chunkserver *metadata.Chunkserver, oldChunkserver *metadata.Chunkserver) error {
	if oldChunkserver.Status == INIT_STATUS {
		err := mdDriver.UpdateChunkserverInfo(chunkserver, INIT_STATUS, RW_STATUS)
		if err != nil {
			return err
		}
		chunkserver.Status = RW_STATUS
	}

	if RW_STATUS == oldChunkserver.Status {
		err := mdDriver.UpdateChunkserverInfo(chunkserver, RW_STATUS, RW_STATUS)
		if err != nil {
			return err
		}
		chunkserver.Status = RW_STATUS
	}

	if RO_STATUS == oldChunkserver.Status {
		err := mdDriver.UpdateChunkserverInfo(chunkserver, RO_STATUS, RW_STATUS)
		if err != nil {
			return err
		}
		chunkserver.Status = RW_STATUS
	}

	chunkserver.UpdateTime = time.Now()

	lock.Lock()
	defer lock.Unlock()
	_, ok := serverInfo[key]
	if !ok {
		lock.Unlock()
		return fmt.Errorf("do not exist: %v", chunkserver)
	}
	serverInfo[key] = chunkserver
	return nil
}

func initChunkserverHandler(resp http.ResponseWriter, req *http.Request) {
	if err := util.ContentTypeCheck(req); err != nil {
		util.HandleError(resp, "", err, http.StatusBadRequest)
		return;
	}

	reqData, err := ioutil.ReadAll(req.Body)
	defer req.Body.Close()
	if err != nil {
		util.HandleError(resp, "", err, http.StatusBadRequest)
		return
	}
	log.Infof("[initserverHandler] read reqData %v", string(reqData))

	jsonMap, err := util.DecodeJson(reqData)
	if err != nil {
		util.HandleError(resp, "", err, http.StatusBadRequest)
		return
	}
	log.Infof("[initserverHandler] change json to map %v", jsonMap)

	groupId := uint16((jsonMap["GroupId"]).(float64))
	ip := jsonMap["Ip"].(string)
	port := int((jsonMap["Port"]).(float64))
	chunkserver := new(metadata.Chunkserver)
	chunkserver.GroupId = groupId
	chunkserver.Ip = ip
	chunkserver.Port = port
	chunkserver.Status = INIT_STATUS
	chunkserver.TotalFreeSpace = 0
	chunkserver.MaxFreeSpace = 0
	chunkserver.PendingWrites = 0
	chunkserver.WritingCount = 0
	chunkserver.DataDir = ""
	chunkserver.ReadingCount = 0
	chunkserver.TotalChunks = 0
	chunkserver.ConnectionsCount = 0

	err = addChunkserver(chunkserver)
	if err != nil {
		util.HandleError(resp, "", err, http.StatusInternalServerError)
		return
	}

	util.Response(nil, http.StatusOK, resp)
}

func loadChunkserverInfoHandler(resp http.ResponseWriter, req *http.Request) {
	err := LoadChunkserverInfo()
	if err != nil {
		util.HandleError(resp, "", err, http.StatusInternalServerError)
		return
	}
	log.Infof("[loadChunkserverInfoHandler] load chunkserver info success")
	util.Response(nil, http.StatusOK, resp)
}

func LoadChunkserverInfo() error {
	chunkserverList, err := mdDriver.ListChunkserver()
	log.Infof("[LoadChunkserverInfo] chunkserverList: %v", chunkserverList)
	if err != nil {
		return err
	}
	now := time.Now()
	serverInfoTemp := make(map[string]*metadata.Chunkserver)
	for _, server := range chunkserverList {
		if server.Status == ERR_STATUS {
			continue
		}

		key := fmt.Sprintf("%s:%d", server.Ip, server.Port)
		server.UpdateTime = now
		serverInfoTemp[key] = server
	}

	lock.Lock()
	serverInfo = serverInfoTemp
	lock.Unlock()
	log.Infof("[LoadChunkserverInfo]: %v", serverInfo)
	return nil
}

func addChunkserver(chunkserver *metadata.Chunkserver) error {
	err := mdDriver.AddChunkserver(chunkserver)
	if err != nil {
		return err
	}
	lock.Lock()
	defer lock.Unlock()

	serverInfoOld := serverInfo
	serverInfoTemp := make(map[string]*metadata.Chunkserver)
	for key, chunkserver := range serverInfoOld {
		serverInfoTemp[key] = chunkserver
	}

	key := fmt.Sprintf("%s:%d", chunkserver.Ip, chunkserver.Port)
	chunkserver.UpdateTime = time.Now()
	serverInfoTemp[key] = chunkserver
	serverInfo = serverInfoTemp

	return nil
}

func chunkserverCheckError(resp http.ResponseWriter, req *http.Request) {
	chunkservers, err := mdDriver.ListChunkserver()
	if err != nil {
		util.HandleError(resp, "", err, http.StatusInternalServerError)
		return
	}

	existErrChunkserver := false
	for _, chunkserver := range chunkservers {
		if chunkserver.Status == ERR_STATUS {
			existErrChunkserver = true
			break
		}
	}

	respData := []byte("0")
	if existErrChunkserver {
		respData = []byte("1")
	}

	util.Response(respData, http.StatusOK, resp)
}

func chunkmasterRouteHandler(resp http.ResponseWriter, req *http.Request) {
	lock.RLock()
	serverInfoTemp := serverInfo

	chunkGroup := make(map[string]metadata.Chunkservers)
	for _, chunkserver := range serverInfoTemp {
		groupId := fmt.Sprintf("%v", chunkserver.GroupId)

		list, ok := chunkGroup[groupId]
		if !ok {
			list = make(metadata.Chunkservers, 0, 3)
		}

		chunkGroup[groupId] = append(list, chunkserver)
	}

	lock.RUnlock()

	respData, err := util.EncodeJson(chunkGroup)
	if err != nil {
		util.HandleError(resp, "", err, http.StatusInternalServerError)
		return
	}

	resp.Header().Set("Content-Type", "application/json")
	util.Response(respData, http.StatusOK, resp)
}

func chunkmasterFidHandler(resp http.ResponseWriter, req *http.Request) {

	fidBegin, fidEnd, err := allocFid()
	if err != nil {
		util.HandleError(resp, "", err, http.StatusInternalServerError)
		return
	}
	log.Debugf("[chunkmasterFidHandle] allocate Fid fidBegin %v, fidEnd %v", fidBegin, fidEnd)

	jsonMap := make(map[string]interface{})
	jsonMap["FidBegin"] = fidBegin
	jsonMap["FidEnd"]   = fidEnd
	respData, err := util.EncodeJson(jsonMap)
	if err != nil {
		util.HandleError(resp, "", err, http.StatusInternalServerError)
		return
	}

	resp.Header().Set("Content-Type", "application/json")
	util.Response(respData, http.StatusOK, resp)
}

func allocFid() (uint64, uint64, error) {
	fid.Lock()
	defer fid.Unlock()

	var (
		fidBegin uint64
		fidEnd   uint64
		err error
	)

	if fid.Begin == fid.End {
		fid.Begin, err = mdDriver.GetFid()

		if err != nil {
			return 0, 0, err
		}

		fid.End = fid.Begin + MAX_RANGE
		err = mdDriver.UpdateFid(fid.End)
		if err != nil {
			fid.End = 0
			fid.Begin = 0
			return 0, 0, err
		}
	}

	fidBegin  = fid.Begin
	fidEnd    = fid.Begin + ALLOCATE_RANGE
	fid.Begin = fidEnd

	return fidBegin, fidEnd, nil
}
<commit_msg>adjust log level<commit_after>package api

import (
	"fmt"
	"net/http"
	"io/ioutil"
	"time"

	"github.com/jcloudpub/speedy/chunkmaster/util"
	"github.com/jcloudpub/speedy/chunkmaster/util/log"
	"github.com/jcloudpub/speedy/chunkmaster/metadata"
)

const (
	MAX_RANGE = 10000
	ALLOCATE_RANGE = MAX_RANGE / 100

	INIT_STATUS = 0
	RW_STATUS = 1
	RO_STATUS = 2
	ERR_STATUS = 3

	GLOBAL_NORMAL_STATUS = 0
	GLOBAL_TRANSFER_STAUS = 8

	RO_STATUS_COUNT_MAX = 5
)

func MonitorTicker(intervalSecond int, timeoutSecond int) {
	timer := time.NewTicker(time.Duration(intervalSecond) * time.Second)
	for {
		select {
			case <- timer.C:
				chunkserverMonitor(timeoutSecond)
		}
	}
}

func chunkserverMonitor(timeoutSecond int) {
	serverInfoAbnormal := make(map[string]*metadata.Chunkserver)
	now := time.Now()
	timeout := time.Duration(timeoutSecond)*time.Second

	lock.RLock()

	for key, chunkserver := range serverInfo {
		if RW_STATUS == chunkserver.Status {
			if now.Sub(chunkserver.UpdateTime) > timeout {
				serverInfoAbnormal[key] = chunkserver
			}
		}
	}
	lock.RUnlock()

	if len(serverInfoAbnormal) == 0 {
		return
	}

	for key, chunkserver := range serverInfoAbnormal {
		err := mdDriver.UpdateChunkserverStatus(chunkserver, RW_STATUS, RO_STATUS)
		if err != nil {
			log.Errorf("[ChunkServerMonitor] update chunkserver failed: %v, %v", chunkserver, err)
		} else {
			updateServerInfo(key, RO_STATUS)
		}
	}
}

func updateServerInfo(key string, status int) {
	lock.Lock()
	chunkserver, ok := serverInfo[key]
	if !ok {
		log.Errorf("[updateServerInfo] key: %v do not exist", key)
		lock.Unlock()
		return
	}

	chunkserver.Status = status
	lock.Unlock()
	log.Errorf("[updateServerInfo] update RW_STATUS to RO_STATUS key: %v", key)
}

func reportChunkserverInfoHandler(resp http.ResponseWriter, req *http.Request) {
	reqData, err := ioutil.ReadAll(req.Body)
	if err != nil {
		util.HandleError(resp, "", err, http.StatusBadRequest)
		return
	}
	defer req.Body.Close()

	log.Debugf("[reportChunkserverInfoHandler] reqData: %v", string(reqData))
	jsonMap, err := util.DecodeJson(reqData)
	if err != nil {
		util.HandleError(resp, "", err, http.StatusBadRequest)
		return
	}

	chunkserver, err := metadata.GenChunkserver(jsonMap)
	if err != nil {
		util.HandleError(resp, "", err, http.StatusBadRequest)
		return
	}

	key := fmt.Sprintf("%s:%d", chunkserver.Ip, chunkserver.Port)
	log.Debugf("key: %v", key)
	log.Debugf("serverInfo: %v", serverInfo)

	lock.RLock()
	oldChunkserver, ok := serverInfo[key]
	if !ok {
		lock.RUnlock()
		util.HandleError(resp, "", fmt.Errorf("not exist: %v ", chunkserver), http.StatusBadRequest)
		return
	}
	lock.RUnlock()

	err = reportChunkserverInfo(key, chunkserver, oldChunkserver)
	if err != nil {
		util.HandleError(resp, "", err, http.StatusInternalServerError)
	}

	log.Debugf("[reportInfoHandler] update chunkserver success: %v", chunkserver)
	util.Response(nil, http.StatusOK, resp)
}

func reportChunkserverInfo(key string, chunkserver *metadata.Chunkserver, oldChunkserver *metadata.Chunkserver) error {
	if oldChunkserver.Status == INIT_STATUS {
		err := mdDriver.UpdateChunkserverInfo(chunkserver, INIT_STATUS, RW_STATUS)
		if err != nil {
			return err
		}
		chunkserver.Status = RW_STATUS
	}

	if RW_STATUS == oldChunkserver.Status {
		err := mdDriver.UpdateChunkserverInfo(chunkserver, RW_STATUS, RW_STATUS)
		if err != nil {
			return err
		}
		chunkserver.Status = RW_STATUS
	}

	if RO_STATUS == oldChunkserver.Status {
		err := mdDriver.UpdateChunkserverInfo(chunkserver, RO_STATUS, RW_STATUS)
		if err != nil {
			return err
		}
		chunkserver.Status = RW_STATUS
	}

	chunkserver.UpdateTime = time.Now()

	lock.Lock()
	defer lock.Unlock()
	_, ok := serverInfo[key]
	if !ok {
		lock.Unlock()
		return fmt.Errorf("do not exist: %v", chunkserver)
	}
	serverInfo[key] = chunkserver
	return nil
}

func initChunkserverHandler(resp http.ResponseWriter, req *http.Request) {
	/*
	if err := util.ContentTypeCheck(req); err != nil {
		util.HandleError(resp, "", err, http.StatusBadRequest)
		return;
	}
	*/

	reqData, err := ioutil.ReadAll(req.Body)
	defer req.Body.Close()
	if err != nil {
		util.HandleError(resp, "", err, http.StatusBadRequest)
		return
	}
	log.Infof("[initserverHandler] read reqData %v", string(reqData))

	jsonMap, err := util.DecodeJson(reqData)
	if err != nil {
		util.HandleError(resp, "", err, http.StatusBadRequest)
		return
	}
	log.Infof("[initserverHandler] change json to map %v", jsonMap)

	groupId := uint16((jsonMap["GroupId"]).(float64))
	ip := jsonMap["Ip"].(string)
	port := int((jsonMap["Port"]).(float64))
	chunkserver := new(metadata.Chunkserver)
	chunkserver.GroupId = groupId
	chunkserver.Ip = ip
	chunkserver.Port = port
	chunkserver.Status = INIT_STATUS
	chunkserver.TotalFreeSpace = 0
	chunkserver.MaxFreeSpace = 0
	chunkserver.PendingWrites = 0
	chunkserver.WritingCount = 0
	chunkserver.DataDir = ""
	chunkserver.ReadingCount = 0
	chunkserver.TotalChunks = 0
	chunkserver.ConnectionsCount = 0

	err = addChunkserver(chunkserver)
	if err != nil {
		util.HandleError(resp, "", err, http.StatusInternalServerError)
		return
	}

	util.Response(nil, http.StatusOK, resp)
}

func loadChunkserverInfoHandler(resp http.ResponseWriter, req *http.Request) {
	err := LoadChunkserverInfo()
	if err != nil {
		util.HandleError(resp, "", err, http.StatusInternalServerError)
		return
	}
	log.Infof("[loadChunkserverInfoHandler] load chunkserver info success")
	util.Response(nil, http.StatusOK, resp)
}

func LoadChunkserverInfo() error {
	chunkserverList, err := mdDriver.ListChunkserver()
	log.Infof("[LoadChunkserverInfo] chunkserverList: %v", chunkserverList)
	if err != nil {
		return err
	}
	now := time.Now()
	serverInfoTemp := make(map[string]*metadata.Chunkserver)
	for _, server := range chunkserverList {
		if server.Status == ERR_STATUS {
			continue
		}

		key := fmt.Sprintf("%s:%d", server.Ip, server.Port)
		server.UpdateTime = now
		serverInfoTemp[key] = server
	}

	lock.Lock()
	serverInfo = serverInfoTemp
	lock.Unlock()
	log.Infof("[LoadChunkserverInfo]: %v", serverInfo)
	return nil
}

func addChunkserver(chunkserver *metadata.Chunkserver) error {
	err := mdDriver.AddChunkserver(chunkserver)
	if err != nil {
		return err
	}
	lock.Lock()
	defer lock.Unlock()

	serverInfoOld := serverInfo
	serverInfoTemp := make(map[string]*metadata.Chunkserver)
	for key, chunkserver := range serverInfoOld {
		serverInfoTemp[key] = chunkserver
	}

	key := fmt.Sprintf("%s:%d", chunkserver.Ip, chunkserver.Port)
	chunkserver.UpdateTime = time.Now()
	serverInfoTemp[key] = chunkserver
	serverInfo = serverInfoTemp

	return nil
}

func chunkserverCheckError(resp http.ResponseWriter, req *http.Request) {
	chunkservers, err := mdDriver.ListChunkserver()
	if err != nil {
		util.HandleError(resp, "", err, http.StatusInternalServerError)
		return
	}

	existErrChunkserver := false
	for _, chunkserver := range chunkservers {
		if chunkserver.Status == ERR_STATUS {
			existErrChunkserver = true
			break
		}
	}

	respData := []byte("0")
	if existErrChunkserver {
		respData = []byte("1")
	}

	util.Response(respData, http.StatusOK, resp)
}

func chunkmasterRouteHandler(resp http.ResponseWriter, req *http.Request) {
	lock.RLock()
	serverInfoTemp := serverInfo

	chunkGroup := make(map[string]metadata.Chunkservers)
	for _, chunkserver := range serverInfoTemp {
		groupId := fmt.Sprintf("%v", chunkserver.GroupId)

		list, ok := chunkGroup[groupId]
		if !ok {
			list = make(metadata.Chunkservers, 0, 3)
		}

		chunkGroup[groupId] = append(list, chunkserver)
	}

	lock.RUnlock()

	respData, err := util.EncodeJson(chunkGroup)
	if err != nil {
		util.HandleError(resp, "", err, http.StatusInternalServerError)
		return
	}

	resp.Header().Set("Content-Type", "application/json")
	util.Response(respData, http.StatusOK, resp)
}

func chunkmasterFidHandler(resp http.ResponseWriter, req *http.Request) {

	fidBegin, fidEnd, err := allocFid()
	if err != nil {
		util.HandleError(resp, "", err, http.StatusInternalServerError)
		return
	}
	log.Debugf("[chunkmasterFidHandle] allocate Fid fidBegin %v, fidEnd %v", fidBegin, fidEnd)

	jsonMap := make(map[string]interface{})
	jsonMap["FidBegin"] = fidBegin
	jsonMap["FidEnd"]   = fidEnd
	respData, err := util.EncodeJson(jsonMap)
	if err != nil {
		util.HandleError(resp, "", err, http.StatusInternalServerError)
		return
	}

	resp.Header().Set("Content-Type", "application/json")
	util.Response(respData, http.StatusOK, resp)
}

func allocFid() (uint64, uint64, error) {
	fid.Lock()
	defer fid.Unlock()

	var (
		fidBegin uint64
		fidEnd   uint64
		err error
	)

	if fid.Begin == fid.End {
		fid.Begin, err = mdDriver.GetFid()

		if err != nil {
			return 0, 0, err
		}

		fid.End = fid.Begin + MAX_RANGE
		err = mdDriver.UpdateFid(fid.End)
		if err != nil {
			fid.End = 0
			fid.Begin = 0
			return 0, 0, err
		}
	}

	fidBegin  = fid.Begin
	fidEnd    = fid.Begin + ALLOCATE_RANGE
	fid.Begin = fidEnd

	return fidBegin, fidEnd, nil
}
<|endoftext|>