<commit_before>package scanner

import (
	"container/list"
	"fmt"
	"log"
	"math/big"
	"regexp"
	"sync"
	"sync/atomic"
	"time"

	"github.com/google/certificate-transparency/go"
	"github.com/google/certificate-transparency/go/client"
	"github.com/google/certificate-transparency/go/x509"
)

// Clients wishing to implement their own Matchers should implement this interface:
type Matcher interface {
	// CertificateMatches is called by the scanner for each X509 Certificate found in the log.
	// The implementation should return |true| if the passed Certificate is interesting, and |false| otherwise.
	CertificateMatches(*x509.Certificate) bool

	// PrecertificateMatches is called by the scanner for each CT Precertificate found in the log.
	// The implementation should return |true| if the passed Precertificate is interesting, and |false| otherwise.
	PrecertificateMatches(*ct.Precertificate) bool
}

// MatchAll is a Matcher which will match every possible Certificate and Precertificate.
type MatchAll struct{}

func (m MatchAll) CertificateMatches(_ *x509.Certificate) bool {
	return true
}

func (m MatchAll) PrecertificateMatches(_ *ct.Precertificate) bool {
	return true
}

// MatchNone is a Matcher which will never match any Certificate or Precertificate.
type MatchNone struct{}

func (m MatchNone) CertificateMatches(_ *x509.Certificate) bool {
	return false
}

func (m MatchNone) PrecertificateMatches(_ *ct.Precertificate) bool {
	return false
}

type MatchSerialNumber struct {
	SerialNumber big.Int
}

func (m MatchSerialNumber) CertificateMatches(c *x509.Certificate) bool {
	return c.SerialNumber.String() == m.SerialNumber.String()
}

func (m MatchSerialNumber) PrecertificateMatches(p *ct.Precertificate) bool {
	return p.TBSCertificate.SerialNumber.String() == m.SerialNumber.String()
}

// MatchSubjectRegex is a Matcher which will use |CertificateSubjectRegex| and |PrecertificateSubjectRegex|
// to determine whether Certificates and Precertificates are interesting.
// The two regexes are tested against Subject Common Name as well as all
// Subject Alternative Names
type MatchSubjectRegex struct {
	CertificateSubjectRegex    *regexp.Regexp
	PrecertificateSubjectRegex *regexp.Regexp
}

// Returns true if either CN or any SAN of |c| matches |CertificateSubjectRegex|.
func (m MatchSubjectRegex) CertificateMatches(c *x509.Certificate) bool {
	if m.CertificateSubjectRegex.FindStringIndex(c.Subject.CommonName) != nil {
		return true
	}
	for _, alt := range c.DNSNames {
		if m.CertificateSubjectRegex.FindStringIndex(alt) != nil {
			return true
		}
	}
	return false
}

// Returns true if either CN or any SAN of |p| matches |PrecertificatesubjectRegex|.
func (m MatchSubjectRegex) PrecertificateMatches(p *ct.Precertificate) bool {
	if m.PrecertificateSubjectRegex.FindStringIndex(p.TBSCertificate.Subject.CommonName) != nil {
		return true
	}
	for _, alt := range p.TBSCertificate.DNSNames {
		if m.PrecertificateSubjectRegex.FindStringIndex(alt) != nil {
			return true
		}
	}
	return false
}

// Clients wishing to implement their own Tickers should implement this interface
type Ticker interface {
    HandleTick(s *Scanner, startTime time.Time, sth *ct.SignedTreeHead)
}

// LogTicker prints the progress and throughput
type LogTicker struct{}

func (t LogTicker) HandleTick(s *Scanner, startTime time.Time, sth *ct.SignedTreeHead) {
	throughput := float64(s.certsProcessed) / time.Since(startTime).Seconds()
	remainingCerts := int64(sth.TreeSize) - int64(s.opts.StartIndex) - s.certsProcessed
	remainingSeconds := int(float64(remainingCerts) / throughput)
	remainingString := humanTime(remainingSeconds)
	s.Log(fmt.Sprintf("Processed: %d certs (to index %d). Throughput: %3.2f ETA: %s\n", s.certsProcessed,
		s.opts.StartIndex+int64(s.certsProcessed), throughput, remainingString))
}

// ScannerOptions holds configuration options for the Scanner
type ScannerOptions struct {
	// Custom matcher for x509 Certificates, functor will be called for each
	// Certificate found during scanning.
	Matcher Matcher

	// Match precerts only (Matcher still applies to precerts)
	PrecertOnly bool

	// Number of entries to request in one batch from the Log
	BatchSize int

	// Number of concurrent matchers to run
	NumWorkers int

	// Number of concurrent fethers to run
	ParallelFetch int

	// Log entry index to start fetching & matching at
	StartIndex int64

	// Don't print any status messages to stdout
	Quiet bool

	// The length of time to wait before calling Tickers
	TickTime time.Duration

	// Custom ticker functions, will be called on every tick
	Tickers []Ticker
}

// Creates a new ScannerOptions struct with sensible defaults
func DefaultScannerOptions() *ScannerOptions {
	return &ScannerOptions{
		Matcher:       &MatchAll{},
		PrecertOnly:   false,
		BatchSize:     1000,
		NumWorkers:    1,
		ParallelFetch: 1,
		StartIndex:    0,
		Quiet:         false,
		Tickers:       []Ticker{LogTicker{}},
	}
}

// Scanner is a tool to scan all the entries in a CT Log.
type Scanner struct {
	// Client used to talk to the CT log instance
	logClient *client.LogClient

	// Configuration options for this Scanner instance
	opts ScannerOptions

	// Counter of the number of certificates scanned
	certsProcessed int64

	// Counter of the number of precertificates encountered during the scan.
	precertsSeen int64

	unparsableEntries         int64
	entriesWithNonFatalErrors int64
}

// matcherJob represents the context for an individual matcher job.
type matcherJob struct {
	// The log entry returned by the log server
	entry ct.LogEntry
	// The index of the entry containing the LeafInput in the log
	index int64
}

// fetchRange represents a range of certs to fetch from a CT log
type fetchRange struct {
	start int64
	end   int64
}

// Takes the error returned by either x509.ParseCertificate() or
// x509.ParseTBSCertificate() and determines if it's non-fatal or otherwise.
// In the case of non-fatal errors, the error will be logged,
// entriesWithNonFatalErrors will be incremented, and the return value will be
// nil.
// Fatal errors will be logged, unparsableEntires will be incremented, and the
// fatal error itself will be returned.
// When |err| is nil, this method does nothing.
func (s *Scanner) handleParseEntryError(err error, entryType ct.LogEntryType, index int64) error {
	if err == nil {
		// No error to handle
		return nil
	}
	switch err.(type) {
	case x509.NonFatalErrors:
		s.entriesWithNonFatalErrors++
		// We'll make a note, but continue.
		s.Log(fmt.Sprintf("Non-fatal error in %+v at index %d: %s", entryType, index, err.Error()))
	default:
		s.unparsableEntries++
		s.Log(fmt.Sprintf("Failed to parse in %+v at index %d : %s", entryType, index, err.Error()))
		return err
	}
	return nil
}

// Processes the given |entry| in the specified log.
func (s *Scanner) processEntry(entry ct.LogEntry, foundCert func(*ct.LogEntry), foundPrecert func(*ct.LogEntry)) {
	atomic.AddInt64(&s.certsProcessed, 1)
	switch entry.Leaf.TimestampedEntry.EntryType {
	case ct.X509LogEntryType:
		if s.opts.PrecertOnly {
			// Only interested in precerts and this is an X.509 cert, early-out.
			return
		}
		cert, err := x509.ParseCertificate(entry.Leaf.TimestampedEntry.X509Entry)
		if err = s.handleParseEntryError(err, entry.Leaf.TimestampedEntry.EntryType, entry.Index); err != nil {
			// We hit an unparseable entry, already logged inside handleParseEntryError()
			return
		}
		if s.opts.Matcher.CertificateMatches(cert) {
			entry.X509Cert = cert
			foundCert(&entry)
		}
	case ct.PrecertLogEntryType:
		c, err := x509.ParseTBSCertificate(entry.Leaf.TimestampedEntry.PrecertEntry.TBSCertificate)
		if err = s.handleParseEntryError(err, entry.Leaf.TimestampedEntry.EntryType, entry.Index); err != nil {
			// We hit an unparseable entry, already logged inside handleParseEntryError()
			return
		}
		precert := &ct.Precertificate{
			Raw:            entry.Chain[0],
			TBSCertificate: *c,
			IssuerKeyHash:  entry.Leaf.TimestampedEntry.PrecertEntry.IssuerKeyHash}
		if s.opts.Matcher.PrecertificateMatches(precert) {
			entry.Precert = precert
			foundPrecert(&entry)
		}
		s.precertsSeen++
	}
}

// Worker function to match certs.
// Accepts MatcherJobs over the |entries| channel, and processes them.
// Returns true over the |done| channel when the |entries| channel is closed.
func (s *Scanner) matcherJob(id int, entries <-chan matcherJob, foundCert func(*ct.LogEntry), foundPrecert func(*ct.LogEntry), wg *sync.WaitGroup) {
	for e := range entries {
		s.processEntry(e.entry, foundCert, foundPrecert)
	}
	s.Log(fmt.Sprintf("Matcher %d finished", id))
	wg.Done()
}

// Worker function for fetcher jobs.
// Accepts cert ranges to fetch over the |ranges| channel, and if the fetch is
// successful sends the individual LeafInputs out (as MatcherJobs) into the
// |entries| channel for the matchers to chew on.
// Will retry failed attempts to retrieve ranges indefinitely.
// Sends true over the |done| channel when the |ranges| channel is closed.
func (s *Scanner) fetcherJob(id int, ranges <-chan fetchRange, entries chan<- matcherJob, wg *sync.WaitGroup) {
	for r := range ranges {
		success := false
		// TODO(alcutter): give up after a while:
		for !success {
			logEntries, err := s.logClient.GetEntries(r.start, r.end)
			if err != nil {
				s.Log(fmt.Sprintf("Problem fetching from log: %s", err.Error()))
				continue
			}
			for _, logEntry := range logEntries {
				logEntry.Index = r.start
				entries <- matcherJob{logEntry, r.start}
				r.start++
			}
			if r.start > r.end {
				// Only complete if we actually got all the leaves we were
				// expecting -- Logs MAY return fewer than the number of
				// leaves requested.
				success = true
			}
		}
	}
	s.Log(fmt.Sprintf("Fetcher %d finished", id))
	wg.Done()
}

// Returns the smaller of |a| and |b|
func min(a int64, b int64) int64 {
	if a < b {
		return a
	} else {
		return b
	}
}

// Returns the larger of |a| and |b|
func max(a int64, b int64) int64 {
	if a > b {
		return a
	} else {
		return b
	}
}

// Pretty prints the passed in number of |seconds| into a more human readable
// string.
func humanTime(seconds int) string {
	nanos := time.Duration(seconds) * time.Second
	hours := int(nanos / (time.Hour))
	nanos %= time.Hour
	minutes := int(nanos / time.Minute)
	nanos %= time.Minute
	seconds = int(nanos / time.Second)
	s := ""
	if hours > 0 {
		s += fmt.Sprintf("%d hours ", hours)
	}
	if minutes > 0 {
		s += fmt.Sprintf("%d minutes ", minutes)
	}
	if seconds > 0 {
		s += fmt.Sprintf("%d seconds ", seconds)
	}
	return s
}

func (s Scanner) Log(msg string) {
	if !s.opts.Quiet {
		log.Print(msg)
	}
}

// Performs a scan against the Log.
// For each x509 certificate found, |foundCert| will be called with the
// index of the entry and certificate itself as arguments.  For each precert
// found, |foundPrecert| will be called with the index of the entry and the raw
// precert string as the arguments.
//
// This method blocks until the scan is complete.
func (s *Scanner) Scan(foundCert func(*ct.LogEntry),
	foundPrecert func(*ct.LogEntry)) error {
	s.Log("Starting up...\n")
	s.certsProcessed = 0
	s.precertsSeen = 0
	s.unparsableEntries = 0
	s.entriesWithNonFatalErrors = 0

	latestSth, err := s.logClient.GetSTH()
	if err != nil {
		return err
	}
	s.Log(fmt.Sprintf("Got STH with %d certs", latestSth.TreeSize))

	ticker := time.NewTicker(time.Second)
	startTime := time.Now()
	fetches := make(chan fetchRange, 1000)
	jobs := make(chan matcherJob, 100000)
	go func() {
		for range ticker.C {
			for _, t := range s.opts.Tickers {
				go t.HandleTick(s, startTime, latestSth)
			}
		}
	}()

	var ranges list.List
	for start := s.opts.StartIndex; start < int64(latestSth.TreeSize); {
		end := min(start+int64(s.opts.BatchSize), int64(latestSth.TreeSize)) - 1
		ranges.PushBack(fetchRange{start, end})
		start = end + 1
	}
	var fetcherWG sync.WaitGroup
	var matcherWG sync.WaitGroup
	// Start matcher workers
	for w := 0; w < s.opts.NumWorkers; w++ {
		matcherWG.Add(1)
		go s.matcherJob(w, jobs, foundCert, foundPrecert, &matcherWG)
	}
	// Start fetcher workers
	for w := 0; w < s.opts.ParallelFetch; w++ {
		fetcherWG.Add(1)
		go s.fetcherJob(w, fetches, jobs, &fetcherWG)
	}
	for r := ranges.Front(); r != nil; r = r.Next() {
		fetches <- r.Value.(fetchRange)
	}
	close(fetches)
	fetcherWG.Wait()
	close(jobs)
	matcherWG.Wait()

	s.Log(fmt.Sprintf("Completed %d certs in %s", s.certsProcessed, humanTime(int(time.Since(startTime).Seconds()))))
	s.Log(fmt.Sprintf("Saw %d precerts", s.precertsSeen))
	s.Log(fmt.Sprintf("%d unparsable entries, %d non-fatal errors", s.unparsableEntries, s.entriesWithNonFatalErrors))
	return nil
}

// Creates a new Scanner instance using |client| to talk to the log, and taking
// configuration options from |opts|.
func NewScanner(client *client.LogClient, opts ScannerOptions) *Scanner {
	var scanner Scanner
	scanner.logClient = client
	// Set a default match-everything regex if none was provided:
	if opts.Matcher == nil {
		opts.Matcher = &MatchAll{}
	}
	scanner.opts = opts
	return &scanner
}
<commit_msg>Fix: TickTime option was ignored<commit_after>package scanner

import (
	"container/list"
	"fmt"
	"log"
	"math/big"
	"regexp"
	"sync"
	"sync/atomic"
	"time"

	"github.com/google/certificate-transparency/go"
	"github.com/google/certificate-transparency/go/client"
	"github.com/google/certificate-transparency/go/x509"
)

// Clients wishing to implement their own Matchers should implement this interface:
type Matcher interface {
	// CertificateMatches is called by the scanner for each X509 Certificate found in the log.
	// The implementation should return |true| if the passed Certificate is interesting, and |false| otherwise.
	CertificateMatches(*x509.Certificate) bool

	// PrecertificateMatches is called by the scanner for each CT Precertificate found in the log.
	// The implementation should return |true| if the passed Precertificate is interesting, and |false| otherwise.
	PrecertificateMatches(*ct.Precertificate) bool
}

// MatchAll is a Matcher which will match every possible Certificate and Precertificate.
type MatchAll struct{}

func (m MatchAll) CertificateMatches(_ *x509.Certificate) bool {
	return true
}

func (m MatchAll) PrecertificateMatches(_ *ct.Precertificate) bool {
	return true
}

// MatchNone is a Matcher which will never match any Certificate or Precertificate.
type MatchNone struct{}

func (m MatchNone) CertificateMatches(_ *x509.Certificate) bool {
	return false
}

func (m MatchNone) PrecertificateMatches(_ *ct.Precertificate) bool {
	return false
}

type MatchSerialNumber struct {
	SerialNumber big.Int
}

func (m MatchSerialNumber) CertificateMatches(c *x509.Certificate) bool {
	return c.SerialNumber.String() == m.SerialNumber.String()
}

func (m MatchSerialNumber) PrecertificateMatches(p *ct.Precertificate) bool {
	return p.TBSCertificate.SerialNumber.String() == m.SerialNumber.String()
}

// MatchSubjectRegex is a Matcher which will use |CertificateSubjectRegex| and |PrecertificateSubjectRegex|
// to determine whether Certificates and Precertificates are interesting.
// The two regexes are tested against Subject Common Name as well as all
// Subject Alternative Names
type MatchSubjectRegex struct {
	CertificateSubjectRegex    *regexp.Regexp
	PrecertificateSubjectRegex *regexp.Regexp
}

// Returns true if either CN or any SAN of |c| matches |CertificateSubjectRegex|.
func (m MatchSubjectRegex) CertificateMatches(c *x509.Certificate) bool {
	if m.CertificateSubjectRegex.FindStringIndex(c.Subject.CommonName) != nil {
		return true
	}
	for _, alt := range c.DNSNames {
		if m.CertificateSubjectRegex.FindStringIndex(alt) != nil {
			return true
		}
	}
	return false
}

// Returns true if either CN or any SAN of |p| matches |PrecertificatesubjectRegex|.
func (m MatchSubjectRegex) PrecertificateMatches(p *ct.Precertificate) bool {
	if m.PrecertificateSubjectRegex.FindStringIndex(p.TBSCertificate.Subject.CommonName) != nil {
		return true
	}
	for _, alt := range p.TBSCertificate.DNSNames {
		if m.PrecertificateSubjectRegex.FindStringIndex(alt) != nil {
			return true
		}
	}
	return false
}

// Clients wishing to implement their own Tickers should implement this interface
type Ticker interface {
    HandleTick(s *Scanner, startTime time.Time, sth *ct.SignedTreeHead)
}

// LogTicker prints the progress and throughput
type LogTicker struct{}

func (t LogTicker) HandleTick(s *Scanner, startTime time.Time, sth *ct.SignedTreeHead) {
	throughput := float64(s.certsProcessed) / time.Since(startTime).Seconds()
	remainingCerts := int64(sth.TreeSize) - int64(s.opts.StartIndex) - s.certsProcessed
	remainingSeconds := int(float64(remainingCerts) / throughput)
	remainingString := humanTime(remainingSeconds)
	s.Log(fmt.Sprintf("Processed: %d certs (to index %d). Throughput: %3.2f ETA: %s\n", s.certsProcessed,
		s.opts.StartIndex+int64(s.certsProcessed), throughput, remainingString))
}

// ScannerOptions holds configuration options for the Scanner
type ScannerOptions struct {
	// Custom matcher for x509 Certificates, functor will be called for each
	// Certificate found during scanning.
	Matcher Matcher

	// Match precerts only (Matcher still applies to precerts)
	PrecertOnly bool

	// Number of entries to request in one batch from the Log
	BatchSize int

	// Number of concurrent matchers to run
	NumWorkers int

	// Number of concurrent fethers to run
	ParallelFetch int

	// Log entry index to start fetching & matching at
	StartIndex int64

	// Don't print any status messages to stdout
	Quiet bool

	// The length of time to wait before calling Tickers
	TickTime time.Duration

	// Custom ticker functions, will be called on every tick
	Tickers []Ticker
}

// Creates a new ScannerOptions struct with sensible defaults
func DefaultScannerOptions() *ScannerOptions {
	return &ScannerOptions{
		Matcher:       &MatchAll{},
		PrecertOnly:   false,
		BatchSize:     1000,
		NumWorkers:    1,
		ParallelFetch: 1,
		StartIndex:    0,
		Quiet:         false,
		TickTime:      time.Second,
		Tickers:       []Ticker{LogTicker{}},
	}
}

// Scanner is a tool to scan all the entries in a CT Log.
type Scanner struct {
	// Client used to talk to the CT log instance
	logClient *client.LogClient

	// Configuration options for this Scanner instance
	opts ScannerOptions

	// Counter of the number of certificates scanned
	certsProcessed int64

	// Counter of the number of precertificates encountered during the scan.
	precertsSeen int64

	unparsableEntries         int64
	entriesWithNonFatalErrors int64
}

// matcherJob represents the context for an individual matcher job.
type matcherJob struct {
	// The log entry returned by the log server
	entry ct.LogEntry
	// The index of the entry containing the LeafInput in the log
	index int64
}

// fetchRange represents a range of certs to fetch from a CT log
type fetchRange struct {
	start int64
	end   int64
}

// Takes the error returned by either x509.ParseCertificate() or
// x509.ParseTBSCertificate() and determines if it's non-fatal or otherwise.
// In the case of non-fatal errors, the error will be logged,
// entriesWithNonFatalErrors will be incremented, and the return value will be
// nil.
// Fatal errors will be logged, unparsableEntires will be incremented, and the
// fatal error itself will be returned.
// When |err| is nil, this method does nothing.
func (s *Scanner) handleParseEntryError(err error, entryType ct.LogEntryType, index int64) error {
	if err == nil {
		// No error to handle
		return nil
	}
	switch err.(type) {
	case x509.NonFatalErrors:
		s.entriesWithNonFatalErrors++
		// We'll make a note, but continue.
		s.Log(fmt.Sprintf("Non-fatal error in %+v at index %d: %s", entryType, index, err.Error()))
	default:
		s.unparsableEntries++
		s.Log(fmt.Sprintf("Failed to parse in %+v at index %d : %s", entryType, index, err.Error()))
		return err
	}
	return nil
}

// Processes the given |entry| in the specified log.
func (s *Scanner) processEntry(entry ct.LogEntry, foundCert func(*ct.LogEntry), foundPrecert func(*ct.LogEntry)) {
	atomic.AddInt64(&s.certsProcessed, 1)
	switch entry.Leaf.TimestampedEntry.EntryType {
	case ct.X509LogEntryType:
		if s.opts.PrecertOnly {
			// Only interested in precerts and this is an X.509 cert, early-out.
			return
		}
		cert, err := x509.ParseCertificate(entry.Leaf.TimestampedEntry.X509Entry)
		if err = s.handleParseEntryError(err, entry.Leaf.TimestampedEntry.EntryType, entry.Index); err != nil {
			// We hit an unparseable entry, already logged inside handleParseEntryError()
			return
		}
		if s.opts.Matcher.CertificateMatches(cert) {
			entry.X509Cert = cert
			foundCert(&entry)
		}
	case ct.PrecertLogEntryType:
		c, err := x509.ParseTBSCertificate(entry.Leaf.TimestampedEntry.PrecertEntry.TBSCertificate)
		if err = s.handleParseEntryError(err, entry.Leaf.TimestampedEntry.EntryType, entry.Index); err != nil {
			// We hit an unparseable entry, already logged inside handleParseEntryError()
			return
		}
		precert := &ct.Precertificate{
			Raw:            entry.Chain[0],
			TBSCertificate: *c,
			IssuerKeyHash:  entry.Leaf.TimestampedEntry.PrecertEntry.IssuerKeyHash}
		if s.opts.Matcher.PrecertificateMatches(precert) {
			entry.Precert = precert
			foundPrecert(&entry)
		}
		s.precertsSeen++
	}
}

// Worker function to match certs.
// Accepts MatcherJobs over the |entries| channel, and processes them.
// Returns true over the |done| channel when the |entries| channel is closed.
func (s *Scanner) matcherJob(id int, entries <-chan matcherJob, foundCert func(*ct.LogEntry), foundPrecert func(*ct.LogEntry), wg *sync.WaitGroup) {
	for e := range entries {
		s.processEntry(e.entry, foundCert, foundPrecert)
	}
	s.Log(fmt.Sprintf("Matcher %d finished", id))
	wg.Done()
}

// Worker function for fetcher jobs.
// Accepts cert ranges to fetch over the |ranges| channel, and if the fetch is
// successful sends the individual LeafInputs out (as MatcherJobs) into the
// |entries| channel for the matchers to chew on.
// Will retry failed attempts to retrieve ranges indefinitely.
// Sends true over the |done| channel when the |ranges| channel is closed.
func (s *Scanner) fetcherJob(id int, ranges <-chan fetchRange, entries chan<- matcherJob, wg *sync.WaitGroup) {
	for r := range ranges {
		success := false
		// TODO(alcutter): give up after a while:
		for !success {
			logEntries, err := s.logClient.GetEntries(r.start, r.end)
			if err != nil {
				s.Log(fmt.Sprintf("Problem fetching from log: %s", err.Error()))
				continue
			}
			for _, logEntry := range logEntries {
				logEntry.Index = r.start
				entries <- matcherJob{logEntry, r.start}
				r.start++
			}
			if r.start > r.end {
				// Only complete if we actually got all the leaves we were
				// expecting -- Logs MAY return fewer than the number of
				// leaves requested.
				success = true
			}
		}
	}
	s.Log(fmt.Sprintf("Fetcher %d finished", id))
	wg.Done()
}

// Returns the smaller of |a| and |b|
func min(a int64, b int64) int64 {
	if a < b {
		return a
	} else {
		return b
	}
}

// Returns the larger of |a| and |b|
func max(a int64, b int64) int64 {
	if a > b {
		return a
	} else {
		return b
	}
}

// Pretty prints the passed in number of |seconds| into a more human readable
// string.
func humanTime(seconds int) string {
	nanos := time.Duration(seconds) * time.Second
	hours := int(nanos / (time.Hour))
	nanos %= time.Hour
	minutes := int(nanos / time.Minute)
	nanos %= time.Minute
	seconds = int(nanos / time.Second)
	s := ""
	if hours > 0 {
		s += fmt.Sprintf("%d hours ", hours)
	}
	if minutes > 0 {
		s += fmt.Sprintf("%d minutes ", minutes)
	}
	if seconds > 0 {
		s += fmt.Sprintf("%d seconds ", seconds)
	}
	return s
}

func (s Scanner) Log(msg string) {
	if !s.opts.Quiet {
		log.Print(msg)
	}
}

// Performs a scan against the Log.
// For each x509 certificate found, |foundCert| will be called with the
// index of the entry and certificate itself as arguments.  For each precert
// found, |foundPrecert| will be called with the index of the entry and the raw
// precert string as the arguments.
//
// This method blocks until the scan is complete.
func (s *Scanner) Scan(foundCert func(*ct.LogEntry),
	foundPrecert func(*ct.LogEntry)) error {
	s.Log("Starting up...\n")
	s.certsProcessed = 0
	s.precertsSeen = 0
	s.unparsableEntries = 0
	s.entriesWithNonFatalErrors = 0

	latestSth, err := s.logClient.GetSTH()
	if err != nil {
		return err
	}
	s.Log(fmt.Sprintf("Got STH with %d certs", latestSth.TreeSize))

	ticker := time.NewTicker(s.opts.TickTime)
	startTime := time.Now()
	fetches := make(chan fetchRange, 1000)
	jobs := make(chan matcherJob, 100000)
	go func() {
		for range ticker.C {
			for _, t := range s.opts.Tickers {
				go t.HandleTick(s, startTime, latestSth)
			}
		}
	}()

	var ranges list.List
	for start := s.opts.StartIndex; start < int64(latestSth.TreeSize); {
		end := min(start+int64(s.opts.BatchSize), int64(latestSth.TreeSize)) - 1
		ranges.PushBack(fetchRange{start, end})
		start = end + 1
	}
	var fetcherWG sync.WaitGroup
	var matcherWG sync.WaitGroup
	// Start matcher workers
	for w := 0; w < s.opts.NumWorkers; w++ {
		matcherWG.Add(1)
		go s.matcherJob(w, jobs, foundCert, foundPrecert, &matcherWG)
	}
	// Start fetcher workers
	for w := 0; w < s.opts.ParallelFetch; w++ {
		fetcherWG.Add(1)
		go s.fetcherJob(w, fetches, jobs, &fetcherWG)
	}
	for r := ranges.Front(); r != nil; r = r.Next() {
		fetches <- r.Value.(fetchRange)
	}
	close(fetches)
	fetcherWG.Wait()
	close(jobs)
	matcherWG.Wait()

	s.Log(fmt.Sprintf("Completed %d certs in %s", s.certsProcessed, humanTime(int(time.Since(startTime).Seconds()))))
	s.Log(fmt.Sprintf("Saw %d precerts", s.precertsSeen))
	s.Log(fmt.Sprintf("%d unparsable entries, %d non-fatal errors", s.unparsableEntries, s.entriesWithNonFatalErrors))
	return nil
}

// Creates a new Scanner instance using |client| to talk to the log, and taking
// configuration options from |opts|.
func NewScanner(client *client.LogClient, opts ScannerOptions) *Scanner {
	var scanner Scanner
	scanner.logClient = client
	// Set a default match-everything regex if none was provided:
	if opts.Matcher == nil {
		opts.Matcher = &MatchAll{}
	}
	scanner.opts = opts
	return &scanner
}
<|endoftext|>


<commit_before>package s

import (
	"os"
	"path/filepath"
	"unsafe"

	"github.com/asp2insp/go-misc/utils"
	"github.com/asp2insp/runnel-go/i"
	"github.com/edsrzf/mmap-go"
)

type fileStorage struct {
	fileId       string
	rootPath     string
	file         *os.File
	headerFile   *os.File
	mappedMemory mmap.MMap
	headerMemory mmap.MMap
	header       *i.StreamHeader
}

func NewFileStorage(root string) *fileStorage {
	return &fileStorage{
		rootPath: root,
	}
}

// STORAGE
func (store *fileStorage) Init(id string) i.Storage {
	store.fileId = id

	// Update the header
	headerSize := uint64(unsafe.Sizeof(&i.StreamHeader{}))
	store.headerMemory, store.headerFile = mmapFile(fheader(store.fileId, store.rootPath), headerSize, os.O_RDWR|os.O_CREATE, mmap.RDWR)
	store.header = mmapToHeader(store.headerMemory)

	// Map in the data
	store.Resize(uint64(os.Getpagesize()))
	return store
}

func (store *fileStorage) Resize(size uint64) i.Storage {
	var fileName string
	if store.file != nil {
		err := store.file.Truncate(int64(size))
		utils.Check(err)
		fileName = store.file.Name()
	} else {
		fileName = fname(store.fileId, store.rootPath)
	}
	// Re-map our data
	tmpMap := store.mappedMemory
	tmpFile := store.file
	store.mappedMemory, store.file = mmapFile(fileName, size, os.O_APPEND|os.O_RDWR|os.O_CREATE, mmap.RDWR)
	if len(tmpMap) > 0 {
		tmpMap.Unmap()
	}
	if tmpFile != nil {
		tmpFile.Close()
	}
	store.header.FileSize = size
	return store
}

func (store *fileStorage) GetBytes(start, end int64) []byte {
	return store.mappedMemory
}

func (store *fileStorage) Capacity() uint64 {
	return store.header.FileSize
}

func (store *fileStorage) Header() *i.StreamHeader {
	return store.header
}

func (store *fileStorage) Utilization() int {
	return int(store.header.Tail * 100 / (store.Capacity()))
}

func (store *fileStorage) Flush() {
	store.mappedMemory.Flush()
	store.headerMemory.Flush()
}

func (store *fileStorage) Refresh() {
	// TODO: Refresh header then refresh file.
	// 			 This will be much easier once I have the single
	//       file reference refactor in place
	// tmpMap := store.mappedMemory
	// tmpFile := store.file
	// store.mappedMemory, store.file = mmapFile(fileName, size, os.O_APPEND|os.O_RDWR|os.O_CREATE, mmap.RDWR)
	// if len(tmpMap) > 0 {
	// 	tmpMap.Unmap()
	// }
	// if tmpFile != nil {
	// 	tmpFile.Close()
	// }
}

// CLOSABLE

// Close this storage, by closing the file
// pointers and unmapping all memory
func (store *fileStorage) Close() {
	store.header = &i.StreamHeader{} // Empty the header so calls to Size() return 0
	// Release the memory
	store.mappedMemory.Unmap()
	store.file.Close()
	store.headerMemory.Unmap()
	store.headerFile.Close()
}

// UTILS

// Open the given file with the given flags
func open(path string, fileFlags int) *File {
	file, err := os.OpenFile(path, fileFlags, 0666)
	utils.Check(err)
	if utils.Filesize(file) == 0 {
		err = file.Truncate(int64(os.Getpagesize()))
		utils.Check(err)
	}
	return file
}

// Map the file at the given path into memory with the given flags.
// Panics if the given file cannot be opened or mmapped
func mmapFile(path string, size uint64, fileFlags, mmapFlags int) (mmap.MMap, *os.File) {

	mapData, err := mmap.Map(file, mmapFlags, 0)
	utils.Check(err)
	return mapData, file
}

// Return a path to the file named with the given id.
// If a root dir is provided, the file will be relative
// to that root. Otherwise it is placed in the tmpdir
func fname(id, root string) string {
	if root != "" {
		return filepath.Join(root, id)
	} else {
		return filepath.Join(os.TempDir(), id)
	}
}

// Return a path to the header file for the given id.
// Will always be co-located with the file returned by fname
func fheader(id, root string) string {
	return fname(id, root) + "_header"
}

// Unsafe cast the []byte represented by the mmapped region
// to a streamHeader
func mmapToHeader(data mmap.MMap) *i.StreamHeader {
	return (*i.StreamHeader)(unsafe.Pointer(&data[0]))
}
<commit_msg>Finished refactor to single file ref<commit_after>package s

import (
	"os"
	"path/filepath"
	"unsafe"

	"github.com/asp2insp/go-misc/utils"
	"github.com/asp2insp/runnel-go/i"
	"github.com/edsrzf/mmap-go"
)

type fileStorage struct {
	fileId       string
	rootPath     string
	file         *os.File
	headerFile   *os.File
	mappedMemory mmap.MMap
	headerMemory mmap.MMap
	header       *i.StreamHeader
}

func NewFileStorage(root string) *fileStorage {
	return &fileStorage{
		rootPath: root,
	}
}

// STORAGE
func (store *fileStorage) Init(id string) i.Storage {
	store.fileId = id

	// Init the header
	headerSize := int64(unsafe.Sizeof(&i.StreamHeader{}))
	store.headerFile = open(fheader(store.fileId, store.rootPath), os.O_RDWR|os.O_CREATE|os.O_APPEND)
	store.headerFile.Truncate(headerSize)
	store.headerMemory = mmapFile(store.headerFile, mmap.RDWR)
	store.header = mmapToHeader(store.headerMemory)

	// Init the data
	store.file = open(fname(store.fileId, store.rootPath), os.O_RDWR|os.O_CREATE|os.O_APPEND)
	store.mappedMemory = mmapFile(store.file, mmap.RDWR)
	return store
}

func (store *fileStorage) Resize(size uint64) i.Storage {
	if store.file != nil {
		err := store.file.Truncate(int64(size))
		utils.Check(err)
	}
	// Re-map our data
	tmpMap := store.mappedMemory
	store.mappedMemory = mmapFile(store.file, mmap.RDWR)
	if len(tmpMap) > 0 {
		tmpMap.Unmap()
	}
	store.header.FileSize = size
	return store
}

func (store *fileStorage) GetBytes(start, end int64) []byte {
	return store.mappedMemory
}

func (store *fileStorage) Capacity() uint64 {
	return store.header.FileSize
}

func (store *fileStorage) Header() *i.StreamHeader {
	return store.header
}

func (store *fileStorage) Utilization() int {
	return int(store.header.Tail * 100 / (store.Capacity()))
}

func (store *fileStorage) Flush() {
	store.mappedMemory.Flush()
	store.headerMemory.Flush()
}

func (store *fileStorage) Refresh() {
	store.mappedMemory = mmapFile(store.file, mmap.RDWR)
	store.headerMemory = mmapFile(store.headerFile, mmap.RDWR)
}

// CLOSABLE

// Close this storage, by closing the file
// pointers and unmapping all memory
func (store *fileStorage) Close() {
	store.header = &i.StreamHeader{} // Empty the header so calls to Size() return 0
	// Release the memory
	store.mappedMemory.Unmap()
	store.file.Close()
	store.headerMemory.Unmap()
	store.headerFile.Close()
}

// UTILS

// Open the given file with the given flags
func open(path string, fileFlags int) *os.File {
	file, err := os.OpenFile(path, fileFlags, 0666)
	utils.Check(err)
	if utils.Filesize(file) == 0 {
		err = file.Truncate(int64(os.Getpagesize()))
		utils.Check(err)
	}
	return file
}

// Map the file at the given path into memory with the given flags.
// Panics if the given file cannot be opened or mmapped
func mmapFile(file *os.File, mmapFlags int) mmap.MMap {
	mapData, err := mmap.Map(file, mmapFlags, 0)
	utils.Check(err)
	return mapData
}

// Return a path to the file named with the given id.
// If a root dir is provided, the file will be relative
// to that root. Otherwise it is placed in the tmpdir
func fname(id, root string) string {
	if root != "" {
		return filepath.Join(root, id)
	} else {
		return filepath.Join(os.TempDir(), id)
	}
}

// Return a path to the header file for the given id.
// Will always be co-located with the file returned by fname
func fheader(id, root string) string {
	return fname(id, root) + "_header"
}

// Unsafe cast the []byte represented by the mmapped region
// to a streamHeader
func mmapToHeader(data mmap.MMap) *i.StreamHeader {
	return (*i.StreamHeader)(unsafe.Pointer(&data[0]))
}
<|endoftext|>


<commit_before>// Copyright 2012 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package auth

import (
	"github.com/jacobsa/aws/s3/http"
)

// A Signer knows how to create signatures suitable for inclusion in an HTTP
// request to S3.
type Signer interface {
	// Add an appropriate signature header to the supplied HTTP request.
	Sign(r *http.Request) error
}
<commit_msg>Declared the NewSigner function.<commit_after>// Copyright 2012 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package auth

import (
	"errors"
	"github.com/jacobsa/aws"
	"github.com/jacobsa/aws/s3/http"
)

// A Signer knows how to create signatures suitable for inclusion in an HTTP
// request to S3.
type Signer interface {
	// Add an appropriate signature header to the supplied HTTP request.
	Sign(r *http.Request) error
}

// NewSigner creates a Signer using the supplied access key.
func NewSigner(key aws.AccessKey) (Signer, error) {
	return nil, errors.New("TODO: Implement NewSigner.")
}
<|endoftext|>


<commit_before>// Copyright 2012 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package s3

import (
	. "github.com/jacobsa/ogletest"
	"testing"
)

func TestBucket(t *testing.T) { RunTests(t) }

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

type bucketTest struct {
	httpConn mock_http.MockConn
	signer mock_auth.MockSigner
	bucket Bucket
}

func (t *ConnTest) SetUp(i *TestInfo) {
	var err error

	t.httpConn = mock_http.NewMockConn(i.MockController, "httpConn")
	t.signer = mock_auth.NewMockSigner(i.MockController, "signer")
	t.bucket, err = openBucket("some.bucket", t.httpConn, t.signer)
	AssertEq(nil, err)
}

////////////////////////////////////////////////////////////////////////
// GetObject
////////////////////////////////////////////////////////////////////////

type GetObjectTest struct {
	bucketTest
}

func init() { RegisterTestSuite(&GetObjectTest{}) }

func (t *GetObjectTest) DoesFoo() {
	ExpectEq("TODO", "")
}

////////////////////////////////////////////////////////////////////////
// StoreObject
////////////////////////////////////////////////////////////////////////

type StoreObjectTest struct {
	bucketTest
}

func init() { RegisterTestSuite(&StoreObjectTest{}) }

func (t *StoreObjectTest) DoesFoo() {
	ExpectEq("TODO", "")
}
<commit_msg>Fixed build errors.<commit_after>// Copyright 2012 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package s3

import (
	"github.com/jacobsa/aws/s3/auth/mock"
	"github.com/jacobsa/aws/s3/http/mock"
	. "github.com/jacobsa/ogletest"
	"testing"
)

func TestBucket(t *testing.T) { RunTests(t) }

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

type bucketTest struct {
	httpConn mock_http.MockConn
	signer mock_auth.MockSigner
	bucket Bucket
}

func (t *bucketTest) SetUp(i *TestInfo) {
	var err error

	t.httpConn = mock_http.NewMockConn(i.MockController, "httpConn")
	t.signer = mock_auth.NewMockSigner(i.MockController, "signer")
	t.bucket, err = openBucket("some.bucket", t.httpConn, t.signer)
	AssertEq(nil, err)
}

////////////////////////////////////////////////////////////////////////
// GetObject
////////////////////////////////////////////////////////////////////////

type GetObjectTest struct {
	bucketTest
}

func init() { RegisterTestSuite(&GetObjectTest{}) }

func (t *GetObjectTest) DoesFoo() {
	ExpectEq("TODO", "")
}

////////////////////////////////////////////////////////////////////////
// StoreObject
////////////////////////////////////////////////////////////////////////

type StoreObjectTest struct {
	bucketTest
}

func init() { RegisterTestSuite(&StoreObjectTest{}) }

func (t *StoreObjectTest) DoesFoo() {
	ExpectEq("TODO", "")
}
<|endoftext|>


<commit_before>package common

import (
	"fmt"
	"math"
)

func Multiple(a []int, b []int, r []int) {

	pos := len(r) - 1

	for i := 0; i < len(r); i++ {
		r[i] = 0
	}

	for i := len(a) - 1; i >= 0; i-- {
		off := len(a) - 1 - i

		for j := len(b) - 1; j >= 0; j-- {
			tmp := b[j] * a[i]
			r[pos-off] += tmp % 10
			r[pos-off-1] += r[pos-off]/10 + tmp/10
			r[pos-off] %= 10
			off++
		}
	}
}

func MultipleWithResult(f []int, g []int) []int {

	if len(f) < len(g) {
		t := f
		f = g
		g = t
	}

	result := make([]int, len(f)+len(g)+1)

	for j := len(g) - 1; j >= 0; j-- {
		//initially pos (index of result ) has the same relative postion as j in array
		pos := len(result) - (len(g) - j)
		for i := len(f) - 1; i >= 0; i-- {
			temp := g[j] * f[i]
			result[pos] += temp % 10
			result[pos-1] += temp / 10
			pos--
		}
		//	fmt.Println(result)
	}

	for i := len(result) - 1; i > 0; i-- {
		temp := result[i]
		result[i] = temp % 10
		result[i-1] += temp / 10
	}

	return result
}

func Divisor(n int) []int {

	nums := make([]int, 500)

	for i := 0; i < len(nums); i++ {
		nums[i] = 0
	}

	i, c := 1, 0

	for {
		if i*i > n {
			break
		}

		tmp := n % i
		if tmp == 0 {
			nums[c] = i
			// only collect 1, ingore the number itself
			if i != 1 && i != n/i {
				c++
				nums[c] = n / i
			}
		}
		i++
		c++
		//fmt.Println("c:  ", c)
	}

	return nums
}

func Fib(n int) int {
	if n <= 2 {
		return 1
	}

	return Fib(n-1) + Fib(n-2)
}

// https://stackoverflow.com/questions/33923/what-is-tail-recursion
// 0, 1, 1, 2, 3, 5, 8, 13, 21, 34, 55, 89, 144, 233, 377, 610, 987, 1597, 2584, 4181, 6765, 10946, 17711, 28657, 46368, 75025, 121393, 196418, 317811, ...

func TailFib(n int) int {
	if n <= 2 {
		return 1
	}
	return tailRecursiveAux(n, 1, 1)
}

func tailRecursiveAux(n int, iter int, acc int) int {
	fmt.Println(n, iter, acc)
	if iter == n {
		return acc
	}
	return tailRecursiveAux(n, iter+1, acc+iter)
}

func GoldenFib(n int) float64 {
	f := 1.618034
	m := math.Pow(f, 6) - math.Pow(1-f, 6)
	l := math.Round(m / math.Sqrt(5))
	fmt.Println(f, m, l)
	return l

}
<commit_msg>update<commit_after>package common

import (
	"fmt"
	"math"
)

func Plus(a []int, b []int, r []int) {
	for i := 0; i < len(r); i++ {
		r[i] = 0
	}

	pos := len(r) - 1

	j := pos
	for {

	}
}

func Multiple(a []int, b []int, r []int) {

	pos := len(r) - 1

	for i := 0; i < len(r); i++ {
		r[i] = 0
	}

	for i := len(a) - 1; i >= 0; i-- {
		off := len(a) - 1 - i

		for j := len(b) - 1; j >= 0; j-- {
			tmp := b[j] * a[i]
			r[pos-off] += tmp % 10
			r[pos-off-1] += r[pos-off]/10 + tmp/10
			r[pos-off] %= 10
			off++
		}
	}
}

func MultipleWithResult(f []int, g []int) []int {

	if len(f) < len(g) {
		t := f
		f = g
		g = t
	}

	result := make([]int, len(f)+len(g)+1)

	for j := len(g) - 1; j >= 0; j-- {
		//initially pos (index of result ) has the same relative postion as j in array
		pos := len(result) - (len(g) - j)
		for i := len(f) - 1; i >= 0; i-- {
			temp := g[j] * f[i]
			result[pos] += temp % 10
			result[pos-1] += temp / 10
			pos--
		}
		//	fmt.Println(result)
	}

	for i := len(result) - 1; i > 0; i-- {
		temp := result[i]
		result[i] = temp % 10
		result[i-1] += temp / 10
	}

	return result
}

func Divisor(n int) []int {

	nums := make([]int, 500)

	for i := 0; i < len(nums); i++ {
		nums[i] = 0
	}

	i, c := 1, 0

	for {
		if i*i > n {
			break
		}

		tmp := n % i
		if tmp == 0 {
			nums[c] = i
			// only collect 1, ingore the number itself
			if i != 1 && i != n/i {
				c++
				nums[c] = n / i
			}
		}
		i++
		c++
		//fmt.Println("c:  ", c)
	}

	return nums
}

func Fib(n int) int {
	if n <= 2 {
		return 1
	}

	return Fib(n-1) + Fib(n-2)
}

// https://stackoverflow.com/questions/33923/what-is-tail-recursion
// 0, 1, 1, 2, 3, 5, 8, 13, 21, 34, 55, 89, 144, 233, 377, 610, 987, 1597, 2584, 4181, 6765, 10946, 17711, 28657, 46368, 75025, 121393, 196418, 317811, ...

func TailFib(n int) int {
	if n <= 2 {
		return 1
	}
	return tailRecursiveAux(n, 1, 1)
}

func tailRecursiveAux(n int, iter int, acc int) int {
	//	fmt.Println(n, iter, acc)
	if iter == n {
		return acc
	}
	//return tailRecursiveAux(n, iter++, acc+iter)
	iter++
	return tailRecursiveAux(n, iter, acc+iter)
}

func GoldenFib(n int) float64 {
	f := 1.618034
	m := math.Pow(f, 6) - math.Pow(1-f, 6)
	l := math.Round(m / math.Sqrt(5))
	fmt.Println(f, m, l)
	return l

}

func TailFibLarge(n []int) []int {
	if n <= 2 {
		return 1
	}
	return tailRecursiveAuxLarge(n, 1, 1)
}

func tailRecursiveAuxLarge(n []int, iter int, acc int) int {
	//	fmt.Println(n, iter, acc)
	if iter == n {
		return acc
	}
	//return tailRecursiveAux(n, iter++, acc+iter)
	iter++
	return tailRecursiveAuxLarge(n, iter, acc+iter)
}
<|endoftext|>


<commit_before>// Copyright 2012 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package s3

import (
	"bytes"
	"crypto/md5"
	"encoding/base64"
	"fmt"
	"github.com/jacobsa/aws"
	"github.com/jacobsa/aws/s3/auth"
	"github.com/jacobsa/aws/s3/http"
	"github.com/jacobsa/aws/s3/time"
	"net/url"
	"strings"
	sys_time "time"
	"unicode/utf8"
)

// Bucket represents an S3 bucket, which is a collection of objects keyed on
// Unicode strings. The UTF-8 encoding of a key must be no more than 1024 bytes
// long.
//
// See here for more info:
//
//     http://goo.gl/Nd63t
//
type Bucket interface {
	// Retrieve data for the object with the given key.
	GetObject(key string) (data []byte, err error)

	// Store the supplied data with the given key, overwriting any previous
	// version. The object is created with the default ACL of "private".
	StoreObject(key string, data []byte) error

	// Return an ordered set of contiguous object keys in the bucket that are
	// greater than or equal to minKey. It is guaranteed that as some time during
	// the request there were no keys between minKey and the first key returned.
	//
	// There may be more keys beyond the last key returned. If no keys are
	// returned (and the error is nil), it is guaranteed that at some time during
	// the request there were the bucket contained no keys in [minKey, inf).
	ListKeys(minKey string) (keys []string, err error)
}

// OpenBucket returns a Bucket tied to a given name in whe given region. You
// must have previously created the bucket in the region, and the supplied
// access key must have access to it.
//
// To easily create a bucket, use the AWS Console:
//
//     http://aws.amazon.com/console/
//
func OpenBucket(name string, region Region, key aws.AccessKey) (Bucket, error) {
	// Create a connection to the given region's endpoint.
	endpoint := &url.URL{Scheme: "https", Host: string(region)}
	httpConn, err := http.NewConn(endpoint)
	if err != nil {
		return nil, fmt.Errorf("http.NewConn: %v", err)
	}

	// Create an appropriate request signer.
	signer, err := auth.NewSigner(&key)
	if err != nil {
		return nil, fmt.Errorf("auth.NewSigner: %v", err)
	}

	return openBucket(name, httpConn, signer, time.RealClock())
}

// A version of OpenBucket with the ability to inject dependencies, for
// testability.
func openBucket(
	name string,
	httpConn http.Conn,
	signer auth.Signer,
	clock time.Clock) (Bucket, error) {
	return &bucket{name, httpConn, signer, clock}, nil
}

type bucket struct {
	name     string
	httpConn http.Conn
	signer   auth.Signer
	clock    time.Clock
}

func (b *bucket) GetObject(key string) (data []byte, err error) {
	// Validate the key.
	if err := validateKey(key); err != nil {
		return nil, err
	}

	// Build an appropriate HTTP request.
	//
	// Reference:
	//     http://docs.amazonwebservices.com/AmazonS3/latest/API/RESTObjectGET.html
	httpReq := &http.Request{
		Verb: "GET",
		Path: fmt.Sprintf("/%s/%s", b.name, key),
		Headers: map[string]string{
			"Date": b.clock.Now().UTC().Format(sys_time.RFC1123),
		},
	}

	// Sign the request.
	if err := b.signer.Sign(httpReq); err != nil {
		return nil, fmt.Errorf("Sign: %v", err)
	}

	// Send the request.
	httpResp, err := b.httpConn.SendRequest(httpReq)
	if err != nil {
		return nil, fmt.Errorf("SendRequest: %v", err)
	}

	// Check the response.
	if httpResp.StatusCode != 200 {
		return nil, fmt.Errorf("Error from server: %d %s", httpResp.StatusCode, httpResp.Body)
	}

	return httpResp.Body, nil
}

func (b *bucket) StoreObject(key string, data []byte) error {
	// Validate the key.
	if err := validateKey(key); err != nil {
		return err
	}

	// Build an appropriate HTTP request.
	//
	// Reference:
	//     http://docs.amazonwebservices.com/AmazonS3/latest/API/RESTObjectPUT.html
	httpReq := &http.Request{
		Verb: "PUT",
		Path: fmt.Sprintf("/%s/%s", b.name, key),
		Body: data,
		Headers: map[string]string{
			"Date": b.clock.Now().UTC().Format(sys_time.RFC1123),
		},
	}

	// Add a Content-MD5 header, as advised in the Amazon docs.
	if err := addMd5Header(httpReq, data); err != nil {
		return err
	}

	// Sign the request.
	if err := b.signer.Sign(httpReq); err != nil {
		return fmt.Errorf("Sign: %v", err)
	}

	// Send the request.
	httpResp, err := b.httpConn.SendRequest(httpReq)
	if err != nil {
		return fmt.Errorf("SendRequest: %v", err)
	}

	// Check the response.
	if httpResp.StatusCode != 200 {
		return fmt.Errorf("Error from server: %d %s", httpResp.StatusCode, httpResp.Body)
	}

	return nil
}

func (b *bucket) ListKeys(minKey string) (keys []string, err error) {
	return nil, fmt.Errorf("TODO(jacobsa): Implement ListKeys.")
}

func validateKey(key string) error {
	// Keys must be valid UTF-8 no more than 1024 bytes long.
	if len(key) > 1024 {
		return fmt.Errorf("Keys may be no longer than 1024 bytes.")
	}

	if !utf8.ValidString(key) {
		return fmt.Errorf("Keys must be valid UTF-8.")
	}

	// The Amazon docs only put the above restrictions on keys. However as of
	// 2012-09, sending a request for a bucket with a null character in its name
	// fails with a silent HTTP 400, despite the fact that it is a valid Unicode
	// character.
	if strings.ContainsRune(key, 0x00) {
		return fmt.Errorf("Keys may not contain null characters.")
	}

	// An empty sequence is also a sequence, but as of 2012-09 it fails in the
	// same way.
	if key == "" {
		return fmt.Errorf("Keys must be non-empty.")
	}

	return nil
}

func addMd5Header(r *http.Request, body []byte) error {
	md5Hash := md5.New()
	if _, err := md5Hash.Write(body); err != nil {
		return fmt.Errorf("md5Hash.Write: %v", err)
	}

	base64Md5Buf := new(bytes.Buffer)
	base64Encoder := base64.NewEncoder(base64.StdEncoding, base64Md5Buf)
	if _, err := base64Encoder.Write(md5Hash.Sum(nil)); err != nil {
		return fmt.Errorf("base64Encoder.Write: %v", err)
	}

	base64Encoder.Close()
	r.Headers["Content-MD5"] = base64Md5Buf.String()

	return nil
}
<commit_msg>Renamed parameter.<commit_after>// Copyright 2012 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package s3

import (
	"bytes"
	"crypto/md5"
	"encoding/base64"
	"fmt"
	"github.com/jacobsa/aws"
	"github.com/jacobsa/aws/s3/auth"
	"github.com/jacobsa/aws/s3/http"
	"github.com/jacobsa/aws/s3/time"
	"net/url"
	"strings"
	sys_time "time"
	"unicode/utf8"
)

// Bucket represents an S3 bucket, which is a collection of objects keyed on
// Unicode strings. The UTF-8 encoding of a key must be no more than 1024 bytes
// long.
//
// See here for more info:
//
//     http://goo.gl/Nd63t
//
type Bucket interface {
	// Retrieve data for the object with the given key.
	GetObject(key string) (data []byte, err error)

	// Store the supplied data with the given key, overwriting any previous
	// version. The object is created with the default ACL of "private".
	StoreObject(key string, data []byte) error

	// Return an ordered set of contiguous object keys in the bucket that are
	// greater than or equal to min. It is guaranteed that as some time during
	// the request there were no keys between min and the first key returned.
	//
	// There may be more keys beyond the last key returned. If no keys are
	// returned (and the error is nil), it is guaranteed that at some time during
	// the request there were the bucket contained no keys in [min, inf).
	ListKeys(min string) (keys []string, err error)
}

// OpenBucket returns a Bucket tied to a given name in whe given region. You
// must have previously created the bucket in the region, and the supplied
// access key must have access to it.
//
// To easily create a bucket, use the AWS Console:
//
//     http://aws.amazon.com/console/
//
func OpenBucket(name string, region Region, key aws.AccessKey) (Bucket, error) {
	// Create a connection to the given region's endpoint.
	endpoint := &url.URL{Scheme: "https", Host: string(region)}
	httpConn, err := http.NewConn(endpoint)
	if err != nil {
		return nil, fmt.Errorf("http.NewConn: %v", err)
	}

	// Create an appropriate request signer.
	signer, err := auth.NewSigner(&key)
	if err != nil {
		return nil, fmt.Errorf("auth.NewSigner: %v", err)
	}

	return openBucket(name, httpConn, signer, time.RealClock())
}

// A version of OpenBucket with the ability to inject dependencies, for
// testability.
func openBucket(
	name string,
	httpConn http.Conn,
	signer auth.Signer,
	clock time.Clock) (Bucket, error) {
	return &bucket{name, httpConn, signer, clock}, nil
}

type bucket struct {
	name     string
	httpConn http.Conn
	signer   auth.Signer
	clock    time.Clock
}

func (b *bucket) GetObject(key string) (data []byte, err error) {
	// Validate the key.
	if err := validateKey(key); err != nil {
		return nil, err
	}

	// Build an appropriate HTTP request.
	//
	// Reference:
	//     http://docs.amazonwebservices.com/AmazonS3/latest/API/RESTObjectGET.html
	httpReq := &http.Request{
		Verb: "GET",
		Path: fmt.Sprintf("/%s/%s", b.name, key),
		Headers: map[string]string{
			"Date": b.clock.Now().UTC().Format(sys_time.RFC1123),
		},
	}

	// Sign the request.
	if err := b.signer.Sign(httpReq); err != nil {
		return nil, fmt.Errorf("Sign: %v", err)
	}

	// Send the request.
	httpResp, err := b.httpConn.SendRequest(httpReq)
	if err != nil {
		return nil, fmt.Errorf("SendRequest: %v", err)
	}

	// Check the response.
	if httpResp.StatusCode != 200 {
		return nil, fmt.Errorf("Error from server: %d %s", httpResp.StatusCode, httpResp.Body)
	}

	return httpResp.Body, nil
}

func (b *bucket) StoreObject(key string, data []byte) error {
	// Validate the key.
	if err := validateKey(key); err != nil {
		return err
	}

	// Build an appropriate HTTP request.
	//
	// Reference:
	//     http://docs.amazonwebservices.com/AmazonS3/latest/API/RESTObjectPUT.html
	httpReq := &http.Request{
		Verb: "PUT",
		Path: fmt.Sprintf("/%s/%s", b.name, key),
		Body: data,
		Headers: map[string]string{
			"Date": b.clock.Now().UTC().Format(sys_time.RFC1123),
		},
	}

	// Add a Content-MD5 header, as advised in the Amazon docs.
	if err := addMd5Header(httpReq, data); err != nil {
		return err
	}

	// Sign the request.
	if err := b.signer.Sign(httpReq); err != nil {
		return fmt.Errorf("Sign: %v", err)
	}

	// Send the request.
	httpResp, err := b.httpConn.SendRequest(httpReq)
	if err != nil {
		return fmt.Errorf("SendRequest: %v", err)
	}

	// Check the response.
	if httpResp.StatusCode != 200 {
		return fmt.Errorf("Error from server: %d %s", httpResp.StatusCode, httpResp.Body)
	}

	return nil
}

func (b *bucket) ListKeys(min string) (keys []string, err error) {
	return nil, fmt.Errorf("TODO(jacobsa): Implement ListKeys.")
}

func validateKey(key string) error {
	// Keys must be valid UTF-8 no more than 1024 bytes long.
	if len(key) > 1024 {
		return fmt.Errorf("Keys may be no longer than 1024 bytes.")
	}

	if !utf8.ValidString(key) {
		return fmt.Errorf("Keys must be valid UTF-8.")
	}

	// The Amazon docs only put the above restrictions on keys. However as of
	// 2012-09, sending a request for a bucket with a null character in its name
	// fails with a silent HTTP 400, despite the fact that it is a valid Unicode
	// character.
	if strings.ContainsRune(key, 0x00) {
		return fmt.Errorf("Keys may not contain null characters.")
	}

	// An empty sequence is also a sequence, but as of 2012-09 it fails in the
	// same way.
	if key == "" {
		return fmt.Errorf("Keys must be non-empty.")
	}

	return nil
}

func addMd5Header(r *http.Request, body []byte) error {
	md5Hash := md5.New()
	if _, err := md5Hash.Write(body); err != nil {
		return fmt.Errorf("md5Hash.Write: %v", err)
	}

	base64Md5Buf := new(bytes.Buffer)
	base64Encoder := base64.NewEncoder(base64.StdEncoding, base64Md5Buf)
	if _, err := base64Encoder.Write(md5Hash.Sum(nil)); err != nil {
		return fmt.Errorf("base64Encoder.Write: %v", err)
	}

	base64Encoder.Close()
	r.Headers["Content-MD5"] = base64Md5Buf.String()

	return nil
}
<|endoftext|>


<commit_before>package gqltesting

import (
	"bytes"
	"context"
	"encoding/json"
	"reflect"
	"strconv"
	"testing"

	graphql "github.com/graph-gophers/graphql-go"
	"github.com/graph-gophers/graphql-go/errors"
)

// Test is a GraphQL test case to be used with RunTest(s).
type Test struct {
	Context        context.Context
	Schema         *graphql.Schema
	Query          string
	OperationName  string
	Variables      map[string]interface{}
	ExpectedResult string
	ExpectedErrors []*errors.QueryError
}

// RunTests runs the given GraphQL test cases as subtests.
func RunTests(t *testing.T, tests []*Test) {
	if len(tests) == 1 {
		RunTest(t, tests[0])
		return
	}

	for i, test := range tests {
		t.Run(strconv.Itoa(i+1), func(t *testing.T) {
			RunTest(t, test)
		})
	}
}

// RunTest runs a single GraphQL test case.
func RunTest(t *testing.T, test *Test) {
	if test.Context == nil {
		test.Context = context.Background()
	}
	result := test.Schema.Exec(test.Context, test.Query, test.OperationName, test.Variables)
	// Verify JSON to avoid red herring errors.
	got, err := formatJSON(result.Data)
	if err != nil {
		t.Fatalf("got: invalid JSON: %s", err)
	}
	want, err := formatJSON([]byte(test.ExpectedResult))
	if err != nil {
		t.Fatalf("want: invalid JSON: %s", err)
	}

	checkErrors(t, test.ExpectedErrors, result.Errors)

	if !bytes.Equal(got, want) {
		t.Logf("got:  %s", got)
		t.Logf("want: %s", want)
		t.Fail()
	}
}

func formatJSON(data []byte) ([]byte, error) {
	var v interface{}
	if err := json.Unmarshal(data, &v); err != nil {
		return nil, err
	}
	formatted, err := json.Marshal(v)
	if err != nil {
		return nil, err
	}
	return formatted, nil
}

func checkErrors(t *testing.T, expected, actual []*errors.QueryError) {
	expectedCount, actualCount := len(expected), len(actual)

	if expectedCount != actualCount {
		t.Fatalf("unexpected number of errors: got %d, want %d", actualCount, expectedCount)
	}

	if expectedCount > 0 {
		for i, want := range expected {
			got := actual[i]

			if !reflect.DeepEqual(got, want) {
				t.Fatalf("unexpected error: got %+v, want %+v", got, want)
			}
		}

		// Return because we're done checking.
		return
	}

	for _, err := range actual {
		t.Errorf("unexpected error: '%s'", err)
	}
}
<commit_msg>simpler error comparisons<commit_after>package gqltesting

import (
	"bytes"
	"context"
	"encoding/json"
	"reflect"
	"strconv"
	"testing"

	graphql "github.com/graph-gophers/graphql-go"
	"github.com/graph-gophers/graphql-go/errors"
)

// Test is a GraphQL test case to be used with RunTest(s).
type Test struct {
	Context        context.Context
	Schema         *graphql.Schema
	Query          string
	OperationName  string
	Variables      map[string]interface{}
	ExpectedResult string
	ExpectedErrors []*errors.QueryError
}

// RunTests runs the given GraphQL test cases as subtests.
func RunTests(t *testing.T, tests []*Test) {
	if len(tests) == 1 {
		RunTest(t, tests[0])
		return
	}

	for i, test := range tests {
		t.Run(strconv.Itoa(i+1), func(t *testing.T) {
			RunTest(t, test)
		})
	}
}

// RunTest runs a single GraphQL test case.
func RunTest(t *testing.T, test *Test) {
	if test.Context == nil {
		test.Context = context.Background()
	}
	result := test.Schema.Exec(test.Context, test.Query, test.OperationName, test.Variables)
	// Verify JSON to avoid red herring errors.
	got, err := formatJSON(result.Data)
	if err != nil {
		t.Fatalf("got: invalid JSON: %s", err)
	}
	want, err := formatJSON([]byte(test.ExpectedResult))
	if err != nil {
		t.Fatalf("want: invalid JSON: %s", err)
	}

	checkErrors(t, test.ExpectedErrors, result.Errors)

	if !bytes.Equal(got, want) {
		t.Logf("got:  %s", got)
		t.Logf("want: %s", want)
		t.Fail()
	}
}

func formatJSON(data []byte) ([]byte, error) {
	var v interface{}
	if err := json.Unmarshal(data, &v); err != nil {
		return nil, err
	}
	formatted, err := json.Marshal(v)
	if err != nil {
		return nil, err
	}
	return formatted, nil
}

func checkErrors(t *testing.T, want, got []*errors.QueryError) {
	if !reflect.DeepEqual(got, want) {
		t.Fatalf("unexpected error: got %+v, want %+v", got, want)
	}
}
<|endoftext|>


<commit_before>package geom

// Vector represents a point or a direction vector in a 3D cartesian coordinate system.
type Vector struct {
	// X, Y and Z are the coordinates of a point in each of the three
	// dimensions or alternatively the direction defined by a vector
	// with origin (0,0,0) and end at X, Y and Z.
	X, Y, Z float64
}

// NewVector returns a Vector with coordinates `x`, `y` and `z`.
func NewVector(x, y, z float64) Vector {
	return Vector{
		X: x,
		Y: y,
		Z: z,
	}
}

// Cross returns a Vector, the cross product of v1 and v2.
func Cross(v1, v2 Vector) (v Vector) {
	v.X = v1.Y*v2.Z - v1.Z*v2.Y
	v.Y = v1.Z*v2.X - v1.X*v2.Z
	v.Z = v1.X*v2.Y - v1.Y*v2.X
	return
}

// Dot returns a float64, the dot product of v1 and v2.
func Dot(v1, v2 Vector) (d float64) {
	d = v1.X*v2.X + v1.Y*v2.Y + v1.Z*v2.Z
	return
}

// Sub returns a Vector, the result of subtracting v2 from v1.
func Sub(v1, v2 Vector) (v Vector) {
	v.X = v1.X - v2.X
	v.Y = v1.Y - v2.Y
	v.Z = v1.Z - v2.Z
	return
}
<commit_msg>Add Add and Mul vector methods<commit_after>package geom

// Vector represents a point or a direction vector in a 3D cartesian coordinate system.
type Vector struct {
	// X, Y and Z are the coordinates of a point in each of the three
	// dimensions or alternatively the direction defined by a vector
	// with origin (0,0,0) and end at X, Y and Z.
	X, Y, Z float64
}

// NewVector returns a Vector with coordinates `x`, `y` and `z`.
func NewVector(x, y, z float64) Vector {
	return Vector{
		X: x,
		Y: y,
		Z: z,
	}
}

// Cross returns a Vector, the cross product of v1 and v2.
func Cross(v1, v2 Vector) (v Vector) {
	v.X = v1.Y*v2.Z - v1.Z*v2.Y
	v.Y = v1.Z*v2.X - v1.X*v2.Z
	v.Z = v1.X*v2.Y - v1.Y*v2.X
	return
}

// Dot returns a float64, the dot product of v1 and v2.
func Dot(v1, v2 Vector) (d float64) {
	d = v1.X*v2.X + v1.Y*v2.Y + v1.Z*v2.Z
	return
}

// Sub returns a Vector, the result of subtracting v2 from v1.
func Sub(v1, v2 Vector) (v Vector) {
	v.X = v1.X - v2.X
	v.Y = v1.Y - v2.Y
	v.Z = v1.Z - v2.Z
	return
}

// Add returns a Vector, the result of subtracting v2 from v1.
func Add(v1, v2 Vector) (v Vector) {
	v.X = v1.X + v2.X
	v.Y = v1.Y + v2.Y
	v.Z = v1.Z + v2.Z
	return
}

// Mul returns a Vector, multiplied by the scalar value n
func Mul(v Vector, n float64) (result Vector) {
	result.X = v.X * n
	result.Y = v.Y * n
	result.Z = v.Z * n
	return
}
<|endoftext|>


<commit_before>// The MIT License (MIT)
//
// Copyright (c) 2014 Jamie Alquiza
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.
package main

import (
	"flag"
	"fmt"
	"math/rand"
	"os"
	"os/signal"
	"runtime"
	"sort"
	"strconv"
	"strings"
	"syscall"
	"time"

	kafka "github.com/jamiealquiza/sangrenel/vendor/github.com/Shopify/sarama"
)

var (
	// Configs.
	brokers       []string
	topic         string
	msgSize       int
	msgRate       int64
	clientWorkers int
	noop          bool

	// Character selection from which random messages are generated.
	chars = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890!@#$%^&*(){}][:<>.")

	// Counters / misc.
	sig_chan        = make(chan os.Signal)
	clientKill_chan = make(chan bool, 24)
	sentCntr        = make(chan int64, 1)
	latency         []float64
	latency_chan    = make(chan float64, 1)
	resetLat_chan   = make(chan bool, 1)
)

func init() {
	flag.StringVar(&topic, "topic", "sangrenel", "Topic to publish to")
	flag.IntVar(&msgSize, "size", 300, "Message size in bytes")
	flag.Int64Var(&msgRate, "rate", 100000000, "Apply a global message rate limit")
	flag.BoolVar(&noop, "noop", false, "Test message generation performance, do not transmit messages")
	flag.IntVar(&clientWorkers, "workers", 1, "Number of Kafka client workers")
	brokerString := flag.String("brokers", "localhost:9092", "Comma delimited list of Kafka brokers")
	flag.Parse()

	brokers = strings.Split(*brokerString, ",")

	sentCntr <- 0
	runtime.GOMAXPROCS(runtime.NumCPU())
}

// clientWorker generates random messages and writes to Kafka.
// Workers track and limit message rates using incrSent() and fetchSent().
// 5 instances of clientWorker are created under each Kafka client.
func clientWorker(c kafka.Client) {
	producer, err := kafka.NewSimpleProducer(&c, nil)
	if err != nil {
		fmt.Println(err.Error())
	}
	defer producer.Close()

	// Instantiate rand per producer to avoid mutex contention.
	source := rand.NewSource(time.Now().UnixNano())
	generator := rand.New(source)
	// Create msg object once, reuse.
	msg := make([]rune, msgSize)

	// Use a local accumulator then periodically update global counter.
	// Global counter can become a bottleneck with too many threads.
	tick := time.Tick(1 * time.Second)
	var n int64

	for {
		// Message rate limit works by having all clientWorker loops incrementing
		// a global counter and tracking the aggregate per-second progress.
		// If the configured rate is met, the worker will sleep
		// for the remainder of the 1 second window.
		rateEnd := time.Now().Add(time.Second)
		countStart := fetchSent()
		var start time.Time
		for fetchSent()-countStart < msgRate {
			data := randMsg(msg, *generator)
			// We start timing after the message is created.
			// This ensures latency metering from the time between message sent and receiving an ack.
			start = time.Now()
			err = producer.SendMessage(topic, nil, kafka.StringEncoder(data))
			if err != nil {
				fmt.Println(err)
			} else {
				// Increment global sent count and fire off time since start value into the latency channel.
				n++
				select {
				case <-tick:
					incrSent(n)
					n = 0
				default:
					break
				}
				latency_chan <- time.Since(start).Seconds() * 1000
			}
		}
		// If the global per-second rate limit was met,
		// the inner loop breaks and the outer loop sleeps for the second remainder.
		time.Sleep(rateEnd.Sub(time.Now()) + time.Since(start))
	}
}

// kafkaClient initializes a connection to a Kafka cluster.
// Each client manages 5 clientWorker() (producer instances).
// Fixed value since this seem to flatline throughput at ~5.
func kafkaClient(n int) {
	switch noop {
	// If not noop, actually fire up Kafka connections and send messages.
	case false:
		cId := "client_" + strconv.Itoa(n)
		client, err := kafka.NewClient(cId, brokers, kafka.NewClientConfig())
		if err != nil {
			panic(err)
		} else {
			fmt.Printf("%s connected\n", cId)
		}
		for i := 0; i < 5; i++ {
			go clientWorker(*client)
		}
	// If noop, we're not creating connections at all.
	// Just generate messages and burn CPU.
	default:
		for i := 0; i < 5; i++ {
			go genMsgs()
		}
	}
	<-clientKill_chan
}

// genMsgs is a dummy function that kafkaClient calls if noop is True.
// It is used in place of starting actual Kafka client connections to test message creation performance.
func genMsgs() {
	// Instantiate 'rand' per producer to avoid mutex contention.
	source := rand.NewSource(time.Now().UnixNano())
	generator := rand.New(source)
	// Define msg objet once, reuse.
	msg := make([]rune, msgSize)

	tick := time.Tick(1 * time.Second)
	var n int64

	for {
		// Generate a random message and increment the global counter.
		randMsg(msg, *generator)
		n++
		select {
		case <-tick:
			incrSent(n)
			n = 0
		default:
			break
		}
	}
}

// Returns a random message generated from the chars rune slice.
// Message length of m bytes as defined by msgSize.
func randMsg(m []rune, generator rand.Rand) string {
	for i := range m {
		m[i] = chars[generator.Intn(len(chars))]
	}
	return string(m)
}

// Thread-safe global counter functions.
func incrSent(n int64) {
	i := <-sentCntr
	sentCntr <- i + n
}
func fetchSent() int64 {
	i := <-sentCntr
	sentCntr <- i
	return i
}

// Thread-safe receiver for latency values captured by all producer goroutines.
// May want to do something smart about this to limit time to sort
// huge slices in high-throughput configurations where lots of latency values are received.
func latencyAggregator() {
	for {
		select {
		case i := <-latency_chan:
			latency = append(latency, i)
		case <-resetLat_chan:
			latency = latency[:0]
		}
	}
}

// Calculates aggregate raw message output in networking friendly units.
// Gives an idea of minimum network traffic being generated.
func calcOutput(n int64) string {
	m := (float64(n) / 5) * float64(msgSize)
	var o string
	switch {
	case m >= 131072:
		o = strconv.FormatFloat(m/131072, 'f', 0, 64) + "Mb/sec"
	case m < 131072:
		o = strconv.FormatFloat(m/1024, 'f', 0, 64) + "KB/sec"
	}
	return o
}

// Fetches & resets current latencies set held by 'latencyAggregator()'.
// Sorts then averages the 90th percentile worse latencies.
func calcLatency() float64 {
	var avg float64
	// With 'noop', we don't have latencies to operate on.
	switch noop {
	case true:
		break
	default:
		// Fetch values.
		lat := latency
		// Issue the current values to be cleared.
		resetLat_chan <- true
		// Sort and sum values.
		sort.Float64s(lat)
		var sum float64
		// Get percentile count and values, sum values.
		topn := int(float64(len(lat)) * 0.90)
		for i := topn; i < len(lat); i++ {
			sum += lat[i]
		}
		// Calc average.
		avg = sum / float64(len(lat)-topn)
	}
	return avg
}

func main() {
	// Listens for signals.
	signal.Notify(sig_chan, syscall.SIGINT, syscall.SIGTERM)
	// Fires up 'latencyAggregator()' to background.
	go latencyAggregator()

	// Print Sangrenel startup info.
	fmt.Printf("\n::: Sangrenel :::\nStarting %s workers\nMessage size %s bytes\n\n",
		strconv.Itoa(clientWorkers),
		strconv.Itoa(msgSize))

	// Start client workers.
	for i := 0; i < clientWorkers; i++ {
		go kafkaClient(i + 1)
	}

	// Start Sangrenel periodic info output.
	tick := time.Tick(5 * time.Second)
	// Count mile-markers for tracking message rates.
	var currCnt, lastCnt int64
	for {
		select {
		case <-tick:
			// Set last and current to last read sent count.
			lastCnt = currCnt
			// Get actual current sent count, then delta from last count.
			// Delta is divided by update interval (5s) for per-second rate over output updates.
			currCnt = fetchSent()
			deltaCnt := currCnt - lastCnt
			fmt.Printf("%s Generating %s @ %d messages/sec | topic: %s | %.2fms avg latency\n",
				// Always be RFC'ing.
				time.Now().Format(time.RFC3339),
				calcOutput(deltaCnt),
				deltaCnt/5,
				topic,
				// Well, this technically appends a small latency to the 5s interval.
				calcLatency())
		// Waits for signals. Currently just brutally kills Sangrenel.
		case <-sig_chan:
			fmt.Println("Killing Connections")
			for i := 0; i < clientWorkers; i++ {
				clientKill_chan <- true
			}
			close(clientKill_chan)
			time.Sleep(2 * time.Second)
			os.Exit(0)
		}
	}
}
<commit_msg>lowers global counter update interval to avoid loss of accuracy<commit_after>// The MIT License (MIT)
//
// Copyright (c) 2014 Jamie Alquiza
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.
package main

import (
	"flag"
	"fmt"
	"math/rand"
	"os"
	"os/signal"
	"runtime"
	"sort"
	"strconv"
	"strings"
	"syscall"
	"time"

	kafka "github.com/jamiealquiza/sangrenel/vendor/github.com/Shopify/sarama"
)

var (
	// Configs.
	brokers       []string
	topic         string
	msgSize       int
	msgRate       int64
	clientWorkers int
	noop          bool

	// Character selection from which random messages are generated.
	chars = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890!@#$%^&*(){}][:<>.")

	// Counters / misc.
	sig_chan        = make(chan os.Signal)
	clientKill_chan = make(chan bool, 24)
	sentCntr        = make(chan int64, 1)
	latency         []float64
	latency_chan    = make(chan float64, 1)
	resetLat_chan   = make(chan bool, 1)
)

func init() {
	flag.StringVar(&topic, "topic", "sangrenel", "Topic to publish to")
	flag.IntVar(&msgSize, "size", 300, "Message size in bytes")
	flag.Int64Var(&msgRate, "rate", 100000000, "Apply a global message rate limit")
	flag.BoolVar(&noop, "noop", false, "Test message generation performance, do not transmit messages")
	flag.IntVar(&clientWorkers, "workers", 1, "Number of Kafka client workers")
	brokerString := flag.String("brokers", "localhost:9092", "Comma delimited list of Kafka brokers")
	flag.Parse()

	brokers = strings.Split(*brokerString, ",")

	sentCntr <- 0
	runtime.GOMAXPROCS(runtime.NumCPU())
}

// clientWorker generates random messages and writes to Kafka.
// Workers track and limit message rates using incrSent() and fetchSent().
// 5 instances of clientWorker are created under each Kafka client.
func clientWorker(c kafka.Client) {
	producer, err := kafka.NewSimpleProducer(&c, nil)
	if err != nil {
		fmt.Println(err.Error())
	}
	defer producer.Close()

	// Instantiate rand per producer to avoid mutex contention.
	source := rand.NewSource(time.Now().UnixNano())
	generator := rand.New(source)
	// Create msg object once, reuse.
	msg := make([]rune, msgSize)

	// Use a local accumulator then periodically update global counter.
	// Global counter can become a bottleneck with too many threads.
	tick := time.Tick(50 * time.Millisecond)
	var n int64

	for {
		// Message rate limit works by having all clientWorker loops incrementing
		// a global counter and tracking the aggregate per-second progress.
		// If the configured rate is met, the worker will sleep
		// for the remainder of the 1 second window.
		rateEnd := time.Now().Add(time.Second)
		countStart := fetchSent()
		var start time.Time
		for fetchSent()-countStart < msgRate {
			data := randMsg(msg, *generator)
			// We start timing after the message is created.
			// This ensures latency metering from the time between message sent and receiving an ack.
			start = time.Now()
			err = producer.SendMessage(topic, nil, kafka.StringEncoder(data))
			if err != nil {
				fmt.Println(err)
			} else {
				// Increment global sent count and fire off time since start value into the latency channel.
				n++
				select {
				case <-tick:
					incrSent(n)
					n = 0
				default:
					break
				}
				latency_chan <- time.Since(start).Seconds() * 1000
			}
		}
		// If the global per-second rate limit was met,
		// the inner loop breaks and the outer loop sleeps for the second remainder.
		time.Sleep(rateEnd.Sub(time.Now()) + time.Since(start))
	}
}

// kafkaClient initializes a connection to a Kafka cluster.
// Each client manages 5 clientWorker() (producer instances).
// Fixed value since this seem to flatline throughput at ~5.
func kafkaClient(n int) {
	switch noop {
	// If not noop, actually fire up Kafka connections and send messages.
	case false:
		cId := "client_" + strconv.Itoa(n)
		client, err := kafka.NewClient(cId, brokers, kafka.NewClientConfig())
		if err != nil {
			panic(err)
		} else {
			fmt.Printf("%s connected\n", cId)
		}
		for i := 0; i < 5; i++ {
			go clientWorker(*client)
		}
	// If noop, we're not creating connections at all.
	// Just generate messages and burn CPU.
	default:
		for i := 0; i < 5; i++ {
			go genMsgs()
		}
	}
	<-clientKill_chan
}

// genMsgs is a dummy function that kafkaClient calls if noop is True.
// It is used in place of starting actual Kafka client connections to test message creation performance.
func genMsgs() {
	// Instantiate 'rand' per producer to avoid mutex contention.
	source := rand.NewSource(time.Now().UnixNano())
	generator := rand.New(source)
	// Define msg objet once, reuse.
	msg := make([]rune, msgSize)

	tick := time.Tick(50 * time.Millisecond)
	var n int64

	for {
		// Generate a random message and increment the global counter.
		randMsg(msg, *generator)
		n++
		select {
		case <-tick:
			incrSent(n)
			n = 0
		default:
			break
		}
	}
}

// Returns a random message generated from the chars rune slice.
// Message length of m bytes as defined by msgSize.
func randMsg(m []rune, generator rand.Rand) string {
	for i := range m {
		m[i] = chars[generator.Intn(len(chars))]
	}
	return string(m)
}

// Thread-safe global counter functions.
func incrSent(n int64) {
	i := <-sentCntr
	sentCntr <- i + n
}
func fetchSent() int64 {
	i := <-sentCntr
	sentCntr <- i
	return i
}

// Thread-safe receiver for latency values captured by all producer goroutines.
// May want to do something smart about this to limit time to sort
// huge slices in high-throughput configurations where lots of latency values are received.
func latencyAggregator() {
	for {
		select {
		case i := <-latency_chan:
			latency = append(latency, i)
		case <-resetLat_chan:
			latency = latency[:0]
		}
	}
}

// Calculates aggregate raw message output in networking friendly units.
// Gives an idea of minimum network traffic being generated.
func calcOutput(n int64) string {
	m := (float64(n) / 5) * float64(msgSize)
	var o string
	switch {
	case m >= 131072:
		o = strconv.FormatFloat(m/131072, 'f', 0, 64) + "Mb/sec"
	case m < 131072:
		o = strconv.FormatFloat(m/1024, 'f', 0, 64) + "KB/sec"
	}
	return o
}

// Fetches & resets current latencies set held by 'latencyAggregator()'.
// Sorts then averages the 90th percentile worse latencies.
func calcLatency() float64 {
	var avg float64
	// With 'noop', we don't have latencies to operate on.
	switch noop {
	case true:
		break
	default:
		// Fetch values.
		lat := latency
		// Issue the current values to be cleared.
		resetLat_chan <- true
		// Sort and sum values.
		sort.Float64s(lat)
		var sum float64
		// Get percentile count and values, sum values.
		topn := int(float64(len(lat)) * 0.90)
		for i := topn; i < len(lat); i++ {
			sum += lat[i]
		}
		// Calc average.
		avg = sum / float64(len(lat)-topn)
	}
	return avg
}

func main() {
	// Listens for signals.
	signal.Notify(sig_chan, syscall.SIGINT, syscall.SIGTERM)
	// Fires up 'latencyAggregator()' to background.
	go latencyAggregator()

	// Print Sangrenel startup info.
	fmt.Printf("\n::: Sangrenel :::\nStarting %s workers\nMessage size %s bytes\n\n",
		strconv.Itoa(clientWorkers),
		strconv.Itoa(msgSize))

	// Start client workers.
	for i := 0; i < clientWorkers; i++ {
		go kafkaClient(i + 1)
	}

	// Start Sangrenel periodic info output.
	tick := time.Tick(5 * time.Second)
	// Count mile-markers for tracking message rates.
	var currCnt, lastCnt int64
	for {
		select {
		case <-tick:
			// Set last and current to last read sent count.
			lastCnt = currCnt
			// Get actual current sent count, then delta from last count.
			// Delta is divided by update interval (5s) for per-second rate over output updates.
			currCnt = fetchSent()
			deltaCnt := currCnt - lastCnt
			fmt.Printf("%s Generating %s @ %d messages/sec | topic: %s | %.2fms avg latency\n",
				// Always be RFC'ing.
				time.Now().Format(time.RFC3339),
				calcOutput(deltaCnt),
				deltaCnt/5,
				topic,
				// Well, this technically appends a small latency to the 5s interval.
				calcLatency())
		// Waits for signals. Currently just brutally kills Sangrenel.
		case <-sig_chan:
			fmt.Println("Killing Connections")
			for i := 0; i < clientWorkers; i++ {
				clientKill_chan <- true
			}
			close(clientKill_chan)
			time.Sleep(2 * time.Second)
			os.Exit(0)
		}
	}
}
<|endoftext|>


<commit_before>package dns

import (
	"runtime"
	"strings"

	"github.com/datawire/teleproxy/pkg/tpu"
)

type searchDomains struct {
	Interface string
	Domains   string
}

func OverrideSearchDomains(domains string) func() {
	if runtime.GOOS != "darwin" {
		return func() {}
	}

	ifaces, err := getIfaces()
	if err != nil {
		panic(err)
	}
	previous := []searchDomains{}

	for _, iface := range ifaces {
		// setup dns search path
		domain, _ := getSearchDomains(iface)
		setSearchDomains(iface, domains)
		previous = append(previous, searchDomains{iface, domain})
	}

	// return function to restore dns search paths
	return func() {
		for _, prev := range previous {
			setSearchDomains(prev.Interface, prev.Domains)
		}
	}
}

func getIfaces() (ifaces []string, err error) {
	lines, err := tpu.CmdLogf([]string{"networksetup", "-listallnetworkservices"}, log)
	if err != nil {
		return
	}
	for _, line := range strings.Split(lines, "\n") {
		if strings.Contains(line, "*") {
			continue
		}
		line = strings.TrimSpace(line)
		if len(line) > 0 {
			ifaces = append(ifaces, line)
		}
	}
	return
}

func getSearchDomains(iface string) (domains string, err error) {
	domains, err = tpu.CmdLogf([]string{"networksetup", "-getsearchdomains", iface}, log)
	domains = strings.TrimSpace(domains)
	return
}

func setSearchDomains(iface, domains string) (err error) {
	_, err = tpu.CmdLogf([]string{"networksetup", "-setsearchdomains", iface, domains}, log)
	return
}
<commit_msg>pay attention to more exit codes<commit_after>package dns

import (
	"runtime"
	"strings"

	"github.com/datawire/teleproxy/pkg/tpu"
)

type searchDomains struct {
	Interface string
	Domains   string
}

func OverrideSearchDomains(domains string) func() {
	if runtime.GOOS != "darwin" {
		return func() {}
	}

	ifaces, err := getIfaces()
	if err != nil {
		panic(err)
	}
	previous := []searchDomains{}

	for _, iface := range ifaces {
		// setup dns search path
		domain, err := getSearchDomains(iface)
		if err != nil {
			log("DNS: error getting search domain for interface %v: %v", iface, err)
		} else {
			setSearchDomains(iface, domains)
			previous = append(previous, searchDomains{iface, domain})
		}
	}

	// return function to restore dns search paths
	return func() {
		for _, prev := range previous {
			setSearchDomains(prev.Interface, prev.Domains)
		}
	}
}

func getIfaces() (ifaces []string, err error) {
	lines, err := tpu.CmdLogf([]string{"networksetup", "-listallnetworkservices"}, log)
	if err != nil {
		return
	}
	for _, line := range strings.Split(lines, "\n") {
		if strings.Contains(line, "*") {
			continue
		}
		line = strings.TrimSpace(line)
		if len(line) > 0 {
			ifaces = append(ifaces, line)
		}
	}
	return
}

func getSearchDomains(iface string) (domains string, err error) {
	domains, err = tpu.CmdLogf([]string{"networksetup", "-getsearchdomains", iface}, log)
	domains = strings.TrimSpace(domains)
	return
}

func setSearchDomains(iface, domains string) (err error) {
	_, err = tpu.CmdLogf([]string{"networksetup", "-setsearchdomains", iface, domains}, log)
	return
}
<|endoftext|>


<commit_before>package version

import (
	"fmt"
	"runtime"
)

const Binary = "0.3.7-HA.1.5.6"

func String(app string) string {
	return fmt.Sprintf("%s v%s (built w/%s)", app, Binary, runtime.Version())
}
<commit_msg>bump version<commit_after>package version

import (
	"fmt"
	"runtime"
)

const Binary = "0.3.7-HA.1.5.7"

func String(app string) string {
	return fmt.Sprintf("%s v%s (built w/%s)", app, Binary, runtime.Version())
}
<|endoftext|>