<commit_before>// Copyright (c) 2015 Janoš Guljaš <janos@resenje.org>
// All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

/*
Package boltdbpool implements a pool container for BoltDB github.com/coreos/bbolt databases.
Pool elements called connections keep reference counts for each database to close it
when it when the count is 0. Database is reused or opened based on database file path. Closing
the database must not be done directly, instead Connection.Close() method should be used.
Database is removed form the pool and closed by the goroutine in the background in respect to
reference count and delay in time if it is specified.

Example:

    package main

    import (
        "fmt"
        "time"

        "resenje.org/boltdbpool"
    )

    func main() {
        pool := boltdbpool.New(&boltdbpool.Options{
            ConnectionExpires: 5 * time.Second,
            ErrorHandler: boltdbpool.ErrorHandlerFunc(func(err error) {
                fmt.Printf("error: %v", err)
            }),
        })
        defer p.Close()

        ...

        c, err := pool.Get("/tmp/db.bolt")
        if err != nil {
            panic(err)
        }
        defer c.Close()

        ...

        c.DB.Update(func(tx *bolt.TX) error {
            ...
        })
    }
*/
package boltdbpool // import "resenje.org/boltdbpool"

import (
	"fmt"
	"log"
	"os"
	"path/filepath"
	"sync"
	"time"

	bolt "go.etcd.io/bbolt"
)

var (
	// DefaultFileMode is used in bolt.Open() as file mode for database file
	// if FileMode is not specified in boltdbpool.Options.
	DefaultFileMode = os.FileMode(0666)

	// DefaultDirMode is used in os.MkdirAll() as file mode for database directories
	// if DirMode is not specified in boltdbpool.Options.
	DefaultDirMode = os.FileMode(0777)

	// DefaultErrorHandler accepts errors from
	// goroutine that closes the databases if ErrorHandler is not specified in
	// boltdbpool.Options.
	DefaultErrorHandler = ErrorHandlerFunc(func(err error) {
		log.Printf("error: %v", err)
	})
)

// Options are used when a new pool is created that.
type Options struct {
	// BoltOptions is used on bolt.Open().
	BoltOptions *bolt.Options

	// FileMode is used in bolt.Open() as file mode for database file. Default: 0640.
	FileMode os.FileMode

	// DirMode is used in os.MkdirAll() as file mode for database directories. Default: 0750.
	DirMode os.FileMode

	// ConnectionExpires is a duration between the reference count drops to 0 and
	// the time when the database is closed. It is useful to avoid frequent
	// openings of the same database. If the value is 0 (default), no caching is done.
	ConnectionExpires time.Duration

	// ErrorHandler represents interface that accepts errors from goroutine that closes the databases.
	ErrorHandler ErrorHandler
}

// Pool keeps track of connections.
type Pool struct {
	options       *Options
	connections   map[string]*Connection
	mu            sync.RWMutex
	removeTrigger chan struct{}
	quit          chan struct{}
}

// New creates new pool with provided options and also starts database closing goroutone
// and goroutine for errors handling to ErrorHandler.
func New(options *Options) *Pool {
	if options == nil {
		options = &Options{}
	}
	if options.FileMode == 0 {
		options.FileMode = DefaultFileMode
	}
	if options.DirMode == 0 {
		options.DirMode = DefaultDirMode
	}
	if options.ErrorHandler == nil {
		options.ErrorHandler = DefaultErrorHandler
	}
	p := &Pool{
		options:       options,
		connections:   map[string]*Connection{},
		removeTrigger: make(chan struct{}, 1),
		quit:          make(chan struct{}),
	}
	go func() {
		for {
			select {
			case <-p.removeTrigger:
				select {
				case <-time.After(p.options.ConnectionExpires):
				case <-p.quit:
					return
				}
				p.mu.Lock()
				for _, c := range p.connections {
					c.mu.RLock()
					if !c.closeTime.IsZero() && c.closeTime.Before(time.Now()) {
						p.handleError(c.removeFromPool())
					}
					c.mu.RUnlock()
				}
				p.mu.Unlock()
			case <-p.quit:
				return
			}
		}
	}()
	return p
}

// Get returns a connection that contains a database or creates a new connection
// with newly opened database based on options specified on pool creation.
func (p *Pool) Get(path string) (*Connection, error) {
	p.mu.Lock()
	defer p.mu.Unlock()

	if c, ok := p.connections[path]; ok {
		c.mu.Lock()
		c.increment()
		c.mu.Unlock()
		return c, nil
	}
	if _, err := os.Stat(filepath.Dir(path)); os.IsNotExist(err) {
		if err := os.MkdirAll(filepath.Dir(path), p.options.DirMode); err != nil {
			return nil, err
		}
	} else if err != nil {
		return nil, err
	}
	db, err := bolt.Open(path, p.options.FileMode, p.options.BoltOptions)
	if err != nil {
		return nil, err
	}
	c := &Connection{
		DB:   db,
		path: path,
		pool: p,
	}
	c.mu.Lock()
	c.increment()
	p.connections[path] = c
	c.mu.Unlock()
	return c, nil
}

// Has returns true if a database with a file path is in the pool.
func (p *Pool) Has(path string) bool {
	p.mu.RLock()
	defer p.mu.RUnlock()

	_, ok := p.connections[path]
	return ok
}

// Close function closes and removes from the pool all databases. After the execution
// pool is not usable.
func (p *Pool) Close() {
	p.mu.Lock()
	defer p.mu.Unlock()

	for _, c := range p.connections {
		p.handleError(c.removeFromPool())
	}
	close(p.quit)
}

func (p *Pool) remove(path string) error {
	c, ok := p.connections[path]
	if !ok {
		return fmt.Errorf("boltdbpool: Unknown DB %s", path)
	}
	delete(p.connections, path)
	return c.DB.Close()
}

func (p *Pool) handleError(err error) {
	if err != nil {
		p.options.ErrorHandler.HandleError(err)
	}
}

// Connection encapsulates bolt.DB and keeps reference counter and closing time information.
type Connection struct {
	DB *bolt.DB

	pool      *Pool
	path      string
	count     int64
	closeTime time.Time
	mu        sync.RWMutex
}

// Close function on Connection decrements reference counter and closes the database if needed.
func (c *Connection) Close() {
	c.mu.Lock()
	defer c.mu.Unlock()

	c.decrement()

	if c.count > 0 {
		return
	}

	if c.pool.options.ConnectionExpires == 0 {
		c.pool.mu.Lock()
		c.pool.handleError(c.removeFromPool())
		c.pool.mu.Unlock()
		return
	}

	c.closeTime = time.Now().Add(c.pool.options.ConnectionExpires)
	select {
	case c.pool.removeTrigger <- struct{}{}:
	default:
	}
}

func (c *Connection) increment() {
	// Reset the closing time
	c.closeTime = time.Time{}
	c.count++
}

func (c *Connection) decrement() {
	c.count--
}

func (c *Connection) removeFromPool() error {
	return c.pool.remove(c.path)
}

// ErrorHandler interface can be used for objects that log or panic on error
type ErrorHandler interface {
	HandleError(err error)
}

// The ErrorHandlerFunc type is an adapter to allow the use of
// ordinary functions as error handlers.
type ErrorHandlerFunc func(err error)

// HandleError calls f(err).
func (f ErrorHandlerFunc) HandleError(err error) {
	f(err)
}
<commit_msg>Better error message<commit_after>// Copyright (c) 2015 Janoš Guljaš <janos@resenje.org>
// All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

/*
Package boltdbpool implements a pool container for BoltDB github.com/coreos/bbolt databases.
Pool elements called connections keep reference counts for each database to close it
when it when the count is 0. Database is reused or opened based on database file path. Closing
the database must not be done directly, instead Connection.Close() method should be used.
Database is removed form the pool and closed by the goroutine in the background in respect to
reference count and delay in time if it is specified.

Example:

    package main

    import (
        "fmt"
        "time"

        "resenje.org/boltdbpool"
    )

    func main() {
        pool := boltdbpool.New(&boltdbpool.Options{
            ConnectionExpires: 5 * time.Second,
            ErrorHandler: boltdbpool.ErrorHandlerFunc(func(err error) {
                fmt.Printf("error: %v", err)
            }),
        })
        defer p.Close()

        ...

        c, err := pool.Get("/tmp/db.bolt")
        if err != nil {
            panic(err)
        }
        defer c.Close()

        ...

        c.DB.Update(func(tx *bolt.TX) error {
            ...
        })
    }
*/
package boltdbpool // import "resenje.org/boltdbpool"

import (
	"fmt"
	"log"
	"os"
	"path/filepath"
	"sync"
	"time"

	bolt "go.etcd.io/bbolt"
)

var (
	// DefaultFileMode is used in bolt.Open() as file mode for database file
	// if FileMode is not specified in boltdbpool.Options.
	DefaultFileMode = os.FileMode(0666)

	// DefaultDirMode is used in os.MkdirAll() as file mode for database directories
	// if DirMode is not specified in boltdbpool.Options.
	DefaultDirMode = os.FileMode(0777)

	// DefaultErrorHandler accepts errors from
	// goroutine that closes the databases if ErrorHandler is not specified in
	// boltdbpool.Options.
	DefaultErrorHandler = ErrorHandlerFunc(func(err error) {
		log.Printf("error: %v", err)
	})
)

// Options are used when a new pool is created that.
type Options struct {
	// BoltOptions is used on bolt.Open().
	BoltOptions *bolt.Options

	// FileMode is used in bolt.Open() as file mode for database file. Default: 0640.
	FileMode os.FileMode

	// DirMode is used in os.MkdirAll() as file mode for database directories. Default: 0750.
	DirMode os.FileMode

	// ConnectionExpires is a duration between the reference count drops to 0 and
	// the time when the database is closed. It is useful to avoid frequent
	// openings of the same database. If the value is 0 (default), no caching is done.
	ConnectionExpires time.Duration

	// ErrorHandler represents interface that accepts errors from goroutine that closes the databases.
	ErrorHandler ErrorHandler
}

// Pool keeps track of connections.
type Pool struct {
	options       *Options
	connections   map[string]*Connection
	mu            sync.RWMutex
	removeTrigger chan struct{}
	quit          chan struct{}
}

// New creates new pool with provided options and also starts database closing goroutone
// and goroutine for errors handling to ErrorHandler.
func New(options *Options) *Pool {
	if options == nil {
		options = &Options{}
	}
	if options.FileMode == 0 {
		options.FileMode = DefaultFileMode
	}
	if options.DirMode == 0 {
		options.DirMode = DefaultDirMode
	}
	if options.ErrorHandler == nil {
		options.ErrorHandler = DefaultErrorHandler
	}
	p := &Pool{
		options:       options,
		connections:   map[string]*Connection{},
		removeTrigger: make(chan struct{}, 1),
		quit:          make(chan struct{}),
	}
	go func() {
		for {
			select {
			case <-p.removeTrigger:
				select {
				case <-time.After(p.options.ConnectionExpires):
				case <-p.quit:
					return
				}
				p.mu.Lock()
				for _, c := range p.connections {
					c.mu.RLock()
					if !c.closeTime.IsZero() && c.closeTime.Before(time.Now()) {
						p.handleError(c.removeFromPool())
					}
					c.mu.RUnlock()
				}
				p.mu.Unlock()
			case <-p.quit:
				return
			}
		}
	}()
	return p
}

// Get returns a connection that contains a database or creates a new connection
// with newly opened database based on options specified on pool creation.
func (p *Pool) Get(path string) (*Connection, error) {
	p.mu.Lock()
	defer p.mu.Unlock()

	if c, ok := p.connections[path]; ok {
		c.mu.Lock()
		c.increment()
		c.mu.Unlock()
		return c, nil
	}
	if _, err := os.Stat(filepath.Dir(path)); os.IsNotExist(err) {
		if err := os.MkdirAll(filepath.Dir(path), p.options.DirMode); err != nil {
			return nil, err
		}
	} else if err != nil {
		return nil, err
	}
	db, err := bolt.Open(path, p.options.FileMode, p.options.BoltOptions)
	if err != nil {
		return nil, err
	}
	c := &Connection{
		DB:   db,
		path: path,
		pool: p,
	}
	c.mu.Lock()
	c.increment()
	p.connections[path] = c
	c.mu.Unlock()
	return c, nil
}

// Has returns true if a database with a file path is in the pool.
func (p *Pool) Has(path string) bool {
	p.mu.RLock()
	defer p.mu.RUnlock()

	_, ok := p.connections[path]
	return ok
}

// Close function closes and removes from the pool all databases. After the execution
// pool is not usable.
func (p *Pool) Close() {
	p.mu.Lock()
	defer p.mu.Unlock()

	for _, c := range p.connections {
		p.handleError(c.removeFromPool())
	}
	close(p.quit)
}

func (p *Pool) remove(path string) error {
	c, ok := p.connections[path]
	if !ok {
		return fmt.Errorf("boltdbpool: unknown db %s", path)
	}
	delete(p.connections, path)
	return c.DB.Close()
}

func (p *Pool) handleError(err error) {
	if err != nil {
		p.options.ErrorHandler.HandleError(err)
	}
}

// Connection encapsulates bolt.DB and keeps reference counter and closing time information.
type Connection struct {
	DB *bolt.DB

	pool      *Pool
	path      string
	count     int64
	closeTime time.Time
	mu        sync.RWMutex
}

// Close function on Connection decrements reference counter and closes the database if needed.
func (c *Connection) Close() {
	c.mu.Lock()
	defer c.mu.Unlock()

	c.decrement()

	if c.count > 0 {
		return
	}

	if c.pool.options.ConnectionExpires == 0 {
		c.pool.mu.Lock()
		c.pool.handleError(c.removeFromPool())
		c.pool.mu.Unlock()
		return
	}

	c.closeTime = time.Now().Add(c.pool.options.ConnectionExpires)
	select {
	case c.pool.removeTrigger <- struct{}{}:
	default:
	}
}

func (c *Connection) increment() {
	// Reset the closing time
	c.closeTime = time.Time{}
	c.count++
}

func (c *Connection) decrement() {
	c.count--
}

func (c *Connection) removeFromPool() error {
	return c.pool.remove(c.path)
}

// ErrorHandler interface can be used for objects that log or panic on error
type ErrorHandler interface {
	HandleError(err error)
}

// The ErrorHandlerFunc type is an adapter to allow the use of
// ordinary functions as error handlers.
type ErrorHandlerFunc func(err error)

// HandleError calls f(err).
func (f ErrorHandlerFunc) HandleError(err error) {
	f(err)
}
<|endoftext|>


<commit_before>package sudoku

import (
	"fmt"
	"log"
	"strconv"
)

type forcingChainsTechnique struct {
	*basicSolveTechnique
}

func (self *forcingChainsTechnique) HumanLikelihood() float64 {
	//TODO: figure out what the baseDifficulty should be
	return self.difficultyHelper(200.0)
}

func (self *forcingChainsTechnique) Description(step *SolveStep) string {
	//TODO: implement this
	return "ERROR: NOT IMPLEMENTED"
}

func (self *forcingChainsTechnique) Find(grid *Grid, results chan *SolveStep, done chan bool) {
	//TODO: test that this will find multiple if they exist.
	//TODO: Implement this.

	getter := grid.queue().DefaultGetter()

	_MAX_IMPLICATION_STEPS := 6

	for {

		//Check if it's time to stop.
		select {
		case <-done:
			return
		default:
		}

		candidate := getter.GetSmallerThan(3)

		if candidate == nil {
			break
		}

		candidateCell := candidate.(*Cell)

		if len(candidateCell.Possibilities()) != 2 {
			//We found one with 1 possibility, which isn't interesting for us--nakedSingle should do that one.
			continue
		}

		firstPossibilityNum := candidateCell.Possibilities()[0]
		secondPossibilityNum := candidateCell.Possibilities()[1]

		firstGrid := grid.Copy()
		secondGrid := grid.Copy()

		//Check that the neighbor isn't just already having a single possibility, because then this technique is overkill.

		firstAccumulator := makeChainSeacherAccumulator(_MAX_IMPLICATION_STEPS)
		secondAccumulator := makeChainSeacherAccumulator(_MAX_IMPLICATION_STEPS)

		chainSearcher(_MAX_IMPLICATION_STEPS,
			candidateCell.InGrid(firstGrid),
			firstPossibilityNum,
			firstAccumulator)

		chainSearcher(_MAX_IMPLICATION_STEPS,
			candidateCell.InGrid(secondGrid),
			secondPossibilityNum,
			secondAccumulator)

		//TODO:Check if the sets overlap.

		doPrint := candidateCell.Row() == 1 && candidateCell.Col() == 0

		//For these debugging purposes, only print out the candidateCell we know to be interesting in the test case.
		if doPrint {
			log.Println(firstAccumulator)
			log.Println(secondAccumulator)
		}

		//See if either branch, at some generation, has the same cell forced to the same number in either generation.

		//accumulate forward, so the last generation has ALL cells affected in any generation on that branch
		firstAccumulator.accumulateGenerations()
		secondAccumulator.accumulateGenerations()

		if doPrint {
			log.Println("Accumulators after accumulating generations:")
			log.Println(firstAccumulator)
			log.Println(secondAccumulator)
		}

		foundOne := false

		for generation := _MAX_IMPLICATION_STEPS - 1; generation >= 0 && !foundOne; generation-- {

			//Check for any overlap at the last generation
			firstAffectedCells := firstAccumulator[generation]
			secondAffectedCells := secondAccumulator[generation]

			for key, val := range firstAffectedCells {

				//Skip the candidateCell, because that's not a meaningful overlap--we set that one as a way of branching!
				if key == candidateCell.ref() {
					continue
				}

				if num, ok := secondAffectedCells[key]; ok {
					//Found cell overlap! ... is the forced number the same?
					if val == num {
						//Yup, seems like we've found a cell that is forced to the same value on either branch.
						step := &SolveStep{self,
							CellSlice{key.Cell(grid)},
							IntSlice{val},
							CellSlice{candidateCell},
							candidateCell.Possibilities(),
						}

						if doPrint {
							log.Println(step)
							log.Println("Candidate Cell", candidateCell.ref())
						}

						if step.IsUseful(grid) {
							foundOne = true
							if doPrint {
								log.Println("Found solution on generation: ", generation)
							}
							select {
							case results <- step:
							case <-done:
								return
							}
						}
					}
				}
			}
		}

		//TODO: figure out why the tests are coming back with different answers, even when only looking at the key cell
		//that should work from the example.
		//TODO: we should prefer solutions where the total implications on both branches are minimized.
		//For example, if only one implication is requried on left, but 4 are on right, that's preferable to one where
		//three implications are required on both sides.
		//TODO: figure out a way to only compute a generation if required on each branch (don't compute all the way to _MAX_IMPLICATIONS to start)

	}
}

type chainSearcherGenerationDetails map[cellRef]int

func (c chainSearcherGenerationDetails) String() string {
	result := "Begin map\n"
	for cell, num := range c {
		result += "\t" + cell.String() + " : " + strconv.Itoa(num) + "\n"
	}
	result += "End map\n"
	return result
}

type chainSearcherAccumulator []chainSearcherGenerationDetails

func (c chainSearcherAccumulator) String() string {
	result := "Accumulator[\n"
	for _, rec := range c {
		result += fmt.Sprintf("%s\n", rec)
	}
	result += "]\n"
	return result
}

//accumulateGenerations goes through each generation (youngest to newest)
//and squaches older generation maps into each generation, so each
//generation's map represents the totality of all cells seen at that point.
func (c chainSearcherAccumulator) accumulateGenerations() {
	for i := len(c) - 2; i >= 0; i-- {
		lastGeneration := c[i+1]
		currentGeneration := c[i]
		for key, val := range lastGeneration {
			if currentVal, ok := currentGeneration[key]; ok {
				if currentVal != val {
					//No, this should be expected to happen when we get to an invalid grid state,
					//which we should expect to happen down one of the two branches (at least as explore
					//far enough.)
					log.Println("We were about to overwrite a value from an earlier generation... this shouldn't happen.")
				}
			}
			currentGeneration[key] = val
		}
	}
}

func makeChainSeacherAccumulator(size int) chainSearcherAccumulator {
	result := make(chainSearcherAccumulator, size)
	for i := 0; i < size; i++ {
		result[i] = make(map[cellRef]int)
	}
	return result
}

func chainSearcher(i int, cell *Cell, numToApply int, accumulator chainSearcherAccumulator) {

	//TODO: we should change this implementation so that it's not DFS but BFS.
	//the first time we cross over into a new generation, we should do a one-time copy of the old generation
	//into the new.
	//At any write, if we notice that we'd be overwriting to a different value, we can bail out (how would
	//we mark that we bailed early), since we've run into an inconsistency down this branch and following
	//it further is not useful.

	if i <= 0 || cell == nil {
		//Base case
		return
	}

	if i-1 >= len(accumulator) {
		panic("The accumulator provided was not big enough for the i provided.")
	}

	generationDetails := accumulator[i-1]

	//Find the nextCells that WILL have their numbers forced by the cell we're thinking of fillint.
	cellsToVisit := cell.Neighbors().FilterByPossible(numToApply).FilterByNumPossibilities(2)

	//Now that we know which cells will be affected and what their next number will be,
	//set the number in the given cell and then recurse downward down each branch.
	cell.SetNumber(numToApply)

	generationDetails[cell.ref()] = numToApply

	for _, cellToVisit := range cellsToVisit {

		possibilities := cellToVisit.Possibilities()

		if len(possibilities) != 1 {
			panic("Expected the cell to have one possibility")
		}

		forcedNum := possibilities[0]

		//Each branch modifies the grid, so create a new copy
		newGrid := cellToVisit.grid.Copy()
		cellToVisit = cellToVisit.InGrid(newGrid)

		//Recurse downward
		chainSearcher(i-1, cellToVisit, forcedNum, accumulator)

	}

}
<commit_msg>TESTS FAIL. chainSearcher switched to an equivalent BFS, which, once we do a few TODOs in the code, should have better properties.<commit_after>package sudoku

import (
	"container/list"
	"fmt"
	"log"
	"strconv"
)

type forcingChainsTechnique struct {
	*basicSolveTechnique
}

func (self *forcingChainsTechnique) HumanLikelihood() float64 {
	//TODO: figure out what the baseDifficulty should be
	return self.difficultyHelper(200.0)
}

func (self *forcingChainsTechnique) Description(step *SolveStep) string {
	//TODO: implement this
	return "ERROR: NOT IMPLEMENTED"
}

func (self *forcingChainsTechnique) Find(grid *Grid, results chan *SolveStep, done chan bool) {
	//TODO: test that this will find multiple if they exist.
	//TODO: Implement this.

	getter := grid.queue().DefaultGetter()

	_MAX_IMPLICATION_STEPS := 6

	for {

		//Check if it's time to stop.
		select {
		case <-done:
			return
		default:
		}

		candidate := getter.GetSmallerThan(3)

		if candidate == nil {
			break
		}

		candidateCell := candidate.(*Cell)

		if len(candidateCell.Possibilities()) != 2 {
			//We found one with 1 possibility, which isn't interesting for us--nakedSingle should do that one.
			continue
		}

		firstPossibilityNum := candidateCell.Possibilities()[0]
		secondPossibilityNum := candidateCell.Possibilities()[1]

		firstGrid := grid.Copy()
		secondGrid := grid.Copy()

		//Check that the neighbor isn't just already having a single possibility, because then this technique is overkill.

		firstAccumulator := makeChainSeacherAccumulator(_MAX_IMPLICATION_STEPS)
		secondAccumulator := makeChainSeacherAccumulator(_MAX_IMPLICATION_STEPS)

		chainSearcher(_MAX_IMPLICATION_STEPS,
			candidateCell.InGrid(firstGrid),
			firstPossibilityNum,
			firstAccumulator)

		chainSearcher(_MAX_IMPLICATION_STEPS,
			candidateCell.InGrid(secondGrid),
			secondPossibilityNum,
			secondAccumulator)

		//TODO:Check if the sets overlap.

		doPrint := candidateCell.Row() == 1 && candidateCell.Col() == 0

		//For these debugging purposes, only print out the candidateCell we know to be interesting in the test case.
		if doPrint {
			log.Println(firstAccumulator)
			log.Println(secondAccumulator)
		}

		//See if either branch, at some generation, has the same cell forced to the same number in either generation.

		//accumulate forward, so the last generation has ALL cells affected in any generation on that branch
		firstAccumulator.accumulateGenerations()
		secondAccumulator.accumulateGenerations()

		if doPrint {
			log.Println("Accumulators after accumulating generations:")
			log.Println(firstAccumulator)
			log.Println(secondAccumulator)
		}

		foundOne := false

		for generation := _MAX_IMPLICATION_STEPS - 1; generation >= 0 && !foundOne; generation-- {

			//Check for any overlap at the last generation
			firstAffectedCells := firstAccumulator[generation]
			secondAffectedCells := secondAccumulator[generation]

			for key, val := range firstAffectedCells {

				//Skip the candidateCell, because that's not a meaningful overlap--we set that one as a way of branching!
				if key == candidateCell.ref() {
					continue
				}

				if num, ok := secondAffectedCells[key]; ok {
					//Found cell overlap! ... is the forced number the same?
					if val == num {
						//Yup, seems like we've found a cell that is forced to the same value on either branch.
						step := &SolveStep{self,
							CellSlice{key.Cell(grid)},
							IntSlice{val},
							CellSlice{candidateCell},
							candidateCell.Possibilities(),
						}

						if doPrint {
							log.Println(step)
							log.Println("Candidate Cell", candidateCell.ref())
						}

						if step.IsUseful(grid) {
							foundOne = true
							if doPrint {
								log.Println("Found solution on generation: ", generation)
							}
							select {
							case results <- step:
							case <-done:
								return
							}
						}
					}
				}
			}
		}

		//TODO: figure out why the tests are coming back with different answers, even when only looking at the key cell
		//that should work from the example.
		//TODO: we should prefer solutions where the total implications on both branches are minimized.
		//For example, if only one implication is requried on left, but 4 are on right, that's preferable to one where
		//three implications are required on both sides.
		//TODO: figure out a way to only compute a generation if required on each branch (don't compute all the way to _MAX_IMPLICATIONS to start)

	}
}

type chainSearcherGenerationDetails map[cellRef]int

func (c chainSearcherGenerationDetails) String() string {
	result := "Begin map\n"
	for cell, num := range c {
		result += "\t" + cell.String() + " : " + strconv.Itoa(num) + "\n"
	}
	result += "End map\n"
	return result
}

type chainSearcherAccumulator []chainSearcherGenerationDetails

func (c chainSearcherAccumulator) String() string {
	result := "Accumulator[\n"
	for _, rec := range c {
		result += fmt.Sprintf("%s\n", rec)
	}
	result += "]\n"
	return result
}

//accumulateGenerations goes through each generation (youngest to newest)
//and squaches older generation maps into each generation, so each
//generation's map represents the totality of all cells seen at that point.
func (c chainSearcherAccumulator) accumulateGenerations() {
	for i := len(c) - 2; i >= 0; i-- {
		lastGeneration := c[i+1]
		currentGeneration := c[i]
		for key, val := range lastGeneration {
			if currentVal, ok := currentGeneration[key]; ok {
				if currentVal != val {
					//No, this should be expected to happen when we get to an invalid grid state,
					//which we should expect to happen down one of the two branches (at least as explore
					//far enough.)
					log.Println("We were about to overwrite a value from an earlier generation... this shouldn't happen.")
				}
			}
			currentGeneration[key] = val
		}
	}
}

func makeChainSeacherAccumulator(size int) chainSearcherAccumulator {
	result := make(chainSearcherAccumulator, size)
	for i := 0; i < size; i++ {
		result[i] = make(map[cellRef]int)
	}
	return result
}

func chainSearcher(i int, cell *Cell, numToApply int, accumulator chainSearcherAccumulator) {

	//Chainsearcher implements a BFS over implications forward given the starting point.
	//It collects its results in the provided chainSearcherAccumulator.

	//TODO: why doesn't this just return its own chainSearcherAccumulator.

	//the first time we cross over into a new generation, we should do a one-time copy of the old generation
	//into the new.
	//At any write, if we notice that we'd be overwriting to a different value, we can bail out (how would
	//we mark that we bailed early), since we've run into an inconsistency down this branch and following
	//it further is not useful.

	type modificationToMake struct {
		generation int
		cell       *Cell
		numToApply int
	}

	workSteps := list.New()

	//Add the first workstep.
	workSteps.PushBack(modificationToMake{
		i,
		cell,
		numToApply,
	})

	var step modificationToMake

	e := workSteps.Front()

	for e != nil {

		workSteps.Remove(e)

		switch t := e.Value.(type) {
		case modificationToMake:
			step = t
		default:
			panic("Found unexpected type in workSteps list")
		}

		if step.generation <= 0 {
			break
		}

		generationDetails := accumulator[step.generation-1]

		//TODO: Check here if we crossed a generation boundary. If so, accumulate last generation.
		cellsToVisit := step.cell.Neighbors().FilterByPossible(step.numToApply).FilterByNumPossibilities(2)

		step.cell.SetNumber(step.numToApply)

		//TODO: check here if we're overwriting a value; if so, don't process anymore work steps.
		generationDetails[step.cell.ref()] = step.numToApply

		for _, cellToVisit := range cellsToVisit {
			possibilities := cellToVisit.Possibilities()

			if len(possibilities) != 1 {
				panic("Expected the cell to have one possibility")
			}

			forcedNum := possibilities[0]

			//Each branch modifies the grid, so create a new copy
			newGrid := cellToVisit.grid.Copy()
			cellToVisit = cellToVisit.InGrid(newGrid)

			workSteps.PushBack(modificationToMake{
				step.generation - 1,
				cellToVisit,
				forcedNum,
			})

		}

		e = workSteps.Front()

	}

}
<|endoftext|>


<commit_before>package main

import (
	"os"
	"strings"

	log "github.com/sirupsen/logrus"
	"github.com/spf13/cobra"

	"github.com/authelia/authelia/internal/utils"
)

func buildAutheliaBinary(xflags []string) {
	cmd := utils.CommandWithStdout("go", "build", "-o", "../../"+OutputDir+"/authelia", "-ldflags", strings.Join(xflags, " "))
	cmd.Dir = "cmd/authelia"

	cmd.Env = append(os.Environ(),
		"GOOS=linux", "GOARCH=amd64", "CGO_ENABLED=0")

	err := cmd.Run()
	if err != nil {
		log.Fatal(err)
	}
}

func buildFrontend() {
	cmd := utils.CommandWithStdout("yarn", "install")
	cmd.Dir = webDirectory

	err := cmd.Run()
	if err != nil {
		log.Fatal(err)
	}

	cmd = utils.CommandWithStdout("yarn", "build")
	cmd.Dir = webDirectory

	cmd.Env = append(os.Environ(), "INLINE_RUNTIME_CHUNK=false")

	err = cmd.Run()
	if err != nil {
		log.Fatal(err)
	}
}

func buildSwagger() {
	swaggerVer := "3.51.2"
	cmd := utils.CommandWithStdout("bash", "-c", "wget -q https://github.com/swagger-api/swagger-ui/archive/v"+swaggerVer+".tar.gz -O ./v"+swaggerVer+".tar.gz")

	err := cmd.Run()
	if err != nil {
		log.Fatal(err)
	}

	cmd = utils.CommandWithStdout("cp", "-r", "api", "internal/server/public_html")

	err = cmd.Run()
	if err != nil {
		log.Fatal(err)
	}

	cmd = utils.CommandWithStdout("tar", "-C", "internal/server/public_html/api", "--exclude=index.html", "--strip-components=2", "-xf", "v"+swaggerVer+".tar.gz", "swagger-ui-"+swaggerVer+"/dist")

	err = cmd.Run()
	if err != nil {
		log.Fatal(err)
	}

	cmd = utils.CommandWithStdout("rm", "./v"+swaggerVer+".tar.gz")

	err = cmd.Run()
	if err != nil {
		log.Fatal(err)
	}
}

func cleanAssets() {
	if err := os.Rename("internal/server/public_html", OutputDir+"/public_html"); err != nil {
		log.Fatal(err)
	}

	cmd := utils.CommandWithStdout("mkdir", "-p", "internal/server/public_html/api")

	if err := cmd.Run(); err != nil {
		log.Fatal(err)
	}

	cmd = utils.CommandWithStdout("bash", "-c", "touch internal/server/public_html/{index.html,api/index.html,api/openapi.yml}")

	if err := cmd.Run(); err != nil {
		log.Fatal(err)
	}
}

// Build build Authelia.
func Build(cobraCmd *cobra.Command, args []string) {
	log.Info("Building Authelia...")

	Clean(cobraCmd, args)

	xflags, err := getXFlags("", "0", "")
	if err != nil {
		log.Fatal(err)
	}

	log.Debug("Creating `" + OutputDir + "` directory")
	err = os.MkdirAll(OutputDir, os.ModePerm)

	if err != nil {
		log.Fatal(err)
	}

	log.Debug("Building Authelia frontend...")
	buildFrontend()

	log.Debug("Building swagger-ui frontend...")
	buildSwagger()

	buildkite, _ := cobraCmd.Flags().GetBool("buildkite")
	if buildkite {
		log.Debug("Buildkite job detected, skipping Authelia Go binary build")
	} else {
		log.Debug("Building Authelia Go binary...")
		buildAutheliaBinary(xflags)
	}

	cleanAssets()
}
<commit_msg>build(deps): update swagger-ui to v3.52.0 (#2276)<commit_after>package main

import (
	"os"
	"strings"

	log "github.com/sirupsen/logrus"
	"github.com/spf13/cobra"

	"github.com/authelia/authelia/internal/utils"
)

func buildAutheliaBinary(xflags []string) {
	cmd := utils.CommandWithStdout("go", "build", "-o", "../../"+OutputDir+"/authelia", "-ldflags", strings.Join(xflags, " "))
	cmd.Dir = "cmd/authelia"

	cmd.Env = append(os.Environ(),
		"GOOS=linux", "GOARCH=amd64", "CGO_ENABLED=0")

	err := cmd.Run()
	if err != nil {
		log.Fatal(err)
	}
}

func buildFrontend() {
	cmd := utils.CommandWithStdout("yarn", "install")
	cmd.Dir = webDirectory

	err := cmd.Run()
	if err != nil {
		log.Fatal(err)
	}

	cmd = utils.CommandWithStdout("yarn", "build")
	cmd.Dir = webDirectory

	cmd.Env = append(os.Environ(), "INLINE_RUNTIME_CHUNK=false")

	err = cmd.Run()
	if err != nil {
		log.Fatal(err)
	}
}

func buildSwagger() {
	swaggerVer := "3.52.0"
	cmd := utils.CommandWithStdout("bash", "-c", "wget -q https://github.com/swagger-api/swagger-ui/archive/v"+swaggerVer+".tar.gz -O ./v"+swaggerVer+".tar.gz")

	err := cmd.Run()
	if err != nil {
		log.Fatal(err)
	}

	cmd = utils.CommandWithStdout("cp", "-r", "api", "internal/server/public_html")

	err = cmd.Run()
	if err != nil {
		log.Fatal(err)
	}

	cmd = utils.CommandWithStdout("tar", "-C", "internal/server/public_html/api", "--exclude=index.html", "--strip-components=2", "-xf", "v"+swaggerVer+".tar.gz", "swagger-ui-"+swaggerVer+"/dist")

	err = cmd.Run()
	if err != nil {
		log.Fatal(err)
	}

	cmd = utils.CommandWithStdout("rm", "./v"+swaggerVer+".tar.gz")

	err = cmd.Run()
	if err != nil {
		log.Fatal(err)
	}
}

func cleanAssets() {
	if err := os.Rename("internal/server/public_html", OutputDir+"/public_html"); err != nil {
		log.Fatal(err)
	}

	cmd := utils.CommandWithStdout("mkdir", "-p", "internal/server/public_html/api")

	if err := cmd.Run(); err != nil {
		log.Fatal(err)
	}

	cmd = utils.CommandWithStdout("bash", "-c", "touch internal/server/public_html/{index.html,api/index.html,api/openapi.yml}")

	if err := cmd.Run(); err != nil {
		log.Fatal(err)
	}
}

// Build build Authelia.
func Build(cobraCmd *cobra.Command, args []string) {
	log.Info("Building Authelia...")

	Clean(cobraCmd, args)

	xflags, err := getXFlags("", "0", "")
	if err != nil {
		log.Fatal(err)
	}

	log.Debug("Creating `" + OutputDir + "` directory")
	err = os.MkdirAll(OutputDir, os.ModePerm)

	if err != nil {
		log.Fatal(err)
	}

	log.Debug("Building Authelia frontend...")
	buildFrontend()

	log.Debug("Building swagger-ui frontend...")
	buildSwagger()

	buildkite, _ := cobraCmd.Flags().GetBool("buildkite")
	if buildkite {
		log.Debug("Buildkite job detected, skipping Authelia Go binary build")
	} else {
		log.Debug("Building Authelia Go binary...")
		buildAutheliaBinary(xflags)
	}

	cleanAssets()
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"os"
	"runtime"
	"testing"
)

type mockTripper struct {
}

func (m *mockTripper) RoundTrip(r *http.Request) (*http.Response, error) {
	return &http.Response{StatusCode: 200, Body: ioutil.NopCloser(bytes.NewReader([]byte("{}")))}, nil
}

var mockClient = &http.Client{Transport: &mockTripper{}}

func TestCreateInstanceConfigs(t *testing.T) {
	for _, v := range []struct {
		desc string
		//inputs
		dir          string
		useFuse      bool
		instances    []string
		instancesSrc string
		// We don't need to check the []instancesConfig return value, we already
		// have a TestParseInstanceConfig.
		wantErr bool

		skipFailedInstanceConfig bool
	}{
		{
			desc:                     "setting -fuse and -dir",
			dir:                      "dir",
			useFuse:                  true,
			instances:                nil,
			instancesSrc:             "",
			wantErr:                  false,
			skipFailedInstanceConfig: false,
		},
		{
			desc:                     "setting -fuse",
			dir:                      "",
			useFuse:                  true,
			instances:                nil,
			instancesSrc:             "",
			wantErr:                  true,
			skipFailedInstanceConfig: false,
		},
		{
			desc:                     "setting -fuse, -dir, and -instances",
			dir:                      "dir",
			useFuse:                  true,
			instances:                []string{"proj:reg:x"},
			instancesSrc:             "",
			wantErr:                  true,
			skipFailedInstanceConfig: false,
		},
		{
			desc:                     "setting -fuse, -dir, and -instances_metadata",
			dir:                      "dir",
			useFuse:                  true,
			instances:                nil,
			instancesSrc:             "md",
			wantErr:                  true,
			skipFailedInstanceConfig: false,
		},
		{
			desc:                     "setting -dir and -instances (unix socket)",
			dir:                      "dir",
			useFuse:                  false,
			instances:                []string{"proj:reg:x"},
			instancesSrc:             "",
			wantErr:                  false,
			skipFailedInstanceConfig: false,
		},
		{
			// tests for the case where invalid configs can still exist, when skipped
			desc:                     "setting -dir and -instances (unix socket) w/ something invalid",
			dir:                      "dir",
			useFuse:                  false,
			instances:                []string{"proj:reg:x", "INVALID_PROJECT_STRING"},
			instancesSrc:             "",
			wantErr:                  false,
			skipFailedInstanceConfig: true,
		},
		{
			desc:                     "Seting -instance (unix socket)",
			dir:                      "",
			useFuse:                  false,
			instances:                []string{"proj:reg:x"},
			instancesSrc:             "",
			wantErr:                  true,
			skipFailedInstanceConfig: false,
		},
		{
			desc:                     "setting -instance (tcp socket)",
			dir:                      "",
			useFuse:                  false,
			instances:                []string{"proj:reg:x=tcp:1234"},
			instancesSrc:             "",
			wantErr:                  false,
			skipFailedInstanceConfig: false,
		},
		{
			desc:                     "setting -instance (tcp socket) and -instances_metadata",
			dir:                      "",
			useFuse:                  false,
			instances:                []string{"proj:reg:x=tcp:1234"},
			instancesSrc:             "md",
			wantErr:                  true,
			skipFailedInstanceConfig: false,
		},
		{
			desc:                     "setting -dir, -instance (tcp socket), and -instances_metadata",
			dir:                      "dir",
			useFuse:                  false,
			instances:                []string{"proj:reg:x=tcp:1234"},
			instancesSrc:             "md",
			wantErr:                  false,
			skipFailedInstanceConfig: false,
		},
		{
			desc:                     "setting -dir, -instance (unix socket), and -instances_metadata",
			dir:                      "dir",
			useFuse:                  false,
			instances:                []string{"proj:reg:x"},
			instancesSrc:             "md",
			wantErr:                  false,
			skipFailedInstanceConfig: false,
		},
		{
			desc:                     "setting -dir and -instances_metadata",
			dir:                      "dir",
			useFuse:                  false,
			instances:                nil,
			instancesSrc:             "md",
			wantErr:                  false,
			skipFailedInstanceConfig: false,
		},
		{
			desc:                     "setting -instances_metadata",
			dir:                      "",
			useFuse:                  false,
			instances:                nil,
			instancesSrc:             "md",
			wantErr:                  true,
			skipFailedInstanceConfig: false,
		},
	} {
		// fuse is not supported in CI for darwin
		if runtime.GOOS == "darwin" && v.useFuse {
			continue
		}
		// fuse and unix sockets are not supported on windows
		if runtime.GOOS == "windows" && (v.useFuse || v.dir != "") {
			continue
		}
		if v.useFuse && testing.Short() {
			t.Skip("skipping fuse tests in short mode.")
		}
		_, err := CreateInstanceConfigs(v.dir, v.useFuse, v.instances, v.instancesSrc, mockClient, v.skipFailedInstanceConfig)
		if v.wantErr {
			if err == nil {
				t.Errorf("CreateInstanceConfigs passed when %s, wanted error", v.desc)
			}
			continue
		}
		if err != nil {
			t.Errorf("CreateInstanceConfigs gave error when %s: %v", v.desc, err)
		}
	}
}

func TestParseInstanceConfig(t *testing.T) {
	// sentinel values
	var (
		anyLoopbackAddress = "<any loopback address>"
		wantErr            = instanceConfig{"<want error>", "", ""}
	)

	tcs := []struct {
		// inputs
		dir, instance string

		wantCfg instanceConfig
	}{
		{
			"/x", "domain.com:my-proj:my-reg:my-instance",
			instanceConfig{"domain.com:my-proj:my-reg:my-instance", "unix", "/x/domain.com:my-proj:my-reg:my-instance"},
		}, {
			"/x", "my-proj:my-reg:my-instance",
			instanceConfig{"my-proj:my-reg:my-instance", "unix", "/x/my-proj:my-reg:my-instance"},
		}, {
			"/x", "my-proj:my-reg:my-instance=unix:socket_name",
			instanceConfig{"my-proj:my-reg:my-instance", "unix", "/x/socket_name"},
		}, {
			"/x", "my-proj:my-reg:my-instance=unix:/my/custom/sql-socket",
			instanceConfig{"my-proj:my-reg:my-instance", "unix", "/my/custom/sql-socket"},
		}, {
			"/x", "my-proj:my-reg:my-instance=tcp:1234",
			instanceConfig{"my-proj:my-reg:my-instance", "tcp", anyLoopbackAddress},
		}, {
			"/x", "my-proj:my-reg:my-instance=tcp4:1234",
			instanceConfig{"my-proj:my-reg:my-instance", "tcp4", "127.0.0.1:1234"},
		}, {
			"/x", "my-proj:my-reg:my-instance=tcp6:1234",
			instanceConfig{"my-proj:my-reg:my-instance", "tcp6", "[::1]:1234"},
		}, {
			"/x", "my-proj:my-reg:my-instance=tcp:my-host:1111",
			instanceConfig{"my-proj:my-reg:my-instance", "tcp", "my-host:1111"},
		}, {
			"/x", "my-proj:my-reg:my-instance=",
			wantErr,
		}, {
			"/x", "my-proj:my-reg:my-instance=cool network",
			wantErr,
		}, {
			"/x", "my-proj:my-reg:my-instance=cool network:1234",
			wantErr,
		}, {
			"/x", "my-proj:my-reg:my-instance=oh:so:many:colons",
			wantErr,
		},
	}

	for _, tc := range tcs {
		t.Run(fmt.Sprintf("parseInstanceConfig(%q, %q)", tc.dir, tc.instance), func(t *testing.T) {
			if os.Getenv("EXPECT_IPV4_AND_IPV6") != "true" {
				// Skip ipv4 and ipv6 if they are not supported by the machine.
				// (assumption is that validNets isn't buggy)
				if tc.wantCfg.Network == "tcp4" || tc.wantCfg.Network == "tcp6" {
					if !validNets[tc.wantCfg.Network] {
						t.Skipf("%q net not supported, skipping", tc.wantCfg.Network)
					}
				}
				// Skip unix sockets on Windows
				if runtime.GOOS == "windows" && tc.wantCfg.Network == "unix" {
					t.Skipf("%q net not supported on Windows, skipping", tc.wantCfg.Network)
				}
			}

			got, err := parseInstanceConfig(tc.dir, tc.instance, mockClient)
			if tc.wantCfg == wantErr {
				if err != nil {
					return // pass. an error was expected and returned.
				}
				t.Fatalf("parseInstanceConfig(%s, %s) = %+v, wanted error", tc.dir, tc.instance, got)
			}
			if err != nil {
				t.Fatalf("parseInstanceConfig(%s, %s) had unexpected error: %v", tc.dir, tc.instance, err)
			}

			if tc.wantCfg.Address == anyLoopbackAddress {
				host, _, err := net.SplitHostPort(got.Address)
				if err != nil {
					t.Fatalf("net.SplitHostPort(%v): %v", got.Address, err)
				}
				ip := net.ParseIP(host)
				if !ip.IsLoopback() {
					t.Fatalf("want loopback, got addr: %v", got.Address)
				}

				// use a placeholder address, so the rest of the config can be compared
				got.Address = "<loopback>"
				tc.wantCfg.Address = got.Address
			}

			if got != tc.wantCfg {
				t.Errorf("parseInstanceConfig(%s, %s) = %+v, want %+v", tc.dir, tc.instance, got, tc.wantCfg)
			}
		})
	}
}
<commit_msg>chore: use subtest (#1275)<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"os"
	"runtime"
	"testing"
)

type mockTripper struct {
}

func (m *mockTripper) RoundTrip(r *http.Request) (*http.Response, error) {
	return &http.Response{StatusCode: 200, Body: ioutil.NopCloser(bytes.NewReader([]byte("{}")))}, nil
}

var mockClient = &http.Client{Transport: &mockTripper{}}

func TestCreateInstanceConfigs(t *testing.T) {
	for _, v := range []struct {
		desc string
		//inputs
		dir          string
		useFuse      bool
		instances    []string
		instancesSrc string
		// We don't need to check the []instancesConfig return value, we already
		// have a TestParseInstanceConfig.
		wantErr bool

		skipFailedInstanceConfig bool
	}{
		{
			desc:                     "setting -fuse and -dir",
			dir:                      "dir",
			useFuse:                  true,
			instances:                nil,
			instancesSrc:             "",
			wantErr:                  false,
			skipFailedInstanceConfig: false,
		},
		{
			desc:                     "setting -fuse",
			dir:                      "",
			useFuse:                  true,
			instances:                nil,
			instancesSrc:             "",
			wantErr:                  true,
			skipFailedInstanceConfig: false,
		},
		{
			desc:                     "setting -fuse, -dir, and -instances",
			dir:                      "dir",
			useFuse:                  true,
			instances:                []string{"proj:reg:x"},
			instancesSrc:             "",
			wantErr:                  true,
			skipFailedInstanceConfig: false,
		},
		{
			desc:                     "setting -fuse, -dir, and -instances_metadata",
			dir:                      "dir",
			useFuse:                  true,
			instances:                nil,
			instancesSrc:             "md",
			wantErr:                  true,
			skipFailedInstanceConfig: false,
		},
		{
			desc:                     "setting -dir and -instances (unix socket)",
			dir:                      "dir",
			useFuse:                  false,
			instances:                []string{"proj:reg:x"},
			instancesSrc:             "",
			wantErr:                  false,
			skipFailedInstanceConfig: false,
		},
		{
			// tests for the case where invalid configs can still exist, when skipped
			desc:                     "setting -dir and -instances (unix socket) w/ something invalid",
			dir:                      "dir",
			useFuse:                  false,
			instances:                []string{"proj:reg:x", "INVALID_PROJECT_STRING"},
			instancesSrc:             "",
			wantErr:                  false,
			skipFailedInstanceConfig: true,
		},
		{
			desc:                     "Seting -instance (unix socket)",
			dir:                      "",
			useFuse:                  false,
			instances:                []string{"proj:reg:x"},
			instancesSrc:             "",
			wantErr:                  true,
			skipFailedInstanceConfig: false,
		},
		{
			desc:                     "setting -instance (tcp socket)",
			dir:                      "",
			useFuse:                  false,
			instances:                []string{"proj:reg:x=tcp:1234"},
			instancesSrc:             "",
			wantErr:                  false,
			skipFailedInstanceConfig: false,
		},
		{
			desc:                     "setting -instance (tcp socket) and -instances_metadata",
			dir:                      "",
			useFuse:                  false,
			instances:                []string{"proj:reg:x=tcp:1234"},
			instancesSrc:             "md",
			wantErr:                  true,
			skipFailedInstanceConfig: false,
		},
		{
			desc:                     "setting -dir, -instance (tcp socket), and -instances_metadata",
			dir:                      "dir",
			useFuse:                  false,
			instances:                []string{"proj:reg:x=tcp:1234"},
			instancesSrc:             "md",
			wantErr:                  false,
			skipFailedInstanceConfig: false,
		},
		{
			desc:                     "setting -dir, -instance (unix socket), and -instances_metadata",
			dir:                      "dir",
			useFuse:                  false,
			instances:                []string{"proj:reg:x"},
			instancesSrc:             "md",
			wantErr:                  false,
			skipFailedInstanceConfig: false,
		},
		{
			desc:                     "setting -dir and -instances_metadata",
			dir:                      "dir",
			useFuse:                  false,
			instances:                nil,
			instancesSrc:             "md",
			wantErr:                  false,
			skipFailedInstanceConfig: false,
		},
		{
			desc:                     "setting -instances_metadata",
			dir:                      "",
			useFuse:                  false,
			instances:                nil,
			instancesSrc:             "md",
			wantErr:                  true,
			skipFailedInstanceConfig: false,
		},
	} {
		t.Run(v.desc, func(t *testing.T) {
			// fuse is not supported in CI for darwin
			if runtime.GOOS == "darwin" && v.useFuse {
				t.Skip("skipping Darwin and FUSE test")
			}
			// fuse and unix sockets are not supported on windows
			if runtime.GOOS == "windows" && (v.useFuse || v.dir != "") {
				t.Skip("skipping Windows and FUSE/Unix socket test")
			}
			if v.useFuse && testing.Short() {
				t.Skip("skipping fuse tests in short mode.")
			}
			_, err := CreateInstanceConfigs(v.dir, v.useFuse, v.instances, v.instancesSrc, mockClient, v.skipFailedInstanceConfig)
			if v.wantErr {
				if err == nil {
					t.Errorf("CreateInstanceConfigs passed when %s, wanted error", v.desc)
				}
				return
			}
			if err != nil {
				t.Errorf("CreateInstanceConfigs gave error when %s: %v", v.desc, err)
			}
		})
	}
}

func TestParseInstanceConfig(t *testing.T) {
	// sentinel values
	var (
		anyLoopbackAddress = "<any loopback address>"
		wantErr            = instanceConfig{"<want error>", "", ""}
	)

	tcs := []struct {
		// inputs
		dir, instance string

		wantCfg instanceConfig
	}{
		{
			"/x", "domain.com:my-proj:my-reg:my-instance",
			instanceConfig{"domain.com:my-proj:my-reg:my-instance", "unix", "/x/domain.com:my-proj:my-reg:my-instance"},
		}, {
			"/x", "my-proj:my-reg:my-instance",
			instanceConfig{"my-proj:my-reg:my-instance", "unix", "/x/my-proj:my-reg:my-instance"},
		}, {
			"/x", "my-proj:my-reg:my-instance=unix:socket_name",
			instanceConfig{"my-proj:my-reg:my-instance", "unix", "/x/socket_name"},
		}, {
			"/x", "my-proj:my-reg:my-instance=unix:/my/custom/sql-socket",
			instanceConfig{"my-proj:my-reg:my-instance", "unix", "/my/custom/sql-socket"},
		}, {
			"/x", "my-proj:my-reg:my-instance=tcp:1234",
			instanceConfig{"my-proj:my-reg:my-instance", "tcp", anyLoopbackAddress},
		}, {
			"/x", "my-proj:my-reg:my-instance=tcp4:1234",
			instanceConfig{"my-proj:my-reg:my-instance", "tcp4", "127.0.0.1:1234"},
		}, {
			"/x", "my-proj:my-reg:my-instance=tcp6:1234",
			instanceConfig{"my-proj:my-reg:my-instance", "tcp6", "[::1]:1234"},
		}, {
			"/x", "my-proj:my-reg:my-instance=tcp:my-host:1111",
			instanceConfig{"my-proj:my-reg:my-instance", "tcp", "my-host:1111"},
		}, {
			"/x", "my-proj:my-reg:my-instance=",
			wantErr,
		}, {
			"/x", "my-proj:my-reg:my-instance=cool network",
			wantErr,
		}, {
			"/x", "my-proj:my-reg:my-instance=cool network:1234",
			wantErr,
		}, {
			"/x", "my-proj:my-reg:my-instance=oh:so:many:colons",
			wantErr,
		},
	}

	for _, tc := range tcs {
		t.Run(fmt.Sprintf("parseInstanceConfig(%q, %q)", tc.dir, tc.instance), func(t *testing.T) {
			if os.Getenv("EXPECT_IPV4_AND_IPV6") != "true" {
				// Skip ipv4 and ipv6 if they are not supported by the machine.
				// (assumption is that validNets isn't buggy)
				if tc.wantCfg.Network == "tcp4" || tc.wantCfg.Network == "tcp6" {
					if !validNets[tc.wantCfg.Network] {
						t.Skipf("%q net not supported, skipping", tc.wantCfg.Network)
					}
				}
				// Skip unix sockets on Windows
				if runtime.GOOS == "windows" && tc.wantCfg.Network == "unix" {
					t.Skipf("%q net not supported on Windows, skipping", tc.wantCfg.Network)
				}
			}

			got, err := parseInstanceConfig(tc.dir, tc.instance, mockClient)
			if tc.wantCfg == wantErr {
				if err != nil {
					return // pass. an error was expected and returned.
				}
				t.Fatalf("parseInstanceConfig(%s, %s) = %+v, wanted error", tc.dir, tc.instance, got)
			}
			if err != nil {
				t.Fatalf("parseInstanceConfig(%s, %s) had unexpected error: %v", tc.dir, tc.instance, err)
			}

			if tc.wantCfg.Address == anyLoopbackAddress {
				host, _, err := net.SplitHostPort(got.Address)
				if err != nil {
					t.Fatalf("net.SplitHostPort(%v): %v", got.Address, err)
				}
				ip := net.ParseIP(host)
				if !ip.IsLoopback() {
					t.Fatalf("want loopback, got addr: %v", got.Address)
				}

				// use a placeholder address, so the rest of the config can be compared
				got.Address = "<loopback>"
				tc.wantCfg.Address = got.Address
			}

			if got != tc.wantCfg {
				t.Errorf("parseInstanceConfig(%s, %s) = %+v, want %+v", tc.dir, tc.instance, got, tc.wantCfg)
			}
		})
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"flag"
	"log"
	"os"
	"os/exec"
	"os/signal"
	"strings"
	"text/template"
	"time"

	"github.com/nullstyle/testy-mctesterton/pkgwatch"
	"github.com/nullstyle/testy-mctesterton/pkgwork"
)

var done = make(chan os.Signal, 1)
var cmdStr = flag.String("c", "", "command line to execute upon package source change")
var cmdTmpl *template.Template

func main() {
	var err error

	flag.Parse()
	signal.Notify(done, os.Interrupt, os.Kill)

	cmdTmpl, err = template.New("cmd").Parse(*cmdStr)
	if err != nil {
		log.Fatal(err)
	}

	dir, err := os.Getwd()
	if err != nil {
		log.Fatal(err)
	}

	watcher := &pkgwatch.Watcher{
		Dir:      dir,
		Debounce: 300 * time.Millisecond,
	}

	worker := &pkgwork.Worker{
		Fn:       execute,
		Cooldown: 5 * time.Second,
	}

	if err := watcher.Run(); err != nil {
		log.Fatal(err)
	}
	defer watcher.Close()

	log.Println("waiting for changes")

	for {
		select {
		case pkg := <-watcher.Changes():
			go func() {
				if err := worker.Run(pkg); err != nil {
					log.Fatal(err)
				}
			}()
		case _ = <-done:
			log.Println("shutting down")
			os.Exit(0)
		}
	}
}

func execute(pkg string) error {
	var cmdBuf bytes.Buffer
	err := cmdTmpl.Execute(&cmdBuf, struct{ Pkg string }{pkg})
	if err != nil {
		return err
	}

	full := cmdBuf.String()
	split := strings.Split(full, " ")
	cmd := exec.Command(split[0], split[1:]...)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr

	return cmd.Run()
}
<commit_msg>Add configurable debounce and cooldown<commit_after>package main

import (
	"bytes"
	"flag"
	"log"
	"os"
	"os/exec"
	"os/signal"
	"strings"
	"text/template"
	"time"

	"github.com/nullstyle/testy-mctesterton/pkgwatch"
	"github.com/nullstyle/testy-mctesterton/pkgwork"
)

var done = make(chan os.Signal, 1)

var cmdStr = flag.String("cmd", "", "command line to execute upon package source change")
var cmdTmpl *template.Template

var debounce = flag.Duration("debounce", 500*time.Millisecond, "how long to debounce package changes")
var cooldown = flag.Duration("cooldown", 4*time.Second, "how long to cooldown each command execution")

func main() {
	var err error

	flag.Parse()
	signal.Notify(done, os.Interrupt, os.Kill)

	cmdTmpl, err = template.New("cmd").Parse(*cmdStr)
	if err != nil {
		log.Fatal(err)
	}

	dir, err := os.Getwd()
	if err != nil {
		log.Fatal(err)
	}

	watcher := &pkgwatch.Watcher{
		Dir:      dir,
		Debounce: *debounce,
	}

	worker := &pkgwork.Worker{
		Fn:       execute,
		Cooldown: *cooldown,
	}

	if err := watcher.Run(); err != nil {
		log.Fatal(err)
	}
	defer watcher.Close()

	log.Println("waiting for changes")

	for {
		select {
		case pkg := <-watcher.Changes():
			go func() {
				if err := worker.Run(pkg); err != nil {
					log.Fatal(err)
				}
			}()
		case _ = <-done:
			log.Println("shutting down")
			os.Exit(0)
		}
	}
}

func execute(pkg string) error {
	var cmdBuf bytes.Buffer
	err := cmdTmpl.Execute(&cmdBuf, struct{ Pkg string }{pkg})
	if err != nil {
		return err
	}

	full := cmdBuf.String()
	split := strings.Split(full, " ")
	cmd := exec.Command(split[0], split[1:]...)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr

	return cmd.Run()
}
<|endoftext|>


<commit_before>package canal

import (
	"time"

	"golang.org/x/net/context"

	"github.com/juju/errors"
	"github.com/ngaut/log"
	"github.com/siddontang/go-mysql/mysql"
	"github.com/siddontang/go-mysql/replication"
)

func (c *Canal) startSyncBinlog() error {
	pos := mysql.Position{c.master.Name, c.master.Position}

	log.Infof("start sync binlog at %v", pos)

	s, err := c.syncer.StartSync(pos)
	if err != nil {
		return errors.Errorf("start sync replication at %v error %v", pos, err)
	}

	timeout := time.Second
	forceSavePos := false
	for {
		ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
		ev, err := s.GetEvent(ctx)
		cancel()

		if err == context.DeadlineExceeded {
			timeout = 2 * timeout
			continue
		}

		if err != nil {
			return errors.Trace(err)
		}

		timeout = time.Second

		//next binlog pos
		pos.Pos = ev.Header.LogPos

		forceSavePos = false

		switch e := ev.Event.(type) {
		case *replication.RotateEvent:
			pos.Name = string(e.NextLogName)
			pos.Pos = uint32(e.Position)
			// r.ev <- pos
			forceSavePos = true
			log.Infof("rotate binlog to %v", pos)
		case *replication.RowsEvent:
			// we only focus row based event
			if err = c.handleRowsEvent(ev); err != nil {
				log.Errorf("handle rows event error %v", err)
				return errors.Trace(err)
			}
		case *replication.TableMapEvent:
			continue
		case *replication.FormatDescriptionEvent:
			continue
		default:
		}

		c.master.Update(pos.Name, pos.Pos)
		c.master.Save(forceSavePos)
	}

	return nil
}

func (c *Canal) handleRowsEvent(e *replication.BinlogEvent) error {
	ev := e.Event.(*replication.RowsEvent)

	// Caveat: table may be altered at runtime.
	schema := string(ev.Table.Schema)
	table := string(ev.Table.Table)

	t, err := c.GetTable(schema, table)
	if err != nil {
		return errors.Trace(err)
	}
	var action string
	switch e.Header.EventType {
	case replication.WRITE_ROWS_EVENTv1, replication.WRITE_ROWS_EVENTv2:
		action = InsertAction
	case replication.DELETE_ROWS_EVENTv1, replication.DELETE_ROWS_EVENTv2:
		action = DeleteAction
	case replication.UPDATE_ROWS_EVENTv1, replication.UPDATE_ROWS_EVENTv2:
		action = UpdateAction
	default:
		return errors.Errorf("%s not supported now", e.Header.EventType)
	}
	events := newRowsEvent(t, action, ev.Rows)
	return c.travelRowsEventHandler(events)
}

func (c *Canal) WaitUntilPos(pos mysql.Position, timeout int) error {
	if timeout <= 0 {
		timeout = 60
	}

	timer := time.NewTimer(time.Duration(timeout) * time.Second)
	for {
		select {
		case <-timer.C:
			return errors.Errorf("wait position %v err", pos)
		default:
			curpos := c.master.Pos()
			if curpos.Compare(pos) >= 0 {
				return nil
			} else {
				time.Sleep(100 * time.Millisecond)
			}
		}
	}

	return nil
}

func (c *Canal) CatchMasterPos(timeout int) error {
	rr, err := c.Execute("SHOW MASTER STATUS")
	if err != nil {
		return errors.Trace(err)
	}

	name, _ := rr.GetString(0, 0)
	pos, _ := rr.GetInt(0, 1)

	return c.WaitUntilPos(mysql.Position{name, uint32(pos)}, timeout)
}
<commit_msg>canel: update sync position save. (#101)<commit_after>package canal

import (
	"time"

	"golang.org/x/net/context"

	"github.com/juju/errors"
	"github.com/ngaut/log"
	"github.com/siddontang/go-mysql/mysql"
	"github.com/siddontang/go-mysql/replication"
)

func (c *Canal) startSyncBinlog() error {
	pos := mysql.Position{c.master.Name, c.master.Position}

	log.Infof("start sync binlog at %v", pos)

	s, err := c.syncer.StartSync(pos)
	if err != nil {
		return errors.Errorf("start sync replication at %v error %v", pos, err)
	}

	timeout := time.Second
	forceSavePos := false
	for {
		ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
		ev, err := s.GetEvent(ctx)
		cancel()

		if err == context.DeadlineExceeded {
			timeout = 2 * timeout
			continue
		}

		if err != nil {
			return errors.Trace(err)
		}

		timeout = time.Second

		//next binlog pos
		pos.Pos = ev.Header.LogPos

		forceSavePos = false

		// We only save position with RotateEvent and XIDEvent.
		// For RowsEvent, we can't save the position until meeting XIDEvent
		// which tells the whole transaction is over.
		// TODO: If we meet any DDL query, we must save too.
		switch e := ev.Event.(type) {
		case *replication.RotateEvent:
			pos.Name = string(e.NextLogName)
			pos.Pos = uint32(e.Position)
			// r.ev <- pos
			forceSavePos = true
			log.Infof("rotate binlog to %v", pos)
		case *replication.RowsEvent:
			// we only focus row based event
			if err = c.handleRowsEvent(ev); err != nil {
				log.Errorf("handle rows event error %v", err)
				return errors.Trace(err)
			}
			continue
		case *replication.XIDEvent:
			// try to save the position later
		default:
			continue
		}

		c.master.Update(pos.Name, pos.Pos)
		c.master.Save(forceSavePos)
	}

	return nil
}

func (c *Canal) handleRowsEvent(e *replication.BinlogEvent) error {
	ev := e.Event.(*replication.RowsEvent)

	// Caveat: table may be altered at runtime.
	schema := string(ev.Table.Schema)
	table := string(ev.Table.Table)

	t, err := c.GetTable(schema, table)
	if err != nil {
		return errors.Trace(err)
	}
	var action string
	switch e.Header.EventType {
	case replication.WRITE_ROWS_EVENTv1, replication.WRITE_ROWS_EVENTv2:
		action = InsertAction
	case replication.DELETE_ROWS_EVENTv1, replication.DELETE_ROWS_EVENTv2:
		action = DeleteAction
	case replication.UPDATE_ROWS_EVENTv1, replication.UPDATE_ROWS_EVENTv2:
		action = UpdateAction
	default:
		return errors.Errorf("%s not supported now", e.Header.EventType)
	}
	events := newRowsEvent(t, action, ev.Rows)
	return c.travelRowsEventHandler(events)
}

func (c *Canal) WaitUntilPos(pos mysql.Position, timeout int) error {
	if timeout <= 0 {
		timeout = 60
	}

	timer := time.NewTimer(time.Duration(timeout) * time.Second)
	for {
		select {
		case <-timer.C:
			return errors.Errorf("wait position %v err", pos)
		default:
			curpos := c.master.Pos()
			if curpos.Compare(pos) >= 0 {
				return nil
			} else {
				time.Sleep(100 * time.Millisecond)
			}
		}
	}

	return nil
}

func (c *Canal) CatchMasterPos(timeout int) error {
	rr, err := c.Execute("SHOW MASTER STATUS")
	if err != nil {
		return errors.Trace(err)
	}

	name, _ := rr.GetString(0, 0)
	pos, _ := rr.GetInt(0, 1)

	return c.WaitUntilPos(mysql.Position{name, uint32(pos)}, timeout)
}
<|endoftext|>


<commit_before>// Copyright 2017 The go-hep Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package hepevt

import (
	"fmt"
	"io"
)

type AsciiEncoder struct {
	w io.Writer
}

func NewAsciiEncoder(w io.Writer) *AsciiEncoder {
	return &AsciiEncoder{w: w}
}

func (enc *AsciiEncoder) Encode(evt *Event) error {
	var err error
	_, err = fmt.Fprintf(enc.w, "%d %d\n", evt.Nevhep, evt.Nhep)
	if err != nil {
		return err
	}

	for i := 0; i < evt.Nhep; i++ {
		_, err = fmt.Fprintf(
			enc.w,
			"%d %d %d %d %d %d %E %E %E %E %E %E %E %E %E\n",
			evt.Isthep[i],
			evt.Idhep[i],
			// convert 0-based indices to 1-based ones
			evt.Jmohep[i][0]+1, evt.Jmohep[i][1]+1,
			evt.Jdahep[i][0]+1, evt.Jdahep[i][1]+1,
			// <---
			evt.Phep[i][0], evt.Phep[i][1], evt.Phep[i][2], evt.Phep[i][3],
			evt.Phep[i][4],
			evt.Vhep[i][0], evt.Vhep[i][1], evt.Vhep[i][2], evt.Vhep[i][3],
		)
		if err != nil {
			return err
		}
	}
	return err
}

type AsciiDecoder struct {
	r io.Reader
}

func NewAsciiDecoder(r io.Reader) *AsciiDecoder {
	return &AsciiDecoder{r: r}
}

func (dec *AsciiDecoder) Decode(evt *Event) error {
	var err error

	_, err = fmt.Fscanf(dec.r, "%d %d\n", &evt.Nevhep, &evt.Nhep)
	if err != nil {
		return err
	}

	// resize
	if len(evt.Isthep) > evt.Nhep {
		evt.Isthep = evt.Isthep[:evt.Nhep]
		evt.Idhep = evt.Idhep[:evt.Nhep]
		evt.Jmohep = evt.Jmohep[:evt.Nhep]
		evt.Jdahep = evt.Jdahep[:evt.Nhep]
		evt.Phep = evt.Phep[:evt.Nhep]
		evt.Vhep = evt.Vhep[:evt.Nhep]
	} else {
		sz := evt.Nhep - len(evt.Isthep)
		evt.Isthep = append(evt.Isthep, make([]int, sz)...)
		evt.Idhep = append(evt.Idhep, make([]int, sz)...)
		evt.Jmohep = append(evt.Jmohep, make([][2]int, sz)...)
		evt.Jdahep = append(evt.Jdahep, make([][2]int, sz)...)
		evt.Phep = append(evt.Phep, make([][5]float64, sz)...)
		evt.Vhep = append(evt.Vhep, make([][4]float64, sz)...)
	}

	for i := 0; i < evt.Nhep; i++ {
		_, err = fmt.Fscanf(
			dec.r,
			"%d %d %d %d %d %d %E %E %E %E %E %E %E %E %E\n",
			&evt.Isthep[i],
			&evt.Idhep[i],
			&evt.Jmohep[i][0], &evt.Jmohep[i][1],
			&evt.Jdahep[i][0], &evt.Jdahep[i][1],
			&evt.Phep[i][0], &evt.Phep[i][1], &evt.Phep[i][2], &evt.Phep[i][3],
			&evt.Phep[i][4],
			&evt.Vhep[i][0], &evt.Vhep[i][1], &evt.Vhep[i][2], &evt.Vhep[i][3],
		)
		if err != nil {
			return err
		}
		// convert 0-based indices to 1-based ones
		evt.Jmohep[i][0] -= 1
		evt.Jmohep[i][1] -= 1
		evt.Jdahep[i][0] -= 1
		evt.Jdahep[i][1] -= 1
	}
	return err
}
<commit_msg>hepevt: use pkg/errors.WithStack<commit_after>// Copyright 2017 The go-hep Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package hepevt

import (
	"fmt"
	"io"

	"github.com/pkg/errors"
)

type AsciiEncoder struct {
	w io.Writer
}

func NewAsciiEncoder(w io.Writer) *AsciiEncoder {
	return &AsciiEncoder{w: w}
}

func (enc *AsciiEncoder) Encode(evt *Event) error {
	var err error
	_, err = fmt.Fprintf(enc.w, "%d %d\n", evt.Nevhep, evt.Nhep)
	if err != nil {
		return errors.WithStack(err)
	}

	for i := 0; i < evt.Nhep; i++ {
		_, err = fmt.Fprintf(
			enc.w,
			"%d %d %d %d %d %d %E %E %E %E %E %E %E %E %E\n",
			evt.Isthep[i],
			evt.Idhep[i],
			// convert 0-based indices to 1-based ones
			evt.Jmohep[i][0]+1, evt.Jmohep[i][1]+1,
			evt.Jdahep[i][0]+1, evt.Jdahep[i][1]+1,
			// <---
			evt.Phep[i][0], evt.Phep[i][1], evt.Phep[i][2], evt.Phep[i][3],
			evt.Phep[i][4],
			evt.Vhep[i][0], evt.Vhep[i][1], evt.Vhep[i][2], evt.Vhep[i][3],
		)
		if err != nil {
			return errors.WithStack(err)
		}
	}
	return errors.WithStack(err)
}

type AsciiDecoder struct {
	r io.Reader
}

func NewAsciiDecoder(r io.Reader) *AsciiDecoder {
	return &AsciiDecoder{r: r}
}

func (dec *AsciiDecoder) Decode(evt *Event) error {
	var err error

	_, err = fmt.Fscanf(dec.r, "%d %d\n", &evt.Nevhep, &evt.Nhep)
	if err != nil {
		return errors.WithStack(err)
	}

	// resize
	if len(evt.Isthep) > evt.Nhep {
		evt.Isthep = evt.Isthep[:evt.Nhep]
		evt.Idhep = evt.Idhep[:evt.Nhep]
		evt.Jmohep = evt.Jmohep[:evt.Nhep]
		evt.Jdahep = evt.Jdahep[:evt.Nhep]
		evt.Phep = evt.Phep[:evt.Nhep]
		evt.Vhep = evt.Vhep[:evt.Nhep]
	} else {
		sz := evt.Nhep - len(evt.Isthep)
		evt.Isthep = append(evt.Isthep, make([]int, sz)...)
		evt.Idhep = append(evt.Idhep, make([]int, sz)...)
		evt.Jmohep = append(evt.Jmohep, make([][2]int, sz)...)
		evt.Jdahep = append(evt.Jdahep, make([][2]int, sz)...)
		evt.Phep = append(evt.Phep, make([][5]float64, sz)...)
		evt.Vhep = append(evt.Vhep, make([][4]float64, sz)...)
	}

	for i := 0; i < evt.Nhep; i++ {
		_, err = fmt.Fscanf(
			dec.r,
			"%d %d %d %d %d %d %E %E %E %E %E %E %E %E %E\n",
			&evt.Isthep[i],
			&evt.Idhep[i],
			&evt.Jmohep[i][0], &evt.Jmohep[i][1],
			&evt.Jdahep[i][0], &evt.Jdahep[i][1],
			&evt.Phep[i][0], &evt.Phep[i][1], &evt.Phep[i][2], &evt.Phep[i][3],
			&evt.Phep[i][4],
			&evt.Vhep[i][0], &evt.Vhep[i][1], &evt.Vhep[i][2], &evt.Vhep[i][3],
		)
		if err != nil {
			return errors.WithStack(err)
		}
		// convert 0-based indices to 1-based ones
		evt.Jmohep[i][0] -= 1
		evt.Jmohep[i][1] -= 1
		evt.Jdahep[i][0] -= 1
		evt.Jdahep[i][1] -= 1
	}
	return errors.WithStack(err)
}
<|endoftext|>


<commit_before>package bsw

type Result struct {
	Source   string `json:"src"`
	IP       string `json:"ip"`
	Hostname string `json:"hostname"`
}
<commit_msg>Implements sorting<commit_after>package bsw

import (
	"net"
	"encoding/binary"
)

type Result struct {
	Source   string `json:"src"`
	IP       string `json:"ip"`
	Hostname string `json:"hostname"`
}

type Results []Result

func (r Results) Len() int { return len(r) }
func (r Results) Swap(i, j int) { r[i], r[j] = r[j], r[i] }

// Sorts by IPv4 address, IPv6 addresses will be show first and will be unsorted.
func (r Results) Less(i, j int) bool {
	first := net.ParseIP(r[i].IP).To4()
	second := net.ParseIP(r[j].IP).To4()
	if first == nil {
		return true
	}
	if second == nil {
		return false
	}
	return binary.BigEndian.Uint32(first) < binary.BigEndian.Uint32(second)
}
<|endoftext|>


<commit_before>package deduplicate

import (
	"encoding/json"
	// "fmt"
	"github.com/henrylee2cn/pholcus/common/mgo"
	"github.com/henrylee2cn/pholcus/common/mysql"
	"github.com/henrylee2cn/pholcus/common/util"
	"github.com/henrylee2cn/pholcus/config"
	"github.com/henrylee2cn/pholcus/logs"
	"gopkg.in/mgo.v2/bson"
	"io/ioutil"
	"os"
	"path"
	"sync"
)

type Deduplicate interface {
	// 采集非重复样本并返回对比结果，重复为true
	Compare(obj interface{}) bool
	// 提交去重记录至指定输出方式(数据库或文件)
	Submit(provider ...string)
	// 从指定输出方式(数据库或文件)读取更新去重记录
	Update(provider ...string)
	// 取消指定去重样本
	Remove(obj interface{})
	// 清空样本缓存
	CleanCache()
}

type Deduplication struct {
	sampling map[string]bool
	provider string
	sync.Mutex
}

func New() Deduplicate {
	return &Deduplication{
		sampling: make(map[string]bool),
	}
}

// 对比是否已存在，不存在则采样
func (self *Deduplication) Compare(obj interface{}) bool {
	self.Mutex.Lock()
	defer self.Unlock()

	s := util.MakeUnique(obj)
	if !self.sampling[s] {
		self.sampling[s] = true
		return false
	}
	return true
}

// 取消指定去重样本
func (self *Deduplication) Remove(obj interface{}) {
	self.Mutex.Lock()
	defer self.Unlock()

	s := util.MakeUnique(obj)
	if self.sampling[s] {
		delete(self.sampling, s)
	}
}

func (self *Deduplication) Submit(provider ...string) {
	if len(provider) > 0 && provider[0] != "" {
		self.provider = provider[0]
	}
	if len(self.sampling) == 0 {
		return
	}
	switch self.provider {
	case "mgo":
		var docs = make([]map[string]interface{}, len(self.sampling))
		var i int
		for key := range self.sampling {
			docs[i] = map[string]interface{}{"_id": key}
			i++
		}
		// fmt.Println(docs)
		mgo.Mgo(nil, "insert", map[string]interface{}{
			"Database":   config.MGO.DB,
			"Collection": config.DEDUPLICATION.FILE_NAME,
			"Docs":       docs,
		})

	case "mysql":
		db, ok := mysql.MysqlPool.GetOne().(*mysql.MysqlSrc)
		if !ok || db == nil {
			logs.Log.Error("链接Mysql数据库超时，无法保存去重记录！")
			return
		}
		defer mysql.MysqlPool.Free(db)
		table := mysql.New(db.DB).
			SetTableName(config.DEDUPLICATION.FILE_NAME).
			CustomPrimaryKey(`id VARCHAR(255) not null primary key`).
			Create()
		for key := range self.sampling {
			table.AddRow(key).Update()
		}

	default:
		p, _ := path.Split(config.COMM_PATH.CACHE + "/" + config.DEDUPLICATION.FILE_NAME)
		// 创建/打开目录
		d, err := os.Stat(p)
		if err != nil || !d.IsDir() {
			if err := os.MkdirAll(p, 0777); err != nil {
				logs.Log.Error("Error: %v\n", err)
			}
		}

		// 创建并写入文件
		f, _ := os.Create(config.COMM_PATH.CACHE + "/" + config.DEDUPLICATION.FILE_NAME)
		b, _ := json.Marshal(self.sampling)
		f.Write(b)
		f.Close()
	}
	// fmt.Printf(" *     新增 %v 条去重样本\n", len(self.sampling))
}

func (self *Deduplication) Update(provider ...string) {
	if len(provider) == 0 || provider[0] == self.provider {
		return
	}
	self.CleanCache()
	self.provider = provider[0]

	switch self.provider {
	case "mgo":
		var docs = map[string]interface{}{}
		err := mgo.Mgo(&docs, "find", map[string]interface{}{
			"Database":   config.MGO.DB,
			"Collection": config.DEDUPLICATION.FILE_NAME,
		})
		if err != nil {
			logs.Log.Error("去重读取mgo: %v", err)
			return
		}
		for _, v := range docs["Docs"].([]interface{}) {
			self.sampling[v.(bson.M)["_id"].(string)] = true
		}

	case "mysql":
		db, ok := mysql.MysqlPool.GetOne().(*mysql.MysqlSrc)
		if !ok || db == nil {
			logs.Log.Error("链接Mysql数据库超时，无法读取去重记录！")
			return
		}
		defer mysql.MysqlPool.Free(db)
		rows, err := mysql.New(db.DB).
			SetTableName("`" + config.DEDUPLICATION.FILE_NAME + "`").
			SelectAll()
		if err != nil {
			// logs.Log.Error("读取Mysql数据库中去重记录失败：%v", err)
			return
		}

		for rows.Next() {
			var id string
			err = rows.Scan(&id)
			self.sampling[id] = true
		}

	default:
		f, err := os.Open(config.COMM_PATH.CACHE + "/" + config.DEDUPLICATION.FILE_NAME)
		if err != nil {
			return
		}
		defer f.Close()
		b, _ := ioutil.ReadAll(f)
		json.Unmarshal(b, &self.sampling)
	}
	// fmt.Printf(" *     读出 %v 条去重样本\n", len(self.sampling))
}

func (self *Deduplication) CleanCache() {
	self.sampling = make(map[string]bool)
}
<commit_msg>提升去重性能<commit_after>package deduplicate

import (
	"encoding/json"
	"github.com/henrylee2cn/pholcus/common/mgo"
	"github.com/henrylee2cn/pholcus/common/mysql"
	"github.com/henrylee2cn/pholcus/common/util"
	"github.com/henrylee2cn/pholcus/config"
	"github.com/henrylee2cn/pholcus/logs"
	"gopkg.in/mgo.v2/bson"
	"io/ioutil"
	"os"
	"path"
	"sync"
)

type Deduplicate interface {
	// 采集非重复样本并返回对比结果，重复为true
	Compare(obj interface{}) bool
	// 提交去重记录至指定输出方式(数据库或文件)
	Submit(provider ...string)
	// 从指定输出方式(数据库或文件)读取更新去重记录
	Update(provider ...string)
	// 取消指定去重样本
	Remove(obj interface{})
	// 清空样本缓存
	CleanCache()
}

type Deduplication struct {
	sampling struct {
		new map[string]bool
		old map[string]bool
	}
	provider string
	sync.Mutex
}

func New() Deduplicate {
	return &Deduplication{
		sampling: struct {
			new map[string]bool
			old map[string]bool
		}{
			new: make(map[string]bool),
			old: make(map[string]bool),
		},
	}
}

// 对比是否已存在，不存在则采样
func (self *Deduplication) Compare(obj interface{}) (duplicate bool) {
	self.Mutex.Lock()
	defer self.Unlock()

	s := util.MakeUnique(obj)
	if self.sampling.old[s] {
		return true
	}
	if self.sampling.new[s] {
		return true
	}
	self.sampling.new[s] = true
	return false
}

// 取消指定去重样本
func (self *Deduplication) Remove(obj interface{}) {
	self.Mutex.Lock()
	defer self.Unlock()

	s := util.MakeUnique(obj)
	if self.sampling.new[s] {
		delete(self.sampling.new, s)
	}
}

func (self *Deduplication) Submit(provider ...string) {
	if len(provider) > 0 && provider[0] != "" {
		self.provider = provider[0]
	}
	if len(self.sampling.new) == 0 {
		return
	}

	self.Mutex.Lock()
	defer self.Unlock()

	switch self.provider {
	case "mgo":
		var docs = make([]map[string]interface{}, len(self.sampling.new))
		var i int
		for key := range self.sampling.new {
			docs[i] = map[string]interface{}{"_id": key}
			self.sampling.old[key] = true
			i++
		}
		mgo.Mgo(nil, "insert", map[string]interface{}{
			"Database":   config.MGO.DB,
			"Collection": config.DEDUPLICATION.FILE_NAME,
			"Docs":       docs,
		})

	case "mysql":
		db, ok := mysql.MysqlPool.GetOne().(*mysql.MysqlSrc)
		if !ok || db == nil {
			logs.Log.Error("链接Mysql数据库超时，无法保存去重记录！")
			return
		}
		defer mysql.MysqlPool.Free(db)
		table := mysql.New(db.DB).
			SetTableName(config.DEDUPLICATION.FILE_NAME).
			CustomPrimaryKey(`id VARCHAR(255) not null primary key`).
			Create()
		for key := range self.sampling.new {
			table.AddRow(key).Update()
			self.sampling.old[key] = true
		}

	default:
		p, _ := path.Split(config.COMM_PATH.CACHE + "/" + config.DEDUPLICATION.FILE_NAME)
		// 创建/打开目录
		d, err := os.Stat(p)
		if err != nil || !d.IsDir() {
			if err := os.MkdirAll(p, 0777); err != nil {
				logs.Log.Error("Error: %v\n", err)
			}
		}

		f, _ := os.OpenFile(config.COMM_PATH.CACHE+"/"+config.DEDUPLICATION.FILE_NAME, os.O_CREATE|os.O_APPEND|os.O_RDWR, 0660)

		b, _ := json.Marshal(self.sampling.new)
		b[0] = ','
		f.Write(b[:len(b)-1])
		f.Close()

		for key := range self.sampling.new {
			self.sampling.old[key] = true
		}
	}
	logs.Log.Informational(" *     新增 %v 条去重样本\n", len(self.sampling.new))
	self.sampling.new = make(map[string]bool)
}

func (self *Deduplication) Update(provider ...string) {
	if len(provider) == 0 || provider[0] == self.provider {
		return
	}
	self.CleanCache()
	self.provider = provider[0]

	switch self.provider {
	case "mgo":
		var docs = map[string]interface{}{}
		err := mgo.Mgo(&docs, "find", map[string]interface{}{
			"Database":   config.MGO.DB,
			"Collection": config.DEDUPLICATION.FILE_NAME,
		})
		if err != nil {
			logs.Log.Error("去重读取mgo: %v", err)
			return
		}
		for _, v := range docs["Docs"].([]interface{}) {
			self.sampling.old[v.(bson.M)["_id"].(string)] = true
		}

	case "mysql":
		db, ok := mysql.MysqlPool.GetOne().(*mysql.MysqlSrc)
		if !ok || db == nil {
			logs.Log.Error("链接Mysql数据库超时，无法读取去重记录！")
			return
		}
		defer mysql.MysqlPool.Free(db)
		rows, err := mysql.New(db.DB).
			SetTableName("`" + config.DEDUPLICATION.FILE_NAME + "`").
			SelectAll()
		if err != nil {
			// logs.Log.Error("读取Mysql数据库中去重记录失败：%v", err)
			return
		}

		for rows.Next() {
			var id string
			err = rows.Scan(&id)
			self.sampling.old[id] = true
		}

	default:
		f, err := os.Open(config.COMM_PATH.CACHE + "/" + config.DEDUPLICATION.FILE_NAME)
		if err != nil {
			return
		}
		defer f.Close()
		b, _ := ioutil.ReadAll(f)
		b[0] = '{'
		json.Unmarshal(
			append(b, '}'),
			&self.sampling.old,
		)
	}
	logs.Log.Informational(" *     读出 %v 条去重样本\n", len(self.sampling.old))
}

func (self *Deduplication) CleanCache() {
	self.Mutex.Lock()
	defer self.Unlock()
	self.sampling.old = make(map[string]bool)
	self.sampling.new = make(map[string]bool)
}
<|endoftext|>


<commit_before>package backend

import (
	"fmt"
	"net/http"
)

func init() {
	http.HandleFunc("/api/helloword", helloworld)
}

func helloworld(w http.ResponseWriter, r *http.Request) {
	fmt.Fprint(w, "hello world")
}<commit_msg>implements part of the fetching of comments<commit_after>package backend

import (
"fmt"
"net/http"
"encoding/json"

"appengine"
"appengine/urlfetch"
)

//struct of reddit json api
//Respons is a slice because for some reason, the reddit api starts as an array
type Response []struct {
	Parent struct {
		Data struct {
			Children []struct {
				Com Comment
			}
		}
	}
}
//struct of comment
type Comment struct {
	Name string `json:"body"`
}

func init() {
	http.HandleFunc("/api/getcomments", getComments)
}

func getComments(w http.ResponseWriter, r *http.Request) {

	url := "https://www.reddit.com/r/videos/comments/3vgdsb/recruitment_2016.json"
	//starting newContext and getting data from the url
	c := appengine.NewContext(r)
	client := urlfetch.Client(c)
	resp, err := client.Get(url)
	if err != nil { fmt.Fprint(w, "Error client.Get(): ", err) }
	//creating new response object
	re := new(Response)
	fmt.Fprint(w, "Response struct: ", re, "\n")
	//decoding the data to json into our Response array
	errTwo := json.NewDecoder(resp.Body).Decode(&re)
	if errTwo != nil {  fmt.Fprint(w, "Error decoding: ", errTwo, "\n") }

	fmt.Fprint(w, "Response struct: ", re)
}

<|endoftext|>


<commit_before>package bufit

import (
	"bytes"
	"crypto/rand"
	"io"
	"io/ioutil"
	"os"
	"sync"
	"testing"
	"time"
)

func ExampleWriter() {
	buf := newWriter(make([]byte, 0, 10))
	io.Copy(os.Stdout, buf)
	io.Copy(os.Stdout, io.NewSectionReader(*&buf, 0, 100))

	io.WriteString(buf, "Hello ")
	r := io.NewSectionReader(*&buf, 0, int64(buf.Len()))
	io.CopyN(os.Stdout, r, 5)
	io.CopyN(os.Stdout, buf, 5)
	io.WriteString(buf, "World")
	r = io.NewSectionReader(*&buf, 0, int64(buf.Len()))
	io.CopyN(os.Stdout, r, 6)

	io.WriteString(buf, "abcdefg")
	io.Copy(os.Stdout, buf)
	io.Copy(os.Stdout, buf)

	io.WriteString(buf, "Hello World")
	r = io.NewSectionReader(*&buf, 0, int64(buf.Len()))
	io.CopyN(os.Stdout, r, 5)
	io.CopyN(os.Stdout, buf, 4)

	io.WriteString(buf, "abcdefg")
	io.Copy(os.Stdout, buf)
	io.Copy(os.Stdout, buf)
	//Output:
	// HelloHello World WorldabcdefgHelloHello Worldabcdefg
}

type badBuffer []byte

func (b *badBuffer) Write(p []byte) (int, error) {
	*b = append(*b, p...)
	return len(p), nil
}

func (b *badBuffer) Read(p []byte) (n int, err error) {
	n = copy(p, *b)
	*b = (*b)[n:]
	if len(*b) == 0 {
		err = io.EOF
	}
	return n, err
}

func BenchmarkBufferNoPressure(b *testing.B) {
	data, _ := ioutil.ReadAll(io.LimitReader(rand.Reader, 32*1024))

	buf := NewCapped(len(data))

	go func() {
		for i := 0; i < 1000; i++ {
			buf.Write(data)
		}
		buf.Close()
	}()

	b.RunParallel(func(pb *testing.PB) {
		for pb.Next() {
			rr := buf.NextReader()
			io.Copy(ioutil.Discard, rr)
			rr.Close()
		}
	})

	b.ReportAllocs()
}

func BenchmarkBuffer(b *testing.B) {
	buf := New()
	data, _ := ioutil.ReadAll(io.LimitReader(rand.Reader, 32*1024))

	go func() {
		for i := 0; i < 1000; i++ {
			buf.Write(data)
		}
		buf.Close()
	}()

	r := buf.NextReader()

	b.RunParallel(func(pb *testing.PB) {
		for pb.Next() {
			rr := buf.NextReader()
			io.Copy(ioutil.Discard, rr)
			rr.Close()
		}
	})

	r.Close()
	b.ReportAllocs()
}

func BenchmarkReadWriter(b *testing.B) {
	buf := newWriter(nil)
	data, _ := ioutil.ReadAll(io.LimitReader(rand.Reader, 32*1024))
	temp := make([]byte, 32*1024)
	for i := 0; i < b.N; i++ {
		buf.Write(data)
		io.CopyBuffer(ioutil.Discard, buf, temp)
	}
	b.ReportAllocs()
}

func TestCappedBuffer(t *testing.T) {
	data := []byte("Hello World")
	buf := NewCapped(5)

	go func() {
		for i := 0; i < 5; i++ {
			n, err := buf.Write(data)
			l := buf.Len()
			if err != nil {
				t.Errorf("expected no error, got %s", err)
			}
			if n != len(data) {
				t.Errorf("expected %d bytes to be written, got %d", len(data), n)
			}
			if l > 5 {
				t.Errorf("length shouldn't exceed cap, but got %d", l)
			}

		}
		buf.Close()
	}()

	r := buf.NextReader()
	res, err := ioutil.ReadAll(r)
	if err != nil {
		t.Error(err)
	}

	if !bytes.Equal(res, bytes.Repeat(data, 5)) {
		t.Errorf("expected %s got %s", string(bytes.Repeat(data, 5)), string(res))
	}
}

func TestReadReaderAfterClose(t *testing.T) {
	buf := New()
	r := buf.NextReader()
	r.Close()
	p := make([]byte, 10)
	n, err := r.Read(p)
	if err != io.EOF {
		t.Errorf("expected io.EOF got %v, %v", n, err)
	}
}

func TestCloseReaderTwice(t *testing.T) {
	buf := New()
	r := buf.NextReader()
	r.Close()
	r.Close()
}

func TestNextReaderFromNowOnlySeesNewWrites(t *testing.T) {
	data := []byte("hello world")
	buf := New()
	r := buf.NextReader()

	buf.Write(data)
	if buf.Len() != len(data) {
		t.Errorf("expected len to be %d but got %d", len(data), buf.Len())
	}

	r2 := buf.NextReaderFromNow()

	buf.Write(data)
	buf.Close()

	out, err := ioutil.ReadAll(r2)
	if err != nil {
		t.Errorf("expected no error, got %s", err)
	}

	if !bytes.Equal(out, data) {
		t.Errorf("expected %s, got %s", data, out)
	}

	if buf.Len() != len(data)*2 {
		t.Errorf("expected len to be %d but got %d", len(data)*2, buf.Len())
	}

	r.Close()

	if buf.Len() != 0 {
		t.Errorf("expected len to be %d but got %d", 0, buf.Len())
	}
}

func TestDontPanicWhenLastReaderDropsOnNonEmptyBuffer(t *testing.T) {
	data := []byte("hello world")
	buf := New()
	r := buf.NextReader()

	buf.Write(data)
	if buf.Len() != len(data) {
		t.Errorf("expected len to be %d but got %d", len(data), buf.Len())
	}

	buf.Close()
	r.Close()
}

func TestNextReaderKeepNewWrites(t *testing.T) {
	data := []byte("123456789a")
	buf := NewCapped(10)
	keep := 5
	expect := data[len(data)-keep:]
	buf.Keep(keep)
	buf.Keep(11) // Ignore, bigger than cap
	buf.Keep(-1) // Ignore, not positive

	buf.Write(data)
	if buf.Len() != len(data) {
		t.Errorf("expected len to be %d but got %d", len(data), buf.Len())
	}
	buf.Close()

	r := buf.NextReader()
	io.Copy(ioutil.Discard, r)

	if buf.Len() != len(expect) {
		t.Errorf("expected len to be %d but got %d", len(expect), buf.Len())
	}

	r.Close()

	for i := 0; i < 5; i++ {
		r2 := buf.NextReader()
		out, err := ioutil.ReadAll(r2)
		if err != nil {
			t.Errorf("expected no error, got %s", err)
		}

		if !bytes.Equal(out, expect) {
			t.Errorf("expected %s, got %s", expect, out)
		}

		if buf.Len() != len(expect) {
			t.Errorf("expected len to be %d but got %d", len(expect), buf.Len())
		}

		r2.Close()

		if buf.Len() != len(expect) {
			t.Errorf("expected len to be %d but got %d", len(expect), buf.Len())
		}
	}
}

func TestReaderClosesWriter(t *testing.T) {
	buf := NewCapped(1)
	defer buf.Close()

	wait := make(chan struct{})

	go func() {
		// make sure blocking writes get canceled
		if _, err := io.WriteString(buf, "hello"); err != io.ErrClosedPipe {
			t.Errorf("expected %s got %s", io.ErrClosedPipe, err)
		}
		close(wait)
	}()

	buf.OnLastReaderClose(buf.Close)
	buf.NextReader().Close() // should close the blocking write

	select {
	case <-wait:
	case <-time.After(100 * time.Millisecond):
		t.Errorf("write didn't break in time")
	}
}

func assertNumReaders(n int, buf *Buffer, t *testing.T) {
	if v := buf.NumReaders(); v != n {
		t.Errorf("expected %d readers, found %d", n, v)
	}
}

func TestCloseCallback(t *testing.T) {
	called := false
	buf := New()
	buf.OnLastReaderClose(func() error {
		called = true
		return nil
	})

	assertNumReaders(0, buf, t)

	lr := buf.NextReader()
	assertNumReaders(1, buf, t)
	lr.Close()
	assertNumReaders(0, buf, t)
	if !called {
		t.Errorf("expected callback to be called")
	}

	lr = buf.NextReader()
	assertNumReaders(1, buf, t)
	called = false

	for i := 0; i < 10; i++ {
		r := buf.NextReader()
		assertNumReaders(2, buf, t)
		r.Close()
		assertNumReaders(1, buf, t)
		if called {
			t.Errorf("expected callback to not be called")
		}
	}

	assertNumReaders(1, buf, t)
	lr.Close()
	assertNumReaders(0, buf, t)
	if !called {
		t.Errorf("expected callback to be called")
	}
}

func TestConcurrent(t *testing.T) {
	var grp sync.WaitGroup
	buf := New()

	var rs []io.ReadCloser
	for i := 0; i < 1000; i++ {
		rs = append(rs, buf.NextReader())
	}

	testData := bytes.NewBuffer(nil)
	io.CopyN(testData, rand.Reader, 32*1024*10)

	for _, r := range rs {
		grp.Add(1)
		go func(r io.ReadCloser) {
			defer grp.Done()
			defer r.Close()
			data, err := ioutil.ReadAll(r)
			if err != nil {
				t.Error(err)
			}
			if !bytes.Equal(testData.Bytes(), data) {
				t.Error("unexpected result...", testData.Len(), len(data))
			}
		}(r)
	}

	r := bytes.NewReader(testData.Bytes())
	for r.Len() > 0 {
		io.CopyN(buf, r, 32*1024*2)
		<-time.After(100 * time.Millisecond)
	}
	buf.Close()
	grp.Wait()
}

func TestQuitReader(t *testing.T) {
	buf := New()
	r := buf.NextReader()

	wait := make(chan struct{})
	go func() {
		io.Copy(ioutil.Discard, r)
		close(wait)
	}()

	r.Close()
	select {
	case <-wait:
	case <-time.After(100 * time.Millisecond):
		t.Error("timed out waiting for Reader to Close")
	}
}

func TestQuitWriter(t *testing.T) {
	buf := New()
	buf.Close()
	_, err := io.WriteString(buf, ".")
	if err != io.ErrClosedPipe {
		t.Errorf("Writer after Close expected io.ErrClosedPipe but got %v", err)
	}
}

func TestQuitCappedWriter(t *testing.T) {
	buf := NewCapped(2)
	go func() {
		// wait until blocking write has started
		<-time.After(100 * time.Millisecond)
		buf.Close()
	}()
	_, err := io.WriteString(buf, "hello world")
	if err != io.ErrClosedPipe {
		t.Errorf("Writer after Close expected io.ErrClosedPipe but got %v", err)
	}
}

func ExampleBuffer() {
	// Start a new buffer
	buf := New()

	// Create two readers
	r1, r2 := buf.NextReader(), buf.NextReader()

	// Broadcast a message
	io.WriteString(buf, "Hello World\n")

	// Wait
	var grp sync.WaitGroup
	grp.Add(4)

	// Read fast
	go func() {
		defer grp.Done()
		io.Copy(os.Stdout, r1) // "Hello World\n"
	}()

	// Read slow
	go func() {
		defer grp.Done()
		<-time.After(100 * time.Millisecond)
		io.CopyN(os.Stdout, r2, 5) // "Hello"
		<-time.After(time.Second)
		io.Copy(os.Stdout, r2) // "World\n"
	}()

	// Both readers will read the entire buffer! The slow reader
	// won't block the fast one from reading ahead either.

	// Late reader
	// Since this reader joins after all existing readers have Read "Hello"
	// "Hello" has already been cleared from the Buffer, this Reader will only see
	// "World\n" and beyond.
	go func() {
		defer grp.Done()
		<-time.After(500 * time.Millisecond)
		r3 := buf.NextReader()
		io.Copy(os.Stdout, r3) // "World\n"
	}()

	// Short Reader
	// **Important!** if your reader isn't going to read until the buffer is empty
	// you'll need to call Close() when you are done with it to tell the buffer
	// it's done reading data.
	go func() {
		defer grp.Done()
		<-time.After(100 * time.Millisecond)
		r4 := buf.NextReader()
		io.CopyN(os.Stdout, r4, 5) // "Hello"
		r4.Close()                 // tell the buffer you're done reading
	}()

	// **Important!** mark close so that readers can ret. io.EOF
	buf.Close()

	grp.Wait()
	// Output:
	// Hello World
	// HelloHelloHello World
	//  World
}
<commit_msg>Remove unused badBuffer and *&<commit_after>package bufit

import (
	"bytes"
	"crypto/rand"
	"io"
	"io/ioutil"
	"os"
	"sync"
	"testing"
	"time"
)

func ExampleWriter() {
	buf := newWriter(make([]byte, 0, 10))
	io.Copy(os.Stdout, buf)
	io.Copy(os.Stdout, io.NewSectionReader(buf, 0, 100))

	io.WriteString(buf, "Hello ")
	r := io.NewSectionReader(buf, 0, int64(buf.Len()))
	io.CopyN(os.Stdout, r, 5)
	io.CopyN(os.Stdout, buf, 5)
	io.WriteString(buf, "World")
	r = io.NewSectionReader(buf, 0, int64(buf.Len()))
	io.CopyN(os.Stdout, r, 6)

	io.WriteString(buf, "abcdefg")
	io.Copy(os.Stdout, buf)
	io.Copy(os.Stdout, buf)

	io.WriteString(buf, "Hello World")
	r = io.NewSectionReader(buf, 0, int64(buf.Len()))
	io.CopyN(os.Stdout, r, 5)
	io.CopyN(os.Stdout, buf, 4)

	io.WriteString(buf, "abcdefg")
	io.Copy(os.Stdout, buf)
	io.Copy(os.Stdout, buf)
	//Output:
	// HelloHello World WorldabcdefgHelloHello Worldabcdefg
}

func BenchmarkBufferNoPressure(b *testing.B) {
	data, _ := ioutil.ReadAll(io.LimitReader(rand.Reader, 32*1024))

	buf := NewCapped(len(data))

	go func() {
		for i := 0; i < 1000; i++ {
			buf.Write(data)
		}
		buf.Close()
	}()

	b.RunParallel(func(pb *testing.PB) {
		for pb.Next() {
			rr := buf.NextReader()
			io.Copy(ioutil.Discard, rr)
			rr.Close()
		}
	})

	b.ReportAllocs()
}

func BenchmarkBuffer(b *testing.B) {
	buf := New()
	data, _ := ioutil.ReadAll(io.LimitReader(rand.Reader, 32*1024))

	go func() {
		for i := 0; i < 1000; i++ {
			buf.Write(data)
		}
		buf.Close()
	}()

	r := buf.NextReader()

	b.RunParallel(func(pb *testing.PB) {
		for pb.Next() {
			rr := buf.NextReader()
			io.Copy(ioutil.Discard, rr)
			rr.Close()
		}
	})

	r.Close()
	b.ReportAllocs()
}

func BenchmarkReadWriter(b *testing.B) {
	buf := newWriter(nil)
	data, _ := ioutil.ReadAll(io.LimitReader(rand.Reader, 32*1024))
	temp := make([]byte, 32*1024)
	for i := 0; i < b.N; i++ {
		buf.Write(data)
		io.CopyBuffer(ioutil.Discard, buf, temp)
	}
	b.ReportAllocs()
}

func TestCappedBuffer(t *testing.T) {
	data := []byte("Hello World")
	buf := NewCapped(5)

	go func() {
		for i := 0; i < 5; i++ {
			n, err := buf.Write(data)
			l := buf.Len()
			if err != nil {
				t.Errorf("expected no error, got %s", err)
			}
			if n != len(data) {
				t.Errorf("expected %d bytes to be written, got %d", len(data), n)
			}
			if l > 5 {
				t.Errorf("length shouldn't exceed cap, but got %d", l)
			}

		}
		buf.Close()
	}()

	r := buf.NextReader()
	res, err := ioutil.ReadAll(r)
	if err != nil {
		t.Error(err)
	}

	if !bytes.Equal(res, bytes.Repeat(data, 5)) {
		t.Errorf("expected %s got %s", string(bytes.Repeat(data, 5)), string(res))
	}
}

func TestReadReaderAfterClose(t *testing.T) {
	buf := New()
	r := buf.NextReader()
	r.Close()
	p := make([]byte, 10)
	n, err := r.Read(p)
	if err != io.EOF {
		t.Errorf("expected io.EOF got %v, %v", n, err)
	}
}

func TestCloseReaderTwice(t *testing.T) {
	buf := New()
	r := buf.NextReader()
	r.Close()
	r.Close()
}

func TestNextReaderFromNowOnlySeesNewWrites(t *testing.T) {
	data := []byte("hello world")
	buf := New()
	r := buf.NextReader()

	buf.Write(data)
	if buf.Len() != len(data) {
		t.Errorf("expected len to be %d but got %d", len(data), buf.Len())
	}

	r2 := buf.NextReaderFromNow()

	buf.Write(data)
	buf.Close()

	out, err := ioutil.ReadAll(r2)
	if err != nil {
		t.Errorf("expected no error, got %s", err)
	}

	if !bytes.Equal(out, data) {
		t.Errorf("expected %s, got %s", data, out)
	}

	if buf.Len() != len(data)*2 {
		t.Errorf("expected len to be %d but got %d", len(data)*2, buf.Len())
	}

	r.Close()

	if buf.Len() != 0 {
		t.Errorf("expected len to be %d but got %d", 0, buf.Len())
	}
}

func TestDontPanicWhenLastReaderDropsOnNonEmptyBuffer(t *testing.T) {
	data := []byte("hello world")
	buf := New()
	r := buf.NextReader()

	buf.Write(data)
	if buf.Len() != len(data) {
		t.Errorf("expected len to be %d but got %d", len(data), buf.Len())
	}

	buf.Close()
	r.Close()
}

func TestNextReaderKeepNewWrites(t *testing.T) {
	data := []byte("123456789a")
	buf := NewCapped(10)
	keep := 5
	expect := data[len(data)-keep:]
	buf.Keep(keep)
	buf.Keep(11) // Ignore, bigger than cap
	buf.Keep(-1) // Ignore, not positive

	buf.Write(data)
	if buf.Len() != len(data) {
		t.Errorf("expected len to be %d but got %d", len(data), buf.Len())
	}
	buf.Close()

	r := buf.NextReader()
	io.Copy(ioutil.Discard, r)

	if buf.Len() != len(expect) {
		t.Errorf("expected len to be %d but got %d", len(expect), buf.Len())
	}

	r.Close()

	for i := 0; i < 5; i++ {
		r2 := buf.NextReader()
		out, err := ioutil.ReadAll(r2)
		if err != nil {
			t.Errorf("expected no error, got %s", err)
		}

		if !bytes.Equal(out, expect) {
			t.Errorf("expected %s, got %s", expect, out)
		}

		if buf.Len() != len(expect) {
			t.Errorf("expected len to be %d but got %d", len(expect), buf.Len())
		}

		r2.Close()

		if buf.Len() != len(expect) {
			t.Errorf("expected len to be %d but got %d", len(expect), buf.Len())
		}
	}
}

func TestReaderClosesWriter(t *testing.T) {
	buf := NewCapped(1)
	defer buf.Close()

	wait := make(chan struct{})

	go func() {
		// make sure blocking writes get canceled
		if _, err := io.WriteString(buf, "hello"); err != io.ErrClosedPipe {
			t.Errorf("expected %s got %s", io.ErrClosedPipe, err)
		}
		close(wait)
	}()

	buf.OnLastReaderClose(buf.Close)
	buf.NextReader().Close() // should close the blocking write

	select {
	case <-wait:
	case <-time.After(100 * time.Millisecond):
		t.Errorf("write didn't break in time")
	}
}

func assertNumReaders(n int, buf *Buffer, t *testing.T) {
	if v := buf.NumReaders(); v != n {
		t.Errorf("expected %d readers, found %d", n, v)
	}
}

func TestCloseCallback(t *testing.T) {
	called := false
	buf := New()
	buf.OnLastReaderClose(func() error {
		called = true
		return nil
	})

	assertNumReaders(0, buf, t)

	lr := buf.NextReader()
	assertNumReaders(1, buf, t)
	lr.Close()
	assertNumReaders(0, buf, t)
	if !called {
		t.Errorf("expected callback to be called")
	}

	lr = buf.NextReader()
	assertNumReaders(1, buf, t)
	called = false

	for i := 0; i < 10; i++ {
		r := buf.NextReader()
		assertNumReaders(2, buf, t)
		r.Close()
		assertNumReaders(1, buf, t)
		if called {
			t.Errorf("expected callback to not be called")
		}
	}

	assertNumReaders(1, buf, t)
	lr.Close()
	assertNumReaders(0, buf, t)
	if !called {
		t.Errorf("expected callback to be called")
	}
}

func TestConcurrent(t *testing.T) {
	var grp sync.WaitGroup
	buf := New()

	var rs []io.ReadCloser
	for i := 0; i < 1000; i++ {
		rs = append(rs, buf.NextReader())
	}

	testData := bytes.NewBuffer(nil)
	io.CopyN(testData, rand.Reader, 32*1024*10)

	for _, r := range rs {
		grp.Add(1)
		go func(r io.ReadCloser) {
			defer grp.Done()
			defer r.Close()
			data, err := ioutil.ReadAll(r)
			if err != nil {
				t.Error(err)
			}
			if !bytes.Equal(testData.Bytes(), data) {
				t.Error("unexpected result...", testData.Len(), len(data))
			}
		}(r)
	}

	r := bytes.NewReader(testData.Bytes())
	for r.Len() > 0 {
		io.CopyN(buf, r, 32*1024*2)
		<-time.After(100 * time.Millisecond)
	}
	buf.Close()
	grp.Wait()
}

func TestQuitReader(t *testing.T) {
	buf := New()
	r := buf.NextReader()

	wait := make(chan struct{})
	go func() {
		io.Copy(ioutil.Discard, r)
		close(wait)
	}()

	r.Close()
	select {
	case <-wait:
	case <-time.After(100 * time.Millisecond):
		t.Error("timed out waiting for Reader to Close")
	}
}

func TestQuitWriter(t *testing.T) {
	buf := New()
	buf.Close()
	_, err := io.WriteString(buf, ".")
	if err != io.ErrClosedPipe {
		t.Errorf("Writer after Close expected io.ErrClosedPipe but got %v", err)
	}
}

func TestQuitCappedWriter(t *testing.T) {
	buf := NewCapped(2)
	go func() {
		// wait until blocking write has started
		<-time.After(100 * time.Millisecond)
		buf.Close()
	}()
	_, err := io.WriteString(buf, "hello world")
	if err != io.ErrClosedPipe {
		t.Errorf("Writer after Close expected io.ErrClosedPipe but got %v", err)
	}
}

func ExampleBuffer() {
	// Start a new buffer
	buf := New()

	// Create two readers
	r1, r2 := buf.NextReader(), buf.NextReader()

	// Broadcast a message
	io.WriteString(buf, "Hello World\n")

	// Wait
	var grp sync.WaitGroup
	grp.Add(4)

	// Read fast
	go func() {
		defer grp.Done()
		io.Copy(os.Stdout, r1) // "Hello World\n"
	}()

	// Read slow
	go func() {
		defer grp.Done()
		<-time.After(100 * time.Millisecond)
		io.CopyN(os.Stdout, r2, 5) // "Hello"
		<-time.After(time.Second)
		io.Copy(os.Stdout, r2) // "World\n"
	}()

	// Both readers will read the entire buffer! The slow reader
	// won't block the fast one from reading ahead either.

	// Late reader
	// Since this reader joins after all existing readers have Read "Hello"
	// "Hello" has already been cleared from the Buffer, this Reader will only see
	// "World\n" and beyond.
	go func() {
		defer grp.Done()
		<-time.After(500 * time.Millisecond)
		r3 := buf.NextReader()
		io.Copy(os.Stdout, r3) // "World\n"
	}()

	// Short Reader
	// **Important!** if your reader isn't going to read until the buffer is empty
	// you'll need to call Close() when you are done with it to tell the buffer
	// it's done reading data.
	go func() {
		defer grp.Done()
		<-time.After(100 * time.Millisecond)
		r4 := buf.NextReader()
		io.CopyN(os.Stdout, r4, 5) // "Hello"
		r4.Close()                 // tell the buffer you're done reading
	}()

	// **Important!** mark close so that readers can ret. io.EOF
	buf.Close()

	grp.Wait()
	// Output:
	// Hello World
	// HelloHelloHello World
	//  World
}
<|endoftext|>