<commit_before>package replication

import (
	"errors"
	"fmt"
	"gopkg.in/fsnotify.v1"
	"io"
	"net"
	"net/http"
	"os"
	"path"
	"time"

	"github.com/omniscale/imposm3"
	"github.com/omniscale/imposm3/logging"
)

var log = logging.NewLogger("replication")

var NotAvailable = errors.New("file not available")

type Sequence struct {
	Filename      string
	StateFilename string
	Time          time.Time
	Sequence      int
}

type Source interface {
	Sequences() <-chan Sequence
}

// N = AAA*1000000 + BBB*1000 + CCC
func seqPath(seq int) string {
	c := seq % 1000
	b := seq / 1000 % 1000
	a := seq / 1000000

	return fmt.Sprintf("%03d/%03d/%03d", a, b, c)
}

var _ Source = &downloader{}

type downloader struct {
	baseUrl      string
	dest         string
	fileExt      string
	stateExt     string
	lastSequence int
	stateTime    func(string) (time.Time, error)
	interval     time.Duration
	errWaittime  time.Duration
	naWaittime   time.Duration
	sequences    chan Sequence
	client       *http.Client
}

func newDownloader(dest, url string, seq int, interval time.Duration) *downloader {
	client := &http.Client{
		Transport: &http.Transport{
			Proxy: http.ProxyFromEnvironment,
			Dial: (&net.Dialer{
				Timeout:   30 * time.Second,
				KeepAlive: 1 * time.Second, // do not keep alive till next interval
			}).Dial,
			TLSHandshakeTimeout:   10 * time.Second,
			ResponseHeaderTimeout: 10 * time.Second,
			ExpectContinueTimeout: 1 * time.Second,
		},
	}

	dl := &downloader{
		baseUrl:      url,
		dest:         dest,
		lastSequence: seq,
		interval:     interval,
		errWaittime:  60 * time.Second,
		naWaittime:   10 * time.Second,
		sequences:    make(chan Sequence, 1),
		client:       client,
	}

	return dl
}

func (d *downloader) Sequences() <-chan Sequence {
	return d.sequences
}

func (d *downloader) download(seq int, ext string) error {
	dest := path.Join(d.dest, seqPath(seq)+ext)
	url := d.baseUrl + seqPath(seq) + ext

	if _, err := os.Stat(dest); err == nil {
		return nil
	}

	if err := os.MkdirAll(path.Dir(dest), 0755); err != nil {
		return err
	}

	tmpDest := fmt.Sprintf("%s~%d", dest, os.Getpid())
	out, err := os.Create(tmpDest)
	if err != nil {
		return err
	}
	defer out.Close()

	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return err
	}
	req.Header.Set("User-Agent", "Imposm3 "+imposm3.Version)
	resp, err := d.client.Do(req)
	if err != nil {
		return err
	}

	defer resp.Body.Close()

	if resp.StatusCode == 404 {
		return NotAvailable
	}

	if resp.StatusCode != 200 {
		return errors.New(fmt.Sprintf("invalid repsonse: %v", resp))
	}

	_, err = io.Copy(out, resp.Body)
	if err != nil {
		return err
	}
	out.Close()

	err = os.Rename(tmpDest, dest)
	if err != nil {
		return err
	}

	return nil
}

func (d *downloader) downloadTillSuccess(seq int, ext string) {
	for {
		err := d.download(seq, ext)
		if err == nil {
			break
		}
		if err == NotAvailable {
			time.Sleep(d.naWaittime)
		} else {
			log.Warn(err)
			time.Sleep(d.errWaittime)
		}
	}
}

func (d *downloader) fetchNextLoop() {
	stateFile := path.Join(d.dest, seqPath(d.lastSequence)+d.stateExt)
	lastTime, err := d.stateTime(stateFile)
	for {
		if err == nil {
			nextDiffTime := lastTime.Add(d.interval)
			if nextDiffTime.After(time.Now()) {
				// we catched up and the next diff file is in the future.
				// wait till last diff time + interval, before fetching next
				nextDiffTime = lastTime.Add(d.interval + 2*time.Second /* allow small time diff between server*/)
				waitFor := nextDiffTime.Sub(time.Now())
				time.Sleep(waitFor)
			}
		}
		nextSeq := d.lastSequence + 1
		// download will retry until they succeed
		d.downloadTillSuccess(nextSeq, d.stateExt)
		d.downloadTillSuccess(nextSeq, d.fileExt)
		d.lastSequence = nextSeq
		base := path.Join(d.dest, seqPath(d.lastSequence))
		lastTime, _ = d.stateTime(base + d.stateExt)
		d.sequences <- Sequence{
			Sequence:      d.lastSequence,
			Filename:      base + d.fileExt,
			StateFilename: base + d.stateExt,
			Time:          lastTime,
		}
	}
}

var _ Source = &reader{}

type reader struct {
	dest         string
	fileExt      string
	stateExt     string
	lastSequence int
	stateTime    func(string) (time.Time, error)
	sequences    chan Sequence
}

func newReader(dest string, seq int) *reader {
	r := &reader{
		dest:         dest,
		lastSequence: seq,
		sequences:    make(chan Sequence, 1),
	}

	return r
}

func (d *reader) Sequences() <-chan Sequence {
	return d.sequences
}

func (d *reader) waitTillPresent(seq int, ext string) error {
	filename := path.Join(d.dest, seqPath(seq)+ext)
	if _, err := os.Stat(filename); err == nil {
		return nil
	}

	w, err := fsnotify.NewWatcher()
	if err != nil {
		return err
	}
	defer w.Close()
	w.Add(filename)

	// check again, in case file was created before we added the file
	if _, err := os.Stat(filename); err == nil {
		return nil
	}

	for {
		select {
		case evt := <-w.Events:
			if evt.Op == fsnotify.Create {
				return nil
			}
		}
	}
	return nil
}

func (d *reader) fetchNextLoop() {
	for {
		nextSeq := d.lastSequence + 1
		d.waitTillPresent(nextSeq, d.stateExt)
		d.waitTillPresent(nextSeq, d.fileExt)
		d.lastSequence = nextSeq
		base := path.Join(d.dest, seqPath(d.lastSequence))
		lastTime, _ := d.stateTime(base + d.stateExt)
		d.sequences <- Sequence{
			Sequence:      d.lastSequence,
			Filename:      base + d.fileExt,
			StateFilename: base + d.stateExt,
			Time:          lastTime,
		}
	}
}
<commit_msg>improve diff reader from directory<commit_after>package replication

import (
	"errors"
	"fmt"
	"io"
	"net"
	"net/http"
	"os"
	"path"
	"time"

	"gopkg.in/fsnotify.v1"

	"github.com/omniscale/imposm3"
	"github.com/omniscale/imposm3/logging"
)

var log = logging.NewLogger("replication")

var NotAvailable = errors.New("file not available")

type Sequence struct {
	Filename      string
	StateFilename string
	Time          time.Time
	Sequence      int
}

type Source interface {
	Sequences() <-chan Sequence
}

// N = AAA*1000000 + BBB*1000 + CCC
func seqPath(seq int) string {
	c := seq % 1000
	b := seq / 1000 % 1000
	a := seq / 1000000

	return fmt.Sprintf("%03d/%03d/%03d", a, b, c)
}

var _ Source = &downloader{}

type downloader struct {
	baseUrl      string
	dest         string
	fileExt      string
	stateExt     string
	lastSequence int
	stateTime    func(string) (time.Time, error)
	interval     time.Duration
	errWaittime  time.Duration
	naWaittime   time.Duration
	sequences    chan Sequence
	client       *http.Client
}

func newDownloader(dest, url string, seq int, interval time.Duration) *downloader {
	client := &http.Client{
		Transport: &http.Transport{
			Proxy: http.ProxyFromEnvironment,
			Dial: (&net.Dialer{
				Timeout:   30 * time.Second,
				KeepAlive: 1 * time.Second, // do not keep alive till next interval
			}).Dial,
			TLSHandshakeTimeout:   10 * time.Second,
			ResponseHeaderTimeout: 10 * time.Second,
			ExpectContinueTimeout: 1 * time.Second,
		},
	}

	dl := &downloader{
		baseUrl:      url,
		dest:         dest,
		lastSequence: seq,
		interval:     interval,
		errWaittime:  60 * time.Second,
		naWaittime:   10 * time.Second,
		sequences:    make(chan Sequence, 1),
		client:       client,
	}

	return dl
}

func (d *downloader) Sequences() <-chan Sequence {
	return d.sequences
}

func (d *downloader) download(seq int, ext string) error {
	dest := path.Join(d.dest, seqPath(seq)+ext)
	url := d.baseUrl + seqPath(seq) + ext

	if _, err := os.Stat(dest); err == nil {
		return nil
	}

	if err := os.MkdirAll(path.Dir(dest), 0755); err != nil {
		return err
	}

	tmpDest := fmt.Sprintf("%s~%d", dest, os.Getpid())
	out, err := os.Create(tmpDest)
	if err != nil {
		return err
	}
	defer out.Close()

	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return err
	}
	req.Header.Set("User-Agent", "Imposm3 "+imposm3.Version)
	resp, err := d.client.Do(req)
	if err != nil {
		return err
	}

	defer resp.Body.Close()

	if resp.StatusCode == 404 {
		return NotAvailable
	}

	if resp.StatusCode != 200 {
		return errors.New(fmt.Sprintf("invalid repsonse: %v", resp))
	}

	_, err = io.Copy(out, resp.Body)
	if err != nil {
		return err
	}
	out.Close()

	err = os.Rename(tmpDest, dest)
	if err != nil {
		return err
	}

	return nil
}

func (d *downloader) downloadTillSuccess(seq int, ext string) {
	for {
		err := d.download(seq, ext)
		if err == nil {
			break
		}
		if err == NotAvailable {
			time.Sleep(d.naWaittime)
		} else {
			log.Warn(err)
			time.Sleep(d.errWaittime)
		}
	}
}

func (d *downloader) fetchNextLoop() {
	stateFile := path.Join(d.dest, seqPath(d.lastSequence)+d.stateExt)
	lastTime, err := d.stateTime(stateFile)
	for {
		if err == nil {
			nextDiffTime := lastTime.Add(d.interval)
			if nextDiffTime.After(time.Now()) {
				// we catched up and the next diff file is in the future.
				// wait till last diff time + interval, before fetching next
				nextDiffTime = lastTime.Add(d.interval + 2*time.Second /* allow small time diff between server*/)
				waitFor := nextDiffTime.Sub(time.Now())
				time.Sleep(waitFor)
			}
		}
		nextSeq := d.lastSequence + 1
		// download will retry until they succeed
		d.downloadTillSuccess(nextSeq, d.stateExt)
		d.downloadTillSuccess(nextSeq, d.fileExt)
		d.lastSequence = nextSeq
		base := path.Join(d.dest, seqPath(d.lastSequence))
		lastTime, _ = d.stateTime(base + d.stateExt)
		d.sequences <- Sequence{
			Sequence:      d.lastSequence,
			Filename:      base + d.fileExt,
			StateFilename: base + d.stateExt,
			Time:          lastTime,
		}
	}
}

var _ Source = &reader{}

type reader struct {
	dest         string
	fileExt      string
	stateExt     string
	lastSequence int
	stateTime    func(string) (time.Time, error)
	sequences    chan Sequence
}

func newReader(dest string, seq int) *reader {
	r := &reader{
		dest:         dest,
		lastSequence: seq,
		sequences:    make(chan Sequence, 1),
	}

	return r
}

func (d *reader) Sequences() <-chan Sequence {
	return d.sequences
}

func (d *reader) waitTillPresent(seq int, ext string) error {
	filename := path.Join(d.dest, seqPath(seq)+ext)
	if _, err := os.Stat(filename); err == nil {
		return nil
	}

	w, err := fsnotify.NewWatcher()
	if err != nil {
		return err
	}
	defer w.Close()
	w.Add(path.Dir(filename))

	// check again, in case file was created before we added the file
	if _, err := os.Stat(filename); err == nil {
		return nil
	}

	for {
		select {
		case evt := <-w.Events:
			if evt.Op&fsnotify.Create == fsnotify.Create && evt.Name == filename {
				return nil
			}
		}
	}
	return nil
}

func (d *reader) fetchNextLoop() {
	for {
		nextSeq := d.lastSequence + 1
		if err := d.waitTillPresent(nextSeq, d.stateExt); err != nil {
			log.Error(err)
		}
		if err := d.waitTillPresent(nextSeq, d.fileExt); err != nil {
			log.Error(err)
		}
		d.lastSequence = nextSeq
		base := path.Join(d.dest, seqPath(d.lastSequence))
		lastTime, _ := d.stateTime(base + d.stateExt)
		d.sequences <- Sequence{
			Sequence:      d.lastSequence,
			Filename:      base + d.fileExt,
			StateFilename: base + d.stateExt,
			Time:          lastTime,
		}
	}
}
<|endoftext|>


<commit_before>package seq
// Copyright ©2011 Dan Kortschak <dan.kortschak@adelaide.edu.au>
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

import (
	"fmt"
	"github.com/kortschak/BioGo/bio"
	"github.com/kortschak/BioGo/feat"
	"strings"
)

// ACGCTGACTTGGTGCACGT dna
// ACGCTGACTTGGTGCACGT rna
//     ^ first invalid RNA position
func ExampleSeq_New() {
	d := New("example sequence", []byte("ACGCTGACTTGGTGCACGT"), nil) // Default to bio.DNA
	fmt.Println(d, d.Moltype)
	// Alternative using struct literal
	r := &Seq{ID: "example RNA", Seq: d.Seq, Moltype: bio.RNA}
	fmt.Println(r, r.Moltype)
	if ok, pos := bio.ValidR.Check(r.Seq); ok {
		fmt.Println("valid RNA")
	} else {
		fmt.Println(strings.Repeat(" ", pos-1), "^ first invalid RNA position")
	}
}

// ACGCTGACTTGGTGCACGT
// GACTTGG
func ExampleSeq_Trunc_1() {
	s := &Seq{Seq: []byte("ACGCTGACTTGGTGCACGT")}
	fmt.Println(s)
	if t, err := s.Trunc(5, 12); err == nil {
		fmt.Println(t)
	}
}

// ACGCTGACTTGGTGCACGT Circular = true
// TGCACGTACGCT Circular = false
// ACGCTGACTTGGTGCACGT Circular = false
// Error: Start position greater than end position for non-circular molecule.
func ExampleSeq_Trunc_2() {
	s := &Seq{Seq: []byte("ACGCTGACTTGGTGCACGT"), Circular: true}
	fmt.Println(s, "Circular =", s.Circular)
	if t, err := s.Trunc(12, 5); err == nil {
		fmt.Println(t, "Circular =", t.Circular)
	} else {
		fmt.Println("Error:", err)
	}
	s.Circular = false
	fmt.Println(s, "Circular =", s.Circular)
	if t, err := s.Trunc(12, 5); err == nil {
		fmt.Println(t, "Circular =", t.Circular)
	} else {
		fmt.Println("Error:", err)
	}
}

// ATGCtGACTTGGTGCACGT
// ACGTGCACCAAGTCaGCAT
func ExampleSeq_RevComp_1() {
	s := &Seq{Seq: []byte("ATGCtGACTTGGTGCACGT")}
	fmt.Println(s)
	if t, err := s.RevComp(); err == nil {
		fmt.Println(t)
	}
}

// Forward:
// NTTTCTTCTATATCCTTTTCATCTTTTAATCCATTCACCATTTTTTTCCCTCCACCTACCTNTCCTTCTCTTTCT
// #.47435885169773237879795033445-3255530396.)05545553111+0333,#,54331+-7!!!!
// Reverse:
// AGAAAGAGAAGGANAGGTAGGTGGAGGGAAAAAAATGGTGAATGGATTAAAAGATGAAAAGGATATAGAAGAAAN
// !!!!7-+13345,#,3330+11135554550).6930355523-54433059797873237796158853474.#
func ExampleSeq_RevComp_2() {
	q := &Quality{Qual: []Qsanger{
		2, 13, 19, 22, 19, 18, 20, 23, 23, 20, 16, 21, 24, 22, 22, 18, 17, 18, 22, 23, 22, 24, 22, 24, 20, 15,
		18, 18, 19, 19, 20, 12, 18, 17, 20, 20, 20, 18, 15, 18, 24, 21, 13, 8, 15, 20, 20, 19, 20, 20, 20, 18,
		16, 16, 16, 10, 15, 18, 18, 18, 11, 2, 11, 20, 19, 18, 18, 16, 10, 12, 22, 0, 0, 0, 0}}
	s := &Seq{Seq: []byte("NTTTCTTCTATATCCTTTTCATCTTTTAATCCATTCACCATTTTTTTCCCTCCACCTACCTNTCCTTCTCTTTCT"), Quality: q}
	fmt.Println("Forward:")
	fmt.Println(s)
	fmt.Println(s.Quality)
	if t, err := s.RevComp(); err == nil {
	fmt.Println("Reverse:")
		fmt.Println(t)
		fmt.Println(t.Quality)
	}
}

// IIIHIEGA6.*!!!
// !!!*.6AGEIHIII
func ExampleQuality_Reverse() {
	q := &Quality{Qual: []Qsanger{40, 40, 40, 39, 40, 36, 38, 32, 21, 13, 9, 0, 0, 0}}
	fmt.Println(q)
	t := q.Reverse()
	fmt.Println(t)
}

// agctgtgctga CGTGCAGTCATGAGTGA
// agctgtgctgaCGTGCAGTCATGAGTGA
// CGTGCAGTCATGAGTGAagctgtgctga
func ExampleJoin() {
	var s1, s2 *Seq
	s1 = &Seq{Seq: []byte("agctgtgctga")}
	s2 = &Seq{Seq: []byte("CGTGCAGTCATGAGTGA")}
	fmt.Println(s1, s2)
	if t, err := s1.Join(s2, Append); err == nil {
		fmt.Println(t)
	}
	s1 = &Seq{Seq: []byte("agctgtgctga")}
	s2 = &Seq{Seq: []byte("CGTGCAGTCATGAGTGA")}
	if t, err := s1.Join(s2, Prepend); err == nil {
		fmt.Println(t)
	}
}

// aAGTATAAgtcagtgcagtgtctggcagTGCTCGTGCgtagtgaagtagGGTTAGTTTa
// AGTATAATGCTCGTGCGGTTAGTTT
func ExampleStitch() {
	s := &Seq{Seq: []byte("aAGTATAAgtcagtgcagtgtctggcagTGCTCGTGCgtagtgaagtagGGTTAGTTTa")}
	f := feat.FeatureSet{
		&feat.Feature{Start: 1, End: 8},
		&feat.Feature{Start: 28, End: 37},
		&feat.Feature{Start: 49, End: s.Len() - 1},
	}
	fmt.Println(s)
	if t, err := s.Stitch(f); err == nil {
		fmt.Println(t)
	}
}
<commit_msg>Missed failing test.<commit_after>package seq
// Copyright ©2011 Dan Kortschak <dan.kortschak@adelaide.edu.au>
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

import (
	"fmt"
	"github.com/kortschak/BioGo/bio"
	"github.com/kortschak/BioGo/feat"
	"strings"
)

// ACGCTGACTTGGTGCACGT DNA
// ACGCTGACTTGGTGCACGT RNA
//     ^ first invalid RNA position
func ExampleSeq_New() {
	d := New("example sequence", []byte("ACGCTGACTTGGTGCACGT"), nil) // Default to bio.DNA
	fmt.Println(d, d.Moltype)
	// Alternative using struct literal
	r := &Seq{ID: "example RNA", Seq: d.Seq, Moltype: bio.RNA}
	fmt.Println(r, r.Moltype)
	if ok, pos := bio.ValidR.Check(r.Seq); ok {
		fmt.Println("valid RNA")
	} else {
		fmt.Println(strings.Repeat(" ", pos-1), "^ first invalid RNA position")
	}
}

// ACGCTGACTTGGTGCACGT
// GACTTGG
func ExampleSeq_Trunc_1() {
	s := &Seq{Seq: []byte("ACGCTGACTTGGTGCACGT")}
	fmt.Println(s)
	if t, err := s.Trunc(5, 12); err == nil {
		fmt.Println(t)
	}
}

// ACGCTGACTTGGTGCACGT Circular = true
// TGCACGTACGCT Circular = false
// ACGCTGACTTGGTGCACGT Circular = false
// Error: Start position greater than end position for non-circular molecule.
func ExampleSeq_Trunc_2() {
	s := &Seq{Seq: []byte("ACGCTGACTTGGTGCACGT"), Circular: true}
	fmt.Println(s, "Circular =", s.Circular)
	if t, err := s.Trunc(12, 5); err == nil {
		fmt.Println(t, "Circular =", t.Circular)
	} else {
		fmt.Println("Error:", err)
	}
	s.Circular = false
	fmt.Println(s, "Circular =", s.Circular)
	if t, err := s.Trunc(12, 5); err == nil {
		fmt.Println(t, "Circular =", t.Circular)
	} else {
		fmt.Println("Error:", err)
	}
}

// ATGCtGACTTGGTGCACGT
// ACGTGCACCAAGTCaGCAT
func ExampleSeq_RevComp_1() {
	s := &Seq{Seq: []byte("ATGCtGACTTGGTGCACGT")}
	fmt.Println(s)
	if t, err := s.RevComp(); err == nil {
		fmt.Println(t)
	}
}

// Forward:
// NTTTCTTCTATATCCTTTTCATCTTTTAATCCATTCACCATTTTTTTCCCTCCACCTACCTNTCCTTCTCTTTCT
// #.47435885169773237879795033445-3255530396.)05545553111+0333,#,54331+-7!!!!
// Reverse:
// AGAAAGAGAAGGANAGGTAGGTGGAGGGAAAAAAATGGTGAATGGATTAAAAGATGAAAAGGATATAGAAGAAAN
// !!!!7-+13345,#,3330+11135554550).6930355523-54433059797873237796158853474.#
func ExampleSeq_RevComp_2() {
	q := &Quality{Qual: []Qsanger{
		2, 13, 19, 22, 19, 18, 20, 23, 23, 20, 16, 21, 24, 22, 22, 18, 17, 18, 22, 23, 22, 24, 22, 24, 20, 15,
		18, 18, 19, 19, 20, 12, 18, 17, 20, 20, 20, 18, 15, 18, 24, 21, 13, 8, 15, 20, 20, 19, 20, 20, 20, 18,
		16, 16, 16, 10, 15, 18, 18, 18, 11, 2, 11, 20, 19, 18, 18, 16, 10, 12, 22, 0, 0, 0, 0}}
	s := &Seq{Seq: []byte("NTTTCTTCTATATCCTTTTCATCTTTTAATCCATTCACCATTTTTTTCCCTCCACCTACCTNTCCTTCTCTTTCT"), Quality: q}
	fmt.Println("Forward:")
	fmt.Println(s)
	fmt.Println(s.Quality)
	if t, err := s.RevComp(); err == nil {
	fmt.Println("Reverse:")
		fmt.Println(t)
		fmt.Println(t.Quality)
	}
}

// IIIHIEGA6.*!!!
// !!!*.6AGEIHIII
func ExampleQuality_Reverse() {
	q := &Quality{Qual: []Qsanger{40, 40, 40, 39, 40, 36, 38, 32, 21, 13, 9, 0, 0, 0}}
	fmt.Println(q)
	t := q.Reverse()
	fmt.Println(t)
}

// agctgtgctga CGTGCAGTCATGAGTGA
// agctgtgctgaCGTGCAGTCATGAGTGA
// CGTGCAGTCATGAGTGAagctgtgctga
func ExampleJoin() {
	var s1, s2 *Seq
	s1 = &Seq{Seq: []byte("agctgtgctga")}
	s2 = &Seq{Seq: []byte("CGTGCAGTCATGAGTGA")}
	fmt.Println(s1, s2)
	if t, err := s1.Join(s2, Append); err == nil {
		fmt.Println(t)
	}
	s1 = &Seq{Seq: []byte("agctgtgctga")}
	s2 = &Seq{Seq: []byte("CGTGCAGTCATGAGTGA")}
	if t, err := s1.Join(s2, Prepend); err == nil {
		fmt.Println(t)
	}
}

// aAGTATAAgtcagtgcagtgtctggcagTGCTCGTGCgtagtgaagtagGGTTAGTTTa
// AGTATAATGCTCGTGCGGTTAGTTT
func ExampleStitch() {
	s := &Seq{Seq: []byte("aAGTATAAgtcagtgcagtgtctggcagTGCTCGTGCgtagtgaagtagGGTTAGTTTa")}
	f := feat.FeatureSet{
		&feat.Feature{Start: 1, End: 8},
		&feat.Feature{Start: 28, End: 37},
		&feat.Feature{Start: 49, End: s.Len() - 1},
	}
	fmt.Println(s)
	if t, err := s.Stitch(f); err == nil {
		fmt.Println(t)
	}
}
<|endoftext|>


<commit_before>package web_test

import (
	"fmt"

	"github.com/concourse/atc"
	"github.com/concourse/go-concourse/concourse"
	"github.com/concourse/testflight/helpers"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/sclevine/agouti"

	"testing"
)

var atcURL = helpers.AtcURL()
var pipelineName string
var publicBuild, privateBuild atc.Build
var brokenResource atc.Resource

var agoutiDriver *agouti.WebDriver
var page *agouti.Page

var client concourse.Client

var _ = SynchronizedBeforeSuite(func() []byte {
	Eventually(helpers.ErrorPolling(atcURL)).ShouldNot(HaveOccurred())

	data, err := helpers.FirstNodeClientSetup(atcURL)
	Expect(err).NotTo(HaveOccurred())

	return data
}, func(data []byte) {
	var err error
	client, err = helpers.AllNodeClientSetup(data)
	Expect(err).NotTo(HaveOccurred())

	pipelineName = fmt.Sprintf("test-pipeline-%d", GinkgoParallelNode())

	agoutiDriver = helpers.AgoutiDriver()
	Expect(agoutiDriver.Start()).To(Succeed())
})

var _ = AfterSuite(func() {
	Expect(agoutiDriver.Stop()).To(Succeed())
})

var _ = BeforeEach(func() {
	_, err := client.DeletePipeline(pipelineName)
	Expect(err).ToNot(HaveOccurred())

	pushMainPipeline()

	page, err = agoutiDriver.NewPage()
	Expect(err).NotTo(HaveOccurred())
})

var _ = AfterEach(func() {
	Expect(page.Destroy()).To(Succeed())

	_, err := client.DeletePipeline(pipelineName)
	Expect(err).ToNot(HaveOccurred())
})

func TestWeb(t *testing.T) {
	RegisterFailHandler(Fail)
	RunSpecs(t, "Authentication Web Suite")
}

func pushMainPipeline() {
	_, _, err := client.CreateOrUpdatePipelineConfig(pipelineName, "0", atc.Config{
		Jobs: []atc.JobConfig{
			{
				Name:   "public-job",
				Public: true,
				Plan: atc.PlanSequence{
					{
						Task: "some-task",
						TaskConfig: &atc.TaskConfig{
							Run: atc.TaskRunConfig{
								Path: "sh",
								Args: []string{"-c", "sleep 30 && echo public job info"},
							},
						},
					},
				},
			},
			{
				Name:   "private-job",
				Public: false,
				Plan: atc.PlanSequence{
					{
						Task: "some-task",
						TaskConfig: &atc.TaskConfig{
							Run: atc.TaskRunConfig{
								Path: "echo",
								Args: []string{"private job info"},
							},
						},
					},
				},
			},
		},
		Resources: []atc.ResourceConfig{
			{
				Name: "broken-resource",
				Type: "git",
				Source: atc.Source{
					"branch": "master",
					"uri":    "git@github.com:concourse/deployments.git",
				},
				CheckEvery: "",
			},
		},
	})
	Expect(err).NotTo(HaveOccurred())

	_, err = client.UnpausePipeline(pipelineName)
	Expect(err).NotTo(HaveOccurred())

	publicBuild, err = client.CreateJobBuild(pipelineName, "public-job")
	Expect(err).NotTo(HaveOccurred())

	privateBuild, err = client.CreateJobBuild(pipelineName, "private-job")
	Expect(err).NotTo(HaveOccurred())

	var found bool
	brokenResource, found, err = client.Resource(pipelineName, "broken-resource")
	Expect(found).To(BeTrue())
	Expect(err).NotTo(HaveOccurred())
}
<commit_msg>add moar missing platforms<commit_after>package web_test

import (
	"fmt"

	"github.com/concourse/atc"
	"github.com/concourse/go-concourse/concourse"
	"github.com/concourse/testflight/helpers"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/sclevine/agouti"

	"testing"
)

var atcURL = helpers.AtcURL()
var pipelineName string
var publicBuild, privateBuild atc.Build
var brokenResource atc.Resource

var agoutiDriver *agouti.WebDriver
var page *agouti.Page

var client concourse.Client

var _ = SynchronizedBeforeSuite(func() []byte {
	Eventually(helpers.ErrorPolling(atcURL)).ShouldNot(HaveOccurred())

	data, err := helpers.FirstNodeClientSetup(atcURL)
	Expect(err).NotTo(HaveOccurred())

	return data
}, func(data []byte) {
	var err error
	client, err = helpers.AllNodeClientSetup(data)
	Expect(err).NotTo(HaveOccurred())

	pipelineName = fmt.Sprintf("test-pipeline-%d", GinkgoParallelNode())

	agoutiDriver = helpers.AgoutiDriver()
	Expect(agoutiDriver.Start()).To(Succeed())
})

var _ = AfterSuite(func() {
	Expect(agoutiDriver.Stop()).To(Succeed())
})

var _ = BeforeEach(func() {
	_, err := client.DeletePipeline(pipelineName)
	Expect(err).ToNot(HaveOccurred())

	pushMainPipeline()

	page, err = agoutiDriver.NewPage()
	Expect(err).NotTo(HaveOccurred())
})

var _ = AfterEach(func() {
	Expect(page.Destroy()).To(Succeed())

	_, err := client.DeletePipeline(pipelineName)
	Expect(err).ToNot(HaveOccurred())
})

func TestWeb(t *testing.T) {
	RegisterFailHandler(Fail)
	RunSpecs(t, "Authentication Web Suite")
}

func pushMainPipeline() {
	_, _, err := client.CreateOrUpdatePipelineConfig(pipelineName, "0", atc.Config{
		Jobs: []atc.JobConfig{
			{
				Name:   "public-job",
				Public: true,
				Plan: atc.PlanSequence{
					{
						Task: "some-task",
						TaskConfig: &atc.TaskConfig{
							Platform: "linux",
							Run: atc.TaskRunConfig{
								Path: "sh",
								Args: []string{"-c", "sleep 30 && echo public job info"},
							},
						},
					},
				},
			},
			{
				Name:   "private-job",
				Public: false,
				Plan: atc.PlanSequence{
					{
						Task: "some-task",
						TaskConfig: &atc.TaskConfig{
							Platform: "linux",
							Run: atc.TaskRunConfig{
								Path: "echo",
								Args: []string{"private job info"},
							},
						},
					},
				},
			},
		},
		Resources: []atc.ResourceConfig{
			{
				Name: "broken-resource",
				Type: "git",
				Source: atc.Source{
					"branch": "master",
					"uri":    "git@github.com:concourse/deployments.git",
				},
				CheckEvery: "",
			},
		},
	})
	Expect(err).NotTo(HaveOccurred())

	_, err = client.UnpausePipeline(pipelineName)
	Expect(err).NotTo(HaveOccurred())

	publicBuild, err = client.CreateJobBuild(pipelineName, "public-job")
	Expect(err).NotTo(HaveOccurred())

	privateBuild, err = client.CreateJobBuild(pipelineName, "private-job")
	Expect(err).NotTo(HaveOccurred())

	var found bool
	brokenResource, found, err = client.Resource(pipelineName, "broken-resource")
	Expect(found).To(BeTrue())
	Expect(err).NotTo(HaveOccurred())
}
<|endoftext|>


<commit_before>package cloudformation

var AWSTemplateFormatVersion = "2010-09-09"

func NewTemplate(desc string) *Template {
	return &Template{
		AWSTemplateFormatVersion: AWSTemplateFormatVersion,
		Description:              desc,
	}
}

type Template struct {
	AWSTemplateFormatVersion string `json:"AWSTemplateFormatVersion,omitempty"`
	Description              string `json:"Description,omitempty"`

	Parameters map[string]*Parameter `json:"Parameters,omitempty"`
	Resources  Resources             `json:"Resources,omitempty"`
}

type Resources map[string]interface{}

type Resource struct {
	Type         string      `json:"Type,omitempty"`
	Properties   interface{} `json:"Properties,omitempty"`
	UpdatePolicy interface{} `json:"UpdatePolicy,omitempty"`
}

func NewResource(theType string, properties interface{}) *Resource {
	return &Resource{
		Type: theType, Properties: properties,
	}
}

type Properties struct {
	Type       string      `json:"Type,omitempty"`
	Properties *Properties `json:"Properties,omitempty"`
	MinValue   int         `json:"MinValue,omitempty"`
}

// AWS::ElasticLoadBalancing::LoadBalancer

type LoadBalancer struct {
	LoadBalancerName interface{}  `json:"LoadBalancerName,omitempty"`
	CrossZone        interface{}  `json:"CrossZone,omitempty"`
	HealthCheck      *HealthCheck `json:"HealthCheck,omitempty"`
	Listeners        []*Listener  `json:"Listeners,omitempty"`
}

type HealthCheck struct {
	HealthyThreshold   interface{} `json:"HealthyThreshold,omitempty"`
	Interval           interface{} `json:"Interval,omitempty"`
	Target             interface{} `json:"Target,omitempty"`
	Timeout            interface{} `json:"Timeout,omitempty"`
	UnhealthyThreshold interface{} `json:"UnhealthyThreshold,omitempty"`
}

type Listener struct {
	InstancePort     interface{}   `json:"InstancePort,omitempty"`
	LoadBalancerPort interface{}   `json:"LoadBalancerPort,omitempty"`
	Protocol         interface{}   `json:"Protocol,omitempty"`
	InstanceProtocol interface{}   `json:"InstanceProtocol,omitempty"`
	SSLCertificateId interface{}   `json:"SSLCertificateId,omitempty"`
	PolicyNames      []interface{} `json:"PolicyNames,omitempty"`
}

type Policy struct {
	PolicyName interface{}        `json:"PolicyName,omitempty"`
	PolicyType interface{}        `json:"PolicyType,omitempty"`
	Attributes []*PolicyAttribute `json:"Attributes,omitempty"`
}

// AWS::AutoScaling::LaunchConfiguration
type LaunchConfiguration struct {
	InstanceType        interface{}           `json:"InstanceType,omitempty"`
	ImageId             interface{}           `json:"ImageId,omitempty"`
	KeyName             interface{}           `json:"KeyName,omitempty"`
	SecurityGroups      []interface{}         `json:"SecurityGroups,omitempty"`
	UserData            interface{}           `json:"UserData,omitempty"`
	BlockDeviceMappings []*BlockDeviceMapping `json:"BlockDeviceMappings,omitempty"`
}

// AWS::AutoScaling::AutoScalingGroup",
type AutoScalingGroup struct {
	AvailabilityZones       []interface{} `json:"AvailabilityZones,omitempty"`
	Cooldown                interface{}   `json:"Cooldown,omitempty"`
	DesiredCapacity         interface{}   `json:"DesiredCapacity,omitempty"`
	HealthCheckGracePeriod  interface{}   `json:"HealthCheckGracePeriod,omitempty"`
	HealthCheckType         interface{}   `json:"HealthCheckType,omitempty"`
	LaunchConfigurationName interface{}   `json:"LaunchConfigurationName,omitempty"`
	LoadBalancerNames       []interface{} `json:"LoadBalancerNames,omitempty"`
	MaxSize                 interface{}   `json:"MaxSize,omitempty"`
	MinSize                 interface{}   `json:"MinSize,omitempty"`
}

type NotificationConfiguration struct {
	TopicARN          interface{}   `json:"TopicARN,omitempty"`
	NotificationTypes []interface{} `json:"NotificationTypes,omitempty"`
	Tags              []*Tag        `json:"Tags,omitempty"`
}

type Tag struct {
	Key               interface{} `json:"Key,omitempty"`
	Value             interface{} `json:"Value,omitempty"`
	PropagateAtLaunch interface{} `json:"PropagateAtLaunch,omitempty"`
	VPCZoneIdentifier interface{} `json:"VPCZoneIdentifier,omitempty"`
}

// AWS::CloudWatch::Alarm
type Alarm struct {
	ActionsEnabled     interface{}   `json:"ActionsEnabled,omitempty"`
	ComparisonOperator interface{}   `json:"ComparisonOperator,omitempty"`
	EvaluationPeriods  interface{}   `json:"EvaluationPeriods,omitempty"`
	MetricName         interface{}   `json:"MetricName,omitempty"`
	Namespace          interface{}   `json:"Namespace,omitempty"`
	Period             interface{}   `json:"Period,omitempty"`
	Statistic          interface{}   `json:"Statistic,omitempty"`
	Threshold          interface{}   `json:"Threshold,omitempty"`
	AlarmActions       []interface{} `json:"AlarmActions,omitempty"`
}

type BlockDeviceMapping struct {
	DeviceName interface{} `json:"DeviceName,omitempty"`
	Ebs        *Ebs        `json:"Ebs,omitempty"`
}

type Ebs struct {
	VolumeSize interface{} `json:"VolumeSize,omitempty"`
}

type PolicyAttribute struct {
	Name           interface{}   `json:"Name,omitempty"`
	Value          interface{}   `json:"Value,omitempty"`
	SecurityGroups []interface{} `json:"SecurityGroups,omitempty"`
	Subnets        interface{}   `json:"Subnets,omitempty"`
}

// AWS::EC2::SecurityGroup
type SecurityGroupProperties struct {
	GroupDescription     interface{}     `json:"GroupDescription,omitempty"`
	VpcId                interface{}     `json:"VpcId,omitempty"`
	SecurityGroupIngress []SecurityGroup `json:"SecurityGroupIngress,omitempty"`
	SecurityGroupEgress  []SecurityGroup `json:"SecurityGroupEgress,omitempty"`
}

type SecurityGroup struct {
	VpcId                interface{}          `json:"VpcId,omitempty"`
	GroupDescription     interface{}          `json:"GroupDescription,omitempty"`
	SecurityGroupEgress  []*SecurityGroupRule `json:"SecurityGroupEgress,omitempty"`
	SecurityGroupIngress []*SecurityGroupRule `json:"SecurityGroupIngress,omitempty"`
	Tags                 []*Tag               `json:"Tags,omitempty"`
}

type SecurityGroupRule struct {
	GroupId                    interface{} `json:"GroupId,omitempty"`
	IpProtocol                 interface{} `json:"IpProtocol,omitempty"`
	FromPort                   interface{} `json:"FromPort,omitempty"`
	ToPort                     interface{} `json:"ToPort,omitempty"`
	CidrIp                     interface{} `json:"CidrIp,omitempty"`
	SourceSecurityGroupId      interface{} `json:"SourceSecurityGroupId,omitempty"`
	SourceSecurityGroupOwnerId interface{} `json:"SourceSecurityGroupOwnerId,omitempty"`
}

// AWS::RDS::DBSubnetGroup
type DBSubnetGroup struct {
	DBSubnetGroupDescription interface{} `json:"DBSubnetGroupDescription,omitempty"`
	SubnetIds                interface{} `json:"SubnetIds,omitempty"`
}

// AWS::SNS::Topic
type Topic struct {
	DisplayName string `json:"DisplayName,omitempty"`
}

// AWS::ElastiCache::CacheCluster
type CacheClusterProperties struct {
	AutoMinorVersionUpgrade    interface{}   `json:"AutoMinorVersionUpgrade,omitempty"`
	CacheNodeType              interface{}   `json:"CacheNodeType,omitempty"`
	ClusterName                interface{}   `json:"ClusterName,omitempty"`
	Engine                     interface{}   `json:"Engine,omitempty"`
	EngineVersion              interface{}   `json:"EngineVersion,omitempty"`
	NotificationTopicArn       interface{}   `json:"NotificationTopicArn,omitempty"`
	NumCacheNodes              interface{}   `json:"NumCacheNodes,omitempty"`
	PreferredAvailabilityZone  interface{}   `json:"PreferredAvailabilityZone,omitempty"`
	PreferredMaintenanceWindow interface{}   `json:"PreferredMaintenanceWindow,omitempty"`
	VpcSecurityGroupIds        []interface{} `json:"VpcSecurityGroupIds,omitempty"`
}

// AWS::RDS::DBParameterGroup
type DBParameterGroup struct {
	Description                interface{}       `json:"Description,omitempty"`
	Family                     interface{}       `json:"Family,omitempty"`
	DBParameterGroupParameters map[string]string `json:"DBParameterGroupParameters,omitempty"`
}

// AWS::RDS::DBInstance
type DBInstance struct {
	AllocatedStorage           interface{}   `json:"	AllocatedStorage,omitempty"`
	AutoMinorVersionUpgrade    interface{}   `json:"	AutoMinorVersionUpgrade,omitempty"`
	BackupRetentionPeriod      interface{}   `json:"	BackupRetentionPeriod,omitempty"`
	DBSubnetGroupName          interface{}   `json:"	DBSubnetGroupName,omitempty"`
	DBInstanceClass            interface{}   `json:"	DBInstanceClass,omitempty"`
	DBInstanceIdentifier       interface{}   `json:"	DBInstanceIdentifier,omitempty"`
	DBName                     interface{}   `json:"	DBName,omitempty"`
	Engine                     interface{}   `json:"	Engine,omitempty"`
	EngineVersion              interface{}   `json:"	EngineVersion,omitempty"`
	LicenseModel               interface{}   `json:"	LicenseModel,omitempty"`
	MasterUsername             interface{}   `json:"	MasterUsername,omitempty"`
	MasterUserPassword         interface{}   `json:"	MasterUserPassword,omitempty"`
	MultiAZ                    interface{}   `json:"	MultiAZ,omitempty"`
	Port                       interface{}   `json:"	Port,omitempty"`
	PreferredBackupWindow      interface{}   `json:"	PreferredBackupWindow,omitempty"`
	PreferredMaintenanceWindow interface{}   `json:"	PreferredMaintenanceWindow,omitempty"`
	Tags                       []interface{} `json:"	Tags,omitempty"`
	VPCSecurityGroups          []interface{} `json:"	VPCSecurityGroups,omitempty"`
}

type Property struct {
	Type string `json:"Type,omitempty"`
}

type Parameter struct {
	Type    string `json:"Type,omitempty"`
	Default string `json:"Default,omitempty"`
}
<commit_msg>fix cloudformation mappings<commit_after>package cloudformation

var AWSTemplateFormatVersion = "2010-09-09"

func NewTemplate(desc string) *Template {
	return &Template{
		AWSTemplateFormatVersion: AWSTemplateFormatVersion,
		Description:              desc,
	}
}

type Template struct {
	AWSTemplateFormatVersion string `json:"AWSTemplateFormatVersion,omitempty"`
	Description              string `json:"Description,omitempty"`

	Parameters map[string]*Parameter `json:"Parameters,omitempty"`
	Resources  Resources             `json:"Resources,omitempty"`
}

type Resources map[string]interface{}

type Resource struct {
	Type         string      `json:"Type,omitempty"`
	Properties   interface{} `json:"Properties,omitempty"`
	UpdatePolicy interface{} `json:"UpdatePolicy,omitempty"`
}

func NewResource(theType string, properties interface{}) *Resource {
	return &Resource{
		Type: theType, Properties: properties,
	}
}

type Properties struct {
	Type       string      `json:"Type,omitempty"`
	Properties *Properties `json:"Properties,omitempty"`
	MinValue   int         `json:"MinValue,omitempty"`
}

// AWS::ElasticLoadBalancing::LoadBalancer

type LoadBalancer struct {
	LoadBalancerName interface{}  `json:"LoadBalancerName,omitempty"`
	CrossZone        interface{}  `json:"CrossZone,omitempty"`
	HealthCheck      *HealthCheck `json:"HealthCheck,omitempty"`
	Listeners        []*Listener  `json:"Listeners,omitempty"`
	SecurityGroups   interface{}  `json:"SecurityGroups,omitempty"`
	Subnets          interface{}  `json:"Subnets,omitempty"`
}

type HealthCheck struct {
	HealthyThreshold   interface{} `json:"HealthyThreshold,omitempty"`
	Interval           interface{} `json:"Interval,omitempty"`
	Target             interface{} `json:"Target,omitempty"`
	Timeout            interface{} `json:"Timeout,omitempty"`
	UnhealthyThreshold interface{} `json:"UnhealthyThreshold,omitempty"`
}

type Listener struct {
	InstancePort     interface{}   `json:"InstancePort,omitempty"`
	LoadBalancerPort interface{}   `json:"LoadBalancerPort,omitempty"`
	Protocol         interface{}   `json:"Protocol,omitempty"`
	InstanceProtocol interface{}   `json:"InstanceProtocol,omitempty"`
	SSLCertificateId interface{}   `json:"SSLCertificateId,omitempty"`
	PolicyNames      []interface{} `json:"PolicyNames,omitempty"`
}

type Policy struct {
	PolicyName interface{}        `json:"PolicyName,omitempty"`
	PolicyType interface{}        `json:"PolicyType,omitempty"`
	Attributes []*PolicyAttribute `json:"Attributes,omitempty"`
}

// AWS::AutoScaling::LaunchConfiguration
type LaunchConfiguration struct {
	InstanceType        interface{}           `json:"InstanceType,omitempty"`
	ImageId             interface{}           `json:"ImageId,omitempty"`
	KeyName             interface{}           `json:"KeyName,omitempty"`
	SecurityGroups      []interface{}         `json:"SecurityGroups,omitempty"`
	UserData            interface{}           `json:"UserData,omitempty"`
	BlockDeviceMappings []*BlockDeviceMapping `json:"BlockDeviceMappings,omitempty"`
}

// AWS::AutoScaling::AutoScalingGroup",
type AutoScalingGroup struct {
	AvailabilityZones       []interface{} `json:"AvailabilityZones,omitempty"`
	Cooldown                interface{}   `json:"Cooldown,omitempty"`
	DesiredCapacity         interface{}   `json:"DesiredCapacity,omitempty"`
	HealthCheckGracePeriod  interface{}   `json:"HealthCheckGracePeriod,omitempty"`
	HealthCheckType         interface{}   `json:"HealthCheckType,omitempty"`
	LaunchConfigurationName interface{}   `json:"LaunchConfigurationName,omitempty"`
	LoadBalancerNames       []interface{} `json:"LoadBalancerNames,omitempty"`
	MaxSize                 interface{}   `json:"MaxSize,omitempty"`
	MinSize                 interface{}   `json:"MinSize,omitempty"`
}

type NotificationConfiguration struct {
	TopicARN          interface{}   `json:"TopicARN,omitempty"`
	NotificationTypes []interface{} `json:"NotificationTypes,omitempty"`
	Tags              []*Tag        `json:"Tags,omitempty"`
}

type Tag struct {
	Key               interface{} `json:"Key,omitempty"`
	Value             interface{} `json:"Value,omitempty"`
	PropagateAtLaunch interface{} `json:"PropagateAtLaunch,omitempty"`
	VPCZoneIdentifier interface{} `json:"VPCZoneIdentifier,omitempty"`
}

// AWS::CloudWatch::Alarm
type Alarm struct {
	ActionsEnabled     interface{}   `json:"ActionsEnabled,omitempty"`
	ComparisonOperator interface{}   `json:"ComparisonOperator,omitempty"`
	EvaluationPeriods  interface{}   `json:"EvaluationPeriods,omitempty"`
	MetricName         interface{}   `json:"MetricName,omitempty"`
	Namespace          interface{}   `json:"Namespace,omitempty"`
	Period             interface{}   `json:"Period,omitempty"`
	Statistic          interface{}   `json:"Statistic,omitempty"`
	Threshold          interface{}   `json:"Threshold,omitempty"`
	AlarmActions       []interface{} `json:"AlarmActions,omitempty"`
}

type BlockDeviceMapping struct {
	DeviceName interface{} `json:"DeviceName,omitempty"`
	Ebs        *Ebs        `json:"Ebs,omitempty"`
}

type Ebs struct {
	VolumeSize interface{} `json:"VolumeSize,omitempty"`
}

type PolicyAttribute struct {
	Name           interface{}   `json:"Name,omitempty"`
	Value          interface{}   `json:"Value,omitempty"`
	SecurityGroups []interface{} `json:"SecurityGroups,omitempty"`
	Subnets        interface{}   `json:"Subnets,omitempty"`
}

// AWS::EC2::SecurityGroup
type SecurityGroupProperties struct {
	GroupDescription     interface{}     `json:"GroupDescription,omitempty"`
	VpcId                interface{}     `json:"VpcId,omitempty"`
	SecurityGroupIngress []SecurityGroup `json:"SecurityGroupIngress,omitempty"`
	SecurityGroupEgress  []SecurityGroup `json:"SecurityGroupEgress,omitempty"`
}

type SecurityGroup struct {
	VpcId                interface{}          `json:"VpcId,omitempty"`
	GroupDescription     interface{}          `json:"GroupDescription,omitempty"`
	SecurityGroupEgress  []*SecurityGroupRule `json:"SecurityGroupEgress,omitempty"`
	SecurityGroupIngress []*SecurityGroupRule `json:"SecurityGroupIngress,omitempty"`
	Tags                 []*Tag               `json:"Tags,omitempty"`
}

type SecurityGroupRule struct {
	IpProtocol                 interface{} `json:"IpProtocol,omitempty"`
	FromPort                   interface{} `json:"FromPort,omitempty"`
	ToPort                     interface{} `json:"ToPort,omitempty"`
	CidrIp                     interface{} `json:"CidrIp,omitempty"`
	SourceSecurityGroupId      interface{} `json:"SourceSecurityGroupId,omitempty"`
	SourceSecurityGroupOwnerId interface{} `json:"SourceSecurityGroupOwnerId,omitempty"`
}

// AWS::RDS::DBSubnetGroup
type DBSubnetGroup struct {
	DBSubnetGroupDescription interface{} `json:"DBSubnetGroupDescription,omitempty"`
	SubnetIds                interface{} `json:"SubnetIds,omitempty"`
}

// AWS::SNS::Topic
type Topic struct {
	DisplayName string `json:"DisplayName,omitempty"`
}

// AWS::ElastiCache::CacheCluster
type CacheClusterProperties struct {
	AutoMinorVersionUpgrade    interface{}   `json:"AutoMinorVersionUpgrade,omitempty"`
	CacheNodeType              interface{}   `json:"CacheNodeType,omitempty"`
	ClusterName                interface{}   `json:"ClusterName,omitempty"`
	Engine                     interface{}   `json:"Engine,omitempty"`
	EngineVersion              interface{}   `json:"EngineVersion,omitempty"`
	NotificationTopicArn       interface{}   `json:"NotificationTopicArn,omitempty"`
	NumCacheNodes              interface{}   `json:"NumCacheNodes,omitempty"`
	PreferredAvailabilityZone  interface{}   `json:"PreferredAvailabilityZone,omitempty"`
	PreferredMaintenanceWindow interface{}   `json:"PreferredMaintenanceWindow,omitempty"`
	VpcSecurityGroupIds        []interface{} `json:"VpcSecurityGroupIds,omitempty"`
}

// AWS::RDS::DBParameterGroup
type DBParameterGroup struct {
	Description                interface{}       `json:"Description,omitempty"`
	Family                     interface{}       `json:"Family,omitempty"`
	DBParameterGroupParameters map[string]string `json:"DBParameterGroupParameters,omitempty"`
}

// AWS::RDS::DBInstance
type DBInstance struct {
	AllocatedStorage           interface{}   `json:"	AllocatedStorage,omitempty"`
	AutoMinorVersionUpgrade    interface{}   `json:"	AutoMinorVersionUpgrade,omitempty"`
	BackupRetentionPeriod      interface{}   `json:"	BackupRetentionPeriod,omitempty"`
	DBSubnetGroupName          interface{}   `json:"	DBSubnetGroupName,omitempty"`
	DBInstanceClass            interface{}   `json:"	DBInstanceClass,omitempty"`
	DBInstanceIdentifier       interface{}   `json:"	DBInstanceIdentifier,omitempty"`
	DBName                     interface{}   `json:"	DBName,omitempty"`
	Engine                     interface{}   `json:"	Engine,omitempty"`
	EngineVersion              interface{}   `json:"	EngineVersion,omitempty"`
	LicenseModel               interface{}   `json:"	LicenseModel,omitempty"`
	MasterUsername             interface{}   `json:"	MasterUsername,omitempty"`
	MasterUserPassword         interface{}   `json:"	MasterUserPassword,omitempty"`
	MultiAZ                    interface{}   `json:"	MultiAZ,omitempty"`
	Port                       interface{}   `json:"	Port,omitempty"`
	PreferredBackupWindow      interface{}   `json:"	PreferredBackupWindow,omitempty"`
	PreferredMaintenanceWindow interface{}   `json:"	PreferredMaintenanceWindow,omitempty"`
	Tags                       []interface{} `json:"	Tags,omitempty"`
	VPCSecurityGroups          []interface{} `json:"	VPCSecurityGroups,omitempty"`
}

type Property struct {
	Type string `json:"Type,omitempty"`
}

type Parameter struct {
	Type    string `json:"Type,omitempty"`
	Default string `json:"Default,omitempty"`
}
<|endoftext|>


<commit_before>package aws

import (
	"fmt"
	"log"
	"strings"
	"time"

	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/helper/schema"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/s3"
)

func resourceAwsS3BucketMetric() *schema.Resource {
	return &schema.Resource{
		Create: resourceAwsS3BucketMetricPut,
		Read:   resourceAwsS3BucketMetricRead,
		Update: resourceAwsS3BucketMetricPut,
		Delete: resourceAwsS3BucketMetricDelete,
		Importer: &schema.ResourceImporter{
			State: schema.ImportStatePassthrough,
		},

		Schema: map[string]*schema.Schema{
			"bucket": {
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},
			"filter": {
				Type:     schema.TypeList,
				Optional: true,
				MaxItems: 1,
				Elem: &schema.Resource{
					Schema: map[string]*schema.Schema{
						"prefix": {
							Type:     schema.TypeString,
							Optional: true,
						},
						"tags": tagsSchema(),
					},
				},
			},
			"name": {
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},
		},
	}
}

func resourceAwsS3BucketMetricPut(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).s3conn
	bucket := d.Get("bucket").(string)
	name := d.Get("name").(string)

	metricsConfiguration := &s3.MetricsConfiguration{
		Id: aws.String(name),
	}

	if v, ok := d.GetOk("filter"); ok {
		metricsConfiguration.Filter = expandS3MetricsFilter(v.([]interface{})[0].(map[string]interface{}))
	}

	input := &s3.PutBucketMetricsConfigurationInput{
		Bucket:               aws.String(bucket),
		Id:                   aws.String(name),
		MetricsConfiguration: metricsConfiguration,
	}

	log.Printf("[DEBUG] Putting metric configuration: %s", input)
	err := resource.Retry(1*time.Minute, func() *resource.RetryError {
		_, err := conn.PutBucketMetricsConfiguration(input)
		if err != nil {
			if isAWSErr(err, s3.ErrCodeNoSuchBucket, "") {
				return resource.RetryableError(err)
			}
			return resource.NonRetryableError(err)
		}
		return nil
	})
	if err != nil {
		return fmt.Errorf("Error putting S3 metric configuration: %s", err)
	}

	d.SetId(fmt.Sprintf("%s:%s", bucket, name))

	return resourceAwsS3BucketMetricRead(d, meta)
}

func resourceAwsS3BucketMetricDelete(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).s3conn

	bucket, name, err := resourceAwsS3BucketMetricParseID(d.Id())
	if err != nil {
		return err
	}

	input := &s3.DeleteBucketMetricsConfigurationInput{
		Bucket: aws.String(bucket),
		Id:     aws.String(name),
	}

	log.Printf("[DEBUG] Deleting S3 bucket metric configuration: %s", input)
	_, err = conn.DeleteBucketMetricsConfiguration(input)
	if err != nil {
		if isAWSErr(err, s3.ErrCodeNoSuchBucket, "") || isAWSErr(err, "NoSuchConfiguration", "The specified configuration does not exist.") {
			log.Printf("[WARN] %s S3 bucket metrics configuration not found, removing from state.", d.Id())
			d.SetId("")
			return nil
		}
		return fmt.Errorf("Error deleting S3 metric configuration: %s", err)
	}

	d.SetId("")
	return nil
}

func resourceAwsS3BucketMetricRead(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).s3conn

	bucket, name, err := resourceAwsS3BucketMetricParseID(d.Id())
	if err != nil {
		return err
	}

	d.Set("bucket", bucket)
	d.Set("name", name)

	input := &s3.GetBucketMetricsConfigurationInput{
		Bucket: aws.String(bucket),
		Id:     aws.String(name),
	}

	log.Printf("[DEBUG] Reading S3 bucket metrics configuration: %s", input)
	output, err := conn.GetBucketMetricsConfiguration(input)
	if err != nil {
		if isAWSErr(err, s3.ErrCodeNoSuchBucket, "") || isAWSErr(err, "NoSuchConfiguration", "The specified configuration does not exist.") {
			log.Printf("[WARN] %s S3 bucket metrics configuration not found, removing from state.", d.Id())
			d.SetId("")
			return nil
		}
		return err
	}

	if output.MetricsConfiguration.Filter != nil {
		if err := d.Set("filter", []interface{}{flattenS3MetricsFilter(output.MetricsConfiguration.Filter)}); err != nil {
			return err
		}
	}

	return nil
}

func expandS3MetricsFilter(m map[string]interface{}) *s3.MetricsFilter {
	var prefix string
	if v, ok := m["prefix"]; ok {
		prefix = v.(string)
	}

	var tags []*s3.Tag
	if v, ok := m["tags"]; ok {
		tags = tagsFromMapS3(v.(map[string]interface{}))
	}

	metricsFilter := &s3.MetricsFilter{}
	if prefix != "" && len(tags) > 0 {
		metricsFilter.And = &s3.MetricsAndOperator{
			Prefix: aws.String(prefix),
			Tags:   tags,
		}
	} else if len(tags) > 1 {
		metricsFilter.And = &s3.MetricsAndOperator{
			Tags: tags,
		}
	} else if len(tags) == 1 {
		metricsFilter.Tag = tags[0]
	} else {
		metricsFilter.Prefix = aws.String(prefix)
	}
	return metricsFilter
}

func flattenS3MetricsFilter(metricsFilter *s3.MetricsFilter) map[string]interface{} {
	m := make(map[string]interface{})

	if metricsFilter.And != nil {
		and := *metricsFilter.And
		if and.Prefix != nil {
			m["prefix"] = *and.Prefix
		}
		if and.Tags != nil {
			m["tags"] = tagsToMapS3(and.Tags)
		}
	} else if metricsFilter.Prefix != nil {
		m["prefix"] = *metricsFilter.Prefix
	} else if metricsFilter.Tag != nil {
		tags := []*s3.Tag{
			metricsFilter.Tag,
		}
		m["tags"] = tagsToMapS3(tags)
	}
	return m
}

func resourceAwsS3BucketMetricParseID(id string) (string, string, error) {
	idParts := strings.Split(id, ":")
	if len(idParts) != 2 {
		return "", "", fmt.Errorf("please make sure the ID is in the form BUCKET:NAME (i.e. my-bucket:EntireBucket")
	}
	bucket := idParts[0]
	name := idParts[1]
	return bucket, name, nil
}
<commit_msg>resource/aws_s3_bucket_metric: Fix import ordering and spread filter casting across lines<commit_after>package aws

import (
	"fmt"
	"log"
	"strings"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/helper/schema"
)

func resourceAwsS3BucketMetric() *schema.Resource {
	return &schema.Resource{
		Create: resourceAwsS3BucketMetricPut,
		Read:   resourceAwsS3BucketMetricRead,
		Update: resourceAwsS3BucketMetricPut,
		Delete: resourceAwsS3BucketMetricDelete,
		Importer: &schema.ResourceImporter{
			State: schema.ImportStatePassthrough,
		},

		Schema: map[string]*schema.Schema{
			"bucket": {
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},
			"filter": {
				Type:     schema.TypeList,
				Optional: true,
				MaxItems: 1,
				Elem: &schema.Resource{
					Schema: map[string]*schema.Schema{
						"prefix": {
							Type:     schema.TypeString,
							Optional: true,
						},
						"tags": tagsSchema(),
					},
				},
			},
			"name": {
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},
		},
	}
}

func resourceAwsS3BucketMetricPut(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).s3conn
	bucket := d.Get("bucket").(string)
	name := d.Get("name").(string)

	metricsConfiguration := &s3.MetricsConfiguration{
		Id: aws.String(name),
	}

	if v, ok := d.GetOk("filter"); ok {
		filterList := v.([]interface{})
		filterMap := filterList[0].(map[string]interface{})
		metricsConfiguration.Filter = expandS3MetricsFilter(filterMap)
	}

	input := &s3.PutBucketMetricsConfigurationInput{
		Bucket:               aws.String(bucket),
		Id:                   aws.String(name),
		MetricsConfiguration: metricsConfiguration,
	}

	log.Printf("[DEBUG] Putting metric configuration: %s", input)
	err := resource.Retry(1*time.Minute, func() *resource.RetryError {
		_, err := conn.PutBucketMetricsConfiguration(input)
		if err != nil {
			if isAWSErr(err, s3.ErrCodeNoSuchBucket, "") {
				return resource.RetryableError(err)
			}
			return resource.NonRetryableError(err)
		}
		return nil
	})
	if err != nil {
		return fmt.Errorf("Error putting S3 metric configuration: %s", err)
	}

	d.SetId(fmt.Sprintf("%s:%s", bucket, name))

	return resourceAwsS3BucketMetricRead(d, meta)
}

func resourceAwsS3BucketMetricDelete(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).s3conn

	bucket, name, err := resourceAwsS3BucketMetricParseID(d.Id())
	if err != nil {
		return err
	}

	input := &s3.DeleteBucketMetricsConfigurationInput{
		Bucket: aws.String(bucket),
		Id:     aws.String(name),
	}

	log.Printf("[DEBUG] Deleting S3 bucket metric configuration: %s", input)
	_, err = conn.DeleteBucketMetricsConfiguration(input)
	if err != nil {
		if isAWSErr(err, s3.ErrCodeNoSuchBucket, "") || isAWSErr(err, "NoSuchConfiguration", "The specified configuration does not exist.") {
			log.Printf("[WARN] %s S3 bucket metrics configuration not found, removing from state.", d.Id())
			d.SetId("")
			return nil
		}
		return fmt.Errorf("Error deleting S3 metric configuration: %s", err)
	}

	d.SetId("")
	return nil
}

func resourceAwsS3BucketMetricRead(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).s3conn

	bucket, name, err := resourceAwsS3BucketMetricParseID(d.Id())
	if err != nil {
		return err
	}

	d.Set("bucket", bucket)
	d.Set("name", name)

	input := &s3.GetBucketMetricsConfigurationInput{
		Bucket: aws.String(bucket),
		Id:     aws.String(name),
	}

	log.Printf("[DEBUG] Reading S3 bucket metrics configuration: %s", input)
	output, err := conn.GetBucketMetricsConfiguration(input)
	if err != nil {
		if isAWSErr(err, s3.ErrCodeNoSuchBucket, "") || isAWSErr(err, "NoSuchConfiguration", "The specified configuration does not exist.") {
			log.Printf("[WARN] %s S3 bucket metrics configuration not found, removing from state.", d.Id())
			d.SetId("")
			return nil
		}
		return err
	}

	if output.MetricsConfiguration.Filter != nil {
		if err := d.Set("filter", []interface{}{flattenS3MetricsFilter(output.MetricsConfiguration.Filter)}); err != nil {
			return err
		}
	}

	return nil
}

func expandS3MetricsFilter(m map[string]interface{}) *s3.MetricsFilter {
	var prefix string
	if v, ok := m["prefix"]; ok {
		prefix = v.(string)
	}

	var tags []*s3.Tag
	if v, ok := m["tags"]; ok {
		tags = tagsFromMapS3(v.(map[string]interface{}))
	}

	metricsFilter := &s3.MetricsFilter{}
	if prefix != "" && len(tags) > 0 {
		metricsFilter.And = &s3.MetricsAndOperator{
			Prefix: aws.String(prefix),
			Tags:   tags,
		}
	} else if len(tags) > 1 {
		metricsFilter.And = &s3.MetricsAndOperator{
			Tags: tags,
		}
	} else if len(tags) == 1 {
		metricsFilter.Tag = tags[0]
	} else {
		metricsFilter.Prefix = aws.String(prefix)
	}
	return metricsFilter
}

func flattenS3MetricsFilter(metricsFilter *s3.MetricsFilter) map[string]interface{} {
	m := make(map[string]interface{})

	if metricsFilter.And != nil {
		and := *metricsFilter.And
		if and.Prefix != nil {
			m["prefix"] = *and.Prefix
		}
		if and.Tags != nil {
			m["tags"] = tagsToMapS3(and.Tags)
		}
	} else if metricsFilter.Prefix != nil {
		m["prefix"] = *metricsFilter.Prefix
	} else if metricsFilter.Tag != nil {
		tags := []*s3.Tag{
			metricsFilter.Tag,
		}
		m["tags"] = tagsToMapS3(tags)
	}
	return m
}

func resourceAwsS3BucketMetricParseID(id string) (string, string, error) {
	idParts := strings.Split(id, ":")
	if len(idParts) != 2 {
		return "", "", fmt.Errorf("please make sure the ID is in the form BUCKET:NAME (i.e. my-bucket:EntireBucket")
	}
	bucket := idParts[0]
	name := idParts[1]
	return bucket, name, nil
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"flag"
	"fmt"
	"math/rand"
	"os"
)

var (
	numRead int
	readLen int
	numGene int
	geneLen int

	reads []string
)

func generateReads() {

	fid, err := os.Create("reads.fastq")
	if err != nil {
		panic(err)
	}
	defer fid.Close()

	var buf bytes.Buffer

	for i := 0; i < numRead; i++ {
		fid.WriteString(fmt.Sprintf("read_%d\n", i))

		buf.Reset()
		for j := 0; j < readLen; j++ {
			x := rand.Float64()
			switch {
			case x < 0.25:
				buf.Write([]byte("A"))
			case x < 0.5:
				buf.Write([]byte("T"))
			case x < 0.75:
				buf.Write([]byte("G"))
			default:
				buf.Write([]byte("C"))
			}
		}
		fid.Write(buf.Bytes())

		if i < 10 {
			reads = append(reads, string(buf.Bytes()))
		}

		fid.WriteString("\n+\n")
		for j := 0; j < readLen; j++ {
			fid.WriteString("!")
		}
		fid.WriteString("\n")
	}
}

func generateGenes() {

	fid, err := os.Create("genes.txt")
	if err != nil {
		panic(err)
	}
	defer fid.Close()

	for i := 0; i < numGene; i++ {
		fid.WriteString(fmt.Sprintf("gene_%d\t", i))

		m := geneLen
		if i < len(reads) {
			fid.WriteString(reads[i])
			m = geneLen - readLen
		}

		for j := 0; j < m; j++ {
			x := rand.Float64()
			switch {
			case x < 0.25:
				fid.WriteString("A")
			case x < 0.5:
				fid.WriteString("T")
			case x < 0.75:
				fid.WriteString("G")
			default:
				fid.WriteString("C")
			}
		}
		fid.WriteString("\n")
	}
}

func main() {

	flag.IntVar(&numRead, "NumRead", 10000, "Number of reads")
	flag.IntVar(&readLen, "ReadLen", 100, "Read length")
	flag.IntVar(&numGene, "NumGene", 10000, "Number of genes")
	flag.IntVar(&geneLen, "GeneLen", 1000, "Gene length")

	flag.Parse()

	generateReads()
	generateGenes()
}
<commit_msg>update data generation<commit_after>package main

import (
	"bytes"
	"flag"
	"fmt"
	"io"
	"math/rand"
	"os"
)

var (
	numRead int
	readLen int
	numGene int
	geneLen int

	reads []string
)

func generateReads() {

	fid, err := os.Create("reads.fastq")
	if err != nil {
		panic(err)
	}
	defer fid.Close()

	var buf bytes.Buffer

	for i := 0; i < numRead; i++ {
		fid.WriteString(fmt.Sprintf("read_%d\n", i))

		buf.Reset()
		for j := 0; j < readLen; j++ {
			x := rand.Float64()
			switch {
			case x < 0.25:
				buf.Write([]byte("A"))
			case x < 0.5:
				buf.Write([]byte("T"))
			case x < 0.75:
				buf.Write([]byte("G"))
			default:
				buf.Write([]byte("C"))
			}
		}
		fid.Write(buf.Bytes())

		if i < 10 {
			reads = append(reads, string(buf.Bytes()))
		}

		fid.WriteString("\n+\n")
		for j := 0; j < readLen; j++ {
			fid.WriteString("!")
		}
		fid.WriteString("\n")
	}
}

func writeRand(w io.Writer, n int) {

	for j := 0; j < n; j++ {
		x := rand.Float64()
		switch {
		case x < 0.25:
			w.Write([]byte("A"))
		case x < 0.5:
			w.Write([]byte("T"))
		case x < 0.75:
			w.Write([]byte("G"))
		default:
			w.Write([]byte("C"))
		}
	}
}

func generateGenes() {

	fid, err := os.Create("genes.txt")
	if err != nil {
		panic(err)
	}
	defer fid.Close()

	for i := 0; i < numGene; i++ {
		fid.WriteString(fmt.Sprintf("gene_%d\t", i))

		if i < len(reads) {
			writeRand(fid, i)
			fid.WriteString(reads[i])
			writeRand(fid, geneLen-(readLen+i))
		} else {
			writeRand(fid, geneLen)
		}
		fid.WriteString("\n")
	}
}

func main() {

	flag.IntVar(&numRead, "NumRead", 10000, "Number of reads")
	flag.IntVar(&readLen, "ReadLen", 100, "Read length")
	flag.IntVar(&numGene, "NumGene", 10000, "Number of genes")
	flag.IntVar(&geneLen, "GeneLen", 1000, "Gene length")

	flag.Parse()

	generateReads()
	generateGenes()
}
<|endoftext|>


<commit_before>package peco

import (
	"encoding/json"
	"fmt"
	"os"
	"unicode"

	"github.com/nsf/termbox-go"
)

type KeymapHandler func(*Input, termbox.Event)
type Keymap map[termbox.Key]KeymapHandler
type KeymapStringKey string
type KeymapStringHandler string

// This map is populated using some magic numbers, which must match
// the values defined in termbox-go. Verification against the actual
// termbox constants are done in the test
var stringToKey = map[string]termbox.Key{}

func init() {
	fidx := 12
	for k := termbox.KeyF1; k > termbox.KeyF12; k-- {
		sk := fmt.Sprintf("F%d", fidx)
		stringToKey[sk] = k
		fidx--
	}

	names := []string{
		"Insert",
		"Delete",
		"Home",
		"End",
		"Pgup",
		"Pgdn",
		"ArrowUp",
		"ArrowDown",
		"ArrowLeft",
		"ArrowRight",
	}
	for i, n := range names {
		stringToKey[n] = termbox.Key(int(termbox.KeyF12) - (i + 1))
	}

	names = []string{
		"Left",
		"Middle",
		"Right",
	}
	for i, n := range names {
		sk := fmt.Sprintf("Mouse%s", n)
		stringToKey[sk] = termbox.Key(int(termbox.KeyArrowRight) - (i + 2))
	}

	whacky := [][]string{
		{"~", "2", "Space"},
		{"a"},
		{"b"},
		{"c"},
		{"d"},
		{"e"},
		{"f"},
		{"g"},
		{"h"},
		{"i"},
		{"j"},
		{"k"},
		{"l"},
		{"m"},
		{"n"},
		{"o"},
		{"p"},
		{"q"},
		{"r"},
		{"s"},
		{"t"},
		{"u"},
		{"v"},
		{"w"},
		{"x"},
		{"y"},
		{"z"},
		{"[", "3"},
		{"4", "\\"},
		{"5", "]"},
		{"6"},
		{"7", "/", "_"},
	}
	for i, list := range whacky {
		for _, n := range list {
			sk := fmt.Sprintf("C-%s", n)
			stringToKey[sk] = termbox.Key(int(termbox.KeyCtrlTilde) + i)
		}
	}

	stringToKey["BS"] = termbox.KeyBackspace
	stringToKey["Tab"] = termbox.KeyTab
	stringToKey["Enter"] = termbox.KeyEnter
	stringToKey["Esc"] = termbox.KeyEsc
	stringToKey["Space"] = termbox.KeySpace
	stringToKey["BS2"] = termbox.KeyBackspace2
	stringToKey["C-8"] = termbox.KeyCtrl8

	//	panic(fmt.Sprintf("%#q", stringToKey))
}

func handleAcceptChar(i *Input, ev termbox.Event) {
	if ev.Key == termbox.KeySpace {
		ev.Ch = ' '
	}

	if ev.Ch > 0 {
		if len(i.query) == i.caretPos {
			i.query = append(i.query, ev.Ch)
		} else {
			buf := make([]rune, len(i.query)+1)
			copy(buf, i.query[:i.caretPos])
			buf[i.caretPos] = ev.Ch
			copy(buf[i.caretPos+1:], i.query[i.caretPos:])
			i.query = buf
		}
		i.caretPos++
		i.ExecQuery(string(i.query))
	}
}

// peco.Finish -> end program, exit with success
func handleFinish(i *Input, _ termbox.Event) {
	if len(i.current) == 1 {
		i.result = i.current[0].line
	} else if i.selectedLine > 0 && i.selectedLine < len(i.current) {
		i.result = i.current[i.selectedLine-1].line
	}
	i.Finish()
}

// peco.Cancel -> end program, exit with failure
func handleCancel(i *Input, ev termbox.Event) {
	i.ExitStatus = 1
	i.Finish()
}

func handleSelectPrevious(i *Input, ev termbox.Event) {
	i.PagingCh() <- ToPrevLine
	i.DrawMatches(nil)
}

func handleSelectNext(i *Input, ev termbox.Event) {
	i.PagingCh() <- ToNextLine
	i.DrawMatches(nil)
}

func handleSelectPreviousPage(i *Input, ev termbox.Event) {
	i.PagingCh() <- ToPrevPage
	i.DrawMatches(nil)
}

func handleSelectNextPage(i *Input, ev termbox.Event) {
	i.PagingCh() <- ToNextPage
	i.DrawMatches(nil)
}

func handleForwardWord(i *Input, _ termbox.Event) {
	if i.caretPos >= len(i.query) {
		return
	}

	foundSpace := false
	for pos := i.caretPos; pos < len(i.query); pos++ {
		r := i.query[pos]
		if foundSpace {
			if !unicode.IsSpace(r) {
				i.caretPos = pos
				i.DrawMatches(nil)
				return
			}
		} else {
			if unicode.IsSpace(r) {
				foundSpace = true
			}
		}
	}

	// not found. just move to the end of the buffer
	i.caretPos = len(i.query)
	i.DrawMatches(nil)

}

func handleForwardChar(i *Input, _ termbox.Event) {
	if i.caretPos >= len(i.query) {
		return
	}
	i.caretPos++
	i.DrawMatches(nil)
}

func handleBackwardChar(i *Input, _ termbox.Event) {
	if i.caretPos <= 0 {
		return
	}
	i.caretPos--
	i.DrawMatches(nil)
}

func handleBeginningOfLine(i *Input, _ termbox.Event) {
	i.caretPos = 0
	i.DrawMatches(nil)
}

func handleEndOfLine(i *Input, _ termbox.Event) {
	i.caretPos = len(i.query)
	i.DrawMatches(nil)
}

func handleKillEndOfLine(i *Input, _ termbox.Event) {
	if len(i.query) <= i.caretPos {
		return
	}

	i.query = i.query[0:i.caretPos]
	i.DrawMatches(nil)
}

func handleDeleteForwardChar(i *Input, ev termbox.Event) {
	if len(i.query) == i.caretPos {
		return
	}

	buf := make([]rune, len(i.query)-1)
	copy(buf, i.query[:i.caretPos])
	copy(buf[i.caretPos:], i.query[i.caretPos+1:])
	i.query = buf
	if len(i.query) > 0 {
		i.ExecQuery(string(i.query))
		return
	}

	i.current = nil
	i.DrawMatches(nil)
}

func handleDeleteBackwardChar(i *Input, ev termbox.Event) {
	if len(i.query) <= 0 {
		return
	}

	switch i.caretPos {
	case 0:
		// No op
		return
	case len(i.query):
		i.query = i.query[:len(i.query)-1]
	default:
		buf := make([]rune, len(i.query)-1)
		copy(buf, i.query[:i.caretPos])
		copy(buf[i.caretPos-1:], i.query[i.caretPos:])
		i.query = buf
	}
	i.caretPos--
	if len(i.query) > 0 {
		i.ExecQuery(string(i.query))
		return
	}

	i.current = nil
	i.DrawMatches(nil)
}

func (ksk KeymapStringKey) ToKey() (k termbox.Key, err error) {
	k, ok := stringToKey[string(ksk)]
	if !ok {
		err = fmt.Errorf("No such key %s", ksk)
	}
	return
}

func (ksh KeymapStringHandler) ToHandler() (h KeymapHandler, err error) {
	switch ksh {
	case "peco.KillEndOfLine":
		h = handleKillEndOfLine
	case "peco.BeginningOfLine":
		h = handleBeginningOfLine
	case "peco.EndOfLine":
		h = handleEndOfLine
	case "peco.ForwardChar":
		h = handleForwardChar
	case "peco.BackwardChar":
		h = handleBackwardChar
	case "peco.ForwardWord":
		h = handleForwardWord
	case "peco.DeleteForwardChar":
		h = handleDeleteForwardChar
	case "peco.DeleteBackwardChar":
		h = handleDeleteBackwardChar
	case "peco.SelectPreviousPage":
		h = handleSelectPreviousPage
	case "peco.SelectNextPage":
		h = handleSelectNextPage
	case "peco.SelectPrevious":
		h = handleSelectPrevious
	case "peco.SelectNext":
		h = handleSelectNext
	case "peco.Finish":
		h = handleFinish
	case "peco.Cancel":
		h = handleCancel
	default:
		err = fmt.Errorf("No such handler %s", ksh)
	}
	return
}

func NewKeymap() Keymap {
	return Keymap{
		termbox.KeyEsc:        handleCancel,
		termbox.KeyEnter:      handleFinish,
		termbox.KeyArrowUp:    handleSelectPrevious,
		termbox.KeyCtrlK:      handleSelectPrevious,
		termbox.KeyArrowDown:  handleSelectNext,
		termbox.KeyCtrlJ:      handleSelectNext,
		termbox.KeyArrowLeft:  handleSelectPreviousPage,
		termbox.KeyArrowRight: handleSelectNextPage,
		termbox.KeyBackspace:  handleDeleteBackwardChar,
		termbox.KeyBackspace2: handleDeleteBackwardChar,
	}
}

func (km Keymap) Handler(k termbox.Key) KeymapHandler {
	h, ok := km[k]
	if ok {
		return h
	}
	return handleAcceptChar
}

func (km Keymap) UnmarshalJSON(buf []byte) error {
	raw := map[string]string{}
	if err := json.Unmarshal(buf, &raw); err != nil {
		return err
	}

	for ks, vs := range raw {
		k, err := KeymapStringKey(ks).ToKey()
		if err != nil {
			fmt.Fprintf(os.Stderr, "Unknown key %s", ks)
			continue
		}

		v, err := KeymapStringHandler(vs).ToHandler()
		if err != nil {
			fmt.Fprintf(os.Stderr, "Unknown handler %s", vs)
			continue
		}

		km[k] = v
	}

	return nil
}
<commit_msg>Implement BackwardWord<commit_after>package peco

import (
	"encoding/json"
	"fmt"
	"os"
	"unicode"

	"github.com/nsf/termbox-go"
)

type KeymapHandler func(*Input, termbox.Event)
type Keymap map[termbox.Key]KeymapHandler
type KeymapStringKey string
type KeymapStringHandler string

// This map is populated using some magic numbers, which must match
// the values defined in termbox-go. Verification against the actual
// termbox constants are done in the test
var stringToKey = map[string]termbox.Key{}

func init() {
	fidx := 12
	for k := termbox.KeyF1; k > termbox.KeyF12; k-- {
		sk := fmt.Sprintf("F%d", fidx)
		stringToKey[sk] = k
		fidx--
	}

	names := []string{
		"Insert",
		"Delete",
		"Home",
		"End",
		"Pgup",
		"Pgdn",
		"ArrowUp",
		"ArrowDown",
		"ArrowLeft",
		"ArrowRight",
	}
	for i, n := range names {
		stringToKey[n] = termbox.Key(int(termbox.KeyF12) - (i + 1))
	}

	names = []string{
		"Left",
		"Middle",
		"Right",
	}
	for i, n := range names {
		sk := fmt.Sprintf("Mouse%s", n)
		stringToKey[sk] = termbox.Key(int(termbox.KeyArrowRight) - (i + 2))
	}

	whacky := [][]string{
		{"~", "2", "Space"},
		{"a"},
		{"b"},
		{"c"},
		{"d"},
		{"e"},
		{"f"},
		{"g"},
		{"h"},
		{"i"},
		{"j"},
		{"k"},
		{"l"},
		{"m"},
		{"n"},
		{"o"},
		{"p"},
		{"q"},
		{"r"},
		{"s"},
		{"t"},
		{"u"},
		{"v"},
		{"w"},
		{"x"},
		{"y"},
		{"z"},
		{"[", "3"},
		{"4", "\\"},
		{"5", "]"},
		{"6"},
		{"7", "/", "_"},
	}
	for i, list := range whacky {
		for _, n := range list {
			sk := fmt.Sprintf("C-%s", n)
			stringToKey[sk] = termbox.Key(int(termbox.KeyCtrlTilde) + i)
		}
	}

	stringToKey["BS"] = termbox.KeyBackspace
	stringToKey["Tab"] = termbox.KeyTab
	stringToKey["Enter"] = termbox.KeyEnter
	stringToKey["Esc"] = termbox.KeyEsc
	stringToKey["Space"] = termbox.KeySpace
	stringToKey["BS2"] = termbox.KeyBackspace2
	stringToKey["C-8"] = termbox.KeyCtrl8

	//	panic(fmt.Sprintf("%#q", stringToKey))
}

func handleAcceptChar(i *Input, ev termbox.Event) {
	if ev.Key == termbox.KeySpace {
		ev.Ch = ' '
	}

	if ev.Ch > 0 {
		if len(i.query) == i.caretPos {
			i.query = append(i.query, ev.Ch)
		} else {
			buf := make([]rune, len(i.query)+1)
			copy(buf, i.query[:i.caretPos])
			buf[i.caretPos] = ev.Ch
			copy(buf[i.caretPos+1:], i.query[i.caretPos:])
			i.query = buf
		}
		i.caretPos++
		i.ExecQuery(string(i.query))
	}
}

// peco.Finish -> end program, exit with success
func handleFinish(i *Input, _ termbox.Event) {
	if len(i.current) == 1 {
		i.result = i.current[0].line
	} else if i.selectedLine > 0 && i.selectedLine < len(i.current) {
		i.result = i.current[i.selectedLine-1].line
	}
	i.Finish()
}

// peco.Cancel -> end program, exit with failure
func handleCancel(i *Input, ev termbox.Event) {
	i.ExitStatus = 1
	i.Finish()
}

func handleSelectPrevious(i *Input, ev termbox.Event) {
	i.PagingCh() <- ToPrevLine
	i.DrawMatches(nil)
}

func handleSelectNext(i *Input, ev termbox.Event) {
	i.PagingCh() <- ToNextLine
	i.DrawMatches(nil)
}

func handleSelectPreviousPage(i *Input, ev termbox.Event) {
	i.PagingCh() <- ToPrevPage
	i.DrawMatches(nil)
}

func handleSelectNextPage(i *Input, ev termbox.Event) {
	i.PagingCh() <- ToNextPage
	i.DrawMatches(nil)
}

func handleForwardWord(i *Input, _ termbox.Event) {
	if i.caretPos >= len(i.query) {
		return
	}

	foundSpace := false
	for pos := i.caretPos; pos < len(i.query); pos++ {
		r := i.query[pos]
		if foundSpace {
			if !unicode.IsSpace(r) {
				i.caretPos = pos
				i.DrawMatches(nil)
				return
			}
		} else {
			if unicode.IsSpace(r) {
				foundSpace = true
			}
		}
	}

	// not found. just move to the end of the buffer
	i.caretPos = len(i.query)
	i.DrawMatches(nil)

}

func handleBackwardWord(i *Input, _ termbox.Event) {
	if i.caretPos == 0 {
		return
	}

	if i.caretPos >= len(i.query) {
		i.caretPos--
	}

	// if we start from a whitespace-ish position, we should
	// rewind to the end of the previous word, and then do the
	// search all over again
SEARCH_PREV_WORD:
	if unicode.IsSpace(i.query[i.caretPos]) {
		for pos := i.caretPos; pos > 0; pos-- {
			if !unicode.IsSpace(i.query[pos]) {
				i.caretPos = pos
				break
			}
		}
	}

	// if we start from the first character of a word, we
	// should attempt to move back and search for the previous word
	if i.caretPos > 0 && unicode.IsSpace(i.query[i.caretPos-1]) {
		i.caretPos--
		goto SEARCH_PREV_WORD
	}

	// Now look for a space
	for pos := i.caretPos; pos > 0; pos-- {
		if unicode.IsSpace(i.query[pos]) {
			i.caretPos = pos + 1
			i.DrawMatches(nil)
			return
		}
	}

	// not found. just move to the beginning of the buffer
	i.caretPos = 0
	i.DrawMatches(nil)
}

func handleForwardChar(i *Input, _ termbox.Event) {
	if i.caretPos >= len(i.query) {
		return
	}
	i.caretPos++
	i.DrawMatches(nil)
}

func handleBackwardChar(i *Input, _ termbox.Event) {
	if i.caretPos <= 0 {
		return
	}
	i.caretPos--
	i.DrawMatches(nil)
}

func handleBeginningOfLine(i *Input, _ termbox.Event) {
	i.caretPos = 0
	i.DrawMatches(nil)
}

func handleEndOfLine(i *Input, _ termbox.Event) {
	i.caretPos = len(i.query)
	i.DrawMatches(nil)
}

func handleKillEndOfLine(i *Input, _ termbox.Event) {
	if len(i.query) <= i.caretPos {
		return
	}

	i.query = i.query[0:i.caretPos]
	i.DrawMatches(nil)
}

func handleDeleteForwardChar(i *Input, ev termbox.Event) {
	if len(i.query) == i.caretPos {
		return
	}

	buf := make([]rune, len(i.query)-1)
	copy(buf, i.query[:i.caretPos])
	copy(buf[i.caretPos:], i.query[i.caretPos+1:])
	i.query = buf
	if len(i.query) > 0 {
		i.ExecQuery(string(i.query))
		return
	}

	i.current = nil
	i.DrawMatches(nil)
}

func handleDeleteBackwardChar(i *Input, ev termbox.Event) {
	if len(i.query) <= 0 {
		return
	}

	switch i.caretPos {
	case 0:
		// No op
		return
	case len(i.query):
		i.query = i.query[:len(i.query)-1]
	default:
		buf := make([]rune, len(i.query)-1)
		copy(buf, i.query[:i.caretPos])
		copy(buf[i.caretPos-1:], i.query[i.caretPos:])
		i.query = buf
	}
	i.caretPos--
	if len(i.query) > 0 {
		i.ExecQuery(string(i.query))
		return
	}

	i.current = nil
	i.DrawMatches(nil)
}

func (ksk KeymapStringKey) ToKey() (k termbox.Key, err error) {
	k, ok := stringToKey[string(ksk)]
	if !ok {
		err = fmt.Errorf("No such key %s", ksk)
	}
	return
}

func (ksh KeymapStringHandler) ToHandler() (h KeymapHandler, err error) {
	switch ksh {
	case "peco.KillEndOfLine":
		h = handleKillEndOfLine
	case "peco.BeginningOfLine":
		h = handleBeginningOfLine
	case "peco.EndOfLine":
		h = handleEndOfLine
	case "peco.ForwardChar":
		h = handleForwardChar
	case "peco.BackwardChar":
		h = handleBackwardChar
	case "peco.ForwardWord":
		h = handleForwardWord
	case "peco.BackwardWord":
		h = handleBackwardWord
	case "peco.DeleteForwardChar":
		h = handleDeleteForwardChar
	case "peco.DeleteBackwardChar":
		h = handleDeleteBackwardChar
	case "peco.SelectPreviousPage":
		h = handleSelectPreviousPage
	case "peco.SelectNextPage":
		h = handleSelectNextPage
	case "peco.SelectPrevious":
		h = handleSelectPrevious
	case "peco.SelectNext":
		h = handleSelectNext
	case "peco.Finish":
		h = handleFinish
	case "peco.Cancel":
		h = handleCancel
	default:
		err = fmt.Errorf("No such handler %s", ksh)
	}
	return
}

func NewKeymap() Keymap {
	return Keymap{
		termbox.KeyEsc:        handleCancel,
		termbox.KeyEnter:      handleFinish,
		termbox.KeyArrowUp:    handleSelectPrevious,
		termbox.KeyCtrlK:      handleSelectPrevious,
		termbox.KeyArrowDown:  handleSelectNext,
		termbox.KeyCtrlJ:      handleSelectNext,
		termbox.KeyArrowLeft:  handleSelectPreviousPage,
		termbox.KeyArrowRight: handleSelectNextPage,
		termbox.KeyBackspace:  handleDeleteBackwardChar,
		termbox.KeyBackspace2: handleDeleteBackwardChar,
	}
}

func (km Keymap) Handler(k termbox.Key) KeymapHandler {
	h, ok := km[k]
	if ok {
		return h
	}
	return handleAcceptChar
}

func (km Keymap) UnmarshalJSON(buf []byte) error {
	raw := map[string]string{}
	if err := json.Unmarshal(buf, &raw); err != nil {
		return err
	}

	for ks, vs := range raw {
		k, err := KeymapStringKey(ks).ToKey()
		if err != nil {
			fmt.Fprintf(os.Stderr, "Unknown key %s", ks)
			continue
		}

		v, err := KeymapStringHandler(vs).ToHandler()
		if err != nil {
			fmt.Fprintf(os.Stderr, "Unknown handler %s", vs)
			continue
		}

		km[k] = v
	}

	return nil
}
<|endoftext|>


<commit_before>package dht

import (
	"bytes"
	"testing"

	context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context"

	ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go"
	ma "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr"

	ci "github.com/jbenet/go-ipfs/crypto"
	spipe "github.com/jbenet/go-ipfs/crypto/spipe"
	inet "github.com/jbenet/go-ipfs/net"
	mux "github.com/jbenet/go-ipfs/net/mux"
	netservice "github.com/jbenet/go-ipfs/net/service"
	peer "github.com/jbenet/go-ipfs/peer"
	u "github.com/jbenet/go-ipfs/util"

	"fmt"
	"time"
)

func setupDHT(t *testing.T, p *peer.Peer) *IpfsDHT {
	ctx := context.Background()

	peerstore := peer.NewPeerstore()

	dhts := netservice.NewService(nil) // nil handler for now, need to patch it
	if err := dhts.Start(ctx); err != nil {
		t.Fatal(err)
	}

	net, err := inet.NewIpfsNetwork(ctx, p, peerstore, &mux.ProtocolMap{
		mux.ProtocolID_Routing: dhts,
	})
	if err != nil {
		t.Fatal(err)
	}

	d := NewDHT(p, peerstore, net, dhts, ds.NewMapDatastore())
	dhts.Handler = d
	return d
}

func setupDHTS(n int, t *testing.T) ([]ma.Multiaddr, []*peer.Peer, []*IpfsDHT) {
	var addrs []ma.Multiaddr
	for i := 0; i < n; i++ {
		a, err := ma.NewMultiaddr(fmt.Sprintf("/ip4/127.0.0.1/tcp/%d", 5000+i))
		if err != nil {
			t.Fatal(err)
		}
		addrs = append(addrs, a)
	}

	var peers []*peer.Peer
	for i := 0; i < n; i++ {
		p := makePeer(addrs[i])
		peers = append(peers, p)
	}

	dhts := make([]*IpfsDHT, n)
	for i := 0; i < n; i++ {
		dhts[i] = setupDHT(t, peers[i])
	}

	return addrs, peers, dhts
}

func makePeer(addr ma.Multiaddr) *peer.Peer {
	p := new(peer.Peer)
	p.AddAddress(addr)
	sk, pk, err := ci.GenerateKeyPair(ci.RSA, 512)
	if err != nil {
		panic(err)
	}
	p.PrivKey = sk
	p.PubKey = pk
	id, err := spipe.IDFromPubKey(pk)
	if err != nil {
		panic(err)
	}

	p.ID = id
	return p
}

func TestPing(t *testing.T) {
	// t.Skip("skipping test to debug another")

	u.Debug = false
	addrA, err := ma.NewMultiaddr("/ip4/127.0.0.1/tcp/2222")
	if err != nil {
		t.Fatal(err)
	}
	addrB, err := ma.NewMultiaddr("/ip4/127.0.0.1/tcp/5678")
	if err != nil {
		t.Fatal(err)
	}

	peerA := makePeer(addrA)
	peerB := makePeer(addrB)

	dhtA := setupDHT(t, peerA)
	dhtB := setupDHT(t, peerB)

	defer dhtA.Halt()
	defer dhtB.Halt()
	defer dhtA.network.Close()
	defer dhtB.network.Close()

	_, err = dhtA.Connect(context.Background(), peerB)
	if err != nil {
		t.Fatal(err)
	}

	//Test that we can ping the node
	ctx, _ := context.WithTimeout(context.Background(), 5*time.Millisecond)
	err = dhtA.Ping(ctx, peerB)
	if err != nil {
		t.Fatal(err)
	}

	ctx, _ = context.WithTimeout(context.Background(), 5*time.Millisecond)
	err = dhtB.Ping(ctx, peerA)
	if err != nil {
		t.Fatal(err)
	}
}

func TestValueGetSet(t *testing.T) {
	// t.Skip("skipping test to debug another")

	u.Debug = false
	addrA, err := ma.NewMultiaddr("/ip4/127.0.0.1/tcp/1235")
	if err != nil {
		t.Fatal(err)
	}
	addrB, err := ma.NewMultiaddr("/ip4/127.0.0.1/tcp/5679")
	if err != nil {
		t.Fatal(err)
	}

	peerA := makePeer(addrA)
	peerB := makePeer(addrB)

	dhtA := setupDHT(t, peerA)
	dhtB := setupDHT(t, peerB)

	defer dhtA.Halt()
	defer dhtB.Halt()
	defer dhtA.network.Close()
	defer dhtB.network.Close()

	_, err = dhtA.Connect(context.Background(), peerB)
	if err != nil {
		t.Fatal(err)
	}

	ctxT, _ := context.WithTimeout(context.Background(), time.Second)
	dhtA.PutValue(ctxT, "hello", []byte("world"))

	ctxT, _ = context.WithTimeout(context.Background(), time.Second*2)
	val, err := dhtA.GetValue(ctxT, "hello")
	if err != nil {
		t.Fatal(err)
	}

	if string(val) != "world" {
		t.Fatalf("Expected 'world' got '%s'", string(val))
	}

	ctxT, _ = context.WithTimeout(context.Background(), time.Second*2)
	val, err = dhtB.GetValue(ctxT, "hello")
	if err != nil {
		t.Fatal(err)
	}

	if string(val) != "world" {
		t.Fatalf("Expected 'world' got '%s'", string(val))
	}
}

func TestProvides(t *testing.T) {
	// t.Skip("skipping test to debug another")

	u.Debug = false

	_, peers, dhts := setupDHTS(4, t)
	defer func() {
		for i := 0; i < 4; i++ {
			dhts[i].Halt()
			defer dhts[i].network.Close()
		}
	}()

	_, err := dhts[0].Connect(context.Background(), peers[1])
	if err != nil {
		t.Fatal(err)
	}

	_, err = dhts[1].Connect(context.Background(), peers[2])
	if err != nil {
		t.Fatal(err)
	}

	_, err = dhts[1].Connect(context.Background(), peers[3])
	if err != nil {
		t.Fatal(err)
	}

	err = dhts[3].putLocal(u.Key("hello"), []byte("world"))
	if err != nil {
		t.Fatal(err)
	}

	bits, err := dhts[3].getLocal(u.Key("hello"))
	if err != nil && bytes.Equal(bits, []byte("world")) {
		t.Fatal(err)
	}

	err = dhts[3].Provide(context.Background(), u.Key("hello"))
	if err != nil {
		t.Fatal(err)
	}

	time.Sleep(time.Millisecond * 60)

	ctxT, _ := context.WithTimeout(context.Background(), time.Second)
	provs, err := dhts[0].FindProviders(ctxT, u.Key("hello"))
	if err != nil {
		t.Fatal(err)
	}

	if len(provs) != 1 {
		t.Fatal("Didnt get back providers")
	}
}

func TestProvidesAsync(t *testing.T) {
	// t.Skip("skipping test to debug another")

	u.Debug = false

	_, peers, dhts := setupDHTS(4, t)
	defer func() {
		for i := 0; i < 4; i++ {
			dhts[i].Halt()
			defer dhts[i].network.Close()
		}
	}()

	_, err := dhts[0].Connect(context.Background(), peers[1])
	if err != nil {
		t.Fatal(err)
	}

	_, err = dhts[1].Connect(context.Background(), peers[2])
	if err != nil {
		t.Fatal(err)
	}

	_, err = dhts[1].Connect(context.Background(), peers[3])
	if err != nil {
		t.Fatal(err)
	}

	err = dhts[3].putLocal(u.Key("hello"), []byte("world"))
	if err != nil {
		t.Fatal(err)
	}

	bits, err := dhts[3].getLocal(u.Key("hello"))
	if err != nil && bytes.Equal(bits, []byte("world")) {
		t.Fatal(err)
	}

	err = dhts[3].Provide(context.Background(), u.Key("hello"))
	if err != nil {
		t.Fatal(err)
	}

	time.Sleep(time.Millisecond * 60)

	ctx, _ := context.WithTimeout(context.TODO(), time.Millisecond*300)
	provs := dhts[0].FindProvidersAsync(ctx, u.Key("hello"), 5)
	select {
	case p := <-provs:
		if !p.ID.Equal(dhts[3].self.ID) {
			t.Fatalf("got a provider, but not the right one. %s", p)
		}
	case <-ctx.Done():
		t.Fatal("Didnt get back providers")
	}
}

func TestLayeredGet(t *testing.T) {
	// t.Skip("skipping test to debug another")

	u.Debug = false
	_, peers, dhts := setupDHTS(4, t)
	defer func() {
		for i := 0; i < 4; i++ {
			dhts[i].Halt()
			defer dhts[i].network.Close()
		}
	}()

	_, err := dhts[0].Connect(context.Background(), peers[1])
	if err != nil {
		t.Fatalf("Failed to connect: %s", err)
	}

	_, err = dhts[1].Connect(context.Background(), peers[2])
	if err != nil {
		t.Fatal(err)
	}

	_, err = dhts[1].Connect(context.Background(), peers[3])
	if err != nil {
		t.Fatal(err)
	}

	err = dhts[3].putLocal(u.Key("hello"), []byte("world"))
	if err != nil {
		t.Fatal(err)
	}

	err = dhts[3].Provide(context.Background(), u.Key("hello"))
	if err != nil {
		t.Fatal(err)
	}

	time.Sleep(time.Millisecond * 60)

	ctxT, _ := context.WithTimeout(context.Background(), time.Second)
	val, err := dhts[0].GetValue(ctxT, u.Key("hello"))
	if err != nil {
		t.Fatal(err)
	}

	if string(val) != "world" {
		t.Fatal("Got incorrect value.")
	}

}

func TestFindPeer(t *testing.T) {
	// t.Skip("skipping test to debug another")

	u.Debug = false

	_, peers, dhts := setupDHTS(4, t)
	defer func() {
		for i := 0; i < 4; i++ {
			dhts[i].Halt()
			dhts[i].network.Close()
		}
	}()

	_, err := dhts[0].Connect(context.Background(), peers[1])
	if err != nil {
		t.Fatal(err)
	}

	_, err = dhts[1].Connect(context.Background(), peers[2])
	if err != nil {
		t.Fatal(err)
	}

	_, err = dhts[1].Connect(context.Background(), peers[3])
	if err != nil {
		t.Fatal(err)
	}

	ctxT, _ := context.WithTimeout(context.Background(), time.Second)
	p, err := dhts[0].FindPeer(ctxT, peers[2].ID)
	if err != nil {
		t.Fatal(err)
	}

	if p == nil {
		t.Fatal("Failed to find peer.")
	}

	if !p.ID.Equal(peers[2].ID) {
		t.Fatal("Didnt find expected peer.")
	}
}
<commit_msg>handler fixes for tests<commit_after>package dht

import (
	"bytes"
	"testing"

	context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context"

	ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go"
	ma "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr"

	ci "github.com/jbenet/go-ipfs/crypto"
	spipe "github.com/jbenet/go-ipfs/crypto/spipe"
	inet "github.com/jbenet/go-ipfs/net"
	mux "github.com/jbenet/go-ipfs/net/mux"
	netservice "github.com/jbenet/go-ipfs/net/service"
	peer "github.com/jbenet/go-ipfs/peer"
	u "github.com/jbenet/go-ipfs/util"

	"fmt"
	"time"
)

func setupDHT(t *testing.T, p *peer.Peer) *IpfsDHT {
	ctx := context.Background()

	peerstore := peer.NewPeerstore()

	dhts := netservice.NewService(nil) // nil handler for now, need to patch it
	if err := dhts.Start(ctx); err != nil {
		t.Fatal(err)
	}

	net, err := inet.NewIpfsNetwork(ctx, p, peerstore, &mux.ProtocolMap{
		mux.ProtocolID_Routing: dhts,
	})
	if err != nil {
		t.Fatal(err)
	}

	d := NewDHT(p, peerstore, net, dhts, ds.NewMapDatastore())
	dhts.SetHandler(d)
	return d
}

func setupDHTS(n int, t *testing.T) ([]ma.Multiaddr, []*peer.Peer, []*IpfsDHT) {
	var addrs []ma.Multiaddr
	for i := 0; i < n; i++ {
		a, err := ma.NewMultiaddr(fmt.Sprintf("/ip4/127.0.0.1/tcp/%d", 5000+i))
		if err != nil {
			t.Fatal(err)
		}
		addrs = append(addrs, a)
	}

	var peers []*peer.Peer
	for i := 0; i < n; i++ {
		p := makePeer(addrs[i])
		peers = append(peers, p)
	}

	dhts := make([]*IpfsDHT, n)
	for i := 0; i < n; i++ {
		dhts[i] = setupDHT(t, peers[i])
	}

	return addrs, peers, dhts
}

func makePeer(addr ma.Multiaddr) *peer.Peer {
	p := new(peer.Peer)
	p.AddAddress(addr)
	sk, pk, err := ci.GenerateKeyPair(ci.RSA, 512)
	if err != nil {
		panic(err)
	}
	p.PrivKey = sk
	p.PubKey = pk
	id, err := spipe.IDFromPubKey(pk)
	if err != nil {
		panic(err)
	}

	p.ID = id
	return p
}

func TestPing(t *testing.T) {
	// t.Skip("skipping test to debug another")

	u.Debug = false
	addrA, err := ma.NewMultiaddr("/ip4/127.0.0.1/tcp/2222")
	if err != nil {
		t.Fatal(err)
	}
	addrB, err := ma.NewMultiaddr("/ip4/127.0.0.1/tcp/5678")
	if err != nil {
		t.Fatal(err)
	}

	peerA := makePeer(addrA)
	peerB := makePeer(addrB)

	dhtA := setupDHT(t, peerA)
	dhtB := setupDHT(t, peerB)

	defer dhtA.Halt()
	defer dhtB.Halt()
	defer dhtA.network.Close()
	defer dhtB.network.Close()

	_, err = dhtA.Connect(context.Background(), peerB)
	if err != nil {
		t.Fatal(err)
	}

	//Test that we can ping the node
	ctx, _ := context.WithTimeout(context.Background(), 5*time.Millisecond)
	err = dhtA.Ping(ctx, peerB)
	if err != nil {
		t.Fatal(err)
	}

	ctx, _ = context.WithTimeout(context.Background(), 5*time.Millisecond)
	err = dhtB.Ping(ctx, peerA)
	if err != nil {
		t.Fatal(err)
	}
}

func TestValueGetSet(t *testing.T) {
	// t.Skip("skipping test to debug another")

	u.Debug = false
	addrA, err := ma.NewMultiaddr("/ip4/127.0.0.1/tcp/1235")
	if err != nil {
		t.Fatal(err)
	}
	addrB, err := ma.NewMultiaddr("/ip4/127.0.0.1/tcp/5679")
	if err != nil {
		t.Fatal(err)
	}

	peerA := makePeer(addrA)
	peerB := makePeer(addrB)

	dhtA := setupDHT(t, peerA)
	dhtB := setupDHT(t, peerB)

	defer dhtA.Halt()
	defer dhtB.Halt()
	defer dhtA.network.Close()
	defer dhtB.network.Close()

	_, err = dhtA.Connect(context.Background(), peerB)
	if err != nil {
		t.Fatal(err)
	}

	ctxT, _ := context.WithTimeout(context.Background(), time.Second)
	dhtA.PutValue(ctxT, "hello", []byte("world"))

	ctxT, _ = context.WithTimeout(context.Background(), time.Second*2)
	val, err := dhtA.GetValue(ctxT, "hello")
	if err != nil {
		t.Fatal(err)
	}

	if string(val) != "world" {
		t.Fatalf("Expected 'world' got '%s'", string(val))
	}

	ctxT, _ = context.WithTimeout(context.Background(), time.Second*2)
	val, err = dhtB.GetValue(ctxT, "hello")
	if err != nil {
		t.Fatal(err)
	}

	if string(val) != "world" {
		t.Fatalf("Expected 'world' got '%s'", string(val))
	}
}

func TestProvides(t *testing.T) {
	// t.Skip("skipping test to debug another")

	u.Debug = false

	_, peers, dhts := setupDHTS(4, t)
	defer func() {
		for i := 0; i < 4; i++ {
			dhts[i].Halt()
			defer dhts[i].network.Close()
		}
	}()

	_, err := dhts[0].Connect(context.Background(), peers[1])
	if err != nil {
		t.Fatal(err)
	}

	_, err = dhts[1].Connect(context.Background(), peers[2])
	if err != nil {
		t.Fatal(err)
	}

	_, err = dhts[1].Connect(context.Background(), peers[3])
	if err != nil {
		t.Fatal(err)
	}

	err = dhts[3].putLocal(u.Key("hello"), []byte("world"))
	if err != nil {
		t.Fatal(err)
	}

	bits, err := dhts[3].getLocal(u.Key("hello"))
	if err != nil && bytes.Equal(bits, []byte("world")) {
		t.Fatal(err)
	}

	err = dhts[3].Provide(context.Background(), u.Key("hello"))
	if err != nil {
		t.Fatal(err)
	}

	time.Sleep(time.Millisecond * 60)

	ctxT, _ := context.WithTimeout(context.Background(), time.Second)
	provs, err := dhts[0].FindProviders(ctxT, u.Key("hello"))
	if err != nil {
		t.Fatal(err)
	}

	if len(provs) != 1 {
		t.Fatal("Didnt get back providers")
	}
}

func TestProvidesAsync(t *testing.T) {
	// t.Skip("skipping test to debug another")

	u.Debug = false

	_, peers, dhts := setupDHTS(4, t)
	defer func() {
		for i := 0; i < 4; i++ {
			dhts[i].Halt()
			defer dhts[i].network.Close()
		}
	}()

	_, err := dhts[0].Connect(context.Background(), peers[1])
	if err != nil {
		t.Fatal(err)
	}

	_, err = dhts[1].Connect(context.Background(), peers[2])
	if err != nil {
		t.Fatal(err)
	}

	_, err = dhts[1].Connect(context.Background(), peers[3])
	if err != nil {
		t.Fatal(err)
	}

	err = dhts[3].putLocal(u.Key("hello"), []byte("world"))
	if err != nil {
		t.Fatal(err)
	}

	bits, err := dhts[3].getLocal(u.Key("hello"))
	if err != nil && bytes.Equal(bits, []byte("world")) {
		t.Fatal(err)
	}

	err = dhts[3].Provide(context.Background(), u.Key("hello"))
	if err != nil {
		t.Fatal(err)
	}

	time.Sleep(time.Millisecond * 60)

	ctx, _ := context.WithTimeout(context.TODO(), time.Millisecond*300)
	provs := dhts[0].FindProvidersAsync(ctx, u.Key("hello"), 5)
	select {
	case p := <-provs:
		if !p.ID.Equal(dhts[3].self.ID) {
			t.Fatalf("got a provider, but not the right one. %s", p)
		}
	case <-ctx.Done():
		t.Fatal("Didnt get back providers")
	}
}

func TestLayeredGet(t *testing.T) {
	// t.Skip("skipping test to debug another")

	u.Debug = false
	_, peers, dhts := setupDHTS(4, t)
	defer func() {
		for i := 0; i < 4; i++ {
			dhts[i].Halt()
			defer dhts[i].network.Close()
		}
	}()

	_, err := dhts[0].Connect(context.Background(), peers[1])
	if err != nil {
		t.Fatalf("Failed to connect: %s", err)
	}

	_, err = dhts[1].Connect(context.Background(), peers[2])
	if err != nil {
		t.Fatal(err)
	}

	_, err = dhts[1].Connect(context.Background(), peers[3])
	if err != nil {
		t.Fatal(err)
	}

	err = dhts[3].putLocal(u.Key("hello"), []byte("world"))
	if err != nil {
		t.Fatal(err)
	}

	err = dhts[3].Provide(context.Background(), u.Key("hello"))
	if err != nil {
		t.Fatal(err)
	}

	time.Sleep(time.Millisecond * 60)

	ctxT, _ := context.WithTimeout(context.Background(), time.Second)
	val, err := dhts[0].GetValue(ctxT, u.Key("hello"))
	if err != nil {
		t.Fatal(err)
	}

	if string(val) != "world" {
		t.Fatal("Got incorrect value.")
	}

}

func TestFindPeer(t *testing.T) {
	// t.Skip("skipping test to debug another")

	u.Debug = false

	_, peers, dhts := setupDHTS(4, t)
	defer func() {
		for i := 0; i < 4; i++ {
			dhts[i].Halt()
			dhts[i].network.Close()
		}
	}()

	_, err := dhts[0].Connect(context.Background(), peers[1])
	if err != nil {
		t.Fatal(err)
	}

	_, err = dhts[1].Connect(context.Background(), peers[2])
	if err != nil {
		t.Fatal(err)
	}

	_, err = dhts[1].Connect(context.Background(), peers[3])
	if err != nil {
		t.Fatal(err)
	}

	ctxT, _ := context.WithTimeout(context.Background(), time.Second)
	p, err := dhts[0].FindPeer(ctxT, peers[2].ID)
	if err != nil {
		t.Fatal(err)
	}

	if p == nil {
		t.Fatal("Failed to find peer.")
	}

	if !p.ID.Equal(peers[2].ID) {
		t.Fatal("Didnt find expected peer.")
	}
}
<|endoftext|>


<commit_before>package gocluster

import (
	"math"
	"math/rand"
	"time"
)

// Function to randomly initialize clusters from input dataset.
// After initializing k clusters, runs Lloyd's Algorithm to find clusters
func Km(calcDist func(p1, p2 interface{}) float64, updateClust func([]interface{}) interface{}) func([]interface{}, int, int) ([]interface{}, [][]interface{}, error) {
	return func(entities []interface{}, k, maxIters int) ([]interface{}, [][]interface{}, error) {
		// create new random generator to avoid using the shared global object's Mutex lock
		r := rand.New(rand.NewSource(time.Now().UnixNano()))
		numEntities := len(entities)

		// initialize entities
		centers := make([]interface{}, k)
		for i := 0; i < k; i++ {
			centers[i] = entities[r.Intn(numEntities)]
		}
		return lloydsAlgo(entities, centers, maxIters, calcDist, updateClust)
	}
}

func Kmpp(calcDist func(p1, p2 interface{}) float64, updateClust func([]interface{}) interface{}) func([]interface{}, int, int) ([]interface{}, [][]interface{}, error) {
	return func(entities []interface{}, k, maxIters int) ([]interface{}, [][]interface{}, error) {
		// create new random generator to avoid using the shared global object's Mutex lock
		r := rand.New(rand.NewSource(time.Now().UnixNano()))
		numEntities := len(entities)
		centers := make([]interface{}, k)
		d2 := make([]float64, numEntities) // distance squared
		centers[0] = entities[r.Intn(numEntities)]

		// iterate through rest of k to initialize other centers
		for i := 1; i < k; i++ {
			sum := 0.0
			for j := range entities {
				_, minDist := nearest(entities[j], centers[:i], calcDist)
				d2[j] = minDist * minDist
				sum += d2[j]
			}

			// use find random number less than sum then iterate through d2 until index is found
			target := r.Float64() * sum
			j := 0
			for sum = d2[0]; sum < target; sum += d2[j] {
				j++
			}
			centers[i] = entities[j]
		}
		return lloydsAlgo(entities, centers, maxIters, calcDist, updateClust)
	}
}

// Lloye's Algorithm for calculating k-means clustering
// called from Km
func lloydsAlgo(entities, centers []interface{}, maxIters int, calcDist func(p1, p2 interface{}) float64, updateClust func([]interface{}) interface{}) ([]interface{}, [][]interface{}, error) {
	numEntities := len(entities)

	// setup cluster groups
	clusters := make([][]interface{}, len(centers))
	prevVariance := math.MaxFloat64

	for i := 0; i < maxIters; i++ {
		// setup clusters
		for i := range centers {
			clusters[i] = make([]interface{}, 0, 10)
		}
		curVariance := 0. // current variance
		for j := 0; j < numEntities; j++ {
			// get best cluster for each point
			minPt, minDist := nearest(entities[j], centers, calcDist)
			clusters[minPt] = append(clusters[minPt], entities[j])
			curVariance += minDist
		}

		if prevVariance == curVariance {
			return centers, clusters, nil
		}

		// compute new centers
		for j := range clusters {
			centers[j] = updateClust(clusters[j])
		}
		prevVariance = curVariance

	}
	return centers, clusters, nil
}

// fine nearest center to given point
func nearest(val interface{}, centers []interface{}, calcDist func(p1, p2 interface{}) float64) (int, float64) {
	minDist, minPt := math.MaxFloat64, 0
	for l := range centers {
		dist := calcDist(val, centers[l])
		if dist < minDist {
			minDist, minPt = dist, l
		}
	}
	return minPt, minDist
}
<commit_msg>Requires less frequent GC<commit_after>package gocluster

import (
	"math"
	"math/rand"
	"time"
)

// Function to randomly initialize clusters from input dataset.
// After initializing k clusters, runs Lloyd's Algorithm to find clusters
func Km(calcDist func(p1, p2 interface{}) float64, updateClust func([]interface{}) interface{}) func([]interface{}, int, int) ([]interface{}, [][]interface{}, error) {
	return func(entities []interface{}, k, maxIters int) ([]interface{}, [][]interface{}, error) {
		// create new random generator to avoid using the shared global object's Mutex lock
		r := rand.New(rand.NewSource(time.Now().UnixNano()))
		numEntities := len(entities)

		// initialize entities
		centers := make([]interface{}, k)
		for i := 0; i < k; i++ {
			centers[i] = entities[r.Intn(numEntities)]
		}
		return lloydsAlgo(entities, centers, maxIters, calcDist, updateClust)
	}
}

func Kmpp(calcDist func(p1, p2 interface{}) float64, updateClust func([]interface{}) interface{}) func([]interface{}, int, int) ([]interface{}, [][]interface{}, error) {
	return func(entities []interface{}, k, maxIters int) ([]interface{}, [][]interface{}, error) {
		// create new random generator to avoid using the shared global object's Mutex lock
		r := rand.New(rand.NewSource(time.Now().UnixNano()))
		numEntities := len(entities)
		centers := make([]interface{}, k)
		d2 := make([]float64, numEntities) // distance squared
		centers[0] = entities[r.Intn(numEntities)]

		// iterate through rest of k to initialize other centers
		for i := 1; i < k; i++ {
			sum := 0.0
			for j := range entities {
				_, minDist := nearest(entities[j], centers[:i], calcDist)
				d2[j] = minDist * minDist
				sum += d2[j]
			}

			// use find random number less than sum then iterate through d2 until index is found
			target := r.Float64() * sum
			j := 0
			for sum = d2[0]; sum < target; sum += d2[j] {
				j++
			}
			centers[i] = entities[j]
		}
		return lloydsAlgo(entities, centers, maxIters, calcDist, updateClust)
	}
}

// Lloye's Algorithm for calculating k-means clustering
// called from Km
func lloydsAlgo(entities, centers []interface{}, maxIters int, calcDist func(p1, p2 interface{}) float64, updateClust func([]interface{}) interface{}) ([]interface{}, [][]interface{}, error) {
	numEntities := len(entities)

	// setup cluster groups
	clusters := make([][]interface{}, len(centers))
	prevVariance := math.MaxFloat64

	// setup clusters
	for i := range centers {
		clusters[i] = make([]interface{}, 0, 10)
	}

	for i := 0; i < maxIters; i++ {
		// reset slices to zero
		for i := range centers {
			clusters[i] = clusters[i][:0]
		}

		curVariance := 0. // current variance
		for j := 0; j < numEntities; j++ {
			// get best cluster for each point
			minPt, minDist := nearest(entities[j], centers, calcDist)
			clusters[minPt] = append(clusters[minPt], entities[j])
			curVariance += minDist
		}

		if prevVariance == curVariance {
			return centers, clusters, nil
		}

		// compute new centers
		for j := range clusters {
			centers[j] = updateClust(clusters[j])
		}
		prevVariance = curVariance

	}
	return centers, clusters, nil
}

// fine nearest center to given point
func nearest(val interface{}, centers []interface{}, calcDist func(p1, p2 interface{}) float64) (int, float64) {
	minDist, minPt := math.MaxFloat64, 0
	for l := range centers {
		dist := calcDist(val, centers[l])
		if dist < minDist {
			minDist, minPt = dist, l
		}
	}
	return minPt, minDist
}
<|endoftext|>