<commit_before>package units

import (
	"fmt"
	"reflect"
	"runtime"
	"strings"
	"testing"
)

func ExampleBytesSize() {
	fmt.Println(BytesSize(1024))
	fmt.Println(BytesSize(1024 * 1024))
	fmt.Println(BytesSize(1048576))
	fmt.Println(BytesSize(2 * MiB))
	fmt.Println(BytesSize(3.42 * GiB))
	fmt.Println(BytesSize(5.372 * TiB))
	fmt.Println(BytesSize(2.22 * PiB))
}

func ExampleHumanSize() {
	fmt.Println(HumanSize(1000))
	fmt.Println(HumanSize(1024))
	fmt.Println(HumanSize(1000000))
	fmt.Println(HumanSize(1048576))
	fmt.Println(HumanSize(2 * MB))
	fmt.Println(HumanSize(float64(3.42 * GB)))
	fmt.Println(HumanSize(float64(5.372 * TB)))
	fmt.Println(HumanSize(float64(2.22 * PB)))
}

func ExampleFromHumanSize() {
	fmt.Println(FromHumanSize("32"))
	fmt.Println(FromHumanSize("32b"))
	fmt.Println(FromHumanSize("32B"))
	fmt.Println(FromHumanSize("32k"))
	fmt.Println(FromHumanSize("32K"))
	fmt.Println(FromHumanSize("32kb"))
	fmt.Println(FromHumanSize("32Kb"))
	fmt.Println(FromHumanSize("32Mb"))
	fmt.Println(FromHumanSize("32Gb"))
	fmt.Println(FromHumanSize("32Tb"))
	fmt.Println(FromHumanSize("32Pb"))
}

func ExampleRAMInBytes() {
	fmt.Println(RAMInBytes("32"))
	fmt.Println(RAMInBytes("32b"))
	fmt.Println(RAMInBytes("32B"))
	fmt.Println(RAMInBytes("32k"))
	fmt.Println(RAMInBytes("32K"))
	fmt.Println(RAMInBytes("32kb"))
	fmt.Println(RAMInBytes("32Kb"))
	fmt.Println(RAMInBytes("32Mb"))
	fmt.Println(RAMInBytes("32Gb"))
	fmt.Println(RAMInBytes("32Tb"))
	fmt.Println(RAMInBytes("32Pb"))
	fmt.Println(RAMInBytes("32PB"))
	fmt.Println(RAMInBytes("32P"))
}

func TestBytesSize(t *testing.T) {
	assertEquals(t, "1KiB", BytesSize(1024))
	assertEquals(t, "1MiB", BytesSize(1024*1024))
	assertEquals(t, "1MiB", BytesSize(1048576))
	assertEquals(t, "2MiB", BytesSize(2*MiB))
	assertEquals(t, "3.42GiB", BytesSize(3.42*GiB))
	assertEquals(t, "5.372TiB", BytesSize(5.372*TiB))
	assertEquals(t, "2.22PiB", BytesSize(2.22*PiB))
	assertEquals(t, "1.049e+06YiB", BytesSize(KiB*KiB*KiB*KiB*KiB*PiB))
}

func TestHumanSize(t *testing.T) {
	assertEquals(t, "1kB", HumanSize(1000))
	assertEquals(t, "1.024kB", HumanSize(1024))
	assertEquals(t, "1MB", HumanSize(1000000))
	assertEquals(t, "1.049MB", HumanSize(1048576))
	assertEquals(t, "2MB", HumanSize(2*MB))
	assertEquals(t, "3.42GB", HumanSize(float64(3.42*GB)))
	assertEquals(t, "5.372TB", HumanSize(float64(5.372*TB)))
	assertEquals(t, "2.22PB", HumanSize(float64(2.22*PB)))
	assertEquals(t, "1e+04YB", HumanSize(float64(10000000000000*PB)))
}

func TestFromHumanSize(t *testing.T) {
	assertSuccessEquals(t, 32, FromHumanSize, "32")
	assertSuccessEquals(t, 32, FromHumanSize, "32b")
	assertSuccessEquals(t, 32, FromHumanSize, "32B")
	assertSuccessEquals(t, 32*KB, FromHumanSize, "32k")
	assertSuccessEquals(t, 32*KB, FromHumanSize, "32K")
	assertSuccessEquals(t, 32*KB, FromHumanSize, "32kb")
	assertSuccessEquals(t, 32*KB, FromHumanSize, "32Kb")
	assertSuccessEquals(t, 32*MB, FromHumanSize, "32Mb")
	assertSuccessEquals(t, 32*GB, FromHumanSize, "32Gb")
	assertSuccessEquals(t, 32*TB, FromHumanSize, "32Tb")
	assertSuccessEquals(t, 32*PB, FromHumanSize, "32Pb")

	assertSuccessEquals(t, 32.5*KB, FromHumanSize, "32.5kB")
	assertSuccessEquals(t, 32.5*KB, FromHumanSize, "32.5 kB")
	assertSuccessEquals(t, 32, FromHumanSize, "32.5 B")
	assertSuccessEquals(t, 300, FromHumanSize, "0.3 K")

	assertError(t, FromHumanSize, "")
	assertError(t, FromHumanSize, "hello")
	assertError(t, FromHumanSize, "-32")
	assertError(t, FromHumanSize, ".3kB")
	assertError(t, FromHumanSize, " 32 ")
	assertError(t, FromHumanSize, "32m b")
	assertError(t, FromHumanSize, "32bm")
}

func TestRAMInBytes(t *testing.T) {
	assertSuccessEquals(t, 32, RAMInBytes, "32")
	assertSuccessEquals(t, 32, RAMInBytes, "32b")
	assertSuccessEquals(t, 32, RAMInBytes, "32B")
	assertSuccessEquals(t, 32*KiB, RAMInBytes, "32k")
	assertSuccessEquals(t, 32*KiB, RAMInBytes, "32K")
	assertSuccessEquals(t, 32*KiB, RAMInBytes, "32kb")
	assertSuccessEquals(t, 32*KiB, RAMInBytes, "32Kb")
	assertSuccessEquals(t, 32*KiB, RAMInBytes, "32Kib")
	assertSuccessEquals(t, 32*KiB, RAMInBytes, "32KIB")
	assertSuccessEquals(t, 32*MiB, RAMInBytes, "32Mb")
	assertSuccessEquals(t, 32*GiB, RAMInBytes, "32Gb")
	assertSuccessEquals(t, 32*TiB, RAMInBytes, "32Tb")
	assertSuccessEquals(t, 32*PiB, RAMInBytes, "32Pb")
	assertSuccessEquals(t, 32*PiB, RAMInBytes, "32PB")
	assertSuccessEquals(t, 32*PiB, RAMInBytes, "32P")

	assertSuccessEquals(t, 32, RAMInBytes, "32.3")
	tmp := 32.3 * MiB
	assertSuccessEquals(t, int64(tmp), RAMInBytes, "32.3 mb")
	tmp = 0.3 * MiB
	assertSuccessEquals(t, int64(tmp), RAMInBytes, "0.3MB")

	assertError(t, RAMInBytes, "")
	assertError(t, RAMInBytes, "hello")
	assertError(t, RAMInBytes, "-32")
	assertError(t, RAMInBytes, " 32 ")
	assertError(t, RAMInBytes, "32m b")
	assertError(t, RAMInBytes, "32bm")
}

func assertEquals(t *testing.T, expected, actual interface{}) {
	if expected != actual {
		t.Errorf("Expected '%v' but got '%v'", expected, actual)
	}
}

// func that maps to the parse function signatures as testing abstraction
type parseFn func(string) (int64, error)

// Define 'String()' for pretty-print
func (fn parseFn) String() string {
	fnName := runtime.FuncForPC(reflect.ValueOf(fn).Pointer()).Name()
	return fnName[strings.LastIndex(fnName, ".")+1:]
}

func assertSuccessEquals(t *testing.T, expected int64, fn parseFn, arg string) {
	res, err := fn(arg)
	if err != nil || res != expected {
		t.Errorf("%s(\"%s\") -> expected '%d' but got '%d' with error '%v'", fn, arg, expected, res, err)
	}
}

func assertError(t *testing.T, fn parseFn, arg string) {
	res, err := fn(arg)
	if err == nil && res != -1 {
		t.Errorf("%s(\"%s\") -> expected error but got '%d'", fn, arg, res)
	}
}
<commit_msg>size_test: add parseSize benchmark<commit_after>package units

import (
	"fmt"
	"reflect"
	"runtime"
	"strings"
	"testing"
)

func ExampleBytesSize() {
	fmt.Println(BytesSize(1024))
	fmt.Println(BytesSize(1024 * 1024))
	fmt.Println(BytesSize(1048576))
	fmt.Println(BytesSize(2 * MiB))
	fmt.Println(BytesSize(3.42 * GiB))
	fmt.Println(BytesSize(5.372 * TiB))
	fmt.Println(BytesSize(2.22 * PiB))
}

func ExampleHumanSize() {
	fmt.Println(HumanSize(1000))
	fmt.Println(HumanSize(1024))
	fmt.Println(HumanSize(1000000))
	fmt.Println(HumanSize(1048576))
	fmt.Println(HumanSize(2 * MB))
	fmt.Println(HumanSize(float64(3.42 * GB)))
	fmt.Println(HumanSize(float64(5.372 * TB)))
	fmt.Println(HumanSize(float64(2.22 * PB)))
}

func ExampleFromHumanSize() {
	fmt.Println(FromHumanSize("32"))
	fmt.Println(FromHumanSize("32b"))
	fmt.Println(FromHumanSize("32B"))
	fmt.Println(FromHumanSize("32k"))
	fmt.Println(FromHumanSize("32K"))
	fmt.Println(FromHumanSize("32kb"))
	fmt.Println(FromHumanSize("32Kb"))
	fmt.Println(FromHumanSize("32Mb"))
	fmt.Println(FromHumanSize("32Gb"))
	fmt.Println(FromHumanSize("32Tb"))
	fmt.Println(FromHumanSize("32Pb"))
}

func ExampleRAMInBytes() {
	fmt.Println(RAMInBytes("32"))
	fmt.Println(RAMInBytes("32b"))
	fmt.Println(RAMInBytes("32B"))
	fmt.Println(RAMInBytes("32k"))
	fmt.Println(RAMInBytes("32K"))
	fmt.Println(RAMInBytes("32kb"))
	fmt.Println(RAMInBytes("32Kb"))
	fmt.Println(RAMInBytes("32Mb"))
	fmt.Println(RAMInBytes("32Gb"))
	fmt.Println(RAMInBytes("32Tb"))
	fmt.Println(RAMInBytes("32Pb"))
	fmt.Println(RAMInBytes("32PB"))
	fmt.Println(RAMInBytes("32P"))
}

func TestBytesSize(t *testing.T) {
	assertEquals(t, "1KiB", BytesSize(1024))
	assertEquals(t, "1MiB", BytesSize(1024*1024))
	assertEquals(t, "1MiB", BytesSize(1048576))
	assertEquals(t, "2MiB", BytesSize(2*MiB))
	assertEquals(t, "3.42GiB", BytesSize(3.42*GiB))
	assertEquals(t, "5.372TiB", BytesSize(5.372*TiB))
	assertEquals(t, "2.22PiB", BytesSize(2.22*PiB))
	assertEquals(t, "1.049e+06YiB", BytesSize(KiB*KiB*KiB*KiB*KiB*PiB))
}

func TestHumanSize(t *testing.T) {
	assertEquals(t, "1kB", HumanSize(1000))
	assertEquals(t, "1.024kB", HumanSize(1024))
	assertEquals(t, "1MB", HumanSize(1000000))
	assertEquals(t, "1.049MB", HumanSize(1048576))
	assertEquals(t, "2MB", HumanSize(2*MB))
	assertEquals(t, "3.42GB", HumanSize(float64(3.42*GB)))
	assertEquals(t, "5.372TB", HumanSize(float64(5.372*TB)))
	assertEquals(t, "2.22PB", HumanSize(float64(2.22*PB)))
	assertEquals(t, "1e+04YB", HumanSize(float64(10000000000000*PB)))
}

func TestFromHumanSize(t *testing.T) {
	assertSuccessEquals(t, 32, FromHumanSize, "32")
	assertSuccessEquals(t, 32, FromHumanSize, "32b")
	assertSuccessEquals(t, 32, FromHumanSize, "32B")
	assertSuccessEquals(t, 32*KB, FromHumanSize, "32k")
	assertSuccessEquals(t, 32*KB, FromHumanSize, "32K")
	assertSuccessEquals(t, 32*KB, FromHumanSize, "32kb")
	assertSuccessEquals(t, 32*KB, FromHumanSize, "32Kb")
	assertSuccessEquals(t, 32*MB, FromHumanSize, "32Mb")
	assertSuccessEquals(t, 32*GB, FromHumanSize, "32Gb")
	assertSuccessEquals(t, 32*TB, FromHumanSize, "32Tb")
	assertSuccessEquals(t, 32*PB, FromHumanSize, "32Pb")

	assertSuccessEquals(t, 32.5*KB, FromHumanSize, "32.5kB")
	assertSuccessEquals(t, 32.5*KB, FromHumanSize, "32.5 kB")
	assertSuccessEquals(t, 32, FromHumanSize, "32.5 B")
	assertSuccessEquals(t, 300, FromHumanSize, "0.3 K")

	assertError(t, FromHumanSize, "")
	assertError(t, FromHumanSize, "hello")
	assertError(t, FromHumanSize, "-32")
	assertError(t, FromHumanSize, ".3kB")
	assertError(t, FromHumanSize, " 32 ")
	assertError(t, FromHumanSize, "32m b")
	assertError(t, FromHumanSize, "32bm")
}

func TestRAMInBytes(t *testing.T) {
	assertSuccessEquals(t, 32, RAMInBytes, "32")
	assertSuccessEquals(t, 32, RAMInBytes, "32b")
	assertSuccessEquals(t, 32, RAMInBytes, "32B")
	assertSuccessEquals(t, 32*KiB, RAMInBytes, "32k")
	assertSuccessEquals(t, 32*KiB, RAMInBytes, "32K")
	assertSuccessEquals(t, 32*KiB, RAMInBytes, "32kb")
	assertSuccessEquals(t, 32*KiB, RAMInBytes, "32Kb")
	assertSuccessEquals(t, 32*KiB, RAMInBytes, "32Kib")
	assertSuccessEquals(t, 32*KiB, RAMInBytes, "32KIB")
	assertSuccessEquals(t, 32*MiB, RAMInBytes, "32Mb")
	assertSuccessEquals(t, 32*GiB, RAMInBytes, "32Gb")
	assertSuccessEquals(t, 32*TiB, RAMInBytes, "32Tb")
	assertSuccessEquals(t, 32*PiB, RAMInBytes, "32Pb")
	assertSuccessEquals(t, 32*PiB, RAMInBytes, "32PB")
	assertSuccessEquals(t, 32*PiB, RAMInBytes, "32P")

	assertSuccessEquals(t, 32, RAMInBytes, "32.3")
	tmp := 32.3 * MiB
	assertSuccessEquals(t, int64(tmp), RAMInBytes, "32.3 mb")
	tmp = 0.3 * MiB
	assertSuccessEquals(t, int64(tmp), RAMInBytes, "0.3MB")

	assertError(t, RAMInBytes, "")
	assertError(t, RAMInBytes, "hello")
	assertError(t, RAMInBytes, "-32")
	assertError(t, RAMInBytes, " 32 ")
	assertError(t, RAMInBytes, "32m b")
	assertError(t, RAMInBytes, "32bm")
}

func BenchmarkParseSize(b *testing.B) {
	for i := 0; i < b.N; i++ {
		for _, s := range []string{
			"", "32", "32b", "32 B", "32k", "32.5 K", "32kb", "32 Kb",
			"32.8Mb", "32.9Gb", "32.777Tb", "32Pb", "0.3Mb", "-1",
		} {
			FromHumanSize(s)
			RAMInBytes(s)
		}
	}
}

func assertEquals(t *testing.T, expected, actual interface{}) {
	if expected != actual {
		t.Errorf("Expected '%v' but got '%v'", expected, actual)
	}
}

// func that maps to the parse function signatures as testing abstraction
type parseFn func(string) (int64, error)

// Define 'String()' for pretty-print
func (fn parseFn) String() string {
	fnName := runtime.FuncForPC(reflect.ValueOf(fn).Pointer()).Name()
	return fnName[strings.LastIndex(fnName, ".")+1:]
}

func assertSuccessEquals(t *testing.T, expected int64, fn parseFn, arg string) {
	res, err := fn(arg)
	if err != nil || res != expected {
		t.Errorf("%s(\"%s\") -> expected '%d' but got '%d' with error '%v'", fn, arg, expected, res, err)
	}
}

func assertError(t *testing.T, fn parseFn, arg string) {
	res, err := fn(arg)
	if err == nil && res != -1 {
		t.Errorf("%s(\"%s\") -> expected error but got '%d'", fn, arg, res)
	}
}
<|endoftext|>


<commit_before><commit_msg>minor refactor<commit_after><|endoftext|>


<commit_before>package shapeio_test

import (
	"bytes"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"testing"
	"time"

	"github.com/dustin/go-humanize"
	"github.com/fujiwara/shapeio"
)

var rates = []float64{
	500 * 1024,       // 500KB/sec
	1024 * 1024,      // 1MB/sec
	10 * 1024 * 1024, // 10MB/sec
	50 * 1024 * 1024, // 50MB/sec
}

var srcs = []*bytes.Reader{
	bytes.NewReader(bytes.Repeat([]byte{0}, 64*1024)),   // 64KB
	bytes.NewReader(bytes.Repeat([]byte{1}, 256*1024)),  // 256KB
	bytes.NewReader(bytes.Repeat([]byte{2}, 1024*1024)), // 1MB
}

func ExampleReader() {
	// example for downloading http body with rate limit.
	resp, _ := http.Get("http://example.com")
	defer resp.Body.Close()

	reader := shapeio.NewReader(resp.Body)
	reader.SetRateLimit(1024 * 10) // 10KB/sec
	io.Copy(ioutil.Discard, reader)
}

func ExampleWriter() {
	// example for writing file with rate limit.
	src := bytes.NewReader(bytes.Repeat([]byte{0}, 32*1024)) // 32KB
	f, _ := os.Create("/tmp/foo")
	writer := shapeio.NewWriter(f)
	writer.SetRateLimit(1024 * 10) // 10KB/sec
	io.Copy(writer, src)
	f.Close()
}

func TestRead(t *testing.T) {
	for _, src := range srcs {
		for _, limit := range rates {
			src.Seek(0, 0)
			sio := shapeio.NewReader(src)
			sio.SetRateLimit(limit)
			start := time.Now()
			n, err := io.Copy(ioutil.Discard, sio)
			elapsed := time.Since(start)
			if err != nil {
				t.Errorf("io.Copy failed", err)
			}
			realRate := float64(n) / elapsed.Seconds()
			if realRate > limit {
				t.Errorf("Limit %f but real rate %f", limit, realRate)
			}
			t.Logf(
				"read %s / %s: Real %s/sec Limit %s/sec. (%f %%)",
				humanize.IBytes(uint64(n)),
				elapsed,
				humanize.IBytes(uint64(realRate)),
				humanize.IBytes(uint64(limit)),
				realRate/limit*100,
			)
		}
	}
}

func TestWrite(t *testing.T) {
	for _, src := range srcs {
		for _, limit := range rates {
			src.Seek(0, 0)
			sio := shapeio.NewWriter(ioutil.Discard)
			sio.SetRateLimit(limit)
			start := time.Now()
			n, err := io.Copy(sio, src)
			elapsed := time.Since(start)
			if err != nil {
				t.Errorf("io.Copy failed", err)
			}
			realRate := float64(n) / elapsed.Seconds()
			if realRate > limit {
				t.Errorf("Limit %f but real rate %f", limit, realRate)
			}
			t.Logf(
				"write %s / %s: Real %s/sec Limit %s/sec. (%f %%)",
				humanize.IBytes(uint64(n)),
				elapsed,
				humanize.IBytes(uint64(realRate)),
				humanize.IBytes(uint64(limit)),
				realRate/limit*100,
			)
		}
	}
}
<commit_msg>change errorf to error when not using directives<commit_after>package shapeio_test

import (
	"bytes"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"testing"
	"time"

	"github.com/dustin/go-humanize"
	"github.com/fujiwara/shapeio"
)

var rates = []float64{
	500 * 1024,       // 500KB/sec
	1024 * 1024,      // 1MB/sec
	10 * 1024 * 1024, // 10MB/sec
	50 * 1024 * 1024, // 50MB/sec
}

var srcs = []*bytes.Reader{
	bytes.NewReader(bytes.Repeat([]byte{0}, 64*1024)),   // 64KB
	bytes.NewReader(bytes.Repeat([]byte{1}, 256*1024)),  // 256KB
	bytes.NewReader(bytes.Repeat([]byte{2}, 1024*1024)), // 1MB
}

func ExampleReader() {
	// example for downloading http body with rate limit.
	resp, _ := http.Get("http://example.com")
	defer resp.Body.Close()

	reader := shapeio.NewReader(resp.Body)
	reader.SetRateLimit(1024 * 10) // 10KB/sec
	io.Copy(ioutil.Discard, reader)
}

func ExampleWriter() {
	// example for writing file with rate limit.
	src := bytes.NewReader(bytes.Repeat([]byte{0}, 32*1024)) // 32KB
	f, _ := os.Create("/tmp/foo")
	writer := shapeio.NewWriter(f)
	writer.SetRateLimit(1024 * 10) // 10KB/sec
	io.Copy(writer, src)
	f.Close()
}

func TestRead(t *testing.T) {
	for _, src := range srcs {
		for _, limit := range rates {
			src.Seek(0, 0)
			sio := shapeio.NewReader(src)
			sio.SetRateLimit(limit)
			start := time.Now()
			n, err := io.Copy(ioutil.Discard, sio)
			elapsed := time.Since(start)
			if err != nil {
				t.Error("io.Copy failed", err)
			}
			realRate := float64(n) / elapsed.Seconds()
			if realRate > limit {
				t.Errorf("Limit %f but real rate %f", limit, realRate)
			}
			t.Logf(
				"read %s / %s: Real %s/sec Limit %s/sec. (%f %%)",
				humanize.IBytes(uint64(n)),
				elapsed,
				humanize.IBytes(uint64(realRate)),
				humanize.IBytes(uint64(limit)),
				realRate/limit*100,
			)
		}
	}
}

func TestWrite(t *testing.T) {
	for _, src := range srcs {
		for _, limit := range rates {
			src.Seek(0, 0)
			sio := shapeio.NewWriter(ioutil.Discard)
			sio.SetRateLimit(limit)
			start := time.Now()
			n, err := io.Copy(sio, src)
			elapsed := time.Since(start)
			if err != nil {
				t.Error("io.Copy failed", err)
			}
			realRate := float64(n) / elapsed.Seconds()
			if realRate > limit {
				t.Errorf("Limit %f but real rate %f", limit, realRate)
			}
			t.Logf(
				"write %s / %s: Real %s/sec Limit %s/sec. (%f %%)",
				humanize.IBytes(uint64(n)),
				elapsed,
				humanize.IBytes(uint64(realRate)),
				humanize.IBytes(uint64(limit)),
				realRate/limit*100,
			)
		}
	}
}
<|endoftext|>


<commit_before>package shell

import (
	"bufio"
	"fmt"
	"io"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"strings"
	"syscall"

	"github.com/zetamatta/go-texts/mbcs"

	"github.com/zetamatta/nyagos/dos"
)

func readEnv(scan *bufio.Scanner, verbose io.Writer) (int, error) {
	errorlevel := -1
	for scan.Scan() {
		line := strings.TrimSpace(scan.Text())
		eqlPos := strings.Index(line, "=")
		if eqlPos > 0 {
			left := line[:eqlPos]
			right := line[eqlPos+1:]
			if left == "ERRORLEVEL_" {
				value, err := strconv.ParseInt(right, 10, 32)
				if err != nil {
					if verbose != nil {
						fmt.Fprintf(verbose, "Could not read ERRORLEVEL(%s)\n", right)
					}
				} else {
					errorlevel = int(value)
				}
			} else {
				orig := os.Getenv(left)
				if verbose != nil {
					fmt.Fprintf(verbose, "%s=%s\n", left, right)
				}
				if orig != right {
					// fmt.Fprintf(os.Stderr, "%s:=%s\n", left, right)
					os.Setenv(left, right)
				}
			}
		}
	}
	return errorlevel, scan.Err()
}

func readPwd(scan *bufio.Scanner, verbose io.Writer) error {
	if !scan.Scan() {
		if err := scan.Err(); err != nil {
			return err
		} else {
			return io.EOF
		}
	}
	line := strings.TrimSpace(scan.Text())
	if verbose != nil {
		fmt.Fprintf(verbose, "cd \"%s\"\n", line)
	}
	os.Chdir(line)
	return nil
}

// loadTmpFile - read update the current-directory and environment-variables from tmp-file.
func loadTmpFile(fname string, verbose io.Writer) (int, error) {
	fp, err := os.Open(fname)
	if err != nil {
		return -1, err
	}
	defer fp.Close()

	scan := bufio.NewScanner(mbcs.NewAtoUReader(fp, mbcs.ConsoleCP()))
	if err := readPwd(scan, verbose); err != nil {
		return -1, err
	}
	return readEnv(scan, verbose)
}

func callBatch(
	args []string,
	tmpfile string,
	verbose io.Writer,
	stdin io.Reader,
	stdout io.Writer,
	stderr io.Writer) (int, error) {

	var cmdline strings.Builder

	cmdline.WriteString(`/V:ON /S /C "call`)
	for _, arg1 := range args {
		cmdline.WriteByte(' ')
		cmdline.WriteString(arg1)
	}
	cmdline.WriteString(` & set "ERRORLEVEL_=!ERRORLEVEL!" & (cd & set) > "`)
	cmdline.WriteString(tmpfile)
	cmdline.WriteString(`" "`)

	cmd := exec.Cmd{
		Path:        os.Getenv("COMSPEC"),
		Stdin:       stdin,
		Stdout:      stdout,
		Stderr:      stderr,
		SysProcAttr: &syscall.SysProcAttr{CmdLine: cmdline.String()},
	}
	if err := cmd.Run(); err != nil {
		return 1, err
	}
	errorlevel, errorlevelOk := dos.GetErrorLevel(&cmd)
	if !errorlevelOk {
		errorlevel = 255
	}
	return errorlevel, nil
}

// RawSource calls the batchfiles and load the changed variable the batchfile has done.
func RawSource(args []string, verbose io.Writer, debug bool, stdin io.Reader, stdout io.Writer, stderr io.Writer) (int, error) {
	tempDir := os.TempDir()
	pid := os.Getpid()
	tmpfile := filepath.Join(tempDir, fmt.Sprintf("nyagos-%d.tmp", pid))

	errorlevel, err := callBatch(
		args,
		tmpfile,
		verbose,
		stdin,
		stdout,
		stderr)

	if err != nil {
		return errorlevel, err
	}

	if !debug {
		defer os.Remove(tmpfile)
	}

	if errorlevel, err = loadTmpFile(tmpfile, verbose); err != nil {
		if os.IsNotExist(err) {
			return 1, fmt.Errorf("%s: the batch file may use `exit` without `/b` option. Could not find the change of the environment variables", args[0])
		}
		return 1, err
	}
	if err != nil {
		return errorlevel, err
	}
	if errorlevel != 0 {
		return errorlevel, fmt.Errorf("exit status %d", errorlevel)
	}
	return 0, nil
}
<commit_msg>(#343) When %COMSPEC% is empty, use CMD.EXE<commit_after>package shell

import (
	"bufio"
	"fmt"
	"io"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"strings"
	"syscall"

	"github.com/zetamatta/go-texts/mbcs"

	"github.com/zetamatta/nyagos/dos"
)

func readEnv(scan *bufio.Scanner, verbose io.Writer) (int, error) {
	errorlevel := -1
	for scan.Scan() {
		line := strings.TrimSpace(scan.Text())
		eqlPos := strings.Index(line, "=")
		if eqlPos > 0 {
			left := line[:eqlPos]
			right := line[eqlPos+1:]
			if left == "ERRORLEVEL_" {
				value, err := strconv.ParseInt(right, 10, 32)
				if err != nil {
					if verbose != nil {
						fmt.Fprintf(verbose, "Could not read ERRORLEVEL(%s)\n", right)
					}
				} else {
					errorlevel = int(value)
				}
			} else {
				orig := os.Getenv(left)
				if verbose != nil {
					fmt.Fprintf(verbose, "%s=%s\n", left, right)
				}
				if orig != right {
					// fmt.Fprintf(os.Stderr, "%s:=%s\n", left, right)
					os.Setenv(left, right)
				}
			}
		}
	}
	return errorlevel, scan.Err()
}

func readPwd(scan *bufio.Scanner, verbose io.Writer) error {
	if !scan.Scan() {
		if err := scan.Err(); err != nil {
			return err
		} else {
			return io.EOF
		}
	}
	line := strings.TrimSpace(scan.Text())
	if verbose != nil {
		fmt.Fprintf(verbose, "cd \"%s\"\n", line)
	}
	os.Chdir(line)
	return nil
}

// loadTmpFile - read update the current-directory and environment-variables from tmp-file.
func loadTmpFile(fname string, verbose io.Writer) (int, error) {
	fp, err := os.Open(fname)
	if err != nil {
		return -1, err
	}
	defer fp.Close()

	scan := bufio.NewScanner(mbcs.NewAtoUReader(fp, mbcs.ConsoleCP()))
	if err := readPwd(scan, verbose); err != nil {
		return -1, err
	}
	return readEnv(scan, verbose)
}

func callBatch(
	args []string,
	tmpfile string,
	verbose io.Writer,
	stdin io.Reader,
	stdout io.Writer,
	stderr io.Writer) (int, error) {

	var cmdline strings.Builder

	cmdline.WriteString(`/V:ON /S /C "call`)
	for _, arg1 := range args {
		cmdline.WriteByte(' ')
		cmdline.WriteString(arg1)
	}
	cmdline.WriteString(` & set "ERRORLEVEL_=!ERRORLEVEL!" & (cd & set) > "`)
	cmdline.WriteString(tmpfile)
	cmdline.WriteString(`" "`)

	cmdexe := os.Getenv("COMSPEC")

	if cmdexe == "" {
		cmdexe = "cmd.exe"
	}

	cmd := exec.Cmd{
		Path:        cmdexe,
		Stdin:       stdin,
		Stdout:      stdout,
		Stderr:      stderr,
		SysProcAttr: &syscall.SysProcAttr{CmdLine: cmdline.String()},
	}
	if err := cmd.Run(); err != nil {
		return 1, err
	}
	errorlevel, errorlevelOk := dos.GetErrorLevel(&cmd)
	if !errorlevelOk {
		errorlevel = 255
	}
	return errorlevel, nil
}

// RawSource calls the batchfiles and load the changed variable the batchfile has done.
func RawSource(args []string, verbose io.Writer, debug bool, stdin io.Reader, stdout io.Writer, stderr io.Writer) (int, error) {
	tempDir := os.TempDir()
	pid := os.Getpid()
	tmpfile := filepath.Join(tempDir, fmt.Sprintf("nyagos-%d.tmp", pid))

	errorlevel, err := callBatch(
		args,
		tmpfile,
		verbose,
		stdin,
		stdout,
		stderr)

	if err != nil {
		return errorlevel, err
	}

	if !debug {
		defer os.Remove(tmpfile)
	}

	if errorlevel, err = loadTmpFile(tmpfile, verbose); err != nil {
		if os.IsNotExist(err) {
			return 1, fmt.Errorf("%s: the batch file may use `exit` without `/b` option. Could not find the change of the environment variables", args[0])
		}
		return 1, err
	}
	if err != nil {
		return errorlevel, err
	}
	if errorlevel != 0 {
		return errorlevel, fmt.Errorf("exit status %d", errorlevel)
	}
	return 0, nil
}
<|endoftext|>


<commit_before>package slug

import (
	"testing"
)

var (
	generateTests = []struct {
		iS string
		oS string
	}{
		{"", ""},
		{" ", ""},
		{"a lovely day", "a-lovely-day"},
		{"A Lovely Day", "a-lovely-day"},
		{"---a lovely day---", "a-lovely-day"},
		{"wo ai 北京", "wo-ai-bei-jing"},
		{"Antonín Dvořák", "antonin-dvorak"},
		{"Schindler's List", "schindlers-list"},
	}

	generateWithTests = []struct {
		iS     string
		iChars string
		iSep   string
		iL     int
		oS     string
	}{
		{"a lovely day", "", "-", 100, "alovelyday"},
		{"a lovely day", " ", "*123*", 100, "a*123*lovely*123*day"},
		{"a lovely day", " ", "_", 10, "a_lovely_d"},
		{"a lovely day", " ", "_", 9, "a_lovely"},
		{"a lovely day", " ", "_", 8, "a_lovely"},
		{"wo ai 北京", " ", "-", 12, "wo-ai-bei-ji"},
		{"wo ai 北京", " ", "-", 10, "wo-ai-bei"},
		{"wo ai 北京", " ", "-", 7, "wo-ai-b"},
		{"wo ai 北京", " ", "-", 1, "w"},
		{"-a -- b__c", " -", "-", 50, "a-bc"},
		{"Schindler's List", " ", "-", 100, "schindlers-list"},
		{"Schindler's List", " ", "-", 10, "schindlers"},
		{"Schindler's List", " '", "-", 100, "schindler-s-list"},
		{"Schindler's List", " '", "-", 10, "schindler"},
	}
)

func TestGenerate(t *testing.T) {
	for _, test := range generateTests {
		if out := Generate(test.iS); out != test.oS {
			t.Errorf("For input string %q, expected output %q (len %d), was %q (len %d)",
				test.iS, test.oS, len(test.oS), out, len(out))
		}
	}
}

func TestGenerateWith(t *testing.T) {
	for _, test := range generateWithTests {
		if out := GenerateWith(test.iS, test.iChars, test.iSep, test.iL); out != test.oS {
			t.Errorf("For input string %q, chars %q, separator %q and maximum length %d, "+
				"expected output %q (len %d), was %q (len %d)",
				test.iS, test.iChars, test.iSep, test.iL,
				test.oS, len(test.oS), out, len(out))
		}
	}
}
<commit_msg>Cleanup tests<commit_after>package slug

import "testing"

func TestGenerate(t *testing.T) {
	tests := []struct {
		iS string
		oS string
	}{
		{"", ""},
		{" ", ""},
		{"a lovely day", "a-lovely-day"},
		{"A Lovely Day", "a-lovely-day"},
		{"---a lovely day---", "a-lovely-day"},
		{"wo ai 北京", "wo-ai-bei-jing"},
		{"Antonín Dvořák", "antonin-dvorak"},
		{"Schindler's List", "schindlers-list"},
	}

	for _, test := range tests {
		if out := Generate(test.iS); out != test.oS {
			t.Errorf("For input string %q, expected output %q (len %d), was %q (len %d)",
				test.iS, test.oS, len(test.oS), out, len(out))
		}
	}
}

func TestGenerateWith(t *testing.T) {
	tests := []struct {
		iS     string
		iChars string
		iSep   string
		iL     int
		oS     string
	}{
		{"a lovely day", "", "-", 100, "alovelyday"},
		{"a lovely day", " ", "*123*", 100, "a*123*lovely*123*day"},
		{"a lovely day", " ", "_", 10, "a_lovely_d"},
		{"a lovely day", " ", "_", 9, "a_lovely"},
		{"a lovely day", " ", "_", 8, "a_lovely"},
		{"wo ai 北京", " ", "-", 12, "wo-ai-bei-ji"},
		{"wo ai 北京", " ", "-", 10, "wo-ai-bei"},
		{"wo ai 北京", " ", "-", 7, "wo-ai-b"},
		{"wo ai 北京", " ", "-", 1, "w"},
		{"-a -- b__c", " -", "-", 50, "a-bc"},
		{"Schindler's List", " ", "-", 100, "schindlers-list"},
		{"Schindler's List", " ", "-", 10, "schindlers"},
		{"Schindler's List", " '", "-", 100, "schindler-s-list"},
		{"Schindler's List", " '", "-", 10, "schindler"},
	}

	for _, test := range tests {
		if out := GenerateWith(test.iS, test.iChars, test.iSep, test.iL); out != test.oS {
			t.Errorf("For input string %q, chars %q, separator %q and maximum length %d, "+
				"expected output %q (len %d), was %q (len %d)",
				test.iS, test.iChars, test.iSep, test.iL,
				test.oS, len(test.oS), out, len(out))
		}
	}
}
<|endoftext|>


<commit_before>package slack

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"

	"../timeline"
	"github.com/pkg/errors"

	"golang.org/x/net/websocket"
)

var rtmStartURL = "https://slack.com/api/rtm.start"

var slackAPIEndpoint = "https://slack.com/api/"

var origin = "http://localhost"

type rtmStartResponse struct {
	OK    bool   `json:"ok"`
	URL   string `json:"url"`
	Error string `json:"error"`
}

type deletedEvent struct {
	ChannelID string       `json:"channel"`
	Message   SlackMessage `json:"previous_message"`
}

type SlackMessage struct {
	Raw       string `json:"-"`
	Type      string `json:"type"`
	UserID    string `json:"user"`
	Text      string `json:"text"`
	ChannelID string `json:"channel"`
	TimeStamp string `json:"ts"`
	SubType   string `json:"subtype"`
}

func (m *SlackMessage) ToInternal() timeline.Message {
	return timeline.NewMessage(m.Text, m.UserID, m.ChannelID, m.TimeStamp)
}

type userListResponse struct {
	OK    bool   `json:"ok"`
	User  User   `json:"user"`
	Error string `json:"error"`
}

type allUserResponse struct {
	OK      bool   `json:"ok"`
	Members []User `json:"members"`
	Error   string `json:"error"`
}

type User struct {
	ID      string  `json:"id"`
	Name    string  `json:"name"`
	Profile profile `json:"profile"`
}

func (u User) ToInternal() timeline.User {
	return timeline.NewUser(u.ID, u.Name, u.Profile.ImageURL)
}

type profile struct {
	ImageURL string `json:"image_48"`
}

type channelCreated struct {
	Type    string  `json:"type"`
	Channel channel `json:"channel"`
}

type channel struct {
	ID      string `json:"id"`
	Name    string `json:"name"`
	Creator string `json:"creator"`
}

type SlackClient struct {
	Token string
}

func isValidJson(b []byte) bool {
	j := map[string]interface{}{}
	return json.Unmarshal(b, &j) == nil
}

func (cli *SlackClient) connectToRTM() (*websocket.Conn, error) {
	v := url.Values{
		"token": {cli.Token},
	}
	resp, e := http.Get(rtmStartURL + "?" + v.Encode())
	if e != nil {
		return nil, e
	}
	defer resp.Body.Close()
	byteArray, e := ioutil.ReadAll(resp.Body)
	if e != nil {
		return nil, e
	}
	res := rtmStartResponse{}
	e = json.Unmarshal(byteArray, &res)
	if e != nil {
		return nil, e
	}
	if !res.OK {
		return nil, fmt.Errorf(res.Error)
	}
	ws, e := websocket.Dial(res.URL, "", origin)
	if e != nil {
		return nil, e
	}
	return ws, nil
}

func receive(ws *websocket.Conn) ([]byte, error) {
	var msg = make([]byte, 4096)
	n, e := ws.Read(msg)
	if e != nil {
		return nil, e
	}
	return msg[:n], nil
}

func (cli *SlackClient) Polling(
	messageChan, deletedMessageChan chan *timeline.Message,
	warnChan, errorChan chan error,
	restartChan chan bool,
) {
	ws, e := cli.connectToRTM()
	if e != nil {
		errorChan <- e
		return
	}
	defer ws.Close()
	prev := make([]byte, 0)
	for {
		received, e := receive(ws)
		if e != nil {
			errorChan <- e
		}
		msg := append(prev, received...)
		if !isValidJson(msg) {
			prev = msg
			continue
		}
		prev = make([]byte, 0)
		message := SlackMessage{}
		errOnMessage := json.Unmarshal(msg, &message)

		if errOnMessage != nil {
			event := channelCreated{}
			errOnEvent := json.Unmarshal(msg, &event)
			if errOnEvent != nil && errOnMessage {
				warnChan <- errors.Wrap(errOnMessage, fmt.Sprintf("failed to unmarshal to message. json: '%s'", msg))
				warnChan <- errors.Wrap(errOnEvent, fmt.Sprintf("failed to unmarshal to channel created. json: '%s'", msg))
				continue
			}
			restartChan <- true
			return
		}

		message.Raw = string(msg)
		if message.Type != "message" {
			warnChan <- fmt.Errorf("not message: '%+v'", message)
			continue
		}

		switch message.SubType {
		case "":
			m := message.ToInternal()
			messageChan <- &m
		case "message_deleted":
			d := deletedEvent{}
			e := json.Unmarshal([]byte(msg), &d)
			if e != nil {
				warnChan <- errors.Wrap(err, fmt.Sprintf("failed to unmarshal to deleted event. json: '%s'", msg))
				continue
			}
			d.Message.ChannelID = d.ChannelID
			m := d.Message.ToInternal()
			deletedMessageChan <- &m
		}
	}
}

func (cli *SlackClient) postMessage(channelID, text, userName, iconURL string) ([]byte, error) {
	res, e := http.PostForm(slackAPIEndpoint+"chat.postMessage", url.Values{
		"token":      {cli.Token},
		"channel":    {channelID},
		"text":       {text},
		"username":   {userName},
		"as_user":    {"false"},
		"icon_url":   {iconURL},
		"link_names": {"0"},
	})
	if e != nil {
		return nil, e
	}
	defer res.Body.Close()
	byteArray, e := ioutil.ReadAll(res.Body)
	if e != nil {
		return nil, e
	}
	return byteArray, nil
}

func (cli *SlackClient) getUser(userID string) (*User, error) {
	res, e := http.PostForm(slackAPIEndpoint+"users.info", url.Values{
		"token": {cli.Token},
		"user":  {userID},
	})
	if e != nil {
		return nil, e
	}
	defer res.Body.Close()
	b, e := ioutil.ReadAll(res.Body)
	if e != nil {
		return nil, e
	}
	r := userListResponse{}
	e = json.Unmarshal(b, &r)
	if e != nil {
		return nil, e
	}
	if !r.OK {
		return nil, fmt.Errorf(r.Error)
	}
	u := r.User
	return &u, nil
}

func (cli *SlackClient) getAllUsers() ([]User, error) {
	res, e := http.PostForm(slackAPIEndpoint+"users.list", url.Values{
		"token": {cli.Token},
	})
	if e != nil {
		return nil, e
	}
	defer res.Body.Close()
	b, e := ioutil.ReadAll(res.Body)
	if e != nil {
		return nil, e
	}
	r := allUserResponse{}
	e = json.Unmarshal(b, &r)
	if e != nil {
		return nil, e
	}
	if !r.OK {
		return nil, fmt.Errorf(r.Error)
	}
	return r.Members, nil
}

func (cli *SlackClient) deleteMessage(ts, channel string) ([]byte, error) {
	res, e := http.PostForm(slackAPIEndpoint+"chat.delete", url.Values{
		"token":   {cli.Token},
		"ts":      {ts},
		"channel": {channel},
	})
	if e != nil {
		return nil, e
	}
	defer res.Body.Close()
	byteArray, e := ioutil.ReadAll(res.Body)
	if e != nil {
		return nil, e
	}
	return byteArray, nil
}
<commit_msg>:imp: Fix :bug:<commit_after>package slack

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"

	"../timeline"
	"github.com/pkg/errors"

	"golang.org/x/net/websocket"
)

var rtmStartURL = "https://slack.com/api/rtm.start"

var slackAPIEndpoint = "https://slack.com/api/"

var origin = "http://localhost"

type rtmStartResponse struct {
	OK    bool   `json:"ok"`
	URL   string `json:"url"`
	Error string `json:"error"`
}

type deletedEvent struct {
	ChannelID string       `json:"channel"`
	Message   SlackMessage `json:"previous_message"`
}

type SlackMessage struct {
	Raw       string `json:"-"`
	Type      string `json:"type"`
	UserID    string `json:"user"`
	Text      string `json:"text"`
	ChannelID string `json:"channel"`
	TimeStamp string `json:"ts"`
	SubType   string `json:"subtype"`
}

func (m *SlackMessage) ToInternal() timeline.Message {
	return timeline.NewMessage(m.Text, m.UserID, m.ChannelID, m.TimeStamp)
}

type userListResponse struct {
	OK    bool   `json:"ok"`
	User  User   `json:"user"`
	Error string `json:"error"`
}

type allUserResponse struct {
	OK      bool   `json:"ok"`
	Members []User `json:"members"`
	Error   string `json:"error"`
}

type User struct {
	ID      string  `json:"id"`
	Name    string  `json:"name"`
	Profile profile `json:"profile"`
}

func (u User) ToInternal() timeline.User {
	return timeline.NewUser(u.ID, u.Name, u.Profile.ImageURL)
}

type profile struct {
	ImageURL string `json:"image_48"`
}

type channelCreated struct {
	Type    string  `json:"type"`
	Channel channel `json:"channel"`
}

type channel struct {
	ID      string `json:"id"`
	Name    string `json:"name"`
	Creator string `json:"creator"`
}

type SlackClient struct {
	Token string
}

func isValidJson(b []byte) bool {
	j := map[string]interface{}{}
	return json.Unmarshal(b, &j) == nil
}

func (cli *SlackClient) connectToRTM() (*websocket.Conn, error) {
	v := url.Values{
		"token": {cli.Token},
	}
	resp, e := http.Get(rtmStartURL + "?" + v.Encode())
	if e != nil {
		return nil, e
	}
	defer resp.Body.Close()
	byteArray, e := ioutil.ReadAll(resp.Body)
	if e != nil {
		return nil, e
	}
	res := rtmStartResponse{}
	e = json.Unmarshal(byteArray, &res)
	if e != nil {
		return nil, e
	}
	if !res.OK {
		return nil, fmt.Errorf(res.Error)
	}
	ws, e := websocket.Dial(res.URL, "", origin)
	if e != nil {
		return nil, e
	}
	return ws, nil
}

func receive(ws *websocket.Conn) ([]byte, error) {
	var msg = make([]byte, 4096)
	n, e := ws.Read(msg)
	if e != nil {
		return nil, e
	}
	return msg[:n], nil
}

func (cli *SlackClient) Polling(
	messageChan, deletedMessageChan chan *timeline.Message,
	warnChan, errorChan chan error,
	restartChan chan bool,
) {
	ws, e := cli.connectToRTM()
	if e != nil {
		errorChan <- e
		return
	}
	defer ws.Close()
	prev := make([]byte, 0)
	for {
		received, e := receive(ws)
		if e != nil {
			errorChan <- e
		}
		msg := append(prev, received...)
		if !isValidJson(msg) {
			prev = msg
			continue
		}
		prev = make([]byte, 0)
		message := SlackMessage{}
		errOnMessage := json.Unmarshal(msg, &message)

		if errOnMessage != nil {
			event := channelCreated{}
			errOnEvent := json.Unmarshal(msg, &event)
			if errOnEvent != nil && errOnMessage != nil {
				warnChan <- errors.Wrap(errOnMessage, fmt.Sprintf("failed to unmarshal to message. json: '%s'", msg))
				warnChan <- errors.Wrap(errOnEvent, fmt.Sprintf("failed to unmarshal to channel created. json: '%s'", msg))
				continue
			}
			restartChan <- true
			return
		}

		message.Raw = string(msg)
		if message.Type != "message" {
			warnChan <- fmt.Errorf("not message: '%+v'", message)
			continue
		}

		switch message.SubType {
		case "":
			m := message.ToInternal()
			messageChan <- &m
		case "message_deleted":
			d := deletedEvent{}
			e := json.Unmarshal([]byte(msg), &d)
			if e != nil {
				warnChan <- errors.Wrap(e, fmt.Sprintf("failed to unmarshal to deleted event. json: '%s'", msg))
				continue
			}
			d.Message.ChannelID = d.ChannelID
			m := d.Message.ToInternal()
			deletedMessageChan <- &m
		}
	}
}

func (cli *SlackClient) postMessage(channelID, text, userName, iconURL string) ([]byte, error) {
	res, e := http.PostForm(slackAPIEndpoint+"chat.postMessage", url.Values{
		"token":      {cli.Token},
		"channel":    {channelID},
		"text":       {text},
		"username":   {userName},
		"as_user":    {"false"},
		"icon_url":   {iconURL},
		"link_names": {"0"},
	})
	if e != nil {
		return nil, e
	}
	defer res.Body.Close()
	byteArray, e := ioutil.ReadAll(res.Body)
	if e != nil {
		return nil, e
	}
	return byteArray, nil
}

func (cli *SlackClient) getUser(userID string) (*User, error) {
	res, e := http.PostForm(slackAPIEndpoint+"users.info", url.Values{
		"token": {cli.Token},
		"user":  {userID},
	})
	if e != nil {
		return nil, e
	}
	defer res.Body.Close()
	b, e := ioutil.ReadAll(res.Body)
	if e != nil {
		return nil, e
	}
	r := userListResponse{}
	e = json.Unmarshal(b, &r)
	if e != nil {
		return nil, e
	}
	if !r.OK {
		return nil, fmt.Errorf(r.Error)
	}
	u := r.User
	return &u, nil
}

func (cli *SlackClient) getAllUsers() ([]User, error) {
	res, e := http.PostForm(slackAPIEndpoint+"users.list", url.Values{
		"token": {cli.Token},
	})
	if e != nil {
		return nil, e
	}
	defer res.Body.Close()
	b, e := ioutil.ReadAll(res.Body)
	if e != nil {
		return nil, e
	}
	r := allUserResponse{}
	e = json.Unmarshal(b, &r)
	if e != nil {
		return nil, e
	}
	if !r.OK {
		return nil, fmt.Errorf(r.Error)
	}
	return r.Members, nil
}

func (cli *SlackClient) deleteMessage(ts, channel string) ([]byte, error) {
	res, e := http.PostForm(slackAPIEndpoint+"chat.delete", url.Values{
		"token":   {cli.Token},
		"ts":      {ts},
		"channel": {channel},
	})
	if e != nil {
		return nil, e
	}
	defer res.Body.Close()
	byteArray, e := ioutil.ReadAll(res.Body)
	if e != nil {
		return nil, e
	}
	return byteArray, nil
}
<|endoftext|>


<commit_before>package simpleamqp

import (
	"log"
	"time"

	"github.com/streadway/amqp"
)

const (
	timeToReconnect = 5 * time.Second
)

type QueueOptions struct {
	Durable   bool
	Delete    bool
	Exclusive bool
}

var DefaultQueueOptions = QueueOptions{
	Durable:   true,
	Delete:    false,
	Exclusive: false,
}

func setup(url string) (*amqp.Connection, *amqp.Channel) {
	for {
		conn, err := amqp.Dial(url)
		if err != nil {
			log.Println("Error dial", err)
			time.Sleep(timeToReconnect)
			continue
		}

		ch, err := conn.Channel()
		if err != nil {
			log.Println("Error channel", err)
			time.Sleep(timeToReconnect)
			continue
		}
		return conn, ch
	}
}

func exchangeDeclare(ch *amqp.Channel, exchange string) {
	for {
		log.Println("Exchange declare", exchange)
		err := ch.ExchangeDeclare(exchange, "topic", true, false, false, false, nil)
		if err != nil {
			log.Println("Error declaring exchange", err)
			time.Sleep(timeToReconnect)
			continue
		} else {
			return
		}
	}
}

func queueDeclare(ch *amqp.Channel, queue string, queueOptions QueueOptions) amqp.Queue {
	for {
		log.Println("Queue declare", queue)
		q, err := ch.QueueDeclare(
			queue, // name of the queue
			queueOptions.Durable,
			queueOptions.Delete,
			queueOptions.Exclusive,
			false, // noWait
			nil,   // arguments
		)
		if err != nil {
			log.Println("Error declaring queue", err)
			time.Sleep(timeToReconnect)
			continue
		} else {
			return q
		}
	}
}
<commit_msg>Support rabbitmq:// in the broker uri<commit_after>package simpleamqp

import (
	"log"
	"time"
	"strings"

	"github.com/streadway/amqp"
)

const (
	timeToReconnect = 5 * time.Second
)

type QueueOptions struct {
	Durable   bool
	Delete    bool
	Exclusive bool
}

var DefaultQueueOptions = QueueOptions{
	Durable:   true,
	Delete:    false,
	Exclusive: false,
}

func normalizeURL(url string) string{
     return strings.Replace(url, "rabbitmq://", "amqp://", -1)
}

func setup(url string) (*amqp.Connection, *amqp.Channel) {
	for {
		conn, err := amqp.Dial(normalizeURL(url))
		if err != nil {
			log.Println("Error dial", err)
			time.Sleep(timeToReconnect)
			continue
		}

		ch, err := conn.Channel()
		if err != nil {
			log.Println("Error channel", err)
			time.Sleep(timeToReconnect)
			continue
		}
		return conn, ch
	}
}

func exchangeDeclare(ch *amqp.Channel, exchange string) {
	for {
		log.Println("Exchange declare", exchange)
		err := ch.ExchangeDeclare(exchange, "topic", true, false, false, false, nil)
		if err != nil {
			log.Println("Error declaring exchange", err)
			time.Sleep(timeToReconnect)
			continue
		} else {
			return
		}
	}
}

func queueDeclare(ch *amqp.Channel, queue string, queueOptions QueueOptions) amqp.Queue {
	for {
		log.Println("Queue declare", queue)
		q, err := ch.QueueDeclare(
			queue, // name of the queue
			queueOptions.Durable,
			queueOptions.Delete,
			queueOptions.Exclusive,
			false, // noWait
			nil,   // arguments
		)
		if err != nil {
			log.Println("Error declaring queue", err)
			time.Sleep(timeToReconnect)
			continue
		} else {
			return q
		}
	}
}
<|endoftext|>


<commit_before>package simplejson

import (
	"encoding/json"
	"errors"
	"log"
)

// returns the current implementation version
func Version() string {
	return "0.3"
}

type Json struct {
	data interface{}
}

// NewJson returns a pointer to a new `Json` object
// after unmarshaling `body` bytes
func NewJson(body []byte) (*Json, error) {
	j := new(Json)
	err := json.Unmarshal(body, &j.data)
	if err != nil {
		return nil, err
	}
	return j, nil
}

// Encode returns its marshaled data as `[]byte`
func (j *Json) Encode() ([]byte, error) {
	return json.Marshal(&j.data)
}

// Get returns a pointer to a new `Json` object
// for `key` in its `map` representation
//
// useful for chaining operations (to traverse a nested JSON):
//    js.Get("top_level").Get("dict").Get("value").Int()
func (j *Json) Get(key string) *Json {
	m, err := j.Map()
	if err == nil {
		if val, ok := m[key]; ok {
			return &Json{val}
		}
	}
	return &Json{nil}
}

// GetIndex resturns a pointer to a new `Json` object
// for `index` in its `array` representation
//
// this is the analog to Get when accessing elements of
// a json array instead of a json object:
//    js.Get("top_level").Get("array").GetIndex(1).Get("key").Int()
func (j *Json) GetIndex(index int) *Json {
  a, err := j.Array()
  if err == nil {
    if len(a) > index {
      return &Json{a[index]}
    }
  }
  return &Json{nil}
}

// CheckGet returns a pointer to a new `Json` object and
// a `bool` identifying success or failure
//
// useful for chained operations when success is important:
//    if data, ok := js.Get("top_level").CheckGet("inner"); ok {
//        log.Println(data)
//    }
func (j *Json) CheckGet(key string) (*Json, bool) {
	m, err := j.Map()
	if err == nil {
		if val, ok := m[key]; ok {
			return &Json{val}, true
		}
	}
	return nil, false
}

// Map type asserts to `map`
func (j *Json) Map() (map[string]interface{}, error) {
	if m, ok := (j.data).(map[string]interface{}); ok {
		return m, nil
	}
	return nil, errors.New("type assertion to map[string]interface{} failed")
}

// Array type asserts to an `array`
func (j *Json) Array() ([]interface{}, error) {
	if a, ok := (j.data).([]interface{}); ok {
		return a, nil
	}
	return nil, errors.New("type assertion to []interface{} failed")
}

// Bool type asserts to `bool`
func (j *Json) Bool() (bool, error) {
	if s, ok := (j.data).(bool); ok {
		return s, nil
	}
	return false, errors.New("type assertion to bool failed")
}

// String type asserts to `string`
func (j *Json) String() (string, error) {
	if s, ok := (j.data).(string); ok {
		return s, nil
	}
	return "", errors.New("type assertion to string failed")
}

// Float64 type asserts to `float64`
func (j *Json) Float64() (float64, error) {
	if i, ok := (j.data).(float64); ok {
		return i, nil
	}
	return -1, errors.New("type assertion to float64 failed")
}

// Int type asserts to `float64` then converts to `int`
func (j *Json) Int() (int, error) {
	if f, ok := (j.data).(float64); ok {
		return int(f), nil
	}

	return -1, errors.New("type assertion to float64 failed")
}

// Int type asserts to `float64` then converts to `int64`
func (j *Json) Int64() (int64, error) {
	if f, ok := (j.data).(float64); ok {
		return int64(f), nil
	}

	return -1, errors.New("type assertion to float64 failed")
}

// Bytes type asserts to `[]byte`
func (j *Json) Bytes() ([]byte, error) {
	if s, ok := (j.data).(string); ok {
		return []byte(s), nil
	}
	return nil, errors.New("type assertion to []byte failed")
}

// MustString guarantees the return of a `string` (with optional default)
//
// useful when you explicitly want a `string` in a single value return context:
//     myFunc(js.Get("param1").MustString(), js.Get("optional_param").MustString("my_default"))
func (j *Json) MustString(args ...string) string {
	var def string

	switch len(args) {
	case 0:
		break
	case 1:
		def = args[0]
	default:
		log.Panicf("MustString() received too many arguments %d", len(args))
	}

	s, err := j.String()
	if err == nil {
		return s
	}

	return def
}

// MustInt guarantees the return of an `int` (with optional default)
//
// useful when you explicitly want an `int` in a single value return context:
//     myFunc(js.Get("param1").MustInt(), js.Get("optional_param").MustInt(5150))
func (j *Json) MustInt(args ...int) int {
	var def int

	switch len(args) {
	case 0:
		break
	case 1:
		def = args[0]
	default:
		log.Panicf("MustInt() received too many arguments %d", len(args))
	}

	i, err := j.Int()
	if err == nil {
		return i
	}

	return def
}

// MustFloat64 guarantees the return of a `float64` (with optional default)
//
// useful when you explicitly want a `float64` in a single value return context:
//     myFunc(js.Get("param1").MustFloat64(), js.Get("optional_param").MustFloat64(5.150))
func (j *Json) MustFloat64(args ...float64) float64 {
	var def float64

	switch len(args) {
	case 0:
		break
	case 1:
		def = args[0]
	default:
		log.Panicf("MustFloat64() received too many arguments %d", len(args))
	}

	i, err := j.Float64()
	if err == nil {
		return i
	}

	return def
}
<commit_msg>bump version 0.4<commit_after>package simplejson

import (
	"encoding/json"
	"errors"
	"log"
)

// returns the current implementation version
func Version() string {
	return "0.4"
}

type Json struct {
	data interface{}
}

// NewJson returns a pointer to a new `Json` object
// after unmarshaling `body` bytes
func NewJson(body []byte) (*Json, error) {
	j := new(Json)
	err := json.Unmarshal(body, &j.data)
	if err != nil {
		return nil, err
	}
	return j, nil
}

// Encode returns its marshaled data as `[]byte`
func (j *Json) Encode() ([]byte, error) {
	return json.Marshal(&j.data)
}

// Get returns a pointer to a new `Json` object
// for `key` in its `map` representation
//
// useful for chaining operations (to traverse a nested JSON):
//    js.Get("top_level").Get("dict").Get("value").Int()
func (j *Json) Get(key string) *Json {
	m, err := j.Map()
	if err == nil {
		if val, ok := m[key]; ok {
			return &Json{val}
		}
	}
	return &Json{nil}
}

// GetIndex resturns a pointer to a new `Json` object
// for `index` in its `array` representation
//
// this is the analog to Get when accessing elements of
// a json array instead of a json object:
//    js.Get("top_level").Get("array").GetIndex(1).Get("key").Int()
func (j *Json) GetIndex(index int) *Json {
  a, err := j.Array()
  if err == nil {
    if len(a) > index {
      return &Json{a[index]}
    }
  }
  return &Json{nil}
}

// CheckGet returns a pointer to a new `Json` object and
// a `bool` identifying success or failure
//
// useful for chained operations when success is important:
//    if data, ok := js.Get("top_level").CheckGet("inner"); ok {
//        log.Println(data)
//    }
func (j *Json) CheckGet(key string) (*Json, bool) {
	m, err := j.Map()
	if err == nil {
		if val, ok := m[key]; ok {
			return &Json{val}, true
		}
	}
	return nil, false
}

// Map type asserts to `map`
func (j *Json) Map() (map[string]interface{}, error) {
	if m, ok := (j.data).(map[string]interface{}); ok {
		return m, nil
	}
	return nil, errors.New("type assertion to map[string]interface{} failed")
}

// Array type asserts to an `array`
func (j *Json) Array() ([]interface{}, error) {
	if a, ok := (j.data).([]interface{}); ok {
		return a, nil
	}
	return nil, errors.New("type assertion to []interface{} failed")
}

// Bool type asserts to `bool`
func (j *Json) Bool() (bool, error) {
	if s, ok := (j.data).(bool); ok {
		return s, nil
	}
	return false, errors.New("type assertion to bool failed")
}

// String type asserts to `string`
func (j *Json) String() (string, error) {
	if s, ok := (j.data).(string); ok {
		return s, nil
	}
	return "", errors.New("type assertion to string failed")
}

// Float64 type asserts to `float64`
func (j *Json) Float64() (float64, error) {
	if i, ok := (j.data).(float64); ok {
		return i, nil
	}
	return -1, errors.New("type assertion to float64 failed")
}

// Int type asserts to `float64` then converts to `int`
func (j *Json) Int() (int, error) {
	if f, ok := (j.data).(float64); ok {
		return int(f), nil
	}

	return -1, errors.New("type assertion to float64 failed")
}

// Int type asserts to `float64` then converts to `int64`
func (j *Json) Int64() (int64, error) {
	if f, ok := (j.data).(float64); ok {
		return int64(f), nil
	}

	return -1, errors.New("type assertion to float64 failed")
}

// Bytes type asserts to `[]byte`
func (j *Json) Bytes() ([]byte, error) {
	if s, ok := (j.data).(string); ok {
		return []byte(s), nil
	}
	return nil, errors.New("type assertion to []byte failed")
}

// MustString guarantees the return of a `string` (with optional default)
//
// useful when you explicitly want a `string` in a single value return context:
//     myFunc(js.Get("param1").MustString(), js.Get("optional_param").MustString("my_default"))
func (j *Json) MustString(args ...string) string {
	var def string

	switch len(args) {
	case 0:
		break
	case 1:
		def = args[0]
	default:
		log.Panicf("MustString() received too many arguments %d", len(args))
	}

	s, err := j.String()
	if err == nil {
		return s
	}

	return def
}

// MustInt guarantees the return of an `int` (with optional default)
//
// useful when you explicitly want an `int` in a single value return context:
//     myFunc(js.Get("param1").MustInt(), js.Get("optional_param").MustInt(5150))
func (j *Json) MustInt(args ...int) int {
	var def int

	switch len(args) {
	case 0:
		break
	case 1:
		def = args[0]
	default:
		log.Panicf("MustInt() received too many arguments %d", len(args))
	}

	i, err := j.Int()
	if err == nil {
		return i
	}

	return def
}

// MustFloat64 guarantees the return of a `float64` (with optional default)
//
// useful when you explicitly want a `float64` in a single value return context:
//     myFunc(js.Get("param1").MustFloat64(), js.Get("optional_param").MustFloat64(5.150))
func (j *Json) MustFloat64(args ...float64) float64 {
	var def float64

	switch len(args) {
	case 0:
		break
	case 1:
		def = args[0]
	default:
		log.Panicf("MustFloat64() received too many arguments %d", len(args))
	}

	i, err := j.Float64()
	if err == nil {
		return i
	}

	return def
}
<|endoftext|>


<commit_before>package freetree

import (
	"runtime"
	"runtime/debug"
	"sort"
)

// -----------------------------------------------------------------------------

// SimpleTree implements a simple binary search tree.
type SimpleTree struct {
	root  *simpleNode
	nodes uint
}

// NewSimpleTree returns an empty SimpleTree.
func NewSimpleTree() *SimpleTree {
	return &SimpleTree{}
}

// Insert inserts the given Comparables in the tree.
// It does not rebalance the tree, use Rebalance() for that.
//
// If the tree is currently empty and the passed-in Comparable are already
// sorted in increasing order, the tree will be perfectly balanced.
// This means you don't have to Rebalance() the tree if you've inserted all
// your pre-sorted data in one Insert() call.
func (st *SimpleTree) Insert(cs ...Comparable) *SimpleTree {
	return st.InsertArray(cs)
}

// InsertArray is a helper to use Insert() with a ComparableArray.
func (st *SimpleTree) InsertArray(ca ComparableArray) *SimpleTree {
	st.insert(ca)
	st.nodes += uint(len(ca))

	return st
}

func (st *SimpleTree) insert(ca ComparableArray) {
	l := len(ca)
	if l == 0 {
		return
	}

	st.root = st.root.insert(ca[l/2])

	if l > 1 {
		st.insert(ca[:l/2])
		st.insert(ca[l/2+1:])
	}
}

// Ascend returns the first element in the tree that is == `pivot`.
func (st SimpleTree) Ascend(pivot Comparable) Comparable {
	return st.ascend(pivot)
}

func (st SimpleTree) ascend(pivot Comparable) Comparable {
	return st.root.ascend(pivot)
}

// Rebalance rebalances the tree to guarantee O(log(n)) search complexity.
//
// Rebalancing is implemented as straightforwardly as possible: it's dumb.
// I strongly suggest running the garbage collector and scavenger once it's done.
//   runtime.GC()
//   debug.FreeOSMemory()
func (st *SimpleTree) Rebalance() *SimpleTree {
	flat := st.flatten()
	sort.Sort(flat)

	st.root = nil
	st.insert(flat)

	return st
}

// Rebalance rebalances the tree and runs the garbage collector.
func (st *SimpleTree) RebalanceGC() *SimpleTree {
	st.Rebalance()
	runtime.GC()
	debug.FreeOSMemory()

	return st
}

// Flatten returns the content of the tree as a ComparableArray.
func (st SimpleTree) Flatten() ComparableArray {
	return st.flatten()
}

func (st SimpleTree) flatten() ComparableArray {
	ca := make(ComparableArray, 0, st.nodes)
	return st.root.flatten(ca)
}

// -----------------------------------------------------------------------------

type simpleNode struct {
	left, right *simpleNode
	data        Comparable
}

func (sn *simpleNode) insert(c Comparable) *simpleNode {
	if sn == nil {
		return &simpleNode{data: c}
	}

	if c.Less(sn.data) {
		sn.left = sn.left.insert(c)
	} else {
		sn.right = sn.right.insert(c)
	}

	return sn
}

func (sn *simpleNode) ascend(pivot Comparable) Comparable {
	if sn == nil {
		return nil
	}

	if pivot.Less(sn.data) {
		return sn.left.ascend(pivot)
	} else if sn.data.Less(pivot) {
		return sn.right.ascend(pivot)
	}

	return sn.data
}

func (sn *simpleNode) flatten(ca ComparableArray) ComparableArray {
	if sn == nil {
		return ca
	}

	ca = sn.left.flatten(ca)
	ca = sn.right.flatten(ca)

	return append(ca, sn.data)
}
<commit_msg>SimpleTree - added node IDs<commit_after>package freetree

import (
	"runtime"
	"runtime/debug"
	"sort"
)

// -----------------------------------------------------------------------------

// SimpleTree implements a simple binary search tree.
type SimpleTree struct {
	root  *simpleNode
	nodes uint
}

// NewSimpleTree returns an empty SimpleTree.
func NewSimpleTree() *SimpleTree {
	return &SimpleTree{}
}

// Insert inserts the given Comparables in the tree.
// It does not rebalance the tree, use Rebalance() for that.
//
// If the tree is currently empty and the passed-in Comparable are already
// sorted in increasing order, the tree will be perfectly balanced.
// This means you don't have to Rebalance() the tree if you've inserted all
// your pre-sorted data in one Insert() call.
func (st *SimpleTree) Insert(cs ...Comparable) *SimpleTree {
	return st.InsertArray(cs)
}

// InsertArray is a helper to use Insert() with a ComparableArray.
func (st *SimpleTree) InsertArray(ca ComparableArray) *SimpleTree {
	st.insert(ca, st.nodes)
	st.nodes += uint(len(ca))

	return st
}

func (st *SimpleTree) insert(ca ComparableArray, id uint) {
	l := len(ca)
	if l == 0 {
		return
	}

	st.root = st.root.insert(ca[l/2], id)

	if l > 1 {
		st.insert(ca[:l/2], id)
		st.insert(ca[l/2+1:], id)
	}
}

// Ascend returns the first element in the tree that is == `pivot`.
func (st SimpleTree) Ascend(pivot Comparable) Comparable {
	return st.ascend(pivot)
}

func (st SimpleTree) ascend(pivot Comparable) Comparable {
	return st.root.ascend(pivot)
}

// Rebalance rebalances the tree to guarantee O(log(n)) search complexity.
//
// Rebalancing is implemented as straightforwardly as possible: it's dumb.
// I strongly suggest running the garbage collector and scavenger once it's done.
//   runtime.GC()
//   debug.FreeOSMemory()
func (st *SimpleTree) Rebalance() *SimpleTree {
	flat := st.flatten()
	sort.Sort(flat)

	st.root = nil
	st.nodes = 0
	st.insert(flat, 0)

	return st
}

// Rebalance rebalances the tree and runs the garbage collector.
func (st *SimpleTree) RebalanceGC() *SimpleTree {
	st.Rebalance()
	runtime.GC()
	debug.FreeOSMemory()

	return st
}

// Flatten returns the content of the tree as a ComparableArray.
func (st SimpleTree) Flatten() ComparableArray {
	return st.flatten()
}

func (st SimpleTree) flatten() ComparableArray {
	ca := make(ComparableArray, 0, st.nodes)
	return st.root.flatten(ca)
}

// -----------------------------------------------------------------------------

type simpleNode struct {
	id          uint
	left, right *simpleNode
	data        Comparable
}

func (sn *simpleNode) insert(c Comparable, id uint) *simpleNode {
	if sn == nil {
		return &simpleNode{id: id, data: c}
	}

	if c.Less(sn.data) {
		sn.left = sn.left.insert(c, id)
	} else {
		sn.right = sn.right.insert(c, id)
	}

	return sn
}

func (sn *simpleNode) ascend(pivot Comparable) Comparable {
	if sn == nil {
		return nil
	}

	if pivot.Less(sn.data) {
		return sn.left.ascend(pivot)
	} else if sn.data.Less(pivot) {
		return sn.right.ascend(pivot)
	}

	return sn.data
}

func (sn *simpleNode) flatten(ca ComparableArray) ComparableArray {
	if sn == nil {
		return ca
	}

	ca = sn.left.flatten(ca)
	ca = sn.right.flatten(ca)

	return append(ca, sn.data)
}
<|endoftext|>


<commit_before><commit_msg>Added PUT test for the session<commit_after><|endoftext|>


<commit_before>package qsim

// An event scheduled to occur in the simulation We'll run the function F at
// tick Time. F will be called at time Time with the current clock time as its
// argument.
type simEvent struct {
	T int
	F func(clock int)
}

// Schedule holds simEvents in the order that they need to be run. Events that
// have already occurred are removed.
type Schedule struct {
	// The list of events that have yet to occur. This is kept in ascending time
	// order.
	events []simEvent
}

// Add puts a new event in the schedule.
func (sch *Schedule) Add(newEv simEvent) {
	var i int

	if len(sch.events) == 0 {
		sch.events = append(sch.events, newEv)
		return
	}

	for i = len(sch.events) - 1; i >= 0; i-- {
		if sch.events[i].T <= newEv.T {
			sch.insertEvent(i+1, newEv)
			return
		}
	}
	// Fell off the beginning of the schedule, so just insert at the beginning
	sch.insertEvent(0, newEv)
	return
}

// Next returns the next event in the schedule and removes that event from the
// schedule.
func (sch *Schedule) Next() simEvent {
	var ev simEvent

	// This should never happen, which means it definitely will.
	if len(sch.events) == 0 {
		panic("next Schedule event requested but Schedule is empty")
	}

	ev = sch.events[0]
	sch.events = sch.events[1:]
	return ev
}

// insertEvent places a simEvent at the given index in sch.events.
func (sch *Schedule) insertEvent(idx int, newEv simEvent) {
	sch.events = append(sch.events, simEvent{})
	copy(sch.events[idx+1:], sch.events[idx:])
	sch.events[idx] = newEv
	return
}

// NewSchedule creates a new, empty Schedule struct.
func NewSchedule() *Schedule {
	return new(Schedule)
}

// RunSimulation simulates a queueing system for a certain number of ticks.
//
// The internal operations of a queuing system take care of themselves, so
// this function is only responsible for things going in and out of the
// system. It keeps track of the clock and triggers arrivals and
// job-finishes at the appropriate times.
/*func RunSimulation(ap ArrProc, procs []*Processor, ticks int) {
	var schedule *Schedule
	var clock int
}*/
<commit_msg>Added RunSimulation<commit_after>package qsim

// An event scheduled to occur in the simulation We'll run the function F at
// tick Time. F will be called at time Time with the current clock time as its
// argument.
type simEvent struct {
	T int
	F func(clock int)
}

// Schedule holds simEvents in the order that they need to be run. Events that
// have already occurred are removed.
type Schedule struct {
	// The list of events that have yet to occur. This is kept in ascending time
	// order.
	events []simEvent
}

// Add puts a new event in the schedule.
func (sch *Schedule) Add(newEv simEvent) {
	var i int

	if len(sch.events) == 0 {
		sch.events = append(sch.events, newEv)
		return
	}

	for i = len(sch.events) - 1; i >= 0; i-- {
		if sch.events[i].T <= newEv.T {
			sch.insertEvent(i+1, newEv)
			return
		}
	}
	// Fell off the beginning of the schedule, so just insert at the beginning
	sch.insertEvent(0, newEv)
	return
}

// Next returns the next event in the schedule and removes that event from the
// schedule.
func (sch *Schedule) Next() simEvent {
	var ev simEvent

	// This should never happen, which means it definitely will.
	if len(sch.events) == 0 {
		panic("next Schedule event requested but Schedule is empty")
	}

	ev = sch.events[0]
	sch.events = sch.events[1:]
	return ev
}

// insertEvent places a simEvent at the given index in sch.events.
func (sch *Schedule) insertEvent(idx int, newEv simEvent) {
	sch.events = append(sch.events, simEvent{})
	copy(sch.events[idx+1:], sch.events[idx:])
	sch.events[idx] = newEv
	return
}

// NewSchedule creates a new, empty Schedule struct.
func NewSchedule() *Schedule {
	return new(Schedule)
}

// RunSimulation simulates a queueing system for a certain number of ticks.
//
// The internal operations of a queuing system take care of themselves, so
// this function is only responsible for things going into and out of the
// system. It keeps track of the clock and triggers arrivals and
// job-finishes at the appropriate times.
func RunSimulation(ap ArrProc, procs []*Processor, maxTicks int) {
	var sch *Schedule
	var p *Processor
	var clock int
	var ev simEvent
	sch = NewSchedule()

	// Schedule Processor-finish events. Each Processor gets an AfterStart
	// callback that schedules a Finish() call for that processor to occur
	// when the processing time has elapsed.
	cbAfterStart := func(cbProcessor *Processor, cbJob *Job, cbProcTime int) {
		eventCb := func(cbClock int) {
			cbProcessor.Finish()
		}
		sch.Add(simEvent{clock + cbProcTime, eventCb})
	}
	for _, p = range procs {
		p.AfterStart(cbAfterStart)
	}

	// Schedule arrival events, including the initial one.
	cbAfterArrive := func(cbArrProc ArrProc, cbJob *Job, cbInterval int) {
		eventCb := func(cbClock int) {
			ap.Arrive()
		}
		sch.Add(simEvent{clock + cbInterval, eventCb})
	}
	ap.AfterArrive(cbAfterArrive)
	sch.Add(simEvent{0, func(cbClock int) { ap.Arrive() }})

	// Run the simulation.
	for clock = 0; clock <= maxTicks; {
		ev = sch.Next()
		clock = ev.T
		ev.F(clock)
	}
}
<|endoftext|>


<commit_before>package search

import (
	"encoding/json"
	"time"
)

func Aggregate(name string) *AggregateDsl {
	return &AggregateDsl{Name: name}
}

type AggregateDsl struct {
	Name          string
	TypeName      string
	Type          interface{}
	Filters       *FilterWrap              `json:"filters,omitempty"`
	AggregatesVal map[string]*AggregateDsl `json:"aggregations,omitempty"`
}

type FieldAggregate struct {
	Field   string `json:"field"`
	SizeVal *int   `json:"size,omitempty"`
}

/**
 * Aggregates accepts n "sub-aggregates" to be applied to this aggregate
 *
 * agg := Aggregate("user").Term("user_id")
 * agg.Aggregates(
 *   Aggregate("total_spent").Sum("price"),
 *   Aggregate("total_saved").Sum("discount"),
 * )
 */
func (d *AggregateDsl) Aggregates(aggs ...*AggregateDsl) *AggregateDsl {
	if len(aggs) < 1 {
		return d
	}
	if len(d.AggregatesVal) == 0 {
		d.AggregatesVal = make(map[string]*AggregateDsl)
	}

	for _, agg := range aggs {
		d.AggregatesVal[agg.Name] = agg
	}
	return d
}

func (d *AggregateDsl) Min(field string) *AggregateDsl {
	d.Type = FieldAggregate{Field: field}
	d.TypeName = "min"
	return d
}

func (d *AggregateDsl) Max(field string) *AggregateDsl {
	d.Type = FieldAggregate{Field: field}
	d.TypeName = "max"
	return d
}

func (d *AggregateDsl) Sum(field string) *AggregateDsl {
	d.Type = FieldAggregate{Field: field}
	d.TypeName = "sum"
	return d
}

func (d *AggregateDsl) Avg(field string) *AggregateDsl {
	d.Type = FieldAggregate{Field: field}
	d.TypeName = "avg"
	return d
}

func (d *AggregateDsl) Stats(field string) *AggregateDsl {
	d.Type = FieldAggregate{Field: field}
	d.TypeName = "stats"
	return d
}

func (d *AggregateDsl) ExtendedStats(field string) *AggregateDsl {
	d.Type = FieldAggregate{Field: field}
	d.TypeName = "extended_stats"
	return d
}

func (d *AggregateDsl) ValueCount(field string) *AggregateDsl {
	d.Type = FieldAggregate{Field: field}
	d.TypeName = "value_count"
	return d
}

func (d *AggregateDsl) Percentiles(field string) *AggregateDsl {
	d.Type = FieldAggregate{Field: field}
	d.TypeName = "percentiles"
	return d
}

type Cardinality struct {
	Field              string  `json:"field"`
	PrecisionThreshold float64 `json:"precision_threshold,omitempty"`
	Rehash             bool    `json:"rehash,omitempty"`
}

/**
 * Cardinality(
 *	 "field_name",
 *	 true,
 *   0,
 * )
 */
func (d *AggregateDsl) Cardinality(field string, rehash bool, threshold int) *AggregateDsl {
	c := Cardinality{Field: field}

	// Only set if it's false, since the default is true
	if !rehash {
		c.Rehash = false
	}

	if threshold > 0 {
		c.PrecisionThreshold = float64(threshold)
	}
	d.Type = c
	d.TypeName = "cardinality"
	return d
}

func (d *AggregateDsl) Global() *AggregateDsl {
	d.Type = struct{}{}
	d.TypeName = "global"
	return d
}

func (d *AggregateDsl) Filter(filters ...interface{}) *AggregateDsl {

	if len(filters) == 0 {
		return d
	}

	if d.Filters == nil {
		d.Filters = NewFilterWrap()
	}

	d.Filters.addFilters(filters)
	return d
}

func (d *AggregateDsl) Missing(field string) *AggregateDsl {
	d.Type = FieldAggregate{Field: field}
	d.TypeName = "missing"
	return d
}

func (d *AggregateDsl) Terms(field string) *AggregateDsl {
	d.Type = FieldAggregate{Field: field}
	d.TypeName = "terms"
	return d
}

func (d *AggregateDsl) SignificantTerms(field string) *AggregateDsl {
	d.Type = FieldAggregate{Field: field}
	d.TypeName = "significant_terms"
	return d
}

type Histogram struct {
	Field          string      `json:"field"`
	Interval       float64     `json:"interval"`
	MinDocCount    float64     `json:"min_doc_count"`
	ExtendedBounds interface{} `json:"extended_bounds,omitempty"`
}

func (d *AggregateDsl) Histogram(field string, interval int) *AggregateDsl {
	d.Type = Histogram{
		Field:       field,
		Interval:    float64(interval),
		MinDocCount: 1,
	}
	d.TypeName = "histogram"
	return d
}

type DateHistogram struct {
	Field          string      `json:"field"`
	Interval       string      `json:"interval"`
	MinDocCount    float64     `json:"min_doc_count"`
	ExtendedBounds interface{} `json:"extended_bounds,omitempty"`
}

func (d *AggregateDsl) DateHistogram(field, interval string) *AggregateDsl {
	d.Type = DateHistogram{
		Field:       field,
		Interval:    interval,
		MinDocCount: 1,
	}
	d.TypeName = "date_histogram"
	return d
}

func (d *AggregateDsl) Size(size int) *AggregateDsl {
	switch d.Type.(type) {
	case FieldAggregate:
		typ := d.Type.(FieldAggregate)
		typ.SizeVal = &size
		d.Type = typ
	}
	return d
}

// Sets the min doc count for a date histogram or histogram
// This will no-op if used on an inappropriate dsl type
func (d *AggregateDsl) MinDocCount(i float64) *AggregateDsl {

	if d.TypeName == "date_histogram" {
		t := d.Type.(DateHistogram)
		t.MinDocCount = i
		d.Type = t
	} else if d.TypeName == "histogram" {
		t := d.Type.(Histogram)
		t.MinDocCount = i
		d.Type = t
	}

	return d
}

// Hackety hack function that expects different types depending on the type of aggregate
// Not very idiomatic, but fits the elastigo DSL
func (d *AggregateDsl) ExtendedBounds(min, max interface{}) *AggregateDsl {
	if min == nil && max == nil {
		return d
	}

	if d.TypeName == "date_histogram" {
		var n time.Time
		var x time.Time
		t := d.Type.(DateHistogram)
		if min != nil {
			switch min.(type) {
			case time.Time:
				n = min.(time.Time)
			}
		}
		if max != nil {
			switch max.(type) {
			case time.Time:
				x = max.(time.Time)
			}
		}

		if min == nil {
			bounds := struct {
				Max time.Time `json:"max"`
			}{x}
			t.ExtendedBounds = &bounds
		} else if max == nil {
			bounds := struct {
				Min time.Time `json:"min"`
			}{n}
			t.ExtendedBounds = &bounds
		} else {
			bounds := struct {
				Min time.Time `json:"min"`
				Max time.Time `json:"max"`
			}{n, x}
			t.ExtendedBounds = &bounds
		}

		d.Type = t
	}
	if d.TypeName == "histogram" {
		var n float64
		var x float64
		t := d.Type.(Histogram)
		if min != nil {
			switch min.(type) {
			case time.Time:
				n = min.(float64)
			}
		}
		if max != nil {
			switch max.(type) {
			case time.Time:
				x = max.(float64)
			}
		}

		if min == nil {
			bounds := struct {
				Max float64 `json:"max"`
			}{x}
			t.ExtendedBounds = &bounds
		} else if max == nil {
			bounds := struct {
				Min float64 `json:"min"`
			}{n}
			t.ExtendedBounds = &bounds
		} else {
			bounds := struct {
				Min float64 `json:"min"`
				Max float64 `json:"max"`
			}{n, x}
			t.ExtendedBounds = &bounds
		}

		d.Type = t
	}

	return d
}
func (d *AggregateDsl) MarshalJSON() ([]byte, error) {
	return json.Marshal(d.toMap())
}

func (d *AggregateDsl) toMap() map[string]interface{} {
	root := map[string]interface{}{}

	if d.Type != nil {
		root[d.TypeName] = d.Type
	}
	aggregates := d.aggregatesMap()

	if d.Filters != nil {
		root["filter"] = d.Filters
	}

	if len(aggregates) > 0 {
		root["aggregations"] = aggregates
	}
	return root

}
func (d *AggregateDsl) aggregatesMap() map[string]interface{} {
	root := map[string]interface{}{}

	if len(d.AggregatesVal) > 0 {
		for _, agg := range d.AggregatesVal {
			root[agg.Name] = agg.toMap()
		}
	}
	return root
}
<commit_msg>Make ToMap and AggregatesMap public<commit_after>package search

import (
	"encoding/json"
	"time"
)

func Aggregate(name string) *AggregateDsl {
	return &AggregateDsl{Name: name}
}

type AggregateDsl struct {
	Name          string
	TypeName      string
	Type          interface{}
	Filters       *FilterWrap              `json:"filters,omitempty"`
	AggregatesVal map[string]*AggregateDsl `json:"aggregations,omitempty"`
}

type FieldAggregate struct {
	Field   string `json:"field"`
	SizeVal *int   `json:"size,omitempty"`
}

/**
 * Aggregates accepts n "sub-aggregates" to be applied to this aggregate
 *
 * agg := Aggregate("user").Term("user_id")
 * agg.Aggregates(
 *   Aggregate("total_spent").Sum("price"),
 *   Aggregate("total_saved").Sum("discount"),
 * )
 */
func (d *AggregateDsl) Aggregates(aggs ...*AggregateDsl) *AggregateDsl {
	if len(aggs) < 1 {
		return d
	}
	if len(d.AggregatesVal) == 0 {
		d.AggregatesVal = make(map[string]*AggregateDsl)
	}

	for _, agg := range aggs {
		d.AggregatesVal[agg.Name] = agg
	}
	return d
}

func (d *AggregateDsl) Min(field string) *AggregateDsl {
	d.Type = FieldAggregate{Field: field}
	d.TypeName = "min"
	return d
}

func (d *AggregateDsl) Max(field string) *AggregateDsl {
	d.Type = FieldAggregate{Field: field}
	d.TypeName = "max"
	return d
}

func (d *AggregateDsl) Sum(field string) *AggregateDsl {
	d.Type = FieldAggregate{Field: field}
	d.TypeName = "sum"
	return d
}

func (d *AggregateDsl) Avg(field string) *AggregateDsl {
	d.Type = FieldAggregate{Field: field}
	d.TypeName = "avg"
	return d
}

func (d *AggregateDsl) Stats(field string) *AggregateDsl {
	d.Type = FieldAggregate{Field: field}
	d.TypeName = "stats"
	return d
}

func (d *AggregateDsl) ExtendedStats(field string) *AggregateDsl {
	d.Type = FieldAggregate{Field: field}
	d.TypeName = "extended_stats"
	return d
}

func (d *AggregateDsl) ValueCount(field string) *AggregateDsl {
	d.Type = FieldAggregate{Field: field}
	d.TypeName = "value_count"
	return d
}

func (d *AggregateDsl) Percentiles(field string) *AggregateDsl {
	d.Type = FieldAggregate{Field: field}
	d.TypeName = "percentiles"
	return d
}

type Cardinality struct {
	Field              string  `json:"field"`
	PrecisionThreshold float64 `json:"precision_threshold,omitempty"`
	Rehash             bool    `json:"rehash,omitempty"`
}

/**
 * Cardinality(
 *	 "field_name",
 *	 true,
 *   0,
 * )
 */
func (d *AggregateDsl) Cardinality(field string, rehash bool, threshold int) *AggregateDsl {
	c := Cardinality{Field: field}

	// Only set if it's false, since the default is true
	if !rehash {
		c.Rehash = false
	}

	if threshold > 0 {
		c.PrecisionThreshold = float64(threshold)
	}
	d.Type = c
	d.TypeName = "cardinality"
	return d
}

func (d *AggregateDsl) Global() *AggregateDsl {
	d.Type = struct{}{}
	d.TypeName = "global"
	return d
}

func (d *AggregateDsl) Filter(filters ...interface{}) *AggregateDsl {

	if len(filters) == 0 {
		return d
	}

	if d.Filters == nil {
		d.Filters = NewFilterWrap()
	}

	d.Filters.addFilters(filters)
	return d
}

func (d *AggregateDsl) Missing(field string) *AggregateDsl {
	d.Type = FieldAggregate{Field: field}
	d.TypeName = "missing"
	return d
}

func (d *AggregateDsl) Terms(field string) *AggregateDsl {
	d.Type = FieldAggregate{Field: field}
	d.TypeName = "terms"
	return d
}

func (d *AggregateDsl) SignificantTerms(field string) *AggregateDsl {
	d.Type = FieldAggregate{Field: field}
	d.TypeName = "significant_terms"
	return d
}

type Histogram struct {
	Field          string      `json:"field"`
	Interval       float64     `json:"interval"`
	MinDocCount    float64     `json:"min_doc_count"`
	ExtendedBounds interface{} `json:"extended_bounds,omitempty"`
}

func (d *AggregateDsl) Histogram(field string, interval int) *AggregateDsl {
	d.Type = Histogram{
		Field:       field,
		Interval:    float64(interval),
		MinDocCount: 1,
	}
	d.TypeName = "histogram"
	return d
}

type DateHistogram struct {
	Field          string      `json:"field"`
	Interval       string      `json:"interval"`
	MinDocCount    float64     `json:"min_doc_count"`
	ExtendedBounds interface{} `json:"extended_bounds,omitempty"`
}

func (d *AggregateDsl) DateHistogram(field, interval string) *AggregateDsl {
	d.Type = DateHistogram{
		Field:       field,
		Interval:    interval,
		MinDocCount: 1,
	}
	d.TypeName = "date_histogram"
	return d
}

func (d *AggregateDsl) Size(size int) *AggregateDsl {
	switch d.Type.(type) {
	case FieldAggregate:
		typ := d.Type.(FieldAggregate)
		typ.SizeVal = &size
		d.Type = typ
	}
	return d
}

// Sets the min doc count for a date histogram or histogram
// This will no-op if used on an inappropriate dsl type
func (d *AggregateDsl) MinDocCount(i float64) *AggregateDsl {

	if d.TypeName == "date_histogram" {
		t := d.Type.(DateHistogram)
		t.MinDocCount = i
		d.Type = t
	} else if d.TypeName == "histogram" {
		t := d.Type.(Histogram)
		t.MinDocCount = i
		d.Type = t
	}

	return d
}

// Hackety hack function that expects different types depending on the type of aggregate
// Not very idiomatic, but fits the elastigo DSL
func (d *AggregateDsl) ExtendedBounds(min, max interface{}) *AggregateDsl {
	if min == nil && max == nil {
		return d
	}

	if d.TypeName == "date_histogram" {
		var n time.Time
		var x time.Time
		t := d.Type.(DateHistogram)
		if min != nil {
			switch min.(type) {
			case time.Time:
				n = min.(time.Time)
			}
		}
		if max != nil {
			switch max.(type) {
			case time.Time:
				x = max.(time.Time)
			}
		}

		if min == nil {
			bounds := struct {
				Max time.Time `json:"max"`
			}{x}
			t.ExtendedBounds = &bounds
		} else if max == nil {
			bounds := struct {
				Min time.Time `json:"min"`
			}{n}
			t.ExtendedBounds = &bounds
		} else {
			bounds := struct {
				Min time.Time `json:"min"`
				Max time.Time `json:"max"`
			}{n, x}
			t.ExtendedBounds = &bounds
		}

		d.Type = t
	}
	if d.TypeName == "histogram" {
		var n float64
		var x float64
		t := d.Type.(Histogram)
		if min != nil {
			switch min.(type) {
			case time.Time:
				n = min.(float64)
			}
		}
		if max != nil {
			switch max.(type) {
			case time.Time:
				x = max.(float64)
			}
		}

		if min == nil {
			bounds := struct {
				Max float64 `json:"max"`
			}{x}
			t.ExtendedBounds = &bounds
		} else if max == nil {
			bounds := struct {
				Min float64 `json:"min"`
			}{n}
			t.ExtendedBounds = &bounds
		} else {
			bounds := struct {
				Min float64 `json:"min"`
				Max float64 `json:"max"`
			}{n, x}
			t.ExtendedBounds = &bounds
		}

		d.Type = t
	}

	return d
}
func (d *AggregateDsl) MarshalJSON() ([]byte, error) {
	return json.Marshal(d.ToMap())
}

func (d *AggregateDsl) ToMap() map[string]interface{} {
	root := map[string]interface{}{}

	if d.Type != nil {
		root[d.TypeName] = d.Type
	}
	aggregates := d.AggregatesMap()

	if d.Filters != nil {
		root["filter"] = d.Filters
	}

	if len(aggregates) > 0 {
		root["aggregations"] = aggregates
	}
	return root

}
func (d *AggregateDsl) AggregatesMap() map[string]interface{} {
	root := map[string]interface{}{}

	if len(d.AggregatesVal) > 0 {
		for _, agg := range d.AggregatesVal {
			root[agg.Name] = agg.ToMap()
		}
	}
	return root
}
<|endoftext|>


<commit_before>package log

import (
	"os"
	"path/filepath"
	"runtime"
	"strings"
)

var (
	pid         = os.Getpid()
	hostname, _ = os.Hostname()
	appname     = filepath.Base(os.Args[0])
)

// CallerInfo encapsulates information about a piece of code that called a certain log function,
// such as file name, line number, etc.
type CallerInfo struct {
	FileName string
	FilePath string
	FuncName string
	LineNo   int
}

// getCallerInfo returns information about a certain log function invoker
// such as file name, function name and line number
func getCallerInfo(depth int) *CallerInfo {
	if pc, filePath, lineNo, ok := runtime.Caller(depth + 1); !ok {
		return &CallerInfo{"unknown_file", "unknown_path", "unknown_func", 0}
	} else {
		var fileName string
		slashPos := strings.LastIndex(filePath, "/")
		if slashPos >= 0 {
			fileName = filePath[slashPos+1:]
		}
		return &CallerInfo{fileName, filePath, runtime.FuncForPC(pc).Name(), lineNo}
	}
}
<commit_msg>One more fix<commit_after>package log

import (
	"os"
	"path/filepath"
	"runtime"
)

var (
	pid         = os.Getpid()
	hostname, _ = os.Hostname()
	appname     = filepath.Base(os.Args[0])
)

// CallerInfo encapsulates information about a piece of code that called a certain log function,
// such as file name, line number, etc.
type CallerInfo struct {
	FileName string
	FilePath string
	FuncName string
	LineNo   int
}

// getCallerInfo returns information about a certain log function invoker
// such as file name, function name and line number
func getCallerInfo(depth int) *CallerInfo {
	if pc, filePath, lineNo, ok := runtime.Caller(depth + 1); !ok {
		return &CallerInfo{"unknown_file", "unknown_path", "unknown_func", 0}
	} else {
		return &CallerInfo{filepath.Base(filePath), filePath, runtime.FuncForPC(pc).Name(), lineNo}
	}
}
<|endoftext|>


<commit_before>package canal

import (
	"time"

	"github.com/juju/errors"
	"github.com/siddontang/go-mysql/mysql"
	"github.com/siddontang/go-mysql/replication"
	"github.com/siddontang/go/log"
)

func (c *Canal) startSyncBinlog() error {
	pos := mysql.Position{c.master.Name, c.master.Position}

	log.Infof("start sync binlog at %v", pos)

	s, err := c.syncer.StartSync(pos)
	if err != nil {
		return errors.Errorf("start sync replication at %v error %v", pos, err)
	}

	timeout := time.Second
	forceSavePos := false
	for {
		ev, err := s.GetEventTimeout(timeout)
		if err != nil && err != replication.ErrGetEventTimeout {
			return errors.Trace(err)
		} else if err == replication.ErrGetEventTimeout {
			timeout = 2 * timeout
			continue
		}

		timeout = time.Second

		//next binlog pos
		pos.Pos = ev.Header.LogPos

		forceSavePos = false

		switch e := ev.Event.(type) {
		case *replication.RotateEvent:
			pos.Name = string(e.NextLogName)
			pos.Pos = uint32(e.Position)
			// r.ev <- pos
			forceSavePos = true
			log.Infof("rotate binlog to %v", pos)
		case *replication.RowsEvent:
			// we only focus row based event
			if err = c.handleRowsEvent(ev); err != nil {
				log.Errorf("handle rows event error %v", err)
				return errors.Trace(err)
			}
		default:
		}

		c.master.Update(pos.Name, pos.Pos)
		c.master.Save(forceSavePos)
	}

	return nil
}

func (c *Canal) handleRowsEvent(e *replication.BinlogEvent) error {
	ev := e.Event.(*replication.RowsEvent)

	// Caveat: table may be altered at runtime.
	schema := string(ev.Table.Schema)
	table := string(ev.Table.Table)

	t, err := c.GetTable(schema, table)
	if err != nil {
		return errors.Trace(err)
	}
	var action string
	switch e.Header.EventType {
	case replication.WRITE_ROWS_EVENTv1, replication.WRITE_ROWS_EVENTv2:
		action = InsertAction
	case replication.DELETE_ROWS_EVENTv1, replication.DELETE_ROWS_EVENTv2:
		action = DeleteAction
	case replication.UPDATE_ROWS_EVENTv1, replication.UPDATE_ROWS_EVENTv2:
		action = UpdateAction
	default:
		return errors.Errorf("%s not supported now", e.Header.EventType)
	}
	events := newRowsEvent(t, action, ev.Rows)
	return c.travelRowsEventHandler(events)
}

func (c *Canal) WaitUntilPos(pos mysql.Position, timeout int) error {
	if timeout <= 0 {
		timeout = 60
	}

	timer := time.NewTimer(time.Duration(timeout) * time.Second)
	for {
		select {
		case <-timer.C:
			return errors.Errorf("wait position %v err", pos)
		default:
			curpos := c.master.Pos()
			if curpos.Compare(pos) >= 0 {
				return nil
			} else {
				time.Sleep(100 * time.Millisecond)
			}
		}
	}

	return nil
}

func (c *Canal) CatchMasterPos(timeout int) error {
	rr, err := c.Execute("SHOW MASTER STATUS")
	if err != nil {
		return errors.Trace(err)
	}

	name, _ := rr.GetString(0, 0)
	pos, _ := rr.GetInt(0, 1)

	return c.WaitUntilPos(mysql.Position{name, uint32(pos)}, timeout)
}
<commit_msg>fix table id not found<commit_after>package canal

import (
	"time"

	"github.com/juju/errors"
	"github.com/siddontang/go-mysql/mysql"
	"github.com/siddontang/go-mysql/replication"
	"github.com/siddontang/go/log"
)

func (c *Canal) startSyncBinlog() error {
	pos := mysql.Position{c.master.Name, c.master.Position}

	log.Infof("start sync binlog at %v", pos)

	s, err := c.syncer.StartSync(pos)
	if err != nil {
		return errors.Errorf("start sync replication at %v error %v", pos, err)
	}

	timeout := time.Second
	forceSavePos := false
	for {
		ev, err := s.GetEventTimeout(timeout)
		if err != nil && err != replication.ErrGetEventTimeout {
			return errors.Trace(err)
		} else if err == replication.ErrGetEventTimeout {
			timeout = 2 * timeout
			continue
		}

		timeout = time.Second

		//next binlog pos
		pos.Pos = ev.Header.LogPos

		forceSavePos = false

		switch e := ev.Event.(type) {
		case *replication.RotateEvent:
			pos.Name = string(e.NextLogName)
			pos.Pos = uint32(e.Position)
			// r.ev <- pos
			forceSavePos = true
			log.Infof("rotate binlog to %v", pos)
		case *replication.RowsEvent:
			// we only focus row based event
			if err = c.handleRowsEvent(ev); err != nil {
				log.Errorf("handle rows event error %v", err)
				return errors.Trace(err)
			}
		case *replication.TableMapEvent:
			continue
		default:
		}

		c.master.Update(pos.Name, pos.Pos)
		c.master.Save(forceSavePos)
	}

	return nil
}

func (c *Canal) handleRowsEvent(e *replication.BinlogEvent) error {
	ev := e.Event.(*replication.RowsEvent)

	// Caveat: table may be altered at runtime.
	schema := string(ev.Table.Schema)
	table := string(ev.Table.Table)

	t, err := c.GetTable(schema, table)
	if err != nil {
		return errors.Trace(err)
	}
	var action string
	switch e.Header.EventType {
	case replication.WRITE_ROWS_EVENTv1, replication.WRITE_ROWS_EVENTv2:
		action = InsertAction
	case replication.DELETE_ROWS_EVENTv1, replication.DELETE_ROWS_EVENTv2:
		action = DeleteAction
	case replication.UPDATE_ROWS_EVENTv1, replication.UPDATE_ROWS_EVENTv2:
		action = UpdateAction
	default:
		return errors.Errorf("%s not supported now", e.Header.EventType)
	}
	events := newRowsEvent(t, action, ev.Rows)
	return c.travelRowsEventHandler(events)
}

func (c *Canal) WaitUntilPos(pos mysql.Position, timeout int) error {
	if timeout <= 0 {
		timeout = 60
	}

	timer := time.NewTimer(time.Duration(timeout) * time.Second)
	for {
		select {
		case <-timer.C:
			return errors.Errorf("wait position %v err", pos)
		default:
			curpos := c.master.Pos()
			if curpos.Compare(pos) >= 0 {
				return nil
			} else {
				time.Sleep(100 * time.Millisecond)
			}
		}
	}

	return nil
}

func (c *Canal) CatchMasterPos(timeout int) error {
	rr, err := c.Execute("SHOW MASTER STATUS")
	if err != nil {
		return errors.Trace(err)
	}

	name, _ := rr.GetString(0, 0)
	pos, _ := rr.GetInt(0, 1)

	return c.WaitUntilPos(mysql.Position{name, uint32(pos)}, timeout)
}
<|endoftext|>


<commit_before>// Copyright 2014 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package cas implements a content-addressable-store on disk.
// It leverages the `diskv` package to store items in a simple
// key-value blob store: https://github.com/peterbourgon/diskv
package cas

import (
	"database/sql"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"strings"
	"time"

	"github.com/coreos/rocket/pkg/keystore"

	"github.com/coreos/rocket/Godeps/_workspace/src/github.com/appc/docker2aci/lib"
	"github.com/coreos/rocket/Godeps/_workspace/src/github.com/appc/spec/aci"

	"github.com/coreos/rocket/Godeps/_workspace/src/github.com/mitchellh/ioprogress"
	"github.com/coreos/rocket/Godeps/_workspace/src/golang.org/x/crypto/openpgp"
)

func NewRemote(aciurl, sigurl string) *Remote {
	r := &Remote{
		ACIURL: aciurl,
		SigURL: sigurl,
	}
	return r
}

type Remote struct {
	ACIURL string
	SigURL string
	ETag   string
	// The key in the blob store under which the ACI has been saved.
	BlobKey string
}

// Download downloads and verifies the remote ACI.
// If Keystore is nil signature verification will be skipped.
// Download returns the signer, an *os.File representing the ACI, and an error if any.
// err will be nil if the ACI downloads successfully and the ACI is verified.
func (r Remote) Download(ds Store, ks *keystore.Keystore) (*openpgp.Entity, *os.File, error) {
	var entity *openpgp.Entity
	u, err := url.Parse(r.ACIURL)
	if err != nil {
		return nil, nil, fmt.Errorf("error parsing ACI url: %v", err)
	}
	if u.Scheme == "docker" {
		registryURL := strings.TrimPrefix(r.ACIURL, "docker://")

		tmpDir, err := ds.tmpDir()
		if err != nil {
			return nil, nil, fmt.Errorf("error creating temporary dir for docker to ACI conversion: %v", err)
		}

		acis, err := docker2aci.Convert(registryURL, true, tmpDir)
		if err != nil {
			return nil, nil, fmt.Errorf("error converting docker image to ACI: %v", err)
		}

		aciFile, err := os.Open(acis[0])
		if err != nil {
			return nil, nil, fmt.Errorf("error opening squashed ACI file: %v", err)
		}

		return nil, aciFile, nil
	}

	acif, err := downloadACI(ds, r.ACIURL)
	if err != nil {
		return nil, acif, fmt.Errorf("error downloading the aci image: %v", err)
	}

	if ks != nil {
		fmt.Printf("Downloading signature from %v\n", r.SigURL)
		sigTempFile, err := downloadSignatureFile(r.SigURL)
		if err != nil {
			return nil, acif, fmt.Errorf("error downloading the signature file: %v", err)
		}
		defer sigTempFile.Close()
		defer os.Remove(sigTempFile.Name())

		manifest, err := aci.ManifestFromImage(acif)
		if err != nil {
			return nil, acif, err
		}

		if _, err := acif.Seek(0, 0); err != nil {
			return nil, acif, err
		}
		if _, err := sigTempFile.Seek(0, 0); err != nil {
			return nil, acif, err
		}
		if entity, err = ks.CheckSignature(manifest.Name.String(), acif, sigTempFile); err != nil {
			return nil, acif, err
		}
	}

	if _, err := acif.Seek(0, 0); err != nil {
		return nil, acif, err
	}
	return entity, acif, nil
}

// TODO: add locking
// Store stores the ACI represented by r in the target data store.
func (r Remote) Store(ds Store, aci io.Reader) (*Remote, error) {
	key, err := ds.WriteACI(aci)
	if err != nil {
		return nil, err
	}
	r.BlobKey = key
	err = ds.WriteRemote(&r)
	if err != nil {
		return nil, err
	}
	return &r, nil
}

// downloadACI gets the aci specified at aciurl
func downloadACI(ds Store, aciurl string) (*os.File, error) {
	return downloadHTTP(aciurl, "ACI", ds.tmpFile)
}

// downloadSignatureFile gets the signature specified at sigurl
func downloadSignatureFile(sigurl string) (*os.File, error) {
	getTemp := func() (*os.File, error) {
		return ioutil.TempFile("", "")
	}

	return downloadHTTP(sigurl, "signature", getTemp)
}

// downloadHTTP retrieves url, creating a temp file using getTempFile
// file:// http:// and https:// urls supported
func downloadHTTP(url, label string, getTempFile func() (*os.File, error)) (*os.File, error) {
	tmp, err := getTempFile()
	if err != nil {
		return nil, fmt.Errorf("error downloading %s: %v", label, err)
	}
	defer func() {
		if err != nil {
			os.Remove(tmp.Name())
			tmp.Close()
		}
	}()

	res, err := http.Get(url)
	if err != nil {
		return nil, err
	}
	defer res.Body.Close()

	prefix := "Downloading " + label
	fmtBytesSize := 18
	barSize := int64(80 - len(prefix) - fmtBytesSize)
	bar := ioprogress.DrawTextFormatBar(barSize)
	fmtfunc := func(progress, total int64) string {
		return fmt.Sprintf(
			"%s: %s %s",
			prefix,
			bar(progress, total),
			ioprogress.DrawTextFormatBytes(progress, total),
		)
	}

	reader := &ioprogress.Reader{
		Reader:       res.Body,
		Size:         res.ContentLength,
		DrawFunc:     ioprogress.DrawTerminalf(os.Stdout, fmtfunc),
		DrawInterval: time.Second,
	}

	// TODO(jonboulle): handle http more robustly (redirects?)
	if res.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("bad HTTP status code: %d", res.StatusCode)
	}

	if _, err := io.Copy(tmp, reader); err != nil {
		return nil, fmt.Errorf("error copying %s: %v", label, err)
	}

	if err := tmp.Sync(); err != nil {
		return nil, fmt.Errorf("error writing %s: %v", label, err)
	}

	return tmp, nil
}

// GetRemote tries to retrieve a remote with the given aciURL. found will be
// false if remote doesn't exist.
func GetRemote(tx *sql.Tx, aciURL string) (remote *Remote, found bool, err error) {
	remote = &Remote{}
	rows, err := tx.Query("SELECT sigurl, etag, blobkey FROM remote WHERE aciurl == $1", aciURL)
	if err != nil {
		return nil, false, err
	}
	for rows.Next() {
		found = true
		if err := rows.Scan(&remote.SigURL, &remote.ETag, &remote.BlobKey); err != nil {
			return nil, false, err
		}
	}
	if err := rows.Err(); err != nil {
		return nil, false, err
	}

	return remote, found, err
}

// WriteRemote adds or updates the provided Remote.
func WriteRemote(tx *sql.Tx, remote *Remote) error {
	// ql doesn't have an INSERT OR UPDATE function so
	// it's faster to remove and reinsert the row
	_, err := tx.Exec("DELETE FROM remote WHERE aciurl == $1", remote.ACIURL)
	if err != nil {
		return err
	}
	_, err = tx.Exec("INSERT INTO remote VALUES ($1, $2, $3, $4)", remote.ACIURL, remote.SigURL, remote.ETag, remote.BlobKey)
	if err != nil {
		return err
	}
	return nil
}
<commit_msg>cas: remote: download the sig before the ACI<commit_after>// Copyright 2014 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package cas implements a content-addressable-store on disk.
// It leverages the `diskv` package to store items in a simple
// key-value blob store: https://github.com/peterbourgon/diskv
package cas

import (
	"database/sql"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"strings"
	"time"

	"github.com/coreos/rocket/pkg/keystore"

	"github.com/coreos/rocket/Godeps/_workspace/src/github.com/appc/docker2aci/lib"
	"github.com/coreos/rocket/Godeps/_workspace/src/github.com/appc/spec/aci"

	"github.com/coreos/rocket/Godeps/_workspace/src/github.com/mitchellh/ioprogress"
	"github.com/coreos/rocket/Godeps/_workspace/src/golang.org/x/crypto/openpgp"
)

func NewRemote(aciurl, sigurl string) *Remote {
	r := &Remote{
		ACIURL: aciurl,
		SigURL: sigurl,
	}
	return r
}

type Remote struct {
	ACIURL string
	SigURL string
	ETag   string
	// The key in the blob store under which the ACI has been saved.
	BlobKey string
}

// Download downloads and verifies the remote ACI.
// If Keystore is nil signature verification will be skipped.
// Download returns the signer, an *os.File representing the ACI, and an error if any.
// err will be nil if the ACI downloads successfully and the ACI is verified.
func (r Remote) Download(ds Store, ks *keystore.Keystore) (*openpgp.Entity, *os.File, error) {
	var entity *openpgp.Entity
	u, err := url.Parse(r.ACIURL)
	if err != nil {
		return nil, nil, fmt.Errorf("error parsing ACI url: %v", err)
	}
	if u.Scheme == "docker" {
		registryURL := strings.TrimPrefix(r.ACIURL, "docker://")

		tmpDir, err := ds.tmpDir()
		if err != nil {
			return nil, nil, fmt.Errorf("error creating temporary dir for docker to ACI conversion: %v", err)
		}

		acis, err := docker2aci.Convert(registryURL, true, tmpDir)
		if err != nil {
			return nil, nil, fmt.Errorf("error converting docker image to ACI: %v", err)
		}

		aciFile, err := os.Open(acis[0])
		if err != nil {
			return nil, nil, fmt.Errorf("error opening squashed ACI file: %v", err)
		}

		return nil, aciFile, nil
	}

	var sigTempFile *os.File
	if ks != nil {
		fmt.Printf("Downloading signature from %v\n", r.SigURL)
		sigTempFile, err = downloadSignatureFile(r.SigURL)
		if err != nil {
			return nil, nil, fmt.Errorf("error downloading the signature file: %v", err)
		}
		defer sigTempFile.Close()
		defer os.Remove(sigTempFile.Name())
	}

	acif, err := downloadACI(ds, r.ACIURL)
	if err != nil {
		return nil, acif, fmt.Errorf("error downloading the aci image: %v", err)
	}

	if ks != nil {
		manifest, err := aci.ManifestFromImage(acif)
		if err != nil {
			return nil, acif, err
		}

		if _, err := acif.Seek(0, 0); err != nil {
			return nil, acif, err
		}
		if _, err := sigTempFile.Seek(0, 0); err != nil {
			return nil, acif, err
		}
		if entity, err = ks.CheckSignature(manifest.Name.String(), acif, sigTempFile); err != nil {
			return nil, acif, err
		}
	}

	if _, err := acif.Seek(0, 0); err != nil {
		return nil, acif, err
	}
	return entity, acif, nil
}

// TODO: add locking
// Store stores the ACI represented by r in the target data store.
func (r Remote) Store(ds Store, aci io.Reader) (*Remote, error) {
	key, err := ds.WriteACI(aci)
	if err != nil {
		return nil, err
	}
	r.BlobKey = key
	err = ds.WriteRemote(&r)
	if err != nil {
		return nil, err
	}
	return &r, nil
}

// downloadACI gets the aci specified at aciurl
func downloadACI(ds Store, aciurl string) (*os.File, error) {
	return downloadHTTP(aciurl, "ACI", ds.tmpFile)
}

// downloadSignatureFile gets the signature specified at sigurl
func downloadSignatureFile(sigurl string) (*os.File, error) {
	getTemp := func() (*os.File, error) {
		return ioutil.TempFile("", "")
	}

	return downloadHTTP(sigurl, "signature", getTemp)
}

// downloadHTTP retrieves url, creating a temp file using getTempFile
// file:// http:// and https:// urls supported
func downloadHTTP(url, label string, getTempFile func() (*os.File, error)) (*os.File, error) {
	tmp, err := getTempFile()
	if err != nil {
		return nil, fmt.Errorf("error downloading %s: %v", label, err)
	}
	defer func() {
		if err != nil {
			os.Remove(tmp.Name())
			tmp.Close()
		}
	}()

	res, err := http.Get(url)
	if err != nil {
		return nil, err
	}
	defer res.Body.Close()

	prefix := "Downloading " + label
	fmtBytesSize := 18
	barSize := int64(80 - len(prefix) - fmtBytesSize)
	bar := ioprogress.DrawTextFormatBar(barSize)
	fmtfunc := func(progress, total int64) string {
		return fmt.Sprintf(
			"%s: %s %s",
			prefix,
			bar(progress, total),
			ioprogress.DrawTextFormatBytes(progress, total),
		)
	}

	reader := &ioprogress.Reader{
		Reader:       res.Body,
		Size:         res.ContentLength,
		DrawFunc:     ioprogress.DrawTerminalf(os.Stdout, fmtfunc),
		DrawInterval: time.Second,
	}

	// TODO(jonboulle): handle http more robustly (redirects?)
	if res.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("bad HTTP status code: %d", res.StatusCode)
	}

	if _, err := io.Copy(tmp, reader); err != nil {
		return nil, fmt.Errorf("error copying %s: %v", label, err)
	}

	if err := tmp.Sync(); err != nil {
		return nil, fmt.Errorf("error writing %s: %v", label, err)
	}

	return tmp, nil
}

// GetRemote tries to retrieve a remote with the given aciURL. found will be
// false if remote doesn't exist.
func GetRemote(tx *sql.Tx, aciURL string) (remote *Remote, found bool, err error) {
	remote = &Remote{}
	rows, err := tx.Query("SELECT sigurl, etag, blobkey FROM remote WHERE aciurl == $1", aciURL)
	if err != nil {
		return nil, false, err
	}
	for rows.Next() {
		found = true
		if err := rows.Scan(&remote.SigURL, &remote.ETag, &remote.BlobKey); err != nil {
			return nil, false, err
		}
	}
	if err := rows.Err(); err != nil {
		return nil, false, err
	}

	return remote, found, err
}

// WriteRemote adds or updates the provided Remote.
func WriteRemote(tx *sql.Tx, remote *Remote) error {
	// ql doesn't have an INSERT OR UPDATE function so
	// it's faster to remove and reinsert the row
	_, err := tx.Exec("DELETE FROM remote WHERE aciurl == $1", remote.ACIURL)
	if err != nil {
		return err
	}
	_, err = tx.Exec("INSERT INTO remote VALUES ($1, $2, $3, $4)", remote.ACIURL, remote.SigURL, remote.ETag, remote.BlobKey)
	if err != nil {
		return err
	}
	return nil
}
<|endoftext|>