<commit_before>package ginkgo

import (
	"bytes"
	"context"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/signal"
	"path/filepath"
	"sort"
	"strings"
	"syscall"
	"time"

	"github.com/openshift/origin/pkg/monitor"

	"github.com/onsi/ginkgo/config"
)

// Options is used to run a suite of tests by invoking each test
// as a call to a child worker (the run-tests command).
type Options struct {
	Parallelism int
	Count       int
	Timeout     time.Duration
	JUnitDir    string
	TestFile    string
	OutFile     string

	// Regex allows a selection of a subset of tests
	Regex string
	// MatchFn if set is also used to filter the suite contents
	MatchFn func(name string) bool

	// SyntheticEventTests allows the caller to translate events or outside
	// context into a failure.
	SyntheticEventTests JUnitsForEvents

	IncludeSuccessOutput bool

	FromRepository string
	Provider       string
	SuiteOptions   string

	Suites []*TestSuite

	DryRun        bool
	PrintCommands bool
	Out, ErrOut   io.Writer

	StartTime time.Time
}

func (opt *Options) AsEnv() []string {
	var args []string
	args = append(args, "KUBE_TEST_REPO_LIST=") // explicitly prevent selective override
	args = append(args, fmt.Sprintf("KUBE_TEST_REPO=%s", opt.FromRepository))
	args = append(args, fmt.Sprintf("TEST_PROVIDER=%s", opt.Provider))
	args = append(args, fmt.Sprintf("TEST_SUITE_OPTIONS=%s", opt.SuiteOptions))
	args = append(args, fmt.Sprintf("TEST_SUITE_START_TIME=%d", opt.StartTime.Unix()))
	return args
}

func (opt *Options) Run(args []string) error {
	var suite *TestSuite

	if len(opt.TestFile) > 0 {
		var in []byte
		var err error
		if opt.TestFile == "-" {
			in, err = ioutil.ReadAll(os.Stdin)
			if err != nil {
				return err
			}
		} else {
			in, err = ioutil.ReadFile(opt.TestFile)
		}
		if err != nil {
			return err
		}
		suite, err = newSuiteFromFile("files", in)
		if err != nil {
			return fmt.Errorf("could not read test suite from input: %v", err)
		}
	}

	if suite == nil && len(args) == 0 {
		fmt.Fprintf(opt.ErrOut, SuitesString(opt.Suites, "Select a test suite to run against the server:\n\n"))
		return fmt.Errorf("specify a test suite to run, for example: %s run %s", filepath.Base(os.Args[0]), opt.Suites[0].Name)
	}
	if suite == nil && len(args) > 0 {
		for _, s := range opt.Suites {
			if s.Name == args[0] {
				suite = s
				break
			}
		}
	}
	if suite == nil {
		fmt.Fprintf(opt.ErrOut, SuitesString(opt.Suites, "Select a test suite to run against the server:\n\n"))
		return fmt.Errorf("suite %q does not exist", args[0])
	}

	if len(opt.Regex) > 0 {
		if err := filterWithRegex(suite, opt.Regex); err != nil {
			return err
		}
	}
	if opt.MatchFn != nil {
		original := suite.Matches
		suite.Matches = func(name string) bool {
			return original(name) && opt.MatchFn(name)
		}
	}

	syntheticEventTests := JUnitsForAllEvents{
		opt.SyntheticEventTests,
		suite.SyntheticEventTests,
	}

	tests, err := testsForSuite(config.GinkgoConfig)
	if err != nil {
		return err
	}

	// This ensures that tests in the identified paths do not run in parallel, because
	// the test suite reuses shared resources without considering whether another test
	// could be running at the same time. While these are technically [Serial], ginkgo
	// parallel mode provides this guarantee. Doing this for all suites would be too
	// slow.
	setTestExclusion(tests, func(suitePath string, t *testCase) bool {
		for _, name := range []string{
			"/k8s.io/kubernetes/test/e2e/apps/disruption.go",
		} {
			if strings.HasSuffix(suitePath, name) {
				return true
			}
		}
		return false
	})

	tests = suite.Filter(tests)
	if len(tests) == 0 {
		return fmt.Errorf("suite %q does not contain any tests", suite.Name)
	}

	count := opt.Count
	if count < 1 {
		count = suite.Count
	}
	if count > 1 {
		var newTests []*testCase
		for i := 0; i < count; i++ {
			for _, t := range tests {
				copied := *t
				newTests = append(newTests, &copied)
			}
		}
		tests = newTests
	}

	start := time.Now()
	if opt.StartTime.IsZero() {
		opt.StartTime = start
	}

	if opt.PrintCommands {
		status := newTestStatus(opt.Out, true, len(tests), time.Minute, &monitor.Monitor{}, opt.AsEnv())
		newParallelTestQueue(tests).Execute(context.Background(), 1, status.OutputCommand)
		return nil
	}
	if opt.DryRun {
		for _, test := range sortedTests(tests) {
			fmt.Fprintf(opt.Out, "%q\n", test.name)
		}
		return nil
	}

	if len(opt.JUnitDir) > 0 {
		if _, err := os.Stat(opt.JUnitDir); err != nil {
			if !os.IsNotExist(err) {
				return fmt.Errorf("could not access --junit-dir: %v", err)
			}
			if err := os.MkdirAll(opt.JUnitDir, 0755); err != nil {
				return fmt.Errorf("could not create --junit-dir: %v", err)
			}
		}
	}

	parallelism := opt.Parallelism
	if parallelism == 0 {
		parallelism = suite.Parallelism
	}
	if parallelism == 0 {
		parallelism = 10
	}
	timeout := opt.Timeout
	if timeout == 0 {
		timeout = suite.TestTimeout
	}
	if timeout == 0 {
		timeout = 15 * time.Minute
	}

	ctx, cancelFn := context.WithCancel(context.Background())
	defer cancelFn()
	abortCh := make(chan os.Signal)
	go func() {
		<-abortCh
		fmt.Fprintf(opt.ErrOut, "Interrupted, terminating tests\n")
		cancelFn()
		sig := <-abortCh
		fmt.Fprintf(opt.ErrOut, "Interrupted twice, exiting (%s)\n", sig)
		switch sig {
		case syscall.SIGINT:
			os.Exit(130)
		default:
			os.Exit(0)
		}
	}()
	signal.Notify(abortCh, syscall.SIGINT, syscall.SIGTERM)

	m, err := monitor.Start(ctx)
	if err != nil {
		return err
	}
	// if we run a single test, always include success output
	includeSuccess := opt.IncludeSuccessOutput
	if len(tests) == 1 {
		includeSuccess = true
	}
	status := newTestStatus(opt.Out, includeSuccess, len(tests), timeout, m, opt.AsEnv())

	early, normal := splitTests(tests, func(t *testCase) bool {
		return strings.Contains(t.name, "[Early]")
	})

	late, normal := splitTests(normal, func(t *testCase) bool {
		return strings.Contains(t.name, "[Late]")
	})

	// run the tests

	// run our Early tests first
	q := newParallelTestQueue(early)
	q.Execute(ctx, parallelism, status.Run)

	// run other tests next
	q = newParallelTestQueue(normal)
	q.Execute(ctx, parallelism, status.Run)

	// run Late test suits after everything else
	q = newParallelTestQueue(late)
	q.Execute(ctx, parallelism, status.Run)

	duration := time.Now().Sub(start).Round(time.Second / 10)
	if duration > time.Minute {
		duration = duration.Round(time.Second)
	}

	pass, fail, skip, failing := summarizeTests(tests)

	// monitor the cluster while the tests are running and report any detected anomalies
	var syntheticTestResults []*JUnitTestCase
	var syntheticFailure bool
	if events := m.Events(time.Time{}, time.Time{}); len(events) > 0 {
		eventsForTests := createEventsForTests(tests)

		var buf *bytes.Buffer
		syntheticTestResults, buf, _ = createSyntheticTestsFromMonitor(m, eventsForTests, duration)
		testCases, passed := syntheticEventTests.JUnitsForEvents(events, duration)
		syntheticTestResults = append(syntheticTestResults, testCases...)
		if !passed {
			syntheticFailure = true
		}

		fmt.Fprintln(buf, "Synthetic test results:")
		for _, test := range syntheticTestResults {
			status := "passed"
			if test.FailureOutput != nil {
				status = "failed"
			}
			fmt.Fprintf(buf, "%s: %s\n", status, test.Name)
		}
		fmt.Fprintln(buf, "(duplicates are used to mark some failures as flake and not to fail the whole suite")
		fmt.Fprintln(buf)

		opt.Out.Write(buf.Bytes())
	}

	// attempt to retry failures to do flake detection
	if fail > 0 && fail <= suite.MaximumAllowedFlakes {
		var retries []*testCase
		for _, test := range failing {
			retry := test.Retry()
			retries = append(retries, retry)
			tests = append(tests, retry)
			if len(retries) > suite.MaximumAllowedFlakes {
				break
			}
		}

		q := newParallelTestQueue(retries)
		status := newTestStatus(ioutil.Discard, opt.IncludeSuccessOutput, len(retries), timeout, m, opt.AsEnv())
		q.Execute(ctx, parallelism, status.Run)
		var flaky []string
		var repeatFailures []*testCase
		for _, test := range retries {
			if test.success {
				flaky = append(flaky, test.name)
			} else {
				repeatFailures = append(repeatFailures, test)
			}
		}
		if len(flaky) > 0 {
			failing = repeatFailures
			sort.Strings(flaky)
			fmt.Fprintf(opt.Out, "Flaky tests:\n\n%s\n\n", strings.Join(flaky, "\n"))
		}
	}

	if len(failing) > 0 {
		names := testNames(failing)
		sort.Strings(names)
		fmt.Fprintf(opt.Out, "Failing tests:\n\n%s\n\n", strings.Join(names, "\n"))
	}

	if len(opt.JUnitDir) > 0 {
		if err := writeJUnitReport("junit_e2e", "openshift-tests", tests, opt.JUnitDir, duration, opt.ErrOut, syntheticTestResults...); err != nil {
			fmt.Fprintf(opt.Out, "error: Unable to write e2e JUnit results: %v", err)
		}
	}

	if fail > 0 {
		if len(failing) > 0 || suite.MaximumAllowedFlakes == 0 {
			return fmt.Errorf("%d fail, %d pass, %d skip (%s)", fail, pass, skip, duration)
		}
		fmt.Fprintf(opt.Out, "%d flakes detected, suite allows passing with only flakes\n\n", fail)
	}

	if syntheticFailure {
		return fmt.Errorf("failed because an invariant was violated, %d pass, %d skip (%s)\n", pass, skip, duration)
	}

	fmt.Fprintf(opt.Out, "%d pass, %d skip (%s)\n", pass, skip, duration)
	return ctx.Err()
}
<commit_msg>test/ginkgo: Improve output when multiple tests are being run<commit_after>package ginkgo

import (
	"bytes"
	"context"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/signal"
	"path/filepath"
	"sort"
	"strings"
	"syscall"
	"time"

	"github.com/openshift/origin/pkg/monitor"
	"k8s.io/apimachinery/pkg/util/sets"

	"github.com/onsi/ginkgo/config"
)

// Options is used to run a suite of tests by invoking each test
// as a call to a child worker (the run-tests command).
type Options struct {
	Parallelism int
	Count       int
	Timeout     time.Duration
	JUnitDir    string
	TestFile    string
	OutFile     string

	// Regex allows a selection of a subset of tests
	Regex string
	// MatchFn if set is also used to filter the suite contents
	MatchFn func(name string) bool

	// SyntheticEventTests allows the caller to translate events or outside
	// context into a failure.
	SyntheticEventTests JUnitsForEvents

	IncludeSuccessOutput bool

	FromRepository string
	Provider       string
	SuiteOptions   string

	Suites []*TestSuite

	DryRun        bool
	PrintCommands bool
	Out, ErrOut   io.Writer

	StartTime time.Time
}

func (opt *Options) AsEnv() []string {
	var args []string
	args = append(args, "KUBE_TEST_REPO_LIST=") // explicitly prevent selective override
	args = append(args, fmt.Sprintf("KUBE_TEST_REPO=%s", opt.FromRepository))
	args = append(args, fmt.Sprintf("TEST_PROVIDER=%s", opt.Provider))
	args = append(args, fmt.Sprintf("TEST_SUITE_OPTIONS=%s", opt.SuiteOptions))
	args = append(args, fmt.Sprintf("TEST_SUITE_START_TIME=%d", opt.StartTime.Unix()))
	return args
}

func (opt *Options) Run(args []string) error {
	var suite *TestSuite

	if len(opt.TestFile) > 0 {
		var in []byte
		var err error
		if opt.TestFile == "-" {
			in, err = ioutil.ReadAll(os.Stdin)
			if err != nil {
				return err
			}
		} else {
			in, err = ioutil.ReadFile(opt.TestFile)
		}
		if err != nil {
			return err
		}
		suite, err = newSuiteFromFile("files", in)
		if err != nil {
			return fmt.Errorf("could not read test suite from input: %v", err)
		}
	}

	if suite == nil && len(args) == 0 {
		fmt.Fprintf(opt.ErrOut, SuitesString(opt.Suites, "Select a test suite to run against the server:\n\n"))
		return fmt.Errorf("specify a test suite to run, for example: %s run %s", filepath.Base(os.Args[0]), opt.Suites[0].Name)
	}
	if suite == nil && len(args) > 0 {
		for _, s := range opt.Suites {
			if s.Name == args[0] {
				suite = s
				break
			}
		}
	}
	if suite == nil {
		fmt.Fprintf(opt.ErrOut, SuitesString(opt.Suites, "Select a test suite to run against the server:\n\n"))
		return fmt.Errorf("suite %q does not exist", args[0])
	}

	if len(opt.Regex) > 0 {
		if err := filterWithRegex(suite, opt.Regex); err != nil {
			return err
		}
	}
	if opt.MatchFn != nil {
		original := suite.Matches
		suite.Matches = func(name string) bool {
			return original(name) && opt.MatchFn(name)
		}
	}

	syntheticEventTests := JUnitsForAllEvents{
		opt.SyntheticEventTests,
		suite.SyntheticEventTests,
	}

	tests, err := testsForSuite(config.GinkgoConfig)
	if err != nil {
		return err
	}

	// This ensures that tests in the identified paths do not run in parallel, because
	// the test suite reuses shared resources without considering whether another test
	// could be running at the same time. While these are technically [Serial], ginkgo
	// parallel mode provides this guarantee. Doing this for all suites would be too
	// slow.
	setTestExclusion(tests, func(suitePath string, t *testCase) bool {
		for _, name := range []string{
			"/k8s.io/kubernetes/test/e2e/apps/disruption.go",
		} {
			if strings.HasSuffix(suitePath, name) {
				return true
			}
		}
		return false
	})

	tests = suite.Filter(tests)
	if len(tests) == 0 {
		return fmt.Errorf("suite %q does not contain any tests", suite.Name)
	}

	count := opt.Count
	if count < 1 {
		count = suite.Count
	}
	if count > 1 {
		var newTests []*testCase
		for i := 0; i < count; i++ {
			for _, t := range tests {
				copied := *t
				newTests = append(newTests, &copied)
			}
		}
		tests = newTests
	}

	start := time.Now()
	if opt.StartTime.IsZero() {
		opt.StartTime = start
	}

	if opt.PrintCommands {
		status := newTestStatus(opt.Out, true, len(tests), time.Minute, &monitor.Monitor{}, opt.AsEnv())
		newParallelTestQueue(tests).Execute(context.Background(), 1, status.OutputCommand)
		return nil
	}
	if opt.DryRun {
		for _, test := range sortedTests(tests) {
			fmt.Fprintf(opt.Out, "%q\n", test.name)
		}
		return nil
	}

	if len(opt.JUnitDir) > 0 {
		if _, err := os.Stat(opt.JUnitDir); err != nil {
			if !os.IsNotExist(err) {
				return fmt.Errorf("could not access --junit-dir: %v", err)
			}
			if err := os.MkdirAll(opt.JUnitDir, 0755); err != nil {
				return fmt.Errorf("could not create --junit-dir: %v", err)
			}
		}
	}

	parallelism := opt.Parallelism
	if parallelism == 0 {
		parallelism = suite.Parallelism
	}
	if parallelism == 0 {
		parallelism = 10
	}
	timeout := opt.Timeout
	if timeout == 0 {
		timeout = suite.TestTimeout
	}
	if timeout == 0 {
		timeout = 15 * time.Minute
	}

	ctx, cancelFn := context.WithCancel(context.Background())
	defer cancelFn()
	abortCh := make(chan os.Signal)
	go func() {
		<-abortCh
		fmt.Fprintf(opt.ErrOut, "Interrupted, terminating tests\n")
		cancelFn()
		sig := <-abortCh
		fmt.Fprintf(opt.ErrOut, "Interrupted twice, exiting (%s)\n", sig)
		switch sig {
		case syscall.SIGINT:
			os.Exit(130)
		default:
			os.Exit(0)
		}
	}()
	signal.Notify(abortCh, syscall.SIGINT, syscall.SIGTERM)

	m, err := monitor.Start(ctx)
	if err != nil {
		return err
	}
	// if we run a single test, always include success output
	includeSuccess := opt.IncludeSuccessOutput
	if len(tests) == 1 {
		includeSuccess = true
	}
	status := newTestStatus(opt.Out, includeSuccess, len(tests), timeout, m, opt.AsEnv())

	early, normal := splitTests(tests, func(t *testCase) bool {
		return strings.Contains(t.name, "[Early]")
	})

	late, normal := splitTests(normal, func(t *testCase) bool {
		return strings.Contains(t.name, "[Late]")
	})

	// run the tests

	// run our Early tests first
	q := newParallelTestQueue(early)
	q.Execute(ctx, parallelism, status.Run)

	// run other tests next
	q = newParallelTestQueue(normal)
	q.Execute(ctx, parallelism, status.Run)

	// run Late test suits after everything else
	q = newParallelTestQueue(late)
	q.Execute(ctx, parallelism, status.Run)

	duration := time.Now().Sub(start).Round(time.Second / 10)
	if duration > time.Minute {
		duration = duration.Round(time.Second)
	}

	pass, fail, skip, failing := summarizeTests(tests)

	// monitor the cluster while the tests are running and report any detected anomalies
	var syntheticTestResults []*JUnitTestCase
	var syntheticFailure bool
	if events := m.Events(time.Time{}, time.Time{}); len(events) > 0 {
		eventsForTests := createEventsForTests(tests)

		var buf *bytes.Buffer
		syntheticTestResults, buf, _ = createSyntheticTestsFromMonitor(m, eventsForTests, duration)
		testCases, passed := syntheticEventTests.JUnitsForEvents(events, duration)
		syntheticTestResults = append(syntheticTestResults, testCases...)
		if !passed {
			syntheticFailure = true
		}

		if len(syntheticTestResults) > 0 {
			// mark any failures by name
			failing, flaky := sets.NewString(), sets.NewString()
			for _, test := range syntheticTestResults {
				if test.FailureOutput != nil {
					failing.Insert(test.Name)
				}
			}
			// if a test has both a pass and a failure, flag it
			// as a flake
			for _, test := range syntheticTestResults {
				if test.FailureOutput == nil {
					if failing.Has(test.Name) {
						flaky.Insert(test.Name)
					}
				}
			}
			failing = failing.Difference(flaky)
			if failing.Len() > 0 {
				fmt.Fprintf(buf, "Failing invariants:\n\n%s\n\n", strings.Join(failing.List(), "\n"))
			}
			if flaky.Len() > 0 {
				fmt.Fprintf(buf, "Flaky invariants:\n\n%s\n\n", strings.Join(flaky.List(), "\n"))
			}
		}

		opt.Out.Write(buf.Bytes())
	}

	// attempt to retry failures to do flake detection
	if fail > 0 && fail <= suite.MaximumAllowedFlakes {
		var retries []*testCase
		for _, test := range failing {
			retry := test.Retry()
			retries = append(retries, retry)
			tests = append(tests, retry)
			if len(retries) > suite.MaximumAllowedFlakes {
				break
			}
		}

		q := newParallelTestQueue(retries)
		status := newTestStatus(ioutil.Discard, opt.IncludeSuccessOutput, len(retries), timeout, m, opt.AsEnv())
		q.Execute(ctx, parallelism, status.Run)
		var flaky []string
		var repeatFailures []*testCase
		for _, test := range retries {
			if test.success {
				flaky = append(flaky, test.name)
			} else {
				repeatFailures = append(repeatFailures, test)
			}
		}
		if len(flaky) > 0 {
			failing = repeatFailures
			sort.Strings(flaky)
			fmt.Fprintf(opt.Out, "Flaky tests:\n\n%s\n\n", strings.Join(flaky, "\n"))
		}
	}

	// report the outcome of the test
	if len(failing) > 0 {
		names := sets.NewString(testNames(failing)...).List()
		fmt.Fprintf(opt.Out, "Failing tests:\n\n%s\n\n", strings.Join(names, "\n"))
	}

	if len(opt.JUnitDir) > 0 {
		if err := writeJUnitReport("junit_e2e", "openshift-tests", tests, opt.JUnitDir, duration, opt.ErrOut, syntheticTestResults...); err != nil {
			fmt.Fprintf(opt.Out, "error: Unable to write e2e JUnit results: %v", err)
		}
	}

	if fail > 0 {
		if len(failing) > 0 || suite.MaximumAllowedFlakes == 0 {
			return fmt.Errorf("%d fail, %d pass, %d skip (%s)", fail, pass, skip, duration)
		}
		fmt.Fprintf(opt.Out, "%d flakes detected, suite allows passing with only flakes\n\n", fail)
	}

	if syntheticFailure {
		return fmt.Errorf("failed because an invariant was violated, %d pass, %d skip (%s)\n", pass, skip, duration)
	}

	fmt.Fprintf(opt.Out, "%d pass, %d skip (%s)\n", pass, skip, duration)
	return ctx.Err()
}
<|endoftext|>


<commit_before>/*
Package key contains a single Key interface. The key package was
created to prevent cicular depedencies betwee "github.com/lleo/go-hamt" and
either "github.com/lleo/go-hamt/hamt32" or "github.com/lleo/go-hamt/hamt64".

However the key pacakge is also used by the functional HAMT variation in
"github.com/lleo/go-hamt-functional".
*/
package key

import (
	"fmt"
	"hash/fnv"
	"strings"
)

type Key interface {
	String() string // ISA/statisfies fmt.Stringer interface
	Equals(Key) bool
	Hash30() uint32
	Hash60() uint64
}

// Base struct of all structs that satisfy Key interface
type KeyBase struct {
	hash30 uint32
	hash60 uint64
}

// INTERESTING IDEA JUST NOT YET...
//
// type Hash30 uint32
//
// func (h30 Hash30) String() {
// 	return hash30String(uint32(h30))
// }
//
// func (h30 Hash30) index(depth uint) {
// 	return index(uint32(h30), depth)
// }
//
// type Hash60 uint64
//
// func (h Hash60) String() {
// 	return hash60String(uint64(h))
// }
//
// func (h60 Hash60) index(depth uint) {
// 	return index(uint64(h60), depth)
// }

//
// 30bit hash values & functions
//

const NBITS30 uint = 5

func indexMask30(depth uint) uint32 {
	return uint32(uint8(1<<NBITS30)-1) << (depth * NBITS30)
}

func index30(h30 uint32, depth uint) uint {
	var idxMask = indexMask30(depth)
	var idx = uint((h30 & idxMask) >> (depth * NBITS30))
	return idx
}

func hashPathString30(hashPath uint32, depth uint) string {
	if depth == 0 {
		return "/"
	}
	var strs = make([]string, depth+1)

	for d := uint(0); d <= depth; d++ {
		var idx = index30(hashPath, d)
		strs[d] = fmt.Sprintf("%02d", idx)
	}

	return "/" + strings.Join(strs, "/")
}

func hash30String(h30 uint32) string {
	return hashPathString30(h30, 5)
}

//
// 60bit hash values & functions
//

const NBITS60 uint = 6

func indexMask60(depth uint) uint64 {
	return uint64(uint64(1<<NBITS60)-1) << (depth * NBITS60)
}

func index60(h60 uint64, depth uint) uint {
	var idxMask = indexMask60(depth)
	var idx = uint((h60 & idxMask) >> (depth * NBITS60))
	return idx
}

func hashPathString60(hashPath uint64, depth uint) string {
	if depth == 0 {
		return "/"
	}
	var strs = make([]string, depth+1)

	for d := uint(0); d <= depth; d++ {
		var idx = index60(hashPath, d)
		strs[d] = fmt.Sprintf("%02d", idx)
	}

	return "/" + strings.Join(strs, "/")
}

func hash60String(h60 uint64) string {
	return hashPathString60(h60, 9)
}

const mask30 = uint32(1<<30) - 1

// Fold the top two bits into the bottom 30 to get 30bits of hash
// from: http://www.isthe.com/chongo/tech/comp/fnv/index.html#xor-fold
func fold30(h32 uint32) uint32 {
	return h32>>30 ^ h32&mask30
}

func hash32(bs []byte) uint32 {
	var h = fnv.New32()
	h.Write(bs)
	return h.Sum32()
}

const mask60 = uint64(1<<60) - 1

// Fold the top four bits into the bottom 60 to get 60bits of hash
// from: http://www.isthe.com/chongo/tech/comp/fnv/index.html#xor-fold
func fold60(h64 uint64) uint64 {
	return h64>>60 ^ h64&mask60
}

func hash64(bs []byte) uint64 {
	var h = fnv.New64()
	h.Write(bs)
	return h.Sum64()
}

func (kb *KeyBase) String() string {
	return fmt.Sprintf("KeyBase{hash30:(%d)%s, hash60:(%d)%s}",
		kb.hash30, hash30String(kb.hash30),
		kb.hash60, hash60String(kb.hash60))
}

// Initialize the KeyBase part of any struct that has the KeyBase struct embeded.
// will be called by the New() function of any decendent class.
//     k.Initialize(bs)
// where k is interpreted as a *KeyBase and bs is a unique []byte to calculate
// the 30bit hash from.
func (kb *KeyBase) Initialize(bs []byte) {
	kb.hash30 = fold30(hash32(bs))
	kb.hash60 = fold60(hash64(bs))
}

func (kb *KeyBase) Hash30() uint32 {
	return kb.hash30
}

func (kb *KeyBase) Hash60() uint64 {
	return kb.hash60
}
<commit_msg>updated docs to conform to golint.<commit_after>/*
Package key contains a single Key interface. The key package was
created to prevent cicular dependencies between "github.com/lleo/go-hamt" and
either "github.com/lleo/go-hamt/hamt32" or "github.com/lleo/go-hamt/hamt64".

Additionally, the github.com/lleo/go-hamt/key provides a Base structure.
The Base structure if added to a derivative key type will provide the
Hamt30() and Hamt60() methods. Base needs to be populated by the derivative
key constructor calling the Initialize([]byte) method.

Any key created useing the Key interface must be read only after construction.

The key package is used by the functional HAMT variation in
"github.com/lleo/go-hamt-functional".
*/
package key

import (
	"fmt"
	"hash/fnv"
	"strings"
)

// Key interface descibes the methods any struct needs to implement to be used
// in either the github.com/lleo/go-hamt or github.com/lleo/go-hamt-functional
// packages.
type Key interface {
	String() string // ISA/statisfies fmt.Stringer interface
	Equals(Key) bool
	Hash30() uint32
	Hash60() uint64
}

// Base struct is intended to be the base struct of all structs that satisfy
// Key interface. It caches the calculated hash30 and hash60 generated when
// the method Initialize([]byte) is called.
type Base struct {
	hash30 uint32
	hash60 uint64
}

// INTERESTING IDEA JUST NOT YET...
//
// type Hash30 uint32
//
// func (h30 Hash30) String() {
// 	return hash30String(uint32(h30))
// }
//
// func (h30 Hash30) index(depth uint) {
// 	return index(uint32(h30), depth)
// }
//
// type Hash60 uint64
//
// func (h Hash60) String() {
// 	return hash60String(uint64(h))
// }
//
// func (h60 Hash60) index(depth uint) {
// 	return index(uint64(h60), depth)
// }

//
// 30bit hash value and string generator functions
// Hash values are generated when Initialize([]byte) is called.
// Hash strings are generated as needed.
//

const nBits30 uint = 5

func indexMask30(depth uint) uint32 {
	return uint32(uint8(1<<nBits30)-1) << (depth * nBits30)
}

func index30(h30 uint32, depth uint) uint {
	var idxMask = indexMask30(depth)
	var idx = uint((h30 & idxMask) >> (depth * nBits30))
	return idx
}

func hashPathString30(hashPath uint32, depth uint) string {
	if depth == 0 {
		return "/"
	}
	var strs = make([]string, depth+1)

	for d := uint(0); d <= depth; d++ {
		var idx = index30(hashPath, d)
		strs[d] = fmt.Sprintf("%02d", idx)
	}

	return "/" + strings.Join(strs, "/")
}

func hash30String(h30 uint32) string {
	return hashPathString30(h30, 5)
}

//
// 60bit hash value and string generator functions.
// Hash values are generated when Initialize([]byte) is called.
// Hash strings are generated as needed.
//

const nBits60 uint = 6

func indexMask60(depth uint) uint64 {
	return uint64(uint64(1<<nBits60)-1) << (depth * nBits60)
}

func index60(h60 uint64, depth uint) uint {
	var idxMask = indexMask60(depth)
	var idx = uint((h60 & idxMask) >> (depth * nBits60))
	return idx
}

func hashPathString60(hashPath uint64, depth uint) string {
	if depth == 0 {
		return "/"
	}
	var strs = make([]string, depth+1)

	for d := uint(0); d <= depth; d++ {
		var idx = index60(hashPath, d)
		strs[d] = fmt.Sprintf("%02d", idx)
	}

	return "/" + strings.Join(strs, "/")
}

func hash60String(h60 uint64) string {
	return hashPathString60(h60, 9)
}

const mask30 = uint32(1<<30) - 1

// fold30 folds the top 2 bits into the bottom 30 to get 30bits of hash
// from: http://www.isthe.com/chongo/tech/comp/fnv/index.html#xor-fold
func fold30(h32 uint32) uint32 {
	return h32>>30 ^ h32&mask30
}

// hash64 calculates the 64bit hash value of a byte string using hash/fnv.
func hash32(bs []byte) uint32 {
	var h = fnv.New32()
	h.Write(bs)
	return h.Sum32()
}

const mask60 = uint64(1<<60) - 1

// fold60 folds the top 4 bits into the bottom 60 to get 60bits of hash
// from: http://www.isthe.com/chongo/tech/comp/fnv/index.html#xor-fold
func fold60(h64 uint64) uint64 {
	return h64>>60 ^ h64&mask60
}

// hash64 calculates the 64bit hash value of a byte string using hash/fnv.
func hash64(bs []byte) uint64 {
	var h = fnv.New64()
	h.Write(bs)
	return h.Sum64()
}

// String provides a human readable form of the Base's hash30 and hash60
// values. Both in raw integer and slash '/' separated integers of the 5 and
// 6 bit path values; for hash30 and hash60 respectively.
func (kb *Base) String() string {
	return fmt.Sprintf("Base{(%d)%s, (%d)%s}",
		kb.hash30, hash30String(kb.hash30),
		kb.hash60, hash60String(kb.hash60))
}

// Initialize the Base part of any struct that has the Base struct embeded.
// will be called by the New() function of any decendent class.
//     k.Initialize(bs)
// where k is interpreted as a *Base and bs is a unique []byte to calculate
// the 30bit hash from.
func (kb *Base) Initialize(bs []byte) {
	kb.hash30 = fold30(hash32(bs))
	kb.hash60 = fold60(hash64(bs))
}

// Hash30 returns the uint32 that contains the 30bit hash value.
func (kb *Base) Hash30() uint32 {
	return kb.hash30
}

// Hash60 returns the uint32 that contains the 60bit hash value.
func (kb *Base) Hash60() uint64 {
	return kb.hash60
}
<|endoftext|>


<commit_before>// Copyright 2014 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License.
//
// Author: Spencer Kimball (spencer.kimball@gmail.com)

package kv

import (
	"fmt"
	"io"
	"math/rand"
	"time"

	"golang.org/x/net/context"
	"google.golang.org/grpc"

	"github.com/cockroachdb/cockroach/roachpb"
	"github.com/cockroachdb/cockroach/rpc"
	"github.com/cockroachdb/cockroach/util"
	"github.com/cockroachdb/cockroach/util/envutil"
	"github.com/cockroachdb/cockroach/util/log"
	"github.com/cockroachdb/cockroach/util/retry"
)

// orderingPolicy is an enum for ordering strategies when there
// are multiple endpoints available.
type orderingPolicy int

const (
	// orderStable uses endpoints in the order provided.
	orderStable orderingPolicy = iota
	// orderRandom randomly orders available endpoints.
	orderRandom
)

// A SendOptions structure describes the algorithm for sending RPCs to one or
// more replicas, depending on error conditions and how many successful
// responses are required.
type SendOptions struct {
	context.Context // must not be nil
	// Ordering indicates how the available endpoints are ordered when
	// deciding which to send to (if there are more than one).
	Ordering orderingPolicy
	// SendNextTimeout is the duration after which RPCs are sent to
	// other replicas in a set.
	SendNextTimeout time.Duration
	// Timeout is the maximum duration of an RPC before failure.
	// 0 for no timeout.
	Timeout time.Duration
}

func (so SendOptions) contextWithTimeout() (context.Context, func()) {
	if so.Timeout != 0 {
		return context.WithTimeout(so.Context, so.Timeout)
	}
	return so.Context, func() {}
}

// An rpcError indicates a failure to send the RPC. rpcErrors are
// retryable.
type rpcError struct {
	error
}

func newRPCError(err error) rpcError {
	return rpcError{err}
}

// CanRetry implements the Retryable interface.
// TODO(tschottdorf): the way this is used by rpc/send suggests that it
// may be better if these weren't retriable - they are returned when the
// connection fails, i.e. for example when a node is down or the network
// fails. Retrying on such errors keeps the caller waiting for a long time
// and without a positive outlook.
func (r rpcError) CanRetry() bool { return true }

type batchClient struct {
	remoteAddr string
	conn       *grpc.ClientConn
	client     roachpb.InternalClient
	args       roachpb.BatchRequest
}

func shuffleClients(clients []batchClient) {
	for i, n := 0, len(clients); i < n-1; i++ {
		j := rand.Intn(n-i) + i
		clients[i], clients[j] = clients[j], clients[i]
	}
}

type batchCall struct {
	reply *roachpb.BatchResponse
	err   error
}

// Send sends one or more RPCs to clients specified by the slice of
// replicas. On success, Send returns the first successful reply. Otherwise,
// Send returns an error if and as soon as the number of failed RPCs exceeds
// the available endpoints less the number of required replies.
func send(opts SendOptions, replicas ReplicaSlice,
	args roachpb.BatchRequest, rpcContext *rpc.Context) (*roachpb.BatchResponse, error) {

	if len(replicas) < 1 {
		return nil, roachpb.NewSendError(
			fmt.Sprintf("insufficient replicas (%d) to satisfy send request of %d",
				len(replicas), 1), false)
	}

	done := make(chan batchCall, len(replicas))

	clients := make([]batchClient, 0, len(replicas))
	for _, replica := range replicas {
		conn, err := rpcContext.GRPCDial(replica.NodeDesc.Address.String())
		if err != nil {
			return nil, err
		}
		argsCopy := args
		argsCopy.Replica = replica.ReplicaDescriptor
		clients = append(clients, batchClient{
			remoteAddr: replica.NodeDesc.Address.String(),
			conn:       conn,
			client:     roachpb.NewInternalClient(conn),
			args:       argsCopy,
		})
	}

	// Put known-unhealthy clients last.
	nHealthy, err := splitHealthy(clients)
	if err != nil {
		return nil, err
	}

	var orderedClients []batchClient
	switch opts.Ordering {
	case orderStable:
		orderedClients = clients
	case orderRandom:
		// Randomly permute order, but keep known-unhealthy clients last.
		shuffleClients(clients[:nHealthy])
		shuffleClients(clients[nHealthy:])

		orderedClients = clients
	}
	// TODO(spencer): going to need to also sort by affinity; closest
	// ping time should win. Makes sense to have the rpc client/server
	// heartbeat measure ping times. With a bit of seasoning, each
	// node will be able to order the healthy replicas based on latency.

	// Send the first request.
	sendOneFn(opts, rpcContext, orderedClients[0], done)
	orderedClients = orderedClients[1:]

	var errors, retryableErrors int

	// Wait for completions.
	var sendNextTimer util.Timer
	defer sendNextTimer.Stop()
	for {
		sendNextTimer.Reset(opts.SendNextTimeout)
		select {
		case <-sendNextTimer.C:
			sendNextTimer.Read = true
			// On successive RPC timeouts, send to additional replicas if available.
			if len(orderedClients) > 0 {
				log.Trace(opts.Context, "timeout, trying next peer")
				sendOneFn(opts, rpcContext, orderedClients[0], done)
				orderedClients = orderedClients[1:]
			}

		case call := <-done:
			err := call.err
			if err == nil {
				if log.V(2) {
					log.Infof("successful reply: %+v", call.reply)
				}

				return call.reply, nil
			}

			// Error handling.
			if log.V(1) {
				log.Warningf("error reply: %s", err)
			}

			errors++

			// Since we have a reconnecting client here, disconnect errors are retryable.
			disconnected := err == io.ErrUnexpectedEOF
			if retryErr, ok := err.(retry.Retryable); disconnected || (ok && retryErr.CanRetry()) {
				retryableErrors++
			}

			if remainingNonErrorRPCs := len(replicas) - errors; remainingNonErrorRPCs < 1 {
				return nil, roachpb.NewSendError(
					fmt.Sprintf("too many errors encountered (%d of %d total): %v",
						errors, len(clients), err), remainingNonErrorRPCs+retryableErrors >= 1)
			}
			// Send to additional replicas if available.
			if len(orderedClients) > 0 {
				log.Trace(opts.Context, "error, trying next peer")
				sendOneFn(opts, rpcContext, orderedClients[0], done)
				orderedClients = orderedClients[1:]
			}
		}
	}
}

// splitHealthy splits the provided client slice into healthy clients and
// unhealthy clients, based on their connection state. Healthy clients will
// be rearranged first in the slice, and unhealthy clients will be rearranged
// last. Within these two groups, the rearrangement will be stable. The function
// will then return the number of healthy clients.
func splitHealthy(clients []batchClient) (int, error) {
	var nHealthy int
	for i, client := range clients {
		clientState, err := client.conn.State()
		if err != nil {
			// This should not currently happen with the default grpc.Picker.
			return 0, err
		}
		if clientState == grpc.Ready {
			clients[i], clients[nHealthy] = clients[nHealthy], clients[i]
			nHealthy++
		}
	}
	return nHealthy, nil
}

// Allow local calls to be dispatched directly to the local server without
// sending an RPC.
var enableLocalCalls = envutil.EnvOrDefaultBool("enable_local_calls", true)

// sendOneFn is overwritten in tests to mock sendOne.
var sendOneFn = sendOne

// sendOne invokes the specified RPC on the supplied client when the
// client is ready. On success, the reply is sent on the channel;
// otherwise an error is sent.
//
// Do not call directly, but instead use sendOneFn. Tests mock out this method
// via sendOneFn in order to test various error cases.
func sendOne(opts SendOptions, rpcContext *rpc.Context, client batchClient, done chan batchCall) {
	addr := client.remoteAddr
	if log.V(2) {
		log.Infof("sending request to %s: %+v", addr, client.args)
	}

	if localServer := rpcContext.GetLocalInternalServerForAddr(addr); enableLocalCalls && localServer != nil {
		ctx, cancel := opts.contextWithTimeout()
		defer cancel()

		reply, err := localServer.Batch(ctx, &client.args)
		done <- batchCall{reply: reply, err: err}
		return
	}

	go func() {
		ctx, cancel := opts.contextWithTimeout()
		defer cancel()

		c := client.conn
		for state, err := c.State(); state != grpc.Ready; state, err = c.WaitForStateChange(ctx, state) {
			if err != nil {
				done <- batchCall{err: newRPCError(
					util.Errorf("rpc to %s failed: %s", addr, err))}
				return
			}
			if state == grpc.Shutdown {
				done <- batchCall{err: newRPCError(
					util.Errorf("rpc to %s failed as client connection was closed", addr))}
				return
			}
		}

		reply, err := client.client.Batch(ctx, &client.args)
		done <- batchCall{reply: reply, err: err}
	}()
}
<commit_msg>kv: Replace rpcError with roachpb.SendError<commit_after>// Copyright 2014 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License.
//
// Author: Spencer Kimball (spencer.kimball@gmail.com)

package kv

import (
	"fmt"
	"io"
	"math/rand"
	"time"

	"golang.org/x/net/context"
	"google.golang.org/grpc"

	"github.com/cockroachdb/cockroach/roachpb"
	"github.com/cockroachdb/cockroach/rpc"
	"github.com/cockroachdb/cockroach/util"
	"github.com/cockroachdb/cockroach/util/envutil"
	"github.com/cockroachdb/cockroach/util/log"
	"github.com/cockroachdb/cockroach/util/retry"
)

// orderingPolicy is an enum for ordering strategies when there
// are multiple endpoints available.
type orderingPolicy int

const (
	// orderStable uses endpoints in the order provided.
	orderStable orderingPolicy = iota
	// orderRandom randomly orders available endpoints.
	orderRandom
)

// A SendOptions structure describes the algorithm for sending RPCs to one or
// more replicas, depending on error conditions and how many successful
// responses are required.
type SendOptions struct {
	context.Context // must not be nil
	// Ordering indicates how the available endpoints are ordered when
	// deciding which to send to (if there are more than one).
	Ordering orderingPolicy
	// SendNextTimeout is the duration after which RPCs are sent to
	// other replicas in a set.
	SendNextTimeout time.Duration
	// Timeout is the maximum duration of an RPC before failure.
	// 0 for no timeout.
	Timeout time.Duration
}

func (so SendOptions) contextWithTimeout() (context.Context, func()) {
	if so.Timeout != 0 {
		return context.WithTimeout(so.Context, so.Timeout)
	}
	return so.Context, func() {}
}

type batchClient struct {
	remoteAddr string
	conn       *grpc.ClientConn
	client     roachpb.InternalClient
	args       roachpb.BatchRequest
}

func shuffleClients(clients []batchClient) {
	for i, n := 0, len(clients); i < n-1; i++ {
		j := rand.Intn(n-i) + i
		clients[i], clients[j] = clients[j], clients[i]
	}
}

type batchCall struct {
	reply *roachpb.BatchResponse
	err   error
}

// Send sends one or more RPCs to clients specified by the slice of
// replicas. On success, Send returns the first successful reply. Otherwise,
// Send returns an error if and as soon as the number of failed RPCs exceeds
// the available endpoints less the number of required replies.
func send(opts SendOptions, replicas ReplicaSlice,
	args roachpb.BatchRequest, rpcContext *rpc.Context) (*roachpb.BatchResponse, error) {

	if len(replicas) < 1 {
		return nil, roachpb.NewSendError(
			fmt.Sprintf("insufficient replicas (%d) to satisfy send request of %d",
				len(replicas), 1), false)
	}

	done := make(chan batchCall, len(replicas))

	clients := make([]batchClient, 0, len(replicas))
	for _, replica := range replicas {
		conn, err := rpcContext.GRPCDial(replica.NodeDesc.Address.String())
		if err != nil {
			return nil, err
		}
		argsCopy := args
		argsCopy.Replica = replica.ReplicaDescriptor
		clients = append(clients, batchClient{
			remoteAddr: replica.NodeDesc.Address.String(),
			conn:       conn,
			client:     roachpb.NewInternalClient(conn),
			args:       argsCopy,
		})
	}

	// Put known-unhealthy clients last.
	nHealthy, err := splitHealthy(clients)
	if err != nil {
		return nil, err
	}

	var orderedClients []batchClient
	switch opts.Ordering {
	case orderStable:
		orderedClients = clients
	case orderRandom:
		// Randomly permute order, but keep known-unhealthy clients last.
		shuffleClients(clients[:nHealthy])
		shuffleClients(clients[nHealthy:])

		orderedClients = clients
	}
	// TODO(spencer): going to need to also sort by affinity; closest
	// ping time should win. Makes sense to have the rpc client/server
	// heartbeat measure ping times. With a bit of seasoning, each
	// node will be able to order the healthy replicas based on latency.

	// Send the first request.
	sendOneFn(opts, rpcContext, orderedClients[0], done)
	orderedClients = orderedClients[1:]

	var errors, retryableErrors int

	// Wait for completions.
	var sendNextTimer util.Timer
	defer sendNextTimer.Stop()
	for {
		sendNextTimer.Reset(opts.SendNextTimeout)
		select {
		case <-sendNextTimer.C:
			sendNextTimer.Read = true
			// On successive RPC timeouts, send to additional replicas if available.
			if len(orderedClients) > 0 {
				log.Trace(opts.Context, "timeout, trying next peer")
				sendOneFn(opts, rpcContext, orderedClients[0], done)
				orderedClients = orderedClients[1:]
			}

		case call := <-done:
			err := call.err
			if err == nil {
				if log.V(2) {
					log.Infof("successful reply: %+v", call.reply)
				}

				return call.reply, nil
			}

			// Error handling.
			if log.V(1) {
				log.Warningf("error reply: %s", err)
			}

			errors++

			// Since we have a reconnecting client here, disconnect errors are retryable.
			disconnected := err == io.ErrUnexpectedEOF
			if retryErr, ok := err.(retry.Retryable); disconnected || (ok && retryErr.CanRetry()) {
				retryableErrors++
			}

			if remainingNonErrorRPCs := len(replicas) - errors; remainingNonErrorRPCs < 1 {
				return nil, roachpb.NewSendError(
					fmt.Sprintf("too many errors encountered (%d of %d total): %v",
						errors, len(clients), err), remainingNonErrorRPCs+retryableErrors >= 1)
			}
			// Send to additional replicas if available.
			if len(orderedClients) > 0 {
				log.Trace(opts.Context, "error, trying next peer")
				sendOneFn(opts, rpcContext, orderedClients[0], done)
				orderedClients = orderedClients[1:]
			}
		}
	}
}

// splitHealthy splits the provided client slice into healthy clients and
// unhealthy clients, based on their connection state. Healthy clients will
// be rearranged first in the slice, and unhealthy clients will be rearranged
// last. Within these two groups, the rearrangement will be stable. The function
// will then return the number of healthy clients.
func splitHealthy(clients []batchClient) (int, error) {
	var nHealthy int
	for i, client := range clients {
		clientState, err := client.conn.State()
		if err != nil {
			// This should not currently happen with the default grpc.Picker.
			return 0, err
		}
		if clientState == grpc.Ready {
			clients[i], clients[nHealthy] = clients[nHealthy], clients[i]
			nHealthy++
		}
	}
	return nHealthy, nil
}

// Allow local calls to be dispatched directly to the local server without
// sending an RPC.
var enableLocalCalls = envutil.EnvOrDefaultBool("enable_local_calls", true)

// sendOneFn is overwritten in tests to mock sendOne.
var sendOneFn = sendOne

// sendOne invokes the specified RPC on the supplied client when the
// client is ready. On success, the reply is sent on the channel;
// otherwise an error is sent.
//
// Do not call directly, but instead use sendOneFn. Tests mock out this method
// via sendOneFn in order to test various error cases.
func sendOne(opts SendOptions, rpcContext *rpc.Context, client batchClient, done chan batchCall) {
	addr := client.remoteAddr
	if log.V(2) {
		log.Infof("sending request to %s: %+v", addr, client.args)
	}

	if localServer := rpcContext.GetLocalInternalServerForAddr(addr); enableLocalCalls && localServer != nil {
		ctx, cancel := opts.contextWithTimeout()
		defer cancel()

		reply, err := localServer.Batch(ctx, &client.args)
		done <- batchCall{reply: reply, err: err}
		return
	}

	go func() {
		ctx, cancel := opts.contextWithTimeout()
		defer cancel()

		c := client.conn
		for state, err := c.State(); state != grpc.Ready; state, err = c.WaitForStateChange(ctx, state) {
			if err != nil {
				done <- batchCall{err: roachpb.NewSendError(
					fmt.Sprintf("rpc to %s failed: %s", addr, err), true)}
				return
			}
			if state == grpc.Shutdown {
				done <- batchCall{err: roachpb.NewSendError(
					fmt.Sprintf("rpc to %s failed as client connection was closed", addr), true)}
				return
			}
		}

		reply, err := client.client.Batch(ctx, &client.args)
		done <- batchCall{reply: reply, err: err}
	}()
}
<|endoftext|>


<commit_before>package forestdb

//  Copyright (c) 2014 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
//  except in compliance with the License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing, software distributed under the
//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
//  either express or implied. See the License for the specific language governing permissions
//  and limitations under the License.

//#include <stdlib.h>
//#include <string.h>
//#include <libforestdb/forestdb.h>
import "C"
import "unsafe"

type batchOp struct {
	del  bool
	k    unsafe.Pointer
	klen C.size_t
	v    unsafe.Pointer
	vlen C.size_t
}

type KVBatch struct {
	ops []*batchOp
}

func NewKVBatch() *KVBatch {
	return &KVBatch{
		ops: make([]*batchOp, 0, 100),
	}
}

func (b *KVBatch) Set(k, v []byte) {
	klen := C.size_t(len(k))
	kc := C.malloc(klen)
	C.memmove(kc, unsafe.Pointer(&k[0]), klen)
	vlen := C.size_t(len(v))
	var vc unsafe.Pointer
	if vlen > 0 {
		vc = C.malloc(vlen)
		C.memmove(vc, unsafe.Pointer(&v[0]), vlen)
	}
	b.ops = append(b.ops, &batchOp{
		del:  false,
		k:    unsafe.Pointer(kc),
		klen: klen,
		v:    unsafe.Pointer(vc),
		vlen: vlen,
	})
}

func (b *KVBatch) Delete(k []byte) {
	klen := C.size_t(len(k))
	kc := C.malloc(klen)
	C.memmove(kc, unsafe.Pointer(&k[0]), klen)
	b.ops = append(b.ops, &batchOp{
		del:  true,
		k:    unsafe.Pointer(kc),
		klen: klen,
	})
}

func (b *KVBatch) Reset() {
	for _, op := range b.ops {
		if op.klen > 0 {
			C.free(op.k)
		}
		if op.vlen > 0 {
			C.free(op.v)
		}
	}
	b.ops = b.ops[:0]
}

func (k *KVStore) ExecuteBatch(b *KVBatch, opt CommitOpt) (err error) {

	err = k.File().BeginTransaction(ISOLATION_READ_COMMITTED)
	if err != nil {
		return
	}
	// defer function to ensure that once started,
	// we either commit transaction or abort it
	defer func() {
		// if nothing went wrong, commit
		if err == nil {
			// careful to catch error here too
			err = k.File().EndTransaction(opt)
		} else {
			// caller should see error that caused abort,
			// not success or failure of abort itself
			_ = k.File().AbortTransaction()
		}
	}()

	for _, op := range b.ops {
		if op.del {
			Log.Tracef("fdb_del_kv call k:%p db:%p kk:%v", k, k.db, op.k)
			errNo := C.fdb_del_kv(k.db, op.k, op.klen)
			Log.Tracef("fdb_del_kv retn k:%p errNo:%v", k, errNo)
			if errNo != RESULT_SUCCESS {
				err = Error(errNo)
				return
			}
		} else {
			Log.Tracef("fdb_set_kv call k:%p db:%p kk:%v v:%v", k, k.db, op.k, op.v)
			errNo := C.fdb_set_kv(k.db, op.k, op.klen, op.v, op.vlen)
			Log.Tracef("fdb_set_kv retn k:%p errNo:%v", k, errNo)
			if errNo != RESULT_SUCCESS {
				err = Error(errNo)
				return
			}
		}
	}
	return
}
<commit_msg>change implementation of kvbatch copying []byte into C<commit_after>package forestdb

//  Copyright (c) 2014 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
//  except in compliance with the License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing, software distributed under the
//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
//  either express or implied. See the License for the specific language governing permissions
//  and limitations under the License.

//#include <stdlib.h>
//#include <libforestdb/forestdb.h>
import "C"
import (
	"reflect"
	"unsafe"
)

type batchOp struct {
	del  bool
	k    unsafe.Pointer
	klen C.size_t
	v    unsafe.Pointer
	vlen C.size_t
}

type KVBatch struct {
	ops []*batchOp
}

func NewKVBatch() *KVBatch {
	return &KVBatch{
		ops: make([]*batchOp, 0, 100),
	}
}

func copySliceToC(in []byte) (C.size_t, unsafe.Pointer) {
	outl := C.size_t(len(in))
	outv := C.malloc(outl)
	var tmpSlice []byte
	shdr := (*reflect.SliceHeader)(unsafe.Pointer(&tmpSlice))
	shdr.Data = uintptr(outv)
	shdr.Len = int(outl)
	shdr.Cap = shdr.Len
	copy(tmpSlice, in)
	return outl, outv
}

func (b *KVBatch) Set(k, v []byte) {
	bo := batchOp{del: false}
	bo.klen, bo.k = copySliceToC(k)
	if len(v) > 0 {
		bo.vlen, bo.v = copySliceToC(v)
	}
	b.ops = append(b.ops, &bo)
}

func (b *KVBatch) Delete(k []byte) {
	bo := batchOp{del: true}
	bo.klen, bo.k = copySliceToC(k)
	b.ops = append(b.ops, &bo)
}

func (b *KVBatch) Reset() {
	for _, op := range b.ops {
		if op.klen > 0 {
			C.free(op.k)
		}
		if op.vlen > 0 {
			C.free(op.v)
		}
	}
	b.ops = b.ops[:0]
}

func (k *KVStore) ExecuteBatch(b *KVBatch, opt CommitOpt) (err error) {

	err = k.File().BeginTransaction(ISOLATION_READ_COMMITTED)
	if err != nil {
		return
	}
	// defer function to ensure that once started,
	// we either commit transaction or abort it
	defer func() {
		// if nothing went wrong, commit
		if err == nil {
			// careful to catch error here too
			err = k.File().EndTransaction(opt)
		} else {
			// caller should see error that caused abort,
			// not success or failure of abort itself
			_ = k.File().AbortTransaction()
		}
	}()

	for _, op := range b.ops {
		if op.del {
			Log.Tracef("fdb_del_kv call k:%p db:%p kk:%v", k, k.db, op.k)
			errNo := C.fdb_del_kv(k.db, op.k, op.klen)
			Log.Tracef("fdb_del_kv retn k:%p errNo:%v", k, errNo)
			if errNo != RESULT_SUCCESS {
				err = Error(errNo)
				return
			}
		} else {
			Log.Tracef("fdb_set_kv call k:%p db:%p kk:%v v:%v", k, k.db, op.k, op.v)
			errNo := C.fdb_set_kv(k.db, op.k, op.klen, op.v, op.vlen)
			Log.Tracef("fdb_set_kv retn k:%p errNo:%v", k, errNo)
			if errNo != RESULT_SUCCESS {
				err = Error(errNo)
				return
			}
		}
	}
	return
}
<|endoftext|>


<commit_before>package geo

import(
	"fmt"
	"math"
)

type Latlong struct {
	Lat  float64 `json:"lat"` // JSON format to match the Google Maps object definition
	Long float64 `json:"lng"`
}
func (ll Latlong)String() string { return fmt.Sprintf("(%.4f,%.4f)", ll.Lat, ll.Long) }

// We often treat latlong as a simple (x,y) space. We take Long as x, to make horiz/vert look normal
func (ll Latlong)x() float64 { return ll.Long }
func (ll Latlong)y() float64 { return ll.Lat }

// The square of the distance; useful for deciding closest approach
func (from Latlong)LatlongDistSq(to Latlong) float64 {
	x,y := (to.Long - from.Long), (to.Lat - from.Lat)
	return x*x + y*y
}

// ApproxDist treats the latlongs as y,x coords, and returns the 'latlong dist'
func (from Latlong)LatlongDist(to Latlong) float64 {
	return math.Sqrt(from.LatlongDistSq(to))
}

// Maybe it's safe to compare a nil-float64 directly to 0.0, and this is just a paranoid bodge.
func (ll Latlong)IsNil() bool {
	return math.Abs(ll.Lat)<0.01 && math.Abs(ll.Long)<0.01
}

// This probably isn't what you want
func (from Latlong)Equal(to Latlong) bool {
	return from.Lat == to.Lat && from.Long == to.Long
}

// Dist is the great-circle distance, in KM
func (from Latlong)Dist(to Latlong) float64 { return from.DistKM(to) }
func (from Latlong)DistKM(to Latlong) float64 {
	return haversine(from.Long,from.Lat,  to.Long,to.Lat)
}
func (from Latlong)DistNM(to Latlong) float64 {
	return from.DistKM(to) * KNauticalMilePerKM
}

func (from Latlong)BearingTowards(to Latlong) float64 {
	return forwardAzimuth(from.Long,from.Lat,  to.Long,to.Lat)
}

func (at Latlong)MapsUrl() string {
	return fmt.Sprintf("https://www.google.com/maps/@%.6f,%.6f,9z", at.Lat, at.Long)
}

// We have the two right angle sides of the triangle !
func (from Latlong)Dist3(to Latlong, altitude float64) float64 {
	horizDist := haversine(from.Long,from.Lat,  to.Long,to.Lat)
	vertDist  := altitude / 3280.84
	return math.Sqrt(horizDist*horizDist + vertDist*vertDist)
}

func (from Latlong)InterpolateTo(to Latlong, ratio float64) Latlong {
	interpFunc := func(from,to float64) float64 { return from + (to-from)*ratio }

	return Latlong{
		Lat: interpFunc(from.Lat, to.Lat),
		Long: interpFunc(from.Long, to.Long),
	}
}

// {{{ -------------------------={ E N D }=----------------------------------

// Local variables:
// folded-file: t
// end:

// }}}
<commit_msg>Git rid of the confusing JSON tag name compatibility thing for Google Maps<commit_after>package geo

import(
	"fmt"
	"math"
)

type Latlong struct {
	Lat  float64
	Long float64
}
func (ll Latlong)String() string { return fmt.Sprintf("(%.4f,%.4f)", ll.Lat, ll.Long) }

// We often treat latlong as a simple (x,y) space. We take Long as x, to make horiz/vert look normal
func (ll Latlong)x() float64 { return ll.Long }
func (ll Latlong)y() float64 { return ll.Lat }

// The square of the distance; useful for deciding closest approach
func (from Latlong)LatlongDistSq(to Latlong) float64 {
	x,y := (to.Long - from.Long), (to.Lat - from.Lat)
	return x*x + y*y
}

// ApproxDist treats the latlongs as y,x coords, and returns the 'latlong dist'
func (from Latlong)LatlongDist(to Latlong) float64 {
	return math.Sqrt(from.LatlongDistSq(to))
}

// Maybe it's safe to compare a nil-float64 directly to 0.0, and this is just a paranoid bodge.
func (ll Latlong)IsNil() bool {
	return math.Abs(ll.Lat)<0.01 && math.Abs(ll.Long)<0.01
}

// This probably isn't what you want
func (from Latlong)Equal(to Latlong) bool {
	return from.Lat == to.Lat && from.Long == to.Long
}

// Dist is the great-circle distance, in KM
func (from Latlong)Dist(to Latlong) float64 { return from.DistKM(to) }
func (from Latlong)DistKM(to Latlong) float64 {
	return haversine(from.Long,from.Lat,  to.Long,to.Lat)
}
func (from Latlong)DistNM(to Latlong) float64 {
	return from.DistKM(to) * KNauticalMilePerKM
}

func (from Latlong)BearingTowards(to Latlong) float64 {
	return forwardAzimuth(from.Long,from.Lat,  to.Long,to.Lat)
}

func (at Latlong)MapsUrl() string {
	return fmt.Sprintf("https://www.google.com/maps/@%.6f,%.6f,9z", at.Lat, at.Long)
}

// We have the two right angle sides of the triangle !
func (from Latlong)Dist3(to Latlong, altitude float64) float64 {
	horizDist := haversine(from.Long,from.Lat,  to.Long,to.Lat)
	vertDist  := altitude / 3280.84
	return math.Sqrt(horizDist*horizDist + vertDist*vertDist)
}

func (from Latlong)InterpolateTo(to Latlong, ratio float64) Latlong {
	interpFunc := func(from,to float64) float64 { return from + (to-from)*ratio }

	return Latlong{
		Lat: interpFunc(from.Lat, to.Lat),
		Long: interpFunc(from.Long, to.Long),
	}
}

// {{{ -------------------------={ E N D }=----------------------------------

// Local variables:
// folded-file: t
// end:

// }}}
<|endoftext|>


<commit_before>package lcd

import (
	"fmt"

	"github.com/scottyw/tetromino/mem"
)

const (
	bit7 = 1 << iota
	bit6 = 1 << iota
	bit5 = 1 << iota
	bit4 = 1 << iota
	bit3 = 1 << iota
	bit2 = 1 << iota
	bit1 = 1 << iota
	bit0 = 1 << iota
)

// LCD represents the LCD display of the Gameboy
type LCD struct {
	data [23040]uint8
}

// NewLCD returns the configured LCD
func NewLCD() *LCD {
	return &LCD{}
}

// Tick runs the LCD driver for one machine cycle i.e. 4 clock cycles
func (lcd *LCD) Tick(memory mem.Memory, cycle int) {
	ly := uint8(cycle / 114)
	lyRemainder := cycle % 114
	var stat uint8
	// Set mode on stat register
	switch {
	case ly == 144:
		// V-Blank period starts
		stat = 1
		*memory.IF |= 0x01
	case lyRemainder == 0:
		// OAM period starts
		stat = 2
	case lyRemainder == 20:
		// LCD data transfer period starts
		stat = 3
	case lyRemainder == 63:
		// H-Blank period starts
		stat = 0
		if ly < 144 {
			lcd.updateLcdLine(memory, ly)
		}
	default:
		stat = *memory.STAT
	}
	// Set coincidence flag and coincidence interrupt on stat register
	if ly == uint8(*memory.Read(0xff45)) {
		stat |= 0x44
	} else {
		stat &^= 0x44
	}
	// Set interrupts on stat register
	switch {
	case ly == 144:
		stat |= 0x10
	case lyRemainder == 0:
		stat |= 0x20
	case lyRemainder == 63:
		stat |= 0x08
	}
	*memory.STAT = stat
	*memory.LY = ly
}

// FrameData returns the frame data as a 160x144 array of bytes where each element is a colour value between 0 and 3
func (lcd *LCD) FrameData() [23040]uint8 {
	return lcd.data
}

func lowTileAbsoluteAddress(tileNumber uint8) uint16 {
	return 0x8000 + uint16(tileNumber)*16
}

func highTileAbsoluteAddress(tileNumber int8) uint16 {
	return uint16(0x9000 + int(tileNumber)*16)
}

// Returns 16 bytes representing one 8x8 tile
func tileData(mem mem.Memory, tile uint16) []byte {
	var tileAddr uint16
	if highBgTileMapDisplaySelect(mem) {
		tileAddr = 0x9c00 + tile
	} else {
		tileAddr = 0x9800 + tile
	}
	tileNumber := *mem.Read(tileAddr)
	if lowTileDataSelect(mem) {
		return mem.ReadRegion(lowTileAbsoluteAddress(tileNumber), 16)
	}
	return mem.ReadRegion(highTileAbsoluteAddress(int8(tileNumber)), 16)
}

func pixel(mem mem.Memory, memoryAddr uint16, bit uint8) uint8 {
	var a, b, pixel uint8
	a = uint8(*mem.Read(memoryAddr))
	b = uint8(*mem.Read(memoryAddr + 1))
	switch bit {
	case 0:
		pixel = (a&bit0)>>7 | (b&bit0)>>6
	case 1:
		pixel = (a&bit1)>>6 | (b&bit1)>>5
	case 2:
		pixel = (a&bit2)>>5 | (b&bit2)>>4
	case 3:
		pixel = (a&bit3)>>4 | (b&bit3)>>3
	case 4:
		pixel = (a&bit4)>>3 | (b&bit4)>>2
	case 5:
		pixel = (a&bit5)>>2 | (b&bit5)>>1
	case 6:
		pixel = (a&bit6)>>1 | (b & bit6)
	case 7:
		pixel = (a & bit7) | (b&bit7)<<1
	default:
		panic(fmt.Sprintf("Bad bit in pixel(): %v", bit))
	}
	return pixel
}

// Returns the memory address of the tile
func tileDataAddr(mem mem.Memory, tileX, tileY uint8, highTileMap, lowTileData bool) uint16 {
	var tileNumberAddr, tileIndex uint16
	tileIndex = uint16(tileY)*32 + uint16(tileX)
	if highTileMap {
		tileNumberAddr = 0x9c00 + tileIndex
	} else {
		tileNumberAddr = 0x9800 + tileIndex
	}
	tileNumber := *mem.Read(tileNumberAddr)
	if lowTileData {
		return lowTileAbsoluteAddress(tileNumber)
	}
	return highTileAbsoluteAddress(int8(tileNumber))
}

func deriveSpritePixel(mem mem.Memory, vramX, vramY uint8) (uint8, bool) {
	return 0, false
}

func deriveTilePixel(mem mem.Memory, vramX, vramY uint8, highTileMap, lowTileData bool) uint8 {
	var tileX, tileY, tileOffsetX, tileOffsetY uint8
	var tileAddr, memoryAddr uint16
	tileX = vramX / 8
	tileY = vramY / 8
	tileAddr = tileDataAddr(mem, tileX, tileY, highTileMap, lowTileData)
	tileOffsetX = vramX % 8
	tileOffsetY = vramY % 8
	memoryAddr = tileAddr + uint16(tileOffsetY)*2
	return pixel(mem, memoryAddr, tileOffsetX)
}

func deriveWindowPixel(mem mem.Memory, vramX, vramY uint8) (uint8, bool) {
	if windowDisplayEnable(mem) {
		highTileMap := highWindowTileMapDisplaySelect(mem)
		lowTileData := lowTileDataSelect(mem)
		return deriveTilePixel(mem, vramX, vramY, highTileMap, lowTileData), true
	}
	return 0, false
}

func deriveBackgroundPixel(mem mem.Memory, vramX, vramY uint8) uint8 {
	highTileMap := highBgTileMapDisplaySelect(mem)
	lowTileData := lowTileDataSelect(mem)
	return deriveTilePixel(mem, vramX, vramY, highTileMap, lowTileData)
}

func derivePixel(mem mem.Memory, vramX, vramY uint8) uint8 {
	if pixel, found := deriveSpritePixel(mem, vramX, vramY); found {
		return pixel
	}
	if pixel, found := deriveWindowPixel(mem, vramX, vramY); found {
		return pixel
	}
	return deriveBackgroundPixel(mem, vramX, vramY)
}

func (lcd *LCD) updateLcdLine(mem mem.Memory, lcdY uint8) {
	var lcdX, scrX, scrY, vramX, vramY uint8
	var index uint16
	for lcdX = 0; lcdX < 160; lcdX++ {
		index = uint16(lcdY)*160 + uint16(lcdX)
		scrX = 0            // FIXME scroll register
		scrY = 0            // FIXME scroll register
		vramX = lcdX + scrX // Overflows deliberately
		vramY = lcdY + scrY // Overflows deliberately
		lcd.data[index] = derivePixel(mem, vramX, vramY)
	}
}
<commit_msg>More LCDC support<commit_after>package lcd

import (
	"fmt"

	"github.com/scottyw/tetromino/mem"
)

const (
	bit7 = 1 << iota
	bit6 = 1 << iota
	bit5 = 1 << iota
	bit4 = 1 << iota
	bit3 = 1 << iota
	bit2 = 1 << iota
	bit1 = 1 << iota
	bit0 = 1 << iota
)

// LCD represents the LCD display of the Gameboy
type LCD struct {
	data [23040]uint8
}

// NewLCD returns the configured LCD
func NewLCD() *LCD {
	return &LCD{}
}

// Tick runs the LCD driver for one machine cycle i.e. 4 clock cycles
func (lcd *LCD) Tick(memory mem.Memory, cycle int) {
	ly := uint8(cycle / 114)
	lyRemainder := cycle % 114
	var stat uint8
	// Set mode on stat register
	switch {
	case ly == 144:
		// V-Blank period starts
		stat = 1
		*memory.IF |= 0x01
	case lyRemainder == 0:
		// OAM period starts
		stat = 2
	case lyRemainder == 20:
		// LCD data transfer period starts
		stat = 3
	case lyRemainder == 63:
		// H-Blank period starts
		stat = 0
		if ly < 144 {
			lcd.updateLcdLine(memory, ly)
		}
	default:
		stat = *memory.STAT
	}
	// Set coincidence flag and coincidence interrupt on stat register
	if ly == uint8(*memory.Read(0xff45)) {
		stat |= 0x44
	} else {
		stat &^= 0x44
	}
	// Set interrupts on stat register
	switch {
	case ly == 144:
		stat |= 0x10
	case lyRemainder == 0:
		stat |= 0x20
	case lyRemainder == 63:
		stat |= 0x08
	}
	*memory.STAT = stat
	*memory.LY = ly
}

// FrameData returns the frame data as a 160x144 array of bytes where each element is a colour value between 0 and 3
func (lcd *LCD) FrameData() [23040]uint8 {
	return lcd.data
}

func lowTileAbsoluteAddress(tileNumber uint8) uint16 {
	return 0x8000 + uint16(tileNumber)*16
}

func highTileAbsoluteAddress(tileNumber int8) uint16 {
	return uint16(0x9000 + int(tileNumber)*16)
}

// Returns 16 bytes representing one 8x8 tile
func tileData(mem mem.Memory, tile uint16) []byte {
	var tileAddr uint16
	if highBgTileMapDisplaySelect(mem) {
		tileAddr = 0x9c00 + tile
	} else {
		tileAddr = 0x9800 + tile
	}
	tileNumber := *mem.Read(tileAddr)
	if lowTileDataSelect(mem) {
		return mem.ReadRegion(lowTileAbsoluteAddress(tileNumber), 16)
	}
	return mem.ReadRegion(highTileAbsoluteAddress(int8(tileNumber)), 16)
}

func pixel(mem mem.Memory, memoryAddr uint16, bit uint8) uint8 {
	var a, b, pixel uint8
	a = uint8(*mem.Read(memoryAddr))
	b = uint8(*mem.Read(memoryAddr + 1))
	switch bit {
	case 0:
		pixel = (a&bit0)>>7 | (b&bit0)>>6
	case 1:
		pixel = (a&bit1)>>6 | (b&bit1)>>5
	case 2:
		pixel = (a&bit2)>>5 | (b&bit2)>>4
	case 3:
		pixel = (a&bit3)>>4 | (b&bit3)>>3
	case 4:
		pixel = (a&bit4)>>3 | (b&bit4)>>2
	case 5:
		pixel = (a&bit5)>>2 | (b&bit5)>>1
	case 6:
		pixel = (a&bit6)>>1 | (b & bit6)
	case 7:
		pixel = (a & bit7) | (b&bit7)<<1
	default:
		panic(fmt.Sprintf("Bad bit in pixel(): %v", bit))
	}
	return pixel
}

// Returns the memory address of the tile
func tileDataAddr(mem mem.Memory, tileX, tileY uint8, highTileMap, lowTileData bool) uint16 {
	var tileNumberAddr, tileIndex uint16
	tileIndex = uint16(tileY)*32 + uint16(tileX)
	if highTileMap {
		tileNumberAddr = 0x9c00 + tileIndex
	} else {
		tileNumberAddr = 0x9800 + tileIndex
	}
	tileNumber := *mem.Read(tileNumberAddr)
	if lowTileData {
		return lowTileAbsoluteAddress(tileNumber)
	}
	return highTileAbsoluteAddress(int8(tileNumber))
}

func deriveSpritePixel(mem mem.Memory, vramX, vramY uint8) (uint8, bool) {
	return 0, false
}

func deriveTilePixel(mem mem.Memory, vramX, vramY uint8, highTileMap, lowTileData bool) uint8 {
	var tileX, tileY, tileOffsetX, tileOffsetY uint8
	var tileAddr, memoryAddr uint16
	tileX = vramX / 8
	tileY = vramY / 8
	tileAddr = tileDataAddr(mem, tileX, tileY, highTileMap, lowTileData)
	tileOffsetX = vramX % 8
	tileOffsetY = vramY % 8
	memoryAddr = tileAddr + uint16(tileOffsetY)*2
	return pixel(mem, memoryAddr, tileOffsetX)
}

func deriveWindowPixel(mem mem.Memory, vramX, vramY uint8) (uint8, bool) {
	if windowDisplayEnable(mem) && vramX >= *mem.WX && vramY >= *mem.WY+7 {
		highTileMap := highWindowTileMapDisplaySelect(mem)
		lowTileData := lowTileDataSelect(mem)
		return deriveTilePixel(mem, vramX, vramY, highTileMap, lowTileData), true
	}
	return 0, false
}

func deriveBackgroundPixel(mem mem.Memory, vramX, vramY uint8) uint8 {
	if !bgDisplay(mem) {
		return 0
	}
	highTileMap := highBgTileMapDisplaySelect(mem)
	lowTileData := lowTileDataSelect(mem)
	return deriveTilePixel(mem, vramX, vramY, highTileMap, lowTileData)
}

func derivePixel(mem mem.Memory, vramX, vramY uint8) uint8 {
	if !lcdDisplayEnable(mem) {
		return 0
	}
	if pixel, found := deriveSpritePixel(mem, vramX, vramY); found {
		return pixel
	}
	if pixel, found := deriveWindowPixel(mem, vramX, vramY); found {
		return pixel
	}
	return deriveBackgroundPixel(mem, vramX, vramY)
}

func (lcd *LCD) updateLcdLine(mem mem.Memory, lcdY uint8) {
	var lcdX, vramX, vramY uint8
	var index uint16
	for lcdX = 0; lcdX < 160; lcdX++ {
		index = uint16(lcdY)*160 + uint16(lcdX)
		vramX = lcdX + *mem.SCX // Overflows deliberately
		vramY = lcdY + *mem.SCY // Overflows deliberately
		lcd.data[index] = derivePixel(mem, vramX, vramY)
	}
}
<|endoftext|>


<commit_before>package zenhan

import (
	"strings"
)

type flag uint

const (
	// ASCII converts only ascii
	ASCII flag = 1 << iota
	// DIGIT converts only digit
	DIGIT
	// KANA converts only kana
	KANA
	// ALL converts ascii, digit, and kana
	ALL = ASCII | DIGIT | KANA
)

const (
	dakuten    = "ﾞ"
	handakuten = "ﾟ"
)

var (
	zhASCII map[string]string
	hzASCII map[string]string
	zhDIGIT map[string]string
	hzDIGIT map[string]string
	zhKANA  map[string]string
	hzKANA  map[string]string
)

func join(m1 map[string]string, m2 map[string]string) map[string]string {
	m := make(map[string]string)
	for k, v := range m1 {
		m[k] = v
	}
	for k, v := range m2 {
		m[k] = v
	}
	return m
}

func init() {
	zASCII := []string{"ａ", "ｂ", "ｃ", "ｄ", "ｅ", "ｆ", "ｇ", "ｈ", "ｉ",
		"ｊ", "ｋ", "ｌ", "ｍ", "ｎ", "ｏ", "ｐ", "ｑ", "ｒ",
		"ｓ", "ｔ", "ｕ", "ｖ", "ｗ", "ｘ", "ｙ", "ｚ",
		"Ａ", "Ｂ", "Ｃ", "Ｄ", "Ｅ", "Ｆ", "Ｇ", "Ｈ", "Ｉ",
		"Ｊ", "Ｋ", "Ｌ", "Ｍ", "Ｎ", "Ｏ", "Ｐ", "Ｑ", "Ｒ",
		"Ｓ", "Ｔ", "Ｕ", "Ｖ", "Ｗ", "Ｘ", "Ｙ", "Ｚ",
		"！", "”", "＃", "＄", "％", "＆", "’", "（", "）",
		"＊", "＋", "，", "−", "．", "／", "：", "；", "＜",
		"＝", "＞", "？", "＠", "［", "￥", "］", "＾", "＿",
		"‘", "｛", "｜", "｝", "〜", "　"}
	zDigit := []string{"０", "１", "２", "３", "４",
		"５", "６", "７", "８", "９"}
	zKana := []string{"ア", "イ", "ウ", "エ", "オ",
		"カ", "キ", "ク", "ケ", "コ",
		"サ", "シ", "ス", "セ", "ソ",
		"タ", "チ", "ツ", "テ", "ト",
		"ナ", "ニ", "ヌ", "ネ", "ノ",
		"ハ", "ヒ", "フ", "ヘ", "ホ",
		"マ", "ミ", "ム", "メ", "モ",
		"ヤ", "ユ", "ヨ",
		"ラ", "リ", "ル", "レ", "ロ",
		"ワ", "ヲ", "ン",
		"ァ", "ィ", "ゥ", "ェ", "ォ",
		"ッ", "ャ", "ュ", "ョ", "ヴ",
		"ガ", "ギ", "グ", "ゲ", "ゴ",
		"ザ", "ジ", "ズ", "ゼ", "ゾ",
		"ダ", "ヂ", "ヅ", "デ", "ド",
		"バ", "ビ", "ブ", "ベ", "ボ",
		"パ", "ピ", "プ", "ペ", "ポ",
		"。", "、", "・", "゛", "゜", "「", "」", "ー"}
	hASCII := []string{"a", "b", "c", "d", "e", "f", "g", "h", "i",
		"j", "k", "l", "m", "n", "o", "p", "q", "r",
		"s", "t", "u", "v", "w", "x", "y", "z",
		"A", "B", "C", "D", "E", "F", "G", "H", "I",
		"J", "K", "L", "M", "N", "O", "P", "Q", "R",
		"S", "T", "U", "V", "W", "X", "Y", "Z",
		"!", `"`, "#", "$", "%", "&", "'", "(", ")",
		"*", "+", ",", "-", ".", "/", ":", ";", "<",
		"=", ">", "?", "@", "[", "\\", "]", "^", "_",
		"`", "{", "|", "}", "~", " "}
	hDigit := []string{"0", "1", "2", "3", "4", "5", "6", "7", "8", "9"}
	hKana := []string{"ｱ", "ｲ", "ｳ", "ｴ", "ｵ",
		"ｶ", "ｷ", "ｸ", "ｹ", "ｺ",
		"ｻ", "ｼ", "ｽ", "ｾ", "ｿ",
		"ﾀ", "ﾁ", "ﾂ", "ﾃ", "ﾄ",
		"ﾅ", "ﾆ", "ﾇ", "ﾈ", "ﾉ",
		"ﾊ", "ﾋ", "ﾌ", "ﾍ", "ﾎ",
		"ﾏ", "ﾐ", "ﾑ", "ﾒ", "ﾓ",
		"ﾔ", "ﾕ", "ﾖ",
		"ﾗ", "ﾘ", "ﾙ", "ﾚ", "ﾛ",
		"ﾜ", "ｦ", "ﾝ",
		"ｧ", "ｨ", "ｩ", "ｪ", "ｫ",
		"ｯ", "ｬ", "ｭ", "ｮ", "ｳﾞ",
		"ｶﾞ", "ｷﾞ", "ｸﾞ", "ｹﾞ", "ｺﾞ",
		"ｻﾞ", "ｼﾞ", "ｽﾞ", "ｾﾞ", "ｿﾞ",
		"ﾀﾞ", "ﾁﾞ", "ﾂﾞ", "ﾃﾞ", "ﾄﾞ",
		"ﾊﾞ", "ﾋﾞ", "ﾌﾞ", "ﾍﾞ", "ﾎﾞ",
		"ﾊﾟ", "ﾋﾟ", "ﾌﾟ", "ﾍﾟ", "ﾎﾟ",
		"｡", "､", "･", "ﾞ", "ﾟ", "｢", "｣", "ｰ"}

	zhASCII = make(map[string]string)
	for i, s := range zASCII {
		zhASCII[s] = hASCII[i]
	}
	hzASCII = make(map[string]string)
	for i, s := range hASCII {
		hzASCII[s] = zASCII[i]
	}

	zhDIGIT = make(map[string]string)
	for i, s := range zDigit {
		zhDIGIT[s] = hDigit[i]
	}
	hzDIGIT = make(map[string]string)
	for i, s := range hDigit {
		hzDIGIT[s] = zDigit[i]
	}

	zhKANA = make(map[string]string)
	for i, s := range zKana {
		zhKANA[s] = hKana[i]
	}
	hzKANA = make(map[string]string)
	for i, s := range hKana {
		hzKANA[s] = zKana[i]
	}
}

func makeHan2zenDict(mode flag) map[string]string {
	m := make(map[string]string)
	if mode&ASCII == ASCII {
		m = join(m, hzASCII)
	}
	if mode&DIGIT == DIGIT {
		m = join(m, hzDIGIT)
	}
	if mode&KANA == KANA {
		m = join(m, hzKANA)
	}
	return m
}

func makeZen2hanDict(mode flag) map[string]string {
	m := make(map[string]string)
	if mode&ASCII == ASCII {
		m = join(m, zhASCII)
	}
	if mode&DIGIT == DIGIT {
		m = join(m, zhDIGIT)
	}
	if mode&KANA == KANA {
		m = join(m, zhKANA)
	}
	return m
}

func getValueFromMap(m map[string]string, key string, defaultValue string) string {
	if v, ok := m[key]; ok {
		return v
	} else {
		return defaultValue
	}
}

func any(array []string, value string) bool {
	for _, v := range array {
		if value == v {
			return true
		}
	}
	return false
}

// H2z converts string from hankaku to zenkaku
func H2z(text string, mode flag, ignore ...string) string {
	m := makeHan2zenDict(mode)
	t := []rune(text)
	converted := make([]string, 0, len(t))
	for i, v := range t {
		curr := string(v)
		if any(ignore, curr) {
			converted = append(converted, curr)
		} else if curr == dakuten || curr == handakuten && i > 0 {
			prev := string(t[i-1])
			if z, ok := m[prev+curr]; ok {
				converted = converted[:len(converted)-1]
				converted = append(converted, z)
			} else {
				converted = append(converted, getValueFromMap(m, curr, curr))
			}
		} else {
			converted = append(converted, getValueFromMap(m, curr, curr))
		}
	}
	return strings.Join(converted, "")
}

// Z2h converts string from zenkaku to hankaku
func Z2h(text string, mode flag, ignore ...string) string {
	m := makeZen2hanDict(mode)
	t := []rune(text)
	converted := make([]string, 0, len(t))
	for _, v := range t {
		curr := string(v)
		if any(ignore, curr) {
			converted = append(converted, curr)
		} else {
			converted = append(converted, getValueFromMap(m, curr, curr))
		}
	}
	return strings.Join(converted, "")
}
<commit_msg>Remove else statement<commit_after>package zenhan

import (
	"strings"
)

type flag uint

const (
	// ASCII converts only ascii
	ASCII flag = 1 << iota
	// DIGIT converts only digit
	DIGIT
	// KANA converts only kana
	KANA
	// ALL converts ascii, digit, and kana
	ALL = ASCII | DIGIT | KANA
)

const (
	dakuten    = "ﾞ"
	handakuten = "ﾟ"
)

var (
	zhASCII map[string]string
	hzASCII map[string]string
	zhDIGIT map[string]string
	hzDIGIT map[string]string
	zhKANA  map[string]string
	hzKANA  map[string]string
)

func join(m1 map[string]string, m2 map[string]string) map[string]string {
	m := make(map[string]string)
	for k, v := range m1 {
		m[k] = v
	}
	for k, v := range m2 {
		m[k] = v
	}
	return m
}

func init() {
	zASCII := []string{"ａ", "ｂ", "ｃ", "ｄ", "ｅ", "ｆ", "ｇ", "ｈ", "ｉ",
		"ｊ", "ｋ", "ｌ", "ｍ", "ｎ", "ｏ", "ｐ", "ｑ", "ｒ",
		"ｓ", "ｔ", "ｕ", "ｖ", "ｗ", "ｘ", "ｙ", "ｚ",
		"Ａ", "Ｂ", "Ｃ", "Ｄ", "Ｅ", "Ｆ", "Ｇ", "Ｈ", "Ｉ",
		"Ｊ", "Ｋ", "Ｌ", "Ｍ", "Ｎ", "Ｏ", "Ｐ", "Ｑ", "Ｒ",
		"Ｓ", "Ｔ", "Ｕ", "Ｖ", "Ｗ", "Ｘ", "Ｙ", "Ｚ",
		"！", "”", "＃", "＄", "％", "＆", "’", "（", "）",
		"＊", "＋", "，", "−", "．", "／", "：", "；", "＜",
		"＝", "＞", "？", "＠", "［", "￥", "］", "＾", "＿",
		"‘", "｛", "｜", "｝", "〜", "　"}
	zDigit := []string{"０", "１", "２", "３", "４",
		"５", "６", "７", "８", "９"}
	zKana := []string{"ア", "イ", "ウ", "エ", "オ",
		"カ", "キ", "ク", "ケ", "コ",
		"サ", "シ", "ス", "セ", "ソ",
		"タ", "チ", "ツ", "テ", "ト",
		"ナ", "ニ", "ヌ", "ネ", "ノ",
		"ハ", "ヒ", "フ", "ヘ", "ホ",
		"マ", "ミ", "ム", "メ", "モ",
		"ヤ", "ユ", "ヨ",
		"ラ", "リ", "ル", "レ", "ロ",
		"ワ", "ヲ", "ン",
		"ァ", "ィ", "ゥ", "ェ", "ォ",
		"ッ", "ャ", "ュ", "ョ", "ヴ",
		"ガ", "ギ", "グ", "ゲ", "ゴ",
		"ザ", "ジ", "ズ", "ゼ", "ゾ",
		"ダ", "ヂ", "ヅ", "デ", "ド",
		"バ", "ビ", "ブ", "ベ", "ボ",
		"パ", "ピ", "プ", "ペ", "ポ",
		"。", "、", "・", "゛", "゜", "「", "」", "ー"}
	hASCII := []string{"a", "b", "c", "d", "e", "f", "g", "h", "i",
		"j", "k", "l", "m", "n", "o", "p", "q", "r",
		"s", "t", "u", "v", "w", "x", "y", "z",
		"A", "B", "C", "D", "E", "F", "G", "H", "I",
		"J", "K", "L", "M", "N", "O", "P", "Q", "R",
		"S", "T", "U", "V", "W", "X", "Y", "Z",
		"!", `"`, "#", "$", "%", "&", "'", "(", ")",
		"*", "+", ",", "-", ".", "/", ":", ";", "<",
		"=", ">", "?", "@", "[", "\\", "]", "^", "_",
		"`", "{", "|", "}", "~", " "}
	hDigit := []string{"0", "1", "2", "3", "4", "5", "6", "7", "8", "9"}
	hKana := []string{"ｱ", "ｲ", "ｳ", "ｴ", "ｵ",
		"ｶ", "ｷ", "ｸ", "ｹ", "ｺ",
		"ｻ", "ｼ", "ｽ", "ｾ", "ｿ",
		"ﾀ", "ﾁ", "ﾂ", "ﾃ", "ﾄ",
		"ﾅ", "ﾆ", "ﾇ", "ﾈ", "ﾉ",
		"ﾊ", "ﾋ", "ﾌ", "ﾍ", "ﾎ",
		"ﾏ", "ﾐ", "ﾑ", "ﾒ", "ﾓ",
		"ﾔ", "ﾕ", "ﾖ",
		"ﾗ", "ﾘ", "ﾙ", "ﾚ", "ﾛ",
		"ﾜ", "ｦ", "ﾝ",
		"ｧ", "ｨ", "ｩ", "ｪ", "ｫ",
		"ｯ", "ｬ", "ｭ", "ｮ", "ｳﾞ",
		"ｶﾞ", "ｷﾞ", "ｸﾞ", "ｹﾞ", "ｺﾞ",
		"ｻﾞ", "ｼﾞ", "ｽﾞ", "ｾﾞ", "ｿﾞ",
		"ﾀﾞ", "ﾁﾞ", "ﾂﾞ", "ﾃﾞ", "ﾄﾞ",
		"ﾊﾞ", "ﾋﾞ", "ﾌﾞ", "ﾍﾞ", "ﾎﾞ",
		"ﾊﾟ", "ﾋﾟ", "ﾌﾟ", "ﾍﾟ", "ﾎﾟ",
		"｡", "､", "･", "ﾞ", "ﾟ", "｢", "｣", "ｰ"}

	zhASCII = make(map[string]string)
	for i, s := range zASCII {
		zhASCII[s] = hASCII[i]
	}
	hzASCII = make(map[string]string)
	for i, s := range hASCII {
		hzASCII[s] = zASCII[i]
	}

	zhDIGIT = make(map[string]string)
	for i, s := range zDigit {
		zhDIGIT[s] = hDigit[i]
	}
	hzDIGIT = make(map[string]string)
	for i, s := range hDigit {
		hzDIGIT[s] = zDigit[i]
	}

	zhKANA = make(map[string]string)
	for i, s := range zKana {
		zhKANA[s] = hKana[i]
	}
	hzKANA = make(map[string]string)
	for i, s := range hKana {
		hzKANA[s] = zKana[i]
	}
}

func makeHan2zenDict(mode flag) map[string]string {
	m := make(map[string]string)
	if mode&ASCII == ASCII {
		m = join(m, hzASCII)
	}
	if mode&DIGIT == DIGIT {
		m = join(m, hzDIGIT)
	}
	if mode&KANA == KANA {
		m = join(m, hzKANA)
	}
	return m
}

func makeZen2hanDict(mode flag) map[string]string {
	m := make(map[string]string)
	if mode&ASCII == ASCII {
		m = join(m, zhASCII)
	}
	if mode&DIGIT == DIGIT {
		m = join(m, zhDIGIT)
	}
	if mode&KANA == KANA {
		m = join(m, zhKANA)
	}
	return m
}

func getValueFromMap(m map[string]string, key string, defaultValue string) string {
	if v, ok := m[key]; ok {
		return v
	}
	return defaultValue
}

func any(array []string, value string) bool {
	for _, v := range array {
		if value == v {
			return true
		}
	}
	return false
}

// H2z converts string from hankaku to zenkaku
func H2z(text string, mode flag, ignore ...string) string {
	m := makeHan2zenDict(mode)
	t := []rune(text)
	converted := make([]string, 0, len(t))
	for i, v := range t {
		curr := string(v)
		if any(ignore, curr) {
			converted = append(converted, curr)
		} else if curr == dakuten || curr == handakuten && i > 0 {
			prev := string(t[i-1])
			if z, ok := m[prev+curr]; ok {
				converted = converted[:len(converted)-1]
				converted = append(converted, z)
			} else {
				converted = append(converted, getValueFromMap(m, curr, curr))
			}
		} else {
			converted = append(converted, getValueFromMap(m, curr, curr))
		}
	}
	return strings.Join(converted, "")
}

// Z2h converts string from zenkaku to hankaku
func Z2h(text string, mode flag, ignore ...string) string {
	m := makeZen2hanDict(mode)
	t := []rune(text)
	converted := make([]string, 0, len(t))
	for _, v := range t {
		curr := string(v)
		if any(ignore, curr) {
			converted = append(converted, curr)
		} else {
			converted = append(converted, getValueFromMap(m, curr, curr))
		}
	}
	return strings.Join(converted, "")
}
<|endoftext|>


<commit_before>package lib

import (
	"sync"

	"github.com/dop251/goja"
)

type VM struct {
	*goja.Runtime
	sync.Mutex
}

func (vm *VM) Lock() {
	vm.Mutex.Lock()
}

func (vm *VM) Unlock() {
	vm.Mutex.Unlock()
}

func (vm *VM) RunString(str string) (goja.Value, error) {
	vm.Lock()
	defer vm.Unlock()

	return vm.Runtime.RunString(str)
}

func (vm *VM) RunScript(name, value string) (goja.Value, error) {
	vm.Lock()
	defer vm.Unlock()

	return vm.Runtime.RunScript(name, value)
}

func Register(vm *VM) {
	initAsync(vm)
	initTimers(vm)
	initUUID(vm)
	initProcess(vm)
	initSSDP(vm)
}
<commit_msg>Remove mutexes<commit_after>package lib

import (
	"sync"

	"github.com/dop251/goja"
)

type VM struct {
	*goja.Runtime
	sync.Mutex
}

func (vm *VM) RunString(str string) (goja.Value, error) {
	vm.Lock()
	defer vm.Unlock()

	return vm.Runtime.RunString(str)
}

func (vm *VM) RunScript(name, value string) (goja.Value, error) {
	vm.Lock()
	defer vm.Unlock()

	return vm.Runtime.RunScript(name, value)
}

func Register(vm *VM) {
	initAsync(vm)
	initTimers(vm)
	initUUID(vm)
	initProcess(vm)
	initSSDP(vm)
}
<|endoftext|>


<commit_before>package containers

import (
	"context"
	"encoding/json"

	"github.com/boltdb/bolt"
	"github.com/pkg/errors"
)

var (
	ErrExists   = errors.New("images: exists")
	ErrNotFound = errors.New("images: not found")
)

// IsNotFound returns true if the error is due to a missing image.
func IsNotFound(err error) bool {
	return errors.Cause(err) == ErrNotFound
}

func IsExists(err error) bool {
	return errors.Cause(err) == ErrExists
}

var (
	bucketKeyStorageVersion = []byte("v1")
	bucketKeyContainers     = []byte("containers")
)

type storage struct {
	tx *bolt.Tx
}

func NewStore(tx *bolt.Tx) Store {
	return &storage{
		tx: tx,
	}
}

func (s *storage) Get(ctx context.Context, id string) (Container, error) {
	bkt := s.bucket()
	if bkt == nil {
		return Container{}, errors.Wrap(ErrNotFound, "bucket does not exist")
	}

	cb := bkt.Get([]byte(id))
	if len(cb) == 0 {
		return Container{}, errors.Wrap(ErrNotFound, "no content for id")
	}

	var container Container
	if err := json.Unmarshal(cb, &container); err != nil {
		return Container{}, errors.Wrap(err, "failed to unmarshal container")
	}

	return container, nil
}

func (s *storage) List(ctx context.Context, filter string) ([]Container, error) {
	containers := []Container{}
	bkt := s.bucket()
	if bkt == nil {
		return containers, nil
	}
	err := bkt.ForEach(func(k, v []byte) error {
		container := Container{ID: string(k)}
		if err := json.Unmarshal(v, &container); err != nil {
			return errors.Wrap(err, "failed to unmarshal container")
		}
		containers = append(containers, container)
		return nil
	})
	if err != nil {
		return nil, err
	}

	return containers, nil
}

func (s *storage) Create(ctx context.Context, container Container) (Container, error) {
	bkt, err := s.createBucket()
	if err != nil {
		return Container{}, err
	}

	b := bkt.Get([]byte(container.ID))
	if len(b) > 0 {
		return Container{}, errors.Wrap(ErrExists, "content for id already exists")
	}

	b, err = json.Marshal(container)
	if err != nil {
		return Container{}, err
	}

	return container, bkt.Put([]byte(container.ID), b)
}

func (s *storage) Update(ctx context.Context, container Container) (Container, error) {
	bkt, err := s.createBucket()
	if err != nil {
		return Container{}, err
	}

	b := bkt.Get([]byte(container.ID))
	if len(b) == 0 {
		return Container{}, errors.Wrap(ErrNotFound, "no content for id")
	}

	b, err = json.Marshal(container)
	if err != nil {
		return Container{}, err
	}

	return container, bkt.Put([]byte(container.ID), b)
}

func (s *storage) Delete(ctx context.Context, id string) error {
	bkt, err := s.createBucket()
	if err != nil {
		return err
	}

	b := bkt.Get([]byte(id))
	if len(b) == 0 {
		return errors.Wrap(ErrNotFound, "no content for id")
	}

	return bkt.Delete([]byte(id))
}

func (s *storage) bucket() *bolt.Bucket {
	bkt := s.tx.Bucket(bucketKeyStorageVersion)
	if bkt == nil {
		return nil
	}
	return bkt.Bucket(bucketKeyContainers)
}

func (s *storage) createBucket() (*bolt.Bucket, error) {
	bkt, err := s.tx.CreateBucketIfNotExists(bucketKeyStorageVersion)
	if err != nil {
		return nil, err
	}
	return bkt.CreateBucketIfNotExists(bucketKeyContainers)
}
<commit_msg>Store container metadata in boltdb buckets<commit_after>package containers

import (
	"context"

	"github.com/boltdb/bolt"
	"github.com/pkg/errors"
)

var (
	ErrExists   = errors.New("images: exists")
	ErrNotFound = errors.New("images: not found")
)

// IsNotFound returns true if the error is due to a missing image.
func IsNotFound(err error) bool {
	return errors.Cause(err) == ErrNotFound
}

func IsExists(err error) bool {
	return errors.Cause(err) == ErrExists
}

var (
	bucketKeyStorageVersion = []byte("v1")
	bucketKeyContainers     = []byte("containers")
	bucketKeyLabels         = []byte("labels")
	bucketKeyImage          = []byte("image")
	bucketKeyRuntime        = []byte("runtime")
	bucketKeySpec           = []byte("spec")
	bucketKeyRootFS         = []byte("rootfs")
)

type storage struct {
	tx *bolt.Tx
}

func NewStore(tx *bolt.Tx) Store {
	return &storage{
		tx: tx,
	}
}

func (s *storage) Get(ctx context.Context, id string) (Container, error) {
	bkt := getContainerBucket(s.tx, id)
	if bkt == nil {
		return Container{}, errors.Wrap(ErrNotFound, "bucket does not exist")
	}

	container := Container{ID: id}
	if err := readContainer(&container, bkt); err != nil {
		return Container{}, errors.Wrap(err, "failed to read container")
	}

	return container, nil
}

func (s *storage) List(ctx context.Context, filter string) ([]Container, error) {
	containers := []Container{}
	bkt := getContainersBucket(s.tx)
	if bkt == nil {
		return containers, nil
	}
	err := bkt.ForEach(func(k, v []byte) error {
		cbkt := bkt.Bucket(k)
		if cbkt == nil {
			return nil
		}
		container := Container{ID: string(k)}

		if err := readContainer(&container, cbkt); err != nil {
			return errors.Wrap(err, "failed to read container")
		}
		containers = append(containers, container)
		return nil
	})
	if err != nil {
		return nil, err
	}

	return containers, nil
}

func (s *storage) Create(ctx context.Context, container Container) (Container, error) {
	bkt, err := createContainersBucket(s.tx)
	if err != nil {
		return Container{}, err
	}

	cbkt, err := bkt.CreateBucket([]byte(container.ID))
	if err != nil {
		if err == bolt.ErrBucketExists {
			err = errors.Wrap(ErrExists, "content for id already exists")
		}
		return Container{}, err
	}

	if err := writeContainer(&container, cbkt); err != nil {
		return Container{}, errors.Wrap(err, "failed to write container")
	}

	return container, nil
}

func (s *storage) Update(ctx context.Context, container Container) (Container, error) {
	bkt := getContainersBucket(s.tx)
	if bkt == nil {
		return Container{}, errors.Wrap(ErrNotFound, "no containers")
	}

	cbkt := bkt.Bucket([]byte(container.ID))
	if cbkt == nil {
		return Container{}, errors.Wrap(ErrNotFound, "no content for id")
	}

	if err := writeContainer(&container, cbkt); err != nil {
		return Container{}, errors.Wrap(err, "failed to write container")
	}

	return container, nil
}

func (s *storage) Delete(ctx context.Context, id string) error {
	bkt := getContainersBucket(s.tx)
	if bkt == nil {
		return errors.Wrap(ErrNotFound, "no containers")
	}

	err := bkt.DeleteBucket([]byte(id))
	if err == bolt.ErrBucketNotFound {
		return errors.Wrap(ErrNotFound, "no content for id")
	}
	return err
}

func readContainer(container *Container, bkt *bolt.Bucket) error {
	return bkt.ForEach(func(k, v []byte) error {
		switch string(k) {
		case string(bucketKeyImage):
			container.Image = string(v)
		case string(bucketKeyRuntime):
			container.Runtime = string(v)
		case string(bucketKeySpec):
			container.Spec = v
		case string(bucketKeyRootFS):
			container.RootFS = string(v)
		case string(bucketKeyLabels):
			lbkt := bkt.Bucket(bucketKeyLabels)
			if lbkt == nil {
				return nil
			}
			container.Labels = map[string]string{}
			if err := lbkt.ForEach(func(k, v []byte) error {
				container.Labels[string(k)] = string(v)
				return nil
			}); err != nil {
				return err
			}
		}

		return nil
	})
}

func writeContainer(container *Container, bkt *bolt.Bucket) error {
	for _, v := range [][2][]byte{
		{bucketKeyImage, []byte(container.Image)},
		{bucketKeyRuntime, []byte(container.Runtime)},
		{bucketKeySpec, container.Spec},
		{bucketKeyRootFS, []byte(container.RootFS)},
	} {
		if err := bkt.Put(v[0], v[1]); err != nil {
			return err
		}
	}
	// Remove existing labels to keep from merging
	if lbkt := bkt.Bucket(bucketKeyLabels); lbkt != nil {
		if err := bkt.DeleteBucket(bucketKeyLabels); err != nil {
			return err
		}
	}
	lbkt, err := bkt.CreateBucket(bucketKeyLabels)
	if err != nil {
		return err
	}
	for k, v := range container.Labels {
		if err := lbkt.Put([]byte(k), []byte(v)); err != nil {
			return err
		}
	}
	return nil
}

func createContainersBucket(tx *bolt.Tx) (*bolt.Bucket, error) {
	bkt, err := tx.CreateBucketIfNotExists(bucketKeyStorageVersion)
	if err != nil {
		return nil, err
	}
	return bkt.CreateBucketIfNotExists(bucketKeyContainers)
}

func getContainersBucket(tx *bolt.Tx) *bolt.Bucket {
	return getBucket(tx, bucketKeyStorageVersion, bucketKeyContainers)
}

func getContainerBucket(tx *bolt.Tx, id string) *bolt.Bucket {
	return getBucket(tx, bucketKeyStorageVersion, bucketKeyContainers, []byte(id))
}

func getBucket(tx *bolt.Tx, keys ...[]byte) *bolt.Bucket {
	bkt := tx.Bucket(keys[0])

	for _, key := range keys[1:] {
		if bkt == nil {
			break
		}
		bkt = bkt.Bucket(key)
	}

	return bkt
}
<|endoftext|>


<commit_before>package main

import (
    "code.google.com/p/go-tour/wc"
)

func WordCount(s string) map[string]int {
    return map[string]int{"x": 1}
}

func main() {
    wc.Test(WordCount)
}
<commit_msg>Implement `WordCount`.<commit_after>package main

import (
	"code.google.com/p/go-tour/wc"
	"strings"
)

func WordCount(s string) map[string]int {
	ret := make(map[string]int)
	for _, c := range strings.Fields(s) {
		ret[c]++
	}
	return ret
}

func main() {
	wc.Test(WordCount)
}
<|endoftext|>


<commit_before>package metadata

import (
	"errors"
	"fmt"
	. "github.com/dbdd4us/qcloudapi-sdk-go/util"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"net/url"
	"time"
)

type Request struct {
}

const (
	ENDPOINT = "http://metadata.tencentyun.com/meta-data"

	INSTANCE_ID  = "instance-id"
	UUID         = "uuid"
	MAC          = "mac"
	PRIVATE_IPV4 = "local-ipv4"
	REGION       = "placement/region"
	ZONE         = "placement/zone"
	PUBLIC_IPV4  = "public-ipv4"
)

type IMetaDataClient interface {
	Resource(resource string) IMetaDataClient
	Go() (string, error)
	Url() (string, error)
}

type MetaData struct {
	c IMetaDataClient
}

func NewMetaData(client *http.Client) *MetaData {
	if client == nil {
		client = &http.Client{}
	}
	return &MetaData{
		c: &MetaDataClient{client: client},
	}
}

func (m *MetaData) UUID() (string, error) {

	uuid, err := m.c.Resource(UUID).Go()
	if err != nil {
		return "", err
	}
	return uuid, err
}

func (m *MetaData) InstanceID() (string, error) {

	instanceId, err := m.c.Resource(INSTANCE_ID).Go()
	if err != nil {
		return "", err
	}
	return instanceId, err
}

func (m *MetaData) Mac() (string, error) {

	mac, err := m.c.Resource(MAC).Go()
	if err != nil {
		return "", err
	}
	return mac, nil
}

func (m *MetaData) PrivateIPv4() (string, error) {

	ip, err := m.c.Resource(PRIVATE_IPV4).Go()
	if err != nil {
		return "", err
	}
	return ip, nil
}

func (m *MetaData) PublicIPv4() (string, error) {

	ip, err := m.c.Resource(PUBLIC_IPV4).Go()
	if err != nil {
		return "", err
	}
	return ip, nil
}

func (m *MetaData) Region() (string, error) {

	region, err := m.c.Resource(REGION).Go()
	if err != nil {
		return "", err
	}
	return region, nil
}

func (m *MetaData) Zone() (string, error) {

	zone, err := m.c.Resource(ZONE).Go()
	if err != nil {
		return "", err
	}
	return zone, nil
}

//
type MetaDataClient struct {
	resource string
	client   *http.Client
}

func (m *MetaDataClient) Resource(resource string) IMetaDataClient {
	m.resource = resource
	return m
}

func (m *MetaDataClient) Url() (string, error) {
	if m.resource == "" {
		return "", errors.New("the resource you want to visit must not be nil!")
	}
	return fmt.Sprintf("%s/%s", ENDPOINT, m.resource), nil
}

func (m *MetaDataClient) send() (string, error) {
	u, err := m.Url()
	if err != nil {
		return "", err
	}
	requ, err := http.NewRequest(http.MethodGet, u, nil)

	if err != nil {
		return "", err
	}
	resp, err := m.client.Do(requ)
	if err != nil {
		return "", err
	}
	if resp.StatusCode != 200 {
		return "", err
	}
	defer resp.Body.Close()

	data, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return "", err
	}
	return string(data), nil

}

var retry = AttemptStrategy{
	Min:   5,
	Total: 5 * time.Second,
	Delay: 200 * time.Millisecond,
}

func (vpc *MetaDataClient) Go() (resu string, err error) {
	for r := retry.Start(); r.Next(); {
		resu, err = vpc.send()
		if !shouldRetry(err) {
			break
		}
	}
	return resu, err
}

type TimeoutError interface {
	error
	Timeout() bool // Is the error a timeout?
}

func shouldRetry(err error) bool {
	if err == nil {
		return false
	}

	_, ok := err.(TimeoutError)
	if ok {
		return true
	}

	switch err {
	case io.ErrUnexpectedEOF, io.EOF:
		return true
	}
	switch e := err.(type) {
	case *net.DNSError:
		return true
	case *net.OpError:
		switch e.Op {
		case "read", "write":
			return true
		}
	case *url.Error:
		// url.Error can be returned either by net/url if a URL cannot be
		// parsed, or by net/http if the response is closed before the headers
		// are received or parsed correctly. In that later case, e.Op is set to
		// the HTTP method name with the first letter uppercased. We don't want
		// to retry on POST operations, since those are not idempotent, all the
		// other ones should be safe to retry.
		switch e.Op {
		case "Get", "Put", "Delete", "Head":
			return shouldRetry(e.Err)
		default:
			return false
		}
	}
	return false
}
<commit_msg>fix connection leak<commit_after>package metadata

import (
	"errors"
	"fmt"
	. "github.com/dbdd4us/qcloudapi-sdk-go/util"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"net/url"
	"time"
)

type Request struct {
}

const (
	ENDPOINT = "http://metadata.tencentyun.com/meta-data"

	INSTANCE_ID  = "instance-id"
	UUID         = "uuid"
	MAC          = "mac"
	PRIVATE_IPV4 = "local-ipv4"
	REGION       = "placement/region"
	ZONE         = "placement/zone"
	PUBLIC_IPV4  = "public-ipv4"
)

type IMetaDataClient interface {
	Resource(resource string) IMetaDataClient
	Go() (string, error)
	Url() (string, error)
}

type MetaData struct {
	c IMetaDataClient
}

func NewMetaData(client *http.Client) *MetaData {
	if client == nil {
		client = &http.Client{}
	}
	return &MetaData{
		c: &MetaDataClient{client: client},
	}
}

func (m *MetaData) UUID() (string, error) {

	uuid, err := m.c.Resource(UUID).Go()
	if err != nil {
		return "", err
	}
	return uuid, err
}

func (m *MetaData) InstanceID() (string, error) {

	instanceId, err := m.c.Resource(INSTANCE_ID).Go()
	if err != nil {
		return "", err
	}
	return instanceId, err
}

func (m *MetaData) Mac() (string, error) {

	mac, err := m.c.Resource(MAC).Go()
	if err != nil {
		return "", err
	}
	return mac, nil
}

func (m *MetaData) PrivateIPv4() (string, error) {

	ip, err := m.c.Resource(PRIVATE_IPV4).Go()
	if err != nil {
		return "", err
	}
	return ip, nil
}

func (m *MetaData) PublicIPv4() (string, error) {

	ip, err := m.c.Resource(PUBLIC_IPV4).Go()
	if err != nil {
		return "", err
	}
	return ip, nil
}

func (m *MetaData) Region() (string, error) {

	region, err := m.c.Resource(REGION).Go()
	if err != nil {
		return "", err
	}
	return region, nil
}

func (m *MetaData) Zone() (string, error) {

	zone, err := m.c.Resource(ZONE).Go()
	if err != nil {
		return "", err
	}
	return zone, nil
}

//
type MetaDataClient struct {
	resource string
	client   *http.Client
}

func (m *MetaDataClient) Resource(resource string) IMetaDataClient {
	m.resource = resource
	return m
}

func (m *MetaDataClient) Url() (string, error) {
	if m.resource == "" {
		return "", errors.New("the resource you want to visit must not be nil!")
	}
	return fmt.Sprintf("%s/%s", ENDPOINT, m.resource), nil
}

func (m *MetaDataClient) send() (string, error) {
	u, err := m.Url()
	if err != nil {
		return "", err
	}
	requ, err := http.NewRequest(http.MethodGet, u, nil)

	if err != nil {
		return "", err
	}
	resp, err := m.client.Do(requ)
	if err != nil {
		return "", err
	}
	defer resp.Body.Close()
	if resp.StatusCode != 200 {
		return "", err
	}

	data, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return "", err
	}
	return string(data), nil

}

var retry = AttemptStrategy{
	Min:   5,
	Total: 5 * time.Second,
	Delay: 200 * time.Millisecond,
}

func (vpc *MetaDataClient) Go() (resu string, err error) {
	for r := retry.Start(); r.Next(); {
		resu, err = vpc.send()
		if !shouldRetry(err) {
			break
		}
	}
	return resu, err
}

type TimeoutError interface {
	error
	Timeout() bool // Is the error a timeout?
}

func shouldRetry(err error) bool {
	if err == nil {
		return false
	}

	_, ok := err.(TimeoutError)
	if ok {
		return true
	}

	switch err {
	case io.ErrUnexpectedEOF, io.EOF:
		return true
	}
	switch e := err.(type) {
	case *net.DNSError:
		return true
	case *net.OpError:
		switch e.Op {
		case "read", "write":
			return true
		}
	case *url.Error:
		// url.Error can be returned either by net/url if a URL cannot be
		// parsed, or by net/http if the response is closed before the headers
		// are received or parsed correctly. In that later case, e.Op is set to
		// the HTTP method name with the first letter uppercased. We don't want
		// to retry on POST operations, since those are not idempotent, all the
		// other ones should be safe to retry.
		switch e.Op {
		case "Get", "Put", "Delete", "Head":
			return shouldRetry(e.Err)
		default:
			return false
		}
	}
	return false
}
<|endoftext|>


<commit_before>package interfaces

import (
	"encoding/json"
	"fmt"
	"github.com/iaas-engine/domain"
	"io"
	"log"
	"net/http"
)

type EngineInteractor interface {
	CreateConf(server domain.Server, zipFile io.Writer)
}

type WebServiceHandler struct {
	EngineInteractor EngineInteractor
}

func (handler WebServiceHandler) CreateConfig(res http.ResponseWriter, req *http.Request) {
	defer req.Body.Close()

	decoder := json.NewDecoder(req.Body)
	mainJSON := domain.Myjson{}
	if e := decoder.Decode(&mainJSON); e != nil {
		log.Fatal(e)
	}

	server := domain.Server{}
	server = mainJSON.Server

	res.Header().Set("Content-Type", "application/zip")
	res.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", "algo"))
	handler.EngineInteractor.CreateConf(server, res)

}
<commit_msg>Renaming zip file for download<commit_after>package interfaces

import (
	"encoding/json"
	"fmt"
	"github.com/iaas-engine/domain"
	"io"
	"log"
	"net/http"
)

type EngineInteractor interface {
	CreateConf(server domain.Server, zipFile io.Writer)
}

type WebServiceHandler struct {
	EngineInteractor EngineInteractor
}

func (handler WebServiceHandler) CreateConfig(res http.ResponseWriter, req *http.Request) {
	defer req.Body.Close()

	decoder := json.NewDecoder(req.Body)
	mainJSON := domain.Myjson{}
	if e := decoder.Decode(&mainJSON); e != nil {
		log.Fatal(e)
	}

	server := domain.Server{}
	server = mainJSON.Server

	res.Header().Set("Content-Type", "application/zip")
	res.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", "puppet.zip"))
	handler.EngineInteractor.CreateConf(server, res)

}
<|endoftext|>


<commit_before>package internal

// VERSION of sail
const VERSION = "0.0.1"
<commit_msg>chore : inc version<commit_after>package internal

// VERSION of sail
const VERSION = "0.0.2"
<|endoftext|>