<commit_before>// Oh the stack trace scanning!
// The density of comments in this file is evidence that
// the code doesn't exactly explain itself. Tread with care...
package convey

import (
	"fmt"
	"github.com/jtolds/gls"
	"reflect"
	"runtime"
	"strings"

	"github.com/smartystreets/goconvey/convey/reporting"
)

const (
	missingGoTest = `Top-level calls to Convey(...) need a reference to the *testing.T.
		Hint: Convey("description here", t, func() { /* notice that the second argument was the *testing.T (t)! */ }) `
	extraGoTest = `Only the top-level call to Convey(...) needs a reference to the *testing.T.`

	failureHalt = "___FAILURE_HALT___"

	nodeKey = "node"
)

///////////////////////////////// Stack Context /////////////////////////////////

func getCurrentContext() *context {
	ctx, ok := ctxMgr.GetValue(nodeKey)
	if ok {
		return ctx.(*context)
	}
	return nil
}

func mustGetCurrentContext() *context {
	ctx := getCurrentContext()
	if ctx == nil {
		panic("cannot perform operation outside of a convey context")
	}
	return ctx
}

//////////////////////////////////// Context ////////////////////////////////////

// context magically handles all coordination of Convey's and So assertions.
//
// It is tracked on the stack as goroutine-local-storage with the gls package,
// or explicitly if the user decides to call convey like:
//
//   Convey(..., func(c C) {
//     c.So(...)
//   })
//
// This implements the `C` interface.
type context struct {
	reporter reporting.Reporter
	test     t

	name string

	curIdx   int
	children []*context

	resets []func()

	executedOnce   bool
	expectChildRun *bool
	complete       bool

	focus       bool
	failureMode FailureMode
}

// rootConvey is the main entry point to a test suite. This is called when
// there's no context in the stack already, and items must contain a `t` object,
// or this panics.
func rootConvey(items ...interface{}) {
	entry := discover(items)

	if entry.Test == nil {
		panic(missingGoTest)
	}

	expectChildRun := true
	ctx := &context{
		name: entry.Situation,

		test:     entry.Test,
		reporter: buildReporter(),

		expectChildRun: &expectChildRun,

		focus:       entry.Focus,
		failureMode: defaultFailureMode.combine(entry.FailMode),
	}
	ctxMgr.SetValues(gls.Values{nodeKey: ctx}, func() {
		ctx.reporter.BeginStory(reporting.NewStoryReport(ctx.test))
		defer ctx.reporter.EndStory()

		for ctx.shouldVisit() {
			ctx.conveyInner(entry.Situation, entry.Func)
			expectChildRun = true
		}
	})
}

//////////////////////////////////// Methods ////////////////////////////////////

func (ctx *context) SkipConvey(items ...interface{}) {
	ctx.Convey(items, skipConvey)
}

func (ctx *context) FocusConvey(items ...interface{}) {
	ctx.Convey(items, focusConvey)
}

func (ctx *context) Convey(items ...interface{}) {
	entry := discover(items)

	// we're a branch, or leaf (on the wind)
	if entry.Test != nil {
		panic(extraGoTest)
	}
	if ctx.focus && !entry.Focus {
		return
	}

	var inner_ctx *context
	if ctx.executedOnce {
		if ctx.curIdx >= len(ctx.children) {
			panic("different set of Convey statements on subsequent pass!")
		}
		inner_ctx = ctx.children[ctx.curIdx]
		if inner_ctx.name != entry.Situation {
			panic("different set of Convey statements on subsequent pass!")
		}
		ctx.curIdx++
	} else {
		inner_ctx = &context{
			name:     entry.Situation,
			test:     ctx.test,
			reporter: ctx.reporter,

			expectChildRun: ctx.expectChildRun,

			focus:       entry.Focus,
			failureMode: ctx.failureMode.combine(entry.FailMode),
		}
		ctx.children = append(ctx.children, inner_ctx)
	}

	if inner_ctx.shouldVisit() {
		ctxMgr.SetValues(gls.Values{nodeKey: inner_ctx}, func() {
			inner_ctx.conveyInner(entry.Situation, entry.Func)
		})
	}
}

func (ctx *context) SkipSo(stuff ...interface{}) {
	ctx.assertionReport(reporting.NewSkipReport())
}

func (ctx *context) So(actual interface{}, assert assertion, expected ...interface{}) {
	if result := assert(actual, expected...); result == assertionSuccess {
		ctx.assertionReport(reporting.NewSuccessReport())
	} else {
		ctx.assertionReport(reporting.NewFailureReport(result))
	}
}

func (ctx *context) Reset(action func()) {
	/* TODO: Failure mode configuration */
	ctx.resets = append(ctx.resets, action)
}

func (ctx *context) Print(items ...interface{}) (int, error) {
	fmt.Fprint(ctx.reporter, items...)
	return fmt.Print(items...)
}

func (ctx *context) Println(items ...interface{}) (int, error) {
	fmt.Fprintln(ctx.reporter, items...)
	return fmt.Println(items...)
}

func (ctx *context) Printf(format string, items ...interface{}) (int, error) {
	fmt.Fprintf(ctx.reporter, format, items...)
	return fmt.Printf(format, items...)
}

//////////////////////////////////// Private ////////////////////////////////////

// shouldVisit returns true iff we should traverse down into a Convey. Note
// that just because we don't traverse a Convey this time, doesn't mean that
// we may not traverse it on a subsequent pass.
func (c *context) shouldVisit() bool {
	return !c.complete && *c.expectChildRun
}

// conveyInner is the function which actually executes the user's anonymous test
// function body. At this point, Convey or RootConvey has decided that this
// function should actually run.
func (ctx *context) conveyInner(situation string, f func(C)) {
	// Record/Reset state for next time.
	defer func() {
		ctx.executedOnce = true
		ctx.curIdx = 0

		// This is only needed at the leaves, but there's no harm in also setting it
		// when returning from branch Convey's
		*ctx.expectChildRun = false
	}()

	// Set up+tear down our scope for the reporter
	fname := runtime.FuncForPC(reflect.ValueOf(f).Pointer()).Name()
	ctx.reporter.Enter(reporting.NewScopeReport(situation, fname))
	defer ctx.reporter.Exit()

	// Resets are registered as the `f` function executes, so nil them here.
	// All resets are run in registration order (FIFO).
	// TODO(riannucci): Should these be run in LIFO?
	ctx.resets = []func(){}
	defer func() {
		for _, r := range ctx.resets {
			// TODO(riannucci): handle panics?
			r()
		}
	}()

	// Recover from any panics in f, and assign the `complete` status for this
	// node of the tree.
	defer func() {
		if problem := recover(); problem != nil {
			if strings.HasPrefix(fmt.Sprintf("%v", problem), extraGoTest) {
				panic(problem)
			}
			if problem != failureHalt {
				ctx.reporter.Report(reporting.NewErrorReport(problem))
			}
			ctx.complete = true
		} else {
			ctx.complete = true
			for _, child := range ctx.children {
				if !child.complete {
					ctx.complete = false
					return
				}
			}
		}
	}()

	if f == nil {
		// if f is nil, this was either a Convey(..., nil), or a SkipConvey
		ctx.reporter.Report(reporting.NewSkipReport())
	} else {
		f(ctx)
	}
}

// assertionReport is a helper for So and SkipSo which makes the report and
// then possibly panics, depending on the current context's failureMode.
func (ctx *context) assertionReport(r *reporting.AssertionResult) {
	ctx.reporter.Report(r)
	if r.Failure != "" && ctx.failureMode == FailureHalts {
		panic(failureHalt)
	}
}
<commit_msg>Fix import ordering<commit_after>// Oh the stack trace scanning!
// The density of comments in this file is evidence that
// the code doesn't exactly explain itself. Tread with care...
package convey

import (
	"fmt"
	"reflect"
	"runtime"
	"strings"

	"github.com/jtolds/gls"
	"github.com/smartystreets/goconvey/convey/reporting"
)

const (
	missingGoTest = `Top-level calls to Convey(...) need a reference to the *testing.T.
		Hint: Convey("description here", t, func() { /* notice that the second argument was the *testing.T (t)! */ }) `
	extraGoTest = `Only the top-level call to Convey(...) needs a reference to the *testing.T.`

	failureHalt = "___FAILURE_HALT___"

	nodeKey = "node"
)

///////////////////////////////// Stack Context /////////////////////////////////

func getCurrentContext() *context {
	ctx, ok := ctxMgr.GetValue(nodeKey)
	if ok {
		return ctx.(*context)
	}
	return nil
}

func mustGetCurrentContext() *context {
	ctx := getCurrentContext()
	if ctx == nil {
		panic("cannot perform operation outside of a convey context")
	}
	return ctx
}

//////////////////////////////////// Context ////////////////////////////////////

// context magically handles all coordination of Convey's and So assertions.
//
// It is tracked on the stack as goroutine-local-storage with the gls package,
// or explicitly if the user decides to call convey like:
//
//   Convey(..., func(c C) {
//     c.So(...)
//   })
//
// This implements the `C` interface.
type context struct {
	reporter reporting.Reporter
	test     t

	name string

	curIdx   int
	children []*context

	resets []func()

	executedOnce   bool
	expectChildRun *bool
	complete       bool

	focus       bool
	failureMode FailureMode
}

// rootConvey is the main entry point to a test suite. This is called when
// there's no context in the stack already, and items must contain a `t` object,
// or this panics.
func rootConvey(items ...interface{}) {
	entry := discover(items)

	if entry.Test == nil {
		panic(missingGoTest)
	}

	expectChildRun := true
	ctx := &context{
		name: entry.Situation,

		test:     entry.Test,
		reporter: buildReporter(),

		expectChildRun: &expectChildRun,

		focus:       entry.Focus,
		failureMode: defaultFailureMode.combine(entry.FailMode),
	}
	ctxMgr.SetValues(gls.Values{nodeKey: ctx}, func() {
		ctx.reporter.BeginStory(reporting.NewStoryReport(ctx.test))
		defer ctx.reporter.EndStory()

		for ctx.shouldVisit() {
			ctx.conveyInner(entry.Situation, entry.Func)
			expectChildRun = true
		}
	})
}

//////////////////////////////////// Methods ////////////////////////////////////

func (ctx *context) SkipConvey(items ...interface{}) {
	ctx.Convey(items, skipConvey)
}

func (ctx *context) FocusConvey(items ...interface{}) {
	ctx.Convey(items, focusConvey)
}

func (ctx *context) Convey(items ...interface{}) {
	entry := discover(items)

	// we're a branch, or leaf (on the wind)
	if entry.Test != nil {
		panic(extraGoTest)
	}
	if ctx.focus && !entry.Focus {
		return
	}

	var inner_ctx *context
	if ctx.executedOnce {
		if ctx.curIdx >= len(ctx.children) {
			panic("different set of Convey statements on subsequent pass!")
		}
		inner_ctx = ctx.children[ctx.curIdx]
		if inner_ctx.name != entry.Situation {
			panic("different set of Convey statements on subsequent pass!")
		}
		ctx.curIdx++
	} else {
		inner_ctx = &context{
			name:     entry.Situation,
			test:     ctx.test,
			reporter: ctx.reporter,

			expectChildRun: ctx.expectChildRun,

			focus:       entry.Focus,
			failureMode: ctx.failureMode.combine(entry.FailMode),
		}
		ctx.children = append(ctx.children, inner_ctx)
	}

	if inner_ctx.shouldVisit() {
		ctxMgr.SetValues(gls.Values{nodeKey: inner_ctx}, func() {
			inner_ctx.conveyInner(entry.Situation, entry.Func)
		})
	}
}

func (ctx *context) SkipSo(stuff ...interface{}) {
	ctx.assertionReport(reporting.NewSkipReport())
}

func (ctx *context) So(actual interface{}, assert assertion, expected ...interface{}) {
	if result := assert(actual, expected...); result == assertionSuccess {
		ctx.assertionReport(reporting.NewSuccessReport())
	} else {
		ctx.assertionReport(reporting.NewFailureReport(result))
	}
}

func (ctx *context) Reset(action func()) {
	/* TODO: Failure mode configuration */
	ctx.resets = append(ctx.resets, action)
}

func (ctx *context) Print(items ...interface{}) (int, error) {
	fmt.Fprint(ctx.reporter, items...)
	return fmt.Print(items...)
}

func (ctx *context) Println(items ...interface{}) (int, error) {
	fmt.Fprintln(ctx.reporter, items...)
	return fmt.Println(items...)
}

func (ctx *context) Printf(format string, items ...interface{}) (int, error) {
	fmt.Fprintf(ctx.reporter, format, items...)
	return fmt.Printf(format, items...)
}

//////////////////////////////////// Private ////////////////////////////////////

// shouldVisit returns true iff we should traverse down into a Convey. Note
// that just because we don't traverse a Convey this time, doesn't mean that
// we may not traverse it on a subsequent pass.
func (c *context) shouldVisit() bool {
	return !c.complete && *c.expectChildRun
}

// conveyInner is the function which actually executes the user's anonymous test
// function body. At this point, Convey or RootConvey has decided that this
// function should actually run.
func (ctx *context) conveyInner(situation string, f func(C)) {
	// Record/Reset state for next time.
	defer func() {
		ctx.executedOnce = true
		ctx.curIdx = 0

		// This is only needed at the leaves, but there's no harm in also setting it
		// when returning from branch Convey's
		*ctx.expectChildRun = false
	}()

	// Set up+tear down our scope for the reporter
	fname := runtime.FuncForPC(reflect.ValueOf(f).Pointer()).Name()
	ctx.reporter.Enter(reporting.NewScopeReport(situation, fname))
	defer ctx.reporter.Exit()

	// Resets are registered as the `f` function executes, so nil them here.
	// All resets are run in registration order (FIFO).
	// TODO(riannucci): Should these be run in LIFO?
	ctx.resets = []func(){}
	defer func() {
		for _, r := range ctx.resets {
			// TODO(riannucci): handle panics?
			r()
		}
	}()

	// Recover from any panics in f, and assign the `complete` status for this
	// node of the tree.
	defer func() {
		if problem := recover(); problem != nil {
			if strings.HasPrefix(fmt.Sprintf("%v", problem), extraGoTest) {
				panic(problem)
			}
			if problem != failureHalt {
				ctx.reporter.Report(reporting.NewErrorReport(problem))
			}
			ctx.complete = true
		} else {
			ctx.complete = true
			for _, child := range ctx.children {
				if !child.complete {
					ctx.complete = false
					return
				}
			}
		}
	}()

	if f == nil {
		// if f is nil, this was either a Convey(..., nil), or a SkipConvey
		ctx.reporter.Report(reporting.NewSkipReport())
	} else {
		f(ctx)
	}
}

// assertionReport is a helper for So and SkipSo which makes the report and
// then possibly panics, depending on the current context's failureMode.
func (ctx *context) assertionReport(r *reporting.AssertionResult) {
	ctx.reporter.Report(r)
	if r.Failure != "" && ctx.failureMode == FailureHalts {
		panic(failureHalt)
	}
}
<|endoftext|>


<commit_before>// +build real

package carbonserver

import (
	"flag"
	"io/ioutil"
	"log"
	"reflect"
	"sort"
	"strings"
	"sync"
	"testing"
	"time"

	"github.com/OneOfOne/go-utils/memory"
)

var testDataPath = flag.String("testdata", "files.txt", "path to test file")
var checkMemory = flag.Bool("memory-size", false, "show index memory size")
var targetQueryPath = flag.String("query-data", "queries.txt", "queries for testing")
var carbonPath = flag.String("carbon", "/var/lib/carbon/whisper", "carbon data path")
var noTrigram = flag.Bool("no-trigram", false, "disable trigram search")
var pureTrie = flag.Bool("no-trie-with-trigram", false, "enable trigram in trie")
var localTest = flag.Bool("local-test", false, "trim prefix and replace / with . on trigram result")

func readFile(path string) []string {
	data, err := ioutil.ReadFile(path)
	if err != nil {
		panic(err)
	}

	// if strings.HasSuffix(path, ".tar.gz") || strings.HasSuffix(path, ".tgz") {
	// 	tar.NewReader()
	// }

	lines := strings.Split(string(data), "\n")
	if lines[len(lines)-1] == "" {
		return lines[:len(lines)-1]
	}
	return lines
}

// TestTrieGlobRealData is built for benchmarking against real data, for two reasons:
// 		* trigram index solution invokes stat syscall
// 		* can't share production data in open source project
func TestTrieGlobRealData(t *testing.T) {
	files := readFile(*testDataPath)
	var trieServer, trigramServer *CarbonserverListener

	var wg sync.WaitGroup
	wg.Add(1)
	go func() {
		trieServer = newTrieServer(files, !*pureTrie)
		trieServer.whisperData = *carbonPath

		if *checkMemory {
			start := time.Now()
			t.Logf("memory.Sizeof(btrieServer)\t\t= %+v took %s\n", memory.Sizeof(trieServer), time.Now().Sub(start))
		}
		wg.Done()
	}()

	if !*noTrigram {
		wg.Add(1)
		go func() {
			trigramServer = newTrigramServer(files)
			trigramServer.whisperData = *carbonPath

			if *checkMemory {
				start := time.Now()
				t.Logf("memory.Sizeof(btrigramServer)\t\t= %+v took %s\n", memory.Sizeof(trigramServer), time.Now().Sub(start))
			}
			wg.Done()
		}()
	}
	wg.Wait()

	queries := readFile(*targetQueryPath)
	for _, query := range queries {
		t.Run(query, func(t *testing.T) {
			defer func() {
				if r := recover(); r != nil {
					t.Errorf("%q: %s", query, r)
				}
			}()
			start1 := time.Now()
			trieFiles, trieLeafs, err := trieServer.expandGlobsTrie(query)
			if err != nil {
				t.Errorf("trie search errro: %s", err)
			}
			trieTime := time.Now().Sub(start1)
			t.Logf("trie took %s", trieTime)

			// for i, str := range trieFiles {
			// 	log.Printf("%d: %s\n", i, str)
			// }
			log.Printf("len(trieFiles) = %+v\n", len(trieFiles))

			if *noTrigram {
				return
			}

			start2 := time.Now()
			trigramFiles, trigramLeafs, err := trigramServer.expandGlobs(query)
			if err != nil {
				t.Errorf("trigram search errro: %s", err)
			}
			trigramTime := time.Now().Sub(start2)
			t.Logf("trigram took %s", trigramTime)

			if *localTest {
				for i := range trigramFiles {
					trigramFiles[i] = strings.TrimPrefix(trigramFiles[i], *carbonPath+"/")
					trigramFiles[i] = strings.TrimSuffix(trigramFiles[i], ".wsp")
					trigramFiles[i] = strings.Replace(trigramFiles[i], "/", ".", -1)
				}
			}

			if trieTime < trigramTime {
				t.Logf("trie is %f times faster", float64(trigramTime)/float64(trieTime))
			} else {
				t.Logf("trie is %f times slower", float64(trieTime)/float64(trigramTime))
			}

			t.Logf("trie %d trigram %d", len(trieFiles), len(trigramFiles))

			sortFilesLeafs(trieFiles, trieLeafs)
			sortFilesLeafs(trigramFiles, trigramLeafs)

			if !reflect.DeepEqual(trieFiles, trigramFiles) {
				t.Errorf("files diff\n\ttrie:    %v\n\ttrigram: %v\n", trieFiles, trigramFiles)
			}
			if trieLeafs == nil {
				trieLeafs = []bool{}
			}
			if !reflect.DeepEqual(trieLeafs, trigramLeafs[:]) {
				t.Errorf("leafs diff\n\ttrie:    %#v\n\ttrigram: %#v\n", trieLeafs, trigramLeafs)
			}
		})
	}
}

func sortFilesLeafs(files []string, leafs []bool) {
	oldIndex := map[string]int{}
	for i, f := range files {
		oldIndex[f] = i
	}
	sort.Strings(files)
	oldLeafs := make([]bool, len(leafs))
	copy(leafs, oldLeafs)
	for i, f := range files {
		leafs[i] = oldLeafs[oldIndex[f]]
	}
}
<commit_msg>trie: fix typos<commit_after>// +build real

package carbonserver

import (
	"flag"
	"io/ioutil"
	"log"
	"reflect"
	"sort"
	"strings"
	"sync"
	"testing"
	"time"

	"github.com/OneOfOne/go-utils/memory"
)

var testDataPath = flag.String("testdata", "files.txt", "path to test file")
var checkMemory = flag.Bool("memory-size", false, "show index memory size")
var targetQueryPath = flag.String("query-data", "queries.txt", "queries for testing")
var carbonPath = flag.String("carbon", "/var/lib/carbon/whisper", "carbon data path")
var noTrigram = flag.Bool("no-trigram", false, "disable trigram search")
var pureTrie = flag.Bool("no-trie-with-trigram", false, "enable trigram in trie")
var localTest = flag.Bool("local-test", false, "trim prefix and replace / with . on trigram result")

func readFile(path string) []string {
	data, err := ioutil.ReadFile(path)
	if err != nil {
		panic(err)
	}

	// if strings.HasSuffix(path, ".tar.gz") || strings.HasSuffix(path, ".tgz") {
	// 	tar.NewReader()
	// }

	lines := strings.Split(string(data), "\n")
	if lines[len(lines)-1] == "" {
		return lines[:len(lines)-1]
	}
	return lines
}

// TestTrieGlobRealData is built for benchmarking against real data, for two reasons:
// 		* trigram index solution invokes stat syscall
// 		* can't share production data in open source project
func TestTrieGlobRealData(t *testing.T) {
	files := readFile(*testDataPath)
	var trieServer, trigramServer *CarbonserverListener

	var wg sync.WaitGroup
	wg.Add(1)
	go func() {
		trieServer = newTrieServer(files, !*pureTrie)
		trieServer.whisperData = *carbonPath

		if *checkMemory {
			start := time.Now()
			t.Logf("memory.Sizeof(btrieServer)\t\t= %+v took %s\n", memory.Sizeof(trieServer), time.Now().Sub(start))
		}
		wg.Done()
	}()

	if !*noTrigram {
		wg.Add(1)
		go func() {
			trigramServer = newTrigramServer(files)
			trigramServer.whisperData = *carbonPath

			if *checkMemory {
				start := time.Now()
				t.Logf("memory.Sizeof(btrigramServer)\t\t= %+v took %s\n", memory.Sizeof(trigramServer), time.Now().Sub(start))
			}
			wg.Done()
		}()
	}
	wg.Wait()

	queries := readFile(*targetQueryPath)
	for _, query := range queries {
		t.Run(query, func(t *testing.T) {
			defer func() {
				if r := recover(); r != nil {
					t.Errorf("%q: %s", query, r)
				}
			}()
			start1 := time.Now()
			trieFiles, trieLeafs, err := trieServer.expandGlobsTrie(query)
			if err != nil {
				t.Errorf("trie search error: %s", err)
			}
			trieTime := time.Now().Sub(start1)
			t.Logf("trie took %s", trieTime)

			// for i, str := range trieFiles {
			// 	log.Printf("%d: %s\n", i, str)
			// }
			log.Printf("len(trieFiles) = %+v\n", len(trieFiles))

			if *noTrigram {
				return
			}

			start2 := time.Now()
			trigramFiles, trigramLeafs, err := trigramServer.expandGlobs(query)
			if err != nil {
				t.Errorf("trigram search error: %s", err)
			}
			trigramTime := time.Now().Sub(start2)
			t.Logf("trigram took %s", trigramTime)

			if *localTest {
				for i := range trigramFiles {
					trigramFiles[i] = strings.TrimPrefix(trigramFiles[i], *carbonPath+"/")
					trigramFiles[i] = strings.TrimSuffix(trigramFiles[i], ".wsp")
					trigramFiles[i] = strings.Replace(trigramFiles[i], "/", ".", -1)
				}
			}

			if trieTime < trigramTime {
				t.Logf("trie is %f times faster", float64(trigramTime)/float64(trieTime))
			} else {
				t.Logf("trie is %f times slower", float64(trieTime)/float64(trigramTime))
			}

			t.Logf("trie %d trigram %d", len(trieFiles), len(trigramFiles))

			sortFilesLeafs(trieFiles, trieLeafs)
			sortFilesLeafs(trigramFiles, trigramLeafs)

			if !reflect.DeepEqual(trieFiles, trigramFiles) {
				t.Errorf("files diff\n\ttrie:    %v\n\ttrigram: %v\n", trieFiles, trigramFiles)
			}
			if trieLeafs == nil {
				trieLeafs = []bool{}
			}
			if !reflect.DeepEqual(trieLeafs, trigramLeafs[:]) {
				t.Errorf("leafs diff\n\ttrie:    %#v\n\ttrigram: %#v\n", trieLeafs, trigramLeafs)
			}
		})
	}
}

func sortFilesLeafs(files []string, leafs []bool) {
	oldIndex := map[string]int{}
	for i, f := range files {
		oldIndex[f] = i
	}
	sort.Strings(files)
	oldLeafs := make([]bool, len(leafs))
	copy(leafs, oldLeafs)
	for i, f := range files {
		leafs[i] = oldLeafs[oldIndex[f]]
	}
}
<|endoftext|>


<commit_before>package gcs_test

import (
	"context"
	"os"
	"testing"

	"github.com/kopia/repo/internal/storagetesting"

	"github.com/kopia/repo/storage"
	"github.com/kopia/repo/storage/gcs"
)

func TestGCSStorage(t *testing.T) {
	bucket := os.Getenv("KOPIA_GCS_TEST_BUCKET")
	if bucket == "" {
		t.Skip("KOPIA_GCS_TEST_BUCKET not provided")
	}

	ctx := context.Background()
	st, err := gcs.New(ctx, &gcs.Options{
		BucketName:                bucket,
		ServiceAccountCredentials: os.Getenv("KOPIA_GCS_CREDENTIALS_FILE"),
	})

	if err != nil {
		t.Fatalf("unable to connect to GCS")
	}

	if err := st.ListBlocks(ctx, "", func(bm storage.BlockMetadata) error {
		return st.DeleteBlock(ctx, bm.BlockID)
	}); err != nil {
		t.Fatalf("unable to clear GCS bucket: %v", err)
	}

	storagetesting.VerifyStorage(ctx, t, st)
}
<commit_msg>storage/gcs: additional logging for gcs test<commit_after>package gcs_test

import (
	"context"
	"os"
	"testing"

	"github.com/kopia/repo/internal/storagetesting"

	"github.com/kopia/repo/storage"
	"github.com/kopia/repo/storage/gcs"
)

func TestGCSStorage(t *testing.T) {
	bucket := os.Getenv("KOPIA_GCS_TEST_BUCKET")
	if bucket == "" {
		t.Skip("KOPIA_GCS_TEST_BUCKET not provided")
	}

	ctx := context.Background()
	st, err := gcs.New(ctx, &gcs.Options{
		BucketName:                bucket,
		ServiceAccountCredentials: os.Getenv("KOPIA_GCS_CREDENTIALS_FILE"),
	})

	if err != nil {
		t.Fatalf("unable to connect to GCS: %v", err)
	}

	if err := st.ListBlocks(ctx, "", func(bm storage.BlockMetadata) error {
		return st.DeleteBlock(ctx, bm.BlockID)
	}); err != nil {
		t.Fatalf("unable to clear GCS bucket: %v", err)
	}

	storagetesting.VerifyStorage(ctx, t, st)
}
<|endoftext|>


<commit_before>/*
 * // Copyright (c) 2018 Cisco and/or its affiliates.
 * //
 * // Licensed under the Apache License, Version 2.0 (the "License");
 * // you may not use this file except in compliance with the License.
 * // You may obtain a copy of the License at:
 * //
 * //     http://www.apache.org/licenses/LICENSE-2.0
 * //
 * // Unless required by applicable law or agreed to in writing, software
 * // distributed under the License is distributed on an "AS IS" BASIS,
 * // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * // See the License for the specific language governing permissions and
 * // limitations under the License.
 */

package renderer

import (
	"fmt"
	"net"

	svcmodel "github.com/contiv/vpp/plugins/ksr/model/service"
)

// ServiceRendererAPI defines the API of Service Renderer.
//
// Service Renderer is a pluggable component of the service plugin, sitting below
// the processor and implementing rendering of ContivService instances into
// the configuration of the target network stack. The idea is to allow to have
// alternative and/or complementary Kubernetes service implementations. For example,
// IPv4 and IPv6 protocols could be handled separately by two different renderers.
// Similarly, the set of supported vswitches in the data plane can be extended
// simply by adding new renderers for services and policies. Another use-case is
// to provide alternative implementation of Kubernetes services for the same
// stack - one being the default, others possibly experimental or tailor-made
// for specific applications. The set of renderers to be activated can be
// configurable or determined from the environment. Every active renderer is
// given the same set of data from the processor.
//
// ServiceRendererAPI is the interface that connects processor's southbound
// with the renderer's northbound. For a single Kubernetes service, all relevant
// configuration and state data are wrapped into a single instance of ContivService.
// The renderer learns the set of external IPs and ports on which the service
// should be exposed together with corresponding sets of backends (= endpoints).
// AddService(), DeleteService(), UpdateService() are called every time a service
// is created, removed or its state/configuration data has changed, respectively.
// ContivService is referenced by ID unique across all namespaces.
//
// The processor also monitors the set of all node IPs in the cluster. Every time
// a new IP is assigned to any node inside the cluster or an existing node has been
// deleted, UpdateNodePortServices() is called with the latest list of all node
// IPs together with a set of NodePort services.
//
// Additionally, the processor distinguishes interfaces connecting VPP with
// potential service clients - denoted as Frontends - from those connecting
// vswitch with service endpoints - denoted as Backends. Interface can be both
// Frontend and Backend. The sets of Frontend and Backend interfaces are refreshed
// by the processor every time a pod is created, destroyed, assigned to a service
// for the first time, or no longer acting as a service endpoint. For Renderer
// this classification of interfaces may or may not be useful. It is up to the
// renderer to either leverage or completely ignore events UpdateLocalFrontendIfs()
// and UpdateLocalBackendIfs(). Please note that the sets of Frontend & Backend
// interfaces are specific to the VPP-based networking and not relevant for
// a different vswitch / network stack.
//
// Resync() is used to pass the current snapshot of all data provided
// by the processor. Upon receipt, the renderer is supposed to make sure that
// the renderered configuration matches the state of Kubernetes services and
// to resolve any discrepancies. Resync() is always called on the agent startup,
// but may also be triggered during the runtime - in case a potential data loss
// between the agent and the data store or the vswitch has been detected.
//
// To integrate (VPP-specific) renderer with the ACL-based policies
// (plugins/policy/renderer/acl), it is required to perform the service address
// translation for both directions in-between the VPP nodes: `acl-plugin-in-ip4-fa`
// and `acl-plugin-out-ip4-fa` (i.e. after ingress ACLs, but before egress ACLs).
type ServiceRendererAPI interface {
	// AddService is called for a newly added service.
	AddService(service *ContivService) error

	// UpdateService informs renderer about a change in the configuration
	// or in the state of a service.
	UpdateService(oldService, newService *ContivService) error

	// DeleteService is called for every removed service.
	DeleteService(service *ContivService) error

	// UpdateNodePortServices is called whenever the set of node IPs in the cluster
	// changes.
	UpdateNodePortServices(nodeIPs *IPAddresses, npServices []*ContivService) error

	// UpdateLocalFrontendIfs gives an update about a changed set of Frontend
	// interfaces (VPP specific).
	UpdateLocalFrontendIfs(oldIfNames, newIfNames Interfaces) error

	// UpdateLocalBackendIfs gives an updated about a changed set of backend
	// interfaces (VPP specific).
	UpdateLocalBackendIfs(oldIfNames, newIfNames Interfaces) error

	// Resync provides a complete snapshot of all service-related data.
	// The render should resolve any discrepancies between the state of K8s
	// services and the currently rendered configuration.
	Resync(resyncEv *ResyncEventData) error
}

// ContivService is a less-abstract, free of indirect references representation
// of K8s Service.
// It has:
//   - endpoints combined with services
//   - the full list of IP addresses on which the service should be exposed
//     on this node
type ContivService struct {
	// ID uniquely identifies service across all namespaces.
	ID svcmodel.ID

	// TrafficPolicy decides if traffic is routed cluster-wide or node-local only.
	TrafficPolicy TrafficPolicyType

	// ExternalIPs is a set of all IP addresses on which the service
	// should be exposed on this node (aside from node IPs for NodePorts, which
	// are provided separately via the ServiceRendererAPI.UpdateNodePortServices()
	// method).
	ExternalIPs *IPAddresses

	// Ports is a map of all ports exposed for this service.
	Ports map[string] /* service port name */ *ServicePort

	// Backends map external service ports with corresponding backends (= endpoints).
	Backends map[string] /*service port name */ []*ServiceBackend
}

// TrafficPolicyType is either Cluster-wide routing or Node-local only routing.
type TrafficPolicyType int

const (
	// ClusterWide allows to load-balance traffic across all backends.
	ClusterWide TrafficPolicyType = 0

	// NodeLocal allows to load-balance traffic only across node-local backends.
	NodeLocal TrafficPolicyType = 1
)

// NewContivService is a constructor for ContivService.
func NewContivService() *ContivService {
	return &ContivService{
		ExternalIPs: NewIPAddresses(),
		Ports:       make(map[string]*ServicePort),
		Backends:    make(map[string][]*ServiceBackend),
	}
}

// String converts ContivService into a human-readable string.
func (cs ContivService) String() string {
	externalIPs := ""
	for idx, ip := range cs.ExternalIPs.list {
		externalIPs += ip.String()
		if idx < len(cs.ExternalIPs.list)-1 {
			externalIPs += ", "
		}
	}
	allBackends := ""
	idx := 0
	for port, svcBackends := range cs.Backends {
		backends := ""
		for idx2, svcBackend := range svcBackends {
			backends += svcBackend.String()
			if idx2 < len(svcBackends)-1 {
				backends += ", "
			}
		}
		allBackends += fmt.Sprintf("%s->[%s]", cs.Ports[port].String(), backends)
		if idx < len(cs.Backends)-1 {
			allBackends += ", "
		}
		idx++
	}
	return fmt.Sprintf("ContivService %s <Traffic-Policy:%s ExternalIPs:[%s] Backends:{%s}>",
		cs.ID.String(), cs.TrafficPolicy.String(), externalIPs, allBackends)
}

// String converts TrafficPolicyType into a human-readable string.
func (tpt TrafficPolicyType) String() string {
	switch tpt {
	case ClusterWide:
		return "cluster-wide"
	case NodeLocal:
		return "node-local"
	}
	return "INVALID"
}

// HasNodePort returns true if service is also exposed on the Node IP.
func (cs ContivService) HasNodePort() bool {
	for _, port := range cs.Ports {
		if port.NodePort != 0 {
			return true
		}
	}
	return false
}

// ServicePort contains information on service's port.
type ServicePort struct {
	Protocol ProtocolType /* protocol type */
	Port     uint16       /* port that will be exposed by this service */
	NodePort uint16       /* port on which this service is exposed for Node IP (0 if none) */
}

// String converts ServicePort into a human-readable string.
func (sp ServicePort) String() string {
	if sp.NodePort == 0 {
		return fmt.Sprintf("%d/%s", sp.Port, sp.Protocol.String())
	}
	return fmt.Sprintf("%d:%d/%s", sp.Port, sp.NodePort, sp.Protocol.String())
}

// ProtocolType is either TCP or UDP.
type ProtocolType int

const (
	// TCP protocol.
	TCP ProtocolType = 6

	// UDP protocol.
	UDP ProtocolType = 17
)

// String converts ProtocolType into a human-readable string.
func (pt ProtocolType) String() string {
	switch pt {
	case TCP:
		return "TCP"
	case UDP:
		return "UDP"
	}
	return "INVALID"
}

// ServiceBackend represents a single service backend (= endpoint).
type ServiceBackend struct {
	IP          net.IP /* internal IP address of the backend */
	Port        uint16 /* backend-local port on which the service listens */
	Local       bool   /* true if the backend is deployed on this node (can be leveraged for smart load-balancing) */
	HostNetwork bool   /* true if the backend uses host networking */
}

// String converts Backend into a human-readable string.
func (sb ServiceBackend) String() string {
	return fmt.Sprintf("<IP:%s Port:%d, Local:%t>", sb.IP, sb.Port, sb.Local)
}

// IPAddresses is a set of IP addresses.
type IPAddresses struct {
	list []net.IP
}

// NewIPAddresses is a constructor for IPAddresses.
func NewIPAddresses(addrs ...net.IP) *IPAddresses {
	ipAddresses := &IPAddresses{
		list: []net.IP{},
	}
	for _, addr := range addrs {
		ipAddresses.list = append(ipAddresses.list, addr)
	}
	return ipAddresses
}

// List returns the set as a slice which can be iterated through.
func (addrs *IPAddresses) List() []net.IP {
	return addrs.list
}

// Add IP address into the set.
func (addrs *IPAddresses) Add(addr net.IP) {
	if !addrs.Has(addr) {
		addrs.list = append(addrs.list, addr)
	}
}

// Del IP address from the set.
func (addrs *IPAddresses) Del(addr net.IP) {
	newAddrs := []net.IP{}
	for _, addr2 := range addrs.list {
		if !addr2.Equal(addr) {
			newAddrs = append(newAddrs, addr2)
		}
	}
	addrs.list = newAddrs
}

// Copy creates a deep copy of the set.
func (addrs *IPAddresses) Copy() *IPAddresses {
	addrsCopy := NewIPAddresses()
	for _, addr := range addrs.list {
		addrsCopy.list = append(addrsCopy.list, addr)
	}
	return addrsCopy
}

// Has returns true if the given IP address is in the set.
func (addrs *IPAddresses) Has(addr net.IP) bool {
	for _, addr2 := range addrs.list {
		if addr2.Equal(addr) {
			return true
		}
	}
	return false
}

// String converts a set of IP addresses into a human-readable string.
func (addrs IPAddresses) String() string {
	str := "{"
	for idx, addr := range addrs.list {
		str += addr.String()
		if idx < len(addrs.list)-1 {
			str += ", "
		}
	}
	str += "}"
	return str
}

// Interfaces is a set of interface names.
type Interfaces map[string]struct{}

// NewInterfaces is a constructor for Interfaces.
func NewInterfaces(ifNames ...string) Interfaces {
	interfaces := make(Interfaces)
	for _, ifName := range ifNames {
		interfaces.Add(ifName)
	}
	return interfaces
}

// Add interface name into the set.
func (ifs Interfaces) Add(ifName string) {
	ifs[ifName] = struct{}{}
}

// Del interface name from the set.
func (ifs Interfaces) Del(ifName string) {
	if ifs.Has(ifName) {
		delete(ifs, ifName)
	}
}

// Copy creates a deep copy of the set.
func (ifs Interfaces) Copy() Interfaces {
	ifsCopy := NewInterfaces()
	for intf := range ifs {
		ifsCopy.Add(intf)
	}
	return ifsCopy
}

// Has returns true if the given interface name is in the set.
func (ifs Interfaces) Has(ifName string) bool {
	_, has := ifs[ifName]
	return has
}

// String converts a set of interface names into a human-readable string.
func (ifs Interfaces) String() string {
	str := "{"
	idx := 0
	for ifName := range ifs {
		str += ifName
		if idx < len(ifs)-1 {
			str += ", "
		}
		idx++
	}
	str += "}"
	return str
}

// ResyncEventData wraps an entire state of K8s services as provided by the Processor.
type ResyncEventData struct {
	// NodeIPs is a list of IP addresses of all nodes in the cluster.
	NodeIPs *IPAddresses

	// Services is a list of all currently deployed services.
	Services []*ContivService

	// FrontendIfs is a set of all interfaces connecting clients with VPP
	// (VPP specific).
	FrontendIfs Interfaces

	// BackendIfs is a set of all interfaces connecting service backends with VPP
	// (VPP specific).
	BackendIfs Interfaces
}

// NewResyncEventData is a constructor for ResyncEventData.
func NewResyncEventData() *ResyncEventData {
	return &ResyncEventData{
		NodeIPs:     NewIPAddresses(),
		Services:    []*ContivService{},
		FrontendIfs: NewInterfaces(),
		BackendIfs:  NewInterfaces(),
	}
}

// String converts ResyncEventData into a human-readable string.
func (red ResyncEventData) String() string {
	services := ""
	for idx, service := range red.Services {
		services += service.String()
		if idx < len(red.Services)-1 {
			services += ", "
		}
	}
	return fmt.Sprintf("ResyncEventData <NodeIPs:[%s] Services:[%s], FrontendIfs:%s BackendIfs:%s>",
		red.NodeIPs.String(), services, red.FrontendIfs.String(), red.BackendIfs.String())
}
<commit_msg>Fix typo<commit_after>/*
 * // Copyright (c) 2018 Cisco and/or its affiliates.
 * //
 * // Licensed under the Apache License, Version 2.0 (the "License");
 * // you may not use this file except in compliance with the License.
 * // You may obtain a copy of the License at:
 * //
 * //     http://www.apache.org/licenses/LICENSE-2.0
 * //
 * // Unless required by applicable law or agreed to in writing, software
 * // distributed under the License is distributed on an "AS IS" BASIS,
 * // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * // See the License for the specific language governing permissions and
 * // limitations under the License.
 */

package renderer

import (
	"fmt"
	"net"

	svcmodel "github.com/contiv/vpp/plugins/ksr/model/service"
)

// ServiceRendererAPI defines the API of Service Renderer.
//
// Service Renderer is a pluggable component of the service plugin, sitting below
// the processor and implementing rendering of ContivService instances into
// the configuration of the target network stack. The idea is to allow to have
// alternative and/or complementary Kubernetes service implementations. For example,
// IPv4 and IPv6 protocols could be handled separately by two different renderers.
// Similarly, the set of supported vswitches in the data plane can be extended
// simply by adding new renderers for services and policies. Another use-case is
// to provide alternative implementation of Kubernetes services for the same
// stack - one being the default, others possibly experimental or tailor-made
// for specific applications. The set of renderers to be activated can be
// configurable or determined from the environment. Every active renderer is
// given the same set of data from the processor.
//
// ServiceRendererAPI is the interface that connects processor's southbound
// with the renderer's northbound. For a single Kubernetes service, all relevant
// configuration and state data are wrapped into a single instance of ContivService.
// The renderer learns the set of external IPs and ports on which the service
// should be exposed together with corresponding sets of backends (= endpoints).
// AddService(), DeleteService(), UpdateService() are called every time a service
// is created, removed or its state/configuration data has changed, respectively.
// ContivService is referenced by ID unique across all namespaces.
//
// The processor also monitors the set of all node IPs in the cluster. Every time
// a new IP is assigned to any node inside the cluster or an existing node has been
// deleted, UpdateNodePortServices() is called with the latest list of all node
// IPs together with a set of NodePort services.
//
// Additionally, the processor distinguishes interfaces connecting VPP with
// potential service clients - denoted as Frontends - from those connecting
// vswitch with service endpoints - denoted as Backends. Interface can be both
// Frontend and Backend. The sets of Frontend and Backend interfaces are refreshed
// by the processor every time a pod is created, destroyed, assigned to a service
// for the first time, or no longer acting as a service endpoint. For Renderer
// this classification of interfaces may or may not be useful. It is up to the
// renderer to either leverage or completely ignore events UpdateLocalFrontendIfs()
// and UpdateLocalBackendIfs(). Please note that the sets of Frontend & Backend
// interfaces are specific to the VPP-based networking and not relevant for
// a different vswitch / network stack.
//
// Resync() is used to pass the current snapshot of all data provided
// by the processor. Upon receipt, the renderer is supposed to make sure that
// the rendered configuration matches the state of Kubernetes services and
// to resolve any discrepancies. Resync() is always called on the agent startup,
// but may also be triggered during the runtime - in case a potential data loss
// between the agent and the data store or the vswitch has been detected.
//
// To integrate (VPP-specific) renderer with the ACL-based policies
// (plugins/policy/renderer/acl), it is required to perform the service address
// translation for both directions in-between the VPP nodes: `acl-plugin-in-ip4-fa`
// and `acl-plugin-out-ip4-fa` (i.e. after ingress ACLs, but before egress ACLs).
type ServiceRendererAPI interface {
	// AddService is called for a newly added service.
	AddService(service *ContivService) error

	// UpdateService informs renderer about a change in the configuration
	// or in the state of a service.
	UpdateService(oldService, newService *ContivService) error

	// DeleteService is called for every removed service.
	DeleteService(service *ContivService) error

	// UpdateNodePortServices is called whenever the set of node IPs in the cluster
	// changes.
	UpdateNodePortServices(nodeIPs *IPAddresses, npServices []*ContivService) error

	// UpdateLocalFrontendIfs gives an update about a changed set of Frontend
	// interfaces (VPP specific).
	UpdateLocalFrontendIfs(oldIfNames, newIfNames Interfaces) error

	// UpdateLocalBackendIfs gives an updated about a changed set of backend
	// interfaces (VPP specific).
	UpdateLocalBackendIfs(oldIfNames, newIfNames Interfaces) error

	// Resync provides a complete snapshot of all service-related data.
	// The render should resolve any discrepancies between the state of K8s
	// services and the currently rendered configuration.
	Resync(resyncEv *ResyncEventData) error
}

// ContivService is a less-abstract, free of indirect references representation
// of K8s Service.
// It has:
//   - endpoints combined with services
//   - the full list of IP addresses on which the service should be exposed
//     on this node
type ContivService struct {
	// ID uniquely identifies service across all namespaces.
	ID svcmodel.ID

	// TrafficPolicy decides if traffic is routed cluster-wide or node-local only.
	TrafficPolicy TrafficPolicyType

	// ExternalIPs is a set of all IP addresses on which the service
	// should be exposed on this node (aside from node IPs for NodePorts, which
	// are provided separately via the ServiceRendererAPI.UpdateNodePortServices()
	// method).
	ExternalIPs *IPAddresses

	// Ports is a map of all ports exposed for this service.
	Ports map[string] /* service port name */ *ServicePort

	// Backends map external service ports with corresponding backends (= endpoints).
	Backends map[string] /*service port name */ []*ServiceBackend
}

// TrafficPolicyType is either Cluster-wide routing or Node-local only routing.
type TrafficPolicyType int

const (
	// ClusterWide allows to load-balance traffic across all backends.
	ClusterWide TrafficPolicyType = 0

	// NodeLocal allows to load-balance traffic only across node-local backends.
	NodeLocal TrafficPolicyType = 1
)

// NewContivService is a constructor for ContivService.
func NewContivService() *ContivService {
	return &ContivService{
		ExternalIPs: NewIPAddresses(),
		Ports:       make(map[string]*ServicePort),
		Backends:    make(map[string][]*ServiceBackend),
	}
}

// String converts ContivService into a human-readable string.
func (cs ContivService) String() string {
	externalIPs := ""
	for idx, ip := range cs.ExternalIPs.list {
		externalIPs += ip.String()
		if idx < len(cs.ExternalIPs.list)-1 {
			externalIPs += ", "
		}
	}
	allBackends := ""
	idx := 0
	for port, svcBackends := range cs.Backends {
		backends := ""
		for idx2, svcBackend := range svcBackends {
			backends += svcBackend.String()
			if idx2 < len(svcBackends)-1 {
				backends += ", "
			}
		}
		allBackends += fmt.Sprintf("%s->[%s]", cs.Ports[port].String(), backends)
		if idx < len(cs.Backends)-1 {
			allBackends += ", "
		}
		idx++
	}
	return fmt.Sprintf("ContivService %s <Traffic-Policy:%s ExternalIPs:[%s] Backends:{%s}>",
		cs.ID.String(), cs.TrafficPolicy.String(), externalIPs, allBackends)
}

// String converts TrafficPolicyType into a human-readable string.
func (tpt TrafficPolicyType) String() string {
	switch tpt {
	case ClusterWide:
		return "cluster-wide"
	case NodeLocal:
		return "node-local"
	}
	return "INVALID"
}

// HasNodePort returns true if service is also exposed on the Node IP.
func (cs ContivService) HasNodePort() bool {
	for _, port := range cs.Ports {
		if port.NodePort != 0 {
			return true
		}
	}
	return false
}

// ServicePort contains information on service's port.
type ServicePort struct {
	Protocol ProtocolType /* protocol type */
	Port     uint16       /* port that will be exposed by this service */
	NodePort uint16       /* port on which this service is exposed for Node IP (0 if none) */
}

// String converts ServicePort into a human-readable string.
func (sp ServicePort) String() string {
	if sp.NodePort == 0 {
		return fmt.Sprintf("%d/%s", sp.Port, sp.Protocol.String())
	}
	return fmt.Sprintf("%d:%d/%s", sp.Port, sp.NodePort, sp.Protocol.String())
}

// ProtocolType is either TCP or UDP.
type ProtocolType int

const (
	// TCP protocol.
	TCP ProtocolType = 6

	// UDP protocol.
	UDP ProtocolType = 17
)

// String converts ProtocolType into a human-readable string.
func (pt ProtocolType) String() string {
	switch pt {
	case TCP:
		return "TCP"
	case UDP:
		return "UDP"
	}
	return "INVALID"
}

// ServiceBackend represents a single service backend (= endpoint).
type ServiceBackend struct {
	IP          net.IP /* internal IP address of the backend */
	Port        uint16 /* backend-local port on which the service listens */
	Local       bool   /* true if the backend is deployed on this node (can be leveraged for smart load-balancing) */
	HostNetwork bool   /* true if the backend uses host networking */
}

// String converts Backend into a human-readable string.
func (sb ServiceBackend) String() string {
	return fmt.Sprintf("<IP:%s Port:%d, Local:%t>", sb.IP, sb.Port, sb.Local)
}

// IPAddresses is a set of IP addresses.
type IPAddresses struct {
	list []net.IP
}

// NewIPAddresses is a constructor for IPAddresses.
func NewIPAddresses(addrs ...net.IP) *IPAddresses {
	ipAddresses := &IPAddresses{
		list: []net.IP{},
	}
	for _, addr := range addrs {
		ipAddresses.list = append(ipAddresses.list, addr)
	}
	return ipAddresses
}

// List returns the set as a slice which can be iterated through.
func (addrs *IPAddresses) List() []net.IP {
	return addrs.list
}

// Add IP address into the set.
func (addrs *IPAddresses) Add(addr net.IP) {
	if !addrs.Has(addr) {
		addrs.list = append(addrs.list, addr)
	}
}

// Del IP address from the set.
func (addrs *IPAddresses) Del(addr net.IP) {
	newAddrs := []net.IP{}
	for _, addr2 := range addrs.list {
		if !addr2.Equal(addr) {
			newAddrs = append(newAddrs, addr2)
		}
	}
	addrs.list = newAddrs
}

// Copy creates a deep copy of the set.
func (addrs *IPAddresses) Copy() *IPAddresses {
	addrsCopy := NewIPAddresses()
	for _, addr := range addrs.list {
		addrsCopy.list = append(addrsCopy.list, addr)
	}
	return addrsCopy
}

// Has returns true if the given IP address is in the set.
func (addrs *IPAddresses) Has(addr net.IP) bool {
	for _, addr2 := range addrs.list {
		if addr2.Equal(addr) {
			return true
		}
	}
	return false
}

// String converts a set of IP addresses into a human-readable string.
func (addrs IPAddresses) String() string {
	str := "{"
	for idx, addr := range addrs.list {
		str += addr.String()
		if idx < len(addrs.list)-1 {
			str += ", "
		}
	}
	str += "}"
	return str
}

// Interfaces is a set of interface names.
type Interfaces map[string]struct{}

// NewInterfaces is a constructor for Interfaces.
func NewInterfaces(ifNames ...string) Interfaces {
	interfaces := make(Interfaces)
	for _, ifName := range ifNames {
		interfaces.Add(ifName)
	}
	return interfaces
}

// Add interface name into the set.
func (ifs Interfaces) Add(ifName string) {
	ifs[ifName] = struct{}{}
}

// Del interface name from the set.
func (ifs Interfaces) Del(ifName string) {
	if ifs.Has(ifName) {
		delete(ifs, ifName)
	}
}

// Copy creates a deep copy of the set.
func (ifs Interfaces) Copy() Interfaces {
	ifsCopy := NewInterfaces()
	for intf := range ifs {
		ifsCopy.Add(intf)
	}
	return ifsCopy
}

// Has returns true if the given interface name is in the set.
func (ifs Interfaces) Has(ifName string) bool {
	_, has := ifs[ifName]
	return has
}

// String converts a set of interface names into a human-readable string.
func (ifs Interfaces) String() string {
	str := "{"
	idx := 0
	for ifName := range ifs {
		str += ifName
		if idx < len(ifs)-1 {
			str += ", "
		}
		idx++
	}
	str += "}"
	return str
}

// ResyncEventData wraps an entire state of K8s services as provided by the Processor.
type ResyncEventData struct {
	// NodeIPs is a list of IP addresses of all nodes in the cluster.
	NodeIPs *IPAddresses

	// Services is a list of all currently deployed services.
	Services []*ContivService

	// FrontendIfs is a set of all interfaces connecting clients with VPP
	// (VPP specific).
	FrontendIfs Interfaces

	// BackendIfs is a set of all interfaces connecting service backends with VPP
	// (VPP specific).
	BackendIfs Interfaces
}

// NewResyncEventData is a constructor for ResyncEventData.
func NewResyncEventData() *ResyncEventData {
	return &ResyncEventData{
		NodeIPs:     NewIPAddresses(),
		Services:    []*ContivService{},
		FrontendIfs: NewInterfaces(),
		BackendIfs:  NewInterfaces(),
	}
}

// String converts ResyncEventData into a human-readable string.
func (red ResyncEventData) String() string {
	services := ""
	for idx, service := range red.Services {
		services += service.String()
		if idx < len(red.Services)-1 {
			services += ", "
		}
	}
	return fmt.Sprintf("ResyncEventData <NodeIPs:[%s] Services:[%s], FrontendIfs:%s BackendIfs:%s>",
		red.NodeIPs.String(), services, red.FrontendIfs.String(), red.BackendIfs.String())
}
<|endoftext|>


<commit_before>package cli

import (
	"context"
	"fmt"
	"sort"
	"sync"
	"time"

	"github.com/kopia/kopia/snapshot"
	"github.com/kopia/kopia/snapshot/snapshotfs"
	"github.com/kopia/repo"
)

var (
	migrateCommand      = snapshotCommands.Command("migrate", "Migrate snapshots from another repository")
	migrateSourceConfig = migrateCommand.Flag("source-config", "Configuration file for the source repository").Required().ExistingFile()
	migrateSources      = migrateCommand.Flag("sources", "List of sources to migrate").Strings()
	migrateAll          = migrateCommand.Flag("all", "Migrate all sources").Bool()
	migrateLatestOnly   = migrateCommand.Flag("latest-only", "Only migrate the latest snapshot").Bool()
	migrateIgnoreErrors = migrateCommand.Flag("ignore-errors", "Ignore errors when reading source backup").Bool()
	migrateParallelism  = migrateCommand.Flag("parallelism", "Number of sources to migrate in parallel").Default("1").Int()
)

func runMigrateCommand(ctx context.Context, destRepo *repo.Repository) error {
	sourceRepo, err := repo.Open(ctx, *migrateSourceConfig, mustGetPasswordFromFlags(false, false), applyOptionsFromFlags(nil))
	if err != nil {
		return fmt.Errorf("can't open source repository: %v", err)
	}

	sources, err := getSourcesToMigrate(ctx, sourceRepo)
	if err != nil {
		return fmt.Errorf("can't retrieve sources: %v", err)
	}

	semaphore := make(chan struct{}, *migrateParallelism)
	var wg sync.WaitGroup

	var (
		mu              sync.Mutex
		cancelled       bool
		activeUploaders = map[snapshot.SourceInfo]*snapshotfs.Uploader{}
	)

	onCtrlC(func() {
		mu.Lock()
		mu.Unlock()

		if !cancelled {
			cancelled = true
			for s, u := range activeUploaders {
				log.Warningf("cancelling active uploader for %v", s)
				u.Cancel()
			}
		}
	})

	for _, s := range sources {
		// start a new uploader unless already cancelled
		mu.Lock()
		if cancelled {
			mu.Unlock()
			break
		}

		uploader := snapshotfs.NewUploader(destRepo)
		uploader.Progress = cliProgress
		uploader.IgnoreFileErrors = *migrateIgnoreErrors
		activeUploaders[s] = uploader
		mu.Unlock()

		wg.Add(1)
		semaphore <- struct{}{}
		go func(s snapshot.SourceInfo) {
			defer func() {
				mu.Lock()
				delete(activeUploaders, s)
				mu.Unlock()

				<-semaphore
				wg.Done()
			}()

			if err := migrateSingleSource(ctx, uploader, sourceRepo, destRepo, s); err != nil {
				log.Warningf("unable to migrate source: %v", err)
			}
		}(s)
	}
	wg.Wait()

	return nil
}

func findPreviousSnapshotManifestWithStartTime(ctx context.Context, rep *repo.Repository, sourceInfo snapshot.SourceInfo, startTime time.Time) (*snapshot.Manifest, error) {
	previous, err := snapshot.ListSnapshots(ctx, rep, sourceInfo)
	if err != nil {
		return nil, fmt.Errorf("error listing previous backups: %v", err)
	}

	for _, p := range previous {
		if p.StartTime == startTime {
			return p, nil
		}
	}

	return nil, nil
}

func migrateSingleSource(ctx context.Context, uploader *snapshotfs.Uploader, sourceRepo, destRepo *repo.Repository, s snapshot.SourceInfo) error {
	manifests, err := snapshot.ListSnapshotManifests(ctx, sourceRepo, &s)
	if err != nil {
		return err
	}
	snapshots, err := snapshot.LoadSnapshots(ctx, sourceRepo, manifests)
	if err != nil {
		return fmt.Errorf("unable to load snapshot manifests for %v: %v", s, err)
	}

	sort.Slice(snapshots, func(i, j int) bool {
		return snapshots[i].StartTime.Before(snapshots[j].StartTime)
	})

	for _, m := range filterSnapshotsToMigrate(snapshots) {
		if uploader.IsCancelled() {
			break
		}
		if m.IncompleteReason != "" {
			log.Infof("ignoring incomplete %v at %v", s, formatTimestamp(m.StartTime))
			continue
		}
		sourceEntry := snapshotfs.DirectoryEntry(sourceRepo, m.RootObjectID(), nil)

		existing, err := findPreviousSnapshotManifestWithStartTime(ctx, destRepo, m.Source, m.StartTime)
		if err != nil {
			return err
		}
		if existing != nil {
			log.Infof("already migrated %v at %v", s, formatTimestamp(m.StartTime))
			continue
		}

		log.Infof("migrating snapshot of %v at %v", s, formatTimestamp(m.StartTime))
		previousManifest, err := findPreviousSnapshotManifest(ctx, destRepo, m.Source, &m.StartTime)
		if err != nil {
			return err
		}

		newm, err := uploader.Upload(ctx, sourceEntry, m.Source, previousManifest)
		if err != nil {
			return fmt.Errorf("error migrating shapshot %v @ %v: %v", m.Source, m.StartTime, err)
		}

		newm.StartTime = m.StartTime
		newm.EndTime = m.EndTime
		newm.Description = m.Description
		if newm.IncompleteReason == "" {
			if _, err := snapshot.SaveSnapshot(ctx, destRepo, newm); err != nil {
				return fmt.Errorf("cannot save manifest: %v", err)
			}
		}
	}

	return nil
}

func filterSnapshotsToMigrate(s []*snapshot.Manifest) []*snapshot.Manifest {
	if *migrateLatestOnly && len(s) > 0 {
		s = s[0:1]
	}
	return s
}

func getSourcesToMigrate(ctx context.Context, rep *repo.Repository) ([]snapshot.SourceInfo, error) {
	if len(*migrateSources) > 0 {
		var result []snapshot.SourceInfo

		for _, s := range *migrateSources {
			si, err := snapshot.ParseSourceInfo(s, getHostName(), getUserName())
			if err != nil {
				return nil, err
			}

			result = append(result, si)
		}

		return result, nil
	}

	if *migrateAll {
		return snapshot.ListSources(ctx, rep)
	}

	return nil, fmt.Errorf("must specify either --all or --sources")
}

func init() {
	migrateCommand.Action(repositoryAction(runMigrateCommand))
}
<commit_msg>cli: fixed linter issues<commit_after>package cli

import (
	"context"
	"fmt"
	"sort"
	"sync"
	"time"

	"github.com/kopia/kopia/snapshot"
	"github.com/kopia/kopia/snapshot/snapshotfs"
	"github.com/kopia/repo"
)

var (
	migrateCommand      = snapshotCommands.Command("migrate", "Migrate snapshots from another repository")
	migrateSourceConfig = migrateCommand.Flag("source-config", "Configuration file for the source repository").Required().ExistingFile()
	migrateSources      = migrateCommand.Flag("sources", "List of sources to migrate").Strings()
	migrateAll          = migrateCommand.Flag("all", "Migrate all sources").Bool()
	migrateLatestOnly   = migrateCommand.Flag("latest-only", "Only migrate the latest snapshot").Bool()
	migrateIgnoreErrors = migrateCommand.Flag("ignore-errors", "Ignore errors when reading source backup").Bool()
	migrateParallelism  = migrateCommand.Flag("parallelism", "Number of sources to migrate in parallel").Default("1").Int()
)

func runMigrateCommand(ctx context.Context, destRepo *repo.Repository) error {
	sourceRepo, err := repo.Open(ctx, *migrateSourceConfig, mustGetPasswordFromFlags(false, false), applyOptionsFromFlags(nil))
	if err != nil {
		return fmt.Errorf("can't open source repository: %v", err)
	}

	sources, err := getSourcesToMigrate(ctx, sourceRepo)
	if err != nil {
		return fmt.Errorf("can't retrieve sources: %v", err)
	}

	semaphore := make(chan struct{}, *migrateParallelism)
	var wg sync.WaitGroup

	var (
		mu              sync.Mutex
		cancelled       bool
		activeUploaders = map[snapshot.SourceInfo]*snapshotfs.Uploader{}
	)

	onCtrlC(func() {
		mu.Lock()
		defer mu.Unlock()

		if !cancelled {
			cancelled = true
			for s, u := range activeUploaders {
				log.Warningf("cancelling active uploader for %v", s)
				u.Cancel()
			}
		}
	})

	for _, s := range sources {
		// start a new uploader unless already cancelled
		mu.Lock()
		if cancelled {
			mu.Unlock()
			break
		}

		uploader := snapshotfs.NewUploader(destRepo)
		uploader.Progress = cliProgress
		uploader.IgnoreFileErrors = *migrateIgnoreErrors
		activeUploaders[s] = uploader
		mu.Unlock()

		wg.Add(1)
		semaphore <- struct{}{}
		go func(s snapshot.SourceInfo) {
			defer func() {
				mu.Lock()
				delete(activeUploaders, s)
				mu.Unlock()

				<-semaphore
				wg.Done()
			}()

			if err := migrateSingleSource(ctx, uploader, sourceRepo, destRepo, s); err != nil {
				log.Warningf("unable to migrate source: %v", err)
			}
		}(s)
	}
	wg.Wait()

	return nil
}

func findPreviousSnapshotManifestWithStartTime(ctx context.Context, rep *repo.Repository, sourceInfo snapshot.SourceInfo, startTime time.Time) (*snapshot.Manifest, error) {
	previous, err := snapshot.ListSnapshots(ctx, rep, sourceInfo)
	if err != nil {
		return nil, fmt.Errorf("error listing previous backups: %v", err)
	}

	for _, p := range previous {
		if p.StartTime == startTime {
			return p, nil
		}
	}

	return nil, nil
}

func migrateSingleSource(ctx context.Context, uploader *snapshotfs.Uploader, sourceRepo, destRepo *repo.Repository, s snapshot.SourceInfo) error {
	manifests, err := snapshot.ListSnapshotManifests(ctx, sourceRepo, &s)
	if err != nil {
		return err
	}
	snapshots, err := snapshot.LoadSnapshots(ctx, sourceRepo, manifests)
	if err != nil {
		return fmt.Errorf("unable to load snapshot manifests for %v: %v", s, err)
	}

	sort.Slice(snapshots, func(i, j int) bool {
		return snapshots[i].StartTime.Before(snapshots[j].StartTime)
	})

	for _, m := range filterSnapshotsToMigrate(snapshots) {
		if uploader.IsCancelled() {
			break
		}

		if err := migrateSingleSourceSnapshot(ctx, uploader, sourceRepo, destRepo, s, m); err != nil {
			return err
		}

	}

	return nil
}

func migrateSingleSourceSnapshot(ctx context.Context, uploader *snapshotfs.Uploader, sourceRepo, destRepo *repo.Repository, s snapshot.SourceInfo, m *snapshot.Manifest) error {
	if m.IncompleteReason != "" {
		log.Infof("ignoring incomplete %v at %v", s, formatTimestamp(m.StartTime))
		return nil
	}
	sourceEntry := snapshotfs.DirectoryEntry(sourceRepo, m.RootObjectID(), nil)

	existing, err := findPreviousSnapshotManifestWithStartTime(ctx, destRepo, m.Source, m.StartTime)
	if err != nil {
		return err
	}
	if existing != nil {
		log.Infof("already migrated %v at %v", s, formatTimestamp(m.StartTime))
		return nil
	}

	log.Infof("migrating snapshot of %v at %v", s, formatTimestamp(m.StartTime))
	previousManifest, err := findPreviousSnapshotManifest(ctx, destRepo, m.Source, &m.StartTime)
	if err != nil {
		return err
	}

	newm, err := uploader.Upload(ctx, sourceEntry, m.Source, previousManifest)
	if err != nil {
		return fmt.Errorf("error migrating shapshot %v @ %v: %v", m.Source, m.StartTime, err)
	}

	newm.StartTime = m.StartTime
	newm.EndTime = m.EndTime
	newm.Description = m.Description
	if newm.IncompleteReason == "" {
		if _, err := snapshot.SaveSnapshot(ctx, destRepo, newm); err != nil {
			return fmt.Errorf("cannot save manifest: %v", err)
		}
	}
	return nil
}

func filterSnapshotsToMigrate(s []*snapshot.Manifest) []*snapshot.Manifest {
	if *migrateLatestOnly && len(s) > 0 {
		s = s[0:1]
	}
	return s
}

func getSourcesToMigrate(ctx context.Context, rep *repo.Repository) ([]snapshot.SourceInfo, error) {
	if len(*migrateSources) > 0 {
		var result []snapshot.SourceInfo

		for _, s := range *migrateSources {
			si, err := snapshot.ParseSourceInfo(s, getHostName(), getUserName())
			if err != nil {
				return nil, err
			}

			result = append(result, si)
		}

		return result, nil
	}

	if *migrateAll {
		return snapshot.ListSources(ctx, rep)
	}

	return nil, fmt.Errorf("must specify either --all or --sources")
}

func init() {
	migrateCommand.Action(repositoryAction(runMigrateCommand))
}
<|endoftext|>


<commit_before>// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
// vespa-deploy command
// Author: arnej

package deploy

import (
	// "fmt"
	"os"
	"path/filepath"
	"strconv"
	"strings"
)

const (
	cloudconfigDir              = ".cloudconfig"
	configsourceUrlUsedFileName = "deploy-configsource-url-used"
	sessionIdFileName           = "deploy-session-id"
)

func createCloudconfigDir() (string, error) {
	userHome, err := os.UserHomeDir()
	if err != nil {
		return "", err
	}
	home := filepath.Join(userHome, cloudconfigDir)
	if err := os.MkdirAll(home, 0700); err != nil {
		return "", err
	}
	return home, nil
}

func configsourceUrlUsedFile() string {
	home, err := createCloudconfigDir()
	if err != nil {
		home = "/tmp"
	}
	return filepath.Join(home, configsourceUrlUsedFileName)
}

func createTenantDir(tenant string) string {
	home, err := createCloudconfigDir()
	if err != nil {
		panic(err)
	}
	tdir := filepath.Join(home, tenant)
	if err := os.MkdirAll(tdir, 0700); err != nil {
		panic(err)
	}
	return tdir
}

func writeConfigsourceUrlUsed(url string) {
	fn := configsourceUrlUsedFile()
	os.WriteFile(fn, []byte(url), 0600)
}

func getConfigsourceUrlUsed() string {
	fn := configsourceUrlUsedFile()
	bytes, err := os.ReadFile(fn)
	if err != nil {
		return ""
	}
	return string(bytes)
}

func writeSessionIdFromResponseToFile(tenant, response string) {
	newSessionId := getSessionIdFromResponse(response)
	writeSessionIdToFile(tenant, newSessionId)
}

func writeSessionIdToFile(tenant, newSessionId string) {
	if newSessionId != "" {
		dir := createTenantDir(tenant)
		fn := filepath.Join(dir, sessionIdFileName)
		os.WriteFile(fn, []byte(newSessionId), 0600)
		// fmt.Printf("wrote %s to %s\n", newSessionId, fn)
	}
}

func getSessionIdFromResponse(response string) string {
	_, after, found := strings.Cut(response, `"session-id":"`)
	if !found {
		return ""
	}
	digits, _, found := strings.Cut(after, `"`)
	if !found {
		return ""
	}
	if _, err := strconv.Atoi(digits); err != nil {
		return ""
	}
	return digits
}

func getSessionIdFromFile(tenant string) string {
	dir := createTenantDir(tenant)
	fn := filepath.Join(dir, sessionIdFileName)
	bytes, err := os.ReadFile(fn)
	if err == nil {
		// fmt.Printf("Session-id '%s' found from file %s\n", string(bytes), fn)
		return string(bytes)
	}
	panic("Could not read session id from file, and no session id supplied as argument. Exiting.")
}
<commit_msg>remove some unused code<commit_after>// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
// vespa-deploy command
// Author: arnej

package deploy

import (
	// "fmt"
	"os"
	"path/filepath"
)

const (
	cloudconfigDir              = ".cloudconfig"
	configsourceUrlUsedFileName = "deploy-configsource-url-used"
	sessionIdFileName           = "deploy-session-id"
)

func createCloudconfigDir() (string, error) {
	userHome, err := os.UserHomeDir()
	if err != nil {
		return "", err
	}
	home := filepath.Join(userHome, cloudconfigDir)
	if err := os.MkdirAll(home, 0700); err != nil {
		return "", err
	}
	return home, nil
}

func configsourceUrlUsedFile() string {
	home, err := createCloudconfigDir()
	if err != nil {
		home = "/tmp"
	}
	return filepath.Join(home, configsourceUrlUsedFileName)
}

func createTenantDir(tenant string) string {
	home, err := createCloudconfigDir()
	if err != nil {
		panic(err)
	}
	tdir := filepath.Join(home, tenant)
	if err := os.MkdirAll(tdir, 0700); err != nil {
		panic(err)
	}
	return tdir
}

func writeConfigsourceUrlUsed(url string) {
	fn := configsourceUrlUsedFile()
	os.WriteFile(fn, []byte(url), 0600)
}

func getConfigsourceUrlUsed() string {
	fn := configsourceUrlUsedFile()
	bytes, err := os.ReadFile(fn)
	if err != nil {
		return ""
	}
	return string(bytes)
}

func writeSessionIdToFile(tenant, newSessionId string) {
	if newSessionId != "" {
		dir := createTenantDir(tenant)
		fn := filepath.Join(dir, sessionIdFileName)
		os.WriteFile(fn, []byte(newSessionId), 0600)
		// fmt.Printf("wrote %s to %s\n", newSessionId, fn)
	}
}

func getSessionIdFromFile(tenant string) string {
	dir := createTenantDir(tenant)
	fn := filepath.Join(dir, sessionIdFileName)
	bytes, err := os.ReadFile(fn)
	if err == nil {
		// fmt.Printf("Session-id '%s' found from file %s\n", string(bytes), fn)
		return string(bytes)
	}
	panic("Could not read session id from file, and no session id supplied as argument. Exiting.")
}
<|endoftext|>


<commit_before>package generation

import (
	"os"

	"chunkymonkey/chunkstore"
	"chunkymonkey/gamerules"
	. "chunkymonkey/types"
	"nbt"
	"perlin"
	"rand"
	"time"
)

const SeaLevel = 63

// ChunkData implements chunkstore.IChunkReader.
type ChunkData struct {
	loc        ChunkXz
	blocks     []byte
	blockData  []byte
	blockLight []byte
	skyLight   []byte
	heightMap  []byte
}

func newChunkData(loc ChunkXz) *ChunkData {
	return &ChunkData{
		loc:        loc,
		blocks:     make([]byte, ChunkSizeH*ChunkSizeH*ChunkSizeY),
		blockData:  make([]byte, (ChunkSizeH*ChunkSizeH*ChunkSizeY)>>1),
		skyLight:   make([]byte, (ChunkSizeH*ChunkSizeH*ChunkSizeY)>>1),
		blockLight: make([]byte, (ChunkSizeH*ChunkSizeH*ChunkSizeY)>>1),
		heightMap:  make([]byte, ChunkSizeH*ChunkSizeH),
	}
}

func (data *ChunkData) ChunkLoc() ChunkXz {
	return data.loc
}

func (data *ChunkData) Blocks() []byte {
	return data.blocks
}

func (data *ChunkData) BlockData() []byte {
	return data.blockData
}

func (data *ChunkData) BlockLight() []byte {
	return data.blockLight
}

func (data *ChunkData) SkyLight() []byte {
	return data.skyLight
}

func (data *ChunkData) HeightMap() []byte {
	return data.heightMap
}

func (data *ChunkData) Entities() []gamerules.INonPlayerEntity {
	return nil
}

func (data *ChunkData) RootTag() nbt.ITag {
	return nil
}

// TestGenerator implements chunkstore.IChunkStore.
type TestGenerator struct {
	heightSource ISource
	randSource   rand.Source
	randGen      *rand.Rand
}

func NewTestGenerator(seed int64) *TestGenerator {
	perlin := perlin.NewPerlinNoise(seed)

	randSource := rand.NewSource(time.Nanoseconds())
	randGen := rand.New(randSource)

	return &TestGenerator{
		randSource: randSource,
		randGen:    randGen,
		heightSource: &Sum{
			Inputs: []ISource{
				&Turbulence{
					Dx:     &Scale{50, 1, &Offset{20.1, 0, perlin}},
					Dy:     &Scale{50, 1, &Offset{10.1, 0, perlin}},
					Factor: 50,
					Source: &Scale{
						Wavelength: 200,
						Amplitude:  50,
						Source:     perlin,
					},
				},
				&Turbulence{
					Dx:     &Scale{40, 1, &Offset{20.1, 0, perlin}},
					Dy:     &Scale{40, 1, &Offset{10.1, 0, perlin}},
					Factor: 10,
					Source: &Mult{
						A: &Scale{
							Wavelength: 40,
							Amplitude:  20,
							Source:     perlin,
						},
						// Local steepness.
						B: &Scale{
							Wavelength: 200,
							Amplitude:  1,
							Source:     &Add{perlin, 0.6},
						},
					},
				},
				&Scale{
					Wavelength: 5,
					Amplitude:  2,
					Source:     perlin,
				},
			},
		},
	}
}

func (gen *TestGenerator) LoadChunk(chunkLoc ChunkXz) (reader chunkstore.IChunkReader, err os.Error) {
	baseBlockXyz := chunkLoc.ChunkCornerBlockXY()

	baseX, baseZ := baseBlockXyz.X, baseBlockXyz.Z

	data := newChunkData(chunkLoc)

	baseIndex := BlockIndex(0)
	heightMapIndex := 0
	for x := 0; x < ChunkSizeH; x++ {
		for z := 0; z < ChunkSizeH; z++ {
			xf, zf := float64(x)+float64(baseX), float64(z)+float64(baseZ)
			height := int(SeaLevel + gen.heightSource.At2d(xf, zf))

			if height < 0 {
				height = 0
			} else if height >= ChunkSizeY {
				height = ChunkSizeY - 1
			}

			skyLightHeight := gen.setBlockStack(
				height,
				data.blocks[baseIndex:baseIndex+ChunkSizeY])

			data.heightMap[heightMapIndex] = byte(skyLightHeight)

			heightMapIndex++
			baseIndex += ChunkSizeY
		}
	}

	// The chunk has been generated, now add some trees if appropriate
	gen.addSaplings(data)
	gen.setSkylight(data)

	return data, nil
}

func (gen *TestGenerator) setBlockStack(height int, blocks []byte) (skyLightHeight int) {
	var topBlockType byte
	if height < SeaLevel+1 {
		skyLightHeight = SeaLevel + 1

		for y := SeaLevel; y > height; y-- {
			blocks[y] = 9 // stationary water
		}
		blocks[height] = 12 // sand
		topBlockType = 12
	} else {
		skyLightHeight = height + 1
		if height <= SeaLevel+1 {
			blocks[height] = 12 // sand
			topBlockType = 12
		} else {
			blocks[height] = 2 // grass
			topBlockType = 3   // dirt
		}
	}

	for y := height - 1; y > height-3 && y > 0; y-- {
		blocks[y] = topBlockType
	}
	for y := height - 3; y > 0; y-- {
		blocks[y] = 1 // stone
	}

	if skyLightHeight < 0 {
		skyLightHeight = 0
	}

	return
}

func (gen *TestGenerator) setSkyLightStack(skyLightHeight int, blocks []byte, skyLight []byte) {
	for y := ChunkSizeY - 1; y >= skyLightHeight; y-- {
		BlockIndex(y).SetBlockData(skyLight, 15)
	}

	var lightLevel int8 = 15

	for y := skyLightHeight; y >= 0 && lightLevel > 0; y-- {
		blockType, ok := gamerules.Blocks.Get(BlockId(blocks[y]))
		if lightLevel > 0 && ok && blockType.Opacity > 0 {
			lightLevel -= blockType.Opacity
		}

		BlockIndex(y).SetBlockData(skyLight, byte(lightLevel))
	}
}

func (gen *TestGenerator) setSkylight(data *ChunkData) {
	baseIndex := 0
	heightMapIndex := 0

	for x := 0; x < ChunkSizeH; x++ {
		for z := 0; z < ChunkSizeH; z++ {
			lightBase := baseIndex >> 1

			gen.setSkyLightStack(
				int(data.heightMap[heightMapIndex]),
				data.blocks[baseIndex:baseIndex+ChunkSizeY],
				data.skyLight[lightBase:lightBase+ChunkSizeY/2])
			heightMapIndex++
			baseIndex += ChunkSizeY
		}
	}

}

func (gen *TestGenerator) addSaplings(data *ChunkData) {
	baseIndex := 0
	heightMapIndex := 0

	// Move throughout the chunk, but refuse to create trees on chunk
	// boundaries.
	for x := 0; x < ChunkSizeH; x++ {
		for z := 0; z < ChunkSizeH; z++ {
			topBlock := int(data.heightMap[heightMapIndex] - 1)
			blockIndex := baseIndex + int(topBlock)

			if data.blocks[blockIndex] == 2 {
				// We could add a tree, check to see if we want to
				addTree := gen.randGen.Intn(100) > 95
				if addTree && x > 0 && x < ChunkSizeH-1 && z > 0 && z < ChunkSizeH-1 {
					if !adjacentBlockIs(data, x, topBlock, z, 2, 2, 2, 6) {
						// Check if an adjacent block has a sapling already
						data.blocks[blockIndex+1] = 6
					}
				}
			}

			heightMapIndex++
			baseIndex += ChunkSizeY
		}
	}
}

// adjacentBlockIs returns whether or not the blocks that are adjacent to
// the block at (bx,by,bz) are of type 'blockType'. The area which this
// function checks is specified by dx, dy, dz. So to check 1 block in all
// directions, you can specify 1,1,1.
func adjacentBlockIs(data *ChunkData, bx, by, bz, dx, dy, dz int, blockType byte) bool {
	subChunk := new(SubChunkXyz)

	for x := bx - dx; x <= bx+dx; x++ {
		for z := bz - dz; z <= bz+dz; z++ {
			for y := by - dy; y <= by+dy; y++ {
				if x != bx || y != by || z != bz {
					subChunk.X = SubChunkCoord(x)
					subChunk.Y = SubChunkCoord(y)
					subChunk.Z = SubChunkCoord(z)
					index, ok := subChunk.BlockIndex()
					if ok {
						if data.blocks[index] == blockType {
							return true
						}
					}
				}
			}
		}
	}

	return false
}
<commit_msg>Clarify comment for adjacentBlockIs<commit_after>package generation

import (
	"os"

	"chunkymonkey/chunkstore"
	"chunkymonkey/gamerules"
	. "chunkymonkey/types"
	"nbt"
	"perlin"
	"rand"
	"time"
)

const SeaLevel = 63

// ChunkData implements chunkstore.IChunkReader.
type ChunkData struct {
	loc        ChunkXz
	blocks     []byte
	blockData  []byte
	blockLight []byte
	skyLight   []byte
	heightMap  []byte
}

func newChunkData(loc ChunkXz) *ChunkData {
	return &ChunkData{
		loc:        loc,
		blocks:     make([]byte, ChunkSizeH*ChunkSizeH*ChunkSizeY),
		blockData:  make([]byte, (ChunkSizeH*ChunkSizeH*ChunkSizeY)>>1),
		skyLight:   make([]byte, (ChunkSizeH*ChunkSizeH*ChunkSizeY)>>1),
		blockLight: make([]byte, (ChunkSizeH*ChunkSizeH*ChunkSizeY)>>1),
		heightMap:  make([]byte, ChunkSizeH*ChunkSizeH),
	}
}

func (data *ChunkData) ChunkLoc() ChunkXz {
	return data.loc
}

func (data *ChunkData) Blocks() []byte {
	return data.blocks
}

func (data *ChunkData) BlockData() []byte {
	return data.blockData
}

func (data *ChunkData) BlockLight() []byte {
	return data.blockLight
}

func (data *ChunkData) SkyLight() []byte {
	return data.skyLight
}

func (data *ChunkData) HeightMap() []byte {
	return data.heightMap
}

func (data *ChunkData) Entities() []gamerules.INonPlayerEntity {
	return nil
}

func (data *ChunkData) RootTag() nbt.ITag {
	return nil
}

// TestGenerator implements chunkstore.IChunkStore.
type TestGenerator struct {
	heightSource ISource
	randSource   rand.Source
	randGen      *rand.Rand
}

func NewTestGenerator(seed int64) *TestGenerator {
	perlin := perlin.NewPerlinNoise(seed)

	randSource := rand.NewSource(time.Nanoseconds())
	randGen := rand.New(randSource)

	return &TestGenerator{
		randSource: randSource,
		randGen:    randGen,
		heightSource: &Sum{
			Inputs: []ISource{
				&Turbulence{
					Dx:     &Scale{50, 1, &Offset{20.1, 0, perlin}},
					Dy:     &Scale{50, 1, &Offset{10.1, 0, perlin}},
					Factor: 50,
					Source: &Scale{
						Wavelength: 200,
						Amplitude:  50,
						Source:     perlin,
					},
				},
				&Turbulence{
					Dx:     &Scale{40, 1, &Offset{20.1, 0, perlin}},
					Dy:     &Scale{40, 1, &Offset{10.1, 0, perlin}},
					Factor: 10,
					Source: &Mult{
						A: &Scale{
							Wavelength: 40,
							Amplitude:  20,
							Source:     perlin,
						},
						// Local steepness.
						B: &Scale{
							Wavelength: 200,
							Amplitude:  1,
							Source:     &Add{perlin, 0.6},
						},
					},
				},
				&Scale{
					Wavelength: 5,
					Amplitude:  2,
					Source:     perlin,
				},
			},
		},
	}
}

func (gen *TestGenerator) LoadChunk(chunkLoc ChunkXz) (reader chunkstore.IChunkReader, err os.Error) {
	baseBlockXyz := chunkLoc.ChunkCornerBlockXY()

	baseX, baseZ := baseBlockXyz.X, baseBlockXyz.Z

	data := newChunkData(chunkLoc)

	baseIndex := BlockIndex(0)
	heightMapIndex := 0
	for x := 0; x < ChunkSizeH; x++ {
		for z := 0; z < ChunkSizeH; z++ {
			xf, zf := float64(x)+float64(baseX), float64(z)+float64(baseZ)
			height := int(SeaLevel + gen.heightSource.At2d(xf, zf))

			if height < 0 {
				height = 0
			} else if height >= ChunkSizeY {
				height = ChunkSizeY - 1
			}

			skyLightHeight := gen.setBlockStack(
				height,
				data.blocks[baseIndex:baseIndex+ChunkSizeY])

			data.heightMap[heightMapIndex] = byte(skyLightHeight)

			heightMapIndex++
			baseIndex += ChunkSizeY
		}
	}

	// The chunk has been generated, now add some trees if appropriate
	gen.addSaplings(data)
	gen.setSkylight(data)

	return data, nil
}

func (gen *TestGenerator) setBlockStack(height int, blocks []byte) (skyLightHeight int) {
	var topBlockType byte
	if height < SeaLevel+1 {
		skyLightHeight = SeaLevel + 1

		for y := SeaLevel; y > height; y-- {
			blocks[y] = 9 // stationary water
		}
		blocks[height] = 12 // sand
		topBlockType = 12
	} else {
		skyLightHeight = height + 1
		if height <= SeaLevel+1 {
			blocks[height] = 12 // sand
			topBlockType = 12
		} else {
			blocks[height] = 2 // grass
			topBlockType = 3   // dirt
		}
	}

	for y := height - 1; y > height-3 && y > 0; y-- {
		blocks[y] = topBlockType
	}
	for y := height - 3; y > 0; y-- {
		blocks[y] = 1 // stone
	}

	if skyLightHeight < 0 {
		skyLightHeight = 0
	}

	return
}

func (gen *TestGenerator) setSkyLightStack(skyLightHeight int, blocks []byte, skyLight []byte) {
	for y := ChunkSizeY - 1; y >= skyLightHeight; y-- {
		BlockIndex(y).SetBlockData(skyLight, 15)
	}

	var lightLevel int8 = 15

	for y := skyLightHeight; y >= 0 && lightLevel > 0; y-- {
		blockType, ok := gamerules.Blocks.Get(BlockId(blocks[y]))
		if lightLevel > 0 && ok && blockType.Opacity > 0 {
			lightLevel -= blockType.Opacity
		}

		BlockIndex(y).SetBlockData(skyLight, byte(lightLevel))
	}
}

func (gen *TestGenerator) setSkylight(data *ChunkData) {
	baseIndex := 0
	heightMapIndex := 0

	for x := 0; x < ChunkSizeH; x++ {
		for z := 0; z < ChunkSizeH; z++ {
			lightBase := baseIndex >> 1

			gen.setSkyLightStack(
				int(data.heightMap[heightMapIndex]),
				data.blocks[baseIndex:baseIndex+ChunkSizeY],
				data.skyLight[lightBase:lightBase+ChunkSizeY/2])
			heightMapIndex++
			baseIndex += ChunkSizeY
		}
	}

}

func (gen *TestGenerator) addSaplings(data *ChunkData) {
	baseIndex := 0
	heightMapIndex := 0

	// Move throughout the chunk, but refuse to create trees on chunk
	// boundaries.
	for x := 0; x < ChunkSizeH; x++ {
		for z := 0; z < ChunkSizeH; z++ {
			topBlock := int(data.heightMap[heightMapIndex] - 1)
			blockIndex := baseIndex + int(topBlock)

			if data.blocks[blockIndex] == 2 {
				// We could add a tree, check to see if we want to
				addTree := gen.randGen.Intn(100) > 95
				if addTree && x > 0 && x < ChunkSizeH-1 && z > 0 && z < ChunkSizeH-1 {
					if !adjacentBlockIs(data, x, topBlock, z, 2, 2, 2, 6) {
						// Check if an adjacent block has a sapling already
						data.blocks[blockIndex+1] = 6
					}
				}
			}

			heightMapIndex++
			baseIndex += ChunkSizeY
		}
	}
}

// adjacentBlockIs return whether or not at least one block adjacent to
// (bx,by,bz) is of type 'blockType'. The area which this function checks is
// specified by dx,dy,dz. Blocks outside the given chunk are not checked.
func adjacentBlockIs(data *ChunkData, bx, by, bz, dx, dy, dz int, blockType byte) bool {
	subChunk := new(SubChunkXyz)

	for x := bx - dx; x <= bx+dx; x++ {
		for z := bz - dz; z <= bz+dz; z++ {
			for y := by - dy; y <= by+dy; y++ {
				if x != bx || y != by || z != bz {
					subChunk.X = SubChunkCoord(x)
					subChunk.Y = SubChunkCoord(y)
					subChunk.Z = SubChunkCoord(z)
					index, ok := subChunk.BlockIndex()
					if ok {
						if data.blocks[index] == blockType {
							return true
						}
					}
				}
			}
		}
	}

	return false
}
<|endoftext|>


<commit_before>package memory

import (
	"bytes"
	"container/list"
	"errors"
	"fmt"
	"strings"
	"sync"
	"time"

	"github.com/micro/go-micro/v3/config/loader"
	"github.com/micro/go-micro/v3/config/reader"
	"github.com/micro/go-micro/v3/config/reader/json"
	"github.com/micro/go-micro/v3/config/source"
)

type memory struct {
	exit chan bool
	opts loader.Options

	sync.RWMutex
	// the current snapshot
	snap *loader.Snapshot
	// the current values
	vals reader.Values
	// all the sets
	sets []*source.ChangeSet
	// all the sources
	sources []source.Source

	watchers *list.List
}

type updateValue struct {
	version string
	value   reader.Value
}

type watcher struct {
	exit    chan bool
	path    []string
	value   reader.Value
	reader  reader.Reader
	version string
	updates chan updateValue
}

func (m *memory) watch(idx int, s source.Source) {
	// watches a source for changes
	watch := func(idx int, s source.Watcher) error {
		for {
			// get changeset
			cs, err := s.Next()
			if err != nil {
				return err
			}

			m.Lock()

			// save
			m.sets[idx] = cs

			// merge sets
			set, err := m.opts.Reader.Merge(m.sets...)
			if err != nil {
				m.Unlock()
				return err
			}

			// set values
			m.vals, _ = m.opts.Reader.Values(set)
			m.snap = &loader.Snapshot{
				ChangeSet: set,
				Version:   genVer(),
			}
			m.Unlock()

			// send watch updates
			m.update()
		}
	}

	for {
		// watch the source
		w, err := s.Watch()
		if err != nil {
			time.Sleep(time.Second)
			continue
		}

		done := make(chan bool)

		// the stop watch func
		go func() {
			select {
			case <-done:
			case <-m.exit:
			}
			w.Stop()
		}()

		// block watch
		if err := watch(idx, w); err != nil {
			// do something better
			time.Sleep(time.Second)
		}

		// close done chan
		close(done)

		// if the config is closed exit
		select {
		case <-m.exit:
			return
		default:
		}
	}
}

func (m *memory) loaded() bool {
	var loaded bool
	m.RLock()
	if m.vals != nil {
		loaded = true
	}
	m.RUnlock()
	return loaded
}

// reload reads the sets and creates new values
func (m *memory) reload() error {
	m.Lock()

	// merge sets
	set, err := m.opts.Reader.Merge(m.sets...)
	if err != nil {
		m.Unlock()
		return err
	}

	// set values
	m.vals, _ = m.opts.Reader.Values(set)
	m.snap = &loader.Snapshot{
		ChangeSet: set,
		Version:   genVer(),
	}

	m.Unlock()

	// update watchers
	m.update()

	return nil
}

func (m *memory) update() {
	watchers := make([]*watcher, 0, m.watchers.Len())

	m.RLock()
	for e := m.watchers.Front(); e != nil; e = e.Next() {
		watchers = append(watchers, e.Value.(*watcher))
	}

	vals := m.vals
	snap := m.snap
	m.RUnlock()

	for _, w := range watchers {
		if w.version >= snap.Version {
			continue
		}

		uv := updateValue{
			version: m.snap.Version,
			value:   vals.Get(w.path...),
		}

		select {
		case w.updates <- uv:
		default:
		}
	}
}

// Snapshot returns a snapshot of the current loaded config
func (m *memory) Snapshot() (*loader.Snapshot, error) {
	if m.loaded() {
		m.RLock()
		snap := loader.Copy(m.snap)
		m.RUnlock()
		return snap, nil
	}

	// not loaded, sync
	if err := m.Sync(); err != nil {
		return nil, err
	}

	// make copy
	m.RLock()
	snap := loader.Copy(m.snap)
	m.RUnlock()

	return snap, nil
}

// Sync loads all the sources, calls the parser and updates the config
func (m *memory) Sync() error {
	//nolint:prealloc
	var sets []*source.ChangeSet

	m.Lock()

	// read the source
	var gerr []string

	for _, source := range m.sources {
		ch, err := source.Read()
		if err != nil {
			gerr = append(gerr, err.Error())
			continue
		}
		sets = append(sets, ch)
	}

	// merge sets
	set, err := m.opts.Reader.Merge(sets...)
	if err != nil {
		m.Unlock()
		return err
	}

	// set values
	vals, err := m.opts.Reader.Values(set)
	if err != nil {
		m.Unlock()
		return err
	}
	m.vals = vals
	m.snap = &loader.Snapshot{
		ChangeSet: set,
		Version:   genVer(),
	}

	m.Unlock()

	// update watchers
	m.update()

	if len(gerr) > 0 {
		return fmt.Errorf("source loading errors: %s", strings.Join(gerr, "\n"))
	}

	return nil
}

func (m *memory) Close() error {
	select {
	case <-m.exit:
		return nil
	default:
		close(m.exit)
	}
	return nil
}

func (m *memory) Get(path ...string) (reader.Value, error) {
	if !m.loaded() {
		if err := m.Sync(); err != nil {
			return nil, err
		}
	}

	m.Lock()
	defer m.Unlock()

	// did sync actually work?
	if m.vals != nil {
		return m.vals.Get(path...), nil
	}

	// assuming vals is nil
	// create new vals

	ch := m.snap.ChangeSet

	// we are truly screwed, trying to load in a hacked way
	v, err := m.opts.Reader.Values(ch)
	if err != nil {
		return nil, err
	}

	// lets set it just because
	m.vals = v

	if m.vals != nil {
		return m.vals.Get(path...), nil
	}

	// ok we're going hardcore now

	return nil, errors.New("no values")
}

func (m *memory) Load(sources ...source.Source) error {
	var gerrors []string

	for _, source := range sources {
		set, err := source.Read()
		if err != nil {
			gerrors = append(gerrors,
				fmt.Sprintf("error loading source %s: %v",
					source,
					err))
			// continue processing
			continue
		}
		m.Lock()
		m.sources = append(m.sources, source)
		m.sets = append(m.sets, set)
		idx := len(m.sets) - 1
		m.Unlock()
		go m.watch(idx, source)
	}

	if err := m.reload(); err != nil {
		gerrors = append(gerrors, err.Error())
	}

	// Return errors
	if len(gerrors) != 0 {
		return errors.New(strings.Join(gerrors, "\n"))
	}
	return nil
}

func (m *memory) Watch(path ...string) (loader.Watcher, error) {
	value, err := m.Get(path...)
	if err != nil {
		return nil, err
	}

	m.Lock()

	w := &watcher{
		exit:    make(chan bool),
		path:    path,
		value:   value,
		reader:  m.opts.Reader,
		updates: make(chan updateValue, 1),
		version: m.snap.Version,
	}

	e := m.watchers.PushBack(w)

	m.Unlock()

	go func() {
		<-w.exit
		m.Lock()
		m.watchers.Remove(e)
		m.Unlock()
	}()

	return w, nil
}

func (m *memory) String() string {
	return "memory"
}

func (w *watcher) Next() (*loader.Snapshot, error) {
	update := func(v reader.Value) *loader.Snapshot {
		w.value = v

		cs := &source.ChangeSet{
			Data:      v.Bytes(),
			Format:    w.reader.String(),
			Source:    "memory",
			Timestamp: time.Now(),
		}
		cs.Checksum = cs.Sum()

		return &loader.Snapshot{
			ChangeSet: cs,
			Version:   w.version,
		}

	}

	for {
		select {
		case <-w.exit:
			return nil, errors.New("watcher stopped")

		case uv := <-w.updates:
			if uv.version <= w.version {
				continue
			}

			v := uv.value

			w.version = uv.version

			if bytes.Equal(w.value.Bytes(), v.Bytes()) {
				continue
			}

			return update(v), nil
		}
	}
}

func (w *watcher) Stop() error {
	select {
	case <-w.exit:
	default:
		close(w.exit)
		close(w.updates)
	}

	return nil
}

func genVer() string {
	return fmt.Sprintf("%d", time.Now().UnixNano())
}

func NewLoader(opts ...loader.Option) loader.Loader {
	options := loader.Options{
		Reader: json.NewReader(),
	}

	for _, o := range opts {
		o(&options)
	}

	m := &memory{
		exit:     make(chan bool),
		opts:     options,
		watchers: list.New(),
		sources:  options.Source,
	}

	m.sets = make([]*source.ChangeSet, len(options.Source))

	for i, s := range options.Source {
		m.sets[i] = &source.ChangeSet{Source: s.String()}
		go m.watch(i, s)
	}

	return m
}
<commit_msg>Check chan not closed before sending updates for memory loader (#1894)<commit_after>package memory

import (
	"bytes"
	"container/list"
	"errors"
	"fmt"
	"strings"
	"sync"
	"time"

	"github.com/micro/go-micro/v3/config/loader"
	"github.com/micro/go-micro/v3/config/reader"
	"github.com/micro/go-micro/v3/config/reader/json"
	"github.com/micro/go-micro/v3/config/source"
)

type memory struct {
	exit chan bool
	opts loader.Options

	sync.RWMutex
	// the current snapshot
	snap *loader.Snapshot
	// the current values
	vals reader.Values
	// all the sets
	sets []*source.ChangeSet
	// all the sources
	sources []source.Source

	watchers *list.List
}

type updateValue struct {
	version string
	value   reader.Value
}

type watcher struct {
	exit    chan bool
	path    []string
	value   reader.Value
	reader  reader.Reader
	version string
	updates chan updateValue
}

func (m *memory) watch(idx int, s source.Source) {
	// watches a source for changes
	watch := func(idx int, s source.Watcher) error {
		for {
			// get changeset
			cs, err := s.Next()
			if err != nil {
				return err
			}

			m.Lock()

			// save
			m.sets[idx] = cs

			// merge sets
			set, err := m.opts.Reader.Merge(m.sets...)
			if err != nil {
				m.Unlock()
				return err
			}

			// set values
			m.vals, _ = m.opts.Reader.Values(set)
			m.snap = &loader.Snapshot{
				ChangeSet: set,
				Version:   genVer(),
			}
			m.Unlock()

			// send watch updates
			m.update()
		}
	}

	for {
		// watch the source
		w, err := s.Watch()
		if err != nil {
			time.Sleep(time.Second)
			continue
		}

		done := make(chan bool)

		// the stop watch func
		go func() {
			select {
			case <-done:
			case <-m.exit:
			}
			w.Stop()
		}()

		// block watch
		if err := watch(idx, w); err != nil {
			// do something better
			time.Sleep(time.Second)
		}

		// close done chan
		close(done)

		// if the config is closed exit
		select {
		case <-m.exit:
			return
		default:
		}
	}
}

func (m *memory) loaded() bool {
	var loaded bool
	m.RLock()
	if m.vals != nil {
		loaded = true
	}
	m.RUnlock()
	return loaded
}

// reload reads the sets and creates new values
func (m *memory) reload() error {
	m.Lock()

	// merge sets
	set, err := m.opts.Reader.Merge(m.sets...)
	if err != nil {
		m.Unlock()
		return err
	}

	// set values
	m.vals, _ = m.opts.Reader.Values(set)
	m.snap = &loader.Snapshot{
		ChangeSet: set,
		Version:   genVer(),
	}

	m.Unlock()

	// update watchers
	m.update()

	return nil
}

func (m *memory) update() {
	watchers := make([]*watcher, 0, m.watchers.Len())

	m.RLock()
	for e := m.watchers.Front(); e != nil; e = e.Next() {
		watchers = append(watchers, e.Value.(*watcher))
	}

	vals := m.vals
	snap := m.snap
	m.RUnlock()

	for _, w := range watchers {
		if w.version >= snap.Version {
			continue
		}

		uv := updateValue{
			version: m.snap.Version,
			value:   vals.Get(w.path...),
		}

		select {
		case <-w.exit:
			continue
		default:
		}
		select {
		case w.updates <- uv:
		default:
		}
	}
}

// Snapshot returns a snapshot of the current loaded config
func (m *memory) Snapshot() (*loader.Snapshot, error) {
	if m.loaded() {
		m.RLock()
		snap := loader.Copy(m.snap)
		m.RUnlock()
		return snap, nil
	}

	// not loaded, sync
	if err := m.Sync(); err != nil {
		return nil, err
	}

	// make copy
	m.RLock()
	snap := loader.Copy(m.snap)
	m.RUnlock()

	return snap, nil
}

// Sync loads all the sources, calls the parser and updates the config
func (m *memory) Sync() error {
	//nolint:prealloc
	var sets []*source.ChangeSet

	m.Lock()

	// read the source
	var gerr []string

	for _, source := range m.sources {
		ch, err := source.Read()
		if err != nil {
			gerr = append(gerr, err.Error())
			continue
		}
		sets = append(sets, ch)
	}

	// merge sets
	set, err := m.opts.Reader.Merge(sets...)
	if err != nil {
		m.Unlock()
		return err
	}

	// set values
	vals, err := m.opts.Reader.Values(set)
	if err != nil {
		m.Unlock()
		return err
	}
	m.vals = vals
	m.snap = &loader.Snapshot{
		ChangeSet: set,
		Version:   genVer(),
	}

	m.Unlock()

	// update watchers
	m.update()

	if len(gerr) > 0 {
		return fmt.Errorf("source loading errors: %s", strings.Join(gerr, "\n"))
	}

	return nil
}

func (m *memory) Close() error {
	select {
	case <-m.exit:
		return nil
	default:
		close(m.exit)
	}
	return nil
}

func (m *memory) Get(path ...string) (reader.Value, error) {
	if !m.loaded() {
		if err := m.Sync(); err != nil {
			return nil, err
		}
	}

	m.Lock()
	defer m.Unlock()

	// did sync actually work?
	if m.vals != nil {
		return m.vals.Get(path...), nil
	}

	// assuming vals is nil
	// create new vals

	ch := m.snap.ChangeSet

	// we are truly screwed, trying to load in a hacked way
	v, err := m.opts.Reader.Values(ch)
	if err != nil {
		return nil, err
	}

	// lets set it just because
	m.vals = v

	if m.vals != nil {
		return m.vals.Get(path...), nil
	}

	// ok we're going hardcore now

	return nil, errors.New("no values")
}

func (m *memory) Load(sources ...source.Source) error {
	var gerrors []string

	for _, source := range sources {
		set, err := source.Read()
		if err != nil {
			gerrors = append(gerrors,
				fmt.Sprintf("error loading source %s: %v",
					source,
					err))
			// continue processing
			continue
		}
		m.Lock()
		m.sources = append(m.sources, source)
		m.sets = append(m.sets, set)
		idx := len(m.sets) - 1
		m.Unlock()
		go m.watch(idx, source)
	}

	if err := m.reload(); err != nil {
		gerrors = append(gerrors, err.Error())
	}

	// Return errors
	if len(gerrors) != 0 {
		return errors.New(strings.Join(gerrors, "\n"))
	}
	return nil
}

func (m *memory) Watch(path ...string) (loader.Watcher, error) {
	value, err := m.Get(path...)
	if err != nil {
		return nil, err
	}

	m.Lock()

	w := &watcher{
		exit:    make(chan bool),
		path:    path,
		value:   value,
		reader:  m.opts.Reader,
		updates: make(chan updateValue, 1),
		version: m.snap.Version,
	}

	e := m.watchers.PushBack(w)

	m.Unlock()

	go func() {
		<-w.exit
		m.Lock()
		m.watchers.Remove(e)
		m.Unlock()
	}()

	return w, nil
}

func (m *memory) String() string {
	return "memory"
}

func (w *watcher) Next() (*loader.Snapshot, error) {
	update := func(v reader.Value) *loader.Snapshot {
		w.value = v

		cs := &source.ChangeSet{
			Data:      v.Bytes(),
			Format:    w.reader.String(),
			Source:    "memory",
			Timestamp: time.Now(),
		}
		cs.Checksum = cs.Sum()

		return &loader.Snapshot{
			ChangeSet: cs,
			Version:   w.version,
		}

	}

	for {
		select {
		case <-w.exit:
			return nil, errors.New("watcher stopped")

		case uv := <-w.updates:
			if uv.version <= w.version {
				continue
			}

			v := uv.value

			w.version = uv.version

			if bytes.Equal(w.value.Bytes(), v.Bytes()) {
				continue
			}

			return update(v), nil
		}
	}
}

func (w *watcher) Stop() error {
	select {
	case <-w.exit:
	default:
		close(w.exit)
		close(w.updates)
	}

	return nil
}

func genVer() string {
	return fmt.Sprintf("%d", time.Now().UnixNano())
}

func NewLoader(opts ...loader.Option) loader.Loader {
	options := loader.Options{
		Reader: json.NewReader(),
	}

	for _, o := range opts {
		o(&options)
	}

	m := &memory{
		exit:     make(chan bool),
		opts:     options,
		watchers: list.New(),
		sources:  options.Source,
	}

	m.sets = make([]*source.ChangeSet, len(options.Source))

	for i, s := range options.Source {
		m.sets[i] = &source.ChangeSet{Source: s.String()}
		go m.watch(i, s)
	}

	return m
}
<|endoftext|>


<commit_before>package constants

const (
	HOST = "0.0.0.0" // All host names
	PORT = "9000"

	LENGTH_HEADER_SIZE = 3 // Measured in bytes

	// Packet constants for plug communication
	HEAD   = "@H\n"
	ACK    = "\x06\n"
	OKAY   = "@K\n"
	CONFIG = "@SLITIsEP00000000500000C00001\n"
	//"@SLITVsEP00000000500000C00001\n"
	//"@SLITWsEP00000000500000C00001\n"
	//"@SLITTsEP00000000500000C00001\n"

	READ_TIME_LIMIT = 5 // Measured in seconds

	HEADER_REGEX = "^(?:THS)(\\d+)(?:t\\d+X)$"

	// Database
	DB_SOCKET   = "/var/run/postgresql"
	DB_PORT     = 5432
	DB_USER     = "landingzone"
	DB_NAME     = "seads"
	DB_PASSWORD = ""
)
<commit_msg>Try increasing timeout<commit_after>package constants

const (
	HOST = "0.0.0.0" // All host names
	PORT = "9000"

	LENGTH_HEADER_SIZE = 3 // Measured in bytes

	// Packet constants for plug communication
	HEAD   = "@H\n"
	ACK    = "\x06\n"
	OKAY   = "@K\n"
	CONFIG = "@SLITIsEP00000000500000C00001\n"
	//"@SLITVsEP00000000500000C00001\n"
	//"@SLITWsEP00000000500000C00001\n"
	//"@SLITTsEP00000000500000C00001\n"

	READ_TIME_LIMIT = 100 // Measured in seconds

	HEADER_REGEX = "^(?:THS)(\\d+)(?:t\\d+X)$"

	// Database
	DB_SOCKET   = "/var/run/postgresql"
	DB_PORT     = 5432
	DB_USER     = "landingzone"
	DB_NAME     = "seads"
	DB_PASSWORD = ""
)
<|endoftext|>