<commit_before>package gocd

import (
	"context"
	"fmt"
	"github.com/stretchr/testify/assert"
	"io/ioutil"
	"net/http"
	"testing"
)

func TestServerVersion(t *testing.T) {
	t.Run("ServerVersion", testServerVersion)
	t.Run("BadServerVersion", testBadServerVersion)
}

func testServerVersion(t *testing.T) {
	setup()
	defer teardown()

	mux.HandleFunc("/api/version", func(w http.ResponseWriter, r *http.Request) {
		assert.Equal(t, r.Method, "GET", "Unexpected HTTP method")
		assert.Equal(t, apiV1, r.Header.Get("Accept"))

		j, _ := ioutil.ReadFile("test/resources/server-version.v1.1.json")

		fmt.Fprint(w, string(j))
	})

	cachedServerVersion = nil
	v, _, err := client.ServerVersion.Get(context.Background())

	assert.NoError(t, err)

	assert.Equal(t, &ServerVersion{
		Version:     "16.6.0",
		BuildNumber: "3348",
		GitSha:      "a7a5717cbd60c30006314fb8dd529796c93adaf0",
		FullVersion: "16.6.0 (3348-a7a5717cbd60c30006314fb8dd529796c93adaf0)",
		CommitURL:   "https://github.com/gocd/gocd/commits/a7a5717cbd60c30006314fb8dd529796c93adaf0",
		VersionParts: &ServerVersionParts{
			Major: 16,
			Minor: 6,
			Patch: 0,
		},
	}, v)

	// Verify that the server version is cached
	assert.Equal(t, cachedServerVersion, v)

}

func testBadServerVersion(t *testing.T) {
	for _, test := range []struct {
		name      string
		id        int
		errString string
	}{
		{name: "Major", id: 2, errString: "strconv.Atoi: parsing \"a\": invalid syntax"},
		{name: "Minor", id: 3, errString: "strconv.Atoi: parsing \"b\": invalid syntax"},
		{name: "Patch", id: 4, errString: "strconv.Atoi: parsing \"c\": invalid syntax"},
	} {
		cachedServerVersion = nil
		t.Run(test.name, func(t *testing.T) { testBadServerVersionMajor(t, test.id, test.errString) })
	}
}

func testBadServerVersionMajor(t *testing.T, i int, errString string) {
	setup()
	defer teardown()

	mux.HandleFunc("/api/version", func(w http.ResponseWriter, r *http.Request) {
		assert.Equal(t, r.Method, "GET", "Unexpected HTTP method")
		assert.Equal(t, apiV1, r.Header.Get("Accept"))

		j, _ := ioutil.ReadFile(
			fmt.Sprintf("test/resources/server-version.v1.%d.json", i),
		)

		fmt.Fprint(w, string(j))
	})

	cachedServerVersion = nil
	_, _, err := client.ServerVersion.Get(context.Background())

	assert.EqualError(t, err, errString)
}

func testServerVersionCaching(t *testing.T) {
	setup()
	defer teardown()
	// Note that this test should not do an API call

	cachedServerVersion = &ServerVersion{
		Version:     "18.7.0",
		BuildNumber: "7121",
		GitSha:      "75d1247f58ab8bcde3c5b43392a87347979f82c5",
		FullVersion: "18.7.0 (7121-75d1247f58ab8bcde3c5b43392a87347979f82c5)",
		CommitURL:   "https://github.com/gocd/gocd/commits/75d1247f58ab8bcde3c5b43392a87347979f82c5",
		VersionParts: &ServerVersionParts{
			Major: 18,
			Minor: 7,
			Patch: 0,
		},
	}
	v, b, err := client.ServerVersion.Get(context.Background())

	assert.NoError(t, err)
	assert.Nil(t, b)

	assert.Equal(t, &ServerVersion{
		Version:     "18.7.0",
		BuildNumber: "7121",
		GitSha:      "75d1247f58ab8bcde3c5b43392a87347979f82c5",
		FullVersion: "18.7.0 (7121-75d1247f58ab8bcde3c5b43392a87347979f82c5)",
		CommitURL:   "https://github.com/gocd/gocd/commits/75d1247f58ab8bcde3c5b43392a87347979f82c5",
		VersionParts: &ServerVersionParts{
			Major: 18,
			Minor: 7,
			Patch: 0,
		},
	}, v)
}
<commit_msg>Make sure the new test is run<commit_after>package gocd

import (
	"context"
	"fmt"
	"github.com/stretchr/testify/assert"
	"io/ioutil"
	"net/http"
	"testing"
)

func TestServerVersion(t *testing.T) {
	t.Run("ServerVersion", testServerVersion)
	t.Run("BadServerVersion", testBadServerVersion)
	t.Run("ServerVersionCaching", testServerVersionCaching)
}

func testServerVersion(t *testing.T) {
	setup()
	defer teardown()

	mux.HandleFunc("/api/version", func(w http.ResponseWriter, r *http.Request) {
		assert.Equal(t, r.Method, "GET", "Unexpected HTTP method")
		assert.Equal(t, apiV1, r.Header.Get("Accept"))

		j, _ := ioutil.ReadFile("test/resources/server-version.v1.1.json")

		fmt.Fprint(w, string(j))
	})

	cachedServerVersion = nil
	v, _, err := client.ServerVersion.Get(context.Background())

	assert.NoError(t, err)

	assert.Equal(t, &ServerVersion{
		Version:     "16.6.0",
		BuildNumber: "3348",
		GitSha:      "a7a5717cbd60c30006314fb8dd529796c93adaf0",
		FullVersion: "16.6.0 (3348-a7a5717cbd60c30006314fb8dd529796c93adaf0)",
		CommitURL:   "https://github.com/gocd/gocd/commits/a7a5717cbd60c30006314fb8dd529796c93adaf0",
		VersionParts: &ServerVersionParts{
			Major: 16,
			Minor: 6,
			Patch: 0,
		},
	}, v)

	// Verify that the server version is cached
	assert.Equal(t, cachedServerVersion, v)

}

func testBadServerVersion(t *testing.T) {
	for _, test := range []struct {
		name      string
		id        int
		errString string
	}{
		{name: "Major", id: 2, errString: "strconv.Atoi: parsing \"a\": invalid syntax"},
		{name: "Minor", id: 3, errString: "strconv.Atoi: parsing \"b\": invalid syntax"},
		{name: "Patch", id: 4, errString: "strconv.Atoi: parsing \"c\": invalid syntax"},
	} {
		cachedServerVersion = nil
		t.Run(test.name, func(t *testing.T) { testBadServerVersionMajor(t, test.id, test.errString) })
	}
}

func testBadServerVersionMajor(t *testing.T, i int, errString string) {
	setup()
	defer teardown()

	mux.HandleFunc("/api/version", func(w http.ResponseWriter, r *http.Request) {
		assert.Equal(t, r.Method, "GET", "Unexpected HTTP method")
		assert.Equal(t, apiV1, r.Header.Get("Accept"))

		j, _ := ioutil.ReadFile(
			fmt.Sprintf("test/resources/server-version.v1.%d.json", i),
		)

		fmt.Fprint(w, string(j))
	})

	_, _, err := client.ServerVersion.Get(context.Background())

	assert.EqualError(t, err, errString)
}

func testServerVersionCaching(t *testing.T) {
	setup()
	defer teardown()
	// Note that this test should not do an API call

	cachedServerVersion = &ServerVersion{
		Version:     "18.7.0",
		BuildNumber: "7121",
		GitSha:      "75d1247f58ab8bcde3c5b43392a87347979f82c5",
		FullVersion: "18.7.0 (7121-75d1247f58ab8bcde3c5b43392a87347979f82c5)",
		CommitURL:   "https://github.com/gocd/gocd/commits/75d1247f58ab8bcde3c5b43392a87347979f82c5",
		VersionParts: &ServerVersionParts{
			Major: 18,
			Minor: 7,
			Patch: 0,
		},
	}
	v, b, err := client.ServerVersion.Get(context.Background())

	assert.NoError(t, err)
	assert.Nil(t, b)

	assert.Equal(t, &ServerVersion{
		Version:     "18.7.0",
		BuildNumber: "7121",
		GitSha:      "75d1247f58ab8bcde3c5b43392a87347979f82c5",
		FullVersion: "18.7.0 (7121-75d1247f58ab8bcde3c5b43392a87347979f82c5)",
		CommitURL:   "https://github.com/gocd/gocd/commits/75d1247f58ab8bcde3c5b43392a87347979f82c5",
		VersionParts: &ServerVersionParts{
			Major: 18,
			Minor: 7,
			Patch: 0,
		},
	}, v)
}
<|endoftext|>


<commit_before>package release

import (
	"time"

	"github.com/go-kit/kit/log"

	"github.com/weaveworks/flux/update"
)

type Changes interface {
	CalculateRelease(update.ReleaseContext, log.Logger) ([]*update.ControllerUpdate, update.Result, error)
	ReleaseKind() update.ReleaseKind
	ReleaseType() update.ReleaseType
	CommitMessage(update.Result) string
}

func Release(rc *ReleaseContext, changes Changes, logger log.Logger) (results update.Result, err error) {
	defer func(start time.Time) {
		update.ObserveRelease(
			start,
			err == nil,
			changes.ReleaseType(),
			changes.ReleaseKind(),
		)
	}(time.Now())

	logger = log.With(logger, "type", "release")

	updates, results, err := changes.CalculateRelease(rc, logger)
	if err != nil {
		return nil, err
	}

	err = ApplyChanges(rc, updates, logger)
	return results, err
}

func ApplyChanges(rc *ReleaseContext, updates []*update.ControllerUpdate, logger log.Logger) error {
	logger.Log("updates", len(updates))
	if len(updates) == 0 {
		logger.Log("exit", "no images to update for services given")
		return nil
	}

	timer := update.NewStageTimer("write_changes")
	err := rc.WriteUpdates(updates)
	timer.ObserveDuration()
	return err
}
<commit_msg>Add stub verification step to releaser<commit_after>package release

import (
	"time"

	"github.com/go-kit/kit/log"

	"github.com/weaveworks/flux/resource"
	"github.com/weaveworks/flux/update"
)

type Changes interface {
	CalculateRelease(update.ReleaseContext, log.Logger) ([]*update.ControllerUpdate, update.Result, error)
	ReleaseKind() update.ReleaseKind
	ReleaseType() update.ReleaseType
	CommitMessage(update.Result) string
}

func Release(rc *ReleaseContext, changes Changes, logger log.Logger) (results update.Result, err error) {
	defer func(start time.Time) {
		update.ObserveRelease(
			start,
			err == nil,
			changes.ReleaseType(),
			changes.ReleaseKind(),
		)
	}(time.Now())

	logger = log.With(logger, "type", "release")

	before, err := rc.manifests.LoadManifests(rc.repo.Dir(), rc.repo.ManifestDir())
	updates, results, err := changes.CalculateRelease(rc, logger)
	if err != nil {
		return nil, err
	}

	err = ApplyChanges(rc, updates, logger)
	if err == nil {
		var after map[string]resource.Resource
		after, err = rc.manifests.LoadManifests(rc.repo.Dir(), rc.repo.ManifestDir())
		if err == nil {
			err = VerifyChanges(before, updates, after)
		}
	}
	return results, err
}

func ApplyChanges(rc *ReleaseContext, updates []*update.ControllerUpdate, logger log.Logger) error {
	logger.Log("updates", len(updates))
	if len(updates) == 0 {
		logger.Log("exit", "no images to update for services given")
		return nil
	}

	timer := update.NewStageTimer("write_changes")
	err := rc.WriteUpdates(updates)
	timer.ObserveDuration()
	return err
}

func VerifyChanges(before map[string]resource.Resource, updates []*update.ControllerUpdate, after map[string]resource.Resource) error {
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2012 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package s3

import (
	"errors"
	"github.com/jacobsa/aws"
	"github.com/jacobsa/aws/s3/auth"
	"github.com/jacobsa/aws/s3/http"
)

// NonExistentBucketError represents an error due to an attempt to work with a
// bucket that doesn't exist according to S3.
type NonExistentBucketError struct {
	s string
}

// Bucket represents an S3 bucket, which is a collection of objects keyed on
// Unicode strings. The UTF-8 encoding of a key must be no more than 1024 bytes
// long.
//
// See here for more info:
//
//     http://goo.gl/Nd63t
//
type Bucket interface {
	// Retrieve data for the object with the given key.
	GetObject(key string) (data []byte, err error)

	// Store the supplied data with the given key, overwriting any previous
	// version.
	StoreObject(key string, data []byte) error
}

// OpenBucket returns a Bucket tied to a given name in whe given region. You
// must have previously created the bucket in the region, and the supplied
// access key must have access to it.
//
// If the supplied bucket doesn't exist, a *NonExistentBucketError is returned.
//
// To easily create a bucket, use the AWS Console:
//
//     http://aws.amazon.com/console/
//
func OpenBucket(name string, region Region, key aws.AccessKey) (Bucket, error) {
	return nil, errors.New("TODO: Implement OpenBucket.")
}

func (e *NonExistentBucketError) Error() string {
	return e.s
}

// A version of OpenBucket with the ability to inject dependencies, for
// testability.
func openBucket(
	name string,
	httpConn http.Conn,
	signer auth.Signer) (Bucket, error) {
	return nil, errors.New("TODO: Implement openBucket.")
}
<commit_msg>Implemented OpenBucket.<commit_after>// Copyright 2012 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package s3

import (
	"errors"
	"fmt"
	"github.com/jacobsa/aws"
	"github.com/jacobsa/aws/s3/auth"
	"github.com/jacobsa/aws/s3/http"
	"net/url"
)

// NonExistentBucketError represents an error due to an attempt to work with a
// bucket that doesn't exist according to S3.
type NonExistentBucketError struct {
	s string
}

// Bucket represents an S3 bucket, which is a collection of objects keyed on
// Unicode strings. The UTF-8 encoding of a key must be no more than 1024 bytes
// long.
//
// See here for more info:
//
//     http://goo.gl/Nd63t
//
type Bucket interface {
	// Retrieve data for the object with the given key.
	GetObject(key string) (data []byte, err error)

	// Store the supplied data with the given key, overwriting any previous
	// version.
	StoreObject(key string, data []byte) error
}

// OpenBucket returns a Bucket tied to a given name in whe given region. You
// must have previously created the bucket in the region, and the supplied
// access key must have access to it.
//
// If the supplied bucket doesn't exist, a *NonExistentBucketError is returned.
//
// To easily create a bucket, use the AWS Console:
//
//     http://aws.amazon.com/console/
//
func OpenBucket(name string, region Region, key aws.AccessKey) (Bucket, error) {
	// Create a connection to the given region's endpoint.
	endpoint := &url.URL{Scheme: "https", Host: string(region)}
	httpConn, err := http.NewConn(endpoint)
	if err != nil {
		return nil, fmt.Errorf("http.NewConn: %v", err)
	}

	// Create an appropriate request signer.
	signer, err := auth.NewSigner(&key)
	if err != nil {
		return nil, fmt.Errorf("auth.NewSigner: %v", err)
	}

	return openBucket(name, httpConn, signer)
}

func (e *NonExistentBucketError) Error() string {
	return e.s
}

// A version of OpenBucket with the ability to inject dependencies, for
// testability.
func openBucket(
	name string,
	httpConn http.Conn,
	signer auth.Signer) (Bucket, error) {
	return nil, errors.New("TODO: Implement openBucket.")
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"reflect"
	"strings"
)

func parseTag(tag string) []string {

	return strings.Split(tag, ",")
}

func contains(domains []string, domain string) bool {

	for _, d := range domains {

		if d == domain {

			return true
		}
	}

	return false
}

func Sanitise(obj interface{}, domain string) error {

	var v reflect.Value
	var t reflect.Type
	var ok bool

	fmt.Printf("%v(type %T)\n", obj, obj)

	// make sure this is a pointer, so that we can update the contents if needed
	if v, ok = obj.(reflect.Value); !ok {

		v = reflect.ValueOf(obj)
	}

	for v.Kind() == reflect.Ptr || v.Kind() == reflect.Interface {

		fmt.Println("object is a pointer or an interface, calling Elem()")
		v = v.Elem()
	}

	if !v.IsValid() {

		return nil
	}

	fmt.Printf("%v(type %T)\n", v, v)

	t = reflect.TypeOf(v.Interface())
	k := t.Kind()

	if k == reflect.Map {

		keys := v.MapKeys()
		for _, key := range keys {

			Sanitise(v.MapIndex(key), domain)
		}
	} else if k == reflect.Struct {

		for i := 0; i < t.NumField(); i++ {

			fmt.Printf("Processing field %v(%v)\n", t.Field(i).Type, t.Field(i).Name)
			field := t.Field(i)
			field_kind := field.Type.Kind()

			if tag := field.Tag.Get("sanitise"); len(tag) > 0 {

				// the sanitise tag's value should be a comma-separated list of
				// domains
				fmt.Printf("Field %v(type %T) has a sanitise tag\n", field.Name, v.Field(i))
				domains := parseTag(tag)
				if contains(domains, domain) || contains(domains, "*") {
					// sanitise this field
					if !v.Field(i).CanSet() {

						return fmt.Errorf("Unable to set zero value for %v", t.Field(i).Name)
					}

					fmt.Printf("Sanitising field %v\n", t.Field(i).Name)
					v.Field(i).Set(reflect.New(t.Field(i).Type).Elem())
				}
			} else if field_kind == reflect.Struct || field_kind == reflect.Interface {

				sv := v.Field(i)
				fmt.Printf("%v(type %T)\n", sv, sv)

				if err := Sanitise(sv, domain); err != nil {

					return err
				}
			}
		}
	}

	return nil
}
<commit_msg>Save a hierarchy string of traversed objects<commit_after>package sanitiser

import (
	"fmt"
	"reflect"
	"strings"
)

func parseTag(tag string) []string {

	return strings.Split(tag, ",")
}

func contains(domains []string, domain string) bool {

	for _, d := range domains {

		if d == domain {

			return true
		}
	}

	return false
}

func traverseObjects(obj interface{}, domain string, hierarchy string) error {

	var v reflect.Value
	var t reflect.Type
	var ok bool

	fmt.Printf("%v.%v(type %T)\n", hierarchy, obj, obj)

	// make sure this is a pointer, so that we can update the contents if needed
	if v, ok = obj.(reflect.Value); !ok {

		v = reflect.ValueOf(obj)
	}

	for v.Kind() == reflect.Ptr || v.Kind() == reflect.Interface {

		fmt.Println("object is a pointer or an interface, calling Elem()")
		v = v.Elem()
	}

	if !v.IsValid() {

		return nil
	}

	fmt.Printf("%v.%v(type %T)\n", hierarchy, v, v)

	t = reflect.TypeOf(v.Interface())
	k := t.Kind()

	if k == reflect.Map {

		keys := v.MapKeys()
		for _, key := range keys {

			if err := traverseObjects(v.MapIndex(key), domain, hierarchy+"["+fmt.Sprintf("%v", key.Elem())+"]"); err != nil {

				return err
			}
		}
	} else if k == reflect.Struct {

		for i := 0; i < t.NumField(); i++ {

			fmt.Printf("Processing field %v.%v(%v)\n", hierarchy, t.Field(i).Name, t.Field(i).Type)
			field := t.Field(i)
			field_kind := field.Type.Kind()

			if tag := field.Tag.Get("sanitise"); len(tag) > 0 {

				// the sanitise tag's value should be a comma-separated list of
				// domains
				fmt.Printf("Field %v.%v(type %T) has a sanitise tag\n", hierarchy, field.Name, v.Field(i))
				domains := parseTag(tag)
				if contains(domains, domain) || contains(domains, "*") {
					// sanitise this field
					if !v.Field(i).CanSet() {

						return fmt.Errorf("Unable to set zero value for %v.%v", hierarchy, t.Field(i).Name)
					}

					fmt.Printf("Sanitising field %v.%v\n", hierarchy, t.Field(i).Name)
					v.Field(i).Set(reflect.New(t.Field(i).Type).Elem())
				}
			} else if field_kind == reflect.Struct || field_kind == reflect.Interface {

				sv := v.Field(i)
				fmt.Printf("Processing object %v.%v(type %T)\n", hierarchy, sv, sv)

				if err := traverseObjects(sv, domain, hierarchy+"."+t.Field(i).Name); err != nil {

					return err
				}
			}
		}
	}

	return nil
}

func Sanitise(obj interface{}, domain string) error {

	return traverseObjects(obj, domain, "")
}
<|endoftext|>


<commit_before>package scheduler

import (
	"github.com/caelifer/scheduler/job"
	"github.com/caelifer/scheduler/worker"
)

// Public interfaces

// Scheduler is an interface type to provide an abstraction around load-balancing operation scheduling.
type Scheduler interface {
	Schedule(job.Interface)
}

// New() builds a new Scheduler object. It starts its scheduler on the background and make sure
// all awailable workers are always running on the background waiting for the work unit to come in. The
// work units are managed in a separate buffered channel of job.Interfaces. New() takes two paramters.
// nworkers - a number of background workers, and njobs - a number of queued jobs, before scheduler would
// block.
func New(nworkers, njobs int) Scheduler {
	s := new(simpleScheduler) // Heap
	s.workPool = make(chan worker.Interface, nworkers)
	s.done = make(chan worker.Interface)
	s.jobs = make(chan job.Interface, njobs)

	// Populate our pool of workers
	for i := 0; i < nworkers; i++ {
		s.workPool <- worker.New(s.done)
	}

	// Start our scheduler on the background
	go func() {
		// Run until the main program finishes
		for {
			select {
			case w := <-s.workPool:
				// Schedule new worker on the background "thread"
				go func() { w.Run(<-s.jobs) }()
			case w := <-s.done:
				s.workPool <- w // Never blocks
			}
		}
	}()

	return s
}

// Private concreat implementation

// simpleScheduler is an value type that implementes Scheduler interface.
type simpleScheduler struct {
	workPool chan worker.Interface // Buffered channel of workers
	done     chan worker.Interface // Unbuffered channel for workers to signal they are done
	jobs     chan job.Interface    // Buffered channel of pending work units
}

// Schedule is an implementation of Schedlue interface for simpleSchedule value type.
func (s *simpleScheduler) Schedule(j job.Interface) {
	s.jobs <- j // Could block if jobs buffer is full
}
<commit_msg>Fixed documenting comment<commit_after>package scheduler

import (
	"github.com/caelifer/scheduler/job"
	"github.com/caelifer/scheduler/worker"
)

// Public interfaces

// Scheduler is an interface type to provide an abstraction around load-balancing operation scheduling.
type Scheduler interface {
	Schedule(job.Interface)
}

// New builds a new Scheduler object. It starts its internal scheduling process
// on the background. This scheduling process makes sure that all available workers
// are always running on the background waiting for the work unit to come in. The
// work units are managed in a separate buffered channel of job.Interfaces. New
// takes two paramters.  nworkers - a number of background workers, and njobs -
// a number of queued jobs, before scheduler would block.
func New(nworkers, njobs int) Scheduler {
	s := new(simpleScheduler) // Heap
	s.workPool = make(chan worker.Interface, nworkers)
	s.done = make(chan worker.Interface)
	s.jobs = make(chan job.Interface, njobs)

	// Populate our pool of workers
	for i := 0; i < nworkers; i++ {
		s.workPool <- worker.New(s.done)
	}

	// Start our scheduler on the background
	go func() {
		// Run until the main program finishes
		for {
			select {
			case w := <-s.workPool:
				// Schedule new worker on the background "thread"
				go func() { w.Run(<-s.jobs) }()
			case w := <-s.done:
				s.workPool <- w // Never blocks
			}
		}
	}()

	return s
}

// Private concreat implementation

// simpleScheduler is an value type that implementes Scheduler interface.
type simpleScheduler struct {
	workPool chan worker.Interface // Buffered channel of workers
	done     chan worker.Interface // Unbuffered channel for workers to signal they are done
	jobs     chan job.Interface    // Buffered channel of pending work units
}

// Schedule is an implementation of Schedlue interface for simpleSchedule value type.
func (s *simpleScheduler) Schedule(j job.Interface) {
	s.jobs <- j // Could block if jobs buffer is full
}
<|endoftext|>


<commit_before>package main

import (
	"testing"

	. "github.com/VonC/godbg"
	"github.com/VonC/godbg/exit"
	"github.com/VonC/senvgo/prgs"
	. "github.com/smartystreets/goconvey/convey"
)

type testGetter0Prg struct{}
type testPrg struct{ name string }

func (tp *testPrg) Name() string { return tp.name }

func (tg0 testGetter0Prg) Get() []prgs.Prg {
	return []prgs.Prg{}
}

type testGetter3Prgs struct{}

func (tg3 testGetter3Prgs) Get() []prgs.Prg {
	return []prgs.Prg{&testPrg{name: "prg1"}, &testPrg{name: "prg2"}, &testPrg{name: "prg3"}}
}

func TestMain(t *testing.T) {

	exiter = exit.New(func(int) {})

	Convey("senvgo main installation scenario with no command", t, func() {
		SetBuffers(nil)
		prgsGetter = testGetter0Prg{}
		main()
		So(ErrString(), ShouldEqualNL, `  [main] (func)
    senvgo
`)
		So(exiter.Status(), ShouldEqual, 0)

		Convey("No prg means no prgs installed", func() {
			SetBuffers(nil)
			main()
			So(OutString(), ShouldEqual, `No program to install: nothing to do`)
			So(ErrString(), ShouldEqualNL, `  [main] (func)
    senvgo
`)
			So(exiter.Status(), ShouldEqual, 0)
			prgsGetter = testGetter3Prgs{}
			SetBuffers(nil)
			main()
			So(OutString(), ShouldNotEqual, `No program to install: nothing to do`)
		})

		// Convey("A program already installed means nothing to do", func() {
	})
}
<commit_msg>Prepare 'already installed' test<commit_after>package main

import (
	"testing"

	. "github.com/VonC/godbg"
	"github.com/VonC/godbg/exit"
	"github.com/VonC/senvgo/prgs"
	. "github.com/smartystreets/goconvey/convey"
)

type testGetter0Prg struct{}
type testPrg struct{ name string }

func (tp *testPrg) Name() string { return tp.name }

func (tg0 testGetter0Prg) Get() []prgs.Prg {
	return []prgs.Prg{}
}

type testGetter3Prgs struct{}

func (tg3 testGetter3Prgs) Get() []prgs.Prg {
	return []prgs.Prg{&testPrg{name: "prg1"}, &testPrg{name: "prg2"}, &testPrg{name: "prg3"}}
}

func TestMain(t *testing.T) {

	exiter = exit.New(func(int) {})

	Convey("senvgo main installation scenario with no command", t, func() {
		SetBuffers(nil)
		prgsGetter = testGetter0Prg{}
		main()
		So(ErrString(), ShouldEqualNL, `  [main] (func)
    senvgo
`)
		So(exiter.Status(), ShouldEqual, 0)

		Convey("No prg means no prgs installed", func() {
			SetBuffers(nil)
			main()
			So(OutString(), ShouldEqual, `No program to install: nothing to do`)
			So(ErrString(), ShouldEqualNL, `  [main] (func)
    senvgo
`)
			So(exiter.Status(), ShouldEqual, 0)
			prgsGetter = testGetter3Prgs{}
			SetBuffers(nil)
			main()
			So(OutString(), ShouldNotEqual, `No program to install: nothing to do`)
		})

		Convey("A program already installed means nothing to do", func() {
			prgsGetter = testGetter3Prgs{}
			SetBuffers(nil)
			main()
			So(OutString(), ShouldNotEqual, `'prg1' (1/3)... already installed: nothing to do
'prg3' (2/3)... already installed: nothing to do
'prg3' (3/3)... already installed: nothing to do
				`)
		})
	})
}
<|endoftext|>


<commit_before>package serf

import (
	"fmt"
	"github.com/hashicorp/memberlist"
	"log"
	"sync"
	"time"
)

type MemberStatus int

const (
	StatusNone MemberStatus = iota
	StatusAlive
	StatusLeaving
	StatusLeft
	StatusFailed
	StatusPartitioned // Partitioned is a best guess, should be treated as failed
)

type Serf struct {
	conf       *Config
	memberlist *memberlist.Memberlist
	joinCh     chan *memberlist.Node
	leaveCh    chan *memberlist.Node
	shutdownCh chan struct{}

	memberLock sync.RWMutex
	members    []*Member
	memberMap  map[string]*Member

	detector partitionDetector

	broadcasts *memberlist.TransmitLimitedQueue

	changeCh chan statusChange
}

// Member represents a single member in the gossip pool
type Member struct {
	Name   string
	Addr   []byte
	Role   string
	Status MemberStatus
}

type EventDelegate interface {
	// MembersJoined invoked when members have joined the cluster
	MembersJoined([]*Member)

	// MembersLeft invoked when members have left the cluster gracefully
	MembersLeft([]*Member)

	// MembersFailed invoked when members are unreachable by the cluster without
	// previously announcing their intention to leave
	MembersFailed([]*Member)

	// MembersPartitioned invoked when members are unreachable by cluster without
	// previously announcing their intention to leave. Partitions are impossible
	// to tell apart from failures, so this is more of a heuristic based on the
	// likelyhood of simultaneous failures. It should not be treated as exact.
	MembersPartitioned([]*Member)
}

// Start is used to initialize a new Serf instance
func Start(conf *Config) (*Serf, error) {
	serf := &Serf{
		conf:       conf,
		joinCh:     make(chan *memberlist.Node, 64),
		leaveCh:    make(chan *memberlist.Node, 64),
		shutdownCh: make(chan struct{}, 4),
		changeCh:   make(chan statusChange, 1024),
	}

	// Select a partition detector
	if conf.PartitionCount > 0 && conf.PartitionInterval > 0 {
		serf.detector = newPartitionRing(conf.PartitionCount, conf.PartitionInterval)
	} else {
		// Parititon detection disabled
		serf.detector = noopDetector{}
	}

	// Create the broadcast queue
	serf.broadcasts = &memberlist.TransmitLimitedQueue{
		NumNodes:       func() int { return len(serf.members) },
		RetransmitMult: conf.RetransmitMult,
	}

	// Create the memberlist config
	mlConf := memberlistConfig(conf)
	mlConf.JoinCh = serf.joinCh
	mlConf.LeaveCh = serf.leaveCh
	mlConf.UserDelegate = serf

	// Attempt to create the
	memb, err := memberlist.Create(mlConf)
	if err != nil {
		return nil, err
	}
	serf.memberlist = memb

	// Start the event handelr
	go serf.eventHandler()

	// Start the change handler
	go serf.changeHandler()

	// Done
	return serf, nil
}

// Join is used to attempt to join an existing gossip pool
// Returns an error if none of the existing nodes could be contacted
func (s *Serf) Join(existing []string) error {
	// Ensure we have some input
	if len(existing) == 0 {
		return fmt.Errorf("must specify at least one node to join")
	}

	// Use memberlist to perform the join
	_, err := s.memberlist.Join(existing)
	return err
}

// Members provides a point-in-time snapshot of the members
func (s *Serf) Members() []*Member {
	s.memberLock.RLock()
	defer s.memberLock.RUnlock()

	members := make([]*Member, len(s.members))
	for idx, m := range s.members {
		newM := &Member{}
		*newM = *m
		members[idx] = newM
	}

	return members
}

// Leave allows a node to gracefully leave the cluster. This
// should be followed by a call to Shutdown
func (s *Serf) Leave() error {
	var notifyCh chan struct{}
	var l leave

	// No need to broadcast if there is nobody else
	if len(s.members) <= 1 {
		goto AFTER_BROADCAST
	}

	// Create a channel to get notified
	notifyCh = make(chan struct{}, 1)

	// Broadcast leave intention
	l = leave{s.conf.Hostname}
	if err := s.encodeBroadcastNotify(leaveMsg, &l, notifyCh); err != nil {
		return err
	}

	select {
	case <-notifyCh:
	case <-time.After(s.conf.LeaveTimeout):
		log.Printf("[WARN] Timed out broadcasting leave intention")
	}

AFTER_BROADCAST:
	// Broadcast our own death
	return s.memberlist.Leave()
}

// Shutdown is used to shutdown all the listeners. It is not graceful,
// and should be preceeded by a call to Leave.
func (s *Serf) Shutdown() error {
	// Emit once per background routine (eventHandler, changeHandler)
	for i := 0; i < 2; i++ {
		s.shutdownCh <- struct{}{}
	}
	return s.memberlist.Shutdown()
}
<commit_msg>Close shutdown channel instead of sending messages<commit_after>package serf

import (
	"fmt"
	"github.com/hashicorp/memberlist"
	"log"
	"sync"
	"time"
)

type MemberStatus int

const (
	StatusNone MemberStatus = iota
	StatusAlive
	StatusLeaving
	StatusLeft
	StatusFailed
	StatusPartitioned // Partitioned is a best guess, should be treated as failed
)

type Serf struct {
	conf       *Config
	memberlist *memberlist.Memberlist
	joinCh     chan *memberlist.Node
	leaveCh    chan *memberlist.Node
	shutdownCh chan struct{}

	memberLock sync.RWMutex
	members    []*Member
	memberMap  map[string]*Member

	detector partitionDetector

	broadcasts *memberlist.TransmitLimitedQueue

	changeCh chan statusChange
}

// Member represents a single member in the gossip pool
type Member struct {
	Name   string
	Addr   []byte
	Role   string
	Status MemberStatus
}

type EventDelegate interface {
	// MembersJoined invoked when members have joined the cluster
	MembersJoined([]*Member)

	// MembersLeft invoked when members have left the cluster gracefully
	MembersLeft([]*Member)

	// MembersFailed invoked when members are unreachable by the cluster without
	// previously announcing their intention to leave
	MembersFailed([]*Member)

	// MembersPartitioned invoked when members are unreachable by cluster without
	// previously announcing their intention to leave. Partitions are impossible
	// to tell apart from failures, so this is more of a heuristic based on the
	// likelyhood of simultaneous failures. It should not be treated as exact.
	MembersPartitioned([]*Member)
}

// Start is used to initialize a new Serf instance
func Start(conf *Config) (*Serf, error) {
	serf := &Serf{
		conf:       conf,
		joinCh:     make(chan *memberlist.Node, 64),
		leaveCh:    make(chan *memberlist.Node, 64),
		shutdownCh: make(chan struct{}),
		changeCh:   make(chan statusChange, 1024),
	}

	// Select a partition detector
	if conf.PartitionCount > 0 && conf.PartitionInterval > 0 {
		serf.detector = newPartitionRing(conf.PartitionCount, conf.PartitionInterval)
	} else {
		// Parititon detection disabled
		serf.detector = noopDetector{}
	}

	// Create the broadcast queue
	serf.broadcasts = &memberlist.TransmitLimitedQueue{
		NumNodes:       func() int { return len(serf.members) },
		RetransmitMult: conf.RetransmitMult,
	}

	// Create the memberlist config
	mlConf := memberlistConfig(conf)
	mlConf.JoinCh = serf.joinCh
	mlConf.LeaveCh = serf.leaveCh
	mlConf.UserDelegate = serf

	// Attempt to create the
	memb, err := memberlist.Create(mlConf)
	if err != nil {
		return nil, err
	}
	serf.memberlist = memb

	// Start the event handelr
	go serf.eventHandler()

	// Start the change handler
	go serf.changeHandler()

	// Done
	return serf, nil
}

// Join is used to attempt to join an existing gossip pool
// Returns an error if none of the existing nodes could be contacted
func (s *Serf) Join(existing []string) error {
	// Ensure we have some input
	if len(existing) == 0 {
		return fmt.Errorf("must specify at least one node to join")
	}

	// Use memberlist to perform the join
	_, err := s.memberlist.Join(existing)
	return err
}

// Members provides a point-in-time snapshot of the members
func (s *Serf) Members() []*Member {
	s.memberLock.RLock()
	defer s.memberLock.RUnlock()

	members := make([]*Member, len(s.members))
	for idx, m := range s.members {
		newM := &Member{}
		*newM = *m
		members[idx] = newM
	}

	return members
}

// Leave allows a node to gracefully leave the cluster. This
// should be followed by a call to Shutdown
func (s *Serf) Leave() error {
	var notifyCh chan struct{}
	var l leave

	// No need to broadcast if there is nobody else
	if len(s.members) <= 1 {
		goto AFTER_BROADCAST
	}

	// Create a channel to get notified
	notifyCh = make(chan struct{}, 1)

	// Broadcast leave intention
	l = leave{s.conf.Hostname}
	if err := s.encodeBroadcastNotify(leaveMsg, &l, notifyCh); err != nil {
		return err
	}

	select {
	case <-notifyCh:
	case <-time.After(s.conf.LeaveTimeout):
		log.Printf("[WARN] Timed out broadcasting leave intention")
	}

AFTER_BROADCAST:
	// Broadcast our own death
	return s.memberlist.Leave()
}

// Shutdown is used to shutdown all the listeners. It is not graceful,
// and should be preceeded by a call to Leave.
func (s *Serf) Shutdown() error {
	// Emit once per background routine (eventHandler, changeHandler)
	close(s.shutdownCh)
	return s.memberlist.Shutdown()
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/gob"
	"encoding/json"
	"fmt"
	"os"
	"sort"
	"strconv"
)

type SerializableVar struct {
	Type     string
	V        string
	Origin   string
	Children []SerializableVar
}

type SerializableDepNode struct {
	Output             string
	Cmds               []string
	Deps               []string
	HasRule            bool
	IsOrderOnly        bool
	IsPhony            bool
	ActualInputs       []string
	TargetSpecificVars []int
	Filename           string
	Lineno             int
}

type SerializableTargetSpecificVar struct {
	Name  string
	Value SerializableVar
}

type SerializableGraph struct {
	Nodes []*SerializableDepNode
	Vars  map[string]SerializableVar
	Tsvs  []SerializableTargetSpecificVar
}

func encGob(v interface{}) string {
	var buf bytes.Buffer
	e := gob.NewEncoder(&buf)
	err := e.Encode(v)
	if err != nil {
		panic(err)
	}
	return buf.String()
}

type DepNodesSerializer struct {
	nodes  []*SerializableDepNode
	tsvs   []SerializableTargetSpecificVar
	tsvMap map[string]int
	done   map[string]bool
}

func NewDepNodesSerializer() *DepNodesSerializer {
	return &DepNodesSerializer{
		tsvMap: make(map[string]int),
		done:   make(map[string]bool),
	}
}

func (ns *DepNodesSerializer) SerializeDepNodes(nodes []*DepNode) {
	for _, n := range nodes {
		if ns.done[n.Output] {
			continue
		}
		ns.done[n.Output] = true

		var deps []string
		for _, d := range n.Deps {
			deps = append(deps, d.Output)
		}

		// Sort keys for consistent serialization.
		var tsvKeys []string
		for k := range n.TargetSpecificVars {
			tsvKeys = append(tsvKeys, k)
		}
		sort.Strings(tsvKeys)

		var vars []int
		for _, k := range tsvKeys {
			v := n.TargetSpecificVars[k]
			sv := SerializableTargetSpecificVar{Name: k, Value: v.Serialize()}
			gob := encGob(sv)
			id, present := ns.tsvMap[gob]
			if !present {
				id = len(ns.tsvs)
				ns.tsvMap[gob] = id
				ns.tsvs = append(ns.tsvs, sv)
			}
			vars = append(vars, id)
		}

		ns.nodes = append(ns.nodes, &SerializableDepNode{
			Output:             n.Output,
			Cmds:               n.Cmds,
			Deps:               deps,
			HasRule:            n.HasRule,
			IsOrderOnly:        n.IsOrderOnly,
			IsPhony:            n.IsPhony,
			ActualInputs:       n.ActualInputs,
			TargetSpecificVars: vars,
			Filename:           n.Filename,
			Lineno:             n.Lineno,
		})
		ns.SerializeDepNodes(n.Deps)
	}
}

func MakeSerializableVars(vars Vars) (r map[string]SerializableVar) {
	r = make(map[string]SerializableVar)
	for k, v := range vars {
		r[k] = v.Serialize()
	}
	return r
}

func MakeSerializableGraph(nodes []*DepNode, vars Vars) SerializableGraph {
	ns := NewDepNodesSerializer()
	ns.SerializeDepNodes(nodes)
	v := MakeSerializableVars(vars)
	return SerializableGraph{Nodes: ns.nodes, Vars: v, Tsvs: ns.tsvs}
}

func DumpDepGraphAsJson(nodes []*DepNode, vars Vars, filename string) {
	o, err := json.MarshalIndent(MakeSerializableGraph(nodes, vars), " ", " ")
	if err != nil {
		panic(err)
	}
	f, err2 := os.Create(filename)
	if err2 != nil {
		panic(err2)
	}
	f.Write(o)
}

func DumpDepGraph(nodes []*DepNode, vars Vars, filename string) {
	f, err := os.Create(filename)
	if err != nil {
		panic(err)
	}
	e := gob.NewEncoder(f)
	e.Encode(MakeSerializableGraph(nodes, vars))
}

func DeserializeSingleChild(sv SerializableVar) Value {
	if len(sv.Children) != 1 {
		panic(fmt.Sprintf("unexpected number of children: %q", sv))
	}
	return DeserializeVar(sv.Children[0])
}

func DeserializeVar(sv SerializableVar) (r Value) {
	switch sv.Type {
	case "literal":
		return literal(sv.V)
	case "tmpval":
		return tmpval([]byte(sv.V))
	case "expr":
		var e Expr
		for _, v := range sv.Children {
			e = append(e, DeserializeVar(v))
		}
		return e
	case "varref":
		return varref{varname: DeserializeSingleChild(sv)}
	case "paramref":
		v, err := strconv.Atoi(sv.V)
		if err != nil {
			panic(err)
		}
		return paramref(v)
	case "varsubst":
		return varsubst{
			varname: DeserializeVar(sv.Children[0]),
			pat:     DeserializeVar(sv.Children[1]),
			subst:   DeserializeVar(sv.Children[2]),
		}

	case "func":
		name := DeserializeVar(sv.Children[0]).(literal)
		f := funcMap[string(name[1:])]()
		f.AddArg(name)
		for _, a := range sv.Children[1:] {
			f.AddArg(DeserializeVar(a))
		}
		return f
	case "funcEvalAssign":
		return &funcEvalAssign{
			lhs: sv.Children[0].V,
			op:  sv.Children[1].V,
			rhs: DeserializeVar(sv.Children[2]),
		}
	case "funcNop":
		return &funcNop{expr: sv.V}

	case "simple":
		return SimpleVar{
			value:  []byte(sv.V),
			origin: sv.Origin,
		}
	case "recursive":
		return RecursiveVar{
			expr:   DeserializeSingleChild(sv),
			origin: sv.Origin,
		}

	case ":=", "=", "+=", "?=":
		return TargetSpecificVar{
			v:  DeserializeSingleChild(sv).(Var),
			op: sv.Type,
		}

	default:
		panic(fmt.Sprintf("unknown serialized variable type: %q", sv))
	}
	return UndefinedVar{}
}

func DeserializeVars(vars map[string]SerializableVar) Vars {
	r := make(Vars)
	for k, v := range vars {
		r[k] = DeserializeVar(v).(Var)
	}
	return r
}

func DeserializeNodes(nodes []*SerializableDepNode, tsvs []SerializableTargetSpecificVar) (r []*DepNode) {
	nodeMap := make(map[string]*DepNode)
	for _, n := range nodes {
		d := &DepNode{
			Output:             n.Output,
			Cmds:               n.Cmds,
			HasRule:            n.HasRule,
			IsOrderOnly:        n.IsOrderOnly,
			IsPhony:            n.IsPhony,
			ActualInputs:       n.ActualInputs,
			Filename:           n.Filename,
			Lineno:             n.Lineno,
			TargetSpecificVars: make(Vars),
		}

		for _, id := range n.TargetSpecificVars {
			sv := tsvs[id]
			d.TargetSpecificVars[sv.Name] = DeserializeVar(sv.Value).(Var)
		}

		nodeMap[n.Output] = d
		r = append(r, d)
	}

	for _, n := range nodes {
		d := nodeMap[n.Output]
		for _, o := range n.Deps {
			c, present := nodeMap[o]
			if !present {
				panic(fmt.Sprintf("unknown target: %s", o))
			}
			d.Deps = append(d.Deps, c)
		}
	}

	return r
}

func DeserializeGraph(g SerializableGraph) ([]*DepNode, Vars) {
	nodes := DeserializeNodes(g.Nodes, g.Tsvs)
	vars := DeserializeVars(g.Vars)
	return nodes, vars
}

func LoadDepGraphFromJson(filename string) ([]*DepNode, Vars) {
	f, err := os.Open(filename)
	if err != nil {
		panic(err)
	}

	d := json.NewDecoder(f)
	g := SerializableGraph{Vars: make(map[string]SerializableVar)}
	err = d.Decode(&g)
	if err != nil {
		panic(err)
	}
	return DeserializeGraph(g)
}

func LoadDepGraph(filename string) ([]*DepNode, Vars) {
	f, err := os.Open(filename)
	if err != nil {
		panic(err)
	}

	d := gob.NewDecoder(f)
	g := SerializableGraph{Vars: make(map[string]SerializableVar)}
	err = d.Decode(&g)
	if err != nil {
		panic(err)
	}
	return DeserializeGraph(g)
}
<commit_msg>Do not serialize duplicated TSVs multiple times<commit_after>package main

import (
	"bytes"
	"encoding/gob"
	"encoding/json"
	"fmt"
	"os"
	"sort"
	"strconv"
)

type SerializableVar struct {
	Type     string
	V        string
	Origin   string
	Children []SerializableVar
}

type SerializableDepNode struct {
	Output             string
	Cmds               []string
	Deps               []string
	HasRule            bool
	IsOrderOnly        bool
	IsPhony            bool
	ActualInputs       []string
	TargetSpecificVars []int
	Filename           string
	Lineno             int
}

type SerializableTargetSpecificVar struct {
	Name  string
	Value SerializableVar
}

type SerializableGraph struct {
	Nodes []*SerializableDepNode
	Vars  map[string]SerializableVar
	Tsvs  []SerializableTargetSpecificVar
}

func encGob(v interface{}) string {
	var buf bytes.Buffer
	e := gob.NewEncoder(&buf)
	err := e.Encode(v)
	if err != nil {
		panic(err)
	}
	return buf.String()
}

type DepNodesSerializer struct {
	nodes  []*SerializableDepNode
	tsvs   []SerializableTargetSpecificVar
	tsvMap map[string]int
	done   map[string]bool
}

func NewDepNodesSerializer() *DepNodesSerializer {
	return &DepNodesSerializer{
		tsvMap: make(map[string]int),
		done:   make(map[string]bool),
	}
}

func (ns *DepNodesSerializer) SerializeDepNodes(nodes []*DepNode) {
	for _, n := range nodes {
		if ns.done[n.Output] {
			continue
		}
		ns.done[n.Output] = true

		var deps []string
		for _, d := range n.Deps {
			deps = append(deps, d.Output)
		}

		// Sort keys for consistent serialization.
		var tsvKeys []string
		for k := range n.TargetSpecificVars {
			tsvKeys = append(tsvKeys, k)
		}
		sort.Strings(tsvKeys)

		var vars []int
		for _, k := range tsvKeys {
			v := n.TargetSpecificVars[k]
			sv := SerializableTargetSpecificVar{Name: k, Value: v.Serialize()}
			gob := encGob(sv)
			id, present := ns.tsvMap[gob]
			if !present {
				id = len(ns.tsvs)
				ns.tsvMap[gob] = id
				ns.tsvs = append(ns.tsvs, sv)
			}
			vars = append(vars, id)
		}

		ns.nodes = append(ns.nodes, &SerializableDepNode{
			Output:             n.Output,
			Cmds:               n.Cmds,
			Deps:               deps,
			HasRule:            n.HasRule,
			IsOrderOnly:        n.IsOrderOnly,
			IsPhony:            n.IsPhony,
			ActualInputs:       n.ActualInputs,
			TargetSpecificVars: vars,
			Filename:           n.Filename,
			Lineno:             n.Lineno,
		})
		ns.SerializeDepNodes(n.Deps)
	}
}

func MakeSerializableVars(vars Vars) (r map[string]SerializableVar) {
	r = make(map[string]SerializableVar)
	for k, v := range vars {
		r[k] = v.Serialize()
	}
	return r
}

func MakeSerializableGraph(nodes []*DepNode, vars Vars) SerializableGraph {
	ns := NewDepNodesSerializer()
	ns.SerializeDepNodes(nodes)
	v := MakeSerializableVars(vars)
	return SerializableGraph{Nodes: ns.nodes, Vars: v, Tsvs: ns.tsvs}
}

func DumpDepGraphAsJson(nodes []*DepNode, vars Vars, filename string) {
	o, err := json.MarshalIndent(MakeSerializableGraph(nodes, vars), " ", " ")
	if err != nil {
		panic(err)
	}
	f, err2 := os.Create(filename)
	if err2 != nil {
		panic(err2)
	}
	f.Write(o)
}

func DumpDepGraph(nodes []*DepNode, vars Vars, filename string) {
	f, err := os.Create(filename)
	if err != nil {
		panic(err)
	}
	e := gob.NewEncoder(f)
	e.Encode(MakeSerializableGraph(nodes, vars))
}

func DeserializeSingleChild(sv SerializableVar) Value {
	if len(sv.Children) != 1 {
		panic(fmt.Sprintf("unexpected number of children: %q", sv))
	}
	return DeserializeVar(sv.Children[0])
}

func DeserializeVar(sv SerializableVar) (r Value) {
	switch sv.Type {
	case "literal":
		return literal(sv.V)
	case "tmpval":
		return tmpval([]byte(sv.V))
	case "expr":
		var e Expr
		for _, v := range sv.Children {
			e = append(e, DeserializeVar(v))
		}
		return e
	case "varref":
		return varref{varname: DeserializeSingleChild(sv)}
	case "paramref":
		v, err := strconv.Atoi(sv.V)
		if err != nil {
			panic(err)
		}
		return paramref(v)
	case "varsubst":
		return varsubst{
			varname: DeserializeVar(sv.Children[0]),
			pat:     DeserializeVar(sv.Children[1]),
			subst:   DeserializeVar(sv.Children[2]),
		}

	case "func":
		name := DeserializeVar(sv.Children[0]).(literal)
		f := funcMap[string(name[1:])]()
		f.AddArg(name)
		for _, a := range sv.Children[1:] {
			f.AddArg(DeserializeVar(a))
		}
		return f
	case "funcEvalAssign":
		return &funcEvalAssign{
			lhs: sv.Children[0].V,
			op:  sv.Children[1].V,
			rhs: DeserializeVar(sv.Children[2]),
		}
	case "funcNop":
		return &funcNop{expr: sv.V}

	case "simple":
		return SimpleVar{
			value:  []byte(sv.V),
			origin: sv.Origin,
		}
	case "recursive":
		return RecursiveVar{
			expr:   DeserializeSingleChild(sv),
			origin: sv.Origin,
		}

	case ":=", "=", "+=", "?=":
		return TargetSpecificVar{
			v:  DeserializeSingleChild(sv).(Var),
			op: sv.Type,
		}

	default:
		panic(fmt.Sprintf("unknown serialized variable type: %q", sv))
	}
	return UndefinedVar{}
}

func DeserializeVars(vars map[string]SerializableVar) Vars {
	r := make(Vars)
	for k, v := range vars {
		r[k] = DeserializeVar(v).(Var)
	}
	return r
}

func DeserializeNodes(nodes []*SerializableDepNode, tsvs []SerializableTargetSpecificVar) (r []*DepNode) {
	// Deserialize all TSVs first so that multiple rules can share memory.
	var tsvValues []Var
	for _, sv := range tsvs {
		tsvValues = append(tsvValues, DeserializeVar(sv.Value).(Var))
	}

	nodeMap := make(map[string]*DepNode)
	for _, n := range nodes {
		d := &DepNode{
			Output:             n.Output,
			Cmds:               n.Cmds,
			HasRule:            n.HasRule,
			IsOrderOnly:        n.IsOrderOnly,
			IsPhony:            n.IsPhony,
			ActualInputs:       n.ActualInputs,
			Filename:           n.Filename,
			Lineno:             n.Lineno,
			TargetSpecificVars: make(Vars),
		}

		for _, id := range n.TargetSpecificVars {
			sv := tsvs[id]
			d.TargetSpecificVars[sv.Name] = tsvValues[id]
		}

		nodeMap[n.Output] = d
		r = append(r, d)
	}

	for _, n := range nodes {
		d := nodeMap[n.Output]
		for _, o := range n.Deps {
			c, present := nodeMap[o]
			if !present {
				panic(fmt.Sprintf("unknown target: %s", o))
			}
			d.Deps = append(d.Deps, c)
		}
	}

	return r
}

func DeserializeGraph(g SerializableGraph) ([]*DepNode, Vars) {
	nodes := DeserializeNodes(g.Nodes, g.Tsvs)
	vars := DeserializeVars(g.Vars)
	return nodes, vars
}

func LoadDepGraphFromJson(filename string) ([]*DepNode, Vars) {
	f, err := os.Open(filename)
	if err != nil {
		panic(err)
	}

	d := json.NewDecoder(f)
	g := SerializableGraph{Vars: make(map[string]SerializableVar)}
	err = d.Decode(&g)
	if err != nil {
		panic(err)
	}
	return DeserializeGraph(g)
}

func LoadDepGraph(filename string) ([]*DepNode, Vars) {
	f, err := os.Open(filename)
	if err != nil {
		panic(err)
	}

	d := gob.NewDecoder(f)
	g := SerializableGraph{Vars: make(map[string]SerializableVar)}
	err = d.Decode(&g)
	if err != nil {
		panic(err)
	}
	return DeserializeGraph(g)
}
<|endoftext|>


<commit_before>package main

import (
	"labix.org/v2/mgo"
	"labix.org/v2/mgo/bson"
	"log"
	"errors"
)

var ErrorNotFound = errors.New("No documents found!")
var ErrorNullResponse = errors.New("Got back null response from mgo.")

type Database struct {
	db *mgo.Database
	collections map[string]*Collection
}

func NewDatabase(host string) *Database {
	s,err := mgo.Dial(host)
	if err != nil {
		panic(err)
	}
	mdb := s.DB("askeecs")

	dbs := new(Database)
	dbs.db = mdb
	dbs.collections = make(map[string]*Collection)
	return dbs
}

func (db *Database) Collection(name string, typ I) *Collection {
	c,ok := db.collections[name]
	if ok {
		return c
	}

	c = new(Collection)
	c.col = db.db.C(name)
	c.template = typ
	return c
}

type I interface {
	GetID() bson.ObjectId
	New() I
}

type Collection struct {
	col *mgo.Collection
	cache map[string]I
	template I
}

func (c *Collection) Save(doc I) error {
	//TODO: handle errors?
	log.Printf("Saving document.")
	err := c.col.Insert(doc)
	return err
}

func (c *Collection) FindByID(ID bson.ObjectId) I {
	q := c.col.FindId(ID)
	if q == nil {
		log.Println(ErrorNullResponse)
		return nil
	}
	cnt,err := q.Count()
	if err != nil {
		log.Println(err)
		return nil
	}
	if cnt < 1 {
		log.Println(ErrorNotFound)
		return nil
	}
	obj := c.template.New()
	q.One(obj)
	return obj
}

func (c *Collection) FindWhere(match bson.M) []I {
	log.Println(match)
	q := c.col.Find(match)
	if q == nil {
		log.Println(ErrorNullResponse)
		return nil
	}

	n,err := q.Count()
	if err != nil {
		log.Println(err)
		return nil
	}
	if n == 0 {
		log.Println("Nothing matched the query...")
		return nil
	}

	var out []I
	i := q.Iter()
	v := c.template.New()
	for i.Next(v) {
		out = append(out,v)
	}
	return out
}
<commit_msg>fix bug in findwhere<commit_after>package main

import (
	"labix.org/v2/mgo"
	"labix.org/v2/mgo/bson"
	"log"
	"errors"
)

var ErrorNotFound = errors.New("No documents found!")
var ErrorNullResponse = errors.New("Got back null response from mgo.")

type Database struct {
	db *mgo.Database
	collections map[string]*Collection
}

func NewDatabase(host string) *Database {
	s,err := mgo.Dial(host)
	if err != nil {
		panic(err)
	}
	mdb := s.DB("askeecs")

	dbs := new(Database)
	dbs.db = mdb
	dbs.collections = make(map[string]*Collection)
	return dbs
}

func (db *Database) Collection(name string, typ I) *Collection {
	c,ok := db.collections[name]
	if ok {
		return c
	}

	c = new(Collection)
	c.col = db.db.C(name)
	c.template = typ
	return c
}

type I interface {
	GetID() bson.ObjectId
	New() I
}

type Collection struct {
	col *mgo.Collection
	cache map[string]I
	template I
}

func (c *Collection) Save(doc I) error {
	//TODO: handle errors?
	log.Printf("Saving document.")
	err := c.col.Insert(doc)
	return err
}

func (c *Collection) FindByID(ID bson.ObjectId) I {
	q := c.col.FindId(ID)
	if q == nil {
		log.Println(ErrorNullResponse)
		return nil
	}
	cnt,err := q.Count()
	if err != nil {
		log.Println(err)
		return nil
	}
	if cnt < 1 {
		log.Println(ErrorNotFound)
		return nil
	}
	obj := c.template.New()
	q.One(obj)
	return obj
}

func (c *Collection) FindWhere(match bson.M) []I {
	log.Println(match)
	q := c.col.Find(match)
	if q == nil {
		log.Println(ErrorNullResponse)
		return nil
	}

	n,err := q.Count()
	if err != nil {
		log.Println(err)
		return nil
	}
	if n == 0 {
		log.Println("Nothing matched the query...")
		return nil
	}

	var out []I
	i := q.Iter()
	v := c.template.New()
	for i.Next(v) {
		out = append(out,v)
		v = c.template.New()
	}
	return out
}
<|endoftext|>


<commit_before>package kcp

import (
	"crypto/sha1"
	"fmt"
	"log"
	"sync"
	"testing"
	"time"

	"golang.org/x/crypto/pbkdf2"
)

const port = "127.0.0.1:9999"
const salt = "kcptest"

var key = []byte("testkey")
var fec = 4

func DialTest() (*UDPSession, error) {
	pass := pbkdf2.Key(key, []byte(salt), 4096, 32, sha1.New)
	block, _ := NewNoneBlockCrypt(pass)
	//block, _ := NewSimpleXORBlockCrypt(pass)
	//block, _ := NewTEABlockCrypt(pass[:16])
	//block, _ := NewAESBlockCrypt(pass)
	return DialWithOptions(port, block, 10, 3)
}

func ListenTest() (*Listener, error) {
	pass := pbkdf2.Key(key, []byte(salt), 4096, 32, sha1.New)
	block, _ := NewNoneBlockCrypt(pass)
	//block, _ := NewSimpleXORBlockCrypt(pass)
	//block, _ := NewTEABlockCrypt(pass[:16])
	//block, _ := NewAESBlockCrypt(pass)
	return ListenWithOptions(port, block, 10, 3)
}

func server() {
	l, err := ListenTest()
	if err != nil {
		panic(err)
	}
	log.Println("listening on:", l.Addr())
	for {
		s, err := l.Accept()
		if err != nil {
			panic(err)
		}

		go handle_client(s)
	}
}

func init() {
	go server()
}

func handle_client(conn *UDPSession) {
	conn.SetWindowSize(1024, 1024)
	conn.SetNoDelay(1, 20, 2, 1)
	conn.SetDSCP(46)
	conn.SetMtu(1450)
	conn.SetACKNoDelay(false)
	conn.SetReadDeadline(time.Now().Add(time.Hour))
	conn.SetWriteDeadline(time.Now().Add(time.Hour))
	fmt.Println("new client", conn.RemoteAddr())
	buf := make([]byte, 65536)
	count := 0
	for {
		n, err := conn.Read(buf)
		if err != nil {
			panic(err)
		}
		count++
		conn.Write(buf[:n])
	}
}

func TestSendRecv(t *testing.T) {
	var wg sync.WaitGroup
	const par = 1
	wg.Add(par)
	for i := 0; i < par; i++ {
		go client(&wg)
	}
	wg.Wait()
}

func client(wg *sync.WaitGroup) {
	cli, err := DialTest()
	if err != nil {
		panic(err)
	}
	cli.SetNoDelay(1, 20, 2, 1)
	cli.SetACKNoDelay(true)
	cli.SetDeadline(time.Now().Add(time.Minute))
	const N = 100
	buf := make([]byte, 10)
	for i := 0; i < N; i++ {
		msg := fmt.Sprintf("hello%v", i)
		fmt.Println("sent:", msg)
		cli.Write([]byte(msg))
		if n, err := cli.Read(buf); err == nil {
			fmt.Println("recv:", string(buf[:n]))
		} else {
			panic(err)
		}
	}
	cli.Close()
	wg.Done()
}

func TestBigPacket(t *testing.T) {
	var wg sync.WaitGroup
	wg.Add(1)
	go client2(&wg)
	wg.Wait()
}

func client2(wg *sync.WaitGroup) {
	cli, err := DialTest()
	if err != nil {
		panic(err)
	}
	cli.SetNoDelay(1, 20, 2, 1)
	const N = 10
	buf := make([]byte, 1024*512)
	msg := make([]byte, 1024*512)
	for i := 0; i < N; i++ {
		cli.Write(msg)
	}
	println("total written:", len(msg)*N)

	nrecv := 0
	cli.SetReadDeadline(time.Now().Add(3 * time.Second))
	for {
		n, err := cli.Read(buf)
		if err != nil {
			break
		} else {
			nrecv += n
			if nrecv == len(msg)*N {
				break
			}
		}
	}

	println("total recv:", nrecv)
	cli.Close()
	wg.Done()
}

func TestSpeed(t *testing.T) {
	var wg sync.WaitGroup
	wg.Add(1)
	go client3(&wg)
	wg.Wait()
}

func client3(wg *sync.WaitGroup) {
	cli, err := DialTest()
	if err != nil {
		panic(err)
	}
	log.Println("remote:", cli.RemoteAddr(), "local:", cli.LocalAddr())
	log.Println("conv:", cli.GetConv())
	cli.SetNoDelay(1, 20, 2, 1)
	start := time.Now()

	go func() {
		buf := make([]byte, 1024*1024)
		nrecv := 0
		for {
			n, err := cli.Read(buf)
			if err != nil {
				fmt.Println(err)
				break
			} else {
				nrecv += n
				if nrecv == 4096*4096 {
					break
				}
			}
		}
		println("total recv:", nrecv)
		cli.Close()
		fmt.Println("time for 16MB rtt with encryption", time.Now().Sub(start))
		fmt.Printf("%+v\n", DefaultSnmp.Copy())
		wg.Done()
	}()
	msg := make([]byte, 4096)
	cli.SetWindowSize(1024, 1024)
	for i := 0; i < 4096; i++ {
		cli.Write(msg)
	}
}

func TestParallel(t *testing.T) {
	par := 200
	var wg sync.WaitGroup
	wg.Add(par)
	fmt.Println("testing parallel", par, "connections")
	for i := 0; i < par; i++ {
		go client4(&wg)
	}
	wg.Wait()
}

func client4(wg *sync.WaitGroup) {
	cli, err := DialTest()
	if err != nil {
		panic(err)
	}
	const N = 100
	cli.SetNoDelay(1, 20, 2, 1)
	buf := make([]byte, 10)
	for i := 0; i < N; i++ {
		msg := fmt.Sprintf("hello%v", i)
		cli.Write([]byte(msg))
		if _, err := cli.Read(buf); err != nil {
			break
		}
		<-time.After(10 * time.Millisecond)
	}
	cli.Close()
	wg.Done()
}
<commit_msg>update test<commit_after>package kcp

import (
	"crypto/sha1"
	"fmt"
	"log"
	"sync"
	"testing"
	"time"

	"golang.org/x/crypto/pbkdf2"
)

const port = "127.0.0.1:9999"
const salt = "kcptest"

var key = []byte("testkey")
var fec = 4

func DialTest() (*UDPSession, error) {
	pass := pbkdf2.Key(key, []byte(salt), 4096, 32, sha1.New)
	block, _ := NewNoneBlockCrypt(pass)
	//block, _ := NewSimpleXORBlockCrypt(pass)
	//block, _ := NewTEABlockCrypt(pass[:16])
	//block, _ := NewAESBlockCrypt(pass)
	return DialWithOptions(port, block, 10, 3)
}

func ListenTest() (*Listener, error) {
	pass := pbkdf2.Key(key, []byte(salt), 4096, 32, sha1.New)
	block, _ := NewNoneBlockCrypt(pass)
	//block, _ := NewSimpleXORBlockCrypt(pass)
	//block, _ := NewTEABlockCrypt(pass[:16])
	//block, _ := NewAESBlockCrypt(pass)
	return ListenWithOptions(port, block, 10, 3)
}

func server() {
	l, err := ListenTest()
	if err != nil {
		panic(err)
	}
	log.Println("listening on:", l.Addr())
	for {
		s, err := l.Accept()
		if err != nil {
			panic(err)
		}

		go handle_client(s)
	}
}

func init() {
	go server()
}

func handle_client(conn *UDPSession) {
	conn.SetWindowSize(1024, 1024)
	conn.SetNoDelay(1, 20, 2, 1)
	conn.SetDSCP(46)
	conn.SetMtu(1450)
	conn.SetACKNoDelay(false)
	conn.SetReadDeadline(time.Now().Add(time.Hour))
	conn.SetWriteDeadline(time.Now().Add(time.Hour))
	fmt.Println("new client", conn.RemoteAddr())
	buf := make([]byte, 65536)
	count := 0
	for {
		n, err := conn.Read(buf)
		if err != nil {
			panic(err)
		}
		count++
		conn.Write(buf[:n])
	}
}

func TestTimeout(t *testing.T) {
	cli, err := DialTest()
	if err != nil {
		panic(err)
	}
	buf := make([]byte, 10)

	//timeout
	cli.SetDeadline(time.Now().Add(time.Second))
	<-time.After(2 * time.Second)
	n, err := cli.Read(buf)
	if n != 0 || err == nil {
		t.Fail()
	}
	n, err = cli.Write(buf)
	if n != 0 || err == nil {
		t.Fail()
	}
}

func TestClose(t *testing.T) {
	cli, err := DialTest()
	if err != nil {
		panic(err)
	}
	buf := make([]byte, 10)

	cli.Close()
	if cli.Close() == nil {
		t.Fail()
	}
	n, err := cli.Write(buf)
	if n != 0 || err == nil {
		t.Fail()
	}
	n, err = cli.Read(buf)
	if n != 0 || err == nil {
		t.Fail()
	}
}

func TestSendRecv(t *testing.T) {
	var wg sync.WaitGroup
	const par = 1
	wg.Add(par)
	for i := 0; i < par; i++ {
		go client(&wg)
	}
	wg.Wait()
}

func client(wg *sync.WaitGroup) {
	cli, err := DialTest()
	if err != nil {
		panic(err)
	}
	cli.SetNoDelay(1, 20, 2, 1)
	cli.SetACKNoDelay(true)
	cli.SetDeadline(time.Now().Add(time.Minute))
	const N = 100
	buf := make([]byte, 10)
	for i := 0; i < N; i++ {
		msg := fmt.Sprintf("hello%v", i)
		fmt.Println("sent:", msg)
		cli.Write([]byte(msg))
		if n, err := cli.Read(buf); err == nil {
			fmt.Println("recv:", string(buf[:n]))
		} else {
			panic(err)
		}
	}
	cli.Close()
	wg.Done()
}

func TestBigPacket(t *testing.T) {
	var wg sync.WaitGroup
	wg.Add(1)
	go client2(&wg)
	wg.Wait()
}

func client2(wg *sync.WaitGroup) {
	cli, err := DialTest()
	if err != nil {
		panic(err)
	}
	cli.SetNoDelay(1, 20, 2, 1)
	const N = 10
	buf := make([]byte, 1024*512)
	msg := make([]byte, 1024*512)
	for i := 0; i < N; i++ {
		cli.Write(msg)
	}
	println("total written:", len(msg)*N)

	nrecv := 0
	cli.SetReadDeadline(time.Now().Add(3 * time.Second))
	for {
		n, err := cli.Read(buf)
		if err != nil {
			break
		} else {
			nrecv += n
			if nrecv == len(msg)*N {
				break
			}
		}
	}

	println("total recv:", nrecv)
	cli.Close()
	wg.Done()
}

func TestSpeed(t *testing.T) {
	var wg sync.WaitGroup
	wg.Add(1)
	go client3(&wg)
	wg.Wait()
}

func client3(wg *sync.WaitGroup) {
	cli, err := DialTest()
	if err != nil {
		panic(err)
	}
	log.Println("remote:", cli.RemoteAddr(), "local:", cli.LocalAddr())
	log.Println("conv:", cli.GetConv())
	cli.SetNoDelay(1, 20, 2, 1)
	start := time.Now()

	go func() {
		buf := make([]byte, 1024*1024)
		nrecv := 0
		for {
			n, err := cli.Read(buf)
			if err != nil {
				fmt.Println(err)
				break
			} else {
				nrecv += n
				if nrecv == 4096*4096 {
					break
				}
			}
		}
		println("total recv:", nrecv)
		cli.Close()
		fmt.Println("time for 16MB rtt with encryption", time.Now().Sub(start))
		fmt.Printf("%+v\n", DefaultSnmp.Copy())
		wg.Done()
	}()
	msg := make([]byte, 4096)
	cli.SetWindowSize(1024, 1024)
	for i := 0; i < 4096; i++ {
		cli.Write(msg)
	}
}

func TestParallel(t *testing.T) {
	par := 200
	var wg sync.WaitGroup
	wg.Add(par)
	fmt.Println("testing parallel", par, "connections")
	for i := 0; i < par; i++ {
		go client4(&wg)
	}
	wg.Wait()
}

func client4(wg *sync.WaitGroup) {
	cli, err := DialTest()
	if err != nil {
		panic(err)
	}
	const N = 100
	cli.SetNoDelay(1, 20, 2, 1)
	buf := make([]byte, 10)
	for i := 0; i < N; i++ {
		msg := fmt.Sprintf("hello%v", i)
		cli.Write([]byte(msg))
		if _, err := cli.Read(buf); err != nil {
			break
		}
		<-time.After(10 * time.Millisecond)
	}
	cli.Close()
	wg.Done()
}
<|endoftext|>


<commit_before>package rdfloader

/**
 * This module provides the functions needed to read a file tag by tag.
 * Since the documents are written in rdf/xml,
 * Creating an xml reader for reading rdf tags.
 */

import (
	"bufio"
	"errors"
	"io"
	"os"
	"unicode"
)

const WHITESPACE = 1<<'\t' | 1<<'\n' | 1<<'\r' | 1<<' '

type XMLReader struct {
	fileReader *bufio.Reader
}

type Attribute struct {
	name string
	schemaName string
	value string
}

type pair struct {
	first interface{}
	second interface{}
}

type Tag struct {
	schemaName string
	name string
	attrs []Attribute
}

type Block struct {
	// A block is a valid sub-xml.
	// for example:
	// 		1. <tag />
	// 		2. <tag attr="attr" />
	//      3. <tag> value </tag>
	//      4. <parent> <child> value </child> </parent>
	openingTag Tag
	value string
	children []*Block
}

func (xmlReader *XMLReader) peekARune() (r rune, err error) {
	// returns next character in the file without affecting the file pointer
	singleByte, err := xmlReader.fileReader.Peek(1)
	return rune(singleByte[0]), err
}

func (xmlReader *XMLReader) readARune() (rune, error) {
	singleByteArray := make([]byte, 1)
	_, err := xmlReader.fileReader.Read(singleByteArray)
	return rune(singleByteArray[0]), err
}

func (xmlReader *XMLReader) readTill(delim uint64) ([]rune, error) {
	// reads the input file rune by rune till the target rune is found
	//		or eof is reached.
	// Note: it doesn't include the target rune in the read word.
	var buffer []rune
	for {
		r, err := xmlReader.fileReader.Peek(1)
		if err == nil {
			// checking if the read rune is same as any of the delimiters' mask
			if (delim & (1 << r[0])) != 0 {
				// current char is same as one of the delimiters.
				return buffer, nil
			}

			// moving file pointer one character ahead.
			xmlReader.readARune()

			// current character is not one of the delimiters.
			buffer = append(buffer, rune(r[0]))
		} else {
			return buffer, err
		}
	}
}

func (xmlReader *XMLReader) peekNBytes(n int) ([]byte, error) {
	return xmlReader.fileReader.Peek(n)
}

func (xmlReader *XMLReader) readNBytes(n int) (nextNBytes []byte, err error) {
	next2Bytes := make([]byte, n)
	_, err = xmlReader.fileReader.Read(next2Bytes)
	return next2Bytes, err
}

func (xmlReader *XMLReader) ignoreWhiteSpace() (nWS int, err error) {
	// advance the file pointer until a non-blank character is found.
	// nWS: number of whitespaces which were stripped.
	for {
		char, err := xmlReader.peekARune()
		if err != nil || !unicode.IsSpace(char) {
			return nWS, err
		}
		nWS++
		xmlReader.readARune()
	}
}

func (xmlReader *XMLReader) readColonPair(delim uint64) (pair pair, colonFound bool, err error) {
	// reads a:b into a Pair Object.
	word, err := xmlReader.readTill(delim)
	if err != nil { return }

	for i, r := range word {
		if r == ':' {
			colonFound = true
			pair.first = string(word[:i])
			latter := string(word[i+1:])
			if len(latter) == 0 {
				err = errors.New("expected a word after colon")
				return
			}
			pair.second = latter
			break
		}
	}
	if !colonFound {
		// no colon was found.
		pair.first = string(word)
	}
	return
}

func (xmlReader *XMLReader) readAttribute() (attr Attribute, err error) {
	// assumes the file pointer is pointing to the attribute name
	pair, colonExists, err := xmlReader.readColonPair(WHITESPACE | 1 << '=')
	if err != nil { return attr, err }
	if colonExists {
		attr.schemaName = pair.first.(string)
		attr.name = pair.second.(string)
	} else {
		attr.name = pair.first.(string)
	}
	_, err = xmlReader.ignoreWhiteSpace()
	if err != nil { return attr, err }

	nextRune, err := xmlReader.peekARune()
	if err != nil { return attr, err }
	if nextRune != '=' {
		err = errors.New("expected an assignment sign (=)")
	}
	xmlReader.readARune()

	firstQuote, err := xmlReader.readARune()
	if firstQuote != '\'' && firstQuote != '"' {
		err = errors.New("assignment operator must be followed by an attribute enclosed within quotes")
	}

	// read till next quote.
	word, err := xmlReader.readTill(WHITESPACE | 1 << byte(firstQuote))
	if err != nil { return attr, err }

	secondQuote, _ := xmlReader.readARune()
	if firstQuote != secondQuote {
		return attr, errors.New("unexpected blank char. expected a closing quote")
	}

	attr.value = string(word)
	return attr, err
}

func (xmlReader *XMLReader) readOpeningTag() (tag Tag, blockComplete bool, err error) {
	// Opening Tag can be:
	//		<tag[:schema]
	//			[attr=attr_val]
	//			[attr=attr_val]...	>
	// or
	//		<tag[:schema]
	//			[attr=attr_val]
	//			[attr=attr_val]...	/>
	// Second example is a completed block where no value or internal nodes were found.

	var word []rune

	// forward file pointer until a not-space character is found.
	// removing all blank characters before opening bracket.
	_, err = xmlReader.ignoreWhiteSpace()
	if err != nil { return }

	// find the opening angular bracket.
	// after stripping all the spaces, the next character to be read should be '<'
	//   If the next character is not '<',
	//       there are few chars before opening tag. Which is not allowed.
	word, err = xmlReader.readTill(1 << '<')
	if err == io.EOF {
		// we reached the end of the file while searching for a new tag.
		if len(word) > 0 {
			return tag, blockComplete, errors.New("found stray characters at EOF")
		} else {
			// no new tags were found.
			return tag, blockComplete, io.EOF
		}
	}
	if len(word) != 0 {
		return tag, blockComplete, errors.New("found extra chars before tag start")
	}

	// next char is an opening angular bracket.
	xmlReader.readARune()
	xmlReader.ignoreWhiteSpace() // there shouldn't be any for a well-formed rdf/xml document.

	nextRune, _ := xmlReader.peekARune()
	if nextRune == '/' {
		return tag, blockComplete, errors.New("unexpected closing tag")
	}

	// reading the next word till we reach a colon or a blank-char or a closing angular bracket.
	pair, colonExist, err := xmlReader.readColonPair(1 << '>' | WHITESPACE | 1 << '/')
	if err != nil { return }

	if colonExist {
		tag.schemaName = pair.first.(string)
		tag.name = pair.second.(string)
	} else {
		tag.name = pair.first.(string)
	}

	delim, _ := xmlReader.readARune() // read the delimiter.

	switch delim {
	case '>':
		// found end of tag. entire tag was parsed.
		return

	case '/':
		// "<[schemaName:]tag /" was parsed. expecting next character to be a closing angular bracket.
		blockComplete = true

		nextRune, err := xmlReader.peekARune()
		if err != nil {
			return tag, blockComplete, err
		}

		if nextRune == '>' {
			xmlReader.readARune() // advancing file pointer by 1.
		} else {
			err = errors.New("expected closing angular bracket after /")
		}
		return tag, blockComplete, err
	}

	// "<[schemaName:]tagName WhiteSpace" is parsed till now.

	_, err = xmlReader.ignoreWhiteSpace()
	if err != nil { return }

	nextRune, err = xmlReader.peekARune()
	if err != nil { return }

	if nextRune == '>' {
		// opening tag didn't had any attributes.
		tag.name = string(word)
		return
	}

	// there are some attributes to be read.
	for !(nextRune == '>' || nextRune == '/') {
		attr, err := xmlReader.readAttribute()
		if err != nil { return tag, blockComplete, err }

		tag.attrs = append(tag.attrs, attr)
		_, err = xmlReader.ignoreWhiteSpace()
		if err != nil { return tag, blockComplete, err }

		nextRune, err = xmlReader.peekARune()
		if err != nil { return tag, blockComplete, err }
	}

	nextRune, _ = xmlReader.readARune()

	if nextRune == '/' {
		// "<[schemaName:]tag /" was parsed. expecting next character to be a closing angular bracket.
		blockComplete = true

		nextRune, err := xmlReader.peekARune()
		if err != nil { return tag, blockComplete, err }

		if nextRune == '>' {
			xmlReader.readARune() // advancing file pointer by 1.
		} else {
			err = errors.New("expected closing angular bracket after /")
		}
	}
	return tag, blockComplete, err
}

func (xmlReader *XMLReader) readClosingTag() (closingTag Tag, err error) {
	// expects white space to be stripped before receiving
	next2Bytes, err := xmlReader.readNBytes(2)
	if err != nil {
		return closingTag, err
	}
	if string(next2Bytes) != "</" {
		return closingTag, errors.New("expected a closing tag")
	}

	pair, colonExists, err := xmlReader.readColonPair(1 << '>' | WHITESPACE)
	if err != nil {
		return closingTag, err
	}

	if colonExists {
		closingTag.schemaName = pair.first.(string)
		closingTag.name = pair.second.(string)
	} else {
		closingTag.name = pair.first.(string)
	}

	xmlReader.ignoreWhiteSpace()
	nextChar, err := xmlReader.readARune()
	if err != nil {
		return closingTag, err
	}

	if nextChar != '>' {
		return closingTag, errors.New("expected a > char")
	}

	return closingTag, err
}

func (xmlReader *XMLReader) readBlock() (block Block, err error) {
	openingTag, blockComplete, err := xmlReader.readOpeningTag()
	if err != nil {
		return block, err
	}
	block.openingTag = openingTag

	if blockComplete {
		// tag was of this type: <schemaName:tagName />
		return block, err
	}

	xmlReader.ignoreWhiteSpace()
	// <schemaName:tagName> is read till now.
	nextRune, err := xmlReader.peekARune()
	if err != nil { return block, err }

	if nextRune != '<' {
		//	the tag must be wrapping a string resource within it.
		// tag is of type <schemaName:tagName> value </schemaName:tagName>
		word, err := xmlReader.readTill(1 << '<') // according to example, word = value.
		if err != nil {
			return block, err
		}
		block.value = string(word)
	} else {
		// expecting a new tag or the closing tag of the currently read tag.
		nextTwoBytes, err := xmlReader.peekNBytes(2)
		if err != nil {
			return block, err
		}
		for string(nextTwoBytes) != "</" {
			// a new tag is found.
			childBlock, err := xmlReader.readBlock()
			if err != nil {
				return block, err
			}

			block.children = append(block.children, &childBlock)

			xmlReader.ignoreWhiteSpace()
			nextTwoBytes, err = xmlReader.peekNBytes(2)
			if err != nil {
				return block, err
			}
		}
	}

	closingTag, err := xmlReader.readClosingTag()
	if err != nil {
		return block, err
	}

	if closingTag.name != closingTag.name || closingTag.schemaName != closingTag.schemaName {
		// opening and closing tags are not equal.
		return block, errors.New("opening and closing tags doesn't match")
	}
	return block, err
}

func (xmlReader *XMLReader) Read() (rootBlock Block, err error) {
	rootBlock, err = xmlReader.readBlock()
	return rootBlock, err
}

func XMLReaderFromFileObject(fileObject *bufio.Reader) XMLReader {
	return XMLReader{fileObject}
}

func XMLReaderFromFilePath(filePath string) (xmlReader XMLReader, err error) {
	fileReader, err := os.Open(filePath)
	defer fileReader.Close()
	if err != nil {
		return xmlReader, err
	}
	return XMLReaderFromFileObject(bufio.NewReader(fileReader)), nil
}
<commit_msg>Export Variables To Make It Accessible<commit_after>package rdfloader

/**
 * This module provides the functions needed to read a file tag by tag.
 * Since the documents are written in rdf/xml,
 * Creating an xml reader for reading rdf tags.
 */

import (
	"bufio"
	"errors"
	"io"
	"os"
	"unicode"
)

const WHITESPACE = 1<<'\t' | 1<<'\n' | 1<<'\r' | 1<<' '

type XMLReader struct {
	fileReader *bufio.Reader
}

type Attribute struct {
	Name string
	SchemaName string
	Value string
}

type pair struct {
	first interface{}
	second interface{}
}

type Tag struct {
	SchemaName string
	Name string
	Attrs []Attribute
}

type Block struct {
	// A block is a valid sub-xml.
	// for example:
	// 		1. <tag />
	// 		2. <tag attr="attr" />
	//      3. <tag> value </tag>
	//      4. <parent> <child> value </child> </parent>
	OpeningTag Tag
	Value string
	Children []*Block
}

func (xmlReader *XMLReader) peekARune() (r rune, err error) {
	// returns next character in the file without affecting the file pointer
	singleByte, err := xmlReader.fileReader.Peek(1)
	return rune(singleByte[0]), err
}

func (xmlReader *XMLReader) readARune() (rune, error) {
	singleByteArray := make([]byte, 1)
	_, err := xmlReader.fileReader.Read(singleByteArray)
	return rune(singleByteArray[0]), err
}

func (xmlReader *XMLReader) readTill(delim uint64) ([]rune, error) {
	// reads the input file rune by rune till the target rune is found
	//		or eof is reached.
	// Note: it doesn't include the target rune in the read word.
	var buffer []rune
	for {
		r, err := xmlReader.fileReader.Peek(1)
		if err == nil {
			// checking if the read rune is same as any of the delimiters' mask
			if (delim & (1 << r[0])) != 0 {
				// current char is same as one of the delimiters.
				return buffer, nil
			}

			// moving file pointer one character ahead.
			xmlReader.readARune()

			// current character is not one of the delimiters.
			buffer = append(buffer, rune(r[0]))
		} else {
			return buffer, err
		}
	}
}

func (xmlReader *XMLReader) peekNBytes(n int) ([]byte, error) {
	return xmlReader.fileReader.Peek(n)
}

func (xmlReader *XMLReader) readNBytes(n int) (nextNBytes []byte, err error) {
	next2Bytes := make([]byte, n)
	_, err = xmlReader.fileReader.Read(next2Bytes)
	return next2Bytes, err
}

func (xmlReader *XMLReader) ignoreWhiteSpace() (nWS int, err error) {
	// advance the file pointer until a non-blank character is found.
	// nWS: number of whitespaces which were stripped.
	for {
		char, err := xmlReader.peekARune()
		if err != nil || !unicode.IsSpace(char) {
			return nWS, err
		}
		nWS++
		xmlReader.readARune()
	}
}

func (xmlReader *XMLReader) readColonPair(delim uint64) (pair pair, colonFound bool, err error) {
	// reads a:b into a Pair Object.
	word, err := xmlReader.readTill(delim)
	if err != nil { return }

	for i, r := range word {
		if r == ':' {
			colonFound = true
			pair.first = string(word[:i])
			latter := string(word[i+1:])
			if len(latter) == 0 {
				err = errors.New("expected a word after colon")
				return
			}
			pair.second = latter
			break
		}
	}
	if !colonFound {
		// no colon was found.
		pair.first = string(word)
	}
	return
}

func (xmlReader *XMLReader) readAttribute() (attr Attribute, err error) {
	// assumes the file pointer is pointing to the attribute name
	pair, colonExists, err := xmlReader.readColonPair(WHITESPACE | 1 << '=')
	if err != nil { return attr, err }
	if colonExists {
		attr.SchemaName = pair.first.(string)
		attr.Name = pair.second.(string)
	} else {
		attr.Name = pair.first.(string)
	}
	_, err = xmlReader.ignoreWhiteSpace()
	if err != nil { return attr, err }

	nextRune, err := xmlReader.peekARune()
	if err != nil { return attr, err }
	if nextRune != '=' {
		err = errors.New("expected an assignment sign (=)")
	}
	xmlReader.readARune()

	firstQuote, err := xmlReader.readARune()
	if firstQuote != '\'' && firstQuote != '"' {
		err = errors.New("assignment operator must be followed by an attribute enclosed within quotes")
	}

	// read till next quote.
	word, err := xmlReader.readTill(WHITESPACE | 1 << byte(firstQuote))
	if err != nil { return attr, err }

	secondQuote, _ := xmlReader.readARune()
	if firstQuote != secondQuote {
		return attr, errors.New("unexpected blank char. expected a closing quote")
	}

	attr.Value = string(word)
	return attr, err
}

func (xmlReader *XMLReader) readOpeningTag() (tag Tag, blockComplete bool, err error) {
	// Opening Tag can be:
	//		<tag[:schema]
	//			[attr=attr_val]
	//			[attr=attr_val]...	>
	// or
	//		<tag[:schema]
	//			[attr=attr_val]
	//			[attr=attr_val]...	/>
	// Second example is a completed block where no value or internal nodes were found.

	var word []rune

	// forward file pointer until a not-space character is found.
	// removing all blank characters before opening bracket.
	_, err = xmlReader.ignoreWhiteSpace()
	if err != nil { return }

	// find the opening angular bracket.
	// after stripping all the spaces, the next character to be read should be '<'
	//   If the next character is not '<',
	//       there are few chars before opening tag. Which is not allowed.
	word, err = xmlReader.readTill(1 << '<')
	if err == io.EOF {
		// we reached the end of the file while searching for a new tag.
		if len(word) > 0 {
			return tag, blockComplete, errors.New("found stray characters at EOF")
		} else {
			// no new tags were found.
			return tag, blockComplete, io.EOF
		}
	}
	if len(word) != 0 {
		return tag, blockComplete, errors.New("found extra chars before tag start")
	}

	// next char is an opening angular bracket.
	xmlReader.readARune()
	xmlReader.ignoreWhiteSpace() // there shouldn't be any for a well-formed rdf/xml document.

	nextRune, _ := xmlReader.peekARune()
	if nextRune == '/' {
		return tag, blockComplete, errors.New("unexpected closing tag")
	}

	// reading the next word till we reach a colon or a blank-char or a closing angular bracket.
	pair, colonExist, err := xmlReader.readColonPair(1 << '>' | WHITESPACE | 1 << '/')
	if err != nil { return }

	if colonExist {
		tag.SchemaName = pair.first.(string)
		tag.Name = pair.second.(string)
	} else {
		tag.Name = pair.first.(string)
	}

	delim, _ := xmlReader.readARune() // read the delimiter.

	switch delim {
	case '>':
		// found end of tag. entire tag was parsed.
		return

	case '/':
		// "<[schemaName:]tag /" was parsed. expecting next character to be a closing angular bracket.
		blockComplete = true

		nextRune, err := xmlReader.peekARune()
		if err != nil {
			return tag, blockComplete, err
		}

		if nextRune == '>' {
			xmlReader.readARune() // advancing file pointer by 1.
		} else {
			err = errors.New("expected closing angular bracket after /")
		}
		return tag, blockComplete, err
	}

	// "<[schemaName:]tagName WhiteSpace" is parsed till now.

	_, err = xmlReader.ignoreWhiteSpace()
	if err != nil { return }

	nextRune, err = xmlReader.peekARune()
	if err != nil { return }

	if nextRune == '>' {
		// opening tag didn't had any attributes.
		tag.Name = string(word)
		return
	}

	// there are some attributes to be read.
	for !(nextRune == '>' || nextRune == '/') {
		attr, err := xmlReader.readAttribute()
		if err != nil { return tag, blockComplete, err }

		tag.Attrs = append(tag.Attrs, attr)
		_, err = xmlReader.ignoreWhiteSpace()
		if err != nil { return tag, blockComplete, err }

		nextRune, err = xmlReader.peekARune()
		if err != nil { return tag, blockComplete, err }
	}

	nextRune, _ = xmlReader.readARune()

	if nextRune == '/' {
		// "<[schemaName:]tag /" was parsed. expecting next character to be a closing angular bracket.
		blockComplete = true

		nextRune, err := xmlReader.peekARune()
		if err != nil { return tag, blockComplete, err }

		if nextRune == '>' {
			xmlReader.readARune() // advancing file pointer by 1.
		} else {
			err = errors.New("expected closing angular bracket after /")
		}
	}
	return tag, blockComplete, err
}

func (xmlReader *XMLReader) readClosingTag() (closingTag Tag, err error) {
	// expects white space to be stripped before receiving
	next2Bytes, err := xmlReader.readNBytes(2)
	if err != nil {
		return closingTag, err
	}
	if string(next2Bytes) != "</" {
		return closingTag, errors.New("expected a closing tag")
	}

	pair, colonExists, err := xmlReader.readColonPair(1 << '>' | WHITESPACE)
	if err != nil {
		return closingTag, err
	}

	if colonExists {
		closingTag.SchemaName = pair.first.(string)
		closingTag.Name = pair.second.(string)
	} else {
		closingTag.Name = pair.first.(string)
	}

	xmlReader.ignoreWhiteSpace()
	nextChar, err := xmlReader.readARune()
	if err != nil {
		return closingTag, err
	}

	if nextChar != '>' {
		return closingTag, errors.New("expected a > char")
	}

	return closingTag, err
}

func (xmlReader *XMLReader) readBlock() (block Block, err error) {
	openingTag, blockComplete, err := xmlReader.readOpeningTag()
	if err != nil {
		return block, err
	}
	block.OpeningTag = openingTag

	if blockComplete {
		// tag was of this type: <schemaName:tagName />
		return block, err
	}

	xmlReader.ignoreWhiteSpace()
	// <schemaName:tagName> is read till now.
	nextRune, err := xmlReader.peekARune()
	if err != nil { return block, err }

	if nextRune != '<' {
		//	the tag must be wrapping a string resource within it.
		// tag is of type <schemaName:tagName> value </schemaName:tagName>
		word, err := xmlReader.readTill(1 << '<') // according to example, word = value.
		if err != nil {
			return block, err
		}
		block.Value = string(word)
	} else {
		// expecting a new tag or the closing tag of the currently read tag.
		nextTwoBytes, err := xmlReader.peekNBytes(2)
		if err != nil {
			return block, err
		}
		for string(nextTwoBytes) != "</" {
			// a new tag is found.
			childBlock, err := xmlReader.readBlock()
			if err != nil {
				return block, err
			}

			block.Children = append(block.Children, &childBlock)

			xmlReader.ignoreWhiteSpace()
			nextTwoBytes, err = xmlReader.peekNBytes(2)
			if err != nil {
				return block, err
			}
		}
	}

	closingTag, err := xmlReader.readClosingTag()
	if err != nil {
		return block, err
	}

	if closingTag.Name != closingTag.Name || closingTag.SchemaName != closingTag.SchemaName {
		// opening and closing tags are not equal.
		return block, errors.New("opening and closing tags doesn't match")
	}
	return block, err
}

func (xmlReader *XMLReader) Read() (rootBlock Block, err error) {
	rootBlock, err = xmlReader.readBlock()
	return rootBlock, err
}

func XMLReaderFromFileObject(fileObject *bufio.Reader) XMLReader {
	return XMLReader{fileObject}
}

func XMLReaderFromFilePath(filePath string) (xmlReader XMLReader, err error) {
	fileReader, err := os.Open(filePath)
	defer fileReader.Close()
	if err != nil {
		return xmlReader, err
	}
	return XMLReaderFromFileObject(bufio.NewReader(fileReader)), nil
}
<|endoftext|>


<commit_before>package xmlstream

import (
	"encoding/xml"
	"io"
	"reflect"
	"sync"
)

// Tag is the interface implemented by the objects that can be unmarshalled
// by the Parse function.
//
// The method TagName should returns the local name of the XML element to
// unmarshal.
//
// The struct implementing this interface are similar to those passed to
// the function xml.Unmarshal (package encoding/xml).
type Tag interface {
	TagName() string
}

// Handler is the interface implemented by objects that can handle
// the unmarshalled Tag passed as a parameter.
//
// Note that the parameter t is a pointer to the underlying Tag element.
type Handler interface {
	HandleTag(t interface{})
}

// Parse streams an XML file and unmarshals the xml elements it encounter as soon
// as they match the tag name of one of the tags parameters. The unmarshalled tag
// is then passed to the method HandleTag() of the handler.
//
// If the parameter maxRoutines is equals to zero, HandleTag() is always called
// sequentially. If this parameter is greater than zero, at most maxRoutines go
// routines will be started. When no more go routines are available, the callback
// is called sequentially. If the parameter is negative, the parser will launch
// as many go routines as needed. It is equivalent to set maxRoutines to the
// maximal int32 value.
func Parse(r io.Reader, handler Handler, maxRoutines int32, tags ...Tag) (err error) {
	var (
		// Number of go routines currently running.
		rRoutines int32 = 0

		// Mapping between the xml local name and  the underlying type of a Tag.
		nameToType map[string]reflect.Type = make(map[string]reflect.Type)

		// XML decoder.
		decoder *xml.Decoder = xml.NewDecoder(r)

		// Waiting group used to wait for all the goroutines to finish.
		// See http://golang.org/pkg/sync/#example_WaitGroup
		wg sync.WaitGroup
	)

	// Map the xml local name of a Tag to its underlying type.
	nameToType = make(map[string]reflect.Type)
	for _, tag := range tags {
		t := reflect.TypeOf(tag)
		nameToType[tag.TagName()] = t
	}

	// Negative value for `maxRoutines` is equivalent of assigning it
	// to the maximal int32 value. In other words, allow the parser
	// to launch as many go routines as needed.
	if maxRoutines < 0 {
		maxRoutines = 2147483647 // max int32
	}

	for {
		// Read tokens from the XML document in a stream.
		token, err := decoder.Token()
		if err != nil {
			if err == io.EOF {
				// End of file. Expeted behavior.
				err = nil
			}
			break
		}
		// Inspect the type of the token just read.
		switch el := token.(type) {
		case xml.StartElement:
			// Read the tag name and compare with the XML element.
			if tagType, ok := nameToType[el.Name.Local]; ok {
				// create a new tag
				tag := reflect.New(tagType).Interface()
				// Decode a whole chunk of following XML.
				err := decoder.DecodeElement(tag, &el)
				if err != nil {
					break
				}

				// Do some stuff with the retrieved object...
				if rRoutines < maxRoutines {
					// ...In parallel ツ
					wg.Add(1)
					rRoutines++
					go func() {
						defer func() {
							wg.Done()
							rRoutines--
						}()
						handler.HandleTag(tag)
					}()
				} else {
					// ...Sequentially.
					handler.HandleTag(tag)
				}
			}
		}
	}
	wg.Wait()
	return
}
<commit_msg>wg.Wait() is now deferred<commit_after>package xmlstream

import (
	"encoding/xml"
	"io"
	"reflect"
	"sync"
)

// Tag is the interface implemented by the objects that can be unmarshalled
// by the Parse function.
//
// The method TagName should returns the local name of the XML element to
// unmarshal.
//
// The struct implementing this interface are similar to those passed to
// the function xml.Unmarshal (package encoding/xml).
type Tag interface {
	TagName() string
}

// Handler is the interface implemented by objects that can handle
// the unmarshalled Tag passed as a parameter.
//
// Note that the parameter t is a pointer to the underlying Tag element.
type Handler interface {
	HandleTag(t interface{})
}

// Parse streams an XML file and unmarshals the xml elements it encounter as soon
// as they match the tag name of one of the tags parameters. The unmarshalled tag
// is then passed to the method HandleTag() of the handler.
//
// If the parameter maxRoutines is equals to zero, HandleTag() is always called
// sequentially. If this parameter is greater than zero, at most maxRoutines go
// routines will be started. When no more go routines are available, the callback
// is called sequentially. If the parameter is negative, the parser will launch
// as many go routines as needed. It is equivalent to set maxRoutines to the
// maximal int32 value.
func Parse(r io.Reader, handler Handler, maxRoutines int32, tags ...Tag) (err error) {
	var (
		// Number of go routines currently running.
		rRoutines int32 = 0

		// Mapping between the xml local name and  the underlying type of a Tag.
		nameToType map[string]reflect.Type = make(map[string]reflect.Type)

		// XML decoder.
		decoder *xml.Decoder = xml.NewDecoder(r)

		// Waiting group used to wait for all the goroutines to finish.
		// See http://golang.org/pkg/sync/#example_WaitGroup
		wg sync.WaitGroup
	)

	// Map the xml local name of a Tag to its underlying type.
	nameToType = make(map[string]reflect.Type)
	for _, tag := range tags {
		t := reflect.TypeOf(tag)
		nameToType[tag.TagName()] = t
	}

	// Negative value for `maxRoutines` is equivalent of assigning it
	// to the maximal int32 value. In other words, allow the parser
	// to launch as many go routines as needed.
	if maxRoutines < 0 {
		maxRoutines = 2147483647 // max int32
	}

	// Wait for all the goroutines to complete before returning.
	defer wg.Wait()

	for {
		// Read tokens from the XML document in a stream.
		token, err := decoder.Token()
		if err != nil {
			if err == io.EOF {
				// End of file. Expeted behavior.
				err = nil
			}
			break
		}
		// Inspect the type of the token just read.
		switch el := token.(type) {
		case xml.StartElement:
			// Read the tag name and compare with the XML element.
			if tagType, ok := nameToType[el.Name.Local]; ok {
				// create a new tag
				tag := reflect.New(tagType).Interface()
				// Decode a whole chunk of following XML.
				err := decoder.DecodeElement(tag, &el)
				if err != nil {
					break
				}

				// Do some stuff with the retrieved object...
				if rRoutines < maxRoutines {
					// ...In parallel ツ
					wg.Add(1)
					rRoutines++
					go func() {
						defer func() {
							wg.Done()
							rRoutines--
						}()
						handler.HandleTag(tag)
					}()
				} else {
					// ...Sequentially.
					handler.HandleTag(tag)
				}
			}
		}
	}
	return
}
<|endoftext|>


<commit_before>package xsd

import (
	"encoding/xml"

	"aqwari.net/xml/xmltree"
)

type elementKey struct {
	Name, Type xml.Name
}

type schemaIndex struct {
	eltByID map[int]*xmltree.Element
	// name indices can collide since different element
	// types can have the same node.
	idByName map[elementKey]int
}

func (idx *schemaIndex) ByName(name, typ xml.Name) (*xmltree.Element, bool) {
	if id, ok := idx.idByName[elementKey{name, typ}]; ok {
		if el, ok := idx.eltByID[id]; ok {
			return el, true
		}
		panic("bug building schema index; name map does not match ID map")
	}
	return nil, false
}

func (idx *schemaIndex) ElementID(name, typ xml.Name) (int, bool) {
	id, ok := idx.idByName[elementKey{name, typ}]
	return id, ok
}

func indexSchema(schema []*xmltree.Element) *schemaIndex {
	counter := 0
	index := &schemaIndex{
		eltByID:  make(map[int]*xmltree.Element),
		idByName: make(map[elementKey]int),
	}
	for _, root := range schema {
		tns := root.Attr("", "targetNamespace")
		for _, el := range root.Flatten() {
			index.eltByID[counter] = el
			if name := el.Attr("", "name"); name != "" {
				xmlname := el.ResolveDefault(name, tns)
				index.idByName[elementKey{xmlname, el.Name}] = counter
			}
			counter++
		}
	}
	return index
}
<commit_msg>Simplify index used to flatten xsd references<commit_after>package xsd

import (
	"encoding/xml"

	"aqwari.net/xml/xmltree"
)

type elementKey struct {
	Name, Type xml.Name
}

type schemaIndex struct {
	eltByID []*xmltree.Element
	// name indices can collide since different element
	// types can have the same node.
	idByName map[elementKey]int
}

func (idx *schemaIndex) ByName(name, typ xml.Name) (*xmltree.Element, bool) {
	if id, ok := idx.idByName[elementKey{name, typ}]; ok {
		return idx.eltByID[id], true
	}
	return nil, false
}

func (idx *schemaIndex) ElementID(name, typ xml.Name) (int, bool) {
	id, ok := idx.idByName[elementKey{name, typ}]
	return id, ok
}

func indexSchema(schema []*xmltree.Element) *schemaIndex {
	index := &schemaIndex{
		idByName: make(map[elementKey]int),
	}
	for _, root := range schema {
		tns := root.Attr("", "targetNamespace")
		for _, el := range root.Flatten() {
			id := len(index.eltByID)
			index.eltByID = append(index.eltByID, el)
			if name := el.Attr("", "name"); name != "" {
				xmlname := el.ResolveDefault(name, tns)
				index.idByName[elementKey{xmlname, el.Name}] = id
			}
		}
	}
	return index
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/paurea/dump/dnav"
)

var debug bool

func rdFlags(tIval *dnav.DumpDate) {
	py := flag.Int("y", 0, "# years ago")
	pm := flag.Int("m", 0, "# months ago")
	pd := flag.Int("d", 1, "# days ago")
	ph := flag.Int("h", 0, "# hours ago")
	db := flag.Bool("D", false, "debug flag")
	flag.Parse()
	*tIval = *dnav.NewDumpDate(-*py, -*pm, -*pd, -*ph*100)

	dnav.Debug = *db
	debug = *db
}

func Dprintf(format string, a ...interface{}) (n int, err error) {

	if !debug {
		return 0, nil
	}
	return fmt.Fprintf(os.Stderr, "yest: "+format, a...)
}

func usage() {
	log.Fatal("yest [-y=n] [-m=n] [-d=n] [-h=n] [-D] file_path")
}

func main() {
	var (
		roots dnav.Roots
		tIval dnav.DumpDate
		path  string
	)

	rdFlags(&tIval)
	args := flag.Args()
	if len(flag.Args()) != 1 {
		usage()
	}
	path = args[0]
	if path == "" {
		path = "."
	}
	if path[0] != '/' {
		dir, err := os.Getwd()
		if err != nil {
			log.Fatal(err)
		}
		path = dir + "/" + path
	}
	path = filepath.Clean(path)
	Dprintf("path %s\n", path)
	Dprintf("%s\n", &tIval)
	dnav.RdRoots(&roots)
	Dprintf("mainRoot: %v, dumpRoot: %v, rootName: %v\n", roots.MainRoot, roots.DumpRoot, roots.RootName)

	t := time.Now()
	Dprintf("Now %v\n", t)
	dDate := dnav.TimeAddDate(t, tIval)
	isD := dnav.IsDump(path, roots)
	if isD {
		dDateDmp, err := dnav.ParseDumpPath(path, roots)
		if err == nil {
			dDate = dnav.SumDates(dDateDmp, tIval)
		}
	}
	Dprintf("date %v\n", dDate)
	yestpath := dnav.FindDumpPath(dDate, roots)
	if yestpath == roots.DumpRoot {
		log.Fatal("could not find dump")
	}
	Dprintf("partial %s\n", yestpath)
	if isD {
		yestpath = yestpath + path[len(yestpath):]
	} else {
		suff := strings.TrimPrefix(path, roots.MainRoot)
		Dprintf("suff %s\n", suff)

		yestpath = yestpath + "/" + roots.RootName + suff
	}
	yestpath = filepath.Clean(yestpath)
	if _, err := os.Stat(yestpath); err != nil {
		fmt.Fprintf(os.Stderr, "path does not exist: %s", err)
	}
	zDate := dnav.DumpDate{}
	if zDate != dDate && strings.HasPrefix(yestpath, path) {
		log.Fatal("could not find previous file in dump")
	}

	fmt.Println(yestpath)

}
<commit_msg>fix yest -h 1 mean one hour ago today and not yesterday<commit_after>package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/paurea/dump/dnav"
)

var debug bool

func rdFlags(tIval *dnav.DumpDate) {
	py := flag.Int("y", 0, "# years ago")
	pm := flag.Int("m", 0, "# months ago")
	pd := flag.Int("d", 0, "# days ago")
	ph := flag.Int("h", 0, "# hours ago")
	db := flag.Bool("D", false, "debug flag")
	flag.Parse()
	nDateFl := flag.NFlag()
	if *db {
		nDateFl--
	}
	if nDateFl == 0 {
		*pd =1	//if no flags, yesterday means yesterday
	}
	*tIval = *dnav.NewDumpDate(-*py, -*pm, -*pd, -*ph*100)

	dnav.Debug = *db
	debug = *db
}

func Dprintf(format string, a ...interface{}) (n int, err error) {

	if !debug {
		return 0, nil
	}
	return fmt.Fprintf(os.Stderr, "yest: "+format, a...)
}

func usage() {
	log.Fatal("yest [-y=n] [-m=n] [-d=n] [-h=n] [-D] file_path")
}

func main() {
	var (
		roots dnav.Roots
		tIval dnav.DumpDate
		path  string
	)

	rdFlags(&tIval)
	args := flag.Args()
	if len(flag.Args()) != 1 {
		usage()
	}
	path = args[0]
	if path == "" {
		path = "."
	}
	if path[0] != '/' {
		dir, err := os.Getwd()
		if err != nil {
			log.Fatal(err)
		}
		path = dir + "/" + path
	}
	path = filepath.Clean(path)
	Dprintf("path %s\n", path)
	Dprintf("%s\n", &tIval)
	dnav.RdRoots(&roots)
	Dprintf("mainRoot: %v, dumpRoot: %v, rootName: %v\n", roots.MainRoot, roots.DumpRoot, roots.RootName)

	t := time.Now()
	Dprintf("Now %v\n", t)
	dDate := dnav.TimeAddDate(t, tIval)
	isD := dnav.IsDump(path, roots)
	if isD {
		dDateDmp, err := dnav.ParseDumpPath(path, roots)
		if err == nil {
			dDate = dnav.SumDates(dDateDmp, tIval)
		}
	}
	Dprintf("date %v\n", dDate)
	yestpath := dnav.FindDumpPath(dDate, roots)
	if yestpath == roots.DumpRoot {
		log.Fatal("could not find dump")
	}
	Dprintf("partial %s\n", yestpath)
	if isD {
		yestpath = yestpath + path[len(yestpath):]
	} else {
		suff := strings.TrimPrefix(path, roots.MainRoot)
		Dprintf("suff %s\n", suff)

		yestpath = yestpath + "/" + roots.RootName + suff
	}
	yestpath = filepath.Clean(yestpath)
	if _, err := os.Stat(yestpath); err != nil {
		fmt.Fprintf(os.Stderr, "path does not exist: %s", err)
	}
	zDate := dnav.DumpDate{}
	if zDate != dDate && strings.HasPrefix(yestpath, path) {
		log.Fatal("could not find previous file in dump")
	}

	fmt.Println(yestpath)

}
<|endoftext|>