<commit_before>package kite

import "sync"

// MethodHandling defines how to handle chaining of kite.Handler middlewares.
// An error breaks the chain regardless of what handling is used. Note that all
// Pre and Post handlers are executed regardless the handling logic, only the
// return paramater is defined by the handling mode.
type MethodHandling int

const (
	// ReturnMethod returns main method's response. This is the standard default.
	ReturnMethod MethodHandling = iota

	// ReturnFirst returns the first non-nil response.
	ReturnFirst

	// ReturnLatest returns the latest response (waterfall behaviour)
	ReturnLatest
)

// Objects implementing the Handler interface can be registered to a method.
// The returned result must be marshalable with json package.
type Handler interface {
	ServeKite(*Request) (result interface{}, err error)
}

// HandlerFunc is a type adapter to allow the use of ordinary functions as
// Kite handlers. If h is a function with the appropriate signature,
// HandlerFunc(h) is a Handler object that calls h.
type HandlerFunc func(*Request) (result interface{}, err error)

// ServeKite calls h(r)
func (h HandlerFunc) ServeKite(r *Request) (interface{}, error) {
	return h(r)
}

// Method defines a method and the Handler it is bind to. By default
// "ReturnMethod" handling is used.
type Method struct {
	// name is the method name. Unnamed methods can exist
	name string

	// handler contains the related Handler for the given method
	handler      Handler   // handler is the base handler, the response of it is returned as the final
	preHandlers  []Handler // a list of handlers that are executed before the main handler
	postHandlers []Handler // a list of handlers that are executed after the main handler

	// authenticate defines if a given authenticator function is enabled for
	// the given auth type in the request.
	authenticate bool

	// handling defines how to handle chaining of kite.Handler middlewares.
	handling MethodHandling

	// initialized is used to indicate whether all pre and post handlers are
	// initialized.
	initialized bool

	mu sync.Mutex // protects handler slices
}

// addHandle is an internal method to add a handler
func (k *Kite) addHandle(method string, handler Handler) *Method {
	authenticate := true
	if k.Config.DisableAuthentication {
		authenticate = false
	}

	m := &Method{
		name:         method,
		handler:      handler,
		preHandlers:  make([]Handler, 0),
		postHandlers: make([]Handler, 0),
		authenticate: authenticate,
		handling:     k.MethodHandling,
	}

	k.handlers[method] = m
	return m
}

// DisableAuthentication disables authentication check for this method.
func (m *Method) DisableAuthentication() *Method {
	m.authenticate = false
	return m
}

// PreHandler adds a new kite handler which is executed before the method.
func (m *Method) PreHandle(handler Handler) *Method {
	m.preHandlers = append(m.preHandlers, handler)
	return m
}

// PreHandlerFunc adds a new kite handlerfunc which is executed before the
// method.
func (m *Method) PreHandleFunc(handler HandlerFunc) *Method {
	return m.PreHandle(handler)
}

// PostHandle adds a new kite handler which is executed after the method.
func (m *Method) PostHandle(handler Handler) *Method {
	m.postHandlers = append(m.postHandlers, handler)
	return m
}

// PostHandlerFunc adds a new kite handlerfunc which is executed before the
// method.
func (m *Method) PostHandleFunc(handler HandlerFunc) *Method {
	return m.PostHandle(handler)
}

// Handle registers the handler for the given method. The handler is called
// when a method call is received from a Kite.
func (k *Kite) Handle(method string, handler Handler) *Method {
	return k.addHandle(method, handler)
}

// HandleFunc registers a handler to run when a method call is received from a
// Kite. It returns a *Method option to further modify certain options on a
// method call
func (k *Kite) HandleFunc(method string, handler HandlerFunc) *Method {
	return k.addHandle(method, handler)
}

// PreHandle registers an handler which is executed before a kite.Handler
// method is executed. Calling PreHandle multiple times registers multiple
// handlers. A non-error return triggers the execution of the next handler. The
// execution order is FIFO.
func (k *Kite) PreHandle(handler Handler) {
	k.preHandlers = append(k.preHandlers, handler)
}

// PreHandleFunc is the same as PreHandle. It accepts a HandlerFunc.
func (k *Kite) PreHandleFunc(handler HandlerFunc) {
	k.PreHandle(handler)
}

// PostHandle registers an handler which is executed after a kite.Handler
// method is executed. Calling PostHandler multiple times registers multiple
// handlers. A non-error return triggers the execution of the next handler. The
// execution order is FIFO.
func (k *Kite) PostHandle(handler Handler) {
	k.postHandlers = append(k.postHandlers, handler)
}

// PostHandleFunc is the same as PostHandle. It accepts a HandlerFunc.
func (k *Kite) PostHandleFunc(handler HandlerFunc) {
	k.PostHandle(handler)
}

func (m *Method) ServeKite(r *Request) (interface{}, error) {
	var firstResp interface{}
	var resp interface{}
	var err error

	// first execute preHandlers. make a copy of the handler to avoid race
	// conditions
	m.mu.Lock()
	preHandlers := make([]Handler, len(m.preHandlers))
	for i, handler := range m.preHandlers {
		preHandlers[i] = handler

	}
	m.mu.Unlock()

	for _, handler := range preHandlers {
		resp, err = handler.ServeKite(r)
		if err != nil {
			return nil, err
		}

		if m.handling == ReturnFirst && resp != nil && firstResp == nil {
			firstResp = resp
		}
	}

	preHandlers = nil // garbage collect it

	// now call our base handler
	resp, err = m.handler.ServeKite(r)
	if err != nil {
		return nil, err
	}

	// also save it dependent on the handling mechanism
	methodResp := resp

	if m.handling == ReturnFirst && resp != nil && firstResp == nil {
		firstResp = resp
	}

	// and finally return our postHandlers
	m.mu.Lock()
	postHandlers := make([]Handler, len(m.postHandlers))
	for i, handler := range m.postHandlers {
		postHandlers[i] = handler
	}
	m.mu.Unlock()

	for _, handler := range postHandlers {
		resp, err = handler.ServeKite(r)
		if err != nil {
			return nil, err
		}

		if m.handling == ReturnFirst && resp != nil && firstResp == nil {
			firstResp = resp
		}
	}

	postHandlers = nil // garbage collect it

	switch m.handling {
	case ReturnMethod:
		return methodResp, nil
	case ReturnFirst:
		return firstResp, nil
	case ReturnLatest:
		return resp, nil
	}

	return resp, nil
}
<commit_msg>method: initial implemntation of implementing method based throttling<commit_after>package kite

import (
	"sync"
	"time"

	"github.com/juju/ratelimit"
)

// MethodHandling defines how to handle chaining of kite.Handler middlewares.
// An error breaks the chain regardless of what handling is used. Note that all
// Pre and Post handlers are executed regardless the handling logic, only the
// return paramater is defined by the handling mode.
type MethodHandling int

const (
	// ReturnMethod returns main method's response. This is the standard default.
	ReturnMethod MethodHandling = iota

	// ReturnFirst returns the first non-nil response.
	ReturnFirst

	// ReturnLatest returns the latest response (waterfall behaviour)
	ReturnLatest
)

// Objects implementing the Handler interface can be registered to a method.
// The returned result must be marshalable with json package.
type Handler interface {
	ServeKite(*Request) (result interface{}, err error)
}

// HandlerFunc is a type adapter to allow the use of ordinary functions as
// Kite handlers. If h is a function with the appropriate signature,
// HandlerFunc(h) is a Handler object that calls h.
type HandlerFunc func(*Request) (result interface{}, err error)

// ServeKite calls h(r)
func (h HandlerFunc) ServeKite(r *Request) (interface{}, error) {
	return h(r)
}

// Method defines a method and the Handler it is bind to. By default
// "ReturnMethod" handling is used.
type Method struct {
	// name is the method name. Unnamed methods can exist
	name string

	// handler contains the related Handler for the given method
	handler      Handler   // handler is the base handler, the response of it is returned as the final
	preHandlers  []Handler // a list of handlers that are executed before the main handler
	postHandlers []Handler // a list of handlers that are executed after the main handler

	// authenticate defines if a given authenticator function is enabled for
	// the given auth type in the request.
	authenticate bool

	// handling defines how to handle chaining of kite.Handler middlewares.
	handling MethodHandling

	// initialized is used to indicate whether all pre and post handlers are
	// initialized.
	initialized bool

	// bucket is used for throttling the method by certain rule
	bucket *ratelimit.Bucket

	mu sync.Mutex // protects handler slices
}

// addHandle is an internal method to add a handler
func (k *Kite) addHandle(method string, handler Handler) *Method {
	authenticate := true
	if k.Config.DisableAuthentication {
		authenticate = false
	}

	m := &Method{
		name:         method,
		handler:      handler,
		preHandlers:  make([]Handler, 0),
		postHandlers: make([]Handler, 0),
		authenticate: authenticate,
		handling:     k.MethodHandling,
	}

	k.handlers[method] = m
	return m
}

// DisableAuthentication disables authentication check for this method.
func (m *Method) DisableAuthentication() *Method {
	m.authenticate = false
	return m
}

// Throttle throttles the method for each incoming request. The throttle
// algorithm is based on token bucket implementation:
// http://en.wikipedia.org/wiki/Token_bucket. Rate determines the number of
// request which are allowed per frequency. Example: A rate of 50 and frequency
// of two minutes means that the method can receive 50 request in two minutes,
// if there is more requests in this two minutes they will be rejected. Another
// example would be a rate of 30 and frequency of one second. This basically
// means the method is limited to 30 requests per second.
func (m *Method) Throttle(rate int64, frequency time.Duration) *Method {
	// don't do anything if the bucket is initialized already
	if m.bucket != nil {
		return m
	}

	m.bucket = ratelimit.NewBucketWithQuantum(
		frequency, // interval
		rate,      // capacity
		rate,      // token per interval
	)

	return m
}

// PreHandler adds a new kite handler which is executed before the method.
func (m *Method) PreHandle(handler Handler) *Method {
	m.preHandlers = append(m.preHandlers, handler)
	return m
}

// PreHandlerFunc adds a new kite handlerfunc which is executed before the
// method.
func (m *Method) PreHandleFunc(handler HandlerFunc) *Method {
	return m.PreHandle(handler)
}

// PostHandle adds a new kite handler which is executed after the method.
func (m *Method) PostHandle(handler Handler) *Method {
	m.postHandlers = append(m.postHandlers, handler)
	return m
}

// PostHandlerFunc adds a new kite handlerfunc which is executed before the
// method.
func (m *Method) PostHandleFunc(handler HandlerFunc) *Method {
	return m.PostHandle(handler)
}

// Handle registers the handler for the given method. The handler is called
// when a method call is received from a Kite.
func (k *Kite) Handle(method string, handler Handler) *Method {
	return k.addHandle(method, handler)
}

// HandleFunc registers a handler to run when a method call is received from a
// Kite. It returns a *Method option to further modify certain options on a
// method call
func (k *Kite) HandleFunc(method string, handler HandlerFunc) *Method {
	return k.addHandle(method, handler)
}

// PreHandle registers an handler which is executed before a kite.Handler
// method is executed. Calling PreHandle multiple times registers multiple
// handlers. A non-error return triggers the execution of the next handler. The
// execution order is FIFO.
func (k *Kite) PreHandle(handler Handler) {
	k.preHandlers = append(k.preHandlers, handler)
}

// PreHandleFunc is the same as PreHandle. It accepts a HandlerFunc.
func (k *Kite) PreHandleFunc(handler HandlerFunc) {
	k.PreHandle(handler)
}

// PostHandle registers an handler which is executed after a kite.Handler
// method is executed. Calling PostHandler multiple times registers multiple
// handlers. A non-error return triggers the execution of the next handler. The
// execution order is FIFO.
func (k *Kite) PostHandle(handler Handler) {
	k.postHandlers = append(k.postHandlers, handler)
}

// PostHandleFunc is the same as PostHandle. It accepts a HandlerFunc.
func (k *Kite) PostHandleFunc(handler HandlerFunc) {
	k.PostHandle(handler)
}

func (m *Method) ServeKite(r *Request) (interface{}, error) {
	var firstResp interface{}
	var resp interface{}
	var err error

	// first execute preHandlers. make a copy of the handler to avoid race
	// conditions
	m.mu.Lock()
	preHandlers := make([]Handler, len(m.preHandlers))
	for i, handler := range m.preHandlers {
		preHandlers[i] = handler

	}
	m.mu.Unlock()

	for _, handler := range preHandlers {
		resp, err = handler.ServeKite(r)
		if err != nil {
			return nil, err
		}

		if m.handling == ReturnFirst && resp != nil && firstResp == nil {
			firstResp = resp
		}
	}

	preHandlers = nil // garbage collect it

	// now call our base handler
	resp, err = m.handler.ServeKite(r)
	if err != nil {
		return nil, err
	}

	// also save it dependent on the handling mechanism
	methodResp := resp

	if m.handling == ReturnFirst && resp != nil && firstResp == nil {
		firstResp = resp
	}

	// and finally return our postHandlers
	m.mu.Lock()
	postHandlers := make([]Handler, len(m.postHandlers))
	for i, handler := range m.postHandlers {
		postHandlers[i] = handler
	}
	m.mu.Unlock()

	for _, handler := range postHandlers {
		resp, err = handler.ServeKite(r)
		if err != nil {
			return nil, err
		}

		if m.handling == ReturnFirst && resp != nil && firstResp == nil {
			firstResp = resp
		}
	}

	postHandlers = nil // garbage collect it

	switch m.handling {
	case ReturnMethod:
		return methodResp, nil
	case ReturnFirst:
		return firstResp, nil
	case ReturnLatest:
		return resp, nil
	}

	return resp, nil
}
<|endoftext|>


<commit_before>package autoupdate

import (
	"bytes"
	"compress/bzip2"
	"io"
	"io/ioutil"
	"net/http"
	"os"

	"github.com/blang/semver"
	"github.com/getlantern/flashlight/pro"
	"github.com/getlantern/go-update"
)

type Updater interface {
	// PublishProgress: publish percentage of update already downloaded
	Progress(int)
}

// byteCounter wraps an existing io.Reader and keeps track of the byte
// count while downloading the latest update
type byteCounter struct {
	io.Reader // Underlying io.Reader to track bytes transferred
	Updater
	total    int64   // Total bytes transferred
	length   int64   // Expected length
	progress float64 // How much of the update has been downloaded
}

func (pt *byteCounter) Read(p []byte) (int, error) {
	n, err := pt.Reader.Read(p)
	if n > 0 {
		pt.total += int64(n)
		percentage := float64(pt.total) / float64(pt.length) * float64(100)
		pt.Updater.Progress(int(percentage))
	}
	return n, err
}

func doCheckUpdate(version, URL string, publicKey []byte) (string, error) {

	log.Debugf("Checking for new mobile version; current version: %s", version)

	httpClient := pro.GetHTTPClient()

	// specify go-update should use our httpClient
	update.SetHttpClient(httpClient)

	res, err := checkUpdate(version, URL, publicKey)
	if err != nil {
		log.Errorf("Error checking for update for mobile: %v", err)
		return "", err
	}

	if res == nil {
		log.Debugf("No new version available!")
		return "", nil
	}

	v, err := semver.Make(version)
	if err != nil {
		log.Errorf("Error checking for update; could not parse version number: %v", err)
		return "", err
	}

	if isNewerVersion(v, res.Version) {
		log.Debugf("Newer version of Lantern mobile available! %s at %s", res.Version, res.Url)
		return res.Url, nil
	}

	return "", nil
}

// CheckMobileUpdate checks if a new update is available for mobile.
func CheckMobileUpdate(updateServer, appVersion string) (string, error) {
	return doCheckUpdate(appVersion,
		updateServer+"/update", []byte(PackagePublicKey))
}

// UpdateMobile downloads the latest APK from the given url to file apkPath.
func UpdateMobile(url, apkPath string, updater Updater) error {
	out, err := os.Create(apkPath)
	if err != nil {
		log.Error(err)
		return err
	}
	defer out.Close()
	return doUpdateMobile(url, out, updater)
}

func doUpdateMobile(url string, out *os.File, updater Updater) error {
	var req *http.Request
	var res *http.Response

	log.Debugf("Attempting to download APK from %s", url)

	httpClient := pro.GetHTTPClient()

	if req, err = http.NewRequest("GET", url, nil); err != nil {
		log.Errorf("Error downloading update: %v", err)
		return err
	}

	req.Header.Add("Accept-Encoding", "gzip")

	if res, err = httpClient.Do(req); err != nil {
		log.Errorf("Error requesting update: %v", err)
		return err
	}

	defer res.Body.Close()

	// We use a special byteCounter that storres a reference
	// to the updater interface to make it easy to publish progress
	// for how much of the update has been downloaded already.
	bytespt := &byteCounter{Updater: updater,
		Reader: res.Body, length: res.ContentLength}

	contents, err := ioutil.ReadAll(bytespt)
	if err != nil {
		log.Errorf("Error reading update: %v", err)
		return err
	}

	_, err = io.Copy(out, bzip2.NewReader(bytes.NewReader(contents)))
	if err != nil {
		log.Errorf("Error copying update: %v", err)
		return err
	}

	return nil
}
<commit_msg>fix error<commit_after>package autoupdate

import (
	"bytes"
	"compress/bzip2"
	"io"
	"io/ioutil"
	"net/http"
	"os"

	"github.com/blang/semver"
	"github.com/getlantern/flashlight/pro"
	"github.com/getlantern/go-update"
)

type Updater interface {
	// PublishProgress: publish percentage of update already downloaded
	Progress(int)
}

// byteCounter wraps an existing io.Reader and keeps track of the byte
// count while downloading the latest update
type byteCounter struct {
	io.Reader // Underlying io.Reader to track bytes transferred
	Updater
	total    int64   // Total bytes transferred
	length   int64   // Expected length
	progress float64 // How much of the update has been downloaded
}

func (pt *byteCounter) Read(p []byte) (int, error) {
	n, err := pt.Reader.Read(p)
	if n > 0 {
		pt.total += int64(n)
		percentage := float64(pt.total) / float64(pt.length) * float64(100)
		pt.Updater.Progress(int(percentage))
	}
	return n, err
}

func doCheckUpdate(version, URL string, publicKey []byte) (string, error) {

	log.Debugf("Checking for new mobile version; current version: %s", version)

	httpClient := pro.GetHTTPClient()

	// specify go-update should use our httpClient
	update.SetHttpClient(httpClient)

	res, err := checkUpdate(version, URL, publicKey)
	if err != nil {
		log.Errorf("Error checking for update for mobile: %v", err)
		return "", err
	}

	if res == nil {
		log.Debugf("No new version available!")
		return "", nil
	}

	v, err := semver.Make(version)
	if err != nil {
		log.Errorf("Error checking for update; could not parse version number: %v", err)
		return "", err
	}

	if isNewerVersion(v, res.Version) {
		log.Debugf("Newer version of Lantern mobile available! %s at %s", res.Version, res.Url)
		return res.Url, nil
	}

	return "", nil
}

// CheckMobileUpdate checks if a new update is available for mobile.
func CheckMobileUpdate(updateServer, appVersion string) (string, error) {
	return doCheckUpdate(appVersion,
		updateServer+"/update", []byte(PackagePublicKey))
}

// UpdateMobile downloads the latest APK from the given url to file apkPath.
func UpdateMobile(url, apkPath string, updater Updater) error {
	out, err := os.Create(apkPath)
	if err != nil {
		log.Error(err)
		return err
	}
	defer out.Close()
	return doUpdateMobile(url, out, updater)
}

func doUpdateMobile(url string, out *os.File, updater Updater) error {
	var req *http.Request
	var res *http.Response

	log.Debugf("Attempting to download APK from %s", url)

	httpClient := pro.GetHTTPClient()

	if req, err := http.NewRequest("GET", url, nil); err != nil {
		log.Errorf("Error downloading update: %v", err)
		return err
	}

	req.Header.Add("Accept-Encoding", "gzip")

	if res, err := httpClient.Do(req); err != nil {
		log.Errorf("Error requesting update: %v", err)
		return err
	}

	defer res.Body.Close()

	// We use a special byteCounter that storres a reference
	// to the updater interface to make it easy to publish progress
	// for how much of the update has been downloaded already.
	bytespt := &byteCounter{Updater: updater,
		Reader: res.Body, length: res.ContentLength}

	contents, err := ioutil.ReadAll(bytespt)
	if err != nil {
		log.Errorf("Error reading update: %v", err)
		return err
	}

	_, err = io.Copy(out, bzip2.NewReader(bytes.NewReader(contents)))
	if err != nil {
		log.Errorf("Error copying update: %v", err)
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>// toopenapi3 relies on `swagger2openapi` to convert
// Swagger 2.0 specs to OpenAPI 3.0 specs.
package toopenapi3

import (
	"errors"
	"path/filepath"
	"strings"

	"github.com/grokify/gotilla/cmd/cmdutil"
	"github.com/grokify/gotilla/errors/errorsutil"
	"github.com/grokify/gotilla/type/stringsutil"
)

const cmdSwagger2OpenAPI = "swagger2openapi"

func Convert(filenames []string, outdir string, renameFunc func(string) string) errorsutil.ErrorInfos {
	errinfos := errorsutil.ErrorInfos{}
	for _, srcpath := range filenames {
		_, srcfile := filepath.Split(srcpath)
		outfile := renameFunc(srcfile)
		outpath := filepath.Join(outdir, outfile)

		qtr := stringsutil.Quoter{Beg: "", End: ""}
		cmd := strings.Join([]string{
			cmdSwagger2OpenAPI,
			qtr.Quote(srcpath)}, " ")

		_, stderr, err := cmdutil.ExecToFiles(cmd, outpath, "", 0644)
		if err != nil {
			if err.Error() == "exit status 1" {
				ei := errorsutil.ErrorInfo{
					Input:   srcpath,
					Correct: outpath,
					Display: cmd,
					Error:   errors.New(stderr.String())}
				errinfos = append(errinfos, &ei)
			} else {
				ei := errorsutil.ErrorInfo{
					Input:   srcpath,
					Correct: outpath,
					Display: cmd,
					Error:   err}
				errinfos = append(errinfos, &ei)
			}
		} else {
			ei := errorsutil.ErrorInfo{
				Input:   srcpath,
				Correct: outpath,
				Display: cmd,
				Error:   nil}
			errinfos = append(errinfos, &ei)
		}
	}
	errinfos.Inflate()
	return errinfos
}
<commit_msg>dependencies: swagger2/toopenapi3: update for errorsutil<commit_after>// toopenapi3 relies on `swagger2openapi` to convert
// Swagger 2.0 specs to OpenAPI 3.0 specs.
package toopenapi3

import (
	"errors"
	"path/filepath"
	"strings"

	"github.com/grokify/gotilla/cmd/cmdutil"
	"github.com/grokify/gotilla/errors/errorsutil"
	"github.com/grokify/gotilla/type/stringsutil"
)

const cmdSwagger2OpenAPI = "swagger2openapi"

func Convert(filenames []string, outdir string, renameFunc func(string) string) errorsutil.ErrorInfos {
	errinfos := errorsutil.ErrorInfos{}
	for _, srcpath := range filenames {
		_, srcfile := filepath.Split(srcpath)
		outfile := renameFunc(srcfile)
		outpath := filepath.Join(outdir, outfile)

		qtr := stringsutil.Quoter{Beg: "", End: ""}
		cmd := strings.Join([]string{
			cmdSwagger2OpenAPI,
			qtr.Quote(srcpath)}, " ")

		_, stderr, err := cmdutil.ExecToFiles(cmd, outpath, "", 0644)
		if err != nil {
			if err.Error() == "exit status 1" {
				ei := errorsutil.ErrorInfo{
					Input:       srcpath,
					Correct:     outpath,
					Description: cmd,
					Error:       errors.New(stderr.String())}
				errinfos = append(errinfos, &ei)
			} else {
				ei := errorsutil.ErrorInfo{
					Input:       srcpath,
					Correct:     outpath,
					Description: cmd,
					Error:       err}
				errinfos = append(errinfos, &ei)
			}
		} else {
			ei := errorsutil.ErrorInfo{
				Input:       srcpath,
				Correct:     outpath,
				Description: cmd,
				Error:       nil}
			errinfos = append(errinfos, &ei)
		}
	}
	errinfos.Inflate()
	return errinfos
}
<|endoftext|>


<commit_before>package completer

import (
	"reflect"
	"sort"
	"testing"
)

func TestCompleterAddAndLookup(t *testing.T) {
	for _, tc := range []struct {
		add          []string
		wantLookup   map[string]string
		wantComplete map[string][]string
	}{
		{
			add: []string{
				"foo",
			},
			wantComplete: map[string][]string{
				"f":   []string{"foo"},
				"fo":  []string{"foo"},
				"foo": []string{"foo"},
			},
			wantLookup: map[string]string{
				"":    "",
				"f":   "foo",
				"fo":  "foo",
				"foo": "foo",
			},
		},
		{
			add: []string{
				"bar",
				"foo",
			},
			wantComplete: map[string][]string{
				"b":   []string{"bar"},
				"ba":  []string{"bar"},
				"bar": []string{"bar"},
				"f":   []string{"foo"},
				"fo":  []string{"foo"},
				"foo": []string{"foo"},
			},
			wantLookup: map[string]string{
				"":    "",
				"b":   "bar",
				"ba":  "bar",
				"bar": "bar",
				"f":   "foo",
				"fo":  "foo",
				"foo": "foo",
			},
		},
		{
			add: []string{
				"bar",
				"baz",
				"foo",
			},
			wantComplete: map[string][]string{
				"b":   []string{"bar", "baz"},
				"ba":  []string{"bar", "baz"},
				"bar": []string{"bar"},
				"baz": []string{"baz"},
				"f":   []string{"foo"},
				"fo":  []string{"foo"},
				"foo": []string{"foo"},
			},
			wantLookup: map[string]string{
				"":    "",
				"b":   "",
				"ba":  "",
				"bar": "bar",
				"baz": "baz",
				"f":   "foo",
				"fo":  "foo",
				"foo": "foo",
			},
		},
		{
			add: []string{
				"bar",
				"baz",
				"foo",
				"fux",
			},
			wantLookup: map[string]string{
				"":    "",
				"b":   "",
				"ba":  "",
				"bar": "bar",
				"baz": "baz",
				"f":   "",
				"fo":  "foo",
				"foo": "foo",
				"fu":  "fux",
				"fux": "fux",
			},
		},
		{
			add: []string{
				"foo",
				"foobar",
			},
			wantLookup: map[string]string{
				"":       "",
				"f":      "",
				"fo":     "",
				"foo":    "foo",
				"foob":   "foobar",
				"fooba":  "foobar",
				"foobar": "foobar",
			},
		},
		{
			add: []string{
				"foo",
				"foobar",
				"foobaz",
			},
			wantComplete: map[string][]string{
				"f":      []string{"foo", "foobar", "foobaz"},
				"fo":     []string{"foo", "foobar", "foobaz"},
				"foo":    []string{"foo", "foobar", "foobaz"},
				"foob":   []string{"foobar", "foobaz"},
				"fooba":  []string{"foobar", "foobaz"},
				"foobar": []string{"foobar"},
			},
		},
	} {
		c := NewCompleter()
		for _, s := range tc.add {
			if err := c.Add(s); err != nil {
				t.Errorf("%+v.Add(%q) == %s, want <nil>", c, s)
			}
		}
		for prefix, wantLookup := range tc.wantLookup {
			if got, ok := c.Lookup(prefix); got != wantLookup || (got == "" && ok) {
				t.Errorf("%+v.Lookup(%q) == %q, %t, want %q", c, prefix, got, ok, wantLookup)
			}
		}
		for prefix, wantComplete := range tc.wantComplete {
			gotComplete := c.Complete(prefix)
			sort.Strings(gotComplete)
			if !reflect.DeepEqual(gotComplete, wantComplete) {
				t.Errorf("%+v.Complete(%q) == %v, want %v", c, prefix, gotComplete, wantComplete)
			}
		}
	}
}

func TestCompleterSubstringLookup(t *testing.T) {
	c := NewCompleter()
	c.Add("foo")
	if err := c.Add("fo"); err != nil {
		t.Errorf("%+v.Add(\"fo\") == %v, want <nil>", c, err)
	}
	prefix := "f"
	if got, ok := c.Lookup(prefix); ok {
		t.Errorf("%+v.Lookup(%q) == %q, %t, want \"\", false", c, prefix, got, ok)
	}
}

func TestCompleterAddDuplicate(t *testing.T) {
	c := NewCompleter()
	c.Add("foo")
	if err := c.Add("foo"); err == nil {
		t.Errorf("%+v.Add(\"foo\") == <nil>, want !<nil>", c)
	}
}
<commit_msg>Add more tests<commit_after>package completer

import (
	"reflect"
	"sort"
	"testing"
)

func TestCompleterAddAndLookup(t *testing.T) {
	for _, tc := range []struct {
		add          []string
		wantLookup   map[string]string
		wantComplete map[string][]string
	}{
		{
			add: []string{
				"foo",
			},
			wantComplete: map[string][]string{
				"f":   []string{"foo"},
				"fo":  []string{"foo"},
				"foo": []string{"foo"},
			},
			wantLookup: map[string]string{
				"":    "",
				"f":   "foo",
				"fo":  "foo",
				"foo": "foo",
			},
		},
		{
			add: []string{
				"bar",
				"foo",
			},
			wantComplete: map[string][]string{
				"b":   []string{"bar"},
				"ba":  []string{"bar"},
				"bar": []string{"bar"},
				"f":   []string{"foo"},
				"fo":  []string{"foo"},
				"foo": []string{"foo"},
			},
			wantLookup: map[string]string{
				"":    "",
				"b":   "bar",
				"ba":  "bar",
				"bar": "bar",
				"f":   "foo",
				"fo":  "foo",
				"foo": "foo",
			},
		},
		{
			add: []string{
				"bar",
				"baz",
				"foo",
			},
			wantComplete: map[string][]string{
				"b":   []string{"bar", "baz"},
				"ba":  []string{"bar", "baz"},
				"bar": []string{"bar"},
				"baz": []string{"baz"},
				"f":   []string{"foo"},
				"fo":  []string{"foo"},
				"foo": []string{"foo"},
			},
			wantLookup: map[string]string{
				"":    "",
				"b":   "",
				"ba":  "",
				"bar": "bar",
				"baz": "baz",
				"f":   "foo",
				"fo":  "foo",
				"foo": "foo",
			},
		},
		{
			add: []string{
				"bar",
				"baz",
				"foo",
				"fux",
			},
			wantComplete: map[string][]string{
				"b":   []string{"bar", "baz"},
				"ba":  []string{"bar", "baz"},
				"bar": []string{"bar"},
				"baz": []string{"baz"},
				"f":   []string{"foo", "fux"},
				"fo":  []string{"foo"},
				"foo": []string{"foo"},
				"fu":  []string{"fux"},
				"fux": []string{"fux"},
			},
			wantLookup: map[string]string{
				"":    "",
				"b":   "",
				"ba":  "",
				"bar": "bar",
				"baz": "baz",
				"f":   "",
				"fo":  "foo",
				"foo": "foo",
				"fu":  "fux",
				"fux": "fux",
			},
		},
		{
			add: []string{
				"foo",
				"foobar",
			},
			wantComplete: map[string][]string{
				"f":      []string{"foo", "foobar"},
				"fo":     []string{"foo", "foobar"},
				"foo":    []string{"foo", "foobar"},
				"foob":   []string{"foobar"},
				"fooba":  []string{"foobar"},
				"foobar": []string{"foobar"},
			},
			wantLookup: map[string]string{
				"":       "",
				"f":      "",
				"fo":     "",
				"foo":    "foo",
				"foob":   "foobar",
				"fooba":  "foobar",
				"foobar": "foobar",
			},
		},
		{
			add: []string{
				"foo",
				"foobar",
				"foobaz",
			},
			wantComplete: map[string][]string{
				"f":      []string{"foo", "foobar", "foobaz"},
				"fo":     []string{"foo", "foobar", "foobaz"},
				"foo":    []string{"foo", "foobar", "foobaz"},
				"foob":   []string{"foobar", "foobaz"},
				"fooba":  []string{"foobar", "foobaz"},
				"foobar": []string{"foobar"},
			},
			wantLookup: map[string]string{
				"": "",
				//"f":      "", // FIXME this fails
				//"fo":     "", // FIXME this fails
				"foo":    "foo",
				"foob":   "",
				"fooba":  "",
				"foobar": "foobar",
				"foobaz": "foobaz",
			},
		},
	} {
		c := NewCompleter()
		for _, s := range tc.add {
			if err := c.Add(s); err != nil {
				t.Errorf("%+v.Add(%q) == %s, want <nil>", c, s)
			}
		}
		for prefix, wantLookup := range tc.wantLookup {
			if got, ok := c.Lookup(prefix); got != wantLookup || (got == "" && ok) {
				t.Errorf("%+v.Lookup(%q) == %q, %t, want %q", c, prefix, got, ok, wantLookup)
			}
		}
		for prefix, wantComplete := range tc.wantComplete {
			gotComplete := c.Complete(prefix)
			sort.Strings(gotComplete)
			if !reflect.DeepEqual(gotComplete, wantComplete) {
				t.Errorf("%+v.Complete(%q) == %v, want %v", c, prefix, gotComplete, wantComplete)
			}
		}
	}
}

func TestCompleterSubstringLookup(t *testing.T) {
	c := NewCompleter()
	c.Add("foo")
	if err := c.Add("fo"); err != nil {
		t.Errorf("%+v.Add(\"fo\") == %v, want <nil>", c, err)
	}
	prefix := "f"
	if got, ok := c.Lookup(prefix); ok {
		t.Errorf("%+v.Lookup(%q) == %q, %t, want \"\", false", c, prefix, got, ok)
	}
}

func TestCompleterAddDuplicate(t *testing.T) {
	c := NewCompleter()
	c.Add("foo")
	if err := c.Add("foo"); err == nil {
		t.Errorf("%+v.Add(\"foo\") == <nil>, want !<nil>", c)
	}
}
<|endoftext|>


<commit_before>package conf

import (
	"errors"
	"fmt"
	"math"
	"strconv"
	"strings"

	"github.com/grafana/metrictank/schema"

	"github.com/grafana/metrictank/mdata/chunk"
	"github.com/raintank/dur"
)

const Month_sec = 60 * 60 * 24 * 28

var errReadyFormat = errors.New("'ready' field must be a bool or unsigned integer")

type Retentions struct {
	Orig string
	Rets []Retention
}

// Sub returns a "subslice" of Retentions starting at the given pos.
func (r Retentions) Sub(pos int) Retentions {
	origSplit := strings.Split(r.Orig, ",")
	return Retentions{
		Orig: strings.Join(origSplit[pos:], ":"),
		Rets: r.Rets[pos:],
	}
}

func BuildFromRetentions(rets ...Retention) Retentions {
	return Retentions{
		Orig: buildOrigFromRetentions(rets),
		Rets: rets,
	}
}
func buildOrigFromRetentions(rets []Retention) string {
	var out []string
	for _, r := range rets {
		out = append(out, r.String())
	}
	return strings.Join(out, ",")
}

// Validate assures the retentions are sane.  As the whisper source code says:
// An ArchiveList must:
// 1. Have at least one archive config. Example: (60, 86400)
// 2. No archive may be a duplicate of another.
// 3. Higher precision archives' precision must evenly divide all lower precision archives' precision.
// 4. Lower precision archives must cover larger time intervals than higher precision archives.
// 5. Each archive must have at least enough points to consolidate to the next archive
func (r Retentions) Validate() error {
	if len(r.Rets) == 0 {
		return fmt.Errorf("No retentions")
	}
	for i := 1; i < len(r.Rets); i++ {
		prev := r.Rets[i-1]
		ret := r.Rets[i]

		if prev.SecondsPerPoint >= ret.SecondsPerPoint {
			return fmt.Errorf("retention must have lower resolution than prior retention")
		}

		if ret.SecondsPerPoint%prev.SecondsPerPoint != 0 {
			return fmt.Errorf("lower resolution retentions must be evenly divisible by higher resolution retentions (%d does not divide by %d)", ret.SecondsPerPoint, prev.SecondsPerPoint)
		}

		if prev.MaxRetention() >= ret.MaxRetention() {
			return fmt.Errorf("lower resolution archives must have longer retention than higher resolution archives")
		}

		if prev.NumberOfPoints < (ret.SecondsPerPoint / prev.SecondsPerPoint) {
			return fmt.Errorf("Each archive must have at least enough points to consolidate to the next archive (archive%v consolidates %v of archive%v's points but it has only %v total points)", i, ret.SecondsPerPoint/prev.SecondsPerPoint, i-1, prev.NumberOfPoints)
		}
	}
	return nil
}

/*
  A retention level.

  Retention levels describe a given archive in the database. How detailed it is and how far back
  it records.
*/
type Retention struct {
	SecondsPerPoint int    // interval in seconds
	NumberOfPoints  int    // ~ttl
	ChunkSpan       uint32 // duration of chunk of aggregated metric for storage, controls how many aggregated points go into 1 chunk
	NumChunks       uint32 // number of chunks to keep in memory. remember, for a query from now until 3 months ago, we will end up querying the memory server as well.
	Ready           uint32 // ready for reads for data as of this timestamp (or as of now-TTL, whichever is highest)
}

func (r Retention) MaxRetention() int {
	return r.SecondsPerPoint * r.NumberOfPoints
}

func (r Retention) String() string {
	s := dur.FormatDuration(uint32(r.SecondsPerPoint))
	s += ":" + dur.FormatDuration(uint32(r.NumberOfPoints*r.SecondsPerPoint))
	s += ":" + dur.FormatDuration(r.ChunkSpan)
	s += ":" + strconv.Itoa(int(r.NumChunks))
	switch r.Ready {
	case 0:
		s += ":true"
	case math.MaxUint32:
		s += ":false"
	default:
		s += ":" + strconv.FormatUint(uint64(r.Ready), 10)
	}
	return s
}

func NewRetention(secondsPerPoint, numberOfPoints int) Retention {
	return Retention{
		SecondsPerPoint: secondsPerPoint,
		NumberOfPoints:  numberOfPoints,
	}
}

func NewRetentionMT(secondsPerPoint int, ttl, chunkSpan, numChunks, ready uint32) Retention {
	return Retention{
		SecondsPerPoint: secondsPerPoint,
		NumberOfPoints:  int(ttl) / secondsPerPoint,
		ChunkSpan:       chunkSpan,
		NumChunks:       numChunks,
		Ready:           ready,
	}
}

func MustParseRetentions(defs string) Retentions {
	r, err := ParseRetentions(defs)
	if err != nil {
		panic(err)
	}
	return r
}

// ParseRetentions parses retention definitions into a Retentions structure
func ParseRetentions(defs string) (Retentions, error) {
	retentions := Retentions{
		Orig: defs,
	}
	cnt := strings.Count(defs, ",")
	if cnt > 254 {
		return retentions, errors.New("no more than 255 individual retensions per rule supported")
	}
	for i, def := range strings.Split(defs, ",") {
		def = strings.TrimSpace(def)
		parts := strings.Split(def, ":")
		if len(parts) < 2 || len(parts) > 5 {
			return retentions, fmt.Errorf("bad retentions spec %q", def)
		}

		// try old format
		val1, err1 := strconv.ParseInt(parts[0], 10, 0)
		val2, err2 := strconv.ParseInt(parts[1], 10, 0)

		var retention Retention
		var err error
		if err1 == nil && err2 == nil {
			retention = NewRetention(int(val1), int(val2))
		} else {
			// try new format
			retention, err = ParseRetentionNew(def)
			if err != nil {
				return retentions, err
			}
		}
		if i != 0 && !schema.IsSpanValid(uint32(retention.SecondsPerPoint)) {
			return retentions, fmt.Errorf("invalid retention: can't encode span of %d", retention.SecondsPerPoint)

		}
		if len(parts) >= 3 {
			retention.ChunkSpan, err = dur.ParseNDuration(parts[2])
			if err != nil {
				return retentions, err
			}
			if (Month_sec % retention.ChunkSpan) != 0 {
				return retentions, errors.New("chunkSpan must fit without remainders into month_sec (28*24*60*60)")
			}
			_, ok := chunk.RevChunkSpans[retention.ChunkSpan]
			if !ok {
				return retentions, fmt.Errorf("chunkSpan %s is not a valid value (https://github.com/grafana/metrictank/blob/master/docs/memory-server.md#valid-chunk-spans)", parts[2])
			}
		} else {
			// default to a valid chunkspan that can hold at least 100 points, or select the largest one otherwise.
			approxSpan := uint32(retention.SecondsPerPoint * 100)
			var span uint32
			for _, span = range chunk.ChunkSpans {
				if span >= approxSpan {
					break
				}
			}
			retention.ChunkSpan = span
		}
		retention.NumChunks = 2
		if len(parts) >= 4 {
			i, err := strconv.Atoi(parts[3])
			if err != nil {
				return retentions, err
			}
			retention.NumChunks = uint32(i)
		}
		if len(parts) == 5 {
			// user is allowed to specify both a bool or a timestamp.
			// internally we map both to timestamp.
			// 0 (default) is effectively the same as 'true'
			// math.MaxUint32 is effectively the same as 'false'
			readyInt, err := strconv.ParseUint(parts[4], 10, 32)
			if err == nil {
				retention.Ready = uint32(readyInt)
			} else {
				readyBool, err := strconv.ParseBool(parts[4])
				if err != nil {
					return retentions, errReadyFormat
				}
				if !readyBool {
					retention.Ready = math.MaxUint32
				}
			}
		}

		retentions.Rets = append(retentions.Rets, retention)
	}
	return retentions, retentions.Validate()
}

func ParseRetentionNew(def string) (Retention, error) {
	parts := strings.Split(def, ":")
	if len(parts) < 2 {
		return Retention{}, fmt.Errorf("Not enough parts in retentionDef %q", def)
	}
	interval, err := dur.ParseDuration(parts[0])
	if err != nil {
		return Retention{}, fmt.Errorf("Failed to parse interval in %q: %s", def, err)
	}

	ttl, err := dur.ParseDuration(parts[1])
	if err != nil {
		return Retention{}, fmt.Errorf("Failed to parse TTL in %q: %s", def, err)
	}

	return Retention{
		SecondsPerPoint: int(interval),
		NumberOfPoints:  int(ttl / interval)}, err
}
<commit_msg>bugfix: list multiple retentions using ',' as separator<commit_after>package conf

import (
	"errors"
	"fmt"
	"math"
	"strconv"
	"strings"

	"github.com/grafana/metrictank/schema"

	"github.com/grafana/metrictank/mdata/chunk"
	"github.com/raintank/dur"
)

const Month_sec = 60 * 60 * 24 * 28

var errReadyFormat = errors.New("'ready' field must be a bool or unsigned integer")

type Retentions struct {
	Orig string
	Rets []Retention
}

// Sub returns a "subslice" of Retentions starting at the given pos.
func (r Retentions) Sub(pos int) Retentions {
	origSplit := strings.Split(r.Orig, ",")
	return Retentions{
		Orig: strings.Join(origSplit[pos:], ","),
		Rets: r.Rets[pos:],
	}
}

func BuildFromRetentions(rets ...Retention) Retentions {
	return Retentions{
		Orig: buildOrigFromRetentions(rets),
		Rets: rets,
	}
}
func buildOrigFromRetentions(rets []Retention) string {
	var out []string
	for _, r := range rets {
		out = append(out, r.String())
	}
	return strings.Join(out, ",")
}

// Validate assures the retentions are sane.  As the whisper source code says:
// An ArchiveList must:
// 1. Have at least one archive config. Example: (60, 86400)
// 2. No archive may be a duplicate of another.
// 3. Higher precision archives' precision must evenly divide all lower precision archives' precision.
// 4. Lower precision archives must cover larger time intervals than higher precision archives.
// 5. Each archive must have at least enough points to consolidate to the next archive
func (r Retentions) Validate() error {
	if len(r.Rets) == 0 {
		return fmt.Errorf("No retentions")
	}
	for i := 1; i < len(r.Rets); i++ {
		prev := r.Rets[i-1]
		ret := r.Rets[i]

		if prev.SecondsPerPoint >= ret.SecondsPerPoint {
			return fmt.Errorf("retention must have lower resolution than prior retention")
		}

		if ret.SecondsPerPoint%prev.SecondsPerPoint != 0 {
			return fmt.Errorf("lower resolution retentions must be evenly divisible by higher resolution retentions (%d does not divide by %d)", ret.SecondsPerPoint, prev.SecondsPerPoint)
		}

		if prev.MaxRetention() >= ret.MaxRetention() {
			return fmt.Errorf("lower resolution archives must have longer retention than higher resolution archives")
		}

		if prev.NumberOfPoints < (ret.SecondsPerPoint / prev.SecondsPerPoint) {
			return fmt.Errorf("Each archive must have at least enough points to consolidate to the next archive (archive%v consolidates %v of archive%v's points but it has only %v total points)", i, ret.SecondsPerPoint/prev.SecondsPerPoint, i-1, prev.NumberOfPoints)
		}
	}
	return nil
}

/*
  A retention level.

  Retention levels describe a given archive in the database. How detailed it is and how far back
  it records.
*/
type Retention struct {
	SecondsPerPoint int    // interval in seconds
	NumberOfPoints  int    // ~ttl
	ChunkSpan       uint32 // duration of chunk of aggregated metric for storage, controls how many aggregated points go into 1 chunk
	NumChunks       uint32 // number of chunks to keep in memory. remember, for a query from now until 3 months ago, we will end up querying the memory server as well.
	Ready           uint32 // ready for reads for data as of this timestamp (or as of now-TTL, whichever is highest)
}

func (r Retention) MaxRetention() int {
	return r.SecondsPerPoint * r.NumberOfPoints
}

func (r Retention) String() string {
	s := dur.FormatDuration(uint32(r.SecondsPerPoint))
	s += ":" + dur.FormatDuration(uint32(r.NumberOfPoints*r.SecondsPerPoint))
	s += ":" + dur.FormatDuration(r.ChunkSpan)
	s += ":" + strconv.Itoa(int(r.NumChunks))
	switch r.Ready {
	case 0:
		s += ":true"
	case math.MaxUint32:
		s += ":false"
	default:
		s += ":" + strconv.FormatUint(uint64(r.Ready), 10)
	}
	return s
}

func NewRetention(secondsPerPoint, numberOfPoints int) Retention {
	return Retention{
		SecondsPerPoint: secondsPerPoint,
		NumberOfPoints:  numberOfPoints,
	}
}

func NewRetentionMT(secondsPerPoint int, ttl, chunkSpan, numChunks, ready uint32) Retention {
	return Retention{
		SecondsPerPoint: secondsPerPoint,
		NumberOfPoints:  int(ttl) / secondsPerPoint,
		ChunkSpan:       chunkSpan,
		NumChunks:       numChunks,
		Ready:           ready,
	}
}

func MustParseRetentions(defs string) Retentions {
	r, err := ParseRetentions(defs)
	if err != nil {
		panic(err)
	}
	return r
}

// ParseRetentions parses retention definitions into a Retentions structure
func ParseRetentions(defs string) (Retentions, error) {
	retentions := Retentions{
		Orig: defs,
	}
	cnt := strings.Count(defs, ",")
	if cnt > 254 {
		return retentions, errors.New("no more than 255 individual retensions per rule supported")
	}
	for i, def := range strings.Split(defs, ",") {
		def = strings.TrimSpace(def)
		parts := strings.Split(def, ":")
		if len(parts) < 2 || len(parts) > 5 {
			return retentions, fmt.Errorf("bad retentions spec %q", def)
		}

		// try old format
		val1, err1 := strconv.ParseInt(parts[0], 10, 0)
		val2, err2 := strconv.ParseInt(parts[1], 10, 0)

		var retention Retention
		var err error
		if err1 == nil && err2 == nil {
			retention = NewRetention(int(val1), int(val2))
		} else {
			// try new format
			retention, err = ParseRetentionNew(def)
			if err != nil {
				return retentions, err
			}
		}
		if i != 0 && !schema.IsSpanValid(uint32(retention.SecondsPerPoint)) {
			return retentions, fmt.Errorf("invalid retention: can't encode span of %d", retention.SecondsPerPoint)

		}
		if len(parts) >= 3 {
			retention.ChunkSpan, err = dur.ParseNDuration(parts[2])
			if err != nil {
				return retentions, err
			}
			if (Month_sec % retention.ChunkSpan) != 0 {
				return retentions, errors.New("chunkSpan must fit without remainders into month_sec (28*24*60*60)")
			}
			_, ok := chunk.RevChunkSpans[retention.ChunkSpan]
			if !ok {
				return retentions, fmt.Errorf("chunkSpan %s is not a valid value (https://github.com/grafana/metrictank/blob/master/docs/memory-server.md#valid-chunk-spans)", parts[2])
			}
		} else {
			// default to a valid chunkspan that can hold at least 100 points, or select the largest one otherwise.
			approxSpan := uint32(retention.SecondsPerPoint * 100)
			var span uint32
			for _, span = range chunk.ChunkSpans {
				if span >= approxSpan {
					break
				}
			}
			retention.ChunkSpan = span
		}
		retention.NumChunks = 2
		if len(parts) >= 4 {
			i, err := strconv.Atoi(parts[3])
			if err != nil {
				return retentions, err
			}
			retention.NumChunks = uint32(i)
		}
		if len(parts) == 5 {
			// user is allowed to specify both a bool or a timestamp.
			// internally we map both to timestamp.
			// 0 (default) is effectively the same as 'true'
			// math.MaxUint32 is effectively the same as 'false'
			readyInt, err := strconv.ParseUint(parts[4], 10, 32)
			if err == nil {
				retention.Ready = uint32(readyInt)
			} else {
				readyBool, err := strconv.ParseBool(parts[4])
				if err != nil {
					return retentions, errReadyFormat
				}
				if !readyBool {
					retention.Ready = math.MaxUint32
				}
			}
		}

		retentions.Rets = append(retentions.Rets, retention)
	}
	return retentions, retentions.Validate()
}

func ParseRetentionNew(def string) (Retention, error) {
	parts := strings.Split(def, ":")
	if len(parts) < 2 {
		return Retention{}, fmt.Errorf("Not enough parts in retentionDef %q", def)
	}
	interval, err := dur.ParseDuration(parts[0])
	if err != nil {
		return Retention{}, fmt.Errorf("Failed to parse interval in %q: %s", def, err)
	}

	ttl, err := dur.ParseDuration(parts[1])
	if err != nil {
		return Retention{}, fmt.Errorf("Failed to parse TTL in %q: %s", def, err)
	}

	return Retention{
		SecondsPerPoint: int(interval),
		NumberOfPoints:  int(ttl / interval)}, err
}
<|endoftext|>


<commit_before>package ginDoi

import (
	"bytes"
	log "github.com/Sirupsen/logrus"
	"net/http"
	"path/filepath"
	"text/template"
)

var LOGPREFIX = "GnodeDoiProvider"

type GnodeDoiProvider struct {
	//https://mds.datacite.org/static/apidoc
	ApiURI  string
	Pwd     string
	DOIBase string
}

func (dp GnodeDoiProvider) MakeDoi(doiInfo *CBerry) string {
	doiInfo.DOI = dp.DOIBase + "/" + "G-NODE." + doiInfo.UUID[:6]
	return doiInfo.DOI
}

func (dp GnodeDoiProvider) GetXml(doiInfo *CBerry) (string, error) {
	dp.MakeDoi(doiInfo)
	t, err := template.ParseFiles(filepath.Join("tmpl", "datacite.xml"))
	if err != nil {
		log.WithFields(log.Fields{
			"source": LOGPREFIX,
			"error":  err,
		}).Error("Could not parse template")
		return "", err
	}
	buff := bytes.Buffer{}
	err = t.Execute(&buff, doiInfo)
	if err != nil {
		log.WithFields(log.Fields{
			"source": LOGPREFIX,
			"error":  err,
		}).Error("Template execution failed")
		return "", err
	}
	return buff.String(), err
}

func (dp GnodeDoiProvider) RegDoi(doiInfo CBerry) (string, error) {
	data, err := dp.GetXml(&doiInfo)
	if err != nil {
		return "", err
	}
	if r, err := http.Post(dp.ApiURI+"/metadata", "text/plain;charset=UTF-8",
		bytes.NewReader([]byte(data))); err != nil {
		return "", err
	} else {
		return r.Status, nil
	}
}
<commit_msg>Doi generation with lower case g-node<commit_after>package ginDoi

import (
	"bytes"
	log "github.com/Sirupsen/logrus"
	"net/http"
	"path/filepath"
	"text/template"
)

var LOGPREFIX = "GnodeDoiProvider"

type GnodeDoiProvider struct {
	//https://mds.datacite.org/static/apidoc
	ApiURI  string
	Pwd     string
	DOIBase string
}

func (dp GnodeDoiProvider) MakeDoi(doiInfo *CBerry) string {
	doiInfo.DOI = dp.DOIBase + "/" + "g-node." + doiInfo.UUID[:6]
	return doiInfo.DOI
}

func (dp GnodeDoiProvider) GetXml(doiInfo *CBerry) (string, error) {
	dp.MakeDoi(doiInfo)
	t, err := template.ParseFiles(filepath.Join("tmpl", "datacite.xml"))
	if err != nil {
		log.WithFields(log.Fields{
			"source": LOGPREFIX,
			"error":  err,
		}).Error("Could not parse template")
		return "", err
	}
	buff := bytes.Buffer{}
	err = t.Execute(&buff, doiInfo)
	if err != nil {
		log.WithFields(log.Fields{
			"source": LOGPREFIX,
			"error":  err,
		}).Error("Template execution failed")
		return "", err
	}
	return buff.String(), err
}

func (dp GnodeDoiProvider) RegDoi(doiInfo CBerry) (string, error) {
	data, err := dp.GetXml(&doiInfo)
	if err != nil {
		return "", err
	}
	if r, err := http.Post(dp.ApiURI+"/metadata", "text/plain;charset=UTF-8",
		bytes.NewReader([]byte(data))); err != nil {
		return "", err
	} else {
		return r.Status, nil
	}
}
<|endoftext|>


<commit_before>package token

import (
	"github.com/devfeel/dotweb"
	"github.com/devfeel/tokenserver/config"
	"github.com/devfeel/tokenserver/const"
	"github.com/devfeel/tokenserver/framework/crypto"
	"github.com/devfeel/tokenserver/framework/json"
	"github.com/devfeel/tokenserver/framework/log"
	"github.com/devfeel/tokenserver/framework/redis"
	"github.com/devfeel/tokenserver/httpserver/model"
	"strconv"
)

type TokenInfo struct {
	Token       string
	AppID       string
	TokenBody   string
	LifeSeconds int //有效时间，单位为秒
}

type VerifyTokenRequest struct {
	Token       string
	AppID       string
	TokenBody   string
	IsCheckBody bool //是否需要验证Body是否一致
}

const (
	Token_DefaultLifeSeconds = 60 * 30 //token默认有效期，默认30分钟有效期
)

/*创建token
Author: panxinming
CreateTime: 2017-02-10 12:00
HttpMethod：Post
Post Param：TokenInfo
返回结构：HandlerResponse
0: 成功
-100001：post data not legal
-100002：AppID is empty
-200001：no config RedisID_Token Redis
-200002：create token error [uuid]
-200003：create token error [redis set]
Response.Message = TokenInfo

UPDATE LOG:
1、初始版本 --2017-02-10 12:00 by pxm
*/
func CreateToken(ctx *dotweb.HttpContext) {
	result := &models.HandlerResponse{RetCode: 0, RetMsg: ""}
	var tokenInfo TokenInfo
	//获取Post内容
	postContent := string(ctx.PostBody())

	defer func() {
		logger.Info("TokenServer::CreateToken["+postContent+"] => "+jsonutil.GetJsonString(result), constdefine.LogTarget_Token)
		ctx.WriteJson(result)
	}()

	//解析提交数据
	err_jsonunmar := jsonutil.Unmarshal(postContent, &tokenInfo)
	if err_jsonunmar != nil {
		result.RetCode = -100001
		result.RetMsg = "post data not legal => " + err_jsonunmar.Error()
		return
	}
	if tokenInfo.AppID == "" {
		result.RetCode = -100002
		result.RetMsg = "AppID is empty"
		return
	}
	//默认值处理
	if tokenInfo.LifeSeconds <= 0 {
		tokenInfo.LifeSeconds = Token_DefaultLifeSeconds
	}

	//获取redis配置
	redisInfo, exists := config.GetRedisInfo(constdefine.RedisID_Token)
	if !exists {
		result.RetCode = -200001
		result.RetMsg = "no config RedisID_Token Redis"
		return
	}

	//创建token
	token := cryptos.GetGuid()
	if token == "" {
		result.RetCode = -200002
		result.RetMsg = "create token error [uuid]"
		return
	}
	tokenInfo.Token = token

	key := redisInfo.KeyPre + ":Token:" + tokenInfo.AppID + "_" + tokenInfo.Token
	value := jsonutil.GetJsonString(tokenInfo)
	redisClient := redisutil.GetRedisClient(redisInfo.ServerIP)
	val, err := redisClient.SetWithExpire(key, value, tokenInfo.LifeSeconds)
	if err != nil {
		result.RetCode = -200003
		result.RetMsg = "create token error [redis set] => " + err.Error()
		return
	}

	result.RetCode = 0
	result.RetMsg = "ok[" + val + "]"
	result.Message = tokenInfo
	//设置原子锁，默认有效期为token的两倍，初始值为1
	key = redisInfo.KeyPre + ":TokenLock:" + tokenInfo.AppID + "_" + tokenInfo.Token
	redisClient.SetWithExpire(key, "1", tokenInfo.LifeSeconds*2)
}

/*验证token
Author: panxinming
CreateTime: 2017-02-10 12:00
HttpMethod：Post
Post Param：VerifyTokenRequest
返回结构：HandlerResponse
0: 成功
-100001：post data not legal
-100002：AppID is empty
-200001：no config RedisID_Token Redis
-201001: query token exists error
-201002: redis token not exists
-202001: get token-locker error
-202002: token-locker locked by other
-203001: query token error
-203002: redis token data not legal
-203003: token body is not match

Response.Message = TokenInfo

UPDATE LOG:
1、初始版本 --2017-02-10 12:00 by pxm
*/
func VerifyToken(ctx *dotweb.HttpContext) {
	result := &models.HandlerResponse{RetCode: 0, RetMsg: ""}
	var verifyToken VerifyTokenRequest
	//获取Post内容
	postContent := string(ctx.PostBody())

	defer func() {
		logger.Info("TokenServer::VerifyToken["+postContent+"] => "+jsonutil.GetJsonString(result), constdefine.LogTarget_Token)
		ctx.WriteJson(result)
	}()

	//解析提交数据
	err_jsonunmar := jsonutil.Unmarshal(postContent, &verifyToken)
	if err_jsonunmar != nil {
		result.RetCode = -100001
		result.RetMsg = "post data not legal => " + err_jsonunmar.Error()
		return
	}
	if verifyToken.AppID == "" {
		result.RetCode = -100002
		result.RetMsg = "AppID is empty"
		return
	}

	//获取Redis配置
	redisInfo, exists := config.GetRedisInfo(constdefine.RedisID_Token)
	if !exists {
		result.RetCode = -200001
		result.RetMsg = "no config RedisID_Token Redis"
		return
	}

	keyLocker := redisInfo.KeyPre + ":TokenLock:" + verifyToken.AppID + "_" + verifyToken.Token
	keyToken := redisInfo.KeyPre + ":Token:" + verifyToken.AppID + "_" + verifyToken.Token

	//创建Redis链接
	var redisClient *redisutil.RedisClient
	redisClient = redisutil.GetRedisClient(redisInfo.ServerIP)

	//检查token是否存在
	numExists, errExists := redisClient.Exists(keyToken)
	if errExists != nil {
		result.RetCode = -201001
		result.RetMsg = "query token exists error => " + errExists.Error()
		return
	}
	if numExists == 0 {
		result.RetCode = -201002
		result.RetMsg = "redis token not exists"
		return
	}

	//加锁
	valIncr, errIncr := redisClient.DECR(keyLocker)
	if errIncr != nil {
		result.RetCode = -202001
		result.RetMsg = "get token-locker error => " + errIncr.Error()
		return
	}
	if valIncr != 0 {
		result.RetCode = -202002
		result.RetMsg = "token-locker locked by other [" + strconv.Itoa(valIncr) + "]"
		//归还当前锁
		redisClient.INCR(keyLocker)
		return
	}

	var redisToken TokenInfo
	//检查Token
	valToken, errToken := redisClient.Get(keyToken)
	if errToken != nil {
		result.RetCode = -203001
		result.RetMsg = "query token error => " + errToken.Error()
		//归还当前锁
		redisClient.INCR(keyLocker)
		return
	}
	err_jsonunmar = jsonutil.Unmarshal(valToken, &redisToken)
	if err_jsonunmar != nil {
		result.RetCode = -203002
		result.RetMsg = "redis token data not legal [" + valToken + "] => " + err_jsonunmar.Error()
		//归还当前锁
		redisClient.INCR(keyLocker)
		return
	}
	if verifyToken.IsCheckBody {
		if verifyToken.TokenBody != redisToken.TokenBody {
			result.RetCode = -201003
			result.RetMsg = "token body is not match [" + valToken + "]"
			//归还当前锁
			redisClient.INCR(keyLocker)
			return
		}
	}

	result.RetCode = 0
	result.RetMsg = "ok"
	result.Message = redisToken
	//一切正常，删除相关token信息
	redisClient.Del(keyLocker)
	redisClient.Del(keyToken)
}

/*查询token
Author: panxinming
CreateTime: 2017-02-10 12:00
HttpMethod：Get
Get Param：
appid: 应用编码
token: token
返回结构：HandlerResponse
0: 成功
-100001：querystring token|appid is empty
-200001：no config RedisID_Token Redis
-200002：query token error [redis get]
Response.Message = TokenInfo

UPDATE LOG:
1、初始版本 --2017-02-10 12:00 by pxm
*/
func QueryToken(ctx *dotweb.HttpContext) {
	result := &models.HandlerResponse{RetCode: 0, RetMsg: ""}
	token := ctx.QueryString("token")
	appId := ctx.QueryString("appid")

	defer func() {
		logger.Info("TokenServer::QueryToken["+ctx.RawQuery()+"] => "+jsonutil.GetJsonString(result), constdefine.LogTarget_Token)
		ctx.WriteJson(result)
	}()

	if token == "" || appId == "" {
		result.RetCode = -100001
		result.RetMsg = "querystring token|appid is empty"
		return
	}

	//处理数据
	redisInfo, exists := config.GetRedisInfo(constdefine.RedisID_Token)
	if !exists {
		result.RetCode = -200001
		result.RetMsg = "no config RedisID_Token Redis"
		return
	}
	key := redisInfo.KeyPre + ":Token:" + appId + "_" + token
	redisClient := redisutil.GetRedisClient(redisInfo.ServerIP)
	val, err := redisClient.Get(key)
	if err != nil {
		result.RetCode = -200002
		result.RetMsg = "query token error [get] => " + err.Error()
		return
	}

	if val == "" {
		result.RetCode = -200003
		result.RetMsg = "query token not exists"
		return
	}

	result.RetCode = 0
	result.RetMsg = "ok"
	result.Message = val
}
<commit_msg>update if format<commit_after>package token

import (
	"github.com/devfeel/dotweb"
	"github.com/devfeel/tokenserver/config"
	"github.com/devfeel/tokenserver/const"
	"github.com/devfeel/tokenserver/framework/crypto"
	"github.com/devfeel/tokenserver/framework/json"
	"github.com/devfeel/tokenserver/framework/log"
	"github.com/devfeel/tokenserver/framework/redis"
	"github.com/devfeel/tokenserver/httpserver/model"
	"strconv"
)

type TokenInfo struct {
	Token       string
	AppID       string
	TokenBody   string
	LifeSeconds int //有效时间，单位为秒
}

type VerifyTokenRequest struct {
	Token       string
	AppID       string
	TokenBody   string
	IsCheckBody bool //是否需要验证Body是否一致
}

const (
	Token_DefaultLifeSeconds = 60 * 30 //token默认有效期，默认30分钟有效期
)

/*创建token
Author: panxinming
CreateTime: 2017-02-10 12:00
HttpMethod：Post
Post Param：TokenInfo
返回结构：HandlerResponse
0: 成功
-100001：post data not legal
-100002：AppID is empty
-200001：no config RedisID_Token Redis
-200002：create token error [uuid]
-200003：create token error [redis set]
Response.Message = TokenInfo

UPDATE LOG:
1、初始版本 --2017-02-10 12:00 by pxm
*/
func CreateToken(ctx *dotweb.HttpContext) {
	result := &models.HandlerResponse{RetCode: 0, RetMsg: ""}
	var tokenInfo TokenInfo
	//获取Post内容
	postContent := string(ctx.PostBody())

	defer func() {
		logger.Info("TokenServer::CreateToken["+postContent+"] => "+jsonutil.GetJsonString(result), constdefine.LogTarget_Token)
		ctx.WriteJson(result)
	}()

	//解析提交数据
	if err_jsonunmar := jsonutil.Unmarshal(postContent, &tokenInfo); err_jsonunmar != nil {
		result.RetCode = -100001
		result.RetMsg = "post data not legal => " + err_jsonunmar.Error()
		return
	}
	if tokenInfo.AppID == "" {
		result.RetCode = -100002
		result.RetMsg = "AppID is empty"
		return
	}
	//默认值处理
	if tokenInfo.LifeSeconds <= 0 {
		tokenInfo.LifeSeconds = Token_DefaultLifeSeconds
	}

	//获取redis配置
	redisInfo, exists := config.GetRedisInfo(constdefine.RedisID_Token)
	if !exists {
		result.RetCode = -200001
		result.RetMsg = "no config RedisID_Token Redis"
		return
	}

	//创建token
	token := cryptos.GetGuid()
	if token == "" {
		result.RetCode = -200002
		result.RetMsg = "create token error [uuid]"
		return
	}
	tokenInfo.Token = token

	key := redisInfo.KeyPre + ":Token:" + tokenInfo.AppID + "_" + tokenInfo.Token
	value := jsonutil.GetJsonString(tokenInfo)
	redisClient := redisutil.GetRedisClient(redisInfo.ServerIP)
	val, err := redisClient.SetWithExpire(key, value, tokenInfo.LifeSeconds)
	if err != nil {
		result.RetCode = -200003
		result.RetMsg = "create token error [redis set] => " + err.Error()
		return
	}

	result.RetCode = 0
	result.RetMsg = "ok[" + val + "]"
	result.Message = tokenInfo
	//设置原子锁，默认有效期为token的两倍，初始值为1
	key = redisInfo.KeyPre + ":TokenLock:" + tokenInfo.AppID + "_" + tokenInfo.Token
	redisClient.SetWithExpire(key, "1", tokenInfo.LifeSeconds*2)
}

/*验证token
Author: panxinming
CreateTime: 2017-02-10 12:00
HttpMethod：Post
Post Param：VerifyTokenRequest
返回结构：HandlerResponse
0: 成功
-100001：post data not legal
-100002：AppID is empty
-200001：no config RedisID_Token Redis
-201001: query token exists error
-201002: redis token not exists
-202001: get token-locker error
-202002: token-locker locked by other
-203001: query token error
-203002: redis token data not legal
-203003: token body is not match

Response.Message = TokenInfo

UPDATE LOG:
1、初始版本 --2017-02-10 12:00 by pxm
*/
func VerifyToken(ctx *dotweb.HttpContext) {
	result := &models.HandlerResponse{RetCode: 0, RetMsg: ""}
	var verifyToken VerifyTokenRequest
	//获取Post内容
	postContent := string(ctx.PostBody())

	defer func() {
		logger.Info("TokenServer::VerifyToken["+postContent+"] => "+jsonutil.GetJsonString(result), constdefine.LogTarget_Token)
		ctx.WriteJson(result)
	}()

	//解析提交数据
	err_jsonunmar := jsonutil.Unmarshal(postContent, &verifyToken)
	if err_jsonunmar != nil {
		result.RetCode = -100001
		result.RetMsg = "post data not legal => " + err_jsonunmar.Error()
		return
	}
	if verifyToken.AppID == "" {
		result.RetCode = -100002
		result.RetMsg = "AppID is empty"
		return
	}

	//获取Redis配置
	redisInfo, exists := config.GetRedisInfo(constdefine.RedisID_Token)
	if !exists {
		result.RetCode = -200001
		result.RetMsg = "no config RedisID_Token Redis"
		return
	}

	keyLocker := redisInfo.KeyPre + ":TokenLock:" + verifyToken.AppID + "_" + verifyToken.Token
	keyToken := redisInfo.KeyPre + ":Token:" + verifyToken.AppID + "_" + verifyToken.Token

	//创建Redis链接
	var redisClient *redisutil.RedisClient
	redisClient = redisutil.GetRedisClient(redisInfo.ServerIP)

	//检查token是否存在
	numExists, errExists := redisClient.Exists(keyToken)
	if errExists != nil {
		result.RetCode = -201001
		result.RetMsg = "query token exists error => " + errExists.Error()
		return
	}
	if numExists == 0 {
		result.RetCode = -201002
		result.RetMsg = "redis token not exists"
		return
	}

	//加锁
	valIncr, errIncr := redisClient.DECR(keyLocker)
	if errIncr != nil {
		result.RetCode = -202001
		result.RetMsg = "get token-locker error => " + errIncr.Error()
		return
	}
	if valIncr != 0 {
		result.RetCode = -202002
		result.RetMsg = "token-locker locked by other [" + strconv.Itoa(valIncr) + "]"
		//归还当前锁
		redisClient.INCR(keyLocker)
		return
	}

	var redisToken TokenInfo
	//检查Token
	valToken, errToken := redisClient.Get(keyToken)
	if errToken != nil {
		result.RetCode = -203001
		result.RetMsg = "query token error => " + errToken.Error()
		//归还当前锁
		redisClient.INCR(keyLocker)
		return
	}
	err_jsonunmar = jsonutil.Unmarshal(valToken, &redisToken)
	if err_jsonunmar != nil {
		result.RetCode = -203002
		result.RetMsg = "redis token data not legal [" + valToken + "] => " + err_jsonunmar.Error()
		//归还当前锁
		redisClient.INCR(keyLocker)
		return
	}
	if verifyToken.IsCheckBody {
		if verifyToken.TokenBody != redisToken.TokenBody {
			result.RetCode = -201003
			result.RetMsg = "token body is not match [" + valToken + "]"
			//归还当前锁
			redisClient.INCR(keyLocker)
			return
		}
	}

	result.RetCode = 0
	result.RetMsg = "ok"
	result.Message = redisToken
	//一切正常，删除相关token信息
	redisClient.Del(keyLocker)
	redisClient.Del(keyToken)
}

/*查询token
Author: panxinming
CreateTime: 2017-02-10 12:00
HttpMethod：Get
Get Param：
appid: 应用编码
token: token
返回结构：HandlerResponse
0: 成功
-100001：querystring token|appid is empty
-200001：no config RedisID_Token Redis
-200002：query token error [redis get]
Response.Message = TokenInfo

UPDATE LOG:
1、初始版本 --2017-02-10 12:00 by pxm
*/
func QueryToken(ctx *dotweb.HttpContext) {
	result := &models.HandlerResponse{RetCode: 0, RetMsg: ""}
	token := ctx.QueryString("token")
	appId := ctx.QueryString("appid")

	defer func() {
		logger.Info("TokenServer::QueryToken["+ctx.RawQuery()+"] => "+jsonutil.GetJsonString(result), constdefine.LogTarget_Token)
		ctx.WriteJson(result)
	}()

	if token == "" || appId == "" {
		result.RetCode = -100001
		result.RetMsg = "querystring token|appid is empty"
		return
	}

	//处理数据
	redisInfo, exists := config.GetRedisInfo(constdefine.RedisID_Token)
	if !exists {
		result.RetCode = -200001
		result.RetMsg = "no config RedisID_Token Redis"
		return
	}
	key := redisInfo.KeyPre + ":Token:" + appId + "_" + token
	redisClient := redisutil.GetRedisClient(redisInfo.ServerIP)
	val, err := redisClient.Get(key)
	if err != nil {
		result.RetCode = -200002
		result.RetMsg = "query token error [get] => " + err.Error()
		return
	}

	if val == "" {
		result.RetCode = -200003
		result.RetMsg = "query token not exists"
		return
	}

	result.RetCode = 0
	result.RetMsg = "ok"
	result.Message = val
}
<|endoftext|>


<commit_before>package effio

/*
 * effio provides ways to build a suite of fio tests & devices, combine them
 * then generate reports / graphs.
 *
 * The fio config file name is stripped of its path and .fio extension to be
 * used in composite test names. A test against the above device with
 * "rand_512b_write_iops.fio" will be named
 * "rand_512b_write_iops-samsung_840_pro_256".
 */

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path"
	"strings"
	"time"
)

// test data generated on the fly based on info above
type Test struct {
	Name        string      // name to be used in tests, files, etc.
	Dir         string      // directory for writing configs, logs, etc.
	FioArgs     []string    // the arguments to the fio command for the test
	FioFile     string      // generated fio config file name
	FioJson     string      // generated fio json output file name
	FioBWLog    string      // filename for the bandwidth log
	FioLatLog   string      // filename for the latency log
	FioIopsLog  string      // filename for the iops log
	TestJson    string      // dump the test data (this struct) to this file
	CmdFile     string      // write the exact fio command used to this file
	FioConfTmpl FioConfTmpl // template info struct
	Device      Device      // device info struct
}

// a test suite has a global id that is also used as a directory name
type Suite struct {
	Id        string
	Created   time.Time // time the test was generated / run
	EffioCmd  []string  // os.Args() of the effio command used to generate the suite
	SuiteJson string
	Tests     []Test
}

// NewSuite returns an initialized Suite with the given
// id and the Created field set to the current time.
func NewSuite(id string) Suite {
	now := time.Now()
	fname := path.Join(id, "suite.json")
	return Suite{id, now, os.Args, fname, []Test{}}
}

// LoadSuiteJson loads a suite from JSON. Argument is a path to a
// JSON file that has a complete suite's information in it.
func LoadSuiteJson(spath string) (suite Suite) {
	data, err := ioutil.ReadFile(spath)
	if err != nil {
		log.Fatalf("Could not read suite JSON file '%s': %s", spath, err)
	}

	err = json.Unmarshal(data, &suite)
	if err != nil {
		log.Fatalf("Could not parse suite JSON in file '%s': %s", spath, err)
	}

	return suite
}

// Run the whole suite one at a time letting fio write its output into
// the suite directories. Repeated runs will overwrite files; behavior
// is dependent on what fio does with existing files for now.
func (suite *Suite) Run(spath string) {
	for _, test := range suite.Tests {
		test.Run(spath)
	}
}

// Populate the test suite with the (cartesian) product of
// Devices x FioConfTmpls to get all combinations.
// This does not modify the filesystem.
func (suite *Suite) Populate(dl Devices, ftl FioConfTmpls) {
	for _, tp := range ftl {
		for _, dev := range dl {
			// I suppose these conventions could be defined higher up in the call stack
			// but this makes things a little easier to modify down the road.
			testName := fmt.Sprintf("%s-%s", dev.Name, tp.Name)
			testDir := path.Join(suite.Id, testName)
			args := []string{"--output-format=json", "--output=output.json", "config.fio"}

			// fio adds _$type.log to log file names so only provide the base name
			test := Test{
				Name:        testName,
				Dir:         testDir,
				FioArgs:     args,
				FioFile:     "config.fio",
				FioJson:     "output.json",
				FioBWLog:    "bw",
				FioLatLog:   "lat",
				FioIopsLog:  "iops",
				TestJson:    "test.json",
				CmdFile:     "run.sh",
				FioConfTmpl: tp,
				Device:      dev,
			}

			suite.Tests = append(suite.Tests, test)
		}
	}
}

// WriteAll(path) writes a suite out to a set of directories and files.
func (suite *Suite) WriteAll(basePath string) {
	suite.mkdirAll(basePath)

	suite.WriteSuiteJson(basePath)

	for _, test := range suite.Tests {
		test.WriteFioFile(basePath)
		test.WriteTestJson(basePath)
		test.WriteCmdFile(basePath)
	}
}

// WriteSuiteJson dumps the suite data structure to a JSON file. This
// file is used by some effio subcommands, such as run_suite and various
// reports.
// <basePath>/<suite id>/suite.json
func (suite *Suite) WriteSuiteJson(basePath string) {
	outfile := path.Join(basePath, suite.SuiteJson)

	js, err := json.MarshalIndent(suite, "", "  ")
	if err != nil {
		log.Fatalf("Failed to encode suite data as JSON: %s\n", err)
	}

	// MarshalIndent does not follow the final brace with a newline
	js = append(js, byte('\n'))

	err = ioutil.WriteFile(outfile, js, 0644)
	if err != nil {
		log.Fatalf("Failed to write suite JSON data file '%s': %s\n", outfile, err)
	}
}

// mkdirAll(path) creates the directory structure of a test suite
// under directory 'path'. This must be called before the Write*()
// methods or they will fail. It only makes sense to call this after
// Populate().
func (suite *Suite) mkdirAll(basePath string) {
	sdir := path.Join(basePath, suite.Id)

	for _, t := range suite.Tests {
		tdir := path.Join(sdir, t.Name)
		err := os.MkdirAll(tdir, 0755)
		if err != nil {
			log.Fatalf("Failed to create test directory '%s': %s\n", tdir, err)
		}
	}
}

func (test *Test) Run(spath string) {
	tpath := path.Join(spath, test.Dir)
	err := os.Chdir(tpath)
	if err != nil {
		log.Fatalf("Could not chdir to '%s': %s\n", tpath, err)
	}

	fioPath, err := exec.LookPath("fio")
	if err != nil {
		log.Fatalf("Could not locate an fio command in PATH: %s\n", err)
	}

	cmd := exec.Command(fioPath, test.FioArgs...)
	before := time.Now()

	stderr, err := cmd.StderrPipe()
	if err != nil {
		log.Fatal(err)
	}

	if err := cmd.Start(); err != nil {
		log.Fatalf("Could not run '%s %s': %s\n", fioPath, strings.Join(test.FioArgs, " "))
	}

	// grab stderr in case something goes wrong
	// TODO: switch this to io.Copy?
	errors, err := ioutil.ReadAll(stderr)
	if err != nil {
		log.Fatal(err)
	}

	err = cmd.Wait()

	// TODO: figure out if this is worth recording and record it
	elapsed := time.Since(before)

	// it might be OK to let 1 fio command out of a suite fail?
	if err != nil {
		log.Printf(string(errors))
		log.Fatalf("Command '%s %s' failed: %s\n", fioPath, strings.Join(test.FioArgs, " "), err)
	}
	log.Printf("Elapsed: %s\n", elapsed)
}

// DumpFioFile writes the fio configuration file.
// <basePath>/<suite id>/<generated test name>/config.fio
func (test *Test) WriteFioFile(basePath string) {
	outfile := path.Join(basePath, test.Dir, test.FioFile)

	fd, err := os.OpenFile(outfile, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
	if err != nil {
		log.Fatalf("Failed to create fio config file '%s': %s\n", outfile, err)
	}
	defer fd.Close()

	err = test.FioConfTmpl.tmpl.Execute(fd, test)
	if err != nil {
		log.Fatalf("Template execution failed: %s\n", err)
	}
}

// WriteTestJson dumps the suite data structure to a JSON file for posterity (and debugging).
// <basePath>/<suite id>/<generated test name>/test.json
func (test *Test) WriteTestJson(basePath string) {
	outfile := path.Join(basePath, test.Dir, test.TestJson)

	js, err := json.MarshalIndent(test, "", "  ")
	if err != nil {
		log.Fatalf("Failed to encode test data as JSON: %s\n", err)
	}

	// MarshalIndent does not follow the final brace with a newline
	js = append(js, byte('\n'))

	err = ioutil.WriteFile(outfile, js, 0644)
	if err != nil {
		log.Fatalf("Failed to write test JSON data file '%s': %s\n", outfile, err)
	}
}

// WriteCmdFile writes the command to a file as a mini shell script.
// <basePath>/<suite id>/<test name>/run.sh
func (test *Test) WriteCmdFile(basePath string) {
	outfile := path.Join(basePath, test.Dir, test.CmdFile)

	fd, err := os.OpenFile(outfile, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0755)
	if err != nil {
		log.Fatalf("Failed to create command file '%s': %s\n", outfile, err)
	}
	defer fd.Close()

	// just use 'fio' if it isn't found on the path
	fioPath, err := exec.LookPath("fio")
	if err != nil {
		fioPath = "fio"
	}
	fmt.Fprintf(fd, "#!/bin/bash -x\n%s %s\n", fioPath, strings.Join(test.FioArgs, " "))
}
<commit_msg>Implement the 'ignore' flag.<commit_after>package effio

/*
 * effio provides ways to build a suite of fio tests & devices, combine them
 * then generate reports / graphs.
 *
 * The fio config file name is stripped of its path and .fio extension to be
 * used in composite test names. A test against the above device with
 * "rand_512b_write_iops.fio" will be named
 * "rand_512b_write_iops-samsung_840_pro_256".
 */

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path"
	"strings"
	"time"
)

// test data generated on the fly based on info above
type Test struct {
	Name        string      // name to be used in tests, files, etc.
	Dir         string      // directory for writing configs, logs, etc.
	FioArgs     []string    // the arguments to the fio command for the test
	FioFile     string      // generated fio config file name
	FioJson     string      // generated fio json output file name
	FioBWLog    string      // filename for the bandwidth log
	FioLatLog   string      // filename for the latency log
	FioIopsLog  string      // filename for the iops log
	TestJson    string      // dump the test data (this struct) to this file
	CmdFile     string      // write the exact fio command used to this file
	FioConfTmpl FioConfTmpl // template info struct
	Device      Device      // device info struct
}

// a test suite has a global id that is also used as a directory name
type Suite struct {
	Id        string
	Created   time.Time // time the test was generated / run
	EffioCmd  []string  // os.Args() of the effio command used to generate the suite
	SuiteJson string
	Tests     []Test
}

// NewSuite returns an initialized Suite with the given
// id and the Created field set to the current time.
func NewSuite(id string) Suite {
	now := time.Now()
	fname := path.Join(id, "suite.json")
	return Suite{id, now, os.Args, fname, []Test{}}
}

// LoadSuiteJson loads a suite from JSON. Argument is a path to a
// JSON file that has a complete suite's information in it.
func LoadSuiteJson(spath string) (suite Suite) {
	data, err := ioutil.ReadFile(spath)
	if err != nil {
		log.Fatalf("Could not read suite JSON file '%s': %s", spath, err)
	}

	err = json.Unmarshal(data, &suite)
	if err != nil {
		log.Fatalf("Could not parse suite JSON in file '%s': %s", spath, err)
	}

	return suite
}

// Run the whole suite one at a time letting fio write its output into
// the suite directories. Repeated runs will overwrite files; behavior
// is dependent on what fio does with existing files for now.
func (suite *Suite) Run(spath string) {
	for _, test := range suite.Tests {
		test.Run(spath)
	}
}

// Populate the test suite with the (cartesian) product of
// Devices x FioConfTmpls to get all combinations.
// This does not modify the filesystem.
func (suite *Suite) Populate(dl Devices, ftl FioConfTmpls) {
	for _, tp := range ftl {
		for _, dev := range dl {
			if dev.Ignore {
				continue
			}

			// I suppose these conventions could be defined higher up in the call stack
			// but this makes things a little easier to modify down the road.
			testName := fmt.Sprintf("%s-%s", dev.Name, tp.Name)
			testDir := path.Join(suite.Id, testName)
			args := []string{"--output-format=json", "--output=output.json", "config.fio"}

			// fio adds _$type.log to log file names so only provide the base name
			test := Test{
				Name:        testName,
				Dir:         testDir,
				FioArgs:     args,
				FioFile:     "config.fio",
				FioJson:     "output.json",
				FioBWLog:    "bw",
				FioLatLog:   "lat",
				FioIopsLog:  "iops",
				TestJson:    "test.json",
				CmdFile:     "run.sh",
				FioConfTmpl: tp,
				Device:      dev,
			}

			suite.Tests = append(suite.Tests, test)
		}
	}
}

// WriteAll(path) writes a suite out to a set of directories and files.
func (suite *Suite) WriteAll(basePath string) {
	suite.mkdirAll(basePath)

	suite.WriteSuiteJson(basePath)

	for _, test := range suite.Tests {
		test.WriteFioFile(basePath)
		test.WriteTestJson(basePath)
		test.WriteCmdFile(basePath)
	}
}

// WriteSuiteJson dumps the suite data structure to a JSON file. This
// file is used by some effio subcommands, such as run_suite and various
// reports.
// <basePath>/<suite id>/suite.json
func (suite *Suite) WriteSuiteJson(basePath string) {
	outfile := path.Join(basePath, suite.SuiteJson)

	js, err := json.MarshalIndent(suite, "", "  ")
	if err != nil {
		log.Fatalf("Failed to encode suite data as JSON: %s\n", err)
	}

	// MarshalIndent does not follow the final brace with a newline
	js = append(js, byte('\n'))

	err = ioutil.WriteFile(outfile, js, 0644)
	if err != nil {
		log.Fatalf("Failed to write suite JSON data file '%s': %s\n", outfile, err)
	}
}

// mkdirAll(path) creates the directory structure of a test suite
// under directory 'path'. This must be called before the Write*()
// methods or they will fail. It only makes sense to call this after
// Populate().
func (suite *Suite) mkdirAll(basePath string) {
	sdir := path.Join(basePath, suite.Id)

	for _, t := range suite.Tests {
		tdir := path.Join(sdir, t.Name)
		err := os.MkdirAll(tdir, 0755)
		if err != nil {
			log.Fatalf("Failed to create test directory '%s': %s\n", tdir, err)
		}
	}
}

func (test *Test) Run(spath string) {
	tpath := path.Join(spath, test.Dir)
	err := os.Chdir(tpath)
	if err != nil {
		log.Fatalf("Could not chdir to '%s': %s\n", tpath, err)
	}

	fioPath, err := exec.LookPath("fio")
	if err != nil {
		log.Fatalf("Could not locate an fio command in PATH: %s\n", err)
	}

	cmd := exec.Command(fioPath, test.FioArgs...)
	before := time.Now()

	stderr, err := cmd.StderrPipe()
	if err != nil {
		log.Fatal(err)
	}

	if err := cmd.Start(); err != nil {
		log.Fatalf("Could not run '%s %s': %s\n", fioPath, strings.Join(test.FioArgs, " "))
	}

	// grab stderr in case something goes wrong
	// TODO: switch this to io.Copy?
	errors, err := ioutil.ReadAll(stderr)
	if err != nil {
		log.Fatal(err)
	}

	err = cmd.Wait()

	// TODO: figure out if this is worth recording and record it
	elapsed := time.Since(before)

	// it might be OK to let 1 fio command out of a suite fail?
	if err != nil {
		log.Printf(string(errors))
		log.Fatalf("Command '%s %s' failed: %s\n", fioPath, strings.Join(test.FioArgs, " "), err)
	}
	log.Printf("Elapsed: %s\n", elapsed)
}

// DumpFioFile writes the fio configuration file.
// <basePath>/<suite id>/<generated test name>/config.fio
func (test *Test) WriteFioFile(basePath string) {
	outfile := path.Join(basePath, test.Dir, test.FioFile)

	fd, err := os.OpenFile(outfile, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
	if err != nil {
		log.Fatalf("Failed to create fio config file '%s': %s\n", outfile, err)
	}
	defer fd.Close()

	err = test.FioConfTmpl.tmpl.Execute(fd, test)
	if err != nil {
		log.Fatalf("Template execution failed: %s\n", err)
	}
}

// WriteTestJson dumps the suite data structure to a JSON file for posterity (and debugging).
// <basePath>/<suite id>/<generated test name>/test.json
func (test *Test) WriteTestJson(basePath string) {
	outfile := path.Join(basePath, test.Dir, test.TestJson)

	js, err := json.MarshalIndent(test, "", "  ")
	if err != nil {
		log.Fatalf("Failed to encode test data as JSON: %s\n", err)
	}

	// MarshalIndent does not follow the final brace with a newline
	js = append(js, byte('\n'))

	err = ioutil.WriteFile(outfile, js, 0644)
	if err != nil {
		log.Fatalf("Failed to write test JSON data file '%s': %s\n", outfile, err)
	}
}

// WriteCmdFile writes the command to a file as a mini shell script.
// <basePath>/<suite id>/<test name>/run.sh
func (test *Test) WriteCmdFile(basePath string) {
	outfile := path.Join(basePath, test.Dir, test.CmdFile)

	fd, err := os.OpenFile(outfile, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0755)
	if err != nil {
		log.Fatalf("Failed to create command file '%s': %s\n", outfile, err)
	}
	defer fd.Close()

	// just use 'fio' if it isn't found on the path
	fioPath, err := exec.LookPath("fio")
	if err != nil {
		fioPath = "fio"
	}
	fmt.Fprintf(fd, "#!/bin/bash -x\n%s %s\n", fioPath, strings.Join(test.FioArgs, " "))
}
<|endoftext|>


<commit_before>package terraform

import "errors"

// Credentials holds all the information that are necessary to
// make TerraForm authenticate against various providers.
type Credentials struct {
	*AWSCredentials `json:",inline"`
}

// ToEnvironment returns the environment variables that are expected by
// TerraForm's provider.
func (c *Credentials) ToEnvironment() (map[string]string, error) {
	env := make(map[string]string)

	if c.AWSCredentials != nil {
		if err := c.AWSCredentials.Validate(); err != nil {
			return nil, err
		}
		mergeEnvMap(env, c.AWSCredentials.ToEnvironment())
	}

	return env, nil
}

// AWSCredentials represents the credentials required by TerraForm's AWS
// provider.
type AWSCredentials struct {
	AWSAccessKeyID     string `json:"AWSAccessKeyID"`
	AWSSecretAccessKey string `json:"AWSSecretAccessKey"`
	AWSSessionToken    string `json:"AWSSessionToken"`
	AWSRegion          string `json:"AWSRegion"`
}

// Validate verifies that the given credentials are valid.
func (a *AWSCredentials) Validate() error {
	if a.AWSAccessKeyID == "" || a.AWSSecretAccessKey == "" || a.AWSRegion == "" {
		return errors.New("AWSAccessKeyID, AWSSecretAccessKey, AWSRegion must be" +
			"specified.")
	}
	return nil
}

// ToEnvironment returns the environment variables that are expected by
// TerraForm's provider.
func (a *AWSCredentials) ToEnvironment() map[string]string {
	return map[string]string{
		"AWS_ACCESS_KEY_ID":     a.AWSAccessKeyID,
		"AWS_SECRET_ACCESS_KEY": a.AWSSecretAccessKey,
		"AWS_SESSION_TOKEN":     a.AWSSessionToken,
		"AWS_REGION":            a.AWSRegion,
	}
}

// mergeEnvMap is an utility function that write all keys from the src string
// map to the destination one, overriding any existing one.
func mergeEnvMap(dst, src map[string]string) {
	for k, v := range src {
		dst[k] = v
	}
}
<commit_msg>installer/server/terraform: remove unused AWS Region<commit_after>package terraform

import "errors"

// Credentials holds all the information that are necessary to
// make TerraForm authenticate against various providers.
type Credentials struct {
	*AWSCredentials `json:",inline"`
}

// ToEnvironment returns the environment variables that are expected by
// TerraForm's provider.
func (c *Credentials) ToEnvironment() (map[string]string, error) {
	env := make(map[string]string)

	if c.AWSCredentials != nil {
		if err := c.AWSCredentials.Validate(); err != nil {
			return nil, err
		}
		mergeEnvMap(env, c.AWSCredentials.ToEnvironment())
	}

	return env, nil
}

// AWSCredentials represents the credentials required by TerraForm's AWS
// provider.
type AWSCredentials struct {
	AWSAccessKeyID     string `json:"AWSAccessKeyID"`
	AWSSecretAccessKey string `json:"AWSSecretAccessKey"`
	AWSSessionToken    string `json:"AWSSessionToken"`
}

// Validate verifies that the given credentials are valid.
func (a *AWSCredentials) Validate() error {
	if a.AWSAccessKeyID == "" || a.AWSSecretAccessKey == "" {
		return errors.New("AWSAccessKeyID & AWSSecretAccessKey must be specified.")
	}
	return nil
}

// ToEnvironment returns the environment variables that are expected by
// TerraForm's provider.
func (a *AWSCredentials) ToEnvironment() map[string]string {
	return map[string]string{
		"AWS_ACCESS_KEY_ID":     a.AWSAccessKeyID,
		"AWS_SECRET_ACCESS_KEY": a.AWSSecretAccessKey,
		"AWS_SESSION_TOKEN":     a.AWSSessionToken,
	}
}

// mergeEnvMap is an utility function that write all keys from the src string
// map to the destination one, overriding any existing one.
func mergeEnvMap(dst, src map[string]string) {
	for k, v := range src {
		dst[k] = v
	}
}
<|endoftext|>


<commit_before>package lr

import (
	"fmt"
	"sort"
	"strings"
	"text/template"

	"gen/codegen"
)

// Action is an entry in the action table, indicating which way to transition
// the state machine.
type Action interface{}

// Shift is an action that means "accept the next token and change states".
type Shift struct {
	state int
}

// Reduce is an action that means "pop up the stack based on a rule".
// Reducing to the root rule means the input is accepted.
type Reduce struct {
	rule *Rule
}

// ActionTable maps parser states to rows; each row maps tokens to actions.
type ActionTable []map[string]Action

func (at ActionTable) Dump(log Logger) {
	log.Println("parsing table:")
	for i, actions := range at {
		line := fmt.Sprintf("%2d", i)
		for term, action := range actions {
			line += fmt.Sprintf(" %s:%s", term, action)
		}
		log.Println(line)
	}
}

// Item is a partially-parsed production.
type Item struct {
	rule *Rule
	// pos represents the character offset that the dot is to the left of,
	// representing a parse up to that character.  If pos == len(rule.pattern)
	// the Item represents a dot to the right of all characters.
	pos int
}

// NextSym returns the next symbol the Item would match and whether
// the Item is at the end of its pattern.
func (i Item) NextSym() (sym string, end bool) {
	if i.pos == len(i.rule.pattern) {
		return "", true
	}
	return i.rule.pattern[i.pos], false
}

// ItemSet is a set of Items.
type ItemSet map[Item]bool

func (is ItemSet) Add(it Item)      { is[it] = true }
func (is ItemSet) Has(it Item) bool { return is[it] }
func (is ItemSet) Empty() bool      { return len(is) == 0 }
func (is ItemSet) Equals(os ItemSet) bool {
	// This isn't generic set equality, but rather relies on the maps
	// only containing true values.
	if len(is) != len(os) {
		return false
	}
	for k := range is {
		if !os[k] {
			return false
		}
	}
	return true
}

func (is ItemSet) Dump(log Logger) {
	for item := range is {
		log.Println(" ", item.rule.Show("->", item.pos))
	}
}

// Closure computes the closure of an ItemSet.
// For example, if the set contains
//    x -> a.b
// then the closure should also contain all expansions of b.
func (is ItemSet) Closure(grammar *Grammar) {
	added := make(map[string]bool)

	for changed := true; changed; {
		changed = false
		for item := range is {
			// Given an item like x -> a.b, grab b.
			sym, end := item.NextSym()
			if end {
				continue
			}
			// If we haven't yet added b, find its expansions in the grammar.
			if !added[sym] {
				for _, rule := range grammar.rules {
					if rule.symbol == sym {
						is.Add(Item{rule, 0})
						changed = true
						added[sym] = true
					}
				}
			}
		}
	}

	// TODO: prune nonkernel items?
}

// Goto computes the resulting set of states given an input token.
func (is ItemSet) Goto(grammar *Grammar, x string) ItemSet {
	out := make(ItemSet)
	for item := range is {
		if sym, end := item.NextSym(); !end && sym == x {
			out.Add(Item{item.rule, item.pos + 1})
		}
	}
	out.Closure(grammar)
	return out
}

func ComputeActions(grammar *Grammar, trace Logger) ActionTable {
	first := grammar.First(trace)
	follow := grammar.Follow(first)
	if trace != nil {
		follow.Dump(trace, "follow set")
	}

	var allActions ActionTable

	states := []ItemSet{
		ItemSet{Item{grammar.rules[0], 0}: true},
	}
	states[0].Closure(grammar)

	// Construct the parsing states list by computing goto() for each
	// state and terminal.
	for i := 0; i < len(states); i++ {
		set := states[i]
		actions := make(map[string]Action)
		allActions = append(allActions, actions)

		for term := range grammar.symbols {
			c := set.Goto(grammar, term)
			if c.Empty() {
				continue
			}

			// Save this set if new.
			id := -1
			for j, oset := range states {
				if c.Equals(oset) {
					id = j
					break
				}
			}
			if id == -1 {
				states = append(states, c)
				id = len(states) - 1
			}

			actions[term] = Shift{state: id}
		}
	}

	// Add a reduce action for all items that have consumed the full rule.
	for i, set := range states {
		actions := allActions[i]
		for item := range set {
			if _, end := item.NextSym(); !end {
				// Still more terminals on this item.
				continue
			}

			f := follow[item.rule.symbol]
			for term := range f {
				if actions[term] != nil {
					// TODO: don't use traceLog
					traceLog.Println("reduce conflict!")
					set.Dump(traceLog)
					traceLog.Printf("in state %d on input %s, %#v vs %v", i, term, actions[term], item.rule)
				}
				actions[term] = Reduce{rule: item.rule}
			}
		}
	}

	if trace != nil {
		for i, set := range states {
			trace.Printf("set %d:\n", i)
			set.Dump(trace)
		}
	}

	return allActions
}

func writeTables(w *codegen.Writer, prefix string, grammar *Grammar, table ActionTable) {
	types := make(map[string]string)
	for _, rule := range grammar.rules {
		types[rule.symbol] = rule.typ
	}

	ruleIds := make(map[*Rule]int)

	w.Linef(`var %sRules = []*%sRule{`, prefix, prefix)
	for i, rule := range grammar.rules {
		ruleIds[rule] = i
		w.Linef(`{%q, %#v,`, rule.symbol, rule.pattern)
		if rule.code != "" {
			w.Line("func(data []interface{}) interface{} {")
			for j, varname := range rule.vars {
				if varname != "" {
					typ := types[rule.pattern[j]]
					if typ == "" {
						typ = "Tok"
					}
					w.Linef("%s := data[%d].(%s)", varname, j, typ)
				}
			}
			w.Line(strings.Trim(rule.code, " \t\n"))
			w.Line("},")
		} else {
			w.Line("nil,")
		}
		w.Line(`},`)
	}
	w.Line(`}`)

	w.Line("")

	w.Linef(`var %sActions = %sActionTable{`, prefix, prefix)
	for _, state := range table {
		w.Line(`{`)
		var keys []string
		for k := range state {
			keys = append(keys, k)
		}
		sort.Strings(keys)
		for _, tok := range keys {
			action := state[tok]
			var str string
			switch a := action.(type) {
			case Shift:
				str = fmt.Sprintf("%d", a.state)
			case Reduce:
				str = fmt.Sprintf("%d", -ruleIds[a.rule])
			default:
				panic("unhandled case")
			}
			w.Linef(`%q: %s,`, tok, str)
		}
		w.Line(`},`)
	}
	w.Line(`}`)
}

func Main(infile string, verbose bool) ([]byte, error) {
	var trace Logger
	if verbose {
		trace = traceLog
	}

	params, rules, err := Parse(infile)
	if err != nil {
		return nil, err
	}

	if trace != nil {
		trace.Println("loaded rule table")
		for i, rule := range rules {
			trace.Printf("  %d: %s\n", i, rule.Show("->", -1))
		}
	}

	g := &Grammar{rules:rules}
	actions := ComputeActions(g, trace)

	// Graph(g, actions)
	// return

	w := &codegen.Writer{}
	tmpl := template.Must(template.New("parse").Parse(
		strings.Replace(parseTemplate, "$", params.Prefix, -1)))
	tmpl.Execute(w, params)

	w.Line("// Result returns the final result of a successful parse.")
	w.Linef("func (p *%sParser) Result() %s {", params.Prefix, g.rules[0].typ)
	w.Linef("return p.data[0].(%s)", g.rules[0].typ)
	w.Line("}")

	writeTables(w, params.Prefix, g, actions)

	code, err := w.Fmt()
	if err != nil {
		return nil, fmt.Errorf("error formatting code: %s\ncode: %s\n", err, w.Raw())
	}
	return code, nil
}
<commit_msg>use TokenType everywhere<commit_after>package lr

import (
	"fmt"
	"sort"
	"strings"
	"text/template"

	"gen/codegen"
)

// Action is an entry in the action table, indicating which way to transition
// the state machine.
type Action interface{}

// Shift is an action that means "accept the next token and change states".
type Shift struct {
	state int
}

// Reduce is an action that means "pop up the stack based on a rule".
// Reducing to the root rule means the input is accepted.
type Reduce struct {
	rule *Rule
}

// ActionTable maps parser states to rows; each row maps tokens to actions.
type ActionTable []map[string]Action

func (at ActionTable) Dump(log Logger) {
	log.Println("parsing table:")
	for i, actions := range at {
		line := fmt.Sprintf("%2d", i)
		for term, action := range actions {
			line += fmt.Sprintf(" %s:%s", term, action)
		}
		log.Println(line)
	}
}

// Item is a partially-parsed production.
type Item struct {
	rule *Rule
	// pos represents the character offset that the dot is to the left of,
	// representing a parse up to that character.  If pos == len(rule.pattern)
	// the Item represents a dot to the right of all characters.
	pos int
}

// NextSym returns the next symbol the Item would match and whether
// the Item is at the end of its pattern.
func (i Item) NextSym() (sym string, end bool) {
	if i.pos == len(i.rule.pattern) {
		return "", true
	}
	return i.rule.pattern[i.pos], false
}

// ItemSet is a set of Items.
type ItemSet map[Item]bool

func (is ItemSet) Add(it Item)      { is[it] = true }
func (is ItemSet) Has(it Item) bool { return is[it] }
func (is ItemSet) Empty() bool      { return len(is) == 0 }
func (is ItemSet) Equals(os ItemSet) bool {
	// This isn't generic set equality, but rather relies on the maps
	// only containing true values.
	if len(is) != len(os) {
		return false
	}
	for k := range is {
		if !os[k] {
			return false
		}
	}
	return true
}

func (is ItemSet) Dump(log Logger) {
	for item := range is {
		log.Println(" ", item.rule.Show("->", item.pos))
	}
}

// Closure computes the closure of an ItemSet.
// For example, if the set contains
//    x -> a.b
// then the closure should also contain all expansions of b.
func (is ItemSet) Closure(grammar *Grammar) {
	added := make(map[string]bool)

	for changed := true; changed; {
		changed = false
		for item := range is {
			// Given an item like x -> a.b, grab b.
			sym, end := item.NextSym()
			if end {
				continue
			}
			// If we haven't yet added b, find its expansions in the grammar.
			if !added[sym] {
				for _, rule := range grammar.rules {
					if rule.symbol == sym {
						is.Add(Item{rule, 0})
						changed = true
						added[sym] = true
					}
				}
			}
		}
	}

	// TODO: prune nonkernel items?
}

// Goto computes the resulting set of states given an input token.
func (is ItemSet) Goto(grammar *Grammar, x string) ItemSet {
	out := make(ItemSet)
	for item := range is {
		if sym, end := item.NextSym(); !end && sym == x {
			out.Add(Item{item.rule, item.pos + 1})
		}
	}
	out.Closure(grammar)
	return out
}

func ComputeActions(grammar *Grammar, trace Logger) ActionTable {
	first := grammar.First(trace)
	follow := grammar.Follow(first)
	if trace != nil {
		follow.Dump(trace, "follow set")
	}

	var allActions ActionTable

	states := []ItemSet{
		ItemSet{Item{grammar.rules[0], 0}: true},
	}
	states[0].Closure(grammar)

	// Construct the parsing states list by computing goto() for each
	// state and terminal.
	for i := 0; i < len(states); i++ {
		set := states[i]
		actions := make(map[string]Action)
		allActions = append(allActions, actions)

		for term := range grammar.symbols {
			c := set.Goto(grammar, term)
			if c.Empty() {
				continue
			}

			// Save this set if new.
			id := -1
			for j, oset := range states {
				if c.Equals(oset) {
					id = j
					break
				}
			}
			if id == -1 {
				states = append(states, c)
				id = len(states) - 1
			}

			actions[term] = Shift{state: id}
		}
	}

	// Add a reduce action for all items that have consumed the full rule.
	for i, set := range states {
		actions := allActions[i]
		for item := range set {
			if _, end := item.NextSym(); !end {
				// Still more terminals on this item.
				continue
			}

			f := follow[item.rule.symbol]
			for term := range f {
				if actions[term] != nil {
					// TODO: don't use traceLog
					traceLog.Println("reduce conflict!")
					set.Dump(traceLog)
					traceLog.Printf("in state %d on input %s, %#v vs %v", i, term, actions[term], item.rule)
				}
				actions[term] = Reduce{rule: item.rule}
			}
		}
	}

	if trace != nil {
		for i, set := range states {
			trace.Printf("set %d:\n", i)
			set.Dump(trace)
		}
	}

	return allActions
}

func writeTables(w *codegen.Writer, params *Params, grammar *Grammar, table ActionTable) {
	types := make(map[string]string)
	for _, rule := range grammar.rules {
		types[rule.symbol] = rule.typ
	}

	ruleIds := make(map[*Rule]int)

	w.Linef(`var %sRules = []*%sRule{`, params.Prefix, params.Prefix)
	for i, rule := range grammar.rules {
		ruleIds[rule] = i
		w.Linef(`{%q, %#v,`, rule.symbol, rule.pattern)
		if rule.code != "" {
			w.Line("func(data []interface{}) interface{} {")
			for j, varname := range rule.vars {
				if varname != "" {
					typ := types[rule.pattern[j]]
					if typ == "" {
						typ = params.TokenType
					}
					w.Linef("%s := data[%d].(%s)", varname, j, typ)
				}
			}
			w.Line(strings.Trim(rule.code, " \t\n"))
			w.Line("},")
		} else {
			w.Line("nil,")
		}
		w.Line(`},`)
	}
	w.Line(`}`)

	w.Line("")

	w.Linef(`var %sActions = %sActionTable{`, params.Prefix, params.Prefix)
	for _, state := range table {
		w.Line(`{`)
		var keys []string
		for k := range state {
			keys = append(keys, k)
		}
		sort.Strings(keys)
		for _, tok := range keys {
			action := state[tok]
			var str string
			switch a := action.(type) {
			case Shift:
				str = fmt.Sprintf("%d", a.state)
			case Reduce:
				str = fmt.Sprintf("%d", -ruleIds[a.rule])
			default:
				panic("unhandled case")
			}
			w.Linef(`%q: %s,`, tok, str)
		}
		w.Line(`},`)
	}
	w.Line(`}`)
}

func Main(infile string, verbose bool) ([]byte, error) {
	var trace Logger
	if verbose {
		trace = traceLog
	}

	params, rules, err := Parse(infile)
	if err != nil {
		return nil, err
	}

	if trace != nil {
		trace.Println("loaded rule table")
		for i, rule := range rules {
			trace.Printf("  %d: %s\n", i, rule.Show("->", -1))
		}
	}

	g := &Grammar{rules:rules}
	actions := ComputeActions(g, trace)

	// Graph(g, actions)
	// return

	w := &codegen.Writer{}
	tmpl := template.Must(template.New("parse").Parse(
		strings.Replace(parseTemplate, "$", params.Prefix, -1)))
	tmpl.Execute(w, params)

	w.Line("// Result returns the final result of a successful parse.")
	w.Linef("func (p *%sParser) Result() %s {", params.Prefix, g.rules[0].typ)
	w.Linef("return p.data[0].(%s)", g.rules[0].typ)
	w.Line("}")

	writeTables(w, params, g, actions)

	code, err := w.Fmt()
	if err != nil {
		return nil, fmt.Errorf("error formatting code: %s\ncode: %s\n", err, w.Raw())
	}
	return code, nil
}
<|endoftext|>