<commit_before>package manager

import (
	"bytes"
	"encoding/json"
	"io"
	"net/http"

	"github.com/dchest/uniuri"
	"github.com/julienschmidt/httprouter"
	"gopkg.in/vinxi/vinxi.v0"
	"gopkg.in/vinxi/vinxi.v0/config"
	"gopkg.in/vinxi/vinxi.v0/layer"
	"gopkg.in/vinxi/vinxi.v0/plugin"
	"gopkg.in/vinxi/vinxi.v0/rule"

	// An empty import is required to load all the rules subpackages
	_ "gopkg.in/vinxi/vinxi.v0/plugins"
	_ "gopkg.in/vinxi/vinxi.v0/rules"
)

type Instance struct {
	ID          string `json:"id"`
	Name        string `json:"name,omitempty"`
	Description string `json:"description,omitempty"`
	scopes      []*Scope
	instance    *vinxi.Vinxi
}

type Manager struct {
	Server    *http.Server
	Router    *httprouter.Router
	plugins   *plugin.Layer
	scopes    []*Scope
	instances []*Instance
}

// New creates a new manager able to manage
// and configure multiple vinxi proxy instance.
func New() *Manager {
	return &Manager{plugins: plugin.NewLayer(), Router: httprouter.New()}
}

// Manage creates a new empty manage and
// starts managing the given vinxi proxy instance.
func Manage(name, description string, proxy *vinxi.Vinxi) *Manager {
	m := New()
	m.Manage(name, description, proxy)
	return m
}

// Manage adds a new vinxi proxy instance to be
// managed by the current manager instance.
func (m *Manager) Manage(name, description string, proxy *vinxi.Vinxi) {
	// Register manager middleware in the proxy
	proxy.Layer.UsePriority("request", layer.Tail, m)

	// Register the managed Vinxi instance
	instance := &Instance{ID: uniuri.New(), Name: name, Description: description, instance: proxy}
	m.instances = append(m.instances, instance)
}

// ListenAndServe creates a new admin HTTP server and starts listening on
// the network based on the given server options.
func (m *Manager) ListenAndServe(opts ServerOptions) (*http.Server, error) {
	m.Server = NewServer(opts)
	m.configure()
	return m.Server, Listen(m.Server, opts)
}

// ServeDefault creates a new admin HTTP server and starts listening
// on the network based on the default server settings.
func (m *Manager) ServeDefault() (*http.Server, error) {
	return m.ListenAndServe(ServerOptions{})
}

// NewScope creates a new scope based on the given name
// and optional description.
func (m *Manager) NewScope(name, description string) *Scope {
	scope := NewScope(name, description)
	m.scopes = append(m.scopes, scope)
	return scope
}

// NewScope creates a new default scope.
func (m *Manager) NewDefaultScope(rules ...rule.Rule) *Scope {
	scope := m.NewScope("default", "Default generic scope")
	scope.UseRule(rules...)
	return scope
}

// HandleHTTP is triggered by the vinxi middleware layer on incoming HTTP request.
func (m *Manager) HandleHTTP(w http.ResponseWriter, r *http.Request, h http.Handler) {
	next := h

	for _, scope := range m.scopes {
		next = http.HandlerFunc(scope.HandleHTTP(next))
	}

	next.ServeHTTP(w, r)
}

// configure is used to configure the HTTP API.
func (m *Manager) configure() error {
	m.Server.Handler = m.Router

	// Define route handlers
	for _, r := range routes {
		m.Router.Handler(r.Method, r.Path, r)
		r.Manager = m // Expose manager instance in routes
	}

	return nil
}

type JSONRule struct {
	ID          string        `json:"id"`
	Name        string        `json:"name,omitempty"`
	Description string        `json:"description,omitempty"`
	Config      config.Config `json:"config,omitempty"`
}

type JSONPlugin struct {
	ID          string        `json:"id"`
	Name        string        `json:"name,omitempty"`
	Description string        `json:"description,omitempty"`
	Enabled     bool          `json:"enabled,omitempty"`
	Config      config.Config `json:"config,omitempty"`
}

type JSONScope struct {
	ID      string       `json:"id"`
	Name    string       `json:"name,omitempty"`
	Rules   []JSONRule   `json:"rules,omitempty"`
	Plugins []JSONPlugin `json:"plugins,omitempty"`
}

type ControllerHandler func(http.ResponseWriter, *http.Request, *Controller)

type Controller struct {
	Path    string
	Method  string
	Manager *Manager
	Handler ControllerHandler
}

func (c *Controller) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	c.Handler(w, r, c)
}

var routes = []*Controller{}

func AddRoute(method, path string, fn ControllerHandler) {
	route := &Controller{
		Path:    path,
		Method:  method,
		Handler: fn,
	}
	routes = append(routes, route)
}

func reply(w http.ResponseWriter) func([]byte, error) {
	return func(buf []byte, err error) {
		if err != nil {
			w.WriteHeader(500)
			w.Write([]byte(err.Error()))
			return
		}
		w.Write(buf)
	}
}

func init() {
	AddRoute("GET", "/", func(w http.ResponseWriter, req *http.Request, c *Controller) {
		io.WriteString(w, "vinxi HTTP API manager "+vinxi.Version)
	})

	AddRoute("GET", "/version", func(w http.ResponseWriter, req *http.Request, c *Controller) {
		versions := struct {
			Vinxi string `json:"vinxi"`
		}{
			Vinxi: vinxi.Version,
		}

		reply(w)(json.Marshal(versions))
	})

	AddRoute("GET", "/health", func(w http.ResponseWriter, req *http.Request, c *Controller) {
		w.Header().Set("Content-Type", "application/json")
		io.WriteString(w, "{}")
	})

	AddRoute("GET", "/catalog", func(w http.ResponseWriter, req *http.Request, c *Controller) {
		links := struct {
			Links map[string]string `json:"links,omitempty"`
		}{
			Links: map[string]string{
				"self":    "/catalog",
				"plugins": "/catalog/plugins",
				"rules":   "/catalog/rules",
			},
		}

		reply(w)(json.Marshal(links))
	})

	AddRoute("GET", "/catalog/plugins", func(w http.ResponseWriter, req *http.Request, c *Controller) {
		reply(w)(json.Marshal(plugin.Plugins))
	})

	AddRoute("GET", "/catalog/rules", func(w http.ResponseWriter, req *http.Request, c *Controller) {
		reply(w)(json.Marshal(rule.Rules))
	})

	AddRoute("GET", "/instances", func(w http.ResponseWriter, req *http.Request, c *Controller) {
		buf := &bytes.Buffer{}

		err := json.NewEncoder(buf).Encode(c.Manager.instances)
		if err != nil {
			w.WriteHeader(500)
			w.Write([]byte(err.Error()))
			return
		}

		w.Write(buf.Bytes())
	})

	AddRoute("GET", "/instances/:instance", func(w http.ResponseWriter, req *http.Request, c *Controller) {
		buf := &bytes.Buffer{}
		id := req.URL.Query().Get(":instance")

		mgr := c.Manager
		for _, instance := range mgr.instances {
			if instance.ID == id || instance.Name == id {
				scopes := createScopes(instance.scopes)

				err := json.NewEncoder(buf).Encode(scopes)
				if err != nil {
					w.WriteHeader(500)
					w.Write([]byte(err.Error()))
					return
				}

				w.Write(buf.Bytes())
				return
			}
		}

		w.WriteHeader(404)
		w.Write([]byte("Not found"))
		return
	})

	AddRoute("GET", "/plugins", func(w http.ResponseWriter, req *http.Request, c *Controller) {
		buf := &bytes.Buffer{}
		scopes := createScopes(c.Manager.scopes)

		err := json.NewEncoder(buf).Encode(scopes)
		if err != nil {
			w.WriteHeader(500)
			w.Write([]byte(err.Error()))
			return
		}

		w.Write(buf.Bytes())
	})

	AddRoute("GET", "/scopes", func(w http.ResponseWriter, req *http.Request, c *Controller) {
		buf := &bytes.Buffer{}
		scopes := createScopes(c.Manager.scopes)

		err := json.NewEncoder(buf).Encode(scopes)
		if err != nil {
			w.WriteHeader(500)
			w.Write([]byte(err.Error()))
			return
		}

		w.Write(buf.Bytes())
	})

	AddRoute("GET", "/scopes/:scope", func(w http.ResponseWriter, req *http.Request, c *Controller) {
		id := req.URL.Query().Get(":scope")

		// Find scope by ID
		for _, scope := range c.Manager.scopes {
			if scope.ID == id {
				data, err := encodeJSON(createScope(scope))
				if err != nil {
					w.WriteHeader(500)
					w.Write([]byte(err.Error()))
					return
				}
				w.Write(data)
				return
			}
		}

		w.WriteHeader(404)
		w.Write([]byte("not found"))
	})
}

func encodeJSON(data interface{}) ([]byte, error) {
	buf := &bytes.Buffer{}
	err := json.NewEncoder(buf).Encode(data)
	return buf.Bytes(), err
}

func createScope(scope *Scope) JSONScope {
	return JSONScope{
		ID:      scope.ID,
		Name:    scope.Name,
		Rules:   createRules(scope),
		Plugins: createPlugins(scope),
	}
}

func createScopes(scopes []*Scope) []JSONScope {
	buf := make([]JSONScope, len(scopes))
	for i, scope := range scopes {
		buf[i] = createScope(scope)
	}
	return buf
}

func createRules(scope *Scope) []JSONRule {
	rules := make([]JSONRule, scope.Rules.Len())
	for i, rule := range scope.Rules.Get() {
		rules[i] = JSONRule{ID: rule.ID(), Name: rule.Name(), Description: rule.Description(), Config: rule.Config()}
	}
	return rules
}

func createPlugins(scope *Scope) []JSONPlugin {
	plugins := make([]JSONPlugin, scope.Plugins.Len())
	for i, plugin := range scope.Plugins.Get() {
		plugins[i] = JSONPlugin{ID: plugin.ID(), Name: plugin.Name(), Description: plugin.Description(), Config: plugin.Config()}
	}
	return plugins
}
<commit_msg>feat(manager): add built-in middleware layer<commit_after>package manager

import (
	"bytes"
	"encoding/json"
	"io"
	"net/http"

	"github.com/dchest/uniuri"
	"github.com/julienschmidt/httprouter"
	"gopkg.in/vinxi/vinxi.v0"
	"gopkg.in/vinxi/vinxi.v0/config"
	"gopkg.in/vinxi/vinxi.v0/layer"
	"gopkg.in/vinxi/vinxi.v0/plugin"
	"gopkg.in/vinxi/vinxi.v0/rule"

	// An empty import is required to load all the rules subpackages
	_ "gopkg.in/vinxi/vinxi.v0/plugins"
	_ "gopkg.in/vinxi/vinxi.v0/rules"
)

type Instance struct {
	ID          string `json:"id"`
	Name        string `json:"name,omitempty"`
	Description string `json:"description,omitempty"`
	scopes      []*Scope
	instance    *vinxi.Vinxi
}

type Manager struct {
	Server    *http.Server
	Router    *httprouter.Router
	plugins   *plugin.Layer
	scopes    []*Scope
	instances []*Instance
	layer     *layer.Layer
}

// New creates a new manager able to manage
// and configure multiple vinxi proxy instance.
func New() *Manager {
	return &Manager{layer: layer.New(), plugins: plugin.NewLayer(), Router: httprouter.New()}
}

// Manage creates a new empty manage and
// starts managing the given vinxi proxy instance.
func Manage(name, description string, proxy *vinxi.Vinxi) *Manager {
	m := New()
	m.Manage(name, description, proxy)
	return m
}

// Manage adds a new vinxi proxy instance to be
// managed by the current manager instance.
func (m *Manager) Manage(name, description string, proxy *vinxi.Vinxi) {
	// Register manager middleware in the proxy
	proxy.Layer.UsePriority("request", layer.Tail, m)

	// Register the managed Vinxi instance
	instance := &Instance{ID: uniuri.New(), Name: name, Description: description, instance: proxy}
	m.instances = append(m.instances, instance)
}

// Use attaches a new middleware handler for incoming HTTP traffic.
func (m *Manager) Use(handler ...interface{}) {
	m.layer.Use(layer.RequestPhase, handler...)
}

// UsePhase attaches a new middleware handler to a specific phase.
func (m *Manager) UsePhase(phase string, handler ...interface{}) {
	m.layer.Use(phase, handler...)
}

// UseFinalHandler uses a new middleware handler function as final handler.
func (m *Manager) UseFinalHandler(fn http.Handler) {
	m.layer.UseFinalHandler(fn)
}

// ListenAndServe creates a new admin HTTP server and starts listening on
// the network based on the given server options.
func (m *Manager) ListenAndServe(opts ServerOptions) (*http.Server, error) {
	m.Server = NewServer(opts)
	m.configure()
	return m.Server, Listen(m.Server, opts)
}

// ServeDefault creates a new admin HTTP server and starts listening
// on the network based on the default server settings.
func (m *Manager) ServeDefault() (*http.Server, error) {
	return m.ListenAndServe(ServerOptions{})
}

// NewScope creates a new scope based on the given name
// and optional description.
func (m *Manager) NewScope(name, description string) *Scope {
	scope := NewScope(name, description)
	m.scopes = append(m.scopes, scope)
	return scope
}

// NewScope creates a new default scope.
func (m *Manager) NewDefaultScope(rules ...rule.Rule) *Scope {
	scope := m.NewScope("default", "Default generic scope")
	scope.UseRule(rules...)
	return scope
}

// HandleHTTP is triggered by the vinxi middleware layer on incoming HTTP request.
func (m *Manager) HandleHTTP(w http.ResponseWriter, r *http.Request, h http.Handler) {
	next := h

	for _, scope := range m.scopes {
		next = http.HandlerFunc(scope.HandleHTTP(next))
	}

	next.ServeHTTP(w, r)
}

// serveHTTP is used to handle HTTP traffic via admin API.
func (m *Manager) serveHTTP(w http.ResponseWriter, r *http.Request) {
	// trigger middleware layer first, then the router
	m.layer.Run(layer.RequestPhase, w, r, m.Router)
}

// configure is used to configure the HTTP API.
func (m *Manager) configure() {
	// bind the admin http.Handler
	m.Server.Handler = http.HandlerFunc(m.serveHTTP)

	// Define route handlers
	for _, r := range routes {
		m.Router.Handler(r.Method, r.Path, r)
		r.Manager = m // Expose manager instance in routes
	}
}

type JSONRule struct {
	ID          string        `json:"id"`
	Name        string        `json:"name,omitempty"`
	Description string        `json:"description,omitempty"`
	Config      config.Config `json:"config,omitempty"`
}

type JSONPlugin struct {
	ID          string        `json:"id"`
	Name        string        `json:"name,omitempty"`
	Description string        `json:"description,omitempty"`
	Enabled     bool          `json:"enabled,omitempty"`
	Config      config.Config `json:"config,omitempty"`
}

type JSONScope struct {
	ID      string       `json:"id"`
	Name    string       `json:"name,omitempty"`
	Rules   []JSONRule   `json:"rules,omitempty"`
	Plugins []JSONPlugin `json:"plugins,omitempty"`
}

type ControllerHandler func(http.ResponseWriter, *http.Request, *Controller)

type Controller struct {
	Path    string
	Method  string
	Manager *Manager
	Handler ControllerHandler
}

func (c *Controller) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	c.Handler(w, r, c)
}

var routes = []*Controller{}

func AddRoute(method, path string, fn ControllerHandler) {
	route := &Controller{
		Path:    path,
		Method:  method,
		Handler: fn,
	}
	routes = append(routes, route)
}

func reply(w http.ResponseWriter) func([]byte, error) {
	return func(buf []byte, err error) {
		if err != nil {
			w.WriteHeader(500)
			w.Write([]byte(err.Error()))
			return
		}
		w.Write(buf)
	}
}

func init() {
	AddRoute("GET", "/", func(w http.ResponseWriter, req *http.Request, c *Controller) {
		io.WriteString(w, "vinxi HTTP API manager "+vinxi.Version)
	})

	AddRoute("GET", "/version", func(w http.ResponseWriter, req *http.Request, c *Controller) {
		versions := struct {
			Vinxi string `json:"vinxi"`
		}{
			Vinxi: vinxi.Version,
		}

		reply(w)(json.Marshal(versions))
	})

	AddRoute("GET", "/health", func(w http.ResponseWriter, req *http.Request, c *Controller) {
		w.Header().Set("Content-Type", "application/json")
		io.WriteString(w, "{}")
	})

	AddRoute("GET", "/catalog", func(w http.ResponseWriter, req *http.Request, c *Controller) {
		links := struct {
			Links map[string]string `json:"links,omitempty"`
		}{
			Links: map[string]string{
				"self":    "/catalog",
				"plugins": "/catalog/plugins",
				"rules":   "/catalog/rules",
			},
		}

		reply(w)(json.Marshal(links))
	})

	AddRoute("GET", "/catalog/plugins", func(w http.ResponseWriter, req *http.Request, c *Controller) {
		reply(w)(json.Marshal(plugin.Plugins))
	})

	AddRoute("GET", "/catalog/rules", func(w http.ResponseWriter, req *http.Request, c *Controller) {
		reply(w)(json.Marshal(rule.Rules))
	})

	AddRoute("GET", "/instances", func(w http.ResponseWriter, req *http.Request, c *Controller) {
		buf := &bytes.Buffer{}

		err := json.NewEncoder(buf).Encode(c.Manager.instances)
		if err != nil {
			w.WriteHeader(500)
			w.Write([]byte(err.Error()))
			return
		}

		w.Write(buf.Bytes())
	})

	AddRoute("GET", "/instances/:instance", func(w http.ResponseWriter, req *http.Request, c *Controller) {
		buf := &bytes.Buffer{}
		id := req.URL.Query().Get(":instance")

		mgr := c.Manager
		for _, instance := range mgr.instances {
			if instance.ID == id || instance.Name == id {
				scopes := createScopes(instance.scopes)

				err := json.NewEncoder(buf).Encode(scopes)
				if err != nil {
					w.WriteHeader(500)
					w.Write([]byte(err.Error()))
					return
				}

				w.Write(buf.Bytes())
				return
			}
		}

		w.WriteHeader(404)
		w.Write([]byte("Not found"))
		return
	})

	AddRoute("GET", "/plugins", func(w http.ResponseWriter, req *http.Request, c *Controller) {
		buf := &bytes.Buffer{}
		scopes := createScopes(c.Manager.scopes)

		err := json.NewEncoder(buf).Encode(scopes)
		if err != nil {
			w.WriteHeader(500)
			w.Write([]byte(err.Error()))
			return
		}

		w.Write(buf.Bytes())
	})

	AddRoute("GET", "/scopes", func(w http.ResponseWriter, req *http.Request, c *Controller) {
		buf := &bytes.Buffer{}
		scopes := createScopes(c.Manager.scopes)

		err := json.NewEncoder(buf).Encode(scopes)
		if err != nil {
			w.WriteHeader(500)
			w.Write([]byte(err.Error()))
			return
		}

		w.Write(buf.Bytes())
	})

	AddRoute("GET", "/scopes/:scope", func(w http.ResponseWriter, req *http.Request, c *Controller) {
		id := req.URL.Query().Get(":scope")

		// Find scope by ID
		for _, scope := range c.Manager.scopes {
			if scope.ID == id {
				data, err := encodeJSON(createScope(scope))
				if err != nil {
					w.WriteHeader(500)
					w.Write([]byte(err.Error()))
					return
				}
				w.Write(data)
				return
			}
		}

		w.WriteHeader(404)
		w.Write([]byte("not found"))
	})
}

func encodeJSON(data interface{}) ([]byte, error) {
	buf := &bytes.Buffer{}
	err := json.NewEncoder(buf).Encode(data)
	return buf.Bytes(), err
}

func createScope(scope *Scope) JSONScope {
	return JSONScope{
		ID:      scope.ID,
		Name:    scope.Name,
		Rules:   createRules(scope),
		Plugins: createPlugins(scope),
	}
}

func createScopes(scopes []*Scope) []JSONScope {
	buf := make([]JSONScope, len(scopes))
	for i, scope := range scopes {
		buf[i] = createScope(scope)
	}
	return buf
}

func createRules(scope *Scope) []JSONRule {
	rules := make([]JSONRule, scope.Rules.Len())
	for i, rule := range scope.Rules.Get() {
		rules[i] = JSONRule{ID: rule.ID(), Name: rule.Name(), Description: rule.Description(), Config: rule.Config()}
	}
	return rules
}

func createPlugins(scope *Scope) []JSONPlugin {
	plugins := make([]JSONPlugin, scope.Plugins.Len())
	for i, plugin := range scope.Plugins.Get() {
		plugins[i] = JSONPlugin{ID: plugin.ID(), Name: plugin.Name(), Description: plugin.Description(), Config: plugin.Config()}
	}
	return plugins
}
<|endoftext|>


<commit_before>package module

import (
	"github.com/v4lproik/no-name/data"
	"html/template"
	"io"
	"bytes"
	"os"
	"fmt"
	"time"
	"github.com/google/uuid"
	"path/filepath"
	"strings"
)

type reportModule struct {
	io io.Writer
	format string
	templateDir string

	next Module
}

const REPORT_FOLDER = "./report"

func NewReportModuleWithSource(rootDir string, format data.ReportFormat, io io.Writer) *reportModule{

	formatChoosen := ""
	switch format {
	case data.HTML:
		formatChoosen = "html"
	case data.GREPABLE:
		formatChoosen = "txt"
	}

	createReportFolder()

	return &reportModule{io, formatChoosen, rootDir, nil}
}

func NewReportModule(templateDir string, format data.ReportFormat) *reportModule{

	formatChoosen := ""
	switch format {
	case data.HTML:
		formatChoosen = "html"
	case data.GREPABLE:
		formatChoosen = "txt"
	}

	var buf bytes.Buffer

	createReportFolder()

	return &reportModule{&buf, formatChoosen, templateDir, nil}
}

func createReportFolder() {
	if _, err := os.Stat(REPORT_FOLDER); os.IsNotExist(err) {
		err := os.Mkdir(REPORT_FOLDER, os.FileMode(0755))
		if err != nil {
			logger.Criticalf(err.Error())
		}
	}
}

func (m *reportModule) 	Request(flag bool, wi *data.WebInterface) {

	type reportModule struct {
		Domain string
		UrlForm string
		UrlToSubmit string
		FaviconMD5Hash string
		FaviconPath string
		MethodSubmitArg string
		OtherArgWithValue map[string]string
		PasswordArg string
		UsernameArg string
		PotentialPassword string
		PotentialUsername string
		SubmitArg string
		CsrfArg string
	}

	info := reportModule{
		wi.Form.Domain,
		wi.Form.UrlForm,
		wi.Form.UrlToSubmit,
		wi.Form.FaviconMD5Hash,
		wi.Form.FaviconPath,
		wi.Form.MethodSubmitArg,
		wi.Form.OtherArgWithValue,
		wi.Form.PasswordArg,
		wi.Form.UsernameArg,
		wi.Form.PotentialPassword,
		wi.Form.PotentialUsername,
		wi.Form.SubmitArg,
		wi.Form.CsrfArg,
	}

	var templates = template.Must(template.ParseFiles(filepath.Join(m.templateDir, "./template/report" + "." + m.format)))
	err := templates.ExecuteTemplate(m.io, "report" + "." + m.format, info)
	if err != nil {
		logger.Criticalf("Cannot Get View ", err)
	}

	filename := filepath.Join(m.templateDir, REPORT_FOLDER) + "/" + "report-" + strings.Replace(time.Now().String() + "_" + uuid.New().String() + "." + m.format, " ", "_", -1)
	f, err := os.Create(filename)
	if err != nil {
		logger.Criticalf("Cannot Create File for Report ", err)
	}else{
		fmt.Fprintf(f, "%s", m.io)
		logger.Infof("Report has been created at " + filename)

		wi.ReportPath = filename
	}
}

func (m *reportModule) SetNextModule(next Module){
	m.next = next
}<commit_msg>Add domain name in report's name<commit_after>package module

import (
	"github.com/v4lproik/no-name/data"
	"html/template"
	"io"
	"bytes"
	"os"
	"fmt"
	"time"
	"github.com/google/uuid"
	"path/filepath"
	"strings"
)

type reportModule struct {
	io io.Writer
	format string
	templateDir string

	next Module
}

const REPORT_FOLDER = "./report"

func NewReportModuleWithSource(rootDir string, format data.ReportFormat, io io.Writer) *reportModule{

	formatChoosen := ""
	switch format {
	case data.HTML:
		formatChoosen = "html"
	case data.GREPABLE:
		formatChoosen = "txt"
	}

	createReportFolder()

	return &reportModule{io, formatChoosen, rootDir, nil}
}

func NewReportModule(templateDir string, format data.ReportFormat) *reportModule{

	formatChoosen := ""
	switch format {
	case data.HTML:
		formatChoosen = "html"
	case data.GREPABLE:
		formatChoosen = "txt"
	}

	var buf bytes.Buffer

	createReportFolder()

	return &reportModule{&buf, formatChoosen, templateDir, nil}
}

func createReportFolder() {
	if _, err := os.Stat(REPORT_FOLDER); os.IsNotExist(err) {
		err := os.Mkdir(REPORT_FOLDER, os.FileMode(0755))
		if err != nil {
			logger.Criticalf(err.Error())
		}
	}
}

func (m *reportModule) 	Request(flag bool, wi *data.WebInterface) {

	type reportModule struct {
		Domain string
		UrlForm string
		UrlToSubmit string
		FaviconMD5Hash string
		FaviconPath string
		MethodSubmitArg string
		OtherArgWithValue map[string]string
		PasswordArg string
		UsernameArg string
		PotentialPassword string
		PotentialUsername string
		SubmitArg string
		CsrfArg string
	}

	info := reportModule{
		wi.Form.Domain,
		wi.Form.UrlForm,
		wi.Form.UrlToSubmit,
		wi.Form.FaviconMD5Hash,
		wi.Form.FaviconPath,
		wi.Form.MethodSubmitArg,
		wi.Form.OtherArgWithValue,
		wi.Form.PasswordArg,
		wi.Form.UsernameArg,
		wi.Form.PotentialPassword,
		wi.Form.PotentialUsername,
		wi.Form.SubmitArg,
		wi.Form.CsrfArg,
	}

	var templates = template.Must(template.ParseFiles(filepath.Join(m.templateDir, "./template/report" + "." + m.format)))
	err := templates.ExecuteTemplate(m.io, "report" + "." + m.format, info)
	if err != nil {
		logger.Criticalf("Cannot Get View ", err)
	}

	filename := filepath.Join(m.templateDir, REPORT_FOLDER) + "/" + "report-" + strings.Replace(time.Now().String() + "_" + wi.Form.Domain + "_" + uuid.New().String() + "." + m.format, " ", "_", -1)
	f, err := os.Create(filename)
	if err != nil {
		logger.Criticalf("Cannot Create File for Report ", err)
	}else{
		fmt.Fprintf(f, "%s", m.io)
		logger.Infof("Report has been created at " + filename)

		wi.ReportPath = filename
	}
}

func (m *reportModule) SetNextModule(next Module){
	m.next = next
}<|endoftext|>


<commit_before>/*
Copyright 2015 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package mungers

import (
	"math"
	"math/rand"
	"os"
	"strings"

	"k8s.io/contrib/mungegithub/github"
	"k8s.io/kubernetes/pkg/util/yaml"

	"github.com/golang/glog"
	github_api "github.com/google/go-github/github"
	"github.com/spf13/cobra"
)

// weightMap is a map of user to a weight for that user.
type weightMap map[string]int64

// A BlunderbussConfig maps a set of file prefixes to a set of owner names (github users)
type BlunderbussConfig struct {
	PrefixMap map[string][]string `json:"prefixMap,omitempty" yaml:"prefixMap,omitempty"`
}

func (b *BlunderbussConfig) findOwners(filename string) weightMap {
	wm := weightMap{}
	for prefix, ownersList := range b.PrefixMap {
		if strings.HasPrefix(filename, prefix) {
			// Give one point for each directory-- so that more specific directories get more weight.
			weight := int64(1 + strings.Count(prefix, "/"))
			for _, owner := range ownersList {
				wm[owner] = wm[owner] + weight
			}
		}
	}
	return wm
}

// BlunderbussMunger will assign issues to users based on the config file
// provided by --blunderbuss-config.
type BlunderbussMunger struct {
	config                *BlunderbussConfig
	blunderbussConfigFile string
	blunderbussReassign   bool
}

func init() {
	blunderbuss := &BlunderbussMunger{}
	RegisterMungerOrDie(blunderbuss)
}

// Name is the name usable in --pr-mungers
func (b *BlunderbussMunger) Name() string { return "blunderbuss" }

// Initialize will initialize the munger
func (b *BlunderbussMunger) Initialize(config *github.Config) error {
	if len(b.blunderbussConfigFile) == 0 {
		glog.Fatalf("--blunderbuss-config is required with the blunderbuss munger")
	}
	file, err := os.Open(b.blunderbussConfigFile)
	if err != nil {
		glog.Fatalf("Failed to load blunderbuss config: %v", err)
	}
	defer file.Close()

	b.config = &BlunderbussConfig{}
	if err := yaml.NewYAMLToJSONDecoder(file).Decode(b.config); err != nil {
		glog.Fatalf("Failed to load blunderbuss config: %v", err)
	}
	glog.V(4).Infof("Loaded config from %s", b.blunderbussConfigFile)
	return nil
}

// EachLoop is called at the start of every munge loop
func (b *BlunderbussMunger) EachLoop() error { return nil }

// AddFlags will add any request flags to the cobra `cmd`
func (b *BlunderbussMunger) AddFlags(cmd *cobra.Command, config *github.Config) {
	cmd.Flags().StringVar(&b.blunderbussConfigFile, "blunderbuss-config", "./blunderbuss.yml", "Path to the blunderbuss config file")
	cmd.Flags().BoolVar(&b.blunderbussReassign, "blunderbuss-reassign", false, "Assign PRs even if they're already assigned; use with -dry-run to judge changes to the assignment algorithm")
	b.addBlunderbussCommand(cmd)
}

// u may be nil.
func describeUser(u *github_api.User) string {
	if u != nil && u.Login != nil {
		return *u.Login
	}
	return "<nil>"
}

// Munge is the workhorse the will actually make updates to the PR
func (b *BlunderbussMunger) Munge(obj *github.MungeObject) {
	if !obj.IsPR() {
		return
	}

	issue := obj.Issue
	if !b.blunderbussReassign && issue.Assignee != nil {
		glog.V(6).Infof("skipping %v: reassign: %v assignee: %v", *issue.Number, b.blunderbussReassign, describeUser(issue.Assignee))
		return
	}

	commits, err := obj.GetCommits()
	if err != nil {
		return
	}

	potentialOwners := weightMap{}
	weightSum := int64(0)
	for _, commit := range commits {
		if commit.Author == nil || commit.Author.Login == nil || commit.SHA == nil {
			glog.Warningf("Skipping invalid commit for %d: %#v", *issue.Number, commit)
			continue
		}
		for _, file := range commit.Files {
			fileWeight := int64(1)
			if file.Changes != nil && *file.Changes != 0 {
				fileWeight = int64(*file.Changes)
			}
			// Judge file size on a log scale-- effectively this
			// makes three buckets, we shouldn't have many 10k+
			// line changes.
			fileWeight = int64(math.Log10(float64(fileWeight))) + 1
			fileOwners := b.config.findOwners(*file.Filename)
			if len(fileOwners) == 0 {
				glog.Warningf("Couldn't find an owner for: %s", *file.Filename)
			}
			for owner, ownerWeight := range fileOwners {
				if owner == *issue.User.Login {
					continue
				}
				potentialOwners[owner] = potentialOwners[owner] + fileWeight*ownerWeight
				weightSum += fileWeight * ownerWeight
			}
		}
	}
	if len(potentialOwners) == 0 {
		glog.Errorf("No owners found for PR %d", *issue.Number)
		return
	}
	glog.V(4).Infof("Weights: %#v\nSum: %v", potentialOwners, weightSum)

	if issue.Assignee != nil {
		cur := *issue.Assignee.Login
		glog.Infof("Current assignee %v has a %02.2f%% chance of having been chosen", cur, 100.0*float64(potentialOwners[cur])/float64(weightSum))
	}
	selection := rand.Int63n(weightSum)
	owner := ""
	for o, w := range potentialOwners {
		owner = o
		selection -= w
		if selection <= 0 {
			break
		}
	}
	glog.Infof("Assigning %v to %v (previously assigned to %v)", *issue.Number, owner, describeUser(issue.Assignee))
	obj.AssignPR(owner)
}
<commit_msg>Remove useless complaint in blunderbuss<commit_after>/*
Copyright 2015 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package mungers

import (
	"math"
	"math/rand"
	"os"
	"strings"

	"k8s.io/contrib/mungegithub/github"
	"k8s.io/kubernetes/pkg/util/yaml"

	"github.com/golang/glog"
	github_api "github.com/google/go-github/github"
	"github.com/spf13/cobra"
)

// weightMap is a map of user to a weight for that user.
type weightMap map[string]int64

// A BlunderbussConfig maps a set of file prefixes to a set of owner names (github users)
type BlunderbussConfig struct {
	PrefixMap map[string][]string `json:"prefixMap,omitempty" yaml:"prefixMap,omitempty"`
}

func (b *BlunderbussConfig) findOwners(filename string) weightMap {
	wm := weightMap{}
	for prefix, ownersList := range b.PrefixMap {
		if strings.HasPrefix(filename, prefix) {
			// Give one point for each directory-- so that more specific directories get more weight.
			weight := int64(1 + strings.Count(prefix, "/"))
			for _, owner := range ownersList {
				wm[owner] = wm[owner] + weight
			}
		}
	}
	return wm
}

// BlunderbussMunger will assign issues to users based on the config file
// provided by --blunderbuss-config.
type BlunderbussMunger struct {
	config                *BlunderbussConfig
	blunderbussConfigFile string
	blunderbussReassign   bool
}

func init() {
	blunderbuss := &BlunderbussMunger{}
	RegisterMungerOrDie(blunderbuss)
}

// Name is the name usable in --pr-mungers
func (b *BlunderbussMunger) Name() string { return "blunderbuss" }

// Initialize will initialize the munger
func (b *BlunderbussMunger) Initialize(config *github.Config) error {
	if len(b.blunderbussConfigFile) == 0 {
		glog.Fatalf("--blunderbuss-config is required with the blunderbuss munger")
	}
	file, err := os.Open(b.blunderbussConfigFile)
	if err != nil {
		glog.Fatalf("Failed to load blunderbuss config: %v", err)
	}
	defer file.Close()

	b.config = &BlunderbussConfig{}
	if err := yaml.NewYAMLToJSONDecoder(file).Decode(b.config); err != nil {
		glog.Fatalf("Failed to load blunderbuss config: %v", err)
	}
	glog.V(4).Infof("Loaded config from %s", b.blunderbussConfigFile)
	return nil
}

// EachLoop is called at the start of every munge loop
func (b *BlunderbussMunger) EachLoop() error { return nil }

// AddFlags will add any request flags to the cobra `cmd`
func (b *BlunderbussMunger) AddFlags(cmd *cobra.Command, config *github.Config) {
	cmd.Flags().StringVar(&b.blunderbussConfigFile, "blunderbuss-config", "./blunderbuss.yml", "Path to the blunderbuss config file")
	cmd.Flags().BoolVar(&b.blunderbussReassign, "blunderbuss-reassign", false, "Assign PRs even if they're already assigned; use with -dry-run to judge changes to the assignment algorithm")
	b.addBlunderbussCommand(cmd)
}

// u may be nil.
func describeUser(u *github_api.User) string {
	if u != nil && u.Login != nil {
		return *u.Login
	}
	return "<nil>"
}

// Munge is the workhorse the will actually make updates to the PR
func (b *BlunderbussMunger) Munge(obj *github.MungeObject) {
	if !obj.IsPR() {
		return
	}

	issue := obj.Issue
	if !b.blunderbussReassign && issue.Assignee != nil {
		glog.V(6).Infof("skipping %v: reassign: %v assignee: %v", *issue.Number, b.blunderbussReassign, describeUser(issue.Assignee))
		return
	}

	commits, err := obj.GetCommits()
	if err != nil {
		return
	}

	potentialOwners := weightMap{}
	weightSum := int64(0)
	for _, commit := range commits {
		for _, file := range commit.Files {
			fileWeight := int64(1)
			if file.Changes != nil && *file.Changes != 0 {
				fileWeight = int64(*file.Changes)
			}
			// Judge file size on a log scale-- effectively this
			// makes three buckets, we shouldn't have many 10k+
			// line changes.
			fileWeight = int64(math.Log10(float64(fileWeight))) + 1
			fileOwners := b.config.findOwners(*file.Filename)
			if len(fileOwners) == 0 {
				glog.Warningf("Couldn't find an owner for: %s", *file.Filename)
			}
			for owner, ownerWeight := range fileOwners {
				if owner == *issue.User.Login {
					continue
				}
				potentialOwners[owner] = potentialOwners[owner] + fileWeight*ownerWeight
				weightSum += fileWeight * ownerWeight
			}
		}
	}
	if len(potentialOwners) == 0 {
		glog.Errorf("No owners found for PR %d", *issue.Number)
		return
	}
	glog.V(4).Infof("Weights: %#v\nSum: %v", potentialOwners, weightSum)

	if issue.Assignee != nil {
		cur := *issue.Assignee.Login
		glog.Infof("Current assignee %v has a %02.2f%% chance of having been chosen", cur, 100.0*float64(potentialOwners[cur])/float64(weightSum))
	}
	selection := rand.Int63n(weightSum)
	owner := ""
	for o, w := range potentialOwners {
		owner = o
		selection -= w
		if selection <= 0 {
			break
		}
	}
	glog.Infof("Assigning %v to %v (previously assigned to %v)", *issue.Number, owner, describeUser(issue.Assignee))
	obj.AssignPR(owner)
}
<|endoftext|>


<commit_before>package node

import (
	"DNA/common/config"
	"DNA/common/log"
	"DNA/core/ledger"
	. "DNA/net/message"
	. "DNA/net/protocol"
	"fmt"
	"net"
	"strconv"
	"time"
)

func keepAlive(from *Noder, dst *Noder) {
	// Need move to node function or keep here?
}

func (node *node) GetBlkHdrs() {
	if node.local.GetNbrNodeCnt() < MINCONNCNT {
		return
	}

	noders := node.local.GetNeighborNoder()
	for _, n := range noders {
		if uint64(ledger.DefaultLedger.Store.GetHeaderHeight()) < n.GetHeight() {
			if n.LocalNode().IsSyncFailed() == false {
				SendMsgSyncHeaders(n)
				n.StartRetryTimer()
				break
			}
		}
	}
}

func (node *node) SyncBlk() {
	headerHeight := ledger.DefaultLedger.Store.GetHeaderHeight()
	currentBlkHeight := ledger.DefaultLedger.Blockchain.BlockHeight
	if currentBlkHeight >= headerHeight {
		return
	}
	var dValue int32
	var reqCnt uint32
	var i uint32
	noders := node.local.GetNeighborNoder()

	for _, n := range noders {
		n.RemoveFlightHeightLessThan(currentBlkHeight)
		count := MAXREQBLKONCE - uint32(n.GetFlightHeightCnt())
		dValue = int32(headerHeight - currentBlkHeight - reqCnt)
		flights := n.GetFlightHeights()
		if count == 0 {
			for _, f := range flights {

				hash := ledger.DefaultLedger.Store.GetHeaderHashByHeight(f)
				ReqBlkData(n, hash)
			}

		}
		for i = 1; i <= count && dValue >= 0; i++ {
			hash := ledger.DefaultLedger.Store.GetHeaderHashByHeight(currentBlkHeight + reqCnt)
			ReqBlkData(n, hash)
			n.StoreFlightHeight(currentBlkHeight + reqCnt)
			reqCnt++
			dValue--
		}
	}
}

func (node *node) SendPingToNbr() {
	noders := node.local.GetNeighborNoder()
	for _, n := range noders {
		t := n.GetLastRXTime()
		if time.Since(t).Seconds() > PERIODUPDATETIME {
			if n.GetState() == ESTABLISH {
				buf, err := NewPingMsg()
				if err != nil {
					log.Error("failed build a new ping message")
				} else {
					go n.Tx(buf)
				}
			}
		}
	}
}

func (node *node) HeartBeatMonitor() {
	noders := node.local.GetNeighborNoder()
	for _, n := range noders {
		if n.GetState() == ESTABLISH {
			t := n.GetLastRXTime()
			if time.Since(t).Seconds() > (PERIODUPDATETIME * KEEPALIVETIMEOUT) {
				log.Warn("keepalive timeout!!!")
				n.SetState(INACTIVITY)
				//n.CloseConn()
			}
		}
	}
}

func (node *node) ReqNeighborList() {
	buf, _ := NewMsg("getaddr", node.local)
	go node.Tx(buf)
}

func (node *node) ConnectSeeds() {
	if node.nbrNodes.GetConnectionCnt() == 0 {
		seedNodes := config.Parameters.SeedList
		for _, nodeAddr := range seedNodes {
			go node.Connect(nodeAddr)
		}
	}
}

func getNodeAddr(n *node) NodeAddr {
	var addr NodeAddr
	addr.IpAddr, _ = n.GetAddr16()
	addr.Time = n.GetTime()
	addr.Services = n.Services()
	addr.Port = n.GetPort()
	addr.ID = n.GetID()
	return addr
}

func (node *node) reconnect(peer *node) error {
	isTls := config.Parameters.IsTLS
	addr := getNodeAddr(peer)
	var ip net.IP
	ip = addr.IpAddr[:]
	nodeAddr := ip.To16().String() + ":" + strconv.Itoa(int(addr.Port))
	log.Info("try to reconnect peer, peer addr is ", nodeAddr)
	var conn net.Conn
	var err error
	if isTls {
		conn, err = TLSDial(nodeAddr)
		if err != nil {
			return nil
		}
	} else {
		conn, err = NonTLSDial(nodeAddr)
		if err != nil {
			return nil
		}
	}
	peer.tryTimes = 0
	peer.conn = conn
	peer.addr, err = parseIPaddr(conn.RemoteAddr().String())
	peer.local = node

	log.Info(fmt.Sprintf("Reconnect node %s connect with %s with %s",
		conn.LocalAddr().String(), conn.RemoteAddr().String(),
		conn.RemoteAddr().Network()))
	go peer.rx()

	peer.SetState(ESTABLISH)

	return nil
}

func (node *node) TryConnect() {
	node.nbrNodes.RLock()
	defer node.nbrNodes.RUnlock()
	for _, n := range node.nbrNodes.List {
		if n.GetState() == INACTIVITY && n.tryTimes < 3 {
			//try to connect
			n.tryTimes++
			node.reconnect(n)
		}
	}

}

// FIXME part of node info update function could be a node method itself intead of
// a node map method
// Fixme the Nodes should be a parameter
func (node *node) updateNodeInfo() {
	ticker := time.NewTicker(time.Second * PERIODUPDATETIME)
	quit := make(chan struct{})
	for {
		select {
		case <-ticker.C:
			node.ConnectSeeds()
			node.SendPingToNbr()
			node.GetBlkHdrs()
			node.SyncBlk()
			node.HeartBeatMonitor()
			node.TryConnect()
		case <-quit:
			ticker.Stop()
			return
		}
	}
	// TODO when to close the timer
	//close(quit)
}
<commit_msg>Close the connection when the node is inactivity<commit_after>package node

import (
	"DNA/common/config"
	"DNA/common/log"
	"DNA/core/ledger"
	. "DNA/net/message"
	. "DNA/net/protocol"
	"fmt"
	"net"
	"strconv"
	"time"
)

func keepAlive(from *Noder, dst *Noder) {
	// Need move to node function or keep here?
}

func (node *node) GetBlkHdrs() {
	if node.local.GetNbrNodeCnt() < MINCONNCNT {
		return
	}

	noders := node.local.GetNeighborNoder()
	for _, n := range noders {
		if uint64(ledger.DefaultLedger.Store.GetHeaderHeight()) < n.GetHeight() {
			if n.LocalNode().IsSyncFailed() == false {
				SendMsgSyncHeaders(n)
				n.StartRetryTimer()
				break
			}
		}
	}
}

func (node *node) SyncBlk() {
	headerHeight := ledger.DefaultLedger.Store.GetHeaderHeight()
	currentBlkHeight := ledger.DefaultLedger.Blockchain.BlockHeight
	if currentBlkHeight >= headerHeight {
		return
	}
	var dValue int32
	var reqCnt uint32
	var i uint32
	noders := node.local.GetNeighborNoder()

	for _, n := range noders {
		n.RemoveFlightHeightLessThan(currentBlkHeight)
		count := MAXREQBLKONCE - uint32(n.GetFlightHeightCnt())
		dValue = int32(headerHeight - currentBlkHeight - reqCnt)
		flights := n.GetFlightHeights()
		if count == 0 {
			for _, f := range flights {

				hash := ledger.DefaultLedger.Store.GetHeaderHashByHeight(f)
				ReqBlkData(n, hash)
			}

		}
		for i = 1; i <= count && dValue >= 0; i++ {
			hash := ledger.DefaultLedger.Store.GetHeaderHashByHeight(currentBlkHeight + reqCnt)
			ReqBlkData(n, hash)
			n.StoreFlightHeight(currentBlkHeight + reqCnt)
			reqCnt++
			dValue--
		}
	}
}

func (node *node) SendPingToNbr() {
	noders := node.local.GetNeighborNoder()
	for _, n := range noders {
		t := n.GetLastRXTime()
		if time.Since(t).Seconds() > PERIODUPDATETIME {
			if n.GetState() == ESTABLISH {
				buf, err := NewPingMsg()
				if err != nil {
					log.Error("failed build a new ping message")
				} else {
					go n.Tx(buf)
				}
			}
		}
	}
}

func (node *node) HeartBeatMonitor() {
	noders := node.local.GetNeighborNoder()
	for _, n := range noders {
		if n.GetState() == ESTABLISH {
			t := n.GetLastRXTime()
			if time.Since(t).Seconds() > (PERIODUPDATETIME * KEEPALIVETIMEOUT) {
				log.Warn("keepalive timeout!!!")
				n.SetState(INACTIVITY)
				n.CloseConn()
			}
		}
	}
}

func (node *node) ReqNeighborList() {
	buf, _ := NewMsg("getaddr", node.local)
	go node.Tx(buf)
}

func (node *node) ConnectSeeds() {
	if node.nbrNodes.GetConnectionCnt() == 0 {
		seedNodes := config.Parameters.SeedList
		for _, nodeAddr := range seedNodes {
			go node.Connect(nodeAddr)
		}
	}
}

func getNodeAddr(n *node) NodeAddr {
	var addr NodeAddr
	addr.IpAddr, _ = n.GetAddr16()
	addr.Time = n.GetTime()
	addr.Services = n.Services()
	addr.Port = n.GetPort()
	addr.ID = n.GetID()
	return addr
}

func (node *node) reconnect(peer *node) error {
	isTls := config.Parameters.IsTLS
	addr := getNodeAddr(peer)
	var ip net.IP
	ip = addr.IpAddr[:]
	nodeAddr := ip.To16().String() + ":" + strconv.Itoa(int(addr.Port))
	log.Info("try to reconnect peer, peer addr is ", nodeAddr)
	var conn net.Conn
	var err error
	if isTls {
		conn, err = TLSDial(nodeAddr)
		if err != nil {
			return nil
		}
	} else {
		conn, err = NonTLSDial(nodeAddr)
		if err != nil {
			return nil
		}
	}
	t := time.Now()
	peer.UpdateRXTime(t)
	peer.tryTimes = 0
	peer.conn = conn
	peer.addr, err = parseIPaddr(conn.RemoteAddr().String())
	peer.local = node

	log.Info(fmt.Sprintf("Reconnect node %s connect with %s with %s",
		conn.LocalAddr().String(), conn.RemoteAddr().String(),
		conn.RemoteAddr().Network()))
	go peer.rx()

	peer.SetState(ESTABLISH)

	return nil
}

func (node *node) TryConnect() {
	node.nbrNodes.RLock()
	defer node.nbrNodes.RUnlock()
	for _, n := range node.nbrNodes.List {
		if n.GetState() == INACTIVITY && n.tryTimes < 3 {
			//try to connect
			n.tryTimes++
			node.reconnect(n)
		}
	}

}

// FIXME part of node info update function could be a node method itself intead of
// a node map method
// Fixme the Nodes should be a parameter
func (node *node) updateNodeInfo() {
	ticker := time.NewTicker(time.Second * PERIODUPDATETIME)
	quit := make(chan struct{})
	for {
		select {
		case <-ticker.C:
			node.ConnectSeeds()
			node.SendPingToNbr()
			node.GetBlkHdrs()
			node.SyncBlk()
			node.HeartBeatMonitor()
			node.TryConnect()
		case <-quit:
			ticker.Stop()
			return
		}
	}
	// TODO when to close the timer
	//close(quit)
}
<|endoftext|>


<commit_before>package taskfile

import (
	"fmt"
	"strings"
)

// NamespaceSeparator contains the character that separates namescapes
const NamespaceSeparator = ":"

// Merge merges the second Taskfile into the first
func Merge(t1, t2 *Taskfile, includedTaskfile *IncludedTaskfile, namespaces ...string) error {
	if t1.Version != t2.Version {
		return fmt.Errorf(`task: Taskfiles versions should match. First is "%s" but second is "%s"`, t1.Version, t2.Version)
	}

	if t2.Expansions != 0 && t2.Expansions != 2 {
		t1.Expansions = t2.Expansions
	}
	if t2.Output.IsSet() {
		t1.Output = t2.Output
	}

	if t1.Includes == nil {
		t1.Includes = &IncludedTaskfiles{}
	}
	t1.Includes.Merge(t2.Includes)

	if t1.Vars == nil {
		t1.Vars = &Vars{}
	}
	if t1.Env == nil {
		t1.Env = &Vars{}
	}
	t1.Vars.Merge(t2.Vars)
	t1.Env.Merge(t2.Env)

	if t1.Tasks == nil {
		t1.Tasks = make(Tasks)
	}
	for k, v := range t2.Tasks {
		// We do a deep copy of the task struct here to ensure that no data can
		// be changed elsewhere once the taskfile is merged.
		task := v.DeepCopy()

		// Set the task to internal if EITHER the included task or the included
		// taskfile are marked as internal
		task.Internal = task.Internal || includedTaskfile.Internal

		// Add namespaces to dependencies, commands and aliases
		for _, dep := range task.Deps {
			dep.Task = taskNameWithNamespace(dep.Task, namespaces...)
		}
		for _, cmd := range task.Cmds {
			if cmd != nil && cmd.Task != "" {
				cmd.Task = taskNameWithNamespace(cmd.Task, namespaces...)
			}
		}
		for i, alias := range task.Aliases {
			task.Aliases[i] = taskNameWithNamespace(alias, namespaces...)
		}
		// Add namespace aliases
		if includedTaskfile != nil {
			for _, namespaceAlias := range includedTaskfile.Aliases {
				for _, alias := range v.Aliases {
					task.Aliases = append(task.Aliases, taskNameWithNamespace(alias, namespaceAlias))
				}
			}
		}

		// Add the task to the merged taskfile
		t1.Tasks[taskNameWithNamespace(k, namespaces...)] = task
	}

	return nil
}

func taskNameWithNamespace(taskName string, namespaces ...string) string {
	if strings.HasPrefix(taskName, ":") {
		return strings.TrimPrefix(taskName, ":")
	}
	return strings.Join(append(namespaces, taskName), NamespaceSeparator)
}
<commit_msg>For include aliases, consider the main task name as well<commit_after>package taskfile

import (
	"fmt"
	"strings"
)

// NamespaceSeparator contains the character that separates namescapes
const NamespaceSeparator = ":"

// Merge merges the second Taskfile into the first
func Merge(t1, t2 *Taskfile, includedTaskfile *IncludedTaskfile, namespaces ...string) error {
	if t1.Version != t2.Version {
		return fmt.Errorf(`task: Taskfiles versions should match. First is "%s" but second is "%s"`, t1.Version, t2.Version)
	}

	if t2.Expansions != 0 && t2.Expansions != 2 {
		t1.Expansions = t2.Expansions
	}
	if t2.Output.IsSet() {
		t1.Output = t2.Output
	}

	if t1.Includes == nil {
		t1.Includes = &IncludedTaskfiles{}
	}
	t1.Includes.Merge(t2.Includes)

	if t1.Vars == nil {
		t1.Vars = &Vars{}
	}
	if t1.Env == nil {
		t1.Env = &Vars{}
	}
	t1.Vars.Merge(t2.Vars)
	t1.Env.Merge(t2.Env)

	if t1.Tasks == nil {
		t1.Tasks = make(Tasks)
	}
	for k, v := range t2.Tasks {
		// We do a deep copy of the task struct here to ensure that no data can
		// be changed elsewhere once the taskfile is merged.
		task := v.DeepCopy()

		// Set the task to internal if EITHER the included task or the included
		// taskfile are marked as internal
		task.Internal = task.Internal || includedTaskfile.Internal

		// Add namespaces to dependencies, commands and aliases
		for _, dep := range task.Deps {
			dep.Task = taskNameWithNamespace(dep.Task, namespaces...)
		}
		for _, cmd := range task.Cmds {
			if cmd != nil && cmd.Task != "" {
				cmd.Task = taskNameWithNamespace(cmd.Task, namespaces...)
			}
		}
		for i, alias := range task.Aliases {
			task.Aliases[i] = taskNameWithNamespace(alias, namespaces...)
		}
		// Add namespace aliases
		if includedTaskfile != nil {
			for _, namespaceAlias := range includedTaskfile.Aliases {
				task.Aliases = append(task.Aliases, taskNameWithNamespace(task.Task, namespaceAlias))
				for _, alias := range v.Aliases {
					task.Aliases = append(task.Aliases, taskNameWithNamespace(alias, namespaceAlias))
				}
			}
		}

		// Add the task to the merged taskfile
		t1.Tasks[taskNameWithNamespace(k, namespaces...)] = task
	}

	return nil
}

func taskNameWithNamespace(taskName string, namespaces ...string) string {
	if strings.HasPrefix(taskName, ":") {
		return strings.TrimPrefix(taskName, ":")
	}
	return strings.Join(append(namespaces, taskName), NamespaceSeparator)
}
<|endoftext|>


<commit_before>// Copyright 2014 by caixw, All rights reserved.
// Use of this source code is governed by a MIT
// license that can be found in the LICENSE file.

package util

import (
	"database/sql"
	"fmt"
	"os"
	"reflect"
	"testing"

	"github.com/caixw/lib.go/assert"
	_ "github.com/mattn/go-sqlite3"
)

type FetchUser struct {
	Id       int    `orm:"name(id);ai(1,2);"`
	Email    string `orm:"unique(unique_index);nullable;pk(pk_name)"`
	Username string `orm:"index(index)"`
	Group    int    `orm:"name(group);fk(fk_group,group,id)"`

	Regdate int `orm:"-"`
}

func TestParseObj(t *testing.T) {
	a := assert.New(t)
	obj := &FetchUser{Id: 5}
	mapped := map[string]reflect.Value{}

	v := reflect.ValueOf(obj).Elem()
	a.True(v.IsValid())

	err := parseObj(v, &mapped)
	a.NotError(err).Equal(4, len(mapped))

	// 忽略的字段
	_, found := mapped["Regdate"]
	a.False(found)

	// 判断字段是否存在
	vi, found := mapped["id"]
	a.True(found).True(vi.IsValid())

	// 设置字段的值
	mapped["id"].Set(reflect.ValueOf(36))
	a.Equal(36, obj.Id)
	mapped["Email"].SetString("email")
	a.Equal("email", obj.Email)
	mapped["Username"].SetString("username")
	a.Equal("username", obj.Username)
	mapped["group"].SetInt(1)
	a.Equal(1, obj.Group)
}

// 初始化一个sql.DB(sqlite3)，方便后面的测试用例使用。
func initDB(a *assert.Assertion) *sql.DB {
	db, err := sql.Open("sqlite3", "./testdata/test")
	a.NotError(err).NotNil(db)

	/* 创建表 */
	sql := `create table user (
        id integer not null primary key,
        Email text,
        Username text,
        [group] interger)`
	_, err = db.Exec(sql)
	a.NotError(err)

	/* 插入数据 */
	tx, err := db.Begin()
	a.NotError(err).NotNil(tx)

	stmt, err := tx.Prepare("insert into user(id, Email,Username,[group]) values(?, ?, ?, ?)")
	a.NotError(err).NotNil(stmt)

	for i := 0; i < 100; i++ {
		_, err = stmt.Exec(i, fmt.Sprintf("email-%d", i), fmt.Sprintf("username-%d", i), 1)
		a.NotError(err)
	}
	tx.Commit()
	stmt.Close()

	return db
}

// 关闭sql.DB(sqlite3)的数据库连结。
func closeDB(db *sql.DB, a *assert.Assertion) {
	db.Close()
	a.NotError(os.Remove("./testdata/test"))
	a.FileNotExists("./testdata/test")
}

func TestFetch2Objs(t *testing.T) {
	a := assert.New(t)
	db := initDB(a)
	defer closeDB(db, a)

	sql := `SELECT id,Email FROM user WHERE id<2 ORDER BY id`

	// test1:objs的长度与导出的数据长度相等
	rows, err := db.Query(sql)
	a.NotError(err).NotNil(rows)

	objs := []*FetchUser{
		&FetchUser{},
		&FetchUser{},
	}

	a.NotError(Fetch2Objs(&objs, rows))

	a.Equal([]*FetchUser{
		&FetchUser{Id: 0, Email: "email-0"},
		&FetchUser{Id: 1, Email: "email-1"},
	}, objs)

	// test2:objs的长度小于导出数据的长度，objs应该自动增加长度。
	rows, err = db.Query(sql)
	a.NotError(err).NotNil(rows)
	objs = []*FetchUser{
		&FetchUser{},
	}
	a.NotError(Fetch2Objs(&objs, rows))
	a.Equal(len(objs), 2)
	a.Equal([]*FetchUser{
		&FetchUser{Id: 0, Email: "email-0"},
		&FetchUser{Id: 1, Email: "email-1"},
	}, objs)

	// test3:objs的长度小于导出数据的长度，objs不会增加长度。
	rows, err = db.Query(sql)
	a.NotError(err).NotNil(rows)
	objs = []*FetchUser{
		&FetchUser{},
	}
	a.NotError(Fetch2Objs(objs, rows)) // 非指针传递
	a.Equal(len(objs), 1)
	a.Equal([]*FetchUser{
		&FetchUser{Id: 0, Email: "email-0"},
	}, objs)

	// test4:objs的长度大于导出数据的长度。
	rows, err = db.Query(sql)
	objs = []*FetchUser{
		&FetchUser{},
		&FetchUser{},
		&FetchUser{},
	}
	a.NotError(Fetch2Objs(&objs, rows))
	a.Equal([]*FetchUser{
		&FetchUser{Id: 0, Email: "email-0"},
		&FetchUser{Id: 1, Email: "email-1"},
		&FetchUser{},
	}, objs)

	// test5:非数组指针传递。
	rows, err = db.Query(sql)
	array := [1]*FetchUser{
		&FetchUser{},
	}
	a.Error(Fetch2Objs(array, rows)) // 非指针传递，出错

	// test6:数组指针传递，不会增长数组长度。
	rows, err = db.Query(sql)
	array = [1]*FetchUser{
		&FetchUser{},
	}
	a.NotError(Fetch2Objs(&array, rows))
	a.Equal([1]*FetchUser{
		&FetchUser{Id: 0, Email: "email-0"},
	}, array)

	// test7:obj为一个struct指针。
	rows, err = db.Query(sql)
	obj := FetchUser{}
	a.NotError(Fetch2Objs(&obj, rows))
	a.Equal(FetchUser{Id: 0, Email: "email-0"}, obj)

	// test8:obj为一个struct。这将返回错误信息
	rows, err = db.Query(sql)
	obj = FetchUser{}
	a.Error(Fetch2Objs(obj, rows))
}

func TestFetch2Maps(t *testing.T) {
	a := assert.New(t)
	db := initDB(a)
	defer closeDB(db, a)

	sql := `SELECT id,Email FROM user WHERE id<2 ORDER BY id`
	rows, err := db.Query(sql)
	a.NotError(err).NotNil(rows)

	mapped, err := Fetch2Maps(false, rows)
	a.NotError(err).NotNil(mapped)

	a.Equal([]map[string]interface{}{
		map[string]interface{}{"id": 0, "Email": "email-0"},
		map[string]interface{}{"id": 1, "Email": "email-1"},
	}, mapped)

	// 读取一行
	rows, err = db.Query(sql)
	a.NotError(err).NotNil(rows)

	mapped, err = Fetch2Maps(true, rows)
	a.NotError(err).NotNil(mapped)

	a.Equal([]map[string]interface{}{
		map[string]interface{}{"id": 0, "Email": "email-0"},
	}, mapped)
}

func TestFetch2MapsString(t *testing.T) {
	a := assert.New(t)
	db := initDB(a)
	defer closeDB(db, a)

	sql := `SELECT id,Email FROM user WHERE id<2 ORDER BY id`
	rows, err := db.Query(sql)
	a.NotError(err).NotNil(rows)

	mapped, err := Fetch2MapsString(false, rows)
	a.NotError(err).NotNil(mapped)

	a.Equal(mapped, []map[string]string{
		map[string]string{"id": "0", "Email": "email-0"},
		map[string]string{"id": "1", "Email": "email-1"},
	})

	// 读取一行
	rows, err = db.Query(sql)
	a.NotError(err).NotNil(rows)

	mapped, err = Fetch2MapsString(true, rows)
	a.NotError(err).NotNil(mapped)

	a.Equal(mapped, []map[string]string{
		map[string]string{"id": "0", "Email": "email-0"},
	})
}

func TestFetchColumns(t *testing.T) {
	a := assert.New(t)
	db := initDB(a)
	defer closeDB(db, a)

	sql := `SELECT id FROM user WHERE id<2 ORDER BY id`
	rows, err := db.Query(sql)
	a.NotError(err).NotNil(rows)

	cols, err := FetchColumns(false, "id", rows)
	a.NotError(err).NotNil(cols)

	a.Equal([]interface{}{0, 1}, cols)

	// 读取一行
	rows, err = db.Query(sql)
	a.NotError(err).NotNil(rows)

	cols, err = FetchColumns(true, "id", rows)
	a.NotError(err).NotNil(cols)

	a.Equal([]interface{}{0}, cols)
}

func TestFetchColumnsString(t *testing.T) {
	a := assert.New(t)
	db := initDB(a)
	defer closeDB(db, a)

	sql := `SELECT id FROM user WHERE id<2 ORDER BY id`
	rows, err := db.Query(sql)
	a.NotError(err).NotNil(rows)

	cols, err := FetchColumnsString(false, "id", rows)
	a.NotError(err).NotNil(cols)

	a.Equal([]string{"0", "1"}, cols)

	// 读取一行
	rows, err = db.Query(sql)
	a.NotError(err).NotNil(rows)

	cols, err = FetchColumnsString(false, "id", rows)
	a.NotError(err).NotNil(cols)

	a.Equal([]string{"0"}, cols)
}
<commit_msg>修正orm/util测试错误<commit_after>// Copyright 2014 by caixw, All rights reserved.
// Use of this source code is governed by a MIT
// license that can be found in the LICENSE file.

package util

import (
	"database/sql"
	"fmt"
	"os"
	"reflect"
	"testing"

	"github.com/caixw/lib.go/assert"
	_ "github.com/mattn/go-sqlite3"
)

type FetchUser struct {
	Id       int    `orm:"name(id);ai(1,2);"`
	Email    string `orm:"unique(unique_index);nullable;pk(pk_name)"`
	Username string `orm:"index(index)"`
	Group    int    `orm:"name(group);fk(fk_group,group,id)"`

	Regdate int `orm:"-"`
}

func TestParseObj(t *testing.T) {
	a := assert.New(t)
	obj := &FetchUser{Id: 5}
	mapped := map[string]reflect.Value{}

	v := reflect.ValueOf(obj).Elem()
	a.True(v.IsValid())

	err := parseObj(v, &mapped)
	a.NotError(err).Equal(4, len(mapped))

	// 忽略的字段
	_, found := mapped["Regdate"]
	a.False(found)

	// 判断字段是否存在
	vi, found := mapped["id"]
	a.True(found).True(vi.IsValid())

	// 设置字段的值
	mapped["id"].Set(reflect.ValueOf(36))
	a.Equal(36, obj.Id)
	mapped["Email"].SetString("email")
	a.Equal("email", obj.Email)
	mapped["Username"].SetString("username")
	a.Equal("username", obj.Username)
	mapped["group"].SetInt(1)
	a.Equal(1, obj.Group)
}

// 初始化一个sql.DB(sqlite3)，方便后面的测试用例使用。
func initDB(a *assert.Assertion) *sql.DB {
	db, err := sql.Open("sqlite3", "./testdata/test")
	a.NotError(err).NotNil(db)

	/* 创建表 */
	sql := `create table user (
        id integer not null primary key,
        Email text,
        Username text,
        [group] interger)`
	_, err = db.Exec(sql)
	a.NotError(err)

	/* 插入数据 */
	tx, err := db.Begin()
	a.NotError(err).NotNil(tx)

	stmt, err := tx.Prepare("insert into user(id, Email,Username,[group]) values(?, ?, ?, ?)")
	a.NotError(err).NotNil(stmt)

	for i := 0; i < 100; i++ {
		_, err = stmt.Exec(i, fmt.Sprintf("email-%d", i), fmt.Sprintf("username-%d", i), 1)
		a.NotError(err)
	}
	tx.Commit()
	stmt.Close()

	return db
}

// 关闭sql.DB(sqlite3)的数据库连结。
func closeDB(db *sql.DB, a *assert.Assertion) {
	a.NotError(db.Close()).
		NotError(os.Remove("./testdata/test")).
		FileNotExists("./testdata/test")
}

func TestFetch2Objs(t *testing.T) {
	a := assert.New(t)
	db := initDB(a)
	defer closeDB(db, a)

	sql := `SELECT id,Email FROM user WHERE id<2 ORDER BY id`

	// test1:objs的长度与导出的数据长度相等
	rows, err := db.Query(sql)
	a.NotError(err).NotNil(rows)

	objs := []*FetchUser{
		&FetchUser{},
		&FetchUser{},
	}

	a.NotError(Fetch2Objs(&objs, rows))

	a.Equal([]*FetchUser{
		&FetchUser{Id: 0, Email: "email-0"},
		&FetchUser{Id: 1, Email: "email-1"},
	}, objs)
	a.NotError(rows.Close())

	// test2:objs的长度小于导出数据的长度，objs应该自动增加长度。
	rows, err = db.Query(sql)
	a.NotError(err).NotNil(rows)
	objs = []*FetchUser{
		&FetchUser{},
	}
	a.NotError(Fetch2Objs(&objs, rows))
	a.Equal(len(objs), 2)
	a.Equal([]*FetchUser{
		&FetchUser{Id: 0, Email: "email-0"},
		&FetchUser{Id: 1, Email: "email-1"},
	}, objs)
	a.NotError(rows.Close())

	// test3:objs的长度小于导出数据的长度，objs不会增加长度。
	rows, err = db.Query(sql)
	a.NotError(err).NotNil(rows)
	objs = []*FetchUser{
		&FetchUser{},
	}
	a.NotError(Fetch2Objs(objs, rows)) // 非指针传递
	a.Equal(len(objs), 1)
	a.Equal([]*FetchUser{
		&FetchUser{Id: 0, Email: "email-0"},
	}, objs)
	a.NotError(rows.Close())

	// test4:objs的长度大于导出数据的长度。
	rows, err = db.Query(sql)
	objs = []*FetchUser{
		&FetchUser{},
		&FetchUser{},
		&FetchUser{},
	}
	a.NotError(Fetch2Objs(&objs, rows))
	a.Equal([]*FetchUser{
		&FetchUser{Id: 0, Email: "email-0"},
		&FetchUser{Id: 1, Email: "email-1"},
		&FetchUser{},
	}, objs)
	a.NotError(rows.Close())

	// test5:非数组指针传递。
	rows, err = db.Query(sql)
	array := [1]*FetchUser{
		&FetchUser{},
	}
	a.Error(Fetch2Objs(array, rows)) // 非指针传递，出错
	a.NotError(rows.Close())

	// test6:数组指针传递，不会增长数组长度。
	rows, err = db.Query(sql)
	array = [1]*FetchUser{
		&FetchUser{},
	}
	a.NotError(Fetch2Objs(&array, rows))
	a.Equal([1]*FetchUser{
		&FetchUser{Id: 0, Email: "email-0"},
	}, array)
	a.NotError(rows.Close())

	// test7:obj为一个struct指针。
	rows, err = db.Query(sql)
	obj := FetchUser{}
	a.NotError(Fetch2Objs(&obj, rows))
	a.Equal(FetchUser{Id: 0, Email: "email-0"}, obj)
	a.NotError(rows.Close())

	// test8:obj为一个struct。这将返回错误信息
	rows, err = db.Query(sql)
	obj = FetchUser{}
	a.Error(Fetch2Objs(obj, rows))
	a.NotError(rows.Close())
}

func TestFetch2Maps(t *testing.T) {
	a := assert.New(t)
	db := initDB(a)
	defer closeDB(db, a)

	sql := `SELECT id,Email FROM user WHERE id<2 ORDER BY id`
	rows, err := db.Query(sql)
	a.NotError(err).NotNil(rows)

	mapped, err := Fetch2Maps(false, rows)
	a.NotError(err).NotNil(mapped)

	a.Equal([]map[string]interface{}{
		map[string]interface{}{"id": 0, "Email": "email-0"},
		map[string]interface{}{"id": 1, "Email": "email-1"},
	}, mapped)
	a.NotError(rows.Close())

	// 读取一行
	rows, err = db.Query(sql)
	a.NotError(err).NotNil(rows)

	mapped, err = Fetch2Maps(true, rows)
	a.NotError(err).NotNil(mapped)

	a.Equal([]map[string]interface{}{
		map[string]interface{}{"id": 0, "Email": "email-0"},
	}, mapped)
	a.NotError(rows.Close())
}

func TestFetch2MapsString(t *testing.T) {
	a := assert.New(t)
	db := initDB(a)
	defer closeDB(db, a)

	sql := `SELECT id,Email FROM user WHERE id<2 ORDER BY id`
	rows, err := db.Query(sql)
	a.NotError(err).NotNil(rows)

	mapped, err := Fetch2MapsString(false, rows)
	a.NotError(err).NotNil(mapped)

	a.Equal(mapped, []map[string]string{
		map[string]string{"id": "0", "Email": "email-0"},
		map[string]string{"id": "1", "Email": "email-1"},
	})

	a.NotError(rows.Close())

	// 读取一行
	rows, err = db.Query(sql)
	a.NotError(err).NotNil(rows)

	mapped, err = Fetch2MapsString(true, rows)
	a.NotError(err).NotNil(mapped)

	a.Equal(mapped, []map[string]string{
		map[string]string{"id": "0", "Email": "email-0"},
	})
	a.NotError(rows.Close())
}

func TestFetchColumns(t *testing.T) {
	a := assert.New(t)
	db := initDB(a)
	defer closeDB(db, a)

	sql := `SELECT id FROM user WHERE id<2 ORDER BY id`
	rows, err := db.Query(sql)
	a.NotError(err).NotNil(rows)

	cols, err := FetchColumns(false, "id", rows)
	a.NotError(err).NotNil(cols)

	a.Equal([]interface{}{0, 1}, cols)
	a.NotError(rows.Close())

	// 读取一行
	rows, err = db.Query(sql)
	a.NotError(err).NotNil(rows)

	cols, err = FetchColumns(true, "id", rows)
	a.NotError(err).NotNil(cols)

	a.Equal([]interface{}{0}, cols)
	a.NotError(rows.Close())
}

func TestFetchColumnsString(t *testing.T) {
	a := assert.New(t)
	db := initDB(a)
	defer closeDB(db, a)

	sql := `SELECT id FROM user WHERE id<2 ORDER BY id`
	rows, err := db.Query(sql)
	a.NotError(err).NotNil(rows)

	cols, err := FetchColumnsString(false, "id", rows)
	a.NotError(err).NotNil(cols)

	a.Equal([]string{"0", "1"}, cols)
	a.NotError(rows.Close())

	// 读取一行
	rows, err = db.Query(sql)
	a.NotError(err).NotNil(rows)

	cols, err = FetchColumnsString(true, "id", rows)
	a.NotError(err).NotNil(cols)

	a.Equal([]string{"0"}, cols)
	a.NotError(rows.Close())
}
<|endoftext|>


<commit_before>package state

import (
	"bytes"
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"math/rand"
	"os"
	"os/user"
	"strings"
	"text/template"
	"time"

	uuid "github.com/hashicorp/go-uuid"
	"github.com/hashicorp/terraform/terraform"
)

var rngSource *rand.Rand

func init() {
	rngSource = rand.New(rand.NewSource(time.Now().UnixNano()))
}

// State is the collection of all state interfaces.
type State interface {
	StateReader
	StateWriter
	StateRefresher
	StatePersister
	Locker
}

// StateReader is the interface for things that can return a state. Retrieving
// the state here must not error. Loading the state fresh (an operation that
// can likely error) should be implemented by RefreshState. If a state hasn't
// been loaded yet, it is okay for State to return nil.
type StateReader interface {
	State() *terraform.State
}

// StateWriter is the interface that must be implemented by something that
// can write a state. Writing the state can be cached or in-memory, as
// full persistence should be implemented by StatePersister.
type StateWriter interface {
	WriteState(*terraform.State) error
}

// StateRefresher is the interface that is implemented by something that
// can load a state. This might be refreshing it from a remote location or
// it might simply be reloading it from disk.
type StateRefresher interface {
	RefreshState() error
}

// StatePersister is implemented to truly persist a state. Whereas StateWriter
// is allowed to perhaps be caching in memory, PersistState must write the
// state to some durable storage.
type StatePersister interface {
	PersistState() error
}

// Locker is implemented to lock state during command execution.
// The info parameter can be recorded with the lock, but the
// implementation should not depend in its value. The string returned by Lock
// is an ID corresponding to the lock acquired, and must be passed to Unlock to
// ensure that the correct lock is being released.
//
// Lock and Unlock may return an error value of type LockError which in turn
// can contain the LockInfo of a conflicting lock.
type Locker interface {
	Lock(info *LockInfo) (string, error)
	Unlock(id string) error
}

// Lock the state, using the provided context for timeout and cancellation.
// TODO: this should probably backoff somewhat.
func LockWithContext(ctx context.Context, s State, info *LockInfo) (string, error) {
	for {
		id, err := s.Lock(info)
		if err == nil {
			return id, nil
		}

		le, ok := err.(*LockError)
		if !ok {
			// not a lock error, so we can't retry
			return "", err
		}

		if le.Info.ID == "" {
			// the lock has no ID, something is wrong so don't keep trying
			return "", fmt.Errorf("lock error missing ID: %s", err)
		}

		// there's an existing lock, wait and try again
		select {
		case <-ctx.Done():
			// return the last lock error with the info
			return "", err
		case <-time.After(time.Second):
		}
	}
}

// Generate a LockInfo structure, populating the required fields.
func NewLockInfo() *LockInfo {
	// this doesn't need to be cryptographically secure, just unique.
	// Using math/rand alleviates the need to check handle the read error.
	// Use a uuid format to match other IDs used throughout Terraform.
	buf := make([]byte, 16)
	rngSource.Read(buf)

	id, err := uuid.FormatUUID(buf)
	if err != nil {
		// this of course shouldn't happen
		panic(err)
	}

	// don't error out on user and hostname, as we don't require them
	userName := ""
	if userInfo, err := user.Current(); err == nil {
		userName = userInfo.Username
	}
	host, _ := os.Hostname()

	info := &LockInfo{
		ID:      id,
		Who:     fmt.Sprintf("%s@%s", userName, host),
		Version: terraform.Version,
		Created: time.Now().UTC(),
	}
	return info
}

// LockInfo stores lock metadata.
//
// Only Operation and Info are required to be set by the caller of Lock.
type LockInfo struct {
	// Unique ID for the lock. NewLockInfo provides a random ID, but this may
	// be overridden by the lock implementation. The final value if ID will be
	// returned by the call to Lock.
	ID string

	// Terraform operation, provided by the caller.
	Operation string
	// Extra information to store with the lock, provided by the caller.
	Info string

	// user@hostname when available
	Who string
	// Terraform version
	Version string
	// Time that the lock was taken.
	Created time.Time

	// Path to the state file when applicable. Set by the Lock implementation.
	Path string
}

// Err returns the lock info formatted in an error
func (l *LockInfo) Err() error {
	return errors.New(l.String())
}

// Marshal returns a string json representation of the LockInfo
func (l *LockInfo) Marshal() []byte {
	js, err := json.Marshal(l)
	if err != nil {
		panic(err)
	}
	return js
}

// String return a multi-line string representation of LockInfo
func (l *LockInfo) String() string {
	tmpl := `Lock Info:
  ID:        {{.ID}}
  Path:      {{.Path}}
  Operation: {{.Operation}}
  Who:       {{.Who}}
  Version:   {{.Version}}
  Created:   {{.Created}}
  Info:      {{.Info}}
`

	t := template.Must(template.New("LockInfo").Parse(tmpl))
	var out bytes.Buffer
	if err := t.Execute(&out, l); err != nil {
		panic(err)
	}
	return out.String()
}

type LockError struct {
	Info *LockInfo
	Err  error
}

func (e *LockError) Error() string {
	var out []string
	if e.Err != nil {
		out = append(out, e.Err.Error())
	}

	if e.Info != nil {
		out = append(out, e.Info.String())
	}
	return strings.Join(out, "\n")
}
<commit_msg>give LockWithContext a little backoff<commit_after>package state

import (
	"bytes"
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"math/rand"
	"os"
	"os/user"
	"strings"
	"text/template"
	"time"

	uuid "github.com/hashicorp/go-uuid"
	"github.com/hashicorp/terraform/terraform"
)

var rngSource *rand.Rand

func init() {
	rngSource = rand.New(rand.NewSource(time.Now().UnixNano()))
}

// State is the collection of all state interfaces.
type State interface {
	StateReader
	StateWriter
	StateRefresher
	StatePersister
	Locker
}

// StateReader is the interface for things that can return a state. Retrieving
// the state here must not error. Loading the state fresh (an operation that
// can likely error) should be implemented by RefreshState. If a state hasn't
// been loaded yet, it is okay for State to return nil.
type StateReader interface {
	State() *terraform.State
}

// StateWriter is the interface that must be implemented by something that
// can write a state. Writing the state can be cached or in-memory, as
// full persistence should be implemented by StatePersister.
type StateWriter interface {
	WriteState(*terraform.State) error
}

// StateRefresher is the interface that is implemented by something that
// can load a state. This might be refreshing it from a remote location or
// it might simply be reloading it from disk.
type StateRefresher interface {
	RefreshState() error
}

// StatePersister is implemented to truly persist a state. Whereas StateWriter
// is allowed to perhaps be caching in memory, PersistState must write the
// state to some durable storage.
type StatePersister interface {
	PersistState() error
}

// Locker is implemented to lock state during command execution.
// The info parameter can be recorded with the lock, but the
// implementation should not depend in its value. The string returned by Lock
// is an ID corresponding to the lock acquired, and must be passed to Unlock to
// ensure that the correct lock is being released.
//
// Lock and Unlock may return an error value of type LockError which in turn
// can contain the LockInfo of a conflicting lock.
type Locker interface {
	Lock(info *LockInfo) (string, error)
	Unlock(id string) error
}

// Lock the state, using the provided context for timeout and cancellation.
// This backs off slightly to an upper limit.
func LockWithContext(ctx context.Context, s State, info *LockInfo) (string, error) {
	delay := time.Second
	maxDelay := 16 * time.Second
	for {
		id, err := s.Lock(info)
		if err == nil {
			return id, nil
		}

		le, ok := err.(*LockError)
		if !ok {
			// not a lock error, so we can't retry
			return "", err
		}

		if le.Info.ID == "" {
			// the lock has no ID, something is wrong so don't keep trying
			return "", fmt.Errorf("lock error missing ID: %s", err)
		}

		// there's an existing lock, wait and try again
		select {
		case <-ctx.Done():
			// return the last lock error with the info
			return "", err
		case <-time.After(delay):
			if delay < maxDelay {
				delay *= 2
			}
		}
	}
}

// Generate a LockInfo structure, populating the required fields.
func NewLockInfo() *LockInfo {
	// this doesn't need to be cryptographically secure, just unique.
	// Using math/rand alleviates the need to check handle the read error.
	// Use a uuid format to match other IDs used throughout Terraform.
	buf := make([]byte, 16)
	rngSource.Read(buf)

	id, err := uuid.FormatUUID(buf)
	if err != nil {
		// this of course shouldn't happen
		panic(err)
	}

	// don't error out on user and hostname, as we don't require them
	userName := ""
	if userInfo, err := user.Current(); err == nil {
		userName = userInfo.Username
	}
	host, _ := os.Hostname()

	info := &LockInfo{
		ID:      id,
		Who:     fmt.Sprintf("%s@%s", userName, host),
		Version: terraform.Version,
		Created: time.Now().UTC(),
	}
	return info
}

// LockInfo stores lock metadata.
//
// Only Operation and Info are required to be set by the caller of Lock.
type LockInfo struct {
	// Unique ID for the lock. NewLockInfo provides a random ID, but this may
	// be overridden by the lock implementation. The final value if ID will be
	// returned by the call to Lock.
	ID string

	// Terraform operation, provided by the caller.
	Operation string
	// Extra information to store with the lock, provided by the caller.
	Info string

	// user@hostname when available
	Who string
	// Terraform version
	Version string
	// Time that the lock was taken.
	Created time.Time

	// Path to the state file when applicable. Set by the Lock implementation.
	Path string
}

// Err returns the lock info formatted in an error
func (l *LockInfo) Err() error {
	return errors.New(l.String())
}

// Marshal returns a string json representation of the LockInfo
func (l *LockInfo) Marshal() []byte {
	js, err := json.Marshal(l)
	if err != nil {
		panic(err)
	}
	return js
}

// String return a multi-line string representation of LockInfo
func (l *LockInfo) String() string {
	tmpl := `Lock Info:
  ID:        {{.ID}}
  Path:      {{.Path}}
  Operation: {{.Operation}}
  Who:       {{.Who}}
  Version:   {{.Version}}
  Created:   {{.Created}}
  Info:      {{.Info}}
`

	t := template.Must(template.New("LockInfo").Parse(tmpl))
	var out bytes.Buffer
	if err := t.Execute(&out, l); err != nil {
		panic(err)
	}
	return out.String()
}

type LockError struct {
	Info *LockInfo
	Err  error
}

func (e *LockError) Error() string {
	var out []string
	if e.Err != nil {
		out = append(out, e.Err.Error())
	}

	if e.Info != nil {
		out = append(out, e.Info.String())
	}
	return strings.Join(out, "\n")
}
<|endoftext|>


<commit_before>package pgx_test

// import (
// 	"context"
// 	"fmt"
// 	"math/rand"
// 	"os"
// 	"strconv"
// 	"testing"
// 	"time"

// 	errors "golang.org/x/xerrors"

// 	"github.com/jackc/fake"
// 	"github.com/jackc/pgx/v4"
// 	"github.com/jackc/pgconn"
// )

// type execer interface {
// 	Exec(ctx context.Context, sql string, arguments ...interface{}) (commandTag pgconn.CommandTag, err error)
// }
// type queryer interface {
// 	Query(sql string, args ...interface{}) (*pgx.Rows, error)
// }
// type queryRower interface {
// 	QueryRow(sql string, args ...interface{}) *pgx.Row
// }

// func TestStressConnPool(t *testing.T) {
// 	t.Parallel()

// 	maxConnections := 8
// 	pool := createConnPool(t, maxConnections)
// 	defer pool.Close()

// 	setupStressDB(t, pool)

// 	actions := []struct {
// 		name string
// 		fn   func(*pgx.ConnPool, int) error
// 	}{
// 		{"insertUnprepared", func(p *pgx.ConnPool, n int) error { return insertUnprepared(p, n) }},
// 		{"queryRowWithoutParams", func(p *pgx.ConnPool, n int) error { return queryRowWithoutParams(p, n) }},
// 		{"query", func(p *pgx.ConnPool, n int) error { return queryCloseEarly(p, n) }},
// 		{"queryCloseEarly", func(p *pgx.ConnPool, n int) error { return query(p, n) }},
// 		{"queryErrorWhileReturningRows", func(p *pgx.ConnPool, n int) error { return queryErrorWhileReturningRows(p, n) }},
// 		{"txInsertRollback", txInsertRollback},
// 		{"txInsertCommit", txInsertCommit},
// 		{"txMultipleQueries", txMultipleQueries},
// 		{"notify", notify},
// 		{"listenAndPoolUnlistens", listenAndPoolUnlistens},
// 		{"reset", func(p *pgx.ConnPool, n int) error { p.Reset(); return nil }},
// 		{"poolPrepareUseAndDeallocate", poolPrepareUseAndDeallocate},
// 		{"canceledQueryExContext", canceledQueryExContext},
// 		{"canceledExecExContext", canceledExecExContext},
// 	}

// 	actionCount := 1000
// 	if s := os.Getenv("STRESS_FACTOR"); s != "" {
// 		stressFactor, err := strconv.ParseInt(s, 10, 64)
// 		if err != nil {
// 			t.Fatalf("failed to parse STRESS_FACTOR: %v", s)
// 		}
// 		actionCount *= int(stressFactor)
// 	}

// 	workerCount := 16

// 	workChan := make(chan int)
// 	doneChan := make(chan struct{})
// 	errChan := make(chan error)

// 	work := func() {
// 		for n := range workChan {
// 			action := actions[rand.Intn(len(actions))]
// 			err := action.fn(pool, n)
// 			if err != nil {
// 				errChan <- errors.Errorf("%s: %v", action.name, err)
// 				break
// 			}
// 		}
// 		doneChan <- struct{}{}
// 	}

// 	for i := 0; i < workerCount; i++ {
// 		go work()
// 	}

// 	for i := 0; i < actionCount; i++ {
// 		select {
// 		case workChan <- i:
// 		case err := <-errChan:
// 			close(workChan)
// 			t.Fatal(err)
// 		}
// 	}
// 	close(workChan)

// 	for i := 0; i < workerCount; i++ {
// 		<-doneChan
// 	}
// }

// func setupStressDB(t *testing.T, pool *pgx.ConnPool) {
// 	_, err := pool.Exec(context.Background(), `
// 		drop table if exists widgets;
// 		create table widgets(
// 			id serial primary key,
// 			name varchar not null,
// 			description text,
// 			creation_time timestamptz
// 		);
// `)
// 	if err != nil {
// 		t.Fatal(err)
// 	}
// }

// func insertUnprepared(e execer, actionNum int) error {
// 	sql := `
// 		insert into widgets(name, description, creation_time)
// 		values($1, $2, $3)`

// 	_, err := e.Exec(context.Background(), sql, fake.ProductName(), fake.Sentences(), time.Now())
// 	return err
// }

// func queryRowWithoutParams(qr queryRower, actionNum int) error {
// 	var id int32
// 	var name, description string
// 	var creationTime time.Time

// 	sql := `select * from widgets order by random() limit 1`

// 	err := qr.QueryRow(sql).Scan(&id, &name, &description, &creationTime)
// 	if err == pgx.ErrNoRows {
// 		return nil
// 	}
// 	return err
// }

// func query(q queryer, actionNum int) error {
// 	sql := `select * from widgets order by random() limit $1`

// 	rows, err := q.Query(sql, 10)
// 	if err != nil {
// 		return err
// 	}
// 	defer rows.Close()

// 	for rows.Next() {
// 		var id int32
// 		var name, description string
// 		var creationTime time.Time
// 		rows.Scan(&id, &name, &description, &creationTime)
// 	}

// 	return rows.Err()
// }

// func queryCloseEarly(q queryer, actionNum int) error {
// 	sql := `select * from generate_series(1,$1)`

// 	rows, err := q.Query(sql, 100)
// 	if err != nil {
// 		return err
// 	}
// 	defer rows.Close()

// 	for i := 0; i < 10 && rows.Next(); i++ {
// 		var n int32
// 		rows.Scan(&n)
// 	}
// 	rows.Close()

// 	return rows.Err()
// }

// func queryErrorWhileReturningRows(q queryer, actionNum int) error {
// 	// This query should divide by 0 within the first number of rows
// 	sql := `select 42 / (random() * 20)::integer from generate_series(1,100000)`

// 	rows, err := q.Query(sql)
// 	if err != nil {
// 		return nil
// 	}
// 	defer rows.Close()

// 	for rows.Next() {
// 		var n int32
// 		rows.Scan(&n)
// 	}

// 	if _, ok := rows.Err().(*pgconn.PgError); ok {
// 		return nil
// 	}
// 	return rows.Err()
// }

// func notify(pool *pgx.ConnPool, actionNum int) error {
// 	_, err := pool.Exec(context.Background(), "notify stress")
// 	return err
// }

// func listenAndPoolUnlistens(pool *pgx.ConnPool, actionNum int) error {
// 	conn, err := pool.Acquire()
// 	if err != nil {
// 		return err
// 	}
// 	defer pool.Release(conn)

// 	err = conn.Listen("stress")
// 	if err != nil {
// 		return err
// 	}

// 	ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
// 	defer cancel()
// 	_, err = conn.WaitForNotification(ctx)
// 	if err == context.DeadlineExceeded {
// 		return nil
// 	}
// 	return err
// }

// func poolPrepareUseAndDeallocate(pool *pgx.ConnPool, actionNum int) error {
// 	psName := fmt.Sprintf("poolPreparedStatement%d", actionNum)

// 	_, err := pool.Prepare(psName, "select $1::text")
// 	if err != nil {
// 		return err
// 	}

// 	var s string
// 	err = pool.QueryRow(psName, "hello").Scan(&s)
// 	if err != nil {
// 		return err
// 	}

// 	if s != "hello" {
// 		return errors.Errorf("Prepared statement did not return expected value: %v", s)
// 	}

// 	return pool.Deallocate(psName)
// }

// func txInsertRollback(pool *pgx.ConnPool, actionNum int) error {
// 	tx, err := pool.Begin()
// 	if err != nil {
// 		return err
// 	}

// 	sql := `
// 		insert into widgets(name, description, creation_time)
// 		values($1, $2, $3)`

// 	_, err = tx.Exec(context.Background(), sql, fake.ProductName(), fake.Sentences(), time.Now())
// 	if err != nil {
// 		return err
// 	}

// 	return tx.Rollback()
// }

// func txInsertCommit(pool *pgx.ConnPool, actionNum int) error {
// 	tx, err := pool.Begin()
// 	if err != nil {
// 		return err
// 	}

// 	sql := `
// 		insert into widgets(name, description, creation_time)
// 		values($1, $2, $3)`

// 	_, err = tx.Exec(context.Background(), sql, fake.ProductName(), fake.Sentences(), time.Now())
// 	if err != nil {
// 		tx.Rollback()
// 		return err
// 	}

// 	return tx.Commit(context.Background())
// }

// func txMultipleQueries(pool *pgx.ConnPool, actionNum int) error {
// 	tx, err := pool.Begin()
// 	if err != nil {
// 		return err
// 	}
// 	defer tx.Rollback()

// 	errExpectedTxDeath := errors.New("Expected tx death")

// 	actions := []struct {
// 		name string
// 		fn   func() error
// 	}{
// 		{"insertUnprepared", func() error { return insertUnprepared(tx, actionNum) }},
// 		{"queryRowWithoutParams", func() error { return queryRowWithoutParams(tx, actionNum) }},
// 		{"query", func() error { return query(tx, actionNum) }},
// 		{"queryCloseEarly", func() error { return queryCloseEarly(tx, actionNum) }},
// 		{"queryErrorWhileReturningRows", func() error {
// 			err := queryErrorWhileReturningRows(tx, actionNum)
// 			if err != nil {
// 				return err
// 			}
// 			return errExpectedTxDeath
// 		}},
// 	}

// 	for i := 0; i < 20; i++ {
// 		action := actions[rand.Intn(len(actions))]
// 		err := action.fn()
// 		if err == errExpectedTxDeath {
// 			return nil
// 		} else if err != nil {
// 			return err
// 		}
// 	}

// 	return tx.Commit(context.Background())
// }

// func canceledQueryExContext(pool *pgx.ConnPool, actionNum int) error {
// 	ctx, cancelFunc := context.WithCancel(context.Background())
// 	go func() {
// 		time.Sleep(time.Duration(rand.Intn(50)) * time.Millisecond)
// 		cancelFunc()
// 	}()

// 	rows, err := pool.QueryEx(ctx, "select pg_sleep(2)", nil)
// 	if err == context.Canceled {
// 		return nil
// 	} else if err != nil {
// 		return errors.Errorf("Only allowed error is context.Canceled, got %v", err)
// 	}

// 	for rows.Next() {
// 		return errors.New("should never receive row")
// 	}

// 	if rows.Err() != context.Canceled {
// 		return errors.Errorf("Expected context.Canceled error, got %v", rows.Err())
// 	}

// 	return nil
// }

// func canceledExecExContext(pool *pgx.ConnPool, actionNum int) error {
// 	ctx, cancelFunc := context.WithCancel(context.Background())
// 	go func() {
// 		time.Sleep(time.Duration(rand.Intn(50)) * time.Millisecond)
// 		cancelFunc()
// 	}()

// 	_, err := pool.Exec(ctx, "select pg_sleep(2)")
// 	if err != context.Canceled {
// 		return errors.Errorf("Expected context.Canceled error, got %v", err)
// 	}

// 	return nil
// }
<commit_msg>Remove obsolete test file<commit_after><|endoftext|>


<commit_before>package workflow

import (
	"errors"
	"fmt"
	"os"
	"path/filepath"

	yaml "gopkg.in/yaml.v2"

	configgenerator "github.com/coreos/tectonic-installer/installer/pkg/config-generator"
)

const (
	kubeSystemPath             = "generated/manifests"
	kubeSystemFileName         = "cluster-config.yaml"
	tectonicSystemPath         = "generated/tectonic"
	tectonicSystemFileName     = "cluster-config.yaml"
	terraformVariablesFileName = "terraform.tfvars"
)

// NewInitWorkflow creates new instances of the 'init' workflow,
// responsible for initializing a new cluster.
func NewInitWorkflow(configFilePath string) Workflow {
	return Workflow{
		metadata: metadata{configFilePath: configFilePath},
		steps: []Step{
			readClusterConfigStep,
			prepareWorspaceStep,
			buildInternalStep,
			generateTerraformVariablesStep,
		},
	}
}

func buildInternalStep(m *metadata) error {
	if m.clusterDir == "" {
		return errors.New("no clusterDir path set in metadata")
	}

	// fill the internal struct
	clusterId, err := configgenerator.GenerateClusterID(16)
	if err != nil {
		return err
	}
	m.cluster.Internal.ClusterID = clusterId

	// store the content
	internalFileContent, err := yaml.Marshal(m.cluster.Internal)
	if err != nil {
		return err
	}
	return writeFile(filepath.Join(m.clusterDir, internalFileName), string(internalFileContent))
}

func generateTerraformVariablesStep(m *metadata) error {
	vars, err := m.cluster.TFVars()
	if err != nil {
		return err
	}

	terraformVariablesFilePath := filepath.Join(m.clusterDir, terraformVariablesFileName)
	return writeFile(terraformVariablesFilePath, vars)
}

func prepareWorspaceStep(m *metadata) error {
	dir, err := os.Getwd()
	if err != nil {
		return fmt.Errorf("Failed to get current directory because: %s", err)
	}

	m.clusterDir = filepath.Join(dir, m.cluster.Name)
	if stat, err := os.Stat(m.clusterDir); err == nil && stat.IsDir() {
		return fmt.Errorf("cluster directory already exists at %s", m.clusterDir)
	}

	if err := os.MkdirAll(m.clusterDir, os.ModeDir|0755); err != nil {
		return fmt.Errorf("Failed to create cluster directory at %s", m.clusterDir)
	}

	configFilePath := filepath.Join(m.clusterDir, configFileName)
	if err := copyFile(m.configFilePath, configFilePath); err != nil {
		return err
	}

	return nil
}
<commit_msg>add warning meesage to yaml internal<commit_after>package workflow

import (
	"errors"
	"fmt"
	"os"
	"path/filepath"

	yaml "gopkg.in/yaml.v2"

	configgenerator "github.com/coreos/tectonic-installer/installer/pkg/config-generator"
)

const (
	kubeSystemPath             = "generated/manifests"
	kubeSystemFileName         = "cluster-config.yaml"
	tectonicSystemPath         = "generated/tectonic"
	tectonicSystemFileName     = "cluster-config.yaml"
	terraformVariablesFileName = "terraform.tfvars"
)

// NewInitWorkflow creates new instances of the 'init' workflow,
// responsible for initializing a new cluster.
func NewInitWorkflow(configFilePath string) Workflow {
	return Workflow{
		metadata: metadata{configFilePath: configFilePath},
		steps: []Step{
			readClusterConfigStep,
			prepareWorspaceStep,
			buildInternalStep,
			generateTerraformVariablesStep,
		},
	}
}

func buildInternalStep(m *metadata) error {
	if m.clusterDir == "" {
		return errors.New("no clusterDir path set in metadata")
	}

	// fill the internal struct
	clusterId, err := configgenerator.GenerateClusterID(16)
	if err != nil {
		return err
	}
	m.cluster.Internal.ClusterID = clusterId

	// store the content
	yamlContent, err := yaml.Marshal(m.cluster.Internal)
	internalFileContent := []byte("# Do not touch, auto-generated\n")
	internalFileContent = append(internalFileContent, yamlContent...)
	if err != nil {
		return err
	}
	return writeFile(filepath.Join(m.clusterDir, internalFileName), string(internalFileContent))
}

func generateTerraformVariablesStep(m *metadata) error {
	vars, err := m.cluster.TFVars()
	if err != nil {
		return err
	}

	terraformVariablesFilePath := filepath.Join(m.clusterDir, terraformVariablesFileName)
	return writeFile(terraformVariablesFilePath, vars)
}

func prepareWorspaceStep(m *metadata) error {
	dir, err := os.Getwd()
	if err != nil {
		return fmt.Errorf("Failed to get current directory because: %s", err)
	}

	m.clusterDir = filepath.Join(dir, m.cluster.Name)
	if stat, err := os.Stat(m.clusterDir); err == nil && stat.IsDir() {
		return fmt.Errorf("cluster directory already exists at %s", m.clusterDir)
	}

	if err := os.MkdirAll(m.clusterDir, os.ModeDir|0755); err != nil {
		return fmt.Errorf("Failed to create cluster directory at %s", m.clusterDir)
	}

	configFilePath := filepath.Join(m.clusterDir, configFileName)
	if err := copyFile(m.configFilePath, configFilePath); err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>package packed

import (
	"bytes"
	"encoding/hex"
	"io"
	"testing"
)

var compressionTests = []struct {
	name       string
	original   []byte
	compressed []byte
}{
	{
		"empty",
		[]byte{},
		[]byte{},
	},
	{
		"one zero word",
		[]byte{0, 0, 0, 0, 0, 0, 0, 0},
		[]byte{0, 0},
	},
	{
		"one word with mixed zero bytes",
		[]byte{0, 0, 12, 0, 0, 34, 0, 0},
		[]byte{0x24, 12, 34},
	},
	{
		"two words with mixed zero bytes",
		[]byte{
			0x08, 0x00, 0x00, 0x00, 0x03, 0x00, 0x02, 0x00,
			0x19, 0x00, 0x00, 0x00, 0xaa, 0x01, 0x00, 0x00,
		},
		[]byte{0x51, 0x08, 0x03, 0x02, 0x31, 0x19, 0xaa, 0x01},
	},
	{
		"two words with mixed zero bytes",
		[]byte{0x8, 0, 0, 0, 0x3, 0, 0x2, 0, 0x19, 0, 0, 0, 0xaa, 0x1, 0, 0},
		[]byte{0x51, 0x08, 0x03, 0x02, 0x31, 0x19, 0xaa, 0x01},
	},
	{
		"four zero words",
		[]byte{
			0, 0, 0, 0, 0, 0, 0, 0,
			0, 0, 0, 0, 0, 0, 0, 0,
			0, 0, 0, 0, 0, 0, 0, 0,
			0, 0, 0, 0, 0, 0, 0, 0,
		},
		[]byte{0x00, 0x03},
	},
	{
		"four words without zero bytes",
		[]byte{
			0x8a, 0x8a, 0x8a, 0x8a, 0x8a, 0x8a, 0x8a, 0x8a,
			0x8a, 0x8a, 0x8a, 0x8a, 0x8a, 0x8a, 0x8a, 0x8a,
			0x8a, 0x8a, 0x8a, 0x8a, 0x8a, 0x8a, 0x8a, 0x8a,
			0x8a, 0x8a, 0x8a, 0x8a, 0x8a, 0x8a, 0x8a, 0x8a,
		},
		[]byte{
			0xff,
			0x8a, 0x8a, 0x8a, 0x8a, 0x8a, 0x8a, 0x8a, 0x8a,
			0x03,
			0x8a, 0x8a, 0x8a, 0x8a, 0x8a, 0x8a, 0x8a, 0x8a,
			0x8a, 0x8a, 0x8a, 0x8a, 0x8a, 0x8a, 0x8a, 0x8a,
			0x8a, 0x8a, 0x8a, 0x8a, 0x8a, 0x8a, 0x8a, 0x8a,
		},
	},
	{
		"one word without zero bytes",
		[]byte{1, 3, 2, 4, 5, 7, 6, 8},
		[]byte{0xff, 1, 3, 2, 4, 5, 7, 6, 8, 0},
	},
	{
		"one zero word followed by one word without zero bytes",
		[]byte{0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 2, 4, 5, 7, 6, 8},
		[]byte{0, 0, 0xff, 1, 3, 2, 4, 5, 7, 6, 8, 0},
	},
	{
		"one word with mixed zero bytes followed by one word without zero bytes",
		[]byte{0, 0, 12, 0, 0, 34, 0, 0, 1, 3, 2, 4, 5, 7, 6, 8},
		[]byte{0x24, 12, 34, 0xff, 1, 3, 2, 4, 5, 7, 6, 8, 0},
	},
	{
		"two words with no zero bytes",
		[]byte{1, 3, 2, 4, 5, 7, 6, 8, 8, 6, 7, 4, 5, 2, 3, 1},
		[]byte{0xff, 1, 3, 2, 4, 5, 7, 6, 8, 1, 8, 6, 7, 4, 5, 2, 3, 1},
	},
	{
		"five words, with only the last containing zero bytes",
		[]byte{
			1, 2, 3, 4, 5, 6, 7, 8,
			1, 2, 3, 4, 5, 6, 7, 8,
			1, 2, 3, 4, 5, 6, 7, 8,
			1, 2, 3, 4, 5, 6, 7, 8,
			0, 2, 4, 0, 9, 0, 5, 1,
		},
		[]byte{
			0xff, 1, 2, 3, 4, 5, 6, 7, 8,
			3,
			1, 2, 3, 4, 5, 6, 7, 8,
			1, 2, 3, 4, 5, 6, 7, 8,
			1, 2, 3, 4, 5, 6, 7, 8,
			0xd6, 2, 4, 9, 5, 1,
		},
	},
	{
		"five words, with the middle and last words containing zero bytes",
		[]byte{
			1, 2, 3, 4, 5, 6, 7, 8,
			1, 2, 3, 4, 5, 6, 7, 8,
			6, 2, 4, 3, 9, 0, 5, 1,
			1, 2, 3, 4, 5, 6, 7, 8,
			0, 2, 4, 0, 9, 0, 5, 1,
		},
		[]byte{
			0xff, 1, 2, 3, 4, 5, 6, 7, 8,
			3,
			1, 2, 3, 4, 5, 6, 7, 8,
			6, 2, 4, 3, 9, 0, 5, 1,
			1, 2, 3, 4, 5, 6, 7, 8,
			0xd6, 2, 4, 9, 5, 1,
		},
	},
	{
		"words with mixed zeroes sandwiching zero words",
		[]byte{
			8, 0, 100, 6, 0, 1, 1, 2,
			0, 0, 0, 0, 0, 0, 0, 0,
			0, 0, 0, 0, 0, 0, 0, 0,
			0, 0, 0, 0, 0, 0, 0, 0,
			0, 0, 1, 0, 2, 0, 3, 1,
		},
		[]byte{
			0xed, 8, 100, 6, 1, 1, 2,
			0, 2,
			0xd4, 1, 2, 3, 1,
		},
	},
	{
		"real-world Cap'n Proto data",
		[]byte{
			0x0, 0x0, 0x0, 0x0, 0x5, 0x0, 0x0, 0x0,
			0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x1, 0x0,
			0x25, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
			0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
			0x1, 0x0, 0x0, 0x0, 0xc, 0x0, 0x0, 0x0,
			0xd4, 0x7, 0xc, 0x7, 0x0, 0x0, 0x0, 0x0,
		},
		[]byte{
			0x10, 0x5,
			0x50, 0x2, 0x1,
			0x1, 0x25,
			0x0, 0x0,
			0x11, 0x1, 0xc,
			0xf, 0xd4, 0x7, 0xc, 0x7,
		},
	},
}

func TestPack(t *testing.T) {
	for _, test := range compressionTests {
		orig := make([]byte, len(test.original))
		copy(orig, test.original)
		compressed := Pack(nil, orig)
		if !bytes.Equal(compressed, test.compressed) {
			t.Errorf("%s: Pack(nil,\n%s\n) =\n%s\n; want\n%s", test.name, hex.Dump(test.original), hex.Dump(compressed), hex.Dump(test.compressed))
		}
	}
}

func TestReader(t *testing.T) {
	for _, test := range compressionTests {
		for readSize := 1; readSize <= 8+2*len(test.original); readSize++ {
			r := bytes.NewReader(test.compressed)
			d := NewReader(r)
			buf := make([]byte, readSize)
			var actual []byte
			for {
				n, err := d.Read(buf)
				actual = append(actual, buf[:n]...)
				if err != nil {
					if err == io.EOF {
						break
					}
					t.Fatalf("Read: %v", err)
				}
			}

			if len(test.original) != len(actual) {
				t.Errorf("%s: readSize=%d: expected %d bytes, got %d", test.name, readSize, len(test.original), len(actual))
				continue
			}

			if !bytes.Equal(test.original, actual) {
				t.Errorf("%s: readSize=%d: bytes not equal", test.name, readSize)
			}
		}
	}
}

var result []byte

func BenchmarkPack(b *testing.B) {
	src := bytes.Repeat([]byte{
		8, 0, 100, 6, 0, 1, 1, 2,
		8, 0, 100, 6, 0, 1, 1, 2,
		0, 0, 0, 0, 0, 0, 0, 0,
		0, 0, 0, 0, 0, 0, 0, 0,
		0, 0, 0, 0, 0, 0, 0, 0,
		0, 0, 0, 0, 0, 0, 0, 0,
		0, 0, 1, 0, 2, 0, 3, 1,
		'H', 'e', 'l', 'l', 'o', ',', ' ', 'W',
		'o', 'r', 'l', 'd', '!', ' ', ' ', 'P',
		'a', 'd', ' ', 't', 'e', 'x', 't', '.',
	}, 128)
	dst := make([]byte, 0, len(src))
	b.SetBytes(int64(len(src)))

	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		Pack(dst, src)
	}
	result = dst
}

func BenchmarkDecompressor(b *testing.B) {
	const multiplier = 128
	src := bytes.Repeat([]byte{
		0xa7, 8, 100, 6, 1, 1, 2,
		0xa7, 8, 100, 6, 1, 1, 2,
		0x00, 3,
		0x2a,
		0xff, 'H', 'e', 'l', 'l', 'o', ',', ' ', 'W',
		2,
		'o', 'r', 'l', 'd', '!', ' ', ' ', 'P',
		'a', 'd', ' ', 't', 'e', 'x', 't', '.',
	}, multiplier)
	b.SetBytes(int64(len(src)))
	r := bytes.NewReader(src)

	dst := make([]byte, 10*8*multiplier)

	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		r.Seek(0, 0)
		d := NewReader(r)
		_, err := d.Read(dst)
		if err != nil {
			b.Fatal(err)
		}
	}
	result = dst
}
<commit_msg>internal/packed: add test for data error<commit_after>package packed

import (
	"bytes"
	"encoding/hex"
	"io"
	"testing"
	"testing/iotest"
)

var compressionTests = []struct {
	name       string
	original   []byte
	compressed []byte
}{
	{
		"empty",
		[]byte{},
		[]byte{},
	},
	{
		"one zero word",
		[]byte{0, 0, 0, 0, 0, 0, 0, 0},
		[]byte{0, 0},
	},
	{
		"one word with mixed zero bytes",
		[]byte{0, 0, 12, 0, 0, 34, 0, 0},
		[]byte{0x24, 12, 34},
	},
	{
		"two words with mixed zero bytes",
		[]byte{
			0x08, 0x00, 0x00, 0x00, 0x03, 0x00, 0x02, 0x00,
			0x19, 0x00, 0x00, 0x00, 0xaa, 0x01, 0x00, 0x00,
		},
		[]byte{0x51, 0x08, 0x03, 0x02, 0x31, 0x19, 0xaa, 0x01},
	},
	{
		"two words with mixed zero bytes",
		[]byte{0x8, 0, 0, 0, 0x3, 0, 0x2, 0, 0x19, 0, 0, 0, 0xaa, 0x1, 0, 0},
		[]byte{0x51, 0x08, 0x03, 0x02, 0x31, 0x19, 0xaa, 0x01},
	},
	{
		"four zero words",
		[]byte{
			0, 0, 0, 0, 0, 0, 0, 0,
			0, 0, 0, 0, 0, 0, 0, 0,
			0, 0, 0, 0, 0, 0, 0, 0,
			0, 0, 0, 0, 0, 0, 0, 0,
		},
		[]byte{0x00, 0x03},
	},
	{
		"four words without zero bytes",
		[]byte{
			0x8a, 0x8a, 0x8a, 0x8a, 0x8a, 0x8a, 0x8a, 0x8a,
			0x8a, 0x8a, 0x8a, 0x8a, 0x8a, 0x8a, 0x8a, 0x8a,
			0x8a, 0x8a, 0x8a, 0x8a, 0x8a, 0x8a, 0x8a, 0x8a,
			0x8a, 0x8a, 0x8a, 0x8a, 0x8a, 0x8a, 0x8a, 0x8a,
		},
		[]byte{
			0xff,
			0x8a, 0x8a, 0x8a, 0x8a, 0x8a, 0x8a, 0x8a, 0x8a,
			0x03,
			0x8a, 0x8a, 0x8a, 0x8a, 0x8a, 0x8a, 0x8a, 0x8a,
			0x8a, 0x8a, 0x8a, 0x8a, 0x8a, 0x8a, 0x8a, 0x8a,
			0x8a, 0x8a, 0x8a, 0x8a, 0x8a, 0x8a, 0x8a, 0x8a,
		},
	},
	{
		"one word without zero bytes",
		[]byte{1, 3, 2, 4, 5, 7, 6, 8},
		[]byte{0xff, 1, 3, 2, 4, 5, 7, 6, 8, 0},
	},
	{
		"one zero word followed by one word without zero bytes",
		[]byte{0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 2, 4, 5, 7, 6, 8},
		[]byte{0, 0, 0xff, 1, 3, 2, 4, 5, 7, 6, 8, 0},
	},
	{
		"one word with mixed zero bytes followed by one word without zero bytes",
		[]byte{0, 0, 12, 0, 0, 34, 0, 0, 1, 3, 2, 4, 5, 7, 6, 8},
		[]byte{0x24, 12, 34, 0xff, 1, 3, 2, 4, 5, 7, 6, 8, 0},
	},
	{
		"two words with no zero bytes",
		[]byte{1, 3, 2, 4, 5, 7, 6, 8, 8, 6, 7, 4, 5, 2, 3, 1},
		[]byte{0xff, 1, 3, 2, 4, 5, 7, 6, 8, 1, 8, 6, 7, 4, 5, 2, 3, 1},
	},
	{
		"five words, with only the last containing zero bytes",
		[]byte{
			1, 2, 3, 4, 5, 6, 7, 8,
			1, 2, 3, 4, 5, 6, 7, 8,
			1, 2, 3, 4, 5, 6, 7, 8,
			1, 2, 3, 4, 5, 6, 7, 8,
			0, 2, 4, 0, 9, 0, 5, 1,
		},
		[]byte{
			0xff, 1, 2, 3, 4, 5, 6, 7, 8,
			3,
			1, 2, 3, 4, 5, 6, 7, 8,
			1, 2, 3, 4, 5, 6, 7, 8,
			1, 2, 3, 4, 5, 6, 7, 8,
			0xd6, 2, 4, 9, 5, 1,
		},
	},
	{
		"five words, with the middle and last words containing zero bytes",
		[]byte{
			1, 2, 3, 4, 5, 6, 7, 8,
			1, 2, 3, 4, 5, 6, 7, 8,
			6, 2, 4, 3, 9, 0, 5, 1,
			1, 2, 3, 4, 5, 6, 7, 8,
			0, 2, 4, 0, 9, 0, 5, 1,
		},
		[]byte{
			0xff, 1, 2, 3, 4, 5, 6, 7, 8,
			3,
			1, 2, 3, 4, 5, 6, 7, 8,
			6, 2, 4, 3, 9, 0, 5, 1,
			1, 2, 3, 4, 5, 6, 7, 8,
			0xd6, 2, 4, 9, 5, 1,
		},
	},
	{
		"words with mixed zeroes sandwiching zero words",
		[]byte{
			8, 0, 100, 6, 0, 1, 1, 2,
			0, 0, 0, 0, 0, 0, 0, 0,
			0, 0, 0, 0, 0, 0, 0, 0,
			0, 0, 0, 0, 0, 0, 0, 0,
			0, 0, 1, 0, 2, 0, 3, 1,
		},
		[]byte{
			0xed, 8, 100, 6, 1, 1, 2,
			0, 2,
			0xd4, 1, 2, 3, 1,
		},
	},
	{
		"real-world Cap'n Proto data",
		[]byte{
			0x0, 0x0, 0x0, 0x0, 0x5, 0x0, 0x0, 0x0,
			0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x1, 0x0,
			0x25, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
			0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
			0x1, 0x0, 0x0, 0x0, 0xc, 0x0, 0x0, 0x0,
			0xd4, 0x7, 0xc, 0x7, 0x0, 0x0, 0x0, 0x0,
		},
		[]byte{
			0x10, 0x5,
			0x50, 0x2, 0x1,
			0x1, 0x25,
			0x0, 0x0,
			0x11, 0x1, 0xc,
			0xf, 0xd4, 0x7, 0xc, 0x7,
		},
	},
}

func TestPack(t *testing.T) {
	for _, test := range compressionTests {
		orig := make([]byte, len(test.original))
		copy(orig, test.original)
		compressed := Pack(nil, orig)
		if !bytes.Equal(compressed, test.compressed) {
			t.Errorf("%s: Pack(nil,\n%s\n) =\n%s\n; want\n%s", test.name, hex.Dump(test.original), hex.Dump(compressed), hex.Dump(test.compressed))
		}
	}
}

func TestReader(t *testing.T) {
	for _, test := range compressionTests {
		for readSize := 1; readSize <= 8+2*len(test.original); readSize++ {
			r := bytes.NewReader(test.compressed)
			d := NewReader(r)
			buf := make([]byte, readSize)
			var actual []byte
			for {
				n, err := d.Read(buf)
				actual = append(actual, buf[:n]...)
				if err != nil {
					if err == io.EOF {
						break
					}
					t.Fatalf("Read: %v", err)
				}
			}

			if len(test.original) != len(actual) {
				t.Errorf("%s: readSize=%d: expected %d bytes, got %d", test.name, readSize, len(test.original), len(actual))
				continue
			}

			if !bytes.Equal(test.original, actual) {
				t.Errorf("%s: readSize=%d: bytes not equal", test.name, readSize)
			}
		}
	}
}

func TestReader_DataErr(t *testing.T) {
	const readSize = 3
	for _, test := range compressionTests {
		r := iotest.DataErrReader(bytes.NewReader(test.compressed))
		d := NewReader(r)
		buf := make([]byte, readSize)
		var actual []byte
		for {
			n, err := d.Read(buf)
			actual = append(actual, buf[:n]...)
			if err != nil {
				if err == io.EOF {
					break
				}
				t.Fatalf("Read: %v", err)
			}
		}

		if len(test.original) != len(actual) {
			t.Errorf("%s: expected %d bytes, got %d", test.name, len(test.original), len(actual))
			continue
		}

		if !bytes.Equal(test.original, actual) {
			t.Errorf("%s: bytes not equal", test.name)
		}
	}
}

var result []byte

func BenchmarkPack(b *testing.B) {
	src := bytes.Repeat([]byte{
		8, 0, 100, 6, 0, 1, 1, 2,
		8, 0, 100, 6, 0, 1, 1, 2,
		0, 0, 0, 0, 0, 0, 0, 0,
		0, 0, 0, 0, 0, 0, 0, 0,
		0, 0, 0, 0, 0, 0, 0, 0,
		0, 0, 0, 0, 0, 0, 0, 0,
		0, 0, 1, 0, 2, 0, 3, 1,
		'H', 'e', 'l', 'l', 'o', ',', ' ', 'W',
		'o', 'r', 'l', 'd', '!', ' ', ' ', 'P',
		'a', 'd', ' ', 't', 'e', 'x', 't', '.',
	}, 128)
	dst := make([]byte, 0, len(src))
	b.SetBytes(int64(len(src)))

	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		Pack(dst, src)
	}
	result = dst
}

func BenchmarkDecompressor(b *testing.B) {
	const multiplier = 128
	src := bytes.Repeat([]byte{
		0xa7, 8, 100, 6, 1, 1, 2,
		0xa7, 8, 100, 6, 1, 1, 2,
		0x00, 3,
		0x2a,
		0xff, 'H', 'e', 'l', 'l', 'o', ',', ' ', 'W',
		2,
		'o', 'r', 'l', 'd', '!', ' ', ' ', 'P',
		'a', 'd', ' ', 't', 'e', 'x', 't', '.',
	}, multiplier)
	b.SetBytes(int64(len(src)))
	r := bytes.NewReader(src)

	dst := make([]byte, 10*8*multiplier)

	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		r.Seek(0, 0)
		d := NewReader(r)
		_, err := d.Read(dst)
		if err != nil {
			b.Fatal(err)
		}
	}
	result = dst
}
<|endoftext|>