<commit_before>package config

import (
	"bytes"
	"errors"
	"fmt"
	"io/ioutil"
	"regexp"
	"sort"
	"strconv"
	"strings"

	"github.com/hashicorp/terraform/config/lang/ast"
	"github.com/mitchellh/go-homedir"
)

// Funcs is the mapping of built-in functions for configuration.
var Funcs map[string]ast.Function

func init() {
	Funcs = map[string]ast.Function{
		"file":       interpolationFuncFile(),
		"format":     interpolationFuncFormat(),
		"formatlist": interpolationFuncFormatList(),
		"join":       interpolationFuncJoin(),
		"element":    interpolationFuncElement(),
		"replace":    interpolationFuncReplace(),
		"split":      interpolationFuncSplit(),
		"length":     interpolationFuncLength(),

		// Concat is a little useless now since we supported embeddded
		// interpolations but we keep it around for backwards compat reasons.
		"concat": interpolationFuncConcat(),
	}
}

// interpolationFuncConcat implements the "concat" function that
// concatenates multiple strings. This isn't actually necessary anymore
// since our language supports string concat natively, but for backwards
// compat we do this.
func interpolationFuncConcat() ast.Function {
	return ast.Function{
		ArgTypes:     []ast.Type{ast.TypeString},
		ReturnType:   ast.TypeString,
		Variadic:     true,
		VariadicType: ast.TypeString,
		Callback: func(args []interface{}) (interface{}, error) {
			var b bytes.Buffer
			for _, v := range args {
				b.WriteString(v.(string))
			}

			return b.String(), nil
		},
	}
}

// interpolationFuncFile implements the "file" function that allows
// loading contents from a file.
func interpolationFuncFile() ast.Function {
	return ast.Function{
		ArgTypes:   []ast.Type{ast.TypeString},
		ReturnType: ast.TypeString,
		Callback: func(args []interface{}) (interface{}, error) {
			path, err := homedir.Expand(args[0].(string))
			if err != nil {
				return "", err
			}
			data, err := ioutil.ReadFile(path)
			if err != nil {
				return "", err
			}

			return string(data), nil
		},
	}
}

// interpolationFuncFormat implements the "format" function that does
// string formatting.
func interpolationFuncFormat() ast.Function {
	return ast.Function{
		ArgTypes:     []ast.Type{ast.TypeString},
		Variadic:     true,
		VariadicType: ast.TypeAny,
		ReturnType:   ast.TypeString,
		Callback: func(args []interface{}) (interface{}, error) {
			format := args[0].(string)
			return fmt.Sprintf(format, args[1:]...), nil
		},
	}
}

// interpolationFuncFormatList implements the "formatlist" function that does
// string formatting on lists.
func interpolationFuncFormatList() ast.Function {
	return ast.Function{
		ArgTypes:     []ast.Type{ast.TypeString},
		Variadic:     true,
		VariadicType: ast.TypeAny,
		ReturnType:   ast.TypeString,
		Callback: func(args []interface{}) (interface{}, error) {
			// Make a copy of the variadic part of args
			// to avoid modifying the original.
			varargs := make([]interface{}, len(args)-1)
			copy(varargs, args[1:])

			// Convert arguments that are lists into slices.
			// Confirm along the way that all lists have the same length (n).
			var n int
			for i := 1; i < len(args); i++ {
				s, ok := args[i].(string)
				if !ok {
					continue
				}
				parts := strings.Split(s, InterpSplitDelim)
				if len(parts) == 1 {
					continue
				}
				varargs[i-1] = parts
				if n == 0 {
					// first list we've seen
					n = len(parts)
					continue
				}
				if n != len(parts) {
					return nil, fmt.Errorf("format: mismatched list lengths: %d != %d", n, len(parts))
				}
			}

			if n == 0 {
				return nil, errors.New("no lists in arguments to formatlist")
			}

			// Do the formatting.
			format := args[0].(string)

			// Generate a list of formatted strings.
			list := make([]string, n)
			fmtargs := make([]interface{}, len(varargs))
			for i := 0; i < n; i++ {
				for j, arg := range varargs {
					switch arg := arg.(type) {
					default:
						fmtargs[j] = arg
					case []string:
						fmtargs[j] = arg[i]
					}
				}
				list[i] = fmt.Sprintf(format, fmtargs...)
			}
			return strings.Join(list, InterpSplitDelim), nil
		},
	}
}

// interpolationFuncJoin implements the "join" function that allows
// multi-variable values to be joined by some character.
func interpolationFuncJoin() ast.Function {
	return ast.Function{
		ArgTypes:   []ast.Type{ast.TypeString, ast.TypeString},
		ReturnType: ast.TypeString,
		Callback: func(args []interface{}) (interface{}, error) {
			var list []string
			for _, arg := range args[1:] {
				parts := strings.Split(arg.(string), InterpSplitDelim)
				list = append(list, parts...)
			}

			return strings.Join(list, args[0].(string)), nil
		},
	}
}

// interpolationFuncReplace implements the "replace" function that does
// string replacement.
func interpolationFuncReplace() ast.Function {
	return ast.Function{
		ArgTypes:   []ast.Type{ast.TypeString, ast.TypeString, ast.TypeString},
		ReturnType: ast.TypeString,
		Callback: func(args []interface{}) (interface{}, error) {
			s := args[0].(string)
			search := args[1].(string)
			replace := args[2].(string)

			// We search/replace using a regexp if the string is surrounded
			// in forward slashes.
			if len(search) > 1 && search[0] == '/' && search[len(search)-1] == '/' {
				re, err := regexp.Compile(search[1 : len(search)-1])
				if err != nil {
					return nil, err
				}

				return re.ReplaceAllString(s, replace), nil
			}

			return strings.Replace(s, search, replace, -1), nil
		},
	}
}

func interpolationFuncLength() ast.Function {
	return ast.Function{
		ArgTypes:   []ast.Type{ast.TypeString},
		ReturnType: ast.TypeInt,
		Variadic:   false,
		Callback: func(args []interface{}) (interface{}, error) {
			if !strings.Contains(args[0].(string), InterpSplitDelim) {
				return len(args[0].(string)), nil
			}

			var list []string
			for _, arg := range args {
				parts := strings.Split(arg.(string), InterpSplitDelim)
				for _, part := range parts {
					list = append(list, part)
				}
			}
			return len(list), nil
		},
	}
}

// interpolationFuncSplit implements the "split" function that allows
// strings to split into multi-variable values
func interpolationFuncSplit() ast.Function {
	return ast.Function{
		ArgTypes:   []ast.Type{ast.TypeString, ast.TypeString},
		ReturnType: ast.TypeString,
		Callback: func(args []interface{}) (interface{}, error) {
			return strings.Replace(args[1].(string), args[0].(string), InterpSplitDelim, -1), nil
		},
	}
}

// interpolationFuncLookup implements the "lookup" function that allows
// dynamic lookups of map types within a Terraform configuration.
func interpolationFuncLookup(vs map[string]ast.Variable) ast.Function {
	return ast.Function{
		ArgTypes:   []ast.Type{ast.TypeString, ast.TypeString},
		ReturnType: ast.TypeString,
		Callback: func(args []interface{}) (interface{}, error) {
			k := fmt.Sprintf("var.%s.%s", args[0].(string), args[1].(string))
			v, ok := vs[k]
			if !ok {
				return "", fmt.Errorf(
					"lookup in '%s' failed to find '%s'",
					args[0].(string), args[1].(string))
			}
			if v.Type != ast.TypeString {
				return "", fmt.Errorf(
					"lookup in '%s' for '%s' has bad type %s",
					args[0].(string), args[1].(string), v.Type)
			}

			return v.Value.(string), nil
		},
	}
}

// interpolationFuncElement implements the "element" function that allows
// a specific index to be looked up in a multi-variable value. Note that this will
// wrap if the index is larger than the number of elements in the multi-variable value.
func interpolationFuncElement() ast.Function {
	return ast.Function{
		ArgTypes:   []ast.Type{ast.TypeString, ast.TypeString},
		ReturnType: ast.TypeString,
		Callback: func(args []interface{}) (interface{}, error) {
			list := strings.Split(args[0].(string), InterpSplitDelim)

			index, err := strconv.Atoi(args[1].(string))
			if err != nil {
				return "", fmt.Errorf(
					"invalid number for index, got %s", args[1])
			}

			v := list[index%len(list)]
			return v, nil
		},
	}
}

// interpolationFuncKeys implements the "keys" function that yields a list of
// keys of map types within a Terraform configuration.
func interpolationFuncKeys(vs map[string]ast.Variable) ast.Function {
	return ast.Function{
		ArgTypes:   []ast.Type{ast.TypeString},
		ReturnType: ast.TypeString,
		Callback: func(args []interface{}) (interface{}, error) {
			// Prefix must include ending dot to be a map
			prefix := fmt.Sprintf("var.%s.", args[0].(string))
			keys := make([]string, 0, len(vs))
			for k, _ := range vs {
				if !strings.HasPrefix(k, prefix) {
					continue
				}
				keys = append(keys, k[len(prefix):])
			}

			if len(keys) <= 0 {
				return "", fmt.Errorf(
					"failed to find map '%s'",
					args[0].(string))
			}

			sort.Strings(keys)

			return strings.Join(keys, InterpSplitDelim), nil
		},
	}
}

// interpolationFuncValues implements the "values" function that yields a list of
// keys of map types within a Terraform configuration.
func interpolationFuncValues(vs map[string]ast.Variable) ast.Function {
	return ast.Function{
		ArgTypes:   []ast.Type{ast.TypeString},
		ReturnType: ast.TypeString,
		Callback: func(args []interface{}) (interface{}, error) {
			// Prefix must include ending dot to be a map
			prefix := fmt.Sprintf("var.%s.", args[0].(string))
			keys := make([]string, 0, len(vs))
			for k, _ := range vs {
				if !strings.HasPrefix(k, prefix) {
					continue
				}
				keys = append(keys, k)
			}

			if len(keys) <= 0 {
				return "", fmt.Errorf(
					"failed to find map '%s'",
					args[0].(string))
			}

			sort.Strings(keys)

			vals := make([]string, 0, len(keys))

			for _, k := range keys {
				v := vs[k]
				if v.Type != ast.TypeString {
					return "", fmt.Errorf("values(): %q has bad type %s", k, v.Type)
				}
				vals = append(vals, vs[k].Value.(string))
			}

			return strings.Join(vals, InterpSplitDelim), nil
		},
	}
}
<commit_msg>Sort functions by name alphabetically<commit_after>package config

import (
	"bytes"
	"errors"
	"fmt"
	"io/ioutil"
	"regexp"
	"sort"
	"strconv"
	"strings"

	"github.com/hashicorp/terraform/config/lang/ast"
	"github.com/mitchellh/go-homedir"
)

// Funcs is the mapping of built-in functions for configuration.
var Funcs map[string]ast.Function

func init() {
	Funcs = map[string]ast.Function{
		"element":    interpolationFuncElement(),
		"file":       interpolationFuncFile(),
		"format":     interpolationFuncFormat(),
		"formatlist": interpolationFuncFormatList(),
		"join":       interpolationFuncJoin(),
		"length":     interpolationFuncLength(),
		"replace":    interpolationFuncReplace(),
		"split":      interpolationFuncSplit(),

		// Concat is a little useless now since we supported embeddded
		// interpolations but we keep it around for backwards compat reasons.
		"concat": interpolationFuncConcat(),
	}
}

// interpolationFuncConcat implements the "concat" function that
// concatenates multiple strings. This isn't actually necessary anymore
// since our language supports string concat natively, but for backwards
// compat we do this.
func interpolationFuncConcat() ast.Function {
	return ast.Function{
		ArgTypes:     []ast.Type{ast.TypeString},
		ReturnType:   ast.TypeString,
		Variadic:     true,
		VariadicType: ast.TypeString,
		Callback: func(args []interface{}) (interface{}, error) {
			var b bytes.Buffer
			for _, v := range args {
				b.WriteString(v.(string))
			}

			return b.String(), nil
		},
	}
}

// interpolationFuncFile implements the "file" function that allows
// loading contents from a file.
func interpolationFuncFile() ast.Function {
	return ast.Function{
		ArgTypes:   []ast.Type{ast.TypeString},
		ReturnType: ast.TypeString,
		Callback: func(args []interface{}) (interface{}, error) {
			path, err := homedir.Expand(args[0].(string))
			if err != nil {
				return "", err
			}
			data, err := ioutil.ReadFile(path)
			if err != nil {
				return "", err
			}

			return string(data), nil
		},
	}
}

// interpolationFuncFormat implements the "format" function that does
// string formatting.
func interpolationFuncFormat() ast.Function {
	return ast.Function{
		ArgTypes:     []ast.Type{ast.TypeString},
		Variadic:     true,
		VariadicType: ast.TypeAny,
		ReturnType:   ast.TypeString,
		Callback: func(args []interface{}) (interface{}, error) {
			format := args[0].(string)
			return fmt.Sprintf(format, args[1:]...), nil
		},
	}
}

// interpolationFuncFormatList implements the "formatlist" function that does
// string formatting on lists.
func interpolationFuncFormatList() ast.Function {
	return ast.Function{
		ArgTypes:     []ast.Type{ast.TypeString},
		Variadic:     true,
		VariadicType: ast.TypeAny,
		ReturnType:   ast.TypeString,
		Callback: func(args []interface{}) (interface{}, error) {
			// Make a copy of the variadic part of args
			// to avoid modifying the original.
			varargs := make([]interface{}, len(args)-1)
			copy(varargs, args[1:])

			// Convert arguments that are lists into slices.
			// Confirm along the way that all lists have the same length (n).
			var n int
			for i := 1; i < len(args); i++ {
				s, ok := args[i].(string)
				if !ok {
					continue
				}
				parts := strings.Split(s, InterpSplitDelim)
				if len(parts) == 1 {
					continue
				}
				varargs[i-1] = parts
				if n == 0 {
					// first list we've seen
					n = len(parts)
					continue
				}
				if n != len(parts) {
					return nil, fmt.Errorf("format: mismatched list lengths: %d != %d", n, len(parts))
				}
			}

			if n == 0 {
				return nil, errors.New("no lists in arguments to formatlist")
			}

			// Do the formatting.
			format := args[0].(string)

			// Generate a list of formatted strings.
			list := make([]string, n)
			fmtargs := make([]interface{}, len(varargs))
			for i := 0; i < n; i++ {
				for j, arg := range varargs {
					switch arg := arg.(type) {
					default:
						fmtargs[j] = arg
					case []string:
						fmtargs[j] = arg[i]
					}
				}
				list[i] = fmt.Sprintf(format, fmtargs...)
			}
			return strings.Join(list, InterpSplitDelim), nil
		},
	}
}

// interpolationFuncJoin implements the "join" function that allows
// multi-variable values to be joined by some character.
func interpolationFuncJoin() ast.Function {
	return ast.Function{
		ArgTypes:   []ast.Type{ast.TypeString, ast.TypeString},
		ReturnType: ast.TypeString,
		Callback: func(args []interface{}) (interface{}, error) {
			var list []string
			for _, arg := range args[1:] {
				parts := strings.Split(arg.(string), InterpSplitDelim)
				list = append(list, parts...)
			}

			return strings.Join(list, args[0].(string)), nil
		},
	}
}

// interpolationFuncReplace implements the "replace" function that does
// string replacement.
func interpolationFuncReplace() ast.Function {
	return ast.Function{
		ArgTypes:   []ast.Type{ast.TypeString, ast.TypeString, ast.TypeString},
		ReturnType: ast.TypeString,
		Callback: func(args []interface{}) (interface{}, error) {
			s := args[0].(string)
			search := args[1].(string)
			replace := args[2].(string)

			// We search/replace using a regexp if the string is surrounded
			// in forward slashes.
			if len(search) > 1 && search[0] == '/' && search[len(search)-1] == '/' {
				re, err := regexp.Compile(search[1 : len(search)-1])
				if err != nil {
					return nil, err
				}

				return re.ReplaceAllString(s, replace), nil
			}

			return strings.Replace(s, search, replace, -1), nil
		},
	}
}

func interpolationFuncLength() ast.Function {
	return ast.Function{
		ArgTypes:   []ast.Type{ast.TypeString},
		ReturnType: ast.TypeInt,
		Variadic:   false,
		Callback: func(args []interface{}) (interface{}, error) {
			if !strings.Contains(args[0].(string), InterpSplitDelim) {
				return len(args[0].(string)), nil
			}

			var list []string
			for _, arg := range args {
				parts := strings.Split(arg.(string), InterpSplitDelim)
				for _, part := range parts {
					list = append(list, part)
				}
			}
			return len(list), nil
		},
	}
}

// interpolationFuncSplit implements the "split" function that allows
// strings to split into multi-variable values
func interpolationFuncSplit() ast.Function {
	return ast.Function{
		ArgTypes:   []ast.Type{ast.TypeString, ast.TypeString},
		ReturnType: ast.TypeString,
		Callback: func(args []interface{}) (interface{}, error) {
			return strings.Replace(args[1].(string), args[0].(string), InterpSplitDelim, -1), nil
		},
	}
}

// interpolationFuncLookup implements the "lookup" function that allows
// dynamic lookups of map types within a Terraform configuration.
func interpolationFuncLookup(vs map[string]ast.Variable) ast.Function {
	return ast.Function{
		ArgTypes:   []ast.Type{ast.TypeString, ast.TypeString},
		ReturnType: ast.TypeString,
		Callback: func(args []interface{}) (interface{}, error) {
			k := fmt.Sprintf("var.%s.%s", args[0].(string), args[1].(string))
			v, ok := vs[k]
			if !ok {
				return "", fmt.Errorf(
					"lookup in '%s' failed to find '%s'",
					args[0].(string), args[1].(string))
			}
			if v.Type != ast.TypeString {
				return "", fmt.Errorf(
					"lookup in '%s' for '%s' has bad type %s",
					args[0].(string), args[1].(string), v.Type)
			}

			return v.Value.(string), nil
		},
	}
}

// interpolationFuncElement implements the "element" function that allows
// a specific index to be looked up in a multi-variable value. Note that this will
// wrap if the index is larger than the number of elements in the multi-variable value.
func interpolationFuncElement() ast.Function {
	return ast.Function{
		ArgTypes:   []ast.Type{ast.TypeString, ast.TypeString},
		ReturnType: ast.TypeString,
		Callback: func(args []interface{}) (interface{}, error) {
			list := strings.Split(args[0].(string), InterpSplitDelim)

			index, err := strconv.Atoi(args[1].(string))
			if err != nil {
				return "", fmt.Errorf(
					"invalid number for index, got %s", args[1])
			}

			v := list[index%len(list)]
			return v, nil
		},
	}
}

// interpolationFuncKeys implements the "keys" function that yields a list of
// keys of map types within a Terraform configuration.
func interpolationFuncKeys(vs map[string]ast.Variable) ast.Function {
	return ast.Function{
		ArgTypes:   []ast.Type{ast.TypeString},
		ReturnType: ast.TypeString,
		Callback: func(args []interface{}) (interface{}, error) {
			// Prefix must include ending dot to be a map
			prefix := fmt.Sprintf("var.%s.", args[0].(string))
			keys := make([]string, 0, len(vs))
			for k, _ := range vs {
				if !strings.HasPrefix(k, prefix) {
					continue
				}
				keys = append(keys, k[len(prefix):])
			}

			if len(keys) <= 0 {
				return "", fmt.Errorf(
					"failed to find map '%s'",
					args[0].(string))
			}

			sort.Strings(keys)

			return strings.Join(keys, InterpSplitDelim), nil
		},
	}
}

// interpolationFuncValues implements the "values" function that yields a list of
// keys of map types within a Terraform configuration.
func interpolationFuncValues(vs map[string]ast.Variable) ast.Function {
	return ast.Function{
		ArgTypes:   []ast.Type{ast.TypeString},
		ReturnType: ast.TypeString,
		Callback: func(args []interface{}) (interface{}, error) {
			// Prefix must include ending dot to be a map
			prefix := fmt.Sprintf("var.%s.", args[0].(string))
			keys := make([]string, 0, len(vs))
			for k, _ := range vs {
				if !strings.HasPrefix(k, prefix) {
					continue
				}
				keys = append(keys, k)
			}

			if len(keys) <= 0 {
				return "", fmt.Errorf(
					"failed to find map '%s'",
					args[0].(string))
			}

			sort.Strings(keys)

			vals := make([]string, 0, len(keys))

			for _, k := range keys {
				v := vs[k]
				if v.Type != ast.TypeString {
					return "", fmt.Errorf("values(): %q has bad type %s", k, v.Type)
				}
				vals = append(vals, vs[k].Value.(string))
			}

			return strings.Join(vals, InterpSplitDelim), nil
		},
	}
}
<|endoftext|>


<commit_before>package meli

import (
	"context"
	"io/ioutil"
	"strings"
	"testing"
)

func TestPullDockerImage(t *testing.T) {
	tt := []struct {
		dc          *DockerContainer
		expectedErr error
	}{
		{&DockerContainer{ComposeService: ComposeService{Image: "busybox"}, LogMedium: ioutil.Discard}, nil},
	}
	var ctx = context.Background()
	cli := &mockDockerClient{}
	for _, v := range tt {
		err := PullDockerImage(ctx, cli, v.dc)
		if err != nil {
			t.Errorf("\nCalled PullDockerImage(%#+v) \ngot %s \nwanted %#+v", v.dc, err, v.expectedErr)
		}
	}
}

func TestBuildDockerImage(t *testing.T) {
	tt := []struct {
		dc          *DockerContainer
		expected    string
		expectedErr error
	}{
		{
			&DockerContainer{
				ServiceName:       "myservicename",
				DockerComposeFile: "docker-compose.yml",
				ComposeService: ComposeService{
					Build: Buildstruct{Dockerfile: "testdata/Dockerfile"}},
				LogMedium: ioutil.Discard},
			"meli_myservicename",
			nil},
		{
			&DockerContainer{
				ServiceName:       "myservicename",
				DockerComposeFile: "docker-compose.yml",
				ComposeService: ComposeService{
					Build: Buildstruct{Dockerfile: "testdata/Dockerfile"}},
				LogMedium: ioutil.Discard,
				Rebuild:   true,
			},
			"meli_myservicename",
			nil},
	}

	var ctx = context.Background()
	cli := &mockDockerClient{}
	LoadAuth()
	for _, v := range tt {
		actual, err := BuildDockerImage(ctx, cli, v.dc)
		if err != nil {
			t.Errorf("\nCalled BuildDockerImage(%#+v) \ngot %s \nwanted %#+v", v.dc, err, v.expectedErr)
		}
		if actual != v.expected {
			t.Errorf("\nCalled BuildDockerImage(%#+v) \ngot %s \nwanted %#+v", v.dc, actual, v.expected)
		}
	}
}

func BenchmarkPullDockerImage(b *testing.B) {
	var ctx = context.Background()
	cli := &mockDockerClient{}
	dc := &DockerContainer{ComposeService: ComposeService{Image: "busybox"}, LogMedium: ioutil.Discard}
	LoadAuth()
	for n := 0; n < b.N; n++ {
		_ = PullDockerImage(ctx, cli, dc)
	}
}

func BenchmarkBuildDockerImage(b *testing.B) {
	var ctx = context.Background()
	cli := &mockDockerClient{}
	dc := &DockerContainer{
		ServiceName:       "myservicename",
		DockerComposeFile: "docker-compose.yml",
		ComposeService: ComposeService{
			Build: Buildstruct{Dockerfile: "testdata/Dockerfile"}},
		LogMedium: ioutil.Discard,
		Rebuild:   true,
	}
	LoadAuth()
	for n := 0; n < b.N; n++ {
		_, _ = BuildDockerImage(ctx, cli, dc)
	}
}

func BenchmarkPoolReadFrom(b *testing.B) {
	// go test -run=XXX -bench=BenchmarkPoolReadFrom
	// BenchmarkPoolReadFrom-4   	30000000	        36.0 ns/op
	// after fixing issues/118:                         50.1 ns/op
	r := strings.NewReader("hello")
	b.ResetTimer()
	for n := 0; n < b.N; n++ {
		_, _ = poolReadFrom(r)
	}
}
<commit_msg>rm unused cooment<commit_after>package meli

import (
	"context"
	"io/ioutil"
	"strings"
	"testing"
)

func TestPullDockerImage(t *testing.T) {
	tt := []struct {
		dc          *DockerContainer
		expectedErr error
	}{
		{&DockerContainer{ComposeService: ComposeService{Image: "busybox"}, LogMedium: ioutil.Discard}, nil},
	}
	var ctx = context.Background()
	cli := &mockDockerClient{}
	for _, v := range tt {
		err := PullDockerImage(ctx, cli, v.dc)
		if err != nil {
			t.Errorf("\nCalled PullDockerImage(%#+v) \ngot %s \nwanted %#+v", v.dc, err, v.expectedErr)
		}
	}
}

func TestBuildDockerImage(t *testing.T) {
	tt := []struct {
		dc          *DockerContainer
		expected    string
		expectedErr error
	}{
		{
			&DockerContainer{
				ServiceName:       "myservicename",
				DockerComposeFile: "docker-compose.yml",
				ComposeService: ComposeService{
					Build: Buildstruct{Dockerfile: "testdata/Dockerfile"}},
				LogMedium: ioutil.Discard},
			"meli_myservicename",
			nil},
		{
			&DockerContainer{
				ServiceName:       "myservicename",
				DockerComposeFile: "docker-compose.yml",
				ComposeService: ComposeService{
					Build: Buildstruct{Dockerfile: "testdata/Dockerfile"}},
				LogMedium: ioutil.Discard,
				Rebuild:   true,
			},
			"meli_myservicename",
			nil},
	}

	var ctx = context.Background()
	cli := &mockDockerClient{}
	LoadAuth()
	for _, v := range tt {
		actual, err := BuildDockerImage(ctx, cli, v.dc)
		if err != nil {
			t.Errorf("\nCalled BuildDockerImage(%#+v) \ngot %s \nwanted %#+v", v.dc, err, v.expectedErr)
		}
		if actual != v.expected {
			t.Errorf("\nCalled BuildDockerImage(%#+v) \ngot %s \nwanted %#+v", v.dc, actual, v.expected)
		}
	}
}

func BenchmarkPullDockerImage(b *testing.B) {
	var ctx = context.Background()
	cli := &mockDockerClient{}
	dc := &DockerContainer{ComposeService: ComposeService{Image: "busybox"}, LogMedium: ioutil.Discard}
	LoadAuth()
	for n := 0; n < b.N; n++ {
		_ = PullDockerImage(ctx, cli, dc)
	}
}

func BenchmarkBuildDockerImage(b *testing.B) {
	var ctx = context.Background()
	cli := &mockDockerClient{}
	dc := &DockerContainer{
		ServiceName:       "myservicename",
		DockerComposeFile: "docker-compose.yml",
		ComposeService: ComposeService{
			Build: Buildstruct{Dockerfile: "testdata/Dockerfile"}},
		LogMedium: ioutil.Discard,
		Rebuild:   true,
	}
	LoadAuth()
	for n := 0; n < b.N; n++ {
		_, _ = BuildDockerImage(ctx, cli, dc)
	}
}

func BenchmarkPoolReadFrom(b *testing.B) {
	r := strings.NewReader("hello")
	b.ResetTimer()
	for n := 0; n < b.N; n++ {
		_, _ = poolReadFrom(r)
	}
}
<|endoftext|>


<commit_before>package goping

type ICMPRequestBuilder interface {
}

type ICMPRequestSender interface {
}
<commit_msg>Refactorung: Adding interfaces<commit_after>package goping

import (
	"net"

	"golang.org/x/net/icmp"
)

type ICMPRequest interface {
	//Protocol: The ip protocol to send the ip message  (ICMP=1, TCP=6, UDP=17)
	Protocol() uint

	//IP: The target address of this request
	IP() net.IP

	//The ICMP Message to send
	Message() icmp.Message
}

type ICMPReply interface {
}
<|endoftext|>


<commit_before>// Package buffers provides shared byte buffers based on connmux
package buffers

import (
	"github.com/getlantern/connmux"
)

const (
	maxBuffers = 5000
)

var (
	pool = connmux.NewBufferPool(maxBuffers)
)

// Pool gets the byte pool
func Pool() connmux.BufferPool {
	return pool
}

// Get gets a byte buffer from the pool
func Get() []byte {
	return pool.Get()
}

// Put returns a byte buffer to the pool
func Put(b []byte) {
	pool.Put(b)
}
<commit_msg>Switched to lampshade<commit_after>// Package buffers provides shared byte buffers based on lampshade
package buffers

import (
	"github.com/getlantern/lampshade"
)

const (
	maxBuffers = 5000
)

var (
	pool = lampshade.NewBufferPool(maxBuffers)
)

// Pool gets the byte pool
func Pool() lampshade.BufferPool {
	return pool
}

// Get gets a byte buffer from the pool
func Get() []byte {
	return pool.Get()
}

// Put returns a byte buffer to the pool
func Put(b []byte) {
	pool.Put(b)
}
<|endoftext|>


<commit_before>// Copyright 2016 someonegg. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package bufpool is a wrapper over the sync.Pool for buffer
// objects, it simplifies the use.
package bufpool

import (
	"sync"
)

var bufTypes = [...]int{
	16, 32, 48, 64, 80, 96, 112,
	128, 160, 192, 224,
	256, 320, 384, 448,
	512, 640, 768, 896,
	1024,
}

const bufTypeNum = len(bufTypes)

var bufPools [bufTypeNum]sync.Pool

func init() {
	for i := 0; i < bufTypeNum; i++ {
		l := bufTypes[i]
		bufPools[i].New = func() interface{} {
			return make([]byte, l, l)
		}
	}
}

func BufGet(size int) []byte {
	if size == 0 {
		return nil
	}

	if size <= bufTypes[bufTypeNum-1] {

		for i := 0; i < bufTypeNum; i++ {
			l := bufTypes[i]
			if size <= l {
				b := bufPools[i].Get().([]byte)
				return b[0:size]
			}
		}
	}

	return make([]byte, size, size)
}

func BufPut(b []byte) {
	size := cap(b)

	if size == 0 {
		return
	}

	if size <= bufTypes[bufTypeNum-1] {

		for i := 0; i < bufTypeNum; i++ {
			l := bufTypes[i]
			if size <= l {
				if size == l {
					bufPools[i].Put(b[0:size])
				}
				return
			}
		}
	}

	return
}
<commit_msg>simplify name<commit_after>// Copyright 2016 someonegg. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package bufpool is a wrapper over the sync.Pool for buffer
// objects, it simplifies the use.
package bufpool

import (
	"sync"
)

var bufTypes = [...]int{
	16, 32, 48, 64, 80, 96, 112,
	128, 160, 192, 224,
	256, 320, 384, 448,
	512, 640, 768, 896,
	1024,
}

const bufTypeNum = len(bufTypes)

var bufPools [bufTypeNum]sync.Pool

func init() {
	for i := 0; i < bufTypeNum; i++ {
		l := bufTypes[i]
		bufPools[i].New = func() interface{} {
			return make([]byte, l, l)
		}
	}
}

func Get(size int) []byte {
	if size == 0 {
		return nil
	}

	if size <= bufTypes[bufTypeNum-1] {

		for i := 0; i < bufTypeNum; i++ {
			l := bufTypes[i]
			if size <= l {
				b := bufPools[i].Get().([]byte)
				return b[0:size]
			}
		}
	}

	return make([]byte, size, size)
}

func Put(b []byte) {
	size := cap(b)

	if size == 0 {
		return
	}

	if size <= bufTypes[bufTypeNum-1] {

		for i := 0; i < bufTypeNum; i++ {
			l := bufTypes[i]
			if size <= l {
				if size == l {
					bufPools[i].Put(b[0:size])
				}
				return
			}
		}
	}

	return
}
<|endoftext|>


<commit_before>// Copyright 2014 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package docker

import (
	"bufio"
	"encoding/json"
	"fmt"
	"math"
	"os"
	"path"
	"strings"
	"time"

	"github.com/dotcloud/docker/nat"
	"github.com/dotcloud/docker/pkg/libcontainer/cgroups"
	"github.com/dotcloud/docker/pkg/libcontainer/cgroups/fs"
	"github.com/fsouza/go-dockerclient"
	"github.com/google/cadvisor/container"
	"github.com/google/cadvisor/info"
)

type dockerContainerHandler struct {
	client             *docker.Client
	name               string
	machineInfoFactory info.MachineInfoFactory
	container.NoStatsSummary
}

func (self *dockerContainerHandler) splitName() (string, string, error) {
	parent, id := path.Split(self.name)
	cgroupSelf, err := os.Open("/proc/self/cgroup")
	if err != nil {
		return "", "", err
	}
	scanner := bufio.NewScanner(cgroupSelf)

	subsys := []string{"memory", "cpu"}
	nestedLevels := 0
	for scanner.Scan() {
		line := scanner.Text()
		elems := strings.Split(line, ":")
		if len(elems) < 3 {
			continue
		}
		for _, s := range subsys {
			if elems[1] == s {
				// count how many nested docker containers are there.
				nestedLevels = strings.Count(elems[2], "/docker")
				break
			}
		}
	}
	if nestedLevels > 0 {
		// we are running inside a docker container
		upperLevel := strings.Repeat("../../", nestedLevels)
		//parent = strings.Join([]string{parent, upperLevel}, "/")
		parent = fmt.Sprintf("%v%v", upperLevel, parent)
	}
	return parent, id, nil
}

func (self *dockerContainerHandler) isDockerRoot() bool {
	// TODO(dengnan): Should we consider other cases?
	return self.name == "/docker"
}

func (self *dockerContainerHandler) isRootContainer() bool {
	return self.name == "/"
}

func (self *dockerContainerHandler) isDockerContainer() bool {
	return (!self.isDockerRoot()) && (!self.isRootContainer())
}

type dockerPortBinding struct {
	HostIp   string
	HostPort string
}

type dockerPort string
type dockerPortMap map[dockerPort][]dockerPortBinding

type dockerNetworkSettings struct {
	IPAddress   string
	IPPrefixLen int
	Gateway     string
	Bridge      string
	Ports       dockerPortMap
}

type dockerContainerConfig struct {
	Hostname        string
	Domainname      string
	User            string
	Memory          int64  // Memory limit (in bytes)
	MemorySwap      int64  // Total memory usage (memory + swap); set `-1' to disable swap
	CpuShares       int64  // CPU shares (relative weight vs. other containers)
	Cpuset          string // Cpuset 0-2, 0,1
	AttachStdin     bool
	AttachStdout    bool
	AttachStderr    bool
	PortSpecs       []string // Deprecated - Can be in the format of 8080/tcp
	ExposedPorts    map[nat.Port]struct{}
	Tty             bool // Attach standard streams to a tty, including stdin if it is not closed.
	OpenStdin       bool // Open stdin
	StdinOnce       bool // If true, close stdin after the 1 attached client disconnects.
	Env             []string
	Cmd             []string
	Image           string // Name of the image as it was passed by the operator (eg. could be symbolic)
	Volumes         map[string]struct{}
	WorkingDir      string
	Entrypoint      []string
	NetworkDisabled bool
	OnBuild         []string
}

type dockerState struct {
	Running    bool
	Pid        int
	ExitCode   int
	StartedAt  time.Time
	FinishedAt time.Time
}

type dockerContainerSpec struct {
	ID string

	Created time.Time

	Path string
	Args []string

	Config *dockerContainerConfig
	State  dockerState
	Image  string

	NetworkSettings *dockerNetworkSettings

	ResolvConfPath string
	HostnamePath   string
	HostsPath      string
	Name           string
	Driver         string
	ExecDriver     string

	MountLabel, ProcessLabel string

	Volumes map[string]string
	// Store rw/ro in a separate structure to preserve reverse-compatibility on-disk.
	// Easier than migrating older container configs :)
	VolumesRW map[string]bool
	// contains filtered or unexported fields
}

func readDockerSpec(id string) (spec *dockerContainerSpec, err error) {
	dir := "/var/lib/docker/containers"
	configPath := path.Join(dir, id, "config.json")
	f, err := os.Open(configPath)
	if err != nil {
		return
	}
	defer f.Close()
	d := json.NewDecoder(f)
	ret := new(dockerContainerSpec)
	err = d.Decode(ret)
	if err != nil {
		return
	}
	spec = ret
	return
}

func dockerConfigToContainerSpec(config *dockerContainerSpec, mi *info.MachineInfo) *info.ContainerSpec {
	spec := new(info.ContainerSpec)
	spec.Memory = new(info.MemorySpec)
	spec.Memory.Limit = math.MaxUint64
	spec.Memory.SwapLimit = math.MaxUint64
	if config.Config.Memory > 0 {
		spec.Memory.Limit = uint64(config.Config.Memory)
	}
	if config.Config.MemorySwap > 0 {
		spec.Memory.SwapLimit = uint64(config.Config.MemorySwap - config.Config.Memory)
	}
	if mi != nil {
		spec.Cpu = new(info.CpuSpec)
		spec.Cpu.Limit = math.MaxUint64
		n := mi.NumCores / 64
		if mi.NumCores%64 > 0 {
			n++
		}
		spec.Cpu.Mask.Data = make([]uint64, n)
		for i := 0; i < n; i++ {
			spec.Cpu.Mask.Data[i] = math.MaxUint64
		}
	}
	return spec
}

func (self *dockerContainerHandler) GetSpec() (spec *info.ContainerSpec, err error) {
	if !self.isDockerContainer() {
		spec = new(info.ContainerSpec)
		return
	}
	mi, err := self.machineInfoFactory.GetMachineInfo()
	if err != nil {
		return
	}
	_, id, err := self.splitName()
	if err != nil {
		return
	}
	dspec, err := readDockerSpec(id)
	if err != nil {
		return
	}

	spec = dockerConfigToContainerSpec(dspec, mi)
	return
}

func libcontainerToContainerStats(s *cgroups.Stats, mi *info.MachineInfo) *info.ContainerStats {
	ret := new(info.ContainerStats)
	ret.Timestamp = time.Now()
	ret.Cpu = new(info.CpuStats)
	ret.Cpu.Usage.User = s.CpuStats.CpuUsage.UsageInUsermode
	ret.Cpu.Usage.System = s.CpuStats.CpuUsage.UsageInKernelmode
	n := len(s.CpuStats.CpuUsage.PercpuUsage)
	ret.Cpu.Usage.PerCpu = make([]uint64, n)

	ret.Cpu.Usage.Total = 0
	for i := 0; i < n; i++ {
		ret.Cpu.Usage.PerCpu[i] = s.CpuStats.CpuUsage.PercpuUsage[i]
		ret.Cpu.Usage.Total += s.CpuStats.CpuUsage.PercpuUsage[i]
	}
	ret.Memory = new(info.MemoryStats)
	ret.Memory.Usage = s.MemoryStats.Usage
	if v, ok := s.MemoryStats.Stats["pgfault"]; ok {
		ret.Memory.ContainerData.Pgfault = v
		ret.Memory.HierarchicalData.Pgfault = v
	}
	if v, ok := s.MemoryStats.Stats["pgmajfault"]; ok {
		ret.Memory.ContainerData.Pgmajfault = v
		ret.Memory.HierarchicalData.Pgmajfault = v
	}
	return ret
}

func (self *dockerContainerHandler) GetStats() (stats *info.ContainerStats, err error) {
	if !self.isDockerContainer() {
		return
	}
	mi, err := self.machineInfoFactory.GetMachineInfo()
	if err != nil {
		return
	}
	parent, id, err := self.splitName()
	if err != nil {
		return
	}
	cg := &cgroups.Cgroup{
		Parent: parent,
		Name:   id,
	}
	s, err := fs.GetStats(cg)
	if err != nil {
		return
	}
	stats = libcontainerToContainerStats(s, mi)
	return
}

func (self *dockerContainerHandler) ListContainers(listType container.ListType) ([]string, error) {
	if self.isDockerContainer() {
		return nil, nil
	}
	if self.isRootContainer() && listType == container.LIST_SELF {
		return []string{"/docker"}, nil
	}
	opt := docker.ListContainersOptions{
		All: true,
	}
	containers, err := self.client.ListContainers(opt)
	if err != nil {
		return nil, err
	}
	ret := make([]string, 0, len(containers)+1)
	for _, c := range containers {
		if !strings.HasPrefix(c.Status, "Up ") {
			continue
		}
		path := fmt.Sprintf("/docker/%v", c.ID)
		ret = append(ret, path)
	}
	if self.isRootContainer() {
		ret = append(ret, "/docker")
	}
	return ret, nil
}

func (self *dockerContainerHandler) ListThreads(listType container.ListType) ([]int, error) {
	return nil, nil
}

func (self *dockerContainerHandler) ListProcesses(listType container.ListType) ([]int, error) {
	return nil, nil
}
<commit_msg>Move to using the libcontainer repo directly.<commit_after>// Copyright 2014 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package docker

import (
	"bufio"
	"encoding/json"
	"fmt"
	"math"
	"os"
	"path"
	"strings"
	"time"

	"github.com/docker/libcontainer/cgroups"
	"github.com/docker/libcontainer/cgroups/fs"
	"github.com/dotcloud/docker/nat"
	"github.com/fsouza/go-dockerclient"
	"github.com/google/cadvisor/container"
	"github.com/google/cadvisor/info"
)

type dockerContainerHandler struct {
	client             *docker.Client
	name               string
	machineInfoFactory info.MachineInfoFactory
	container.NoStatsSummary
}

func (self *dockerContainerHandler) splitName() (string, string, error) {
	parent, id := path.Split(self.name)
	cgroupSelf, err := os.Open("/proc/self/cgroup")
	if err != nil {
		return "", "", err
	}
	scanner := bufio.NewScanner(cgroupSelf)

	subsys := []string{"memory", "cpu"}
	nestedLevels := 0
	for scanner.Scan() {
		line := scanner.Text()
		elems := strings.Split(line, ":")
		if len(elems) < 3 {
			continue
		}
		for _, s := range subsys {
			if elems[1] == s {
				// count how many nested docker containers are there.
				nestedLevels = strings.Count(elems[2], "/docker")
				break
			}
		}
	}
	if nestedLevels > 0 {
		// we are running inside a docker container
		upperLevel := strings.Repeat("../../", nestedLevels)
		//parent = strings.Join([]string{parent, upperLevel}, "/")
		parent = fmt.Sprintf("%v%v", upperLevel, parent)
	}
	return parent, id, nil
}

func (self *dockerContainerHandler) isDockerRoot() bool {
	// TODO(dengnan): Should we consider other cases?
	return self.name == "/docker"
}

func (self *dockerContainerHandler) isRootContainer() bool {
	return self.name == "/"
}

func (self *dockerContainerHandler) isDockerContainer() bool {
	return (!self.isDockerRoot()) && (!self.isRootContainer())
}

type dockerPortBinding struct {
	HostIp   string
	HostPort string
}

type dockerPort string
type dockerPortMap map[dockerPort][]dockerPortBinding

type dockerNetworkSettings struct {
	IPAddress   string
	IPPrefixLen int
	Gateway     string
	Bridge      string
	Ports       dockerPortMap
}

type dockerContainerConfig struct {
	Hostname        string
	Domainname      string
	User            string
	Memory          int64  // Memory limit (in bytes)
	MemorySwap      int64  // Total memory usage (memory + swap); set `-1' to disable swap
	CpuShares       int64  // CPU shares (relative weight vs. other containers)
	Cpuset          string // Cpuset 0-2, 0,1
	AttachStdin     bool
	AttachStdout    bool
	AttachStderr    bool
	PortSpecs       []string // Deprecated - Can be in the format of 8080/tcp
	ExposedPorts    map[nat.Port]struct{}
	Tty             bool // Attach standard streams to a tty, including stdin if it is not closed.
	OpenStdin       bool // Open stdin
	StdinOnce       bool // If true, close stdin after the 1 attached client disconnects.
	Env             []string
	Cmd             []string
	Image           string // Name of the image as it was passed by the operator (eg. could be symbolic)
	Volumes         map[string]struct{}
	WorkingDir      string
	Entrypoint      []string
	NetworkDisabled bool
	OnBuild         []string
}

type dockerState struct {
	Running    bool
	Pid        int
	ExitCode   int
	StartedAt  time.Time
	FinishedAt time.Time
}

type dockerContainerSpec struct {
	ID string

	Created time.Time

	Path string
	Args []string

	Config *dockerContainerConfig
	State  dockerState
	Image  string

	NetworkSettings *dockerNetworkSettings

	ResolvConfPath string
	HostnamePath   string
	HostsPath      string
	Name           string
	Driver         string
	ExecDriver     string

	MountLabel, ProcessLabel string

	Volumes map[string]string
	// Store rw/ro in a separate structure to preserve reverse-compatibility on-disk.
	// Easier than migrating older container configs :)
	VolumesRW map[string]bool
	// contains filtered or unexported fields
}

func readDockerSpec(id string) (spec *dockerContainerSpec, err error) {
	dir := "/var/lib/docker/containers"
	configPath := path.Join(dir, id, "config.json")
	f, err := os.Open(configPath)
	if err != nil {
		return
	}
	defer f.Close()
	d := json.NewDecoder(f)
	ret := new(dockerContainerSpec)
	err = d.Decode(ret)
	if err != nil {
		return
	}
	spec = ret
	return
}

func dockerConfigToContainerSpec(config *dockerContainerSpec, mi *info.MachineInfo) *info.ContainerSpec {
	spec := new(info.ContainerSpec)
	spec.Memory = new(info.MemorySpec)
	spec.Memory.Limit = math.MaxUint64
	spec.Memory.SwapLimit = math.MaxUint64
	if config.Config.Memory > 0 {
		spec.Memory.Limit = uint64(config.Config.Memory)
	}
	if config.Config.MemorySwap > 0 {
		spec.Memory.SwapLimit = uint64(config.Config.MemorySwap - config.Config.Memory)
	}
	if mi != nil {
		spec.Cpu = new(info.CpuSpec)
		spec.Cpu.Limit = math.MaxUint64
		n := mi.NumCores / 64
		if mi.NumCores%64 > 0 {
			n++
		}
		spec.Cpu.Mask.Data = make([]uint64, n)
		for i := 0; i < n; i++ {
			spec.Cpu.Mask.Data[i] = math.MaxUint64
		}
	}
	return spec
}

func (self *dockerContainerHandler) GetSpec() (spec *info.ContainerSpec, err error) {
	if !self.isDockerContainer() {
		spec = new(info.ContainerSpec)
		return
	}
	mi, err := self.machineInfoFactory.GetMachineInfo()
	if err != nil {
		return
	}
	_, id, err := self.splitName()
	if err != nil {
		return
	}
	dspec, err := readDockerSpec(id)
	if err != nil {
		return
	}

	spec = dockerConfigToContainerSpec(dspec, mi)
	return
}

func libcontainerToContainerStats(s *cgroups.Stats, mi *info.MachineInfo) *info.ContainerStats {
	ret := new(info.ContainerStats)
	ret.Timestamp = time.Now()
	ret.Cpu = new(info.CpuStats)
	ret.Cpu.Usage.User = s.CpuStats.CpuUsage.UsageInUsermode
	ret.Cpu.Usage.System = s.CpuStats.CpuUsage.UsageInKernelmode
	n := len(s.CpuStats.CpuUsage.PercpuUsage)
	ret.Cpu.Usage.PerCpu = make([]uint64, n)

	ret.Cpu.Usage.Total = 0
	for i := 0; i < n; i++ {
		ret.Cpu.Usage.PerCpu[i] = s.CpuStats.CpuUsage.PercpuUsage[i]
		ret.Cpu.Usage.Total += s.CpuStats.CpuUsage.PercpuUsage[i]
	}
	ret.Memory = new(info.MemoryStats)
	ret.Memory.Usage = s.MemoryStats.Usage
	if v, ok := s.MemoryStats.Stats["pgfault"]; ok {
		ret.Memory.ContainerData.Pgfault = v
		ret.Memory.HierarchicalData.Pgfault = v
	}
	if v, ok := s.MemoryStats.Stats["pgmajfault"]; ok {
		ret.Memory.ContainerData.Pgmajfault = v
		ret.Memory.HierarchicalData.Pgmajfault = v
	}
	return ret
}

func (self *dockerContainerHandler) GetStats() (stats *info.ContainerStats, err error) {
	if !self.isDockerContainer() {
		// Return empty stats for root containers.
		stats = new(info.ContainerStats)
		stats.Timestamp = time.Now()
		return
	}
	mi, err := self.machineInfoFactory.GetMachineInfo()
	if err != nil {
		return
	}
	parent, id, err := self.splitName()
	if err != nil {
		return
	}
	cg := &cgroups.Cgroup{
		Parent: parent,
		Name:   id,
	}
	s, err := fs.GetStats(cg)
	if err != nil {
		return
	}
	stats = libcontainerToContainerStats(s, mi)
	return
}

func (self *dockerContainerHandler) ListContainers(listType container.ListType) ([]string, error) {
	if self.isDockerContainer() {
		return nil, nil
	}
	if self.isRootContainer() && listType == container.LIST_SELF {
		return []string{"/docker"}, nil
	}
	opt := docker.ListContainersOptions{
		All: true,
	}
	containers, err := self.client.ListContainers(opt)
	if err != nil {
		return nil, err
	}
	ret := make([]string, 0, len(containers)+1)
	for _, c := range containers {
		if !strings.HasPrefix(c.Status, "Up ") {
			continue
		}
		path := fmt.Sprintf("/docker/%v", c.ID)
		ret = append(ret, path)
	}
	if self.isRootContainer() {
		ret = append(ret, "/docker")
	}
	return ret, nil
}

func (self *dockerContainerHandler) ListThreads(listType container.ListType) ([]int, error) {
	return nil, nil
}

func (self *dockerContainerHandler) ListProcesses(listType container.ListType) ([]int, error) {
	return nil, nil
}
<|endoftext|>


<commit_before>// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package tar

import (
	"bytes"
	"fmt"
	"strconv"
	"strings"
	"time"
)

// hasNUL reports whether the NUL character exists within s.
func hasNUL(s string) bool {
	return strings.IndexByte(s, 0) >= 0
}

// isASCII reports whether the input is an ASCII C-style string.
func isASCII(s string) bool {
	for _, c := range s {
		if c >= 0x80 || c == 0x00 {
			return false
		}
	}
	return true
}

// toASCII converts the input to an ASCII C-style string.
// This a best effort conversion, so invalid characters are dropped.
func toASCII(s string) string {
	if isASCII(s) {
		return s
	}
	b := make([]byte, 0, len(s))
	for _, c := range s {
		if c < 0x80 && c != 0x00 {
			b = append(b, byte(c))
		}
	}
	return string(b)
}

type parser struct {
	err error // Last error seen
}

type formatter struct {
	err error // Last error seen
}

// parseString parses bytes as a NUL-terminated C-style string.
// If a NUL byte is not found then the whole slice is returned as a string.
func (*parser) parseString(b []byte) string {
	if i := bytes.IndexByte(b, 0); i >= 0 {
		return string(b[:i])
	}
	return string(b)
}

// formatString copies s into b, NUL-terminating if possible.
func (f *formatter) formatString(b []byte, s string) {
	if len(s) > len(b) {
		f.err = ErrFieldTooLong
	}
	copy(b, s)
	if len(s) < len(b) {
		b[len(s)] = 0
	}

	// Some buggy readers treat regular files with a trailing slash
	// in the V7 path field as a directory even though the full path
	// recorded elsewhere (e.g., via PAX record) contains no trailing slash.
	if len(s) > len(b) && b[len(b)-1] == '/' {
		n := len(strings.TrimRight(s[:len(b)], "/"))
		b[n] = 0 // Replace trailing slash with NUL terminator
	}
}

// fitsInBase256 reports whether x can be encoded into n bytes using base-256
// encoding. Unlike octal encoding, base-256 encoding does not require that the
// string ends with a NUL character. Thus, all n bytes are available for output.
//
// If operating in binary mode, this assumes strict GNU binary mode; which means
// that the first byte can only be either 0x80 or 0xff. Thus, the first byte is
// equivalent to the sign bit in two's complement form.
func fitsInBase256(n int, x int64) bool {
	binBits := uint(n-1) * 8
	return n >= 9 || (x >= -1<<binBits && x < 1<<binBits)
}

// parseNumeric parses the input as being encoded in either base-256 or octal.
// This function may return negative numbers.
// If parsing fails or an integer overflow occurs, err will be set.
func (p *parser) parseNumeric(b []byte) int64 {
	// Check for base-256 (binary) format first.
	// If the first bit is set, then all following bits constitute a two's
	// complement encoded number in big-endian byte order.
	if len(b) > 0 && b[0]&0x80 != 0 {
		// Handling negative numbers relies on the following identity:
		//	-a-1 == ^a
		//
		// If the number is negative, we use an inversion mask to invert the
		// data bytes and treat the value as an unsigned number.
		var inv byte // 0x00 if positive or zero, 0xff if negative
		if b[0]&0x40 != 0 {
			inv = 0xff
		}

		var x uint64
		for i, c := range b {
			c ^= inv // Inverts c only if inv is 0xff, otherwise does nothing
			if i == 0 {
				c &= 0x7f // Ignore signal bit in first byte
			}
			if (x >> 56) > 0 {
				p.err = ErrHeader // Integer overflow
				return 0
			}
			x = x<<8 | uint64(c)
		}
		if (x >> 63) > 0 {
			p.err = ErrHeader // Integer overflow
			return 0
		}
		if inv == 0xff {
			return ^int64(x)
		}
		return int64(x)
	}

	// Normal case is base-8 (octal) format.
	return p.parseOctal(b)
}

// formatNumeric encodes x into b using base-8 (octal) encoding if possible.
// Otherwise it will attempt to use base-256 (binary) encoding.
func (f *formatter) formatNumeric(b []byte, x int64) {
	if fitsInOctal(len(b), x) {
		f.formatOctal(b, x)
		return
	}

	if fitsInBase256(len(b), x) {
		for i := len(b) - 1; i >= 0; i-- {
			b[i] = byte(x)
			x >>= 8
		}
		b[0] |= 0x80 // Highest bit indicates binary format
		return
	}

	f.formatOctal(b, 0) // Last resort, just write zero
	f.err = ErrFieldTooLong
}

func (p *parser) parseOctal(b []byte) int64 {
	// Because unused fields are filled with NULs, we need
	// to skip leading NULs. Fields may also be padded with
	// spaces or NULs.
	// So we remove leading and trailing NULs and spaces to
	// be sure.
	b = bytes.Trim(b, " \x00")

	if len(b) == 0 {
		return 0
	}
	x, perr := strconv.ParseUint(p.parseString(b), 8, 64)
	if perr != nil {
		p.err = ErrHeader
	}
	return int64(x)
}

func (f *formatter) formatOctal(b []byte, x int64) {
	if !fitsInOctal(len(b), x) {
		x = 0 // Last resort, just write zero
		f.err = ErrFieldTooLong
	}

	s := strconv.FormatInt(x, 8)
	// Add leading zeros, but leave room for a NUL.
	if n := len(b) - len(s) - 1; n > 0 {
		s = strings.Repeat("0", n) + s
	}
	f.formatString(b, s)
}

// fitsInOctal reports whether the integer x fits in a field n-bytes long
// using octal encoding with the appropriate NUL terminator.
func fitsInOctal(n int, x int64) bool {
	octBits := uint(n-1) * 3
	return x >= 0 && (n >= 22 || x < 1<<octBits)
}

// parsePAXTime takes a string of the form %d.%d as described in the PAX
// specification. Note that this implementation allows for negative timestamps,
// which is allowed for by the PAX specification, but not always portable.
func parsePAXTime(s string) (time.Time, error) {
	const maxNanoSecondDigits = 9

	// Split string into seconds and sub-seconds parts.
	ss, sn := s, ""
	if pos := strings.IndexByte(s, '.'); pos >= 0 {
		ss, sn = s[:pos], s[pos+1:]
	}

	// Parse the seconds.
	secs, err := strconv.ParseInt(ss, 10, 64)
	if err != nil {
		return time.Time{}, ErrHeader
	}
	if len(sn) == 0 {
		return time.Unix(secs, 0), nil // No sub-second values
	}

	// Parse the nanoseconds.
	if strings.Trim(sn, "0123456789") != "" {
		return time.Time{}, ErrHeader
	}
	if len(sn) < maxNanoSecondDigits {
		sn += strings.Repeat("0", maxNanoSecondDigits-len(sn)) // Right pad
	} else {
		sn = sn[:maxNanoSecondDigits] // Right truncate
	}
	nsecs, _ := strconv.ParseInt(sn, 10, 64) // Must succeed
	if len(ss) > 0 && ss[0] == '-' {
		return time.Unix(secs, -1*nsecs), nil // Negative correction
	}
	return time.Unix(secs, nsecs), nil
}

// formatPAXTime converts ts into a time of the form %d.%d as described in the
// PAX specification. This function is capable of negative timestamps.
func formatPAXTime(ts time.Time) (s string) {
	secs, nsecs := ts.Unix(), ts.Nanosecond()
	if nsecs == 0 {
		return strconv.FormatInt(secs, 10)
	}

	// If seconds is negative, then perform correction.
	sign := ""
	if secs < 0 {
		sign = "-"             // Remember sign
		secs = -(secs + 1)     // Add a second to secs
		nsecs = -(nsecs - 1e9) // Take that second away from nsecs
	}
	return strings.TrimRight(fmt.Sprintf("%s%d.%09d", sign, secs, nsecs), "0")
}

// parsePAXRecord parses the input PAX record string into a key-value pair.
// If parsing is successful, it will slice off the currently read record and
// return the remainder as r.
func parsePAXRecord(s string) (k, v, r string, err error) {
	// The size field ends at the first space.
	sp := strings.IndexByte(s, ' ')
	if sp == -1 {
		return "", "", s, ErrHeader
	}

	// Parse the first token as a decimal integer.
	n, perr := strconv.ParseInt(s[:sp], 10, 0) // Intentionally parse as native int
	if perr != nil || n < 5 || int64(len(s)) < n {
		return "", "", s, ErrHeader
	}

	// Extract everything between the space and the final newline.
	rec, nl, rem := s[sp+1:n-1], s[n-1:n], s[n:]
	if nl != "\n" {
		return "", "", s, ErrHeader
	}

	// The first equals separates the key from the value.
	eq := strings.IndexByte(rec, '=')
	if eq == -1 {
		return "", "", s, ErrHeader
	}
	k, v = rec[:eq], rec[eq+1:]

	if !validPAXRecord(k, v) {
		return "", "", s, ErrHeader
	}
	return k, v, rem, nil
}

// formatPAXRecord formats a single PAX record, prefixing it with the
// appropriate length.
func formatPAXRecord(k, v string) (string, error) {
	if !validPAXRecord(k, v) {
		return "", ErrHeader
	}

	const padding = 3 // Extra padding for ' ', '=', and '\n'
	size := len(k) + len(v) + padding
	size += len(strconv.Itoa(size))
	record := strconv.Itoa(size) + " " + k + "=" + v + "\n"

	// Final adjustment if adding size field increased the record size.
	if len(record) != size {
		size = len(record)
		record = strconv.Itoa(size) + " " + k + "=" + v + "\n"
	}
	return record, nil
}

// validPAXRecord reports whether the key-value pair is valid where each
// record is formatted as:
//	"%d %s=%s\n" % (size, key, value)
//
// Keys and values should be UTF-8, but the number of bad writers out there
// forces us to be a more liberal.
// Thus, we only reject all keys with NUL, and only reject NULs in values
// for the PAX version of the USTAR string fields.
// The key must not contain an '=' character.
func validPAXRecord(k, v string) bool {
	if k == "" || strings.IndexByte(k, '=') >= 0 {
		return false
	}
	switch k {
	case paxPath, paxLinkpath, paxUname, paxGname:
		return !hasNUL(v)
	default:
		return !hasNUL(k)
	}
}
<commit_msg>archive/tar: fix typo in comment<commit_after>// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package tar

import (
	"bytes"
	"fmt"
	"strconv"
	"strings"
	"time"
)

// hasNUL reports whether the NUL character exists within s.
func hasNUL(s string) bool {
	return strings.IndexByte(s, 0) >= 0
}

// isASCII reports whether the input is an ASCII C-style string.
func isASCII(s string) bool {
	for _, c := range s {
		if c >= 0x80 || c == 0x00 {
			return false
		}
	}
	return true
}

// toASCII converts the input to an ASCII C-style string.
// This is a best effort conversion, so invalid characters are dropped.
func toASCII(s string) string {
	if isASCII(s) {
		return s
	}
	b := make([]byte, 0, len(s))
	for _, c := range s {
		if c < 0x80 && c != 0x00 {
			b = append(b, byte(c))
		}
	}
	return string(b)
}

type parser struct {
	err error // Last error seen
}

type formatter struct {
	err error // Last error seen
}

// parseString parses bytes as a NUL-terminated C-style string.
// If a NUL byte is not found then the whole slice is returned as a string.
func (*parser) parseString(b []byte) string {
	if i := bytes.IndexByte(b, 0); i >= 0 {
		return string(b[:i])
	}
	return string(b)
}

// formatString copies s into b, NUL-terminating if possible.
func (f *formatter) formatString(b []byte, s string) {
	if len(s) > len(b) {
		f.err = ErrFieldTooLong
	}
	copy(b, s)
	if len(s) < len(b) {
		b[len(s)] = 0
	}

	// Some buggy readers treat regular files with a trailing slash
	// in the V7 path field as a directory even though the full path
	// recorded elsewhere (e.g., via PAX record) contains no trailing slash.
	if len(s) > len(b) && b[len(b)-1] == '/' {
		n := len(strings.TrimRight(s[:len(b)], "/"))
		b[n] = 0 // Replace trailing slash with NUL terminator
	}
}

// fitsInBase256 reports whether x can be encoded into n bytes using base-256
// encoding. Unlike octal encoding, base-256 encoding does not require that the
// string ends with a NUL character. Thus, all n bytes are available for output.
//
// If operating in binary mode, this assumes strict GNU binary mode; which means
// that the first byte can only be either 0x80 or 0xff. Thus, the first byte is
// equivalent to the sign bit in two's complement form.
func fitsInBase256(n int, x int64) bool {
	binBits := uint(n-1) * 8
	return n >= 9 || (x >= -1<<binBits && x < 1<<binBits)
}

// parseNumeric parses the input as being encoded in either base-256 or octal.
// This function may return negative numbers.
// If parsing fails or an integer overflow occurs, err will be set.
func (p *parser) parseNumeric(b []byte) int64 {
	// Check for base-256 (binary) format first.
	// If the first bit is set, then all following bits constitute a two's
	// complement encoded number in big-endian byte order.
	if len(b) > 0 && b[0]&0x80 != 0 {
		// Handling negative numbers relies on the following identity:
		//	-a-1 == ^a
		//
		// If the number is negative, we use an inversion mask to invert the
		// data bytes and treat the value as an unsigned number.
		var inv byte // 0x00 if positive or zero, 0xff if negative
		if b[0]&0x40 != 0 {
			inv = 0xff
		}

		var x uint64
		for i, c := range b {
			c ^= inv // Inverts c only if inv is 0xff, otherwise does nothing
			if i == 0 {
				c &= 0x7f // Ignore signal bit in first byte
			}
			if (x >> 56) > 0 {
				p.err = ErrHeader // Integer overflow
				return 0
			}
			x = x<<8 | uint64(c)
		}
		if (x >> 63) > 0 {
			p.err = ErrHeader // Integer overflow
			return 0
		}
		if inv == 0xff {
			return ^int64(x)
		}
		return int64(x)
	}

	// Normal case is base-8 (octal) format.
	return p.parseOctal(b)
}

// formatNumeric encodes x into b using base-8 (octal) encoding if possible.
// Otherwise it will attempt to use base-256 (binary) encoding.
func (f *formatter) formatNumeric(b []byte, x int64) {
	if fitsInOctal(len(b), x) {
		f.formatOctal(b, x)
		return
	}

	if fitsInBase256(len(b), x) {
		for i := len(b) - 1; i >= 0; i-- {
			b[i] = byte(x)
			x >>= 8
		}
		b[0] |= 0x80 // Highest bit indicates binary format
		return
	}

	f.formatOctal(b, 0) // Last resort, just write zero
	f.err = ErrFieldTooLong
}

func (p *parser) parseOctal(b []byte) int64 {
	// Because unused fields are filled with NULs, we need
	// to skip leading NULs. Fields may also be padded with
	// spaces or NULs.
	// So we remove leading and trailing NULs and spaces to
	// be sure.
	b = bytes.Trim(b, " \x00")

	if len(b) == 0 {
		return 0
	}
	x, perr := strconv.ParseUint(p.parseString(b), 8, 64)
	if perr != nil {
		p.err = ErrHeader
	}
	return int64(x)
}

func (f *formatter) formatOctal(b []byte, x int64) {
	if !fitsInOctal(len(b), x) {
		x = 0 // Last resort, just write zero
		f.err = ErrFieldTooLong
	}

	s := strconv.FormatInt(x, 8)
	// Add leading zeros, but leave room for a NUL.
	if n := len(b) - len(s) - 1; n > 0 {
		s = strings.Repeat("0", n) + s
	}
	f.formatString(b, s)
}

// fitsInOctal reports whether the integer x fits in a field n-bytes long
// using octal encoding with the appropriate NUL terminator.
func fitsInOctal(n int, x int64) bool {
	octBits := uint(n-1) * 3
	return x >= 0 && (n >= 22 || x < 1<<octBits)
}

// parsePAXTime takes a string of the form %d.%d as described in the PAX
// specification. Note that this implementation allows for negative timestamps,
// which is allowed for by the PAX specification, but not always portable.
func parsePAXTime(s string) (time.Time, error) {
	const maxNanoSecondDigits = 9

	// Split string into seconds and sub-seconds parts.
	ss, sn := s, ""
	if pos := strings.IndexByte(s, '.'); pos >= 0 {
		ss, sn = s[:pos], s[pos+1:]
	}

	// Parse the seconds.
	secs, err := strconv.ParseInt(ss, 10, 64)
	if err != nil {
		return time.Time{}, ErrHeader
	}
	if len(sn) == 0 {
		return time.Unix(secs, 0), nil // No sub-second values
	}

	// Parse the nanoseconds.
	if strings.Trim(sn, "0123456789") != "" {
		return time.Time{}, ErrHeader
	}
	if len(sn) < maxNanoSecondDigits {
		sn += strings.Repeat("0", maxNanoSecondDigits-len(sn)) // Right pad
	} else {
		sn = sn[:maxNanoSecondDigits] // Right truncate
	}
	nsecs, _ := strconv.ParseInt(sn, 10, 64) // Must succeed
	if len(ss) > 0 && ss[0] == '-' {
		return time.Unix(secs, -1*nsecs), nil // Negative correction
	}
	return time.Unix(secs, nsecs), nil
}

// formatPAXTime converts ts into a time of the form %d.%d as described in the
// PAX specification. This function is capable of negative timestamps.
func formatPAXTime(ts time.Time) (s string) {
	secs, nsecs := ts.Unix(), ts.Nanosecond()
	if nsecs == 0 {
		return strconv.FormatInt(secs, 10)
	}

	// If seconds is negative, then perform correction.
	sign := ""
	if secs < 0 {
		sign = "-"             // Remember sign
		secs = -(secs + 1)     // Add a second to secs
		nsecs = -(nsecs - 1e9) // Take that second away from nsecs
	}
	return strings.TrimRight(fmt.Sprintf("%s%d.%09d", sign, secs, nsecs), "0")
}

// parsePAXRecord parses the input PAX record string into a key-value pair.
// If parsing is successful, it will slice off the currently read record and
// return the remainder as r.
func parsePAXRecord(s string) (k, v, r string, err error) {
	// The size field ends at the first space.
	sp := strings.IndexByte(s, ' ')
	if sp == -1 {
		return "", "", s, ErrHeader
	}

	// Parse the first token as a decimal integer.
	n, perr := strconv.ParseInt(s[:sp], 10, 0) // Intentionally parse as native int
	if perr != nil || n < 5 || int64(len(s)) < n {
		return "", "", s, ErrHeader
	}

	// Extract everything between the space and the final newline.
	rec, nl, rem := s[sp+1:n-1], s[n-1:n], s[n:]
	if nl != "\n" {
		return "", "", s, ErrHeader
	}

	// The first equals separates the key from the value.
	eq := strings.IndexByte(rec, '=')
	if eq == -1 {
		return "", "", s, ErrHeader
	}
	k, v = rec[:eq], rec[eq+1:]

	if !validPAXRecord(k, v) {
		return "", "", s, ErrHeader
	}
	return k, v, rem, nil
}

// formatPAXRecord formats a single PAX record, prefixing it with the
// appropriate length.
func formatPAXRecord(k, v string) (string, error) {
	if !validPAXRecord(k, v) {
		return "", ErrHeader
	}

	const padding = 3 // Extra padding for ' ', '=', and '\n'
	size := len(k) + len(v) + padding
	size += len(strconv.Itoa(size))
	record := strconv.Itoa(size) + " " + k + "=" + v + "\n"

	// Final adjustment if adding size field increased the record size.
	if len(record) != size {
		size = len(record)
		record = strconv.Itoa(size) + " " + k + "=" + v + "\n"
	}
	return record, nil
}

// validPAXRecord reports whether the key-value pair is valid where each
// record is formatted as:
//	"%d %s=%s\n" % (size, key, value)
//
// Keys and values should be UTF-8, but the number of bad writers out there
// forces us to be a more liberal.
// Thus, we only reject all keys with NUL, and only reject NULs in values
// for the PAX version of the USTAR string fields.
// The key must not contain an '=' character.
func validPAXRecord(k, v string) bool {
	if k == "" || strings.IndexByte(k, '=') >= 0 {
		return false
	}
	switch k {
	case paxPath, paxLinkpath, paxUname, paxGname:
		return !hasNUL(v)
	default:
		return !hasNUL(k)
	}
}
<|endoftext|>


<commit_before>// @author       sigu-399
// @description  An implementation of JSON Schema, draft v4 - Go language
// @created      27-02-2013

package gojsonschema

import (
	"gojsonreference"
)

type JsonSchema struct {
	id          *string
	title       *string
	description *string

	ref *gojsonreference.JsonReference

	definitionsChildren []*JsonSchema
	itemsChild          *JsonSchema
	propertiesChildren  []*JsonSchema

	parent *JsonSchema

	property *string

	schema *gojsonreference.JsonReference
}

func (s *JsonSchema) AddDefinitionChild(child *JsonSchema) {
	s.definitionsChildren = append(s.definitionsChildren, child)
}

func (s *JsonSchema) SetItemsChild(child *JsonSchema) {
	s.itemsChild = child
}

func (s *JsonSchema) AddPropertiesChild(child *JsonSchema) {
	s.propertiesChildren = append(s.propertiesChildren, child)
}
<commit_msg>added schema type (etype)<commit_after>// @author       sigu-399
// @description  An implementation of JSON Schema, draft v4 - Go language
// @created      27-02-2013

package gojsonschema

import (
	"gojsonreference"
)

type JsonSchema struct {
	id          *string
	title       *string
	description *string
	etype       *string

	ref *gojsonreference.JsonReference

	definitionsChildren []*JsonSchema
	itemsChild          *JsonSchema
	propertiesChildren  []*JsonSchema

	parent *JsonSchema

	property *string

	schema *gojsonreference.JsonReference
}

func (s *JsonSchema) AddDefinitionChild(child *JsonSchema) {
	s.definitionsChildren = append(s.definitionsChildren, child)
}

func (s *JsonSchema) SetItemsChild(child *JsonSchema) {
	s.itemsChild = child
}

func (s *JsonSchema) AddPropertiesChild(child *JsonSchema) {
	s.propertiesChildren = append(s.propertiesChildren, child)
}
<|endoftext|>


<commit_before>package buildbox

import (
  "github.com/Sirupsen/logrus"
  "bytes"
  "strings"
  "fmt"
  "time"
)

const (
  nocolor = 0
  red = 31
  green = 32
  yellow = 33
  blue = 34
)

var Logger = initLogger()

type LogFormatter struct {

}

type LoggerFields logrus.Fields;

func (f *LogFormatter) Format(entry *logrus.Entry) ([]byte, error) {
  b := &bytes.Buffer{}

  // Start by printing the time
  now := time.Now()
  fmt.Fprintf(b, "%s ", now.Format("2006-01-02 15:04:05"))

  // Upper case the log level
  levelText := strings.ToUpper(entry.Data["level"].(string))

  // Print the log level, but toggle the color
  if logrus.IsTerminal() {
    levelColor := blue

    if entry.Data["level"] == "debug" {
      levelColor = green
    } else if entry.Data["level"] == "warning" {
      levelColor = yellow
    } else if entry.Data["level"] == "error" || entry.Data["level"] == "fatal" || entry.Data["level"] == "panic" {
      levelColor = red
    }

    fmt.Fprintf(b, "\x1b[%dm[%-5s]\x1b[0m ", levelColor, levelText)
  } else {
    fmt.Fprintf(b, "[%-5s] ", levelText)
  }

  // Now print the message
  fmt.Fprintf(b, "%s ", entry.Data["msg"])

  // Print any extra data. By default, the data map has 3
  // elements.
  if len(entry.Data) > 3 {
    keys := make([]string, 0)
    for key, value := range entry.Data {
      if key != "time" && key != "level" && key != "msg" {
        if _, ok := value.(string); ok {
          keys = append(keys, fmt.Sprintf("%v: %s", key, value))
        } else {
          keys = append(keys, fmt.Sprintf("%v: %v", key, value))
        }
      }
    }

    fmt.Fprintf(b, "(%s)", strings.Join(keys, " "))
  }

  b.WriteByte('\n')

  return b.Bytes(), nil
}

func initLogger() *logrus.Logger {
  // Create a new instance of the logrus logging library
  var log = logrus.New()

  // Use our custom log formatter
  log.Formatter = new(LogFormatter)

  return log
}

func LoggerInitDebug() {
  // Enable debugging
  Logger.Level = logrus.Debug
  Logger.Debug("Debugging enabled")
}
<commit_msg>Tweaked the logging colors.<commit_after>package buildbox

import (
  "github.com/Sirupsen/logrus"
  "bytes"
  "strings"
  "fmt"
  "time"
)

const (
  nocolor = 0
  red = 31
  green = 32
  yellow = 33
  blue = 34
)

var Logger = initLogger()

type LogFormatter struct {

}

type LoggerFields logrus.Fields;

func (f *LogFormatter) Format(entry *logrus.Entry) ([]byte, error) {
  b := &bytes.Buffer{}

  // Start by printing the time
  now := time.Now()
  fmt.Fprintf(b, "%s ", now.Format("2006-01-02 15:04:05"))

  // Upper case the log level
  levelText := strings.ToUpper(entry.Data["level"].(string))

  // Print the log level, but toggle the color
  if logrus.IsTerminal() {
    levelColor := green

    if entry.Data["level"] == "debug" {
      levelColor = blue
    } else if entry.Data["level"] == "warning" {
      levelColor = yellow
    } else if entry.Data["level"] == "error" || entry.Data["level"] == "fatal" || entry.Data["level"] == "panic" {
      levelColor = red
    }

    fmt.Fprintf(b, "\x1b[%dm[%-5s]\x1b[0m ", levelColor, levelText)
  } else {
    fmt.Fprintf(b, "[%-5s] ", levelText)
  }

  // Now print the message
  fmt.Fprintf(b, "%s ", entry.Data["msg"])

  // Print any extra data. By default, the data map has 3
  // elements.
  if len(entry.Data) > 3 {
    keys := make([]string, 0)
    for key, value := range entry.Data {
      if key != "time" && key != "level" && key != "msg" {
        if _, ok := value.(string); ok {
          keys = append(keys, fmt.Sprintf("%v: %s", key, value))
        } else {
          keys = append(keys, fmt.Sprintf("%v: %v", key, value))
        }
      }
    }

    fmt.Fprintf(b, "(%s)", strings.Join(keys, " "))
  }

  b.WriteByte('\n')

  return b.Bytes(), nil
}

func initLogger() *logrus.Logger {
  // Create a new instance of the logrus logging library
  var log = logrus.New()

  // Use our custom log formatter
  log.Formatter = new(LogFormatter)

  return log
}

func LoggerInitDebug() {
  // Enable debugging
  Logger.Level = logrus.Debug
  Logger.Debug("Debugging enabled")
}
<|endoftext|>


<commit_before>package cache

import (
	"fmt"
	"time"

	"go.etcd.io/bbolt"

	log "github.com/Sirupsen/logrus"
)

type oldCacheItem struct {
	Translation string           `json:"t"`
	ErrorCode   translationError `json:"c,omitempty"`
	ErrorText   string           `json:"e,omitempty"`
	Timestamp   int64            `json:"u,omitempty"`
}

const latestVersion = 1

func (c *Cache) migrateDatabase() error {
	if c.meta.Version == latestVersion {
		return nil
	}

	log.Infof("Need to run %d database migrations", latestVersion-c.meta.Version)

	for c.meta.Version < latestVersion {
		c.migrate(c.meta.Version)
	}

	return nil
}

func (c *Cache) migrate(ver int) {
	start := time.Now()

	tgt := ver + 1

	log.Info("Running migration ", tgt)

	var err error

	switch tgt {
	case 1:
		err = c.migration1()
	}

	log := log.WithFields(log.Fields{
		"time": time.Since(start),
		"ver":  ver,
		"tgt":  tgt,
	})

	if err != nil {
		log.Fatal("Migration failed: ", err)
	}

	c.meta.Version = tgt

	log.Info("Finished migration")
}

func (c *Cache) migration1() error {
	// Move raw boltdb to storm structure

	buckets := [][]byte{
		[]byte("Bing"),
		[]byte("Yandex"),
		[]byte("Google"),
	}

	for _, bucketName := range buckets {
		count := 0

		log.Infof("migrating %s", string(bucketName))

		items := []Item{}

		err := c.db.Bolt.Update(func(tx *bbolt.Tx) error {
			b := tx.Bucket([]byte(bucketName))
			if b == nil {
				return fmt.Errorf("bucket doesn't exist: %s", string(bucketName))
			}

			cur := b.Cursor()
			for k, v := cur.First(); k != nil; k, v = cur.Next() {
				if v == nil {
					continue
				}

				oldItem, err := decodeOldCacheItem(v)
				if err != nil {
					log.Error(err)
				}

				if oldItem.ErrorCode == 0 && time.Since(time.Unix(oldItem.Timestamp, 0)) < getCacheExpiration(oldItem.ErrorCode) {
					item := Item{
						Text:        string(k),
						Translation: oldItem.Translation,
						ErrorCode:   oldItem.ErrorCode,
						ErrorText:   oldItem.ErrorText,
						Timestamp:   oldItem.Timestamp,
					}

					items = append(items, item)

					count++

					if count%10000 == 0 {
						log.Info(count)
					}
				}

				b.Delete(k)

			}

			log.Infof("read %d valid records", count)

			return nil
		})

		if count > 0 {
			count = 0

			db := c.db.From(string(bucketName))
			tx, err := db.Begin(true)
			if err != nil {
				return err
			}
			defer tx.Rollback()

			log.Infof("writing new records")

			for _, item := range items {

				err = tx.Save(&item)
				if err != nil {
					log.Fatal(err)
				}

				count++

				if count%10000 == 0 {
					log.Info(count)
				}
			}

			tx.Commit()

			log.Infof("%d records written", count)
		}

		if err != nil {
			log.Error("failed to migrate: ", string(bucketName))
			continue
		}

	}

	// Delete old metadata
	err := c.db.Bolt.Update(func(tx *bbolt.Tx) error {
		err := tx.DeleteBucket([]byte("___metadata"))
		if err != nil {
			log.Error(err)
		}

		return nil
	})

	return err
}
<commit_msg>Fix bolt import<commit_after>package cache

import (
	"fmt"
	"time"

	log "github.com/Sirupsen/logrus"
	"go.etcd.io/bbolt"
)

type oldCacheItem struct {
	Translation string           `json:"t"`
	ErrorCode   translationError `json:"c,omitempty"`
	ErrorText   string           `json:"e,omitempty"`
	Timestamp   int64            `json:"u,omitempty"`
}

const latestVersion = 1

func (c *Cache) migrateDatabase() error {
	if c.meta.Version == latestVersion {
		return nil
	}

	log.Infof("Need to run %d database migrations", latestVersion-c.meta.Version)

	for c.meta.Version < latestVersion {
		c.migrate(c.meta.Version)
	}

	return nil
}

func (c *Cache) migrate(ver int) {
	start := time.Now()

	tgt := ver + 1

	log.Info("Running migration ", tgt)

	var err error

	switch tgt {
	case 1:
		err = c.migration1()
	}

	log := log.WithFields(log.Fields{
		"time": time.Since(start),
		"ver":  ver,
		"tgt":  tgt,
	})

	if err != nil {
		log.Fatal("Migration failed: ", err)
	}

	c.meta.Version = tgt

	log.Info("Finished migration")
}

func (c *Cache) migration1() error {
	// Move raw boltdb to storm structure

	buckets := [][]byte{
		[]byte("Bing"),
		[]byte("Yandex"),
		[]byte("Google"),
	}

	for _, bucketName := range buckets {
		count := 0

		log.Infof("migrating %s", string(bucketName))

		items := []Item{}

		err := c.db.Bolt.Update(func(tx *bolt.Tx) error {
			b := tx.Bucket([]byte(bucketName))
			if b == nil {
				return fmt.Errorf("bucket doesn't exist: %s", string(bucketName))
			}

			cur := b.Cursor()
			for k, v := cur.First(); k != nil; k, v = cur.Next() {
				if v == nil {
					continue
				}

				oldItem, err := decodeOldCacheItem(v)
				if err != nil {
					log.Error(err)
				}

				if oldItem.ErrorCode == 0 && time.Since(time.Unix(oldItem.Timestamp, 0)) < getCacheExpiration(oldItem.ErrorCode) {
					item := Item{
						Text:        string(k),
						Translation: oldItem.Translation,
						ErrorCode:   oldItem.ErrorCode,
						ErrorText:   oldItem.ErrorText,
						Timestamp:   oldItem.Timestamp,
					}

					items = append(items, item)

					count++

					if count%10000 == 0 {
						log.Info(count)
					}
				}

				b.Delete(k)

			}

			log.Infof("read %d valid records", count)

			return nil
		})

		if count > 0 {
			count = 0

			db := c.db.From(string(bucketName))
			tx, err := db.Begin(true)
			if err != nil {
				return err
			}
			defer tx.Rollback()

			log.Infof("writing new records")

			for _, item := range items {

				err = tx.Save(&item)
				if err != nil {
					log.Fatal(err)
				}

				count++

				if count%10000 == 0 {
					log.Info(count)
				}
			}

			tx.Commit()

			log.Infof("%d records written", count)
		}

		if err != nil {
			log.Error("failed to migrate: ", string(bucketName))
			continue
		}

	}

	// Delete old metadata
	err := c.db.Bolt.Update(func(tx *bolt.Tx) error {
		err := tx.DeleteBucket([]byte("___metadata"))
		if err != nil {
			log.Error(err)
		}

		return nil
	})

	return err
}
<|endoftext|>


<commit_before>package routes

import (
	"bytes"
	"encoding/base64"
	"fmt"
	"github.com/Jeffail/gabs"
	log "github.com/Sirupsen/logrus"
	"io"
	"io/ioutil"
	"net/http"
	"strings"

	"github.com/gillesdemey/npm-registry/packages"
	"github.com/gillesdemey/npm-registry/storage"
)

// GetPackageMetadata fetches package meta data
//
// 1. fetch package metadata from NPM upstream
// 2. if 304 -> return 304 and exit
// 3. if 404 or error -> check storage package
// 4. if still not found -> 404
// 5. if all is well, update storage with newer metadata
func GetPackageMetadata(w http.ResponseWriter, req *http.Request) {
	var err error

	pkg := req.URL.Query().Get(":pkg")
	storage := StorageFromContext(req.Context())
	renderer := RendererFromContext(req.Context())

	pr, pw := io.Pipe()
	multiWriter := io.MultiWriter(w, pw)

	resp, err := tryUpstream(pkg)
	if err != nil {
		err = tryMetaStorage(storage, pkg, multiWriter)
		if err != nil {
			renderer.JSON(w, http.StatusNotFound, map[string]string{
				"error": "no such package available",
			})
			return
		}
		return
	}
	defer resp.Body.Close()

	w.WriteHeader(resp.StatusCode)
	if resp.StatusCode == http.StatusNotModified {
		return
	}

	go func() {
		packages.RewriteTarballLocation(resp.Body, multiWriter)
		pw.Close()
	}()

	updateMetaStorage(storage, pkg, pr)
}

func tryUpstream(pkg string) (*http.Response, error) {
	logger := log.WithFields(log.Fields{
		"source":  "upstream",
		"package": pkg,
	})

	logger.Info("Trying upstream...")

	pkgMetaURL := fmt.Sprintf("https://registry.npmjs.org/%s", pkg)
	response, err := http.Get(pkgMetaURL)
	if err != nil {
		logger.Warn("Upstream failed: ", err)
		return nil, err
	}

	if response.StatusCode == http.StatusNotFound {
		err := fmt.Errorf("no such package available")
		logger.Warn(err)
		return nil, err
	}

	// TODO catch 5xx errors from upstream

	return response, nil
}

func tryMetaStorage(s storage.Engine, pkg string, writer io.Writer) error {
	logger := log.WithFields(log.Fields{
		"source":  "storage",
		"package": pkg,
	})

	logger.Info("Trying storage...")

	err := s.RetrieveMetadata(pkg, writer)
	if err != nil {
		logger.Warn("no such package available")
		return err
	}

	return nil
}

func updateMetaStorage(s storage.Engine, pkg string, data io.Reader) error {
	logger := log.WithFields(log.Fields{
		"source":  "storage",
		"package": pkg,
	})
	logger.Info("Updating meta storage")

	err := s.StoreMetadata(pkg, data)
	if err != nil {
		logger.Error("Failed to update meta storage: ", err)
		return err
	}

	return nil
}

// TODO add authentication middleware
func PublishPackage(w http.ResponseWriter, req *http.Request) {
	storage := StorageFromContext(req.Context())
	renderer := RendererFromContext(req.Context())

	pkgInfo, err := gabs.ParseJSONBuffer(req.Body)
	if err != nil {
		renderer.JSON(w, http.StatusBadRequest, map[string]string{"error": "bad request"})
		return
	}
	defer req.Body.Close()

	pkgName, _ := pkgInfo.Path("name").Data().(string)

	// Check if we have 1 dist-tag
	distTags, _ := pkgInfo.Path("dist-tags").ChildrenMap()
	if len(distTags) != 1 {
		renderer.JSON(w, http.StatusBadRequest, map[string]string{"error": "must have 1 dist-tag"})
		return
	}

	// Check if version of this package already exists
	var tag string
	for k, _ := range distTags {
		tag = k
		break
	}
	version := distTags[tag]

	logger := log.WithFields(log.Fields{
		"package": pkgName,
		"version": version,
	})
	logger.Info("Processing package")

	err = storage.RetrieveTarball(
		pkgName,
		// TODO there's probably a better way then manually creating the filename?
		fmt.Sprintf("%s-%s.tgz", pkgName, version),
		ioutil.Discard, // write to /dev/null
	)
	if err == nil {
		renderer.JSON(w, http.StatusBadRequest, map[string]string{"error": "version already exists"})
		return
	}

	attachments, _ := pkgInfo.Path("_attachments").ChildrenMap()
	for filename, attachment := range attachments {
		logger.WithFields(log.Fields{"attachment": filename}).Info("Decoding...")
		data := attachment.Path("data").Data().(string)
		buff, err := base64.StdEncoding.DecodeString(data)
		if err != nil {
			logger.Error(err)
			w.WriteHeader(http.StatusInternalServerError)
			return
		}
		storage.StoreTarball(pkgName, filename, bytes.NewReader(buff))
	}

	// TODO save package metadata
	// 1. check if package metadata already exists
	// 2. add dist-tag version to existing versions
	// 3. delete _attachments from JSON payload
	pkgInfo.Delete("_attachments")
	storage.StoreMetadata(pkgName, strings.NewReader(pkgInfo.String()))
}
<commit_msg>Adds string reflection to version<commit_after>package routes

import (
	"bytes"
	"encoding/base64"
	"fmt"
	"github.com/Jeffail/gabs"
	log "github.com/Sirupsen/logrus"
	"io"
	"io/ioutil"
	"net/http"
	"strings"

	"github.com/gillesdemey/npm-registry/packages"
	"github.com/gillesdemey/npm-registry/storage"
)

// GetPackageMetadata fetches package meta data
//
// 1. fetch package metadata from NPM upstream
// 2. if 304 -> return 304 and exit
// 3. if 404 or error -> check storage package
// 4. if still not found -> 404
// 5. if all is well, update storage with newer metadata
func GetPackageMetadata(w http.ResponseWriter, req *http.Request) {
	var err error

	pkg := req.URL.Query().Get(":pkg")
	storage := StorageFromContext(req.Context())
	renderer := RendererFromContext(req.Context())

	pr, pw := io.Pipe()
	multiWriter := io.MultiWriter(w, pw)

	resp, err := tryUpstream(pkg)
	if err != nil {
		err = tryMetaStorage(storage, pkg, multiWriter)
		if err != nil {
			renderer.JSON(w, http.StatusNotFound, map[string]string{
				"error": "no such package available",
			})
			return
		}
		return
	}
	defer resp.Body.Close()

	w.WriteHeader(resp.StatusCode)
	if resp.StatusCode == http.StatusNotModified {
		return
	}

	go func() {
		packages.RewriteTarballLocation(resp.Body, multiWriter)
		pw.Close()
	}()

	updateMetaStorage(storage, pkg, pr)
}

func tryUpstream(pkg string) (*http.Response, error) {
	logger := log.WithFields(log.Fields{
		"source":  "upstream",
		"package": pkg,
	})

	logger.Info("Trying upstream...")

	pkgMetaURL := fmt.Sprintf("https://registry.npmjs.org/%s", pkg)
	response, err := http.Get(pkgMetaURL)
	if err != nil {
		logger.Warn("Upstream failed: ", err)
		return nil, err
	}

	if response.StatusCode == http.StatusNotFound {
		err := fmt.Errorf("no such package available")
		logger.Warn(err)
		return nil, err
	}

	// TODO catch 5xx errors from upstream

	return response, nil
}

func tryMetaStorage(s storage.Engine, pkg string, writer io.Writer) error {
	logger := log.WithFields(log.Fields{
		"source":  "storage",
		"package": pkg,
	})

	logger.Info("Trying storage...")

	err := s.RetrieveMetadata(pkg, writer)
	if err != nil {
		logger.Warn("no such package available")
		return err
	}

	return nil
}

func updateMetaStorage(s storage.Engine, pkg string, data io.Reader) error {
	logger := log.WithFields(log.Fields{
		"source":  "storage",
		"package": pkg,
	})
	logger.Info("Updating meta storage")

	err := s.StoreMetadata(pkg, data)
	if err != nil {
		logger.Error("Failed to update meta storage: ", err)
		return err
	}

	return nil
}

// TODO add authentication middleware
func PublishPackage(w http.ResponseWriter, req *http.Request) {
	var err error
	storage := StorageFromContext(req.Context())
	renderer := RendererFromContext(req.Context())

	pkgInfo, err := gabs.ParseJSONBuffer(req.Body)
	if err != nil {
		renderer.JSON(w, http.StatusBadRequest, map[string]string{"error": "bad request"})
		return
	}
	defer req.Body.Close()

	pkgName, _ := pkgInfo.Path("name").Data().(string)

	// Check if we have 1 dist-tag
	distTags, _ := pkgInfo.Path("dist-tags").ChildrenMap()
	if len(distTags) != 1 {
		renderer.JSON(w, http.StatusBadRequest, map[string]string{"error": "must have 1 dist-tag"})
		return
	}

	// Check if version of this package already exists
	var tag string
	for k, _ := range distTags {
		tag = k
		break
	}
	version := distTags[tag].Data().(string)

	logger := log.WithFields(log.Fields{
		"package": pkgName,
		"version": version,
	})
	logger.Info("Processing package")

	err = storage.RetrieveTarball(
		pkgName,
		// TODO there's probably a better way then manually creating the filename?
		fmt.Sprintf("%s-%s.tgz", pkgName, version),
		ioutil.Discard, // write to /dev/null
	)

	if err == nil {
		renderer.JSON(w, http.StatusBadRequest, map[string]string{
			"error": "version already exists",
		})
		return
	}

	attachments, _ := pkgInfo.Path("_attachments").ChildrenMap()
	for filename, attachment := range attachments {
		logger.WithFields(log.Fields{"attachment": filename}).Info("Decoding...")
		data := attachment.Path("data").Data().(string)
		buff, err := base64.StdEncoding.DecodeString(data)
		if err != nil {
			logger.Error(err)
			w.WriteHeader(http.StatusInternalServerError)
			return
		}
		storage.StoreTarball(pkgName, filename, bytes.NewReader(buff))
	}

	// TODO save package metadata
	// 1. check if package metadata already exists
	// 2. add dist-tag version to existing versions
	// 3. delete _attachments from JSON payload
	pkgInfo.Delete("_attachments")
	storage.StoreMetadata(pkgName, strings.NewReader(pkgInfo.String()))
}
<|endoftext|>


<commit_before>package accessor

import (
	"github.com/stretchr/testify/assert"
	"testing"
)

func TestParsePath(t *testing.T) {
	type Input struct {
		Path string
	}
	type Expect struct {
		Path Path
		Err  error
	}
	type Test struct {
		Title  string
		Input  Input
		Expect Expect
	}

	table := []Test{
		Test{
			Title: "basic",
			Input: Input{
				Path: "a/b/0",
			},
			Expect: Expect{
				Path: &basicPath{"a", &basicPath{"b", &basicPath{"0", nil}}},
				Err:  nil,
			},
		},
		Test{
			Title: "slash",
			Input: Input{
				Path: "/a/b/0/",
			},
			Expect: Expect{
				Path: &basicPath{"a", &basicPath{"b", &basicPath{"0", nil}}},
				Err:  nil,
			},
		},
		Test{
			Title: "empty",
			Input: Input{
				Path: "",
			},
			Expect: Expect{
				Path: nil,
				Err:  NewInvalidPathError(""),
			},
		},
		Test{
			Title: "empty key",
			Input: Input{
				Path: "//",
			},
			Expect: Expect{
				Path: nil,
				Err:  NewInvalidPathError("//"),
			},
		},
	}

	for _, testCase := range table {
		t.Run(testCase.Title, func(t *testing.T) {
			assert := assert.New(t)

			path, err := ParsePath(testCase.Input.Path)

			assert.Equal(testCase.Expect.Path, path)
			assert.Equal(testCase.Expect.Err, err)
		})
	}
}
<commit_msg>Add test of ParsePath for /<commit_after>package accessor

import (
	"github.com/stretchr/testify/assert"
	"testing"
)

func TestParsePath(t *testing.T) {
	type Input struct {
		Path string
	}
	type Expect struct {
		Path Path
		Err  error
	}
	type Test struct {
		Title  string
		Input  Input
		Expect Expect
	}

	table := []Test{
		Test{
			Title: "basic",
			Input: Input{
				Path: "a/b/0",
			},
			Expect: Expect{
				Path: &basicPath{"a", &basicPath{"b", &basicPath{"0", nil}}},
				Err:  nil,
			},
		},
		Test{
			Title: "many slash",
			Input: Input{
				Path: "/a/b/0/",
			},
			Expect: Expect{
				Path: &basicPath{"a", &basicPath{"b", &basicPath{"0", nil}}},
				Err:  nil,
			},
		},
		Test{
			Title: "only slash",
			Input: Input{
				Path: "/",
			},
			Expect: Expect{
				Path: nil,
				Err:  NewInvalidPathError("/"),
			},
		},
		Test{
			Title: "empty",
			Input: Input{
				Path: "",
			},
			Expect: Expect{
				Path: nil,
				Err:  NewInvalidPathError(""),
			},
		},
		Test{
			Title: "empty key",
			Input: Input{
				Path: "//",
			},
			Expect: Expect{
				Path: nil,
				Err:  NewInvalidPathError("//"),
			},
		},
	}

	for _, testCase := range table {
		t.Run(testCase.Title, func(t *testing.T) {
			assert := assert.New(t)

			path, err := ParsePath(testCase.Input.Path)

			assert.Equal(testCase.Expect.Path, path)
			assert.Equal(testCase.Expect.Err, err)
		})
	}
}
<|endoftext|>