<commit_before>// Copyright 2013 Beego Authors
// Copyright 2014 Unknwon
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

// Package captcha a middleware that provides captcha service for Macaron.
package captcha

import (
	"fmt"
	"html/template"
	"path"
	"strings"
	"net/http"

	"github.com/Unknwon/com"
	"github.com/macaron-contrib/cache"

	"github.com/lunny/tango"
)

var (
	defaultChars = []byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}
)

// Captcha represents a captcha service.
type Captchas struct {
	store            cache.Cache
	SubURL           string
	URLPrefix        string
	FieldIdName      string
	FieldCaptchaName string
	StdWidth         int
	StdHeight        int
	ChallengeNums    int
	Expiration       int64
	CachePrefix      string
}

// NewCaptcha initializes and returns a captcha with given options.
func New(opt Options, c cache.Cache) *Captchas {
	opt = prepareOptions([]Options{opt})
	return &Captchas{
		store: c,
		SubURL:           opt.SubURL,
		URLPrefix:        opt.URLPrefix,
		FieldIdName:      opt.FieldIdName,
		FieldCaptchaName: opt.FieldCaptchaName,
		StdWidth:         opt.Width,
		StdHeight:        opt.Height,
		ChallengeNums:    opt.ChallengeNums,
		Expiration:       opt.Expiration,
		CachePrefix:      opt.CachePrefix,
	}
}

// generate key string
func (c *Captchas) key(id string) string {
	return c.CachePrefix + id
}

// generate rand chars with default chars
func (c *Captchas) genRandChars() string {
	return string(com.RandomCreateBytes(c.ChallengeNums, defaultChars...))
}

// create a new captcha id
func (c *Captchas) CreateCaptcha() (string, error) {
	id := string(com.RandomCreateBytes(15))
	if err := c.store.Put(c.key(id), c.genRandChars(), c.Expiration); err != nil {
		return "", err
	}
	return id, nil
}

// verify from a request
func (c *Captchas) VerifyReq(req *http.Request) bool {
	req.ParseForm()
	return c.Verify(req.FormValue(c.FieldIdName), req.FormValue(c.FieldCaptchaName))
}

// direct verify id and challenge string
func (c *Captchas) Verify(id string, challenge string) bool {
	if len(challenge) == 0 || len(id) == 0 {
		return false
	}

	var chars string

	key := c.key(id)
	if v, ok := c.store.Get(key).(string); ok {
		chars = v
	} else {
		return false
	}

	fmt.Println("chars:", chars)

	defer c.store.Delete(key)

	if len(chars) != len(challenge) {
		return false
	}

	// verify challenge
	for i, c := range []byte(chars) {
		if c != challenge[i]-48 {
			return false
		}
	}

	return true
}

// tempalte func for output html
func (c *Captchas) CreateHtml() template.HTML {
	value, err := c.CreateCaptcha()
	if err != nil {
		panic(fmt.Errorf("fail to create captcha: %v", err))
	}
	return template.HTML(fmt.Sprintf(`<input type="hidden" name="%s" value="%s">

	`, c.FieldIdName, value, c.SubURL, c.URLPrefix, value, c.SubURL, c.URLPrefix, value))
}

type Options struct {
	// Suburl path. Default is empty.
	SubURL string
	// URL prefix of getting captcha pictures. Default is "/captcha/".
	URLPrefix string
	// Hidden input element ID. Default is "captcha_id".
	FieldIdName string
	// User input value element name in request form. Default is "captcha".
	FieldCaptchaName string
	// Challenge number. Default is 6.
	ChallengeNums int
	// Captcha image width. Default is 240.
	Width int
	// Captcha image height. Default is 80.
	Height int
	// Captcha expiration time in seconds. Default is 600.
	Expiration int64
	// Cache key prefix captcha characters. Default is "captcha_".
	CachePrefix string
}

func prepareOptions(options []Options) Options {
	var opt Options
	if len(options) > 0 {
		opt = options[0]
	}

	opt.SubURL = strings.TrimSuffix(opt.SubURL, "/")

	// Defaults.
	if len(opt.URLPrefix) == 0 {
		opt.URLPrefix = "/captcha/"
	} else if opt.URLPrefix[len(opt.URLPrefix)-1] != '/' {
		opt.URLPrefix += "/"
	}
	if len(opt.FieldIdName) == 0 {
		opt.FieldIdName = "captcha_id"
	}
	if len(opt.FieldCaptchaName) == 0 {
		opt.FieldCaptchaName = "captcha"
	}
	if opt.ChallengeNums == 0 {
		opt.ChallengeNums = 6
	}
	if opt.Width == 0 {
		opt.Width = stdWidth
	}
	if opt.Height == 0 {
		opt.Height = stdHeight
	}
	if opt.Expiration == 0 {
		opt.Expiration = 600
	}
	if len(opt.CachePrefix) == 0 {
		opt.CachePrefix = "captcha_"
	}

	return opt
}

type Captcha struct {
	c *Captchas
	req *http.Request
}

func (c *Captcha) SetCaptcha(cpt *Captchas, req *http.Request) {
	c.c = cpt
	c.req = req
}

func (c *Captcha) CreateHtml() template.HTML {
	return c.c.CreateHtml()
}

// verify from a request
func (c *Captcha) Verify() bool {
	return c.c.VerifyReq(c.req)
}

func (c *Captcha) CreateCaptcha() (string, error) {
	return c.c.CreateCaptcha()
}

func (c *Captcha) VerifyCaptcha(id string, challenge string) bool {
	return c.c.Verify(id, challenge)
}

type Captchaer interface {
	SetCaptcha(*Captchas, *http.Request)
}

// Captchaer is a middleware that maps a captcha.Captcha service into the Macaron handler chain.
// An single variadic captcha.Options struct can be optionally provided to configure.
// This should be register after cache.Cacher.
func (c *Captchas) Handle(ctx *tango.Context) {
	if !strings.HasPrefix(ctx.Req().RequestURI, c.URLPrefix) {
		if action := ctx.Action(); action != nil {
			if p, ok := action.(Captchaer); ok {
				p.SetCaptcha(c, ctx.Req())
			}
		}

		ctx.Next()
		return
	}

	var chars string
	id := path.Base(ctx.Req().RequestURI)
	if i := strings.Index(id, "."); i > -1 {
		id = id[:i]
	}
	key := c.key(id)

	// Reload captcha.
	if len(ctx.Req().FormValue("reload")) > 0 {
		chars = c.genRandChars()
		if err := c.store.Put(key, chars, c.Expiration); err != nil {
			ctx.WriteHeader(http.StatusInternalServerError)
			ctx.Write([]byte("captcha reload error"))
			panic(fmt.Errorf("fail to reload captcha: %v", err))
		}
	} else {
		if v, ok := c.store.Get(key).(string); ok {
			chars = v
		} else {
			ctx.WriteHeader(http.StatusNotFound)
			ctx.Write([]byte("captcha not found"))
			return
		}
	}

	if _, err := NewImage([]byte(chars), c.StdWidth, c.StdHeight).WriteTo(ctx.ResponseWriter); err != nil {
		panic(fmt.Errorf("fail to write captcha: %v", err))
	}
}
<commit_msg>remove debug<commit_after>// Copyright 2013 Beego Authors
// Copyright 2014 Unknwon
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

// Package captcha a middleware that provides captcha service for Macaron.
package captcha

import (
	"fmt"
	"html/template"
	"path"
	"strings"
	"net/http"

	"github.com/Unknwon/com"
	"github.com/macaron-contrib/cache"

	"github.com/lunny/tango"
)

var (
	defaultChars = []byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}
)

// Captcha represents a captcha service.
type Captchas struct {
	store            cache.Cache
	SubURL           string
	URLPrefix        string
	FieldIdName      string
	FieldCaptchaName string
	StdWidth         int
	StdHeight        int
	ChallengeNums    int
	Expiration       int64
	CachePrefix      string
}

// NewCaptcha initializes and returns a captcha with given options.
func New(opt Options, c cache.Cache) *Captchas {
	opt = prepareOptions([]Options{opt})
	return &Captchas{
		store: c,
		SubURL:           opt.SubURL,
		URLPrefix:        opt.URLPrefix,
		FieldIdName:      opt.FieldIdName,
		FieldCaptchaName: opt.FieldCaptchaName,
		StdWidth:         opt.Width,
		StdHeight:        opt.Height,
		ChallengeNums:    opt.ChallengeNums,
		Expiration:       opt.Expiration,
		CachePrefix:      opt.CachePrefix,
	}
}

// generate key string
func (c *Captchas) key(id string) string {
	return c.CachePrefix + id
}

// generate rand chars with default chars
func (c *Captchas) genRandChars() string {
	return string(com.RandomCreateBytes(c.ChallengeNums, defaultChars...))
}

// create a new captcha id
func (c *Captchas) CreateCaptcha() (string, error) {
	id := string(com.RandomCreateBytes(15))
	if err := c.store.Put(c.key(id), c.genRandChars(), c.Expiration); err != nil {
		return "", err
	}
	return id, nil
}

// verify from a request
func (c *Captchas) VerifyReq(req *http.Request) bool {
	req.ParseForm()
	return c.Verify(req.FormValue(c.FieldIdName), req.FormValue(c.FieldCaptchaName))
}

// direct verify id and challenge string
func (c *Captchas) Verify(id string, challenge string) bool {
	if len(challenge) == 0 || len(id) == 0 {
		return false
	}

	var chars string

	key := c.key(id)
	if v, ok := c.store.Get(key).(string); ok {
		chars = v
	} else {
		return false
	}

	defer c.store.Delete(key)

	if len(chars) != len(challenge) {
		return false
	}

	// verify challenge
	for i, c := range []byte(chars) {
		if c != challenge[i]-48 {
			return false
		}
	}

	return true
}

// tempalte func for output html
func (c *Captchas) CreateHtml() template.HTML {
	value, err := c.CreateCaptcha()
	if err != nil {
		panic(fmt.Errorf("fail to create captcha: %v", err))
	}
	return template.HTML(fmt.Sprintf(`<input type="hidden" name="%s" value="%s">

	`, c.FieldIdName, value, c.SubURL, c.URLPrefix, value, c.SubURL, c.URLPrefix, value))
}

type Options struct {
	// Suburl path. Default is empty.
	SubURL string
	// URL prefix of getting captcha pictures. Default is "/captcha/".
	URLPrefix string
	// Hidden input element ID. Default is "captcha_id".
	FieldIdName string
	// User input value element name in request form. Default is "captcha".
	FieldCaptchaName string
	// Challenge number. Default is 6.
	ChallengeNums int
	// Captcha image width. Default is 240.
	Width int
	// Captcha image height. Default is 80.
	Height int
	// Captcha expiration time in seconds. Default is 600.
	Expiration int64
	// Cache key prefix captcha characters. Default is "captcha_".
	CachePrefix string
}

func prepareOptions(options []Options) Options {
	var opt Options
	if len(options) > 0 {
		opt = options[0]
	}

	opt.SubURL = strings.TrimSuffix(opt.SubURL, "/")

	// Defaults.
	if len(opt.URLPrefix) == 0 {
		opt.URLPrefix = "/captcha/"
	} else if opt.URLPrefix[len(opt.URLPrefix)-1] != '/' {
		opt.URLPrefix += "/"
	}
	if len(opt.FieldIdName) == 0 {
		opt.FieldIdName = "captcha_id"
	}
	if len(opt.FieldCaptchaName) == 0 {
		opt.FieldCaptchaName = "captcha"
	}
	if opt.ChallengeNums == 0 {
		opt.ChallengeNums = 6
	}
	if opt.Width == 0 {
		opt.Width = stdWidth
	}
	if opt.Height == 0 {
		opt.Height = stdHeight
	}
	if opt.Expiration == 0 {
		opt.Expiration = 600
	}
	if len(opt.CachePrefix) == 0 {
		opt.CachePrefix = "captcha_"
	}

	return opt
}

type Captcha struct {
	c *Captchas
	req *http.Request
}

func (c *Captcha) SetCaptcha(cpt *Captchas, req *http.Request) {
	c.c = cpt
	c.req = req
}

func (c *Captcha) CreateHtml() template.HTML {
	return c.c.CreateHtml()
}

// verify from a request
func (c *Captcha) Verify() bool {
	return c.c.VerifyReq(c.req)
}

func (c *Captcha) CreateCaptcha() (string, error) {
	return c.c.CreateCaptcha()
}

func (c *Captcha) VerifyCaptcha(id string, challenge string) bool {
	return c.c.Verify(id, challenge)
}

type Captchaer interface {
	SetCaptcha(*Captchas, *http.Request)
}

// Captchaer is a middleware that maps a captcha.Captcha service into the Macaron handler chain.
// An single variadic captcha.Options struct can be optionally provided to configure.
// This should be register after cache.Cacher.
func (c *Captchas) Handle(ctx *tango.Context) {
	if !strings.HasPrefix(ctx.Req().RequestURI, c.URLPrefix) {
		if action := ctx.Action(); action != nil {
			if p, ok := action.(Captchaer); ok {
				p.SetCaptcha(c, ctx.Req())
			}
		}

		ctx.Next()
		return
	}

	var chars string
	id := path.Base(ctx.Req().RequestURI)
	if i := strings.Index(id, "."); i > -1 {
		id = id[:i]
	}
	key := c.key(id)

	// Reload captcha.
	if len(ctx.Req().FormValue("reload")) > 0 {
		chars = c.genRandChars()
		if err := c.store.Put(key, chars, c.Expiration); err != nil {
			ctx.WriteHeader(http.StatusInternalServerError)
			ctx.Write([]byte("captcha reload error"))
			panic(fmt.Errorf("fail to reload captcha: %v", err))
		}
	} else {
		if v, ok := c.store.Get(key).(string); ok {
			chars = v
		} else {
			ctx.WriteHeader(http.StatusNotFound)
			ctx.Write([]byte("captcha not found"))
			return
		}
	}

	if _, err := NewImage([]byte(chars), c.StdWidth, c.StdHeight).WriteTo(ctx.ResponseWriter); err != nil {
		panic(fmt.Errorf("fail to write captcha: %v", err))
	}
}
<|endoftext|>


<commit_before>package main

import (
	"code.google.com/p/freetype-go/freetype"
	"code.google.com/p/freetype-go/freetype/truetype"
	"fmt"
	"github.com/errnoh/term.color"
	"image"
	"image/color/palette"
	"image/draw"
	"github.com/james4k/terminal"
	"os"
)

var font *truetype.Font

const fontSize = 18

func init() {
	fontData, err := Asset("font/Anonymous Pro Minus.ttf")
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
	font, err = freetype.ParseFont(fontData)
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
}

// Capture draws virtual terminal and return paletted image
func (g *GifGenerator) Capture(state *terminal.State) (paletted *image.Paletted, err error) {
	fb := font.Bounds(fontSize)
	cursorX, cursorY := state.Cursor()
	paletted = image.NewPaletted(image.Rect(0, 0, g.Col*int(fb.XMax-fb.XMin)+10, g.Row*int(fb.YMax-fb.YMin)+10), palette.WebSafe)

	c := freetype.NewContext()
	c.SetFontSize(fontSize)
	c.SetFont(font)
	c.SetDst(paletted)
	c.SetClip(paletted.Bounds())
	for row := 0; row < g.Row; row++ {
		for col := 0; col < g.Col; col++ {
			ch, fg, bg := state.Cell(col, row)
			var uniform *image.Uniform
			// background color
			if bg != terminal.DefaultBG {
				if bg == terminal.DefaultFG {
					uniform = image.White
				} else {
					uniform = image.NewUniform(color.Term256{Val: uint8(bg)})
				}
			}
			// cursor
			if state.CursorVisible() && (row == cursorY && col == cursorX) {
				uniform = image.White
			}
			if uniform != nil {
				draw.Draw(paletted, image.Rect(5+col*int(fb.XMax-fb.XMin), row*int(fb.YMax-fb.YMin)-int(fb.YMin), 5+(col+1)*int(fb.XMax-fb.XMin), (row+1)*int(fb.YMax-fb.YMin)-int(fb.YMin)), uniform, image.ZP, draw.Src)
			}
			// foreground color
			switch fg {
			case terminal.DefaultFG:
				c.SetSrc(image.White)
			case terminal.DefaultBG:
				c.SetSrc(image.Black)
			default:
				c.SetSrc(image.NewUniform(color.Term256{Val: uint8(fg)}))
			}
			_, err = c.DrawString(string(ch), freetype.Pt(5+col*int(fb.XMax-fb.XMin), (row+1)*int(fb.YMax-fb.YMin)))
			if err != nil {
				return
			}
		}
	}
	return paletted, nil
}
<commit_msg>fix freetype repo<commit_after>package main

import (
	"github.com/golang/freetype"
	"github.com/golang/freetype/truetype"
	"fmt"
	"github.com/errnoh/term.color"
	"image"
	"image/color/palette"
	"image/draw"
	"github.com/james4k/terminal"
	"os"
)

var font *truetype.Font

const fontSize = 18

func init() {
	fontData, err := Asset("font/Anonymous Pro Minus.ttf")
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
	font, err = freetype.ParseFont(fontData)
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
}

// Capture draws virtual terminal and return paletted image
func (g *GifGenerator) Capture(state *terminal.State) (paletted *image.Paletted, err error) {
	fb := font.Bounds(fontSize)
	cursorX, cursorY := state.Cursor()
	paletted = image.NewPaletted(image.Rect(0, 0, g.Col*int(fb.Max.X-fb.Min.X)+10, g.Row*int(fb.Max.Y-fb.Min.Y)+10), palette.WebSafe)

	c := freetype.NewContext()
	c.SetFontSize(fontSize)
	c.SetFont(font)
	c.SetDst(paletted)
	c.SetClip(paletted.Bounds())
	for row := 0; row < g.Row; row++ {
		for col := 0; col < g.Col; col++ {
			ch, fg, bg := state.Cell(col, row)
			var uniform *image.Uniform
			// background color
			if bg != terminal.DefaultBG {
				if bg == terminal.DefaultFG {
					uniform = image.White
				} else {
					uniform = image.NewUniform(color.Term256{Val: uint8(bg)})
				}
			}
			// cursor
			if state.CursorVisible() && (row == cursorY && col == cursorX) {
				uniform = image.White
			}
			if uniform != nil {
				draw.Draw(paletted, image.Rect(5+col*int(fb.Max.X-fb.Min.X), row*int(fb.Max.Y-fb.Min.Y)-int(fb.Min.Y), 5+(col+1)*int(fb.Max.X-fb.Min.X), (row+1)*int(fb.Max.Y-fb.Min.Y)-int(fb.Min.Y)), uniform, image.ZP, draw.Src)
			}
			// foreground color
			switch fg {
			case terminal.DefaultFG:
				c.SetSrc(image.White)
			case terminal.DefaultBG:
				c.SetSrc(image.Black)
			default:
				c.SetSrc(image.NewUniform(color.Term256{Val: uint8(fg)}))
			}
			_, err = c.DrawString(string(ch), freetype.Pt(5+col*int(fb.Max.X-fb.Min.X), (row+1)*int(fb.Max.Y-fb.Min.Y)))
			if err != nil {
				return
			}
		}
	}
	return paletted, nil
}
<|endoftext|>


<commit_before>package core_test

import (
	//	"fmt"
	"github.com/APTrust/bagit/core"
	"github.com/APTrust/bagit/util/testutil"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
	"io/ioutil"
	"os"
	//	"path/filepath"
	//	"runtime"
	"testing"
)

func TestNewBagger(t *testing.T) {
	tempFile, err := ioutil.TempFile("", "bagger")
	defer tempFile.Close()
	defer os.Remove(tempFile.Name())
	require.Nil(t, err)

	payloadDir, err := testutil.GetPathToTestFileDir()
	require.Nil(t, err)

	profilePath, err := testutil.GetPathToTestProfile("aptrust_bagit_profile_2.0.json")
	require.Nil(t, err)
	aptrustProfile, err := core.LoadBagItProfile(profilePath)
	require.Nil(t, err)

	bagger := core.NewBagger(tempFile.Name(), payloadDir, aptrustProfile, nil, false)
	require.NotNil(t, bagger)
	require.NotNil(t, bagger.Bag)
	assert.Equal(t, tempFile.Name(), bagger.Bag.Path)
	assert.Equal(t, payloadDir, bagger.PayloadDir)
	assert.Nil(t, bagger.TagValues)
	assert.False(t, bagger.ForceOverwrite)

	tagValues := map[string]string{
		"one": "first",
		"two": "second",
	}

	bagger = core.NewBagger(tempFile.Name(), payloadDir, aptrustProfile, tagValues, true)
	require.NotNil(t, bagger)
	require.NotNil(t, bagger.Bag)
	assert.Equal(t, tempFile.Name(), bagger.Bag.Path)
	assert.Equal(t, payloadDir, bagger.PayloadDir)
	require.NotNil(t, bagger.TagValues)
	assert.Equal(t, "first", bagger.TagValues["one"])
	assert.Equal(t, "second", bagger.TagValues["two"])
	assert.True(t, bagger.ForceOverwrite)
}

func TestBuildBag(t *testing.T) {

}
<commit_msg>Abstract pre-requisites in test<commit_after>package core_test

import (
	"github.com/APTrust/bagit/core"
	"github.com/APTrust/bagit/util/testutil"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
	"io/ioutil"
	"os"
	"testing"
)

func getBaggerPreReqs(t *testing.T) (tempFile *os.File, payloadDir string, aptrustProfile *core.BagItProfile) {
	var err error
	tempFile, err = ioutil.TempFile("", "bagger")
	require.Nil(t, err)

	payloadDir, err = testutil.GetPathToTestFileDir()
	require.Nil(t, err)

	profilePath, err := testutil.GetPathToTestProfile("aptrust_bagit_profile_2.0.json")
	require.Nil(t, err)
	aptrustProfile, err = core.LoadBagItProfile(profilePath)
	require.Nil(t, err)

	return tempFile, payloadDir, aptrustProfile
}

func TestNewBagger(t *testing.T) {
	tempFile, payloadDir, aptrustProfile := getBaggerPreReqs(t)
	defer tempFile.Close()
	defer os.Remove(tempFile.Name())

	bagger := core.NewBagger(tempFile.Name(), payloadDir, aptrustProfile, nil, false)
	require.NotNil(t, bagger)
	require.NotNil(t, bagger.Bag)
	assert.Equal(t, tempFile.Name(), bagger.Bag.Path)
	assert.Equal(t, payloadDir, bagger.PayloadDir)
	assert.Nil(t, bagger.TagValues)
	assert.False(t, bagger.ForceOverwrite)

	tagValues := map[string]string{
		"one": "first",
		"two": "second",
	}

	bagger = core.NewBagger(tempFile.Name(), payloadDir, aptrustProfile, tagValues, true)
	require.NotNil(t, bagger)
	require.NotNil(t, bagger.Bag)
	assert.Equal(t, tempFile.Name(), bagger.Bag.Path)
	assert.Equal(t, payloadDir, bagger.PayloadDir)
	require.NotNil(t, bagger.TagValues)
	assert.Equal(t, "first", bagger.TagValues["one"])
	assert.Equal(t, "second", bagger.TagValues["two"])
	assert.True(t, bagger.ForceOverwrite)
}

func TestBuildBag(t *testing.T) {

}
<|endoftext|>


<commit_before>package cioutil

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/garyburd/go-oauth/oauth"
	"github.com/pkg/errors"
)

// ClientRequest defines information that can be used to make a request
type ClientRequest struct {
	Method      string
	Path        string
	FormValues  interface{}
	QueryValues interface{}
}

// DoFormRequest makes the actual request
func (cio Cio) DoFormRequest(request ClientRequest, result interface{}) error {

	// Construct the url
	cioURL := cio.Host + request.Path + QueryString(request.QueryValues)

	// Construct the body
	bodyValues := FormValues(request.FormValues)
	bodyString := bodyValues.Encode()
	logRequest(cio.Log, request.Method, cioURL, bodyValues)

	err := cio.createAndSendRequest(request, cioURL, bodyString, bodyValues, result)

	// Retry if Status Code >= 500 and RetryServerErr is set to true
	if cio.RetryServerErr && ErrorStatusCode(err) >= 500 {
		time.Sleep(1 * time.Second)
		err = cio.createAndSendRequest(request, cioURL, bodyString, bodyValues, result)
	}
	return err
}

// createAndSendRequest creates the body io.Reader, the *http.Request, and sends the request, logging the response
func (cio Cio) createAndSendRequest(request ClientRequest, cioURL string, bodyString string, bodyValues url.Values, result interface{}) error {

	var bodyReader io.Reader
	if len(bodyString) > 0 {
		bodyReader = bytes.NewReader([]byte(bodyString))
	}

	// Construct the request
	httpReq, err := cio.createRequest(request, cioURL, bodyReader, bodyValues)
	if err != nil {
		return err
	}

	// Send the request
	return cio.sendRequest(httpReq, result, cioURL)
}

// createRequest creates the *http.Request object
func (cio Cio) createRequest(request ClientRequest, cioURL string, bodyReader io.Reader, bodyValues url.Values) (*http.Request, error) {
	// Construct the request
	httpReq, err := http.NewRequest(request.Method, cioURL, bodyReader)
	if err != nil {
		return httpReq, RequestError{errors.Wrap(err, "CIO: Failed to form request"), ErrorMetaData{Method: request.Method, URL: cioURL}}
	}

	// oAuth signature
	var client oauth.Client
	client.Credentials = oauth.Credentials{Token: cio.apiKey, Secret: cio.apiSecret}

	// Add headers
	httpReq.Header.Set("Content-Type", "application/x-www-form-urlencoded")
	httpReq.Header.Set("Accept", "application/json")
	httpReq.Header.Set("Accept-Charset", "utf-8")
	httpReq.Header.Set("User-Agent", "Golang CIO Library")
	httpReq.Header.Set("Authorization", client.AuthorizationHeader(nil, request.Method, httpReq.URL, bodyValues))

	return httpReq, nil
}

// sendRequest sends the *http.Request
func (cio Cio) sendRequest(httpReq *http.Request, result interface{}, cioURL string) error {
	// Create the HTTP client
	httpClient := &http.Client{
		Transport: http.DefaultTransport,
		Timeout:   cio.RequestTimeout,
	}

	// Make the request
	res, err := httpClient.Do(httpReq)
	if err != nil {
		return RequestError{errors.Wrap(err, "CIO: Failed to make request"), ErrorMetaData{Method: httpReq.Method, URL: cioURL}}
	}

	// Parse the response
	defer func() {
		if closeErr := res.Body.Close(); closeErr != nil {
			logBodyCloseError(cio.Log, closeErr)
		}
	}()

	resBody, err := ioutil.ReadAll(res.Body)
	resBodyString := string(resBody)
	if err != nil {
		return RequestError{errors.Wrap(err, "CIO: Could not read response"), ErrorMetaData{Method: httpReq.Method, URL: cioURL, StatusCode: res.StatusCode, Payload: resBodyString}}
	}

	// Unmarshal result
	err = json.Unmarshal(resBody, &result)

	// Log the response
	logResponse(cio.Log, httpReq.Method, cioURL, res.StatusCode, resBodyString, err)

	// Return own error if Status Code >= 400
	if res.StatusCode >= 400 {
		return RequestError{errors.New("CIO: Status Code >= 400"), ErrorMetaData{Method: httpReq.Method, URL: cioURL, StatusCode: res.StatusCode, Payload: resBodyString}}
	}

	// Return Unmarshal error (if any) if Status Code is < 400
	if err != nil {
		return RequestError{errors.Wrap(err, "CIO: Could not unmarshal payload"), ErrorMetaData{Method: httpReq.Method, URL: cioURL, StatusCode: res.StatusCode, Payload: resBodyString}}
	}
	return nil
}

// logRequest logs the request about to be made to CIO, redacting sensitive information in the body
func logRequest(log Logger, method string, cioURL string, bodyValues url.Values) {
	if log != nil {

		// Copy url.Values
		redactedValues := url.Values{}
		for k, v := range bodyValues {
			redactedValues[k] = v
		}

		// Redact sensitive information
		if val := redactedValues.Get("password"); len(val) > 0 {
			redactedValues.Set("password", "redacted")
		}
		if val := redactedValues.Get("provider_refresh_token"); len(val) > 0 {
			redactedValues.Set("provider_refresh_token", "redacted")
		}
		if val := redactedValues.Get("provider_consumer_key"); len(val) > 0 {
			redactedValues.Set("provider_consumer_key", "redacted")
		}
		if val := redactedValues.Get("provider_consumer_secret"); len(val) > 0 {
			redactedValues.Set("provider_consumer_secret", "redacted")
		}

		// Actually log
		if logrusLogger, ok := log.(*logrus.Logger); ok {
			// If logrus, use structured fields
			logrusLogger.WithFields(logrus.Fields{"httpMethod": method, "url": cioURL, "payload": redactedValues.Encode()}).Debug("Creating new request to CIO")
		} else {
			// Else just log with Println
			log.Printf("Creating new %s request to: %s with payload: %s\n", method, cioURL, redactedValues.Encode())
		}
	}
}

// logBodyCloseError logs any error that happens when trying to close the *http.Response.Body
func logBodyCloseError(log Logger, closeError error) {
	if log != nil {
		if logrusLogger, ok := log.(*logrus.Logger); ok {
			// If logrus, use structured fields
			logrusLogger.WithError(closeError).Warn("Unable to close response body from CIO")
		} else {
			// Else just log with Println
			log.Printf("Unable to close response body from CIO, with error: %s\n", closeError.Error())
		}
	}
}

// logResponse logs the response from CIO, if any logger is set
func logResponse(log Logger, method string, cioURL string, statusCode int, responseBody string, unmarshalError error) {
	if log != nil {

		// TODO: redact access_token and access_token_secret before logging (only occurs with 3-legged oauth [rare])

		// Take only the first 2000 characters from the responseBody, which should be more than enough to debug anything
		if bodyLen := len(responseBody); bodyLen > 2000 {
			responseBody = responseBody[:2000]
		}

		if logrusLogger, ok := log.(*logrus.Logger); ok {
			// If logrus, use structured fields
			logEntry := logrusLogger.WithFields(logrus.Fields{
				"httpMethod":     method,
				"url":            cioURL,
				"statusCode":     fmt.Sprintf("%d", statusCode),
				"payloadSnippet": responseBody})
			if unmarshalError != nil || statusCode >= 400 {
				logEntry.Warn("Received response from CIO")
			} else {
				logEntry.Debug("Received response from CIO")
			}

		} else {
			// Else just log with Println
			log.Printf("Received response from %s to: %s with status code: %d and payload snippet: %s\n", method, cioURL, statusCode, responseBody)
		}
	}
}
<commit_msg>UN-418: retry on nonce collisions<commit_after>package cioutil

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/garyburd/go-oauth/oauth"
	"github.com/pkg/errors"
)

// ClientRequest defines information that can be used to make a request
type ClientRequest struct {
	Method      string
	Path        string
	FormValues  interface{}
	QueryValues interface{}
}

// DoFormRequest makes the actual request
func (cio Cio) DoFormRequest(request ClientRequest, result interface{}) error {

	// Construct the url
	cioURL := cio.Host + request.Path + QueryString(request.QueryValues)

	// Construct the body
	bodyValues := FormValues(request.FormValues)
	bodyString := bodyValues.Encode()
	logRequest(cio.Log, request.Method, cioURL, bodyValues)

	err := cio.createAndSendRequest(request, cioURL, bodyString, bodyValues, result)

	// Retry if Status Code >= 500 and RetryServerErr is set to true
	if cio.RetryServerErr && shouldRetryOnce(err) {
		time.Sleep(1 * time.Second)
		err = cio.createAndSendRequest(request, cioURL, bodyString, bodyValues, result)
	}
	return err
}

// shouldRetryOnce returns true if the request should be retried
func shouldRetryOnce(err error) bool {
	// Retry on CIO Server errors, and also if the nonce has been used
	// (CIO seems to have issues with nonce collisions).
	return ErrorStatusCode(err) >= 500 ||
		(ErrorStatusCode(err) == 401 && strings.Contains(strings.ToLower(ErrorPayload(err)), "nonce"))
}

// createAndSendRequest creates the body io.Reader, the *http.Request, and sends the request, logging the response
func (cio Cio) createAndSendRequest(request ClientRequest, cioURL string, bodyString string, bodyValues url.Values, result interface{}) error {

	var bodyReader io.Reader
	if len(bodyString) > 0 {
		bodyReader = bytes.NewReader([]byte(bodyString))
	}

	// Construct the request
	httpReq, err := cio.createRequest(request, cioURL, bodyReader, bodyValues)
	if err != nil {
		return err
	}

	// Send the request
	return cio.sendRequest(httpReq, result, cioURL)
}

// createRequest creates the *http.Request object
func (cio Cio) createRequest(request ClientRequest, cioURL string, bodyReader io.Reader, bodyValues url.Values) (*http.Request, error) {
	// Construct the request
	httpReq, err := http.NewRequest(request.Method, cioURL, bodyReader)
	if err != nil {
		return httpReq, RequestError{errors.Wrap(err, "CIO: Failed to form request"), ErrorMetaData{Method: request.Method, URL: cioURL}}
	}

	// oAuth signature
	var client oauth.Client
	client.Credentials = oauth.Credentials{Token: cio.apiKey, Secret: cio.apiSecret}

	// Add headers
	httpReq.Header.Set("Content-Type", "application/x-www-form-urlencoded")
	httpReq.Header.Set("Accept", "application/json")
	httpReq.Header.Set("Accept-Charset", "utf-8")
	httpReq.Header.Set("User-Agent", "Golang CIO Library")
	httpReq.Header.Set("Authorization", client.AuthorizationHeader(nil, request.Method, httpReq.URL, bodyValues))

	return httpReq, nil
}

// sendRequest sends the *http.Request
func (cio Cio) sendRequest(httpReq *http.Request, result interface{}, cioURL string) error {
	// Create the HTTP client
	httpClient := &http.Client{
		Transport: http.DefaultTransport,
		Timeout:   cio.RequestTimeout,
	}

	// Make the request
	res, err := httpClient.Do(httpReq)
	if err != nil {
		return RequestError{errors.Wrap(err, "CIO: Failed to make request"), ErrorMetaData{Method: httpReq.Method, URL: cioURL}}
	}

	// Parse the response
	defer func() {
		if closeErr := res.Body.Close(); closeErr != nil {
			logBodyCloseError(cio.Log, closeErr)
		}
	}()

	resBody, err := ioutil.ReadAll(res.Body)
	resBodyString := string(resBody)
	if err != nil {
		return RequestError{errors.Wrap(err, "CIO: Could not read response"), ErrorMetaData{Method: httpReq.Method, URL: cioURL, StatusCode: res.StatusCode, Payload: resBodyString}}
	}

	// Unmarshal result
	err = json.Unmarshal(resBody, &result)

	// Log the response
	logResponse(cio.Log, httpReq.Method, cioURL, res.StatusCode, resBodyString, err)

	// Return own error if Status Code >= 400
	if res.StatusCode >= 400 {
		return RequestError{errors.New("CIO: Status Code >= 400"), ErrorMetaData{Method: httpReq.Method, URL: cioURL, StatusCode: res.StatusCode, Payload: resBodyString}}
	}

	// Return Unmarshal error (if any) if Status Code is < 400
	if err != nil {
		return RequestError{errors.Wrap(err, "CIO: Could not unmarshal payload"), ErrorMetaData{Method: httpReq.Method, URL: cioURL, StatusCode: res.StatusCode, Payload: resBodyString}}
	}
	return nil
}

// logRequest logs the request about to be made to CIO, redacting sensitive information in the body
func logRequest(log Logger, method string, cioURL string, bodyValues url.Values) {
	if log != nil {

		// Copy url.Values
		redactedValues := url.Values{}
		for k, v := range bodyValues {
			redactedValues[k] = v
		}

		// Redact sensitive information
		if val := redactedValues.Get("password"); len(val) > 0 {
			redactedValues.Set("password", "redacted")
		}
		if val := redactedValues.Get("provider_refresh_token"); len(val) > 0 {
			redactedValues.Set("provider_refresh_token", "redacted")
		}
		if val := redactedValues.Get("provider_consumer_key"); len(val) > 0 {
			redactedValues.Set("provider_consumer_key", "redacted")
		}
		if val := redactedValues.Get("provider_consumer_secret"); len(val) > 0 {
			redactedValues.Set("provider_consumer_secret", "redacted")
		}

		// Actually log
		if logrusLogger, ok := log.(*logrus.Logger); ok {
			// If logrus, use structured fields
			logrusLogger.WithFields(logrus.Fields{"httpMethod": method, "url": cioURL, "payload": redactedValues.Encode()}).Debug("Creating new request to CIO")
		} else {
			// Else just log with Println
			log.Printf("Creating new %s request to: %s with payload: %s\n", method, cioURL, redactedValues.Encode())
		}
	}
}

// logBodyCloseError logs any error that happens when trying to close the *http.Response.Body
func logBodyCloseError(log Logger, closeError error) {
	if log != nil {
		if logrusLogger, ok := log.(*logrus.Logger); ok {
			// If logrus, use structured fields
			logrusLogger.WithError(closeError).Warn("Unable to close response body from CIO")
		} else {
			// Else just log with Println
			log.Printf("Unable to close response body from CIO, with error: %s\n", closeError.Error())
		}
	}
}

// logResponse logs the response from CIO, if any logger is set
func logResponse(log Logger, method string, cioURL string, statusCode int, responseBody string, unmarshalError error) {
	if log != nil {

		// TODO: redact access_token and access_token_secret before logging (only occurs with 3-legged oauth [rare])

		// Take only the first 2000 characters from the responseBody, which should be more than enough to debug anything
		if bodyLen := len(responseBody); bodyLen > 2000 {
			responseBody = responseBody[:2000]
		}

		if logrusLogger, ok := log.(*logrus.Logger); ok {
			// If logrus, use structured fields
			logEntry := logrusLogger.WithFields(logrus.Fields{
				"httpMethod":     method,
				"url":            cioURL,
				"statusCode":     fmt.Sprintf("%d", statusCode),
				"payloadSnippet": responseBody})
			if unmarshalError != nil || statusCode >= 400 {
				logEntry.Warn("Received response from CIO")
			} else {
				logEntry.Debug("Received response from CIO")
			}

		} else {
			// Else just log with Println
			log.Printf("Received response from %s to: %s with status code: %d and payload snippet: %s\n", method, cioURL, statusCode, responseBody)
		}
	}
}
<|endoftext|>


<commit_before>// Copyright The OpenTelemetry Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//       http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package tests contains test cases. To run the tests go to tests directory and run:
// RUN_TESTBED=1 go test -v

package tests

import (
	"fmt"
	"path/filepath"
	"testing"
	"time"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"

	"github.com/open-telemetry/opentelemetry-collector-contrib/testbed/testbed"
)

func TestIdleMode(t *testing.T) {

	options := testbed.LoadOptions{DataItemsPerSecond: 10_000, ItemsPerBatch: 10}
	dataProvider := testbed.NewPerfTestDataProvider(options)

	resultDir, err := filepath.Abs(filepath.Join("results", t.Name()))
	require.NoError(t, err)

	sender := testbed.NewOTLPTraceDataSender(testbed.DefaultHost, testbed.GetAvailablePort(t))
	receiver := testbed.NewOTLPDataReceiver(testbed.GetAvailablePort(t))
	cfg := createConfigYaml(t, sender, receiver, resultDir, nil, nil)
	cp := testbed.NewChildProcessCollector()
	cleanup, err := cp.PrepareConfig(cfg)
	require.NoError(t, err)
	t.Cleanup(cleanup)

	tc := testbed.NewTestCase(
		t,
		dataProvider,
		sender,
		receiver,
		cp,
		&testbed.PerfTestValidator{},
		performanceResultsSummary,
		testbed.WithResourceLimits(testbed.ResourceSpec{ExpectedMaxCPU: 20, ExpectedMaxRAM: 83}),
	)
	tc.StartAgent()

	tc.Sleep(tc.Duration)

	tc.Stop()
}

const ballastConfig = `
  memory_ballast:
    size_mib: %d
`

func TestBallastMemory(t *testing.T) {
	tests := []struct {
		ballastSize uint32
		maxRSS      uint32
	}{
		{100, 80},
		{500, 110},
		{1000, 120},
	}

	resultDir, err := filepath.Abs(filepath.Join("results", t.Name()))
	require.NoError(t, err)

	options := testbed.LoadOptions{DataItemsPerSecond: 10_000, ItemsPerBatch: 10}
	dataProvider := testbed.NewPerfTestDataProvider(options)
	for _, test := range tests {
		t.Run(fmt.Sprintf("ballast-size-%d", test.ballastSize), func(t *testing.T) {
			sender := testbed.NewOTLPTraceDataSender(testbed.DefaultHost, testbed.GetAvailablePort(t))
			receiver := testbed.NewOTLPDataReceiver(testbed.GetAvailablePort(t))
			ballastCfg := createConfigYaml(
				t, sender, receiver, resultDir, nil,
				map[string]string{"memory_ballast": fmt.Sprintf(ballastConfig, test.ballastSize)})
			cp := testbed.NewChildProcessCollector()
			cleanup, err := cp.PrepareConfig(ballastCfg)
			require.NoError(t, err)
			tc := testbed.NewTestCase(
				t,
				dataProvider,
				sender,
				receiver,
				cp,
				&testbed.PerfTestValidator{},
				performanceResultsSummary,
				testbed.WithSkipResults(),
				testbed.WithResourceLimits(
					testbed.ResourceSpec{
						ExpectedMaxRAM:         test.maxRSS,
						ResourceCheckPeriod:    time.Second,
						MaxConsecutiveFailures: 5,
					},
				),
			)
			tc.StartAgent()

			var rss, vms uint32
			// It is possible that the process is not ready or the ballast code path
			// is not hit immediately so we give the process up to a couple of seconds
			// to fire up and setup ballast. 2 seconds is a long time for this case but
			// it is short enough to not be annoying if the test fails repeatedly
			tc.WaitForN(func() bool {
				rss, vms, _ = tc.AgentMemoryInfo()
				return vms > test.ballastSize
			}, time.Second*5, fmt.Sprintf("VMS must be greater than %d", test.ballastSize))

			// https://github.com/open-telemetry/opentelemetry-collector/issues/3233
			// given that the maxRSS isn't an absolute maximum and that the actual maximum might be a bit off,
			// we give some room here instead of failing when the memory usage isn't that much higher than the max
			lenientMax := 1.1 * float32(test.maxRSS)

			// https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/6927#issuecomment-1138624098
			// During garbage collection, we may observe the ballast in rss.
			// If this happens, allow a brief window for garbage collection to complete.
			garbageCollectionMax := lenientMax + float32(test.ballastSize)

			rssTooHigh := fmt.Sprintf("The RSS memory usage (%d) is >10%% higher than the limit (%d).", rss, test.maxRSS)

			if rss > test.ballastSize && float32(rss) <= garbageCollectionMax {
				t.Log("Possible garbage collection under way. Remeasuring RSS.")
				tc.WaitForN(func() bool {
					rss, vms, _ = tc.AgentMemoryInfo()
					return float32(rss) <= lenientMax
				}, time.Second*5, rssTooHigh)
			} else {
				assert.LessOrEqual(t, float32(rss), lenientMax, rssTooHigh)
			}

			cleanup()
			tc.Stop()
		})
	}
}
<commit_msg>[testbed] Allow alternate memory range when ballast appears to be in RSS (#12407)<commit_after>// Copyright The OpenTelemetry Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//       http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package tests contains test cases. To run the tests go to tests directory and run:
// RUN_TESTBED=1 go test -v

package tests

import (
	"fmt"
	"path/filepath"
	"testing"
	"time"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"

	"github.com/open-telemetry/opentelemetry-collector-contrib/testbed/testbed"
)

func TestIdleMode(t *testing.T) {

	options := testbed.LoadOptions{DataItemsPerSecond: 10_000, ItemsPerBatch: 10}
	dataProvider := testbed.NewPerfTestDataProvider(options)

	resultDir, err := filepath.Abs(filepath.Join("results", t.Name()))
	require.NoError(t, err)

	sender := testbed.NewOTLPTraceDataSender(testbed.DefaultHost, testbed.GetAvailablePort(t))
	receiver := testbed.NewOTLPDataReceiver(testbed.GetAvailablePort(t))
	cfg := createConfigYaml(t, sender, receiver, resultDir, nil, nil)
	cp := testbed.NewChildProcessCollector()
	cleanup, err := cp.PrepareConfig(cfg)
	require.NoError(t, err)
	t.Cleanup(cleanup)

	tc := testbed.NewTestCase(
		t,
		dataProvider,
		sender,
		receiver,
		cp,
		&testbed.PerfTestValidator{},
		performanceResultsSummary,
		testbed.WithResourceLimits(testbed.ResourceSpec{ExpectedMaxCPU: 20, ExpectedMaxRAM: 83}),
	)
	tc.StartAgent()

	tc.Sleep(tc.Duration)

	tc.Stop()
}

const ballastConfig = `
  memory_ballast:
    size_mib: %d
`

func TestBallastMemory(t *testing.T) {
	tests := []struct {
		ballastSize uint32
		maxRSS      uint32
	}{
		{100, 80},
		{500, 110},
		{1000, 120},
	}

	resultDir, err := filepath.Abs(filepath.Join("results", t.Name()))
	require.NoError(t, err)

	options := testbed.LoadOptions{DataItemsPerSecond: 10_000, ItemsPerBatch: 10}
	dataProvider := testbed.NewPerfTestDataProvider(options)
	for _, test := range tests {
		t.Run(fmt.Sprintf("ballast-size-%d", test.ballastSize), func(t *testing.T) {
			sender := testbed.NewOTLPTraceDataSender(testbed.DefaultHost, testbed.GetAvailablePort(t))
			receiver := testbed.NewOTLPDataReceiver(testbed.GetAvailablePort(t))
			ballastCfg := createConfigYaml(
				t, sender, receiver, resultDir, nil,
				map[string]string{"memory_ballast": fmt.Sprintf(ballastConfig, test.ballastSize)})
			cp := testbed.NewChildProcessCollector()
			cleanup, err := cp.PrepareConfig(ballastCfg)
			require.NoError(t, err)
			tc := testbed.NewTestCase(
				t,
				dataProvider,
				sender,
				receiver,
				cp,
				&testbed.PerfTestValidator{},
				performanceResultsSummary,
				testbed.WithSkipResults(),
				testbed.WithResourceLimits(
					testbed.ResourceSpec{
						ExpectedMaxRAM:         test.maxRSS,
						ResourceCheckPeriod:    time.Second,
						MaxConsecutiveFailures: 5,
					},
				),
			)
			tc.StartAgent()

			var rss, vms uint32
			// It is possible that the process is not ready or the ballast code path
			// is not hit immediately so we give the process up to a couple of seconds
			// to fire up and setup ballast. 2 seconds is a long time for this case but
			// it is short enough to not be annoying if the test fails repeatedly
			tc.WaitForN(func() bool {
				rss, vms, _ = tc.AgentMemoryInfo()
				return vms > test.ballastSize
			}, time.Second*5, fmt.Sprintf("VMS must be greater than %d", test.ballastSize))

			// https://github.com/open-telemetry/opentelemetry-collector/issues/3233
			// given that the maxRSS isn't an absolute maximum and that the actual maximum might be a bit off,
			// we give some room here instead of failing when the memory usage isn't that much higher than the max
			lenientMax := 1.1 * float32(test.maxRSS)

			// https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/6927#issuecomment-1138624098
			// During garbage collection, we may observe the ballast in rss.
			// If this happens, adjust the baseline expectation for RSS size and validate that additional memory is
			// still within the expected limit.
			garbageCollectionMax := lenientMax + float32(test.ballastSize)

			rssTooHigh := fmt.Sprintf("The RSS memory usage (%d) is >10%% higher than the limit (%d).", rss, test.maxRSS)

			if rss > test.ballastSize {
				assert.LessOrEqual(t, float32(rss), garbageCollectionMax, rssTooHigh)
			} else {
				assert.LessOrEqual(t, float32(rss), lenientMax, rssTooHigh)
			}

			cleanup()
			tc.Stop()
		})
	}
}
<|endoftext|>


<commit_before>package tests

import "github.com/thingfu/hub/api"

type MockThingManager struct {
	thingTypes map[string]api.ThingType
	things     map[string]api.Thing

	rulesService api.RulesManager
	container    api.Container
	factory      api.Factory
	dataSource   api.DataSource
}

func (s *MockThingManager) SetDataSource(svc api.DataSource) {
	s.dataSource = svc
}

func (s *MockThingManager) SetRulesManager(svc api.RulesManager) {
	s.rulesService = svc
}

func (s *MockThingManager) SetFactory(o api.Factory) {
	s.factory = o
}

func (d *MockThingManager) GetContainer() api.Container {
	return d.container
}

func (d *MockThingManager) SetContainer(c api.Container) {
	d.container = c
}

func (o *MockThingManager) GetThing(id string) (dev api.Thing, ok bool) {
	dev, ok = o.things[id]

	return
}

func (o *MockThingManager) SaveThing(d api.Thing) {

}

func (o *MockThingManager) GetThingType(id string) api.ThingType {
	return o.thingTypes[id]
}

func (o *MockThingManager) GetThingTypes() map[string]api.ThingType {
	return o.thingTypes
}

func (o *MockThingManager) RegisterThingType(d api.ThingType) {

}

func (o *MockThingManager) RegisterThing(d api.Thing) {

}

func (o *MockThingManager) GetThings() []api.Thing {
	v := make(api.Things, len(o.things))

	return v
}

func (o *MockThingManager) Handle(thing *api.Thing, service *api.ThingService, state map[string]interface{}) {

}

func (o *MockThingManager) Cycle() {

}

func Actuate(t *api.Thing, op string, params map[string]interface{}) {

}

func (s *MockThingManager) ValidateWiring() {

}
<commit_msg>Fix the signature of actuate method<commit_after>package tests

import "github.com/thingfu/hub/api"

type MockThingManager struct {
	thingTypes map[string]api.ThingType
	things     map[string]api.Thing

	rulesService api.RulesManager
	container    api.Container
	factory      api.Factory
	dataSource   api.DataSource
}

func (s *MockThingManager) SetDataSource(svc api.DataSource) {
	s.dataSource = svc
}

func (s *MockThingManager) SetRulesManager(svc api.RulesManager) {
	s.rulesService = svc
}

func (s *MockThingManager) SetFactory(o api.Factory) {
	s.factory = o
}

func (d *MockThingManager) GetContainer() api.Container {
	return d.container
}

func (d *MockThingManager) SetContainer(c api.Container) {
	d.container = c
}

func (o *MockThingManager) GetThing(id string) (dev api.Thing, ok bool) {
	dev, ok = o.things[id]

	return
}

func (o *MockThingManager) SaveThing(d api.Thing) {

}

func (o *MockThingManager) GetThingType(id string) api.ThingType {
	return o.thingTypes[id]
}

func (o *MockThingManager) GetThingTypes() map[string]api.ThingType {
	return o.thingTypes
}

func (o *MockThingManager) RegisterThingType(d api.ThingType) {

}

func (o *MockThingManager) RegisterThing(d api.Thing) {

}

func (o *MockThingManager) GetThings() []api.Thing {
	v := make(api.Things, len(o.things))

	return v
}

func (o *MockThingManager) Handle(thing *api.Thing, service *api.ThingService, state map[string]interface{}) {

}

func (o *MockThingManager) Cycle() {

}

func (o *MockThingManager) Actuate(t *api.Thing, op string, params map[string]interface{}) {

}

func (s *MockThingManager) ValidateWiring() {

}
<|endoftext|>


<commit_before>package cli

import (
	"encoding/json"
	"fmt"
	"io"

	"github.com/codegangsta/cli"
	"polydawn.net/repeatr/executor/dispatch"
	"polydawn.net/repeatr/scheduler/dispatch"
)

func RunCommandPattern(output io.Writer) cli.Command {
	return cli.Command{
		Name:  "run",
		Usage: "Run a formula",
		Flags: []cli.Flag{
			cli.StringFlag{
				Name:  "executor, e",
				Value: "chroot",
				Usage: "Which executor to use",
			},
			cli.StringFlag{
				Name:  "scheduler, s",
				Value: "linear",
				Usage: "Which scheduler to use",
			},
			cli.StringFlag{
				Name:  "input, i",
				Usage: "Location of input formula (json format)",
			},
			cli.BoolFlag{
				Name:  "ignore-job-exit",
				Usage: "If true, repeatr will exit with 0/success even if the job exited nonzero.",
			},
		},
		Action: func(ctx *cli.Context) {
			// Parse args
			executor := executordispatch.Get(ctx.String("executor"))
			scheduler := schedulerdispatch.Get(ctx.String("scheduler"))
			formulaPaths := ctx.String("input")
			ignoreJobExit := ctx.Bool("ignore-job-exit")
			// Parse formula
			formula := LoadFormulaFromFile(formulaPaths)

			// TODO Don't reeeeally want the 'run once' command going through the schedulers.
			//  Having a path that doesn't invoke that complexity unnecessarily, and also is more clearly allowed to use the current terminal, is want.

			// Invoke!
			result := RunFormula(scheduler, executor, formula, ctx.App.Writer)
			// Exit if the job failed collosally (if it just had a nonzero exit code, that's acceptable).
			if result.Error != nil {
				panic(Error.NewWith("job execution errored", SetExitCode(EXIT_USER)))
			}

			// Output.
			// Note that all other logs, progress, terminals, etc are all routed to "journal" (typically, stderr),
			//  while this output is routed to "output" (typically, stdout), so it can be piped and parsed mechanically.
			msg, err := json.MarshalIndent(result.Outputs, "", "\t")
			if err != nil {
				panic(err)
			}
			fmt.Fprintf(output, "%s\n", string(msg))
			// Exit nonzero with our own "your job did not report success" indicator code, if applicable.
			if result.ExitCode != 0 && !ignoreJobExit {
				panic(Exit.NewWith("job finished with non-zero exit status", SetExitCode(EXIT_JOB)))
			}
		},
	}
}
<commit_msg>Emit full formulas from `repeatr run`.<commit_after>package cli

import (
	"io"

	"github.com/codegangsta/cli"
	"github.com/ugorji/go/codec"

	"polydawn.net/repeatr/executor/dispatch"
	"polydawn.net/repeatr/scheduler/dispatch"
)

func RunCommandPattern(output io.Writer) cli.Command {
	return cli.Command{
		Name:  "run",
		Usage: "Run a formula",
		Flags: []cli.Flag{
			cli.StringFlag{
				Name:  "executor, e",
				Value: "chroot",
				Usage: "Which executor to use",
			},
			cli.StringFlag{
				Name:  "scheduler, s",
				Value: "linear",
				Usage: "Which scheduler to use",
			},
			cli.StringFlag{
				Name:  "input, i",
				Usage: "Location of input formula (json format)",
			},
			cli.BoolFlag{
				Name:  "ignore-job-exit",
				Usage: "If true, repeatr will exit with 0/success even if the job exited nonzero.",
			},
		},
		Action: func(ctx *cli.Context) {
			// Parse args
			executor := executordispatch.Get(ctx.String("executor"))
			scheduler := schedulerdispatch.Get(ctx.String("scheduler"))
			formulaPaths := ctx.String("input")
			ignoreJobExit := ctx.Bool("ignore-job-exit")
			// Parse formula
			formula := LoadFormulaFromFile(formulaPaths)

			// TODO Don't reeeeally want the 'run once' command going through the schedulers.
			//  Having a path that doesn't invoke that complexity unnecessarily, and also is more clearly allowed to use the current terminal, is want.

			// Invoke!
			result := RunFormula(scheduler, executor, formula, ctx.App.Writer)
			// Exit if the job failed collosally (if it just had a nonzero exit code, that's acceptable).
			if result.Error != nil {
				panic(Error.NewWith("job execution errored", SetExitCode(EXIT_USER)))
			}

			// Output.
			// Join the results structure with the original formula, and emit the whole thing,
			//  just to keep it traversals consistent.
			// Note that all other logs, progress, terminals, etc are all routed to "journal" (typically, stderr),
			//  while this output is routed to "output" (typically, stdout), so it can be piped and parsed mechanically.
			formula.Outputs = result.Outputs
			err := codec.NewEncoder(output, &codec.JsonHandle{}).Encode(formula)
			output.Write([]byte{'\n'})
			if err != nil {
				panic(err)
			}
			// Exit nonzero with our own "your job did not report success" indicator code, if applicable.
			if result.ExitCode != 0 && !ignoreJobExit {
				panic(Exit.NewWith("job finished with non-zero exit status", SetExitCode(EXIT_JOB)))
			}
		},
	}
}
<|endoftext|>


<commit_before>package gol

import (
	"fmt"
	"io"
	"io/ioutil"
	"log"
)

type channel struct {
	writers []io.Writer
	mapper  mapper
	logger  *log.Logger
}

var channels = map[level]*channel{
	LEVEL_CRITICAL: &channel{writers: make([]io.Writer, 0, 2), logger: log.New(ioutil.Discard, "C ", log.LstdFlags)},
	LEVEL_ERROR:    &channel{writers: make([]io.Writer, 0, 2), logger: log.New(ioutil.Discard, "E ", log.LstdFlags)},
	LEVEL_WARNING:  &channel{writers: make([]io.Writer, 0, 2), logger: log.New(ioutil.Discard, "W ", log.LstdFlags)},
	LEVEL_INFO:     &channel{writers: make([]io.Writer, 0, 2), logger: log.New(ioutil.Discard, "I ", log.LstdFlags)},
	LEVEL_DEBUG:    &channel{writers: make([]io.Writer, 0, 2), logger: log.New(ioutil.Discard, "D ", log.LstdFlags)},
}

func (c *channel) Printf(format string, values ...interface{}) {
	if c.mapper == nil {
		c.logger.Printf(format, values...)
	} else {
		c.logger.Printf(c.mapper(fmt.Sprintf(format, values...)))
	}
}

// Critical uses format and values to generate a log message on the critical channel.
func Critical(format string, values ...interface{}) {
	channels[LEVEL_CRITICAL].Printf(format, values...)
}

// Error uses format and values to generate a log message on the error channel.
func Error(format string, values ...interface{}) {
	channels[LEVEL_ERROR].Printf(format, values...)
}

// Warning uses format and values to generate a log message on the warning channel.
func Warning(format string, values ...interface{}) {
	channels[LEVEL_WARNING].Printf(format, values...)
}

// Info uses format and values to generate a log message on the info channel.
func Info(format string, values ...interface{}) {
	channels[LEVEL_INFO].Printf(format, values...)
}

// Debug uses format and values to generate a log message on the debug channel.
func Debug(format string, values ...interface{}) {
	channels[LEVEL_DEBUG].Printf(format, values...)
}

// Handle takes the given error and writes it to all channels that correspond to the given levels. If no level is
// given, the error will be published on the error channel.
func Handle(error error, levels ...level) {
	if len(levels) == 0 {
		Handle(error, LEVEL_ERROR)
	} else {
		for _, level := range levels {
			channels[level].Printf(error.Error())
		}
	}
}
<commit_msg>improved error handling<commit_after>package gol

import (
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"github.com/juju/errgo"
)

type channel struct {
	writers []io.Writer
	mapper  mapper
	logger  *log.Logger
}

var channels = map[level]*channel{
	LEVEL_CRITICAL: &channel{writers: make([]io.Writer, 0, 2), logger: log.New(ioutil.Discard, "C ", log.LstdFlags)},
	LEVEL_ERROR:    &channel{writers: make([]io.Writer, 0, 2), logger: log.New(ioutil.Discard, "E ", log.LstdFlags)},
	LEVEL_WARNING:  &channel{writers: make([]io.Writer, 0, 2), logger: log.New(ioutil.Discard, "W ", log.LstdFlags)},
	LEVEL_INFO:     &channel{writers: make([]io.Writer, 0, 2), logger: log.New(ioutil.Discard, "I ", log.LstdFlags)},
	LEVEL_DEBUG:    &channel{writers: make([]io.Writer, 0, 2), logger: log.New(ioutil.Discard, "D ", log.LstdFlags)},
}

func (c *channel) Printf(format string, values ...interface{}) {
	if c.mapper == nil {
		c.logger.Printf(format, values...)
	} else {
		c.logger.Printf(c.mapper(fmt.Sprintf(format, values...)))
	}
}

// Critical uses format and values to generate a log message on the critical channel.
func Critical(format string, values ...interface{}) {
	channels[LEVEL_CRITICAL].Printf(format, values...)
}

// Error uses format and values to generate a log message on the error channel.
func Error(format string, values ...interface{}) {
	channels[LEVEL_ERROR].Printf(format, values...)
}

// Warning uses format and values to generate a log message on the warning channel.
func Warning(format string, values ...interface{}) {
	channels[LEVEL_WARNING].Printf(format, values...)
}

// Info uses format and values to generate a log message on the info channel.
func Info(format string, values ...interface{}) {
	channels[LEVEL_INFO].Printf(format, values...)
}

// Debug uses format and values to generate a log message on the debug channel.
func Debug(format string, values ...interface{}) {
	channels[LEVEL_DEBUG].Printf(format, values...)
}

// Handle takes the given error and writes it to all channels that correspond to the given levels. If no level is
// given, the error will be published on the error channel.
func Handle(err error, levels ...level) {
	if len(levels) == 0 {
		Handle(err, LEVEL_ERROR)
	} else {
		for _, level := range levels {
			channels[level].Printf(errgo.Details(err))
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"path"

	"github.com/coreos/coreos-cloudinit/datasource"
	"github.com/coreos/coreos-cloudinit/initialize"
	"github.com/coreos/coreos-cloudinit/network"
	"github.com/coreos/coreos-cloudinit/system"
)

const version = "0.7.7"

func main() {
	var printVersion bool
	flag.BoolVar(&printVersion, "version", false, "Print the version and exit")

	var ignoreFailure bool
	flag.BoolVar(&ignoreFailure, "ignore-failure", false, "Exits with 0 status in the event of malformed input from user-data")

	var file string
	flag.StringVar(&file, "from-file", "", "Read user-data from provided file")

	var configdrive string
	flag.StringVar(&configdrive, "from-configdrive", "", "Read user-data from provided cloud-drive directory")

	var url string
	flag.StringVar(&url, "from-url", "", "Download user-data from provided url")

	var useProcCmdline bool
	flag.BoolVar(&useProcCmdline, "from-proc-cmdline", false, fmt.Sprintf("Parse %s for '%s=<url>', using the cloud-config served by an HTTP GET to <url>", datasource.ProcCmdlineLocation, datasource.ProcCmdlineCloudConfigFlag))

	var convertNetconf string
	flag.StringVar(&convertNetconf, "convert-netconf", "", "Read the network config provided in cloud-drive and translate it from the specified format into networkd unit files (requires the -from-configdrive flag)")

	var workspace string
	flag.StringVar(&workspace, "workspace", "/var/lib/coreos-cloudinit", "Base directory coreos-cloudinit should use to store data")

	var sshKeyName string
	flag.StringVar(&sshKeyName, "ssh-key-name", initialize.DefaultSSHKeyName, "Add SSH keys to the system with the given name")

	flag.Parse()

	if printVersion == true {
		fmt.Printf("coreos-cloudinit version %s\n", version)
		os.Exit(0)
	}

	var ds datasource.Datasource
	if file != "" {
		ds = datasource.NewLocalFile(file)
	} else if url != "" {
		ds = datasource.NewMetadataService(url)
	} else if configdrive != "" {
		ds = datasource.NewConfigDrive(configdrive)
	} else if useProcCmdline {
		ds = datasource.NewProcCmdline()
	} else {
		fmt.Println("Provide one of --from-file, --from-configdrive, --from-url or --from-proc-cmdline")
		os.Exit(1)
	}

	if convertNetconf != "" && configdrive == "" {
		fmt.Println("-convert-netconf flag requires -from-configdrive")
		os.Exit(1)
	}

	switch convertNetconf {
	case "":
	case "debian":
	default:
		fmt.Printf("Invalid option to -convert-netconf: '%s'. Supported options: 'debian'\n", convertNetconf)
		os.Exit(1)
	}

	fmt.Printf("Fetching user-data from datasource of type %q\n", ds.Type())
	userdataBytes, err := ds.Fetch()
	if err != nil {
		fmt.Printf("Failed fetching user-data from datasource: %v\n", err)
		if ignoreFailure {
			os.Exit(0)
		} else {
			os.Exit(1)
		}
	}

	env := initialize.NewEnvironment("/", workspace)
	if len(userdataBytes) > 0 {
		if err := processUserdata(string(userdataBytes), env); err != nil {
			fmt.Printf("Failed resolving user-data: %v\n", err)
			if !ignoreFailure {
				os.Exit(1)
			}
		}
	} else {
		fmt.Println("No user data to handle.")
	}

	if convertNetconf != "" {
		if err := processNetconf(convertNetconf, configdrive); err != nil {
			fmt.Printf("Failed to process network config: %v\n", err)
			if !ignoreFailure {
				os.Exit(1)
			}
		}
	}
}

func processUserdata(userdata string, env *initialize.Environment) error {
	userdata = env.Apply(userdata)

	parsed, err := initialize.ParseUserData(userdata)
	if err != nil {
		fmt.Printf("Failed parsing user-data: %v\n", err)
		return err
	}

	err = initialize.PrepWorkspace(env.Workspace())
	if err != nil {
		fmt.Printf("Failed preparing workspace: %v\n", err)
		return err
	}

	switch t := parsed.(type) {
	case initialize.CloudConfig:
		err = initialize.Apply(t, env)
	case system.Script:
		var path string
		path, err = initialize.PersistScriptInWorkspace(t, env.Workspace())
		if err == nil {
			var name string
			name, err = system.ExecuteScript(path)
			initialize.PersistUnitNameInWorkspace(name, env.Workspace())
		}
	}

	return err
}

func processNetconf(convertNetconf, configdrive string) error {
	openstackRoot := path.Join(configdrive, "openstack")
	metadataFilename := path.Join(openstackRoot, "latest", "meta_data.json")
	metadataBytes, err := ioutil.ReadFile(metadataFilename)
	if err != nil {
		return err
	}

	var metadata struct {
		NetworkConfig struct {
			ContentPath string `json:"content_path"`
		} `json:"network_config"`
	}
	if err := json.Unmarshal(metadataBytes, &metadata); err != nil {
		return err
	}
	configPath := metadata.NetworkConfig.ContentPath
	if configPath == "" {
		fmt.Printf("No network config specified in %q.\n", metadataFilename)
		return nil
	}

	netconfBytes, err := ioutil.ReadFile(path.Join(openstackRoot, configPath))
	if err != nil {
		return err
	}

	var interfaces []network.InterfaceGenerator
	switch convertNetconf {
	case "debian":
		interfaces, err = network.ProcessDebianNetconf(string(netconfBytes))
	default:
		return fmt.Errorf("Unsupported network config format %q", convertNetconf)
	}

	if err != nil {
		return err
	}

	if err := system.WriteNetworkdConfigs(interfaces); err != nil {
		return err
	}
	return system.RestartNetwork(interfaces)
}
<commit_msg>chore(coreos-cloudinit): bump to 0.7.7+git<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"path"

	"github.com/coreos/coreos-cloudinit/datasource"
	"github.com/coreos/coreos-cloudinit/initialize"
	"github.com/coreos/coreos-cloudinit/network"
	"github.com/coreos/coreos-cloudinit/system"
)

const version = "0.7.7+git"

func main() {
	var printVersion bool
	flag.BoolVar(&printVersion, "version", false, "Print the version and exit")

	var ignoreFailure bool
	flag.BoolVar(&ignoreFailure, "ignore-failure", false, "Exits with 0 status in the event of malformed input from user-data")

	var file string
	flag.StringVar(&file, "from-file", "", "Read user-data from provided file")

	var configdrive string
	flag.StringVar(&configdrive, "from-configdrive", "", "Read user-data from provided cloud-drive directory")

	var url string
	flag.StringVar(&url, "from-url", "", "Download user-data from provided url")

	var useProcCmdline bool
	flag.BoolVar(&useProcCmdline, "from-proc-cmdline", false, fmt.Sprintf("Parse %s for '%s=<url>', using the cloud-config served by an HTTP GET to <url>", datasource.ProcCmdlineLocation, datasource.ProcCmdlineCloudConfigFlag))

	var convertNetconf string
	flag.StringVar(&convertNetconf, "convert-netconf", "", "Read the network config provided in cloud-drive and translate it from the specified format into networkd unit files (requires the -from-configdrive flag)")

	var workspace string
	flag.StringVar(&workspace, "workspace", "/var/lib/coreos-cloudinit", "Base directory coreos-cloudinit should use to store data")

	var sshKeyName string
	flag.StringVar(&sshKeyName, "ssh-key-name", initialize.DefaultSSHKeyName, "Add SSH keys to the system with the given name")

	flag.Parse()

	if printVersion == true {
		fmt.Printf("coreos-cloudinit version %s\n", version)
		os.Exit(0)
	}

	var ds datasource.Datasource
	if file != "" {
		ds = datasource.NewLocalFile(file)
	} else if url != "" {
		ds = datasource.NewMetadataService(url)
	} else if configdrive != "" {
		ds = datasource.NewConfigDrive(configdrive)
	} else if useProcCmdline {
		ds = datasource.NewProcCmdline()
	} else {
		fmt.Println("Provide one of --from-file, --from-configdrive, --from-url or --from-proc-cmdline")
		os.Exit(1)
	}

	if convertNetconf != "" && configdrive == "" {
		fmt.Println("-convert-netconf flag requires -from-configdrive")
		os.Exit(1)
	}

	switch convertNetconf {
	case "":
	case "debian":
	default:
		fmt.Printf("Invalid option to -convert-netconf: '%s'. Supported options: 'debian'\n", convertNetconf)
		os.Exit(1)
	}

	fmt.Printf("Fetching user-data from datasource of type %q\n", ds.Type())
	userdataBytes, err := ds.Fetch()
	if err != nil {
		fmt.Printf("Failed fetching user-data from datasource: %v\n", err)
		if ignoreFailure {
			os.Exit(0)
		} else {
			os.Exit(1)
		}
	}

	env := initialize.NewEnvironment("/", workspace)
	if len(userdataBytes) > 0 {
		if err := processUserdata(string(userdataBytes), env); err != nil {
			fmt.Printf("Failed resolving user-data: %v\n", err)
			if !ignoreFailure {
				os.Exit(1)
			}
		}
	} else {
		fmt.Println("No user data to handle.")
	}

	if convertNetconf != "" {
		if err := processNetconf(convertNetconf, configdrive); err != nil {
			fmt.Printf("Failed to process network config: %v\n", err)
			if !ignoreFailure {
				os.Exit(1)
			}
		}
	}
}

func processUserdata(userdata string, env *initialize.Environment) error {
	userdata = env.Apply(userdata)

	parsed, err := initialize.ParseUserData(userdata)
	if err != nil {
		fmt.Printf("Failed parsing user-data: %v\n", err)
		return err
	}

	err = initialize.PrepWorkspace(env.Workspace())
	if err != nil {
		fmt.Printf("Failed preparing workspace: %v\n", err)
		return err
	}

	switch t := parsed.(type) {
	case initialize.CloudConfig:
		err = initialize.Apply(t, env)
	case system.Script:
		var path string
		path, err = initialize.PersistScriptInWorkspace(t, env.Workspace())
		if err == nil {
			var name string
			name, err = system.ExecuteScript(path)
			initialize.PersistUnitNameInWorkspace(name, env.Workspace())
		}
	}

	return err
}

func processNetconf(convertNetconf, configdrive string) error {
	openstackRoot := path.Join(configdrive, "openstack")
	metadataFilename := path.Join(openstackRoot, "latest", "meta_data.json")
	metadataBytes, err := ioutil.ReadFile(metadataFilename)
	if err != nil {
		return err
	}

	var metadata struct {
		NetworkConfig struct {
			ContentPath string `json:"content_path"`
		} `json:"network_config"`
	}
	if err := json.Unmarshal(metadataBytes, &metadata); err != nil {
		return err
	}
	configPath := metadata.NetworkConfig.ContentPath
	if configPath == "" {
		fmt.Printf("No network config specified in %q.\n", metadataFilename)
		return nil
	}

	netconfBytes, err := ioutil.ReadFile(path.Join(openstackRoot, configPath))
	if err != nil {
		return err
	}

	var interfaces []network.InterfaceGenerator
	switch convertNetconf {
	case "debian":
		interfaces, err = network.ProcessDebianNetconf(string(netconfBytes))
	default:
		return fmt.Errorf("Unsupported network config format %q", convertNetconf)
	}

	if err != nil {
		return err
	}

	if err := system.WriteNetworkdConfigs(interfaces); err != nil {
		return err
	}
	return system.RestartNetwork(interfaces)
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"log"
	"os"

	"github.com/coreos/coreos-cloudinit/datasource"
	"github.com/coreos/coreos-cloudinit/initialize"
	"github.com/coreos/coreos-cloudinit/system"
)

const version = "0.6.0+git"

func main() {
	var printVersion bool
	flag.BoolVar(&printVersion, "version", false, "Print the version and exit")

	var ignoreFailure bool
	flag.BoolVar(&ignoreFailure, "ignore-failure", false, "Exits with 0 status in the event of malformed input from user-data")

	var file string
	flag.StringVar(&file, "from-file", "", "Read user-data from provided file")

	var url string
	flag.StringVar(&url, "from-url", "", "Download user-data from provided url")

	var useProcCmdline bool
	flag.BoolVar(&useProcCmdline, "from-proc-cmdline", false, fmt.Sprintf("Parse %s for '%s=<url>', using the cloud-config served by an HTTP GET to <url>", datasource.ProcCmdlineLocation, datasource.ProcCmdlineCloudConfigFlag))

	var workspace string
	flag.StringVar(&workspace, "workspace", "/var/lib/coreos-cloudinit", "Base directory coreos-cloudinit should use to store data")

	var sshKeyName string
	flag.StringVar(&sshKeyName, "ssh-key-name", initialize.DefaultSSHKeyName, "Add SSH keys to the system with the given name")

	flag.Parse()

	if printVersion == true {
		fmt.Printf("coreos-cloudinit version %s\n", version)
		os.Exit(0)
	}

	if file != "" && url != "" && !useProcCmdline {
		fmt.Println("Provide one of --from-file, --from-url or --from-proc-cmdline")
		os.Exit(1)
	}

	var ds datasource.Datasource
	if file != "" {
		ds = datasource.NewLocalFile(file)
	} else if url != "" {
		ds = datasource.NewMetadataService(url)
	} else if useProcCmdline {
		ds = datasource.NewProcCmdline()
	} else {
		fmt.Println("Provide one of --from-file, --from-url or --from-proc-cmdline")
		os.Exit(1)
	}

	log.Printf("Fetching user-data from datasource of type %q", ds.Type())
	userdataBytes, err := ds.Fetch()
	if err != nil {
		log.Printf("Failed fetching user-data from datasource: %v", err)
		if ignoreFailure {
			os.Exit(0)
		} else {
			os.Exit(1)
		}
	}

	if len(userdataBytes) == 0 {
		log.Printf("No user data to handle, exiting.")
		os.Exit(0)
	}

	env := initialize.NewEnvironment("/", workspace)

	userdata := string(userdataBytes)
	userdata = env.Apply(userdata)

	parsed, err := initialize.ParseUserData(userdata)
	if err != nil {
		log.Printf("Failed parsing user-data: %v", err)
		if ignoreFailure {
			os.Exit(0)
		} else {
			os.Exit(1)
		}
	}

	err = initialize.PrepWorkspace(env.Workspace())
	if err != nil {
		log.Fatalf("Failed preparing workspace: %v", err)
	}

	switch t := parsed.(type) {
	case initialize.CloudConfig:
		err = initialize.Apply(t, env)
	case system.Script:
		var path string
		path, err = initialize.PersistScriptInWorkspace(t, env.Workspace())
		if err == nil {
			var name string
			name, err = system.ExecuteScript(path)
			initialize.PersistUnitNameInWorkspace(name, workspace)
		}
	}

	if err != nil {
		log.Fatalf("Failed resolving user-data: %v", err)
	}
}
<commit_msg>chore(cloudinit): remove superfluous check<commit_after>package main

import (
	"flag"
	"fmt"
	"log"
	"os"

	"github.com/coreos/coreos-cloudinit/datasource"
	"github.com/coreos/coreos-cloudinit/initialize"
	"github.com/coreos/coreos-cloudinit/system"
)

const version = "0.6.0+git"

func main() {
	var printVersion bool
	flag.BoolVar(&printVersion, "version", false, "Print the version and exit")

	var ignoreFailure bool
	flag.BoolVar(&ignoreFailure, "ignore-failure", false, "Exits with 0 status in the event of malformed input from user-data")

	var file string
	flag.StringVar(&file, "from-file", "", "Read user-data from provided file")

	var url string
	flag.StringVar(&url, "from-url", "", "Download user-data from provided url")

	var useProcCmdline bool
	flag.BoolVar(&useProcCmdline, "from-proc-cmdline", false, fmt.Sprintf("Parse %s for '%s=<url>', using the cloud-config served by an HTTP GET to <url>", datasource.ProcCmdlineLocation, datasource.ProcCmdlineCloudConfigFlag))

	var workspace string
	flag.StringVar(&workspace, "workspace", "/var/lib/coreos-cloudinit", "Base directory coreos-cloudinit should use to store data")

	var sshKeyName string
	flag.StringVar(&sshKeyName, "ssh-key-name", initialize.DefaultSSHKeyName, "Add SSH keys to the system with the given name")

	flag.Parse()

	if printVersion == true {
		fmt.Printf("coreos-cloudinit version %s\n", version)
		os.Exit(0)
	}

	var ds datasource.Datasource
	if file != "" {
		ds = datasource.NewLocalFile(file)
	} else if url != "" {
		ds = datasource.NewMetadataService(url)
	} else if useProcCmdline {
		ds = datasource.NewProcCmdline()
	} else {
		fmt.Println("Provide one of --from-file, --from-url or --from-proc-cmdline")
		os.Exit(1)
	}

	log.Printf("Fetching user-data from datasource of type %q", ds.Type())
	userdataBytes, err := ds.Fetch()
	if err != nil {
		log.Printf("Failed fetching user-data from datasource: %v", err)
		if ignoreFailure {
			os.Exit(0)
		} else {
			os.Exit(1)
		}
	}

	if len(userdataBytes) == 0 {
		log.Printf("No user data to handle, exiting.")
		os.Exit(0)
	}

	env := initialize.NewEnvironment("/", workspace)

	userdata := string(userdataBytes)
	userdata = env.Apply(userdata)

	parsed, err := initialize.ParseUserData(userdata)
	if err != nil {
		log.Printf("Failed parsing user-data: %v", err)
		if ignoreFailure {
			os.Exit(0)
		} else {
			os.Exit(1)
		}
	}

	err = initialize.PrepWorkspace(env.Workspace())
	if err != nil {
		log.Fatalf("Failed preparing workspace: %v", err)
	}

	switch t := parsed.(type) {
	case initialize.CloudConfig:
		err = initialize.Apply(t, env)
	case system.Script:
		var path string
		path, err = initialize.PersistScriptInWorkspace(t, env.Workspace())
		if err == nil {
			var name string
			name, err = system.ExecuteScript(path)
			initialize.PersistUnitNameInWorkspace(name, workspace)
		}
	}

	if err != nil {
		log.Fatalf("Failed resolving user-data: %v", err)
	}
}
<|endoftext|>


<commit_before>package crest

import (
	"os"
	"testing"
)

var testClient *Client

/* TestMain
 * ****************************************************************************
 * Main entry point for package testing, sets up fixture(s) for testing.      *
 * ****************************************************************************/
func TestMain(m *testing.M) {
	opts := DefaultOptions()
	// Disable TLS verification for Continuous Integration testing.
	opts.DisableTLS = true
	opts.UserAgent = "EveLib Testing Agent"
	testClient, _ = NewClient(opts)
	os.Exit(m.Run())
}

func TestCRESTError(t *testing.T) {
	err := testClient.get("some/endpoint", nil)
	if err == nil {
		t.Error("error returned was unexpectedly nil")
	}
}
<commit_msg>changed test fixture client's user agent<commit_after>package crest

import (
	"os"
	"testing"
)

var testClient *Client

/* TestMain
 * ****************************************************************************
 * Main entry point for package testing, sets up fixture(s) for testing.      *
 * ****************************************************************************/
func TestMain(m *testing.M) {
	opts := DefaultOptions()
	// Disable TLS verification for Continuous Integration testing.
	opts.DisableTLS = true
	opts.UserAgent = "EveLib Testing Agent (https://github.com/vivace-io/evelib)"
	testClient, _ = NewClient(opts)
	os.Exit(m.Run())
}

func TestCRESTError(t *testing.T) {
	/*err := testClient.get("some/endpoint", nil)
	if err == nil {
		t.Error("error returned was unexpectedly nil")
	}*/
}
<|endoftext|>


<commit_before>package gumble

import (
	"encoding/binary"
	"errors"
	"io"
)

const maximumPacketLength = 1024 * 1024 * 10 // 10 megabytes

var (
	errInvalidArgument  = errors.New("invalid argument passed to function")
	errPacketReadType   = errors.New("could not read packet type")
	errPacketReadLength = errors.New("could not read packet length")
	errPacketLength     = errors.New("packet data is too large")
	errPacketRead       = errors.New("packet read error")
)

// serverIncoming reads protobuffer messages from the server.
func clientIncoming(client *Client) {
	defer client.Close()

	conn := client.connection

	for {
		var pType uint16
		var pLength uint32

		if err := binary.Read(conn, binary.BigEndian, &pType); err != nil {
			return
		}
		if err := binary.Read(conn, binary.BigEndian, &pLength); err != nil {
			return
		}
		pLengthInt := int(pLength)
		if pLengthInt > maximumPacketLength {
			return
		}
		data := make([]byte, pLengthInt)
		if _, err := io.ReadFull(conn, data); err != nil {
			return
		}
		if handle, ok := handlers[pType]; ok {
			if err := handle(client, data); err != nil {
				// TODO: log error?
			}
		}
	}
}
<commit_msg>remove unused errors<commit_after>package gumble

import (
	"encoding/binary"
	"io"
)

const maximumPacketLength = 1024 * 1024 * 10 // 10 megabytes

// serverIncoming reads protobuffer messages from the server.
func clientIncoming(client *Client) {
	defer client.Close()

	conn := client.connection

	for {
		var pType uint16
		var pLength uint32

		if err := binary.Read(conn, binary.BigEndian, &pType); err != nil {
			return
		}
		if err := binary.Read(conn, binary.BigEndian, &pLength); err != nil {
			return
		}
		pLengthInt := int(pLength)
		if pLengthInt > maximumPacketLength {
			return
		}
		data := make([]byte, pLengthInt)
		if _, err := io.ReadFull(conn, data); err != nil {
			return
		}
		if handle, ok := handlers[pType]; ok {
			if err := handle(client, data); err != nil {
				// TODO: log error?
			}
		}
	}
}
<|endoftext|>


<commit_before>/*
This package is just a collection of test cases
*/
package main

import (
	"github.com/codegangsta/cli"
	"os"
	"sort"
)

var (
	// flags
	fWant4 bool
	fWant6 bool
	fAllProbes bool
	fAsn string
	fCountry string
	fFieldList string
	fOptFields string
	fSortOrder string
	fVerbose bool
	fWantAnchor bool

	cliCommands []cli.Command
)

type ByAlphabet []cli.Command

func (a ByAlphabet) Len() int           { return len(a) }
func (a ByAlphabet) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a ByAlphabet) Less(i, j int) bool { return a[i].Name < a[j].Name }

// Check global parameters
func checkGlobalFlags(o map[string]string) (map[string]string) {
	opts := o
	if fSortOrder != "" {
		opts["sort"] = fSortOrder
	}

	if fFieldList != "" {
		opts["fields"] = fFieldList
	}

	if fOptFields != "" {
		opts["optional_fields"] = fOptFields
	}
	return opts
}

// main is the starting point (and everything)
func main() {
	app := cli.NewApp()
	app.Name = "atlas"
	app.Usage = "RIPE Atlas cli interface"
	app.Author = "Ollivier Robert <roberto@keltia.net>"
	app.Version = "0.0.1"
	app.HideVersion = true

	// General flags
	app.Flags = []cli.Flag{
		cli.BoolFlag{
			Name: "v",
			Usage: "verbose mode",
			Destination: &fVerbose,
		},
		cli.StringFlag{
			Name: "fields,F",
			Usage: "specify which fields are wanted",
			Destination: &fFieldList,
		},
		cli.StringFlag{
			Name: "opt-fields,O",
			Usage: "specify which optional fields are wanted",
			Destination: &fOptFields,
		},
		cli.StringFlag{
			Name:        "sort,S",
			Usage:       "sort results",
			Value:       "id",
			Destination: &fSortOrder,
		},
	}

	sort.Sort(ByAlphabet(cliCommands))
	app.Commands = cliCommands
	app.Run(os.Args)
}
<commit_msg>Add --format/-f to specify output format. Add func to validate.<commit_after>/*
This package is just a collection of test cases
*/
package main

import (
	"github.com/codegangsta/cli"
	"os"
	"sort"
	"strings"
)

var (
	// flags
	fWant4 bool
	fWant6 bool
	fAllProbes bool
	fAsn string
	fCountry string
	fFieldList string
	fFormat string
	fOptFields string
	fSortOrder string
	fVerbose bool
	fWantAnchor bool

	cliCommands []cli.Command
)

type ByAlphabet []cli.Command

func (a ByAlphabet) Len() int           { return len(a) }
func (a ByAlphabet) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a ByAlphabet) Less(i, j int) bool { return a[i].Name < a[j].Name }

// Check global parameters
func checkGlobalFlags(o map[string]string) (map[string]string) {
	opts := o
	if fSortOrder != "" {
		opts["sort"] = fSortOrder
	}

	if fFieldList != "" {
		opts["fields"] = fFieldList
	}

	if fOptFields != "" {
		opts["optional_fields"] = fOptFields
	}

	if fFormat != "" && validateFormat(fFormat) {
		opts["format"] = fFormat
	}
	return opts
}

// validateFormat allows only supported formats
func validateFormat(fmt string) bool {
	f := strings.ToLower(fmt)
	if f == "json" || f == "xml" || f == "api" || f == "txt" || f == "jsonp" {
		return true
	}
	return false
}

// main is the starting point (and everything)
func main() {
	app := cli.NewApp()
	app.Name = "atlas"
	app.Usage = "RIPE Atlas cli interface"
	app.Author = "Ollivier Robert <roberto@keltia.net>"
	app.Version = "0.0.1"
	app.HideVersion = true

	// General flags
	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name: "format,f",
			Usage: "specify output format",
			Destination: &fFormat,
		},
		cli.BoolFlag{
			Name: "v",
			Usage: "verbose mode",
			Destination: &fVerbose,
		},
		cli.StringFlag{
			Name: "fields,F",
			Usage: "specify which fields are wanted",
			Destination: &fFieldList,
		},
		cli.StringFlag{
			Name: "opt-fields,O",
			Usage: "specify which optional fields are wanted",
			Destination: &fOptFields,
		},
		cli.StringFlag{
			Name:        "sort,S",
			Usage:       "sort results",
			Value:       "id",
			Destination: &fSortOrder,
		},
	}

	sort.Sort(ByAlphabet(cliCommands))
	app.Commands = cliCommands
	app.Run(os.Args)
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/binary"
	kitlog "github.com/go-kit/kit/log"
	"github.com/jstesta/gomidi"
	"github.com/jstesta/gomidi/cfg"
	stdlog "log"
	"os"
)

func main() {

	logger := kitlog.NewLogfmtLogger(os.Stderr)
	stdlog.SetOutput(kitlog.NewStdlibAdapter(logger))

	ctx := kitlog.NewContext(logger).WithPrefix("ts", kitlog.DefaultTimestampUTC)

	m, err := gomidi.ReadMidiFromFile("resource/walzamin.mid", cfg.GomidiConfig{
		ByteOrder:  binary.BigEndian,
		LogContext: ctx,
	})
	if err != nil {
		stdlog.Fatal(err)
	}
	ctx.Log("midi", m)
}
<commit_msg>Main now accepts the input file via command line flags<commit_after>package main

import (
	"encoding/binary"
	kitlog "github.com/go-kit/kit/log"
	"github.com/jstesta/gomidi"
	"github.com/jstesta/gomidi/cfg"
	stdlog "log"
	"os"
	"flag"
)

func main() {
	var (
		midiFile = flag.String("input", "", "Filesystem location of MIDI file to parse")
	)
	flag.Parse()

	logger := kitlog.NewLogfmtLogger(os.Stderr)
	stdlog.SetOutput(kitlog.NewStdlibAdapter(logger))

	logger.Log("midiFile", midiFile)

	ctx := kitlog.NewContext(logger).WithPrefix("ts", kitlog.DefaultTimestampUTC)

	m, err := gomidi.ReadMidiFromFile(*midiFile, cfg.GomidiConfig{
		ByteOrder:  binary.BigEndian,
		LogContext: ctx,
	})
	if err != nil {
		stdlog.Fatal(err)
	}
	ctx.Log("midi", m)
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"os/user"
	"path/filepath"

	"github.com/deis/tiller/pkg/chart"
	"github.com/spf13/cobra"
)

const createDesc = `
This command creates a chart directory along with the common files and
directories used in a chart.

For example, 'helm create foo' will create a directory structure that looks
something like this:

	foo/
	  |- Chart.yaml    # Information about your chart
	  |
	  |- values.toml   # The default values for your templates
	  |
	  |- charts/       # Charts that this chart depends on
	  |
	  |- templates/    # The template files

'helm create' takes a path for an argument. If directories in the given path
do not exist, Helm will attempt to create them as it goes. If the given
destination exists and there are files in that directory, conflicting files
will be overwritten, but other files will be left alone.
`

func init() {
	RootCommand.AddCommand(createCmd)
}

var createCmd = &cobra.Command{
	Use:   "create [PATH]",
	Short: "Create a new chart at the location specified.",
	Long:  createDesc,
	RunE:  runCreate,
}

func runCreate(cmd *cobra.Command, args []string) error {
	if len(args) == 0 {
		return errors.New("the name of the new chart is required")
	}
	cname := args[0]
	cmd.Printf("Creating %s\n", cname)

	chartname := filepath.Base(cname)
	cfile := chart.Chartfile{
		Name:        chartname,
		Description: "A Helm chart for Kubernetes",
		Version:     "0.1.0",
		Maintainers: []*chart.Maintainer{
			{Name: username()},
		},
	}

	if _, err := chart.Create(&cfile, filepath.Dir(cname)); err != nil {
		return err
	}

	return nil
}

func username() string {
	uname := "Unknown"
	u, err := user.Current()
	if err == nil {
		uname = u.Name
		if uname == "" {
			uname = u.Username
		}
	}
	return uname
}
<commit_msg>fix(client): remove auto-generating maintainer<commit_after>package main

import (
	"errors"
	"path/filepath"

	"github.com/deis/tiller/pkg/chart"
	"github.com/spf13/cobra"
)

const createDesc = `
This command creates a chart directory along with the common files and
directories used in a chart.

For example, 'helm create foo' will create a directory structure that looks
something like this:

	foo/
	  |- Chart.yaml    # Information about your chart
	  |
	  |- values.toml   # The default values for your templates
	  |
	  |- charts/       # Charts that this chart depends on
	  |
	  |- templates/    # The template files

'helm create' takes a path for an argument. If directories in the given path
do not exist, Helm will attempt to create them as it goes. If the given
destination exists and there are files in that directory, conflicting files
will be overwritten, but other files will be left alone.
`

func init() {
	RootCommand.AddCommand(createCmd)
}

var createCmd = &cobra.Command{
	Use:   "create [PATH]",
	Short: "Create a new chart at the location specified.",
	Long:  createDesc,
	RunE:  runCreate,
}

func runCreate(cmd *cobra.Command, args []string) error {
	if len(args) == 0 {
		return errors.New("the name of the new chart is required")
	}
	cname := args[0]
	cmd.Printf("Creating %s\n", cname)

	chartname := filepath.Base(cname)
	cfile := chart.Chartfile{
		Name:        chartname,
		Description: "A Helm chart for Kubernetes",
		Version:     "0.1.0",
	}

	if _, err := chart.Create(&cfile, filepath.Dir(cname)); err != nil {
		return err
	}

	return nil
}
<|endoftext|>