<commit_before>package main

import (
	"fmt"
	"net/http"
)

var quit = make(chan int)

func makeDebugServer(name string) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprintln(w, name, ":", r.URL.Path)
	}
}

func main() {
	debug1 := http.NewServeMux()
	debug1.HandleFunc("/", makeDebugServer("whitehall"))
	debug2 := http.NewServeMux()
	debug2.HandleFunc("/", makeDebugServer("frontend"))

	go http.ListenAndServe(":6061", debug1)
	go http.ListenAndServe(":6062", debug2)

	<-quit
}
<commit_msg>Run more servers as part of the test server<commit_after>package main

import (
	"fmt"
	"io"
	"math/rand"
	"net/http"
	"runtime"
	"sync"
	"time"
)

var quit = make(chan int)

type randomDataMaker struct {
	sync.Mutex
	src rand.Source
}

// Read will fill the byte slice with random ASCII bytes
func (r *randomDataMaker) Read(p []byte) (n int, err error) {
	for i := range p {
		p[i] = byte(r.src.Int63() & 0x7f) // 0x7f ensures the result is ASCII
	}
	return len(p), nil
}

func makeDebugServer(name string) http.HandlerFunc {
	var randSrc = &randomDataMaker{src: rand.NewSource(time.Now().UTC().UnixNano())}

	return func(w http.ResponseWriter, r *http.Request) {
		randSrc.Lock()
		defer randSrc.Unlock()

		w.Header().Set("Content-Type", "text/plain")

		fmt.Fprintln(w, name, ":", r.URL.Path)
		fmt.Fprintln(w)

		// pick a size between 8 and 72KiB
		sz := 8192 + rand.Intn(1024*64)
		io.CopyN(w, randSrc, int64(sz))
	}
}

func main() {
	runtime.GOMAXPROCS(runtime.NumCPU())

	startPort := 6061
	servers := []string{"one", "two", "three", "four", "five", "six"}

	for i, s := range servers {
		mux := http.NewServeMux()
		mux.HandleFunc("/", makeDebugServer(s))
		addr := fmt.Sprintf(":%d", startPort+i)
		go http.ListenAndServe(addr, mux)
		fmt.Println("Server", s, "listening on", addr)
	}

	<-quit
}
<|endoftext|>


<commit_before>package expr

import (
	"fmt"
	"math"

	"github.com/grafana/metrictank/api/models"
	"github.com/grafana/metrictank/consolidation"
	"github.com/grafana/metrictank/schema"
	"github.com/grafana/metrictank/util"
)

// Normalize normalizes series to the same common LCM interval - if they don't already have the same interval
// any adjusted series gets created in a series drawn out of the pool and is added to the dataMap so it can be reclaimed
func Normalize(dataMap DataMap, in []models.Series) []models.Series {
	var intervals []uint32
	for _, s := range in {
		if s.Interval == 0 {
			panic("illegal interval 0")
		}
		intervals = append(intervals, s.Interval)
	}
	lcm := util.Lcm(intervals)
	for i, s := range in {
		if s.Interval != lcm {
			in[i] = NormalizeTo(dataMap, s, lcm)
		}
	}
	return in
}

func NormalizeTwo(dataMap DataMap, a, b models.Series) (models.Series, models.Series) {
	if a.Interval == b.Interval {
		return a, b
	}
	intervals := []uint32{a.Interval, b.Interval}
	lcm := util.Lcm(intervals)

	if a.Interval != lcm {
		a = NormalizeTo(dataMap, a, lcm)
	}
	if b.Interval != lcm {
		b = NormalizeTo(dataMap, b, lcm)
	}
	return a, b
}

// NormalizeTo normalizes the given series to the desired interval
// the following MUST be true when calling this:
// * interval > in.Interval
// * interval % in.Interval == 0
func NormalizeTo(dataMap DataMap, in models.Series, interval uint32) models.Series {

	if len(in.Datapoints) == 0 {
		panic(fmt.Sprintf("series %q cannot be normalized from interval %d to %d because it is empty", in.Target, in.Interval, interval))
	}

	// we need to copy the datapoints first because the consolidater will reuse the input slice
	// also, the input may not be pre-canonical. so add nulls in front and at the back to make it pre-canonical.
	// this may make points in front and at the back less accurate when consolidated (e.g. summing when some of the points are null results in a lower value)
	// but this is what graphite does....
	datapoints := pointSlicePool.Get().([]schema.Point)

	// example of how this works:
	// if in.Interval is 5, and interval is 15, then for example, to generate point 15, you want inputs 5, 10 and 15.
	// or more generally (you can follow any example vertically):
	//  5 10 15 20 25 30 35 40 45 50 <-- if any of these timestamps are your first point in `in`
	//  5  5  5 20 20 20 35 35 35 50 <-- then these are the corresponding timestamps of the first values we want as input for the consolidator
	// 15 15 15 30 30 30 45 45 45 60 <-- which, when fed through alignForward(), result in these numbers
	//  5  5  5 20 20 20 35 35 35 50 <-- subtract (aggnum-1)* in.interval or equivalent -interval + in.Interval = -15 + 5 = -10. these are our desired numbers!

	for ts := alignForward(in.Datapoints[0].Ts, interval) - interval + in.Interval; ts < in.Datapoints[0].Ts; ts += interval {
		datapoints = append(datapoints, schema.Point{Val: math.NaN(), Ts: ts})
	}

	datapoints = append(datapoints, in.Datapoints...)

	in.Datapoints = consolidation.Consolidate(datapoints, interval/in.Interval, in.Consolidator)
	in.Interval = interval
	dataMap.Add(Req{}, in)
	return in
}

// alignForward aligns ts to the next timestamp that divides by the interval, except if it is already aligned
func alignForward(ts, interval uint32) uint32 {
	remain := ts % interval
	if remain == 0 {
		return ts
	}
	return ts + interval - remain
}
<commit_msg>Normalize() bugfix: strip end points to assure canonical format<commit_after>package expr

import (
	"fmt"
	"math"

	"github.com/grafana/metrictank/api/models"
	"github.com/grafana/metrictank/consolidation"
	"github.com/grafana/metrictank/schema"
	"github.com/grafana/metrictank/util"
)

// Normalize normalizes series to the same common LCM interval - if they don't already have the same interval
// any adjusted series gets created in a series drawn out of the pool and is added to the dataMap so it can be reclaimed
func Normalize(dataMap DataMap, in []models.Series) []models.Series {
	var intervals []uint32
	for _, s := range in {
		if s.Interval == 0 {
			panic("illegal interval 0")
		}
		intervals = append(intervals, s.Interval)
	}
	lcm := util.Lcm(intervals)
	for i, s := range in {
		if s.Interval != lcm {
			in[i] = NormalizeTo(dataMap, s, lcm)
		}
	}
	return in
}

func NormalizeTwo(dataMap DataMap, a, b models.Series) (models.Series, models.Series) {
	if a.Interval == b.Interval {
		return a, b
	}
	intervals := []uint32{a.Interval, b.Interval}
	lcm := util.Lcm(intervals)

	if a.Interval != lcm {
		a = NormalizeTo(dataMap, a, lcm)
	}
	if b.Interval != lcm {
		b = NormalizeTo(dataMap, b, lcm)
	}
	return a, b
}

// NormalizeTo normalizes the given series to the desired interval
// the following MUST be true when calling this:
// * interval > in.Interval
// * interval % in.Interval == 0
func NormalizeTo(dataMap DataMap, in models.Series, interval uint32) models.Series {

	if len(in.Datapoints) == 0 {
		panic(fmt.Sprintf("series %q cannot be normalized from interval %d to %d because it is empty", in.Target, in.Interval, interval))
	}

	// we need to copy the datapoints first because the consolidater will reuse the input slice
	// also, the input may not be pre-canonical. so add nulls in front and at the back to make it pre-canonical.
	// this may make points in front and at the back less accurate when consolidated (e.g. summing when some of the points are null results in a lower value)
	// but this is what graphite does....
	datapoints := pointSlicePool.Get().([]schema.Point)

	// example of how this works:
	// if in.Interval is 5, and interval is 15, then for example, to generate point 15, you want inputs 5, 10 and 15.
	// or more generally (you can follow any example vertically):
	//  5 10 15 20 25 30 35 40 45 50 <-- if any of these timestamps are your first point in `in`
	//  5  5  5 20 20 20 35 35 35 50 <-- then these are the corresponding timestamps of the first values we want as input for the consolidator
	// 15 15 15 30 30 30 45 45 45 60 <-- which, when fed through alignForward(), result in these numbers
	//  5  5  5 20 20 20 35 35 35 50 <-- subtract (aggnum-1)* in.interval or equivalent -interval + in.Interval = -15 + 5 = -10. these are our desired numbers!

	// now, for the final value, it's important to be aware of cases like this:
	// until=47, interval=10, in.Interval = 5
	// a canonical 10s series would have as last point 40. whereas our input series will have 45, which will consolidate into a point with timestamp 50, which is incorrect
	// (it breaches `to`, and may have more points than other series it needs to be combined with)
	// thus, we also need to potentially trim points from the back until the last point has the same Ts as a canonical series would

	for ts := alignForward(in.Datapoints[0].Ts, interval) - interval + in.Interval; ts < in.Datapoints[0].Ts; ts += interval {
		datapoints = append(datapoints, schema.Point{Val: math.NaN(), Ts: ts})
	}

	datapoints = append(datapoints, in.Datapoints...)

	canonicalTs := (datapoints[len(datapoints)-1].Ts / interval) * interval
	numDrop := int((datapoints[len(datapoints)-1].Ts - canonicalTs) / in.Interval)
	datapoints = datapoints[0 : len(datapoints)-numDrop]

	in.Datapoints = consolidation.Consolidate(datapoints, interval/in.Interval, in.Consolidator)
	in.Interval = interval
	dataMap.Add(Req{}, in)
	return in
}

// alignForward aligns ts to the next timestamp that divides by the interval, except if it is already aligned
func alignForward(ts, interval uint32) uint32 {
	remain := ts % interval
	if remain == 0 {
		return ts
	}
	return ts + interval - remain
}
<|endoftext|>


<commit_before>package controllers

import (
	"net/http"

	"github.com/julienschmidt/httprouter"
	"github.com/unrolled/render"
)

//Convienance for the handle function
type action func(rw http.ResponseWriter, r *http.Request, ps httprouter.Params) error

//Controller is the interface all of our controllers must implement.
type Controller interface {
	//Register takes a router object and allows the controller to add its routes
	Register(router *httprouter.Router)
}

//This is simply a base struct for purposes of any global storage, to define
//handle off of, and for other controllers to inherit from.  It is not exported.
type baseController struct {
	render.Render
}

//This is a helper function for providing generic error handling for any controllers
//that choose to wrap their actions with it.  It's possible we could redefine this as ServeHTTP
//to avoid the need to call it, but we'll see.
func (c *baseController) handle(a action) httprouter.Handle {
	return httprouter.Handle(func(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
		err := a(w, r, ps)
		if err != nil {
			switch e := err.(type) {
			// This refers to controllers.Error
			case Error:
				// We can retrieve the status here and write out a specific
				// HTTP status code.
				// log.Printf("HTTP %d - %s", e.Status(), e)
				http.Error(w, e.Error(), e.Status())
			default:
				// Any error types we don't specifically look out for default
				// to serving a HTTP 500
				http.Error(w, http.StatusText(http.StatusInternalServerError),
					http.StatusInternalServerError)
			}
		}
	})
}
<commit_msg>Cleaned up the inheritence structure in controllers<commit_after>package controllers

import (
	"net/http"

	"github.com/julienschmidt/httprouter"
	"github.com/unrolled/render"
)

//Convienance for the handle function
type action func(rw http.ResponseWriter, r *http.Request, ps httprouter.Params) error

//Controller is the interface all of our controllers must implement.
type Controller interface {
	//Register takes a router object and allows the controller to add its routes
	Register(router *httprouter.Router)
}

//This is simply a base struct for purposes of any global storage, to define
//handle off of, and for other controllers to inherit from.  It is not exported.
type baseController struct {
	renderer render.Render
}

//This is a helper function for providing generic error handling for any controllers
//that choose to wrap their actions with it.  It's possible we could redefine this as ServeHTTP
//to avoid the need to call it, but we'll see.
func (c *baseController) handle(a action) httprouter.Handle {
	return httprouter.Handle(func(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
		err := a(w, r, ps)
		if err != nil {
			switch e := err.(type) {
			// This refers to controllers.Error
			case Error:
				// We can retrieve the status here and write out a specific
				// HTTP status code.
				// log.Printf("HTTP %d - %s", e.Status(), e)
				http.Error(w, e.Error(), e.Status())
			default:
				// Any error types we don't specifically look out for default
				// to serving a HTTP 500
				http.Error(w, http.StatusText(http.StatusInternalServerError),
					http.StatusInternalServerError)
			}
		}
	})
}
<|endoftext|>


<commit_before>package koding

import (
	"fmt"
	"strconv"

	"koding/kites/kloud/machinestate"
	"koding/kites/kloud/protocol"

	"github.com/mitchellh/goamz/ec2"
)

// Resizes increases the current machines underling volume to a larger volume
// without affecting or destroying the users data.
func (p *Provider) Resize(m *protocol.Machine) (resArtifact *protocol.Artifact, resErr error) {
	// Please read the steps before you dig into the code and try to change or
	// fix something. Intented lines are cleanup or self healing procedures
	// which should be called in a defer - arslan:
	//
	// 0. Check if size is eligible (not equal or less than the current size)
	// 1. Prepare/Get volumeId of current instance
	// 2. Prepare/Get availabilityZone of current instance
	// 3. Stop the instance so we can get the snapshot
	// 4. Create new snapshot from the current volumeId of that stopped instance
	//		4a. Delete snapshot after we are done with all following steps (not needed anymore)
	// 5. Create new volume with the desired size from the snapshot and same zone.
	//		5a. Delete volume if something goes wrong in following steps
	// 6. Detach the volume of current stopped instance.
	//    if something goes wrong:
	//		6a. Detach new volume, attach old volume. New volume will be
	//		    attached in the following step, so we are going to rewind it.
	//	  if everything is ok:
	//		6b. Delete old volume (not needed anymore)
	// 7. Attach new volume to current stopped instance
	// 8. Start the stopped instance with the new larger volume
	// 9. Update Default Domain record with the new IP (stopping/starting changes the IP)
	// 11 Update Domain aliases with the new IP (stopping/starting changes the IP)
	// 12. Check if Klient is running

	infoLog := p.GetCustomLogger(m.Id, "info")

	checker, err := p.PlanChecker(m)
	if err != nil {
		return nil, err
	}

	a, err := p.NewClient(m)
	if err != nil {
		return nil, err
	}

	// giving wantStorage as 0 since machine document already
	// has the max amount, we just want to check if its valid or not.
	if err := checker.Storage(0); err != nil {
		return nil, err
	}

	a.Push("Resizing initialized", 10, machinestate.Pending)

	infoLog("checking if size is eligible for instance %s", a.Id())
	instance, err := a.Instance(a.Id())
	if err != nil {
		return nil, err
	}

	if len(instance.BlockDevices) == 0 {
		return nil, fmt.Errorf("fatal error: no block device available")
	}

	// we need it in a lot of places!
	oldVolumeId := instance.BlockDevices[0].VolumeId

	oldVolResp, err := a.Client.Volumes([]string{oldVolumeId}, ec2.NewFilter())
	if err != nil {
		return nil, err
	}

	volSize := oldVolResp.Volumes[0].Size
	currentSize, err := strconv.Atoi(volSize)
	if err != nil {
		return nil, err
	}

	desiredSize := a.Builder.StorageSize

	a.Push("Checking if size is eligible", 20, machinestate.Pending)

	infoLog("user wants size '%dGB'. current storage size: '%dGB'", desiredSize, currentSize)
	if desiredSize <= currentSize {
		return nil, fmt.Errorf("resizing is not allowed. Desired size: %dGB should be larger than current size: %dGB",
			desiredSize, currentSize)
	}

	if desiredSize > 100 {
		return nil, fmt.Errorf("resizing is not allowed. Desired size: %d can't be larger than 100GB",
			desiredSize)
	}

	a.Push("Stopping old instance", 30, machinestate.Pending)
	infoLog("stopping instance %s", a.Id())
	if m.State != machinestate.Stopped {
		err = a.Stop(false)
		if err != nil {
			return nil, err
		}
	}

	a.Push("Creating new snapshot", 40, machinestate.Pending)
	infoLog("creating new snapshot from volume id %s", oldVolumeId)
	snapshotDesc := fmt.Sprintf("Temporary snapshot for instance %s", instance.InstanceId)
	snapshot, err := a.CreateSnapshot(oldVolumeId, snapshotDesc)
	if err != nil {
		return nil, err
	}
	newSnapshotId := snapshot.Id

	defer func() {
		infoLog("deleting snapshot %s (not needed anymore)", newSnapshotId)
		a.DeleteSnapshot(newSnapshotId)
	}()

	a.Push("Creating new volume", 50, machinestate.Pending)
	infoLog("creating volume from snapshot id %s with size: %d", newSnapshotId, desiredSize)

	// Go on with the current volume type. SSD(gp2) or Magnetic(standard)
	volType := oldVolResp.Volumes[0].VolumeType

	volume, err := a.CreateVolume(newSnapshotId, instance.AvailZone, volType, desiredSize)
	if err != nil {
		return nil, err
	}
	newVolumeId := volume.VolumeId

	// delete volume if something goes wrong in following steps
	defer func() {
		if resErr != nil {
			infoLog("(an error occurred) deleting new volume %s ", newVolumeId)
			_, err := a.Client.DeleteVolume(newVolumeId)
			if err != nil {
				a.Log.Error(err.Error())
			}
		}
	}()

	a.Push("Detaching old volume", 60, machinestate.Pending)
	infoLog("detaching current volume id %s", oldVolumeId)
	if err := a.DetachVolume(oldVolumeId); err != nil {
		return nil, err
	}

	// reattach old volume if something goes wrong, if not delete it
	defer func() {
		// if something goes wrong  detach the newly attached volume and attach
		// back the old volume  so it can be used again
		if resErr != nil {
			infoLog("(an error occurred) detaching newly created volume volume %s ", newVolumeId)
			_, err := a.Client.DetachVolume(newVolumeId)
			if err != nil {
				a.Log.Error(err.Error())
			}

			infoLog("(an error occurred) attaching back old volume %s", oldVolumeId)
			_, err = a.Client.AttachVolume(oldVolumeId, a.Id(), "/dev/sda1")
			if err != nil {
				a.Log.Error(err.Error())
			}
		} else {
			// if not just delete, it's not used anymore
			infoLog("deleting old volume %s (not needed anymore)", oldVolumeId)
			go a.Client.DeleteVolume(oldVolumeId)
		}
	}()

	a.Push("Attaching new volume", 70, machinestate.Pending)
	// attach new volume to current stopped instance
	if err := a.AttachVolume(newVolumeId, a.Id(), "/dev/sda1"); err != nil {
		return nil, err
	}

	a.Push("Starting instance", 80, machinestate.Pending)
	// start the stopped instance now as we attached the new volume
	artifact, err := a.Start(false)
	if err != nil {
		return nil, err
	}

	a.Push("Updating domain", 85, machinestate.Pending)
	// update Domain record with the new IP
	if err := p.UpdateDomain(artifact.IpAddress, m.Domain.Name, m.Username); err != nil {
		return nil, err
	}

	a.Push("Updating domain aliases", 87, machinestate.Pending)
	// also get all domain aliases that belongs to this machine and unset
	domains, err := p.DomainStorage.GetByMachine(m.Id)
	if err != nil {
		p.Log.Error("[%s] fetching domains for unsetting err: %s", m.Id, err.Error())
	}

	for _, domain := range domains {
		if err := p.UpdateDomain(artifact.IpAddress, domain.Name, m.Username); err != nil {
			p.Log.Error("[%s] couldn't update domain: %s", m.Id, err.Error())
		}
	}

	infoLog("updating user domain tag %s of instance %s", m.Domain.Name, artifact.InstanceId)
	if err := a.AddTag(artifact.InstanceId, "koding-domain", m.Domain.Name); err != nil {
		return nil, err
	}

	a.Push("Checking connectivity", 90, machinestate.Pending)
	artifact.DomainName = m.Domain.Name

	infoLog("connecting to remote Klient instance")
	if p.IsKlientReady(m.QueryString) {
		p.Log.Debug("[%s] klient is ready.", m.Id)
	} else {
		p.Log.Warning("[%s] klient is not ready. I couldn't connect to it.", m.Id)
	}

	return artifact, nil
}
<commit_msg>Kloud:Resize: storage checker should get the difference between db and aws<commit_after>package koding

import (
	"fmt"
	"strconv"

	"koding/kites/kloud/machinestate"
	"koding/kites/kloud/protocol"

	"github.com/mitchellh/goamz/ec2"
)

// Resizes increases the current machines underling volume to a larger volume
// without affecting or destroying the users data.
func (p *Provider) Resize(m *protocol.Machine) (resArtifact *protocol.Artifact, resErr error) {
	// Please read the steps before you dig into the code and try to change or
	// fix something. Intented lines are cleanup or self healing procedures
	// which should be called in a defer - arslan:
	//
	// 0. Check if size is eligible (not equal or less than the current size)
	// 1. Prepare/Get volumeId of current instance
	// 2. Prepare/Get availabilityZone of current instance
	// 3. Stop the instance so we can get the snapshot
	// 4. Create new snapshot from the current volumeId of that stopped instance
	//		4a. Delete snapshot after we are done with all following steps (not needed anymore)
	// 5. Create new volume with the desired size from the snapshot and same zone.
	//		5a. Delete volume if something goes wrong in following steps
	// 6. Detach the volume of current stopped instance.
	//    if something goes wrong:
	//		6a. Detach new volume, attach old volume. New volume will be
	//		    attached in the following step, so we are going to rewind it.
	//	  if everything is ok:
	//		6b. Delete old volume (not needed anymore)
	// 7. Attach new volume to current stopped instance
	// 8. Start the stopped instance with the new larger volume
	// 9. Update Default Domain record with the new IP (stopping/starting changes the IP)
	// 11 Update Domain aliases with the new IP (stopping/starting changes the IP)
	// 12. Check if Klient is running

	infoLog := p.GetCustomLogger(m.Id, "info")

	a, err := p.NewClient(m)
	if err != nil {
		return nil, err
	}

	a.Push("Resizing initialized", 10, machinestate.Pending)

	infoLog("checking if size is eligible for instance %s", a.Id())
	instance, err := a.Instance(a.Id())
	if err != nil {
		return nil, err
	}

	if len(instance.BlockDevices) == 0 {
		return nil, fmt.Errorf("fatal error: no block device available")
	}

	// we need it in a lot of places!
	oldVolumeId := instance.BlockDevices[0].VolumeId

	oldVolResp, err := a.Client.Volumes([]string{oldVolumeId}, ec2.NewFilter())
	if err != nil {
		return nil, err
	}

	volSize := oldVolResp.Volumes[0].Size
	currentSize, err := strconv.Atoi(volSize)
	if err != nil {
		return nil, err
	}

	desiredSize := a.Builder.StorageSize

	checker, err := p.PlanChecker(m)
	if err != nil {
	  return nil, err
	}

	// Storage is counting all current sizes. So we need ask only for the
	// difference that we want to add. So say if the current size is 3
	// and our desired size is 10, we need to ask if we have still
	// limit for a 7 GB space.
	if err := checker.Storage(desiredSize - currentSize); err != nil {
	  return nil, err
	}

	a.Push("Checking if size is eligible", 20, machinestate.Pending)

	infoLog("user wants size '%dGB'. current storage size: '%dGB'", desiredSize, currentSize)
	if desiredSize <= currentSize {
		return nil, fmt.Errorf("resizing is not allowed. Desired size: %dGB should be larger than current size: %dGB",
			desiredSize, currentSize)
	}

	if desiredSize > 100 {
		return nil, fmt.Errorf("resizing is not allowed. Desired size: %d can't be larger than 100GB",
			desiredSize)
	}

	a.Push("Stopping old instance", 30, machinestate.Pending)
	infoLog("stopping instance %s", a.Id())
	if m.State != machinestate.Stopped {
		err = a.Stop(false)
		if err != nil {
			return nil, err
		}
	}

	a.Push("Creating new snapshot", 40, machinestate.Pending)
	infoLog("creating new snapshot from volume id %s", oldVolumeId)
	snapshotDesc := fmt.Sprintf("Temporary snapshot for instance %s", instance.InstanceId)
	snapshot, err := a.CreateSnapshot(oldVolumeId, snapshotDesc)
	if err != nil {
		return nil, err
	}
	newSnapshotId := snapshot.Id

	defer func() {
		infoLog("deleting snapshot %s (not needed anymore)", newSnapshotId)
		a.DeleteSnapshot(newSnapshotId)
	}()

	a.Push("Creating new volume", 50, machinestate.Pending)
	infoLog("creating volume from snapshot id %s with size: %d", newSnapshotId, desiredSize)

	// Go on with the current volume type. SSD(gp2) or Magnetic(standard)
	volType := oldVolResp.Volumes[0].VolumeType

	volume, err := a.CreateVolume(newSnapshotId, instance.AvailZone, volType, desiredSize)
	if err != nil {
		return nil, err
	}
	newVolumeId := volume.VolumeId

	// delete volume if something goes wrong in following steps
	defer func() {
		if resErr != nil {
			infoLog("(an error occurred) deleting new volume %s ", newVolumeId)
			_, err := a.Client.DeleteVolume(newVolumeId)
			if err != nil {
				a.Log.Error(err.Error())
			}
		}
	}()

	a.Push("Detaching old volume", 60, machinestate.Pending)
	infoLog("detaching current volume id %s", oldVolumeId)
	if err := a.DetachVolume(oldVolumeId); err != nil {
		return nil, err
	}

	// reattach old volume if something goes wrong, if not delete it
	defer func() {
		// if something goes wrong  detach the newly attached volume and attach
		// back the old volume  so it can be used again
		if resErr != nil {
			infoLog("(an error occurred) detaching newly created volume volume %s ", newVolumeId)
			_, err := a.Client.DetachVolume(newVolumeId)
			if err != nil {
				a.Log.Error(err.Error())
			}

			infoLog("(an error occurred) attaching back old volume %s", oldVolumeId)
			_, err = a.Client.AttachVolume(oldVolumeId, a.Id(), "/dev/sda1")
			if err != nil {
				a.Log.Error(err.Error())
			}
		} else {
			// if not just delete, it's not used anymore
			infoLog("deleting old volume %s (not needed anymore)", oldVolumeId)
			go a.Client.DeleteVolume(oldVolumeId)
		}
	}()

	a.Push("Attaching new volume", 70, machinestate.Pending)
	// attach new volume to current stopped instance
	if err := a.AttachVolume(newVolumeId, a.Id(), "/dev/sda1"); err != nil {
		return nil, err
	}

	a.Push("Starting instance", 80, machinestate.Pending)
	// start the stopped instance now as we attached the new volume
	artifact, err := a.Start(false)
	if err != nil {
		return nil, err
	}

	a.Push("Updating domain", 85, machinestate.Pending)
	// update Domain record with the new IP
	if err := p.UpdateDomain(artifact.IpAddress, m.Domain.Name, m.Username); err != nil {
		return nil, err
	}

	a.Push("Updating domain aliases", 87, machinestate.Pending)
	// also get all domain aliases that belongs to this machine and unset
	domains, err := p.DomainStorage.GetByMachine(m.Id)
	if err != nil {
		p.Log.Error("[%s] fetching domains for unsetting err: %s", m.Id, err.Error())
	}

	for _, domain := range domains {
		if err := p.UpdateDomain(artifact.IpAddress, domain.Name, m.Username); err != nil {
			p.Log.Error("[%s] couldn't update domain: %s", m.Id, err.Error())
		}
	}

	infoLog("updating user domain tag %s of instance %s", m.Domain.Name, artifact.InstanceId)
	if err := a.AddTag(artifact.InstanceId, "koding-domain", m.Domain.Name); err != nil {
		return nil, err
	}

	a.Push("Checking connectivity", 90, machinestate.Pending)
	artifact.DomainName = m.Domain.Name

	infoLog("connecting to remote Klient instance")
	if p.IsKlientReady(m.QueryString) {
		p.Log.Debug("[%s] klient is ready.", m.Id)
	} else {
		p.Log.Warning("[%s] klient is not ready. I couldn't connect to it.", m.Id)
	}

	return artifact, nil
}
<|endoftext|>


<commit_before>package weed_server

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"math/rand"
	"net/http"
	"strconv"
	"strings"

	"github.com/chrislusf/seaweedfs/go/glog"
	"github.com/chrislusf/seaweedfs/go/operation"
	"github.com/chrislusf/seaweedfs/go/storage"
	"github.com/chrislusf/seaweedfs/go/topology"
	"github.com/chrislusf/seaweedfs/go/util"
	"github.com/golang/protobuf/proto"
)

func (ms *MasterServer) collectionDeleteHandler(w http.ResponseWriter, r *http.Request) {
	collection, ok := ms.Topo.GetCollection(r.FormValue("collection"))
	if !ok {
		writeJsonError(w, r, http.StatusBadRequest, fmt.Errorf("collection %s does not exist", r.FormValue("collection")))
		return
	}
	for _, server := range collection.ListVolumeServers() {
		_, err := util.Get("http://" + server.Ip + ":" + strconv.Itoa(server.Port) + "/admin/delete_collection?collection=" + r.FormValue("collection"))
		if err != nil {
			writeJsonError(w, r, http.StatusInternalServerError, err)
			return
		}
	}
	ms.Topo.DeleteCollection(r.FormValue("collection"))
}

func (ms *MasterServer) dirJoinHandler(w http.ResponseWriter, r *http.Request) {
	body, err := ioutil.ReadAll(r.Body)
	if err != nil {
		writeJsonError(w, r, http.StatusBadRequest, err)
		return
	}
	joinMessage := &operation.JoinMessage{}
	if err = proto.Unmarshal(body, joinMessage); err != nil {
		writeJsonError(w, r, http.StatusBadRequest, err)
		return
	}
	if *joinMessage.Ip == "" {
		*joinMessage.Ip = r.RemoteAddr[0:strings.LastIndex(r.RemoteAddr, ":")]
	}
	if glog.V(4) {
		if jsonData, jsonError := json.Marshal(joinMessage); jsonError != nil {
			glog.V(0).Infoln("json marshaling error: ", jsonError)
			writeJsonError(w, r, http.StatusBadRequest, jsonError)
			return
		} else {
			glog.V(4).Infoln("Proto size", len(body), "json size", len(jsonData), string(jsonData))
		}
	}

	ms.Topo.ProcessJoinMessage(joinMessage)
	writeJsonQuiet(w, r, http.StatusOK, operation.JoinResult{
		VolumeSizeLimit: uint64(ms.volumeSizeLimitMB) * 1024 * 1024,
		SecretKey:       string(ms.guard.SecretKey),
	})
}

func (ms *MasterServer) dirStatusHandler(w http.ResponseWriter, r *http.Request) {
	m := make(map[string]interface{})
	m["Version"] = util.VERSION
	m["Topology"] = ms.Topo.ToMap()
	writeJsonQuiet(w, r, http.StatusOK, m)
}

func (ms *MasterServer) volumeVacuumHandler(w http.ResponseWriter, r *http.Request) {
	gcThreshold := r.FormValue("garbageThreshold")
	if gcThreshold == "" {
		gcThreshold = ms.garbageThreshold
	}
	glog.Infoln("garbageThreshold =", gcThreshold)
	ms.Topo.Vacuum(gcThreshold)
	ms.dirStatusHandler(w, r)
}

func (ms *MasterServer) volumeGrowHandler(w http.ResponseWriter, r *http.Request) {
	count := 0
	option, err := ms.getVolumeGrowOption(r)
	if err != nil {
		writeJsonError(w, r, http.StatusNotAcceptable, err)
		return
	}
	if err == nil {
		if count, err = strconv.Atoi(r.FormValue("count")); err == nil {
			if ms.Topo.FreeSpace() < count*option.ReplicaPlacement.GetCopyCount() {
				err = errors.New("Only " + strconv.Itoa(ms.Topo.FreeSpace()) + " volumes left! Not enough for " + strconv.Itoa(count*option.ReplicaPlacement.GetCopyCount()))
			} else {
				count, err = ms.vg.GrowByCountAndType(count, option, ms.Topo)
			}
		} else {
			err = errors.New("parameter count is not found")
		}
	}
	if err != nil {
		writeJsonError(w, r, http.StatusNotAcceptable, err)
	} else {
		writeJsonQuiet(w, r, http.StatusOK, map[string]interface{}{"count": count})
	}
}

func (ms *MasterServer) volumeStatusHandler(w http.ResponseWriter, r *http.Request) {
	m := make(map[string]interface{})
	m["Version"] = util.VERSION
	m["Volumes"] = ms.Topo.ToVolumeMap()
	writeJsonQuiet(w, r, http.StatusOK, m)
}

func (ms *MasterServer) redirectHandler(w http.ResponseWriter, r *http.Request) {
	vid, _, _, _, _ := parseURLPath(r.URL.Path)
	volumeId, err := storage.NewVolumeId(vid)
	if err != nil {
		debug("parsing error:", err, r.URL.Path)
		return
	}
	machines := ms.Topo.Lookup("", volumeId)
	if machines != nil && len(machines) > 0 {
		var url string
		if r.URL.RawQuery != "" {
			url = util.NormalizeUrl(machines[rand.Intn(len(machines))].PublicUrl) + r.URL.Path + "?" + r.URL.RawQuery
		} else {
			url = util.NormalizeUrl(machines[rand.Intn(len(machines))].PublicUrl) + r.URL.Path
		}
		http.Redirect(w, r, url, http.StatusMovedPermanently)
	} else {
		writeJsonError(w, r, http.StatusNotFound, fmt.Errorf("volume id %d not found", volumeId))
	}
}

func (ms *MasterServer) selfUrl(r *http.Request) string {
	if r.Host != "" {
		return r.Host
	}
	return "localhost:" + strconv.Itoa(ms.port)
}
func (ms *MasterServer) submitFromMasterServerHandler(w http.ResponseWriter, r *http.Request) {
	if ms.Topo.IsLeader() {
		submitForClientHandler(w, r, ms.selfUrl(r))
	} else {
		masterUrl, err := ms.Topo.Leader()
		if err != nil {
			writeJsonError(w, r, http.StatusInternalServerError, err)
		} else {
			submitForClientHandler(w, r, masterUrl)
		}
	}
}

func (ms *MasterServer) deleteFromMasterServerHandler(w http.ResponseWriter, r *http.Request) {
	if ms.Topo.IsLeader() {
		deleteForClientHandler(w, r, ms.selfUrl(r))
	} else {
		deleteForClientHandler(w, r, ms.Topo.RaftServer.Leader())
	}
}

func (ms *MasterServer) HasWritableVolume(option *topology.VolumeGrowOption) bool {
	vl := ms.Topo.GetVolumeLayout(option.Collection, option.ReplicaPlacement, option.Ttl)
	return vl.GetActiveVolumeCount(option) > 0
}

func (ms *MasterServer) getVolumeGrowOption(r *http.Request) (*topology.VolumeGrowOption, error) {
	replicationString := r.FormValue("replication")
	if replicationString == "" {
		replicationString = ms.defaultReplicaPlacement
	}
	replicaPlacement, err := storage.NewReplicaPlacementFromString(replicationString)
	if err != nil {
		return nil, err
	}
	ttl, err := storage.ReadTTL(r.FormValue("ttl"))
	if err != nil {
		return nil, err
	}
	volumeGrowOption := &topology.VolumeGrowOption{
		Collection:       r.FormValue("collection"),
		ReplicaPlacement: replicaPlacement,
		Ttl:              ttl,
		DataCenter:       r.FormValue("dataCenter"),
		Rack:             r.FormValue("rack"),
		DataNode:         r.FormValue("dataNode"),
	}
	return volumeGrowOption, nil
}
<commit_msg>master: redirect handler accept collection parameter<commit_after>package weed_server

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"math/rand"
	"net/http"
	"strconv"
	"strings"

	"github.com/chrislusf/seaweedfs/go/glog"
	"github.com/chrislusf/seaweedfs/go/operation"
	"github.com/chrislusf/seaweedfs/go/storage"
	"github.com/chrislusf/seaweedfs/go/topology"
	"github.com/chrislusf/seaweedfs/go/util"
	"github.com/golang/protobuf/proto"
)

func (ms *MasterServer) collectionDeleteHandler(w http.ResponseWriter, r *http.Request) {
	collection, ok := ms.Topo.GetCollection(r.FormValue("collection"))
	if !ok {
		writeJsonError(w, r, http.StatusBadRequest, fmt.Errorf("collection %s does not exist", r.FormValue("collection")))
		return
	}
	for _, server := range collection.ListVolumeServers() {
		_, err := util.Get("http://" + server.Ip + ":" + strconv.Itoa(server.Port) + "/admin/delete_collection?collection=" + r.FormValue("collection"))
		if err != nil {
			writeJsonError(w, r, http.StatusInternalServerError, err)
			return
		}
	}
	ms.Topo.DeleteCollection(r.FormValue("collection"))
}

func (ms *MasterServer) dirJoinHandler(w http.ResponseWriter, r *http.Request) {
	body, err := ioutil.ReadAll(r.Body)
	if err != nil {
		writeJsonError(w, r, http.StatusBadRequest, err)
		return
	}
	joinMessage := &operation.JoinMessage{}
	if err = proto.Unmarshal(body, joinMessage); err != nil {
		writeJsonError(w, r, http.StatusBadRequest, err)
		return
	}
	if *joinMessage.Ip == "" {
		*joinMessage.Ip = r.RemoteAddr[0:strings.LastIndex(r.RemoteAddr, ":")]
	}
	if glog.V(4) {
		if jsonData, jsonError := json.Marshal(joinMessage); jsonError != nil {
			glog.V(0).Infoln("json marshaling error: ", jsonError)
			writeJsonError(w, r, http.StatusBadRequest, jsonError)
			return
		} else {
			glog.V(4).Infoln("Proto size", len(body), "json size", len(jsonData), string(jsonData))
		}
	}

	ms.Topo.ProcessJoinMessage(joinMessage)
	writeJsonQuiet(w, r, http.StatusOK, operation.JoinResult{
		VolumeSizeLimit: uint64(ms.volumeSizeLimitMB) * 1024 * 1024,
		SecretKey:       string(ms.guard.SecretKey),
	})
}

func (ms *MasterServer) dirStatusHandler(w http.ResponseWriter, r *http.Request) {
	m := make(map[string]interface{})
	m["Version"] = util.VERSION
	m["Topology"] = ms.Topo.ToMap()
	writeJsonQuiet(w, r, http.StatusOK, m)
}

func (ms *MasterServer) volumeVacuumHandler(w http.ResponseWriter, r *http.Request) {
	gcThreshold := r.FormValue("garbageThreshold")
	if gcThreshold == "" {
		gcThreshold = ms.garbageThreshold
	}
	glog.Infoln("garbageThreshold =", gcThreshold)
	ms.Topo.Vacuum(gcThreshold)
	ms.dirStatusHandler(w, r)
}

func (ms *MasterServer) volumeGrowHandler(w http.ResponseWriter, r *http.Request) {
	count := 0
	option, err := ms.getVolumeGrowOption(r)
	if err != nil {
		writeJsonError(w, r, http.StatusNotAcceptable, err)
		return
	}
	if err == nil {
		if count, err = strconv.Atoi(r.FormValue("count")); err == nil {
			if ms.Topo.FreeSpace() < count*option.ReplicaPlacement.GetCopyCount() {
				err = errors.New("Only " + strconv.Itoa(ms.Topo.FreeSpace()) + " volumes left! Not enough for " + strconv.Itoa(count*option.ReplicaPlacement.GetCopyCount()))
			} else {
				count, err = ms.vg.GrowByCountAndType(count, option, ms.Topo)
			}
		} else {
			err = errors.New("parameter count is not found")
		}
	}
	if err != nil {
		writeJsonError(w, r, http.StatusNotAcceptable, err)
	} else {
		writeJsonQuiet(w, r, http.StatusOK, map[string]interface{}{"count": count})
	}
}

func (ms *MasterServer) volumeStatusHandler(w http.ResponseWriter, r *http.Request) {
	m := make(map[string]interface{})
	m["Version"] = util.VERSION
	m["Volumes"] = ms.Topo.ToVolumeMap()
	writeJsonQuiet(w, r, http.StatusOK, m)
}

func (ms *MasterServer) redirectHandler(w http.ResponseWriter, r *http.Request) {
	vid, _, _, _, _ := parseURLPath(r.URL.Path)
	volumeId, err := storage.NewVolumeId(vid)
	if err != nil {
		debug("parsing error:", err, r.URL.Path)
		return
	}
	collection := r.FormValue("collection")
	machines := ms.Topo.Lookup(collection, volumeId)
	if machines != nil && len(machines) > 0 {
		var url string
		if r.URL.RawQuery != "" {
			url = util.NormalizeUrl(machines[rand.Intn(len(machines))].PublicUrl) + r.URL.Path + "?" + r.URL.RawQuery
		} else {
			url = util.NormalizeUrl(machines[rand.Intn(len(machines))].PublicUrl) + r.URL.Path
		}
		http.Redirect(w, r, url, http.StatusMovedPermanently)
	} else {
		writeJsonError(w, r, http.StatusNotFound, fmt.Errorf("volume id %d or collection %s not found", volumeId, collection))
	}
}

func (ms *MasterServer) selfUrl(r *http.Request) string {
	if r.Host != "" {
		return r.Host
	}
	return "localhost:" + strconv.Itoa(ms.port)
}
func (ms *MasterServer) submitFromMasterServerHandler(w http.ResponseWriter, r *http.Request) {
	if ms.Topo.IsLeader() {
		submitForClientHandler(w, r, ms.selfUrl(r))
	} else {
		masterUrl, err := ms.Topo.Leader()
		if err != nil {
			writeJsonError(w, r, http.StatusInternalServerError, err)
		} else {
			submitForClientHandler(w, r, masterUrl)
		}
	}
}

func (ms *MasterServer) deleteFromMasterServerHandler(w http.ResponseWriter, r *http.Request) {
	if ms.Topo.IsLeader() {
		deleteForClientHandler(w, r, ms.selfUrl(r))
	} else {
		deleteForClientHandler(w, r, ms.Topo.RaftServer.Leader())
	}
}

func (ms *MasterServer) HasWritableVolume(option *topology.VolumeGrowOption) bool {
	vl := ms.Topo.GetVolumeLayout(option.Collection, option.ReplicaPlacement, option.Ttl)
	return vl.GetActiveVolumeCount(option) > 0
}

func (ms *MasterServer) getVolumeGrowOption(r *http.Request) (*topology.VolumeGrowOption, error) {
	replicationString := r.FormValue("replication")
	if replicationString == "" {
		replicationString = ms.defaultReplicaPlacement
	}
	replicaPlacement, err := storage.NewReplicaPlacementFromString(replicationString)
	if err != nil {
		return nil, err
	}
	ttl, err := storage.ReadTTL(r.FormValue("ttl"))
	if err != nil {
		return nil, err
	}
	volumeGrowOption := &topology.VolumeGrowOption{
		Collection:       r.FormValue("collection"),
		ReplicaPlacement: replicaPlacement,
		Ttl:              ttl,
		DataCenter:       r.FormValue("dataCenter"),
		Rack:             r.FormValue("rack"),
		DataNode:         r.FormValue("dataNode"),
	}
	return volumeGrowOption, nil
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"net/http"
	"os"
	"regexp"
	"sort"
	"strconv"
	"time"

	proc "github.com/abulimov/cadvisor-companion/process"
)

var version = "0.0.5"

// set up cli vars
var argIP = flag.String("listen_ip", "", "IP to listen on, defaults to all IPs")
var argPort = flag.Int("port", 8801, "port to listen")
var versionFlag = flag.Bool("version", false, "print cAdvisor-companion version and exit")

// ProcessSnapshot represents slice of processes in time
type ProcessSnapshot struct {
	Timestamp time.Time `json:"timestamp"`
	Processes proc.List `json:"processes"`
}

// HistoryEntry containes all ProcessSnapshots for some moment in time
type HistoryEntry map[string]ProcessSnapshot

// history holds RRD-like array of last 60 history entries
var history [60]HistoryEntry

// getLastData returns data from history with added relative CPU usage
// offset (in seconds) lets us get data from the past.
// interval (in seconds) is used to calculate CPU usage
func getLastData(containerID string, interval, offset int) (*ProcessSnapshot, error) {
	last := len(history) - offset
	first := last - interval
	entry1 := history[first][containerID]
	entry2 := history[last][containerID]
	var procs proc.List
	cpu1 := entry1.Processes.GetCPUTotalUsage()
	cpu2 := entry2.Processes.GetCPUTotalUsage()
	for _, p2 := range entry2.Processes {
		p1 := entry1.Processes.FindProc(p2.Status.Pid)
		if p1 != nil {
			user := int64(p2.Stat.Utime - p1.Stat.Utime)
			system := int64(p2.Stat.Stime - p1.Stat.Stime)
			percent := (float64(user+system) / float64(cpu2-cpu1)) * 100
			p2.RelativeCPUUsage = percent
			procs = append(procs, p2)
		}

	}
	return &ProcessSnapshot{entry2.Timestamp, procs}, nil
}

// getTopCPU returns `limit` entries with top CPU usage
func getTopCPU(containerID string, limit, interval, offset int) (*ProcessSnapshot, error) {
	entry, err := getLastData(containerID, interval, offset)
	if err != nil {
		return nil, err
	}
	sort.Sort(sort.Reverse(proc.ByCPU(entry.Processes)))
	if limit > len(entry.Processes) {
		limit = len(entry.Processes)
	}
	var result ProcessSnapshot
	result.Timestamp = entry.Timestamp
	for _, p := range entry.Processes[:limit] {
		result.Processes = append(result.Processes, p)
	}
	return &result, nil
}

// getTopMem returns `limit` entries with top VmRSS usage
func getTopMem(containerID string, limit, interval, offset int) (*ProcessSnapshot, error) {
	entry, err := getLastData(containerID, interval, offset)
	if err != nil {
		return nil, err
	}
	sort.Sort(sort.Reverse(proc.ByRSS(entry.Processes)))
	if limit > len(entry.Processes) {
		limit = len(entry.Processes)
	}
	var result ProcessSnapshot
	result.Timestamp = entry.Timestamp
	for _, p := range entry.Processes[:limit] {
		result.Processes = append(result.Processes, p)
	}
	return &result, nil
}

// apiHandler handles http requests
func apiHandler(res http.ResponseWriter, req *http.Request) {
	var validPath = regexp.MustCompile("^/api/v1.0/([a-zA-Z0-9/]+)/processes$")

	// validate requested URL
	m := validPath.FindStringSubmatch(req.URL.Path)
	if m == nil {
		http.NotFound(res, req)
		return
	}

	// process get parameters

	// sortStr is used to get top sorted procs
	sortStr := req.URL.Query().Get("sort")

	// limit is used to limit top sorted procs
	limitStr := req.URL.Query().Get("limit")
	limit, err := strconv.Atoi(limitStr)
	if err != nil || limit < 1 {
		limit = 5
	}

	// interval is the interval we use to calculate CPU usage
	// and to iterate back to the past
	intervalStr := req.URL.Query().Get("interval")
	interval, err := strconv.Atoi(intervalStr)
	if err != nil || interval < 1 {
		interval = 1
	}

	// count is the count of resulting points in time
	countStr := req.URL.Query().Get("count")
	count, err := strconv.Atoi(countStr)
	if err != nil || count < 1 {
		count = 1
	}

	// containerID is the name of our container/cgroup
	containerID := "/" + m[1]

	// set response headers
	res.Header().Set(
		"Content-Type",
		"text/json",
	)
	var result []ProcessSnapshot
	var ps *ProcessSnapshot
	// case our possible sort parameters
	switch sortStr {
	case "cpu":
		for i := count - 1; i >= 0; i-- {
			ps, err = getTopCPU(containerID, limit, interval, i*interval+1)
			result = append(result, *ps)
		}
	case "mem":
		for i := count - 1; i >= 0; i-- {
			ps, err = getTopMem(containerID, limit, interval, i*interval+1)
			result = append(result, *ps)
		}
	case "":
		for i := count - 1; i >= 0; i-- {
			ps, err = getLastData(containerID, interval, i*interval+1)
			result = append(result, *ps)
		}
	}
	if err != nil {
		fmt.Println(err)
		res.WriteHeader(500) // HTTP 500
		io.WriteString(res, err.Error())
	}
	jsonResult, _ := json.Marshal(result)
	io.WriteString(res, string(jsonResult))
}

// collectData scrapes procs data for all containers every second
// and keeps it in global history var
func collectData(rootPath string) {
	for {
		timeStamp := time.Now()
		// get all processes without cgroup grouping
		allProcs, _ := proc.GetProcesses(rootPath)
		// group all processes by their cgroups
		cgroupsProcs := allProcs.GetCgroupsMap()

		entry := make(HistoryEntry)
		for e, p := range cgroupsProcs {
			entry[e] = ProcessSnapshot{timeStamp, p}
		}

		// rotate our history
		for i, v := range history[1:] {
			history[i] = v
		}
		history[len(history)-1] = entry
		time.Sleep(time.Second)
	}
}

func getRootPath() string {
	dir, err := os.Stat("/rootfs")
	if err != nil {
		return "/"
	}
	// check if the /rootfs is indeed a directory or not
	if !dir.IsDir() {
		return "/"
	}
	return "/rootfs"
}

func main() {
	flag.Parse()

	if *versionFlag {
		fmt.Printf("cAdvisor-companion version %s\n", version)
		os.Exit(0)
	}

	// rootPath is where our /proc is mounted.
	// When we are inside the container, host's /proc should be mounted
	// at /rootfs/proc, so rootPath is /rootfs
	rootPath := getRootPath()

	// start collecting data
	go collectData(rootPath)

	addr := fmt.Sprintf("%s:%d", *argIP, *argPort)
	fmt.Printf("Starting cAdvisor-companion version: %q on port %d\n", version, *argPort)
	http.HandleFunc("/api/", apiHandler)
	err := http.ListenAndServe(addr, nil)
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
}
<commit_msg>ADD simple api error handling<commit_after>package main

import (
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"io"
	"net/http"
	"os"
	"regexp"
	"sort"
	"strconv"
	"time"

	proc "github.com/abulimov/cadvisor-companion/process"
)

var version = "0.0.5"

// set up cli vars
var argIP = flag.String("listen_ip", "", "IP to listen on, defaults to all IPs")
var argPort = flag.Int("port", 8801, "port to listen")
var versionFlag = flag.Bool("version", false, "print cAdvisor-companion version and exit")

// ProcessSnapshot represents slice of processes in time
type ProcessSnapshot struct {
	Timestamp time.Time `json:"timestamp"`
	Processes proc.List `json:"processes"`
}

// HistoryEntry containes all ProcessSnapshots for some moment in time
type HistoryEntry map[string]ProcessSnapshot

// history holds RRD-like array of last 60 history entries
var history [60]HistoryEntry

// getLastData returns data from history with added relative CPU usage
// offset (in seconds) lets us get data from the past.
// interval (in seconds) is used to calculate CPU usage
func getLastData(containerID string, interval, offset int) (*ProcessSnapshot, error) {
	if len(history) < offset+interval {
		return nil, errors.New("Wrong offset and interval combination")
	}
	last := len(history) - offset
	first := last - interval
	entry1 := history[first][containerID]
	entry2 := history[last][containerID]
	var procs proc.List
	cpu1 := entry1.Processes.GetCPUTotalUsage()
	cpu2 := entry2.Processes.GetCPUTotalUsage()
	for _, p2 := range entry2.Processes {
		p1 := entry1.Processes.FindProc(p2.Status.Pid)
		if p1 != nil {
			user := int64(p2.Stat.Utime - p1.Stat.Utime)
			system := int64(p2.Stat.Stime - p1.Stat.Stime)
			percent := (float64(user+system) / float64(cpu2-cpu1)) * 100
			p2.RelativeCPUUsage = percent
			procs = append(procs, p2)
		}

	}
	return &ProcessSnapshot{entry2.Timestamp, procs}, nil
}

// getTopCPU returns `limit` entries with top CPU usage
func getTopCPU(containerID string, limit, interval, offset int) (*ProcessSnapshot, error) {
	entry, err := getLastData(containerID, interval, offset)
	if err != nil {
		return nil, err
	}
	sort.Sort(sort.Reverse(proc.ByCPU(entry.Processes)))
	if limit > len(entry.Processes) {
		limit = len(entry.Processes)
	}
	var result ProcessSnapshot
	result.Timestamp = entry.Timestamp
	for _, p := range entry.Processes[:limit] {
		result.Processes = append(result.Processes, p)
	}
	return &result, nil
}

// getTopMem returns `limit` entries with top VmRSS usage
func getTopMem(containerID string, limit, interval, offset int) (*ProcessSnapshot, error) {
	entry, err := getLastData(containerID, interval, offset)
	if err != nil {
		return nil, err
	}
	sort.Sort(sort.Reverse(proc.ByRSS(entry.Processes)))
	if limit > len(entry.Processes) {
		limit = len(entry.Processes)
	}
	var result ProcessSnapshot
	result.Timestamp = entry.Timestamp
	for _, p := range entry.Processes[:limit] {
		result.Processes = append(result.Processes, p)
	}
	return &result, nil
}

// apiHandler handles http requests
func apiHandler(res http.ResponseWriter, req *http.Request) {
	var validPath = regexp.MustCompile("^/api/v1.0/([a-zA-Z0-9/]+)/processes$")

	// validate requested URL
	m := validPath.FindStringSubmatch(req.URL.Path)
	if m == nil {
		http.NotFound(res, req)
		return
	}

	// process get parameters

	// sortStr is used to get top sorted procs
	sortStr := req.URL.Query().Get("sort")

	// limit is used to limit top sorted procs
	limitStr := req.URL.Query().Get("limit")
	limit, err := strconv.Atoi(limitStr)
	if err != nil || limit < 1 {
		limit = 5
	}

	// interval is the interval we use to calculate CPU usage
	// and to iterate back to the past
	intervalStr := req.URL.Query().Get("interval")
	interval, err := strconv.Atoi(intervalStr)
	if err != nil || interval < 1 {
		interval = 1
	}

	// count is the count of resulting points in time
	countStr := req.URL.Query().Get("count")
	count, err := strconv.Atoi(countStr)
	if err != nil || count < 1 {
		count = 1
	}

	// containerID is the name of our container/cgroup
	containerID := "/" + m[1]

	// set response headers
	res.Header().Set(
		"Content-Type",
		"text/json",
	)
	var result []ProcessSnapshot
	var ps *ProcessSnapshot
	fail := func(err error) {
		message := map[string]string{"error": err.Error()}
		messageJSON, _ := json.Marshal(message)
		fmt.Printf("Error: %s\n", err.Error())
		res.WriteHeader(500) // HTTP 500
		io.WriteString(res, string(messageJSON))
		return
	}
	// case our possible sort parameters
	switch sortStr {
	case "cpu":
		for i := count - 1; i >= 0; i-- {
			ps, err = getTopCPU(containerID, limit, interval, i*interval+1)
			if err != nil {
				fail(err)
				return
			}
			result = append(result, *ps)
		}
	case "mem":
		for i := count - 1; i >= 0; i-- {
			ps, err = getTopMem(containerID, limit, interval, i*interval+1)
			if err != nil {
				fail(err)
				return
			}
			result = append(result, *ps)
		}
	case "":
		for i := count - 1; i >= 0; i-- {
			ps, err = getLastData(containerID, interval, i*interval+1)
			if err != nil {
				fail(err)
				return
			}
			result = append(result, *ps)
		}
	}
	jsonResult, _ := json.Marshal(result)
	io.WriteString(res, string(jsonResult))
}

// collectData scrapes procs data for all containers every second
// and keeps it in global history var
func collectData(rootPath string) {
	for {
		timeStamp := time.Now()
		// get all processes without cgroup grouping
		allProcs, _ := proc.GetProcesses(rootPath)
		// group all processes by their cgroups
		cgroupsProcs := allProcs.GetCgroupsMap()

		entry := make(HistoryEntry)
		for e, p := range cgroupsProcs {
			entry[e] = ProcessSnapshot{timeStamp, p}
		}

		// rotate our history
		for i, v := range history[1:] {
			history[i] = v
		}
		history[len(history)-1] = entry
		time.Sleep(time.Second)
	}
}

func getRootPath() string {
	dir, err := os.Stat("/rootfs")
	if err != nil {
		return "/"
	}
	// check if the /rootfs is indeed a directory or not
	if !dir.IsDir() {
		return "/"
	}
	return "/rootfs"
}

func main() {
	flag.Parse()

	if *versionFlag {
		fmt.Printf("cAdvisor-companion version %s\n", version)
		os.Exit(0)
	}

	// rootPath is where our /proc is mounted.
	// When we are inside the container, host's /proc should be mounted
	// at /rootfs/proc, so rootPath is /rootfs
	rootPath := getRootPath()

	// start collecting data
	go collectData(rootPath)

	addr := fmt.Sprintf("%s:%d", *argIP, *argPort)
	fmt.Printf("Starting cAdvisor-companion version: %q on port %d\n", version, *argPort)
	http.HandleFunc("/api/", apiHandler)
	err := http.ListenAndServe(addr, nil)
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>package feed

import (
	"bytes"
	"io"
	"io/ioutil"
	"net/http"
	"testing"
	"time"

	"code.google.com/p/go-charset/charset"
	_ "code.google.com/p/go-charset/data"
)

func itemHandler(feed *Feed, ch *Channel, newitems []*Item) {}

func charsetReader(name string, r io.Reader) (io.Reader, error) {
	return charset.NewReader(name, r)
}

func TestFeed(t *testing.T) {
	urilist := []string{
		"http://cyber.law.harvard.edu/rss/examples/sampleRss091.xml", // Non-utf8 encoding.
		"http://store.steampowered.com/feeds/news.xml",               // This feed violates the rss spec.
		"http://cyber.law.harvard.edu/rss/examples/sampleRss092.xml",
		"http://cyber.law.harvard.edu/rss/examples/rss2sample.xml",
		"http://blog.case.edu/news/feed.atom",
	}

	var feed *Feed
	var err error

	for _, uri := range urilist {
		feed = New(5, itemHandler, NewDatabase())

		if _, err = feed.Fetch(uri, &http.Client{Timeout: 5 * time.Second}, charsetReader); err != nil {
			t.Errorf("%s >>> %s", uri, err)
			return
		}
	}
}

func Test_NewItem(t *testing.T) {
	content, _ := ioutil.ReadFile("testdata/initial.atom")
	itemsCh := make(chan []*Item, 2)
	feed := New(1, func(_ *Feed, _ *Channel, newitems []*Item) {
		itemsCh <- newitems
	}, NewDatabase())
	err := feed.load(bytes.NewReader(content), nil)
	if err != nil {
		t.Error(err)
	}

	content, _ = ioutil.ReadFile("testdata/initial_plus_one_new.atom")
	feed.load(bytes.NewReader(content), nil)
	expected := "Second title"

	select {
	case items := <-itemsCh:
		if len(items) != 1 {
			t.Errorf("Expected %s new item, got %s", 1, len(items))
		}

		if "First title" != items[0].Title {
			t.Errorf("Expected %s, got %s", expected, items[0].Title)
		}
	case <-time.After(time.Second):
		t.Fatal("timeout")
	}

	select {
	case items := <-itemsCh:
		if len(items) != 1 {
			t.Errorf("Expected %s new item, got %s", 1, len(items))
		}

		if expected != items[0].Title {
			t.Errorf("Expected %s, got %s", expected, items[0].Title)
		}
	case <-time.After(time.Second):
		t.Fatal("timeout")
	}
}

func Test_AtomAuthor(t *testing.T) {
	content, err := ioutil.ReadFile("testdata/idownload.atom")
	if err != nil {
		t.Errorf("unable to load file")
	}
	itemCh := make(chan *Item, 1)
	feed := New(1, func(f *Feed, ch *Channel, newitems []*Item) {
		itemCh <- newitems[0]
	}, NewDatabase())
	err = feed.load(bytes.NewReader(content), nil)

	select {
	case item := <-itemCh:
		expected := "Cody Lee"
		if item.Author.Name != expected {
			t.Errorf("Expected author to be %s but found %s", expected, item.Author.Name)
		}
	case <-time.After(time.Second):
		t.Fatal("timeout")
	}
}

func Test_RssAuthor(t *testing.T) {
	content, _ := ioutil.ReadFile("testdata/boing.rss")
	itemCh := make(chan *Item, 1)
	feed := New(1, func(f *Feed, ch *Channel, newitems []*Item) {
		itemCh <- newitems[0]
	}, NewDatabase())
	feed.load(bytes.NewReader(content), nil)

	select {
	case item := <-itemCh:
		expected := "Cory Doctorow"
		if item.Author.Name != expected {
			t.Errorf("Expected author to be %s but found %s", expected, item.Author.Name)
		}
	case <-time.After(time.Second):
		t.Fatal("timeout")
	}
}

func Test_ItemExtensions(t *testing.T) {
	content, _ := ioutil.ReadFile("testdata/extension.rss")
	itemCh := make(chan *Item, 1)
	feed := New(1, func(_ *Feed, _ *Channel, newitems []*Item) {
		itemCh <- newitems[0]
	}, NewDatabase())
	feed.load(bytes.NewReader(content), nil)

	select {
	case item := <-itemCh:
		edgarExtensionxbrlFiling := item.Extensions["http://www.sec.gov/Archives/edgar"]["xbrlFiling"][0].Childrens
		companyExpected := "Cellular Biomedicine Group, Inc."
		companyName := edgarExtensionxbrlFiling["companyName"][0]
		if companyName.Value != companyExpected {
			t.Errorf("Expected company to be %s but found %s", companyExpected, companyName.Value)
		}

		files := edgarExtensionxbrlFiling["xbrlFiles"][0].Childrens["xbrlFile"]
		fileSizeExpected := 10
		if len(files) != 10 {
			t.Errorf("Expected files size to be %s but found %s", fileSizeExpected, len(files))
		}

		file := files[0]
		fileExpected := "cbmg_10qa.htm"
		if file.Attrs["file"] != fileExpected {
			t.Errorf("Expected file to be %s but found %s", fileExpected, len(file.Attrs["file"]))
		}
	case <-time.After(time.Second):
		t.Fatal("timeout")
	}
}

func Test_ChannelExtensions(t *testing.T) {
	content, _ := ioutil.ReadFile("testdata/extension.rss")
	channelCh := make(chan *Channel, 1)
	feed := New(1, func(_ *Feed, ch *Channel, _ []*Item) {
		channelCh <- ch
	}, NewDatabase())
	feed.load(bytes.NewReader(content), nil)

	select {
	case channel := <-channelCh:
		itunesExtentions := channel.Extensions["http://www.itunes.com/dtds/podcast-1.0.dtd"]

		authorExptected := "The Author"
		ownerEmailExpected := "test@rss.com"
		categoryExpected := "Politics"
		imageExptected := "http://golang.org/doc/gopher/project.png"

		if itunesExtentions["author"][0].Value != authorExptected {
			t.Errorf("Expected author to be %s but found %s", authorExptected, itunesExtentions["author"][0].Value)
		}

		if itunesExtentions["owner"][0].Childrens["email"][0].Value != ownerEmailExpected {
			t.Errorf("Expected owner email to be %s but found %s", ownerEmailExpected, itunesExtentions["owner"][0].Childrens["email"][0].Value)
		}

		if itunesExtentions["category"][0].Attrs["text"] != categoryExpected {
			t.Errorf("Expected category text to be %s but found %s", categoryExpected, itunesExtentions["category"][0].Attrs["text"])
		}

		if itunesExtentions["image"][0].Attrs["href"] != imageExptected {
			t.Errorf("Expected image href to be %s but found %s", imageExptected, itunesExtentions["image"][0].Attrs["href"])
		}
	case <-time.After(time.Second):
		t.Fatal("timeout")
	}
}

func Test_CData(t *testing.T) {
	content, _ := ioutil.ReadFile("testdata/iosBoardGameGeek.rss")
	itemCh := make(chan *Item, 1)
	feed := New(1, func(_ *Feed, _ *Channel, newitems []*Item) {
		itemCh <- newitems[0]
	}, NewDatabase())
	feed.load(bytes.NewReader(content), nil)

	select {
	case item := <-itemCh:
		expected := `<p>abc<div>"def"</div>ghi`
		if item.Description != expected {
			t.Errorf("Expected item.Description to be [%s] but item.Description=[%s]", expected, item.Description)
		}
	case <-time.After(time.Second):
		t.Fatal("timeout")
	}
}

func Test_Link(t *testing.T) {
	content, _ := ioutil.ReadFile("testdata/nytimes.rss")
	type pair struct {
		Item    *Item
		Channel *Channel
	}
	itemCh := make(chan pair, 1)

	feed := New(1, func(_ *Feed, ch *Channel, newitems []*Item) {
		itemCh <- pair{newitems[0], ch}
	}, NewDatabase())
	feed.load(bytes.NewReader(content), nil)

	select {
	case p := <-itemCh:
		channel := p.Channel
		item := p.Item

		channelLinkExpected := "http://www.nytimes.com/services/xml/rss/nyt/US.xml"
		itemLinkExpected := "http://www.nytimes.com/2014/01/18/technology/in-keeping-grip-on-data-pipeline-obama-does-little-to-reassure-industry.html?partner=rss&emc=rss"

		if channel.Links[0].Href != channelLinkExpected {
			t.Errorf("Expected author to be %s but found %s", channelLinkExpected, channel.Links[0].Href)
		}

		if item.Links[0].Href != itemLinkExpected {
			t.Errorf("Expected author to be %s but found %s", itemLinkExpected, item.Links[0].Href)
		}
	case <-time.After(time.Second):
		t.Fatal("timeout")
	}
}
<commit_msg>Remove ioutil usage in feed tests<commit_after>package feed

import (
	"io"
	"net/http"
	"os"
	"testing"
	"time"

	"code.google.com/p/go-charset/charset"
	_ "code.google.com/p/go-charset/data"
)

func itemHandler(feed *Feed, ch *Channel, newitems []*Item) {}

func charsetReader(name string, r io.Reader) (io.Reader, error) {
	return charset.NewReader(name, r)
}

func TestFeed(t *testing.T) {
	urilist := []string{
		"http://cyber.law.harvard.edu/rss/examples/sampleRss091.xml", // Non-utf8 encoding.
		"http://store.steampowered.com/feeds/news.xml",               // This feed violates the rss spec.
		"http://cyber.law.harvard.edu/rss/examples/sampleRss092.xml",
		"http://cyber.law.harvard.edu/rss/examples/rss2sample.xml",
		"http://blog.case.edu/news/feed.atom",
	}

	var feed *Feed
	var err error

	for _, uri := range urilist {
		feed = New(5, itemHandler, NewDatabase())

		if _, err = feed.Fetch(uri, &http.Client{Timeout: 5 * time.Second}, charsetReader); err != nil {
			t.Errorf("%s >>> %s", uri, err)
			return
		}
	}
}

func Test_NewItem(t *testing.T) {
	file, _ := os.Open("testdata/initial.atom")

	itemsCh := make(chan []*Item, 2)
	feed := New(1, func(_ *Feed, _ *Channel, newitems []*Item) {
		itemsCh <- newitems
	}, NewDatabase())
	err := feed.load(file, nil)
	if err != nil {
		t.Error(err)
	}

	file.Close()

	file, _ = os.Open("testdata/initial_plus_one_new.atom")
	defer file.Close()
	feed.load(file, nil)
	expected := "Second title"

	select {
	case items := <-itemsCh:
		if len(items) != 1 {
			t.Errorf("Expected %s new item, got %s", 1, len(items))
		}

		if "First title" != items[0].Title {
			t.Errorf("Expected %s, got %s", expected, items[0].Title)
		}
	case <-time.After(time.Second):
		t.Fatal("timeout")
	}

	select {
	case items := <-itemsCh:
		if len(items) != 1 {
			t.Errorf("Expected %s new item, got %s", 1, len(items))
		}

		if expected != items[0].Title {
			t.Errorf("Expected %s, got %s", expected, items[0].Title)
		}
	case <-time.After(time.Second):
		t.Fatal("timeout")
	}
}

func Test_AtomAuthor(t *testing.T) {
	file, err := os.Open("testdata/idownload.atom")
	if err != nil {
		t.Errorf("unable to load file")
	}
	defer file.Close()

	itemCh := make(chan *Item, 1)
	feed := New(1, func(f *Feed, ch *Channel, newitems []*Item) {
		itemCh <- newitems[0]
	}, NewDatabase())
	err = feed.load(file, nil)

	select {
	case item := <-itemCh:
		expected := "Cody Lee"
		if item.Author.Name != expected {
			t.Errorf("Expected author to be %s but found %s", expected, item.Author.Name)
		}
	case <-time.After(time.Second):
		t.Fatal("timeout")
	}
}

func Test_RssAuthor(t *testing.T) {
	file, _ := os.Open("testdata/boing.rss")
	defer file.Close()

	itemCh := make(chan *Item, 1)
	feed := New(1, func(f *Feed, ch *Channel, newitems []*Item) {
		itemCh <- newitems[0]
	}, NewDatabase())
	feed.load(file, nil)

	select {
	case item := <-itemCh:
		expected := "Cory Doctorow"
		if item.Author.Name != expected {
			t.Errorf("Expected author to be %s but found %s", expected, item.Author.Name)
		}
	case <-time.After(time.Second):
		t.Fatal("timeout")
	}
}

func Test_ItemExtensions(t *testing.T) {
	file, _ := os.Open("testdata/extension.rss")
	defer file.Close()

	itemCh := make(chan *Item, 1)
	feed := New(1, func(_ *Feed, _ *Channel, newitems []*Item) {
		itemCh <- newitems[0]
	}, NewDatabase())
	feed.load(file, nil)

	select {
	case item := <-itemCh:
		edgarExtensionxbrlFiling := item.Extensions["http://www.sec.gov/Archives/edgar"]["xbrlFiling"][0].Childrens
		companyExpected := "Cellular Biomedicine Group, Inc."
		companyName := edgarExtensionxbrlFiling["companyName"][0]
		if companyName.Value != companyExpected {
			t.Errorf("Expected company to be %s but found %s", companyExpected, companyName.Value)
		}

		files := edgarExtensionxbrlFiling["xbrlFiles"][0].Childrens["xbrlFile"]
		fileSizeExpected := 10
		if len(files) != 10 {
			t.Errorf("Expected files size to be %s but found %s", fileSizeExpected, len(files))
		}

		file := files[0]
		fileExpected := "cbmg_10qa.htm"
		if file.Attrs["file"] != fileExpected {
			t.Errorf("Expected file to be %s but found %s", fileExpected, len(file.Attrs["file"]))
		}
	case <-time.After(time.Second):
		t.Fatal("timeout")
	}
}

func Test_ChannelExtensions(t *testing.T) {
	file, _ := os.Open("testdata/extension.rss")
	defer file.Close()

	channelCh := make(chan *Channel, 1)
	feed := New(1, func(_ *Feed, ch *Channel, _ []*Item) {
		channelCh <- ch
	}, NewDatabase())

	feed.load(file, nil)

	select {
	case channel := <-channelCh:
		itunesExtentions := channel.Extensions["http://www.itunes.com/dtds/podcast-1.0.dtd"]

		authorExptected := "The Author"
		ownerEmailExpected := "test@rss.com"
		categoryExpected := "Politics"
		imageExptected := "http://golang.org/doc/gopher/project.png"

		if itunesExtentions["author"][0].Value != authorExptected {
			t.Errorf("Expected author to be %s but found %s", authorExptected, itunesExtentions["author"][0].Value)
		}

		if itunesExtentions["owner"][0].Childrens["email"][0].Value != ownerEmailExpected {
			t.Errorf("Expected owner email to be %s but found %s", ownerEmailExpected, itunesExtentions["owner"][0].Childrens["email"][0].Value)
		}

		if itunesExtentions["category"][0].Attrs["text"] != categoryExpected {
			t.Errorf("Expected category text to be %s but found %s", categoryExpected, itunesExtentions["category"][0].Attrs["text"])
		}

		if itunesExtentions["image"][0].Attrs["href"] != imageExptected {
			t.Errorf("Expected image href to be %s but found %s", imageExptected, itunesExtentions["image"][0].Attrs["href"])
		}
	case <-time.After(time.Second):
		t.Fatal("timeout")
	}
}

func Test_CData(t *testing.T) {
	file, _ := os.Open("testdata/iosBoardGameGeek.rss")
	defer file.Close()

	itemCh := make(chan *Item, 1)
	feed := New(1, func(_ *Feed, _ *Channel, newitems []*Item) {
		itemCh <- newitems[0]
	}, NewDatabase())

	feed.load(file, nil)

	select {
	case item := <-itemCh:
		expected := `<p>abc<div>"def"</div>ghi`
		if item.Description != expected {
			t.Errorf("Expected item.Description to be [%s] but item.Description=[%s]", expected, item.Description)
		}
	case <-time.After(time.Second):
		t.Fatal("timeout")
	}
}

func Test_Link(t *testing.T) {
	file, _ := os.Open("testdata/nytimes.rss")
	defer file.Close()

	type pair struct {
		Item    *Item
		Channel *Channel
	}
	itemCh := make(chan pair, 1)

	feed := New(1, func(_ *Feed, ch *Channel, newitems []*Item) {
		itemCh <- pair{newitems[0], ch}
	}, NewDatabase())
	feed.load(file, nil)

	select {
	case p := <-itemCh:
		channel := p.Channel
		item := p.Item

		channelLinkExpected := "http://www.nytimes.com/services/xml/rss/nyt/US.xml"
		itemLinkExpected := "http://www.nytimes.com/2014/01/18/technology/in-keeping-grip-on-data-pipeline-obama-does-little-to-reassure-industry.html?partner=rss&emc=rss"

		if channel.Links[0].Href != channelLinkExpected {
			t.Errorf("Expected author to be %s but found %s", channelLinkExpected, channel.Links[0].Href)
		}

		if item.Links[0].Href != itemLinkExpected {
			t.Errorf("Expected author to be %s but found %s", itemLinkExpected, item.Links[0].Href)
		}
	case <-time.After(time.Second):
		t.Fatal("timeout")
	}
}
<|endoftext|>


<commit_before>package calculator

import (
	"fmt"
	"testing"

	"github.com/stretchr/testify/assert"
)

// in organises input value for Calc
type in struct {
	input float64
	op    string
}

// Test organises values for tests
type Test struct {
	name string
	in   in
	out  float64
}

// TestCalcAdd tests the behavior of Calc with Add operation
func TestCalcAdd(t *testing.T) {
	var tests = []Test{
		Test{
			name: "first input",
			in:   in{input: 5, op: "+"},
			out:  5,
		},
		Test{
			name: "second input",
			in:   in{input: 10, op: "+"},
			out:  15,
		},
		Test{
			name: "third input",
			in:   in{input: 35, op: "+"},
			out:  50,
		},
	}
	c := &Calculator{}
	for _, test := range tests {
		// Assertion with assert package
		assert.Equal(
			t,
			test.out,
			c.Do(test.in.input, test.in.op),
			fmt.Sprintf("%s: %v", test.name, test.in),
		)
	}
}

// TestCalcMultipleOps tests the behavior of Calc with multiple operations
func TestCalcMultipleOps(t *testing.T) {
	var tests = []Test{
		Test{
			name: "first input for addition",
			in:   in{input: 10, op: "+"},
			out:  10,
		},
		Test{
			name: "second input for multiplication",
			in:   in{input: 5, op: "*"},
			out:  50,
		},
		Test{
			name: "third input for subtraction",
			in:   in{input: 30, op: "-"},
			out:  20,
		},
		Test{
			name: "third input for subtraction",
			in:   in{input: 4, op: "/"},
			out:  5,
		},
	}
	c := &Calculator{}
	for _, test := range tests {
		// Assertion with assert package
		assert.Equal(
			t,
			test.out,
			c.Do(test.in.input, test.in.op),
			fmt.Sprintf("%s: %v", test.name, test.in),
		)
	}
}

// TestCalcMultipleOpsWithSubTest tests the behavior of Calc with multiple operations
// with sub-level tests
func TestCalcMultipleOpsWithSubTest(t *testing.T) {
	var tests = []Test{
		Test{
			name: "input for addition",
			in:   in{input: 10, op: "+"},
			out:  10,
		},
		Test{
			name: "input for multiplication",
			in:   in{input: 5, op: "*"},
			out:  50,
		},
		Test{
			name: "input for subtraction",
			in:   in{input: 30, op: "-"},
			out:  20,
		},
		Test{
			name: "third input for subtraction",
			in:   in{input: 4, op: "/"},
			out:  5,
		},
	}
	c := &Calculator{}
	for _, test := range tests {
		t.Run(test.name, func(t *testing.T) {
			assert.Equal(
				t,
				test.out,
				c.Do(test.in.input, test.in.op),
				fmt.Sprintf("%s: %v", test.name, test.in),
			)
		})
	}
}

// BenchmarkCalc benchmarks Calc
func BenchmarkCalc(b *testing.B) {
	test := Test{
		in:  in{input: 5, op: "+"},
		out: 5,
	}
	for i := 0; i < b.N; i++ {
		c := &Calculator{}
		c.Do(test.in.input, test.in.op)
	}
}

// BenchmarkCalcMultipleOpsWithSubTest benchmarks with sub-level benchmark
func BenchmarkCalcMultipleOpsWithSubTest(b *testing.B) {
	var tests = []Test{
		Test{
			name: "input for addition",
			in:   in{input: 10, op: "+"},
			out:  10,
		},
		Test{
			name: "input for multiplication",
			in:   in{input: 5, op: "*"},
			out:  50,
		},
		Test{
			name: "input for subtraction",
			in:   in{input: 30, op: "-"},
			out:  20,
		},
		Test{
			name: "third input for subtraction",
			in:   in{input: 4, op: "/"},
			out:  5,
		},
	}
	for _, test := range tests {
		b.Run(test.name, func(b *testing.B) {
			for i := 0; i < b.N; i++ {
				c := &Calculator{}
				c.Do(test.in.input, test.in.op)
			}
		})
	}
}
<commit_msg>Modifications in test examples<commit_after>package calculator

import (
	"fmt"
	"testing"

	"github.com/stretchr/testify/assert"
)

// in organises input value for Calc
type in struct {
	input float64
	op    string
}

// Test organises values for tests
type Test struct {
	name        string
	in          in
	out         float64
	shouldError bool
}

// TestCalcAdd tests the behavior of Calc with Add operation
func TestCalcAdd(t *testing.T) {
	var tests = []Test{
		Test{
			name: "first input",
			in:   in{input: 5, op: "+"},
			out:  5,
		},
		Test{
			name: "second input",
			in:   in{input: 10, op: "+"},
			out:  15,
		},
		Test{
			name: "third input",
			in:   in{input: 35, op: "+"},
			out:  50,
		},
	}
	c := &Calculator{}
	var result float64
	for _, test := range tests {

		result = c.Do(test.in.input, test.in.op)
		//if test.out != result {
		//	t.Errorf(fmt.Sprintf("%s: %v", test.name, test.in))
		//}
		// Assertion with assert package
		assert.Equal(
			t,
			test.out,
			result,
			fmt.Sprintf("%s: %v", test.name, test.in),
		)
	}
}

// TestCalcMultipleOps tests the behavior of Calc with multiple operations
func TestCalcMultipleOps(t *testing.T) {
	var tests = []Test{
		Test{
			name: "first input for addition",
			in:   in{input: 10, op: "+"},
			out:  10,
		},
		Test{
			name: "second input for multiplication",
			in:   in{input: 5, op: "*"},
			out:  50,
		},
		Test{
			name: "third input for subtraction",
			in:   in{input: 30, op: "-"},
			out:  20,
		},
		Test{
			name: "fourth input for subtraction",
			in:   in{input: 4, op: "/"},
			out:  5,
		},
	}
	c := &Calculator{}
	for _, test := range tests {
		// Assertion with assert package
		assert.Equal(
			t,
			test.out,
			c.Do(test.in.input, test.in.op),
			fmt.Sprintf("%s: %v", test.name, test.in),
		)
	}
}

// TestCalcMultipleOpsWithSubTest tests the behavior of Calc with multiple operations
// with sub-level tests
func TestCalcMultipleOpsWithSubTest(t *testing.T) {
	var tests = []Test{
		Test{
			name: "first input for addition",
			in:   in{input: 10, op: "+"},
			out:  10,
		},
		Test{
			name: "second input for multiplication",
			in:   in{input: 5, op: "*"},
			out:  50,
		},
		Test{
			name: "third input for subtraction",
			in:   in{input: 30, op: "-"},
			out:  20,
		},
		Test{
			name: "fourth input for subtraction",
			in:   in{input: 4, op: "/"},
			out:  5,
		},
	}
	c := &Calculator{}
	for _, test := range tests {
		t.Run(test.name, func(t *testing.T) {
			assert.Equal(
				t,
				test.out,
				c.Do(test.in.input, test.in.op),
				fmt.Sprintf("%s: %v", test.name, test.in),
			)
		})
	}
}

//// BenchmarkCalc benchmarks Calc
//func BenchmarkCalc(b *testing.B) {
//	test := Test{
//		in:  in{input: 5, op: "+"},
//		out: 5,
//	}
//	for i := 0; i < b.N; i++ {
//		c := &Calculator{}
//		c.Do(test.in.input, test.in.op)
//	}
//}
//
//// BenchmarkCalcMultipleOpsWithSubTest benchmarks with sub-level benchmark
//func BenchmarkCalcMultipleOpsWithSubTest(b *testing.B) {
//	var tests = []Test{
//		Test{
//			name: "input for addition",
//			in:   in{input: 10, op: "+"},
//			out:  10,
//		},
//		Test{
//			name: "input for multiplication",
//			in:   in{input: 5, op: "*"},
//			out:  50,
//		},
//		Test{
//			name: "input for subtraction",
//			in:   in{input: 30, op: "-"},
//			out:  20,
//		},
//		Test{
//			name: "third input for subtraction",
//			in:   in{input: 4, op: "/"},
//			out:  5,
//		},
//	}
//	for _, test := range tests {
//		b.Run(test.name, func(b *testing.B) {
//			for i := 0; i < b.N; i++ {
//				c := &Calculator{}
//				c.Do(test.in.input, test.in.op)
//			}
//		})
//	}
//}
<|endoftext|>


<commit_before>package integration

import (
	"fmt"
	"net/http"
	"net/http/httptest"
	"net/http/httputil"
	"os/exec"
	"strings"
	"testing"

	"github.com/sclevine/spec"
	"github.com/stretchr/testify/require"
)

var _ = suite("compute/firewall/create", func(t *testing.T, when spec.G, it spec.S) {
	var (
		expect *require.Assertions
		server *httptest.Server
	)

	it.Before(func() {
		expect = require.New(t)

		server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
			switch req.URL.Path {
			case "/v2/firewalls":
				auth := req.Header.Get("Authorization")
				if auth != "Bearer some-magic-token" {
					w.WriteHeader(http.StatusUnauthorized)
					return
				}

				w.Write([]byte(firewallCreateResponse))
			default:
				dump, err := httputil.DumpRequest(req, true)
				if err != nil {
					t.Fatal("failed to dump request")
				}

				t.Fatalf("received unknown request: %s", dump)
			}
		}))
	})

	when("doing create", func() {
		it("does create", func() {
			cmd := exec.Command(builtBinaryPath,
				"-t", "some-magic-token",
				"-u", server.URL,
				"compute",
				"firewall",
				"create",
				"--name", "test-firewall",
				"--inbound-rules", `"protocol:tcp,ports:443"`,
			)

			output, err := cmd.CombinedOutput()
			expect.NoError(err, fmt.Sprintf("received error output: %s", output))
			expect.Equal(strings.TrimSpace(firewallCreateOutput), strings.TrimSpace(string(output)))
		})
	})
})

const firewallCreateOutput = `ID                                      Name             Status       Created At              Inbound Rules              Outbound Rules    Droplet IDs    Tags    Pending Changes
e4b9c960-d385-4950-84f3-d102162e6be5    test-firewall    succeeded    2019-10-24T20:30:26Z    protocol:tcp,ports:443,`

const firewallCreateResponse = `{
  "firewall": {
	"id":"e4b9c960-d385-4950-84f3-d102162e6be5",
	"name":"test-firewall",
	"status":"succeeded",
	"inbound_rules":[{
	  "protocol":"tcp",
	  "ports":"443",
	  "sources":{}
	}],
	"outbound_rules":[],
	"created_at":"2019-10-24T20:30:26Z",
	"droplet_ids":[],
	"tags":[],
	"pending_changes":[]
  }
}`
<commit_msg>Make test case descriptions more informative<commit_after>package integration

import (
	"fmt"
	"net/http"
	"net/http/httptest"
	"net/http/httputil"
	"os/exec"
	"strings"
	"testing"

	"github.com/sclevine/spec"
	"github.com/stretchr/testify/require"
)

var _ = suite("compute/firewall/create", func(t *testing.T, when spec.G, it spec.S) {
	var (
		expect *require.Assertions
		server *httptest.Server
	)

	it.Before(func() {
		expect = require.New(t)

		server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
			switch req.URL.Path {
			case "/v2/firewalls":
				auth := req.Header.Get("Authorization")
				if auth != "Bearer some-magic-token" {
					w.WriteHeader(http.StatusUnauthorized)
					return
				}

				w.Write([]byte(firewallCreateResponse))
			default:
				dump, err := httputil.DumpRequest(req, true)
				if err != nil {
					t.Fatal("failed to dump request")
				}

				t.Fatalf("received unknown request: %s", dump)
			}
		}))
	})

	when("the minimum required flags are provided", func() {
		it("creates a firewall", func() {
			cmd := exec.Command(builtBinaryPath,
				"-t", "some-magic-token",
				"-u", server.URL,
				"compute",
				"firewall",
				"create",
				"--name", "test-firewall",
				"--inbound-rules", `"protocol:tcp,ports:443"`,
			)

			output, err := cmd.CombinedOutput()
			expect.NoError(err, fmt.Sprintf("received error output: %s", output))
			expect.Equal(strings.TrimSpace(firewallCreateOutput), strings.TrimSpace(string(output)))
		})
	})
})

const firewallCreateOutput = `ID                                      Name             Status       Created At              Inbound Rules              Outbound Rules    Droplet IDs    Tags    Pending Changes
e4b9c960-d385-4950-84f3-d102162e6be5    test-firewall    succeeded    2019-10-24T20:30:26Z    protocol:tcp,ports:443,`

const firewallCreateResponse = `{
  "firewall": {
	"id":"e4b9c960-d385-4950-84f3-d102162e6be5",
	"name":"test-firewall",
	"status":"succeeded",
	"inbound_rules":[{
	  "protocol":"tcp",
	  "ports":"443",
	  "sources":{}
	}],
	"outbound_rules":[],
	"created_at":"2019-10-24T20:30:26Z",
	"droplet_ids":[],
	"tags":[],
	"pending_changes":[]
  }
}`
<|endoftext|>


<commit_before>package lifecycle_test

import (
	"bytes"
	"fmt"
	"io"
	"time"

	"code.google.com/p/gogoprotobuf/proto"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"

	"github.com/cloudfoundry-incubator/gordon"
	"github.com/cloudfoundry-incubator/gordon/warden"
)

func readUntilExit(stream <-chan *warden.ProcessPayload) (string, string, uint32) {
	stdout := ""
	stderr := ""
	exitStatus := uint32(12234)

	for payload := range stream {
		switch payload.GetSource() {
		case warden.ProcessPayload_stdout:
			stdout += payload.GetData()

		case warden.ProcessPayload_stderr:
			stderr += payload.GetData()
		}

		exitStatus = payload.GetExitStatus()
	}

	return stdout, stderr, exitStatus
}

var _ = Describe("Through a restart", func() {
	var handle string

	BeforeEach(func() {
		res, err := client.Create(nil)
		Expect(err).ToNot(HaveOccurred())

		handle = res.GetHandle()
	})

	AfterEach(func() {
		err := runner.Stop()
		Expect(err).ToNot(HaveOccurred())

		err = runner.DestroyContainers()
		Expect(err).ToNot(HaveOccurred())

		err = runner.Start()
		Expect(err).ToNot(HaveOccurred())
	})

	restartServer := func() {
		err := runner.Stop()
		Expect(err).ToNot(HaveOccurred())

		err = runner.Start()
		Expect(err).ToNot(HaveOccurred())
	}

	It("retains the container list", func() {
		restartServer()

		res, err := client.List(nil)
		Expect(err).ToNot(HaveOccurred())

		Expect(res.GetHandles()).To(ContainElement(handle))
	})

	Describe("a started job", func() {
		It("continues to stream", func(done Done) {
			processID, runStream, err := client.Run(
				handle,
				"while true; do echo hi; sleep 0.5; done",
				gordon.ResourceLimits{},
				[]gordon.EnvironmentVariable{},
			)

			Expect(err).ToNot(HaveOccurred())

			restartServer()

			Eventually(runStream).Should(BeClosed())

			stream, err := client.Attach(handle, processID)
			Expect(err).ToNot(HaveOccurred())

			Expect((<-stream).GetData()).To(ContainSubstring("hi\n"))

			close(done)
		}, 10.0)

		It("does not have its job ID repeated", func() {
			processID1, _, err := client.Run(handle, "while true; do echo hi; sleep 0.5; done", gordon.ResourceLimits{}, []gordon.EnvironmentVariable{})
			Expect(err).ToNot(HaveOccurred())

			restartServer()

			processID2, _, err := client.Run(handle, "while true; do echo hi; sleep 0.5; done", gordon.ResourceLimits{}, []gordon.EnvironmentVariable{})
			Expect(err).ToNot(HaveOccurred())

			Expect(processID1).ToNot(Equal(processID2))
		})

		Context("that prints monotonously increasing output", func() {
			It("does not duplicate its output on reconnect", func(done Done) {
				receivedNumbers := make(chan int, 2048)

				processID, _, err := client.Run(
					handle,
					"for i in $(seq 10); do echo $i; sleep 0.5; done; echo goodbye; while true; do sleep 1; done",
					gordon.ResourceLimits{},
					[]gordon.EnvironmentVariable{},
				)
				Expect(err).ToNot(HaveOccurred())

				stream, err := client.Attach(handle, processID)
				Expect(err).ToNot(HaveOccurred())

				go streamNumbersTo(receivedNumbers, stream)

				time.Sleep(500 * time.Millisecond)

				restartServer()

				stream, err = client.Attach(handle, processID)
				Expect(err).ToNot(HaveOccurred())

				go streamNumbersTo(receivedNumbers, stream)

				lastNum := 0
				for num := range receivedNumbers {
					Expect(num).To(BeNumerically(">", lastNum))
					lastNum = num
				}

				close(done)
			}, 10.0)
		})
	})

	Describe("a memory limit", func() {
		It("is still enforced", func() {
			_, err := client.LimitMemory(handle, 32*1024*1024)
			Expect(err).ToNot(HaveOccurred())

			restartServer()

			_, stream, err := client.Run(handle, "exec ruby -e '$stdout.sync = true; puts :hello; puts (\"x\" * 64 * 1024 * 1024).size; puts :goodbye; exit 42'", gordon.ResourceLimits{}, []gordon.EnvironmentVariable{})
			Expect(err).ToNot(HaveOccurred())

			// cgroups OOM killer seems to leave no trace of the process;
			// there's no exit status indicator, so just assert that the one
			// we tried to exit with after over-allocating is not seen

			stdout, _, exitStatus := readUntilExit(stream)
			Expect(stdout).To(Equal("hello\n"))
			Expect(exitStatus).ToNot(Equal(uint32(42)))
		})
	})

	Describe("a container's active job", func() {
		It("is still tracked", func() {
			processID, _, err := client.Run(handle, "while true; do echo hi; sleep 0.5; done", gordon.ResourceLimits{}, []gordon.EnvironmentVariable{})
			Expect(err).ToNot(HaveOccurred())

			restartServer()

			info, err := client.Info(handle)
			Expect(err).ToNot(HaveOccurred())

			Expect(info.GetProcessIds()).To(ContainElement(uint64(processID)))
		})
	})

	Describe("a container's list of events", func() {
		It("is still reported", func() {
			_, err := client.LimitMemory(handle, 32*1024*1024)
			Expect(err).ToNot(HaveOccurred())

			// trigger 'out of memory' event
			_, _, err = client.Run(handle, "exec ruby -e '$stdout.sync = true; puts :hello; puts (\"x\" * 64 * 1024 * 1024).size; puts :goodbye; exit 42'", gordon.ResourceLimits{}, []gordon.EnvironmentVariable{})
			Expect(err).ToNot(HaveOccurred())

			Eventually(func() []string {
				info, err := client.Info(handle)
				Expect(err).ToNot(HaveOccurred())

				return info.GetEvents()
			}).Should(ContainElement("out of memory"))

			restartServer()

			info, err := client.Info(handle)
			Expect(err).ToNot(HaveOccurred())

			Expect(info.GetEvents()).To(ContainElement("out of memory"))
		})
	})

	Describe("a container's properties", func() {
		It("are retained", func() {
			createRes, err := client.Create(map[string]string{
				"foo": "bar",
			})
			Expect(err).ToNot(HaveOccurred())

			info, err := client.Info(createRes.GetHandle())
			Expect(err).ToNot(HaveOccurred())

			Expect(info.GetProperties()).To(ContainElement(&warden.Property{
				Key:   proto.String("foo"),
				Value: proto.String("bar"),
			}))

			restartServer()

			info, err = client.Info(createRes.GetHandle())
			Expect(err).ToNot(HaveOccurred())

			Expect(info.GetProperties()).To(ContainElement(&warden.Property{
				Key:   proto.String("foo"),
				Value: proto.String("bar"),
			}))
		})
	})

	Describe("a container's state", func() {
		It("is still reported", func() {
			info, err := client.Info(handle)
			Expect(err).ToNot(HaveOccurred())

			Expect(info.GetState()).To(Equal("active"))

			restartServer()

			info, err = client.Info(handle)
			Expect(err).ToNot(HaveOccurred())

			Expect(info.GetState()).To(Equal("active"))

			_, err = client.Stop(handle, false, false)
			Expect(err).ToNot(HaveOccurred())

			restartServer()

			info, err = client.Info(handle)
			Expect(err).ToNot(HaveOccurred())

			Expect(info.GetState()).To(Equal("stopped"))
		})
	})

	Describe("a container's network", func() {
		It("does not get reused", func() {
			infoA, err := client.Info(handle)
			Expect(err).ToNot(HaveOccurred())

			restartServer()

			res, err := client.Create(nil)
			Expect(err).ToNot(HaveOccurred())

			infoB, err := client.Info(res.GetHandle())
			Expect(err).ToNot(HaveOccurred())

			Expect(infoA.GetHostIp()).ToNot(Equal(infoB.GetHostIp()))
			Expect(infoA.GetContainerIp()).ToNot(Equal(infoB.GetContainerIp()))
		})
	})

	Describe("a container's mapped port", func() {
		It("does not get reused", func() {
			netInA, err := client.NetIn(handle)
			Expect(err).ToNot(HaveOccurred())

			restartServer()

			createRes, err := client.Create(nil)
			Expect(err).ToNot(HaveOccurred())

			netInB, err := client.NetIn(createRes.GetHandle())
			Expect(err).ToNot(HaveOccurred())

			Expect(netInA.GetHostPort()).ToNot(Equal(netInB.GetHostPort()))
			Expect(netInA.GetContainerPort()).ToNot(Equal(netInB.GetContainerPort()))
		})
	})

	Describe("a container's user", func() {
		It("does not get reused", func() {
			idA := ""
			idB := ""

			_, streamA, err := client.Run(handle, "id -u", gordon.ResourceLimits{}, []gordon.EnvironmentVariable{})
			Expect(err).ToNot(HaveOccurred())

			for chunk := range streamA {
				idA += chunk.GetData()
			}

			restartServer()

			createRes, err := client.Create(nil)
			Expect(err).ToNot(HaveOccurred())

			_, streamB, err := client.Run(createRes.GetHandle(), "id -u", gordon.ResourceLimits{}, []gordon.EnvironmentVariable{})
			Expect(err).ToNot(HaveOccurred())

			for chunk := range streamB {
				idB += chunk.GetData()
			}

			Expect(idA).ToNot(Equal(idB))
		})
	})

	Describe("a container's grace time", func() {
		BeforeEach(func() {
			err := runner.Stop()
			Expect(err).ToNot(HaveOccurred())

			err = runner.Start("--containerGraceTime", "5s")
			Expect(err).ToNot(HaveOccurred())

			res, err := client.Create(nil)
			Expect(err).ToNot(HaveOccurred())

			handle = res.GetHandle()
		})

		It("is still enforced", func() {
			restartServer()

			listRes, err := client.List(nil)
			Expect(err).ToNot(HaveOccurred())

			Expect(listRes.GetHandles()).To(ContainElement(handle))

			time.Sleep(6 * time.Second)

			listRes, err = client.List(nil)
			Expect(err).ToNot(HaveOccurred())

			Expect(listRes.GetHandles()).ToNot(ContainElement(handle))
		})
	})
})

func streamNumbersTo(destination chan<- int, source <-chan *warden.ProcessPayload) {
	for out := range source {
		buf := bytes.NewBufferString(out.GetData())

		var num int

		for {
			_, err := fmt.Fscanf(buf, "%d\n", &num)
			if err == io.EOF {
				break
			}

			// got goodbye
			if err != nil {
				close(destination)
				return
			}

			destination <- num
		}
	}
}
<commit_msg>improve test timeout behavior<commit_after>package lifecycle_test

import (
	"bytes"
	"fmt"
	"io"
	"time"

	"code.google.com/p/gogoprotobuf/proto"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"

	"github.com/cloudfoundry-incubator/gordon"
	"github.com/cloudfoundry-incubator/gordon/warden"
)

func readUntilExit(stream <-chan *warden.ProcessPayload) (string, string, uint32) {
	stdout := ""
	stderr := ""
	exitStatus := uint32(12234)

	for payload := range stream {
		switch payload.GetSource() {
		case warden.ProcessPayload_stdout:
			stdout += payload.GetData()

		case warden.ProcessPayload_stderr:
			stderr += payload.GetData()
		}

		exitStatus = payload.GetExitStatus()
	}

	return stdout, stderr, exitStatus
}

var _ = Describe("Through a restart", func() {
	var handle string

	BeforeEach(func() {
		res, err := client.Create(nil)
		Expect(err).ToNot(HaveOccurred())

		handle = res.GetHandle()
	})

	AfterEach(func() {
		err := runner.Stop()
		Expect(err).ToNot(HaveOccurred())

		err = runner.DestroyContainers()
		Expect(err).ToNot(HaveOccurred())

		err = runner.Start()
		Expect(err).ToNot(HaveOccurred())
	})

	restartServer := func() {
		err := runner.Stop()
		Expect(err).ToNot(HaveOccurred())

		err = runner.Start()
		Expect(err).ToNot(HaveOccurred())
	}

	It("retains the container list", func() {
		restartServer()

		res, err := client.List(nil)
		Expect(err).ToNot(HaveOccurred())

		Expect(res.GetHandles()).To(ContainElement(handle))
	})

	Describe("a started job", func() {
		It("continues to stream", func() {
			processID, runStream, err := client.Run(
				handle,
				"while true; do echo hi; sleep 0.5; done",
				gordon.ResourceLimits{},
				[]gordon.EnvironmentVariable{},
			)

			Expect(err).ToNot(HaveOccurred())

			restartServer()

			Eventually(runStream).Should(BeClosed())

			stream, err := client.Attach(handle, processID)
			Expect(err).ToNot(HaveOccurred())

			var chunk *warden.ProcessPayload
			Eventually(stream).Should(Receive(&chunk))
			Expect(chunk.GetData()).To(ContainSubstring("hi\n"))
		})

		It("does not have its job ID repeated", func() {
			processID1, _, err := client.Run(handle, "while true; do echo hi; sleep 0.5; done", gordon.ResourceLimits{}, []gordon.EnvironmentVariable{})
			Expect(err).ToNot(HaveOccurred())

			restartServer()

			processID2, _, err := client.Run(handle, "while true; do echo hi; sleep 0.5; done", gordon.ResourceLimits{}, []gordon.EnvironmentVariable{})
			Expect(err).ToNot(HaveOccurred())

			Expect(processID1).ToNot(Equal(processID2))
		})

		Context("that prints monotonously increasing output", func() {
			It("does not duplicate its output on reconnect", func(done Done) {
				receivedNumbers := make(chan int, 2048)

				processID, _, err := client.Run(
					handle,
					"for i in $(seq 10); do echo $i; sleep 0.5; done; echo goodbye; while true; do sleep 1; done",
					gordon.ResourceLimits{},
					[]gordon.EnvironmentVariable{},
				)
				Expect(err).ToNot(HaveOccurred())

				stream, err := client.Attach(handle, processID)
				Expect(err).ToNot(HaveOccurred())

				go streamNumbersTo(receivedNumbers, stream)

				time.Sleep(500 * time.Millisecond)

				restartServer()

				stream, err = client.Attach(handle, processID)
				Expect(err).ToNot(HaveOccurred())

				go streamNumbersTo(receivedNumbers, stream)

				lastNum := 0
				for num := range receivedNumbers {
					Expect(num).To(BeNumerically(">", lastNum))
					lastNum = num
				}

				close(done)
			}, 10.0)
		})
	})

	Describe("a memory limit", func() {
		It("is still enforced", func() {
			_, err := client.LimitMemory(handle, 32*1024*1024)
			Expect(err).ToNot(HaveOccurred())

			restartServer()

			_, stream, err := client.Run(handle, "exec ruby -e '$stdout.sync = true; puts :hello; puts (\"x\" * 64 * 1024 * 1024).size; puts :goodbye; exit 42'", gordon.ResourceLimits{}, []gordon.EnvironmentVariable{})
			Expect(err).ToNot(HaveOccurred())

			// cgroups OOM killer seems to leave no trace of the process;
			// there's no exit status indicator, so just assert that the one
			// we tried to exit with after over-allocating is not seen

			stdout, _, exitStatus := readUntilExit(stream)
			Expect(stdout).To(Equal("hello\n"))
			Expect(exitStatus).ToNot(Equal(uint32(42)))
		})
	})

	Describe("a container's active job", func() {
		It("is still tracked", func() {
			processID, _, err := client.Run(handle, "while true; do echo hi; sleep 0.5; done", gordon.ResourceLimits{}, []gordon.EnvironmentVariable{})
			Expect(err).ToNot(HaveOccurred())

			restartServer()

			info, err := client.Info(handle)
			Expect(err).ToNot(HaveOccurred())

			Expect(info.GetProcessIds()).To(ContainElement(uint64(processID)))
		})
	})

	Describe("a container's list of events", func() {
		It("is still reported", func() {
			_, err := client.LimitMemory(handle, 32*1024*1024)
			Expect(err).ToNot(HaveOccurred())

			// trigger 'out of memory' event
			_, _, err = client.Run(handle, "exec ruby -e '$stdout.sync = true; puts :hello; puts (\"x\" * 64 * 1024 * 1024).size; puts :goodbye; exit 42'", gordon.ResourceLimits{}, []gordon.EnvironmentVariable{})
			Expect(err).ToNot(HaveOccurred())

			Eventually(func() []string {
				info, err := client.Info(handle)
				Expect(err).ToNot(HaveOccurred())

				return info.GetEvents()
			}).Should(ContainElement("out of memory"))

			restartServer()

			info, err := client.Info(handle)
			Expect(err).ToNot(HaveOccurred())

			Expect(info.GetEvents()).To(ContainElement("out of memory"))
		})
	})

	Describe("a container's properties", func() {
		It("are retained", func() {
			createRes, err := client.Create(map[string]string{
				"foo": "bar",
			})
			Expect(err).ToNot(HaveOccurred())

			info, err := client.Info(createRes.GetHandle())
			Expect(err).ToNot(HaveOccurred())

			Expect(info.GetProperties()).To(ContainElement(&warden.Property{
				Key:   proto.String("foo"),
				Value: proto.String("bar"),
			}))

			restartServer()

			info, err = client.Info(createRes.GetHandle())
			Expect(err).ToNot(HaveOccurred())

			Expect(info.GetProperties()).To(ContainElement(&warden.Property{
				Key:   proto.String("foo"),
				Value: proto.String("bar"),
			}))
		})
	})

	Describe("a container's state", func() {
		It("is still reported", func() {
			info, err := client.Info(handle)
			Expect(err).ToNot(HaveOccurred())

			Expect(info.GetState()).To(Equal("active"))

			restartServer()

			info, err = client.Info(handle)
			Expect(err).ToNot(HaveOccurred())

			Expect(info.GetState()).To(Equal("active"))

			_, err = client.Stop(handle, false, false)
			Expect(err).ToNot(HaveOccurred())

			restartServer()

			info, err = client.Info(handle)
			Expect(err).ToNot(HaveOccurred())

			Expect(info.GetState()).To(Equal("stopped"))
		})
	})

	Describe("a container's network", func() {
		It("does not get reused", func() {
			infoA, err := client.Info(handle)
			Expect(err).ToNot(HaveOccurred())

			restartServer()

			res, err := client.Create(nil)
			Expect(err).ToNot(HaveOccurred())

			infoB, err := client.Info(res.GetHandle())
			Expect(err).ToNot(HaveOccurred())

			Expect(infoA.GetHostIp()).ToNot(Equal(infoB.GetHostIp()))
			Expect(infoA.GetContainerIp()).ToNot(Equal(infoB.GetContainerIp()))
		})
	})

	Describe("a container's mapped port", func() {
		It("does not get reused", func() {
			netInA, err := client.NetIn(handle)
			Expect(err).ToNot(HaveOccurred())

			restartServer()

			createRes, err := client.Create(nil)
			Expect(err).ToNot(HaveOccurred())

			netInB, err := client.NetIn(createRes.GetHandle())
			Expect(err).ToNot(HaveOccurred())

			Expect(netInA.GetHostPort()).ToNot(Equal(netInB.GetHostPort()))
			Expect(netInA.GetContainerPort()).ToNot(Equal(netInB.GetContainerPort()))
		})
	})

	Describe("a container's user", func() {
		It("does not get reused", func() {
			idA := ""
			idB := ""

			_, streamA, err := client.Run(handle, "id -u", gordon.ResourceLimits{}, []gordon.EnvironmentVariable{})
			Expect(err).ToNot(HaveOccurred())

			for chunk := range streamA {
				idA += chunk.GetData()
			}

			restartServer()

			createRes, err := client.Create(nil)
			Expect(err).ToNot(HaveOccurred())

			_, streamB, err := client.Run(createRes.GetHandle(), "id -u", gordon.ResourceLimits{}, []gordon.EnvironmentVariable{})
			Expect(err).ToNot(HaveOccurred())

			for chunk := range streamB {
				idB += chunk.GetData()
			}

			Expect(idA).ToNot(Equal(idB))
		})
	})

	Describe("a container's grace time", func() {
		BeforeEach(func() {
			err := runner.Stop()
			Expect(err).ToNot(HaveOccurred())

			err = runner.Start("--containerGraceTime", "5s")
			Expect(err).ToNot(HaveOccurred())

			res, err := client.Create(nil)
			Expect(err).ToNot(HaveOccurred())

			handle = res.GetHandle()
		})

		It("is still enforced", func() {
			restartServer()

			listRes, err := client.List(nil)
			Expect(err).ToNot(HaveOccurred())

			Expect(listRes.GetHandles()).To(ContainElement(handle))

			time.Sleep(6 * time.Second)

			listRes, err = client.List(nil)
			Expect(err).ToNot(HaveOccurred())

			Expect(listRes.GetHandles()).ToNot(ContainElement(handle))
		})
	})
})

func streamNumbersTo(destination chan<- int, source <-chan *warden.ProcessPayload) {
	for out := range source {
		buf := bytes.NewBufferString(out.GetData())

		var num int

		for {
			_, err := fmt.Fscanf(buf, "%d\n", &num)
			if err == io.EOF {
				break
			}

			// got goodbye
			if err != nil {
				close(destination)
				return
			}

			destination <- num
		}
	}
}
<|endoftext|>