<commit_before>package bbs

import (
	"time"

	"github.com/cloudfoundry-incubator/runtime-schema/bbs/domain_bbs"
	"github.com/cloudfoundry-incubator/runtime-schema/bbs/lock_bbs"
	"github.com/cloudfoundry-incubator/runtime-schema/bbs/lrp_bbs"
	"github.com/cloudfoundry-incubator/runtime-schema/bbs/services_bbs"
	"github.com/cloudfoundry-incubator/runtime-schema/bbs/task_bbs"
	"github.com/cloudfoundry-incubator/runtime-schema/cb"
	"github.com/cloudfoundry-incubator/runtime-schema/models"
	"github.com/cloudfoundry/storeadapter"
	"github.com/pivotal-golang/clock"
	"github.com/pivotal-golang/lager"
	"github.com/tedsuo/ifrit"
)

//Bulletin Board System/Store

//go:generate counterfeiter -o fake_bbs/fake_receptor_bbs.go . ReceptorBBS
type ReceptorBBS interface {
	//task
	DesireTask(lager.Logger, models.Task) error
	Tasks(logger lager.Logger) ([]models.Task, error)
	TasksByDomain(logger lager.Logger, domain string) ([]models.Task, error)
	TaskByGuid(taskGuid string) (models.Task, error)
	ResolvingTask(logger lager.Logger, taskGuid string) error
	ResolveTask(logger lager.Logger, taskGuid string) error
	CancelTask(logger lager.Logger, taskGuid string) error

	//desired lrp
	DesireLRP(lager.Logger, models.DesiredLRP) error
	UpdateDesiredLRP(logger lager.Logger, processGuid string, update models.DesiredLRPUpdate) error
	RemoveDesiredLRPByProcessGuid(logger lager.Logger, processGuid string) error
	DesiredLRPs() ([]models.DesiredLRP, error)
	DesiredLRPsByDomain(domain string) ([]models.DesiredLRP, error)
	DesiredLRPByProcessGuid(processGuid string) (models.DesiredLRP, error)
	WatchForDesiredLRPChanges(logger lager.Logger, created func(models.DesiredLRP), changed func(models.DesiredLRPChange), deleted func(models.DesiredLRP)) (stop chan<- bool, errs <-chan error)

	//actual lrp
	ActualLRPGroups() ([]models.ActualLRPGroup, error)
	ActualLRPGroupsByDomain(domain string) ([]models.ActualLRPGroup, error)
	ActualLRPGroupsByProcessGuid(string) (models.ActualLRPGroupsByIndex, error)
	ActualLRPGroupByProcessGuidAndIndex(processGuid string, index int) (models.ActualLRPGroup, error)
	RetireActualLRPs(lager.Logger, []models.ActualLRP)
	WatchForActualLRPChanges(logger lager.Logger, created func(models.ActualLRP, bool), changed func(models.ActualLRPChange, bool), deleted func(models.ActualLRP, bool)) (stop chan<- bool, errs <-chan error)

	// cells
	Cells() ([]models.CellPresence, error)

	// domains
	UpsertDomain(domain string, ttlInSeconds int) error
	Domains() ([]string, error)

	//services
	NewReceptorHeartbeat(models.ReceptorPresence, time.Duration) ifrit.Runner
}

//go:generate counterfeiter -o fake_bbs/fake_rep_bbs.go . RepBBS
type RepBBS interface {
	//services
	NewCellHeartbeat(cellPresence models.CellPresence, interval time.Duration) ifrit.Runner

	//task
	StartTask(logger lager.Logger, taskGuid string, cellID string) (bool, error)
	TaskByGuid(taskGuid string) (models.Task, error)
	TasksByCellID(logger lager.Logger, cellID string) ([]models.Task, error)
	FailTask(logger lager.Logger, taskGuid string, failureReason string) error
	CompleteTask(logger lager.Logger, taskGuid string, cellID string, failed bool, failureReason string, result string) error

	//lrp
	ActualLRPGroupsByCellID(cellID string) ([]models.ActualLRPGroup, error)
	ClaimActualLRP(lager.Logger, models.ActualLRPKey, models.ActualLRPContainerKey) error
	StartActualLRP(lager.Logger, models.ActualLRPKey, models.ActualLRPContainerKey, models.ActualLRPNetInfo) error
	CrashActualLRP(lager.Logger, models.ActualLRPKey, models.ActualLRPContainerKey) error
	RemoveActualLRP(lager.Logger, models.ActualLRPKey, models.ActualLRPContainerKey) error

	// LRP evacuation
	EvacuateClaimedActualLRP(lager.Logger, models.ActualLRPKey, models.ActualLRPContainerKey) error
	EvacuateRunningActualLRP(lager.Logger, models.ActualLRPKey, models.ActualLRPContainerKey, models.ActualLRPNetInfo, uint64) error
	EvacuateStoppedActualLRP(lager.Logger, models.ActualLRPKey, models.ActualLRPContainerKey) error
	EvacuateCrashedActualLRP(lager.Logger, models.ActualLRPKey, models.ActualLRPContainerKey) error
	RemoveEvacuatingActualLRP(lager.Logger, models.ActualLRPKey, models.ActualLRPContainerKey) error
}

//go:generate counterfeiter -o fake_bbs/fake_converger_bbs.go . ConvergerBBS
type ConvergerBBS interface {
	//lock
	NewConvergeLock(convergerID string, interval time.Duration) ifrit.Runner

	//lrp
	ConvergeLRPs(lager.Logger)

	//task
	ConvergeTasks(logger lager.Logger, timeToClaim, convergenceInterval, timeToResolve time.Duration)

	//cells
	WaitForCellEvent() (services_bbs.CellEvent, error)
}

//go:generate counterfeiter -o fake_bbs/fake_nsync_bbs.go . NsyncBBS
type NsyncBBS interface {
	//lock
	NewNsyncBulkerLock(bulkerID string, interval time.Duration) ifrit.Runner
	NewNsyncListenerLock(listenerID string, interval time.Duration) ifrit.Runner
}

//go:generate counterfeiter -o fake_bbs/fake_auctioneer_bbs.go . AuctioneerBBS
type AuctioneerBBS interface {
	//services
	Cells() ([]models.CellPresence, error)

	// task
	FailTask(logger lager.Logger, taskGuid string, failureReason string) error

	//lock
	NewAuctioneerLock(auctioneerPresence models.AuctioneerPresence, interval time.Duration) (ifrit.Runner, error)

	//lrp
	FailActualLRP(lager.Logger, models.ActualLRPKey, string) error
}

//go:generate counterfeiter -o fake_bbs/fake_metrics_bbs.go . MetricsBBS
type MetricsBBS interface {
	//task
	Tasks(logger lager.Logger) ([]models.Task, error)

	//services
	ServiceRegistrations() (models.ServiceRegistrations, error)

	// domains
	Domains() ([]string, error)

	//lrps
	DesiredLRPs() ([]models.DesiredLRP, error)
	ActualLRPs() ([]models.ActualLRP, error)

	//lock
	NewRuntimeMetricsLock(runtimeMetricsID string, interval time.Duration) ifrit.Runner
}

//go:generate counterfeiter -o fake_bbs/fake_route_emitter_bbs.go . RouteEmitterBBS
type RouteEmitterBBS interface {
	//lock
	NewRouteEmitterLock(emitterID string, interval time.Duration) ifrit.Runner
}

type VeritasBBS interface {
	//task
	Tasks(logger lager.Logger) ([]models.Task, error)

	//lrp
	DesiredLRPs() ([]models.DesiredLRP, error)
	ActualLRPGroups() ([]models.ActualLRPGroup, error)
	DesireLRP(lager.Logger, models.DesiredLRP) error
	RemoveDesiredLRPByProcessGuid(logger lager.Logger, guid string) error

	// domains
	Domains() ([]string, error)
	UpsertDomain(domain string, ttlInSeconds int) error

	//services
	Cells() ([]models.CellPresence, error)
	AuctioneerAddress() (string, error)
}

func NewReceptorBBS(store storeadapter.StoreAdapter, clock clock.Clock, logger lager.Logger) ReceptorBBS {
	return NewBBS(store, clock, logger)
}

func NewRepBBS(store storeadapter.StoreAdapter, clock clock.Clock, logger lager.Logger) RepBBS {
	return NewBBS(store, clock, logger)
}

func NewConvergerBBS(store storeadapter.StoreAdapter, clock clock.Clock, logger lager.Logger) ConvergerBBS {
	return NewBBS(store, clock, logger)
}

func NewNsyncBBS(store storeadapter.StoreAdapter, clock clock.Clock, logger lager.Logger) NsyncBBS {
	return NewBBS(store, clock, logger)
}

func NewAuctioneerBBS(store storeadapter.StoreAdapter, clock clock.Clock, logger lager.Logger) AuctioneerBBS {
	return NewBBS(store, clock, logger)
}

func NewMetricsBBS(store storeadapter.StoreAdapter, clock clock.Clock, logger lager.Logger) MetricsBBS {
	return NewBBS(store, clock, logger)
}

func NewRouteEmitterBBS(store storeadapter.StoreAdapter, clock clock.Clock, logger lager.Logger) RouteEmitterBBS {
	return NewBBS(store, clock, logger)
}

func NewVeritasBBS(store storeadapter.StoreAdapter, clock clock.Clock, logger lager.Logger) VeritasBBS {
	return NewBBS(store, clock, logger)
}

func NewBBS(store storeadapter.StoreAdapter, clock clock.Clock, logger lager.Logger) *BBS {
	services := services_bbs.New(store, clock, logger.Session("services-bbs"))
	auctioneerClient := cb.NewAuctioneerClient()
	retryPolicy := storeadapter.ExponentialRetryPolicy{}

	return &BBS{
		LockBBS:     lock_bbs.New(store, clock, logger.Session("lock-bbs")),
		LRPBBS:      lrp_bbs.New(storeadapter.NewRetryable(store, clock, retryPolicy), clock, cb.NewCellClient(), auctioneerClient, services),
		ServicesBBS: services,
		TaskBBS:     task_bbs.New(storeadapter.NewRetryable(store, clock, retryPolicy), clock, cb.NewTaskClient(), auctioneerClient, cb.NewCellClient(), services),
		DomainBBS:   domain_bbs.New(store, logger),
	}
}

type BBS struct {
	*lock_bbs.LockBBS
	*lrp_bbs.LRPBBS
	*services_bbs.ServicesBBS
	*task_bbs.TaskBBS
	*domain_bbs.DomainBBS
}
<commit_msg>Lets not retry BBS operations<commit_after>package bbs

import (
	"time"

	"github.com/cloudfoundry-incubator/runtime-schema/bbs/domain_bbs"
	"github.com/cloudfoundry-incubator/runtime-schema/bbs/lock_bbs"
	"github.com/cloudfoundry-incubator/runtime-schema/bbs/lrp_bbs"
	"github.com/cloudfoundry-incubator/runtime-schema/bbs/services_bbs"
	"github.com/cloudfoundry-incubator/runtime-schema/bbs/task_bbs"
	"github.com/cloudfoundry-incubator/runtime-schema/cb"
	"github.com/cloudfoundry-incubator/runtime-schema/models"
	"github.com/cloudfoundry/storeadapter"
	"github.com/pivotal-golang/clock"
	"github.com/pivotal-golang/lager"
	"github.com/tedsuo/ifrit"
)

//Bulletin Board System/Store

//go:generate counterfeiter -o fake_bbs/fake_receptor_bbs.go . ReceptorBBS
type ReceptorBBS interface {
	//task
	DesireTask(lager.Logger, models.Task) error
	Tasks(logger lager.Logger) ([]models.Task, error)
	TasksByDomain(logger lager.Logger, domain string) ([]models.Task, error)
	TaskByGuid(taskGuid string) (models.Task, error)
	ResolvingTask(logger lager.Logger, taskGuid string) error
	ResolveTask(logger lager.Logger, taskGuid string) error
	CancelTask(logger lager.Logger, taskGuid string) error

	//desired lrp
	DesireLRP(lager.Logger, models.DesiredLRP) error
	UpdateDesiredLRP(logger lager.Logger, processGuid string, update models.DesiredLRPUpdate) error
	RemoveDesiredLRPByProcessGuid(logger lager.Logger, processGuid string) error
	DesiredLRPs() ([]models.DesiredLRP, error)
	DesiredLRPsByDomain(domain string) ([]models.DesiredLRP, error)
	DesiredLRPByProcessGuid(processGuid string) (models.DesiredLRP, error)
	WatchForDesiredLRPChanges(logger lager.Logger, created func(models.DesiredLRP), changed func(models.DesiredLRPChange), deleted func(models.DesiredLRP)) (stop chan<- bool, errs <-chan error)

	//actual lrp
	ActualLRPGroups() ([]models.ActualLRPGroup, error)
	ActualLRPGroupsByDomain(domain string) ([]models.ActualLRPGroup, error)
	ActualLRPGroupsByProcessGuid(string) (models.ActualLRPGroupsByIndex, error)
	ActualLRPGroupByProcessGuidAndIndex(processGuid string, index int) (models.ActualLRPGroup, error)
	RetireActualLRPs(lager.Logger, []models.ActualLRP)
	WatchForActualLRPChanges(logger lager.Logger, created func(models.ActualLRP, bool), changed func(models.ActualLRPChange, bool), deleted func(models.ActualLRP, bool)) (stop chan<- bool, errs <-chan error)

	// cells
	Cells() ([]models.CellPresence, error)

	// domains
	UpsertDomain(domain string, ttlInSeconds int) error
	Domains() ([]string, error)

	//services
	NewReceptorHeartbeat(models.ReceptorPresence, time.Duration) ifrit.Runner
}

//go:generate counterfeiter -o fake_bbs/fake_rep_bbs.go . RepBBS
type RepBBS interface {
	//services
	NewCellHeartbeat(cellPresence models.CellPresence, interval time.Duration) ifrit.Runner

	//task
	StartTask(logger lager.Logger, taskGuid string, cellID string) (bool, error)
	TaskByGuid(taskGuid string) (models.Task, error)
	TasksByCellID(logger lager.Logger, cellID string) ([]models.Task, error)
	FailTask(logger lager.Logger, taskGuid string, failureReason string) error
	CompleteTask(logger lager.Logger, taskGuid string, cellID string, failed bool, failureReason string, result string) error

	//lrp
	ActualLRPGroupsByCellID(cellID string) ([]models.ActualLRPGroup, error)
	ClaimActualLRP(lager.Logger, models.ActualLRPKey, models.ActualLRPContainerKey) error
	StartActualLRP(lager.Logger, models.ActualLRPKey, models.ActualLRPContainerKey, models.ActualLRPNetInfo) error
	CrashActualLRP(lager.Logger, models.ActualLRPKey, models.ActualLRPContainerKey) error
	RemoveActualLRP(lager.Logger, models.ActualLRPKey, models.ActualLRPContainerKey) error

	// LRP evacuation
	EvacuateClaimedActualLRP(lager.Logger, models.ActualLRPKey, models.ActualLRPContainerKey) error
	EvacuateRunningActualLRP(lager.Logger, models.ActualLRPKey, models.ActualLRPContainerKey, models.ActualLRPNetInfo, uint64) error
	EvacuateStoppedActualLRP(lager.Logger, models.ActualLRPKey, models.ActualLRPContainerKey) error
	EvacuateCrashedActualLRP(lager.Logger, models.ActualLRPKey, models.ActualLRPContainerKey) error
	RemoveEvacuatingActualLRP(lager.Logger, models.ActualLRPKey, models.ActualLRPContainerKey) error
}

//go:generate counterfeiter -o fake_bbs/fake_converger_bbs.go . ConvergerBBS
type ConvergerBBS interface {
	//lock
	NewConvergeLock(convergerID string, interval time.Duration) ifrit.Runner

	//lrp
	ConvergeLRPs(lager.Logger)

	//task
	ConvergeTasks(logger lager.Logger, timeToClaim, convergenceInterval, timeToResolve time.Duration)

	//cells
	WaitForCellEvent() (services_bbs.CellEvent, error)
}

//go:generate counterfeiter -o fake_bbs/fake_nsync_bbs.go . NsyncBBS
type NsyncBBS interface {
	//lock
	NewNsyncBulkerLock(bulkerID string, interval time.Duration) ifrit.Runner
	NewNsyncListenerLock(listenerID string, interval time.Duration) ifrit.Runner
}

//go:generate counterfeiter -o fake_bbs/fake_auctioneer_bbs.go . AuctioneerBBS
type AuctioneerBBS interface {
	//services
	Cells() ([]models.CellPresence, error)

	// task
	FailTask(logger lager.Logger, taskGuid string, failureReason string) error

	//lock
	NewAuctioneerLock(auctioneerPresence models.AuctioneerPresence, interval time.Duration) (ifrit.Runner, error)

	//lrp
	FailActualLRP(lager.Logger, models.ActualLRPKey, string) error
}

//go:generate counterfeiter -o fake_bbs/fake_metrics_bbs.go . MetricsBBS
type MetricsBBS interface {
	//task
	Tasks(logger lager.Logger) ([]models.Task, error)

	//services
	ServiceRegistrations() (models.ServiceRegistrations, error)

	// domains
	Domains() ([]string, error)

	//lrps
	DesiredLRPs() ([]models.DesiredLRP, error)
	ActualLRPs() ([]models.ActualLRP, error)

	//lock
	NewRuntimeMetricsLock(runtimeMetricsID string, interval time.Duration) ifrit.Runner
}

//go:generate counterfeiter -o fake_bbs/fake_route_emitter_bbs.go . RouteEmitterBBS
type RouteEmitterBBS interface {
	//lock
	NewRouteEmitterLock(emitterID string, interval time.Duration) ifrit.Runner
}

type VeritasBBS interface {
	//task
	Tasks(logger lager.Logger) ([]models.Task, error)

	//lrp
	DesiredLRPs() ([]models.DesiredLRP, error)
	ActualLRPGroups() ([]models.ActualLRPGroup, error)
	DesireLRP(lager.Logger, models.DesiredLRP) error
	RemoveDesiredLRPByProcessGuid(logger lager.Logger, guid string) error

	// domains
	Domains() ([]string, error)
	UpsertDomain(domain string, ttlInSeconds int) error

	//services
	Cells() ([]models.CellPresence, error)
	AuctioneerAddress() (string, error)
}

func NewReceptorBBS(store storeadapter.StoreAdapter, clock clock.Clock, logger lager.Logger) ReceptorBBS {
	return NewBBS(store, clock, logger)
}

func NewRepBBS(store storeadapter.StoreAdapter, clock clock.Clock, logger lager.Logger) RepBBS {
	return NewBBS(store, clock, logger)
}

func NewConvergerBBS(store storeadapter.StoreAdapter, clock clock.Clock, logger lager.Logger) ConvergerBBS {
	return NewBBS(store, clock, logger)
}

func NewNsyncBBS(store storeadapter.StoreAdapter, clock clock.Clock, logger lager.Logger) NsyncBBS {
	return NewBBS(store, clock, logger)
}

func NewAuctioneerBBS(store storeadapter.StoreAdapter, clock clock.Clock, logger lager.Logger) AuctioneerBBS {
	return NewBBS(store, clock, logger)
}

func NewMetricsBBS(store storeadapter.StoreAdapter, clock clock.Clock, logger lager.Logger) MetricsBBS {
	return NewBBS(store, clock, logger)
}

func NewRouteEmitterBBS(store storeadapter.StoreAdapter, clock clock.Clock, logger lager.Logger) RouteEmitterBBS {
	return NewBBS(store, clock, logger)
}

func NewVeritasBBS(store storeadapter.StoreAdapter, clock clock.Clock, logger lager.Logger) VeritasBBS {
	return NewBBS(store, clock, logger)
}

func NewBBS(store storeadapter.StoreAdapter, clock clock.Clock, logger lager.Logger) *BBS {
	services := services_bbs.New(store, clock, logger.Session("services-bbs"))
	auctioneerClient := cb.NewAuctioneerClient()

	return &BBS{
		LockBBS:     lock_bbs.New(store, clock, logger.Session("lock-bbs")),
		LRPBBS:      lrp_bbs.New(store, clock, cb.NewCellClient(), auctioneerClient, services),
		ServicesBBS: services,
		TaskBBS:     task_bbs.New(store, clock, cb.NewTaskClient(), auctioneerClient, cb.NewCellClient(), services),
		DomainBBS:   domain_bbs.New(store, logger),
	}
}

type BBS struct {
	*lock_bbs.LockBBS
	*lrp_bbs.LRPBBS
	*services_bbs.ServicesBBS
	*task_bbs.TaskBBS
	*domain_bbs.DomainBBS
}
<|endoftext|>


<commit_before>package ggsearch

import (
	"github.com/stretchr/testify/assert"
	check "gopkg.in/check.v1"
	"math"
	"testing"
)

// Hook up gocheck into the "go test" runner.
func TestGeoUtils(t *testing.T) { check.TestingT(t) }

type GeoUtilsSuite struct {
}

var _ = check.Suite(&GeoUtilsSuite{})

func (s *GeoUtilsSuite) TestDegreesToRadiansZero(c *check.C) {
	c.Assert(degreesToRadians(0), check.Equals, 0.0)
}

func (s *GeoUtilsSuite) TestDegreesToRadiansHalfPi(c *check.C) {
	c.Assert(degreesToRadians(90), check.Equals, math.Pi/2)
}

func (s *GeoUtilsSuite) TestDegreesToRadiansPi(c *check.C) {
	c.Assert(degreesToRadians(180), check.Equals, math.Pi)
}

func TestRadLatToGrid(t *testing.T) {
	assert.Equal(t, radLatToGrid(0, 10), 5)
	assert.Equal(t, radLatToGrid(-0.1, 10), 4)
	assert.Equal(t, radLatToGrid(math.Pi/2-0.1, 10), 9)
	assert.Equal(t, radLatToGrid(-math.Pi/2+0.1, 10), 0)
}

func TestRadLngToGrid(t *testing.T) {
	assert.Equal(t, radLngToGrid(0, 10), 5)
	assert.Equal(t, radLngToGrid(-0.1, 10), 4)
	assert.Equal(t, radLngToGrid(math.Pi-0.1, 10), 9)
	assert.Equal(t, radLngToGrid(-math.Pi+0.1, 10), 0)
}

func (s *GeoUtilsSuite) TestClampGridLat(c *check.C) {
	c.Assert(clampGridLat(10, 11), check.Equals, 10)
	c.Assert(clampGridLat(11, 11), check.Equals, 10)
	c.Assert(clampGridLat(0, 11), check.Equals, 0)
	c.Assert(clampGridLat(-1, 11), check.Equals, 0)
}

func (s *GeoUtilsSuite) TestWrapGridLng(c *check.C) {
	c.Assert(wrapGridLng(10, 11), check.Equals, 10)
	c.Assert(wrapGridLng(11, 11), check.Equals, 0)
	c.Assert(wrapGridLng(0, 11), check.Equals, 0)
	c.Assert(wrapGridLng(-1, 11), check.Equals, 10)
}

func TestHaversine(t *testing.T) {
	assert := assert.New(t)
	distance := haversine(50*math.Pi/180, -70*math.Pi/180,
		51*math.Pi/180, -71*math.Pi/180)
	assert.InDelta(distance, 81.89, 0.01)
}
<commit_msg>Drop check, use testify<commit_after>package ggsearch

import (
	"github.com/stretchr/testify/assert"
	"math"
	"testing"
)

func TestDegreesToRadiansZero(t *testing.T) {
	assert.Equal(t, degreesToRadians(0), 0.0)
}

func TestDegreesToRadiansHalfPi(t *testing.T) {
	assert.Equal(t, degreesToRadians(90), math.Pi/2)
}

func TestDegreesToRadiansPi(t *testing.T) {
	assert.Equal(t, degreesToRadians(180), math.Pi)
}

func TestRadLatToGrid(t *testing.T) {
	assert.Equal(t, radLatToGrid(0, 10), 5)
	assert.Equal(t, radLatToGrid(-0.1, 10), 4)
	assert.Equal(t, radLatToGrid(math.Pi/2-0.1, 10), 9)
	assert.Equal(t, radLatToGrid(-math.Pi/2+0.1, 10), 0)
}

func TestRadLngToGrid(t *testing.T) {
	assert.Equal(t, radLngToGrid(0, 10), 5)
	assert.Equal(t, radLngToGrid(-0.1, 10), 4)
	assert.Equal(t, radLngToGrid(math.Pi-0.1, 10), 9)
	assert.Equal(t, radLngToGrid(-math.Pi+0.1, 10), 0)
}

func TestClampGridLat(t *testing.T) {
	assert.Equal(t, clampGridLat(10, 11), 10)
	assert.Equal(t, clampGridLat(11, 11), 10)
	assert.Equal(t, clampGridLat(0, 11), 0)
	assert.Equal(t, clampGridLat(-1, 11), 0)
}

func TestWrapGridLng(t *testing.T) {
	assert.Equal(t, wrapGridLng(10, 11), 10)
	assert.Equal(t, wrapGridLng(11, 11), 0)
	assert.Equal(t, wrapGridLng(0, 11), 0)
	assert.Equal(t, wrapGridLng(-1, 11), 10)
}

func TestHaversine(t *testing.T) {
	assert := assert.New(t)
	distance := haversine(50*math.Pi/180, -70*math.Pi/180,
		51*math.Pi/180, -71*math.Pi/180)
	assert.InDelta(distance, 81.89, 0.01)
}
<|endoftext|>


<commit_before>package bild

import (
	"image"
	"image/color"
)

// Invert returns a negated version of the image
func Invert(src image.Image) *image.NRGBA {
	img := cloneAsNRGBA(src)

	fn := func(c color.NRGBA) color.NRGBA {
		return color.NRGBA{255 - c.R, 255 - c.G, 255 - c.B, c.A}
	}

	apply(img, fn)

	return img
}
<commit_msg>added brightness method<commit_after>package bild

import (
	"image"
	"image/color"
)

// Invert returns a negated version of the image
func Invert(src image.Image) *image.NRGBA {
	img := cloneAsNRGBA(src)

	fn := func(c color.NRGBA) color.NRGBA {
		return color.NRGBA{255 - c.R, 255 - c.G, 255 - c.B, c.A}
	}

	apply(img, fn)

	return img
}

// Brightness returns a copy of the image with the adjusted brightness
func Brightness(src image.Image, value int) *image.NRGBA {
	img := cloneAsNRGBA(src)

	fn := func(c color.NRGBA) color.NRGBA {
		return color.NRGBA{
			uint8(clamp(int(c.R)+value, 0, 255)),
			uint8(clamp(int(c.G)+value, 0, 255)),
			uint8(clamp(int(c.B)+value, 0, 255)),
			c.A}
	}

	apply(img, fn)

	return img
}
<|endoftext|>


<commit_before>// Package qemu provides implementations of a net.Conn and net.Listener backed by qemu_pipe
package qemu

import (
	"bytes"
	"encoding/binary"
	"errors"
	"io"
	"log"
	"net"
	"os"
	"strings"
	"sync"
	"syscall"
	"time"
)

const (
	qemuDriver = "/dev/qemu_pipe"
	qemuSvc    = "pipe:unix:"
	ioErrMsg   = "input/output error"
	rdyMsg     = "rdy"
)

var errClosed = errors.New("error connection closed")
var errNotImplemented = errors.New("error not implemented")

type qemuAddr string

// Network returns the network type of the connection
func (a qemuAddr) Network() string {
	return qemuDriver
}

// String returns the description of the connection
func (a qemuAddr) String() string {
	return string(a)
}

// Conn implements the net.Conn interface on top of a qemu_pipe
type Conn struct {
	// Backed by qemu_pipe
	conn io.ReadWriteCloser

	// Dummy addr
	addr net.Addr

	// Bytes remaining in the connection buffer
	// it would be tempting to simplify the code and use
	// an intermediate buffer to hold overflow buffer
	// but bytes.Buffer does not shrink which causes memory usage
	// to blow up
	left int

	closedReads  bool
	closedWrites bool

	readsCloseChan  chan struct{}
	writesCloseChan chan struct{}

	readLock  *sync.Mutex
	writeLock *sync.Mutex
}

// Read reads from from the Conn connection.
// Note that each message is prepended with the size, so we need to keep
// track of how many bytes we need to read across Read calls
func (q *Conn) Read(b []byte) (int, error) {
	q.readLock.Lock()
	defer q.readLock.Unlock()

	// Normally this could be done by just checking status != io.EOF, however
	// a read on /dev/qemu-pipe associated with a closed connection will return
	// EIO with the string "input/output error". Since errno isn't visible to us,
	// we check the error message and ensure the target string is not present.
	n, err := q.read(b)
	if err != nil {
		go q.CloseRead()
		if strings.Contains(err.Error(), ioErrMsg) {
			return n, io.EOF
		}
	}
	return n, err

}

func (q *Conn) read(b []byte) (int, error) {
	if q.closedReads {
		return 0, errClosed
	}

	toRead := q.left
	if toRead > len(b) {
		toRead = len(b)
	}

	// read leftovers from previous reads before trying to read the size
	if toRead > 0 {
		n, err := io.ReadFull(q.conn, b[:toRead])
		if err != nil {
			return n, err
		}
		q.left = q.left - n

		// b might be bigger than remaining bytes but conn might be empty and
		// we dont want to block on size read, so return early.
		return n, nil
	}

	var recd uint32
	if err := binary.Read(q.conn, binary.LittleEndian, &recd); err != nil {
		return 0, err
	}

	// The other side has signaled an EOF. Close connection for reads, we might have leftover writes
	if recd == 0 {
		return 0, io.EOF
	}

	toRead = int(recd)
	if toRead > len(b) {
		toRead = len(b)
	}

	n, err := io.ReadFull(q.conn, b[:toRead])
	if err != nil {
		return n, err
	}
	q.left = int(recd) - toRead
	return n, nil
}

// Write writes the contents of b to the underlying connection
func (q *Conn) Write(b []byte) (int, error) {
	q.writeLock.Lock()
	defer q.writeLock.Unlock()

	if q.closedWrites {
		return 0, errClosed
	}

	// Prepend the message with size
	if err := binary.Write(q.conn, binary.LittleEndian, uint32(len(b))); err != nil {
		return 0, err
	}
	return q.conn.Write(b)
}

// Close closes the connection
func (q *Conn) Close() error {
	return q.CloseWrite()
}

// CloseRead closes the read side of the connection
func (q *Conn) CloseRead() error {
	q.readLock.Lock()
	defer q.readLock.Unlock()

	if q.closedReads {
		return errClosed
	}
	q.closedReads = true
	close(q.readsCloseChan)
	return nil
}

// CloseWrite closes the write side of the connection
func (q *Conn) CloseWrite() error {
	q.writeLock.Lock()
	defer q.writeLock.Unlock()

	if q.closedWrites == true {
		return errClosed
	}
	q.closedWrites = true

	err := q.sendClose()
	close(q.writesCloseChan)
	return err
}

func (q *Conn) sendClose() error {
	if err := binary.Write(q.conn, binary.LittleEndian, uint32(0)); err != nil {
		return err
	}
	_, e := q.conn.Write([]byte{})
	return e
}

// LocalAddr returns the qemu address
func (q *Conn) LocalAddr() net.Addr {
	return q.addr
}

// RemoteAddr returns the qemu address
func (q *Conn) RemoteAddr() net.Addr {
	return q.addr
}

// SetDeadline sets the connection deadline
func (q *Conn) SetDeadline(t time.Time) error {
	return errNotImplemented
}

// SetReadDeadline sets the read deadline
func (q *Conn) SetReadDeadline(t time.Time) error {
	return errNotImplemented
}

// SetWriteDeadline sets the write deadline
func (q *Conn) SetWriteDeadline(t time.Time) error {
	return errNotImplemented
}

// closeConn waits for the read end and the write end of th connection
// to be closed and then closes the underlying connection
func (q *Conn) closeConn() {
	// remember the ABC
	<-q.readsCloseChan
	<-q.writesCloseChan
	q.conn.Close()
}

func makeConn(conn io.ReadWriteCloser) *Conn {
	return &Conn{
		conn:            conn,
		addr:            qemuAddr(""),
		readsCloseChan:  make(chan struct{}),
		writesCloseChan: make(chan struct{}),
		readLock:        &sync.Mutex{},
		writeLock:       &sync.Mutex{},
	}
}

// ConnBuilder implements a qemu connection builder. It wraps around listener
// listening on a qemu pipe. It accepts connectsion and sync with the client
// before returning
type ConnBuilder struct {
	lis net.Listener
}

// Close closes the underlying net.Listener
func (b *ConnBuilder) Close() error {
	return b.lis.Close()
}

// Next will connect to the guest and return the connection.
func (b *ConnBuilder) Next() (net.Conn, error) {
	for {
		conn, err := b.lis.Accept()
		if err != nil {
			return nil, err
		}

		// sync with the server
		rdy := []byte(rdyMsg)
		if _, err := conn.Write(rdy); err != nil {
			continue
		}
		if _, err := io.ReadFull(conn, rdy); err != nil {
			continue
		}
		if !bytes.Equal([]byte(rdyMsg), rdy) {
			continue
		}

		q := makeConn(conn)

		go q.closeConn()
		return q, nil
	}
}

// MakeConnBuilder creates a new ConnBuilder struct
func MakeConnBuilder(emuDir, socket string) (*ConnBuilder, error) {
	wd, err := os.Getwd()
	if err != nil {
		return nil, err
	}

	if err := os.Chdir(emuDir); err != nil {
		return nil, err
	}

	if err := os.Remove(socket); err != nil && !os.IsNotExist(err) {
		return nil, err
	}
	lis, err := net.Listen("unix", socket)
	if err != nil {
		return nil, err
	}

	if err := os.Chdir(wd); err != nil {
		return nil, err
	}

	return &ConnBuilder{lis: lis}, nil
}

// Pipe implements a net.Listener on top of a guest qemu pipe
type Pipe struct {
	socketName string
	closed     bool
}

// Accept creates a new net.Conn backed by a qemu_pipe connetion
func (q *Pipe) Accept() (net.Conn, error) {

	log.Println("Creating new conn")
	if q.closed {
		return nil, errClosed
	}

	// Each new file descriptor we open will create a new connection
	// We need to wait on the host to be ready:
	// 1) poll the qemu_pipe driver with the desiered socket name
	// 2) wait until the client is read to send/recv, we do this waiting until we read a rdy message
	var conn *os.File
	var err error
	br := false
	for {
		if conn != nil {
			// Got an error, close connection and try again
			conn.Close()
			time.Sleep(20 * time.Millisecond)
		}

		log.Println("Opening file")
		if conn, err = os.OpenFile(qemuDriver, os.O_RDWR, 0600); err != nil {
			return nil, err
		}
		log.Println("File opened")

		// qemu_pipe does not properly support polling io. Force blocking io
		if err := syscall.SetNonblock(int(conn.Fd()), false); err != nil {
			return nil, err
		}

		svcName := qemuSvc + q.socketName
		buff := make([]byte, len(svcName)+1)
		copy(buff, svcName)

		// retry loop to wait until we can start the service on the qemu_pipe
		for {
			log.Println("Writing service")
			written, err := conn.Write(buff)
			if err != nil {
				// The host has not opened the socket. Sleep and try again
				conn.Close()
				time.Sleep(100 * time.Millisecond)
				break
			}
			buff = buff[written:]
			if len(buff) == 0 {
				log.Println("Wrote service")
				br = true
				break
			}
		}

		if br {
			// Wait for the client to open a socket on the host
			waitBuff := make([]byte, len(rdyMsg))
			log.Println("Reading rdy")
			if _, err := io.ReadFull(conn, waitBuff); err != nil {
				br = false
				continue
			}
			if !bytes.Equal([]byte(rdyMsg), waitBuff) {
				br = false
				continue
			}
			log.Println("Writing rdy")
			if _, err := conn.Write(waitBuff); err != nil {
				br = false
				continue
			}

			log.Println("Done")
			q := makeConn(conn)

			go q.closeConn()
			return q, nil

		}
	}

}

// Close closes the connection
func (q *Pipe) Close() error {
	q.closed = true
	return nil
}

// Addr returns the connection address
func (q *Pipe) Addr() net.Addr {
	return qemuAddr(q.socketName)
}

// MakePipe will return a new net.Listener
// backed by a qemu pipe. Qemu pipes are implemented as virtual
// devices. To get a handle an open("/dev/qemu_pipe") is issued.
// The virtual driver keeps a map of file descriptors to available
// services. In this case we open a unix socket service and return that.
func MakePipe(socketName string) (*Pipe, error) {
	if _, err := os.Stat(qemuDriver); err != nil {
		return nil, err
	}

	return &Pipe{socketName: socketName}, nil
}
<commit_msg>Avoid race condition by opening file in blocking mode as opposed to opening file and then setting it to blocking mode. If the latter is done go with register the file descriptor with the runtime and begin polling, which will cause reads to block if there is no data ready to be read.<commit_after>// Package qemu provides implementations of a net.Conn and net.Listener backed by qemu_pipe
package qemu

import (
	"bytes"
	"encoding/binary"
	"errors"
	"io"
	"log"
	"net"
	"os"
	"strings"
	"sync"
	"syscall"
	"time"
)

const (
	qemuDriver = "/dev/qemu_pipe"
	qemuSvc    = "pipe:unix:"
	ioErrMsg   = "input/output error"
	rdyMsg     = "rdy"
)

var errClosed = errors.New("error connection closed")
var errNotImplemented = errors.New("error not implemented")

type qemuAddr string

// Network returns the network type of the connection
func (a qemuAddr) Network() string {
	return qemuDriver
}

// String returns the description of the connection
func (a qemuAddr) String() string {
	return string(a)
}

// Conn implements the net.Conn interface on top of a qemu_pipe
type Conn struct {
	// Backed by qemu_pipe
	conn io.ReadWriteCloser

	// Dummy addr
	addr net.Addr

	// Bytes remaining in the connection buffer
	// it would be tempting to simplify the code and use
	// an intermediate buffer to hold overflow buffer
	// but bytes.Buffer does not shrink which causes memory usage
	// to blow up
	left int

	closedReads  bool
	closedWrites bool

	readsCloseChan  chan struct{}
	writesCloseChan chan struct{}

	readLock  *sync.Mutex
	writeLock *sync.Mutex
}

// Read reads from from the Conn connection.
// Note that each message is prepended with the size, so we need to keep
// track of how many bytes we need to read across Read calls
func (q *Conn) Read(b []byte) (int, error) {
	q.readLock.Lock()
	defer q.readLock.Unlock()

	// Normally this could be done by just checking status != io.EOF, however
	// a read on /dev/qemu-pipe associated with a closed connection will return
	// EIO with the string "input/output error". Since errno isn't visible to us,
	// we check the error message and ensure the target string is not present.
	n, err := q.read(b)
	if err != nil {
		go q.CloseRead()
		if strings.Contains(err.Error(), ioErrMsg) {
			return n, io.EOF
		}
	}
	return n, err

}

func (q *Conn) read(b []byte) (int, error) {
	if q.closedReads {
		return 0, errClosed
	}

	toRead := q.left
	if toRead > len(b) {
		toRead = len(b)
	}

	// read leftovers from previous reads before trying to read the size
	if toRead > 0 {
		n, err := io.ReadFull(q.conn, b[:toRead])
		if err != nil {
			return n, err
		}
		q.left = q.left - n

		// b might be bigger than remaining bytes but conn might be empty and
		// we dont want to block on size read, so return early.
		return n, nil
	}

	var recd uint32
	if err := binary.Read(q.conn, binary.LittleEndian, &recd); err != nil {
		return 0, err
	}

	// The other side has signaled an EOF. Close connection for reads, we might have leftover writes
	if recd == 0 {
		return 0, io.EOF
	}

	toRead = int(recd)
	if toRead > len(b) {
		toRead = len(b)
	}

	n, err := io.ReadFull(q.conn, b[:toRead])
	if err != nil {
		return n, err
	}
	q.left = int(recd) - toRead
	return n, nil
}

// Write writes the contents of b to the underlying connection
func (q *Conn) Write(b []byte) (int, error) {
	q.writeLock.Lock()
	defer q.writeLock.Unlock()

	if q.closedWrites {
		return 0, errClosed
	}

	// Prepend the message with size
	if err := binary.Write(q.conn, binary.LittleEndian, uint32(len(b))); err != nil {
		return 0, err
	}
	return q.conn.Write(b)
}

// Close closes the connection
func (q *Conn) Close() error {
	return q.CloseWrite()
}

// CloseRead closes the read side of the connection
func (q *Conn) CloseRead() error {
	q.readLock.Lock()
	defer q.readLock.Unlock()

	if q.closedReads {
		return errClosed
	}
	q.closedReads = true
	close(q.readsCloseChan)
	return nil
}

// CloseWrite closes the write side of the connection
func (q *Conn) CloseWrite() error {
	q.writeLock.Lock()
	defer q.writeLock.Unlock()

	if q.closedWrites == true {
		return errClosed
	}
	q.closedWrites = true

	err := q.sendClose()
	close(q.writesCloseChan)
	return err
}

func (q *Conn) sendClose() error {
	if err := binary.Write(q.conn, binary.LittleEndian, uint32(0)); err != nil {
		return err
	}
	_, e := q.conn.Write([]byte{})
	return e
}

// LocalAddr returns the qemu address
func (q *Conn) LocalAddr() net.Addr {
	return q.addr
}

// RemoteAddr returns the qemu address
func (q *Conn) RemoteAddr() net.Addr {
	return q.addr
}

// SetDeadline sets the connection deadline
func (q *Conn) SetDeadline(t time.Time) error {
	return errNotImplemented
}

// SetReadDeadline sets the read deadline
func (q *Conn) SetReadDeadline(t time.Time) error {
	return errNotImplemented
}

// SetWriteDeadline sets the write deadline
func (q *Conn) SetWriteDeadline(t time.Time) error {
	return errNotImplemented
}

// closeConn waits for the read end and the write end of th connection
// to be closed and then closes the underlying connection
func (q *Conn) closeConn() {
	// remember the ABC
	<-q.readsCloseChan
	<-q.writesCloseChan
	q.conn.Close()
}

func makeConn(conn io.ReadWriteCloser) *Conn {
	return &Conn{
		conn:            conn,
		addr:            qemuAddr(""),
		readsCloseChan:  make(chan struct{}),
		writesCloseChan: make(chan struct{}),
		readLock:        &sync.Mutex{},
		writeLock:       &sync.Mutex{},
	}
}

// ConnBuilder implements a qemu connection builder. It wraps around listener
// listening on a qemu pipe. It accepts connectsion and sync with the client
// before returning
type ConnBuilder struct {
	lis net.Listener
}

// Close closes the underlying net.Listener
func (b *ConnBuilder) Close() error {
	return b.lis.Close()
}

// Next will connect to the guest and return the connection.
func (b *ConnBuilder) Next() (net.Conn, error) {
	for {
		conn, err := b.lis.Accept()
		if err != nil {
			return nil, err
		}

		// sync with the server
		rdy := []byte(rdyMsg)
		if _, err := conn.Write(rdy); err != nil {
			conn.Close()
			continue
		}
		if _, err := io.ReadFull(conn, rdy); err != nil {
			conn.Close()
			continue
		}
		if !bytes.Equal([]byte(rdyMsg), rdy) {
			conn.Close()
			continue
		}

		q := makeConn(conn)

		go q.closeConn()
		return q, nil
	}
}

// MakeConnBuilder creates a new ConnBuilder struct
func MakeConnBuilder(emuDir, socket string) (*ConnBuilder, error) {
	wd, err := os.Getwd()
	if err != nil {
		return nil, err
	}

	if err := os.Chdir(emuDir); err != nil {
		return nil, err
	}

	if err := os.Remove(socket); err != nil && !os.IsNotExist(err) {
		return nil, err
	}
	lis, err := net.Listen("unix", socket)
	if err != nil {
		return nil, err
	}

	if err := os.Chdir(wd); err != nil {
		return nil, err
	}

	return &ConnBuilder{lis: lis}, nil
}

func openQemuDevBlocking() (*os.File, error) {
	// Open device manually in blocking mode. Qemu pipes don't support polling io.
	r, err := syscall.Open(qemuDriver, os.O_RDWR|syscall.O_CLOEXEC, 0600)
	if err != nil {
		return nil, err
	}

	if err := syscall.SetNonblock(r, false); err != nil {
		syscall.Close(r)
		return nil, err
	}
	return os.NewFile(uintptr(r), qemuDriver), nil
}

// Pipe implements a net.Listener on top of a guest qemu pipe
type Pipe struct {
	socketName string
	closed     bool
}

// Accept creates a new net.Conn backed by a qemu_pipe connetion
func (q *Pipe) Accept() (net.Conn, error) {

	log.Println("Creating new conn")
	if q.closed {
		return nil, errClosed
	}

	// Each new file descriptor we open will create a new connection
	// We need to wait on the host to be ready:
	// 1) poll the qemu_pipe driver with the desiered socket name
	// 2) wait until the client is read to send/recv, we do this waiting until we read a rdy message
	var conn *os.File
	var err error
	br := false
	for {
		if conn != nil {
			// Got an error, close connection and try again
			conn.Close()
			time.Sleep(20 * time.Millisecond)
		}

		conn, err = openQemuDevBlocking()
		if err != nil {
			return nil, err
		}
		log.Println("File opened")

		svcName := qemuSvc + q.socketName
		buff := make([]byte, len(svcName)+1)
		copy(buff, svcName)

		// retry loop to wait until we can start the service on the qemu_pipe
		for {
			log.Println("Writing service")
			written, err := conn.Write(buff)
			if err != nil {
				// The host has not opened the socket. Sleep and try again
				conn.Close()
				time.Sleep(100 * time.Millisecond)
				break
			}
			buff = buff[written:]
			if len(buff) == 0 {
				log.Println("Wrote service")
				br = true
				break
			}
		}

		if br {
			// Wait for the client to open a socket on the host
			waitBuff := make([]byte, len(rdyMsg))
			log.Println("Reading rdy")
			if _, err := io.ReadFull(conn, waitBuff); err != nil {
				br = false
				continue
			}
			if !bytes.Equal([]byte(rdyMsg), waitBuff) {
				br = false
				continue
			}
			log.Println("Writing rdy")
			if _, err := conn.Write(waitBuff); err != nil {
				br = false
				continue
			}

			log.Println("Done")
			q := makeConn(conn)

			go q.closeConn()
			return q, nil

		}
	}

}

// Close closes the connection
func (q *Pipe) Close() error {
	q.closed = true
	return nil
}

// Addr returns the connection address
func (q *Pipe) Addr() net.Addr {
	return qemuAddr(q.socketName)
}

// MakePipe will return a new net.Listener
// backed by a qemu pipe. Qemu pipes are implemented as virtual
// devices. To get a handle an open("/dev/qemu_pipe") is issued.
// The virtual driver keeps a map of file descriptors to available
// services. In this case we open a unix socket service and return that.
func MakePipe(socketName string) (*Pipe, error) {
	if _, err := os.Stat(qemuDriver); err != nil {
		return nil, err
	}

	return &Pipe{socketName: socketName}, nil
}
<|endoftext|>


<commit_before>/*
Copyright 2020 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"os"
	"os/exec"
	"runtime/debug"
	"strings"

	"github.com/pkg/errors"

	"github.com/spf13/viper"
	"k8s.io/minikube/pkg/minikube/constants"
	"k8s.io/minikube/pkg/minikube/download"
)

const (
	profile      = "generate-preloaded-images-tar"
	minikubePath = "out/minikube"
)

var (
	dockerStorageDriver = "overlay2"
	podmanStorageDriver = "overlay"
	containerRuntimes   = []string{"docker", "containerd", "cri-o"}
	k8sVersions         []string
	k8sVersion          = flag.String("kubernetes-version", "", "desired Kubernetes version, for example `v1.17.2`")
	noUpload            = flag.Bool("no-upload", false, "Do not upload tarballs to GCS")
	force               = flag.Bool("force", false, "Generate the preload tarball even if it's already exists")
	limit               = flag.Int("limit", 0, "Limit the number of tarballs to generate")
)

type preloadCfg struct {
	k8sVer  string
	runtime string
}

func (p preloadCfg) String() string {
	return fmt.Sprintf("%q/%q", p.runtime, p.k8sVer)
}

func main() {
	flag.Parse()
	viper.Set("preload", "true")

	if *k8sVersion != "" {
		k8sVersions = []string{*k8sVersion}
	}

	if err := deleteMinikube(); err != nil {
		fmt.Printf("error cleaning up minikube at start up: %v \n", err)
	}

	k8sVersions, err := collectK8sVers()
	if err != nil {
		exit("Unable to get recent k8s versions: %v\n", err)
	}

	var toGenerate []preloadCfg
	var i int

out:
	for _, kv := range k8sVersions {
		for _, cr := range containerRuntimes {
			if *limit > 0 && i >= *limit {
				break out
			}
			if *force || !download.PreloadExists(kv, cr) {
				toGenerate = append(toGenerate, preloadCfg{kv, cr})
				i++
				fmt.Printf("[%d] A preloaded tarball for k8s version %s - runtime %q does not exist.\n", i, kv, cr)
			} else {
				fmt.Printf("A preloaded tarball for k8s version %s - runtime %q already exists, skipping generation.\n", kv, cr)
			}
		}
	}

	fmt.Printf("Going to generate preloads for %v\n", toGenerate)

	for _, cfg := range toGenerate {
		if err := process(cfg); err != nil {
			exit(err.Error(), err)
		}
	}
}

func collectK8sVers() ([]string, error) {
	if k8sVersions == nil {
		recent, err := recentK8sVersions()
		if err != nil {
			return nil, err
		}
		k8sVersions = recent
	}
	return append([]string{
		constants.DefaultKubernetesVersion,
		constants.NewestKubernetesVersion,
		constants.OldestKubernetesVersion,
	}, k8sVersions...), nil
}

func process(cfg preloadCfg) error {
	kv, cr := cfg.k8sVer, cfg.runtime

	fmt.Printf("A preloaded tarball for k8s version %s - runtime %q doesn't exist, generating now...\n", kv, cr)
	tf := download.TarballName(kv, cr)

	defer func() {
		if err := deleteMinikube(); err != nil {
			fmt.Printf("error cleaning up minikube before finishing up: %v\n", err)
		}
	}()

	if err := generateTarball(kv, cr, tf); err != nil {
		return errors.Wrap(err, fmt.Sprintf("generating tarball for k8s version %s with %s", kv, cr))
	}

	if *noUpload {
		fmt.Printf("skip upload of %q\n", tf)
		return nil
	}
	if err := uploadTarball(tf); err != nil {
		return errors.Wrap(err, fmt.Sprintf("uploading tarball for k8s version %s with %s", kv, cr))
	}
	return nil
}

func verifyDockerStorage() error {
	cmd := exec.Command("docker", "exec", profile, "docker", "info", "-f", "{{.Info.Driver}}")
	var stderr bytes.Buffer
	cmd.Stderr = &stderr
	output, err := cmd.Output()
	if err != nil {
		return fmt.Errorf("%v: %v:\n%s", cmd.Args, err, stderr.String())
	}
	driver := strings.Trim(string(output), " \n")
	if driver != dockerStorageDriver {
		return fmt.Errorf("docker storage driver %s does not match requested %s", driver, dockerStorageDriver)
	}
	return nil
}

func verifyPodmanStorage() error {
	cmd := exec.Command("docker", "exec", profile, "sudo", "podman", "info", "-f", "json")
	var stderr bytes.Buffer
	cmd.Stderr = &stderr
	output, err := cmd.Output()
	if err != nil {
		return fmt.Errorf("%v: %v:\n%s", cmd.Args, err, stderr.String())
	}
	var info map[string]map[string]interface{}
	err = json.Unmarshal(output, &info)
	if err != nil {
		return err
	}
	driver := info["store"]["graphDriverName"]
	if driver != podmanStorageDriver {
		return fmt.Errorf("podman storage driver %s does not match requested %s", driver, podmanStorageDriver)
	}
	return nil
}

// exit will exit and clean up minikube
func exit(msg string, err error) {
	fmt.Printf("WithError(%s)=%v called from:\n%s", msg, err, debug.Stack())
	if err := deleteMinikube(); err != nil {
		fmt.Printf("error cleaning up minikube at start up: %v\n", err)
	}
	os.Exit(60)
}
<commit_msg>add comments<commit_after>/*
Copyright 2020 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"os"
	"os/exec"
	"runtime/debug"
	"strings"

	"github.com/pkg/errors"

	"github.com/spf13/viper"
	"k8s.io/minikube/pkg/minikube/constants"
	"k8s.io/minikube/pkg/minikube/download"
)

const (
	profile      = "generate-preloaded-images-tar"
	minikubePath = "out/minikube"
)

var (
	dockerStorageDriver = "overlay2"
	podmanStorageDriver = "overlay"
	containerRuntimes   = []string{"docker", "containerd", "cri-o"}
	k8sVersions         []string
	k8sVersion          = flag.String("kubernetes-version", "", "desired Kubernetes version, for example `v1.17.2`")
	noUpload            = flag.Bool("no-upload", false, "Do not upload tarballs to GCS")
	force               = flag.Bool("force", false, "Generate the preload tarball even if it's already exists")
	limit               = flag.Int("limit", 0, "Limit the number of tarballs to generate")
)

type preloadCfg struct {
	k8sVer  string
	runtime string
}

func (p preloadCfg) String() string {
	return fmt.Sprintf("%q/%q", p.runtime, p.k8sVer)
}

func main() {
	flag.Parse()

	// used by pkg/minikube/download.PreloadExists()
	viper.Set("preload", "true")

	if *k8sVersion != "" {
		k8sVersions = []string{*k8sVersion}
	}

	if err := deleteMinikube(); err != nil {
		fmt.Printf("error cleaning up minikube at start up: %v \n", err)
	}

	k8sVersions, err := collectK8sVers()
	if err != nil {
		exit("Unable to get recent k8s versions: %v\n", err)
	}

	var toGenerate []preloadCfg
	var i int

out:
	for _, kv := range k8sVersions {
		for _, cr := range containerRuntimes {
			if *limit > 0 && i >= *limit {
				break out
			}
			if !download.PreloadExists(kv, cr) {
				toGenerate = append(toGenerate, preloadCfg{kv, cr})
				i++
				fmt.Printf("[%d] A preloaded tarball for k8s version %s - runtime %q does not exist.\n", i, kv, cr)
			} else if *force {
				// the tarball already exists, but '--force' is passed. we need to overwrite the file
				toGenerate = append(toGenerate, preloadCfg{kv, cr})
				i++
				fmt.Printf("[%d] A preloaded tarball for k8s version %s - runtime %q already exists. Going to overwrite it.\n", i, kv, cr)
			} else {
				fmt.Printf("A preloaded tarball for k8s version %s - runtime %q already exists, skipping generation.\n", kv, cr)
			}
		}
	}

	fmt.Printf("Going to generate preloads for %v\n", toGenerate)

	for _, cfg := range toGenerate {
		if err := makePreload(cfg); err != nil {
			exit(err.Error(), err)
		}
	}
}

func collectK8sVers() ([]string, error) {
	if k8sVersions == nil {
		recent, err := recentK8sVersions()
		if err != nil {
			return nil, err
		}
		k8sVersions = recent
	}
	return append([]string{
		constants.DefaultKubernetesVersion,
		constants.NewestKubernetesVersion,
		constants.OldestKubernetesVersion,
	}, k8sVersions...), nil
}

func makePreload(cfg preloadCfg) error {
	kv, cr := cfg.k8sVer, cfg.runtime

	fmt.Printf("A preloaded tarball for k8s version %s - runtime %q doesn't exist, generating now...\n", kv, cr)
	tf := download.TarballName(kv, cr)

	defer func() {
		if err := deleteMinikube(); err != nil {
			fmt.Printf("error cleaning up minikube before finishing up: %v\n", err)
		}
	}()

	if err := generateTarball(kv, cr, tf); err != nil {
		return errors.Wrap(err, fmt.Sprintf("generating tarball for k8s version %s with %s", kv, cr))
	}

	if *noUpload {
		fmt.Printf("skip upload of %q\n", tf)
		return nil
	}
	if err := uploadTarball(tf); err != nil {
		return errors.Wrap(err, fmt.Sprintf("uploading tarball for k8s version %s with %s", kv, cr))
	}
	return nil
}

func verifyDockerStorage() error {
	cmd := exec.Command("docker", "exec", profile, "docker", "info", "-f", "{{.Info.Driver}}")
	var stderr bytes.Buffer
	cmd.Stderr = &stderr
	output, err := cmd.Output()
	if err != nil {
		return fmt.Errorf("%v: %v:\n%s", cmd.Args, err, stderr.String())
	}
	driver := strings.Trim(string(output), " \n")
	if driver != dockerStorageDriver {
		return fmt.Errorf("docker storage driver %s does not match requested %s", driver, dockerStorageDriver)
	}
	return nil
}

func verifyPodmanStorage() error {
	cmd := exec.Command("docker", "exec", profile, "sudo", "podman", "info", "-f", "json")
	var stderr bytes.Buffer
	cmd.Stderr = &stderr
	output, err := cmd.Output()
	if err != nil {
		return fmt.Errorf("%v: %v:\n%s", cmd.Args, err, stderr.String())
	}
	var info map[string]map[string]interface{}
	err = json.Unmarshal(output, &info)
	if err != nil {
		return err
	}
	driver := info["store"]["graphDriverName"]
	if driver != podmanStorageDriver {
		return fmt.Errorf("podman storage driver %s does not match requested %s", driver, podmanStorageDriver)
	}
	return nil
}

// exit will exit and clean up minikube
func exit(msg string, err error) {
	fmt.Printf("WithError(%s)=%v called from:\n%s", msg, err, debug.Stack())
	if err := deleteMinikube(); err != nil {
		fmt.Printf("error cleaning up minikube at start up: %v\n", err)
	}
	os.Exit(60)
}
<|endoftext|>


<commit_before>package main

import (
	"code.google.com/p/weed-fs/go/glog"
	"code.google.com/p/weed-fs/go/util"
	"code.google.com/p/weed-fs/go/weed/weed_server"
	"net/http"
	"os"
	"runtime"
	"strconv"
	"strings"
	"time"
)

func init() {
	cmdVolume.Run = runVolume // break init cycle
}

var cmdVolume = &Command{
	UsageLine: "volume -port=8080 -dir=/tmp -max=5 -ip=server_name -mserver=localhost:9333",
	Short:     "start a volume server",
	Long: `start a volume server to provide storage spaces

  `,
}

var (
	vport                 = cmdVolume.Flag.Int("port", 8080, "http listen port")
	volumeFolders         = cmdVolume.Flag.String("dir", os.TempDir(), "directories to store data files. dir[,dir]...")
	maxVolumeCounts       = cmdVolume.Flag.String("max", "7", "maximum numbers of volumes, count[,count]...")
	ip                    = cmdVolume.Flag.String("ip", "", "ip or server name")
	publicIp              = cmdVolume.Flag.String("publicIp", "", "Publicly accessible <ip|server_name>")
	masterNode            = cmdVolume.Flag.String("mserver", "localhost:9333", "master server location")
	vpulse                = cmdVolume.Flag.Int("pulseSeconds", 5, "number of seconds between heartbeats, must be smaller than or equal to the master's setting")
	vTimeout              = cmdVolume.Flag.Int("idleTimeout", 10, "connection idle seconds")
	vMaxCpu               = cmdVolume.Flag.Int("maxCpu", 0, "maximum number of CPUs. 0 means all available CPUs")
	dataCenter            = cmdVolume.Flag.String("dataCenter", "", "current volume server's data center name")
	rack                  = cmdVolume.Flag.String("rack", "", "current volume server's rack name")
	volumeWhiteListOption = cmdVolume.Flag.String("whiteList", "", "comma separated Ip addresses having write permission. No limit if empty.")

	volumeWhiteList []string
)

func runVolume(cmd *Command, args []string) bool {
	if *vMaxCpu < 1 {
		*vMaxCpu = runtime.NumCPU()
	}
	runtime.GOMAXPROCS(*vMaxCpu)
	folders := strings.Split(*volumeFolders, ",")
	maxCountStrings := strings.Split(*maxVolumeCounts, ",")
	maxCounts := make([]int, 0)
	for _, maxString := range maxCountStrings {
		if max, e := strconv.Atoi(maxString); e == nil {
			maxCounts = append(maxCounts, max)
		} else {
			glog.Fatalf("The max specified in -max not a valid number %s", maxString)
		}
	}
	if len(folders) != len(maxCounts) {
		glog.Fatalf("%d directories by -dir, but only %d max is set by -max", len(folders), len(maxCounts))
	}
	for _, folder := range folders {
		if err := util.TestFolderWritable(folder); err != nil {
			glog.Fatalf("Check Data Folder(-dir) Writable %s : %s", folder, err)
		}
	}

	if *publicIp == "" {
		if *ip == "" {
			*publicIp = "localhost"
		} else {
			*publicIp = *ip
		}
	}
	if *volumeWhiteListOption != "" {
		volumeWhiteList = strings.Split(*volumeWhiteListOption, ",")
	}

	r := http.NewServeMux()

	volumeServer := weed_server.NewVolumeServer(r, *ip, *vport, *publicIp, folders, maxCounts,
		*masterNode, *vpulse, *dataCenter, *rack, volumeWhiteList,
	)

	listeningAddress := *ip + ":" + strconv.Itoa(*vport)

	glog.V(0).Infoln("Start Weed volume server", util.VERSION, "at", listeningAddress)

	listener, e := util.NewListener(listeningAddress, time.Duration(*vTimeout)*time.Second)
	if e != nil {
		glog.Fatalf(e.Error())
	}

	// deal with control+c
	signalChan := make(chan os.Signal, 1)
	signal.Notify(signalChan, os.Interrupt)
	go func() {
		for _ = range signalChan {
			volumeServer.Shutdown()
			os.Exit(0)
		}
	}()

	if e := http.Serve(listener, r); e != nil {
		glog.Fatalf("Fail to serve:%s", e.Error())
	}
	return true
}
<commit_msg>Add control+c or INT handling<commit_after>package main

import (
	"code.google.com/p/weed-fs/go/glog"
	"code.google.com/p/weed-fs/go/util"
	"code.google.com/p/weed-fs/go/weed/weed_server"
	"net/http"
	"os"
	"os/signal"
	"runtime"
	"strconv"
	"strings"
	"time"
)

func init() {
	cmdVolume.Run = runVolume // break init cycle
}

var cmdVolume = &Command{
	UsageLine: "volume -port=8080 -dir=/tmp -max=5 -ip=server_name -mserver=localhost:9333",
	Short:     "start a volume server",
	Long: `start a volume server to provide storage spaces

  `,
}

var (
	vport                 = cmdVolume.Flag.Int("port", 8080, "http listen port")
	volumeFolders         = cmdVolume.Flag.String("dir", os.TempDir(), "directories to store data files. dir[,dir]...")
	maxVolumeCounts       = cmdVolume.Flag.String("max", "7", "maximum numbers of volumes, count[,count]...")
	ip                    = cmdVolume.Flag.String("ip", "", "ip or server name")
	publicIp              = cmdVolume.Flag.String("publicIp", "", "Publicly accessible <ip|server_name>")
	masterNode            = cmdVolume.Flag.String("mserver", "localhost:9333", "master server location")
	vpulse                = cmdVolume.Flag.Int("pulseSeconds", 5, "number of seconds between heartbeats, must be smaller than or equal to the master's setting")
	vTimeout              = cmdVolume.Flag.Int("idleTimeout", 10, "connection idle seconds")
	vMaxCpu               = cmdVolume.Flag.Int("maxCpu", 0, "maximum number of CPUs. 0 means all available CPUs")
	dataCenter            = cmdVolume.Flag.String("dataCenter", "", "current volume server's data center name")
	rack                  = cmdVolume.Flag.String("rack", "", "current volume server's rack name")
	volumeWhiteListOption = cmdVolume.Flag.String("whiteList", "", "comma separated Ip addresses having write permission. No limit if empty.")

	volumeWhiteList []string
)

func runVolume(cmd *Command, args []string) bool {
	if *vMaxCpu < 1 {
		*vMaxCpu = runtime.NumCPU()
	}
	runtime.GOMAXPROCS(*vMaxCpu)
	folders := strings.Split(*volumeFolders, ",")
	maxCountStrings := strings.Split(*maxVolumeCounts, ",")
	maxCounts := make([]int, 0)
	for _, maxString := range maxCountStrings {
		if max, e := strconv.Atoi(maxString); e == nil {
			maxCounts = append(maxCounts, max)
		} else {
			glog.Fatalf("The max specified in -max not a valid number %s", maxString)
		}
	}
	if len(folders) != len(maxCounts) {
		glog.Fatalf("%d directories by -dir, but only %d max is set by -max", len(folders), len(maxCounts))
	}
	for _, folder := range folders {
		if err := util.TestFolderWritable(folder); err != nil {
			glog.Fatalf("Check Data Folder(-dir) Writable %s : %s", folder, err)
		}
	}

	if *publicIp == "" {
		if *ip == "" {
			*publicIp = "localhost"
		} else {
			*publicIp = *ip
		}
	}
	if *volumeWhiteListOption != "" {
		volumeWhiteList = strings.Split(*volumeWhiteListOption, ",")
	}

	r := http.NewServeMux()

	volumeServer := weed_server.NewVolumeServer(r, *ip, *vport, *publicIp, folders, maxCounts,
		*masterNode, *vpulse, *dataCenter, *rack, volumeWhiteList,
	)

	listeningAddress := *ip + ":" + strconv.Itoa(*vport)

	glog.V(0).Infoln("Start Weed volume server", util.VERSION, "at", listeningAddress)

	listener, e := util.NewListener(listeningAddress, time.Duration(*vTimeout)*time.Second)
	if e != nil {
		glog.Fatalf(e.Error())
	}

	// deal with control+c
	signalChan := make(chan os.Signal, 1)
	signal.Notify(signalChan, os.Interrupt)
	go func() {
		for _ = range signalChan {
			volumeServer.Shutdown()
			os.Exit(0)
		}
	}()

	if e := http.Serve(listener, r); e != nil {
		glog.Fatalf("Fail to serve:%s", e.Error())
	}
	return true
}
<|endoftext|>


<commit_before>/*
 * Copyright (c) 2013 Matt Jibson <matt.jibson@gmail.com>
 *
 * Permission to use, copy, modify, and distribute this software for any
 * purpose with or without fee is hereby granted, provided that the above
 * copyright notice and this permission notice appear in all copies.
 *
 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 */

package goapp

import (
	"bytes"
	"code.google.com/p/go.net/html"
	"io"
	"net/url"
	"regexp"
	"strings"
)

func sanitizeLink(u *url.URL, v string) string {
	p, err := u.Parse(v)
	if err != nil {
		return ""
	}
	if !acceptableUriSchemes[p.Scheme] {
		return ""
	}

	return p.String()
}

func sanitizeStyle(v string) string {
	return v
}

func sanitizeAttributes(u *url.URL, t *html.Token) {
	var attrs []html.Attribute
	var isLink = false
	for _, a := range t.Attr {
		if a.Key == "target" {
		} else if a.Key == "style" {
			a.Val = sanitizeStyle(a.Val)
			attrs = append(attrs, a)
		} else if acceptableAttributes[a.Key] {
			if a.Key == "href" || a.Key == "src" {
				a.Val = sanitizeLink(u, a.Val)
			}
			if a.Key == "href" {
				isLink = true
			}
			attrs = append(attrs, a)
		}
	}
	if isLink {
		attrs = append(attrs, html.Attribute{
			Key: "target",
			Val: "_blank",
		})
	}
	t.Attr = attrs
}

func Sanitize(s string, u *url.URL) (string, string) {
	r := bytes.NewReader([]byte(s))
	r := bytes.NewReader([]byte(strings.TrimSpace(s)))
	z := html.NewTokenizer(r)
	buf := &bytes.Buffer{}
	snip := &bytes.Buffer{}
	skip := 0
	u.RawQuery = ""
	u.Fragment = ""
	for {
		if z.Next() == html.ErrorToken {
			if err := z.Err(); err == io.EOF {
				break
			} else {
				return s, snipper(s)
			}
		}

		t := z.Token()
		if t.Type == html.StartTagToken || t.Type == html.SelfClosingTagToken {
			if !acceptableElements[t.Data] {
				if unacceptableElementsWithEndTag[t.Data] && t.Type != html.SelfClosingTagToken {
					skip += 1
				}
			} else {
				sanitizeAttributes(u, &t)
				buf.WriteString(t.String())
			}
		} else if t.Type == html.EndTagToken {
			if !acceptableElements[t.Data] {
				if unacceptableElementsWithEndTag[t.Data] {
					skip -= 1
				}
			} else {
				buf.WriteString(t.String())
			}
		} else if skip == 0 {
			buf.WriteString(t.String())
			if t.Type == html.TextToken {
				snip.WriteString(t.String())
			}
		}
	}

	return buf.String(), snipper(snip.String())
}

const snipLen = 100

var snipRe = regexp.MustCompile("[\\s]+")

func snipper(s string) string {
	s = snipRe.ReplaceAllString(strings.TrimSpace(s), " ")
	s = html.UnescapeString(s)
	if len(s) <= snipLen {
		return s
	}
	s = s[:snipLen]
	i := strings.LastIndexAny(s, " .-!?")
	if i != -1 {
		return s[:i]
	}
	return cleanNonUTF8(s)
}

// Based on list from MDN's HTML5 element list
// https://developer.mozilla.org/en-US/docs/Web/Guide/HTML/HTML5/HTML5_element_list
var acceptableElements = map[string]bool{
	// Root element
	// "html": true,

	// Document metadata
	// "head":  true,
	// "title": true,
	// "base":  true,
	// "link":  true,
	// "meta":  true,
	// "style": true,

	// Scripting
	"noscript": true,
	// "script":   true,

	// Sections
	// "body":    true,
	"section": true,
	"nav":     true,
	"article": true,
	"aside":   true,
	"h1":      true,
	"h2":      true,
	"h3":      true,
	"h4":      true,
	"h5":      true,
	"h6":      true,
	"header":  true,
	"footer":  true,
	"address": true,
	"main":    true,

	// Grouping content
	"p":          true,
	"hr":         true,
	"pre":        true,
	"blockquote": true,
	"ol":         true,
	"ul":         true,
	"li":         true,
	"dl":         true,
	"dt":         true,
	"dd":         true,
	"figure":     true,
	"figcaption": true,
	"div":        true,

	// Text-level semantics
	"a":      true,
	"em":     true,
	"strong": true,
	"small":  true,
	"s":      true,
	"cite":   true,
	"q":      true,
	"dfn":    true,
	"abbr":   true,
	"data":   true,
	"time":   true,
	"code":   true,
	"var":    true,
	"samp":   true,
	"kbd":    true,
	"sub":    true,
	"sup":    true,
	"i":      true,
	"b":      true,
	"u":      true,
	"mark":   true,
	"ruby":   true,
	"rt":     true,
	"rp":     true,
	"bdi":    true,
	"bdo":    true,
	"span":   true,
	"br":     true,
	"wbr":    true,

	// Edits
	"ins": true,
	"del": true,

	// Embedded content
	"img": true,
	//"iframe":   true,
	//"embed":    true,
	"object": true,
	"param":  true,
	"video":  true,
	"audio":  true,
	"source": true,
	"track":  true,
	"canvas": true,
	"map":    true,
	"area":   true,
	"svg":    true,
	"math":   true,

	// Tabular data
	"table":    true,
	"caption":  true,
	"colgroup": true,
	"col":      true,
	"tbody":    true,
	"thead":    true,
	"tfoot":    true,
	"tr":       true,
	"td":       true,
	"th":       true,

	// Forms
	// "form":     true,
	// "fieldset": true,
	// "legend":   true,
	// "label":    true,
	// "input":    true,
	// "button":   true,
	// "select":   true,
	// "datalist": true,
	// "optgroup": true,
	// "option":   true,
	// "textarea": true,
	// "keygen":   true,
	// "output":   true,
	// "progress": true,
	// "meter":    true,

	// Interactive elements
	// "details":  true,
	// "summary":  true,
	// "menuitem": true,
	// "menu":     true,
}

var unacceptableElementsWithEndTag = map[string]bool{
	"script": true,
	"applet": true,
	"style":  true,
}

// Based on list from MDN's HTML attribute reference
// https://developer.mozilla.org/en-US/docs/Web/HTML/Attributes
var acceptableAttributes = map[string]bool{
	"accept":         true,
	"accept-charset": true,
	// "accesskey":       true,
	"action":       true,
	"align":        true,
	"alt":          true,
	"async":        true,
	"autocomplete": true,
	// "autofocus":       true,
	// "autoplay":        true,
	// "bgcolor":         true,
	// "border":          true,
	"buffered":        true,
	"challenge":       true,
	"charset":         true,
	"checked":         true,
	"cite":            true,
	// "class":           true,
	"code":            true,
	"codebase":        true,
	"color":           true,
	"cols":            true,
	"colspan":         true,
	"content":         true,
	"contenteditable": true,
	"contextmenu":     true,
	"controls":        true,
	"coords":          true,
	"data":            true,
	"data-custom":     true,
	"datetime":        true,
	"default":         true,
	"defer":           true,
	"dir":             true,
	"dirname":         true,
	"disabled":        true,
	"download":        true,
	"draggable":       true,
	"dropzone":        true,
	"enctype":         true,
	// "for":             true,
	"form":       true,
	"headers":    true,
	"height":     true,
	"hidden":     true,
	"high":       true,
	"href":       true,
	"hreflang":   true,
	"http-equiv": true,
	"icon":       true,
	// "id":              true,
	"ismap":       true,
	"itemprop":    true,
	"keytype":     true,
	"kind":        true,
	"label":       true,
	"lang":        true,
	"language":    true,
	"list":        true,
	"loop":        true,
	"low":         true,
	"manifest":    true,
	"max":         true,
	"maxlength":   true,
	"media":       true,
	"method":      true,
	"min":         true,
	"multiple":    true,
	"name":        true,
	"novalidate":  true,
	"open":        true,
	"optimum":     true,
	"pattern":     true,
	"ping":        true,
	"placeholder": true,
	"poster":      true,
	// "preload":         true,
	"pubdate":    true,
	"radiogroup": true,
	"readonly":   true,
	"rel":        true,
	"required":   true,
	"reversed":   true,
	"rows":       true,
	"rowspan":    true,
	"sandbox":    true,
	"spellcheck": true,
	"scope":      true,
	// "scoped":          true,
	// "seamless":        true,
	"selected": true,
	"shape":    true,
	"size":     true,
	"sizes":    true,
	"span":     true,
	"src":      true,
	// "srcdoc":          true,
	"srclang": true,
	"start":   true,
	// "step":            true,
	// "style":           true,
	"summary": true,
	// "tabindex":        true,
	// "target":          true,
	"title": true,
	"type":  true,
	// "usemap":          true,
	"value": true,
	"width": true,
	// "wrap":            true,
}

// Based on list from Wikipedia's URI scheme
// http://en.wikipedia.org/wiki/URI_scheme
var acceptableUriSchemes = map[string]bool{
	"aim":      true,
	"apt":      true,
	"bitcoin":  true,
	"callto":   true,
	"cvs":      true,
	"facetime": true,
	"feed":     true,
	"ftp":      true,
	"git":      true,
	"gopher":   true,
	"gtalk":    true,
	"http":     true,
	"https":    true,
	"imap":     true,
	"irc":      true,
	"itms":     true,
	"jabber":   true,
	"magnet":   true,
	"mailto":   true,
	"mms":      true,
	"msnim":    true,
	"news":     true,
	"nntp":     true,
	"rtmp":     true,
	"rtsp":     true,
	"sftp":     true,
	"skype":    true,
	"svn":      true,
	"ymsgr":    true,
}
<commit_msg>Allow some more tags<commit_after>/*
 * Copyright (c) 2013 Matt Jibson <matt.jibson@gmail.com>
 *
 * Permission to use, copy, modify, and distribute this software for any
 * purpose with or without fee is hereby granted, provided that the above
 * copyright notice and this permission notice appear in all copies.
 *
 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 */

package goapp

import (
	"bytes"
	"code.google.com/p/go.net/html"
	"io"
	"net/url"
	"regexp"
	"strings"
)

func sanitizeLink(u *url.URL, v string) string {
	p, err := u.Parse(v)
	if err != nil {
		return ""
	}
	if !acceptableUriSchemes[p.Scheme] {
		return ""
	}

	return p.String()
}

func sanitizeStyle(v string) string {
	return v
}

func sanitizeAttributes(u *url.URL, t *html.Token) {
	var attrs []html.Attribute
	var isLink = false
	for _, a := range t.Attr {
		if a.Key == "target" {
		} else if a.Key == "style" {
			a.Val = sanitizeStyle(a.Val)
			attrs = append(attrs, a)
		} else if acceptableAttributes[a.Key] {
			if a.Key == "href" || a.Key == "src" {
				a.Val = sanitizeLink(u, a.Val)
			}
			if a.Key == "href" {
				isLink = true
			}
			attrs = append(attrs, a)
		}
	}
	if isLink {
		attrs = append(attrs, html.Attribute{
			Key: "target",
			Val: "_blank",
		})
	}
	t.Attr = attrs
}

func Sanitize(s string, u *url.URL) (string, string) {
	r := bytes.NewReader([]byte(s))
	r := bytes.NewReader([]byte(strings.TrimSpace(s)))
	z := html.NewTokenizer(r)
	buf := &bytes.Buffer{}
	snip := &bytes.Buffer{}
	skip := 0
	u.RawQuery = ""
	u.Fragment = ""
	for {
		if z.Next() == html.ErrorToken {
			if err := z.Err(); err == io.EOF {
				break
			} else {
				return s, snipper(s)
			}
		}

		t := z.Token()
		if t.Type == html.StartTagToken || t.Type == html.SelfClosingTagToken {
			if !acceptableElements[t.Data] {
				if unacceptableElementsWithEndTag[t.Data] && t.Type != html.SelfClosingTagToken {
					skip += 1
				}
			} else {
				sanitizeAttributes(u, &t)
				buf.WriteString(t.String())
			}
		} else if t.Type == html.EndTagToken {
			if !acceptableElements[t.Data] {
				if unacceptableElementsWithEndTag[t.Data] {
					skip -= 1
				}
			} else {
				buf.WriteString(t.String())
			}
		} else if skip == 0 {
			buf.WriteString(t.String())
			if t.Type == html.TextToken {
				snip.WriteString(t.String())
			}
		}
	}

	return buf.String(), snipper(snip.String())
}

const snipLen = 100

var snipRe = regexp.MustCompile("[\\s]+")

func snipper(s string) string {
	s = snipRe.ReplaceAllString(strings.TrimSpace(s), " ")
	s = html.UnescapeString(s)
	if len(s) <= snipLen {
		return s
	}
	s = s[:snipLen]
	i := strings.LastIndexAny(s, " .-!?")
	if i != -1 {
		return s[:i]
	}
	return cleanNonUTF8(s)
}

// Based on list from MDN's HTML5 element list
// https://developer.mozilla.org/en-US/docs/Web/Guide/HTML/HTML5/HTML5_element_list
var acceptableElements = map[string]bool{
	// Root element
	// "html": true,

	// Document metadata
	// "head":  true,
	// "title": true,
	// "base":  true,
	// "link":  true,
	// "meta":  true,
	// "style": true,

	// Scripting
	"noscript": true,
	// "script":   true,

	// Sections
	// "body":    true,
	"section": true,
	"nav":     true,
	"article": true,
	"aside":   true,
	"h1":      true,
	"h2":      true,
	"h3":      true,
	"h4":      true,
	"h5":      true,
	"h6":      true,
	"header":  true,
	"footer":  true,
	"address": true,
	"main":    true,

	// Grouping content
	"p":          true,
	"hr":         true,
	"pre":        true,
	"blockquote": true,
	"ol":         true,
	"ul":         true,
	"li":         true,
	"dl":         true,
	"dt":         true,
	"dd":         true,
	"figure":     true,
	"figcaption": true,
	"div":        true,

	// Text-level semantics
	"a":      true,
	"em":     true,
	"strong": true,
	"small":  true,
	"s":      true,
	"cite":   true,
	"q":      true,
	"dfn":    true,
	"abbr":   true,
	"data":   true,
	"time":   true,
	"code":   true,
	"var":    true,
	"samp":   true,
	"kbd":    true,
	"sub":    true,
	"sup":    true,
	"i":      true,
	"b":      true,
	"u":      true,
	"mark":   true,
	"ruby":   true,
	"rt":     true,
	"rp":     true,
	"bdi":    true,
	"bdo":    true,
	"span":   true,
	"br":     true,
	"wbr":    true,

	// Edits
	"ins": true,
	"del": true,

	// Embedded content
	"img":    true,
	"iframe": true,
	"embed":  true,
	"object": true,
	"param":  true,
	"video":  true,
	"audio":  true,
	"source": true,
	"track":  true,
	"canvas": true,
	"map":    true,
	"area":   true,
	"svg":    true,
	"math":   true,

	// Tabular data
	"table":    true,
	"caption":  true,
	"colgroup": true,
	"col":      true,
	"tbody":    true,
	"thead":    true,
	"tfoot":    true,
	"tr":       true,
	"td":       true,
	"th":       true,

	// Forms
	"form":     true,
	"fieldset": true,
	"legend":   true,
	"label":    true,
	"input":    true,
	"button":   true,
	"select":   true,
	"datalist": true,
	"optgroup": true,
	"option":   true,
	"textarea": true,
	"keygen":   true,
	"output":   true,
	"progress": true,
	"meter":    true,

	// Interactive elements
	// "details":  true,
	// "summary":  true,
	// "menuitem": true,
	// "menu":     true,
}

var unacceptableElementsWithEndTag = map[string]bool{
	"script": true,
	"applet": true,
	"style":  true,
}

// Based on list from MDN's HTML attribute reference
// https://developer.mozilla.org/en-US/docs/Web/HTML/Attributes
var acceptableAttributes = map[string]bool{
	"accept":         true,
	"accept-charset": true,
	// "accesskey":       true,
	"action":       true,
	"align":        true,
	"alt":          true,
	"async":        true,
	"autocomplete": true,
	// "autofocus":       true,
	// "autoplay":        true,
	"bgcolor":         true,
	"border":          true,
	"buffered":        true,
	"challenge":       true,
	"charset":         true,
	"checked":         true,
	"cite":            true,
	"class":           true,
	"code":            true,
	"codebase":        true,
	"color":           true,
	"cols":            true,
	"colspan":         true,
	"content":         true,
	"contenteditable": true,
	"contextmenu":     true,
	"controls":        true,
	"coords":          true,
	"data":            true,
	"data-custom":     true,
	"datetime":        true,
	"default":         true,
	"defer":           true,
	"dir":             true,
	"dirname":         true,
	"disabled":        true,
	"download":        true,
	"draggable":       true,
	"dropzone":        true,
	"enctype":         true,
	"for":             true,
	"form":            true,
	"headers":         true,
	"height":          true,
	"hidden":          true,
	"high":            true,
	"href":            true,
	"hreflang":        true,
	"http-equiv":      true,
	"icon":            true,
	"id":              true,
	"ismap":           true,
	"itemprop":        true,
	"keytype":         true,
	"kind":            true,
	"label":           true,
	"lang":            true,
	"language":        true,
	"list":            true,
	"loop":            true,
	"low":             true,
	"manifest":        true,
	"max":             true,
	"maxlength":       true,
	"media":           true,
	"method":          true,
	"min":             true,
	"multiple":        true,
	"name":            true,
	"novalidate":      true,
	"open":            true,
	"optimum":         true,
	"pattern":         true,
	"ping":            true,
	"placeholder":     true,
	"poster":          true,
	// "preload":         true,
	"pubdate":    true,
	"radiogroup": true,
	"readonly":   true,
	"rel":        true,
	"required":   true,
	"reversed":   true,
	"rows":       true,
	"rowspan":    true,
	"sandbox":    true,
	"spellcheck": true,
	"scope":      true,
	// "scoped":          true,
	// "seamless":        true,
	"selected": true,
	"shape":    true,
	"size":     true,
	"sizes":    true,
	"span":     true,
	"src":      true,
	// "srcdoc":          true,
	"srclang": true,
	"start":   true,
	// "step":            true,
	"style":   true,
	"summary": true,
	// "tabindex":        true,
	// "target":          true,
	"title": true,
	"type":  true,
	// "usemap":          true,
	"value": true,
	"width": true,
	// "wrap":            true,
}

// Based on list from Wikipedia's URI scheme
// http://en.wikipedia.org/wiki/URI_scheme
var acceptableUriSchemes = map[string]bool{
	"aim":      true,
	"apt":      true,
	"bitcoin":  true,
	"callto":   true,
	"cvs":      true,
	"facetime": true,
	"feed":     true,
	"ftp":      true,
	"git":      true,
	"gopher":   true,
	"gtalk":    true,
	"http":     true,
	"https":    true,
	"imap":     true,
	"irc":      true,
	"itms":     true,
	"jabber":   true,
	"magnet":   true,
	"mailto":   true,
	"mms":      true,
	"msnim":    true,
	"news":     true,
	"nntp":     true,
	"rtmp":     true,
	"rtsp":     true,
	"sftp":     true,
	"skype":    true,
	"svn":      true,
	"ymsgr":    true,
}
<|endoftext|>


<commit_before>package emailserver

import (
	"bytes"
	"net"
	"net/mail"
	"testing"
	"gpg-validation-server/email-client"
	"gpg-validation-server/email-server"
	"time"
	"fmt"
)

var received string

func init() {
	server := emailserver.Create("127.0.0.1:2525", mailHandler)
	go server.Run()
	time.Sleep(1 * time.Millisecond)
}

func mailHandler(origin net.Addr, from string, to []string, data []byte) {
	mail.ReadMessage(bytes.NewReader(data))
	received = fmt.Sprintf("%s -> %s", from, to[0])
}

func TestReceiveMail(t *testing.T) {
	received = ""
	expected := "ray@tomlinson.net -> ray.tomlinson@mail.org"
	emailclient.SendMail("ray@tomlinson.net", "ray.tomlinson@mail.org", "QWERTYIOP")
	if received != expected {
		t.Error("Expected:", expected, " Received:", received)
	}
}

<commit_msg>Fix imports in email-server<commit_after>package emailserver

import (
	"bytes"
	"net"
	"net/mail"
	"testing"
	"github.com/TNG/gpg-validation-server/email-client"
	"time"
	"fmt"
)

var received string

func init() {
	server := Create("127.0.0.1:2525", mailHandler)
	go server.Run()
	time.Sleep(1 * time.Millisecond)
}

func mailHandler(origin net.Addr, from string, to []string, data []byte) {
	mail.ReadMessage(bytes.NewReader(data))
	received = fmt.Sprintf("%s -> %s", from, to[0])
}

func TestReceiveMail(t *testing.T) {
	received = ""
	expected := "ray@tomlinson.net -> ray.tomlinson@mail.org"
	emailclient.SendMail("ray@tomlinson.net", "ray.tomlinson@mail.org", "QWERTYIOP")
	if received != expected {
		t.Error("Expected:", expected, " Received:", received)
	}
}

<|endoftext|>