<commit_before>package actor

import (
	"errors"

	"github.com/AsynkronIT/protoactor-go/mailbox"
)

// Props types
type SpawnFunc func(actorSystem *ActorSystem, id string, props *Props, parentContext SpawnerContext) (*PID, error)
type ReceiverMiddleware func(next ReceiverFunc) ReceiverFunc
type SenderMiddleware func(next SenderFunc) SenderFunc
type ContextDecorator func(next ContextDecoratorFunc) ContextDecoratorFunc
type SpawnMiddleware func(next SpawnFunc) SpawnFunc

// Default values
var (
	defaultDispatcher      = mailbox.NewDefaultDispatcher(300)
	defaultMailboxProducer = mailbox.Unbounded()
	defaultSpawner         = func(actorSystem *ActorSystem, id string, props *Props, parentContext SpawnerContext) (*PID, error) {
		ctx := newActorContext(actorSystem, props, parentContext.Self())
		mb := props.produceMailbox()
		dp := props.getDispatcher()
		proc := NewActorProcess(mb)
		pid, absent := actorSystem.ProcessRegistry.Add(proc, id)
		if !absent {
			return pid, ErrNameExists
		}
		ctx.self = pid
		mb.Start()
		mb.RegisterHandlers(ctx, dp)
		mb.PostSystemMessage(startedMessage)

		return pid, nil
	}
	defaultContextDecorator = func(ctx Context) Context {
		return ctx
	}
)

// DefaultSpawner this is a hacking way to allow Proto.Router access default spawner func
var DefaultSpawner SpawnFunc = defaultSpawner

// ErrNameExists is the error used when an existing name is used for spawning an actor.
var ErrNameExists = errors.New("spawn: name exists")

// Props represents configuration to define how an actor should be created
type Props struct {
	spawner                 SpawnFunc
	producer                Producer
	mailboxProducer         mailbox.Producer
	guardianStrategy        SupervisorStrategy
	supervisionStrategy     SupervisorStrategy
	dispatcher              mailbox.Dispatcher
	receiverMiddleware      []ReceiverMiddleware
	senderMiddleware        []SenderMiddleware
	spawnMiddleware         []SpawnMiddleware
	receiverMiddlewareChain ReceiverFunc
	senderMiddlewareChain   SenderFunc
	spawnMiddlewareChain    SpawnFunc
	contextDecorator        []ContextDecorator
	contextDecoratorChain   ContextDecoratorFunc
}

func (props *Props) getSpawner() SpawnFunc {
	if props.spawner == nil {
		return defaultSpawner
	}
	return props.spawner
}

func (props *Props) getDispatcher() mailbox.Dispatcher {
	if props.dispatcher == nil {
		return defaultDispatcher
	}
	return props.dispatcher
}

func (props *Props) getSupervisor() SupervisorStrategy {
	if props.supervisionStrategy == nil {
		return defaultSupervisionStrategy
	}
	return props.supervisionStrategy
}

func (props *Props) getContextDecoratorChain() ContextDecoratorFunc {
	if props.contextDecoratorChain == nil {
		return defaultContextDecorator
	}
	return props.contextDecoratorChain
}

func (props *Props) produceMailbox() mailbox.Mailbox {
	if props.mailboxProducer == nil {
		return defaultMailboxProducer()
	}
	return props.mailboxProducer()
}

func (props *Props) spawn(actorSystem *ActorSystem, name string, parentContext SpawnerContext) (*PID, error) {
	return props.getSpawner()(actorSystem, name, props, parentContext)
}

// WithProducer assigns a actor producer to the props
func (props *Props) WithProducer(p Producer) *Props {
	props.producer = p
	return props
}

// WithDispatcher assigns a dispatcher to the props
func (props *Props) WithDispatcher(dispatcher mailbox.Dispatcher) *Props {
	props.dispatcher = dispatcher
	return props
}

// WithMailbox assigns the desired mailbox producer to the props
func (props *Props) WithMailbox(mailbox mailbox.Producer) *Props {
	props.mailboxProducer = mailbox
	return props
}

// WithContextDecorator assigns context decorator to the props
func (props *Props) WithContextDecorator(contextDecorator ...ContextDecorator) *Props {
	props.contextDecorator = append(props.contextDecorator, contextDecorator...)

	props.contextDecoratorChain = makeContextDecoratorChain(props.contextDecorator, func(ctx Context) Context {
		return ctx
	})

	return props
}

// WithGuardian assigns a guardian strategy to the props
func (props *Props) WithGuardian(guardian SupervisorStrategy) *Props {
	props.guardianStrategy = guardian
	return props
}

// WithSupervisor assigns a supervision strategy to the props
func (props *Props) WithSupervisor(supervisor SupervisorStrategy) *Props {
	props.supervisionStrategy = supervisor
	return props
}

// Assign one or more middleware to the props
func (props *Props) WithReceiverMiddleware(middleware ...ReceiverMiddleware) *Props {
	props.receiverMiddleware = append(props.receiverMiddleware, middleware...)

	// Construct the receiver middleware chain with the final receiver at the end
	props.receiverMiddlewareChain = makeReceiverMiddlewareChain(props.receiverMiddleware, func(ctx ReceiverContext, envelope *MessageEnvelope) {
		ctx.Receive(envelope)
	})

	return props
}

func (props *Props) WithSenderMiddleware(middleware ...SenderMiddleware) *Props {
	props.senderMiddleware = append(props.senderMiddleware, middleware...)

	// Construct the sender middleware chain with the final sender at the end
	props.senderMiddlewareChain = makeSenderMiddlewareChain(props.senderMiddleware, func(sender SenderContext, target *PID, envelope *MessageEnvelope) {
		target.sendUserMessage(sender.ActorSystem(), envelope)
	})

	return props
}

// WithSpawnFunc assigns a custom spawn func to the props, this is mainly for internal usage
func (props *Props) WithSpawnFunc(spawn SpawnFunc) *Props {
	props.spawner = spawn
	return props
}

// WithFunc assigns a receive func to the props
func (props *Props) WithFunc(f ActorFunc) *Props {
	props.producer = func() Actor { return f }
	return props
}

func (props *Props) WithSpawnMiddleware(middleware ...SpawnMiddleware) *Props {
	props.spawnMiddleware = append(props.spawnMiddleware, middleware...)

	// Construct the spawner middleware chain with the final spawner at the end
	props.spawnMiddlewareChain = makeSpawnMiddlewareChain(props.spawnMiddleware, func(actorSystem *ActorSystem, id string, props *Props, parentContext SpawnerContext) (pid *PID, e error) {
		if props.spawner == nil {
			return defaultSpawner(actorSystem, id, props, parentContext)
		}
		return props.spawner(actorSystem, id, props, parentContext)
	})

	return props
}

// PropsFromProducer creates a props with the given actor producer assigned
func PropsFromProducer(producer Producer) *Props {
	return &Props{
		producer:         producer,
		contextDecorator: make([]ContextDecorator, 0),
	}
}

// PropsFromFunc creates a props with the given receive func assigned as the actor producer
func PropsFromFunc(f ActorFunc) *Props {
	return PropsFromProducer(func() Actor { return f })
}

// Deprecated: Use actor.PropsFromProducer instead.
func FromProducer(actorProducer Producer) *Props {
	return PropsFromProducer(actorProducer)
}

// Deprecated: Use actor.PropsFromFunc instead.
func FromFunc(f ActorFunc) *Props {
	return PropsFromFunc(f)
}

// Deprecated: FromSpawnFunc is deprecated.
func FromSpawnFunc(spawn SpawnFunc) *Props {
	return (&Props{}).WithSpawnFunc(spawn)
}
<commit_msg>remove obsolete<commit_after>package actor

import (
	"errors"

	"github.com/AsynkronIT/protoactor-go/mailbox"
)

// Props types
type SpawnFunc func(actorSystem *ActorSystem, id string, props *Props, parentContext SpawnerContext) (*PID, error)
type ReceiverMiddleware func(next ReceiverFunc) ReceiverFunc
type SenderMiddleware func(next SenderFunc) SenderFunc
type ContextDecorator func(next ContextDecoratorFunc) ContextDecoratorFunc
type SpawnMiddleware func(next SpawnFunc) SpawnFunc

// Default values
var (
	defaultDispatcher      = mailbox.NewDefaultDispatcher(300)
	defaultMailboxProducer = mailbox.Unbounded()
	defaultSpawner         = func(actorSystem *ActorSystem, id string, props *Props, parentContext SpawnerContext) (*PID, error) {
		ctx := newActorContext(actorSystem, props, parentContext.Self())
		mb := props.produceMailbox()
		dp := props.getDispatcher()
		proc := NewActorProcess(mb)
		pid, absent := actorSystem.ProcessRegistry.Add(proc, id)
		if !absent {
			return pid, ErrNameExists
		}
		ctx.self = pid
		mb.Start()
		mb.RegisterHandlers(ctx, dp)
		mb.PostSystemMessage(startedMessage)

		return pid, nil
	}
	defaultContextDecorator = func(ctx Context) Context {
		return ctx
	}
)

// DefaultSpawner this is a hacking way to allow Proto.Router access default spawner func
var DefaultSpawner SpawnFunc = defaultSpawner

// ErrNameExists is the error used when an existing name is used for spawning an actor.
var ErrNameExists = errors.New("spawn: name exists")

// Props represents configuration to define how an actor should be created
type Props struct {
	spawner                 SpawnFunc
	producer                Producer
	mailboxProducer         mailbox.Producer
	guardianStrategy        SupervisorStrategy
	supervisionStrategy     SupervisorStrategy
	dispatcher              mailbox.Dispatcher
	receiverMiddleware      []ReceiverMiddleware
	senderMiddleware        []SenderMiddleware
	spawnMiddleware         []SpawnMiddleware
	receiverMiddlewareChain ReceiverFunc
	senderMiddlewareChain   SenderFunc
	spawnMiddlewareChain    SpawnFunc
	contextDecorator        []ContextDecorator
	contextDecoratorChain   ContextDecoratorFunc
}

func (props *Props) getSpawner() SpawnFunc {
	if props.spawner == nil {
		return defaultSpawner
	}
	return props.spawner
}

func (props *Props) getDispatcher() mailbox.Dispatcher {
	if props.dispatcher == nil {
		return defaultDispatcher
	}
	return props.dispatcher
}

func (props *Props) getSupervisor() SupervisorStrategy {
	if props.supervisionStrategy == nil {
		return defaultSupervisionStrategy
	}
	return props.supervisionStrategy
}

func (props *Props) getContextDecoratorChain() ContextDecoratorFunc {
	if props.contextDecoratorChain == nil {
		return defaultContextDecorator
	}
	return props.contextDecoratorChain
}

func (props *Props) produceMailbox() mailbox.Mailbox {
	if props.mailboxProducer == nil {
		return defaultMailboxProducer()
	}
	return props.mailboxProducer()
}

func (props *Props) spawn(actorSystem *ActorSystem, name string, parentContext SpawnerContext) (*PID, error) {
	return props.getSpawner()(actorSystem, name, props, parentContext)
}

// WithProducer assigns a actor producer to the props
func (props *Props) WithProducer(p Producer) *Props {
	props.producer = p
	return props
}

// WithDispatcher assigns a dispatcher to the props
func (props *Props) WithDispatcher(dispatcher mailbox.Dispatcher) *Props {
	props.dispatcher = dispatcher
	return props
}

// WithMailbox assigns the desired mailbox producer to the props
func (props *Props) WithMailbox(mailbox mailbox.Producer) *Props {
	props.mailboxProducer = mailbox
	return props
}

// WithContextDecorator assigns context decorator to the props
func (props *Props) WithContextDecorator(contextDecorator ...ContextDecorator) *Props {
	props.contextDecorator = append(props.contextDecorator, contextDecorator...)

	props.contextDecoratorChain = makeContextDecoratorChain(props.contextDecorator, func(ctx Context) Context {
		return ctx
	})

	return props
}

// WithGuardian assigns a guardian strategy to the props
func (props *Props) WithGuardian(guardian SupervisorStrategy) *Props {
	props.guardianStrategy = guardian
	return props
}

// WithSupervisor assigns a supervision strategy to the props
func (props *Props) WithSupervisor(supervisor SupervisorStrategy) *Props {
	props.supervisionStrategy = supervisor
	return props
}

// Assign one or more middleware to the props
func (props *Props) WithReceiverMiddleware(middleware ...ReceiverMiddleware) *Props {
	props.receiverMiddleware = append(props.receiverMiddleware, middleware...)

	// Construct the receiver middleware chain with the final receiver at the end
	props.receiverMiddlewareChain = makeReceiverMiddlewareChain(props.receiverMiddleware, func(ctx ReceiverContext, envelope *MessageEnvelope) {
		ctx.Receive(envelope)
	})

	return props
}

func (props *Props) WithSenderMiddleware(middleware ...SenderMiddleware) *Props {
	props.senderMiddleware = append(props.senderMiddleware, middleware...)

	// Construct the sender middleware chain with the final sender at the end
	props.senderMiddlewareChain = makeSenderMiddlewareChain(props.senderMiddleware, func(sender SenderContext, target *PID, envelope *MessageEnvelope) {
		target.sendUserMessage(sender.ActorSystem(), envelope)
	})

	return props
}

// WithSpawnFunc assigns a custom spawn func to the props, this is mainly for internal usage
func (props *Props) WithSpawnFunc(spawn SpawnFunc) *Props {
	props.spawner = spawn
	return props
}

// WithFunc assigns a receive func to the props
func (props *Props) WithFunc(f ActorFunc) *Props {
	props.producer = func() Actor { return f }
	return props
}

func (props *Props) WithSpawnMiddleware(middleware ...SpawnMiddleware) *Props {
	props.spawnMiddleware = append(props.spawnMiddleware, middleware...)

	// Construct the spawner middleware chain with the final spawner at the end
	props.spawnMiddlewareChain = makeSpawnMiddlewareChain(props.spawnMiddleware, func(actorSystem *ActorSystem, id string, props *Props, parentContext SpawnerContext) (pid *PID, e error) {
		if props.spawner == nil {
			return defaultSpawner(actorSystem, id, props, parentContext)
		}
		return props.spawner(actorSystem, id, props, parentContext)
	})

	return props
}

// PropsFromProducer creates a props with the given actor producer assigned
func PropsFromProducer(producer Producer) *Props {
	return &Props{
		producer:         producer,
		contextDecorator: make([]ContextDecorator, 0),
	}
}

// PropsFromFunc creates a props with the given receive func assigned as the actor producer
func PropsFromFunc(f ActorFunc) *Props {
	return PropsFromProducer(func() Actor { return f })
}
<|endoftext|>


<commit_before>package automation

import (
	"context"
	"fmt"
	"google.golang.org/api/compute/v1"
	"google.golang.org/api/recommender/v1"
	"log"
)

func listZoneRecommendations(project string, location string, recommenderID string) []*recommender.GoogleCloudRecommenderV1Recommendation {
	ctx := context.Background()
	service, err := recommender.NewService(ctx)
	if err != nil {
		log.Fatal(err)
	}
	recService := recommender.NewProjectsLocationsRecommendersRecommendationsService(service)

	listCall := recService.List(fmt.Sprintf("projects/%s/locations/%s/recommenders/%s", project, location, recommenderID))
	var recommendations []*recommender.GoogleCloudRecommenderV1Recommendation
	addRecommendations := func(response *recommender.GoogleCloudRecommenderV1ListRecommendationsResponse) error {
		recommendations = append(recommendations, response.Recommendations...)
		return nil
	}

	ctx = context.Background()
	err = listCall.Pages(ctx, addRecommendations)
	if err != nil {
		log.Fatal(err)
	}
	return recommendations
}

func listZonesNames(project string) []string {
	ctx := context.Background()
	service, err := compute.NewService(ctx)
	if err != nil {
		log.Fatal(err)
	}
	zonesService := compute.NewZonesService(service)
	listCall := zonesService.List(project)

	var zones []string
	addZones := func(zoneList *compute.ZoneList) error {
		for _, zone := range zoneList.Items {
			zones = append(zones, zone.Name)
		}
		return nil
	}
	ctx = context.Background()
	err = listCall.Pages(ctx, addZones)
	if err != nil {
		log.Fatal(err)
	}
	return zones
}

// ListRecommendations returns the list of recommendations for a Cloud project.
// Requires the
// recommender.*.list
// IAM permission for the specified recommender.
func ListRecommendations(project string, recommenderID string) []*recommender.GoogleCloudRecommenderV1Recommendation {
	zones := listZonesNames(project)
	ch := make(chan []*recommender.GoogleCloudRecommenderV1Recommendation, len(zones))
	for _, zone := range zones {
		go func(zoneName string) {
			ch <- listZoneRecommendations(project, zoneName, recommenderID)
		}(zone)
	}
	var recommendations []*recommender.GoogleCloudRecommenderV1Recommendation
	for range zones {
		recommendations = append(recommendations, <-ch...)
	}
	return recommendations
}
<commit_msg>Added Service interface, optional number of concurrent calls to server<commit_after>package automation

import (
	"context"
	"fmt"
	"log"

	"google.golang.org/api/compute/v1"
	"google.golang.org/api/recommender/v1"
)

// Service is the inferface that prodives the following methods:
// listZoneRecommendations - listing recommendations for specified project, zone and reccommender,
// listZonesNames - listing every zone available for the project methods,
// getNumConcurrentCalls returns the recommended maximum number of concurrent calls to listZonesNames.
type Service interface {
	listZoneRecommendations(project string, location string, recommenderID string) []*recommender.GoogleCloudRecommenderV1Recommendation

	listZonesNames(project string) []string

	getNumConcurrentCalls() int
}

// ZonesRecommendationsService implements Service interface for Recommender and Compute APIs,
// using projects.locations.recommenders.recommendations/list and zones/list methods.
type ZonesRecommendationsService struct {
	ctx                   context.Context
	recommendationService *recommender.ProjectsLocationsRecommendersRecommendationsService
	zonesService          *compute.ZonesService
	numConcurrentCalls    int
}

// NewZonesRecommendationsService creates new ZonesRecommendationsServic.
// If creation failed the error will be non-nil.
// numConcurrentCalls should be positive, otherwise would return error.
func NewZonesRecommendationsService(ctx context.Context, numConcurrentCalls int) (*ZonesRecommendationsService, error) {
	compService, err := recommender.NewService(ctx)
	if err != nil {
		return nil, err
	}

	recommendationService := recommender.NewProjectsLocationsRecommendersRecommendationsService(compService)
	recService, err := compute.NewService(ctx)
	if err != nil {
		return nil, err
	}
	zonesService := compute.NewZonesService(recService)

	if numConcurrentCalls <= 0 {
		return nil, fmt.Errorf("Number of concurrent calls %d should be positive", numConcurrentCalls)
	}

	return &ZonesRecommendationsService{
		ctx:                   ctx,
		recommendationService: recommendationService,
		zonesService:          zonesService,
		numConcurrentCalls:    numConcurrentCalls,
	}, nil
}

func (s *ZonesRecommendationsService) listZoneRecommendations(project string, location string, recommenderID string) []*recommender.GoogleCloudRecommenderV1Recommendation {
	listCall := s.recommendationService.List(fmt.Sprintf("projects/%s/locations/%s/recommenders/%s", project, location, recommenderID))
	var recommendations []*recommender.GoogleCloudRecommenderV1Recommendation
	addRecommendations := func(response *recommender.GoogleCloudRecommenderV1ListRecommendationsResponse) error {
		recommendations = append(recommendations, response.Recommendations...)
		return nil
	}

	err := listCall.Pages(s.ctx, addRecommendations)
	if err != nil {
		log.Fatal(err)
	}
	return recommendations
}

func (s *ZonesRecommendationsService) listZonesNames(project string) []string {
	listCall := s.zonesService.List(project)

	var zones []string
	addZones := func(zoneList *compute.ZoneList) error {
		for _, zone := range zoneList.Items {
			zones = append(zones, zone.Name)
		}
		return nil
	}
	err := listCall.Pages(s.ctx, addZones)
	if err != nil {
		log.Fatal(err)
	}
	return zones
}

func (s *ZonesRecommendationsService) getNumConcurrentCalls() int {
	return s.numConcurrentCalls
}

// ListRecommendations returns the list of recommendations for a Cloud project.
// Requires the recommender.*.list IAM permission for the specified recommender.
func ListRecommendations(service Service, project string, recommenderID string) []*recommender.GoogleCloudRecommenderV1Recommendation {
	zones := service.listZonesNames(project)
	numberOfZones := len(zones)

	numWorkers := service.getNumConcurrentCalls()
	if numberOfZones < numWorkers {
		numWorkers = numberOfZones
	}
	results := make(chan []*recommender.GoogleCloudRecommenderV1Recommendation, numberOfZones)
	zonesJobs := make(chan string, numberOfZones)
	for i := 0; i < numWorkers; i++ {
		go func() {
			for zone := range zonesJobs {
				results <- service.listZoneRecommendations(project, zone, recommenderID)
			}
		}()
	}

	for _, zone := range zones {
		zonesJobs <- zone
	}
	close(zonesJobs)

	var recommendations []*recommender.GoogleCloudRecommenderV1Recommendation
	for range zones {
		recommendations = append(recommendations, <-results...)
	}
	return recommendations
}
<|endoftext|>


<commit_before>package cloudstorage

import (
	"net/http"
	"net/textproto"
	"net/url"
	"strings"
)

// AccessType is either public or private.
type AccessType string

const (
	// AccessTypePublic is public access.
	AccessTypePublic AccessType = "public"
	// AccessTypePrivate requires signature.
	AccessTypePrivate AccessType = "private"
)

// Storage is abstraction over various cloud storage providers.
type Storage interface {
	// PresignPutObject returns an HTTP request that is ready for use.
	PresignPutObject(name string, accessType AccessType, header http.Header) (*http.Request, error)
	// PresignGetObject returns an URL that is ready for use.
	PresignGetObject(name string) (*url.URL, error)
	// AccessType returns AccessType stored in the header.
	AccessType(header http.Header) AccessType
	// StandardToProprietary rewrites any standard headers to proprietary ones
	// so that they can be retrieved later.
	StandardToProprietary(header http.Header) http.Header
	// ProprietaryToStandard does the reverse of StandardToProprietary.
	ProprietaryToStandard(header http.Header) http.Header
}

func RewriteHeaderName(header http.Header, mapping map[string]string) http.Header {
	output := http.Header{}
	for oldName, newName := range mapping {
		newName = textproto.CanonicalMIMEHeaderKey(newName)
		for originalName, value := range header {
			lowerName := strings.ToLower(originalName)
			if oldName == lowerName {
				output[newName] = value
			} else {
				output[originalName] = value
			}
		}
	}
	return output
}
<commit_msg>Add AccessTypeDefault<commit_after>package cloudstorage

import (
	"net/http"
	"net/textproto"
	"net/url"
	"strings"
)

// AccessType is either public or private.
type AccessType string

const (
	// AccessTypePublic is public access.
	AccessTypePublic AccessType = "public"
	// AccessTypePrivate requires signature.
	AccessTypePrivate AccessType = "private"
)

// AccessTypeDefault is public.
const AccessTypeDefault = AccessTypePublic

// Storage is abstraction over various cloud storage providers.
type Storage interface {
	// PresignPutObject returns an HTTP request that is ready for use.
	PresignPutObject(name string, accessType AccessType, header http.Header) (*http.Request, error)
	// PresignGetObject returns an URL that is ready for use.
	PresignGetObject(name string) (*url.URL, error)
	// AccessType returns AccessType stored in the header.
	AccessType(header http.Header) AccessType
	// StandardToProprietary rewrites any standard headers to proprietary ones
	// so that they can be retrieved later.
	StandardToProprietary(header http.Header) http.Header
	// ProprietaryToStandard does the reverse of StandardToProprietary.
	ProprietaryToStandard(header http.Header) http.Header
}

func RewriteHeaderName(header http.Header, mapping map[string]string) http.Header {
	output := http.Header{}
	for oldName, newName := range mapping {
		newName = textproto.CanonicalMIMEHeaderKey(newName)
		for originalName, value := range header {
			lowerName := strings.ToLower(originalName)
			if oldName == lowerName {
				output[newName] = value
			} else {
				output[originalName] = value
			}
		}
	}
	return output
}
<|endoftext|>


<commit_before>/*
Copyright 2014 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cmd

import (
	"bytes"
	"testing"

	cmdtesting "k8s.io/kubernetes/pkg/kubectl/cmd/testing"
)

func TestValidateArgs(t *testing.T) {
	f, _, _, _ := cmdtesting.NewAPIFactory()

	tests := []struct {
		flags     map[string]string
		filenames []string
		args      []string
		expectErr bool
		testName  string
	}{
		{
			expectErr: true,
			testName:  "nothing",
		},
		{
			flags:     map[string]string{},
			args:      []string{"foo"},
			expectErr: true,
			testName:  "no file, no image",
		},
		{
			filenames: []string{"bar.yaml"},
			args:      []string{"foo"},
			testName:  "valid file example",
		},
		{
			flags: map[string]string{
				"image": "foo:v2",
			},
			args:     []string{"foo"},
			testName: "missing second image name",
		},
		{
			flags: map[string]string{
				"image": "foo:v2",
			},
			args:     []string{"foo", "foo-v2"},
			testName: "valid image example",
		},
		{
			flags: map[string]string{
				"image": "foo:v2",
			},
			filenames: []string{"bar.yaml"},
			args:      []string{"foo", "foo-v2"},
			expectErr: true,
			testName:  "both filename and image example",
		},
	}
	for _, test := range tests {
		out := &bytes.Buffer{}
		cmd := NewCmdRollingUpdate(f, out)

		if test.flags != nil {
			for key, val := range test.flags {
				cmd.Flags().Set(key, val)
			}
		}
		err := validateArguments(cmd, test.filenames, test.args)
		if err != nil && !test.expectErr {
			t.Errorf("unexpected error: %v (%s)", err, test.testName)
		}
		if err == nil && test.expectErr {
			t.Errorf("unexpected non-error (%s)", test.testName)
		}
	}
}
<commit_msg>small change to clear<commit_after>/*
Copyright 2014 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cmd

import (
	"bytes"
	"testing"

	cmdtesting "k8s.io/kubernetes/pkg/kubectl/cmd/testing"
)

func TestValidateArgs(t *testing.T) {
	f, _, _, _ := cmdtesting.NewAPIFactory()

	tests := []struct {
		testName  string
		flags     map[string]string
		filenames []string
		args      []string
		expectErr bool
	}{
		{
			testName:  "nothing",
			expectErr: true,
		},
		{
			testName:  "no file, no image",
			flags:     map[string]string{},
			args:      []string{"foo"},
			expectErr: true,
		},
		{
			testName:  "valid file example",
			filenames: []string{"bar.yaml"},
			args:      []string{"foo"},
		},
		{
			testName: "missing second image name",
			flags: map[string]string{
				"image": "foo:v2",
			},
			args: []string{"foo"},
		},
		{
			testName: "valid image example",
			flags: map[string]string{
				"image": "foo:v2",
			},
			args: []string{"foo", "foo-v2"},
		},
		{
			testName: "both filename and image example",
			flags: map[string]string{
				"image": "foo:v2",
			},
			filenames: []string{"bar.yaml"},
			args:      []string{"foo", "foo-v2"},
			expectErr: true,
		},
	}
	for _, test := range tests {
		out := &bytes.Buffer{}
		cmd := NewCmdRollingUpdate(f, out)

		if test.flags != nil {
			for key, val := range test.flags {
				cmd.Flags().Set(key, val)
			}
		}
		err := validateArguments(cmd, test.filenames, test.args)
		if err != nil && !test.expectErr {
			t.Errorf("%s: unexpected error: %v", test.testName, err)
		}
		if err == nil && test.expectErr {
			t.Errorf("%s: unexpected non-error", test.testName)
		}
	}
}
<|endoftext|>


<commit_before>// +build linux

/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package config

import (
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"sync"
	"testing"
	"time"

	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/api/testapi"
	"k8s.io/kubernetes/pkg/api/unversioned"
	"k8s.io/kubernetes/pkg/api/validation"
	kubetypes "k8s.io/kubernetes/pkg/kubelet/types"
	"k8s.io/kubernetes/pkg/runtime"
	"k8s.io/kubernetes/pkg/securitycontext"
	"k8s.io/kubernetes/pkg/types"
	utiltesting "k8s.io/kubernetes/pkg/util/testing"
	"k8s.io/kubernetes/pkg/util/wait"
)

func TestExtractFromNonExistentFile(t *testing.T) {
	ch := make(chan interface{}, 1)
	c := new("/some/fake/file", "localhost", time.Millisecond, ch)
	err := c.watch()
	if err == nil {
		t.Errorf("Expected error")
	}
}

func TestUpdateOnNonExistentFile(t *testing.T) {
	ch := make(chan interface{})
	NewSourceFile("random_non_existent_path", "localhost", time.Millisecond, ch)
	select {
	case got := <-ch:
		update := got.(kubetypes.PodUpdate)
		expected := CreatePodUpdate(kubetypes.SET, kubetypes.FileSource)
		if !api.Semantic.DeepDerivative(expected, update) {
			t.Fatalf("expected %#v, Got %#v", expected, update)
		}

	case <-time.After(wait.ForeverTestTimeout):
		t.Fatalf("expected update, timeout instead")
	}
}

func TestReadPodsFromFileExistAlready(t *testing.T) {
	hostname := types.NodeName("random-test-hostname")
	var testCases = getTestCases(hostname)

	for _, testCase := range testCases {
		func() {
			dirName, err := utiltesting.MkTmpdir("file-test")
			if err != nil {
				t.Fatalf("unable to create temp dir: %v", err)
			}
			defer os.RemoveAll(dirName)
			file := testCase.writeToFile(dirName, "test_pod_config", t)

			ch := make(chan interface{})
			NewSourceFile(file, hostname, time.Millisecond, ch)
			select {
			case got := <-ch:
				update := got.(kubetypes.PodUpdate)
				for _, pod := range update.Pods {
					if errs := validation.ValidatePod(pod); len(errs) > 0 {
						t.Fatalf("%s: Invalid pod %#v, %#v", testCase.desc, pod, errs)
					}
				}
				if !api.Semantic.DeepEqual(testCase.expected, update) {
					t.Fatalf("%s: Expected %#v, Got %#v", testCase.desc, testCase.expected, update)
				}
			case <-time.After(wait.ForeverTestTimeout):
				t.Fatalf("%s: Expected update, timeout instead", testCase.desc)
			}
		}()
	}
}

func TestReadPodsFromFileExistLater(t *testing.T) {
	watchFileAdded(false, t)
}

func TestReadPodsFromFileChanged(t *testing.T) {
	watchFileChanged(false, t)
}

func TestReadPodsFromFileInDirAdded(t *testing.T) {
	watchFileAdded(true, t)
}

func TestReadPodsFromFileInDirChanged(t *testing.T) {
	watchFileChanged(true, t)
}

func TestExtractFromBadDataFile(t *testing.T) {
	dirName, err := utiltesting.MkTmpdir("file-test")
	if err != nil {
		t.Fatalf("unable to create temp dir: %v", err)
	}
	defer os.RemoveAll(dirName)

	fileName := filepath.Join(dirName, "test_pod_config")
	err = ioutil.WriteFile(fileName, []byte{1, 2, 3}, 0555)
	if err != nil {
		t.Fatalf("unable to write test file %#v", err)
	}

	ch := make(chan interface{}, 1)
	c := new(fileName, "localhost", time.Millisecond, ch)
	err = c.resetStoreFromPath()
	if err == nil {
		t.Fatalf("expected error, got nil")
	}
	expectEmptyChannel(t, ch)
}

func TestExtractFromEmptyDir(t *testing.T) {
	dirName, err := utiltesting.MkTmpdir("file-test")
	if err != nil {
		t.Fatalf("unexpected error: %v", err)
	}
	defer os.RemoveAll(dirName)

	ch := make(chan interface{}, 1)
	c := new(dirName, "localhost", time.Millisecond, ch)
	err = c.resetStoreFromPath()
	if err != nil {
		t.Fatalf("unexpected error: %v", err)
	}

	update := (<-ch).(kubetypes.PodUpdate)
	expected := CreatePodUpdate(kubetypes.SET, kubetypes.FileSource)
	if !api.Semantic.DeepEqual(expected, update) {
		t.Fatalf("expected %#v, Got %#v", expected, update)
	}
}

type testCase struct {
	desc     string
	pod      runtime.Object
	expected kubetypes.PodUpdate
}

func getTestCases(hostname types.NodeName) []*testCase {
	grace := int64(30)
	return []*testCase{
		{
			desc: "Simple pod",
			pod: &api.Pod{
				TypeMeta: unversioned.TypeMeta{
					Kind:       "Pod",
					APIVersion: "",
				},
				ObjectMeta: api.ObjectMeta{
					Name:      "test",
					UID:       "12345",
					Namespace: "mynamespace",
				},
				Spec: api.PodSpec{
					Containers:      []api.Container{{Name: "image", Image: "test/image", SecurityContext: securitycontext.ValidSecurityContextWithContainerDefaults()}},
					SecurityContext: &api.PodSecurityContext{},
				},
				Status: api.PodStatus{
					Phase: api.PodPending,
				},
			},
			expected: CreatePodUpdate(kubetypes.SET, kubetypes.FileSource, &api.Pod{
				ObjectMeta: api.ObjectMeta{
					Name:        "test-" + string(hostname),
					UID:         "12345",
					Namespace:   "mynamespace",
					Annotations: map[string]string{kubetypes.ConfigHashAnnotationKey: "12345"},
					SelfLink:    getSelfLink("test-"+string(hostname), "mynamespace"),
				},
				Spec: api.PodSpec{
					NodeName:                      string(hostname),
					RestartPolicy:                 api.RestartPolicyAlways,
					DNSPolicy:                     api.DNSClusterFirst,
					TerminationGracePeriodSeconds: &grace,
					Containers: []api.Container{{
						Name:  "image",
						Image: "test/image",
						TerminationMessagePath: "/dev/termination-log",
						ImagePullPolicy:        "Always",
						SecurityContext:        securitycontext.ValidSecurityContextWithContainerDefaults()}},
					SecurityContext: &api.PodSecurityContext{},
				},
				Status: api.PodStatus{
					Phase: api.PodPending,
				},
			}),
		},
	}
}

func (tc *testCase) writeToFile(dir, name string, t *testing.T) string {
	var versionedPod runtime.Object
	err := testapi.Default.Converter().Convert(&tc.pod, &versionedPod, nil)
	if err != nil {
		t.Fatalf("%s: error in versioning the pod: %v", tc.desc, err)
	}
	fileContents, err := runtime.Encode(testapi.Default.Codec(), versionedPod)
	if err != nil {
		t.Fatalf("%s: error in encoding the pod: %v", tc.desc, err)
	}

	fileName := filepath.Join(dir, name)
	if err := writeFile(fileName, []byte(fileContents)); err != nil {
		t.Fatalf("unable to write test file %#v", err)
	}
	return fileName
}

func watchFileAdded(watchDir bool, t *testing.T) {
	hostname := types.NodeName("random-test-hostname")
	var testCases = getTestCases(hostname)

	fileNamePre := "test_pod_config"
	for index, testCase := range testCases {
		func() {
			dirName, err := utiltesting.MkTmpdir("dir-test")
			if err != nil {
				t.Fatalf("unable to create temp dir: %v", err)
			}
			defer os.RemoveAll(dirName)
			fileName := fmt.Sprintf("%s_%d", fileNamePre, index)

			ch := make(chan interface{})
			if watchDir {
				NewSourceFile(dirName, hostname, 100*time.Millisecond, ch)
			} else {
				NewSourceFile(filepath.Join(dirName, fileName), hostname, 100*time.Millisecond, ch)
			}
			expectEmptyUpdate(t, ch)

			addFile := func() {
				// Add a file
				testCase.writeToFile(dirName, fileName, t)
			}

			if watchDir {
				defer func() {
					// Remove the file
					deleteFile(dirName, fileName, ch, t)
				}()
			}

			go addFile()

			if watchDir {
				// expect an update by CREATE inotify event
				expectUpdate(t, ch, testCase)
				// expect an update by MODIFY inotify event
				expectUpdate(t, ch, testCase)

				from := fileName
				fileName = fileName + "_ch"
				go changeFileName(dirName, from, fileName, t)
				// expect an update by MOVED_FROM inotify event cause changing file name
				expectEmptyUpdate(t, ch)
				// expect an update by MOVED_TO inotify event cause changing file name
				expectUpdate(t, ch, testCase)
			} else {
				// expect an update by SourceFile.resetStoreFromPath()
				expectUpdate(t, ch, testCase)
			}
		}()
	}
}

func watchFileChanged(watchDir bool, t *testing.T) {
	hostname := types.NodeName("random-test-hostname")
	var testCases = getTestCases(hostname)

	fileNamePre := "test_pod_config"
	for index, testCase := range testCases {
		func() {
			dirName, err := utiltesting.MkTmpdir("dir-test")
			fileName := fmt.Sprintf("%s_%d", fileNamePre, index)
			if err != nil {
				t.Fatalf("unable to create temp dir: %v", err)
			}
			defer os.RemoveAll(dirName)

			var file string
			lock := &sync.Mutex{}
			ch := make(chan interface{})
			func() {
				lock.Lock()
				defer lock.Unlock()
				file = testCase.writeToFile(dirName, fileName, t)
			}()

			if watchDir {
				NewSourceFile(dirName, hostname, 100*time.Millisecond, ch)
				defer func() {
					// Remove the file
					deleteFile(dirName, fileName, ch, t)
				}()
			} else {
				NewSourceFile(file, hostname, 100*time.Millisecond, ch)
			}
			// expect an update by SourceFile.resetStoreFromPath()
			expectUpdate(t, ch, testCase)

			changeFile := func() {
				// Edit the file content
				lock.Lock()
				defer lock.Unlock()

				pod := testCase.pod.(*api.Pod)
				pod.Spec.Containers[0].Name = "image2"

				testCase.expected.Pods[0].Spec.Containers[0].Name = "image2"
				testCase.writeToFile(dirName, fileName, t)
			}

			go changeFile()
			// expect an update by MODIFY inotify event
			expectUpdate(t, ch, testCase)

			if watchDir {
				from := fileName
				fileName = fileName + "_ch"
				go changeFileName(dirName, from, fileName, t)
				// expect an update by MOVED_FROM inotify event cause changing file name
				expectEmptyUpdate(t, ch)
				// expect an update by MOVED_TO inotify event cause changing file name
				expectUpdate(t, ch, testCase)
			}
		}()
	}
}

func deleteFile(dir, file string, ch chan interface{}, t *testing.T) {
	go func() {
		path := filepath.Join(dir, file)
		err := os.Remove(path)
		if err != nil {
			t.Errorf("unable to remove test file %s: %s", path, err)
		}
	}()

	expectEmptyUpdate(t, ch)
}

func expectUpdate(t *testing.T, ch chan interface{}, testCase *testCase) {
	timer := time.After(5 * time.Second)
	for {
		select {
		case got := <-ch:
			update := got.(kubetypes.PodUpdate)
			for _, pod := range update.Pods {
				if errs := validation.ValidatePod(pod); len(errs) > 0 {
					t.Fatalf("%s: Invalid pod %#v, %#v", testCase.desc, pod, errs)
				}
			}

			if !api.Semantic.DeepEqual(testCase.expected, update) {
				t.Fatalf("%s: Expected: %#v, Got: %#v", testCase.desc, testCase.expected, update)
			}
			return
		case <-timer:
			t.Fatalf("%s: Expected update, timeout instead", testCase.desc)
		}
	}
}

func expectEmptyUpdate(t *testing.T, ch chan interface{}) {
	timer := time.After(5 * time.Second)
	for {
		select {
		case got := <-ch:
			update := got.(kubetypes.PodUpdate)
			if len(update.Pods) != 0 {
				t.Fatalf("expected empty update, got %#v", update)
			}
			return
		case <-timer:
			t.Fatalf("expected empty update, timeout instead")
		}
	}
}

func writeFile(filename string, data []byte) error {
	f, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE, 0666)
	if err != nil {
		return err
	}
	n, err := f.Write(data)
	if err == nil && n < len(data) {
		err = io.ErrShortWrite
	}
	if err1 := f.Close(); err == nil {
		err = err1
	}
	return err
}

func changeFileName(dir, from, to string, t *testing.T) {
	fromPath := filepath.Join(dir, from)
	toPath := filepath.Join(dir, to)
	if err := exec.Command("mv", fromPath, toPath).Run(); err != nil {
		t.Errorf("Fail to change file name: %s", err)
	}
}
<commit_msg>Fix #33784, IN_CREATE event does not guarantee file content written<commit_after>// +build linux

/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package config

import (
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"sync"
	"testing"
	"time"

	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/api/testapi"
	"k8s.io/kubernetes/pkg/api/unversioned"
	"k8s.io/kubernetes/pkg/api/validation"
	kubetypes "k8s.io/kubernetes/pkg/kubelet/types"
	"k8s.io/kubernetes/pkg/runtime"
	"k8s.io/kubernetes/pkg/securitycontext"
	"k8s.io/kubernetes/pkg/types"
	utiltesting "k8s.io/kubernetes/pkg/util/testing"
	"k8s.io/kubernetes/pkg/util/wait"
)

func TestExtractFromNonExistentFile(t *testing.T) {
	ch := make(chan interface{}, 1)
	c := new("/some/fake/file", "localhost", time.Millisecond, ch)
	err := c.watch()
	if err == nil {
		t.Errorf("Expected error")
	}
}

func TestUpdateOnNonExistentFile(t *testing.T) {
	ch := make(chan interface{})
	NewSourceFile("random_non_existent_path", "localhost", time.Millisecond, ch)
	select {
	case got := <-ch:
		update := got.(kubetypes.PodUpdate)
		expected := CreatePodUpdate(kubetypes.SET, kubetypes.FileSource)
		if !api.Semantic.DeepDerivative(expected, update) {
			t.Fatalf("expected %#v, Got %#v", expected, update)
		}

	case <-time.After(wait.ForeverTestTimeout):
		t.Fatalf("expected update, timeout instead")
	}
}

func TestReadPodsFromFileExistAlready(t *testing.T) {
	hostname := types.NodeName("random-test-hostname")
	var testCases = getTestCases(hostname)

	for _, testCase := range testCases {
		func() {
			dirName, err := utiltesting.MkTmpdir("file-test")
			if err != nil {
				t.Fatalf("unable to create temp dir: %v", err)
			}
			defer os.RemoveAll(dirName)
			file := testCase.writeToFile(dirName, "test_pod_config", t)

			ch := make(chan interface{})
			NewSourceFile(file, hostname, time.Millisecond, ch)
			select {
			case got := <-ch:
				update := got.(kubetypes.PodUpdate)
				for _, pod := range update.Pods {
					if errs := validation.ValidatePod(pod); len(errs) > 0 {
						t.Fatalf("%s: Invalid pod %#v, %#v", testCase.desc, pod, errs)
					}
				}
				if !api.Semantic.DeepEqual(testCase.expected, update) {
					t.Fatalf("%s: Expected %#v, Got %#v", testCase.desc, testCase.expected, update)
				}
			case <-time.After(wait.ForeverTestTimeout):
				t.Fatalf("%s: Expected update, timeout instead", testCase.desc)
			}
		}()
	}
}

func TestReadPodsFromFileExistLater(t *testing.T) {
	watchFileAdded(false, t)
}

func TestReadPodsFromFileChanged(t *testing.T) {
	watchFileChanged(false, t)
}

func TestReadPodsFromFileInDirAdded(t *testing.T) {
	watchFileAdded(true, t)
}

func TestReadPodsFromFileInDirChanged(t *testing.T) {
	watchFileChanged(true, t)
}

func TestExtractFromBadDataFile(t *testing.T) {
	dirName, err := utiltesting.MkTmpdir("file-test")
	if err != nil {
		t.Fatalf("unable to create temp dir: %v", err)
	}
	defer os.RemoveAll(dirName)

	fileName := filepath.Join(dirName, "test_pod_config")
	err = ioutil.WriteFile(fileName, []byte{1, 2, 3}, 0555)
	if err != nil {
		t.Fatalf("unable to write test file %#v", err)
	}

	ch := make(chan interface{}, 1)
	c := new(fileName, "localhost", time.Millisecond, ch)
	err = c.resetStoreFromPath()
	if err == nil {
		t.Fatalf("expected error, got nil")
	}
	expectEmptyChannel(t, ch)
}

func TestExtractFromEmptyDir(t *testing.T) {
	dirName, err := utiltesting.MkTmpdir("file-test")
	if err != nil {
		t.Fatalf("unexpected error: %v", err)
	}
	defer os.RemoveAll(dirName)

	ch := make(chan interface{}, 1)
	c := new(dirName, "localhost", time.Millisecond, ch)
	err = c.resetStoreFromPath()
	if err != nil {
		t.Fatalf("unexpected error: %v", err)
	}

	update := (<-ch).(kubetypes.PodUpdate)
	expected := CreatePodUpdate(kubetypes.SET, kubetypes.FileSource)
	if !api.Semantic.DeepEqual(expected, update) {
		t.Fatalf("expected %#v, Got %#v", expected, update)
	}
}

type testCase struct {
	desc     string
	pod      runtime.Object
	expected kubetypes.PodUpdate
}

func getTestCases(hostname types.NodeName) []*testCase {
	grace := int64(30)
	return []*testCase{
		{
			desc: "Simple pod",
			pod: &api.Pod{
				TypeMeta: unversioned.TypeMeta{
					Kind:       "Pod",
					APIVersion: "",
				},
				ObjectMeta: api.ObjectMeta{
					Name:      "test",
					UID:       "12345",
					Namespace: "mynamespace",
				},
				Spec: api.PodSpec{
					Containers:      []api.Container{{Name: "image", Image: "test/image", SecurityContext: securitycontext.ValidSecurityContextWithContainerDefaults()}},
					SecurityContext: &api.PodSecurityContext{},
				},
				Status: api.PodStatus{
					Phase: api.PodPending,
				},
			},
			expected: CreatePodUpdate(kubetypes.SET, kubetypes.FileSource, &api.Pod{
				ObjectMeta: api.ObjectMeta{
					Name:        "test-" + string(hostname),
					UID:         "12345",
					Namespace:   "mynamespace",
					Annotations: map[string]string{kubetypes.ConfigHashAnnotationKey: "12345"},
					SelfLink:    getSelfLink("test-"+string(hostname), "mynamespace"),
				},
				Spec: api.PodSpec{
					NodeName:                      string(hostname),
					RestartPolicy:                 api.RestartPolicyAlways,
					DNSPolicy:                     api.DNSClusterFirst,
					TerminationGracePeriodSeconds: &grace,
					Containers: []api.Container{{
						Name:  "image",
						Image: "test/image",
						TerminationMessagePath: "/dev/termination-log",
						ImagePullPolicy:        "Always",
						SecurityContext:        securitycontext.ValidSecurityContextWithContainerDefaults()}},
					SecurityContext: &api.PodSecurityContext{},
				},
				Status: api.PodStatus{
					Phase: api.PodPending,
				},
			}),
		},
	}
}

func (tc *testCase) writeToFile(dir, name string, t *testing.T) string {
	var versionedPod runtime.Object
	err := testapi.Default.Converter().Convert(&tc.pod, &versionedPod, nil)
	if err != nil {
		t.Fatalf("%s: error in versioning the pod: %v", tc.desc, err)
	}
	fileContents, err := runtime.Encode(testapi.Default.Codec(), versionedPod)
	if err != nil {
		t.Fatalf("%s: error in encoding the pod: %v", tc.desc, err)
	}

	fileName := filepath.Join(dir, name)
	if err := writeFile(fileName, []byte(fileContents)); err != nil {
		t.Fatalf("unable to write test file %#v", err)
	}
	return fileName
}

func watchFileAdded(watchDir bool, t *testing.T) {
	hostname := types.NodeName("random-test-hostname")
	var testCases = getTestCases(hostname)

	fileNamePre := "test_pod_config"
	for index, testCase := range testCases {
		func() {
			dirName, err := utiltesting.MkTmpdir("dir-test")
			if err != nil {
				t.Fatalf("unable to create temp dir: %v", err)
			}
			defer os.RemoveAll(dirName)
			fileName := fmt.Sprintf("%s_%d", fileNamePre, index)

			ch := make(chan interface{})
			if watchDir {
				NewSourceFile(dirName, hostname, 100*time.Millisecond, ch)
			} else {
				NewSourceFile(filepath.Join(dirName, fileName), hostname, 100*time.Millisecond, ch)
			}
			expectEmptyUpdate(t, ch)

			addFile := func() {
				// Add a file
				testCase.writeToFile(dirName, fileName, t)
			}

			go addFile()

			// For !watchDir: expect an update by SourceFile.resetStoreFromPath().
			// For watchDir: expect at least one update from CREATE & MODIFY inotify event.
			// Shouldn't expect two updates from CREATE & MODIFY because CREATE doesn't guarantee file written.
			// In that case no update will be sent from CREATE event.
			expectUpdate(t, ch, testCase)
		}()
	}
}

func watchFileChanged(watchDir bool, t *testing.T) {
	hostname := types.NodeName("random-test-hostname")
	var testCases = getTestCases(hostname)

	fileNamePre := "test_pod_config"
	for index, testCase := range testCases {
		func() {
			dirName, err := utiltesting.MkTmpdir("dir-test")
			fileName := fmt.Sprintf("%s_%d", fileNamePre, index)
			if err != nil {
				t.Fatalf("unable to create temp dir: %v", err)
			}
			defer os.RemoveAll(dirName)

			var file string
			lock := &sync.Mutex{}
			ch := make(chan interface{})
			func() {
				lock.Lock()
				defer lock.Unlock()
				file = testCase.writeToFile(dirName, fileName, t)
			}()

			if watchDir {
				NewSourceFile(dirName, hostname, 100*time.Millisecond, ch)
				defer func() {
					// Remove the file
					deleteFile(dirName, fileName, ch, t)
				}()
			} else {
				NewSourceFile(file, hostname, 100*time.Millisecond, ch)
			}
			// expect an update by SourceFile.resetStoreFromPath()
			expectUpdate(t, ch, testCase)

			changeFile := func() {
				// Edit the file content
				lock.Lock()
				defer lock.Unlock()

				pod := testCase.pod.(*api.Pod)
				pod.Spec.Containers[0].Name = "image2"

				testCase.expected.Pods[0].Spec.Containers[0].Name = "image2"
				testCase.writeToFile(dirName, fileName, t)
			}

			go changeFile()
			// expect an update by MODIFY inotify event
			expectUpdate(t, ch, testCase)

			if watchDir {
				from := fileName
				fileName = fileName + "_ch"
				go changeFileName(dirName, from, fileName, t)
				// expect an update by MOVED_FROM inotify event cause changing file name
				expectEmptyUpdate(t, ch)
				// expect an update by MOVED_TO inotify event cause changing file name
				expectUpdate(t, ch, testCase)
			}
		}()
	}
}

func deleteFile(dir, file string, ch chan interface{}, t *testing.T) {
	go func() {
		path := filepath.Join(dir, file)
		err := os.Remove(path)
		if err != nil {
			t.Errorf("unable to remove test file %s: %s", path, err)
		}
	}()

	expectEmptyUpdate(t, ch)
}

func expectUpdate(t *testing.T, ch chan interface{}, testCase *testCase) {
	timer := time.After(5 * time.Second)
	for {
		select {
		case got := <-ch:
			update := got.(kubetypes.PodUpdate)
			for _, pod := range update.Pods {
				if errs := validation.ValidatePod(pod); len(errs) > 0 {
					t.Fatalf("%s: Invalid pod %#v, %#v", testCase.desc, pod, errs)
				}
			}

			if !api.Semantic.DeepEqual(testCase.expected, update) {
				t.Fatalf("%s: Expected: %#v, Got: %#v", testCase.desc, testCase.expected, update)
			}
			return
		case <-timer:
			t.Fatalf("%s: Expected update, timeout instead", testCase.desc)
		}
	}
}

func expectEmptyUpdate(t *testing.T, ch chan interface{}) {
	timer := time.After(5 * time.Second)
	for {
		select {
		case got := <-ch:
			update := got.(kubetypes.PodUpdate)
			if len(update.Pods) != 0 {
				t.Fatalf("expected empty update, got %#v", update)
			}
			return
		case <-timer:
			t.Fatalf("expected empty update, timeout instead")
		}
	}
}

func writeFile(filename string, data []byte) error {
	f, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE, 0666)
	if err != nil {
		return err
	}
	n, err := f.Write(data)
	if err == nil && n < len(data) {
		err = io.ErrShortWrite
	}
	if err1 := f.Close(); err == nil {
		err = err1
	}
	return err
}

func changeFileName(dir, from, to string, t *testing.T) {
	fromPath := filepath.Join(dir, from)
	toPath := filepath.Join(dir, to)
	if err := exec.Command("mv", fromPath, toPath).Run(); err != nil {
		t.Errorf("Fail to change file name: %s", err)
	}
}
<|endoftext|>


<commit_before>package migrate

import (
	"flag"
	"runtime"
	"sync"
	"testing"

	"k8s.io/apimachinery/pkg/api/meta"
	"k8s.io/kubernetes/pkg/kubectl/resource"
)

// TestResourceVisitor_Visit is used to check for race conditions
func TestResourceVisitor_Visit(t *testing.T) {
	// set glog levels so we write to Out
	flag.CommandLine.Lookup("v").Value.Set("2")
	flag.CommandLine.Lookup("stderrthreshold").Value.Set("INFO")

	type fields struct {
		Out      mapWriter
		Builder  testBuilder
		SaveFn   *countSaveFn
		PrintFn  MigrateActionFunc
		FilterFn MigrateFilterFunc
		DryRun   bool
		Workers  int
	}
	type args struct {
		fn MigrateVisitFunc
	}
	tests := []struct {
		name    string
		fields  fields
		args    args
		wantErr bool
	}{
		{
			name: "migrate storage race detection",
			fields: fields{
				Out:      make(mapWriter),       // detect writes via multiple goroutines
				Builder:  testBuilder(5000),     // send a specific amount of infos to SaveFn
				SaveFn:   new(countSaveFn),      // make sure we process all resources
				PrintFn:  nil,                   // must be nil to use SaveFn
				FilterFn: nil,                   // we want no filtering
				DryRun:   false,                 // must be false to use SaveFn
				Workers:  32 * runtime.NumCPU(), // same as migrate storage
			},
			args: args{
				fn: AlwaysRequiresMigration, // same as migrate storage
			},
			wantErr: false, // should never error
		},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			o := &ResourceVisitor{
				Out:      tt.fields.Out,
				Builder:  tt.fields.Builder,
				SaveFn:   tt.fields.SaveFn.save,
				PrintFn:  tt.fields.PrintFn,
				FilterFn: tt.fields.FilterFn,
				DryRun:   tt.fields.DryRun,
				Workers:  tt.fields.Workers,
			}
			// how many infos are we expected to process
			expectedInfos := int(tt.fields.Builder)
			// countSaveFn will spawn one goroutine per info it sees
			tt.fields.SaveFn.w.Add(expectedInfos)
			// process the infos
			if err := o.Visit(tt.args.fn); (err != nil) != tt.wantErr {
				t.Errorf("ResourceVisitor.Visit() error = %v, wantErr %v", err, tt.wantErr)
			}
			// wait for all countSaveFn goroutines to finish
			tt.fields.SaveFn.w.Wait()
			// check that we saw the correct amount of infos throughout
			writes := len(tt.fields.Out) - 1 // minus one for the summary output
			saves := tt.fields.SaveFn.n
			if expectedInfos != writes || expectedInfos != saves {
				t.Errorf("ResourceVisitor.Visit() incorrect counts seen, expectedInfos=%d writes=%d saves=%d out=%v",
					expectedInfos, writes, saves, tt.fields.Out)
			}
		})
	}
}

// mapWriter is an io.Writer that is guaranteed to panic if accessed via multiple goroutines at the same time
type mapWriter map[int]string

func (m mapWriter) Write(p []byte) (n int, err error) {
	l := len(m)      // makes it easy to track how many times Write is called
	m[l] = string(p) // string for debugging
	return len(p), nil
}

// countSaveFn is used to build a MigrateActionFunc (SaveFn) that records how many times it was called
// goroutine safe
type countSaveFn struct {
	w sync.WaitGroup
	m sync.Mutex
	n int
}

func (c *countSaveFn) save(_ *resource.Info, _ Reporter) error {
	// do not block workers on the mutex, we do not want to accidentally serialize our code in a way that masks race conditions
	go func() {
		c.m.Lock()
		c.n++
		c.m.Unlock()
		c.w.Done()
	}()
	return nil
}

// testBuilder emits a resource.Visitor that calls resource.VisitorFunc n times
type testBuilder int

func (t testBuilder) Visitor(_ ...resource.ErrMatchFunc) (resource.Visitor, error) {
	infos := make(resource.InfoListVisitor, t) // the resource.VisitorFunc will be called t times
	for i := range infos {
		infos[i] = &resource.Info{Mapping: &meta.RESTMapping{}} // just enough to prevent NPEs
	}
	return infos, nil
}
<commit_msg>Undo global state change in TestResourceVisitor_Visit<commit_after>package migrate

import (
	"flag"
	"runtime"
	"sync"
	"testing"

	"k8s.io/apimachinery/pkg/api/meta"
	"k8s.io/kubernetes/pkg/kubectl/resource"
)

// TestResourceVisitor_Visit is used to check for race conditions
func TestResourceVisitor_Visit(t *testing.T) {
	// get log level flag for glog
	verbosity := flag.CommandLine.Lookup("v").Value
	// save its original value
	origVerbosity := verbosity.String()
	// set log level high enough so we write to ResourceOptions.Out on each success
	verbosity.Set("1")
	// restore the original flag value when we return
	defer func() {
		verbosity.Set(origVerbosity)
	}()

	type fields struct {
		Out      mapWriter
		Builder  testBuilder
		SaveFn   *countSaveFn
		PrintFn  MigrateActionFunc
		FilterFn MigrateFilterFunc
		DryRun   bool
		Workers  int
	}
	type args struct {
		fn MigrateVisitFunc
	}
	tests := []struct {
		name    string
		fields  fields
		args    args
		wantErr bool
	}{
		{
			name: "migrate storage race detection",
			fields: fields{
				Out:      make(mapWriter),       // detect writes via multiple goroutines
				Builder:  testBuilder(5000),     // send a specific amount of infos to SaveFn
				SaveFn:   new(countSaveFn),      // make sure we process all resources
				PrintFn:  nil,                   // must be nil to use SaveFn
				FilterFn: nil,                   // we want no filtering
				DryRun:   false,                 // must be false to use SaveFn
				Workers:  32 * runtime.NumCPU(), // same as migrate storage
			},
			args: args{
				fn: AlwaysRequiresMigration, // same as migrate storage
			},
			wantErr: false, // should never error
		},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			o := &ResourceVisitor{
				Out:      tt.fields.Out,
				Builder:  tt.fields.Builder,
				SaveFn:   tt.fields.SaveFn.save,
				PrintFn:  tt.fields.PrintFn,
				FilterFn: tt.fields.FilterFn,
				DryRun:   tt.fields.DryRun,
				Workers:  tt.fields.Workers,
			}
			// how many infos are we expected to process
			expectedInfos := int(tt.fields.Builder)
			// countSaveFn will spawn one goroutine per info it sees
			tt.fields.SaveFn.w.Add(expectedInfos)
			// process the infos
			if err := o.Visit(tt.args.fn); (err != nil) != tt.wantErr {
				t.Errorf("ResourceVisitor.Visit() error = %v, wantErr %v", err, tt.wantErr)
			}
			// wait for all countSaveFn goroutines to finish
			tt.fields.SaveFn.w.Wait()
			// check that we saw the correct amount of infos throughout
			writes := len(tt.fields.Out) - 1 // minus one for the summary output
			saves := tt.fields.SaveFn.n
			if expectedInfos != writes || expectedInfos != saves {
				t.Errorf("ResourceVisitor.Visit() incorrect counts seen, expectedInfos=%d writes=%d saves=%d out=%v",
					expectedInfos, writes, saves, tt.fields.Out)
			}
		})
	}
}

// mapWriter is an io.Writer that is guaranteed to panic if accessed via multiple goroutines at the same time
type mapWriter map[int]string

func (m mapWriter) Write(p []byte) (n int, err error) {
	l := len(m)      // makes it easy to track how many times Write is called
	m[l] = string(p) // string for debugging
	return len(p), nil
}

// countSaveFn is used to build a MigrateActionFunc (SaveFn) that records how many times it was called
// goroutine safe
type countSaveFn struct {
	w sync.WaitGroup
	m sync.Mutex
	n int
}

func (c *countSaveFn) save(_ *resource.Info, _ Reporter) error {
	// do not block workers on the mutex, we do not want to accidentally serialize our code in a way that masks race conditions
	go func() {
		c.m.Lock()
		c.n++
		c.m.Unlock()
		c.w.Done()
	}()
	return nil
}

// testBuilder emits a resource.Visitor that calls resource.VisitorFunc n times
type testBuilder int

func (t testBuilder) Visitor(_ ...resource.ErrMatchFunc) (resource.Visitor, error) {
	infos := make(resource.InfoListVisitor, t) // the resource.VisitorFunc will be called t times
	for i := range infos {
		infos[i] = &resource.Info{Mapping: &meta.RESTMapping{}} // just enough to prevent NPEs
	}
	return infos, nil
}
<|endoftext|>


<commit_before>// Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"). You may
// not use this file except in compliance with the License. A copy of the
// License is located at
//
//	http://aws.amazon.com/apache2.0/
//
// or in the "license" file accompanying this file. This file is distributed
// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
// express or implied. See the License for the specific language governing
// permissions and limitations under the License.

package main

import (
	"flag"
	mathrand "math/rand"
	"os"
	"runtime"
	"time"

	acshandler "github.com/aws/amazon-ecs-agent/agent/acs/handler"
	"github.com/aws/amazon-ecs-agent/agent/api"
	"github.com/aws/amazon-ecs-agent/agent/config"
	"github.com/aws/amazon-ecs-agent/agent/ec2"
	"github.com/aws/amazon-ecs-agent/agent/engine"
	"github.com/aws/amazon-ecs-agent/agent/eventhandler"
	"github.com/aws/amazon-ecs-agent/agent/handlers"
	"github.com/aws/amazon-ecs-agent/agent/httpclient"
	"github.com/aws/amazon-ecs-agent/agent/logger"
	"github.com/aws/amazon-ecs-agent/agent/sighandlers"
	"github.com/aws/amazon-ecs-agent/agent/sighandlers/exitcodes"
	"github.com/aws/amazon-ecs-agent/agent/statemanager"
	"github.com/aws/amazon-ecs-agent/agent/tcs/handler"
	"github.com/aws/amazon-ecs-agent/agent/utils"
	utilatomic "github.com/aws/amazon-ecs-agent/agent/utils/atomic"
	"github.com/aws/amazon-ecs-agent/agent/version"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	log "github.com/cihub/seelog"
	"golang.org/x/net/context"
)

func init() {
	runtime.GOMAXPROCS(1)
	mathrand.Seed(time.Now().UnixNano())
}

func main() {
	// Use a returning main instead of os.Exiting main to allow defers to run
	// before exit
	os.Exit(_main())
}
func _main() int {
	defer log.Flush()
	versionFlag := flag.Bool("version", false, "Print the agent version information and exit")
	acceptInsecureCert := flag.Bool("k", false, "Do not verify ssl certs")
	logLevel := flag.String("loglevel", "", "Loglevel: [<crit>|<error>|<warn>|<info>|<debug>]")
	flag.Parse()

	logger.SetLevel(*logLevel)

	log.Infof("Starting Agent: %v", version.String())

	log.Info("Loading configuration")
	cfg, err := config.NewConfig()
	// Load cfg before doing 'versionFlag' so that it has the DOCKER_HOST
	// variable loaded if needed
	if *versionFlag {
		versionableEngine := engine.NewTaskEngine(cfg)
		version.PrintVersion(versionableEngine)
		return exitcodes.ExitSuccess
	}

	sighandlers.StartDebugHandler()
	ctx := context.Background()

	if err != nil {
		log.Criticalf("Error loading config: %v", err)
		// All required config values can be inferred from EC2 Metadata, so this error could be transient.
		return exitcodes.ExitError
	}
	log.Debug("Loaded config: " + cfg.String())

	var currentEc2InstanceID, containerInstanceArn string
	var taskEngine engine.TaskEngine

	if cfg.Checkpoint {
		log.Info("Checkpointing is enabled. Attempting to load state")
		var previousCluster, previousEc2InstanceID, previousContainerInstanceArn string
		previousTaskEngine := engine.NewTaskEngine(cfg)
		// previousState is used to verify that our current runtime configuration is
		// compatible with our past configuration as reflected by our state-file
		previousState, err := initializeStateManager(cfg, previousTaskEngine, &previousCluster, &previousContainerInstanceArn, &previousEc2InstanceID, acshandler.SequenceNumber)
		if err != nil {
			log.Criticalf("Error creating state manager: %v", err)
			return exitcodes.ExitTerminal
		}

		err = previousState.Load()
		if err != nil {
			log.Criticalf("Error loading previously saved state: %v", err)
			return exitcodes.ExitTerminal
		}

		if previousCluster != "" {
			// TODO Handle default cluster in a sane and unified way across the codebase
			configuredCluster := cfg.Cluster
			if configuredCluster == "" {
				log.Debug("Setting cluster to default; none configured")
				configuredCluster = config.DEFAULT_CLUSTER_NAME
			}
			if previousCluster != configuredCluster {
				log.Criticalf("Data mismatch; saved cluster '%v' does not match configured cluster '%v'. Perhaps you want to delete the configured checkpoint file?", previousCluster, configuredCluster)
				return exitcodes.ExitTerminal
			}
			cfg.Cluster = previousCluster
			log.Infof("Restored cluster '%v'", cfg.Cluster)
		}

		if instanceIdentityDoc, err := ec2.GetInstanceIdentityDocument(); err == nil {
			currentEc2InstanceID = instanceIdentityDoc.InstanceId
		} else {
			log.Criticalf("Unable to access EC2 Metadata service to determine EC2 ID: %v", err)
		}

		if previousEc2InstanceID != "" && previousEc2InstanceID != currentEc2InstanceID {
			log.Warnf("Data mismatch; saved InstanceID '%v' does not match current InstanceID '%v'. Overwriting old datafile", previousEc2InstanceID, currentEc2InstanceID)

			// Reset taskEngine; all the other values are still default
			taskEngine = engine.NewTaskEngine(cfg)
		} else {
			// Use the values we loaded if there's no issue
			containerInstanceArn = previousContainerInstanceArn
			taskEngine = previousTaskEngine
		}
	} else {
		log.Info("Checkpointing not enabled; a new container instance will be created each time the agent is run")
		taskEngine = engine.NewTaskEngine(cfg)
	}

	stateManager, err := initializeStateManager(cfg, taskEngine, &cfg.Cluster, &containerInstanceArn, &currentEc2InstanceID, acshandler.SequenceNumber)
	if err != nil {
		log.Criticalf("Error creating state manager: %v", err)
		return exitcodes.ExitTerminal
	}

	credentialProvider := aws.DefaultChainCredentials
	// Preflight request to make sure they're good
	if preflightCreds, err := credentialProvider.Get(); err != nil || preflightCreds.AccessKeyID == "" {
		log.Warnf("Error getting valid credentials (AKID %v): %v", preflightCreds.AccessKeyID, err)
	}
	client := api.NewECSClient(credentialProvider, cfg, httpclient.New(api.RoundtripTimeout, *acceptInsecureCert))

	if containerInstanceArn == "" {
		log.Info("Registering Instance with ECS")
		containerInstanceArn, err = client.RegisterContainerInstance("")
		if err != nil {
			log.Errorf("Error registering: %v", err)
			if retriable, ok := err.(utils.Retriable); ok && !retriable.Retry() {
				return exitcodes.ExitTerminal
			}
			return exitcodes.ExitError
		}
		log.Infof("Registration completed successfully. I am running as '%v' in cluster '%v'", containerInstanceArn, cfg.Cluster)
		// Save our shiny new containerInstanceArn
		stateManager.Save()
	} else {
		log.Infof("Restored from checkpoint file. I am running as '%v' in cluster '%v'", containerInstanceArn, cfg.Cluster)
		_, err = client.RegisterContainerInstance(containerInstanceArn)
		if err != nil {
			log.Errorf("Error registering: %v", err)
			if awserr, ok := err.(awserr.Error); ok && api.IsInstanceTypeChangedError(awserr) {
				log.Criticalf("The current instance type does not match the registered instance type. Please revert the instance type change, or alternatively launch a new instance. Error: %v", err)
				return exitcodes.ExitTerminal
			}
		}
	}

	// Begin listening to the docker daemon and saving changes
	taskEngine.SetSaver(stateManager)
	taskEngine.MustInit()

	go sighandlers.StartTerminationHandler(stateManager, taskEngine)

	// Agent introspection api
	go handlers.ServeHttp(&containerInstanceArn, taskEngine, cfg)

	// Start sending events to the backend
	go eventhandler.HandleEngineEvents(taskEngine, client, stateManager)

	telemetrySessionParams := tcshandler.TelemetrySessionParams{
		ContainerInstanceArn: containerInstanceArn,
		CredentialProvider:   credentialProvider,
		Cfg:                  cfg,
		AcceptInvalidCert:    *acceptInsecureCert,
		EcsClient:            client,
		TaskEngine:           taskEngine,
	}

	// Start metrics session in a go routine
	go tcshandler.StartMetricsSession(telemetrySessionParams)

	log.Info("Beginning Polling for updates")
	err = acshandler.StartSession(ctx, acshandler.StartSessionArguments{
		AcceptInvalidCert:    *acceptInsecureCert,
		Config:               cfg,
		ContainerInstanceArn: containerInstanceArn,
		CredentialProvider:   credentialProvider,
		ECSClient:            client,
		StateManager:         stateManager,
		TaskEngine:           taskEngine,
	})
	if err != nil {
		log.Criticalf("Unretriable error starting communicating with ACS: %v", err)
		return exitcodes.ExitTerminal
	}
	log.Critical("ACS Session handler should never exit")
	return exitcodes.ExitError
}

func initializeStateManager(cfg *config.Config, taskEngine engine.TaskEngine, cluster, containerInstanceArn, savedInstanceID *string, sequenceNumber *utilatomic.IncreasingInt64) (statemanager.StateManager, error) {
	if !cfg.Checkpoint {
		return statemanager.NewNoopStateManager(), nil
	}
	stateManager, err := statemanager.NewStateManager(cfg,
		statemanager.AddSaveable("TaskEngine", taskEngine),
		statemanager.AddSaveable("ContainerInstanceArn", containerInstanceArn),
		statemanager.AddSaveable("Cluster", cluster),
		statemanager.AddSaveable("EC2InstanceID", savedInstanceID),
		statemanager.AddSaveable("ACSSeqNum", sequenceNumber),
	)
	if err != nil {
		return nil, err
	}
	return stateManager, nil
}
<commit_msg>Reregistration: Don't ignore errors<commit_after>// Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"). You may
// not use this file except in compliance with the License. A copy of the
// License is located at
//
//	http://aws.amazon.com/apache2.0/
//
// or in the "license" file accompanying this file. This file is distributed
// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
// express or implied. See the License for the specific language governing
// permissions and limitations under the License.

package main

import (
	"flag"
	mathrand "math/rand"
	"os"
	"runtime"
	"time"

	acshandler "github.com/aws/amazon-ecs-agent/agent/acs/handler"
	"github.com/aws/amazon-ecs-agent/agent/api"
	"github.com/aws/amazon-ecs-agent/agent/config"
	"github.com/aws/amazon-ecs-agent/agent/ec2"
	"github.com/aws/amazon-ecs-agent/agent/engine"
	"github.com/aws/amazon-ecs-agent/agent/eventhandler"
	"github.com/aws/amazon-ecs-agent/agent/handlers"
	"github.com/aws/amazon-ecs-agent/agent/httpclient"
	"github.com/aws/amazon-ecs-agent/agent/logger"
	"github.com/aws/amazon-ecs-agent/agent/sighandlers"
	"github.com/aws/amazon-ecs-agent/agent/sighandlers/exitcodes"
	"github.com/aws/amazon-ecs-agent/agent/statemanager"
	"github.com/aws/amazon-ecs-agent/agent/tcs/handler"
	"github.com/aws/amazon-ecs-agent/agent/utils"
	utilatomic "github.com/aws/amazon-ecs-agent/agent/utils/atomic"
	"github.com/aws/amazon-ecs-agent/agent/version"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	log "github.com/cihub/seelog"
	"golang.org/x/net/context"
)

func init() {
	runtime.GOMAXPROCS(1)
	mathrand.Seed(time.Now().UnixNano())
}

func main() {
	// Use a returning main instead of os.Exiting main to allow defers to run
	// before exit
	os.Exit(_main())
}
func _main() int {
	defer log.Flush()
	versionFlag := flag.Bool("version", false, "Print the agent version information and exit")
	acceptInsecureCert := flag.Bool("k", false, "Do not verify ssl certs")
	logLevel := flag.String("loglevel", "", "Loglevel: [<crit>|<error>|<warn>|<info>|<debug>]")
	flag.Parse()

	logger.SetLevel(*logLevel)

	log.Infof("Starting Agent: %v", version.String())

	log.Info("Loading configuration")
	cfg, err := config.NewConfig()
	// Load cfg before doing 'versionFlag' so that it has the DOCKER_HOST
	// variable loaded if needed
	if *versionFlag {
		versionableEngine := engine.NewTaskEngine(cfg)
		version.PrintVersion(versionableEngine)
		return exitcodes.ExitSuccess
	}

	sighandlers.StartDebugHandler()
	ctx := context.Background()

	if err != nil {
		log.Criticalf("Error loading config: %v", err)
		// All required config values can be inferred from EC2 Metadata, so this error could be transient.
		return exitcodes.ExitError
	}
	log.Debug("Loaded config: " + cfg.String())

	var currentEc2InstanceID, containerInstanceArn string
	var taskEngine engine.TaskEngine

	if cfg.Checkpoint {
		log.Info("Checkpointing is enabled. Attempting to load state")
		var previousCluster, previousEc2InstanceID, previousContainerInstanceArn string
		previousTaskEngine := engine.NewTaskEngine(cfg)
		// previousState is used to verify that our current runtime configuration is
		// compatible with our past configuration as reflected by our state-file
		previousState, err := initializeStateManager(cfg, previousTaskEngine, &previousCluster, &previousContainerInstanceArn, &previousEc2InstanceID, acshandler.SequenceNumber)
		if err != nil {
			log.Criticalf("Error creating state manager: %v", err)
			return exitcodes.ExitTerminal
		}

		err = previousState.Load()
		if err != nil {
			log.Criticalf("Error loading previously saved state: %v", err)
			return exitcodes.ExitTerminal
		}

		if previousCluster != "" {
			// TODO Handle default cluster in a sane and unified way across the codebase
			configuredCluster := cfg.Cluster
			if configuredCluster == "" {
				log.Debug("Setting cluster to default; none configured")
				configuredCluster = config.DEFAULT_CLUSTER_NAME
			}
			if previousCluster != configuredCluster {
				log.Criticalf("Data mismatch; saved cluster '%v' does not match configured cluster '%v'. Perhaps you want to delete the configured checkpoint file?", previousCluster, configuredCluster)
				return exitcodes.ExitTerminal
			}
			cfg.Cluster = previousCluster
			log.Infof("Restored cluster '%v'", cfg.Cluster)
		}

		if instanceIdentityDoc, err := ec2.GetInstanceIdentityDocument(); err == nil {
			currentEc2InstanceID = instanceIdentityDoc.InstanceId
		} else {
			log.Criticalf("Unable to access EC2 Metadata service to determine EC2 ID: %v", err)
		}

		if previousEc2InstanceID != "" && previousEc2InstanceID != currentEc2InstanceID {
			log.Warnf("Data mismatch; saved InstanceID '%v' does not match current InstanceID '%v'. Overwriting old datafile", previousEc2InstanceID, currentEc2InstanceID)

			// Reset taskEngine; all the other values are still default
			taskEngine = engine.NewTaskEngine(cfg)
		} else {
			// Use the values we loaded if there's no issue
			containerInstanceArn = previousContainerInstanceArn
			taskEngine = previousTaskEngine
		}
	} else {
		log.Info("Checkpointing not enabled; a new container instance will be created each time the agent is run")
		taskEngine = engine.NewTaskEngine(cfg)
	}

	stateManager, err := initializeStateManager(cfg, taskEngine, &cfg.Cluster, &containerInstanceArn, &currentEc2InstanceID, acshandler.SequenceNumber)
	if err != nil {
		log.Criticalf("Error creating state manager: %v", err)
		return exitcodes.ExitTerminal
	}

	credentialProvider := aws.DefaultChainCredentials
	// Preflight request to make sure they're good
	if preflightCreds, err := credentialProvider.Get(); err != nil || preflightCreds.AccessKeyID == "" {
		log.Warnf("Error getting valid credentials (AKID %v): %v", preflightCreds.AccessKeyID, err)
	}
	client := api.NewECSClient(credentialProvider, cfg, httpclient.New(api.RoundtripTimeout, *acceptInsecureCert))

	if containerInstanceArn == "" {
		log.Info("Registering Instance with ECS")
		containerInstanceArn, err = client.RegisterContainerInstance("")
		if err != nil {
			log.Errorf("Error registering: %v", err)
			if retriable, ok := err.(utils.Retriable); ok && !retriable.Retry() {
				return exitcodes.ExitTerminal
			}
			return exitcodes.ExitError
		}
		log.Infof("Registration completed successfully. I am running as '%v' in cluster '%v'", containerInstanceArn, cfg.Cluster)
		// Save our shiny new containerInstanceArn
		stateManager.Save()
	} else {
		log.Infof("Restored from checkpoint file. I am running as '%v' in cluster '%v'", containerInstanceArn, cfg.Cluster)
		_, err = client.RegisterContainerInstance(containerInstanceArn)
		if err != nil {
			log.Errorf("Error re-registering: %v", err)
			if awserr, ok := err.(awserr.Error); ok && api.IsInstanceTypeChangedError(awserr) {
				log.Criticalf("The current instance type does not match the registered instance type. Please revert the instance type change, or alternatively launch a new instance. Error: %v", err)
				return exitcodes.ExitTerminal
			}
			return exitcodes.ExitError
		}
	}

	// Begin listening to the docker daemon and saving changes
	taskEngine.SetSaver(stateManager)
	taskEngine.MustInit()

	go sighandlers.StartTerminationHandler(stateManager, taskEngine)

	// Agent introspection api
	go handlers.ServeHttp(&containerInstanceArn, taskEngine, cfg)

	// Start sending events to the backend
	go eventhandler.HandleEngineEvents(taskEngine, client, stateManager)

	telemetrySessionParams := tcshandler.TelemetrySessionParams{
		ContainerInstanceArn: containerInstanceArn,
		CredentialProvider:   credentialProvider,
		Cfg:                  cfg,
		AcceptInvalidCert:    *acceptInsecureCert,
		EcsClient:            client,
		TaskEngine:           taskEngine,
	}

	// Start metrics session in a go routine
	go tcshandler.StartMetricsSession(telemetrySessionParams)

	log.Info("Beginning Polling for updates")
	err = acshandler.StartSession(ctx, acshandler.StartSessionArguments{
		AcceptInvalidCert:    *acceptInsecureCert,
		Config:               cfg,
		ContainerInstanceArn: containerInstanceArn,
		CredentialProvider:   credentialProvider,
		ECSClient:            client,
		StateManager:         stateManager,
		TaskEngine:           taskEngine,
	})
	if err != nil {
		log.Criticalf("Unretriable error starting communicating with ACS: %v", err)
		return exitcodes.ExitTerminal
	}
	log.Critical("ACS Session handler should never exit")
	return exitcodes.ExitError
}

func initializeStateManager(cfg *config.Config, taskEngine engine.TaskEngine, cluster, containerInstanceArn, savedInstanceID *string, sequenceNumber *utilatomic.IncreasingInt64) (statemanager.StateManager, error) {
	if !cfg.Checkpoint {
		return statemanager.NewNoopStateManager(), nil
	}
	stateManager, err := statemanager.NewStateManager(cfg,
		statemanager.AddSaveable("TaskEngine", taskEngine),
		statemanager.AddSaveable("ContainerInstanceArn", containerInstanceArn),
		statemanager.AddSaveable("Cluster", cluster),
		statemanager.AddSaveable("EC2InstanceID", savedInstanceID),
		statemanager.AddSaveable("ACSSeqNum", sequenceNumber),
	)
	if err != nil {
		return nil, err
	}
	return stateManager, nil
}
<|endoftext|>


<commit_before>package sparse

import (
	"os"
	"syscall"

	"fmt"

	"github.com/rancher/go-fibmap"
	"github.com/rancher/sparse-tools/log"
)

// Interval [Begin, End) is non-inclusive at the End
type Interval struct {
	Begin, End int64
}

// Len returns length of Interval
func (interval Interval) Len() int64 {
	return interval.End - interval.Begin
}

// String conversion
func (interval Interval) String() string {
	return fmt.Sprintf("[%8d:%8d](%3d)", interval.Begin/Blocks, interval.End/Blocks, interval.Len()/Blocks)
}

// FileIntervalKind distinguishes between data and hole
type FileIntervalKind int

// Sparse file Interval types
const (
	SparseData FileIntervalKind = 1 + iota
	SparseHole
	SparseIgnore // ignore file interval (equal src vs dst part)
)

// FileInterval describes either sparse data Interval or a hole
type FileInterval struct {
	Kind FileIntervalKind
	Interval
}

func (i FileInterval) String() string {
	kind := "?"
	switch i.Kind {
	case SparseData:
		kind = "D"
	case SparseHole:
		kind = " "
	case SparseIgnore:
		kind = "i"
	}
	return fmt.Sprintf("%s%v", kind, i.Interval)
}

// Storage block size in bytes
const (
	Blocks int64 = 4 << 10 // 4k
)

// os.Seek sparse whence values.
const (
	// Adjust the file offset to the next location in the file
	// greater than or equal to offset containing data.  If offset
	// points to data, then the file offset is set to offset.
	seekData int = 3

	// Adjust the file offset to the next hole in the file greater
	// than or equal to offset.  If offset points into the middle of
	// a hole, then the file offset is set to offset.  If there is no
	// hole past offset, then the file offset is adjusted to the End
	// of the file (i.e., there is an implicit hole at the End of any
	// file).
	seekHole int = 4
)

// syscall.Fallocate mode bits
const (
	// default is extend size
	fallocFlKeepSize uint32 = 1

	// de-allocates range
	fallocFlPunchHole uint32 = 2
)

// RetrieveLayoutStream streams sparse file data/hole layout
// Based on fiemap
// To abort: abortStream <- error
// Check status: err := <- errStream
// Usage: go RetrieveLayoutStream(...)
func RetrieveLayoutStream(abortStream <-chan error, file *os.File, r Interval, layoutStream chan<- FileInterval, errStream chan<- error) {
	const extents = 1024
	const chunkSizeMax = 1 /*GB*/ << 30
	chunkSize := r.Len()
	if chunkSize > chunkSizeMax {
		chunkSize = chunkSizeMax
	}

	chunk := Interval{r.Begin, r.Begin + chunkSize}
	// Process file extents for each chunk
	intervalLast := Interval{chunk.Begin, chunk.Begin}
	for chunk.Begin < r.End {
		if chunk.End > r.End {
			chunk.End = r.End
		}

		for more := true; more && chunk.Len() > 0; {
			ext, errno := fibmap.Fiemap(file.Fd(), uint64(chunk.Begin), uint64(chunk.Len()), 1024)
			if errno != 0 {
				close(layoutStream)
				errStream <- &os.PathError{Op: "Fiemap", Path: file.Name(), Err: errno}
				return
			}
			if len(ext) == 0 {
				break
			}

			// Process each extents
			for _, e := range ext {
				interval := Interval{int64(e.Logical), int64(e.Logical + e.Length)}
				log.Debug("Extent:", interval, e.Flags)
				if e.Flags&fibmap.FIEMAP_EXTENT_LAST != 0 {
					more = false
				}
				if intervalLast.End < interval.Begin {
					if intervalLast.Len() > 0 {
						// Pop last Data
						layoutStream <- FileInterval{SparseData, intervalLast}
					}
					// report hole
					intervalLast = Interval{intervalLast.End, interval.Begin}
					layoutStream <- FileInterval{SparseHole, intervalLast}

					// Start data
					intervalLast = interval
				} else {
					// coalesce
					intervalLast.End = interval.End
				}
				chunk.Begin = interval.End
			}
		}
		chunk = Interval{chunk.End, chunk.End + chunkSizeMax}
	}

	if intervalLast.Len() > 0 {
		// Pop last Data
		layoutStream <- FileInterval{SparseData, intervalLast}
	}
	if intervalLast.End < r.End {
		// report hole
		layoutStream <- FileInterval{SparseHole, Interval{intervalLast.End, r.End}}
	}

	close(layoutStream)
	errStream <- nil
	return
}

// RetrieveLayoutStream0 streams sparse file data/hole layout
// Deprecated; Based on file.seek; use RetrieveLayoutStream instead
// To abort: abortStream <- error
// Check status: err := <- errStream
// Usage: go RetrieveLayoutStream(...)
func RetrieveLayoutStream0(abortStream <-chan error, file *os.File, r Interval, layoutStream chan<- FileInterval, errStream chan<- error) {
	curr := r.Begin

	// Data or hole?
	offsetData, errData := file.Seek(curr, seekData)
	offsetHole, errHole := file.Seek(curr, seekHole)
	var interval FileInterval
	if errData != nil {
		// Hole only
		interval = FileInterval{SparseHole, Interval{curr, r.End}}
		if interval.Len() > 0 {
			layoutStream <- interval
		}
		close(layoutStream)
		errStream <- nil
		return
	} else if errHole != nil {
		// Data only
		interval = FileInterval{SparseData, Interval{curr, r.End}}
		if interval.Len() > 0 {
			layoutStream <- interval
		}
		close(layoutStream)
		errStream <- nil
		return
	}

	if offsetData < offsetHole {
		interval = FileInterval{SparseData, Interval{curr, offsetHole}}
		curr = offsetHole
	} else {
		interval = FileInterval{SparseHole, Interval{curr, offsetData}}
		curr = offsetData
	}
	if interval.Len() > 0 {
		layoutStream <- interval
	}

	for curr < r.End {
		// Check abort condition
		select {
		case err := <-abortStream:
			close(layoutStream)
			errStream <- err
			return
		default:
		}

		var whence int
		if SparseData == interval.Kind {
			whence = seekData
		} else {
			whence = seekHole
		}

		// Note: file.Seek masks syscall.ENXIO hence syscall is used instead
		next, errno := syscall.Seek(int(file.Fd()), curr, whence)
		if errno != nil {
			switch errno {
			case syscall.ENXIO:
				// no more intervals
				next = r.End // close the last interval
			default:
				// mimic standard "os"" package error handler
				close(layoutStream)
				errStream <- &os.PathError{Op: "seek", Path: file.Name(), Err: errno}
				return
			}
		}
		if SparseData == interval.Kind {
			// End of data, handle the last hole if any
			interval = FileInterval{SparseHole, Interval{curr, next}}
		} else {
			// End of hole, handle the last data if any
			interval = FileInterval{SparseData, Interval{curr, next}}
		}
		curr = next
		if interval.Len() > 0 {
			layoutStream <- interval
		}
	}
	close(layoutStream)
	errStream <- nil
	return
}

// RetrieveLayout retrieves sparse file hole and data layout
func RetrieveLayout(file *os.File, r Interval) ([]FileInterval, error) {
	layout := make([]FileInterval, 0, 1024)
	abortStream := make(chan error)
	layoutStream := make(chan FileInterval, 128)
	errStream := make(chan error)

	go RetrieveLayoutStream(abortStream, file, r, layoutStream, errStream)
	for interval := range layoutStream {
		layout = append(layout, interval)
	}
	return layout, <-errStream
}

// PunchHole in a sparse file, preserve file size
func PunchHole(file *os.File, hole Interval) error {
	fd := int(file.Fd())
	mode := fallocFlPunchHole | fallocFlKeepSize
	return syscall.Fallocate(fd, mode, hole.Begin, hole.Len())
}
<commit_msg>ssync: fixed ref to go-fibmap<commit_after>package sparse

import (
	"os"
	"syscall"

	"fmt"

	"github.com/frostschutz/go-fibmap"
	"github.com/rancher/sparse-tools/log"
)

// Interval [Begin, End) is non-inclusive at the End
type Interval struct {
	Begin, End int64
}

// Len returns length of Interval
func (interval Interval) Len() int64 {
	return interval.End - interval.Begin
}

// String conversion
func (interval Interval) String() string {
	return fmt.Sprintf("[%8d:%8d](%3d)", interval.Begin/Blocks, interval.End/Blocks, interval.Len()/Blocks)
}

// FileIntervalKind distinguishes between data and hole
type FileIntervalKind int

// Sparse file Interval types
const (
	SparseData FileIntervalKind = 1 + iota
	SparseHole
	SparseIgnore // ignore file interval (equal src vs dst part)
)

// FileInterval describes either sparse data Interval or a hole
type FileInterval struct {
	Kind FileIntervalKind
	Interval
}

func (i FileInterval) String() string {
	kind := "?"
	switch i.Kind {
	case SparseData:
		kind = "D"
	case SparseHole:
		kind = " "
	case SparseIgnore:
		kind = "i"
	}
	return fmt.Sprintf("%s%v", kind, i.Interval)
}

// Storage block size in bytes
const (
	Blocks int64 = 4 << 10 // 4k
)

// os.Seek sparse whence values.
const (
	// Adjust the file offset to the next location in the file
	// greater than or equal to offset containing data.  If offset
	// points to data, then the file offset is set to offset.
	seekData int = 3

	// Adjust the file offset to the next hole in the file greater
	// than or equal to offset.  If offset points into the middle of
	// a hole, then the file offset is set to offset.  If there is no
	// hole past offset, then the file offset is adjusted to the End
	// of the file (i.e., there is an implicit hole at the End of any
	// file).
	seekHole int = 4
)

// syscall.Fallocate mode bits
const (
	// default is extend size
	fallocFlKeepSize uint32 = 1

	// de-allocates range
	fallocFlPunchHole uint32 = 2
)

// RetrieveLayoutStream streams sparse file data/hole layout
// Based on fiemap
// To abort: abortStream <- error
// Check status: err := <- errStream
// Usage: go RetrieveLayoutStream(...)
func RetrieveLayoutStream(abortStream <-chan error, file *os.File, r Interval, layoutStream chan<- FileInterval, errStream chan<- error) {
	const extents = 1024
	const chunkSizeMax = 1 /*GB*/ << 30
	chunkSize := r.Len()
	if chunkSize > chunkSizeMax {
		chunkSize = chunkSizeMax
	}

	chunk := Interval{r.Begin, r.Begin + chunkSize}
	// Process file extents for each chunk
	intervalLast := Interval{chunk.Begin, chunk.Begin}
	for chunk.Begin < r.End {
		if chunk.End > r.End {
			chunk.End = r.End
		}

		for more := true; more && chunk.Len() > 0; {
			ext, errno := fibmap.Fiemap(file.Fd(), uint64(chunk.Begin), uint64(chunk.Len()), 1024)
			if errno != 0 {
				close(layoutStream)
				errStream <- &os.PathError{Op: "Fiemap", Path: file.Name(), Err: errno}
				return
			}
			if len(ext) == 0 {
				break
			}

			// Process each extents
			for _, e := range ext {
				interval := Interval{int64(e.Logical), int64(e.Logical + e.Length)}
				log.Debug("Extent:", interval, e.Flags)
				if e.Flags&fibmap.FIEMAP_EXTENT_LAST != 0 {
					more = false
				}
				if intervalLast.End < interval.Begin {
					if intervalLast.Len() > 0 {
						// Pop last Data
						layoutStream <- FileInterval{SparseData, intervalLast}
					}
					// report hole
					intervalLast = Interval{intervalLast.End, interval.Begin}
					layoutStream <- FileInterval{SparseHole, intervalLast}

					// Start data
					intervalLast = interval
				} else {
					// coalesce
					intervalLast.End = interval.End
				}
				chunk.Begin = interval.End
			}
		}
		chunk = Interval{chunk.End, chunk.End + chunkSizeMax}
	}

	if intervalLast.Len() > 0 {
		// Pop last Data
		layoutStream <- FileInterval{SparseData, intervalLast}
	}
	if intervalLast.End < r.End {
		// report hole
		layoutStream <- FileInterval{SparseHole, Interval{intervalLast.End, r.End}}
	}

	close(layoutStream)
	errStream <- nil
	return
}

// RetrieveLayoutStream0 streams sparse file data/hole layout
// Deprecated; Based on file.seek; use RetrieveLayoutStream instead
// To abort: abortStream <- error
// Check status: err := <- errStream
// Usage: go RetrieveLayoutStream(...)
func RetrieveLayoutStream0(abortStream <-chan error, file *os.File, r Interval, layoutStream chan<- FileInterval, errStream chan<- error) {
	curr := r.Begin

	// Data or hole?
	offsetData, errData := file.Seek(curr, seekData)
	offsetHole, errHole := file.Seek(curr, seekHole)
	var interval FileInterval
	if errData != nil {
		// Hole only
		interval = FileInterval{SparseHole, Interval{curr, r.End}}
		if interval.Len() > 0 {
			layoutStream <- interval
		}
		close(layoutStream)
		errStream <- nil
		return
	} else if errHole != nil {
		// Data only
		interval = FileInterval{SparseData, Interval{curr, r.End}}
		if interval.Len() > 0 {
			layoutStream <- interval
		}
		close(layoutStream)
		errStream <- nil
		return
	}

	if offsetData < offsetHole {
		interval = FileInterval{SparseData, Interval{curr, offsetHole}}
		curr = offsetHole
	} else {
		interval = FileInterval{SparseHole, Interval{curr, offsetData}}
		curr = offsetData
	}
	if interval.Len() > 0 {
		layoutStream <- interval
	}

	for curr < r.End {
		// Check abort condition
		select {
		case err := <-abortStream:
			close(layoutStream)
			errStream <- err
			return
		default:
		}

		var whence int
		if SparseData == interval.Kind {
			whence = seekData
		} else {
			whence = seekHole
		}

		// Note: file.Seek masks syscall.ENXIO hence syscall is used instead
		next, errno := syscall.Seek(int(file.Fd()), curr, whence)
		if errno != nil {
			switch errno {
			case syscall.ENXIO:
				// no more intervals
				next = r.End // close the last interval
			default:
				// mimic standard "os"" package error handler
				close(layoutStream)
				errStream <- &os.PathError{Op: "seek", Path: file.Name(), Err: errno}
				return
			}
		}
		if SparseData == interval.Kind {
			// End of data, handle the last hole if any
			interval = FileInterval{SparseHole, Interval{curr, next}}
		} else {
			// End of hole, handle the last data if any
			interval = FileInterval{SparseData, Interval{curr, next}}
		}
		curr = next
		if interval.Len() > 0 {
			layoutStream <- interval
		}
	}
	close(layoutStream)
	errStream <- nil
	return
}

// RetrieveLayout retrieves sparse file hole and data layout
func RetrieveLayout(file *os.File, r Interval) ([]FileInterval, error) {
	layout := make([]FileInterval, 0, 1024)
	abortStream := make(chan error)
	layoutStream := make(chan FileInterval, 128)
	errStream := make(chan error)

	go RetrieveLayoutStream(abortStream, file, r, layoutStream, errStream)
	for interval := range layoutStream {
		layout = append(layout, interval)
	}
	return layout, <-errStream
}

// PunchHole in a sparse file, preserve file size
func PunchHole(file *os.File, hole Interval) error {
	fd := int(file.Fd())
	mode := fallocFlPunchHole | fallocFlKeepSize
	return syscall.Fallocate(fd, mode, hole.Begin, hole.Len())
}
<|endoftext|>