<commit_before>package core

import (
	"fmt"
	"log"
	"os"
	"os/signal"
	"pilosa/config"
	"pilosa/db"
	"pilosa/hold"
	"pilosa/index"
	"pilosa/interfaces"
	"syscall"

	"github.com/coreos/go-etcd/etcd"
	"github.com/davecgh/go-spew/spew"
	"tux21b.org/v1/gocql/uuid"
)

type Service struct {
	Stopper
	Id             *uuid.UUID
	Etcd           *etcd.Client
	Cluster        *db.Cluster
	TopologyMapper *TopologyMapper
	ProcessMapper  *ProcessMapper
	ProcessMap     *ProcessMap
	Transport      interfaces.Transporter
	Dispatch       interfaces.Dispatcher
	Executor       interfaces.Executorer
	WebService     *WebService
	Index          *index.FragmentContainer
	Hold           *hold.Holder
	version        string
	name           string
}

func NewService() *Service {
	log.Println(spew.Sdump("NewService"))
	service := new(Service)
	service.init_id()
	service.Etcd = etcd.NewClient(nil)
	service.Cluster = db.NewCluster()
	service.TopologyMapper = NewTopologyMapper(service, "/pilosa/0")
	service.ProcessMapper = NewProcessMapper(service, "/pilosa/0")
	service.ProcessMap = NewProcessMap()
	service.WebService = NewWebService(service)
	service.Index = index.NewFragmentContainer()
	service.Hold = hold.NewHolder()
	service.version = "0.0.9"
	service.name = "Cruncher"
	service.PrepareLogging()
	return service
}

func (self *Service) PrepareLogging() {
	base_path := config.GetString("log_path")
	if base_path == "" {
		base_path = "/tmp"
	}
	f, err := os.OpenFile(fmt.Sprintf("%s/%s.%s", base_path, self.name, self.Id), os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
	if err != nil {
		log.Println("error opening file: %v", err)
	}
	//defer f.Close()
	log.SetOutput(f)
}

func (service *Service) init_id() {
	var id uuid.UUID
	var err error
	id_string := config.GetString("id")
	if id_string == "" {
		log.Println("Service id not configured, generating...")
		id = uuid.RandomUUID()
		if err != nil {
			log.Fatal("problem generating uuid")
		}
	} else {
		id, err = uuid.ParseUUID(id_string)
		if err != nil {
			log.Fatalf("Service id '%s' not valid", id_string)
		}
	}
	service.Id = &id
}

func (self *Service) GetProcess() (*db.Process, error) {
	return self.ProcessMap.GetProcess(self.Id)
}

func (service *Service) GetSignals() (chan os.Signal, chan os.Signal) {
	hupChan := make(chan os.Signal, 1)
	termChan := make(chan os.Signal, 1)
	signal.Notify(hupChan, syscall.SIGHUP)
	signal.Notify(termChan, syscall.SIGINT, syscall.SIGTERM)
	return termChan, hupChan
}

func (service *Service) Run() {
	log.Println("Running service...")
	go service.TopologyMapper.Run()
	go service.ProcessMapper.Run()
	go service.WebService.Run()
	go service.Transport.Run()
	go service.Dispatch.Run()
	go service.Executor.Run()
	go service.Hold.Run()

	sigterm, sighup := service.GetSignals()
	for {
		select {
		case <-sighup:
			log.Println("SIGHUP! Reloading configuration...")
			// TODO: reload configuration
		case <-sigterm:
			log.Println("SIGTERM! Cleaning up...")
			service.Index.Shutdown()
			service.Stop()
			return
		}
	}
}

type Message interface {
	Handle(*Service)
}
<commit_msg>added dump version on startup<commit_after>package core

import (
	"fmt"
	"log"
	"os"
	"os/signal"
	"pilosa/config"
	"pilosa/db"
	"pilosa/hold"
	"pilosa/index"
	"pilosa/interfaces"
	"syscall"

	"github.com/coreos/go-etcd/etcd"
	"github.com/davecgh/go-spew/spew"
	"tux21b.org/v1/gocql/uuid"
)

type Service struct {
	Stopper
	Id             *uuid.UUID
	Etcd           *etcd.Client
	Cluster        *db.Cluster
	TopologyMapper *TopologyMapper
	ProcessMapper  *ProcessMapper
	ProcessMap     *ProcessMap
	Transport      interfaces.Transporter
	Dispatch       interfaces.Dispatcher
	Executor       interfaces.Executorer
	WebService     *WebService
	Index          *index.FragmentContainer
	Hold           *hold.Holder
	version        string
	name           string
}

func NewService() *Service {
	log.Println(spew.Sdump("NewService"))
	service := new(Service)
	service.init_id()
	service.Etcd = etcd.NewClient(nil)
	service.Cluster = db.NewCluster()
	service.TopologyMapper = NewTopologyMapper(service, "/pilosa/0")
	service.ProcessMapper = NewProcessMapper(service, "/pilosa/0")
	service.ProcessMap = NewProcessMap()
	service.WebService = NewWebService(service)
	service.Index = index.NewFragmentContainer()
	service.Hold = hold.NewHolder()
	service.version = "0.0.9"
	service.name = "Cruncher"
	service.PrepareLogging()
	return service
}

func (self *Service) PrepareLogging() {
	base_path := config.GetString("log_path")
	if base_path == "" {
		base_path = "/tmp"
	}
	f, err := os.OpenFile(fmt.Sprintf("%s/%s.%s", base_path, self.name, self.Id), os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
	if err != nil {
		log.Println("error opening file: %v", err)
	}
	//defer f.Close()
	log.SetOutput(f)
}

func (service *Service) init_id() {
	var id uuid.UUID
	var err error
	id_string := config.GetString("id")
	if id_string == "" {
		log.Println("Service id not configured, generating...")
		id = uuid.RandomUUID()
		if err != nil {
			log.Fatal("problem generating uuid")
		}
	} else {
		id, err = uuid.ParseUUID(id_string)
		if err != nil {
			log.Fatalf("Service id '%s' not valid", id_string)
		}
	}
	service.Id = &id
}

func (self *Service) GetProcess() (*db.Process, error) {
	return self.ProcessMap.GetProcess(self.Id)
}

func (service *Service) GetSignals() (chan os.Signal, chan os.Signal) {
	hupChan := make(chan os.Signal, 1)
	termChan := make(chan os.Signal, 1)
	signal.Notify(hupChan, syscall.SIGHUP)
	signal.Notify(termChan, syscall.SIGINT, syscall.SIGTERM)
	return termChan, hupChan
}

func (service *Service) Run() {
	log.Println("Running service...", service.version)
	go service.TopologyMapper.Run()
	go service.ProcessMapper.Run()
	go service.WebService.Run()
	go service.Transport.Run()
	go service.Dispatch.Run()
	go service.Executor.Run()
	go service.Hold.Run()

	sigterm, sighup := service.GetSignals()
	for {
		select {
		case <-sighup:
			log.Println("SIGHUP! Reloading configuration...")
			// TODO: reload configuration
		case <-sigterm:
			log.Println("SIGTERM! Cleaning up...")
			service.Index.Shutdown()
			service.Stop()
			return
		}
	}
}

type Message interface {
	Handle(*Service)
}
<|endoftext|>


<commit_before>package zip

import (
	"archive/zip"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"strings"
)

func Unzip(archive, target string, excludedFiles []string) error {
	reader, err := zip.OpenReader(archive)
	if err != nil {
		return err
	}

	defer reader.Close()

	if err := os.MkdirAll(target, 0777); err != nil {
		return err
	}

	for _, file := range reader.File {

		excludeFile := false
		for _, excludedFile := range excludedFiles {
			if excludedFile == file.Name {
				excludeFile = true
				break
			}
		}

		if excludeFile == true {
			fmt.Println("Excluding File:  " + file.Name)
			continue
		}

		path := filepath.Join(target, file.Name)
		if file.FileInfo().IsDir() {
			fmt.Println("Creating Directory:  " + file.Name)
			os.MkdirAll(path, file.Mode())
			continue
		}

		fileReader, err := file.Open()
		if err != nil {
			if fileReader != nil {
                fileReader.Close()
            }
			return err
		}

		targetFile, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, file.Mode())
		if err != nil {
			if fileReader != nil {
                fileReader.Close()
            }
			return err
		}

		fmt.Println("Extracting File:  " + file.Name)
        if _, err := io.Copy(targetFile, fileReader); err != nil {
			fileReader.Close()
			targetFile.Close()
			return err
		}
		fileReader.Close()
		targetFile.Close()
	}

	return nil
}

func Zipit(source, target string) error {
	zipfile, err := os.Create(target)
	if err != nil {
		return err
	}
	defer zipfile.Close()

	archive := zip.NewWriter(zipfile)
	defer archive.Close()

	info, err := os.Stat(source)
	if err != nil {
		return nil
	}

	var baseDir string
	if info.IsDir() {
		baseDir = filepath.Base(source)
	}

	filepath.Walk(source, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}

		header, err := zip.FileInfoHeader(info)
		if err != nil {
			return err
		}

		if baseDir != "" {
			header.Name = filepath.Join(baseDir, strings.TrimPrefix(path, source))
		}

		if info.IsDir() {
			header.Name += "/"
		} else {
			header.Method = zip.Deflate
		}

		writer, err := archive.CreateHeader(header)
		if err != nil {
			return err
		}

		if info.IsDir() {
			return nil
		}

		file, err := os.Open(path)
		if err != nil {
			return err
		}
		defer file.Close()
		_, err = io.Copy(writer, file)
		return err
	})

	return err
}
<commit_msg>Formatting<commit_after>package zip

import (
	"archive/zip"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"strings"
)

func Unzip(archive, target string, excludedFiles []string) error {
	reader, err := zip.OpenReader(archive)
	if err != nil {
		return err
	}

	defer reader.Close()

	if err := os.MkdirAll(target, 0777); err != nil {
		return err
	}

	for _, file := range reader.File {

		excludeFile := false
		for _, excludedFile := range excludedFiles {
			if excludedFile == file.Name {
				excludeFile = true
				break
			}
		}

		if excludeFile == true {
			fmt.Println("Excluding File:  " + file.Name)
			continue
		}

		path := filepath.Join(target, file.Name)
		if file.FileInfo().IsDir() {
			fmt.Println("Creating Directory:  " + file.Name)
			os.MkdirAll(path, file.Mode())
			continue
		}

		fileReader, err := file.Open()
		if err != nil {
			if fileReader != nil {
				fileReader.Close()
			}
			return err
		}

		targetFile, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, file.Mode())
		if err != nil {
			if fileReader != nil {
				fileReader.Close()
			}
			return err
		}

		fmt.Println("Extracting File:  " + file.Name)
		if _, err := io.Copy(targetFile, fileReader); err != nil {
			fileReader.Close()
			targetFile.Close()
			return err
		}
		fileReader.Close()
		targetFile.Close()
	}

	return nil
}

func Zipit(source, target string) error {
	zipfile, err := os.Create(target)
	if err != nil {
		return err
	}
	defer zipfile.Close()

	archive := zip.NewWriter(zipfile)
	defer archive.Close()

	info, err := os.Stat(source)
	if err != nil {
		return nil
	}

	var baseDir string
	if info.IsDir() {
		baseDir = filepath.Base(source)
	}

	filepath.Walk(source, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}

		header, err := zip.FileInfoHeader(info)
		if err != nil {
			return err
		}

		if baseDir != "" {
			header.Name = filepath.Join(baseDir, strings.TrimPrefix(path, source))
		}

		if info.IsDir() {
			header.Name += "/"
		} else {
			header.Method = zip.Deflate
		}

		writer, err := archive.CreateHeader(header)
		if err != nil {
			return err
		}

		if info.IsDir() {
			return nil
		}

		file, err := os.Open(path)
		if err != nil {
			return err
		}
		defer file.Close()
		_, err = io.Copy(writer, file)
		return err
	})

	return err
}
<|endoftext|>


<commit_before>// Package coremain contains the functions for starting CoreDNS.
package coremain

import (
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"runtime"
	"strconv"
	"strings"

	"github.com/mholt/caddy"

	"github.com/coredns/coredns/core/dnsserver"

	// Plug in CoreDNS
	_ "github.com/coredns/coredns/core"
)

func init() {
	// Reset flag.CommandLine to get rid of unwanted flags for instance from glog (used in kubernetes).
	// And readd the once we want to keep.
	flag.VisitAll(func(f *flag.Flag) {
		if _, ok := flagsBlacklist[f.Name]; ok {
			return
		}
		flagsToKeep = append(flagsToKeep, f)
	})

	flag.CommandLine = flag.NewFlagSet(os.Args[0], flag.ExitOnError)
	for _, f := range flagsToKeep {
		flag.Var(f.Value, f.Name, f.Usage)
	}

	caddy.TrapSignals()
	caddy.DefaultConfigFile = "Corefile"
	caddy.Quiet = true // don't show init stuff from caddy
	setVersion()

	flag.StringVar(&conf, "conf", "", "Corefile to load (default \""+caddy.DefaultConfigFile+"\")")
	flag.StringVar(&cpu, "cpu", "100%", "CPU cap")
	flag.BoolVar(&plugins, "plugins", false, "List installed plugins")
	flag.StringVar(&caddy.PidFile, "pidfile", "", "Path to write pid file")
	flag.BoolVar(&version, "version", false, "Show version")
	flag.BoolVar(&dnsserver.Quiet, "quiet", false, "Quiet mode (no initialization output)")
	flag.BoolVar(&logfile, "log", false, "Log to standard output")

	caddy.RegisterCaddyfileLoader("flag", caddy.LoaderFunc(confLoader))
	caddy.SetDefaultCaddyfileLoader("default", caddy.LoaderFunc(defaultLoader))

	caddy.AppName = coreName
	caddy.AppVersion = coreVersion
}

// Run is CoreDNS's main() function.
func Run() {

	flag.Parse()

	// Set up process log before anything bad happens
	if logfile {
		log.SetOutput(os.Stdout)
	}
	log.SetFlags(log.LstdFlags)

	if version {
		showVersion()
		os.Exit(0)
	}
	if plugins {
		fmt.Println(caddy.DescribePlugins())
		os.Exit(0)
	}

	// Set CPU cap
	if err := setCPU(cpu); err != nil {
		mustLogFatal(err)
	}

	// Get Corefile input
	corefile, err := caddy.LoadCaddyfile(serverType)
	if err != nil {
		mustLogFatal(err)
	}

	// Start your engines
	instance, err := caddy.Start(corefile)
	if err != nil {
		mustLogFatal(err)
	}

	logVersion()
	if !dnsserver.Quiet {
		showVersion()
	}

	// Twiddle your thumbs
	instance.Wait()
}

// mustLogFatal wraps log.Fatal() in a way that ensures the
// output is always printed to stderr so the user can see it
// if the user is still there, even if the process log was not
// enabled. If this process is an upgrade, however, and the user
// might not be there anymore, this just logs to the process
// log and exits.
func mustLogFatal(args ...interface{}) {
	if !caddy.IsUpgrade() {
		log.SetOutput(os.Stderr)
	}
	log.Fatal(args...)
}

// confLoader loads the Caddyfile using the -conf flag.
func confLoader(serverType string) (caddy.Input, error) {
	if conf == "" {
		return nil, nil
	}

	if conf == "stdin" {
		return caddy.CaddyfileFromPipe(os.Stdin, "dns")
	}

	contents, err := ioutil.ReadFile(conf)
	if err != nil {
		return nil, err
	}
	return caddy.CaddyfileInput{
		Contents:       contents,
		Filepath:       conf,
		ServerTypeName: serverType,
	}, nil
}

// defaultLoader loads the Corefile from the current working directory.
func defaultLoader(serverType string) (caddy.Input, error) {
	contents, err := ioutil.ReadFile(caddy.DefaultConfigFile)
	if err != nil {
		if os.IsNotExist(err) {
			return nil, nil
		}
		return nil, err
	}
	return caddy.CaddyfileInput{
		Contents:       contents,
		Filepath:       caddy.DefaultConfigFile,
		ServerTypeName: serverType,
	}, nil
}

// logVersion logs the version that is starting.
func logVersion() { log.Print("[INFO] " + versionString()) }

// showVersion prints the version that is starting.
func showVersion() {
	fmt.Print(versionString())
	if devBuild && gitShortStat != "" {
		fmt.Printf("%s\n%s\n", gitShortStat, gitFilesModified)
	}
}

// versionString returns the CoreDNS version as a string.
func versionString() string {
	return fmt.Sprintf("%s-%s\n", caddy.AppName, caddy.AppVersion)
}

// setVersion figures out the version information
// based on variables set by -ldflags.
func setVersion() {
	// A development build is one that's not at a tag or has uncommitted changes
	devBuild = gitTag == "" || gitShortStat != ""

	// Only set the appVersion if -ldflags was used
	if gitNearestTag != "" || gitTag != "" {
		if devBuild && gitNearestTag != "" {
			appVersion = fmt.Sprintf("%s (+%s %s)",
				strings.TrimPrefix(gitNearestTag, "v"), gitCommit, buildDate)
		} else if gitTag != "" {
			appVersion = strings.TrimPrefix(gitTag, "v")
		}
	}
}

// setCPU parses string cpu and sets GOMAXPROCS
// according to its value. It accepts either
// a number (e.g. 3) or a percent (e.g. 50%).
func setCPU(cpu string) error {
	var numCPU int

	availCPU := runtime.NumCPU()

	if strings.HasSuffix(cpu, "%") {
		// Percent
		var percent float32
		pctStr := cpu[:len(cpu)-1]
		pctInt, err := strconv.Atoi(pctStr)
		if err != nil || pctInt < 1 || pctInt > 100 {
			return errors.New("invalid CPU value: percentage must be between 1-100")
		}
		percent = float32(pctInt) / 100
		numCPU = int(float32(availCPU) * percent)
	} else {
		// Number
		num, err := strconv.Atoi(cpu)
		if err != nil || num < 1 {
			return errors.New("invalid CPU value: provide a number or percent greater than 0")
		}
		numCPU = num
	}

	if numCPU > availCPU {
		numCPU = availCPU
	}

	runtime.GOMAXPROCS(numCPU)
	return nil
}

// Flags that control program flow or startup
var (
	conf    string
	cpu     string
	logfile bool
	version bool
	plugins bool
)

// Build information obtained with the help of -ldflags
var (
	appVersion = "(untracked dev build)" // inferred at startup
	devBuild   = true                    // inferred at startup

	buildDate        string // date -u
	gitTag           string // git describe --exact-match HEAD 2> /dev/null
	gitNearestTag    string // git describe --abbrev=0 --tags HEAD
	gitCommit        string // git rev-parse HEAD
	gitShortStat     string // git diff-index --shortstat
	gitFilesModified string // git diff-index --name-only HEAD
)

// flagsBlacklist removes flags with these names from our flagset.
var flagsBlacklist = map[string]bool{
	"logtostderr":      true,
	"alsologtostderr":  true,
	"v":                true,
	"stderrthreshold":  true,
	"vmodule":          true,
	"log_backtrace_at": true,
	"log_dir":          true,
}

var flagsToKeep []*flag.Flag
<commit_msg>core: errors on junk in the command line (#744)<commit_after>// Package coremain contains the functions for starting CoreDNS.
package coremain

import (
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"runtime"
	"strconv"
	"strings"

	"github.com/mholt/caddy"

	"github.com/coredns/coredns/core/dnsserver"

	// Plug in CoreDNS
	_ "github.com/coredns/coredns/core"
)

func init() {
	// Reset flag.CommandLine to get rid of unwanted flags for instance from glog (used in kubernetes).
	// And readd the once we want to keep.
	flag.VisitAll(func(f *flag.Flag) {
		if _, ok := flagsBlacklist[f.Name]; ok {
			return
		}
		flagsToKeep = append(flagsToKeep, f)
	})

	flag.CommandLine = flag.NewFlagSet(os.Args[0], flag.ExitOnError)
	for _, f := range flagsToKeep {
		flag.Var(f.Value, f.Name, f.Usage)
	}

	caddy.TrapSignals()
	caddy.DefaultConfigFile = "Corefile"
	caddy.Quiet = true // don't show init stuff from caddy
	setVersion()

	flag.StringVar(&conf, "conf", "", "Corefile to load (default \""+caddy.DefaultConfigFile+"\")")
	flag.StringVar(&cpu, "cpu", "100%", "CPU cap")
	flag.BoolVar(&plugins, "plugins", false, "List installed plugins")
	flag.StringVar(&caddy.PidFile, "pidfile", "", "Path to write pid file")
	flag.BoolVar(&version, "version", false, "Show version")
	flag.BoolVar(&dnsserver.Quiet, "quiet", false, "Quiet mode (no initialization output)")
	flag.BoolVar(&logfile, "log", false, "Log to standard output")

	caddy.RegisterCaddyfileLoader("flag", caddy.LoaderFunc(confLoader))
	caddy.SetDefaultCaddyfileLoader("default", caddy.LoaderFunc(defaultLoader))

	caddy.AppName = coreName
	caddy.AppVersion = coreVersion
}

// Run is CoreDNS's main() function.
func Run() {

	flag.Parse()

	if len(flag.Args()) > 0 {
		mustLogFatal(fmt.Errorf("extra command line arguments: %s", flag.Args()))
	}

	// Set up process log before anything bad happens
	if logfile {
		log.SetOutput(os.Stdout)
	}
	log.SetFlags(log.LstdFlags)

	if version {
		showVersion()
		os.Exit(0)
	}
	if plugins {
		fmt.Println(caddy.DescribePlugins())
		os.Exit(0)
	}

	// Set CPU cap
	if err := setCPU(cpu); err != nil {
		mustLogFatal(err)
	}

	// Get Corefile input
	corefile, err := caddy.LoadCaddyfile(serverType)
	if err != nil {
		mustLogFatal(err)
	}

	// Start your engines
	instance, err := caddy.Start(corefile)
	if err != nil {
		mustLogFatal(err)
	}

	logVersion()
	if !dnsserver.Quiet {
		showVersion()
	}

	// Twiddle your thumbs
	instance.Wait()
}

// mustLogFatal wraps log.Fatal() in a way that ensures the
// output is always printed to stderr so the user can see it
// if the user is still there, even if the process log was not
// enabled. If this process is an upgrade, however, and the user
// might not be there anymore, this just logs to the process
// log and exits.
func mustLogFatal(args ...interface{}) {
	if !caddy.IsUpgrade() {
		log.SetOutput(os.Stderr)
	}
	log.Fatal(args...)
}

// confLoader loads the Caddyfile using the -conf flag.
func confLoader(serverType string) (caddy.Input, error) {
	if conf == "" {
		return nil, nil
	}

	if conf == "stdin" {
		return caddy.CaddyfileFromPipe(os.Stdin, "dns")
	}

	contents, err := ioutil.ReadFile(conf)
	if err != nil {
		return nil, err
	}
	return caddy.CaddyfileInput{
		Contents:       contents,
		Filepath:       conf,
		ServerTypeName: serverType,
	}, nil
}

// defaultLoader loads the Corefile from the current working directory.
func defaultLoader(serverType string) (caddy.Input, error) {
	contents, err := ioutil.ReadFile(caddy.DefaultConfigFile)
	if err != nil {
		if os.IsNotExist(err) {
			return nil, nil
		}
		return nil, err
	}
	return caddy.CaddyfileInput{
		Contents:       contents,
		Filepath:       caddy.DefaultConfigFile,
		ServerTypeName: serverType,
	}, nil
}

// logVersion logs the version that is starting.
func logVersion() { log.Print("[INFO] " + versionString()) }

// showVersion prints the version that is starting.
func showVersion() {
	fmt.Print(versionString())
	if devBuild && gitShortStat != "" {
		fmt.Printf("%s\n%s\n", gitShortStat, gitFilesModified)
	}
}

// versionString returns the CoreDNS version as a string.
func versionString() string {
	return fmt.Sprintf("%s-%s\n", caddy.AppName, caddy.AppVersion)
}

// setVersion figures out the version information
// based on variables set by -ldflags.
func setVersion() {
	// A development build is one that's not at a tag or has uncommitted changes
	devBuild = gitTag == "" || gitShortStat != ""

	// Only set the appVersion if -ldflags was used
	if gitNearestTag != "" || gitTag != "" {
		if devBuild && gitNearestTag != "" {
			appVersion = fmt.Sprintf("%s (+%s %s)",
				strings.TrimPrefix(gitNearestTag, "v"), gitCommit, buildDate)
		} else if gitTag != "" {
			appVersion = strings.TrimPrefix(gitTag, "v")
		}
	}
}

// setCPU parses string cpu and sets GOMAXPROCS
// according to its value. It accepts either
// a number (e.g. 3) or a percent (e.g. 50%).
func setCPU(cpu string) error {
	var numCPU int

	availCPU := runtime.NumCPU()

	if strings.HasSuffix(cpu, "%") {
		// Percent
		var percent float32
		pctStr := cpu[:len(cpu)-1]
		pctInt, err := strconv.Atoi(pctStr)
		if err != nil || pctInt < 1 || pctInt > 100 {
			return errors.New("invalid CPU value: percentage must be between 1-100")
		}
		percent = float32(pctInt) / 100
		numCPU = int(float32(availCPU) * percent)
	} else {
		// Number
		num, err := strconv.Atoi(cpu)
		if err != nil || num < 1 {
			return errors.New("invalid CPU value: provide a number or percent greater than 0")
		}
		numCPU = num
	}

	if numCPU > availCPU {
		numCPU = availCPU
	}

	runtime.GOMAXPROCS(numCPU)
	return nil
}

// Flags that control program flow or startup
var (
	conf    string
	cpu     string
	logfile bool
	version bool
	plugins bool
)

// Build information obtained with the help of -ldflags
var (
	appVersion = "(untracked dev build)" // inferred at startup
	devBuild   = true                    // inferred at startup

	buildDate        string // date -u
	gitTag           string // git describe --exact-match HEAD 2> /dev/null
	gitNearestTag    string // git describe --abbrev=0 --tags HEAD
	gitCommit        string // git rev-parse HEAD
	gitShortStat     string // git diff-index --shortstat
	gitFilesModified string // git diff-index --name-only HEAD
)

// flagsBlacklist removes flags with these names from our flagset.
var flagsBlacklist = map[string]bool{
	"logtostderr":      true,
	"alsologtostderr":  true,
	"v":                true,
	"stderrthreshold":  true,
	"vmodule":          true,
	"log_backtrace_at": true,
	"log_dir":          true,
}

var flagsToKeep []*flag.Flag
<|endoftext|>


<commit_before>package brdoc

import . "testing"

const (
	checkMark = "\u2713"
	ballotX   = "\u2717"
)

func TestIsCPF(t *T) {
	for i, v := range []struct {
		v string
		r bool
	}{
		// Invalid format.
		{"3467875434578764345789654", false},
		{"", false},
		{"#$%¨&*(ABCDEF", false},

		// Common invalid patterns.
		{"000.000.000-00", false},
		{"111.111.111-11", false},
		{"222.222.222-22", false},
		{"333.333.333-33", false},
		{"444.444.444-44", false},
		{"555.555.555-55", false},
		{"666.666.666-66", false},
		{"777.777.777-77", false},
		{"888.888.888-88", false},
		{"999.999.999-99", false},

		// Invalid digits.
		{"248.438.034-08", false},
		{"099.075.865-06", false},

		// Valid.
		{"248.438.034-80", true},
		{"099.075.865-60", true},
	} {
		t.Logf("#%d CPF validation of %s should return %v: ", i, v.v, v.r)
		if IsCPF(v.v) != v.r {
			t.Fatal(ballotX)
		}
		t.Log(checkMark)
	}
}

func TestIsCNPJ(t *T) {
	for i, v := range []struct {
		v string
		r bool
	}{
		// Invalid format.
		{"3467875434578764345789654", false},
		{"", false},
		{"#$%¨&*(ABCDEF", false},

		// Common invalid patterns.
		{"00.000.000/0000-00", false},
		{"11.111.111/1111-11", false},
		{"22.222.222/2222-22", false},
		{"33.333.333/3333-33", false},
		{"44.444.444/4444-44", false},
		{"55.555.555/5555-55", false},
		{"66.666.666/6666-66", false},
		{"77.777.777/7777-77", false},
		{"88.888.888/8888-88", false},
		{"99.999.999/9999-99", false},

		// Invalid digits.
		{"26.637.142/0001-85", false},
		{"74.221.325/0001-03", false},

		// Valid.
		{"26.637.142/0001-58", true},
		{"74.221.325/0001-30", true},
	} {
		t.Logf("#%d CNPJ validation of %s should return %v: ", i, v.v, v.r)
		if IsCNPJ(v.v) != v.r {
			t.Fatal(ballotX)
		}
		t.Log(checkMark)
	}
}
<commit_msg>Organize tests.<commit_after>package brdoc

import "testing"

const (
	checkMark = "\u2713"
	ballotX   = "\u2717"
)

func TestIsCPF(t *testing.T) {
	for i, v := range []struct {
		v string
		r bool
	}{
		// Invalid format.
		{"3467875434578764345789654", false},
		{"", false},
		{"#$%¨&*(ABCDEF", false},

		// Common invalid patterns.
		{"000.000.000-00", false},
		{"111.111.111-11", false},
		{"222.222.222-22", false},
		{"333.333.333-33", false},
		{"444.444.444-44", false},
		{"555.555.555-55", false},
		{"666.666.666-66", false},
		{"777.777.777-77", false},
		{"888.888.888-88", false},
		{"999.999.999-99", false},

		// Invalid digits.
		{"248.438.034-08", false},
		{"099.075.865-06", false},

		// Valid.
		{"248.438.034-80", true},
		{"099.075.865-60", true},
	} {
		t.Logf("#%d CPF validation of %s should return %v: ", i, v.v, v.r)
		if IsCPF(v.v) != v.r {
			t.Fatal(ballotX)
		}
		t.Log(checkMark)
	}
}

func TestIsCNPJ(t *testing.T) {
	for i, v := range []struct {
		v string
		r bool
	}{
		// Invalid format.
		{"3467875434578764345789654", false},
		{"", false},
		{"#$%¨&*(ABCDEF", false},

		// Common invalid patterns.
		{"00.000.000/0000-00", false},
		{"11.111.111/1111-11", false},
		{"22.222.222/2222-22", false},
		{"33.333.333/3333-33", false},
		{"44.444.444/4444-44", false},
		{"55.555.555/5555-55", false},
		{"66.666.666/6666-66", false},
		{"77.777.777/7777-77", false},
		{"88.888.888/8888-88", false},
		{"99.999.999/9999-99", false},

		// Invalid digits.
		{"26.637.142/0001-85", false},
		{"74.221.325/0001-03", false},

		// Valid.
		{"26.637.142/0001-58", true},
		{"74.221.325/0001-30", true},
	} {
		t.Logf("#%d CNPJ validation of %s should return %v: ", i, v.v, v.r)
		if IsCNPJ(v.v) != v.r {
			t.Fatal(ballotX)
		}
		t.Log(checkMark)
	}
}
<|endoftext|>


<commit_before>package crawl

import (
	"fmt"
	"log"
	"os"
	"regexp"

	"code.google.com/p/goauth2/oauth"
	"github.com/google/go-github/github"

	"github.com/xogeny/impact/parsing"
	"github.com/xogeny/impact/recorder"
)

type GitHubCrawler struct {
	token   string
	pattern string
	re      *regexp.Regexp
	user    string
}

var exclusionList []string

func init() {
	exclusionList = []string{
		"modelica-3rdparty:BrineProp:0.1.9",      // Directory structure is a mess
		"modelica-3rdparty:BondLib:2.3",          // Too large
		"modelica-3rdparty:DESLib:1.6.1",         // Too large
		"modelica-3rdparty:FCSys:0.2.3",          // Tag dir mismatch
		"modelica-3rdparty:FCSys:0.2.2",          // Tag dir mismatch
		"modelica-3rdparty:FCSys:0.2.1",          // Tag dir mismatch
		"modelica-3rdparty:FCSys:0.2.0",          // Tag dir mismatch
		"modelica-3rdparty:FCSys:0.1.2",          // Tag dir mismatch
		"modelica-3rdparty:FCSys:0.1.1",          // Tag dir mismatch
		"modelica-3rdparty:FCSys:0.1.0",          // Tag dir mismatch
		"modelica-3rdparty:HelmholtzMedia:0.9.1", // Unsupported directory structure
		"modelica-3rdparty:HelmholtzMedia:0.9.0", // Unsupported directory structure
		"modelica-3rdparty:HelmholtzMedia:0.8.4", // Unsupported directory structure
		"modelica-3rdparty:HelmholtzMedia:0.8.2", // Unsupported directory structure
		"modelica-3rdparty:HelmholtzMedia:0.8.1", // Unsupported directory structure
		"modelica-3rdparty:HelmholtzMedia:0.8",   // Unsupported directory structure
		"modelica-3rdparty:HelmholtzMedia:0.7.1", // Unsupported directory structure
		"modelica-3rdparty:HelmholtzMedia:0.7",   // Unsupported directory structure
		"modelica-3rdparty:HelmholtzMedia:0.6",   // Unsupported directory structure
		"modelica-3rdparty:HelmholtzMedia:0.6.1", // Unsupported directory structure
		"modelica-3rdparty:HelmholtzMedia:0.5",   // Unsupported directory structure
		"modelica-3rdparty:HelmholtzMedia:0.4",   // Unsupported directory structure
		"modelica-3rdparty:HelmholtzMedia:0.3",   // Unsupported directory structure
		"modelica-3rdparty:HelmholtzMedia:0.2",   // Unsupported directory structure
		"modelica-3rdparty:HelmholtzMedia:0.1",   // Unsupported directory structure
		"modelica-3rdparty:LinearMPC:0.1",        // Tag dir mismatch
		"modelica-3rdparty:ModelicaDEVS:1.0",     // Self reference (and invalid at that)
		"modelica-3rdparty:MotorcycleLib:1.0",    // Too large
		"modelica-3rdparty:NCLib:0.82",           // Missing package.mo
		"modelica:Modelica_LinearSystems2:2.3.1", // Dir name error
	}
}

func exclude(user string, reponame string, tagname string) bool {
	str := fmt.Sprintf("%s:%s:%s", user, reponame, tagname)
	for _, ex := range exclusionList {
		//log.Printf("Comparing '%s' to '%s'", ex, str)
		if ex == str {
			return true
		}
	}
	return false
}

func (c GitHubCrawler) Crawl(r recorder.Recorder, verbose bool, logger *log.Logger) error {
	// Start with whatever token we were given when this crawler was created
	token := c.token

	// If a token wasn't provided with the crawler, look for a token
	// as an environment variable
	if c.token == "" {
		token = os.Getenv("GITHUB_TOKEN")
	}

	// Create a client assuming no authentication
	client := github.NewClient(nil)

	// If we have a token, re-initialize the client with
	// authentication
	if token != "" {
		tok := &oauth.Transport{
			Token: &oauth.Token{AccessToken: token},
		}
		client = github.NewClient(tok.Client())
	}

	lopts := github.RepositoryListOptions{}

	// Get a list of all repositories associated with the specified
	// organization
	repos, _, err := client.Repositories.List(c.user, &lopts)
	if err != nil {
		logger.Printf("Error listing repositories for %s: %v", c.user, err)
		return fmt.Errorf("Error listing repositories for %s: %v", c.user, err)
	}

	// Loop over all repos associated with the given owner
	for _, repo := range repos {
		rname := *repo.Name

		if !c.re.MatchString(rname) {
			if verbose {
				logger.Printf("Skipping: %s (%s), doesn't match pattern '%s'",
					rname, *repo.HTMLURL, c.pattern)
			}
			continue
		}

		if verbose {
			logger.Printf("Processing: %s (%s)", rname, *repo.HTMLURL)
		}

		// Get all the tags associated with this repository
		tags, _, err := client.Repositories.ListTags(c.user, rname, nil)
		if err != nil {
			logger.Printf("Error getting tags for repository %s/%s: %v",
				c.user, rname, err)
			continue
		}

		// Loop over the tags
		for _, tag := range tags {
			// Check if this has a semantic version
			name := *tag.Name
			if name[0] == 'v' {
				name = name[1:]
			}

			// Check for version we know are not supported
			if exclude(c.user, rname, name) {
				continue
			}

			v, verr := parsing.NormalizeVersion(name)
			if verr != nil {
				// If not, ignore it
				if verbose {
					logger.Printf("  %s: Ignoring", name)
				}
				continue
			}

			if verbose {
				logger.Printf("  %s: Recording", name)
			}

			// Formulate directory info (impact.json) for this version of this repository
			di := ExtractInfo(client, c.user, repo, tag, name, logger)

			if len(di.Libraries) == 0 {
				logger.Printf("    No Modelica libraries found in repository %s:%s",
					rname, name)
				continue
			}

			// Loop over all libraries present in this repository
			for _, lib := range di.Libraries {
				if verbose {
					logger.Printf("    Processing library %s @ %s", lib.Name, lib.Path)
				}
				libr := r.GetLibrary(di.Owner, lib.Name)

				if repo.Description != nil {
					libr.SetDescription(*repo.Description)
				}
				if repo.HTMLURL != nil {
					libr.SetHomepage(*repo.HTMLURL)
				}
				libr.SetStars(*repo.StargazersCount)
				if repo.Owner.Email != nil {
					libr.SetEmail(*repo.Owner.Email)
				}

				vr := libr.AddVersion(v)
				vr.SetHash(*tag.Commit.SHA)

				for _, dep := range lib.Dependencies {
					vr.AddDependency(dep.Name, dep.Version)
				}
			}
		}
	}
	return nil
}

func MakeGitHubCrawler(user string, pattern string, token string) (GitHubCrawler, error) {
	if pattern == "" {
		pattern = ".+"
	}

	re, err := regexp.Compile(pattern)
	if err != nil {
		return GitHubCrawler{}, err
	}

	return GitHubCrawler{
		token:   token,
		pattern: pattern,
		re:      re,
		user:    user,
	}, nil
}

var _ Crawler = (*GitHubCrawler)(nil)
<commit_msg>Refactored so that processing is a separate function<commit_after>package crawl

import (
	"fmt"
	"log"
	"os"
	"regexp"

	"code.google.com/p/goauth2/oauth"
	"github.com/google/go-github/github"

	"github.com/xogeny/impact/parsing"
	"github.com/xogeny/impact/recorder"
)

type GitHubCrawler struct {
	token   string
	pattern string
	re      *regexp.Regexp
	user    string
}

var exclusionList []string

func init() {
	exclusionList = []string{
		"modelica-3rdparty:BrineProp:0.1.9",      // Directory structure is a mess
		"modelica-3rdparty:BondLib:2.3",          // Too large
		"modelica-3rdparty:DESLib:1.6.1",         // Too large
		"modelica-3rdparty:FCSys:0.2.3",          // Tag dir mismatch
		"modelica-3rdparty:FCSys:0.2.2",          // Tag dir mismatch
		"modelica-3rdparty:FCSys:0.2.1",          // Tag dir mismatch
		"modelica-3rdparty:FCSys:0.2.0",          // Tag dir mismatch
		"modelica-3rdparty:FCSys:0.1.2",          // Tag dir mismatch
		"modelica-3rdparty:FCSys:0.1.1",          // Tag dir mismatch
		"modelica-3rdparty:FCSys:0.1.0",          // Tag dir mismatch
		"modelica-3rdparty:HelmholtzMedia:0.9.1", // Unsupported directory structure
		"modelica-3rdparty:HelmholtzMedia:0.9.0", // Unsupported directory structure
		"modelica-3rdparty:HelmholtzMedia:0.8.4", // Unsupported directory structure
		"modelica-3rdparty:HelmholtzMedia:0.8.2", // Unsupported directory structure
		"modelica-3rdparty:HelmholtzMedia:0.8.1", // Unsupported directory structure
		"modelica-3rdparty:HelmholtzMedia:0.8",   // Unsupported directory structure
		"modelica-3rdparty:HelmholtzMedia:0.7.1", // Unsupported directory structure
		"modelica-3rdparty:HelmholtzMedia:0.7",   // Unsupported directory structure
		"modelica-3rdparty:HelmholtzMedia:0.6",   // Unsupported directory structure
		"modelica-3rdparty:HelmholtzMedia:0.6.1", // Unsupported directory structure
		"modelica-3rdparty:HelmholtzMedia:0.5",   // Unsupported directory structure
		"modelica-3rdparty:HelmholtzMedia:0.4",   // Unsupported directory structure
		"modelica-3rdparty:HelmholtzMedia:0.3",   // Unsupported directory structure
		"modelica-3rdparty:HelmholtzMedia:0.2",   // Unsupported directory structure
		"modelica-3rdparty:HelmholtzMedia:0.1",   // Unsupported directory structure
		"modelica-3rdparty:LinearMPC:0.1",        // Tag dir mismatch
		"modelica-3rdparty:ModelicaDEVS:1.0",     // Self reference (and invalid at that)
		"modelica-3rdparty:MotorcycleLib:1.0",    // Too large
		"modelica-3rdparty:NCLib:0.82",           // Missing package.mo
		"modelica:Modelica_LinearSystems2:2.3.1", // Dir name error
	}
}

func exclude(user string, reponame string, tagname string) bool {
	str := fmt.Sprintf("%s:%s:%s", user, reponame, tagname)
	for _, ex := range exclusionList {
		//log.Printf("Comparing '%s' to '%s'", ex, str)
		if ex == str {
			return true
		}
	}
	return false
}

func (c GitHubCrawler) processVersion(client *github.Client, r recorder.Recorder,
	repo github.Repository, versionString string, sha string, verbose bool,
	logger *log.Logger) {

	rname := *repo.Name

	v, verr := parsing.NormalizeVersion(versionString)
	if verr != nil {
		// If not, ignore it
		if verbose {
			logger.Printf("  %s: Ignoring", versionString)
		}
		return
	}

	if verbose {
		logger.Printf("  %s: Recording", versionString)
	}

	// Formulate directory info (impact.json) for this version of this repository
	di := ExtractInfo(client, c.user, repo, sha, versionString, logger)

	if len(di.Libraries) == 0 {
		logger.Printf("    No Modelica libraries found in repository %s:%s",
			rname, versionString)
		return
	}

	// Loop over all libraries present in this repository
	for _, lib := range di.Libraries {
		if verbose {
			logger.Printf("    Processing library %s @ %s", lib.Name, lib.Path)
		}
		libr := r.GetLibrary(di.Owner, lib.Name)

		if repo.Description != nil {
			libr.SetDescription(*repo.Description)
		}
		if repo.HTMLURL != nil {
			libr.SetHomepage(*repo.HTMLURL)
		}
		libr.SetStars(*repo.StargazersCount)
		if repo.Owner.Email != nil {
			libr.SetEmail(*repo.Owner.Email)
		}

		vr := libr.AddVersion(v)
		vr.SetHash(sha)

		for _, dep := range lib.Dependencies {
			vr.AddDependency(dep.Name, dep.Version)
		}
	}
}

func (c GitHubCrawler) Crawl(r recorder.Recorder, verbose bool, logger *log.Logger) error {
	// Start with whatever token we were given when this crawler was created
	token := c.token

	// If a token wasn't provided with the crawler, look for a token
	// as an environment variable
	if c.token == "" {
		token = os.Getenv("GITHUB_TOKEN")
	}

	// Create a client assuming no authentication
	client := github.NewClient(nil)

	// If we have a token, re-initialize the client with
	// authentication
	if token != "" {
		tok := &oauth.Transport{
			Token: &oauth.Token{AccessToken: token},
		}
		client = github.NewClient(tok.Client())
	}

	lopts := github.RepositoryListOptions{}

	// Get a list of all repositories associated with the specified
	// organization
	repos, _, err := client.Repositories.List(c.user, &lopts)
	if err != nil {
		logger.Printf("Error listing repositories for %s: %v", c.user, err)
		return fmt.Errorf("Error listing repositories for %s: %v", c.user, err)
	}

	// Loop over all repos associated with the given owner
	for _, repo := range repos {
		rname := *repo.Name

		if !c.re.MatchString(rname) {
			if verbose {
				logger.Printf("Skipping: %s (%s), doesn't match pattern '%s'",
					rname, *repo.HTMLURL, c.pattern)
			}
			continue
		}

		if verbose {
			logger.Printf("Processing: %s (%s)", rname, *repo.HTMLURL)
		}

		// Get all the tags associated with this repository
		tags, _, err := client.Repositories.ListTags(c.user, rname, nil)
		if err != nil {
			logger.Printf("Error getting tags for repository %s/%s: %v",
				c.user, rname, err)
			continue
		}

		// Loop over the tags
		for _, tag := range tags {
			// Check if this has a semantic version
			versionString := *tag.Name
			sha := *tag.Commit.SHA

			if versionString[0] == 'v' {
				versionString = versionString[1:]
			}

			// Check for version we know are not supported
			if exclude(c.user, rname, versionString) {
				continue
			}

			c.processVersion(client, r, repo, versionString, sha, verbose, logger)
		}

		// TODO: Add HEAD of master to list?  But how?  What kind of semantic
		// version number should I associate with it?
	}
	return nil
}

func MakeGitHubCrawler(user string, pattern string, token string) (GitHubCrawler, error) {
	if pattern == "" {
		pattern = ".+"
	}

	re, err := regexp.Compile(pattern)
	if err != nil {
		return GitHubCrawler{}, err
	}

	return GitHubCrawler{
		token:   token,
		pattern: pattern,
		re:      re,
		user:    user,
	}, nil
}

var _ Crawler = (*GitHubCrawler)(nil)
<|endoftext|>


<commit_before>package sudoku

import (
	"math/rand"
)

//Searches for a solution to the puzzle as it currently exists without
//unfilling any cells. If one exists, it will fill in all cells to fit that
//solution and return true. If there are no solutions the grid will remain
//untouched and it will return false.
func (self *Grid) Solve() bool {
	//TODO: Optimization: we only need one, so we can bail as soon as we find a single one.
	solutions := self.Solutions()
	if len(solutions) == 0 {
		return false
	}
	self.Load(solutions[0].DataString())
	return true
}

//Returns the total number of solutions found in the grid. Does not mutate the grid.
func (self *Grid) NumSolutions() int {
	return len(self.Solutions())
}

//Returns true if the grid has at least one solution. Does not mutate the grid.
func (self *Grid) HasSolution() bool {
	//TODO: optimize this to bail as soon as we find a single solution.
	return len(self.nOrFewerSolutions(1)) > 0
}

func (self *Grid) HasMultipleSolutions() bool {
	return len(self.nOrFewerSolutions(2)) >= 2
}

//Returns a slice of grids that represent possible solutions if you were to solve forward this grid. The current grid is not modified.
//If there are no solutions forward from this location it will return a slice with len() 0.
func (self *Grid) Solutions() (solutions []*Grid) {
	return self.nOrFewerSolutions(0)
}

//The actual workhorse of solutions generating. 0 means "as many as you can find". It might return more than you asked for, if it already had more results than requested sitting around.
func (self *Grid) nOrFewerSolutions(max int) []*Grid {

	if self.cachedSolutions == nil || (max > 0 && len(self.cachedSolutions) < max) {
		stackDone := make(chan bool, 1)

		stack := NewChanSyncedStack(stackDone)

		stack.Insert(self.Copy())

		incomingSolutions := make(chan *Grid)

		var solutions []*Grid

		for i := 0; i < NUM_SOLVER_THREADS; i++ {
			go func() {
				//Sovler thread.
				for {
					grid, ok := <-stack.Output
					if !ok {
						return
					}
					result := grid.(*Grid).searchSolutions(stack)
					if result != nil {
						incomingSolutions <- result
					}
					stack.ItemDone()
				}
			}()
		}

	OuterLoop:
		for {
			select {
			case solution := <-incomingSolutions:
				//Add it to results
				solutions = append(solutions, solution)
				if len(solutions) >= max {
					stack.Dispose()
					break OuterLoop
				}
			case <-stackDone:
				//Well, that's as good as it's going to get.
				break
			}
		}

		self.cachedSolutions = solutions

	}

	return self.cachedSolutions

}

func (self *Grid) searchSolutions(stack *ChanSyncedStack) *Grid {
	//This will only be called by Solutions.
	//We will return ourselves if we are a solution, and if not we will return nil.
	//If there are any sub children, we will send them to counter before we're done.

	if self.Invalid() {
		return nil
	}

	self.fillSimpleCells()
	//Have any cells noticed they were invalid while solving forward?
	if self.cellsInvalid() {
		return nil
	}

	if self.Solved() {
		return self
	}

	//Well, looks like we're going to have to branch.
	rankedObject := self.queue.DefaultGetter().Get()
	if rankedObject == nil {
		panic("Queue didn't have any cells.")
	}

	cell, ok := rankedObject.(*Cell)
	if !ok {
		panic("We got back a non-cell from the grid's queue")
	}

	unshuffledPossibilities := cell.Possibilities()

	possibilities := make([]int, len(unshuffledPossibilities))

	for i, j := range rand.Perm(len(unshuffledPossibilities)) {
		possibilities[i] = unshuffledPossibilities[j]
	}

	for _, num := range possibilities {
		copy := self.Copy()
		copy.Cell(cell.Row, cell.Col).SetNumber(num)
		stack.Insert(copy)
	}

	return nil

}

//Fills in all of the cells it can without branching or doing any advanced
//techniques that require anything more than a single cell's possibles list.
func (self *Grid) fillSimpleCells() int {
	count := 0
	getter := self.queue.DefaultGetter()
	obj := getter.GetSmallerThan(2)
	for obj != nil && !self.cellsInvalid() {
		cell, ok := obj.(*Cell)
		if !ok {
			continue
		}
		cell.SetNumber(cell.implicitNumber())
		count++
		obj = getter.GetSmallerThan(2)
	}
	return count
}
<commit_msg>TESTS FAIL. Added another missing label for a break to break out of the infinite loop. Now we send ona  closed channel so tests fail.<commit_after>package sudoku

import (
	"math/rand"
)

//Searches for a solution to the puzzle as it currently exists without
//unfilling any cells. If one exists, it will fill in all cells to fit that
//solution and return true. If there are no solutions the grid will remain
//untouched and it will return false.
func (self *Grid) Solve() bool {
	//TODO: Optimization: we only need one, so we can bail as soon as we find a single one.
	solutions := self.Solutions()
	if len(solutions) == 0 {
		return false
	}
	self.Load(solutions[0].DataString())
	return true
}

//Returns the total number of solutions found in the grid. Does not mutate the grid.
func (self *Grid) NumSolutions() int {
	return len(self.Solutions())
}

//Returns true if the grid has at least one solution. Does not mutate the grid.
func (self *Grid) HasSolution() bool {
	//TODO: optimize this to bail as soon as we find a single solution.
	return len(self.nOrFewerSolutions(1)) > 0
}

func (self *Grid) HasMultipleSolutions() bool {
	return len(self.nOrFewerSolutions(2)) >= 2
}

//Returns a slice of grids that represent possible solutions if you were to solve forward this grid. The current grid is not modified.
//If there are no solutions forward from this location it will return a slice with len() 0.
func (self *Grid) Solutions() (solutions []*Grid) {
	return self.nOrFewerSolutions(0)
}

//The actual workhorse of solutions generating. 0 means "as many as you can find". It might return more than you asked for, if it already had more results than requested sitting around.
func (self *Grid) nOrFewerSolutions(max int) []*Grid {

	if self.cachedSolutions == nil || (max > 0 && len(self.cachedSolutions) < max) {
		stackDone := make(chan bool, 1)

		stack := NewChanSyncedStack(stackDone)

		stack.Insert(self.Copy())

		incomingSolutions := make(chan *Grid)

		var solutions []*Grid

		for i := 0; i < NUM_SOLVER_THREADS; i++ {
			go func() {
				//Sovler thread.
				for {
					grid, ok := <-stack.Output
					if !ok {
						return
					}
					result := grid.(*Grid).searchSolutions(stack)
					if result != nil {
						incomingSolutions <- result
					}
					stack.ItemDone()
				}
			}()
		}

	OuterLoop:
		for {
			select {
			case solution := <-incomingSolutions:
				//Add it to results
				solutions = append(solutions, solution)
				if len(solutions) >= max {
					stack.Dispose()
					break OuterLoop
				}
			case <-stackDone:
				//Well, that's as good as it's going to get.
				break OuterLoop
			}
		}

		self.cachedSolutions = solutions

	}

	return self.cachedSolutions

}

func (self *Grid) searchSolutions(stack *ChanSyncedStack) *Grid {
	//This will only be called by Solutions.
	//We will return ourselves if we are a solution, and if not we will return nil.
	//If there are any sub children, we will send them to counter before we're done.

	if self.Invalid() {
		return nil
	}

	self.fillSimpleCells()
	//Have any cells noticed they were invalid while solving forward?
	if self.cellsInvalid() {
		return nil
	}

	if self.Solved() {
		return self
	}

	//Well, looks like we're going to have to branch.
	rankedObject := self.queue.DefaultGetter().Get()
	if rankedObject == nil {
		panic("Queue didn't have any cells.")
	}

	cell, ok := rankedObject.(*Cell)
	if !ok {
		panic("We got back a non-cell from the grid's queue")
	}

	unshuffledPossibilities := cell.Possibilities()

	possibilities := make([]int, len(unshuffledPossibilities))

	for i, j := range rand.Perm(len(unshuffledPossibilities)) {
		possibilities[i] = unshuffledPossibilities[j]
	}

	for _, num := range possibilities {
		copy := self.Copy()
		copy.Cell(cell.Row, cell.Col).SetNumber(num)
		stack.Insert(copy)
	}

	return nil

}

//Fills in all of the cells it can without branching or doing any advanced
//techniques that require anything more than a single cell's possibles list.
func (self *Grid) fillSimpleCells() int {
	count := 0
	getter := self.queue.DefaultGetter()
	obj := getter.GetSmallerThan(2)
	for obj != nil && !self.cellsInvalid() {
		cell, ok := obj.(*Cell)
		if !ok {
			continue
		}
		cell.SetNumber(cell.implicitNumber())
		count++
		obj = getter.GetSmallerThan(2)
	}
	return count
}
<|endoftext|>


<commit_before>package target

import (
	"math"
	"os"
	"strconv"
)

// Target of keeping the number of items in a queue under a certain length
type QueueLengthTarget struct {
	length     int
	minLength  int
	lastLength int
	vel        []int
	velSamples int
	cumErr     int
	kP         float64
	kI         float64
	kD         float64
	startCount int
}

const queueLengthExceedingPercent float64 = 0.7
const queueAverageSamples int = 1

func NewQueueLengthTarget(length int) Target {
	// TODO!! Better ways to calculate these heuristics and/or pass them in
	kUStr := os.Getenv("F12_KU")
	tUStr := os.Getenv("F12_TU")
	kU, err := strconv.ParseFloat(kUStr, 64)
	if kUStr == "" || err != nil {
		kU = 0.5
	}

	tU, err := strconv.ParseFloat(tUStr, 64)
	if tUStr == "" || err != nil {
		tU = 40.0
	}

	var velSamples int
	velSamplesStr := os.Getenv("F12_VEL_SAMPLES")
	velSamples, err = strconv.Atoi(velSamplesStr)
	if err != nil || velSamples == 0 {
		velSamples = queueAverageSamples
	}

	// Ziegler-Nichols
	kP := 0.6 * kU
	kI := kP * 2.0 / tU
	kD := kP * tU / 8.0
	log.Debugf("[new ql] kU = %f, tU = %f", kU, tU)
	log.Debugf("[new ql] kP = %f, kI = %f, kD = %f", kP, kI, kD)

	return &QueueLengthTarget{
		length:     length,
		minLength:  int(float64(length) * queueLengthExceedingPercent),
		kP:         kP,
		kI:         kI,
		kD:         kD,
		vel:        make([]int, velSamples+1),
		velSamples: velSamples,
	}
}

func (t *QueueLengthTarget) Meeting(current int) bool {
	meeting := (current <= t.length)
	if !meeting {
		log.Debugf("[ql] not meeting: current %d target %d", current, t.length)
	}
	return meeting
}

func (t *QueueLengthTarget) Exceeding(current int) bool {
	exceeding := (current <= t.minLength)
	if exceeding {
		log.Debugf("[ql] exceeding: current %d target %d", current, t.length)
	}
	return exceeding
}

// Number of additional containers
func (t *QueueLengthTarget) Delta(currentLength int) (delta int) {
	var deltafloat float64
	var currErr int
	// If the queue is longer than the target we need a positive number of additional containers
	// We want magnitude of cumErr to decrease if we're within the acceptable range minLength - Length
	if currentLength > t.length {
		currErr = currentLength - t.length
		t.cumErr = t.cumErr + currErr
	} else if currentLength < t.minLength {
		currErr = currentLength - t.minLength
		t.cumErr = t.cumErr + currErr
	} else { // t.minLength <= currentLength <= t.length
		if t.cumErr < 0 {
			t.cumErr = t.cumErr + (t.length - currentLength)
		} else {
			t.cumErr = t.cumErr + (t.minLength - currentLength)
		}
	}

	var aveVel float64
	// Store the new value at the end of the array (this is one more than we need), but only average over the right number of samples
	t.vel[t.velSamples] = currentLength - t.lastLength
	for i := 0; i <= t.velSamples-1; i++ {
		t.vel[i] = t.vel[i+1]
		aveVel = aveVel + float64(t.vel[i])
	}

	aveVel = aveVel / float64(t.velSamples)

	// There is a point beyond which there is no point letting cumErr grow, because our max containers can't
	// necessarily keep up (and also a question of symmetry, since a queue length can't go below 0?)
	if t.cumErr > 10*t.length {
		t.cumErr = 10 * t.length
	}
	if t.cumErr < -10*t.length {
		t.cumErr = -10 * t.length
	}

	t.lastLength = currentLength

	// To start with, velocity isn't valid
	if t.startCount < t.velSamples {
		log.Debugf("[ql] err %d, cumErr %d", currErr, t.cumErr)
		log.Debugf("[ql] err * kp %f, cumErr * kI %f", t.kP*float64(currErr), t.kI*float64(t.cumErr))
		deltafloat = t.kP*float64(currErr) + t.kI*float64(t.cumErr)
		t.startCount = t.startCount + 1
	} else {
		log.Debugf("[ql] err %d, cumErr %d, vel %f", currErr, t.cumErr, aveVel)
		log.Debugf("[ql] err * kp %f, cumErr * kI %f, vel * kd %f", t.kP*float64(currErr), t.kI*float64(t.cumErr), t.kD*float64(aveVel))
		deltafloat = t.kP*float64(currErr) + t.kI*float64(t.cumErr) + t.kD*float64(aveVel)
	}

	log.Debugf("[ql] => deltaf %f", deltafloat)

	// Round to the nearest integer
	delta = int(math.Floor(deltafloat + 0.5))
	return
}
<commit_msg>Rename F12 env vars to MSS.<commit_after>package target

import (
	"math"
	"os"
	"strconv"
)

// Target of keeping the number of items in a queue under a certain length
type QueueLengthTarget struct {
	length     int
	minLength  int
	lastLength int
	vel        []int
	velSamples int
	cumErr     int
	kP         float64
	kI         float64
	kD         float64
	startCount int
}

const queueLengthExceedingPercent float64 = 0.7
const queueAverageSamples int = 1

func NewQueueLengthTarget(length int) Target {
	// TODO!! Better ways to calculate these heuristics and/or pass them in
	kUStr := os.Getenv("MSS_KU")
	tUStr := os.Getenv("MSS_TU")
	kU, err := strconv.ParseFloat(kUStr, 64)
	if kUStr == "" || err != nil {
		kU = 0.5
	}

	tU, err := strconv.ParseFloat(tUStr, 64)
	if tUStr == "" || err != nil {
		tU = 40.0
	}

	var velSamples int
	velSamplesStr := os.Getenv("MSS_VEL_SAMPLES")
	velSamples, err = strconv.Atoi(velSamplesStr)
	if err != nil || velSamples == 0 {
		velSamples = queueAverageSamples
	}

	// Ziegler-Nichols
	kP := 0.6 * kU
	kI := kP * 2.0 / tU
	kD := kP * tU / 8.0
	log.Debugf("[new ql] kU = %f, tU = %f", kU, tU)
	log.Debugf("[new ql] kP = %f, kI = %f, kD = %f", kP, kI, kD)

	return &QueueLengthTarget{
		length:     length,
		minLength:  int(float64(length) * queueLengthExceedingPercent),
		kP:         kP,
		kI:         kI,
		kD:         kD,
		vel:        make([]int, velSamples+1),
		velSamples: velSamples,
	}
}

func (t *QueueLengthTarget) Meeting(current int) bool {
	meeting := (current <= t.length)
	if !meeting {
		log.Debugf("[ql] not meeting: current %d target %d", current, t.length)
	}
	return meeting
}

func (t *QueueLengthTarget) Exceeding(current int) bool {
	exceeding := (current <= t.minLength)
	if exceeding {
		log.Debugf("[ql] exceeding: current %d target %d", current, t.length)
	}
	return exceeding
}

// Number of additional containers
func (t *QueueLengthTarget) Delta(currentLength int) (delta int) {
	var deltafloat float64
	var currErr int
	// If the queue is longer than the target we need a positive number of additional containers
	// We want magnitude of cumErr to decrease if we're within the acceptable range minLength - Length
	if currentLength > t.length {
		currErr = currentLength - t.length
		t.cumErr = t.cumErr + currErr
	} else if currentLength < t.minLength {
		currErr = currentLength - t.minLength
		t.cumErr = t.cumErr + currErr
	} else { // t.minLength <= currentLength <= t.length
		if t.cumErr < 0 {
			t.cumErr = t.cumErr + (t.length - currentLength)
		} else {
			t.cumErr = t.cumErr + (t.minLength - currentLength)
		}
	}

	var aveVel float64
	// Store the new value at the end of the array (this is one more than we need), but only average over the right number of samples
	t.vel[t.velSamples] = currentLength - t.lastLength
	for i := 0; i <= t.velSamples-1; i++ {
		t.vel[i] = t.vel[i+1]
		aveVel = aveVel + float64(t.vel[i])
	}

	aveVel = aveVel / float64(t.velSamples)

	// There is a point beyond which there is no point letting cumErr grow, because our max containers can't
	// necessarily keep up (and also a question of symmetry, since a queue length can't go below 0?)
	if t.cumErr > 10*t.length {
		t.cumErr = 10 * t.length
	}
	if t.cumErr < -10*t.length {
		t.cumErr = -10 * t.length
	}

	t.lastLength = currentLength

	// To start with, velocity isn't valid
	if t.startCount < t.velSamples {
		log.Debugf("[ql] err %d, cumErr %d", currErr, t.cumErr)
		log.Debugf("[ql] err * kp %f, cumErr * kI %f", t.kP*float64(currErr), t.kI*float64(t.cumErr))
		deltafloat = t.kP*float64(currErr) + t.kI*float64(t.cumErr)
		t.startCount = t.startCount + 1
	} else {
		log.Debugf("[ql] err %d, cumErr %d, vel %f", currErr, t.cumErr, aveVel)
		log.Debugf("[ql] err * kp %f, cumErr * kI %f, vel * kd %f", t.kP*float64(currErr), t.kI*float64(t.cumErr), t.kD*float64(aveVel))
		deltafloat = t.kP*float64(currErr) + t.kI*float64(t.cumErr) + t.kD*float64(aveVel)
	}

	log.Debugf("[ql] => deltaf %f", deltafloat)

	// Round to the nearest integer
	delta = int(math.Floor(deltafloat + 0.5))
	return
}
<|endoftext|>


<commit_before>// Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"). You may
// not use this file except in compliance with the License. A copy of the
// License is located at
//
//	http://aws.amazon.com/apache2.0/
//
// or in the "license" file accompanying this file. This file is distributed
// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
// express or implied. See the License for the specific language governing
// permissions and limitations under the License.

package engine

import (
	"fmt"
	"sort"
	"sync"
	"time"

	"github.com/aws/amazon-ecs-agent/agent/api"
	"github.com/aws/amazon-ecs-agent/agent/config"
	"github.com/aws/amazon-ecs-agent/agent/engine/dockerstate"
	"github.com/aws/amazon-ecs-agent/agent/engine/image"
	"github.com/aws/amazon-ecs-agent/agent/statemanager"
	"github.com/cihub/seelog"
	"golang.org/x/net/context"
)

const (
	imageNotFoundForDeletionError = "no such image"
)

// ImageManager is responsible for saving the Image states,
// adding and removing container references to ImageStates
type ImageManager interface {
	RecordContainerReference(container *api.Container) error
	RemoveContainerReferenceFromImageState(container *api.Container) error
	AddAllImageStates(imageStates []*image.ImageState)
	GetImageStateFromImageName(containerImageName string) *image.ImageState
	StartImageCleanupProcess(ctx context.Context)
	SetSaver(stateManager statemanager.Saver)
}

// dockerImageManager accounts all the images and their states in the instance.
// It also has the cleanup policy configuration.
type dockerImageManager struct {
	imageStates                      []*image.ImageState
	client                           DockerClient
	updateLock                       sync.RWMutex
	imageCleanupTicker               *time.Ticker
	state                            dockerstate.TaskEngineState
	saver                            statemanager.Saver
	imageStatesConsideredForDeletion map[string]*image.ImageState
	minimumAgeBeforeDeletion         time.Duration
	numImagesToDelete                int
	imageCleanupTimeInterval         time.Duration
}

// ImageStatesForDeletion is used for implementing the sort interface
type ImageStatesForDeletion []*image.ImageState

// NewImageManager returns a new ImageManager
func NewImageManager(cfg *config.Config, client DockerClient, state dockerstate.TaskEngineState) ImageManager {
	return &dockerImageManager{
		client: client,
		state:  state,
		minimumAgeBeforeDeletion: cfg.MinimumImageDeletionAge,
		numImagesToDelete:        cfg.NumImagesToDeletePerCycle,
		imageCleanupTimeInterval: cfg.ImageCleanupInterval,
	}
}

func (imageManager *dockerImageManager) SetSaver(stateManager statemanager.Saver) {
	imageManager.saver = stateManager
}

func (imageManager *dockerImageManager) AddAllImageStates(imageStates []*image.ImageState) {
	imageManager.updateLock.Lock()
	defer imageManager.updateLock.Unlock()
	for _, imageState := range imageStates {
		imageManager.addImageState(imageState)
	}
}

// RecordContainerReference adds container reference to the corresponding imageState object
func (imageManager *dockerImageManager) RecordContainerReference(container *api.Container) error {
	// the image state has been updated, save the new state
	defer imageManager.saver.ForceSave()
	// On agent restart, container ID was retrieved from agent state file
	// TODO add setter and getter for modifying this
	if container.ImageID != "" {
		if !imageManager.addContainerReferenceToExistingImageState(container) {
			return fmt.Errorf("Failed to add container to existing image state")
		}
		return nil
	}

	if container.Image == "" {
		return fmt.Errorf("Invalid container reference: Empty image name")
	}

	// Inspect image for obtaining Container's Image ID
	imageInspected, err := imageManager.client.InspectImage(container.Image)
	if err != nil {
		seelog.Errorf("Error inspecting image %v: %v", container.Image, err)
		return err
	}

	container.ImageID = imageInspected.ID
	added := imageManager.addContainerReferenceToExistingImageState(container)
	if !added {
		imageManager.addContainerReferenceToNewImageState(container, imageInspected.Size)
	}
	return nil
}

func (imageManager *dockerImageManager) addContainerReferenceToExistingImageState(container *api.Container) bool {
	// this lock is used for reading the image states in the image manager
	imageManager.updateLock.RLock()
	defer imageManager.updateLock.RUnlock()
	imageManager.removeExistingImageNameOfDifferentID(container.Image, container.ImageID)
	imageState, ok := imageManager.getImageState(container.ImageID)
	if ok {
		imageState.UpdateImageState(container)
	}
	return ok
}

func (imageManager *dockerImageManager) addContainerReferenceToNewImageState(container *api.Container, imageSize int64) {
	// this lock is used while creating and adding new image state to image manager
	imageManager.updateLock.Lock()
	defer imageManager.updateLock.Unlock()
	imageManager.removeExistingImageNameOfDifferentID(container.Image, container.ImageID)
	// check to see if a different thread added image state for same image ID
	imageState, ok := imageManager.getImageState(container.ImageID)
	if ok {
		imageState.UpdateImageState(container)
	} else {
		sourceImage := &image.Image{
			ImageID: container.ImageID,
			Size:    imageSize,
		}
		sourceImageState := &image.ImageState{
			Image:      sourceImage,
			PulledAt:   time.Now(),
			LastUsedAt: time.Now(),
		}
		sourceImageState.UpdateImageState(container)
		imageManager.addImageState(sourceImageState)
	}
}

// RemoveContainerReferenceFromImageState removes container reference from the corresponding imageState object
func (imageManager *dockerImageManager) RemoveContainerReferenceFromImageState(container *api.Container) error {
	// the image state has been updated, save the new state
	defer imageManager.saver.ForceSave()
	// this lock is for reading image states and finding the one that the container belongs to
	imageManager.updateLock.RLock()
	defer imageManager.updateLock.RUnlock()
	if container.ImageID == "" {
		return fmt.Errorf("Invalid container reference: Empty image id")
	}

	// Find image state that this container is part of, and remove the reference
	imageState, ok := imageManager.getImageState(container.ImageID)
	if !ok {
		return fmt.Errorf("Cannot find image state for the container to be removed")
	}
	// Found matching ImageState
	return imageState.RemoveContainerReference(container)
}

func (imageManager *dockerImageManager) addImageState(imageState *image.ImageState) {
	imageManager.imageStates = append(imageManager.imageStates, imageState)
}

// getAllImageStates returns the list of imageStates in the instance
func (imageManager *dockerImageManager) getAllImageStates() []*image.ImageState {
	return imageManager.imageStates
}

// getImageState returns the ImageState object that the container is referenced at
func (imageManager *dockerImageManager) getImageState(containerImageID string) (*image.ImageState, bool) {
	for _, imageState := range imageManager.getAllImageStates() {
		if imageState.Image.ImageID == containerImageID {
			return imageState, true
		}
	}
	return nil, false
}

// removeImageState removes the imageState from the list of imageState objects in ImageManager
func (imageManager *dockerImageManager) removeImageState(imageStateToBeRemoved *image.ImageState) {
	for i, imageState := range imageManager.imageStates {
		if imageState.Image.ImageID == imageStateToBeRemoved.Image.ImageID {
			// Image State found; hence remove it
			seelog.Infof("Removing Image State: %v from Image Manager", imageState.Image.ImageID)
			imageManager.imageStates = append(imageManager.imageStates[:i], imageManager.imageStates[i+1:]...)
			return
		}
	}
}

func (imageManager *dockerImageManager) getCandidateImagesForDeletion() []*image.ImageState {
	if len(imageManager.imageStatesConsideredForDeletion) < 1 {
		// no image states present in image manager
		return nil
	}
	var imagesForDeletion []*image.ImageState
	for _, imageState := range imageManager.imageStatesConsideredForDeletion {
		if imageManager.isImageOldEnough(imageState) && imageState.HasNoAssociatedContainers() {
			seelog.Infof("Candidate image for deletion: %+v", imageState)
			imagesForDeletion = append(imagesForDeletion, imageState)
		}
	}
	return imagesForDeletion
}

func (imageManager *dockerImageManager) isImageOldEnough(imageState *image.ImageState) bool {
	ageOfImage := time.Now().Sub(imageState.PulledAt)
	return ageOfImage > imageManager.minimumAgeBeforeDeletion
}

// Implementing sort interface based on last used times of the images
func (imageStates ImageStatesForDeletion) Len() int {
	return len(imageStates)
}

func (imageStates ImageStatesForDeletion) Less(i, j int) bool {
	return imageStates[i].LastUsedAt.Before(imageStates[j].LastUsedAt)
}

func (imageStates ImageStatesForDeletion) Swap(i, j int) {
	imageStates[i], imageStates[j] = imageStates[j], imageStates[i]
}

func (imageManager *dockerImageManager) getLeastRecentlyUsedImage(imagesForDeletion []*image.ImageState) *image.ImageState {
	var candidateImages ImageStatesForDeletion
	for _, imageState := range imagesForDeletion {
		candidateImages = append(candidateImages, imageState)
	}
	// sort images in the order of last used times
	sort.Sort(candidateImages)
	// return only the top LRU image for deletion
	return candidateImages[0]
}

func (imageManager *dockerImageManager) removeExistingImageNameOfDifferentID(containerImageName string, inspectedImageID string) {
	for _, imageState := range imageManager.getAllImageStates() {
		// image with same name pulled in the instance. Untag the already existing image name
		if imageState.Image.ImageID != inspectedImageID {
			imageState.RemoveImageName(containerImageName)
		}
	}
}

func (imageManager *dockerImageManager) StartImageCleanupProcess(ctx context.Context) {
	// passing the cleanup interval as argument which would help during testing
	imageManager.performPeriodicImageCleanup(ctx, imageManager.imageCleanupTimeInterval)
}

func (imageManager *dockerImageManager) performPeriodicImageCleanup(ctx context.Context, imageCleanupInterval time.Duration) {
	imageManager.imageCleanupTicker = time.NewTicker(imageCleanupInterval)
	for {
		select {
		case <-imageManager.imageCleanupTicker.C:
			go imageManager.removeUnusedImages()
		case <-ctx.Done():
			imageManager.imageCleanupTicker.Stop()
			return
		}
	}
}

func (imageManager *dockerImageManager) removeUnusedImages() {
	seelog.Infof("Begin building map of eligible unused images for deletion")
	imageManager.updateLock.Lock()
	defer imageManager.updateLock.Unlock()
	imageManager.imageStatesConsideredForDeletion = make(map[string]*image.ImageState)
	for _, imageState := range imageManager.getAllImageStates() {
		imageManager.imageStatesConsideredForDeletion[imageState.Image.ImageID] = imageState
	}
	for i := 0; i < imageManager.numImagesToDelete; i++ {
		err := imageManager.removeLeastRecentlyUsedImage()
		if err != nil {
			seelog.Infof("End of eligible images for deletion")
			break
		}
	}
}

func (imageManager *dockerImageManager) removeLeastRecentlyUsedImage() error {
	seelog.Debug("Attempting to obtain ImagePullDeleteLock for removing images")
	ImagePullDeleteLock.Lock()
	seelog.Debug("Obtained ImagePullDeleteLock for removing images")
	defer seelog.Debug("Released ImagePullDeleteLock after removing images")
	defer ImagePullDeleteLock.Unlock()
	leastRecentlyUsedImage := imageManager.getUnusedImageForDeletion()
	if leastRecentlyUsedImage == nil {
		return fmt.Errorf("No more eligible images for deletion")
	}
	imageManager.removeImage(leastRecentlyUsedImage)
	return nil
}

func (imageManager *dockerImageManager) getUnusedImageForDeletion() *image.ImageState {
	candidateImageStatesForDeletion := imageManager.getCandidateImagesForDeletion()
	if len(candidateImageStatesForDeletion) < 1 {
		seelog.Infof("No eligible images for deletion for this cleanup cycle")
		return nil
	}
	seelog.Infof("Found %d eligible images for deletion", len(candidateImageStatesForDeletion))
	return imageManager.getLeastRecentlyUsedImage(candidateImageStatesForDeletion)
}

func (imageManager *dockerImageManager) removeImage(leastRecentlyUsedImage *image.ImageState) {
	// Handling deleting while traversing a slice
	imageNames := make([]string, len(leastRecentlyUsedImage.Image.Names))
	copy(imageNames, leastRecentlyUsedImage.Image.Names)
	if len(imageNames) == 0 {
		// potentially untagged image of format <none>:<none>; remove by ID
		imageManager.deleteImage(leastRecentlyUsedImage.Image.ImageID, leastRecentlyUsedImage)
	} else {
		// Image has multiple tags/repos. Untag each name and delete the final reference to image
		for _, imageName := range imageNames {
			imageManager.deleteImage(imageName, leastRecentlyUsedImage)
		}
	}
}

func (imageManager *dockerImageManager) deleteImage(imageID string, imageState *image.ImageState) {
	if imageID == "" {
		seelog.Errorf("Image ID to be deleted is null")
		return
	}
	seelog.Infof("Removing Image: %s", imageID)
	err := imageManager.client.RemoveImage(imageID, removeImageTimeout)
	if err != nil {
		if err.Error() == imageNotFoundForDeletionError {
			seelog.Errorf("Image already removed from the instance: %v", err)
		} else {
			seelog.Errorf("Error removing Image %v - %v", imageID, err)
			delete(imageManager.imageStatesConsideredForDeletion, imageState.Image.ImageID)
			return
		}
	}
	seelog.Infof("Image removed: %v", imageID)
	imageState.RemoveImageName(imageID)
	if len(imageState.Image.Names) == 0 {
		seelog.Infof("Cleaning up all tracking information for image %s as it has zero references", imageID)
		delete(imageManager.imageStatesConsideredForDeletion, imageState.Image.ImageID)
		imageManager.removeImageState(imageState)
		imageManager.state.RemoveImageState(imageState)
		imageManager.saver.Save()
	}
}

func (imageManager *dockerImageManager) GetImageStateFromImageName(containerImageName string) *image.ImageState {
	imageManager.updateLock.Lock()
	defer imageManager.updateLock.Unlock()
	for _, imageState := range imageManager.getAllImageStates() {
		for _, imageName := range imageState.Image.Names {
			if imageName == containerImageName {
				return imageState
			}
		}
	}
	return nil
}
<commit_msg>Fix the deadlock caused by the ImagePullDeleteLock<commit_after>// Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"). You may
// not use this file except in compliance with the License. A copy of the
// License is located at
//
//	http://aws.amazon.com/apache2.0/
//
// or in the "license" file accompanying this file. This file is distributed
// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
// express or implied. See the License for the specific language governing
// permissions and limitations under the License.

package engine

import (
	"fmt"
	"sort"
	"sync"
	"time"

	"github.com/aws/amazon-ecs-agent/agent/api"
	"github.com/aws/amazon-ecs-agent/agent/config"
	"github.com/aws/amazon-ecs-agent/agent/engine/dockerstate"
	"github.com/aws/amazon-ecs-agent/agent/engine/image"
	"github.com/aws/amazon-ecs-agent/agent/statemanager"
	"github.com/cihub/seelog"
	"golang.org/x/net/context"
)

const (
	imageNotFoundForDeletionError = "no such image"
)

// ImageManager is responsible for saving the Image states,
// adding and removing container references to ImageStates
type ImageManager interface {
	RecordContainerReference(container *api.Container) error
	RemoveContainerReferenceFromImageState(container *api.Container) error
	AddAllImageStates(imageStates []*image.ImageState)
	GetImageStateFromImageName(containerImageName string) *image.ImageState
	StartImageCleanupProcess(ctx context.Context)
	SetSaver(stateManager statemanager.Saver)
}

// dockerImageManager accounts all the images and their states in the instance.
// It also has the cleanup policy configuration.
type dockerImageManager struct {
	imageStates                      []*image.ImageState
	client                           DockerClient
	updateLock                       sync.RWMutex
	imageCleanupTicker               *time.Ticker
	state                            dockerstate.TaskEngineState
	saver                            statemanager.Saver
	imageStatesConsideredForDeletion map[string]*image.ImageState
	minimumAgeBeforeDeletion         time.Duration
	numImagesToDelete                int
	imageCleanupTimeInterval         time.Duration
}

// ImageStatesForDeletion is used for implementing the sort interface
type ImageStatesForDeletion []*image.ImageState

// NewImageManager returns a new ImageManager
func NewImageManager(cfg *config.Config, client DockerClient, state dockerstate.TaskEngineState) ImageManager {
	return &dockerImageManager{
		client: client,
		state:  state,
		minimumAgeBeforeDeletion: cfg.MinimumImageDeletionAge,
		numImagesToDelete:        cfg.NumImagesToDeletePerCycle,
		imageCleanupTimeInterval: cfg.ImageCleanupInterval,
	}
}

func (imageManager *dockerImageManager) SetSaver(stateManager statemanager.Saver) {
	imageManager.saver = stateManager
}

func (imageManager *dockerImageManager) AddAllImageStates(imageStates []*image.ImageState) {
	imageManager.updateLock.Lock()
	defer imageManager.updateLock.Unlock()
	for _, imageState := range imageStates {
		imageManager.addImageState(imageState)
	}
}

// RecordContainerReference adds container reference to the corresponding imageState object
func (imageManager *dockerImageManager) RecordContainerReference(container *api.Container) error {
	// the image state has been updated, save the new state
	defer imageManager.saver.ForceSave()
	// On agent restart, container ID was retrieved from agent state file
	// TODO add setter and getter for modifying this
	if container.ImageID != "" {
		if !imageManager.addContainerReferenceToExistingImageState(container) {
			return fmt.Errorf("Failed to add container to existing image state")
		}
		return nil
	}

	if container.Image == "" {
		return fmt.Errorf("Invalid container reference: Empty image name")
	}

	// Inspect image for obtaining Container's Image ID
	imageInspected, err := imageManager.client.InspectImage(container.Image)
	if err != nil {
		seelog.Errorf("Error inspecting image %v: %v", container.Image, err)
		return err
	}

	container.ImageID = imageInspected.ID
	added := imageManager.addContainerReferenceToExistingImageState(container)
	if !added {
		imageManager.addContainerReferenceToNewImageState(container, imageInspected.Size)
	}
	return nil
}

func (imageManager *dockerImageManager) addContainerReferenceToExistingImageState(container *api.Container) bool {
	// this lock is used for reading the image states in the image manager
	imageManager.updateLock.RLock()
	defer imageManager.updateLock.RUnlock()
	imageManager.removeExistingImageNameOfDifferentID(container.Image, container.ImageID)
	imageState, ok := imageManager.getImageState(container.ImageID)
	if ok {
		imageState.UpdateImageState(container)
	}
	return ok
}

func (imageManager *dockerImageManager) addContainerReferenceToNewImageState(container *api.Container, imageSize int64) {
	// this lock is used while creating and adding new image state to image manager
	imageManager.updateLock.Lock()
	defer imageManager.updateLock.Unlock()
	imageManager.removeExistingImageNameOfDifferentID(container.Image, container.ImageID)
	// check to see if a different thread added image state for same image ID
	imageState, ok := imageManager.getImageState(container.ImageID)
	if ok {
		imageState.UpdateImageState(container)
	} else {
		sourceImage := &image.Image{
			ImageID: container.ImageID,
			Size:    imageSize,
		}
		sourceImageState := &image.ImageState{
			Image:      sourceImage,
			PulledAt:   time.Now(),
			LastUsedAt: time.Now(),
		}
		sourceImageState.UpdateImageState(container)
		imageManager.addImageState(sourceImageState)
	}
}

// RemoveContainerReferenceFromImageState removes container reference from the corresponding imageState object
func (imageManager *dockerImageManager) RemoveContainerReferenceFromImageState(container *api.Container) error {
	// the image state has been updated, save the new state
	defer imageManager.saver.ForceSave()
	// this lock is for reading image states and finding the one that the container belongs to
	imageManager.updateLock.RLock()
	defer imageManager.updateLock.RUnlock()
	if container.ImageID == "" {
		return fmt.Errorf("Invalid container reference: Empty image id")
	}

	// Find image state that this container is part of, and remove the reference
	imageState, ok := imageManager.getImageState(container.ImageID)
	if !ok {
		return fmt.Errorf("Cannot find image state for the container to be removed")
	}
	// Found matching ImageState
	return imageState.RemoveContainerReference(container)
}

func (imageManager *dockerImageManager) addImageState(imageState *image.ImageState) {
	imageManager.imageStates = append(imageManager.imageStates, imageState)
}

// getAllImageStates returns the list of imageStates in the instance
func (imageManager *dockerImageManager) getAllImageStates() []*image.ImageState {
	return imageManager.imageStates
}

// getImageState returns the ImageState object that the container is referenced at
func (imageManager *dockerImageManager) getImageState(containerImageID string) (*image.ImageState, bool) {
	for _, imageState := range imageManager.getAllImageStates() {
		if imageState.Image.ImageID == containerImageID {
			return imageState, true
		}
	}
	return nil, false
}

// removeImageState removes the imageState from the list of imageState objects in ImageManager
func (imageManager *dockerImageManager) removeImageState(imageStateToBeRemoved *image.ImageState) {
	for i, imageState := range imageManager.imageStates {
		if imageState.Image.ImageID == imageStateToBeRemoved.Image.ImageID {
			// Image State found; hence remove it
			seelog.Infof("Removing Image State: %v from Image Manager", imageState.Image.ImageID)
			imageManager.imageStates = append(imageManager.imageStates[:i], imageManager.imageStates[i+1:]...)
			return
		}
	}
}

func (imageManager *dockerImageManager) getCandidateImagesForDeletion() []*image.ImageState {
	if len(imageManager.imageStatesConsideredForDeletion) < 1 {
		// no image states present in image manager
		return nil
	}
	var imagesForDeletion []*image.ImageState
	for _, imageState := range imageManager.imageStatesConsideredForDeletion {
		if imageManager.isImageOldEnough(imageState) && imageState.HasNoAssociatedContainers() {
			seelog.Infof("Candidate image for deletion: %+v", imageState)
			imagesForDeletion = append(imagesForDeletion, imageState)
		}
	}
	return imagesForDeletion
}

func (imageManager *dockerImageManager) isImageOldEnough(imageState *image.ImageState) bool {
	ageOfImage := time.Now().Sub(imageState.PulledAt)
	return ageOfImage > imageManager.minimumAgeBeforeDeletion
}

// Implementing sort interface based on last used times of the images
func (imageStates ImageStatesForDeletion) Len() int {
	return len(imageStates)
}

func (imageStates ImageStatesForDeletion) Less(i, j int) bool {
	return imageStates[i].LastUsedAt.Before(imageStates[j].LastUsedAt)
}

func (imageStates ImageStatesForDeletion) Swap(i, j int) {
	imageStates[i], imageStates[j] = imageStates[j], imageStates[i]
}

func (imageManager *dockerImageManager) getLeastRecentlyUsedImage(imagesForDeletion []*image.ImageState) *image.ImageState {
	var candidateImages ImageStatesForDeletion
	for _, imageState := range imagesForDeletion {
		candidateImages = append(candidateImages, imageState)
	}
	// sort images in the order of last used times
	sort.Sort(candidateImages)
	// return only the top LRU image for deletion
	return candidateImages[0]
}

func (imageManager *dockerImageManager) removeExistingImageNameOfDifferentID(containerImageName string, inspectedImageID string) {
	for _, imageState := range imageManager.getAllImageStates() {
		// image with same name pulled in the instance. Untag the already existing image name
		if imageState.Image.ImageID != inspectedImageID {
			imageState.RemoveImageName(containerImageName)
		}
	}
}

func (imageManager *dockerImageManager) StartImageCleanupProcess(ctx context.Context) {
	// passing the cleanup interval as argument which would help during testing
	imageManager.performPeriodicImageCleanup(ctx, imageManager.imageCleanupTimeInterval)
}

func (imageManager *dockerImageManager) performPeriodicImageCleanup(ctx context.Context, imageCleanupInterval time.Duration) {
	imageManager.imageCleanupTicker = time.NewTicker(imageCleanupInterval)
	for {
		select {
		case <-imageManager.imageCleanupTicker.C:
			go imageManager.removeUnusedImages()
		case <-ctx.Done():
			imageManager.imageCleanupTicker.Stop()
			return
		}
	}
}

func (imageManager *dockerImageManager) removeUnusedImages() {
	seelog.Debug("Attempting to obtain ImagePullDeleteLock for removing images")
	ImagePullDeleteLock.Lock()
	seelog.Debug("Obtained ImagePullDeleteLock for removing images")
	defer seelog.Debug("Released ImagePullDeleteLock after removing images")
	defer ImagePullDeleteLock.Unlock()

	seelog.Infof("Begin building map of eligible unused images for deletion")
	imageManager.updateLock.Lock()
	defer imageManager.updateLock.Unlock()
	imageManager.imageStatesConsideredForDeletion = make(map[string]*image.ImageState)
	for _, imageState := range imageManager.getAllImageStates() {
		imageManager.imageStatesConsideredForDeletion[imageState.Image.ImageID] = imageState
	}
	for i := 0; i < imageManager.numImagesToDelete; i++ {
		err := imageManager.removeLeastRecentlyUsedImage()
		if err != nil {
			seelog.Infof("End of eligible images for deletion")
			break
		}
	}
}

func (imageManager *dockerImageManager) removeLeastRecentlyUsedImage() error {
	leastRecentlyUsedImage := imageManager.getUnusedImageForDeletion()
	if leastRecentlyUsedImage == nil {
		return fmt.Errorf("No more eligible images for deletion")
	}
	imageManager.removeImage(leastRecentlyUsedImage)
	return nil
}

func (imageManager *dockerImageManager) getUnusedImageForDeletion() *image.ImageState {
	candidateImageStatesForDeletion := imageManager.getCandidateImagesForDeletion()
	if len(candidateImageStatesForDeletion) < 1 {
		seelog.Infof("No eligible images for deletion for this cleanup cycle")
		return nil
	}
	seelog.Infof("Found %d eligible images for deletion", len(candidateImageStatesForDeletion))
	return imageManager.getLeastRecentlyUsedImage(candidateImageStatesForDeletion)
}

func (imageManager *dockerImageManager) removeImage(leastRecentlyUsedImage *image.ImageState) {
	// Handling deleting while traversing a slice
	imageNames := make([]string, len(leastRecentlyUsedImage.Image.Names))
	copy(imageNames, leastRecentlyUsedImage.Image.Names)
	if len(imageNames) == 0 {
		// potentially untagged image of format <none>:<none>; remove by ID
		imageManager.deleteImage(leastRecentlyUsedImage.Image.ImageID, leastRecentlyUsedImage)
	} else {
		// Image has multiple tags/repos. Untag each name and delete the final reference to image
		for _, imageName := range imageNames {
			imageManager.deleteImage(imageName, leastRecentlyUsedImage)
		}
	}
}

func (imageManager *dockerImageManager) deleteImage(imageID string, imageState *image.ImageState) {
	if imageID == "" {
		seelog.Errorf("Image ID to be deleted is null")
		return
	}
	seelog.Infof("Removing Image: %s", imageID)
	err := imageManager.client.RemoveImage(imageID, removeImageTimeout)
	if err != nil {
		if err.Error() == imageNotFoundForDeletionError {
			seelog.Errorf("Image already removed from the instance: %v", err)
		} else {
			seelog.Errorf("Error removing Image %v - %v", imageID, err)
			delete(imageManager.imageStatesConsideredForDeletion, imageState.Image.ImageID)
			return
		}
	}
	seelog.Infof("Image removed: %v", imageID)
	imageState.RemoveImageName(imageID)
	if len(imageState.Image.Names) == 0 {
		seelog.Infof("Cleaning up all tracking information for image %s as it has zero references", imageID)
		delete(imageManager.imageStatesConsideredForDeletion, imageState.Image.ImageID)
		imageManager.removeImageState(imageState)
		imageManager.state.RemoveImageState(imageState)
		imageManager.saver.Save()
	}
}

func (imageManager *dockerImageManager) GetImageStateFromImageName(containerImageName string) *image.ImageState {
	imageManager.updateLock.Lock()
	defer imageManager.updateLock.Unlock()
	for _, imageState := range imageManager.getAllImageStates() {
		for _, imageName := range imageState.Image.Names {
			if imageName == containerImageName {
				return imageState
			}
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package termite

import (
	"errors"
	"fmt"
	"github.com/hanwen/go-fuse/fuse"
	"github.com/hanwen/termite/attr"
	"github.com/hanwen/termite/fs"
	"io/ioutil"
	"log"
	"os"
	"os/user"
	"path/filepath"
	"strings"
)

type workerFuseFs struct {
	rwDir  string
	tmpDir string
	mount  string

	// without leading /
	writableRoot string
	*fuse.MountState
	fsConnector *fuse.FileSystemConnector
	unionFs     *fs.MemUnionFs
	procFs      *fs.ProcFs
	rpcNodeFs   *fuse.PathNodeFs
	unionNodeFs *fuse.PathNodeFs

	// Protected by Mirror.fsMutex
	id      string
	reaping bool

	// When this reaches zero, we reap the filesystem.
	tasks map[*WorkerTask]bool

	// Task ids that have results pending in this FS.
	taskIds []int
}

func (me *workerFuseFs) Status() (s FuseFsStatus) {
	s.Id = me.id
	s.Mem = me.BufferPoolStats()
	for t := range me.tasks {
		s.Tasks = append(s.Tasks, t.taskInfo)
	}
	return s
}

func (me *workerFuseFs) addTask(task *WorkerTask) {
	me.taskIds = append(me.taskIds, task.req.TaskId)
	me.tasks[task] = true
}

func (me *workerFuseFs) Stop() {
	err := me.MountState.Unmount()
	if err != nil {
		// TODO - Should be fatal?
		log.Println("Unmount fail:", err)
	} else {
		// If the unmount fails, the RemoveAll will stat all
		// of the FUSE file system.
		os.RemoveAll(me.tmpDir)
	}
}

func (me *workerFuseFs) SetDebug(debug bool) {
	me.MountState.Debug = debug
	me.fsConnector.Debug = debug
	me.rpcNodeFs.Debug = debug
}

func newWorkerFuseFs(tmpDir string, rpcFs fuse.FileSystem, writableRoot string, nobody *user.User) (*workerFuseFs, error) {
	tmpDir, err := ioutil.TempDir(tmpDir, "termite-task")
	if err != nil {
		return nil, err
	}
	me := &workerFuseFs{
		tmpDir:       tmpDir,
		writableRoot: strings.TrimLeft(writableRoot, "/"),
		tasks:        map[*WorkerTask]bool{},
	}

	type dirInit struct {
		dst *string
		val string
	}

	tmpBacking := ""
	for _, v := range []dirInit{
		dirInit{&me.rwDir, "rw"},
		dirInit{&me.mount, "mnt"},
		dirInit{&tmpBacking, "tmp-backing"},
	} {
		*v.dst = filepath.Join(me.tmpDir, v.val)
		err = os.Mkdir(*v.dst, 0700)
		if err != nil {
			return nil, err
		}
	}

	fuseOpts := fuse.MountOptions{}
	if os.Geteuid() == 0 {
		fuseOpts.AllowOther = true
	}

	me.rpcNodeFs = fuse.NewPathNodeFs(rpcFs, nil)
	ttl := 30.0
	mOpts := fuse.FileSystemOptions{
		EntryTimeout:    ttl,
		AttrTimeout:     ttl,
		NegativeTimeout: ttl,

		// 32-bit programs have trouble with 64-bit inode
		// numbers.
		PortableInodes: true,
	}

	me.fsConnector = fuse.NewFileSystemConnector(me.rpcNodeFs, &mOpts)
	me.MountState = fuse.NewMountState(me.fsConnector)
	err = me.MountState.Mount(me.mount, &fuseOpts)
	if err != nil {
		return nil, err
	}
	go me.MountState.Loop()

	me.unionFs = fs.NewMemUnionFs(
		me.rwDir, &fuse.PrefixFileSystem{rpcFs, me.writableRoot})

	me.procFs = fs.NewProcFs()
	me.procFs.StripPrefix = me.mount
	if nobody != nil {
		me.procFs.Uid = nobody.Uid
	}
	type submount struct {
		mountpoint string
		fs         fuse.NodeFileSystem
	}

	mounts := []submount{
		{"proc", fuse.NewPathNodeFs(me.procFs, nil)},
		{"sys", fuse.NewPathNodeFs(&fuse.ReadonlyFileSystem{fuse.NewLoopbackFileSystem("/sys")}, nil)},
		{"tmp", fuse.NewMemNodeFs(tmpBacking + "tmp")},
		{"dev", fs.NewDevNullFs()},
		{"var/tmp", fuse.NewMemNodeFs(tmpBacking + "vartmp")},
	}
	for _, s := range mounts {
		subOpts := &mOpts
		if s.mountpoint == "proc" {
			subOpts = nil
		}

		code := me.rpcNodeFs.Mount(s.mountpoint, s.fs, subOpts)
		if !code.Ok() {
			me.MountState.Unmount()
			return nil, errors.New(fmt.Sprintf("submount error for %s: %v", s.mountpoint, code))
		}
	}
	if strings.HasPrefix(me.writableRoot, "tmp/") {
		parent, _ := filepath.Split(me.writableRoot)
		err := os.MkdirAll(filepath.Join(me.mount, parent), 0755)
		if err != nil {
			me.MountState.Unmount()
			return nil, errors.New(fmt.Sprintf("Mkdir of %q in /tmp fail: %v", parent, err))
		}
		// This is hackish, but we don't want rpcfs/fsserver
		// getting confused by asking for tmp/foo/bar
		// directly.
		rpcFs.GetAttr("tmp", nil)
		rpcFs.GetAttr(me.writableRoot, nil)
	}
	code := me.rpcNodeFs.Mount(me.writableRoot, me.unionFs, &mOpts)
	if !code.Ok() {
		me.MountState.Unmount()
		return nil, errors.New(fmt.Sprintf("submount error for %s: %v", me.writableRoot, code))
	}

	return me, nil
}

func (me *workerFuseFs) update(attrs []*attr.FileAttr) {
	updates := map[string]*fs.Result{}
	for _, attr := range attrs {
		path := strings.TrimLeft(attr.Path, "/")
		if !strings.HasPrefix(path, me.writableRoot) {
			me.rpcNodeFs.Notify(path)
			continue
		}
		path = strings.TrimLeft(path[len(me.writableRoot):], "/")

		if attr.Deletion() {
			updates[path] = &fs.Result{}
		} else {
			updates[path] = &fs.Result{
				FileInfo: attr.FileInfo,
				Original: "",
				Backing:  "",
				Link:     attr.Link,
			}
		}
	}
	me.unionFs.Update(updates)
}
<commit_msg>Make FUSE unmount errors fatal.<commit_after>package termite

import (
	"errors"
	"fmt"
	"github.com/hanwen/go-fuse/fuse"
	"github.com/hanwen/termite/attr"
	"github.com/hanwen/termite/fs"
	"io/ioutil"
	"log"
	"os"
	"os/user"
	"path/filepath"
	"strings"
)

type workerFuseFs struct {
	rwDir  string
	tmpDir string
	mount  string

	// without leading /
	writableRoot string
	*fuse.MountState
	fsConnector *fuse.FileSystemConnector
	unionFs     *fs.MemUnionFs
	procFs      *fs.ProcFs
	rpcNodeFs   *fuse.PathNodeFs
	unionNodeFs *fuse.PathNodeFs

	// Protected by Mirror.fsMutex
	id      string
	reaping bool

	// When this reaches zero, we reap the filesystem.
	tasks map[*WorkerTask]bool

	// Task ids that have results pending in this FS.
	taskIds []int
}

func (me *workerFuseFs) Status() (s FuseFsStatus) {
	s.Id = me.id
	s.Mem = me.BufferPoolStats()
	for t := range me.tasks {
		s.Tasks = append(s.Tasks, t.taskInfo)
	}
	return s
}

func (me *workerFuseFs) addTask(task *WorkerTask) {
	me.taskIds = append(me.taskIds, task.req.TaskId)
	me.tasks[task] = true
}

func (me *workerFuseFs) Stop() {
	err := me.MountState.Unmount()
	if err != nil {
		log.Fatal("Unmount fail:", err)
	} else {
		// If the unmount fails, the RemoveAll will stat all
		// of the FUSE file system.
		os.RemoveAll(me.tmpDir)
	}
}

func (me *workerFuseFs) SetDebug(debug bool) {
	me.MountState.Debug = debug
	me.fsConnector.Debug = debug
	me.rpcNodeFs.Debug = debug
}

func newWorkerFuseFs(tmpDir string, rpcFs fuse.FileSystem, writableRoot string, nobody *user.User) (*workerFuseFs, error) {
	tmpDir, err := ioutil.TempDir(tmpDir, "termite-task")
	if err != nil {
		return nil, err
	}
	me := &workerFuseFs{
		tmpDir:       tmpDir,
		writableRoot: strings.TrimLeft(writableRoot, "/"),
		tasks:        map[*WorkerTask]bool{},
	}

	type dirInit struct {
		dst *string
		val string
	}

	tmpBacking := ""
	for _, v := range []dirInit{
		dirInit{&me.rwDir, "rw"},
		dirInit{&me.mount, "mnt"},
		dirInit{&tmpBacking, "tmp-backing"},
	} {
		*v.dst = filepath.Join(me.tmpDir, v.val)
		err = os.Mkdir(*v.dst, 0700)
		if err != nil {
			return nil, err
		}
	}

	fuseOpts := fuse.MountOptions{}
	if os.Geteuid() == 0 {
		fuseOpts.AllowOther = true
	}

	me.rpcNodeFs = fuse.NewPathNodeFs(rpcFs, nil)
	ttl := 30.0
	mOpts := fuse.FileSystemOptions{
		EntryTimeout:    ttl,
		AttrTimeout:     ttl,
		NegativeTimeout: ttl,

		// 32-bit programs have trouble with 64-bit inode
		// numbers.
		PortableInodes: true,
	}

	me.fsConnector = fuse.NewFileSystemConnector(me.rpcNodeFs, &mOpts)
	me.MountState = fuse.NewMountState(me.fsConnector)
	err = me.MountState.Mount(me.mount, &fuseOpts)
	if err != nil {
		return nil, err
	}
	go me.MountState.Loop()

	me.unionFs = fs.NewMemUnionFs(
		me.rwDir, &fuse.PrefixFileSystem{rpcFs, me.writableRoot})

	me.procFs = fs.NewProcFs()
	me.procFs.StripPrefix = me.mount
	if nobody != nil {
		me.procFs.Uid = nobody.Uid
	}
	type submount struct {
		mountpoint string
		fs         fuse.NodeFileSystem
	}

	mounts := []submount{
		{"proc", fuse.NewPathNodeFs(me.procFs, nil)},
		{"sys", fuse.NewPathNodeFs(&fuse.ReadonlyFileSystem{fuse.NewLoopbackFileSystem("/sys")}, nil)},
		{"tmp", fuse.NewMemNodeFs(tmpBacking + "tmp")},
		{"dev", fs.NewDevNullFs()},
		{"var/tmp", fuse.NewMemNodeFs(tmpBacking + "vartmp")},
	}
	for _, s := range mounts {
		subOpts := &mOpts
		if s.mountpoint == "proc" {
			subOpts = nil
		}

		code := me.rpcNodeFs.Mount(s.mountpoint, s.fs, subOpts)
		if !code.Ok() {
			me.MountState.Unmount()
			return nil, errors.New(fmt.Sprintf("submount error for %s: %v", s.mountpoint, code))
		}
	}
	if strings.HasPrefix(me.writableRoot, "tmp/") {
		parent, _ := filepath.Split(me.writableRoot)
		err := os.MkdirAll(filepath.Join(me.mount, parent), 0755)
		if err != nil {
			me.MountState.Unmount()
			return nil, errors.New(fmt.Sprintf("Mkdir of %q in /tmp fail: %v", parent, err))
		}
		// This is hackish, but we don't want rpcfs/fsserver
		// getting confused by asking for tmp/foo/bar
		// directly.
		rpcFs.GetAttr("tmp", nil)
		rpcFs.GetAttr(me.writableRoot, nil)
	}
	code := me.rpcNodeFs.Mount(me.writableRoot, me.unionFs, &mOpts)
	if !code.Ok() {
		me.MountState.Unmount()
		return nil, errors.New(fmt.Sprintf("submount error for %s: %v", me.writableRoot, code))
	}

	return me, nil
}

func (me *workerFuseFs) update(attrs []*attr.FileAttr) {
	updates := map[string]*fs.Result{}
	for _, attr := range attrs {
		path := strings.TrimLeft(attr.Path, "/")
		if !strings.HasPrefix(path, me.writableRoot) {
			me.rpcNodeFs.Notify(path)
			continue
		}
		path = strings.TrimLeft(path[len(me.writableRoot):], "/")

		if attr.Deletion() {
			updates[path] = &fs.Result{}
		} else {
			updates[path] = &fs.Result{
				FileInfo: attr.FileInfo,
				Original: "",
				Backing:  "",
				Link:     attr.Link,
			}
		}
	}
	me.unionFs.Update(updates)
}
<|endoftext|>


<commit_before>// Copyright 2015 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package terror

// Schema error codes
const (
	DatabaseNotExists ErrCode = iota + 1
)

// Executor error codes
const (
	CommitNotInTransaction ErrCode = iota + 1
	RollbackNotInTransaction
)

// KV error codes
const (
	InCompatibleDBFormat ErrCode = iota + 1
	NoDataForHandle
)
<commit_msg>terror: address comment.<commit_after>// Copyright 2015 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package terror

// Schema error codes
const (
	DatabaseNotExists ErrCode = iota + 1
)

// Executor error codes
const (
	CommitNotInTransaction ErrCode = iota + 1
	RollbackNotInTransaction
)

// KV error codes
const (
	IncompatibleDBFormat ErrCode = iota + 1
	NoDataForHandle
)
<|endoftext|>