<commit_before>package main

import (
	"fmt"
	"go/ast"
	"go/build"
	"go/parser"
	"go/token"
	"os"
	"path"
	"path/filepath"
	"strings"
	"sync"

	"code.google.com/p/go.tools/astutil"
)

// importToGroup is a list of functions which map from an import path to
// a group number.
var importToGroup = []func(importPath string) (num int, ok bool){
	func(importPath string) (num int, ok bool) {
		if strings.HasPrefix(importPath, "appengine") {
			return 2, true
		}
		return
	},
	func(importPath string) (num int, ok bool) {
		if strings.Contains(importPath, ".") {
			return 1, true
		}
		return
	},
}

func importGroup(importPath string) int {
	for _, fn := range importToGroup {
		if n, ok := fn(importPath); ok {
			return n
		}
	}
	return 0
}

func fixImports(f *ast.File) (added []string, err error) {
	// refs are a set of possible package references currently unsatisified by imports.
	// first key: either base package (e.g. "fmt") or renamed package
	// second key: referenced package symbol (e.g. "Println")
	refs := make(map[string]map[string]bool)

	// decls are the current package imports. key is base package or renamed package.
	decls := make(map[string]*ast.ImportSpec)

	// collect potential uses of packages.
	var visitor visitFn
	visitor = visitFn(func(node ast.Node) ast.Visitor {
		if node == nil {
			return visitor
		}
		switch v := node.(type) {
		case *ast.ImportSpec:
			if v.Name != nil {
				decls[v.Name.Name] = v
			} else {
				local := path.Base(strings.Trim(v.Path.Value, `\"`))
				decls[local] = v
			}
		case *ast.SelectorExpr:
			xident, ok := v.X.(*ast.Ident)
			if !ok {
				break
			}
			if xident.Obj != nil {
				// if the parser can resolve it, it's not a package ref
				break
			}
			pkgName := xident.Name
			if refs[pkgName] == nil {
				refs[pkgName] = make(map[string]bool)
			}
			if decls[pkgName] == nil {
				refs[pkgName][v.Sel.Name] = true
			}
		}
		return visitor
	})
	ast.Walk(visitor, f)

	// Search for imports matching potential package references.
	searches := 0
	type result struct {
		ipath string
		err   error
	}
	results := make(chan result)
	for pkgName, symbols := range refs {
		if len(symbols) == 0 {
			continue // skip over packages already imported
		}
		go func(pkgName string, symbols map[string]bool) {
			ipath, err := findImport(pkgName, symbols)
			results <- result{ipath, err}
		}(pkgName, symbols)
		searches++
	}
	for i := 0; i < searches; i++ {
		result := <-results
		if result.err != nil {
			return nil, result.err
		}
		if result.ipath != "" {
			astutil.AddImport(f, result.ipath)
			added = append(added, result.ipath)
		}
	}

	// Nil out any unused ImportSpecs, to be removed in following passes
	unusedImport := map[string]bool{}
	for pkg, is := range decls {
		if refs[pkg] == nil && pkg != "_" && pkg != "." {
			unusedImport[strings.Trim(is.Path.Value, `"`)] = true
		}
	}
	for ipath := range unusedImport {
		if ipath == "C" {
			// Don't remove cgo stuff.
			continue
		}
		astutil.DeleteImport(f, ipath)
	}

	return added, nil
}

type pkg struct {
	importpath string // full pkg import path, e.g. "net/http"
	dir        string // absolute file path to pkg directory e.g. "/usr/lib/go/src/fmt"
}

var pkgIndexOnce sync.Once

var pkgIndex struct {
	sync.Mutex
	m map[string][]pkg // shortname => []pkg, e.g "http" => "net/http"
}

func loadPkgIndex() {
	pkgIndex.Lock()
	pkgIndex.m = make(map[string][]pkg)
	pkgIndex.Unlock()

	var wg sync.WaitGroup
	for _, path := range build.Default.SrcDirs() {
		f, err := os.Open(path)
		if err != nil {
			fmt.Fprint(os.Stderr, err)
			continue
		}
		children, err := f.Readdir(-1)
		f.Close()
		if err != nil {
			fmt.Fprint(os.Stderr, err)
			continue
		}
		for _, child := range children {
			if child.IsDir() {
				wg.Add(1)
				go func(path, name string) {
					defer wg.Done()
					loadPkg(&wg, path, name)
				}(path, child.Name())
			}
		}
	}
	wg.Wait()
}

var fset = token.NewFileSet()

func loadPkg(wg *sync.WaitGroup, root, importpath string) {
	shortName := path.Base(importpath)

	dir := filepath.Join(root, importpath)
	pkgIndex.Lock()
	pkgIndex.m[shortName] = append(pkgIndex.m[shortName], pkg{
		importpath: importpath,
		dir:        dir,
	})
	pkgIndex.Unlock()

	pkgDir, err := os.Open(dir)
	if err != nil {
		return
	}
	children, err := pkgDir.Readdir(-1)
	pkgDir.Close()
	if err != nil {
		return
	}
	for _, child := range children {
		if strings.HasPrefix(child.Name(), ".") {
			continue
		}
		if child.IsDir() {
			wg.Add(1)
			go func(root, name string) {
				defer wg.Done()
				loadPkg(wg, root, name)
			}(root, filepath.Join(importpath, child.Name()))
		}
	}
}

// loadExports returns a list exports for a package.
func loadExports(dir string) map[string]bool {
	exports := make(map[string]bool)
	buildPkg, err := build.ImportDir(dir, 0)
	if err != nil {
		if strings.Contains(err.Error(), "no buildable Go source files in") {
			return nil
		}
		fmt.Fprintf(os.Stderr, "could not import %q: %v", dir, err)
		return nil
	}
	for _, file := range buildPkg.GoFiles {
		f, err := parser.ParseFile(fset, filepath.Join(dir, file), nil, 0)
		if err != nil {
			fmt.Fprintf(os.Stderr, "could not parse %q: %v", file, err)
			continue
		}
		for name := range f.Scope.Objects {
			if ast.IsExported(name) {
				exports[name] = true
			}
		}
	}
	return exports
}

// findImport searches for a package with the given symbols.
// If no package is found, findImport returns "".
// Declared as a variable rather than a function so goimports can be easily
// extended by adding a file with an init function.
var findImport = findImportGoPath

func findImportGoPath(pkgName string, symbols map[string]bool) (string, error) {

	pkgIndexOnce.Do(loadPkgIndex)

	// Collect exports for packages with matching names.
	var wg sync.WaitGroup
	var pkgsMu sync.Mutex // guards pkgs
	// full importpath => exported symbol => True
	// e.g. "net/http" => "Client" => True
	pkgs := make(map[string]map[string]bool)
	pkgIndex.Lock()
	for _, pkg := range pkgIndex.m[pkgName] {
		wg.Add(1)
		go func(importpath, dir string) {
			defer wg.Done()
			exports := loadExports(dir)
			if exports != nil {
				pkgsMu.Lock()
				pkgs[importpath] = exports
				pkgsMu.Unlock()
			}
		}(pkg.importpath, pkg.dir)
	}
	pkgIndex.Unlock()
	wg.Wait()

	// Filter out packages missing required exported symbols.
	for symbol := range symbols {
		for importpath, exports := range pkgs {
			if !exports[symbol] {
				delete(pkgs, importpath)
			}
		}
	}
	if len(pkgs) == 0 {
		return "", nil
	}

	// If there are multiple candidate packages, the shortest one wins.
	// This is a heuristic to prefer the standard library (e.g. "bytes")
	// over e.g. "github.com/foo/bar/bytes".
	shortest := ""
	for importPath := range pkgs {
		if shortest == "" || len(importPath) < len(shortest) {
			shortest = importPath
		}
	}
	return shortest, nil
}

type visitFn func(node ast.Node) ast.Visitor

func (fn visitFn) Visit(node ast.Node) ast.Visitor {
	return fn(node)
}
<commit_msg>Skip directories starting with numbers. (I had a bunch)<commit_after>package main

import (
	"fmt"
	"go/ast"
	"go/build"
	"go/parser"
	"go/token"
	"os"
	"path"
	"path/filepath"
	"strings"
	"sync"

	"code.google.com/p/go.tools/astutil"
)

// importToGroup is a list of functions which map from an import path to
// a group number.
var importToGroup = []func(importPath string) (num int, ok bool){
	func(importPath string) (num int, ok bool) {
		if strings.HasPrefix(importPath, "appengine") {
			return 2, true
		}
		return
	},
	func(importPath string) (num int, ok bool) {
		if strings.Contains(importPath, ".") {
			return 1, true
		}
		return
	},
}

func importGroup(importPath string) int {
	for _, fn := range importToGroup {
		if n, ok := fn(importPath); ok {
			return n
		}
	}
	return 0
}

func fixImports(f *ast.File) (added []string, err error) {
	// refs are a set of possible package references currently unsatisified by imports.
	// first key: either base package (e.g. "fmt") or renamed package
	// second key: referenced package symbol (e.g. "Println")
	refs := make(map[string]map[string]bool)

	// decls are the current package imports. key is base package or renamed package.
	decls := make(map[string]*ast.ImportSpec)

	// collect potential uses of packages.
	var visitor visitFn
	visitor = visitFn(func(node ast.Node) ast.Visitor {
		if node == nil {
			return visitor
		}
		switch v := node.(type) {
		case *ast.ImportSpec:
			if v.Name != nil {
				decls[v.Name.Name] = v
			} else {
				local := path.Base(strings.Trim(v.Path.Value, `\"`))
				decls[local] = v
			}
		case *ast.SelectorExpr:
			xident, ok := v.X.(*ast.Ident)
			if !ok {
				break
			}
			if xident.Obj != nil {
				// if the parser can resolve it, it's not a package ref
				break
			}
			pkgName := xident.Name
			if refs[pkgName] == nil {
				refs[pkgName] = make(map[string]bool)
			}
			if decls[pkgName] == nil {
				refs[pkgName][v.Sel.Name] = true
			}
		}
		return visitor
	})
	ast.Walk(visitor, f)

	// Search for imports matching potential package references.
	searches := 0
	type result struct {
		ipath string
		err   error
	}
	results := make(chan result)
	for pkgName, symbols := range refs {
		if len(symbols) == 0 {
			continue // skip over packages already imported
		}
		go func(pkgName string, symbols map[string]bool) {
			ipath, err := findImport(pkgName, symbols)
			results <- result{ipath, err}
		}(pkgName, symbols)
		searches++
	}
	for i := 0; i < searches; i++ {
		result := <-results
		if result.err != nil {
			return nil, result.err
		}
		if result.ipath != "" {
			astutil.AddImport(f, result.ipath)
			added = append(added, result.ipath)
		}
	}

	// Nil out any unused ImportSpecs, to be removed in following passes
	unusedImport := map[string]bool{}
	for pkg, is := range decls {
		if refs[pkg] == nil && pkg != "_" && pkg != "." {
			unusedImport[strings.Trim(is.Path.Value, `"`)] = true
		}
	}
	for ipath := range unusedImport {
		if ipath == "C" {
			// Don't remove cgo stuff.
			continue
		}
		astutil.DeleteImport(f, ipath)
	}

	return added, nil
}

type pkg struct {
	importpath string // full pkg import path, e.g. "net/http"
	dir        string // absolute file path to pkg directory e.g. "/usr/lib/go/src/fmt"
}

var pkgIndexOnce sync.Once

var pkgIndex struct {
	sync.Mutex
	m map[string][]pkg // shortname => []pkg, e.g "http" => "net/http"
}

func loadPkgIndex() {
	pkgIndex.Lock()
	pkgIndex.m = make(map[string][]pkg)
	pkgIndex.Unlock()

	var wg sync.WaitGroup
	for _, path := range build.Default.SrcDirs() {
		f, err := os.Open(path)
		if err != nil {
			fmt.Fprint(os.Stderr, err)
			continue
		}
		children, err := f.Readdir(-1)
		f.Close()
		if err != nil {
			fmt.Fprint(os.Stderr, err)
			continue
		}
		for _, child := range children {
			if child.IsDir() {
				wg.Add(1)
				go func(path, name string) {
					defer wg.Done()
					loadPkg(&wg, path, name)
				}(path, child.Name())
			}
		}
	}
	wg.Wait()
}

var fset = token.NewFileSet()

func loadPkg(wg *sync.WaitGroup, root, importpath string) {
	shortName := path.Base(importpath)

	dir := filepath.Join(root, importpath)
	pkgIndex.Lock()
	pkgIndex.m[shortName] = append(pkgIndex.m[shortName], pkg{
		importpath: importpath,
		dir:        dir,
	})
	pkgIndex.Unlock()

	pkgDir, err := os.Open(dir)
	if err != nil {
		return
	}
	children, err := pkgDir.Readdir(-1)
	pkgDir.Close()
	if err != nil {
		return
	}
	for _, child := range children {
		name := child.Name()
		if name == "" {
			continue
		}
		if c := name[0]; c == '.' || ('0' <= c && c <= '9') {
			continue
		}
		if child.IsDir() {
			wg.Add(1)
			go func(root, name string) {
				defer wg.Done()
				loadPkg(wg, root, name)
			}(root, filepath.Join(importpath, name))
		}
	}
}

// loadExports returns a list exports for a package.
func loadExports(dir string) map[string]bool {
	exports := make(map[string]bool)
	buildPkg, err := build.ImportDir(dir, 0)
	if err != nil {
		if strings.Contains(err.Error(), "no buildable Go source files in") {
			return nil
		}
		fmt.Fprintf(os.Stderr, "could not import %q: %v", dir, err)
		return nil
	}
	for _, file := range buildPkg.GoFiles {
		f, err := parser.ParseFile(fset, filepath.Join(dir, file), nil, 0)
		if err != nil {
			fmt.Fprintf(os.Stderr, "could not parse %q: %v", file, err)
			continue
		}
		for name := range f.Scope.Objects {
			if ast.IsExported(name) {
				exports[name] = true
			}
		}
	}
	return exports
}

// findImport searches for a package with the given symbols.
// If no package is found, findImport returns "".
// Declared as a variable rather than a function so goimports can be easily
// extended by adding a file with an init function.
var findImport = findImportGoPath

func findImportGoPath(pkgName string, symbols map[string]bool) (string, error) {

	pkgIndexOnce.Do(loadPkgIndex)

	// Collect exports for packages with matching names.
	var wg sync.WaitGroup
	var pkgsMu sync.Mutex // guards pkgs
	// full importpath => exported symbol => True
	// e.g. "net/http" => "Client" => True
	pkgs := make(map[string]map[string]bool)
	pkgIndex.Lock()
	for _, pkg := range pkgIndex.m[pkgName] {
		wg.Add(1)
		go func(importpath, dir string) {
			defer wg.Done()
			exports := loadExports(dir)
			if exports != nil {
				pkgsMu.Lock()
				pkgs[importpath] = exports
				pkgsMu.Unlock()
			}
		}(pkg.importpath, pkg.dir)
	}
	pkgIndex.Unlock()
	wg.Wait()

	// Filter out packages missing required exported symbols.
	for symbol := range symbols {
		for importpath, exports := range pkgs {
			if !exports[symbol] {
				delete(pkgs, importpath)
			}
		}
	}
	if len(pkgs) == 0 {
		return "", nil
	}

	// If there are multiple candidate packages, the shortest one wins.
	// This is a heuristic to prefer the standard library (e.g. "bytes")
	// over e.g. "github.com/foo/bar/bytes".
	shortest := ""
	for importPath := range pkgs {
		if shortest == "" || len(importPath) < len(shortest) {
			shortest = importPath
		}
	}
	return shortest, nil
}

type visitFn func(node ast.Node) ast.Visitor

func (fn visitFn) Visit(node ast.Node) ast.Visitor {
	return fn(node)
}
<|endoftext|>


<commit_before>// Package ftp implements a FTP client as described in RFC 959.
package ftp

import (
	"bufio"
	"errors"
	"fmt"
	"io"
	"net"
	"net/textproto"
	"strconv"
	"strings"
)

// EntryType describes the different types of an Entry.
type EntryType int

const (
	EntryTypeFile EntryType = iota
	EntryTypeFolder
	EntryTypeLink
)

// ServerConn represents the connection to a remote FTP server.
type ServerConn struct {
	conn     *textproto.Conn
	host     string
	features map[string]string
}

// Entry describes a file and is returned by List().
type Entry struct {
	Name string
	Type EntryType
	Size uint64
}

// response represent a data-connection
type response struct {
	conn net.Conn
	c    *ServerConn
}

// Connect initializes the connection to the specified ftp server address.
//
// It is generally followed by a call to Login() as most FTP commands require
// an authenticated user.
func Connect(addr string) (*ServerConn, error) {
	conn, err := textproto.Dial("tcp", addr)
	if err != nil {
		return nil, err
	}

	a := strings.SplitN(addr, ":", 2)
	c := &ServerConn{
		conn:     conn,
		host:     a[0],
		features: make(map[string]string),
	}

	_, _, err = c.conn.ReadCodeLine(StatusReady)
	if err != nil {
		c.Quit()
		return nil, err
	}

	err = c.feat()
	if err != nil {
		c.Quit()
		return nil, err
	}

	return c, nil
}

// Login authenticates the client with specified user and password.
//
// "anonymous"/"anonymous" is a common user/password scheme for FTP servers
// that allows anonymous read-only accounts.
func (c *ServerConn) Login(user, password string) error {
	_, _, err := c.cmd(StatusUserOK, "USER %s", user)
	if err != nil {
		return err
	}

	_, _, err = c.cmd(StatusLoggedIn, "PASS %s", password)
	if err != nil {
		return err
	}

	// Switch to binary mode
	_, _, err = c.cmd(StatusCommandOK, "TYPE I")
	if err != nil {
		return err
	}

	return nil
}

// feat issues a FEAT FTP command to list the additional commands supported by
// the remote FTP server.
// FEAT is described in RFC 2389
func (c *ServerConn) feat() error {
	code, message, err := c.cmd(-1, "FEAT")
	if err != nil {
		return err
	}

	if code != StatusSystem {
		// The server does not support the FEAT command. This is not an
		// error: we consider that there is no additional feature.
		return nil
	}

	lines := strings.Split(message, "\n")
	for _, line := range lines {
		if !strings.HasPrefix(line, " ") {
			continue
		}

		line = strings.Trim(line, " ")
		featureElements := strings.SplitN(line, " ", 2)

		command := featureElements[0]

		var commandDesc string
		if len(featureElements) == 2 {
			commandDesc = featureElements[1]
		}

		c.features[command] = commandDesc
	}

	return nil
}

// epsv issues an "EPSV" command to get a port number for a data connection.
func (c *ServerConn) epsv() (port int, err error) {
	_, line, err := c.cmd(StatusExtendedPassiveMode, "EPSV")
	if err != nil {
		return
	}

	start := strings.Index(line, "|||")
	end := strings.LastIndex(line, "|")
	if start == -1 || end == -1 {
		err = errors.New("Invalid EPSV response format")
		return
	}
	port, err = strconv.Atoi(line[start+3 : end])
	return
}

// openDataConn creates a new FTP data connection.
//
// Currently, only EPSV is implemented but a fallback to PASV, and to a lesser
// extent, PORT should be added.
func (c *ServerConn) openDataConn() (net.Conn, error) {
	port, err := c.epsv()
	if err != nil {
		return nil, err
	}

	// Build the new net address string
	addr := fmt.Sprintf("%s:%d", c.host, port)

	conn, err := net.Dial("tcp", addr)
	if err != nil {
		return nil, err
	}

	return conn, nil
}

// cmd is a helper function to execute a command and check for the expected FTP
// return code
func (c *ServerConn) cmd(expected int, format string, args ...interface{}) (int, string, error) {
	_, err := c.conn.Cmd(format, args...)
	if err != nil {
		return 0, "", err
	}

	code, line, err := c.conn.ReadResponse(expected)
	return code, line, err
}

// cmdDataConn executes a command which require a FTP data connection.
func (c *ServerConn) cmdDataConn(format string, args ...interface{}) (net.Conn, error) {
	conn, err := c.openDataConn()
	if err != nil {
		return nil, err
	}

	_, err = c.conn.Cmd(format, args...)
	if err != nil {
		conn.Close()
		return nil, err
	}

	code, msg, err := c.conn.ReadCodeLine(-1)
	if err != nil {
		conn.Close()
		return nil, err
	}
	if code != StatusAlreadyOpen && code != StatusAboutToSend {
		conn.Close()
		return nil, &textproto.Error{code, msg}
	}

	return conn, nil
}

// parseListLine parses the various non-standard format returned by the LIST
// FTP command.
func parseListLine(line string) (*Entry, error) {
	fields := strings.Fields(line)
	if len(fields) < 9 {
		return nil, errors.New("Unsupported LIST line")
	}

	e := &Entry{}
	switch fields[0][0] {
	case '-':
		e.Type = EntryTypeFile
	case 'd':
		e.Type = EntryTypeFolder
	case 'l':
		e.Type = EntryTypeLink
	default:
		return nil, errors.New("Unknown entry type")
	}

	if e.Type == EntryTypeFile {
		size, err := strconv.ParseUint(fields[4], 10, 0)
		if err != nil {
			return nil, err
		}
		e.Size = size
	}

	e.Name = strings.Join(fields[8:], " ")
	return e, nil
}

// List issues a LIST FTP command.
func (c *ServerConn) List(path string) (entries []*Entry, err error) {
	conn, err := c.cmdDataConn("LIST %s", path)
	if err != nil {
		return
	}

	r := &response{conn, c}
	defer r.Close()

	bio := bufio.NewReader(r)
	for {
		line, e := bio.ReadString('\n')
		if e == io.EOF {
			break
		} else if e != nil {
			return nil, e
		}
		entry, err := parseListLine(line)
		if err == nil {
			entries = append(entries, entry)
		}
	}
	return
}

// ChangeDir issues a CWD FTP command, which changes the current directory to
// the specified path.
func (c *ServerConn) ChangeDir(path string) error {
	_, _, err := c.cmd(StatusRequestedFileActionOK, "CWD %s", path)
	return err
}

// ChangeDirToParent issues a CDUP FTP command, which changes the current
// directory to the parent directory.  This is similar to a call to ChangeDir
// with a path set to "..".
func (c *ServerConn) ChangeDirToParent() error {
	_, _, err := c.cmd(StatusRequestedFileActionOK, "CDUP")
	return err
}

// CurrentDir issues a PWD FTP command, which Returns the path of the current
// directory.
func (c *ServerConn) CurrentDir() (string, error) {
	_, msg, err := c.cmd(StatusPathCreated, "PWD")
	if err != nil {
		return "", err
	}

	start := strings.Index(msg, "\"")
	end := strings.LastIndex(msg, "\"")

	if start == -1 || end == -1 {
		return "", errors.New("Unsuported PWD response format")
	}

	return msg[start+1 : end], nil
}

// Retr issues a RETR FTP command to fetch the specified file from the remote
// FTP server.
//
// The returned ReadCloser must be closed to cleanup the FTP data connection.
func (c *ServerConn) Retr(path string) (io.ReadCloser, error) {
	conn, err := c.cmdDataConn("RETR %s", path)
	if err != nil {
		return nil, err
	}

	r := &response{conn, c}
	return r, nil
}

// Stor issues a STOR FTP command to store a file to the remote FTP server.
// Stor creates the specified file with the content of the io.Reader.
//
// Hint: io.Pipe() can be used if an io.Writer is required.
func (c *ServerConn) Stor(path string, r io.Reader) error {
	conn, err := c.cmdDataConn("STOR %s", path)
	if err != nil {
		return err
	}

	_, err = io.Copy(conn, r)
	conn.Close()
	if err != nil {
		return err
	}

	_, _, err = c.conn.ReadCodeLine(StatusClosingDataConnection)
	return err
}

// Rename renames a file on the remote FTP server.
func (c *ServerConn) Rename(from, to string) error {
	_, _, err := c.cmd(StatusRequestFilePending, "RNFR %s", from)
	if err != nil {
		return err
	}

	_, _, err = c.cmd(StatusRequestedFileActionOK, "RNTO %s", to)
	return err
}

// Delete issues a DELE FTP command to delete the specified file from the
// remote FTP server.
func (c *ServerConn) Delete(path string) error {
	_, _, err := c.cmd(StatusRequestedFileActionOK, "DELE %s", path)
	return err
}

// MakeDir issues a MKD FTP command to create the specified directory on the
// remote FTP server.
func (c *ServerConn) MakeDir(path string) error {
	_, _, err := c.cmd(StatusPathCreated, "MKD %s", path)
	return err
}

// RemoveDir issues a RMD FTP command to remove the specified directory from
// the remote FTP server.
func (c *ServerConn) RemoveDir(path string) error {
	_, _, err := c.cmd(StatusRequestedFileActionOK, "RMD %s", path)
	return err
}

// NoOp issues a NOOP FTP command.
// NOOP has no effects and is usually used to prevent the remote FTP server to
// close the otherwise idle connection.
func (c *ServerConn) NoOp() error {
	_, _, err := c.cmd(StatusCommandOK, "NOOP")
	return err
}

// Logout issues a REIN FTP command to logout the current user.
func (c *ServerConn) Logout() error {
	_, _, err := c.cmd(StatusLoggedIn, "REIN")
	return err
}

// Quit issues a QUIT FTP command to properly close the connection from the
// remote FTP server.
func (c *ServerConn) Quit() error {
	c.conn.Cmd("QUIT")
	return c.conn.Close()
}

// Read implements the io.Reader interface on a FTP data connection.
func (r *response) Read(buf []byte) (int, error) {
	n, err := r.conn.Read(buf)
	if err == io.EOF {
		_, _, err2 := r.c.conn.ReadCodeLine(StatusClosingDataConnection)
		if err2 != nil {
			err = err2
		}
	}
	return n, err
}

// Close implements the io.Closer interface on a FTP data connection.
func (r *response) Close() error {
	return r.conn.Close()
}
<commit_msg>Use strings.TrimSpace() instead of strings.Trim().<commit_after>// Package ftp implements a FTP client as described in RFC 959.
package ftp

import (
	"bufio"
	"errors"
	"fmt"
	"io"
	"net"
	"net/textproto"
	"strconv"
	"strings"
)

// EntryType describes the different types of an Entry.
type EntryType int

const (
	EntryTypeFile EntryType = iota
	EntryTypeFolder
	EntryTypeLink
)

// ServerConn represents the connection to a remote FTP server.
type ServerConn struct {
	conn     *textproto.Conn
	host     string
	features map[string]string
}

// Entry describes a file and is returned by List().
type Entry struct {
	Name string
	Type EntryType
	Size uint64
}

// response represent a data-connection
type response struct {
	conn net.Conn
	c    *ServerConn
}

// Connect initializes the connection to the specified ftp server address.
//
// It is generally followed by a call to Login() as most FTP commands require
// an authenticated user.
func Connect(addr string) (*ServerConn, error) {
	conn, err := textproto.Dial("tcp", addr)
	if err != nil {
		return nil, err
	}

	a := strings.SplitN(addr, ":", 2)
	c := &ServerConn{
		conn:     conn,
		host:     a[0],
		features: make(map[string]string),
	}

	_, _, err = c.conn.ReadCodeLine(StatusReady)
	if err != nil {
		c.Quit()
		return nil, err
	}

	err = c.feat()
	if err != nil {
		c.Quit()
		return nil, err
	}

	return c, nil
}

// Login authenticates the client with specified user and password.
//
// "anonymous"/"anonymous" is a common user/password scheme for FTP servers
// that allows anonymous read-only accounts.
func (c *ServerConn) Login(user, password string) error {
	_, _, err := c.cmd(StatusUserOK, "USER %s", user)
	if err != nil {
		return err
	}

	_, _, err = c.cmd(StatusLoggedIn, "PASS %s", password)
	if err != nil {
		return err
	}

	// Switch to binary mode
	_, _, err = c.cmd(StatusCommandOK, "TYPE I")
	if err != nil {
		return err
	}

	return nil
}

// feat issues a FEAT FTP command to list the additional commands supported by
// the remote FTP server.
// FEAT is described in RFC 2389
func (c *ServerConn) feat() error {
	code, message, err := c.cmd(-1, "FEAT")
	if err != nil {
		return err
	}

	if code != StatusSystem {
		// The server does not support the FEAT command. This is not an
		// error: we consider that there is no additional feature.
		return nil
	}

	lines := strings.Split(message, "\n")
	for _, line := range lines {
		if !strings.HasPrefix(line, " ") {
			continue
		}

		line = strings.TrimSpace(line)
		featureElements := strings.SplitN(line, " ", 2)

		command := featureElements[0]

		var commandDesc string
		if len(featureElements) == 2 {
			commandDesc = featureElements[1]
		}

		c.features[command] = commandDesc
	}

	return nil
}

// epsv issues an "EPSV" command to get a port number for a data connection.
func (c *ServerConn) epsv() (port int, err error) {
	_, line, err := c.cmd(StatusExtendedPassiveMode, "EPSV")
	if err != nil {
		return
	}

	start := strings.Index(line, "|||")
	end := strings.LastIndex(line, "|")
	if start == -1 || end == -1 {
		err = errors.New("Invalid EPSV response format")
		return
	}
	port, err = strconv.Atoi(line[start+3 : end])
	return
}

// openDataConn creates a new FTP data connection.
//
// Currently, only EPSV is implemented but a fallback to PASV, and to a lesser
// extent, PORT should be added.
func (c *ServerConn) openDataConn() (net.Conn, error) {
	port, err := c.epsv()
	if err != nil {
		return nil, err
	}

	// Build the new net address string
	addr := fmt.Sprintf("%s:%d", c.host, port)

	conn, err := net.Dial("tcp", addr)
	if err != nil {
		return nil, err
	}

	return conn, nil
}

// cmd is a helper function to execute a command and check for the expected FTP
// return code
func (c *ServerConn) cmd(expected int, format string, args ...interface{}) (int, string, error) {
	_, err := c.conn.Cmd(format, args...)
	if err != nil {
		return 0, "", err
	}

	code, line, err := c.conn.ReadResponse(expected)
	return code, line, err
}

// cmdDataConn executes a command which require a FTP data connection.
func (c *ServerConn) cmdDataConn(format string, args ...interface{}) (net.Conn, error) {
	conn, err := c.openDataConn()
	if err != nil {
		return nil, err
	}

	_, err = c.conn.Cmd(format, args...)
	if err != nil {
		conn.Close()
		return nil, err
	}

	code, msg, err := c.conn.ReadCodeLine(-1)
	if err != nil {
		conn.Close()
		return nil, err
	}
	if code != StatusAlreadyOpen && code != StatusAboutToSend {
		conn.Close()
		return nil, &textproto.Error{code, msg}
	}

	return conn, nil
}

// parseListLine parses the various non-standard format returned by the LIST
// FTP command.
func parseListLine(line string) (*Entry, error) {
	fields := strings.Fields(line)
	if len(fields) < 9 {
		return nil, errors.New("Unsupported LIST line")
	}

	e := &Entry{}
	switch fields[0][0] {
	case '-':
		e.Type = EntryTypeFile
	case 'd':
		e.Type = EntryTypeFolder
	case 'l':
		e.Type = EntryTypeLink
	default:
		return nil, errors.New("Unknown entry type")
	}

	if e.Type == EntryTypeFile {
		size, err := strconv.ParseUint(fields[4], 10, 0)
		if err != nil {
			return nil, err
		}
		e.Size = size
	}

	e.Name = strings.Join(fields[8:], " ")
	return e, nil
}

// List issues a LIST FTP command.
func (c *ServerConn) List(path string) (entries []*Entry, err error) {
	conn, err := c.cmdDataConn("LIST %s", path)
	if err != nil {
		return
	}

	r := &response{conn, c}
	defer r.Close()

	bio := bufio.NewReader(r)
	for {
		line, e := bio.ReadString('\n')
		if e == io.EOF {
			break
		} else if e != nil {
			return nil, e
		}
		entry, err := parseListLine(line)
		if err == nil {
			entries = append(entries, entry)
		}
	}
	return
}

// ChangeDir issues a CWD FTP command, which changes the current directory to
// the specified path.
func (c *ServerConn) ChangeDir(path string) error {
	_, _, err := c.cmd(StatusRequestedFileActionOK, "CWD %s", path)
	return err
}

// ChangeDirToParent issues a CDUP FTP command, which changes the current
// directory to the parent directory.  This is similar to a call to ChangeDir
// with a path set to "..".
func (c *ServerConn) ChangeDirToParent() error {
	_, _, err := c.cmd(StatusRequestedFileActionOK, "CDUP")
	return err
}

// CurrentDir issues a PWD FTP command, which Returns the path of the current
// directory.
func (c *ServerConn) CurrentDir() (string, error) {
	_, msg, err := c.cmd(StatusPathCreated, "PWD")
	if err != nil {
		return "", err
	}

	start := strings.Index(msg, "\"")
	end := strings.LastIndex(msg, "\"")

	if start == -1 || end == -1 {
		return "", errors.New("Unsuported PWD response format")
	}

	return msg[start+1 : end], nil
}

// Retr issues a RETR FTP command to fetch the specified file from the remote
// FTP server.
//
// The returned ReadCloser must be closed to cleanup the FTP data connection.
func (c *ServerConn) Retr(path string) (io.ReadCloser, error) {
	conn, err := c.cmdDataConn("RETR %s", path)
	if err != nil {
		return nil, err
	}

	r := &response{conn, c}
	return r, nil
}

// Stor issues a STOR FTP command to store a file to the remote FTP server.
// Stor creates the specified file with the content of the io.Reader.
//
// Hint: io.Pipe() can be used if an io.Writer is required.
func (c *ServerConn) Stor(path string, r io.Reader) error {
	conn, err := c.cmdDataConn("STOR %s", path)
	if err != nil {
		return err
	}

	_, err = io.Copy(conn, r)
	conn.Close()
	if err != nil {
		return err
	}

	_, _, err = c.conn.ReadCodeLine(StatusClosingDataConnection)
	return err
}

// Rename renames a file on the remote FTP server.
func (c *ServerConn) Rename(from, to string) error {
	_, _, err := c.cmd(StatusRequestFilePending, "RNFR %s", from)
	if err != nil {
		return err
	}

	_, _, err = c.cmd(StatusRequestedFileActionOK, "RNTO %s", to)
	return err
}

// Delete issues a DELE FTP command to delete the specified file from the
// remote FTP server.
func (c *ServerConn) Delete(path string) error {
	_, _, err := c.cmd(StatusRequestedFileActionOK, "DELE %s", path)
	return err
}

// MakeDir issues a MKD FTP command to create the specified directory on the
// remote FTP server.
func (c *ServerConn) MakeDir(path string) error {
	_, _, err := c.cmd(StatusPathCreated, "MKD %s", path)
	return err
}

// RemoveDir issues a RMD FTP command to remove the specified directory from
// the remote FTP server.
func (c *ServerConn) RemoveDir(path string) error {
	_, _, err := c.cmd(StatusRequestedFileActionOK, "RMD %s", path)
	return err
}

// NoOp issues a NOOP FTP command.
// NOOP has no effects and is usually used to prevent the remote FTP server to
// close the otherwise idle connection.
func (c *ServerConn) NoOp() error {
	_, _, err := c.cmd(StatusCommandOK, "NOOP")
	return err
}

// Logout issues a REIN FTP command to logout the current user.
func (c *ServerConn) Logout() error {
	_, _, err := c.cmd(StatusLoggedIn, "REIN")
	return err
}

// Quit issues a QUIT FTP command to properly close the connection from the
// remote FTP server.
func (c *ServerConn) Quit() error {
	c.conn.Cmd("QUIT")
	return c.conn.Close()
}

// Read implements the io.Reader interface on a FTP data connection.
func (r *response) Read(buf []byte) (int, error) {
	n, err := r.conn.Read(buf)
	if err == io.EOF {
		_, _, err2 := r.c.conn.ReadCodeLine(StatusClosingDataConnection)
		if err2 != nil {
			err = err2
		}
	}
	return n, err
}

// Close implements the io.Closer interface on a FTP data connection.
func (r *response) Close() error {
	return r.conn.Close()
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io/ioutil"

	"context"

	"cloud.google.com/go/storage"
)

// A GCSBackend implements Backend and connects to google cloud storage
type GCSBackend struct {
	bucket string
}

// Implements Backend
func (b *GCSBackend) Init() error {
	client, err := storage.NewClient(context.Background())
	if err != nil {
		return err
	}

	handle := client.Bucket(b.bucket)
	_, err = handle.Attrs(context.Background())

	return err
}

// Implements RecordedTape
func (b GCSBackend) RecordedTape(ctx context.Context, name string, i Incrementer) (*RecordedTape, error) {
	client, err := storage.NewClient(ctx)
	handle := client.Bucket(b.bucket)

	return &RecordedTape{
		tape: &GCSTape{
			ctx:    ctx,
			handle: handle,
			name:   name,
			i:      i,
		},
	}, err
}

// Implements BlankTape
func (b GCSBackend) BlankTape(ctx context.Context, name string, i Incrementer) (*BlankTape, error) {
	client, err := storage.NewClient(ctx)
	handle := client.Bucket(b.bucket)

	return &BlankTape{
		tape: &GCSTape{
			handle: handle,
			name:   name,
			i:      i,
		},
	}, err
}

// A GCSTape implements BlankTape and RecordedTape
// and stores entries with an expiration according to TTL
type GCSTape struct {
	name   string
	i      Incrementer
	handle *storage.BucketHandle
	ctx    context.Context
}

func (t *GCSTape) Write(data []byte) error {
	name := fmt.Sprintf("%s/%s.chunk", t.name, t.i.Key())
	w := t.handle.Object(name).NewWriter(t.ctx)
	defer w.Close()

	if _, err := w.Write(data); err != nil {
		return err
	}

	return nil
}

func (t *GCSTape) Read() ([]byte, error) {
	name := fmt.Sprintf("%s/%s.chunk", t.name, t.i.Key())

	r, err := t.handle.Object(name).NewReader(t.ctx)
	if err != nil {
		return []byte{}, err
	}
	defer r.Close()

	return ioutil.ReadAll(r)
}
<commit_msg>Fix gcs context in blank tape<commit_after>package main

import (
	"fmt"
	"io/ioutil"

	"context"

	"cloud.google.com/go/storage"
)

// A GCSBackend implements Backend and connects to google cloud storage
type GCSBackend struct {
	bucket string
}

// Implements Backend
func (b *GCSBackend) Init() error {
	client, err := storage.NewClient(context.Background())
	if err != nil {
		return err
	}

	handle := client.Bucket(b.bucket)
	_, err = handle.Attrs(context.Background())

	return err
}

// Implements RecordedTape
func (b GCSBackend) RecordedTape(ctx context.Context, name string, i Incrementer) (*RecordedTape, error) {
	client, err := storage.NewClient(ctx)
	handle := client.Bucket(b.bucket)

	return &RecordedTape{
		tape: &GCSTape{
			ctx:    ctx,
			handle: handle,
			name:   name,
			i:      i,
		},
	}, err
}

// Implements BlankTape
func (b GCSBackend) BlankTape(ctx context.Context, name string, i Incrementer) (*BlankTape, error) {
	client, err := storage.NewClient(ctx)
	handle := client.Bucket(b.bucket)

	return &BlankTape{
		tape: &GCSTape{
			ctx:    ctx,
			handle: handle,
			name:   name,
			i:      i,
		},
	}, err
}

// A GCSTape implements BlankTape and RecordedTape
// and stores entries with an expiration according to TTL
type GCSTape struct {
	name   string
	i      Incrementer
	handle *storage.BucketHandle
	ctx    context.Context
}

func (t *GCSTape) Write(data []byte) error {
	name := fmt.Sprintf("%s/%s.chunk", t.name, t.i.Key())
	w := t.handle.Object(name).NewWriter(t.ctx)
	defer w.Close()

	if _, err := w.Write(data); err != nil {
		return err
	}

	return nil
}

func (t *GCSTape) Read() ([]byte, error) {
	name := fmt.Sprintf("%s/%s.chunk", t.name, t.i.Key())

	r, err := t.handle.Object(name).NewReader(t.ctx)
	if err != nil {
		return []byte{}, err
	}
	defer r.Close()

	return ioutil.ReadAll(r)
}
<|endoftext|>


<commit_before>package git

import (
	"fmt"
	"io"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"
	"time"

	"golang.org/x/net/context"
)

// Command invokes a `command` in `workdir` with `args`, connecting Stdout and
// Stderr to Stderr.
func Command(workdir, command string, args ...string) *exec.Cmd {
	// log.Printf("wd = %s cmd = %s, args = %q", workdir, command, append([]string{}, args...))
	cmd := exec.Command(command, args...)
	cmd.Dir = workdir
	cmd.Stdout = os.Stderr
	cmd.Stderr = os.Stderr
	return cmd
}

// LocalMirror creates or updates a mirror of `url` at `gitDir` using `git clone
// --mirror`.
func LocalMirror(url, gitDir, ref string, messages io.Writer) error {
	// When mirroring, allow up to two minutes before giving up.
	const MirrorTimeout = 2 * time.Minute
	ctx, done := context.WithTimeout(context.Background(), MirrorTimeout)
	defer done()

	if _, err := os.Stat(gitDir); err == nil {
		// Repo already exists, don't need to clone it.

		if AlreadyHaveRef(gitDir, ref) {
			// Sha already exists, don't need to fetch.
			// log.Printf("Already have ref: %v %v", gitDir, ref)
			return nil
		}

		return Fetch(ctx, gitDir, url, messages)
	}

	err := os.MkdirAll(filepath.Dir(gitDir), 0777)
	if err != nil {
		return err
	}

	return Clone(ctx, url, gitDir, messages)
}

// Clone clones a git repository as mirror.
func Clone(ctx context.Context, url, gitDir string, messages io.Writer) error {
	cmd := Command(".", "git", "clone", "-q", "--mirror", url, gitDir)
	cmd.Stdout = messages
	cmd.Stderr = messages
	return ContextRun(ctx, cmd)
}

// ContextRun runs cmd within a net Context.
// If the context is cancelled or times out, the process is killed.
func ContextRun(ctx context.Context, cmd *exec.Cmd) error {
	errc := make(chan error)

	err := cmd.Start()
	if err != nil {
		return err
	}

	go func() { errc <- cmd.Wait() }()

	select {
	case <-ctx.Done():
		_ = cmd.Process.Kill()
		return ctx.Err()
	case err := <-errc:
		return err // err may be nil
	}
}

// Checkout switches branches or restores working tree files.
func Checkout(gitDir, checkoutDir, ref string) error {
	err := os.MkdirAll(checkoutDir, 0777)
	if err != nil {
		return err
	}

	args := []string{"--work-tree", checkoutDir, "checkout", ref, "--", "."}
	err = Command(gitDir, "git", args...).Run()
	if err != nil {
		return err
	}

	// Set mtimes to time file is most recently affected by a commit.
	// This is annoying but unfortunately git sets the timestamps to now,
	// and docker depends on the mtime for cache invalidation.
	err = SetMTimes(gitDir, checkoutDir, ref)
	if err != nil {
		return err
	}

	return nil
}

// Fetch fetches all branches from a given remote.
func Fetch(ctx context.Context, gitDir, url string, messages io.Writer) error {
	cmd := Command(gitDir, "git", "fetch", "-f", url, "*:*")
	cmd.Stdout = messages
	cmd.Stderr = messages

	err := ContextRun(ctx, cmd)
	if err != nil {
		// git fetch where there is no update is exit status 1.
		if err.Error() != "exit status 1" {
			return err
		}
	}

	return nil
}

// ShaLike specifies a valid git hash.
var ShaLike = regexp.MustCompile("[0-9a-zA-Z]{40}")

// AlreadyHaveRef returns true if ref is sha-like and is in the object database.
// The "sha-like" condition ensures that refs like `master` are always
// freshened.
func AlreadyHaveRef(gitDir, sha string) bool {
	if !ShaLike.MatchString(sha) {
		return false
	}
	cmd := Command(gitDir, "git", "cat-file", "-t", sha)
	cmd.Stdout = nil
	cmd.Stderr = nil

	err := cmd.Run()
	return err == nil
}

// HaveFile checks if a git directory has files checked out.
func HaveFile(gitDir, ref, path string) (ok bool, err error) {
	cmd := Command(gitDir, "git", "show", fmt.Sprintf("%s:%s", ref, path))
	cmd.Stdout = nil // don't want to see the contents
	err = cmd.Run()
	ok = true
	if err != nil {
		ok = false
		if err.Error() == "exit status 128" {
			// This happens if the file doesn't exist.
			err = nil
		}
	}
	return ok, err
}

// RevParse parses and formats the git rev of a given git reference.
func RevParse(gitDir, ref string) (sha string, err error) {
	cmd := Command(gitDir, "git", "rev-parse", ref)
	cmd.Stdout = nil // for cmd.Output

	var stdout []byte
	stdout, err = cmd.Output()
	if err != nil {
		return "", err
	}

	sha = strings.TrimSpace(string(stdout))
	return sha, nil
}

// Describe describes a commit given a reference using the most recent tag
// reachable from it.
func Describe(gitDir, ref string) (desc string, err error) {
	cmd := Command(gitDir, "git", "describe", "--all", "--tags", "--long", ref)
	cmd.Stdout = nil // for cmd.Output

	var stdout []byte
	stdout, err = cmd.Output()
	if err != nil {
		return "", err
	}

	desc = strings.TrimSpace(string(stdout))
	desc = strings.TrimPrefix(desc, "heads/")
	return desc, nil
}

// RecursiveCheckout recursively checks out repositories; similar to "git clone
// --recursive".
func RecursiveCheckout(gitDir, checkoutPath, rev string) error {
	err := Checkout(gitDir, checkoutPath, rev)
	if err != nil {
		return fmt.Errorf("failed to checkout: %v", err)
	}

	err = PrepSubmodules(gitDir, checkoutPath, rev)
	if err != nil {
		return fmt.Errorf("failed to prep submodules: %v", err)
	}
	return nil
}
<commit_msg>Reorder functions by visibility<commit_after>package git

import (
	"fmt"
	"io"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"
	"time"

	"golang.org/x/net/context"
)

// LocalMirror creates or updates a mirror of `url` at `gitDir` using `git clone
// --mirror`.
func LocalMirror(url, gitDir, ref string, messages io.Writer) error {
	// When mirroring, allow up to two minutes before giving up.
	const MirrorTimeout = 2 * time.Minute
	ctx, done := context.WithTimeout(context.Background(), MirrorTimeout)
	defer done()

	if _, err := os.Stat(gitDir); err == nil {
		// Repo already exists, don't need to clone it.

		if AlreadyHaveRef(gitDir, ref) {
			// Sha already exists, don't need to fetch.
			// log.Printf("Already have ref: %v %v", gitDir, ref)
			return nil
		}

		return Fetch(ctx, gitDir, url, messages)
	}

	err := os.MkdirAll(filepath.Dir(gitDir), 0777)
	if err != nil {
		return err
	}

	return Clone(ctx, url, gitDir, messages)
}

// Clone clones a git repository as mirror.
func Clone(ctx context.Context, url, gitDir string, messages io.Writer) error {
	cmd := Command(".", "git", "clone", "-q", "--mirror", url, gitDir)
	cmd.Stdout = messages
	cmd.Stderr = messages
	return ContextRun(ctx, cmd)
}

// Checkout switches branches or restores working tree files.
func Checkout(gitDir, checkoutDir, ref string) error {
	err := os.MkdirAll(checkoutDir, 0777)
	if err != nil {
		return err
	}

	args := []string{"--work-tree", checkoutDir, "checkout", ref, "--", "."}
	err = Command(gitDir, "git", args...).Run()
	if err != nil {
		return err
	}

	// Set mtimes to time file is most recently affected by a commit.
	// This is annoying but unfortunately git sets the timestamps to now,
	// and docker depends on the mtime for cache invalidation.
	err = SetMTimes(gitDir, checkoutDir, ref)
	if err != nil {
		return err
	}

	return nil
}

// Fetch fetches all branches from a given remote.
func Fetch(ctx context.Context, gitDir, url string, messages io.Writer) error {
	cmd := Command(gitDir, "git", "fetch", "-f", url, "*:*")
	cmd.Stdout = messages
	cmd.Stderr = messages

	err := ContextRun(ctx, cmd)
	if err != nil {
		// git fetch where there is no update is exit status 1.
		if err.Error() != "exit status 1" {
			return err
		}
	}

	return nil
}

// ShaLike specifies a valid git hash.
var ShaLike = regexp.MustCompile("[0-9a-zA-Z]{40}")

// AlreadyHaveRef returns true if ref is sha-like and is in the object database.
// The "sha-like" condition ensures that refs like `master` are always
// freshened.
func AlreadyHaveRef(gitDir, sha string) bool {
	if !ShaLike.MatchString(sha) {
		return false
	}
	cmd := Command(gitDir, "git", "cat-file", "-t", sha)
	cmd.Stdout = nil
	cmd.Stderr = nil

	err := cmd.Run()
	return err == nil
}

// HaveFile checks if a git directory has files checked out.
func HaveFile(gitDir, ref, path string) (ok bool, err error) {
	cmd := Command(gitDir, "git", "show", fmt.Sprintf("%s:%s", ref, path))
	cmd.Stdout = nil // don't want to see the contents
	err = cmd.Run()
	ok = true
	if err != nil {
		ok = false
		if err.Error() == "exit status 128" {
			// This happens if the file doesn't exist.
			err = nil
		}
	}
	return ok, err
}

// RevParse parses and formats the git rev of a given git reference.
func RevParse(gitDir, ref string) (sha string, err error) {
	cmd := Command(gitDir, "git", "rev-parse", ref)
	cmd.Stdout = nil // for cmd.Output

	var stdout []byte
	stdout, err = cmd.Output()
	if err != nil {
		return "", err
	}

	sha = strings.TrimSpace(string(stdout))
	return sha, nil
}

// Describe describes a commit given a reference using the most recent tag
// reachable from it.
func Describe(gitDir, ref string) (desc string, err error) {
	cmd := Command(gitDir, "git", "describe", "--all", "--tags", "--long", ref)
	cmd.Stdout = nil // for cmd.Output

	var stdout []byte
	stdout, err = cmd.Output()
	if err != nil {
		return "", err
	}

	desc = strings.TrimSpace(string(stdout))
	desc = strings.TrimPrefix(desc, "heads/")
	return desc, nil
}

// RecursiveCheckout recursively checks out repositories; similar to "git clone
// --recursive".
func RecursiveCheckout(gitDir, checkoutPath, rev string) error {
	err := Checkout(gitDir, checkoutPath, rev)
	if err != nil {
		return fmt.Errorf("failed to checkout: %v", err)
	}

	err = PrepSubmodules(gitDir, checkoutPath, rev)
	if err != nil {
		return fmt.Errorf("failed to prep submodules: %v", err)
	}
	return nil
}

// Command invokes a `command` in `workdir` with `args`, connecting Stdout and
// Stderr to Stderr.
func Command(workdir, command string, args ...string) *exec.Cmd {
	// log.Printf("wd = %s cmd = %s, args = %q", workdir, command, append([]string{}, args...))
	cmd := exec.Command(command, args...)
	cmd.Dir = workdir
	cmd.Stdout = os.Stderr
	cmd.Stderr = os.Stderr
	return cmd
}

// ContextRun runs cmd within a net Context.
// If the context is cancelled or times out, the process is killed.
func ContextRun(ctx context.Context, cmd *exec.Cmd) error {
	errc := make(chan error)

	err := cmd.Start()
	if err != nil {
		return err
	}

	go func() { errc <- cmd.Wait() }()

	select {
	case <-ctx.Done():
		_ = cmd.Process.Kill()
		return ctx.Err()
	case err := <-errc:
		return err // err may be nil
	}
}
<|endoftext|>


<commit_before>// Memcache session support for Gorilla Web Toolkit,
// without Google App Engine dependency.
package gsm

import (
	"bytes"
	"encoding/base32"
	"encoding/gob"
	"encoding/json"
	"fmt"
	"github.com/bradfitz/gomemcache/memcache"
	"github.com/gorilla/securecookie"
	"github.com/gorilla/sessions"
	"log"
	"net/http"
	"strings"
)

// NewMemcacheStore returns a new MemcacheStore.
// You need to provide the memcache client and
// an optional prefix for the keys we store
func NewMemcacheStore(client *memcache.Client, keyPrefix string, keyPairs ...[]byte) *MemcacheStore {

	if client == nil {
		panic("Cannot have nil memcache client")
	}

	return &MemcacheStore{
		Codecs: securecookie.CodecsFromPairs(keyPairs...),
		Options: &sessions.Options{
			Path:   "/",
			MaxAge: 86400 * 30,
		},
		KeyPrefix:   keyPrefix,
		Client:      client,
		StoreMethod: StoreMethodSecureCookie,
	}
}

type StoreMethod string

// take your pick on how to store the values in memcache
const (
	StoreMethodSecureCookie = StoreMethod("securecookie") // security
	StoreMethodGob          = StoreMethod("gob")          // speed
	StoreMethodJson         = StoreMethod("json")         // simplicity; warning: only string keys allowed and rest of data must be JSON.Marshal compatible
)

// MemcacheStore stores sessions in memcache
//
type MemcacheStore struct {
	Codecs      []securecookie.Codec
	Options     *sessions.Options // default configuration
	Client      *memcache.Client
	KeyPrefix   string
	Logging     int // set to > 0 to enable logging (using log.Printf)
	StoreMethod StoreMethod
}

// MaxLength restricts the maximum length of new sessions to l.
// If l is 0 there is no limit to the size of a session, use with caution.
// The default for a new MemcacheStore is 4096.
func (s *MemcacheStore) MaxLength(l int) {
	for _, c := range s.Codecs {
		if codec, ok := c.(*securecookie.SecureCookie); ok {
			codec.MaxLength(l)
		}
	}
}

// Get returns a session for the given name after adding it to the registry.
//
// See CookieStore.Get().
func (s *MemcacheStore) Get(r *http.Request, name string) (*sessions.Session, error) {
	return sessions.GetRegistry(r).Get(s, name)
}

// New returns a session for the given name without adding it to the registry.
//
// See CookieStore.New().
func (s *MemcacheStore) New(r *http.Request, name string) (*sessions.Session, error) {
	session := sessions.NewSession(s, name)
	opts := *s.Options
	session.Options = &opts
	session.IsNew = true
	var err error
	if c, errCookie := r.Cookie(name); errCookie == nil {
		err = securecookie.DecodeMulti(name, c.Value, &session.ID, s.Codecs...)
		if err == nil {
			err = s.load(session)
			if err == nil {
				session.IsNew = false
			}
		}
	}
	return session, err
}

// Save adds a single session to the response.
func (s *MemcacheStore) Save(r *http.Request, w http.ResponseWriter,
	session *sessions.Session) error {
	if session.ID == "" {
		// Because the ID is used in the filename, encode it to
		// use alphanumeric characters only.
		session.ID = strings.TrimRight(
			base32.StdEncoding.EncodeToString(
				securecookie.GenerateRandomKey(32)), "=")
	}
	if err := s.save(session); err != nil {
		return err
	}
	encoded, err := securecookie.EncodeMulti(session.Name(), session.ID,
		s.Codecs...)
	if err != nil {
		return err
	}
	http.SetCookie(w, sessions.NewCookie(session.Name(), encoded, session.Options))
	return nil
}

// save writes encoded session.Values using the memcache client
func (s *MemcacheStore) save(session *sessions.Session) error {

	key := s.KeyPrefix + session.ID

	switch s.StoreMethod {

	case StoreMethodSecureCookie:

		encoded, err := securecookie.EncodeMulti(session.Name(), session.Values,
			s.Codecs...)
		if err != nil {
			if s.Logging > 0 {
				log.Printf("gorilla-sessions-memcache: set (method: securecookie, encoding error: %v)", err)
			}
			return err
		}

		err = s.Client.Set(&memcache.Item{Key: key, Value: []byte(encoded)})
		if s.Logging > 0 {
			log.Printf("gorilla-sessions-memcache: set (method: securecookie, session name: %v, memcache key: %v, memcache value: %v, error: %v)", session.Name(), key, encoded, err)
		}
		if err != nil {
			return err
		}

		return nil

	case StoreMethodGob:

		buf := &bytes.Buffer{}
		enc := gob.NewEncoder(buf)
		if err := enc.Encode(session.Values); err != nil {
			if s.Logging > 0 {
				log.Printf("gorilla-sessions-memcache: set (method: gob, encoding error: %v)", err)
			}
			return err
		}
		bufbytes := buf.Bytes()

		err := s.Client.Set(&memcache.Item{Key: key, Value: bufbytes})
		if s.Logging > 0 {
			log.Printf("gorilla-sessions-memcache: set (method: gob, session name: %v, memcache key: %v, memcache value len: %v, error: %v)", session.Name(), key, len(bufbytes), err)
		}
		if err != nil {
			return err
		}

		return nil

	case StoreMethodJson:

		vals := make(map[string]interface{}, len(session.Values))
		for k, v := range session.Values {
			ks, ok := k.(string)
			if !ok {
				err := fmt.Errorf("Non-string key value, cannot jsonize: %v", k)
				log.Printf("gorilla-sessions-memcache: set (method: json, encoding error: %v)", err)
				return err
			}
			vals[ks] = v
		}

		bufbytes, err := json.Marshal(vals)
		if err != nil {
			if s.Logging > 0 {
				log.Printf("gorilla-sessions-memcache: set (method: json, encoding error: %v)", err)
			}
			return err
		}

		err = s.Client.Set(&memcache.Item{Key: key, Value: bufbytes})
		if s.Logging > 0 {
			log.Printf("gorilla-sessions-memcache: set (method: json, session name: %v, memcache key: %v, memcache value: %v, error: %v)", session.Name(), key, string(bufbytes), err)
		}
		if err != nil {
			return err
		}

		return nil

	default:
		panic("Unknown StoreMethod: " + string(s.StoreMethod))
	}

	panic("Unreachable")
	return nil
}

// load reads a file and decodes its content into session.Values.
func (s *MemcacheStore) load(session *sessions.Session) error {

	key := s.KeyPrefix + session.ID

	it, err := s.Client.Get(key)
	if s.Logging > 0 {
		if s.StoreMethod == StoreMethodJson {
			log.Printf("gorilla-sessions-memcache: get (method: %s, session name: %v, memcache key: %v, memcache value: %v, error: %v)", s.StoreMethod, session.Name(), key, string(it.Value), err)
		} else {
			log.Printf("gorilla-sessions-memcache: get (method: %s, session name: %v, memcache key: %v, memcache value len: %v, error: %v)", s.StoreMethod, session.Name(), key, len(it.Value), err)
		}
	}
	if err != nil {
		return err
	}

	switch s.StoreMethod {

	case StoreMethodSecureCookie:

		if err = securecookie.DecodeMulti(session.Name(), string(it.Value),
			&session.Values, s.Codecs...); err != nil {
			if s.Logging > 0 {
				log.Printf("gorilla-sessions-memcache: get (method: securecookie, decoding error: %v)", err)
			}
			return err
		}
		return nil

	case StoreMethodGob:

		buf := bytes.NewBuffer(it.Value)
		dec := gob.NewDecoder(buf)

		err = dec.Decode(&session.Values)
		if err != nil {
			if s.Logging > 0 {
				log.Printf("gorilla-sessions-memcache: get (method: gob, decoding error: %v)", err)
			}
		}
		return err

	case StoreMethodJson:

		vals := make(map[string]interface{})

		err := json.Unmarshal(it.Value, &vals)
		if err != nil {
			if s.Logging > 0 {
				log.Printf("gorilla-sessions-memcache: get (method: json, decoding error: %v)", err)
			}
			return err
		}

		for k, v := range vals {
			session.Values[k] = v
		}

		return nil

	default:
		panic("Unknown StoreMethod: " + string(s.StoreMethod))

	}

	panic("Unreachable")
	return nil
}
<commit_msg>Session expiration<commit_after>// Memcache session support for Gorilla Web Toolkit,
// without Google App Engine dependency.
package gsm

import (
	"bytes"
	"encoding/base32"
	"encoding/gob"
	"encoding/json"
	"fmt"
	"github.com/bradfitz/gomemcache/memcache"
	"github.com/gorilla/securecookie"
	"github.com/gorilla/sessions"
	"log"
	"net/http"
	"strings"
)

// NewMemcacheStore returns a new MemcacheStore.
// You need to provide the memcache client and
// an optional prefix for the keys we store
func NewMemcacheStore(client *memcache.Client, keyPrefix string, keyPairs ...[]byte) *MemcacheStore {

	if client == nil {
		panic("Cannot have nil memcache client")
	}

	return &MemcacheStore{
		Codecs: securecookie.CodecsFromPairs(keyPairs...),
		Options: &sessions.Options{
			Path:   "/",
			MaxAge: 86400 * 30,
		},
		KeyPrefix:   keyPrefix,
		Client:      client,
		StoreMethod: StoreMethodSecureCookie,
	}
}

type StoreMethod string

// take your pick on how to store the values in memcache
const (
	StoreMethodSecureCookie = StoreMethod("securecookie") // security
	StoreMethodGob          = StoreMethod("gob")          // speed
	StoreMethodJson         = StoreMethod("json")         // simplicity; warning: only string keys allowed and rest of data must be JSON.Marshal compatible
)

// MemcacheStore stores sessions in memcache
//
type MemcacheStore struct {
	Codecs      []securecookie.Codec
	Options     *sessions.Options // default configuration
	Client      *memcache.Client
	KeyPrefix   string
	Logging     int // set to > 0 to enable logging (using log.Printf)
	StoreMethod StoreMethod
}

// MaxLength restricts the maximum length of new sessions to l.
// If l is 0 there is no limit to the size of a session, use with caution.
// The default for a new MemcacheStore is 4096.
func (s *MemcacheStore) MaxLength(l int) {
	for _, c := range s.Codecs {
		if codec, ok := c.(*securecookie.SecureCookie); ok {
			codec.MaxLength(l)
		}
	}
}

// Get returns a session for the given name after adding it to the registry.
//
// See CookieStore.Get().
func (s *MemcacheStore) Get(r *http.Request, name string) (*sessions.Session, error) {
	return sessions.GetRegistry(r).Get(s, name)
}

// New returns a session for the given name without adding it to the registry.
//
// See CookieStore.New().
func (s *MemcacheStore) New(r *http.Request, name string) (*sessions.Session, error) {
	session := sessions.NewSession(s, name)
	opts := *s.Options
	session.Options = &opts
	session.IsNew = true
	var err error
	if c, errCookie := r.Cookie(name); errCookie == nil {
		err = securecookie.DecodeMulti(name, c.Value, &session.ID, s.Codecs...)
		if err == nil {
			err = s.load(session)
			if err == nil {
				session.IsNew = false
			}
		}
	}
	return session, err
}

// Save adds a single session to the response.
func (s *MemcacheStore) Save(r *http.Request, w http.ResponseWriter,
	session *sessions.Session) error {
	if session.ID == "" {
		// Because the ID is used in the filename, encode it to
		// use alphanumeric characters only.
		session.ID = strings.TrimRight(
			base32.StdEncoding.EncodeToString(
				securecookie.GenerateRandomKey(32)), "=")
	}
	if err := s.save(session); err != nil {
		return err
	}
	encoded, err := securecookie.EncodeMulti(session.Name(), session.ID,
		s.Codecs...)
	if err != nil {
		return err
	}
	http.SetCookie(w, sessions.NewCookie(session.Name(), encoded, session.Options))
	return nil
}

// save writes encoded session.Values using the memcache client
func (s *MemcacheStore) save(session *sessions.Session) error {

	key := s.KeyPrefix + session.ID

	switch s.StoreMethod {

	case StoreMethodSecureCookie:

		encoded, err := securecookie.EncodeMulti(session.Name(), session.Values,
			s.Codecs...)
		if err != nil {
			if s.Logging > 0 {
				log.Printf("gorilla-sessions-memcache: set (method: securecookie, encoding error: %v)", err)
			}
			return err
		}

		err = s.Client.Set(&memcache.Item{Key: key, Value: []byte(encoded), Expiration: session.Options.MaxAge})
		if s.Logging > 0 {
			log.Printf("gorilla-sessions-memcache: set (method: securecookie, session name: %v, memcache key: %v, memcache value: %v, error: %v)", session.Name(), key, encoded, err)
		}
		if err != nil {
			return err
		}

		return nil

	case StoreMethodGob:

		buf := &bytes.Buffer{}
		enc := gob.NewEncoder(buf)
		if err := enc.Encode(session.Values); err != nil {
			if s.Logging > 0 {
				log.Printf("gorilla-sessions-memcache: set (method: gob, encoding error: %v)", err)
			}
			return err
		}
		bufbytes := buf.Bytes()

		err := s.Client.Set(&memcache.Item{Key: key, Value: bufbytes, Expiration: session.Options.MaxAge})
		if s.Logging > 0 {
			log.Printf("gorilla-sessions-memcache: set (method: gob, session name: %v, memcache key: %v, memcache value len: %v, error: %v)", session.Name(), key, len(bufbytes), err)
		}
		if err != nil {
			return err
		}

		return nil

	case StoreMethodJson:

		vals := make(map[string]interface{}, len(session.Values))
		for k, v := range session.Values {
			ks, ok := k.(string)
			if !ok {
				err := fmt.Errorf("Non-string key value, cannot jsonize: %v", k)
				log.Printf("gorilla-sessions-memcache: set (method: json, encoding error: %v)", err)
				return err
			}
			vals[ks] = v
		}

		bufbytes, err := json.Marshal(vals)
		if err != nil {
			if s.Logging > 0 {
				log.Printf("gorilla-sessions-memcache: set (method: json, encoding error: %v)", err)
			}
			return err
		}

		err = s.Client.Set(&memcache.Item{Key: key, Value: bufbytes, Expiration: session.Options.MaxAge})
		if s.Logging > 0 {
			log.Printf("gorilla-sessions-memcache: set (method: json, session name: %v, memcache key: %v, memcache value: %v, error: %v)", session.Name(), key, string(bufbytes), err)
		}
		if err != nil {
			return err
		}

		return nil

	default:
		panic("Unknown StoreMethod: " + string(s.StoreMethod))
	}

	panic("Unreachable")
	return nil
}

// load reads a file and decodes its content into session.Values.
func (s *MemcacheStore) load(session *sessions.Session) error {

	key := s.KeyPrefix + session.ID

	it, err := s.Client.Get(key)
	if s.Logging > 0 {
		if s.StoreMethod == StoreMethodJson {
			log.Printf("gorilla-sessions-memcache: get (method: %s, session name: %v, memcache key: %v, memcache value: %v, error: %v)", s.StoreMethod, session.Name(), key, string(it.Value), err)
		} else {
			log.Printf("gorilla-sessions-memcache: get (method: %s, session name: %v, memcache key: %v, memcache value len: %v, error: %v)", s.StoreMethod, session.Name(), key, len(it.Value), err)
		}
	}
	if err != nil {
		return err
	}

	switch s.StoreMethod {

	case StoreMethodSecureCookie:

		if err = securecookie.DecodeMulti(session.Name(), string(it.Value),
			&session.Values, s.Codecs...); err != nil {
			if s.Logging > 0 {
				log.Printf("gorilla-sessions-memcache: get (method: securecookie, decoding error: %v)", err)
			}
			return err
		}
		return nil

	case StoreMethodGob:

		buf := bytes.NewBuffer(it.Value)
		dec := gob.NewDecoder(buf)

		err = dec.Decode(&session.Values)
		if err != nil {
			if s.Logging > 0 {
				log.Printf("gorilla-sessions-memcache: get (method: gob, decoding error: %v)", err)
			}
		}
		return err

	case StoreMethodJson:

		vals := make(map[string]interface{})

		err := json.Unmarshal(it.Value, &vals)
		if err != nil {
			if s.Logging > 0 {
				log.Printf("gorilla-sessions-memcache: get (method: json, decoding error: %v)", err)
			}
			return err
		}

		for k, v := range vals {
			session.Values[k] = v
		}

		return nil

	default:
		panic("Unknown StoreMethod: " + string(s.StoreMethod))

	}

	panic("Unreachable")
	return nil
}
<|endoftext|>


<commit_before>package logging

import (
	"os"
)

const (
	StdoutHandlerName = "stdout"
)

var (
	StdoutHandler Handler
)

func init() {
	bh, err := NewBaseHandler(os.Stdout, DEBUG, DefaultTimeLayout, DefaultFormat)
	if err != nil {
		panic(err)
	}
	StdoutHandler = bh
}

func DisableStdout() {
	delete(DefaultLogger.Handlers, StdoutHandlerName)
}

func EnableStdout() {
	DefaultLogger.Handlers[StdoutHandlerName] = StdoutHandler
}
<commit_msg>enable stdout as default<commit_after>package logging

import (
	"os"
)

const (
	StdoutHandlerName = "stdout"
)

var (
	StdoutHandler Handler
)

func init() {
	bh, err := NewBaseHandler(os.Stdout, DEBUG, DefaultTimeLayout, DefaultFormat)
	if err != nil {
		panic(err)
	}
	StdoutHandler = bh
	EnableStdout()
}

func DisableStdout() {
	delete(DefaultLogger.Handlers, StdoutHandlerName)
}

func EnableStdout() {
	DefaultLogger.Handlers[StdoutHandlerName] = StdoutHandler
}
<|endoftext|>


<commit_before>package cfitsio

// #include "go-cfitsio.h"
import "C"
import "unsafe"

type HduType int

const (
	ImageHdu  HduType = C.IMAGE_HDU  // Primary Array or IMAGE HDU
	AsciiTbl  HduType = C.ASCII_TBL  // ASCII table HDU
	BinaryTbl HduType = C.BINARY_TBL // Binary table HDU
	AnyHdy    HduType = C.ANY_HDU    // matches any HDU type
)

// MoveAbsHdu moves to a different HDU in the file
func (f *File) MoveAbsHdu(hdu int) (HduType, error) {
	c_hdu := C.int(hdu)
	c_htype := C.int(0)
	c_status := C.int(0)

	C.fits_movabs_hdu(f.c, c_hdu, &c_htype, &c_status)
	if c_status > 0 {
		return HduType(c_htype), to_err(c_status)
	}

	return HduType(c_htype), nil
}

// MoveRelHdu moves to a different HDU in the file
func (f *File) MoveRelHdu(n int) (HduType, error) {
	c_n := C.int(n)
	c_htype := C.int(0)
	c_status := C.int(0)

	C.fits_movrel_hdu(f.c, c_n, &c_htype, &c_status)
	if c_status > 0 {
		return HduType(c_htype), to_err(c_status)
	}

	return HduType(c_htype), nil
}

// MoveNamHdu moves to a different HDU in the file
func (f *File) MoveNamHdu(hdu HduType, extname string, extvers int) error {
	c_hdu := C.int(hdu)
	c_name := C.CString(extname)
	defer C.free(unsafe.Pointer(c_name))
	c_vers := C.int(extvers)
	c_status := C.int(0)

	C.fits_movnam_hdu(f.c, c_hdu, c_name, c_vers, &c_status)
	if c_status > 0 {
		return to_err(c_status)
	}

	return nil
}

// NumHdus returns the total number of HDUs in the FITS file.
// This returns the number of completely defined HDUs in the file. If a new HDU has just been added to the FITS file, then that last HDU will only be counted if it has been closed, or if data has been written to the HDU. The current HDU remains unchanged by this routine.
func (f *File) NumHdus() (int, error) {
	c_n := C.int(0)
	c_status := C.int(0)
	C.fits_get_num_hdus(f.c, &c_n, &c_status)
	if c_status > 0 {
		return 0, to_err(c_status)
	}

	return int(c_n), nil
}

// HduNum returns the number of the current HDU (CHDU) in the FITS file (where the primary array = 1). This function returns the HDU number rather than a status value.
func (f *File) HduNum() int {

	c_n := C.int(0)
	C.fits_get_hdu_num(f.c, &c_n)
	return int(c_n)
}

// HduType returns the type of the current HDU in the FITS file. The possible values for hdutype are: IMAGE_HDU, ASCII_TBL, or BINARY_TBL.
func (f *File) HduType() (HduType, error) {
	c_hdu := C.int(0)
	c_status := C.int(0)
	C.fits_get_hdu_type(f.c, &c_hdu, &c_status)
	if c_status > 0 {
		return 0, to_err(c_status)
	}

	return HduType(c_hdu), nil
}

// Copy all or part of the HDUs in the FITS file associated with infptr and append them to the end of the FITS file associated with outfptr. If 'previous' is true, then any HDUs preceding the current HDU in the input file will be copied to the output file. Similarly, 'current' and 'following' determine whether the current HDU, and/or any following HDUs in the input file will be copied to the output file. Thus, if all 3 parameters are true, then the entire input file will be copied. On exit, the current HDU in the input file will be unchanged, and the last HDU in the output file will be the current HDU.
func (f *File) Copy(out *File, previous, current, following bool) error {
	c_previous := C.int(0)
	if previous {
		c_previous = C.int(1)
	}
	c_current := C.int(0)
	if current {
		c_current = C.int(1)
	}
	c_following := C.int(0)
	if following {
		c_following = C.int(1)
	}
	c_status := C.int(0)
	C.fits_copy_file(f.c, out.c, c_previous, c_current, c_following, &c_status)
	if c_status > 0 {
		return to_err(c_status)
	}
	return nil
}

// EOF
<commit_msg>hdu: wrap fits_copy_hdu<commit_after>package cfitsio

// #include "go-cfitsio.h"
import "C"
import "unsafe"

type HduType int

const (
	ImageHdu  HduType = C.IMAGE_HDU  // Primary Array or IMAGE HDU
	AsciiTbl  HduType = C.ASCII_TBL  // ASCII table HDU
	BinaryTbl HduType = C.BINARY_TBL // Binary table HDU
	AnyHdy    HduType = C.ANY_HDU    // matches any HDU type
)

// MoveAbsHdu moves to a different HDU in the file
func (f *File) MoveAbsHdu(hdu int) (HduType, error) {
	c_hdu := C.int(hdu)
	c_htype := C.int(0)
	c_status := C.int(0)

	C.fits_movabs_hdu(f.c, c_hdu, &c_htype, &c_status)
	if c_status > 0 {
		return HduType(c_htype), to_err(c_status)
	}

	return HduType(c_htype), nil
}

// MoveRelHdu moves to a different HDU in the file
func (f *File) MoveRelHdu(n int) (HduType, error) {
	c_n := C.int(n)
	c_htype := C.int(0)
	c_status := C.int(0)

	C.fits_movrel_hdu(f.c, c_n, &c_htype, &c_status)
	if c_status > 0 {
		return HduType(c_htype), to_err(c_status)
	}

	return HduType(c_htype), nil
}

// MoveNamHdu moves to a different HDU in the file
func (f *File) MoveNamHdu(hdu HduType, extname string, extvers int) error {
	c_hdu := C.int(hdu)
	c_name := C.CString(extname)
	defer C.free(unsafe.Pointer(c_name))
	c_vers := C.int(extvers)
	c_status := C.int(0)

	C.fits_movnam_hdu(f.c, c_hdu, c_name, c_vers, &c_status)
	if c_status > 0 {
		return to_err(c_status)
	}

	return nil
}

// NumHdus returns the total number of HDUs in the FITS file.
// This returns the number of completely defined HDUs in the file. If a new HDU has just been added to the FITS file, then that last HDU will only be counted if it has been closed, or if data has been written to the HDU. The current HDU remains unchanged by this routine.
func (f *File) NumHdus() (int, error) {
	c_n := C.int(0)
	c_status := C.int(0)
	C.fits_get_num_hdus(f.c, &c_n, &c_status)
	if c_status > 0 {
		return 0, to_err(c_status)
	}

	return int(c_n), nil
}

// HduNum returns the number of the current HDU (CHDU) in the FITS file (where the primary array = 1). This function returns the HDU number rather than a status value.
func (f *File) HduNum() int {

	c_n := C.int(0)
	C.fits_get_hdu_num(f.c, &c_n)
	return int(c_n)
}

// HduType returns the type of the current HDU in the FITS file. The possible values for hdutype are: IMAGE_HDU, ASCII_TBL, or BINARY_TBL.
func (f *File) HduType() (HduType, error) {
	c_hdu := C.int(0)
	c_status := C.int(0)
	C.fits_get_hdu_type(f.c, &c_hdu, &c_status)
	if c_status > 0 {
		return 0, to_err(c_status)
	}

	return HduType(c_hdu), nil
}

// Copy all or part of the HDUs in the FITS file associated with infptr and append them to the end of the FITS file associated with outfptr. If 'previous' is true, then any HDUs preceding the current HDU in the input file will be copied to the output file. Similarly, 'current' and 'following' determine whether the current HDU, and/or any following HDUs in the input file will be copied to the output file. Thus, if all 3 parameters are true, then the entire input file will be copied. On exit, the current HDU in the input file will be unchanged, and the last HDU in the output file will be the current HDU.
func (f *File) Copy(out *File, previous, current, following bool) error {
	c_previous := C.int(0)
	if previous {
		c_previous = C.int(1)
	}
	c_current := C.int(0)
	if current {
		c_current = C.int(1)
	}
	c_following := C.int(0)
	if following {
		c_following = C.int(1)
	}
	c_status := C.int(0)
	C.fits_copy_file(f.c, out.c, c_previous, c_current, c_following, &c_status)
	if c_status > 0 {
		return to_err(c_status)
	}
	return nil
}

// Copy the current HDU from the FITS file associated with infptr and append it to the end of the FITS file associated with outfptr. Space may be reserved for MOREKEYS additional keywords in the output header.
func (f *File) CopyHdu(out *File, morekeys int) error {
	c_morekeys := C.int(morekeys)
	c_status := C.int(0)
	C.fits_copy_hdu(f.c, out.c, c_morekeys, &c_status)
	if c_status > 0 {
		return to_err(c_status)
	}
	return nil
}

// EOF
<|endoftext|>


<commit_before>package string_finder

type BruteForce struct {
}

func NewBruteForce() *BruteForce {
	return &BruteForce{}
}

func (bf *BruteForce) Find(text, pattern string) int {
	return -1
}
<commit_msg>Implement brute force find<commit_after>package string_finder

type BruteForce struct {
}

func NewBruteForce() *BruteForce {
	return &BruteForce{}
}

func (bf *BruteForce) Find(text, pattern string) int {
	textRunes := []rune(text)
	patternRunes := []rune(pattern)
	var textPos, patternPos int
	for textPos, patternPos = 0, 0; textPos < len(textRunes) && patternPos < len(patternRunes); {
		// Compare text and pattern char
		if textRunes[textPos] == patternRunes[patternPos] {
			textPos++
			patternPos++
		} else {
			// Not matched
			//
			textPos = textPos - patternPos + 1 // Make textPos advance with 1 step
			patternPos = 0                     // Reset pattern position
		}
	}

	if patternPos == len(patternRunes) {
		return textPos - patternPos
	} else {
		return -1
	}
}
<|endoftext|>


<commit_before><commit_msg>strip '#' on creation too<commit_after><|endoftext|>


<commit_before>package vm

import (
	"bytes"
	"fmt"
	"net"
	"net/url"
	"os"
	"strconv"
	"strings"
	"text/template"
	"time"

	bosherr "github.com/cloudfoundry/bosh-utils/errors"
	boshlog "github.com/cloudfoundry/bosh-utils/logger"
	boshsys "github.com/cloudfoundry/bosh-utils/system"
	boshuuid "github.com/cloudfoundry/bosh-utils/uuid"

	bslcommon "github.com/cloudfoundry/bosh-softlayer-cpi/softlayer/common"
	bslcstem "github.com/cloudfoundry/bosh-softlayer-cpi/softlayer/stemcell"
	bslcvmpool "github.com/cloudfoundry/bosh-softlayer-cpi/softlayer/vm/pool"
	datatypes "github.com/maximilien/softlayer-go/data_types"
	sl "github.com/maximilien/softlayer-go/softlayer"

	common "github.com/cloudfoundry/bosh-softlayer-cpi/common"
	util "github.com/cloudfoundry/bosh-softlayer-cpi/util"
)

const SOFTLAYER_VM_CREATOR_LOG_TAG = "SoftLayerVMCreator"

type SoftLayerCreator struct {
	softLayerClient        sl.Client
	agentEnvServiceFactory AgentEnvServiceFactory

	agentOptions  AgentOptions
	logger        boshlog.Logger
	uuidGenerator boshuuid.Generator
	fs            boshsys.FileSystem
}

func NewSoftLayerCreator(softLayerClient sl.Client, agentEnvServiceFactory AgentEnvServiceFactory, agentOptions AgentOptions, logger boshlog.Logger, uuidGenerator boshuuid.Generator, fs boshsys.FileSystem) SoftLayerCreator {
	bslcommon.TIMEOUT = 120 * time.Minute
	bslcommon.POLLING_INTERVAL = 5 * time.Second

	return SoftLayerCreator{
		softLayerClient:        softLayerClient,
		agentEnvServiceFactory: agentEnvServiceFactory,
		agentOptions:           agentOptions,
		logger:                 logger,
		uuidGenerator:          uuidGenerator,
		fs:                     fs,
	}
}

func (c SoftLayerCreator) CreateBySoftlayer(agentID string, stemcell bslcstem.Stemcell, cloudProps VMCloudProperties, networks Networks, env Environment) (VM, error) {

	virtualGuestTemplate, err := CreateVirtualGuestTemplate(stemcell, cloudProps)

	if err != nil {
		return SoftLayerVM{}, bosherr.WrapError(err, "Creating virtual guest template")
	}

	virtualGuestService, err := c.softLayerClient.GetSoftLayer_Virtual_Guest_Service()
	if err != nil {
		return SoftLayerVM{}, bosherr.WrapError(err, "Creating VirtualGuestService from SoftLayer client")
	}

	virtualGuest, err := virtualGuestService.CreateObject(virtualGuestTemplate)
	if err != nil {
		return SoftLayerVM{}, bosherr.WrapError(err, "Creating VirtualGuest from SoftLayer client")
	}

	if cloudProps.EphemeralDiskSize == 0 {
		err = bslcommon.WaitForVirtualGuestLastCompleteTransaction(c.softLayerClient, virtualGuest.Id, "Service Setup")
		if err != nil {
			return SoftLayerVM{}, bosherr.WrapErrorf(err, "Waiting for VirtualGuest `%d` has Service Setup transaction complete", virtualGuest.Id)
		}
	} else {
		err = bslcommon.AttachEphemeralDiskToVirtualGuest(c.softLayerClient, virtualGuest.Id, cloudProps.EphemeralDiskSize, c.logger)
		if err != nil {
			return SoftLayerVM{}, bosherr.WrapError(err, fmt.Sprintf("Attaching ephemeral disk to VirtualGuest `%d`", virtualGuest.Id))
		}
	}

	virtualGuest, err = bslcommon.GetObjectDetailsOnVirtualGuest(c.softLayerClient, virtualGuest.Id)
	if err != nil {
		return SoftLayerVM{}, bosherr.WrapErrorf(err, "Cannot get details from virtual guest with id: %d.", virtualGuest.Id)
	}

	softlayerFileService := NewSoftlayerFileService(util.GetSshClient(), virtualGuest, c.logger, c.uuidGenerator, c.fs)
	agentEnvService := c.agentEnvServiceFactory.New(softlayerFileService, strconv.Itoa(virtualGuest.Id))

	if len(cloudProps.BoshIp) == 0 {
		// update /etc/hosts file of bosh-init vm
		c.updateEtcHostsOfBoshInit(fmt.Sprintf("%s  %s", virtualGuest.PrimaryBackendIpAddress, virtualGuest.FullyQualifiedDomainName))
		// Update mbus url setting for bosh director: construct mbus url with new director ip
		mbus, err := c.parseMbusURL(c.agentOptions.Mbus, virtualGuest.PrimaryBackendIpAddress)
		if err != nil {
			return SoftLayerVM{}, bosherr.WrapErrorf(err, "Cannot construct mbus url.")
		}
		c.agentOptions.Mbus = mbus
	} else {
		// Update mbus url setting
		mbus, err := c.parseMbusURL(c.agentOptions.Mbus, cloudProps.BoshIp)
		if err != nil {
			return SoftLayerVM{}, bosherr.WrapErrorf(err, "Cannot construct mbus url.")
		}
		c.agentOptions.Mbus = mbus
		// Update blobstore setting
		switch c.agentOptions.Blobstore.Provider {
		case BlobstoreTypeDav:
			davConf := DavConfig(c.agentOptions.Blobstore.Options)
			c.updateDavConfig(&davConf, cloudProps.BoshIp)
		}
	}

	agentEnv := CreateAgentUserData(agentID, cloudProps, networks, env, c.agentOptions)
	if err != nil {
		return SoftLayerVM{}, bosherr.WrapErrorf(err, "Cannot agent env for virtual guest with id: %d.", virtualGuest.Id)
	}

	err = agentEnvService.Update(agentEnv)
	if err != nil {
		return SoftLayerVM{}, bosherr.WrapError(err, "Updating VM's agent env")
	}

	vm := NewSoftLayerVM(virtualGuest.Id, c.softLayerClient, util.GetSshClient(), agentEnvService, c.logger)

	if !strings.Contains(cloudProps.VmNamePrefix, "-worker") {
		db, err := bslcvmpool.OpenDB(bslcvmpool.SQLITE_DB_FILE_PATH)
		if err != nil {
			return SoftLayerVM{}, bosherr.WrapError(err, "Opening DB")
		}

		vmInfoDB := bslcvmpool.NewVMInfoDB(vm.ID(), virtualGuestTemplate.Hostname+"."+virtualGuestTemplate.Domain, "t", stemcell.Uuid(), agentID, c.logger, db)
		err = vmInfoDB.InsertVMInfo(bslcvmpool.DB_RETRY_TIMEOUT, bslcvmpool.DB_RETRY_INTERVAL)
		if err != nil {
			return SoftLayerVM{}, bosherr.WrapError(err, "Failed to insert the record into VM pool DB")
		}
	}

	if len(c.agentOptions.VcapPassword) > 0 {
		err = c.SetVcapPassword(vm, virtualGuest, c.agentOptions.VcapPassword)
		if err != nil {
			return SoftLayerVM{}, bosherr.WrapError(err, "Updating VM's vcap password")
		}
	}

	return vm, nil
}

func (c SoftLayerCreator) CreateByOSReload(agentID string, stemcell bslcstem.Stemcell, cloudProps VMCloudProperties, networks Networks, env Environment) (VM, error) {
	virtualGuestService, err := c.softLayerClient.GetSoftLayer_Virtual_Guest_Service()
	if err != nil {
		return SoftLayerVM{}, bosherr.WrapError(err, "Creating VirtualGuestService from SoftLayer client")
	}

	var virtualGuest datatypes.SoftLayer_Virtual_Guest

	if common.IsPrivateSubnet(net.ParseIP(networks.First().IP)) {
		virtualGuest, err = virtualGuestService.GetObjectByPrimaryIpAddress(networks.First().IP)
	} else {
		virtualGuest, err = virtualGuestService.GetObjectByPrimaryBackendIpAddress(networks.First().IP)
	}

	if err != nil || virtualGuest.Id == 0 {
		return SoftLayerVM{}, bosherr.WrapErrorf(err, "Could not find virtual guest by ip address: %s", networks.First().IP)
	}

	c.logger.Info(SOFTLAYER_VM_CREATOR_LOG_TAG, fmt.Sprintf("OS reload on the server id %d with stemcell %d", virtualGuest.Id, stemcell.ID()))

	vm := NewSoftLayerVM(virtualGuest.Id, c.softLayerClient, util.GetSshClient(), nil, c.logger)

	bslcommon.TIMEOUT = 4 * time.Hour
	err = vm.ReloadOS(stemcell)
	if err != nil {
		return SoftLayerVM{}, bosherr.WrapError(err, "Failed to reload OS")
	}

	if cloudProps.EphemeralDiskSize == 0 {
		err = bslcommon.WaitForVirtualGuestLastCompleteTransaction(c.softLayerClient, virtualGuest.Id, "Service Setup")
		if err != nil {
			return SoftLayerVM{}, bosherr.WrapErrorf(err, "Waiting for VirtualGuest `%d` has Service Setup transaction complete", virtualGuest.Id)
		}
	} else {
		err = bslcommon.AttachEphemeralDiskToVirtualGuest(c.softLayerClient, virtualGuest.Id, cloudProps.EphemeralDiskSize, c.logger)
		if err != nil {
			return SoftLayerVM{}, bosherr.WrapError(err, fmt.Sprintf("Attaching ephemeral disk to VirtualGuest `%d`", virtualGuest.Id))
		}
	}

	softlayerFileService := NewSoftlayerFileService(util.GetSshClient(), virtualGuest, c.logger, c.uuidGenerator, c.fs)
	agentEnvService := c.agentEnvServiceFactory.New(softlayerFileService, strconv.Itoa(virtualGuest.Id))

	if len(cloudProps.BoshIp) == 0 {
		// update /etc/hosts file of bosh-init vm
		c.updateEtcHostsOfBoshInit(fmt.Sprintf("%s  %s", virtualGuest.PrimaryBackendIpAddress, virtualGuest.FullyQualifiedDomainName))
		// Update mbus url setting for bosh director: construct mbus url with new director ip
		mbus, err := c.parseMbusURL(c.agentOptions.Mbus, virtualGuest.PrimaryBackendIpAddress)
		if err != nil {
			return SoftLayerVM{}, bosherr.WrapErrorf(err, "Cannot construct mbus url.")
		}
		c.agentOptions.Mbus = mbus
	} else {
		// Update mbus url setting
		mbus, err := c.parseMbusURL(c.agentOptions.Mbus, cloudProps.BoshIp)
		if err != nil {
			return SoftLayerVM{}, bosherr.WrapErrorf(err, "Cannot construct mbus url.")
		}
		c.agentOptions.Mbus = mbus
		// Update blobstore setting
		switch c.agentOptions.Blobstore.Provider {
		case BlobstoreTypeDav:
			davConf := DavConfig(c.agentOptions.Blobstore.Options)
			c.updateDavConfig(&davConf, cloudProps.BoshIp)
		}
	}

	agentEnv := CreateAgentUserData(agentID, cloudProps, networks, env, c.agentOptions)
	if err != nil {
		return SoftLayerVM{}, bosherr.WrapErrorf(err, "Cannot agent env for virtual guest with id: %d.", virtualGuest.Id)
	}

	err = agentEnvService.Update(agentEnv)
	if err != nil {
		return SoftLayerVM{}, bosherr.WrapError(err, "Updating VM's agent env")
	}

	if len(c.agentOptions.VcapPassword) > 0 {
		err = c.SetVcapPassword(vm, virtualGuest, c.agentOptions.VcapPassword)
		if err != nil {
			return SoftLayerVM{}, bosherr.WrapError(err, "Updating VM's vcap password")
		}
	}
	return vm, nil
}

// Private methods
func (c SoftLayerCreator) updateDavConfig(config *DavConfig, directorIP string) (err error) {
	url := (*config)["endpoint"].(string)
	mbus, err := c.parseMbusURL(url, directorIP)
	if err != nil {
		return bosherr.WrapError(err, "Parsing Mbus URL")
	}

	(*config)["endpoint"] = mbus

	return nil
}

func (c SoftLayerCreator) parseMbusURL(mbusURL string, primaryBackendIpAddress string) (string, error) {
	parsedURL, err := url.Parse(mbusURL)
	if err != nil {
		return "", bosherr.WrapError(err, "Parsing Mbus URL")
	}
	var username, password, port string
	_, port, _ = net.SplitHostPort(parsedURL.Host)
	userInfo := parsedURL.User
	if userInfo != nil {
		username = userInfo.Username()
		password, _ = userInfo.Password()
		return fmt.Sprintf("%s://%s:%s@%s:%s", parsedURL.Scheme, username, password, primaryBackendIpAddress, port), nil
	}

	return fmt.Sprintf("%s://%s:%s", parsedURL.Scheme, primaryBackendIpAddress, port), nil
}

func (c SoftLayerCreator) updateEtcHostsOfBoshInit(record string) (err error) {
	buffer := bytes.NewBuffer([]byte{})
	t := template.Must(template.New("etc-hosts").Parse(ETC_HOSTS_TEMPLATE))

	err = t.Execute(buffer, record)
	if err != nil {
		return bosherr.WrapError(err, "Generating config from template")
	}

	err = c.fs.WriteFile("/etc/hosts", buffer.Bytes())
	if err != nil {
		return bosherr.WrapError(err, "Writing to /etc/hosts")
	}

	return nil
}

const ETC_HOSTS_TEMPLATE = `127.0.0.1 localhost
{{.}}
`

func (c SoftLayerCreator) appendRecordToEtcHosts(record string) (err error) {
	file, err := os.OpenFile("/etc/hosts", os.O_RDWR|os.O_APPEND, 0666)
	if err != nil {
		return bosherr.WrapError(err, "Failed to open file /etc/hosts")
	}

	defer file.Close()

	_, err = fmt.Fprintf(file, "\n %s", record)
	if err != nil {
		return bosherr.WrapErrorf(err, "Failed to append new record to /etc/hosts")
	}

	return nil
}

func (c SoftLayerCreator) SetVcapPassword(vm SoftLayerVM, virtualGuest datatypes.SoftLayer_Virtual_Guest, encryptedPwd string) (err error) {
	command := fmt.Sprintf("usermod -p '%s' vcap", c.agentOptions.VcapPassword)
	_, err = vm.sshClient.ExecCommand(ROOT_USER_NAME, vm.getRootPassword(virtualGuest), virtualGuest.PrimaryBackendIpAddress, command)
	if err != nil {
		return bosherr.WrapError(err, "Shelling out to usermod vcap")
	}
	return
}
<commit_msg>update InsertVMInfo<commit_after>package vm

import (
	"bytes"
	"fmt"
	"net"
	"net/url"
	"os"
	"strconv"
	"strings"
	"text/template"
	"time"

	bosherr "github.com/cloudfoundry/bosh-utils/errors"
	boshlog "github.com/cloudfoundry/bosh-utils/logger"
	boshsys "github.com/cloudfoundry/bosh-utils/system"
	boshuuid "github.com/cloudfoundry/bosh-utils/uuid"

	bslcommon "github.com/cloudfoundry/bosh-softlayer-cpi/softlayer/common"
	bslcstem "github.com/cloudfoundry/bosh-softlayer-cpi/softlayer/stemcell"
	bslcvmpool "github.com/cloudfoundry/bosh-softlayer-cpi/softlayer/vm/pool"
	datatypes "github.com/maximilien/softlayer-go/data_types"
	sl "github.com/maximilien/softlayer-go/softlayer"

	common "github.com/cloudfoundry/bosh-softlayer-cpi/common"
	util "github.com/cloudfoundry/bosh-softlayer-cpi/util"
)

const SOFTLAYER_VM_CREATOR_LOG_TAG = "SoftLayerVMCreator"

type SoftLayerCreator struct {
	softLayerClient        sl.Client
	agentEnvServiceFactory AgentEnvServiceFactory

	agentOptions  AgentOptions
	logger        boshlog.Logger
	uuidGenerator boshuuid.Generator
	fs            boshsys.FileSystem
}

func NewSoftLayerCreator(softLayerClient sl.Client, agentEnvServiceFactory AgentEnvServiceFactory, agentOptions AgentOptions, logger boshlog.Logger, uuidGenerator boshuuid.Generator, fs boshsys.FileSystem) SoftLayerCreator {
	bslcommon.TIMEOUT = 120 * time.Minute
	bslcommon.POLLING_INTERVAL = 5 * time.Second

	return SoftLayerCreator{
		softLayerClient:        softLayerClient,
		agentEnvServiceFactory: agentEnvServiceFactory,
		agentOptions:           agentOptions,
		logger:                 logger,
		uuidGenerator:          uuidGenerator,
		fs:                     fs,
	}
}

func (c SoftLayerCreator) CreateBySoftlayer(agentID string, stemcell bslcstem.Stemcell, cloudProps VMCloudProperties, networks Networks, env Environment) (VM, error) {

	virtualGuestTemplate, err := CreateVirtualGuestTemplate(stemcell, cloudProps)

	if err != nil {
		return SoftLayerVM{}, bosherr.WrapError(err, "Creating virtual guest template")
	}

	virtualGuestService, err := c.softLayerClient.GetSoftLayer_Virtual_Guest_Service()
	if err != nil {
		return SoftLayerVM{}, bosherr.WrapError(err, "Creating VirtualGuestService from SoftLayer client")
	}

	virtualGuest, err := virtualGuestService.CreateObject(virtualGuestTemplate)
	if err != nil {
		return SoftLayerVM{}, bosherr.WrapError(err, "Creating VirtualGuest from SoftLayer client")
	}

	if cloudProps.EphemeralDiskSize == 0 {
		err = bslcommon.WaitForVirtualGuestLastCompleteTransaction(c.softLayerClient, virtualGuest.Id, "Service Setup")
		if err != nil {
			return SoftLayerVM{}, bosherr.WrapErrorf(err, "Waiting for VirtualGuest `%d` has Service Setup transaction complete", virtualGuest.Id)
		}
	} else {
		err = bslcommon.AttachEphemeralDiskToVirtualGuest(c.softLayerClient, virtualGuest.Id, cloudProps.EphemeralDiskSize, c.logger)
		if err != nil {
			return SoftLayerVM{}, bosherr.WrapError(err, fmt.Sprintf("Attaching ephemeral disk to VirtualGuest `%d`", virtualGuest.Id))
		}
	}

	virtualGuest, err = bslcommon.GetObjectDetailsOnVirtualGuest(c.softLayerClient, virtualGuest.Id)
	if err != nil {
		return SoftLayerVM{}, bosherr.WrapErrorf(err, "Cannot get details from virtual guest with id: %d.", virtualGuest.Id)
	}

	softlayerFileService := NewSoftlayerFileService(util.GetSshClient(), virtualGuest, c.logger, c.uuidGenerator, c.fs)
	agentEnvService := c.agentEnvServiceFactory.New(softlayerFileService, strconv.Itoa(virtualGuest.Id))

	if len(cloudProps.BoshIp) == 0 {
		// update /etc/hosts file of bosh-init vm
		c.updateEtcHostsOfBoshInit(fmt.Sprintf("%s  %s", virtualGuest.PrimaryBackendIpAddress, virtualGuest.FullyQualifiedDomainName))
		// Update mbus url setting for bosh director: construct mbus url with new director ip
		mbus, err := c.parseMbusURL(c.agentOptions.Mbus, virtualGuest.PrimaryBackendIpAddress)
		if err != nil {
			return SoftLayerVM{}, bosherr.WrapErrorf(err, "Cannot construct mbus url.")
		}
		c.agentOptions.Mbus = mbus
	} else {
		// Update mbus url setting
		mbus, err := c.parseMbusURL(c.agentOptions.Mbus, cloudProps.BoshIp)
		if err != nil {
			return SoftLayerVM{}, bosherr.WrapErrorf(err, "Cannot construct mbus url.")
		}
		c.agentOptions.Mbus = mbus
		// Update blobstore setting
		switch c.agentOptions.Blobstore.Provider {
		case BlobstoreTypeDav:
			davConf := DavConfig(c.agentOptions.Blobstore.Options)
			c.updateDavConfig(&davConf, cloudProps.BoshIp)
		}
	}

	agentEnv := CreateAgentUserData(agentID, cloudProps, networks, env, c.agentOptions)
	if err != nil {
		return SoftLayerVM{}, bosherr.WrapErrorf(err, "Cannot agent env for virtual guest with id: %d.", virtualGuest.Id)
	}

	err = agentEnvService.Update(agentEnv)
	if err != nil {
		return SoftLayerVM{}, bosherr.WrapError(err, "Updating VM's agent env")
	}

	vm := NewSoftLayerVM(virtualGuest.Id, c.softLayerClient, util.GetSshClient(), agentEnvService, c.logger)

	if len(c.agentOptions.VcapPassword) > 0 {
		err = c.SetVcapPassword(vm, virtualGuest, c.agentOptions.VcapPassword)
		if err != nil {
			return SoftLayerVM{}, bosherr.WrapError(err, "Updating VM's vcap password")
		}
	}

	return vm, nil
}

func (c SoftLayerCreator) CreateByOSReload(agentID string, stemcell bslcstem.Stemcell, cloudProps VMCloudProperties, networks Networks, env Environment) (VM, error) {
	virtualGuestService, err := c.softLayerClient.GetSoftLayer_Virtual_Guest_Service()
	if err != nil {
		return SoftLayerVM{}, bosherr.WrapError(err, "Creating VirtualGuestService from SoftLayer client")
	}

	var virtualGuest datatypes.SoftLayer_Virtual_Guest

	if common.IsPrivateSubnet(net.ParseIP(networks.First().IP)) {
		virtualGuest, err = virtualGuestService.GetObjectByPrimaryIpAddress(networks.First().IP)
	} else {
		virtualGuest, err = virtualGuestService.GetObjectByPrimaryBackendIpAddress(networks.First().IP)
	}

	if err != nil || virtualGuest.Id == 0 {
		return SoftLayerVM{}, bosherr.WrapErrorf(err, "Could not find virtual guest by ip address: %s", networks.First().IP)
	}

	c.logger.Info(SOFTLAYER_VM_CREATOR_LOG_TAG, fmt.Sprintf("OS reload on the server id %d with stemcell %d", virtualGuest.Id, stemcell.ID()))

	vm := NewSoftLayerVM(virtualGuest.Id, c.softLayerClient, util.GetSshClient(), nil, c.logger)

	bslcommon.TIMEOUT = 4 * time.Hour
	err = vm.ReloadOS(stemcell)
	if err != nil {
		return SoftLayerVM{}, bosherr.WrapError(err, "Failed to reload OS")
	}

	if cloudProps.EphemeralDiskSize == 0 {
		err = bslcommon.WaitForVirtualGuestLastCompleteTransaction(c.softLayerClient, virtualGuest.Id, "Service Setup")
		if err != nil {
			return SoftLayerVM{}, bosherr.WrapErrorf(err, "Waiting for VirtualGuest `%d` has Service Setup transaction complete", virtualGuest.Id)
		}
	} else {
		err = bslcommon.AttachEphemeralDiskToVirtualGuest(c.softLayerClient, virtualGuest.Id, cloudProps.EphemeralDiskSize, c.logger)
		if err != nil {
			return SoftLayerVM{}, bosherr.WrapError(err, fmt.Sprintf("Attaching ephemeral disk to VirtualGuest `%d`", virtualGuest.Id))
		}
	}

	softlayerFileService := NewSoftlayerFileService(util.GetSshClient(), virtualGuest, c.logger, c.uuidGenerator, c.fs)
	agentEnvService := c.agentEnvServiceFactory.New(softlayerFileService, strconv.Itoa(virtualGuest.Id))

	if len(cloudProps.BoshIp) == 0 {
		// update /etc/hosts file of bosh-init vm
		c.updateEtcHostsOfBoshInit(fmt.Sprintf("%s  %s", virtualGuest.PrimaryBackendIpAddress, virtualGuest.FullyQualifiedDomainName))
		// Update mbus url setting for bosh director: construct mbus url with new director ip
		mbus, err := c.parseMbusURL(c.agentOptions.Mbus, virtualGuest.PrimaryBackendIpAddress)
		if err != nil {
			return SoftLayerVM{}, bosherr.WrapErrorf(err, "Cannot construct mbus url.")
		}
		c.agentOptions.Mbus = mbus
	} else {
		// Update mbus url setting
		mbus, err := c.parseMbusURL(c.agentOptions.Mbus, cloudProps.BoshIp)
		if err != nil {
			return SoftLayerVM{}, bosherr.WrapErrorf(err, "Cannot construct mbus url.")
		}
		c.agentOptions.Mbus = mbus
		// Update blobstore setting
		switch c.agentOptions.Blobstore.Provider {
		case BlobstoreTypeDav:
			davConf := DavConfig(c.agentOptions.Blobstore.Options)
			c.updateDavConfig(&davConf, cloudProps.BoshIp)
		}
	}

	agentEnv := CreateAgentUserData(agentID, cloudProps, networks, env, c.agentOptions)
	if err != nil {
		return SoftLayerVM{}, bosherr.WrapErrorf(err, "Cannot agent env for virtual guest with id: %d.", virtualGuest.Id)
	}

	err = agentEnvService.Update(agentEnv)
	if err != nil {
		return SoftLayerVM{}, bosherr.WrapError(err, "Updating VM's agent env")
	}

	if len(c.agentOptions.VcapPassword) > 0 {
		err = c.SetVcapPassword(vm, virtualGuest, c.agentOptions.VcapPassword)
		if err != nil {
			return SoftLayerVM{}, bosherr.WrapError(err, "Updating VM's vcap password")
		}
	}
	return vm, nil
}

// Private methods
func (c SoftLayerCreator) updateDavConfig(config *DavConfig, directorIP string) (err error) {
	url := (*config)["endpoint"].(string)
	mbus, err := c.parseMbusURL(url, directorIP)
	if err != nil {
		return bosherr.WrapError(err, "Parsing Mbus URL")
	}

	(*config)["endpoint"] = mbus

	return nil
}

func (c SoftLayerCreator) parseMbusURL(mbusURL string, primaryBackendIpAddress string) (string, error) {
	parsedURL, err := url.Parse(mbusURL)
	if err != nil {
		return "", bosherr.WrapError(err, "Parsing Mbus URL")
	}
	var username, password, port string
	_, port, _ = net.SplitHostPort(parsedURL.Host)
	userInfo := parsedURL.User
	if userInfo != nil {
		username = userInfo.Username()
		password, _ = userInfo.Password()
		return fmt.Sprintf("%s://%s:%s@%s:%s", parsedURL.Scheme, username, password, primaryBackendIpAddress, port), nil
	}

	return fmt.Sprintf("%s://%s:%s", parsedURL.Scheme, primaryBackendIpAddress, port), nil
}

func (c SoftLayerCreator) updateEtcHostsOfBoshInit(record string) (err error) {
	buffer := bytes.NewBuffer([]byte{})
	t := template.Must(template.New("etc-hosts").Parse(ETC_HOSTS_TEMPLATE))

	err = t.Execute(buffer, record)
	if err != nil {
		return bosherr.WrapError(err, "Generating config from template")
	}

	err = c.fs.WriteFile("/etc/hosts", buffer.Bytes())
	if err != nil {
		return bosherr.WrapError(err, "Writing to /etc/hosts")
	}

	return nil
}

const ETC_HOSTS_TEMPLATE = `127.0.0.1 localhost
{{.}}
`

func (c SoftLayerCreator) appendRecordToEtcHosts(record string) (err error) {
	file, err := os.OpenFile("/etc/hosts", os.O_RDWR|os.O_APPEND, 0666)
	if err != nil {
		return bosherr.WrapError(err, "Failed to open file /etc/hosts")
	}

	defer file.Close()

	_, err = fmt.Fprintf(file, "\n %s", record)
	if err != nil {
		return bosherr.WrapErrorf(err, "Failed to append new record to /etc/hosts")
	}

	return nil
}

func (c SoftLayerCreator) SetVcapPassword(vm SoftLayerVM, virtualGuest datatypes.SoftLayer_Virtual_Guest, encryptedPwd string) (err error) {
	command := fmt.Sprintf("usermod -p '%s' vcap", c.agentOptions.VcapPassword)
	_, err = vm.sshClient.ExecCommand(ROOT_USER_NAME, vm.getRootPassword(virtualGuest), virtualGuest.PrimaryBackendIpAddress, command)
	if err != nil {
		return bosherr.WrapError(err, "Shelling out to usermod vcap")
	}
	return
}
<|endoftext|>