<commit_before>package kato

//-----------------------------------------------------------------------------
// Package factored import statement:
//-----------------------------------------------------------------------------

import (

	// Stdlib:
	"encoding/json"
	"io/ioutil"
	"net"
	"net/http"
	"os"
	"os/exec"
	"strconv"
	"strings"
	"sync"
)

//-----------------------------------------------------------------------------
// WaitChan stuff:
//-----------------------------------------------------------------------------

// WaitChan is used to handle errors that occur in some goroutines.
type WaitChan struct {
	WaitGrp sync.WaitGroup
	ErrChan chan error
	EndChan chan bool
}

// NewWaitChan initializes a WaitChan struct.
func NewWaitChan(len int) *WaitChan {
	wch := new(WaitChan)
	wch.WaitGrp.Add(len)
	wch.ErrChan = make(chan error, 1)
	wch.EndChan = make(chan bool, 1)
	return wch
}

// WaitErr waits for any error or for all go routines to finish.
func (wch *WaitChan) WaitErr() error {

	// Put the wait group in a go routine:
	go func() {
		wch.WaitGrp.Wait()
		wch.EndChan <- true
	}()

	// This select will block:
	select {
	case <-wch.EndChan:
		return nil
	case err := <-wch.ErrChan:
		return err
	}
}

//-----------------------------------------------------------------------------
// func: DumpState
//-----------------------------------------------------------------------------

func DumpState(s interface{}, id string) error {

	// Marshal the data:
	data, err := json.MarshalIndent(s, "", "  ")
	if err != nil {
		return err
	}

	// Create the state directory:
	path := os.Getenv("HOME") + "/.kato"
	if _, err := os.Stat(path); err != nil {
		if os.IsNotExist(err) {
			err = os.Mkdir(path, 0700)
			if err != nil {
				return err
			}
		}
	}

	// Write the state file:
	err = ioutil.WriteFile(path+"/"+id+".json", data, 0600)
	if err != nil {
		return err
	}

	return nil
}

//-----------------------------------------------------------------------------
// func: ReadState
//-----------------------------------------------------------------------------

// ReadState reads the current ClusterID state file:
func ReadState(clusterID string) ([]byte, error) {

	// Read data from state file:
	stateFile := os.Getenv("HOME") + "/.kato/" + clusterID + ".json"
	raw, err := ioutil.ReadFile(stateFile)
	if err != nil {
		return nil, err
	}

	return raw, nil
}

//-----------------------------------------------------------------------------
// func: CountNodes
//-----------------------------------------------------------------------------

// CountNodes returns the count of <role> nodes defined in <quads>.
func CountNodes(quads []string, role string) (count int) {

	// Default to zero:
	count = 0

	// Get the role count:
	for _, q := range quads {
		if strings.Contains(q, role) {
			s := strings.Split(q, ":")
			count, _ = strconv.Atoi(s[0])
			break
		}
	}

	return
}

//-----------------------------------------------------------------------------
// func: CreateDNSZones
//-----------------------------------------------------------------------------

// CreateDNSZones creates (int|ext).<domain> zones using <provider>.
func CreateDNSZones(wch *WaitChan, provider, apiKey, domain string) error {

	// Decrement:
	if wch != nil {
		defer wch.WaitGrp.Done()
	}

	// Forge the zone command:
	cmd := exec.Command("katoctl", provider,
		"--api-key", apiKey, "zone", "add",
		domain, "int."+domain, "ext."+domain)

	// Execute the zone command:
	cmd.Stderr = os.Stderr
	if err := cmd.Run(); err != nil {
		if wch != nil {
			wch.ErrChan <- err
		}
		return err
	}

	return nil
}

//-----------------------------------------------------------------------------
// func: ExecutePipeline
//-----------------------------------------------------------------------------

// ExecutePipeline takes two commands and pipes the stdout of the first one
// into the stdin of the second one. Returns the output as []byte.
func ExecutePipeline(cmd1, cmd2 *exec.Cmd) ([]byte, error) {

	var err error

	// Adjust the stderr:
	cmd1.Stderr = os.Stderr
	cmd2.Stderr = os.Stderr

	// Connect both commands:
	cmd2.Stdin, err = cmd1.StdoutPipe()
	if err != nil {
		return nil, err
	}

	// Get cmd2 stdout:
	stdout, err := cmd2.StdoutPipe()
	if err != nil {
		return nil, err
	}

	// Execute the pipeline:
	if err = cmd2.Start(); err != nil {
		return nil, err
	}
	if err = cmd1.Run(); err != nil {
		return nil, err
	}

	// Read the cmd2 output:
	out, err := ioutil.ReadAll(stdout)
	if err != nil {
		return nil, err
	}

	// Wait and return:
	if err = cmd2.Wait(); err != nil {
		return nil, err
	}

	return out, nil
}

//-----------------------------------------------------------------------------
// func: NewEtcdToken
//-----------------------------------------------------------------------------

// NewEtcdToken takes quorumCount and returns a valid etcd bootstrap token:
func NewEtcdToken(wch *WaitChan, quorumCount int, token *string) error {

	// Decrement:
	if wch != nil {
		defer wch.WaitGrp.Done()
	}

	// Request an etcd bootstrap token:
	res, err := http.Get("https://discovery.etcd.io/new?size=" + strconv.Itoa(quorumCount))
	if err != nil {
		if wch != nil {
			wch.ErrChan <- err
		}
		return err
	}

	// Retrieve the token URL:
	tokenURL, err := ioutil.ReadAll(res.Body)
	if err != nil {
		if wch != nil {
			wch.ErrChan <- err
		}
		return err
	}

	// Call the close method:
	_ = res.Body.Close()

	// Return the token ID:
	slice := strings.Split(string(tokenURL), "/")
	*token = slice[len(slice)-1]
	return nil
}

//-----------------------------------------------------------------------------
// func: OffsetIP
//-----------------------------------------------------------------------------

// OffsetIP takes a CIDR and an offset and returns the IP address at the offset
// position starting at the beginning of the CIDR's subnet:
func OffsetIP(cidr string, offset int) string {

	// Parse the CIDR:
	ip1, ipnet, err := net.ParseCIDR(cidr)
	if err != nil {
		return ""
	}

	// Compute the IP:
	ip2 := ip1.Mask(ipnet.Mask)
	a := int(ipToI32(ip2[len(ip2)-4:]))

	// Return:
	return i32ToIP(int32(a + offset)).String()
}

func ipToI32(ip net.IP) int32 {
	ip = ip.To4()
	return int32(ip[0])<<24 | int32(ip[1])<<16 | int32(ip[2])<<8 | int32(ip[3])
}

func i32ToIP(a int32) net.IP {
	return net.IPv4(byte(a>>24), byte(a>>16), byte(a>>8), byte(a))
}
<commit_msg>Return errors via channel (only)<commit_after>package kato

//-----------------------------------------------------------------------------
// Package factored import statement:
//-----------------------------------------------------------------------------

import (

	// Stdlib:
	"encoding/json"
	"io/ioutil"
	"net"
	"net/http"
	"os"
	"os/exec"
	"strconv"
	"strings"
	"sync"
)

//-----------------------------------------------------------------------------
// WaitChan stuff:
//-----------------------------------------------------------------------------

// WaitChan is used to handle errors that occur in some goroutines.
type WaitChan struct {
	WaitGrp sync.WaitGroup
	ErrChan chan error
	EndChan chan bool
}

// NewWaitChan initializes a WaitChan struct.
func NewWaitChan(len int) *WaitChan {
	wch := new(WaitChan)
	wch.WaitGrp.Add(len)
	wch.ErrChan = make(chan error, 1)
	wch.EndChan = make(chan bool, 1)
	return wch
}

// WaitErr waits for any error or for all go routines to finish.
func (wch *WaitChan) WaitErr() error {

	// Put the wait group in a go routine:
	go func() {
		wch.WaitGrp.Wait()
		wch.EndChan <- true
	}()

	// This select will block:
	select {
	case <-wch.EndChan:
		return nil
	case err := <-wch.ErrChan:
		return err
	}
}

//-----------------------------------------------------------------------------
// func: DumpState
//-----------------------------------------------------------------------------

func DumpState(s interface{}, id string) error {

	// Marshal the data:
	data, err := json.MarshalIndent(s, "", "  ")
	if err != nil {
		return err
	}

	// Create the state directory:
	path := os.Getenv("HOME") + "/.kato"
	if _, err := os.Stat(path); err != nil {
		if os.IsNotExist(err) {
			err = os.Mkdir(path, 0700)
			if err != nil {
				return err
			}
		}
	}

	// Write the state file:
	err = ioutil.WriteFile(path+"/"+id+".json", data, 0600)
	if err != nil {
		return err
	}

	return nil
}

//-----------------------------------------------------------------------------
// func: ReadState
//-----------------------------------------------------------------------------

// ReadState reads the current ClusterID state file:
func ReadState(clusterID string) ([]byte, error) {

	// Read data from state file:
	stateFile := os.Getenv("HOME") + "/.kato/" + clusterID + ".json"
	raw, err := ioutil.ReadFile(stateFile)
	if err != nil {
		return nil, err
	}

	return raw, nil
}

//-----------------------------------------------------------------------------
// func: CountNodes
//-----------------------------------------------------------------------------

// CountNodes returns the count of <role> nodes defined in <quads>.
func CountNodes(quads []string, role string) (count int) {

	// Default to zero:
	count = 0

	// Get the role count:
	for _, q := range quads {
		if strings.Contains(q, role) {
			s := strings.Split(q, ":")
			count, _ = strconv.Atoi(s[0])
			break
		}
	}

	return
}

//-----------------------------------------------------------------------------
// func: CreateDNSZones
//-----------------------------------------------------------------------------

// CreateDNSZones creates (int|ext).<domain> zones using <provider>.
func CreateDNSZones(wch *WaitChan, provider, apiKey, domain string) {

	// Decrement:
	defer wch.WaitGrp.Done()

	// Forge the zone command:
	cmd := exec.Command("katoctl", provider,
		"--api-key", apiKey, "zone", "add",
		domain, "int."+domain, "ext."+domain)

	// Execute the zone command:
	cmd.Stderr = os.Stderr
	if err := cmd.Run(); err != nil {
		wch.ErrChan <- err
	}
}

//-----------------------------------------------------------------------------
// func: ExecutePipeline
//-----------------------------------------------------------------------------

// ExecutePipeline takes two commands and pipes the stdout of the first one
// into the stdin of the second one. Returns the output as []byte.
func ExecutePipeline(cmd1, cmd2 *exec.Cmd) ([]byte, error) {

	var err error

	// Adjust the stderr:
	cmd1.Stderr = os.Stderr
	cmd2.Stderr = os.Stderr

	// Connect both commands:
	cmd2.Stdin, err = cmd1.StdoutPipe()
	if err != nil {
		return nil, err
	}

	// Get cmd2 stdout:
	stdout, err := cmd2.StdoutPipe()
	if err != nil {
		return nil, err
	}

	// Execute the pipeline:
	if err = cmd2.Start(); err != nil {
		return nil, err
	}
	if err = cmd1.Run(); err != nil {
		return nil, err
	}

	// Read the cmd2 output:
	out, err := ioutil.ReadAll(stdout)
	if err != nil {
		return nil, err
	}

	// Wait and return:
	if err = cmd2.Wait(); err != nil {
		return nil, err
	}

	return out, nil
}

//-----------------------------------------------------------------------------
// func: NewEtcdToken
//-----------------------------------------------------------------------------

// NewEtcdToken takes quorumCount and returns a valid etcd bootstrap token:
func NewEtcdToken(wch *WaitChan, quorumCount int, token *string) {

	// Decrement:
	defer wch.WaitGrp.Done()

	// Request an etcd bootstrap token:
	res, err := http.Get("https://discovery.etcd.io/new?size=" + strconv.Itoa(quorumCount))
	if err != nil {
		wch.ErrChan <- err
	}

	// Retrieve the token URL:
	tokenURL, err := ioutil.ReadAll(res.Body)
	if err != nil {
		wch.ErrChan <- err
	}

	// Call the close method:
	_ = res.Body.Close()

	// Return the token ID:
	slice := strings.Split(string(tokenURL), "/")
	*token = slice[len(slice)-1]
}

//-----------------------------------------------------------------------------
// func: OffsetIP
//-----------------------------------------------------------------------------

// OffsetIP takes a CIDR and an offset and returns the IP address at the offset
// position starting at the beginning of the CIDR's subnet:
func OffsetIP(cidr string, offset int) string {

	// Parse the CIDR:
	ip1, ipnet, err := net.ParseCIDR(cidr)
	if err != nil {
		return ""
	}

	// Compute the IP:
	ip2 := ip1.Mask(ipnet.Mask)
	a := int(ipToI32(ip2[len(ip2)-4:]))

	// Return:
	return i32ToIP(int32(a + offset)).String()
}

func ipToI32(ip net.IP) int32 {
	ip = ip.To4()
	return int32(ip[0])<<24 | int32(ip[1])<<16 | int32(ip[2])<<8 | int32(ip[3])
}

func i32ToIP(a int32) net.IP {
	return net.IPv4(byte(a>>24), byte(a>>16), byte(a>>8), byte(a))
}
<|endoftext|>


<commit_before>package cafs

import (
	"bytes"
	"crypto/sha1"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"sync"

	"github.com/tchaik/tchaik/store"
)

// Index is an interface which contains methods for implementing
// a content addressable file system.
type Index interface {
	// Get returns the real path for the given filename, with true if and only
	// if the path exists in the index.
	Get(path string) (string, bool)

	// Add adds the path to the index, and returns the path to the file
	// and whether the path/content already exists.
	Add(path string, sum string) bool

	// Exists returns true of the sum is in the index.
	Exists(sum string) bool
}

type index struct {
	sync.RWMutex

	files map[string]string // path -> sha1
	index map[string]bool   // {sha1}
}

// NewIndex creates a new file system index.
func NewIndex() *index {
	return &index{
		files: make(map[string]string),
		index: make(map[string]bool),
	}
}

func (i *index) MarshalJSON() ([]byte, error) {
	i.RLock()
	defer i.RUnlock()

	exp := struct {
		Files map[string]string `json:"files"`
	}{
		Files: i.files,
	}
	return json.Marshal(exp)
}

func (i *index) UnmarshalJSON(b []byte) error {
	i.Lock()
	defer i.Unlock()

	var exp struct {
		Files map[string]string
	}
	err := json.Unmarshal(b, &exp)
	if err != nil {
		return err
	}

	i.files = make(map[string]string)
	i.index = make(map[string]bool)
	for k, v := range exp.Files {
		i.index[v] = true
		i.files[k] = v
	}
	return nil
}

// Get implements Index.
func (i *index) Get(path string) (string, bool) {
	i.RLock()
	defer i.RUnlock()

	x, ok := i.files[path]
	return x, ok
}

// Add implements Index.
func (i *index) Add(path, sum string) bool {
	i.Lock()
	defer i.Unlock()

	i.files[path] = sum
	if !i.index[sum] {
		i.index[sum] = true
		return false
	}
	return true
}

// Exists implements Index.
func (i *index) Exists(sum string) bool {
	i.RLock()
	defer i.RUnlock()

	return i.index[sum]
}

// Add the path+data to the index.  Returns the content sum and true if the content
// already existed in the index, false otherwise.
func AddContent(idx Index, path string, content []byte) (string, bool) {
	if x, ok := idx.Get(path); ok {
		return x, true
	}
	s := fmt.Sprintf("%x", sha1.Sum(content))
	return s, idx.Add(path, s)
}

// FileSystem is a type which defines a content addressable filesystem.
type FileSystem struct {
	idx Index

	fs store.RWFileSystem
}

// Open the file with the given path.  Uses the internal index to identify
// which file to open.  NB: Stat on the returned file will refer to the
// internal file.
func (s *FileSystem) Open(path string) (http.File, error) {
	realPath, ok := s.idx.Get(path)
	if !ok {
		return nil, fmt.Errorf("no such file: %v", path)
	}
	return s.open(realPath)
}

// Wait implements RWFileSystem.
func (s *FileSystem) Wait() error { return nil }

func (s *FileSystem) open(path string) (http.File, error) {
	return s.fs.Open(path)
}

func (s *FileSystem) create(path string) (io.WriteCloser, error) {
	return s.fs.Create(path)
}

type file struct {
	bytes.Buffer

	fs   *FileSystem
	path string
}

// Close acts as a signal that all the information has been written to
// the underlying buffer, and the file can be written to the RWFileSystem.
func (a *file) Close() error {
	_, ok := a.fs.idx.Get(a.path)
	if ok {
		return fmt.Errorf("file already exists: %v", a.path)
	}

	path, ok := AddContent(a.fs.idx, a.path, a.Bytes())
	if !ok {
		fmt.Println("creating", path)
		f, err := a.fs.create(path)
		if err != nil {
			return fmt.Errorf("error creating file: %v", err)
		}
		defer f.Close()
		_, err = io.Copy(f, a)
		if err != nil {
			return fmt.Errorf("error copying data into file '%v': %v", path, err)
		}
	}
	err := a.fs.persist()
	if err != nil {
		return err
	}
	return nil
}

// Create a new file with path. We buffer the contents written to the io.WriteCloser
// so that the content can be hashed and then written to the underlying RWFileSystem.
func (s *FileSystem) Create(path string) (io.WriteCloser, error) {
	_, ok := s.idx.Get(path)
	if ok {
		return nil, fmt.Errorf("file already exists for '%v'", path)
	}

	return &file{
		Buffer: bytes.Buffer{},
		path:   path,
		fs:     s,
	}, nil
}

// initIndex initialises the cafs index.
func (s *FileSystem) initIndex() error {
	f, err := s.open(".idx")
	if err != nil {
		// FIXME: Improve this
		// Can't guarantee that we will get an IsNotExist(err) here
		return nil
	}
	defer f.Close()

	b, err := ioutil.ReadAll(f)
	if err != nil {
		return fmt.Errorf("error reading index: %v", err)
	}
	err = json.Unmarshal(b, s.idx)
	if err != nil {
		return fmt.Errorf("error decoding index: %v", err)
	}
	fmt.Printf("Index initialised: %d files (%d paths)", len(s.idx.(*index).index), len(s.idx.(*index).files))
	return nil
}

func (s *FileSystem) persist() error {
	f, err := s.fs.Create(".idx")
	if err != nil {
		return fmt.Errorf("error creating index: %v", err)
	}
	defer f.Close()

	b, err := json.Marshal(s.idx)
	if err != nil {
		return fmt.Errorf("error encoding index: %v", err)
	}

	_, err = f.Write(b)
	if err != nil {
		return fmt.Errorf("error writing index: %v", err)
	}
	return nil
}

// New creates a new content addressable RWFileSystem.
func New(fs store.RWFileSystem) (*FileSystem, error) {
	s := &FileSystem{
		idx: NewIndex(),
		fs:  fs,
	}
	err := s.initIndex()
	if err != nil {
		return nil, err
	}
	return s, nil
}
<commit_msg>Refactor Index.Add().<commit_after>package cafs

import (
	"bytes"
	"crypto/sha1"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"sync"

	"github.com/tchaik/tchaik/store"
)

// Index is an interface which contains methods for implementing
// a content addressable file system.
type Index interface {
	// Get returns the real path for the given filename, with true if and only
	// if the path exists in the index.
	Get(path string) (string, bool)

	// Add adds the path to the index, and returns the path to the file
	// and whether the path/content already exists.
	Add(path string, sum string) bool

	// Exists returns true of the sum is in the index.
	Exists(sum string) bool
}

type index struct {
	sync.RWMutex

	files map[string]string // path -> sha1
	index map[string]bool   // {sha1}
}

// NewIndex creates a new file system index.
func NewIndex() *index {
	return &index{
		files: make(map[string]string),
		index: make(map[string]bool),
	}
}

func (i *index) MarshalJSON() ([]byte, error) {
	i.RLock()
	defer i.RUnlock()

	exp := struct {
		Files map[string]string `json:"files"`
	}{
		Files: i.files,
	}
	return json.Marshal(exp)
}

func (i *index) UnmarshalJSON(b []byte) error {
	i.Lock()
	defer i.Unlock()

	var exp struct {
		Files map[string]string
	}
	err := json.Unmarshal(b, &exp)
	if err != nil {
		return err
	}

	i.files = make(map[string]string)
	i.index = make(map[string]bool)
	for k, v := range exp.Files {
		i.index[v] = true
		i.files[k] = v
	}
	return nil
}

// Get implements Index.
func (i *index) Get(path string) (string, bool) {
	i.RLock()
	defer i.RUnlock()

	x, ok := i.files[path]
	return x, ok
}

// Add implements Index.
func (i *index) Add(path, sum string) bool {
	i.Lock()
	defer i.Unlock()

	i.files[path] = sum
	old := i.index[sum]
	i.index[sum] = true
	return old
}

// Exists implements Index.
func (i *index) Exists(sum string) bool {
	i.RLock()
	defer i.RUnlock()

	return i.index[sum]
}

// Add the path+data to the index.  Returns the content sum and true if the content
// already existed in the index, false otherwise.
func AddContent(idx Index, path string, content []byte) (string, bool) {
	if x, ok := idx.Get(path); ok {
		return x, true
	}
	s := fmt.Sprintf("%x", sha1.Sum(content))
	return s, idx.Add(path, s)
}

// FileSystem is a type which defines a content addressable filesystem.
type FileSystem struct {
	idx Index

	fs store.RWFileSystem
}

// Open the file with the given path.  Uses the internal index to identify
// which file to open.  NB: Stat on the returned file will refer to the
// internal file.
func (s *FileSystem) Open(path string) (http.File, error) {
	realPath, ok := s.idx.Get(path)
	if !ok {
		return nil, fmt.Errorf("no such file: %v", path)
	}
	return s.open(realPath)
}

// Wait implements RWFileSystem.
func (s *FileSystem) Wait() error { return nil }

func (s *FileSystem) open(path string) (http.File, error) {
	return s.fs.Open(path)
}

func (s *FileSystem) create(path string) (io.WriteCloser, error) {
	return s.fs.Create(path)
}

type file struct {
	bytes.Buffer

	fs   *FileSystem
	path string
}

// Close acts as a signal that all the information has been written to
// the underlying buffer, and the file can be written to the RWFileSystem.
func (a *file) Close() error {
	_, ok := a.fs.idx.Get(a.path)
	if ok {
		return fmt.Errorf("file already exists: %v", a.path)
	}

	path, ok := AddContent(a.fs.idx, a.path, a.Bytes())
	if !ok {
		fmt.Println("creating", path)
		f, err := a.fs.create(path)
		if err != nil {
			return fmt.Errorf("error creating file: %v", err)
		}
		defer f.Close()
		_, err = io.Copy(f, a)
		if err != nil {
			return fmt.Errorf("error copying data into file '%v': %v", path, err)
		}
	}
	err := a.fs.persist()
	if err != nil {
		return err
	}
	return nil
}

// Create a new file with path. We buffer the contents written to the io.WriteCloser
// so that the content can be hashed and then written to the underlying RWFileSystem.
func (s *FileSystem) Create(path string) (io.WriteCloser, error) {
	_, ok := s.idx.Get(path)
	if ok {
		return nil, fmt.Errorf("file already exists for '%v'", path)
	}

	return &file{
		Buffer: bytes.Buffer{},
		path:   path,
		fs:     s,
	}, nil
}

// initIndex initialises the cafs index.
func (s *FileSystem) initIndex() error {
	f, err := s.open(".idx")
	if err != nil {
		// FIXME: Improve this
		// Can't guarantee that we will get an IsNotExist(err) here
		return nil
	}
	defer f.Close()

	b, err := ioutil.ReadAll(f)
	if err != nil {
		return fmt.Errorf("error reading index: %v", err)
	}
	err = json.Unmarshal(b, s.idx)
	if err != nil {
		return fmt.Errorf("error decoding index: %v", err)
	}
	fmt.Printf("Index initialised: %d files (%d paths)", len(s.idx.(*index).index), len(s.idx.(*index).files))
	return nil
}

func (s *FileSystem) persist() error {
	f, err := s.fs.Create(".idx")
	if err != nil {
		return fmt.Errorf("error creating index: %v", err)
	}
	defer f.Close()

	b, err := json.Marshal(s.idx)
	if err != nil {
		return fmt.Errorf("error encoding index: %v", err)
	}

	_, err = f.Write(b)
	if err != nil {
		return fmt.Errorf("error writing index: %v", err)
	}
	return nil
}

// New creates a new content addressable RWFileSystem.
func New(fs store.RWFileSystem) (*FileSystem, error) {
	s := &FileSystem{
		idx: NewIndex(),
		fs:  fs,
	}
	err := s.initIndex()
	if err != nil {
		return nil, err
	}
	return s, nil
}
<|endoftext|>


<commit_before>/*
Copyright 2015 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package storage

import (
	"fmt"

	"k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/util/validation/field"
)

const (
	ErrCodeKeyNotFound int = iota + 1
	ErrCodeKeyExists
	ErrCodeResourceVersionConflicts
	ErrCodeInvalidObj
	ErrCodeUnreachable
)

var errCodeToMessage = map[int]string{
	ErrCodeKeyNotFound:              "key not found",
	ErrCodeKeyExists:                "key exists",
	ErrCodeResourceVersionConflicts: "resource version conflicts",
	ErrCodeInvalidObj:               "invalid object",
	ErrCodeUnreachable:              "server unreachable",
}

func NewKeyNotFoundError(key string, rv int64) *StorageError {
	return &StorageError{
		Code:            ErrCodeKeyNotFound,
		Key:             key,
		ResourceVersion: rv,
	}
}

func NewKeyExistsError(key string, rv int64) *StorageError {
	return &StorageError{
		Code:            ErrCodeKeyExists,
		Key:             key,
		ResourceVersion: rv,
	}
}

func NewResourceVersionConflictsError(key string, rv int64) *StorageError {
	return &StorageError{
		Code:            ErrCodeResourceVersionConflicts,
		Key:             key,
		ResourceVersion: rv,
	}
}

func NewUnreachableError(key string, rv int64) *StorageError {
	return &StorageError{
		Code:            ErrCodeUnreachable,
		Key:             key,
		ResourceVersion: rv,
	}
}

func NewInvalidObjError(key, msg string) *StorageError {
	return &StorageError{
		Code:               ErrCodeInvalidObj,
		Key:                key,
		AdditionalErrorMsg: msg,
	}
}

type StorageError struct {
	Code               int
	Key                string
	ResourceVersion    int64
	AdditionalErrorMsg string
}

func (e *StorageError) Error() string {
	return fmt.Sprintf("StorageError: %s, Code: %d, Key: %s, ResourceVersion: %d, AdditionalErrorMsg: %s",
		errCodeToMessage[e.Code], e.Code, e.Key, e.ResourceVersion, e.AdditionalErrorMsg)
}

// IsNotFound returns true if and only if err is "key" not found error.
func IsNotFound(err error) bool {
	return isErrCode(err, ErrCodeKeyNotFound)
}

// IsNodeExist returns true if and only if err is an node already exist error.
func IsNodeExist(err error) bool {
	return isErrCode(err, ErrCodeKeyExists)
}

// IsUnreachable returns true if and only if err indicates the server could not be reached.
func IsUnreachable(err error) bool {
	return isErrCode(err, ErrCodeUnreachable)
}

// IsConflict returns true if and only if err is a write conflict.
func IsConflict(err error) bool {
	return isErrCode(err, ErrCodeResourceVersionConflicts)
}

// IsInvalidObj returns true if and only if err is invalid error
func IsInvalidObj(err error) bool {
	return isErrCode(err, ErrCodeInvalidObj)
}

func isErrCode(err error, code int) bool {
	if err == nil {
		return false
	}
	if e, ok := err.(*StorageError); ok {
		return e.Code == code
	}
	return false
}

// InvalidError is generated when an error caused by invalid API object occurs
// in the storage package.
type InvalidError struct {
	Errs field.ErrorList
}

func (e InvalidError) Error() string {
	return e.Errs.ToAggregate().Error()
}

// IsInvalidError returns true if and only if err is an InvalidError.
func IsInvalidError(err error) bool {
	_, ok := err.(InvalidError)
	return ok
}

func NewInvalidError(errors field.ErrorList) InvalidError {
	return InvalidError{errors}
}

// InternalError is generated when an error occurs in the storage package, i.e.,
// not from the underlying storage backend (e.g., etcd).
type InternalError struct {
	Reason string
}

func (e InternalError) Error() string {
	return e.Reason
}

// IsInternalError returns true if and only if err is an InternalError.
func IsInternalError(err error) bool {
	_, ok := err.(InternalError)
	return ok
}

func NewInternalError(reason string) InternalError {
	return InternalError{reason}
}

func NewInternalErrorf(format string, a ...interface{}) InternalError {
	return InternalError{fmt.Sprintf(format, a...)}
}

var tooLargeResourceVersionCauseMsg = "Too large resource version"

// NewTooLargeResourceVersionError returns a timeout error with the given retrySeconds for a request for
// a minimum resource version that is larger than the largest currently available resource version for a requested resource.
func NewTooLargeResourceVersionError(minimumResourceVersion, currentRevision uint64, retrySeconds int) error {
	err := errors.NewTimeoutError(fmt.Sprintf("Too large resource version: %d, current: %d", minimumResourceVersion, currentRevision), retrySeconds)
	err.ErrStatus.Details.Causes = []metav1.StatusCause{{Message: tooLargeResourceVersionCauseMsg}}
	return err
}

// IsTooLargeResourceVersion returns true if the error is a TooLargeResourceVersion error.
func IsTooLargeResourceVersion(err error) bool {
	if !errors.IsTimeout(err) {
		return false
	}
	switch t := err.(type) {
	case errors.APIStatus:
		if d := t.Status().Details; d != nil {
			for _, cause := range d.Causes {
				if cause.Message == tooLargeResourceVersionCauseMsg {
					return true
				}
			}
		}
	}
	return false
}
<commit_msg>Fix bug in reflector not recovering from "Too large resource version" errors<commit_after>/*
Copyright 2015 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package storage

import (
	"fmt"

	"k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/util/validation/field"
)

const (
	ErrCodeKeyNotFound int = iota + 1
	ErrCodeKeyExists
	ErrCodeResourceVersionConflicts
	ErrCodeInvalidObj
	ErrCodeUnreachable
)

var errCodeToMessage = map[int]string{
	ErrCodeKeyNotFound:              "key not found",
	ErrCodeKeyExists:                "key exists",
	ErrCodeResourceVersionConflicts: "resource version conflicts",
	ErrCodeInvalidObj:               "invalid object",
	ErrCodeUnreachable:              "server unreachable",
}

func NewKeyNotFoundError(key string, rv int64) *StorageError {
	return &StorageError{
		Code:            ErrCodeKeyNotFound,
		Key:             key,
		ResourceVersion: rv,
	}
}

func NewKeyExistsError(key string, rv int64) *StorageError {
	return &StorageError{
		Code:            ErrCodeKeyExists,
		Key:             key,
		ResourceVersion: rv,
	}
}

func NewResourceVersionConflictsError(key string, rv int64) *StorageError {
	return &StorageError{
		Code:            ErrCodeResourceVersionConflicts,
		Key:             key,
		ResourceVersion: rv,
	}
}

func NewUnreachableError(key string, rv int64) *StorageError {
	return &StorageError{
		Code:            ErrCodeUnreachable,
		Key:             key,
		ResourceVersion: rv,
	}
}

func NewInvalidObjError(key, msg string) *StorageError {
	return &StorageError{
		Code:               ErrCodeInvalidObj,
		Key:                key,
		AdditionalErrorMsg: msg,
	}
}

type StorageError struct {
	Code               int
	Key                string
	ResourceVersion    int64
	AdditionalErrorMsg string
}

func (e *StorageError) Error() string {
	return fmt.Sprintf("StorageError: %s, Code: %d, Key: %s, ResourceVersion: %d, AdditionalErrorMsg: %s",
		errCodeToMessage[e.Code], e.Code, e.Key, e.ResourceVersion, e.AdditionalErrorMsg)
}

// IsNotFound returns true if and only if err is "key" not found error.
func IsNotFound(err error) bool {
	return isErrCode(err, ErrCodeKeyNotFound)
}

// IsNodeExist returns true if and only if err is an node already exist error.
func IsNodeExist(err error) bool {
	return isErrCode(err, ErrCodeKeyExists)
}

// IsUnreachable returns true if and only if err indicates the server could not be reached.
func IsUnreachable(err error) bool {
	return isErrCode(err, ErrCodeUnreachable)
}

// IsConflict returns true if and only if err is a write conflict.
func IsConflict(err error) bool {
	return isErrCode(err, ErrCodeResourceVersionConflicts)
}

// IsInvalidObj returns true if and only if err is invalid error
func IsInvalidObj(err error) bool {
	return isErrCode(err, ErrCodeInvalidObj)
}

func isErrCode(err error, code int) bool {
	if err == nil {
		return false
	}
	if e, ok := err.(*StorageError); ok {
		return e.Code == code
	}
	return false
}

// InvalidError is generated when an error caused by invalid API object occurs
// in the storage package.
type InvalidError struct {
	Errs field.ErrorList
}

func (e InvalidError) Error() string {
	return e.Errs.ToAggregate().Error()
}

// IsInvalidError returns true if and only if err is an InvalidError.
func IsInvalidError(err error) bool {
	_, ok := err.(InvalidError)
	return ok
}

func NewInvalidError(errors field.ErrorList) InvalidError {
	return InvalidError{errors}
}

// InternalError is generated when an error occurs in the storage package, i.e.,
// not from the underlying storage backend (e.g., etcd).
type InternalError struct {
	Reason string
}

func (e InternalError) Error() string {
	return e.Reason
}

// IsInternalError returns true if and only if err is an InternalError.
func IsInternalError(err error) bool {
	_, ok := err.(InternalError)
	return ok
}

func NewInternalError(reason string) InternalError {
	return InternalError{reason}
}

func NewInternalErrorf(format string, a ...interface{}) InternalError {
	return InternalError{fmt.Sprintf(format, a...)}
}

var tooLargeResourceVersionCauseMsg = "Too large resource version"

// NewTooLargeResourceVersionError returns a timeout error with the given retrySeconds for a request for
// a minimum resource version that is larger than the largest currently available resource version for a requested resource.
func NewTooLargeResourceVersionError(minimumResourceVersion, currentRevision uint64, retrySeconds int) error {
	err := errors.NewTimeoutError(fmt.Sprintf("Too large resource version: %d, current: %d", minimumResourceVersion, currentRevision), retrySeconds)
	err.ErrStatus.Details.Causes = []metav1.StatusCause{
		{
			Type:    metav1.CauseTypeResourceVersionTooLarge,
			Message: tooLargeResourceVersionCauseMsg,
		},
	}
	return err
}

// IsTooLargeResourceVersion returns true if the error is a TooLargeResourceVersion error.
func IsTooLargeResourceVersion(err error) bool {
	if !errors.IsTimeout(err) {
		return false
	}
	return errors.HasStatusCause(err, metav1.CauseTypeResourceVersionTooLarge)
}
<|endoftext|>


<commit_before>package player

import "testing"

type testPlayer string

func (p testPlayer) Key() string           { return string(p) }
func (testPlayer) Do(a Action) error       { return nil }
func (testPlayer) SetMute(bool) error      { return nil }
func (testPlayer) SetRepeat(bool) error    { return nil }
func (testPlayer) SetVolume(float64) error { return nil }
func (testPlayer) SetTime(float64) error   { return nil }

func TestPlayers(t *testing.T) {
	oneKey := "one"
	onePl := testPlayer(oneKey)

	ps := NewPlayers()

	list := ps.List()
	if len(list) != 0 {
		t.Errorf("len(ps.List()) = %d, expected: %d", len(list), 0)
	}

	ps.Add(onePl)

	p := ps.Get(oneKey)
	if p != onePl {
		t.Errorf("Get(%#v) = %#v, expected: %#v", oneKey, p, onePl)
	}

	list = ps.List()
	if len(list) != 1 {
		t.Errorf("len(ps.List()) = %d, expected: %d", len(list), 1)
	}

	ps.Remove(oneKey)

	p = ps.Get(oneKey)
	if p != nil {
		t.Errorf("Get(%#v) = %#v, expected: %#v", oneKey, p, nil)
	}

	list = ps.List()
	if len(list) != 0 {
		t.Errorf("len(ps.List()) = %d, expected: %d", len(list), 0)
	}
}
<commit_msg>Remove print from TestPlayers<commit_after>package player

import (
	"encoding/json"
	"fmt"
	"testing"
)

type testPlayer string

func (p testPlayer) Key() string           { return string(p) }
func (testPlayer) Do(a Action) error       { return nil }
func (testPlayer) SetMute(bool) error      { return nil }
func (testPlayer) SetRepeat(bool) error    { return nil }
func (testPlayer) SetVolume(float64) error { return nil }
func (testPlayer) SetTime(float64) error   { return nil }

func TestPlayers(t *testing.T) {
	oneKey := "one"
	onePl := testPlayer(oneKey)

	ps := NewPlayers()

	list := ps.List()
	if len(list) != 0 {
		t.Errorf("len(ps.List()) = %d, expected: %d", len(list), 0)
	}

	ps.Add(onePl)

	p := ps.Get(oneKey)
	if p != onePl {
		t.Errorf("Get(%#v) = %#v, expected: %#v", oneKey, p, onePl)
	}

	b, err := json.Marshal(p)
	if err != nil {
		t.Errorf("json.Marshal(p) returned an unexpected error: %v", err)
	}

	list = ps.List()
	if len(list) != 1 {
		t.Errorf("len(ps.List()) = %d, expected: %d", len(list), 1)
	}

	ps.Remove(oneKey)

	p = ps.Get(oneKey)
	if p != nil {
		t.Errorf("Get(%#v) = %#v, expected: %#v", oneKey, p, nil)
	}

	list = ps.List()
	if len(list) != 0 {
		t.Errorf("len(ps.List()) = %d, expected: %d", len(list), 0)
	}
}
<|endoftext|>


<commit_before>package common

import (
	"encoding/json"
	"fmt"
	"time"

	"github.com/oursky/skygear/skydb"
	"github.com/oursky/skygear/skydb/skydbconv"
)

// ExecError is error resulted from application logic of plugin (e.g.
// an exception thrown within a lambda function)
type ExecError struct {
	Name        string `json:"name"`
	Description string `json:"desc"`
}

func (err *ExecError) Error() string {
	return err.Name + "\n" + err.Description
}

// JSONRecord defines a common serialization format for skydb.Record
type JSONRecord skydb.Record

// MarshalJSON implements json.Marshaler
func (record *JSONRecord) MarshalJSON() ([]byte, error) {
	data := map[string]interface{}{}
	for key, value := range record.Data {
		switch v := value.(type) {
		case time.Time:
			data[key] = (skydbconv.MapTime)(v)
		case skydb.Reference:
			data[key] = (skydbconv.MapReference)(v)
		case *skydb.Location:
			data[key] = (*skydbconv.MapLocation)(v)
		case *skydb.Asset:
			data[key] = (*skydbconv.MapAsset)(v)
		default:
			data[key] = value
		}
	}

	m := map[string]interface{}{}
	skydbconv.MapData(data).ToMap(m)

	m["_id"] = record.ID
	m["_ownerID"] = record.OwnerID
	m["_access"] = record.ACL

	if !record.CreatedAt.IsZero() {
		m["_created_at"] = record.CreatedAt
	}
	if record.CreatorID != "" {
		m["_created_by"] = record.CreatorID
	}
	if !record.UpdatedAt.IsZero() {
		m["_updated_at"] = record.UpdatedAt
	}
	if record.UpdaterID != "" {
		m["_updated_by"] = record.UpdaterID
	}

	return json.Marshal(m)
}

// UnmarshalJSON implements json.Unmarshaler
func (record *JSONRecord) UnmarshalJSON(data []byte) (err error) {
	m := map[string]interface{}{}
	if err := json.Unmarshal(data, &m); err != nil {
		return err
	}

	var (
		id      skydb.RecordID
		acl     skydb.RecordACL
		dataMap map[string]interface{}
	)

	extractor := newMapExtractor(m)
	extractor.DoString("_id", func(s string) error {
		return id.UnmarshalText([]byte(s))
	})
	extractor.DoSlice("_access", func(slice []interface{}) error {
		return acl.InitFromJSON(slice)
	})
	if extractor.Err() != nil {
		return extractor.Err()
	}

	m = sanitizedDataMap(m)
	if err := (*skydbconv.MapData)(&dataMap).FromMap(m); err != nil {
		return err
	}

	record.ID = id
	record.ACL = acl
	record.Data = dataMap
	return nil
}

func sanitizedDataMap(m map[string]interface{}) map[string]interface{} {
	mm := map[string]interface{}{}
	for key, value := range m {
		if key[0] != '_' {
			mm[key] = value
		}
	}
	return mm
}

// mapExtractor helps to extract value of a key from a map
//
// potential candicate of a package
type mapExtractor struct {
	m   map[string]interface{}
	err error
}

func newMapExtractor(m map[string]interface{}) *mapExtractor {
	return &mapExtractor{m: m}
}

// Do execute doFunc if key exists in the map
// The key will always be removed no matter error occurred previously
func (e *mapExtractor) Do(key string, doFunc func(interface{}) error) {
	value, ok := e.m[key]
	delete(e.m, key)

	if e.err != nil {
		return
	}

	if ok {
		e.err = doFunc(value)
		delete(e.m, key)
	} else {
		e.err = fmt.Errorf(`no key "%s" in map`, key)
	}
}

func (e *mapExtractor) DoString(key string, doFunc func(string) error) {
	e.Do(key, func(i interface{}) error {
		if m, ok := i.(string); ok {
			return doFunc(m)
		}
		return fmt.Errorf("key %s is of type %T, not string", key, i)
	})
}

func (e *mapExtractor) DoMap(key string, doFunc func(map[string]interface{}) error) {
	e.Do(key, func(i interface{}) error {
		if m, ok := i.(map[string]interface{}); ok {
			return doFunc(m)
		}
		return fmt.Errorf("key %s is of type %T, not map[string]interface{}", key, i)
	})
}

func (e *mapExtractor) DoSlice(key string, doFunc func([]interface{}) error) {
	e.Do(key, func(i interface{}) error {
		switch slice := i.(type) {
		case []interface{}:
			return doFunc(slice)
		case nil:
			return doFunc(nil)
		default:
			return fmt.Errorf("key %s is of type %T, not []interface{}", key, i)
		}
	})
}

func (e *mapExtractor) Err() error {
	return e.err
}
<commit_msg>Hotfix skygear serializing a wrong location field to plugin<commit_after>package common

import (
	"encoding/json"
	"fmt"
	"time"

	"github.com/oursky/skygear/skydb"
	"github.com/oursky/skygear/skydb/skydbconv"
)

// ExecError is error resulted from application logic of plugin (e.g.
// an exception thrown within a lambda function)
type ExecError struct {
	Name        string `json:"name"`
	Description string `json:"desc"`
}

func (err *ExecError) Error() string {
	return err.Name + "\n" + err.Description
}

// JSONRecord defines a common serialization format for skydb.Record
type JSONRecord skydb.Record

// MarshalJSON implements json.Marshaler
func (record *JSONRecord) MarshalJSON() ([]byte, error) {
	data := map[string]interface{}{}
	for key, value := range record.Data {
		switch v := value.(type) {
		case time.Time:
			data[key] = (skydbconv.MapTime)(v)
		case skydb.Reference:
			data[key] = (skydbconv.MapReference)(v)
		case skydb.Location:
			data[key] = (skydbconv.MapLocation)(v)
		case *skydb.Location:
			data[key] = (*skydbconv.MapLocation)(v)
		case *skydb.Asset:
			data[key] = (*skydbconv.MapAsset)(v)
		default:
			data[key] = value
		}
	}

	m := map[string]interface{}{}
	skydbconv.MapData(data).ToMap(m)

	m["_id"] = record.ID
	m["_ownerID"] = record.OwnerID
	m["_access"] = record.ACL

	if !record.CreatedAt.IsZero() {
		m["_created_at"] = record.CreatedAt
	}
	if record.CreatorID != "" {
		m["_created_by"] = record.CreatorID
	}
	if !record.UpdatedAt.IsZero() {
		m["_updated_at"] = record.UpdatedAt
	}
	if record.UpdaterID != "" {
		m["_updated_by"] = record.UpdaterID
	}

	return json.Marshal(m)
}

// UnmarshalJSON implements json.Unmarshaler
func (record *JSONRecord) UnmarshalJSON(data []byte) (err error) {
	m := map[string]interface{}{}
	if err := json.Unmarshal(data, &m); err != nil {
		return err
	}

	var (
		id      skydb.RecordID
		acl     skydb.RecordACL
		dataMap map[string]interface{}
	)

	extractor := newMapExtractor(m)
	extractor.DoString("_id", func(s string) error {
		return id.UnmarshalText([]byte(s))
	})
	extractor.DoSlice("_access", func(slice []interface{}) error {
		return acl.InitFromJSON(slice)
	})
	if extractor.Err() != nil {
		return extractor.Err()
	}

	m = sanitizedDataMap(m)
	if err := (*skydbconv.MapData)(&dataMap).FromMap(m); err != nil {
		return err
	}

	record.ID = id
	record.ACL = acl
	record.Data = dataMap
	return nil
}

func sanitizedDataMap(m map[string]interface{}) map[string]interface{} {
	mm := map[string]interface{}{}
	for key, value := range m {
		if key[0] != '_' {
			mm[key] = value
		}
	}
	return mm
}

// mapExtractor helps to extract value of a key from a map
//
// potential candicate of a package
type mapExtractor struct {
	m   map[string]interface{}
	err error
}

func newMapExtractor(m map[string]interface{}) *mapExtractor {
	return &mapExtractor{m: m}
}

// Do execute doFunc if key exists in the map
// The key will always be removed no matter error occurred previously
func (e *mapExtractor) Do(key string, doFunc func(interface{}) error) {
	value, ok := e.m[key]
	delete(e.m, key)

	if e.err != nil {
		return
	}

	if ok {
		e.err = doFunc(value)
		delete(e.m, key)
	} else {
		e.err = fmt.Errorf(`no key "%s" in map`, key)
	}
}

func (e *mapExtractor) DoString(key string, doFunc func(string) error) {
	e.Do(key, func(i interface{}) error {
		if m, ok := i.(string); ok {
			return doFunc(m)
		}
		return fmt.Errorf("key %s is of type %T, not string", key, i)
	})
}

func (e *mapExtractor) DoMap(key string, doFunc func(map[string]interface{}) error) {
	e.Do(key, func(i interface{}) error {
		if m, ok := i.(map[string]interface{}); ok {
			return doFunc(m)
		}
		return fmt.Errorf("key %s is of type %T, not map[string]interface{}", key, i)
	})
}

func (e *mapExtractor) DoSlice(key string, doFunc func([]interface{}) error) {
	e.Do(key, func(i interface{}) error {
		switch slice := i.(type) {
		case []interface{}:
			return doFunc(slice)
		case nil:
			return doFunc(nil)
		default:
			return fmt.Errorf("key %s is of type %T, not []interface{}", key, i)
		}
	})
}

func (e *mapExtractor) Err() error {
	return e.err
}
<|endoftext|>


<commit_before>package ipamplugin

import (
	"fmt"
	"net"
	"strings"

	"github.com/docker/libnetwork/discoverapi"
	"github.com/docker/libnetwork/netlabel"
	"github.com/weaveworks/weave/api"
	"github.com/weaveworks/weave/common"
)

type Ipam struct {
	weave *api.Client
}

func NewIpam(weave *api.Client) *Ipam {
	return &Ipam{weave: weave}
}

func (i *Ipam) GetDefaultAddressSpaces() (string, string, error) {
	i.logReq("GetDefaultAddressSpaces")
	return "weavelocal", "weaveglobal", nil
}

func (i *Ipam) RequestPool(addressSpace, pool, subPool string, options map[string]string, v6 bool) (poolname string, subnet *net.IPNet, data map[string]string, err error) {
	i.logReq("RequestPool", addressSpace, pool, subPool, options)
	defer func() { i.logRes("RequestPool", poolname, subnet, data, err) }()
	if pool == "" {
		subnet, err = i.weave.DefaultSubnet()
	} else {
		_, subnet, err = net.ParseCIDR(pool)
	}
	if err != nil {
		return
	}
	iprange := subnet
	if subPool != "" {
		if _, iprange, err = net.ParseCIDR(subPool); err != nil {
			return
		}
	}
	// Cunningly-constructed pool "name" which gives us what we need later
	poolname = strings.Join([]string{"weave", subnet.String(), iprange.String()}, "-")
	// Pass back a fake "gateway address"; we don't actually use it,
	// so just give the network address.
	data = map[string]string{netlabel.Gateway: subnet.String()}
	return
}

func (i *Ipam) ReleasePool(poolID string) error {
	i.logReq("ReleasePool", poolID)
	return nil
}

func (i *Ipam) RequestAddress(poolID string, address net.IP, options map[string]string) (ip *net.IPNet, _ map[string]string, err error) {
	i.logReq("RequestAddress", poolID, address, options)
	defer func() { i.logRes("RequestAddress", ip, err) }()
	// If we pass magic string "_" to weave IPAM it stores the address under its own string
	if poolID == "weavepool" { // old-style
		ip, err = i.weave.AllocateIP("_")
		return
	}
	parts := strings.Split(poolID, "-")
	if len(parts) != 3 || parts[0] != "weave" {
		err = fmt.Errorf("Unrecognized pool ID: %s", poolID)
		return
	}
	var subnet, iprange *net.IPNet
	if _, subnet, err = net.ParseCIDR(parts[1]); err != nil {
		return
	}
	if address != nil { // try to claim specific address requested
		ip = &net.IPNet{IP: address, Mask: subnet.Mask}
		if err = i.weave.ClaimIP("_", ip); err != nil {
			return
		}
	} else {
		if _, iprange, err = net.ParseCIDR(parts[2]); err != nil {
			return
		}
		// We are lying slightly to IPAM here: the range is not a subnet
		if ip, err = i.weave.AllocateIPInSubnet("_", iprange); err != nil {
			return
		}
		ip.Mask = subnet.Mask // fix up the subnet we lied about
	}
	return
}

func (i *Ipam) ReleaseAddress(poolID string, address net.IP) error {
	i.logReq("ReleaseAddress", poolID, address)
	return i.weave.ReleaseIPsFor(address.String())
}

// Functions required by ipamapi "contract" but not actually used.

func (i *Ipam) DiscoverNew(discoverapi.DiscoveryType, interface{}) error {
	return nil
}

func (i *Ipam) DiscoverDelete(discoverapi.DiscoveryType, interface{}) error {
	return nil
}

// logging

func (i *Ipam) logReq(fun string, args ...interface{}) {
	common.Log.Debugln(append([]interface{}{fmt.Sprintf("[ipam] %s", fun)}, args...)...)
}

func (i *Ipam) logRes(fun string, args ...interface{}) {
	common.Log.Debugln(append([]interface{}{fmt.Sprintf("[ipam] %s result", fun)}, args...)...)
}
<commit_msg>tweak ipam plugin logging<commit_after>package ipamplugin

import (
	"fmt"
	"net"
	"strings"

	"github.com/docker/libnetwork/discoverapi"
	"github.com/docker/libnetwork/netlabel"
	"github.com/weaveworks/weave/api"
	"github.com/weaveworks/weave/common"
)

type Ipam struct {
	weave *api.Client
}

func NewIpam(weave *api.Client) *Ipam {
	return &Ipam{weave: weave}
}

func (i *Ipam) GetDefaultAddressSpaces() (string, string, error) {
	i.logReq("GetDefaultAddressSpaces")
	return "weavelocal", "weaveglobal", nil
}

func (i *Ipam) RequestPool(addressSpace, pool, subPool string, options map[string]string, v6 bool) (poolname string, subnet *net.IPNet, data map[string]string, err error) {
	i.logReq("RequestPool", addressSpace, pool, subPool, options)
	defer func() { i.logRes("RequestPool", err, poolname, subnet, data) }()
	if pool == "" {
		subnet, err = i.weave.DefaultSubnet()
	} else {
		_, subnet, err = net.ParseCIDR(pool)
	}
	if err != nil {
		return
	}
	iprange := subnet
	if subPool != "" {
		if _, iprange, err = net.ParseCIDR(subPool); err != nil {
			return
		}
	}
	// Cunningly-constructed pool "name" which gives us what we need later
	poolname = strings.Join([]string{"weave", subnet.String(), iprange.String()}, "-")
	// Pass back a fake "gateway address"; we don't actually use it,
	// so just give the network address.
	data = map[string]string{netlabel.Gateway: subnet.String()}
	return
}

func (i *Ipam) ReleasePool(poolID string) error {
	i.logReq("ReleasePool", poolID)
	return nil
}

func (i *Ipam) RequestAddress(poolID string, address net.IP, options map[string]string) (ip *net.IPNet, _ map[string]string, err error) {
	i.logReq("RequestAddress", poolID, address, options)
	defer func() { i.logRes("RequestAddress", err, ip) }()
	// If we pass magic string "_" to weave IPAM it stores the address under its own string
	if poolID == "weavepool" { // old-style
		ip, err = i.weave.AllocateIP("_")
		return
	}
	parts := strings.Split(poolID, "-")
	if len(parts) != 3 || parts[0] != "weave" {
		err = fmt.Errorf("Unrecognized pool ID: %s", poolID)
		return
	}
	var subnet, iprange *net.IPNet
	if _, subnet, err = net.ParseCIDR(parts[1]); err != nil {
		return
	}
	if address != nil { // try to claim specific address requested
		ip = &net.IPNet{IP: address, Mask: subnet.Mask}
		if err = i.weave.ClaimIP("_", ip); err != nil {
			return
		}
	} else {
		if _, iprange, err = net.ParseCIDR(parts[2]); err != nil {
			return
		}
		// We are lying slightly to IPAM here: the range is not a subnet
		if ip, err = i.weave.AllocateIPInSubnet("_", iprange); err != nil {
			return
		}
		ip.Mask = subnet.Mask // fix up the subnet we lied about
	}
	return
}

func (i *Ipam) ReleaseAddress(poolID string, address net.IP) error {
	i.logReq("ReleaseAddress", poolID, address)
	return i.weave.ReleaseIPsFor(address.String())
}

// Functions required by ipamapi "contract" but not actually used.

func (i *Ipam) DiscoverNew(discoverapi.DiscoveryType, interface{}) error {
	return nil
}

func (i *Ipam) DiscoverDelete(discoverapi.DiscoveryType, interface{}) error {
	return nil
}

// logging

func (i *Ipam) logReq(fun string, args ...interface{}) {
	common.Log.Infoln(append([]interface{}{fmt.Sprintf("[ipam] %s", fun)}, args...)...)
}

func (i *Ipam) logRes(fun string, err error, args ...interface{}) {
	if err == nil {
		common.Log.Debugln(append([]interface{}{fmt.Sprintf("[ipam] %s result", fun)}, args...)...)
		return
	}
	common.Log.Errorf("[ipam] %s: %s", fun, err)
}
<|endoftext|>


<commit_before>package iterator

import (
	"math"

	"github.com/cayleygraph/cayley/graph"
	"github.com/cayleygraph/cayley/quad"
)

// Recursive iterator takes a base iterator and a morphism to be applied recursively, for each result.
type Recursive struct {
	uid      uint64
	tags     graph.Tagger
	subIt    graph.Iterator
	result   seenAt
	runstats graph.IteratorStats
	err      error

	qs            graph.QuadStore
	morphism      graph.ApplyMorphism
	seen          map[interface{}]seenAt
	nextIt        graph.Iterator
	depth         int
	pathMap       map[interface{}][]map[string]graph.Value
	pathIndex     int
	containsValue graph.Value
	depthTags     graph.Tagger
	depthCache    []graph.Value
	baseIt        graph.FixedIterator
}

type seenAt struct {
	depth int
	val   graph.Value
}

var _ graph.Iterator = &Recursive{}

var MaxRecursiveSteps = 50

func NewRecursive(qs graph.QuadStore, it graph.Iterator, morphism graph.ApplyMorphism) *Recursive {
	return &Recursive{
		uid:   NextUID(),
		subIt: it,

		qs:            qs,
		morphism:      morphism,
		seen:          make(map[interface{}]seenAt),
		nextIt:        &Null{},
		baseIt:        qs.FixedIterator(),
		pathMap:       make(map[interface{}][]map[string]graph.Value),
		containsValue: nil,
	}
}

func (it *Recursive) UID() uint64 {
	return it.uid
}

func (it *Recursive) Reset() {
	it.result.val = nil
	it.result.depth = 0
	it.err = nil
	it.subIt.Reset()
	it.seen = make(map[interface{}]seenAt)
	it.pathMap = make(map[interface{}][]map[string]graph.Value)
	it.containsValue = nil
	it.pathIndex = 0
	it.nextIt = &Null{}
	it.baseIt = it.qs.FixedIterator()
	it.depth = 0
}

func (it *Recursive) Tagger() *graph.Tagger {
	return &it.tags
}

func (it *Recursive) AddDepthTag(s string) {
	it.depthTags.Add(s)
}

func (it *Recursive) TagResults(dst map[string]graph.Value) {
	for _, tag := range it.tags.Tags() {
		dst[tag] = it.Result()
	}

	for _, tag := range it.depthTags.Tags() {
		dst[tag] = it.qs.ValueOf(quad.Int(it.result.depth))
	}

	for tag, value := range it.tags.Fixed() {
		dst[tag] = value
	}

	for tag, value := range it.depthTags.Fixed() {
		dst[tag] = value
	}
	if it.containsValue != nil {
		paths := it.pathMap[graph.ToKey(it.containsValue)]
		if len(paths) != 0 {
			for k, v := range paths[it.pathIndex] {
				dst[k] = v
			}
		}
	}

}

func (it *Recursive) Clone() graph.Iterator {
	n := NewRecursive(it.qs, it.subIt.Clone(), it.morphism)
	n.tags.CopyFrom(it)
	n.depthTags.CopyFromTagger(&it.depthTags)
	return n
}

func (it *Recursive) SubIterators() []graph.Iterator {
	return []graph.Iterator{it.subIt}
}

func (it *Recursive) Next() bool {
	it.pathIndex = 0
	if it.depth == 0 {
		for it.subIt.Next() {
			res := it.subIt.Result()
			it.depthCache = append(it.depthCache, it.subIt.Result())
			tags := make(map[string]graph.Value)
			it.subIt.TagResults(tags)
			key := graph.ToKey(res)
			it.pathMap[key] = append(it.pathMap[key], tags)
			for it.subIt.NextPath() {
				tags := make(map[string]graph.Value)
				it.subIt.TagResults(tags)
				it.pathMap[key] = append(it.pathMap[key], tags)
			}
		}
	}
	for {
		ok := it.nextIt.Next()
		if !ok {
			if len(it.depthCache) == 0 {
				return graph.NextLogOut(it, false)
			}
			it.depth++
			it.baseIt = it.qs.FixedIterator()
			for _, x := range it.depthCache {
				it.baseIt.Add(x)
			}
			it.baseIt.Tagger().Add("__base_recursive")
			it.depthCache = nil
			it.nextIt = it.morphism(it.qs, it.baseIt)
			continue
		}
		val := it.nextIt.Result()
		results := make(map[string]graph.Value)
		it.nextIt.TagResults(results)
		key := graph.ToKey(val)
		if _, ok := it.seen[key]; ok {
			continue
		}
		it.seen[key] = seenAt{
			val:   results["__base_recursive"],
			depth: it.depth,
		}
		it.result.depth = it.depth
		it.result.val = val
		it.containsValue = it.getBaseValue(val)
		it.depthCache = append(it.depthCache, val)
		break
	}
	return graph.NextLogOut(it, true)
}

func (it *Recursive) Err() error {
	return it.err
}

func (it *Recursive) Result() graph.Value {
	return it.result.val
}

func (it *Recursive) getBaseValue(val graph.Value) graph.Value {
	var at seenAt
	var ok bool
	if at, ok = it.seen[graph.ToKey(val)]; !ok {
		panic("trying to getBaseValue of something unseen")
	}
	for at.depth != 1 {
		if at.depth == 0 {
			panic("seen chain is broken")
		}
		at = it.seen[graph.ToKey(at.val)]
	}
	return at.val
}

func (it *Recursive) Contains(val graph.Value) bool {
	graph.ContainsLogIn(it, val)
	it.pathIndex = 0
	key := graph.ToKey(val)
	if at, ok := it.seen[key]; ok {
		it.containsValue = it.getBaseValue(val)
		it.result.depth = at.depth
		it.result.val = val
		return graph.ContainsLogOut(it, val, true)
	}
	for it.Next() {
		if graph.ToKey(it.Result()) == key {
			return graph.ContainsLogOut(it, val, true)
		}
	}
	return graph.ContainsLogOut(it, val, false)
}

func (it *Recursive) NextPath() bool {
	if it.pathIndex+1 >= len(it.pathMap[graph.ToKey(it.containsValue)]) {
		return false
	}
	it.pathIndex++
	return true
}

func (it *Recursive) Close() error {
	err := it.subIt.Close()
	if err != nil {
		return err
	}
	err = it.nextIt.Close()
	if err != nil {
		return err
	}
	it.seen = nil
	return it.err
}

func (it *Recursive) Type() graph.Type { return graph.Recursive }

func (it *Recursive) Optimize() (graph.Iterator, bool) {
	newIt, optimized := it.subIt.Optimize()
	if optimized {
		it.subIt = newIt
	}
	return it, false
}

func (it *Recursive) Size() (int64, bool) {
	return it.Stats().Size, false
}

func (it *Recursive) Stats() graph.IteratorStats {
	base := it.qs.FixedIterator()
	base.Add(Int64Node(20))
	fanoutit := it.morphism(it.qs, base)
	fanoutStats := fanoutit.Stats()
	subitStats := it.subIt.Stats()

	size := int64(math.Pow(float64(subitStats.Size*fanoutStats.Size), 5))
	return graph.IteratorStats{
		NextCost:     subitStats.NextCost + fanoutStats.NextCost,
		ContainsCost: (subitStats.NextCost+fanoutStats.NextCost)*(size/10) + subitStats.ContainsCost,
		Size:         size,
		Next:         it.runstats.Next,
		Contains:     it.runstats.Contains,
		ContainsNext: it.runstats.ContainsNext,
	}
}

func (it *Recursive) Describe() graph.Description {
	base := it.qs.FixedIterator()
	base.Add(Int64Node(20))
	fanoutdesc := it.morphism(it.qs, base).Describe()
	subIts := []graph.Description{
		it.subIt.Describe(),
		fanoutdesc,
	}

	return graph.Description{
		UID:       it.UID(),
		Type:      it.Type(),
		Tags:      it.tags.Tags(),
		Iterators: subIts,
	}
}
<commit_msg>tag values in recursive iterator correctly<commit_after>package iterator

import (
	"math"

	"github.com/cayleygraph/cayley/graph"
	"github.com/cayleygraph/cayley/quad"
)

// Recursive iterator takes a base iterator and a morphism to be applied recursively, for each result.
type Recursive struct {
	uid      uint64
	tags     graph.Tagger
	subIt    graph.Iterator
	result   seenAt
	runstats graph.IteratorStats
	err      error

	qs            graph.QuadStore
	morphism      graph.ApplyMorphism
	seen          map[interface{}]seenAt
	nextIt        graph.Iterator
	depth         int
	pathMap       map[interface{}][]map[string]graph.Value
	pathIndex     int
	containsValue graph.Value
	depthTags     graph.Tagger
	depthCache    []graph.Value
	baseIt        graph.FixedIterator
}

type seenAt struct {
	depth int
	val   graph.Value
}

var _ graph.Iterator = &Recursive{}

var MaxRecursiveSteps = 50

func NewRecursive(qs graph.QuadStore, it graph.Iterator, morphism graph.ApplyMorphism) *Recursive {
	return &Recursive{
		uid:   NextUID(),
		subIt: it,

		qs:            qs,
		morphism:      morphism,
		seen:          make(map[interface{}]seenAt),
		nextIt:        &Null{},
		baseIt:        qs.FixedIterator(),
		pathMap:       make(map[interface{}][]map[string]graph.Value),
		containsValue: nil,
	}
}

func (it *Recursive) UID() uint64 {
	return it.uid
}

func (it *Recursive) Reset() {
	it.result.val = nil
	it.result.depth = 0
	it.err = nil
	it.subIt.Reset()
	it.seen = make(map[interface{}]seenAt)
	it.pathMap = make(map[interface{}][]map[string]graph.Value)
	it.containsValue = nil
	it.pathIndex = 0
	it.nextIt = &Null{}
	it.baseIt = it.qs.FixedIterator()
	it.depth = 0
}

func (it *Recursive) Tagger() *graph.Tagger {
	return &it.tags
}

func (it *Recursive) AddDepthTag(s string) {
	it.depthTags.Add(s)
}

func (it *Recursive) TagResults(dst map[string]graph.Value) {
	for _, tag := range it.tags.Tags() {
		dst[tag] = it.Result()
	}

	for _, tag := range it.depthTags.Tags() {
		dst[tag] = it.qs.ValueOf(quad.Int(it.result.depth))
	}

	for tag, value := range it.tags.Fixed() {
		dst[tag] = value
	}

	for tag, value := range it.depthTags.Fixed() {
		dst[tag] = value
	}
	if it.containsValue != nil {
		paths := it.pathMap[graph.ToKey(it.containsValue)]
		if len(paths) != 0 {
			for k, v := range paths[it.pathIndex] {
				dst[k] = v
			}
		}
	}
	if it.nextIt != nil {
		it.nextIt.TagResults(dst)
		delete(dst, "__base_recursive")
	}
}

func (it *Recursive) Clone() graph.Iterator {
	n := NewRecursive(it.qs, it.subIt.Clone(), it.morphism)
	n.tags.CopyFrom(it)
	n.depthTags.CopyFromTagger(&it.depthTags)
	return n
}

func (it *Recursive) SubIterators() []graph.Iterator {
	return []graph.Iterator{it.subIt}
}

func (it *Recursive) Next() bool {
	it.pathIndex = 0
	if it.depth == 0 {
		for it.subIt.Next() {
			res := it.subIt.Result()
			it.depthCache = append(it.depthCache, it.subIt.Result())
			tags := make(map[string]graph.Value)
			it.subIt.TagResults(tags)
			key := graph.ToKey(res)
			it.pathMap[key] = append(it.pathMap[key], tags)
			for it.subIt.NextPath() {
				tags := make(map[string]graph.Value)
				it.subIt.TagResults(tags)
				it.pathMap[key] = append(it.pathMap[key], tags)
			}
		}
	}
	for {
		ok := it.nextIt.Next()
		if !ok {
			if len(it.depthCache) == 0 {
				return graph.NextLogOut(it, false)
			}
			it.depth++
			it.baseIt = it.qs.FixedIterator()
			for _, x := range it.depthCache {
				it.baseIt.Add(x)
			}
			it.baseIt.Tagger().Add("__base_recursive")
			it.depthCache = nil
			it.nextIt = it.morphism(it.qs, it.baseIt)
			continue
		}
		val := it.nextIt.Result()
		results := make(map[string]graph.Value)
		it.nextIt.TagResults(results)
		key := graph.ToKey(val)
		if _, ok := it.seen[key]; ok {
			continue
		}
		it.seen[key] = seenAt{
			val:   results["__base_recursive"],
			depth: it.depth,
		}
		it.result.depth = it.depth
		it.result.val = val
		it.containsValue = it.getBaseValue(val)
		it.depthCache = append(it.depthCache, val)
		break
	}
	return graph.NextLogOut(it, true)
}

func (it *Recursive) Err() error {
	return it.err
}

func (it *Recursive) Result() graph.Value {
	return it.result.val
}

func (it *Recursive) getBaseValue(val graph.Value) graph.Value {
	var at seenAt
	var ok bool
	if at, ok = it.seen[graph.ToKey(val)]; !ok {
		panic("trying to getBaseValue of something unseen")
	}
	for at.depth != 1 {
		if at.depth == 0 {
			panic("seen chain is broken")
		}
		at = it.seen[graph.ToKey(at.val)]
	}
	return at.val
}

func (it *Recursive) Contains(val graph.Value) bool {
	graph.ContainsLogIn(it, val)
	it.pathIndex = 0
	key := graph.ToKey(val)
	if at, ok := it.seen[key]; ok {
		it.containsValue = it.getBaseValue(val)
		it.result.depth = at.depth
		it.result.val = val
		return graph.ContainsLogOut(it, val, true)
	}
	for it.Next() {
		if graph.ToKey(it.Result()) == key {
			return graph.ContainsLogOut(it, val, true)
		}
	}
	return graph.ContainsLogOut(it, val, false)
}

func (it *Recursive) NextPath() bool {
	if it.pathIndex+1 >= len(it.pathMap[graph.ToKey(it.containsValue)]) {
		return false
	}
	it.pathIndex++
	return true
}

func (it *Recursive) Close() error {
	err := it.subIt.Close()
	if err != nil {
		return err
	}
	err = it.nextIt.Close()
	if err != nil {
		return err
	}
	it.seen = nil
	return it.err
}

func (it *Recursive) Type() graph.Type { return graph.Recursive }

func (it *Recursive) Optimize() (graph.Iterator, bool) {
	newIt, optimized := it.subIt.Optimize()
	if optimized {
		it.subIt = newIt
	}
	return it, false
}

func (it *Recursive) Size() (int64, bool) {
	return it.Stats().Size, false
}

func (it *Recursive) Stats() graph.IteratorStats {
	base := it.qs.FixedIterator()
	base.Add(Int64Node(20))
	fanoutit := it.morphism(it.qs, base)
	fanoutStats := fanoutit.Stats()
	subitStats := it.subIt.Stats()

	size := int64(math.Pow(float64(subitStats.Size*fanoutStats.Size), 5))
	return graph.IteratorStats{
		NextCost:     subitStats.NextCost + fanoutStats.NextCost,
		ContainsCost: (subitStats.NextCost+fanoutStats.NextCost)*(size/10) + subitStats.ContainsCost,
		Size:         size,
		Next:         it.runstats.Next,
		Contains:     it.runstats.Contains,
		ContainsNext: it.runstats.ContainsNext,
	}
}

func (it *Recursive) Describe() graph.Description {
	base := it.qs.FixedIterator()
	base.Add(Int64Node(20))
	fanoutdesc := it.morphism(it.qs, base).Describe()
	subIts := []graph.Description{
		it.subIt.Describe(),
		fanoutdesc,
	}

	return graph.Description{
		UID:       it.UID(),
		Type:      it.Type(),
		Tags:      it.tags.Tags(),
		Iterators: subIts,
	}
}
<|endoftext|>


<commit_before>package storage

import (
	"errors"
	"log"
	"os"
	"path"
	"sync"
)

const (
	SuperBlockSize = 8
)

type Volume struct {
	Id       VolumeId
	dir      string
	dataFile *os.File
	nm       *NeedleMap

	replicaType ReplicationType

	accessLock sync.Mutex
}

func NewVolume(dirname string, id VolumeId, replicationType ReplicationType) (v *Volume) {
	v = &Volume{dir: dirname, Id: id, replicaType: replicationType}
	v.load()
	return
}
func (v *Volume) load() {
	var e error
	fileName := path.Join(v.dir, v.Id.String())
	v.dataFile, e = os.OpenFile(fileName+".dat", os.O_RDWR|os.O_CREATE, 0644)
	if e != nil {
		log.Fatalf("New Volume [ERROR] %s\n", e)
	}
	if v.replicaType == CopyNil {
		v.readSuperBlock()
	} else {
		v.maybeWriteSuperBlock()
	}
	indexFile, ie := os.OpenFile(fileName+".idx", os.O_RDWR|os.O_CREATE, 0644)
	if ie != nil {
		log.Fatalf("Write Volume Index [ERROR] %s\n", ie)
	}
	v.nm = LoadNeedleMap(indexFile)
}
func (v *Volume) Size() int64 {
	stat, e := v.dataFile.Stat()
	if e == nil {
		return stat.Size()
	}
	return -1
}
func (v *Volume) Close() {
	v.nm.Close()
	v.dataFile.Close()
}
func (v *Volume) maybeWriteSuperBlock() {
	stat, _ := v.dataFile.Stat()
	if stat.Size() == 0 {
		header := make([]byte, SuperBlockSize)
		header[0] = 1
		header[1] = v.replicaType.Byte()
		v.dataFile.Write(header)
	}
}
func (v *Volume) readSuperBlock() {
	v.dataFile.Seek(0, 0)
	header := make([]byte, SuperBlockSize)
	if _, error := v.dataFile.Read(header); error == nil {
		v.replicaType, _ = NewReplicationTypeFromByte(header[1])
	}
}
func (v *Volume) NeedToReplicate() bool {
	return v.replicaType.GetCopyCount() > 1
}

func (v *Volume) write(n *Needle) uint32 {
	v.accessLock.Lock()
	defer v.accessLock.Unlock()
	offset, _ := v.dataFile.Seek(0, 2)
	ret := n.Append(v.dataFile)
	nv, ok := v.nm.Get(n.Id)
	if !ok || int64(nv.Offset)*8 < offset {
		v.nm.Put(n.Id, uint32(offset/8), n.Size)
	}
	return ret
}
func (v *Volume) delete(n *Needle) uint32 {
	v.accessLock.Lock()
	defer v.accessLock.Unlock()
	nv, ok := v.nm.Get(n.Id)
	//log.Println("key", n.Id, "volume offset", nv.Offset, "data_size", n.Size, "cached size", nv.Size)
	if ok {
		v.nm.Delete(n.Id)
		v.dataFile.Seek(int64(nv.Offset*8), 0)
		n.Append(v.dataFile)
		return nv.Size
	}
	return 0
}
func (v *Volume) read(n *Needle) (int, error) {
	v.accessLock.Lock()
	defer v.accessLock.Unlock()
	nv, ok := v.nm.Get(n.Id)
	if ok && nv.Offset > 0 {
		v.dataFile.Seek(int64(nv.Offset)*8, 0)
		return n.Read(v.dataFile, nv.Size)
	}
	return -1, errors.New("Not Found")
}

func (v *Volume) compact() error {
	v.accessLock.Lock()
	defer v.accessLock.Unlock()

	filePath := path.Join(v.dir, v.Id.String())
	return v.copyDataAndGenerateIndexFile(filePath+".dat", filePath+".cpd", filePath+".cpx")
}
func (v *Volume) commitCompact() (int, error) {
	v.accessLock.Lock()
	defer v.accessLock.Unlock()
	v.dataFile.Close()
	os.Rename(path.Join(v.dir, v.Id.String()+".cpd"), path.Join(v.dir, v.Id.String()+".dat"))
	os.Rename(path.Join(v.dir, v.Id.String()+".cpx"), path.Join(v.dir, v.Id.String()+".idx"))
	v.load()
	return 0, nil
}

func (v *Volume) copyDataAndGenerateIndexFile(srcName, dstName, idxName string) (err error) {
	src, err := os.OpenFile(srcName, os.O_RDONLY, 0644)
	if err != nil {
		return err
	}
	defer src.Close()

	dst, err := os.OpenFile(dstName, os.O_WRONLY|os.O_CREATE, 0644)
	if err != nil {
		return err
	}
	defer dst.Close()

	idx, err := os.OpenFile(idxName, os.O_WRONLY|os.O_CREATE, 0644)
	if err != nil {
		return err
	}
	defer idx.Close()

	src.Seek(0, 0)
	header := make([]byte, SuperBlockSize)
	if _, error := src.Read(header); error == nil {
		dst.Write(header)
	}

	n, rest := ReadNeedle(src)
	nm := NewNeedleMap(idx)
	old_offset := uint32(SuperBlockSize)
	new_offset := uint32(SuperBlockSize)
	for n != nil {
		nv, ok := v.nm.Get(n.Id)
		//log.Println("file size is", n.Size, "rest", rest)
		if !ok || nv.Offset*8 != old_offset {
			log.Println("expected offset should be", nv.Offset*8, "skipping", (rest - 16), "key", n.Id, "volume offset", old_offset, "data_size", n.Size, "rest", rest)
			src.Seek(int64(rest), 1)
		} else {
			if nv.Size > 0 {
				nm.Put(n.Id, new_offset/8, n.Size)
				bytes := make([]byte, n.Size+4)
				src.Read(bytes)
				n.Data = bytes[:n.Size]
				n.Checksum = NewCRC(n.Data)
				n.Append(dst)
				new_offset += rest + 16
				log.Println("saving key", n.Id, "volume offset", old_offset, "=>", new_offset, "data_size", n.Size, "rest", rest)
			}
			src.Seek(int64(rest-n.Size-4), 1)
		}
		old_offset += rest + 16
		n, rest = ReadNeedle(src)
	}

	return nil
}
<commit_msg>cleaner code, better error handling<commit_after>package storage

import (
	"errors"
	"fmt"
	"log"
	"os"
	"path"
	"sync"
)

const (
	SuperBlockSize = 8
	Version        = 1
)

type Volume struct {
	Id       VolumeId
	dir      string
	dataFile *os.File
	nm       *NeedleMap

	replicaType ReplicationType

	version uint8

	accessLock sync.Mutex
}

func NewVolume(dirname string, id VolumeId, replicationType ReplicationType) (v *Volume) {
	v = &Volume{dir: dirname, Id: id, replicaType: replicationType}
	v.load()
	return
}
func (v *Volume) load() error {
	var e error
	fileName := path.Join(v.dir, v.Id.String())
	v.dataFile, e = os.OpenFile(fileName+".dat", os.O_RDWR|os.O_CREATE, 0644)
	if e != nil {
		return fmt.Errorf("cannot create Volume Data %s.dat: %s", fileName, e)
	}
	if v.replicaType == CopyNil {
		if e = v.readSuperBlock(); e != nil {
			return e
		}
	} else {
		v.maybeWriteSuperBlock()
	}
	indexFile, ie := os.OpenFile(fileName+".idx", os.O_RDWR|os.O_CREATE, 0644)
	if ie != nil {
		return fmt.Errorf("cannot create Volume Data %s.dat: %s", fileName, e)
	}
	v.nm = LoadNeedleMap(indexFile)
	return nil
}
func (v *Volume) Size() int64 {
	stat, e := v.dataFile.Stat()
	if e == nil {
		return stat.Size()
	}
	return -1
}
func (v *Volume) Close() {
	v.nm.Close()
	v.dataFile.Close()
}
func (v *Volume) maybeWriteSuperBlock() {
	stat, _ := v.dataFile.Stat()
	if stat.Size() == 0 {
		header := make([]byte, SuperBlockSize)
		header[0] = Version
		header[1] = v.replicaType.Byte()
		v.dataFile.Write(header)
	}
}
func (v *Volume) readSuperBlock() error {
	v.dataFile.Seek(0, 0)
	header := make([]byte, SuperBlockSize)
	if _, e := v.dataFile.Read(header); e != nil {
		return fmt.Errorf("cannot read superblock: %s", e)
	}
	v.version = header[0]
	var err error
	if v.replicaType, err = NewReplicationTypeFromByte(header[1]); err != nil {
		return fmt.Errorf("cannot read replica type: %s", err)
	}
	return nil
}
func (v *Volume) NeedToReplicate() bool {
	return v.replicaType.GetCopyCount() > 1
}

func (v *Volume) write(n *Needle) uint32 {
	v.accessLock.Lock()
	defer v.accessLock.Unlock()
	offset, _ := v.dataFile.Seek(0, 2)
	ret := n.Append(v.dataFile)
	nv, ok := v.nm.Get(n.Id)
	if !ok || int64(nv.Offset)*8 < offset {
		v.nm.Put(n.Id, uint32(offset/8), n.Size)
	}
	return ret
}
func (v *Volume) delete(n *Needle) uint32 {
	v.accessLock.Lock()
	defer v.accessLock.Unlock()
	nv, ok := v.nm.Get(n.Id)
	//log.Println("key", n.Id, "volume offset", nv.Offset, "data_size", n.Size, "cached size", nv.Size)
	if ok {
		v.nm.Delete(n.Id)
		v.dataFile.Seek(int64(nv.Offset*8), 0)
		n.Append(v.dataFile)
		return nv.Size
	}
	return 0
}
func (v *Volume) read(n *Needle) (int, error) {
	v.accessLock.Lock()
	defer v.accessLock.Unlock()
	nv, ok := v.nm.Get(n.Id)
	if ok && nv.Offset > 0 {
		v.dataFile.Seek(int64(nv.Offset)*8, 0)
		return n.Read(v.dataFile, nv.Size)
	}
	return -1, errors.New("Not Found")
}

func (v *Volume) compact() error {
	v.accessLock.Lock()
	defer v.accessLock.Unlock()

	filePath := path.Join(v.dir, v.Id.String())
	return v.copyDataAndGenerateIndexFile(filePath+".dat", filePath+".cpd", filePath+".cpx")
}
func (v *Volume) commitCompact() (int, error) {
	v.accessLock.Lock()
	defer v.accessLock.Unlock()
	v.dataFile.Close()
	var e error
	if e = os.Rename(path.Join(v.dir, v.Id.String()+".cpd"), path.Join(v.dir, v.Id.String()+".dat")); e != nil {
		return 0, e
	}
	if e = os.Rename(path.Join(v.dir, v.Id.String()+".cpx"), path.Join(v.dir, v.Id.String()+".idx")); e != nil {
		return 0, e
	}
	if e = v.load(); e != nil {
		return 0, e
	}
	return 0, nil
}

func (v *Volume) copyDataAndGenerateIndexFile(srcName, dstName, idxName string) (err error) {
	src, err := os.OpenFile(srcName, os.O_RDONLY, 0644)
	if err != nil {
		return err
	}
	defer src.Close()

	dst, err := os.OpenFile(dstName, os.O_WRONLY|os.O_CREATE, 0644)
	if err != nil {
		return err
	}
	defer dst.Close()

	idx, err := os.OpenFile(idxName, os.O_WRONLY|os.O_CREATE, 0644)
	if err != nil {
		return err
	}
	defer idx.Close()

	src.Seek(0, 0)
	header := make([]byte, SuperBlockSize)
	if _, error := src.Read(header); error == nil {
		dst.Write(header)
	}

	n, rest := ReadNeedle(src)
	nm := NewNeedleMap(idx)
	old_offset := uint32(SuperBlockSize)
	new_offset := uint32(SuperBlockSize)
	for n != nil {
		nv, ok := v.nm.Get(n.Id)
		//log.Println("file size is", n.Size, "rest", rest)
		if !ok || nv.Offset*8 != old_offset {
			log.Println("expected offset should be", nv.Offset*8, "skipping", (rest - 16), "key", n.Id, "volume offset", old_offset, "data_size", n.Size, "rest", rest)
			src.Seek(int64(rest), 1)
		} else {
			if nv.Size > 0 {
				nm.Put(n.Id, new_offset/8, n.Size)
				bytes := make([]byte, n.Size+4)
				src.Read(bytes)
				n.Data = bytes[:n.Size]
				n.Checksum = NewCRC(n.Data)
				n.Append(dst)
				new_offset += rest + 16
				log.Println("saving key", n.Id, "volume offset", old_offset, "=>", new_offset, "data_size", n.Size, "rest", rest)
			}
			src.Seek(int64(rest-n.Size-4), 1)
		}
		old_offset += rest + 16
		n, rest = ReadNeedle(src)
	}

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"koding/kite-handler/command"
	"koding/kite-handler/fs"
	"koding/kite-handler/terminal"
	"log"
	"net"
	"net/url"
	"os"
	"strconv"

	"github.com/koding/kite"
	"github.com/koding/kite/config"
)

const (
	VERSION = "0.0.1"
	NAME    = "klient"
)

var (
	flagIP          = flag.String("ip", "", "Change public ip")
	flagPort        = flag.Int("port", 3000, "Change running port")
	flagVersion     = flag.Bool("version", false, "Show version and exit")
	flagEnvironment = flag.String("environment", "public-host", "Change environment")
	flagLocal       = flag.Bool("local", false, "Start klient in local environment.")
)

func main() {
	flag.Parse()
	if *flagVersion {
		fmt.Println(VERSION)
		os.Exit(0)
	}

	k := kite.New(NAME, VERSION)
	conf := config.MustGet()
	k.Config = conf
	k.Config.Port = *flagPort
	k.Config.Environment = *flagEnvironment
	k.Config.Id = conf.Id

	k.HandleFunc("fs.readDirectory", fs.ReadDirectory)
	k.HandleFunc("fs.glob", fs.Glob)
	k.HandleFunc("fs.readFile", fs.ReadFile)
	k.HandleFunc("fs.writeFile", fs.WriteFile)
	k.HandleFunc("fs.uniquePath", fs.UniquePath)
	k.HandleFunc("fs.getInfo", fs.GetInfo)
	k.HandleFunc("fs.setPermissions", fs.SetPermissions)
	k.HandleFunc("fs.remove", fs.Remove)
	k.HandleFunc("fs.rename", fs.Rename)
	k.HandleFunc("fs.createDirectory", fs.CreateDirectory)
	k.HandleFunc("fs.move", fs.Move)
	k.HandleFunc("fs.copy", fs.Copy)

	k.HandleFunc("webterm.getSessions", terminal.GetSessions)
	k.HandleFunc("webterm.connect", terminal.Connect)
	k.HandleFunc("webterm.killSession", terminal.KillSession)

	k.HandleFunc("exec", command.Exec)

	if err := k.RegisterForever(registerURL()); err != nil {
		log.Fatal(err)
	}

	k.Run()
}

func registerURL() *url.URL {
	l := &localhost{}

	var ip net.IP
	var err error

	if *flagLocal {
		ip, err = l.LocalIP()
		if err != nil {
			return nil
		}
	} else {
		ip, err = l.PublicIp()
		if err != nil {
			return nil
		}
	}

	return &url.URL{
		Scheme: "ws",
		Host:   ip.String() + ":" + strconv.Itoa(*flagPort),
		Path:   "/" + NAME + "-" + VERSION,
	}
}
<commit_msg>klient: always boot up with the same id<commit_after>package main

import (
	"flag"
	"fmt"
	"koding/kite-handler/command"
	"koding/kite-handler/fs"
	"koding/kite-handler/terminal"
	"log"
	"net"
	"net/url"
	"os"
	"strconv"

	"github.com/koding/kite"
	"github.com/koding/kite/config"
)

const (
	VERSION = "0.0.1"
	NAME    = "klient"
)

var (
	flagIP          = flag.String("ip", "", "Change public ip")
	flagPort        = flag.Int("port", 3000, "Change running port")
	flagVersion     = flag.Bool("version", false, "Show version and exit")
	flagEnvironment = flag.String("environment", "public-host", "Change environment")
	flagLocal       = flag.Bool("local", false, "Start klient in local environment.")
)

func main() {
	flag.Parse()
	if *flagVersion {
		fmt.Println(VERSION)
		os.Exit(0)
	}

	k := kite.New(NAME, VERSION)
	conf := config.MustGet()
	k.Config = conf
	k.Config.Port = *flagPort
	k.Config.Environment = *flagEnvironment

	// always boot up with the same id in the kite.key
	k.Id = conf.Id

	k.HandleFunc("fs.readDirectory", fs.ReadDirectory)
	k.HandleFunc("fs.glob", fs.Glob)
	k.HandleFunc("fs.readFile", fs.ReadFile)
	k.HandleFunc("fs.writeFile", fs.WriteFile)
	k.HandleFunc("fs.uniquePath", fs.UniquePath)
	k.HandleFunc("fs.getInfo", fs.GetInfo)
	k.HandleFunc("fs.setPermissions", fs.SetPermissions)
	k.HandleFunc("fs.remove", fs.Remove)
	k.HandleFunc("fs.rename", fs.Rename)
	k.HandleFunc("fs.createDirectory", fs.CreateDirectory)
	k.HandleFunc("fs.move", fs.Move)
	k.HandleFunc("fs.copy", fs.Copy)

	k.HandleFunc("webterm.getSessions", terminal.GetSessions)
	k.HandleFunc("webterm.connect", terminal.Connect)
	k.HandleFunc("webterm.killSession", terminal.KillSession)

	k.HandleFunc("exec", command.Exec)

	if err := k.RegisterForever(registerURL()); err != nil {
		log.Fatal(err)
	}

	k.Run()
}

func registerURL() *url.URL {
	l := &localhost{}

	var ip net.IP
	var err error

	if *flagLocal {
		ip, err = l.LocalIP()
		if err != nil {
			return nil
		}
	} else {
		ip, err = l.PublicIp()
		if err != nil {
			return nil
		}
	}

	return &url.URL{
		Scheme: "ws",
		Host:   ip.String() + ":" + strconv.Itoa(*flagPort),
		Path:   "/" + NAME + "-" + VERSION,
	}
}
<|endoftext|>


<commit_before>// An example FTP server build on top of graval. graval handles the details
// of the FTP protocol, we just provide a persistence driver for rackspace
// cloud files.
//
// If you're looking to create a custom graval driver, this example is a
// reasonable starting point. I suggest copying this file and changing the
// function bodies as required.
package main

import (
	"errors"
	"github.com/ncw/swift"
	"github.com/yob/graval"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strings"
	"time"
)

// A minimal driver for graval that stores all data on rackspace cloudfiles. The
// authentication are ignored, any username and password will work.
//
// This really just exists as a minimal demonstration of the interface graval
// drivers are required to implement.
type SwiftDriver struct{
	connection *swift.Connection
	container  string
	user       string
}

func (driver *SwiftDriver) Authenticate(user string, pass string) bool {
	log.Printf("Authenticate: %s %s", user, pass)
	driver.user = user
	return true
}
func (driver *SwiftDriver) Bytes(path string) (bytes int) {
	path = scoped_path(driver.user, path)
	log.Printf("Bytes: %s", path)
	object, _, err := driver.connection.Object(driver.container, path)
	if err != nil {
		return -1
	}
	return int(object.Bytes)
}
func (driver *SwiftDriver) ModifiedTime(path string) (time.Time, error) {
	path = scoped_path(driver.user, path)
	log.Printf("ModifiedTime: %s", path)
	object, _, err := driver.connection.Object(driver.container, path)
	if err != nil {
		return time.Now(), err
	}
	return object.LastModified, nil
}
func (driver *SwiftDriver) ChangeDir(path string) bool {
	path = scoped_path(driver.user, path)
	if path == ("/"+driver.user) {
		return true
	}
	log.Printf("ChangeDir: %s", path)
	object, _, err := driver.connection.Object(driver.container, path)
	if err != nil {
		return false
	}
	return object.ContentType == "application/directory"
}
func (driver *SwiftDriver) DirContents(path string) (files []os.FileInfo) {
	path = scoped_path_with_trailing_slash(driver.user, path)
	log.Printf("DirContents: %s", path)
	opts    := &swift.ObjectsOpts{Prefix:path, Delimiter:'/'}
	objects, err := driver.connection.ObjectsAll(driver.container, opts)
	if err != nil {
		return // error connecting to cloud files
	}
	for _, object := range objects {
		tail     := strings.Replace(object.Name, path, "", 1)
        basename := strings.Split(tail, "/")[0]
		if object.ContentType == "application/directory" && object.SubDir == "" {
			files = append(files, graval.NewDirItem(basename))
		} else if object.ContentType != "application/directory"  {
			files = append(files, graval.NewFileItem(basename, int(object.Bytes)))
		}
	}
	return
}

func (driver *SwiftDriver) DeleteDir(path string) bool {
	path = scoped_path(driver.user, path)
	log.Printf("DeleteDir: %s", path)
	err := driver.connection.ObjectDelete(driver.container, path)
	if err != nil {
		return false
	}
	return true
}
func (driver *SwiftDriver) DeleteFile(path string) bool {
	path = scoped_path(driver.user, path)
	log.Printf("DeleteFile: %s", path)
	err := driver.connection.ObjectDelete(driver.container, path)
	if err != nil {
		return false
	}
	return true
}
func (driver *SwiftDriver) Rename(fromPath string, toPath string) bool {
	fromPath = scoped_path(driver.user, fromPath)
	toPath   = scoped_path(driver.user, toPath)
	log.Printf("Rename: %s %s", fromPath, toPath)
	return false
}
func (driver *SwiftDriver) MakeDir(path string) bool {
	path = scoped_path(driver.user, path)
	log.Printf("MakeDir: %s", path)
	opts    := &swift.ObjectsOpts{Prefix:path}
	objects, err := driver.connection.ObjectNames(driver.container, opts)
	if err != nil {
		return false // error connection to cloud files
	}
	if len(objects) > 0 {
		return false // the dir already exists
	}
	driver.connection.ObjectPutString(driver.container, path, "", "application/directory")
	return true
}
func (driver *SwiftDriver) GetFile(path string) (data string, err error) {
	path = scoped_path(driver.user, path)
	log.Printf("GetFile: %s", path)
	data, err = driver.connection.ObjectGetString(driver.container, path)
	if err != nil {
		return "", errors.New("foo")
	}
	return
}
func (driver *SwiftDriver) PutFile(destPath string, data io.Reader) bool {
	destPath = scoped_path(driver.user, destPath)
	log.Printf("PutFile: %s", destPath)
	contents, err := ioutil.ReadAll(data)
	if err != nil {
		return false
	}
	err = driver.connection.ObjectPutBytes(driver.container, destPath, contents, "application/octet-stream")
	if err != nil {
		return false
	}
	return true
}

func scoped_path_with_trailing_slash(user string, path string) string {
	path = scoped_path(user, path)
	if !strings.HasSuffix(path, "/") {
		path += "/"
	}
	if path == "/" {
		return ""
	}
	return path
}

func scoped_path(user string, path string) string {
	if path == "/" {
		path = ""
	}
	return filepath.Join("/", user, path)
}

// graval requires a factory that will create a new driver instance for each
// client connection. Generally the factory will be fairly minimal. This is
// a good place to read any required config for your driver.
type SwiftDriverFactory struct{}

func (factory *SwiftDriverFactory) NewDriver() (graval.FTPDriver, error) {
	driver := &SwiftDriver{}
	driver.container  = "rba-uploads"
	driver.connection = &swift.Connection{
		UserName: os.Getenv("UserName"),
		ApiKey:   os.Getenv("ApiKey"),
		AuthUrl:  os.Getenv("AuthUrl"),
	}
	err := driver.connection.Authenticate()
	if err != nil {
		return nil, err
	}
	return driver, nil
}

// it's alive!
func main() {
	factory := &SwiftDriverFactory{}
	ftpServer := graval.NewFTPServer(&graval.FTPServerOpts{ Factory: factory })
	err := ftpServer.ListenAndServe()
	if err != nil {
		log.Fatal("Error starting server!")
		log.Fatal(err)
	}
}
<commit_msg>target container in the swift example is set by env var<commit_after>// An example FTP server build on top of graval. graval handles the details
// of the FTP protocol, we just provide a persistence driver for rackspace
// cloud files.
//
// If you're looking to create a custom graval driver, this example is a
// reasonable starting point. I suggest copying this file and changing the
// function bodies as required.
package main

import (
	"errors"
	"github.com/ncw/swift"
	"github.com/yob/graval"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strings"
	"time"
)

// A minimal driver for graval that stores all data on rackspace cloudfiles. The
// authentication are ignored, any username and password will work.
//
// This really just exists as a minimal demonstration of the interface graval
// drivers are required to implement.
type SwiftDriver struct{
	connection *swift.Connection
	container  string
	user       string
}

func (driver *SwiftDriver) Authenticate(user string, pass string) bool {
	log.Printf("Authenticate: %s %s", user, pass)
	driver.user = user
	return true
}
func (driver *SwiftDriver) Bytes(path string) (bytes int) {
	path = scoped_path(driver.user, path)
	log.Printf("Bytes: %s", path)
	object, _, err := driver.connection.Object(driver.container, path)
	if err != nil {
		return -1
	}
	return int(object.Bytes)
}
func (driver *SwiftDriver) ModifiedTime(path string) (time.Time, error) {
	path = scoped_path(driver.user, path)
	log.Printf("ModifiedTime: %s", path)
	object, _, err := driver.connection.Object(driver.container, path)
	if err != nil {
		return time.Now(), err
	}
	return object.LastModified, nil
}
func (driver *SwiftDriver) ChangeDir(path string) bool {
	path = scoped_path(driver.user, path)
	if path == ("/"+driver.user) {
		return true
	}
	log.Printf("ChangeDir: %s", path)
	object, _, err := driver.connection.Object(driver.container, path)
	if err != nil {
		return false
	}
	return object.ContentType == "application/directory"
}
func (driver *SwiftDriver) DirContents(path string) (files []os.FileInfo) {
	path = scoped_path_with_trailing_slash(driver.user, path)
	log.Printf("DirContents: %s", path)
	opts    := &swift.ObjectsOpts{Prefix:path, Delimiter:'/'}
	objects, err := driver.connection.ObjectsAll(driver.container, opts)
	if err != nil {
		return // error connecting to cloud files
	}
	for _, object := range objects {
		tail     := strings.Replace(object.Name, path, "", 1)
        basename := strings.Split(tail, "/")[0]
		if object.ContentType == "application/directory" && object.SubDir == "" {
			files = append(files, graval.NewDirItem(basename))
		} else if object.ContentType != "application/directory"  {
			files = append(files, graval.NewFileItem(basename, int(object.Bytes)))
		}
	}
	return
}

func (driver *SwiftDriver) DeleteDir(path string) bool {
	path = scoped_path(driver.user, path)
	log.Printf("DeleteDir: %s", path)
	err := driver.connection.ObjectDelete(driver.container, path)
	if err != nil {
		return false
	}
	return true
}
func (driver *SwiftDriver) DeleteFile(path string) bool {
	path = scoped_path(driver.user, path)
	log.Printf("DeleteFile: %s", path)
	err := driver.connection.ObjectDelete(driver.container, path)
	if err != nil {
		return false
	}
	return true
}
func (driver *SwiftDriver) Rename(fromPath string, toPath string) bool {
	fromPath = scoped_path(driver.user, fromPath)
	toPath   = scoped_path(driver.user, toPath)
	log.Printf("Rename: %s %s", fromPath, toPath)
	return false
}
func (driver *SwiftDriver) MakeDir(path string) bool {
	path = scoped_path(driver.user, path)
	log.Printf("MakeDir: %s", path)
	opts    := &swift.ObjectsOpts{Prefix:path}
	objects, err := driver.connection.ObjectNames(driver.container, opts)
	if err != nil {
		return false // error connection to cloud files
	}
	if len(objects) > 0 {
		return false // the dir already exists
	}
	driver.connection.ObjectPutString(driver.container, path, "", "application/directory")
	return true
}
func (driver *SwiftDriver) GetFile(path string) (data string, err error) {
	path = scoped_path(driver.user, path)
	log.Printf("GetFile: %s", path)
	data, err = driver.connection.ObjectGetString(driver.container, path)
	if err != nil {
		return "", errors.New("foo")
	}
	return
}
func (driver *SwiftDriver) PutFile(destPath string, data io.Reader) bool {
	destPath = scoped_path(driver.user, destPath)
	log.Printf("PutFile: %s", destPath)
	contents, err := ioutil.ReadAll(data)
	if err != nil {
		return false
	}
	err = driver.connection.ObjectPutBytes(driver.container, destPath, contents, "application/octet-stream")
	if err != nil {
		return false
	}
	return true
}

func scoped_path_with_trailing_slash(user string, path string) string {
	path = scoped_path(user, path)
	if !strings.HasSuffix(path, "/") {
		path += "/"
	}
	if path == "/" {
		return ""
	}
	return path
}

func scoped_path(user string, path string) string {
	if path == "/" {
		path = ""
	}
	return filepath.Join("/", user, path)
}

// graval requires a factory that will create a new driver instance for each
// client connection. Generally the factory will be fairly minimal. This is
// a good place to read any required config for your driver.
type SwiftDriverFactory struct{}

func (factory *SwiftDriverFactory) NewDriver() (graval.FTPDriver, error) {
	driver := &SwiftDriver{}
	driver.container  = os.Getenv("Container")
	if driver.container == "" {
		return nil, errors.New("Container env variable not set")
	}
	driver.connection = &swift.Connection{
		UserName: os.Getenv("UserName"),
		ApiKey:   os.Getenv("ApiKey"),
		AuthUrl:  os.Getenv("AuthUrl"),
	}
	err := driver.connection.Authenticate()
	if err != nil {
		return nil, err
	}
	return driver, nil
}

// it's alive!
func main() {
	factory := &SwiftDriverFactory{}
	ftpServer := graval.NewFTPServer(&graval.FTPServerOpts{ Factory: factory })
	err := ftpServer.ListenAndServe()
	if err != nil {
		log.Fatal("Error starting server!")
		log.Fatal(err)
	}
}
<|endoftext|>


<commit_before><commit_msg>Remove empty file (#3456)<commit_after><|endoftext|>


<commit_before>package go_euler

import (
	"container/heap"
	"sync"
)

// An CompositeHeap is a min-heap of (pos, prime) tuples.
// See: http://golang.org/pkg/container/heap/.
type Composite struct {
	Pos   int
	Prime int
}

type CompositeHeap []Composite

func (h CompositeHeap) Len() int           { return len(h) }
func (h CompositeHeap) Less(i, j int) bool { return h[i].Pos < h[j].Pos }
func (h CompositeHeap) Swap(i, j int)      { h[i], h[j] = h[j], h[i] }

func (h *CompositeHeap) Push(x interface{}) {
	// Push and Pop use pointer receivers because they modify the slice's length,
	// not just its contents.
	*h = append(*h, x.(Composite))
}

func (h *CompositeHeap) Pop() interface{} {
	old := *h
	n := len(old)
	x := old[n-1]
	*h = old[0 : n-1]
	return x
}

var compositeHeap *CompositeHeap = &CompositeHeap{
	// Composite{4, 2} is not needed because even numbers are skipped.
	Composite{9, 3},
}

var knownPrimes []int = []int{2, 3}
var primesRwLock sync.RWMutex
var generateLock sync.Mutex

/**
 * Generating primes with sieve.
 * See: http://www.cs.hmc.edu/~oneill/papers/Sieve-JFP.pdf
 *
 * This approach is slightly slower than trial division. Before identifying the
 * memory bottleneck it was significantly slower.
 *
 * primesSieve(2000000) was substantially slower than calling the function in
 * small steps and progressively reaching 2000000 (2 minutes vs 7 seconds).
 *
 * Attempted optimizations:
 * 1) Skipping even numbers. Paper suggests 77% improvement. Observed: ~20%.
 * 2) Generating target primes directly. Substantially slower than calling the
 *    function in small steps and progressively reaching 2000000.
 *    (2 minutes vs 7 seconds.) This mystery applies to trial division as well
 *    and appears to be platform specific (Mac OSX only). Attempts to profile
 *    have failed. See related:
 *    http://godoc.org/code.google.com/p/rsc/cmd/pprof_mac_fix
 * 3) Allocating a full block of memory in advance, rather than append doubling.
 *    No real difference in performance. Removed. Unclear if there is a way to
 *    pre-allocate and use from that pool without tons of hacks.
 *
 * Ultimately the performance problem was from allocating lots of objects:
 *   min := (*compositeHeap)[0]  // New object.
 *   // ...
 *   heap.Push(compositeHeap, min)  // Escapes local scope. Expensive.
 */
func GetPrime(ceil int) int {
	if ceil < len(knownPrimes) {
		return knownPrimes[ceil]
	}
	generateLock.Lock()
	defer generateLock.Unlock()

	min := (*compositeHeap)[0]
	// No lock needed; knownPrimes is append only and GC won't remove value until
	// we release pointer.
	largestPrime := knownPrimes[len(knownPrimes)-1]
	for i := largestPrime + 2; len(knownPrimes) <= ceil; i += 2 {
		for i > min.Pos {
			// Increase the multiple and fix the entry.
			(*compositeHeap)[0].Pos += min.Prime
			heap.Fix(compositeHeap, 0)
			// Look at the new lowest composite.
			min = (*compositeHeap)[0]
		}
		if i < min.Pos {
			// Eg: i == 3, i < 4. Insert new prime 3 as {3*3, 3}.
			// When another prime (eg, 7) is discovered, do the same.
			primesRwLock.Lock()
			knownPrimes = append(knownPrimes, i)
			primesRwLock.Unlock()
			next := Composite{i * i, i}
			heap.Push(compositeHeap, next)
		}
	}
	return knownPrimes[ceil]
}
<commit_msg>Use array of *Composite in primes_sieve.<commit_after>package go_euler

import (
	"container/heap"
	"sync"
)

// An CompositeHeap is a min-heap of (pos, prime) tuples.
// See: http://golang.org/pkg/container/heap/.
type Composite struct {
	Pos   int
	Prime int
}

type CompositeHeap []*Composite

func (h CompositeHeap) Len() int           { return len(h) }
func (h CompositeHeap) Less(i, j int) bool { return h[i].Pos < h[j].Pos }
func (h CompositeHeap) Swap(i, j int)      { h[i], h[j] = h[j], h[i] }

func (h *CompositeHeap) Push(x interface{}) {
	// Push and Pop use pointer receivers because they modify the slice's length,
	// not just its contents.
	*h = append(*h, x.(*Composite))
}

func (h *CompositeHeap) Pop() interface{} {
	old := *h
	n := len(old)
	x := old[n-1]
	*h = old[0 : n-1]
	return x
}

var compositeHeap *CompositeHeap = &CompositeHeap{
	// Not needed because even numbers are skipped:
	// &Composite{4, 2},
	&Composite{9, 3},
}

var knownPrimes []int = []int{2, 3}
var primesRwLock sync.RWMutex
var generateLock sync.Mutex

/**
 * Generating primes with sieve.
 * See: http://www.cs.hmc.edu/~oneill/papers/Sieve-JFP.pdf
 *
 * This approach is slightly slower than trial division. Before identifying the
 * memory bottleneck it was significantly slower.
 *
 * primesSieve(2000000) was substantially slower than calling the function in
 * small steps and progressively reaching 2000000 (2 minutes vs 7 seconds).
 *
 * Attempted optimizations:
 * 1) Skipping even numbers. Paper suggests 77% improvement. Observed: ~20%.
 * 2) Generating target primes directly. Substantially slower than calling the
 *    function in small steps and progressively reaching 2000000.
 *    (2 minutes vs 7 seconds.) This mystery applies to trial division as well
 *    and appears to be platform specific (Mac OSX only). Attempts to profile
 *    have failed. See related:
 *    http://godoc.org/code.google.com/p/rsc/cmd/pprof_mac_fix
 * 3) Allocating a full block of memory in advance, rather than append doubling.
 *    No real difference in performance. Removed. Unclear if there is a way to
 *    pre-allocate and use from that pool without tons of hacks.
 *
 * Ultimately the performance problem was from allocating lots of objects:
 *   min := (*compositeHeap)[0]  // New object.
 *   // ...
 *   heap.Push(compositeHeap, min)  // Escapes local scope. Expensive.
 */
func GetPrime(ceil int) int {
	if ceil < len(knownPrimes) {
		return knownPrimes[ceil]
	}
	generateLock.Lock()
	defer generateLock.Unlock()

	min := (*compositeHeap)[0]
	// No lock needed; knownPrimes is append only and GC won't remove value until
	// we release pointer.
	largestPrime := knownPrimes[len(knownPrimes)-1]
	for i := largestPrime + 2; len(knownPrimes) <= ceil; i += 2 {
		for i > min.Pos {
			// Increase the multiple and fix the entry.
			min.Pos += min.Prime
			heap.Fix(compositeHeap, 0)
			// Look at the new lowest composite.
			min = (*compositeHeap)[0]
		}
		if i < min.Pos {
			// Eg: i == 3, i < 4. Insert new prime 3 as {3*3, 3}.
			// When another prime (eg, 7) is discovered, do the same.
			primesRwLock.Lock()
			knownPrimes = append(knownPrimes, i)
			primesRwLock.Unlock()
			heap.Push(compositeHeap, &Composite{i * i, i})
		}
	}
	return knownPrimes[ceil]
}
<|endoftext|>


<commit_before>package filters

import "testing"
import "github.com/stretchr/testify/assert"
import "github.com/appPlant/alpinepass/src/data"

func TestFilterLocationTrue(t *testing.T) {
	filter := "location=Earth"
	config := data.Config{Location: "Earth"}
	filteredConfig := filterProperty(filter, config)

	assert.Equal(t, true, filteredConfig.IsValid,
		"The Config should match the filter.")
}

func TestFilterLocationFalse(t *testing.T) {
	filter := "location=Earth"
	config := data.Config{Location: "Moon"}
	filteredConfig := filterProperty(filter, config)

	assert.Equal(t, false, filteredConfig.IsValid,
		"The Config should not match the filter.")
}
<commit_msg>Fix tests<commit_after>package filters

import "testing"
import "github.com/stretchr/testify/assert"
import "github.com/appPlant/alpinepass/src/data"

func TestFilterLocationTrue(t *testing.T) {
	filter := "location=Earth"
	config := data.Config{Location: []string{"Solar System", "Earth"}}
	filteredConfig := filterProperty(filter, config)

	assert.Equal(t, true, filteredConfig.IsValid, "The Config should match the filter.")
}

func TestFilterLocationFalse(t *testing.T) {
	filter := "location=Earth"
	config := data.Config{Location: []string{"Solar System", "Mars"}}
	filteredConfig := filterProperty(filter, config)

	assert.Equal(t, false, filteredConfig.IsValid,
		"The Config should not match the filter.")
}
<|endoftext|>