<commit_before>/*
 * Minio Cloud Storage, (C) 2016 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package main

import (
	"encoding/json"
	"fmt"
	"io"
	"path/filepath"
	"strings"
	"sync"

	"github.com/minio/minio/pkg/mimedb"
)

const (
	multipartSuffix   = ".minio.multipart"
	multipartMetaFile = "00000" + multipartSuffix
	formatConfigFile  = "format.json"
)

// xlObjects - Implements fs object layer.
type xlObjects struct {
	storage            StorageAPI
	listObjectMap      map[listParams][]*treeWalker
	listObjectMapMutex *sync.Mutex
}

// isValidFormat - validates input arguments with backend 'format.json'
func isValidFormat(storage StorageAPI, exportPaths ...string) bool {
	// Load saved XL format.json and validate.
	xl, err := loadFormatXL(storage)
	if err != nil {
		log.Errorf("loadFormatXL failed with %s", err)
		return false
	}
	if xl.Version != "1" {
		log.Errorf("Unsupported XL backend format found [%s]", xl.Version)
		return false
	}
	if len(exportPaths) != len(xl.Disks) {
		log.Errorf("Number of disks %d passed at the command-line did not match the backend format %d", len(exportPaths), len(xl.Disks))
		return false
	}
	for index, disk := range xl.Disks {
		if exportPaths[index] != disk {
			log.Errorf("Invalid order of disks detected %s. Required order is %s.", exportPaths, xl.Disks)
			return false
		}
	}
	return true
}

// newXLObjects - initialize new xl object layer.
func newXLObjects(exportPaths ...string) (ObjectLayer, error) {
	storage, err := newXL(exportPaths...)
	if err != nil {
		log.Errorf("newXL failed with %s", err)
		return nil, err
	}

	// Initialize object layer - like creating minioMetaBucket,
	// cleaning up tmp files etc.
	initObjectLayer(storage)

	err = checkFormat(storage)
	if err != nil {
		if err == errFileNotFound {
			// Save new XL format.
			errSave := saveFormatXL(storage, &xlFormat{
				Version: "1",
				Disks:   exportPaths,
			})
			if errSave != nil {
				log.Errorf("saveFormatXL failed with %s", errSave)
				return nil, errSave
			}
		} else {
			log.Errorf("Unable to check backend format %s", err)
			return nil, err
		}
	}

	// Validate if format exists and input arguments are validated
	// with backend format.
	if !isValidFormat(storage, exportPaths...) {
		return nil, fmt.Errorf("Command-line arguments %s is not valid.", exportPaths)
	}

	// Return successfully initialized object layer.
	return xlObjects{
		storage:            storage,
		listObjectMap:      make(map[listParams][]*treeWalker),
		listObjectMapMutex: &sync.Mutex{},
	}, nil
}

/// Bucket operations

// MakeBucket - make a bucket.
func (xl xlObjects) MakeBucket(bucket string) error {
	return makeBucket(xl.storage, bucket)
}

// GetBucketInfo - get bucket info.
func (xl xlObjects) GetBucketInfo(bucket string) (BucketInfo, error) {
	return getBucketInfo(xl.storage, bucket)
}

// ListBuckets - list buckets.
func (xl xlObjects) ListBuckets() ([]BucketInfo, error) {
	return listBuckets(xl.storage)
}

// DeleteBucket - delete a bucket.
func (xl xlObjects) DeleteBucket(bucket string) error {
	return deleteBucket(xl.storage, bucket)
}

/// Object Operations

// GetObject - get an object.
func (xl xlObjects) GetObject(bucket, object string, startOffset int64) (io.ReadCloser, error) {
	// Verify if bucket is valid.
	if !IsValidBucketName(bucket) {
		return nil, BucketNameInvalid{Bucket: bucket}
	}
	// Verify if object is valid.
	if !IsValidObjectName(object) {
		return nil, ObjectNameInvalid{Bucket: bucket, Object: object}
	}
	if ok, err := isMultipartObject(xl.storage, bucket, object); err != nil {
		return nil, toObjectErr(err, bucket, object)
	} else if !ok {
		if _, err = xl.storage.StatFile(bucket, object); err == nil {
			var reader io.ReadCloser
			reader, err = xl.storage.ReadFile(bucket, object, startOffset)
			if err != nil {
				return nil, toObjectErr(err, bucket, object)
			}
			return reader, nil
		}
		return nil, toObjectErr(err, bucket, object)
	}
	fileReader, fileWriter := io.Pipe()
	info, err := getMultipartObjectInfo(xl.storage, bucket, object)
	if err != nil {
		return nil, toObjectErr(err, bucket, object)
	}
	partIndex, offset, err := info.GetPartNumberOffset(startOffset)
	if err != nil {
		return nil, toObjectErr(err, bucket, object)
	}
	go func() {
		for ; partIndex < len(info.Parts); partIndex++ {
			part := info.Parts[partIndex]
			r, err := xl.storage.ReadFile(bucket, pathJoin(object, partNumToPartFileName(part.PartNumber)), offset)
			if err != nil {
				fileWriter.CloseWithError(err)
				return
			}
			if _, err := io.Copy(fileWriter, r); err != nil {
				fileWriter.CloseWithError(err)
				return
			}
		}
		fileWriter.Close()
	}()
	return fileReader, nil
}

// Return the partsInfo of a special multipart object.
func getMultipartObjectInfo(storage StorageAPI, bucket, object string) (info MultipartObjectInfo, err error) {
	offset := int64(0)
	r, err := storage.ReadFile(bucket, pathJoin(object, multipartMetaFile), offset)
	if err != nil {
		return
	}
	decoder := json.NewDecoder(r)
	err = decoder.Decode(&info)
	return
}

// GetObjectInfo - get object info.
func (xl xlObjects) GetObjectInfo(bucket, object string) (ObjectInfo, error) {
	// Verify if bucket is valid.
	if !IsValidBucketName(bucket) {
		return ObjectInfo{}, BucketNameInvalid{Bucket: bucket}
	}
	// Verify if object is valid.
	if !IsValidObjectName(object) {
		return ObjectInfo{}, ObjectNameInvalid{Bucket: bucket, Object: object}
	}
	// Check whether the bucket exists.
	if isExist, err := isBucketExist(xl.storage, bucket); err != nil {
		return ObjectInfo{}, err
	} else if !isExist {
		return ObjectInfo{}, BucketNotFound{Bucket: bucket}
	}
	fi, err := xl.storage.StatFile(bucket, object)
	if err != nil {
		if err != errFileNotFound {
			return ObjectInfo{}, toObjectErr(err, bucket, object)
		}
		var info MultipartObjectInfo
		info, err = getMultipartObjectInfo(xl.storage, bucket, object)
		if err != nil {
			return ObjectInfo{}, toObjectErr(err, bucket, object)
		}
		fi.Size = info.Size
		fi.ModTime = info.ModTime
		fi.MD5Sum = info.MD5Sum
	}
	contentType := "application/octet-stream"
	if objectExt := filepath.Ext(object); objectExt != "" {
		content, ok := mimedb.DB[strings.ToLower(strings.TrimPrefix(objectExt, "."))]
		if ok {
			contentType = content.ContentType
		}
	}
	return ObjectInfo{
		Bucket:      bucket,
		Name:        object,
		ModTime:     fi.ModTime,
		Size:        fi.Size,
		IsDir:       fi.Mode.IsDir(),
		ContentType: contentType,
		MD5Sum:      fi.MD5Sum,
	}, nil
}

// PutObject - create an object.
func (xl xlObjects) PutObject(bucket string, object string, size int64, data io.Reader, metadata map[string]string) (string, error) {
	return putObjectCommon(xl.storage, bucket, object, size, data, metadata)
}

// isMultipartObject - verifies if an object is special multipart file.
func isMultipartObject(storage StorageAPI, bucket, object string) (bool, error) {
	_, err := storage.StatFile(bucket, pathJoin(object, multipartMetaFile))
	if err != nil {
		if err == errFileNotFound {
			return false, nil
		}
		return false, err
	}
	return true, nil
}

func (xl xlObjects) DeleteObject(bucket, object string) error {
	// Verify if bucket is valid.
	if !IsValidBucketName(bucket) {
		return BucketNameInvalid{Bucket: bucket}
	}
	if !IsValidObjectName(object) {
		return ObjectNameInvalid{Bucket: bucket, Object: object}
	}
	// Verify if the object is a multipart object.
	if ok, err := isMultipartObject(xl.storage, bucket, object); err != nil {
		return toObjectErr(err, bucket, object)
	} else if !ok {
		if err = xl.storage.DeleteFile(bucket, object); err != nil {
			return toObjectErr(err, bucket, object)
		}
		return nil
	}
	// Get parts info.
	info, err := getMultipartObjectInfo(xl.storage, bucket, object)
	if err != nil {
		return toObjectErr(err, bucket, object)
	}
	// Range through all files and delete it.
	var wg = &sync.WaitGroup{}
	var errChs = make([]chan error, len(info.Parts))
	for _, part := range info.Parts {
		wg.Add(1)
		go func(part MultipartPartInfo) {
			defer wg.Done()
			err = xl.storage.DeleteFile(bucket, pathJoin(object, partNumToPartFileName(part.PartNumber)))
			errChs[part.PartNumber-1] <- err
		}(part)
	}
	wg.Wait()
	for _, errCh := range errChs {
		if err = <-errCh; err != nil {
			return toObjectErr(err, bucket, object)
		}
	}
	err = xl.storage.DeleteFile(bucket, pathJoin(object, multipartMetaFile))
	if err != nil {
		return toObjectErr(err, bucket, object)
	}
	return nil
}

// ListObjects - list all objects at prefix, delimited by '/'.
func (xl xlObjects) ListObjects(bucket, prefix, marker, delimiter string, maxKeys int) (ListObjectsInfo, error) {
	return listObjectsCommon(xl, bucket, prefix, marker, delimiter, maxKeys)
}
<commit_msg>xl: DeleteObject regression over errChannel. (#1521)<commit_after>/*
 * Minio Cloud Storage, (C) 2016 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package main

import (
	"encoding/json"
	"fmt"
	"io"
	"path/filepath"
	"strings"
	"sync"

	"github.com/minio/minio/pkg/mimedb"
)

const (
	multipartSuffix   = ".minio.multipart"
	multipartMetaFile = "00000" + multipartSuffix
	formatConfigFile  = "format.json"
)

// xlObjects - Implements fs object layer.
type xlObjects struct {
	storage            StorageAPI
	listObjectMap      map[listParams][]*treeWalker
	listObjectMapMutex *sync.Mutex
}

// isValidFormat - validates input arguments with backend 'format.json'
func isValidFormat(storage StorageAPI, exportPaths ...string) bool {
	// Load saved XL format.json and validate.
	xl, err := loadFormatXL(storage)
	if err != nil {
		log.Errorf("loadFormatXL failed with %s", err)
		return false
	}
	if xl.Version != "1" {
		log.Errorf("Unsupported XL backend format found [%s]", xl.Version)
		return false
	}
	if len(exportPaths) != len(xl.Disks) {
		log.Errorf("Number of disks %d passed at the command-line did not match the backend format %d", len(exportPaths), len(xl.Disks))
		return false
	}
	for index, disk := range xl.Disks {
		if exportPaths[index] != disk {
			log.Errorf("Invalid order of disks detected %s. Required order is %s.", exportPaths, xl.Disks)
			return false
		}
	}
	return true
}

// newXLObjects - initialize new xl object layer.
func newXLObjects(exportPaths ...string) (ObjectLayer, error) {
	storage, err := newXL(exportPaths...)
	if err != nil {
		log.Errorf("newXL failed with %s", err)
		return nil, err
	}

	// Initialize object layer - like creating minioMetaBucket,
	// cleaning up tmp files etc.
	initObjectLayer(storage)

	err = checkFormat(storage)
	if err != nil {
		if err == errFileNotFound {
			// Save new XL format.
			errSave := saveFormatXL(storage, &xlFormat{
				Version: "1",
				Disks:   exportPaths,
			})
			if errSave != nil {
				log.Errorf("saveFormatXL failed with %s", errSave)
				return nil, errSave
			}
		} else {
			log.Errorf("Unable to check backend format %s", err)
			return nil, err
		}
	}

	// Validate if format exists and input arguments are validated
	// with backend format.
	if !isValidFormat(storage, exportPaths...) {
		return nil, fmt.Errorf("Command-line arguments %s is not valid.", exportPaths)
	}

	// Return successfully initialized object layer.
	return xlObjects{
		storage:            storage,
		listObjectMap:      make(map[listParams][]*treeWalker),
		listObjectMapMutex: &sync.Mutex{},
	}, nil
}

/// Bucket operations

// MakeBucket - make a bucket.
func (xl xlObjects) MakeBucket(bucket string) error {
	return makeBucket(xl.storage, bucket)
}

// GetBucketInfo - get bucket info.
func (xl xlObjects) GetBucketInfo(bucket string) (BucketInfo, error) {
	return getBucketInfo(xl.storage, bucket)
}

// ListBuckets - list buckets.
func (xl xlObjects) ListBuckets() ([]BucketInfo, error) {
	return listBuckets(xl.storage)
}

// DeleteBucket - delete a bucket.
func (xl xlObjects) DeleteBucket(bucket string) error {
	return deleteBucket(xl.storage, bucket)
}

/// Object Operations

// GetObject - get an object.
func (xl xlObjects) GetObject(bucket, object string, startOffset int64) (io.ReadCloser, error) {
	// Verify if bucket is valid.
	if !IsValidBucketName(bucket) {
		return nil, BucketNameInvalid{Bucket: bucket}
	}
	// Verify if object is valid.
	if !IsValidObjectName(object) {
		return nil, ObjectNameInvalid{Bucket: bucket, Object: object}
	}
	if ok, err := isMultipartObject(xl.storage, bucket, object); err != nil {
		return nil, toObjectErr(err, bucket, object)
	} else if !ok {
		if _, err = xl.storage.StatFile(bucket, object); err == nil {
			var reader io.ReadCloser
			reader, err = xl.storage.ReadFile(bucket, object, startOffset)
			if err != nil {
				return nil, toObjectErr(err, bucket, object)
			}
			return reader, nil
		}
		return nil, toObjectErr(err, bucket, object)
	}
	fileReader, fileWriter := io.Pipe()
	info, err := getMultipartObjectInfo(xl.storage, bucket, object)
	if err != nil {
		return nil, toObjectErr(err, bucket, object)
	}
	partIndex, offset, err := info.GetPartNumberOffset(startOffset)
	if err != nil {
		return nil, toObjectErr(err, bucket, object)
	}
	go func() {
		for ; partIndex < len(info.Parts); partIndex++ {
			part := info.Parts[partIndex]
			r, err := xl.storage.ReadFile(bucket, pathJoin(object, partNumToPartFileName(part.PartNumber)), offset)
			if err != nil {
				fileWriter.CloseWithError(err)
				return
			}
			if _, err := io.Copy(fileWriter, r); err != nil {
				fileWriter.CloseWithError(err)
				return
			}
		}
		fileWriter.Close()
	}()
	return fileReader, nil
}

// Return the partsInfo of a special multipart object.
func getMultipartObjectInfo(storage StorageAPI, bucket, object string) (info MultipartObjectInfo, err error) {
	offset := int64(0)
	r, err := storage.ReadFile(bucket, pathJoin(object, multipartMetaFile), offset)
	if err != nil {
		return
	}
	decoder := json.NewDecoder(r)
	err = decoder.Decode(&info)
	return
}

// GetObjectInfo - get object info.
func (xl xlObjects) GetObjectInfo(bucket, object string) (ObjectInfo, error) {
	// Verify if bucket is valid.
	if !IsValidBucketName(bucket) {
		return ObjectInfo{}, BucketNameInvalid{Bucket: bucket}
	}
	// Verify if object is valid.
	if !IsValidObjectName(object) {
		return ObjectInfo{}, ObjectNameInvalid{Bucket: bucket, Object: object}
	}
	// Check whether the bucket exists.
	if isExist, err := isBucketExist(xl.storage, bucket); err != nil {
		return ObjectInfo{}, err
	} else if !isExist {
		return ObjectInfo{}, BucketNotFound{Bucket: bucket}
	}
	fi, err := xl.storage.StatFile(bucket, object)
	if err != nil {
		if err != errFileNotFound {
			return ObjectInfo{}, toObjectErr(err, bucket, object)
		}
		var info MultipartObjectInfo
		info, err = getMultipartObjectInfo(xl.storage, bucket, object)
		if err != nil {
			return ObjectInfo{}, toObjectErr(err, bucket, object)
		}
		fi.Size = info.Size
		fi.ModTime = info.ModTime
		fi.MD5Sum = info.MD5Sum
	}
	contentType := "application/octet-stream"
	if objectExt := filepath.Ext(object); objectExt != "" {
		content, ok := mimedb.DB[strings.ToLower(strings.TrimPrefix(objectExt, "."))]
		if ok {
			contentType = content.ContentType
		}
	}
	return ObjectInfo{
		Bucket:      bucket,
		Name:        object,
		ModTime:     fi.ModTime,
		Size:        fi.Size,
		IsDir:       fi.Mode.IsDir(),
		ContentType: contentType,
		MD5Sum:      fi.MD5Sum,
	}, nil
}

// PutObject - create an object.
func (xl xlObjects) PutObject(bucket string, object string, size int64, data io.Reader, metadata map[string]string) (string, error) {
	return putObjectCommon(xl.storage, bucket, object, size, data, metadata)
}

// isMultipartObject - verifies if an object is special multipart file.
func isMultipartObject(storage StorageAPI, bucket, object string) (bool, error) {
	_, err := storage.StatFile(bucket, pathJoin(object, multipartMetaFile))
	if err != nil {
		if err == errFileNotFound {
			return false, nil
		}
		return false, err
	}
	return true, nil
}

func (xl xlObjects) DeleteObject(bucket, object string) error {
	// Verify if bucket is valid.
	if !IsValidBucketName(bucket) {
		return BucketNameInvalid{Bucket: bucket}
	}
	if !IsValidObjectName(object) {
		return ObjectNameInvalid{Bucket: bucket, Object: object}
	}
	// Verify if the object is a multipart object.
	if ok, err := isMultipartObject(xl.storage, bucket, object); err != nil {
		return toObjectErr(err, bucket, object)
	} else if !ok {
		if err = xl.storage.DeleteFile(bucket, object); err != nil {
			return toObjectErr(err, bucket, object)
		}
		return nil
	}
	// Get parts info.
	info, err := getMultipartObjectInfo(xl.storage, bucket, object)
	if err != nil {
		return toObjectErr(err, bucket, object)
	}
	// Range through all files and delete it.
	var wg = &sync.WaitGroup{}
	var errs = make([]error, len(info.Parts))
	for index, part := range info.Parts {
		wg.Add(1)
		// Start deleting parts in routine.
		go func(index int, part MultipartPartInfo) {
			defer wg.Done()
			partFileName := partNumToPartFileName(part.PartNumber)
			errs[index] = xl.storage.DeleteFile(bucket, pathJoin(object, partFileName))
		}(index, part)
	}
	// Wait for all the deletes to finish.
	wg.Wait()
	// Loop through and validate if any errors, return back the first
	// error occurred.
	for index, err := range errs {
		if err != nil {
			partFileName := partNumToPartFileName(info.Parts[index].PartNumber)
			return toObjectErr(err, bucket, pathJoin(object, partFileName))
		}
	}
	err = xl.storage.DeleteFile(bucket, pathJoin(object, multipartMetaFile))
	if err != nil {
		return toObjectErr(err, bucket, object)
	}
	return nil
}

// ListObjects - list all objects at prefix, delimited by '/'.
func (xl xlObjects) ListObjects(bucket, prefix, marker, delimiter string, maxKeys int) (ListObjectsInfo, error) {
	return listObjectsCommon(xl, bucket, prefix, marker, delimiter, maxKeys)
}
<|endoftext|>


<commit_before>package app

import (
	"code.google.com/p/gcfg"
	"errors"
	"fmt"
	"github.com/Sirupsen/logrus"
	"github.com/Tapjoy/dynamiq/app/compressor"
	"github.com/Tapjoy/dynamiq/app/stats"
	"github.com/tpjg/goriakpbc"
	"math/rand"
	"strconv"
	"time"
)

var (
	ConfigurationOptionNotFound = errors.New("Configuration Value Not Found")
)

const CONFIGURATION_BUCKET = "config"
const QUEUE_CONFIG_NAME = "queue_config"
const QUEUE_SET_NAME = "queues"

const VISIBILITY_TIMEOUT = "visibility_timeout"
const PARTITION_COUNT = "partition_count"
const MIN_PARTITIONS = "min_partitions"
const MAX_PARTITIONS = "max_partitions"
const MAX_PARTITION_AGE = "max_partition_age"
const COMPRESSED_MESSAGES = "compressed_messages"

// Arrays and maps cannot be made immutable in golang
var SETTINGS = [...]string{VISIBILITY_TIMEOUT, PARTITION_COUNT, MIN_PARTITIONS, MAX_PARTITIONS, MAX_PARTITION_AGE, COMPRESSED_MESSAGES}
var DEFAULT_SETTINGS = map[string]string{VISIBILITY_TIMEOUT: "30", PARTITION_COUNT: "5", MIN_PARTITIONS: "1", MAX_PARTITIONS: "10", MAX_PARTITION_AGE: "432000", COMPRESSED_MESSAGES: "false"}

type Config struct {
	Core     Core
	Stats    Stats
	Queues   *Queues
	RiakPool *riak.Client
	Topics   *Topics
	Core       Core
	Stats      Stats
	Compressor compressor.Compressor
	Queues     *Queues
	RiakPool   *riak.Client
}

type Core struct {
	Name                  string
	Port                  int
	SeedServer            string
	SeedPort              int
	HttpPort              int
	RiakNodes             string
	BackendConnectionPool int
	SyncConfigInterval    time.Duration
	LogLevel              logrus.Level
	LogLevelString        string
}

type Stats struct {
	Type                 string
	FlushInterval        int
	Address              string
	Prefix               string
	Client               stats.StatsClient
	InternalStatsdServer bool
}

func initRiakPool(cfg *Config) *riak.Client {
	rand.Seed(time.Now().UnixNano())
	// TODO this should just be 1 HAProxy
	hosts := []string{cfg.Core.RiakNodes}
	host := hosts[rand.Intn(len(hosts))]
	return riak.NewClientPool(host, cfg.Core.BackendConnectionPool)
}

func GetCoreConfig(config_file *string) (*Config, error) {
	var cfg Config
	err := gcfg.ReadFileInto(&cfg, *config_file)
	if err != nil {
		logrus.Fatal(err)
	}

	cfg.RiakPool = initRiakPool(&cfg)
	cfg.Queues = loadQueuesConfig(&cfg)
	switch cfg.Stats.Type {
	case "statsd":
		cfg.Stats.Client = stats.NewStatsdClient(cfg.Stats.Address, cfg.Stats.Prefix, time.Second*time.Duration(cfg.Stats.FlushInterval))
	default:
		cfg.Stats.Client = stats.NewNOOPClient()
	}

	// Currently we only support zlib, but we may support others
	// Here is where we'd detect and inject
	cfg.Compressor = compressor.NewZlibCompressor()

	cfg.Core.LogLevel, err = logrus.ParseLevel(cfg.Core.LogLevelString)
	if err != nil {
		logrus.Fatal(err)
	}

	go cfg.Queues.syncConfig(&cfg)
	return &cfg, err
}

func loadQueuesConfig(cfg *Config) *Queues {
	// Create the Queues Config struct
	queuesConfig := Queues{
		QueueMap: make(map[string]*Queue),
	}
	// Get the queues
	client := cfg.RiakConnection()
	// TODO: We should be handling errors here
	// Get the bucket holding the map of config data
	configBucket, _ := client.NewBucketType("maps", CONFIGURATION_BUCKET)
	// Fetch the object for holding the set of queues
	config, _ := configBucket.FetchMap(QUEUE_CONFIG_NAME)
	queuesConfig.Config = config

	// AddSet implicitly calls fetch set if the set already exists
	queueSet := config.AddSet(QUEUE_SET_NAME)
	if queueSet == nil {
		queueSet.Add([]byte("default_queue"))
		config.Store()
		config, _ = configBucket.FetchMap(QUEUE_CONFIG_NAME)
	}
	// For each queue we have in the system
	for _, elem := range queueSet.GetValue() {
		// Convert it's name into a string
		name := string(elem[:])
		// Get the Riak RdtMap of settings for this queue
		configMap, _ := configBucket.FetchMap(queueConfigRecordName(name))
		// Pre-warm the settings object
		queue := &Queue{
			Name:   name,
			Config: configMap,
			Parts:  InitPartitions(cfg, name),
		}
		// TODO: We should be handling errors here
		// Set the queue in the queue map
		queuesConfig.QueueMap[name] = queue
	}
	// Return the completed Queue cache of settings
	return &queuesConfig
}

func (cfg *Config) InitializeQueue(queueName string) error {
	// Create the configuration data in Riak first
	// This way it'll be there once the queue is added to the known set
	configMap, err := cfg.createConfigForQueue(queueName)
	if err != nil {
		return err
	}
	// Add to the known set of queues
	err = cfg.addToKnownQueues(queueName)
	// Now, add the queue into our memory-cache of data
	cfg.Queues.QueueMap[queueName] = &Queue{
		Name:   queueName,
		Parts:  InitPartitions(cfg, queueName),
		Config: configMap,
	}
	return err
}

func (cfg *Config) addToKnownQueues(queueName string) error {
	// If we disallow topicless-queues, we can remove this and put it into Topic.AddQueue
	client := cfg.RiakConnection()
	// We purposefully read from Riak here, we'll enventually-consist with the in memory cache
	bucket, _ := client.NewBucketType("maps", CONFIGURATION_BUCKET)
	queueConfig, _ := bucket.FetchMap(QUEUE_CONFIG_NAME)
	queueSet := queueConfig.AddSet(QUEUE_SET_NAME)
	queueSet.Add([]byte(queueName))
	return queueConfig.Store()
}

func (cfg *Config) removeFromKnownQueues(queueName string) error {
	// If we disallow topicless-queues, we can remove this and put it into Topic.RemoveQueue
	client := cfg.RiakConnection()
	// We purposefully read from Riak here, we'll enventually-consist with the in memory cache
	bucket, _ := client.NewBucketType("maps", CONFIGURATION_BUCKET)
	queueConfig, _ := bucket.FetchMap(QUEUE_CONFIG_NAME)
	queueSet := queueConfig.AddSet(QUEUE_SET_NAME)
	queueSet.Remove([]byte(queueName))
	return queueConfig.Store()
}

// TODO: Take in a map which overrides the defaults
func (cfg *Config) createConfigForQueue(queueName string) (*riak.RDtMap, error) {
	client := cfg.RiakConnection()
	// Get the bucket for holding maps of config data
	// TODO: Find a nice way to DRY this up - it's a lil copy/pasty
	bucket, _ := client.NewBucketType("maps", CONFIGURATION_BUCKET)
	// Get the object for this queues settings
	obj, _ := bucket.FetchMap(queueConfigRecordName(queueName))
	// For each known setting
	for _, elem := range SETTINGS {
		// Get the reigster for this setting
		reg := obj.AddRegister(elem)
		// Convert the default value to a bytearray, set it on the Register
		reg.Update([]byte(DEFAULT_SETTINGS[elem]))
	}
	// Save the object, returns an error up the callchain if needed
	return obj, obj.Store()
}

// SETTERS AND GETTERS FOR QUEUE CONFIG

func (cfg *Config) GetVisibilityTimeout(queueName string) (float64, error) {
	val, err := cfg.getQueueSetting(VISIBILITY_TIMEOUT, queueName)
	parsed, err := strconv.ParseFloat(val, 32)
	return parsed, err
}

func (cfg *Config) SetVisibilityTimeout(queueName string, timeout float64) error {
	return cfg.setQueueSetting(VISIBILITY_TIMEOUT, queueName, strconv.FormatFloat(timeout, 'f', -1, 64))
}

func (cfg *Config) GetMinPartitions(queueName string) (int, error) {
	val, _ := cfg.getQueueSetting(MIN_PARTITIONS, queueName)
	return strconv.Atoi(val)
}

func (cfg *Config) SetMinPartitions(queueName string, timeout int) error {
	// TODO do we handle any resizing here? Or does the system "self-adjust"
	return cfg.setQueueSetting(MIN_PARTITIONS, queueName, strconv.Itoa(timeout))
}

func (cfg *Config) GetMaxPartitions(queueName string) (int, error) {
	val, _ := cfg.getQueueSetting(MAX_PARTITIONS, queueName)
	return strconv.Atoi(val)
}

func (cfg *Config) SetMaxPartitions(queueName string, timeout int) error {
	// TODO do we handle any resizing here? Or does the system "self-adjust"
	return cfg.setQueueSetting(MAX_PARTITIONS, queueName, strconv.Itoa(timeout))
}
func (cfg *Config) SetMaxPartitionAge(queueName string, age float64) error {
	return cfg.setQueueSetting(MAX_PARTITION_AGE, queueName, strconv.FormatFloat(age, 'f', -1, 64))
}
func (cfg *Config) GetMaxPartitionAge(queueName string) (float64, error) {
	val, _ := cfg.getQueueSetting(MAX_PARTITION_AGE, queueName)
	return strconv.ParseFloat(val, 32)
}

func (cfg *Config) GetCompressedMessages(queueName string) (bool, error) {
	val, _ := cfg.getQueueSetting(COMPRESSED_MESSAGES, queueName)
	if val == "" {
		// If the queue pre-dated this setting, an empty value is equivalent to not compressing the data
		return false, nil
	}
	return strconv.ParseBool(val)
}

func (cfg *Config) SetCompressedMessages(queueName string, compressedMessages bool) error {
	return cfg.setQueueSetting(COMPRESSED_MESSAGES, queueName, strconv.FormatBool(compressedMessages))
}

// TODO Find a proper way to scope this to a queue VS a topic
func (cfg *Config) getQueueSetting(paramName string, queueName string) (string, error) {
	// Read from local cache
	value := ""
	var err error
	// If cfg.Queues is nil, it means we're in the middle of booting, and we're trying to configure
	// partition counts. If this is the case, skip to reading directly from Riak
	// This means booting up will incur a number of extra reads to Riak
	// We can likely improve this, but it works for the time being

	// If cfg.Queues.QueueMap[queuename] is nil, it means this server hasn't yet synced with Riak
	// While we wait, go and read from Riak directly
	if cfg.Queues != nil {
		if _, ok := cfg.Queues.QueueMap[queueName]; ok {
			value, err = registerValueToString(cfg.Queues.QueueMap[queueName].getQueueConfig().FetchRegister(paramName))
			if err != nil {
				// In the case where a register has been deleted, or for whatver reason gone, return an empty string, and
				// an error stating that it is nil.
				return value, err
			}
		}
	}

	if value == "" {
		// Read from riak
		client := cfg.RiakConnection()
		bucket, _ := client.NewBucketType("maps", CONFIGURATION_BUCKET)
		obj, err := bucket.FetchMap(queueConfigRecordName(queueName))

		// if not found... no config existed for that queue - should not happen hashtagcrossfingers
		if err == riak.NotFound {
			// Log out an error here
			return "", err
		}

		val := obj.FetchRegister(paramName)

		if val != nil {
			// We had a register with this name, return the value
			value, err = registerValueToString(val)
		}
	}
	return value, err
}

// TODO Find a proper way to scope this to a queue VS a topic
func (cfg *Config) setQueueSetting(paramName string, queueName string, value string) error {
	// Write to Riak
	client := cfg.RiakConnection()
	bucket, _ := client.NewBucketType("maps", CONFIGURATION_BUCKET)
	obj, err := bucket.FetchMap(queueConfigRecordName(queueName))
	// if not found... no config existed for that queue - should not happen hashtagcrossfingers
	if err == riak.NotFound {
		// Log out an error here
		return err
	}
	val := obj.AddRegister(paramName)
	val.NewValue = []byte(value)
	// Write to Riak
	return obj.Store()
}

// HELPERS

func registerValueToString(reg *riak.RDtRegister) (string, error) {
	// The register might have been deleted at this point, so handle nil case.
	if reg == nil {
		return "", errors.New("Register is nil.")
	}
	return string(reg.Value[:]), nil
}

func (cfg *Config) RiakConnection() *riak.Client {
	return cfg.RiakPool
}

func queueConfigRecordName(queueName string) string {
	return fmt.Sprintf("queue_%s_config", queueName)
}
<commit_msg>Fixing bug with missing queue params when the queue pre-dated them<commit_after>package app

import (
	"code.google.com/p/gcfg"
	"errors"
	"fmt"
	"github.com/Sirupsen/logrus"
	"github.com/Tapjoy/dynamiq/app/compressor"
	"github.com/Tapjoy/dynamiq/app/stats"
	"github.com/tpjg/goriakpbc"
	"math/rand"
	"strconv"
	"time"
)

var (
	ConfigurationOptionNotFound = errors.New("Configuration Value Not Found")
)

const CONFIGURATION_BUCKET = "config"
const QUEUE_CONFIG_NAME = "queue_config"
const QUEUE_SET_NAME = "queues"

const VISIBILITY_TIMEOUT = "visibility_timeout"
const PARTITION_COUNT = "partition_count"
const MIN_PARTITIONS = "min_partitions"
const MAX_PARTITIONS = "max_partitions"
const MAX_PARTITION_AGE = "max_partition_age"
const COMPRESSED_MESSAGES = "compressed_messages"

// Arrays and maps cannot be made immutable in golang
var SETTINGS = [...]string{VISIBILITY_TIMEOUT, PARTITION_COUNT, MIN_PARTITIONS, MAX_PARTITIONS, MAX_PARTITION_AGE, COMPRESSED_MESSAGES}
var DEFAULT_SETTINGS = map[string]string{VISIBILITY_TIMEOUT: "30", PARTITION_COUNT: "5", MIN_PARTITIONS: "1", MAX_PARTITIONS: "10", MAX_PARTITION_AGE: "432000", COMPRESSED_MESSAGES: "false"}

type Config struct {
	Core     Core
	Stats    Stats
	Queues   *Queues
	RiakPool *riak.Client
	Topics   *Topics
	Core       Core
	Stats      Stats
	Compressor compressor.Compressor
	Queues     *Queues
	RiakPool   *riak.Client
}

type Core struct {
	Name                  string
	Port                  int
	SeedServer            string
	SeedPort              int
	HttpPort              int
	RiakNodes             string
	BackendConnectionPool int
	SyncConfigInterval    time.Duration
	LogLevel              logrus.Level
	LogLevelString        string
}

type Stats struct {
	Type                 string
	FlushInterval        int
	Address              string
	Prefix               string
	Client               stats.StatsClient
	InternalStatsdServer bool
}

func initRiakPool(cfg *Config) *riak.Client {
	rand.Seed(time.Now().UnixNano())
	// TODO this should just be 1 HAProxy
	hosts := []string{cfg.Core.RiakNodes}
	host := hosts[rand.Intn(len(hosts))]
	return riak.NewClientPool(host, cfg.Core.BackendConnectionPool)
}

func GetCoreConfig(config_file *string) (*Config, error) {
	var cfg Config
	err := gcfg.ReadFileInto(&cfg, *config_file)
	if err != nil {
		logrus.Fatal(err)
	}

	cfg.RiakPool = initRiakPool(&cfg)
	cfg.Queues = loadQueuesConfig(&cfg)
	switch cfg.Stats.Type {
	case "statsd":
		cfg.Stats.Client = stats.NewStatsdClient(cfg.Stats.Address, cfg.Stats.Prefix, time.Second*time.Duration(cfg.Stats.FlushInterval))
	default:
		cfg.Stats.Client = stats.NewNOOPClient()
	}

	// Currently we only support zlib, but we may support others
	// Here is where we'd detect and inject
	cfg.Compressor = compressor.NewZlibCompressor()

	cfg.Core.LogLevel, err = logrus.ParseLevel(cfg.Core.LogLevelString)
	if err != nil {
		logrus.Fatal(err)
	}

	go cfg.Queues.syncConfig(&cfg)
	return &cfg, err
}

func loadQueuesConfig(cfg *Config) *Queues {
	// Create the Queues Config struct
	queuesConfig := Queues{
		QueueMap: make(map[string]*Queue),
	}
	// Get the queues
	client := cfg.RiakConnection()
	// TODO: We should be handling errors here
	// Get the bucket holding the map of config data
	configBucket, _ := client.NewBucketType("maps", CONFIGURATION_BUCKET)
	// Fetch the object for holding the set of queues
	config, _ := configBucket.FetchMap(QUEUE_CONFIG_NAME)
	queuesConfig.Config = config

	// AddSet implicitly calls fetch set if the set already exists
	queueSet := config.AddSet(QUEUE_SET_NAME)
	if queueSet == nil {
		queueSet.Add([]byte("default_queue"))
		config.Store()
		config, _ = configBucket.FetchMap(QUEUE_CONFIG_NAME)
	}
	// For each queue we have in the system
	for _, elem := range queueSet.GetValue() {
		// Convert it's name into a string
		name := string(elem[:])
		// Get the Riak RdtMap of settings for this queue
		configMap, _ := configBucket.FetchMap(queueConfigRecordName(name))
		// Pre-warm the settings object
		queue := &Queue{
			Name:   name,
			Config: configMap,
			Parts:  InitPartitions(cfg, name),
		}
		// TODO: We should be handling errors here
		// Set the queue in the queue map
		queuesConfig.QueueMap[name] = queue
	}
	// Return the completed Queue cache of settings
	return &queuesConfig
}

func (cfg *Config) InitializeQueue(queueName string) error {
	// Create the configuration data in Riak first
	// This way it'll be there once the queue is added to the known set
	configMap, err := cfg.createConfigForQueue(queueName)
	if err != nil {
		return err
	}
	// Add to the known set of queues
	err = cfg.addToKnownQueues(queueName)
	// Now, add the queue into our memory-cache of data
	cfg.Queues.QueueMap[queueName] = &Queue{
		Name:   queueName,
		Parts:  InitPartitions(cfg, queueName),
		Config: configMap,
	}
	return err
}

func (cfg *Config) addToKnownQueues(queueName string) error {
	// If we disallow topicless-queues, we can remove this and put it into Topic.AddQueue
	client := cfg.RiakConnection()
	// We purposefully read from Riak here, we'll enventually-consist with the in memory cache
	bucket, _ := client.NewBucketType("maps", CONFIGURATION_BUCKET)
	queueConfig, _ := bucket.FetchMap(QUEUE_CONFIG_NAME)
	queueSet := queueConfig.AddSet(QUEUE_SET_NAME)
	queueSet.Add([]byte(queueName))
	return queueConfig.Store()
}

func (cfg *Config) removeFromKnownQueues(queueName string) error {
	// If we disallow topicless-queues, we can remove this and put it into Topic.RemoveQueue
	client := cfg.RiakConnection()
	// We purposefully read from Riak here, we'll enventually-consist with the in memory cache
	bucket, _ := client.NewBucketType("maps", CONFIGURATION_BUCKET)
	queueConfig, _ := bucket.FetchMap(QUEUE_CONFIG_NAME)
	queueSet := queueConfig.AddSet(QUEUE_SET_NAME)
	queueSet.Remove([]byte(queueName))
	return queueConfig.Store()
}

// TODO: Take in a map which overrides the defaults
func (cfg *Config) createConfigForQueue(queueName string) (*riak.RDtMap, error) {
	client := cfg.RiakConnection()
	// Get the bucket for holding maps of config data
	// TODO: Find a nice way to DRY this up - it's a lil copy/pasty
	bucket, _ := client.NewBucketType("maps", CONFIGURATION_BUCKET)
	// Get the object for this queues settings
	obj, _ := bucket.FetchMap(queueConfigRecordName(queueName))
	// For each known setting
	for _, elem := range SETTINGS {
		// Get the reigster for this setting
		reg := obj.AddRegister(elem)
		// Convert the default value to a bytearray, set it on the Register
		reg.Update([]byte(DEFAULT_SETTINGS[elem]))
	}
	// Save the object, returns an error up the callchain if needed
	return obj, obj.Store()
}

// SETTERS AND GETTERS FOR QUEUE CONFIG

func (cfg *Config) GetVisibilityTimeout(queueName string) (float64, error) {
	val, err := cfg.getQueueSetting(VISIBILITY_TIMEOUT, queueName)
	parsed, err := strconv.ParseFloat(val, 32)
	return parsed, err
}

func (cfg *Config) SetVisibilityTimeout(queueName string, timeout float64) error {
	return cfg.setQueueSetting(VISIBILITY_TIMEOUT, queueName, strconv.FormatFloat(timeout, 'f', -1, 64))
}

func (cfg *Config) GetMinPartitions(queueName string) (int, error) {
	val, _ := cfg.getQueueSetting(MIN_PARTITIONS, queueName)
	return strconv.Atoi(val)
}

func (cfg *Config) SetMinPartitions(queueName string, timeout int) error {
	// TODO do we handle any resizing here? Or does the system "self-adjust"
	return cfg.setQueueSetting(MIN_PARTITIONS, queueName, strconv.Itoa(timeout))
}

func (cfg *Config) GetMaxPartitions(queueName string) (int, error) {
	val, _ := cfg.getQueueSetting(MAX_PARTITIONS, queueName)
	return strconv.Atoi(val)
}

func (cfg *Config) SetMaxPartitions(queueName string, timeout int) error {
	// TODO do we handle any resizing here? Or does the system "self-adjust"
	return cfg.setQueueSetting(MAX_PARTITIONS, queueName, strconv.Itoa(timeout))
}
func (cfg *Config) SetMaxPartitionAge(queueName string, age float64) error {
	return cfg.setQueueSetting(MAX_PARTITION_AGE, queueName, strconv.FormatFloat(age, 'f', -1, 64))
}
func (cfg *Config) GetMaxPartitionAge(queueName string) (float64, error) {
	val, _ := cfg.getQueueSetting(MAX_PARTITION_AGE, queueName)
	return strconv.ParseFloat(val, 32)
}

func (cfg *Config) GetCompressedMessages(queueName string) (bool, error) {
	val, _ := cfg.getQueueSetting(COMPRESSED_MESSAGES, queueName)
	return strconv.ParseBool(val)
}

func (cfg *Config) SetCompressedMessages(queueName string, compressedMessages bool) error {
	return cfg.setQueueSetting(COMPRESSED_MESSAGES, queueName, strconv.FormatBool(compressedMessages))
}

// TODO Find a proper way to scope this to a queue VS a topic
func (cfg *Config) getQueueSetting(paramName string, queueName string) (string, error) {
	// Read from local cache
	value := ""
	var err error
	// If cfg.Queues is nil, it means we're in the middle of booting, and we're trying to configure
	// partition counts. If this is the case, skip to reading directly from Riak
	// This means booting up will incur a number of extra reads to Riak
	// We can likely improve this, but it works for the time being

	// If cfg.Queues.QueueMap[queuename] is nil, it means this server hasn't yet synced with Riak
	// While we wait, go and read from Riak directly
	if cfg.Queues != nil {
		if _, ok := cfg.Queues.QueueMap[queueName]; ok {
			regValue := cfg.Queues.QueueMap[queueName].getQueueConfig().FetchRegister(paramName)
			if regValue != nil {
				value = registerValueToString(regValue)
			} else {
				// There is a chance the queue pre-dated the existence of the given parameter. If so, use the
				// configured default value for now
				// TODO - Need to backfill missing params when they're detected
				value = DEFAULT_SETTINGS[paramName]
			}
		}
	}

	if value == "" {
		// Read from riak
		client := cfg.RiakConnection()
		bucket, _ := client.NewBucketType("maps", CONFIGURATION_BUCKET)
		obj, err := bucket.FetchMap(queueConfigRecordName(queueName))

		// if not found... no config existed for that queue - should not happen hashtagcrossfingers
		if err == riak.NotFound {
			// Log out an error here
			return "", err
		}

		val := obj.FetchRegister(paramName)

		if val != nil {
			// We had a register with this name, return the value
			value, err = registerValueToString(val)
		}
	}
	return value, err
}

// TODO Find a proper way to scope this to a queue VS a topic
func (cfg *Config) setQueueSetting(paramName string, queueName string, value string) error {
	// Write to Riak
	client := cfg.RiakConnection()
	bucket, _ := client.NewBucketType("maps", CONFIGURATION_BUCKET)
	obj, err := bucket.FetchMap(queueConfigRecordName(queueName))
	// if not found... no config existed for that queue - should not happen hashtagcrossfingers
	if err == riak.NotFound {
		// Log out an error here
		return err
	}
	val := obj.AddRegister(paramName)
	val.NewValue = []byte(value)
	// Write to Riak
	return obj.Store()
}

// HELPERS

func registerValueToString(reg *riak.RDtRegister) (string, error) {
	// The register might have been deleted at this point, so handle nil case.
	if reg == nil {
		return "", errors.New("Register is nil.")
	}
	return string(reg.Value[:]), nil
}

func (cfg *Config) RiakConnection() *riak.Client {
	return cfg.RiakPool
}

func queueConfigRecordName(queueName string) string {
	return fmt.Sprintf("queue_%s_config", queueName)
}
<|endoftext|>


<commit_before>package app

import (
	"fmt"
	"os"
	_ "strconv"
	"strings"

	car "github.com/mohae/carchivum"
	"github.com/mohae/contour"
	"github.com/mohae/magicnum"
	jww "github.com/spf13/jwalterweatherman"
)

func Create(dst string, sources ...string) (string, error) {
	var err error
	var message string
	fmt.Printf("\nCreate %q from %v\n", dst, sources)
	switch contour.GetString(Format) {
	case "zip":
		message, err = createZip(dst, sources...)
	default:
		message, err = createTar(dst, sources...)
	}
	if err != nil {
		jww.ERROR.Print(err)
		return "", err
	}
	return message, nil
}

func createZip(dst string, sources ...string) (string, error) {
	jww.INFO.Printf("Creating zip: %s from %s", dst, sources)
	zipper := car.NewZip(dst)
	zipper.UseFullpath = contour.GetBool("usefullpath")
	_, err := zipper.Create(sources...)
	if err != nil {
		jww.ERROR.Print(err)
		return "", err
	}
	return zipper.Message(), nil
}

func createTar(dst string, sources ...string) (string, error) {
	jww.INFO.Printf("Creating tar: %s from %s", dst, sources)
	tballer := car.NewTar(dst)
	t := contour.GetString("type")
	if t != "" {
		f := magicnum.FormatFromString(t)
		if f == magicnum.Unknown {
			err := fmt.Errorf("unknown format: %s", t)
			jww.ERROR.Print(err)
			return "", err
		}
		if !car.IsSupported(f) {
			err := fmt.Errorf("unsupported format: %s is not supported", f)
			jww.ERROR.Print(err)
			return "", err
		}
		tballer.Format = f
	}
	tballer.Owner = contour.GetInt("owner")
	tballer.Group = contour.GetInt("group")
	tballer.FileMode = os.FileMode(contour.GetInt64("mode"))
	//	tabller.Exclude = contour.GetString("exclude")
	tballer.ExcludeAnchored = contour.GetString("exclude-anchored")
	temp := contour.GetString("exclude-ext")
	if temp != "" {
		tballer.ExcludeExt = strings.Split(temp, ",")
		tballer.ExcludeExtCount = len(tballer.ExcludeExt)
	}
	tballer.IncludeAnchored = contour.GetString("include-anchored")
	temp = contour.GetString("include-ext")
	if temp != "" {
		tballer.IncludeExt = strings.Split(temp, ",")
		tballer.IncludeExtCount = len(tballer.IncludeExt)
	}
	// TODO figure out how to convert the incoming time info to time.Time
	// tballer.NewerMTime = contour.GetTime("newer-mtime")
	//	tballer.UseFullpath = contour.GetBool("usefullpath")
	_, err := tballer.Create(sources...)
	if err != nil {
		jww.ERROR.Print(err)
		return "", err
	}
	return tballer.Message(), nil
}
<commit_msg>magicnum -> magicnum/mcompress<commit_after>package app

import (
	"fmt"
	"os"
	_ "strconv"
	"strings"

	car "github.com/mohae/carchivum"
	"github.com/mohae/contour"
	magicnum "github.com/mohae/magicnum/mcompress"
	jww "github.com/spf13/jwalterweatherman"
)

func Create(dst string, sources ...string) (string, error) {
	var err error
	var message string
	fmt.Printf("\nCreate %q from %v\n", dst, sources)
	switch contour.GetString(Format) {
	case "zip":
		message, err = createZip(dst, sources...)
	default:
		message, err = createTar(dst, sources...)
	}
	if err != nil {
		jww.ERROR.Print(err)
		return "", err
	}
	return message, nil
}

func createZip(dst string, sources ...string) (string, error) {
	jww.INFO.Printf("Creating zip: %s from %s", dst, sources)
	zipper := car.NewZip(dst)
	zipper.UseFullpath = contour.GetBool("usefullpath")
	_, err := zipper.Create(sources...)
	if err != nil {
		jww.ERROR.Print(err)
		return "", err
	}
	return zipper.Message(), nil
}

func createTar(dst string, sources ...string) (string, error) {
	jww.INFO.Printf("Creating tar: %s from %s", dst, sources)
	tballer := car.NewTar(dst)
	t := contour.GetString("type")
	if t != "" {
		f := magicnum.FormatFromString(t)
		if f == magicnum.Unknown {
			err := fmt.Errorf("unknown format: %s", t)
			jww.ERROR.Print(err)
			return "", err
		}
		if !car.IsSupported(f) {
			err := fmt.Errorf("unsupported format: %s is not supported", f)
			jww.ERROR.Print(err)
			return "", err
		}
		tballer.Format = f
	}
	tballer.Owner = contour.GetInt("owner")
	tballer.Group = contour.GetInt("group")
	tballer.FileMode = os.FileMode(contour.GetInt64("mode"))
	//	tabller.Exclude = contour.GetString("exclude")
	tballer.ExcludeAnchored = contour.GetString("exclude-anchored")
	temp := contour.GetString("exclude-ext")
	if temp != "" {
		tballer.ExcludeExt = strings.Split(temp, ",")
		tballer.ExcludeExtCount = len(tballer.ExcludeExt)
	}
	tballer.IncludeAnchored = contour.GetString("include-anchored")
	temp = contour.GetString("include-ext")
	if temp != "" {
		tballer.IncludeExt = strings.Split(temp, ",")
		tballer.IncludeExtCount = len(tballer.IncludeExt)
	}
	// TODO figure out how to convert the incoming time info to time.Time
	// tballer.NewerMTime = contour.GetTime("newer-mtime")
	//	tballer.UseFullpath = contour.GetBool("usefullpath")
	_, err := tballer.Create(sources...)
	if err != nil {
		jww.ERROR.Print(err)
		return "", err
	}
	return tballer.Message(), nil
}
<|endoftext|>


<commit_before>package app

import (
	"errors"
	"fmt"
	"sync"
	"time"

	"github.com/koding/klient/Godeps/_workspace/src/github.com/koding/kite"
	"github.com/koding/klient/Godeps/_workspace/src/github.com/koding/kite/config"
	"github.com/koding/klient/collaboration"
	"github.com/koding/klient/command"
	"github.com/koding/klient/docker"
	"github.com/koding/klient/fs"
	"github.com/koding/klient/terminal"
	"github.com/koding/klient/usage"
)

var (
	// we also could use an atomic boolean this is simple for now.
	updating   = false
	updatingMu sync.Mutex // protects updating
)

// Klient is the central app which provides all available methods.
type Klient struct {
	// kite defines the server running and powering Klient.
	kite *kite.Kite

	// collab is used to store authorized third party users and allows them to
	// use the available methods. It provides methods to add or remove users
	// from the storage
	collab *collaboration.Collaboration

	// terminal provides wmethods
	terminal *terminal.Terminal

	// doccker provides the docker related methods.
	docker *docker.Docker

	// usage counts and tracks all called metrics. It also provides a method
	// that return those informations
	usage *usage.Usage

	log kite.Logger

	// disconnectTimer is used track disconnected users and eventually remove
	// them from the collaboration storage.
	disconnectTimer *time.Timer

	// config stores all necessary configuration needed for Klient to work.
	// It's supplied with the NewKlient() function.
	config *KlientConfig
}

// KlientConfig defines a Klient's config
type KlientConfig struct {
	Name    string
	Version string

	IP          string
	Port        int
	Environment string
	Region      string
	RegisterURL string
	Debug       bool

	UpdateInterval time.Duration
	UpdateURL      string
}

// NewKlient returns a new Klient instance
func NewKlient(conf *KlientConfig) *Klient {
	// this is our main reference to count and measure metrics for the klient
	// we count only those methods, please add/remove methods here that will
	// reset the timer of a klient.
	usg := usage.NewUsage(map[string]bool{
		"fs.readDirectory":    true,
		"fs.glob":             true,
		"fs.readFile":         true,
		"fs.writeFile":        true,
		"fs.uniquePath":       true,
		"fs.getInfo":          true,
		"fs.setPermissions":   true,
		"fs.remove":           true,
		"fs.rename":           true,
		"fs.createDirectory":  true,
		"fs.move":             true,
		"fs.copy":             true,
		"webterm.getSessions": true,
		"webterm.connect":     true,
		"webterm.killSession": true,
		"exec":                true,
		"klient.share":        true,
		"klient.unshare":      true,
		"klient.shared":       true,
		"docker.create":       true,
		"docker.connect":      true,
		"docker.stop":         true,
		"docker.start":        true,
		"docker.remove":       true,
		"docker.list":         true,
	})

	k := newKite(conf)
	term := terminal.New(k.Log)
	term.InputHook = usg.Reset

	kl := &Klient{
		kite:     k,
		collab:   collaboration.New(),
		docker:   docker.New("unix://var/run/docker.sock", k.Log),
		terminal: term,
		usage:    usg,
		log:      k.Log,
		config:   conf,
	}

	// This is important, don't forget it
	kl.RegisterMethods()

	return kl
}

// RegisterMethods registers all public available methods
func (k *Klient) RegisterMethods() {
	// don't allow anyone to call a method if we are during an update.
	k.kite.PreHandleFunc(func(r *kite.Request) (interface{}, error) {
		// Koding (kloud) connects to much, don't display it.
		if r.Username != "koding" {
			k.log.Info("Kite '%s/%s/%s' called method: '%s'",
				r.Username, r.Client.Environment, r.Client.Name, r.Method)
		}

		updatingMu.Lock()
		defer updatingMu.Unlock()

		if updating {
			return nil, errors.New("Updating klient. Can't accept any method.")
		}

		return true, nil
	})

	k.kite.PreHandleFunc(k.checkAuth)

	// Metrics, is used by Kloud to get usage so Kloud can stop free VMs
	k.kite.PreHandleFunc(k.usage.Counter) // we measure every incoming request
	k.kite.HandleFunc("klient.usage", k.usage.Current)

	// Collaboration, is used by our Koding.com browser client.
	k.kite.HandleFunc("klient.share", k.collab.Share)
	k.kite.HandleFunc("klient.unshare", k.collab.Unshare)
	k.kite.HandleFunc("klient.shared", k.collab.Shared)

	// Filesystem
	k.kite.HandleFunc("fs.readDirectory", fs.ReadDirectory)
	k.kite.HandleFunc("fs.glob", fs.Glob)
	k.kite.HandleFunc("fs.readFile", fs.ReadFile)
	k.kite.HandleFunc("fs.writeFile", fs.WriteFile)
	k.kite.HandleFunc("fs.uniquePath", fs.UniquePath)
	k.kite.HandleFunc("fs.getInfo", fs.GetInfo)
	k.kite.HandleFunc("fs.setPermissions", fs.SetPermissions)
	k.kite.HandleFunc("fs.remove", fs.Remove)
	k.kite.HandleFunc("fs.rename", fs.Rename)
	k.kite.HandleFunc("fs.createDirectory", fs.CreateDirectory)
	k.kite.HandleFunc("fs.move", fs.Move)
	k.kite.HandleFunc("fs.copy", fs.Copy)

	// Docker
	k.kite.HandleFunc("docker.create", k.docker.Create)
	k.kite.HandleFunc("docker.connect", k.docker.Connect)
	k.kite.HandleFunc("docker.stop", k.docker.Stop)
	k.kite.HandleFunc("docker.start", k.docker.Start)
	k.kite.HandleFunc("docker.remove", k.docker.RemoveContainer)
	k.kite.HandleFunc("docker.list", k.docker.List)

	// Execution
	k.kite.HandleFunc("exec", command.Exec)

	// Terminal
	k.kite.HandleFunc("webterm.getSessions", k.terminal.GetSessions)
	k.kite.HandleFunc("webterm.connect", k.terminal.Connect)
	k.kite.HandleFunc("webterm.killSession", k.terminal.KillSession)
	k.kite.HandleFunc("webterm.killSessions", k.terminal.KillSessions)

	k.kite.OnFirstRequest(func(c *kite.Client) {
		// Koding (kloud) connects to much, don't display it.
		if c.Username != "koding" {
			k.log.Info("Kite '%s/%s/%s' is connected", c.Username, c.Environment, c.Name)
		}

		if c.Username != k.kite.Config.Username {
			return // we don't care for others
		}

		// it's still not initialized, so don't do anything
		if k.disconnectTimer != nil {
			// stop previously started disconnect timer.
			k.log.Info("Disconnection timer is cancelled.")
			k.disconnectTimer.Stop()
		}

	})

	// Unshare collab users if the klient owner disconnects
	k.kite.OnDisconnect(func(c *kite.Client) {
		// Koding (kloud) connects to much, don't display it.
		if c.Username != "koding" {
			k.log.Info("Kite '%s/%s/%s' is disconnected", c.Username, c.Environment, c.Name)
		}

		if c.Username != k.kite.Config.Username {
			return // we don't care for others
		}

		// if there is any previously created timers stop them so we don't leak
		// goroutines
		if k.disconnectTimer != nil {
			k.disconnectTimer.Stop()
		}

		k.log.Info("Disconnection timer of 1 minutes is fired.")
		k.disconnectTimer = time.NewTimer(time.Minute * 1)

		// Close all active sessions of the current. Do not close it
		// immediately, instead of give some time so users can safely exit. If
		// the user reconnects again the timer will be stopped so we don't
		// unshare for network hiccups accidentally.
		go func() {
			select {
			case <-k.disconnectTimer.C:
				sharedUsers, err := k.collab.GetAll()
				if err != nil {
					k.log.Warning("Couldn't unshare users: '%s'", err)
					return
				}

				if len(sharedUsers) == 0 {
					return // nothing to do ...
				}

				k.log.Info("Unsharing users '%s'", sharedUsers)
				for user, option := range sharedUsers {
					// dont touch permanent users
					if option.Permanent {
						k.log.Info("User is permanent, avoiding it: '%s'", user)
						continue
					}

					if err := k.collab.Delete(user); err != nil {
						k.log.Warning("Couldn't delete user from storage: '%s'", err)
					}
					k.terminal.CloseSessions(user)
				}
			}
		}()
	})
}

// Run registers klient to Kontrol and starts the kite server. It also runs any
// necessary workers in the background.
func (k *Klient) Run() {
	k.startUpdater()

	if err := k.register(); err != nil {
		panic(err)
	}

	k.kite.Run()
}

func (k *Klient) startUpdater() {
	if k.config.UpdateInterval < time.Minute {
		k.log.Warning("Update interval can't be less than one minute. Setting to one minute.")
		k.config.UpdateInterval = time.Minute
	}

	// start our updater in the background
	updater := &Updater{
		Endpoint:       k.config.UpdateURL,
		Interval:       k.config.UpdateInterval,
		CurrentVersion: k.config.Version,
		Log:            k.log,
	}
	go updater.Run()
}

func (k *Klient) Close() {
	k.collab.Close()
	k.kite.Close()
}

func newKite(kconf *KlientConfig) *kite.Kite {
	k := kite.New(kconf.Name, kconf.Version)

	if kconf.Debug {
		k.SetLogLevel(kite.DEBUG)
	}

	conf := config.MustGet()
	k.Config = conf
	k.Config.Port = kconf.Port
	k.Config.Environment = kconf.Environment
	k.Config.Region = kconf.Region
	k.Id = conf.Id // always boot up with the same id in the kite.key
	return k
}

// checkAuth checks whether the given incoming request is authenticated or not.
// It don't pass any request if the caller is outside of our scope.
func (k *Klient) checkAuth(r *kite.Request) (interface{}, error) {
	if r.Auth != nil {
		return true, nil
	}

	// lazy return for those, no need to fetch from the DB
	if userIn(r.Username, []string{k.kite.Config.Username, "koding"}...) {
		return true, nil
	}

	// Allow collaboration users as well
	sharedUsers, err := k.collab.GetAll()
	if err != nil {
		return nil, fmt.Errorf("Can't read shared users from the storage. Err: %v", err)
	}

	sharedUsernames := make([]string, 0)
	for username := range sharedUsers {
		sharedUsernames = append(sharedUsernames, username)
	}

	if !userIn(r.Username, sharedUsernames...) {
		return nil, fmt.Errorf("User '%s' is not allowed to make a call to us.", r.Username)
	}

	return true, nil
}

// userIn checks whether the given user exists in the users list or not. It
// returns true if the user exists.
func userIn(user string, users ...string) bool {
	for _, u := range users {
		if u == user {
			return true
		}
	}
	return false
}
<commit_msg>klient: add webterm.rename method to the list of methods<commit_after>package app

import (
	"errors"
	"fmt"
	"sync"
	"time"

	"github.com/koding/klient/Godeps/_workspace/src/github.com/koding/kite"
	"github.com/koding/klient/Godeps/_workspace/src/github.com/koding/kite/config"
	"github.com/koding/klient/collaboration"
	"github.com/koding/klient/command"
	"github.com/koding/klient/docker"
	"github.com/koding/klient/fs"
	"github.com/koding/klient/terminal"
	"github.com/koding/klient/usage"
)

var (
	// we also could use an atomic boolean this is simple for now.
	updating   = false
	updatingMu sync.Mutex // protects updating
)

// Klient is the central app which provides all available methods.
type Klient struct {
	// kite defines the server running and powering Klient.
	kite *kite.Kite

	// collab is used to store authorized third party users and allows them to
	// use the available methods. It provides methods to add or remove users
	// from the storage
	collab *collaboration.Collaboration

	// terminal provides wmethods
	terminal *terminal.Terminal

	// doccker provides the docker related methods.
	docker *docker.Docker

	// usage counts and tracks all called metrics. It also provides a method
	// that return those informations
	usage *usage.Usage

	log kite.Logger

	// disconnectTimer is used track disconnected users and eventually remove
	// them from the collaboration storage.
	disconnectTimer *time.Timer

	// config stores all necessary configuration needed for Klient to work.
	// It's supplied with the NewKlient() function.
	config *KlientConfig
}

// KlientConfig defines a Klient's config
type KlientConfig struct {
	Name    string
	Version string

	IP          string
	Port        int
	Environment string
	Region      string
	RegisterURL string
	Debug       bool

	UpdateInterval time.Duration
	UpdateURL      string
}

// NewKlient returns a new Klient instance
func NewKlient(conf *KlientConfig) *Klient {
	// this is our main reference to count and measure metrics for the klient
	// we count only those methods, please add/remove methods here that will
	// reset the timer of a klient.
	usg := usage.NewUsage(map[string]bool{
		"fs.readDirectory":    true,
		"fs.glob":             true,
		"fs.readFile":         true,
		"fs.writeFile":        true,
		"fs.uniquePath":       true,
		"fs.getInfo":          true,
		"fs.setPermissions":   true,
		"fs.remove":           true,
		"fs.rename":           true,
		"fs.createDirectory":  true,
		"fs.move":             true,
		"fs.copy":             true,
		"webterm.getSessions": true,
		"webterm.connect":     true,
		"webterm.killSession": true,
		"exec":                true,
		"klient.share":        true,
		"klient.unshare":      true,
		"klient.shared":       true,
		"docker.create":       true,
		"docker.connect":      true,
		"docker.stop":         true,
		"docker.start":        true,
		"docker.remove":       true,
		"docker.list":         true,
	})

	k := newKite(conf)
	term := terminal.New(k.Log)
	term.InputHook = usg.Reset

	kl := &Klient{
		kite:     k,
		collab:   collaboration.New(),
		docker:   docker.New("unix://var/run/docker.sock", k.Log),
		terminal: term,
		usage:    usg,
		log:      k.Log,
		config:   conf,
	}

	// This is important, don't forget it
	kl.RegisterMethods()

	return kl
}

// RegisterMethods registers all public available methods
func (k *Klient) RegisterMethods() {
	// don't allow anyone to call a method if we are during an update.
	k.kite.PreHandleFunc(func(r *kite.Request) (interface{}, error) {
		// Koding (kloud) connects to much, don't display it.
		if r.Username != "koding" {
			k.log.Info("Kite '%s/%s/%s' called method: '%s'",
				r.Username, r.Client.Environment, r.Client.Name, r.Method)
		}

		updatingMu.Lock()
		defer updatingMu.Unlock()

		if updating {
			return nil, errors.New("Updating klient. Can't accept any method.")
		}

		return true, nil
	})

	k.kite.PreHandleFunc(k.checkAuth)

	// Metrics, is used by Kloud to get usage so Kloud can stop free VMs
	k.kite.PreHandleFunc(k.usage.Counter) // we measure every incoming request
	k.kite.HandleFunc("klient.usage", k.usage.Current)

	// Collaboration, is used by our Koding.com browser client.
	k.kite.HandleFunc("klient.share", k.collab.Share)
	k.kite.HandleFunc("klient.unshare", k.collab.Unshare)
	k.kite.HandleFunc("klient.shared", k.collab.Shared)

	// Filesystem
	k.kite.HandleFunc("fs.readDirectory", fs.ReadDirectory)
	k.kite.HandleFunc("fs.glob", fs.Glob)
	k.kite.HandleFunc("fs.readFile", fs.ReadFile)
	k.kite.HandleFunc("fs.writeFile", fs.WriteFile)
	k.kite.HandleFunc("fs.uniquePath", fs.UniquePath)
	k.kite.HandleFunc("fs.getInfo", fs.GetInfo)
	k.kite.HandleFunc("fs.setPermissions", fs.SetPermissions)
	k.kite.HandleFunc("fs.remove", fs.Remove)
	k.kite.HandleFunc("fs.rename", fs.Rename)
	k.kite.HandleFunc("fs.createDirectory", fs.CreateDirectory)
	k.kite.HandleFunc("fs.move", fs.Move)
	k.kite.HandleFunc("fs.copy", fs.Copy)

	// Docker
	k.kite.HandleFunc("docker.create", k.docker.Create)
	k.kite.HandleFunc("docker.connect", k.docker.Connect)
	k.kite.HandleFunc("docker.stop", k.docker.Stop)
	k.kite.HandleFunc("docker.start", k.docker.Start)
	k.kite.HandleFunc("docker.remove", k.docker.RemoveContainer)
	k.kite.HandleFunc("docker.list", k.docker.List)

	// Execution
	k.kite.HandleFunc("exec", command.Exec)

	// Terminal
	k.kite.HandleFunc("webterm.getSessions", k.terminal.GetSessions)
	k.kite.HandleFunc("webterm.connect", k.terminal.Connect)
	k.kite.HandleFunc("webterm.killSession", k.terminal.KillSession)
	k.kite.HandleFunc("webterm.killSessions", k.terminal.KillSessions)
	k.kite.HandleFunc("webterm.rename", k.terminal.RenameSession)

	k.kite.OnFirstRequest(func(c *kite.Client) {
		// Koding (kloud) connects to much, don't display it.
		if c.Username != "koding" {
			k.log.Info("Kite '%s/%s/%s' is connected", c.Username, c.Environment, c.Name)
		}

		if c.Username != k.kite.Config.Username {
			return // we don't care for others
		}

		// it's still not initialized, so don't do anything
		if k.disconnectTimer != nil {
			// stop previously started disconnect timer.
			k.log.Info("Disconnection timer is cancelled.")
			k.disconnectTimer.Stop()
		}

	})

	// Unshare collab users if the klient owner disconnects
	k.kite.OnDisconnect(func(c *kite.Client) {
		// Koding (kloud) connects to much, don't display it.
		if c.Username != "koding" {
			k.log.Info("Kite '%s/%s/%s' is disconnected", c.Username, c.Environment, c.Name)
		}

		if c.Username != k.kite.Config.Username {
			return // we don't care for others
		}

		// if there is any previously created timers stop them so we don't leak
		// goroutines
		if k.disconnectTimer != nil {
			k.disconnectTimer.Stop()
		}

		k.log.Info("Disconnection timer of 1 minutes is fired.")
		k.disconnectTimer = time.NewTimer(time.Minute * 1)

		// Close all active sessions of the current. Do not close it
		// immediately, instead of give some time so users can safely exit. If
		// the user reconnects again the timer will be stopped so we don't
		// unshare for network hiccups accidentally.
		go func() {
			select {
			case <-k.disconnectTimer.C:
				sharedUsers, err := k.collab.GetAll()
				if err != nil {
					k.log.Warning("Couldn't unshare users: '%s'", err)
					return
				}

				if len(sharedUsers) == 0 {
					return // nothing to do ...
				}

				k.log.Info("Unsharing users '%s'", sharedUsers)
				for user, option := range sharedUsers {
					// dont touch permanent users
					if option.Permanent {
						k.log.Info("User is permanent, avoiding it: '%s'", user)
						continue
					}

					if err := k.collab.Delete(user); err != nil {
						k.log.Warning("Couldn't delete user from storage: '%s'", err)
					}
					k.terminal.CloseSessions(user)
				}
			}
		}()
	})
}

// Run registers klient to Kontrol and starts the kite server. It also runs any
// necessary workers in the background.
func (k *Klient) Run() {
	k.startUpdater()

	if err := k.register(); err != nil {
		panic(err)
	}

	k.kite.Run()
}

func (k *Klient) startUpdater() {
	if k.config.UpdateInterval < time.Minute {
		k.log.Warning("Update interval can't be less than one minute. Setting to one minute.")
		k.config.UpdateInterval = time.Minute
	}

	// start our updater in the background
	updater := &Updater{
		Endpoint:       k.config.UpdateURL,
		Interval:       k.config.UpdateInterval,
		CurrentVersion: k.config.Version,
		Log:            k.log,
	}
	go updater.Run()
}

func (k *Klient) Close() {
	k.collab.Close()
	k.kite.Close()
}

func newKite(kconf *KlientConfig) *kite.Kite {
	k := kite.New(kconf.Name, kconf.Version)

	if kconf.Debug {
		k.SetLogLevel(kite.DEBUG)
	}

	conf := config.MustGet()
	k.Config = conf
	k.Config.Port = kconf.Port
	k.Config.Environment = kconf.Environment
	k.Config.Region = kconf.Region
	k.Id = conf.Id // always boot up with the same id in the kite.key
	return k
}

// checkAuth checks whether the given incoming request is authenticated or not.
// It don't pass any request if the caller is outside of our scope.
func (k *Klient) checkAuth(r *kite.Request) (interface{}, error) {
	if r.Auth != nil {
		return true, nil
	}

	// lazy return for those, no need to fetch from the DB
	if userIn(r.Username, []string{k.kite.Config.Username, "koding"}...) {
		return true, nil
	}

	// Allow collaboration users as well
	sharedUsers, err := k.collab.GetAll()
	if err != nil {
		return nil, fmt.Errorf("Can't read shared users from the storage. Err: %v", err)
	}

	sharedUsernames := make([]string, 0)
	for username := range sharedUsers {
		sharedUsernames = append(sharedUsernames, username)
	}

	if !userIn(r.Username, sharedUsernames...) {
		return nil, fmt.Errorf("User '%s' is not allowed to make a call to us.", r.Username)
	}

	return true, nil
}

// userIn checks whether the given user exists in the users list or not. It
// returns true if the user exists.
func userIn(user string, users ...string) bool {
	for _, u := range users {
		if u == user {
			return true
		}
	}
	return false
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package inode

import (
	"fmt"
	"os"
	"path"
	"strings"

	"github.com/jacobsa/fuse/fuseops"
	"github.com/jacobsa/fuse/fuseutil"
	"github.com/jacobsa/gcloud/gcs"
	"github.com/jacobsa/gcloud/syncutil"
	"golang.org/x/net/context"
)

type DirInode struct {
	/////////////////////////
	// Dependencies
	/////////////////////////

	bucket gcs.Bucket

	/////////////////////////
	// Constant data
	/////////////////////////

	id           fuseops.InodeID
	implicitDirs bool

	// INVARIANT: name == "" || name[len(name)-1] == '/'
	name string

	/////////////////////////
	// Mutable state
	/////////////////////////

	// A mutex that must be held when calling certain methods. See documentation
	// for each method.
	mu syncutil.InvariantMutex

	// GUARDED_BY(mu)
	lc lookupCount
}

var _ Inode = &DirInode{}

// Create a directory inode for the root of the file system. The initial lookup
// count is zero.
func NewRootInode(
	bucket gcs.Bucket,
	implicitDirs bool) (d *DirInode) {
	d = NewDirInode(bucket, fuseops.RootInodeID, "", implicitDirs)
	return
}

// Create a directory inode for the supplied object name. The name must end
// with a slash unless this is the root directory, in which case it must be
// empty.
//
// If implicitDirs is set, LookUpChild will use ListObjects to find child
// directories that are "implicitly" defined by the existence of their own
// descendents. For example, if there is an object named "foo/bar/baz" and this
// is the directory "foo", a child directory named "bar" will be implied.
//
// The initial lookup count is zero.
//
// REQUIRES: name == "" || name[len(name)-1] == '/'
func NewDirInode(
	bucket gcs.Bucket,
	id fuseops.InodeID,
	name string,
	implicitDirs bool) (d *DirInode) {
	if name != "" && name[len(name)-1] != '/' {
		panic(fmt.Sprintf("Unexpected name: %s", name))
	}

	// Set up the struct.
	d = &DirInode{
		bucket:       bucket,
		id:           id,
		implicitDirs: implicitDirs,
		name:         name,
	}

	d.lc.Init(id)

	// Set up invariant checking.
	d.mu = syncutil.NewInvariantMutex(d.checkInvariants)

	return
}

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

func (d *DirInode) checkInvariants() {
	// INVARIANT: name == "" || name[len(name)-1] == '/'
	if !(d.name == "" || d.name[len(d.name)-1] == '/') {
		panic(fmt.Sprintf("Unexpected name: %s", d.name))
	}
}

func (d *DirInode) lookUpChildFile(
	ctx context.Context,
	name string) (o *gcs.Object, err error) {
	o, err = statObjectMayNotExist(ctx, d.bucket, d.Name()+name)
	if err != nil {
		err = fmt.Errorf("statObjectMayNotExist: %v", err)
		return
	}

	return
}

func (d *DirInode) lookUpChildDir(
	ctx context.Context,
	name string) (o *gcs.Object, err error) {
	b := syncutil.NewBundle(ctx)

	// Stat the placeholder object.
	b.Add(func(ctx context.Context) (err error) {
		o, err = statObjectMayNotExist(ctx, d.bucket, d.Name()+name+"/")
		if err != nil {
			err = fmt.Errorf("statObjectMayNotExist: %v", err)
			return
		}

		return
	})

	// If implicit directories are enabled, find out whether the child name is
	// implicitly defined.
	var implicitlyDefined bool
	if d.implicitDirs {
		b.Add(func(ctx context.Context) (err error) {
			implicitlyDefined, err = objectNamePrefixNonEmpty(
				ctx,
				d.bucket,
				d.Name()+name+"/")

			if err != nil {
				err = fmt.Errorf("objectNamePrefixNonEmpty: %v", err)
				return
			}

			return
		})
	}

	// Wait for both.
	err = b.Join()
	if err != nil {
		return
	}

	// If statting failed by the directory is implicitly defined, fake a source
	// object.
	if o == nil && implicitlyDefined {
		o = &gcs.Object{
			Name: d.Name() + name + "/",
		}
	}

	return
}

// Look up the file for a (file, dir) pair with conflicting names, overriding
// the default behavior. If the file doesn't exist, return a nil record with a
// nil error. If the directory doesn't exist, pretend the file doesn't exist.
//
// REQUIRES: strings.HasSuffix(name, ConflictingFileNameSuffix)
func (d *DirInode) lookUpConflicting(
	ctx context.Context,
	name string) (o *gcs.Object, err error) {
	strippedName := strings.TrimSuffix(name, ConflictingFileNameSuffix)

	// In order to a marked name to be accepted, we require the conflicting
	// directory to exist.
	var dir *gcs.Object
	dir, err = d.lookUpChildDir(ctx, strippedName)
	if err != nil {
		err = fmt.Errorf("lookUpChildDir for stripped name: %v", err)
		return
	}

	if dir == nil {
		return
	}

	// The directory name exists. Find the conflicting file.
	o, err = d.lookUpChildFile(ctx, strippedName)
	if err != nil {
		err = fmt.Errorf("lookUpChildFile for stripped name: %v", err)
		return
	}

	return
}

// List the supplied object name prefix to find out whether it is non-empty.
func objectNamePrefixNonEmpty(
	ctx context.Context,
	bucket gcs.Bucket,
	prefix string) (nonEmpty bool, err error) {
	req := &gcs.ListObjectsRequest{
		Prefix:     prefix,
		MaxResults: 1,
	}

	listing, err := bucket.ListObjects(ctx, req)
	if err != nil {
		err = fmt.Errorf("ListObjects: %v", err)
		return
	}

	nonEmpty = len(listing.Objects) != 0
	return
}

// Stat the object with the given name, returning (nil, nil) if the object
// doesn't exist rather than failing.
func statObjectMayNotExist(
	ctx context.Context,
	bucket gcs.Bucket,
	name string) (o *gcs.Object, err error) {
	// Call the bucket.
	req := &gcs.StatObjectRequest{
		Name: name,
	}

	o, err = bucket.StatObject(ctx, req)

	// Suppress "not found" errors.
	if _, ok := err.(*gcs.NotFoundError); ok {
		err = nil
	}

	// Annotate others.
	if err != nil {
		err = fmt.Errorf("StatObject: %v", err)
		return
	}

	return
}

////////////////////////////////////////////////////////////////////////
// Public interface
////////////////////////////////////////////////////////////////////////

func (d *DirInode) Lock() {
	d.mu.Lock()
}

func (d *DirInode) Unlock() {
	d.mu.Unlock()
}

func (d *DirInode) ID() fuseops.InodeID {
	return d.id
}

// Return the full name of the directory object in GCS, including the trailing
// slash (e.g. "foo/bar/").
func (d *DirInode) Name() string {
	return d.name
}

// LOCKS_REQUIRED(d.mu)
func (d *DirInode) IncrementLookupCount() {
	d.lc.Inc()
}

// LOCKS_REQUIRED(d.mu)
func (d *DirInode) DecrementLookupCount(n uint64) (destroy bool) {
	destroy = d.lc.Dec(n)
	return
}

// LOCKS_REQUIRED(d.mu)
func (d *DirInode) Destroy() (err error) {
	// Nothing interesting to do.
	return
}

func (d *DirInode) Attributes(
	ctx context.Context) (attrs fuseops.InodeAttributes, err error) {
	// Set up basic attributes.
	attrs = fuseops.InodeAttributes{
		Mode:  0700 | os.ModeDir,
		Nlink: 1,
	}

	return
}

// A suffix that can be used to unambiguously tag a file system name.
// (Unambiguous because U+000A is not allowed in GCS object names.) This is
// used to refer to the file in a (file, directory) pair with conflicting
// object names.
//
// See also the notes on DirInode.LookUpChild.
const ConflictingFileNameSuffix = "\n"

// Look up the direct child with the given relative name, returning a record
// for the current object of that name in the GCS bucket. If both a file and a
// directory with the given name exist, the directory is preferred. Return a
// nil record with a nil error if neither is found.
//
// Special case: if the name ends in ConflictingFileNameSuffix, we strip the
// suffix, confirm that a conflicting directory exists, then return a record
// for the file.
//
// If this inode was created with implicitDirs is set, this method will use
// ListObjects to find child directories that are "implicitly" defined by the
// existence of their own descendents. For example, if there is an object named
// "foo/bar/baz" and this is the directory "foo", a child directory named "bar"
// will be implied.
//
// No lock is required.
func (d *DirInode) LookUpChild(
	ctx context.Context,
	name string) (o *gcs.Object, err error) {
	b := syncutil.NewBundle(ctx)

	// Is this a conflict marker name?
	if strings.HasSuffix(name, ConflictingFileNameSuffix) {
		o, err = d.lookUpConflicting(ctx, name)
		return
	}

	// Stat the child as a file.
	var fileRecord *gcs.Object
	b.Add(func(ctx context.Context) (err error) {
		fileRecord, err = d.lookUpChildFile(ctx, name)
		return
	})

	// Stat the child as a directory.
	var dirRecord *gcs.Object
	b.Add(func(ctx context.Context) (err error) {
		dirRecord, err = d.lookUpChildDir(ctx, name)
		return
	})

	// Wait for both.
	err = b.Join()
	if err != nil {
		return
	}

	// Prefer directories over files.
	switch {
	case dirRecord != nil:
		o = dirRecord
	case fileRecord != nil:
		o = fileRecord
	}

	return
}

// Read some number of entries from the directory, returning a continuation
// token that can be used to pick up the read operation where it left off.
// Supply the empty token on the first call.
//
// At the end of the directory, the returned continuation token will be empty.
// Otherwise it will be non-empty. There is no guarantee about the number of
// entries returned; it may be zero even with a non-empty continuation token.
//
// The contents of the Offset and Inode fields for returned entries is
// undefined.
//
// Warning: This method always behaves as if implicit directories are enabled,
// regardless of how the inode was configured. If you want to ensure that
// directories actually exist it non-implicit mode, you must call LookUpChild
// to do so.
func (d *DirInode) ReadEntries(
	ctx context.Context,
	tok string) (entries []fuseutil.Dirent, newTok string, err error) {
	// Ask the bucket to list some objects.
	req := &gcs.ListObjectsRequest{
		Delimiter:         "/",
		Prefix:            d.Name(),
		ContinuationToken: tok,
	}

	listing, err := d.bucket.ListObjects(ctx, req)
	if err != nil {
		err = fmt.Errorf("ListObjects: %v", err)
		return
	}

	// Convert objects to entries for files.
	for _, o := range listing.Objects {
		// Skip the entry for the backing object itself, which of course has its
		// own name as a prefix but which we don't wan tto appear to contain
		// itself.
		if o.Name == d.Name() {
			continue
		}

		e := fuseutil.Dirent{
			Name: path.Base(o.Name),
			Type: fuseutil.DT_File,
		}

		entries = append(entries, e)
	}

	// Convert runs to entries for directories.
	for _, p := range listing.CollapsedRuns {
		e := fuseutil.Dirent{
			Name: path.Base(p),
			Type: fuseutil.DT_Directory,
		}

		entries = append(entries, e)
	}

	// Return an appropriate continuation token, if any.
	newTok = listing.ContinuationToken

	return
}

// Create an empty child file with the supplied (relative) name, failing if a
// backing object already exists in GCS.
//
// No lock is required.
// TODO(jacobsa): Do we really need a lock on dir inodes?
func (d *DirInode) CreateChildFile(
	ctx context.Context,
	name string) (o *gcs.Object, err error) {
	// Create an empty backing object for the child, failing if it already
	// exists.
	var precond int64
	createReq := &gcs.CreateObjectRequest{
		Name:                   path.Join(d.Name(), name),
		Contents:               strings.NewReader(""),
		GenerationPrecondition: &precond,
	}

	o, err = d.bucket.CreateObject(ctx, createReq)
	if err != nil {
		err = fmt.Errorf("CreateObject: %v", err)
		return
	}

	return
}

// Create a backing objet for a child directory with the supplied (relative)
// name, failing if a backing object already exists in GCS.
//
// No lock is required.
func (d *DirInode) CreateChildDir(
	ctx context.Context,
	name string) (o *gcs.Object, err error) {
	// Create an empty backing object for the child, failing if it already
	// exists.
	var precond int64
	createReq := &gcs.CreateObjectRequest{
		Name:                   path.Join(d.Name(), name) + "/",
		Contents:               strings.NewReader(""),
		GenerationPrecondition: &precond,
	}

	o, err = d.bucket.CreateObject(ctx, createReq)
	if err != nil {
		err = fmt.Errorf("CreateObject: %v", err)
		return
	}

	return
}
<commit_msg>Factored out common logic.<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package inode

import (
	"fmt"
	"os"
	"path"
	"strings"

	"github.com/jacobsa/fuse/fuseops"
	"github.com/jacobsa/fuse/fuseutil"
	"github.com/jacobsa/gcloud/gcs"
	"github.com/jacobsa/gcloud/syncutil"
	"golang.org/x/net/context"
)

type DirInode struct {
	/////////////////////////
	// Dependencies
	/////////////////////////

	bucket gcs.Bucket

	/////////////////////////
	// Constant data
	/////////////////////////

	id           fuseops.InodeID
	implicitDirs bool

	// INVARIANT: name == "" || name[len(name)-1] == '/'
	name string

	/////////////////////////
	// Mutable state
	/////////////////////////

	// A mutex that must be held when calling certain methods. See documentation
	// for each method.
	mu syncutil.InvariantMutex

	// GUARDED_BY(mu)
	lc lookupCount
}

var _ Inode = &DirInode{}

// Create a directory inode for the root of the file system. The initial lookup
// count is zero.
func NewRootInode(
	bucket gcs.Bucket,
	implicitDirs bool) (d *DirInode) {
	d = NewDirInode(bucket, fuseops.RootInodeID, "", implicitDirs)
	return
}

// Create a directory inode for the supplied object name. The name must end
// with a slash unless this is the root directory, in which case it must be
// empty.
//
// If implicitDirs is set, LookUpChild will use ListObjects to find child
// directories that are "implicitly" defined by the existence of their own
// descendents. For example, if there is an object named "foo/bar/baz" and this
// is the directory "foo", a child directory named "bar" will be implied.
//
// The initial lookup count is zero.
//
// REQUIRES: name == "" || name[len(name)-1] == '/'
func NewDirInode(
	bucket gcs.Bucket,
	id fuseops.InodeID,
	name string,
	implicitDirs bool) (d *DirInode) {
	if name != "" && name[len(name)-1] != '/' {
		panic(fmt.Sprintf("Unexpected name: %s", name))
	}

	// Set up the struct.
	d = &DirInode{
		bucket:       bucket,
		id:           id,
		implicitDirs: implicitDirs,
		name:         name,
	}

	d.lc.Init(id)

	// Set up invariant checking.
	d.mu = syncutil.NewInvariantMutex(d.checkInvariants)

	return
}

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

func (d *DirInode) checkInvariants() {
	// INVARIANT: name == "" || name[len(name)-1] == '/'
	if !(d.name == "" || d.name[len(d.name)-1] == '/') {
		panic(fmt.Sprintf("Unexpected name: %s", d.name))
	}
}

func (d *DirInode) lookUpChildFile(
	ctx context.Context,
	name string) (o *gcs.Object, err error) {
	o, err = statObjectMayNotExist(ctx, d.bucket, d.Name()+name)
	if err != nil {
		err = fmt.Errorf("statObjectMayNotExist: %v", err)
		return
	}

	return
}

func (d *DirInode) lookUpChildDir(
	ctx context.Context,
	name string) (o *gcs.Object, err error) {
	b := syncutil.NewBundle(ctx)

	// Stat the placeholder object.
	b.Add(func(ctx context.Context) (err error) {
		o, err = statObjectMayNotExist(ctx, d.bucket, d.Name()+name+"/")
		if err != nil {
			err = fmt.Errorf("statObjectMayNotExist: %v", err)
			return
		}

		return
	})

	// If implicit directories are enabled, find out whether the child name is
	// implicitly defined.
	var implicitlyDefined bool
	if d.implicitDirs {
		b.Add(func(ctx context.Context) (err error) {
			implicitlyDefined, err = objectNamePrefixNonEmpty(
				ctx,
				d.bucket,
				d.Name()+name+"/")

			if err != nil {
				err = fmt.Errorf("objectNamePrefixNonEmpty: %v", err)
				return
			}

			return
		})
	}

	// Wait for both.
	err = b.Join()
	if err != nil {
		return
	}

	// If statting failed by the directory is implicitly defined, fake a source
	// object.
	if o == nil && implicitlyDefined {
		o = &gcs.Object{
			Name: d.Name() + name + "/",
		}
	}

	return
}

// Look up the file for a (file, dir) pair with conflicting names, overriding
// the default behavior. If the file doesn't exist, return a nil record with a
// nil error. If the directory doesn't exist, pretend the file doesn't exist.
//
// REQUIRES: strings.HasSuffix(name, ConflictingFileNameSuffix)
func (d *DirInode) lookUpConflicting(
	ctx context.Context,
	name string) (o *gcs.Object, err error) {
	strippedName := strings.TrimSuffix(name, ConflictingFileNameSuffix)

	// In order to a marked name to be accepted, we require the conflicting
	// directory to exist.
	var dir *gcs.Object
	dir, err = d.lookUpChildDir(ctx, strippedName)
	if err != nil {
		err = fmt.Errorf("lookUpChildDir for stripped name: %v", err)
		return
	}

	if dir == nil {
		return
	}

	// The directory name exists. Find the conflicting file.
	o, err = d.lookUpChildFile(ctx, strippedName)
	if err != nil {
		err = fmt.Errorf("lookUpChildFile for stripped name: %v", err)
		return
	}

	return
}

// List the supplied object name prefix to find out whether it is non-empty.
func objectNamePrefixNonEmpty(
	ctx context.Context,
	bucket gcs.Bucket,
	prefix string) (nonEmpty bool, err error) {
	req := &gcs.ListObjectsRequest{
		Prefix:     prefix,
		MaxResults: 1,
	}

	listing, err := bucket.ListObjects(ctx, req)
	if err != nil {
		err = fmt.Errorf("ListObjects: %v", err)
		return
	}

	nonEmpty = len(listing.Objects) != 0
	return
}

// Stat the object with the given name, returning (nil, nil) if the object
// doesn't exist rather than failing.
func statObjectMayNotExist(
	ctx context.Context,
	bucket gcs.Bucket,
	name string) (o *gcs.Object, err error) {
	// Call the bucket.
	req := &gcs.StatObjectRequest{
		Name: name,
	}

	o, err = bucket.StatObject(ctx, req)

	// Suppress "not found" errors.
	if _, ok := err.(*gcs.NotFoundError); ok {
		err = nil
	}

	// Annotate others.
	if err != nil {
		err = fmt.Errorf("StatObject: %v", err)
		return
	}

	return
}

// Fail if the name already exists.
func (d *DirInode) createNewObject(
	ctx context.Context,
	name string) (o *gcs.Object, err error) {
	// Create an empty backing object for the child, failing if it already
	// exists.
	var precond int64
	createReq := &gcs.CreateObjectRequest{
		Name:                   name,
		Contents:               strings.NewReader(""),
		GenerationPrecondition: &precond,
	}

	o, err = d.bucket.CreateObject(ctx, createReq)
	if err != nil {
		err = fmt.Errorf("CreateObject: %v", err)
		return
	}

	return
}

////////////////////////////////////////////////////////////////////////
// Public interface
////////////////////////////////////////////////////////////////////////

func (d *DirInode) Lock() {
	d.mu.Lock()
}

func (d *DirInode) Unlock() {
	d.mu.Unlock()
}

func (d *DirInode) ID() fuseops.InodeID {
	return d.id
}

// Return the full name of the directory object in GCS, including the trailing
// slash (e.g. "foo/bar/").
func (d *DirInode) Name() string {
	return d.name
}

// LOCKS_REQUIRED(d.mu)
func (d *DirInode) IncrementLookupCount() {
	d.lc.Inc()
}

// LOCKS_REQUIRED(d.mu)
func (d *DirInode) DecrementLookupCount(n uint64) (destroy bool) {
	destroy = d.lc.Dec(n)
	return
}

// LOCKS_REQUIRED(d.mu)
func (d *DirInode) Destroy() (err error) {
	// Nothing interesting to do.
	return
}

func (d *DirInode) Attributes(
	ctx context.Context) (attrs fuseops.InodeAttributes, err error) {
	// Set up basic attributes.
	attrs = fuseops.InodeAttributes{
		Mode:  0700 | os.ModeDir,
		Nlink: 1,
	}

	return
}

// A suffix that can be used to unambiguously tag a file system name.
// (Unambiguous because U+000A is not allowed in GCS object names.) This is
// used to refer to the file in a (file, directory) pair with conflicting
// object names.
//
// See also the notes on DirInode.LookUpChild.
const ConflictingFileNameSuffix = "\n"

// Look up the direct child with the given relative name, returning a record
// for the current object of that name in the GCS bucket. If both a file and a
// directory with the given name exist, the directory is preferred. Return a
// nil record with a nil error if neither is found.
//
// Special case: if the name ends in ConflictingFileNameSuffix, we strip the
// suffix, confirm that a conflicting directory exists, then return a record
// for the file.
//
// If this inode was created with implicitDirs is set, this method will use
// ListObjects to find child directories that are "implicitly" defined by the
// existence of their own descendents. For example, if there is an object named
// "foo/bar/baz" and this is the directory "foo", a child directory named "bar"
// will be implied.
//
// No lock is required.
func (d *DirInode) LookUpChild(
	ctx context.Context,
	name string) (o *gcs.Object, err error) {
	b := syncutil.NewBundle(ctx)

	// Is this a conflict marker name?
	if strings.HasSuffix(name, ConflictingFileNameSuffix) {
		o, err = d.lookUpConflicting(ctx, name)
		return
	}

	// Stat the child as a file.
	var fileRecord *gcs.Object
	b.Add(func(ctx context.Context) (err error) {
		fileRecord, err = d.lookUpChildFile(ctx, name)
		return
	})

	// Stat the child as a directory.
	var dirRecord *gcs.Object
	b.Add(func(ctx context.Context) (err error) {
		dirRecord, err = d.lookUpChildDir(ctx, name)
		return
	})

	// Wait for both.
	err = b.Join()
	if err != nil {
		return
	}

	// Prefer directories over files.
	switch {
	case dirRecord != nil:
		o = dirRecord
	case fileRecord != nil:
		o = fileRecord
	}

	return
}

// Read some number of entries from the directory, returning a continuation
// token that can be used to pick up the read operation where it left off.
// Supply the empty token on the first call.
//
// At the end of the directory, the returned continuation token will be empty.
// Otherwise it will be non-empty. There is no guarantee about the number of
// entries returned; it may be zero even with a non-empty continuation token.
//
// The contents of the Offset and Inode fields for returned entries is
// undefined.
//
// Warning: This method always behaves as if implicit directories are enabled,
// regardless of how the inode was configured. If you want to ensure that
// directories actually exist it non-implicit mode, you must call LookUpChild
// to do so.
func (d *DirInode) ReadEntries(
	ctx context.Context,
	tok string) (entries []fuseutil.Dirent, newTok string, err error) {
	// Ask the bucket to list some objects.
	req := &gcs.ListObjectsRequest{
		Delimiter:         "/",
		Prefix:            d.Name(),
		ContinuationToken: tok,
	}

	listing, err := d.bucket.ListObjects(ctx, req)
	if err != nil {
		err = fmt.Errorf("ListObjects: %v", err)
		return
	}

	// Convert objects to entries for files.
	for _, o := range listing.Objects {
		// Skip the entry for the backing object itself, which of course has its
		// own name as a prefix but which we don't wan tto appear to contain
		// itself.
		if o.Name == d.Name() {
			continue
		}

		e := fuseutil.Dirent{
			Name: path.Base(o.Name),
			Type: fuseutil.DT_File,
		}

		entries = append(entries, e)
	}

	// Convert runs to entries for directories.
	for _, p := range listing.CollapsedRuns {
		e := fuseutil.Dirent{
			Name: path.Base(p),
			Type: fuseutil.DT_Directory,
		}

		entries = append(entries, e)
	}

	// Return an appropriate continuation token, if any.
	newTok = listing.ContinuationToken

	return
}

// Create an empty child file with the supplied (relative) name, failing if a
// backing object already exists in GCS.
//
// No lock is required.
// TODO(jacobsa): Do we really need a lock on dir inodes?
func (d *DirInode) CreateChildFile(
	ctx context.Context,
	name string) (o *gcs.Object, err error) {
	o, err = d.createNewObject(ctx, path.Join(d.Name(), name))
	if err != nil {
		return
	}

	return
}

// Create a backing objet for a child directory with the supplied (relative)
// name, failing if a backing object already exists in GCS.
//
// No lock is required.
func (d *DirInode) CreateChildDir(
	ctx context.Context,
	name string) (o *gcs.Object, err error) {
	o, err = d.createNewObject(ctx, path.Join(d.Name(), name)+"/")
	if err != nil {
		return
	}

	return
}
<|endoftext|>


<commit_before>package resource

import (
	"errors"
	"reflect"

	"github.com/qor/qor"
)

var (
	ErrProcessorRecordNotFound = errors.New("resource: record not found")
	ErrProcessorSkipLeft       = errors.New("resource: skip left")
)

type processor struct {
	Result     interface{}
	Resource   Resourcer
	Context    *qor.Context
	MetaValues *MetaValues
	SkipLeft   bool
}

func DecodeToResource(res Resourcer, result interface{}, metaValues *MetaValues, context *qor.Context) *processor {
	return &processor{Resource: res, Result: result, Context: context, MetaValues: metaValues}
}

func (processor *processor) checkSkipLeft(errs ...error) bool {
	if processor.SkipLeft {
		return true
	}

	for _, err := range errs {
		if err == ErrProcessorSkipLeft {
			processor.SkipLeft = true
			break
		}
	}
	return processor.SkipLeft
}

func (processor *processor) Initialize() error {
	err := processor.Resource.CallFindOne(processor.Result, processor.MetaValues, processor.Context)
	processor.checkSkipLeft(err)
	return err
}

func (processor *processor) Validate() (errors []error) {
	if processor.checkSkipLeft() {
		return
	}

	for _, fc := range processor.Resource.GetResource().validators {
		if err := fc(processor.Result, processor.MetaValues, processor.Context); err != nil {
			if processor.checkSkipLeft(err) {
				break
			}
			errors = append(errors, err)
		}
	}
	return
}

func (processor *processor) decode() (errors []error) {
	if processor.checkSkipLeft() {
		return
	}

	for _, metaValue := range processor.MetaValues.Values {
		meta := metaValue.Meta
		if meta == nil {
			continue
		}

		if metaValue.MetaValues == nil {
			if setter := meta.GetSetter(); setter != nil {
				setter(processor.Result, metaValue, processor.Context)
			}
			continue
		}

		res := metaValue.Meta.GetResource()
		if res == nil {
			continue
		}

		field := reflect.Indirect(reflect.ValueOf(processor.Result)).FieldByName(meta.GetFieldName())
		if field.Kind() == reflect.Struct {
			association := field.Addr().Interface()
			DecodeToResource(res, association, metaValue.MetaValues, processor.Context).Start()
		} else if field.Kind() == reflect.Slice {
			value := reflect.New(field.Type().Elem())
			DecodeToResource(res, value.Interface(), metaValue.MetaValues, processor.Context).Start()
			if !reflect.DeepEqual(reflect.Zero(field.Type().Elem()).Interface(), value.Elem().Interface()) {
				field.Set(reflect.Append(field, value.Elem()))
			}
		}
		processor.MetaValues.Errors = append(processor.MetaValues.Errors, metaValue.MetaValues.Errors...)
	}

	return
}

func (processor *processor) Commit() (errors []error) {
	errors = processor.decode()
	if processor.checkSkipLeft(errors...) {
		return
	}

	for _, fc := range processor.Resource.GetResource().processors {
		if err := fc(processor.Result, processor.MetaValues, processor.Context); err != nil {
			if processor.checkSkipLeft(err) {
				break
			}
			errors = append(errors, err)
		}
	}
	return
}

func (processor *processor) Start() (errors []error) {
	processor.Initialize()
	if errors = append(errors, processor.Validate()...); len(errors) == 0 {
		errors = append(errors, processor.Commit()...)
	}
	return
}
<commit_msg>Add permission check when decode resource<commit_after>package resource

import (
	"errors"
	"reflect"

	"github.com/jinzhu/gorm"
	"github.com/qor/qor"
	"github.com/qor/qor/roles"
)

var (
	ErrProcessorRecordNotFound = errors.New("resource: record not found")
	ErrProcessorSkipLeft       = errors.New("resource: skip left")
)

type processor struct {
	Result     interface{}
	Resource   Resourcer
	Context    *qor.Context
	MetaValues *MetaValues
	SkipLeft   bool
	newRecord  bool
}

func DecodeToResource(res Resourcer, result interface{}, metaValues *MetaValues, context *qor.Context) *processor {
	scope := &gorm.Scope{Value: result}
	return &processor{Resource: res, Result: result, Context: context, MetaValues: metaValues, newRecord: scope.PrimaryKeyZero()}
}

func (processor *processor) checkSkipLeft(errs ...error) bool {
	if processor.SkipLeft {
		return true
	}

	for _, err := range errs {
		if err == ErrProcessorSkipLeft {
			processor.SkipLeft = true
			break
		}
	}
	return processor.SkipLeft
}

func (processor *processor) Initialize() error {
	err := processor.Resource.CallFindOne(processor.Result, processor.MetaValues, processor.Context)
	processor.checkSkipLeft(err)
	return err
}

func (processor *processor) Validate() (errors []error) {
	if processor.checkSkipLeft() {
		return
	}

	for _, fc := range processor.Resource.GetResource().validators {
		if err := fc(processor.Result, processor.MetaValues, processor.Context); err != nil {
			if processor.checkSkipLeft(err) {
				break
			}
			errors = append(errors, err)
		}
	}
	return
}

func (processor *processor) decode() (errors []error) {
	if processor.checkSkipLeft() {
		return
	}

	for _, metaValue := range processor.MetaValues.Values {
		meta := metaValue.Meta
		if meta == nil {
			continue
		}

		if processor.newRecord && !meta.HasPermission(roles.Create, processor.Context) {
			continue
		} else if !meta.HasPermission(roles.Update, processor.Context) {
			continue
		}

		if metaValue.MetaValues == nil {
			if setter := meta.GetSetter(); setter != nil {
				setter(processor.Result, metaValue, processor.Context)
			}
			continue
		}

		res := metaValue.Meta.GetResource()
		if res == nil {
			continue
		}

		field := reflect.Indirect(reflect.ValueOf(processor.Result)).FieldByName(meta.GetFieldName())
		if field.Kind() == reflect.Struct {
			association := field.Addr().Interface()
			DecodeToResource(res, association, metaValue.MetaValues, processor.Context).Start()
		} else if field.Kind() == reflect.Slice {
			value := reflect.New(field.Type().Elem())
			DecodeToResource(res, value.Interface(), metaValue.MetaValues, processor.Context).Start()
			if !reflect.DeepEqual(reflect.Zero(field.Type().Elem()).Interface(), value.Elem().Interface()) {
				field.Set(reflect.Append(field, value.Elem()))
			}
		}
		processor.MetaValues.Errors = append(processor.MetaValues.Errors, metaValue.MetaValues.Errors...)
	}

	return
}

func (processor *processor) Commit() (errors []error) {
	errors = processor.decode()
	if processor.checkSkipLeft(errors...) {
		return
	}

	for _, fc := range processor.Resource.GetResource().processors {
		if err := fc(processor.Result, processor.MetaValues, processor.Context); err != nil {
			if processor.checkSkipLeft(err) {
				break
			}
			errors = append(errors, err)
		}
	}
	return
}

func (processor *processor) Start() (errors []error) {
	processor.Initialize()
	if errors = append(errors, processor.Validate()...); len(errors) == 0 {
		errors = append(errors, processor.Commit()...)
	}
	return
}
<|endoftext|>