<commit_before>// Copyright 2017 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
////////////////////////////////////////////////////////////////////////////////

package awskms

import (
	"encoding/csv"
	"errors"
	"fmt"
	"os"
	"regexp"
	"strings"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/kms"
	"github.com/google/tink/go/core/registry"
	"github.com/google/tink/go/tink"
)

const (
	awsPrefix = "aws-kms://"
)

var (
	errCred    = errors.New("invalid credential path")
	errBadFile = errors.New("cannot open credential path")
	errCredCSV = errors.New("malformed credential csv file")
)

// awsClient represents a client that connects to the AWS KMS backend.
type awsClient struct {
	keyURIPrefix string
	kms          *kms.KMS
}

var _ registry.KMSClient = (*awsClient)(nil)

// NewClient returns a new AWS KMS client which will use default
// credentials to handle keys with uriPrefix prefix.
// uriPrefix must have the following format: 'aws-kms://arn:aws:kms:<region>[:path]'.
func NewClient(uriPrefix string) (registry.KMSClient, error) {
	if !strings.HasPrefix(strings.ToLower(uriPrefix), awsPrefix) {
		return nil, fmt.Errorf("uriPrefix must start with %s", awsPrefix)
	}
	r, err := getRegion(uriPrefix)
	if err != nil {
		return nil, err
	}

	session := session.Must(session.NewSession(&aws.Config{
		Region: aws.String(r),
	}))

	return &awsClient{
		keyURIPrefix: uriPrefix,
		kms:          kms.New(session),
	}, nil
}

// NewClientWithCredentials returns a new AWS KMS client which will use given
// credentials to handle keys with uriPrefix prefix.
// uriPrefix must have the following format: 'aws-kms://arn:aws:kms:<region>[:path]'.
func NewClientWithCredentials(uriPrefix string, credentialPath string) (registry.KMSClient, error) {
	if !strings.HasPrefix(strings.ToLower(uriPrefix), awsPrefix) {
		return nil, fmt.Errorf("uriPrefix must start with %s", awsPrefix)
	}
	r, err := getRegion(uriPrefix)
	if err != nil {
		return nil, err
	}

	var creds *credentials.Credentials
	if len(credentialPath) == 0 {
		return nil, errCred
	}
	c, err := extractCredsCSV(credentialPath)
	switch err {
	case nil:
		creds = credentials.NewStaticCredentialsFromCreds(*c)
	case errBadFile, errCredCSV:
		return nil, err
	default:
		// fallback to load the credential path as .ini shared credentials.
		creds = credentials.NewSharedCredentials(credentialPath, "default")
	}
	session := session.Must(session.NewSession(&aws.Config{
		Credentials: creds,
		Region:      aws.String(r),
	}))

	return &awsClient{
		keyURIPrefix: uriPrefix,
		kms:          kms.New(session),
	}, nil
}

// Supported true if this client does support keyURI
func (c *awsClient) Supported(keyURI string) bool {
	return strings.HasPrefix(keyURI, c.keyURIPrefix)
}

// GetAEAD gets an AEAD backend by keyURI.
func (c *awsClient) GetAEAD(keyURI string) (tink.AEAD, error) {
	if !c.Supported(keyURI) {
		return nil, errors.New("unsupported keyURI")
	}

	uri := strings.TrimPrefix(keyURI, awsPrefix)
	return NewAWSAEAD(uri, c.kms), nil
}

func extractCredsCSV(file string) (*credentials.Value, error) {
	f, err := os.Open(file)
	if err != nil {
		return nil, errBadFile
	}
	defer f.Close()

	lines, err := csv.NewReader(f).ReadAll()
	if err != nil {
		return nil, err
	}

	// It is possible that the file is an AWS .ini credential file, and it can be
	// parsed as 1-column CSV file as well. A real AWS credentials.csv is never 1 column.
	if len(lines) > 0 && len(lines[0]) == 1 {
		return nil, errors.New("not a valid CSV credential file")
	}

	// credentials.csv can be obtained when a AWS IAM user is created through IAM console.
	// The first line of the csv file is "User name,Password,Access key ID,Secret access key,Console login link"
	// The 2nd line of it contains 5 comma separated values.
	// Parse the file with a strict format assumption as follows:
	// 1. There must be at least 4 columns and 2 rows.
	// 2. The access key id and the secret access key must be on (0-based) column 2 and 3.
	if len(lines) < 2 {
		return nil, errCredCSV
	}

	if len(lines[1]) < 4 {
		return nil, errCredCSV
	}

	return &credentials.Value{
		AccessKeyID:     lines[1][2],
		SecretAccessKey: lines[1][3],
	}, nil

}

func getRegion(keyURI string) (string, error) {
	re1, err := regexp.Compile(`aws-kms://arn:aws:kms:([a-z0-9-]+):`)
	if err != nil {
		return "", err
	}
	r := re1.FindStringSubmatch(keyURI)
	if len(r) != 2 {
		return "", errors.New("extracting region from URI failed")
	}
	return r[1], nil
}
<commit_msg>Add NewClientWithKMS for AWS KMS client. This allows users to create Tink KMS client from user constructed AWS client (e.g. fake client for testing, or client created with self-managed credential key/id).<commit_after>// Copyright 2017 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
////////////////////////////////////////////////////////////////////////////////

package awskms

import (
	"encoding/csv"
	"errors"
	"fmt"
	"os"
	"regexp"
	"strings"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/kms"
	"github.com/google/tink/go/core/registry"
	"github.com/google/tink/go/tink"
)

const (
	awsPrefix = "aws-kms://"
)

var (
	errCred    = errors.New("invalid credential path")
	errBadFile = errors.New("cannot open credential path")
	errCredCSV = errors.New("malformed credential csv file")
)

// awsClient represents a client that connects to the AWS KMS backend.
type awsClient struct {
	keyURIPrefix string
	kms          *kms.KMS
}

var _ registry.KMSClient = (*awsClient)(nil)

// NewClient returns a new AWS KMS client which will use default
// credentials to handle keys with uriPrefix prefix.
// uriPrefix must have the following format: 'aws-kms://arn:aws:kms:<region>[:path]'.
func NewClient(uriPrefix string) (registry.KMSClient, error) {
	if !strings.HasPrefix(strings.ToLower(uriPrefix), awsPrefix) {
		return nil, fmt.Errorf("uriPrefix must start with %s", awsPrefix)
	}
	r, err := getRegion(uriPrefix)
	if err != nil {
		return nil, err
	}

	session := session.Must(session.NewSession(&aws.Config{
		Region: aws.String(r),
	}))

	return &awsClient{
		keyURIPrefix: uriPrefix,
		kms:          kms.New(session),
	}, nil
}

// NewClientWithCredentials returns a new AWS KMS client which will use given
// credentials to handle keys with uriPrefix prefix.
// uriPrefix must have the following format: 'aws-kms://arn:aws:kms:<region>[:path]'.
func NewClientWithCredentials(uriPrefix string, credentialPath string) (registry.KMSClient, error) {
	if !strings.HasPrefix(strings.ToLower(uriPrefix), awsPrefix) {
		return nil, fmt.Errorf("uriPrefix must start with %s", awsPrefix)
	}
	r, err := getRegion(uriPrefix)
	if err != nil {
		return nil, err
	}

	var creds *credentials.Credentials
	if len(credentialPath) == 0 {
		return nil, errCred
	}
	c, err := extractCredsCSV(credentialPath)
	switch err {
	case nil:
		creds = credentials.NewStaticCredentialsFromCreds(*c)
	case errBadFile, errCredCSV:
		return nil, err
	default:
		// fallback to load the credential path as .ini shared credentials.
		creds = credentials.NewSharedCredentials(credentialPath, "default")
	}
	session := session.Must(session.NewSession(&aws.Config{
		Credentials: creds,
		Region:      aws.String(r),
	}))

	return &awsClient{
		keyURIPrefix: uriPrefix,
		kms:          kms.New(session),
	}, nil
}

// NewClientWithKMS returns a new AWS KMS client with user created KMS client.
// Client is responsible for keeping the region consistency between key URI and KMS client.
// uriPrefix must have the following format: 'aws-kms://arn:aws:kms:<region>[:path]'.
func NewClientWithKMS(uriPrefix string, kms *kms.KMS) (registry.KMSClient, error) {
	if !strings.HasPrefix(strings.ToLower(uriPrefix), awsPrefix) {
		return nil, fmt.Errorf("uriPrefix must start with %s", awsPrefix)
	}

	return &awsClient{
		keyURIPrefix: uriPrefix,
		kms:          kms,
	}, nil
}

// Supported true if this client does support keyURI
func (c *awsClient) Supported(keyURI string) bool {
	return strings.HasPrefix(keyURI, c.keyURIPrefix)
}

// GetAEAD gets an AEAD backend by keyURI.
func (c *awsClient) GetAEAD(keyURI string) (tink.AEAD, error) {
	if !c.Supported(keyURI) {
		return nil, errors.New("unsupported keyURI")
	}

	uri := strings.TrimPrefix(keyURI, awsPrefix)
	return NewAWSAEAD(uri, c.kms), nil
}

func extractCredsCSV(file string) (*credentials.Value, error) {
	f, err := os.Open(file)
	if err != nil {
		return nil, errBadFile
	}
	defer f.Close()

	lines, err := csv.NewReader(f).ReadAll()
	if err != nil {
		return nil, err
	}

	// It is possible that the file is an AWS .ini credential file, and it can be
	// parsed as 1-column CSV file as well. A real AWS credentials.csv is never 1 column.
	if len(lines) > 0 && len(lines[0]) == 1 {
		return nil, errors.New("not a valid CSV credential file")
	}

	// credentials.csv can be obtained when a AWS IAM user is created through IAM console.
	// The first line of the csv file is "User name,Password,Access key ID,Secret access key,Console login link"
	// The 2nd line of it contains 5 comma separated values.
	// Parse the file with a strict format assumption as follows:
	// 1. There must be at least 4 columns and 2 rows.
	// 2. The access key id and the secret access key must be on (0-based) column 2 and 3.
	if len(lines) < 2 {
		return nil, errCredCSV
	}

	if len(lines[1]) < 4 {
		return nil, errCredCSV
	}

	return &credentials.Value{
		AccessKeyID:     lines[1][2],
		SecretAccessKey: lines[1][3],
	}, nil

}

func getRegion(keyURI string) (string, error) {
	re1, err := regexp.Compile(`aws-kms://arn:aws:kms:([a-z0-9-]+):`)
	if err != nil {
		return "", err
	}
	r := re1.FindStringSubmatch(keyURI)
	if len(r) != 2 {
		return "", errors.New("extracting region from URI failed")
	}
	return r[1], nil
}
<|endoftext|>


<commit_before>package scanner

import (
	"bufio"
	"encoding/gob"
	"github.com/Symantec/Dominator/lib/filesystem"
	"github.com/Symantec/Dominator/lib/fsutil"
	"github.com/Symantec/Dominator/lib/hash"
	"io"
	"os"
	"path"
	"time"
)

type unreferencedObject struct {
	Hash   hash.Hash
	Length uint64
	Age    time.Time
}

type unreferencedObjectsEntry struct {
	object unreferencedObject
	prev   *unreferencedObjectsEntry
	next   *unreferencedObjectsEntry
}

type unreferencedObjectsList struct {
	length              uint64
	totalBytes          uint64
	oldest              *unreferencedObjectsEntry
	newest              *unreferencedObjectsEntry
	hashToEntry         map[hash.Hash]*unreferencedObjectsEntry
	lastRegeneratedTime time.Time
}

func loadUnreferencedObjects(fName string) (*unreferencedObjectsList, error) {
	file, err := os.Open(fName)
	if err != nil {
		if !os.IsNotExist(err) {
			return nil, err
		}
		return &unreferencedObjectsList{
			hashToEntry: make(map[hash.Hash]*unreferencedObjectsEntry),
		}, nil
	}
	defer file.Close()
	reader := fsutil.NewChecksumReader(bufio.NewReader(file))
	decoder := gob.NewDecoder(reader)
	var length uint64
	if err := decoder.Decode(&length); err != nil {
		return nil, err
	}
	list := &unreferencedObjectsList{
		length:      length,
		hashToEntry: make(map[hash.Hash]*unreferencedObjectsEntry, length),
	}
	for count := uint64(0); count < length; count++ {
		var object unreferencedObject
		if err := decoder.Decode(&object); err != nil {
			return nil, err
		}
		entry := &unreferencedObjectsEntry{object: object}
		list.addEntry(entry)
	}
	if err := reader.VerifyChecksum(); err != nil {
		return nil, err
	}
	return list, nil
}

func (list *unreferencedObjectsList) write(w io.Writer) error {
	writer := fsutil.NewChecksumWriter(w)
	encoder := gob.NewEncoder(writer)
	if err := encoder.Encode(list.length); err != nil {
		return err
	}
	for entry := list.oldest; entry != nil; entry = entry.next {
		if err := encoder.Encode(entry.object); err != nil {
			return err
		}
	}
	return writer.WriteChecksum()
}

func (list *unreferencedObjectsList) addEntry(entry *unreferencedObjectsEntry) {
	entry.prev = list.newest
	if list.oldest == nil {
		list.oldest = entry
	} else {
		list.newest.next = entry
	}
	list.newest = entry
	list.hashToEntry[entry.object.Hash] = entry
	list.length++
	list.totalBytes += entry.object.Length
}

func (list *unreferencedObjectsList) addObject(hashVal hash.Hash,
	length uint64) {
	if _, ok := list.hashToEntry[hashVal]; !ok {
		object := unreferencedObject{hashVal, length, time.Now()}
		list.addEntry(&unreferencedObjectsEntry{object: object})
	}
}

func (list *unreferencedObjectsList) list() map[hash.Hash]uint64 {
	objectsMap := make(map[hash.Hash]uint64, list.length)
	for entry := list.oldest; entry != nil; entry = entry.next {
		objectsMap[entry.object.Hash] = entry.object.Length
	}
	return objectsMap
}

func (list *unreferencedObjectsList) removeObject(hashVal hash.Hash) bool {
	entry := list.hashToEntry[hashVal]
	if entry == nil {
		return false
	}
	if entry.prev == nil {
		list.oldest = entry.next
	} else {
		entry.prev.next = entry.next
	}
	if entry.next == nil {
		list.newest = entry.prev
	} else {
		entry.next.prev = entry.prev
	}
	list.length--
	list.totalBytes -= entry.object.Length
	return true
}

func (imdb *ImageDataBase) maybeAddToUnreferencedObjectsList(
	inodeTable filesystem.InodeTable) {
	// First get a list of objects in the image being deleted.
	objects := make(map[hash.Hash]uint64)
	for _, inode := range inodeTable {
		if inode, ok := inode.(*filesystem.RegularInode); ok {
			objects[inode.Hash] = inode.Size
		}
	}
	// Scan all remaining images and remove their objects from the list.
	for _, image := range imdb.imageMap {
		for _, inode := range image.FileSystem.InodeTable {
			if inode, ok := inode.(*filesystem.RegularInode); ok {
				delete(objects, inode.Hash)
			}
		}
	}
	for object, size := range objects {
		imdb.unreferencedObjects.addObject(object, size)
	}
}

func (imdb *ImageDataBase) removeFromUnreferencedObjectsList(
	inodeTable filesystem.InodeTable) {
	changed := false
	for _, inode := range inodeTable {
		if inode, ok := inode.(*filesystem.RegularInode); ok {
			if imdb.unreferencedObjects.removeObject(inode.Hash) {
				changed = true
			}
		}
	}
	if changed {
		if err := imdb.writeUnreferencedObjectsList(); err != nil {
			imdb.logger.Printf("Error writing unreferenced objects list: %s\n",
				err)
		}
	}
}

func (imdb *ImageDataBase) writeUnreferencedObjectsList() error {
	filename := path.Join(imdb.baseDir, unreferencedObjectsFile)
	file, err := fsutil.CreateRenamingWriter(filename, filePerms)
	if err != nil {
		return err
	}
	defer file.Close()
	writer := bufio.NewWriter(file)
	defer writer.Flush()
	return imdb.unreferencedObjects.write(writer)
}

func (imdb *ImageDataBase) garbageCollector() {
}

func (imdb *ImageDataBase) maybeRegenerateUnreferencedObjectsList() {
	lastMutationTime := imdb.objectServer.LastMutationTime()
	if lastMutationTime.After(imdb.unreferencedObjects.lastRegeneratedTime) {
		imdb.regenerateUnreferencedObjectsList()
	}
}

func (imdb *ImageDataBase) regenerateUnreferencedObjectsList() {
	scanTime := time.Now()
	// First generate list of currently unused objects.
	objectsMap := imdb.objectServer.ListObjectSizes()
	for _, imageName := range imdb.ListImages() {
		image := imdb.GetImage(imageName)
		if image == nil {
			continue
		}
		for _, inode := range image.FileSystem.InodeTable {
			if inode, ok := inode.(*filesystem.RegularInode); ok {
				delete(objectsMap, inode.Hash)
			}
		}
	}
	// Now add unused objects to cached list.
	for hashVal, length := range objectsMap {
		imdb.unreferencedObjects.addObject(hashVal, length)
	}
	// Finally remove objects from cached list which are no longer unreferenced.
	for entry := imdb.unreferencedObjects.oldest; entry != nil; {
		hashVal := entry.object.Hash
		entry = entry.next
		if _, ok := objectsMap[hashVal]; ok {
			imdb.unreferencedObjects.removeObject(hashVal)
		}
	}
	imdb.unreferencedObjects.lastRegeneratedTime = scanTime
}
<commit_msg>Add garbageCollector() goroutine. Reduce scope of locking.<commit_after>package scanner

import (
	"bufio"
	"encoding/gob"
	"errors"
	"github.com/Symantec/Dominator/lib/filesystem"
	"github.com/Symantec/Dominator/lib/fsutil"
	"github.com/Symantec/Dominator/lib/hash"
	"io"
	"os"
	"path"
	"time"
)

type unreferencedObject struct {
	Hash   hash.Hash
	Length uint64
	Age    time.Time
}

type unreferencedObjectsEntry struct {
	object unreferencedObject
	prev   *unreferencedObjectsEntry
	next   *unreferencedObjectsEntry
}

type unreferencedObjectsList struct {
	length              uint64
	totalBytes          uint64
	oldest              *unreferencedObjectsEntry
	newest              *unreferencedObjectsEntry
	hashToEntry         map[hash.Hash]*unreferencedObjectsEntry
	lastRegeneratedTime time.Time
}

func loadUnreferencedObjects(fName string) (*unreferencedObjectsList, error) {
	file, err := os.Open(fName)
	if err != nil {
		if !os.IsNotExist(err) {
			return nil, err
		}
		return &unreferencedObjectsList{
			hashToEntry: make(map[hash.Hash]*unreferencedObjectsEntry),
		}, nil
	}
	defer file.Close()
	reader := fsutil.NewChecksumReader(bufio.NewReader(file))
	decoder := gob.NewDecoder(reader)
	var length uint64
	if err := decoder.Decode(&length); err != nil {
		return nil, errors.New("error decoding list length: " + err.Error())
	}
	list := &unreferencedObjectsList{
		hashToEntry: make(map[hash.Hash]*unreferencedObjectsEntry, length),
	}
	for count := uint64(0); count < length; count++ {
		var object unreferencedObject
		if err := decoder.Decode(&object); err != nil {
			return nil, errors.New("error decoding object: " + err.Error())
		}
		entry := &unreferencedObjectsEntry{object: object}
		list.addEntry(entry)
	}
	if err := reader.VerifyChecksum(); err != nil {
		return nil, err
	}
	return list, nil
}

func (list *unreferencedObjectsList) write(w io.Writer) error {
	writer := fsutil.NewChecksumWriter(w)
	encoder := gob.NewEncoder(writer)
	if err := encoder.Encode(list.length); err != nil {
		return err
	}
	for entry := list.oldest; entry != nil; entry = entry.next {
		if err := encoder.Encode(entry.object); err != nil {
			return err
		}
	}
	return writer.WriteChecksum()
}

func (list *unreferencedObjectsList) addEntry(entry *unreferencedObjectsEntry) {
	entry.prev = list.newest
	if list.oldest == nil {
		list.oldest = entry
	} else {
		list.newest.next = entry
	}
	list.newest = entry
	list.hashToEntry[entry.object.Hash] = entry
	list.length++
	list.totalBytes += entry.object.Length
}

func (list *unreferencedObjectsList) addObject(hashVal hash.Hash,
	length uint64) bool {
	if _, ok := list.hashToEntry[hashVal]; !ok {
		object := unreferencedObject{hashVal, length, time.Now()}
		list.addEntry(&unreferencedObjectsEntry{object: object})
		return true
	}
	return false
}

func (list *unreferencedObjectsList) list() map[hash.Hash]uint64 {
	objectsMap := make(map[hash.Hash]uint64, list.length)
	for entry := list.oldest; entry != nil; entry = entry.next {
		objectsMap[entry.object.Hash] = entry.object.Length
	}
	return objectsMap
}

func (list *unreferencedObjectsList) removeObject(hashVal hash.Hash) bool {
	entry := list.hashToEntry[hashVal]
	if entry == nil {
		return false
	}
	if entry.prev == nil {
		list.oldest = entry.next
	} else {
		entry.prev.next = entry.next
	}
	if entry.next == nil {
		list.newest = entry.prev
	} else {
		entry.next.prev = entry.prev
	}
	list.length--
	list.totalBytes -= entry.object.Length
	return true
}

func (imdb *ImageDataBase) maybeAddToUnreferencedObjectsList(
	inodeTable filesystem.InodeTable) {
	// First get a list of objects in the image being deleted.
	objects := make(map[hash.Hash]uint64)
	for _, inode := range inodeTable {
		if inode, ok := inode.(*filesystem.RegularInode); ok {
			objects[inode.Hash] = inode.Size
		}
	}
	// Scan all remaining images and remove their objects from the list.
	for _, image := range imdb.imageMap {
		for _, inode := range image.FileSystem.InodeTable {
			if inode, ok := inode.(*filesystem.RegularInode); ok {
				delete(objects, inode.Hash)
			}
		}
	}
	changed := false
	for object, size := range objects {
		if imdb.unreferencedObjects.addObject(object, size) {
			changed = true
		}
	}
	if changed {
		imdb.saveUnreferencedObjectsList()
	}
}

func (imdb *ImageDataBase) removeFromUnreferencedObjectsList(
	inodeTable filesystem.InodeTable) {
	changed := false
	for _, inode := range inodeTable {
		if inode, ok := inode.(*filesystem.RegularInode); ok {
			if imdb.unreferencedObjects.removeObject(inode.Hash) {
				changed = true
			}
		}
	}
	if changed {
		imdb.saveUnreferencedObjectsList()
	}
}

func (imdb *ImageDataBase) saveUnreferencedObjectsList() {
	if err := imdb.writeUnreferencedObjectsList(); err != nil {
		imdb.logger.Printf("Error writing unreferenced objects list: %s\n",
			err)
	}
}

func (imdb *ImageDataBase) writeUnreferencedObjectsList() error {
	filename := path.Join(imdb.baseDir, unreferencedObjectsFile)
	file, err := fsutil.CreateRenamingWriter(filename, filePerms)
	if err != nil {
		return err
	}
	defer file.Close()
	writer := bufio.NewWriter(file)
	defer writer.Flush()
	return imdb.unreferencedObjects.write(writer)
}

func (imdb *ImageDataBase) garbageCollector(bytesToDelete uint64) (
	uint64, error) {
	if bytesToDelete < 1 {
		return 0, nil
	}
	imdb.Lock()
	firstEntry := imdb.unreferencedObjects.oldest
	entry := imdb.unreferencedObjects.oldest
	var nObjects uint64
	var nBytes uint64
	for ; entry != nil && nBytes < bytesToDelete; entry = entry.next {
		nObjects++
		nBytes += entry.object.Length
	}
	imdb.unreferencedObjects.length -= nBytes
	imdb.unreferencedObjects.totalBytes -= nObjects
	imdb.unreferencedObjects.oldest = entry
	if entry == nil {
		imdb.unreferencedObjects.newest = nil
	} else if entry.prev != nil {
		entry.prev.next = nil
		entry.prev = nil
	}
	imdb.Unlock()
	if firstEntry == nil {
		return 0, nil
	}
	nBytes = 0
	var err error
	for entry := firstEntry; entry != nil; entry = entry.next {
		if e := imdb.objectServer.DeleteObject(entry.object.Hash); e != nil {
			if err == nil {
				err = e
			}
		} else {
			nBytes += entry.object.Length
		}
	}
	imdb.saveUnreferencedObjectsList()
	return nBytes, err
}

// This grabs the lock.
func (imdb *ImageDataBase) maybeRegenerateUnreferencedObjectsList() {
	imdb.RLock()
	lastRegeneratedTime := imdb.unreferencedObjects.lastRegeneratedTime
	imdb.RUnlock()
	lastMutationTime := imdb.objectServer.LastMutationTime()
	if lastMutationTime.After(lastRegeneratedTime) {
		imdb.regenerateUnreferencedObjectsList()
	}
}

// This grabs the lock.
func (imdb *ImageDataBase) regenerateUnreferencedObjectsList() {
	scanTime := time.Now()
	// First generate list of currently unused objects.
	objectsMap := imdb.objectServer.ListObjectSizes()
	imdb.Lock()
	defer imdb.Unlock()
	for _, image := range imdb.imageMap {
		for _, inode := range image.FileSystem.InodeTable {
			if inode, ok := inode.(*filesystem.RegularInode); ok {
				delete(objectsMap, inode.Hash)
			}
		}
	}
	changed := false
	// Now add unused objects to cached list.
	for hashVal, length := range objectsMap {
		if imdb.unreferencedObjects.addObject(hashVal, length) {
			changed = true
		}
	}
	// Finally remove objects from cached list which are no longer unreferenced.
	for entry := imdb.unreferencedObjects.oldest; entry != nil; {
		hashVal := entry.object.Hash
		entry = entry.next
		if _, ok := objectsMap[hashVal]; !ok {
			if imdb.unreferencedObjects.removeObject(hashVal) {
				changed = true
			}
		}
	}
	if changed {
		imdb.saveUnreferencedObjectsList()
	}
	imdb.unreferencedObjects.lastRegeneratedTime = scanTime
}
<|endoftext|>


<commit_before>// Copyright 2014, The Serviced Authors. All rights reserved.
// Use of this source code is governed by a
// license that can be found in the LICENSE file.

package elastic

import (
	"github.com/mattbaird/elastigo/api"
	"github.com/zenoss/glog"
	"github.com/zenoss/serviced/datastore"

	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"strings"
	"time"
)

//ElasticDriver describes an the Elastic Search driver
type ElasticDriver interface {
	SetProperty(name string, prop interface{}) error
	// AddMapping add a document mapping to be registered with ElasticSearch
	AddMapping(mapping Mapping) error
	//Initialize the driver, register mappings with elasticserach. Timeout in ms to wait for elastic to be available.
	Initialize(timeout time.Duration) error
	GetConnection() (datastore.Connection, error)
}

// New creates a new ElasticDriver
func New(host string, port uint16, index string) ElasticDriver {
	return newDriver(host, port, index)
}

func newDriver(host string, port uint16, index string) *elasticDriver {
	api.Domain = host
	api.Port = fmt.Sprintf("%v", port)
	//TODO: singleton since elastigo doesn't support multiple endpoints

	driver := &elasticDriver{}
	driver.host = host
	driver.port = port
	driver.index = index
	driver.settings = map[string]interface{}{"number_of_shards": 1}
	driver.mappings = make([]Mapping, 0)
	return driver
}

//Make sure elasticDriver implements datastore.Driver
var _ datastore.Driver = &elasticDriver{}

type elasticDriver struct {
	host     string
	port     uint16
	settings map[string]interface{}
	mappings []Mapping
	index    string
}

func (ed *elasticDriver) GetConnection() (datastore.Connection, error) {
	return &elasticConnection{ed.index}, nil
}

func (ed *elasticDriver) Initialize(timeout time.Duration) error {

	quit := make(chan int)
	healthy := make(chan int)

	go ed.checkHealth(quit, healthy)

	select {
	case <-healthy:
		glog.V(4).Infof("Got response from Elastic")
	case <-time.After(timeout):
		return errors.New("timed Out waiting for response from Elastic")
	}

	if err := ed.postIndex(); err != nil {
		return err
	}

	if err := ed.postMappings(); err != nil {
		return err
	}

	// postMapping and postIndex affect es health
	go ed.checkHealth(quit, healthy)

	select {
	case <-healthy:
		glog.V(4).Infof("Got response from Elastic")
	case <-time.After(timeout):
		return errors.New("timed Out waiting for response from Elastic")
	}

	return nil
}

func (ed *elasticDriver) SetProperty(name string, prop interface{}) error {
	ed.settings[name] = prop
	return nil
}

func (ed *elasticDriver) AddMapping(mapping Mapping) error {
	ed.mappings = append(ed.mappings, mapping)
	return nil
}

func (ed *elasticDriver) AddMappingsFile(path string) error {
	glog.Infof("AddMappingsFiles %v", path)

	bytes, err := ioutil.ReadFile(path)
	if err != nil {
		return err
	}
	glog.V(4).Infof("AddMappingsFiles: content %v", string(bytes))

	type mapFile struct {
		Mappings map[string]map[string]interface{}
		Settings map[string]interface{}
	}
	var allMappings mapFile
	err = json.Unmarshal(bytes, &allMappings)
	if err != nil {
		return err
	}
	for key, val := range allMappings.Settings {
		ed.settings[key] = val
	}
	for key, mapping := range allMappings.Mappings {

		var rawMapping = make(map[string]map[string]interface{})
		rawMapping[key] = mapping
		if value, err := newMapping(rawMapping); err != nil {
			glog.Errorf("%v; could not create mapping from: %v", err, rawMapping)
			return err
		} else {
			ed.AddMapping(value)
		}
	}

	return nil
}

func (ed *elasticDriver) elasticURL() string {
	return fmt.Sprintf("http://%s:%d", ed.host, ed.port)
}

func (ed *elasticDriver) indexURL() string {
	return fmt.Sprintf("%s/%s/", ed.elasticURL(), ed.index)
}

func (ed *elasticDriver) isUp() bool {
	health, err := ed.getHealth()
	if err != nil {
		glog.Errorf("isUp() err=%v", err)
		return false
	}
	status := health["status"]
	return status == "green" || status == "yellow"
}

func (ed *elasticDriver) getHealth() (map[string]interface{}, error) {
	health := make(map[string]interface{})
	healthURL := fmt.Sprintf("%v/_cluster/health", ed.elasticURL())
	resp, err := http.Get(healthURL)
	if err != nil {
		return health, err
	}
	if resp.StatusCode != 200 {
		return health, fmt.Errorf("http status: %v", resp.StatusCode)
	}

	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		glog.Errorf("error reading elastic health: %v", err)
		return health, err
	}
	if err := json.Unmarshal(body, &health); err != nil {
		glog.Errorf("error unmarshalling elastic health: %v; err: %v", string(body), err)
		return health, err
	}
	glog.V(4).Infof("Elastic Health: %v; err: %v", string(body), err)
	return health, nil

}

func (ed *elasticDriver) checkHealth(quit chan int, healthy chan int) {
	for {
		select {
		default:
			if ed.isUp() {
				healthy <- 1
				return
			}
			glog.Infof("Waiting for Elastic Search")
			time.Sleep(1000 * time.Millisecond)

		case <-quit:
			return
		}
	}

}

func (ed *elasticDriver) postMappings() error {

	post := func(typeName string, mappingBytes []byte) error {
		mapURL := fmt.Sprintf("%s/%s/_mapping", ed.indexURL(), typeName)
		glog.V(4).Infof("Posting mapping to %s", mapURL)
		resp, err := http.Post(mapURL, "application/json", bytes.NewReader(mappingBytes))
		if err != nil {
			return fmt.Errorf("error mapping %s: %s", typeName, err)
		}
		glog.V(4).Infof("Response %v", resp)
		defer resp.Body.Close()
		body, err := ioutil.ReadAll(resp.Body)
		glog.V(4).Infof("Post result %s", body)
		if err != nil {
			return err
		}
		if resp.StatusCode < 200 || resp.StatusCode >= 300 {
			return fmt.Errorf("response %d mapping %s: %s", resp.StatusCode, typeName, string(body))
		}
		return nil
	}

	for _, mapping := range ed.mappings {
		mappingBytes, err := json.Marshal(mapping)
		if err != nil {
			return err
		}

		glog.V(4).Infof("mappping %v to  %v", mapping.Name, string(mappingBytes))
		err = post(mapping.Name, mappingBytes)
		if err != nil {
			return err
		}
	}

	return nil
}

func (ed *elasticDriver) deleteIndex() error {
	url := ed.indexURL()
	glog.Infof("Deleting Index %v", url)

	req, err := http.NewRequest("DELETE", url, nil)
	if err != nil {
		return err
	}
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	glog.V(4).Infof("Delete response %s", body)
	if err != nil {
		return err
	}
	return nil
}

func (ed *elasticDriver) postIndex() error {
	url := ed.indexURL()
	glog.V(4).Infof("Posting Index to %v", url)

	config := make(map[string]interface{})
	config["settings"] = ed.settings
	configBytes, err := json.Marshal(config)
	glog.V(4).Infof("Config is %v", string(configBytes))

	if err != nil {
		return err
	}

	resp, err := http.Post(url, "application/json", bytes.NewReader(configBytes))
	if err != nil {
		return err
	}
	glog.V(4).Infof("Response %v", resp)
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return err
	}

	errResponse := true
	if resp.StatusCode == 400 {
		glog.V(4).Info("400 response code")
		//ignore if 400 and IndexAlreadyExistsException
		var result map[string]interface{}
		err = json.Unmarshal(body, &result)
		if err == nil {
			if errString, found := result["error"]; found {
				glog.V(4).Infof("Found error in response: %v", errString)
				switch errString.(type) {
				case string:
					if strings.HasPrefix(errString.(string), "IndexAlreadyExistsException") {
						errResponse = false
					}
				}
			}
		}
	} else if resp.StatusCode >= 200 || resp.StatusCode < 300 {
		errResponse = false
	}
	if errResponse {
		glog.Errorf("Error creating index: %s", string(body))
		return fmt.Errorf("error posting index: %v", resp.Status)
	}
	return nil
}
<commit_msg>Handle another IndexAlreadyExists error message<commit_after>// Copyright 2014, The Serviced Authors. All rights reserved.
// Use of this source code is governed by a
// license that can be found in the LICENSE file.

package elastic

import (
	"github.com/mattbaird/elastigo/api"
	"github.com/zenoss/glog"
	"github.com/zenoss/serviced/datastore"

	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"strings"
	"time"
)

//ElasticDriver describes an the Elastic Search driver
type ElasticDriver interface {
	SetProperty(name string, prop interface{}) error
	// AddMapping add a document mapping to be registered with ElasticSearch
	AddMapping(mapping Mapping) error
	//Initialize the driver, register mappings with elasticserach. Timeout in ms to wait for elastic to be available.
	Initialize(timeout time.Duration) error
	GetConnection() (datastore.Connection, error)
}

// New creates a new ElasticDriver
func New(host string, port uint16, index string) ElasticDriver {
	return newDriver(host, port, index)
}

func newDriver(host string, port uint16, index string) *elasticDriver {
	api.Domain = host
	api.Port = fmt.Sprintf("%v", port)
	//TODO: singleton since elastigo doesn't support multiple endpoints

	driver := &elasticDriver{}
	driver.host = host
	driver.port = port
	driver.index = index
	driver.settings = map[string]interface{}{"number_of_shards": 1}
	driver.mappings = make([]Mapping, 0)
	return driver
}

//Make sure elasticDriver implements datastore.Driver
var _ datastore.Driver = &elasticDriver{}

type elasticDriver struct {
	host     string
	port     uint16
	settings map[string]interface{}
	mappings []Mapping
	index    string
}

func (ed *elasticDriver) GetConnection() (datastore.Connection, error) {
	return &elasticConnection{ed.index}, nil
}

func (ed *elasticDriver) Initialize(timeout time.Duration) error {

	quit := make(chan int)
	healthy := make(chan int)

	go ed.checkHealth(quit, healthy)

	select {
	case <-healthy:
		glog.V(4).Infof("Got response from Elastic")
	case <-time.After(timeout):
		return errors.New("timed Out waiting for response from Elastic")
	}

	if err := ed.postIndex(); err != nil {
		return err
	}

	if err := ed.postMappings(); err != nil {
		return err
	}

	// postMapping and postIndex affect es health
	go ed.checkHealth(quit, healthy)

	select {
	case <-healthy:
		glog.V(4).Infof("Got response from Elastic")
	case <-time.After(timeout):
		return errors.New("timed Out waiting for response from Elastic")
	}

	return nil
}

func (ed *elasticDriver) SetProperty(name string, prop interface{}) error {
	ed.settings[name] = prop
	return nil
}

func (ed *elasticDriver) AddMapping(mapping Mapping) error {
	ed.mappings = append(ed.mappings, mapping)
	return nil
}

func (ed *elasticDriver) AddMappingsFile(path string) error {
	glog.Infof("AddMappingsFiles %v", path)

	bytes, err := ioutil.ReadFile(path)
	if err != nil {
		return err
	}
	glog.V(4).Infof("AddMappingsFiles: content %v", string(bytes))

	type mapFile struct {
		Mappings map[string]map[string]interface{}
		Settings map[string]interface{}
	}
	var allMappings mapFile
	err = json.Unmarshal(bytes, &allMappings)
	if err != nil {
		return err
	}
	for key, val := range allMappings.Settings {
		ed.settings[key] = val
	}
	for key, mapping := range allMappings.Mappings {

		var rawMapping = make(map[string]map[string]interface{})
		rawMapping[key] = mapping
		if value, err := newMapping(rawMapping); err != nil {
			glog.Errorf("%v; could not create mapping from: %v", err, rawMapping)
			return err
		} else {
			ed.AddMapping(value)
		}
	}

	return nil
}

func (ed *elasticDriver) elasticURL() string {
	return fmt.Sprintf("http://%s:%d", ed.host, ed.port)
}

func (ed *elasticDriver) indexURL() string {
	return fmt.Sprintf("%s/%s/", ed.elasticURL(), ed.index)
}

func (ed *elasticDriver) isUp() bool {
	health, err := ed.getHealth()
	if err != nil {
		glog.Errorf("isUp() err=%v", err)
		return false
	}
	status := health["status"]
	return status == "green" || status == "yellow"
}

func (ed *elasticDriver) getHealth() (map[string]interface{}, error) {
	health := make(map[string]interface{})
	healthURL := fmt.Sprintf("%v/_cluster/health", ed.elasticURL())
	resp, err := http.Get(healthURL)
	if err != nil {
		return health, err
	}
	if resp.StatusCode != 200 {
		return health, fmt.Errorf("http status: %v", resp.StatusCode)
	}

	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		glog.Errorf("error reading elastic health: %v", err)
		return health, err
	}
	if err := json.Unmarshal(body, &health); err != nil {
		glog.Errorf("error unmarshalling elastic health: %v; err: %v", string(body), err)
		return health, err
	}
	glog.V(4).Infof("Elastic Health: %v; err: %v", string(body), err)
	return health, nil

}

func (ed *elasticDriver) checkHealth(quit chan int, healthy chan int) {
	for {
		select {
		default:
			if ed.isUp() {
				healthy <- 1
				return
			}
			glog.Infof("Waiting for Elastic Search")
			time.Sleep(1000 * time.Millisecond)

		case <-quit:
			return
		}
	}

}

func (ed *elasticDriver) postMappings() error {

	post := func(typeName string, mappingBytes []byte) error {
		mapURL := fmt.Sprintf("%s/%s/_mapping", ed.indexURL(), typeName)
		glog.V(4).Infof("Posting mapping to %s", mapURL)
		resp, err := http.Post(mapURL, "application/json", bytes.NewReader(mappingBytes))
		if err != nil {
			return fmt.Errorf("error mapping %s: %s", typeName, err)
		}
		glog.V(4).Infof("Response %v", resp)
		defer resp.Body.Close()
		body, err := ioutil.ReadAll(resp.Body)
		glog.V(4).Infof("Post result %s", body)
		if err != nil {
			return err
		}
		if resp.StatusCode < 200 || resp.StatusCode >= 300 {
			return fmt.Errorf("response %d mapping %s: %s", resp.StatusCode, typeName, string(body))
		}
		return nil
	}

	for _, mapping := range ed.mappings {
		mappingBytes, err := json.Marshal(mapping)
		if err != nil {
			return err
		}

		glog.V(4).Infof("mappping %v to  %v", mapping.Name, string(mappingBytes))
		err = post(mapping.Name, mappingBytes)
		if err != nil {
			return err
		}
	}

	return nil
}

func (ed *elasticDriver) deleteIndex() error {
	url := ed.indexURL()
	glog.Infof("Deleting Index %v", url)

	req, err := http.NewRequest("DELETE", url, nil)
	if err != nil {
		return err
	}
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	glog.V(4).Infof("Delete response %s", body)
	if err != nil {
		return err
	}
	return nil
}

func (ed *elasticDriver) postIndex() error {
	url := ed.indexURL()
	glog.V(4).Infof("Posting Index to %v", url)

	config := make(map[string]interface{})
	config["settings"] = ed.settings
	configBytes, err := json.Marshal(config)
	glog.V(4).Infof("Config is %v", string(configBytes))

	if err != nil {
		return err
	}

	resp, err := http.Post(url, "application/json", bytes.NewReader(configBytes))
	if err != nil {
		return err
	}
	glog.V(4).Infof("Response %v", resp)
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return err
	}

	errResponse := true
	if resp.StatusCode == 400 {
		glog.V(4).Info("400 response code")
		//ignore if 400 and IndexAlreadyExistsException
		var result map[string]interface{}
		err = json.Unmarshal(body, &result)
		if err == nil {
			if errString, found := result["error"]; found {
				glog.V(4).Infof("Found error in response: '%v'", errString)
				switch errString.(type) {
				case string:
					if strings.HasPrefix(errString.(string), "IndexAlreadyExistsException") {
						errResponse = false
					}
					msg := fmt.Sprintf("nested: IndexAlreadyExistsException[[%s] already exists]; ", ed.index)
					if strings.HasSuffix(errString.(string), msg) {
						errResponse = false
					}
				}
			}
		}
	} else if resp.StatusCode >= 200 || resp.StatusCode < 300 {
		errResponse = false
	}
	if errResponse {
		glog.Errorf("Error creating index: %s", string(body))
		return fmt.Errorf("error posting index: %v", resp.Status)
	}
	return nil
}
<|endoftext|>


<commit_before>package restart

import (
	"container/list"
	"errors"
	"sync"
	"time"

	"github.com/docker/go-events"
	"github.com/docker/swarmkit/api"
	"github.com/docker/swarmkit/api/defaults"
	"github.com/docker/swarmkit/log"
	"github.com/docker/swarmkit/manager/orchestrator"
	"github.com/docker/swarmkit/manager/state"
	"github.com/docker/swarmkit/manager/state/store"
	gogotypes "github.com/gogo/protobuf/types"
	"golang.org/x/net/context"
)

const defaultOldTaskTimeout = time.Minute

type restartedInstance struct {
	timestamp time.Time
}

type instanceRestartInfo struct {
	// counter of restarts for this instance.
	totalRestarts uint64
	// Linked list of restartedInstance structs. Only used when
	// Restart.MaxAttempts and Restart.Window are both
	// nonzero.
	restartedInstances *list.List
}

type delayedStart struct {
	// cancel is called to cancel the delayed start.
	cancel func()
	doneCh chan struct{}

	// waiter is set to true if the next restart is waiting for this delay
	// to complete.
	waiter bool
}

type instanceTuple struct {
	instance  uint64 // unset for global tasks
	serviceID string
	nodeID    string // unset for replicated tasks
}

// Supervisor initiates and manages restarts. It's responsible for
// delaying restarts when applicable.
type Supervisor struct {
	mu               sync.Mutex
	store            *store.MemoryStore
	delays           map[string]*delayedStart
	historyByService map[string]map[instanceTuple]*instanceRestartInfo
	TaskTimeout      time.Duration
}

// NewSupervisor creates a new RestartSupervisor.
func NewSupervisor(store *store.MemoryStore) *Supervisor {
	return &Supervisor{
		store:            store,
		delays:           make(map[string]*delayedStart),
		historyByService: make(map[string]map[instanceTuple]*instanceRestartInfo),
		TaskTimeout:      defaultOldTaskTimeout,
	}
}

func (r *Supervisor) waitRestart(ctx context.Context, oldDelay *delayedStart, cluster *api.Cluster, taskID string) {
	// Wait for the last restart delay to elapse.
	select {
	case <-oldDelay.doneCh:
	case <-ctx.Done():
		return
	}

	// Start the next restart
	err := r.store.Update(func(tx store.Tx) error {
		t := store.GetTask(tx, taskID)
		if t == nil {
			return nil
		}
		if t.DesiredState > api.TaskStateRunning {
			return nil
		}
		service := store.GetService(tx, t.ServiceID)
		if service == nil {
			return nil
		}
		return r.Restart(ctx, tx, cluster, service, *t)
	})

	if err != nil {
		log.G(ctx).WithError(err).Errorf("failed to restart task after waiting for previous restart")
	}
}

// Restart initiates a new task to replace t if appropriate under the service's
// restart policy.
func (r *Supervisor) Restart(ctx context.Context, tx store.Tx, cluster *api.Cluster, service *api.Service, t api.Task) error {
	// TODO(aluzzardi): This function should not depend on `service`.

	// Is the old task still in the process of restarting? If so, wait for
	// its restart delay to elapse, to avoid tight restart loops (for
	// example, when the image doesn't exist).
	r.mu.Lock()
	oldDelay, ok := r.delays[t.ID]
	if ok {
		if !oldDelay.waiter {
			oldDelay.waiter = true
			go r.waitRestart(ctx, oldDelay, cluster, t.ID)
		}
		r.mu.Unlock()
		return nil
	}
	r.mu.Unlock()

	// Sanity check: was the task shut down already by a separate call to
	// Restart? If so, we must avoid restarting it, because this will create
	// an extra task. This should never happen unless there is a bug.
	if t.DesiredState > api.TaskStateRunning {
		return errors.New("Restart called on task that was already shut down")
	}

	t.DesiredState = api.TaskStateShutdown
	err := store.UpdateTask(tx, &t)
	if err != nil {
		log.G(ctx).WithError(err).Errorf("failed to set task desired state to dead")
		return err
	}

	if !r.shouldRestart(ctx, &t, service) {
		return nil
	}

	var restartTask *api.Task

	if orchestrator.IsReplicatedService(service) {
		restartTask = orchestrator.NewTask(cluster, service, t.Slot, "")
	} else if orchestrator.IsGlobalService(service) {
		restartTask = orchestrator.NewTask(cluster, service, 0, t.NodeID)
	} else {
		log.G(ctx).Error("service not supported by restart supervisor")
		return nil
	}

	n := store.GetNode(tx, t.NodeID)

	restartTask.DesiredState = api.TaskStateReady

	var restartDelay time.Duration
	// Restart delay is not applied to drained nodes
	if n == nil || n.Spec.Availability != api.NodeAvailabilityDrain {
		if t.Spec.Restart != nil && t.Spec.Restart.Delay != nil {
			var err error
			restartDelay, err = gogotypes.DurationFromProto(t.Spec.Restart.Delay)
			if err != nil {
				log.G(ctx).WithError(err).Error("invalid restart delay; using default")
				restartDelay, _ = gogotypes.DurationFromProto(defaults.Service.Task.Restart.Delay)
			}
		} else {
			restartDelay, _ = gogotypes.DurationFromProto(defaults.Service.Task.Restart.Delay)
		}
	}

	waitStop := true

	// Normally we wait for the old task to stop running, but we skip this
	// if the old task is already dead or the node it's assigned to is down.
	if (n != nil && n.Status.State == api.NodeStatus_DOWN) || t.Status.State > api.TaskStateRunning {
		waitStop = false
	}

	if err := store.CreateTask(tx, restartTask); err != nil {
		log.G(ctx).WithError(err).WithField("task.id", restartTask.ID).Error("task create failed")
		return err
	}

	r.recordRestartHistory(restartTask)

	r.DelayStart(ctx, tx, &t, restartTask.ID, restartDelay, waitStop)
	return nil
}

func (r *Supervisor) shouldRestart(ctx context.Context, t *api.Task, service *api.Service) bool {
	// TODO(aluzzardi): This function should not depend on `service`.

	condition := orchestrator.RestartCondition(t)

	if condition != api.RestartOnAny &&
		(condition != api.RestartOnFailure || t.Status.State == api.TaskStateCompleted) {
		return false
	}

	if t.Spec.Restart == nil || t.Spec.Restart.MaxAttempts == 0 {
		return true
	}

	instanceTuple := instanceTuple{
		instance:  t.Slot,
		serviceID: t.ServiceID,
	}

	// Instance is not meaningful for "global" tasks, so they need to be
	// indexed by NodeID.
	if orchestrator.IsGlobalService(service) {
		instanceTuple.nodeID = t.NodeID
	}

	r.mu.Lock()
	defer r.mu.Unlock()

	restartInfo := r.historyByService[t.ServiceID][instanceTuple]
	if restartInfo == nil {
		return true
	}

	if t.Spec.Restart.Window == nil || (t.Spec.Restart.Window.Seconds == 0 && t.Spec.Restart.Window.Nanos == 0) {
		return restartInfo.totalRestarts < t.Spec.Restart.MaxAttempts
	}

	if restartInfo.restartedInstances == nil {
		return true
	}

	window, err := gogotypes.DurationFromProto(t.Spec.Restart.Window)
	if err != nil {
		log.G(ctx).WithError(err).Error("invalid restart lookback window")
		return restartInfo.totalRestarts < t.Spec.Restart.MaxAttempts
	}
	lookback := time.Now().Add(-window)

	var next *list.Element
	for e := restartInfo.restartedInstances.Front(); e != nil; e = next {
		next = e.Next()

		if e.Value.(restartedInstance).timestamp.After(lookback) {
			break
		}
		restartInfo.restartedInstances.Remove(e)
	}

	numRestarts := uint64(restartInfo.restartedInstances.Len())

	if numRestarts == 0 {
		restartInfo.restartedInstances = nil
	}

	return numRestarts < t.Spec.Restart.MaxAttempts
}

func (r *Supervisor) recordRestartHistory(restartTask *api.Task) {
	if restartTask.Spec.Restart == nil || restartTask.Spec.Restart.MaxAttempts == 0 {
		// No limit on the number of restarts, so no need to record
		// history.
		return
	}
	tuple := instanceTuple{
		instance:  restartTask.Slot,
		serviceID: restartTask.ServiceID,
		nodeID:    restartTask.NodeID,
	}

	r.mu.Lock()
	defer r.mu.Unlock()

	if r.historyByService[restartTask.ServiceID] == nil {
		r.historyByService[restartTask.ServiceID] = make(map[instanceTuple]*instanceRestartInfo)
	}
	if r.historyByService[restartTask.ServiceID][tuple] == nil {
		r.historyByService[restartTask.ServiceID][tuple] = &instanceRestartInfo{}
	}

	restartInfo := r.historyByService[restartTask.ServiceID][tuple]
	restartInfo.totalRestarts++

	if restartTask.Spec.Restart.Window != nil && (restartTask.Spec.Restart.Window.Seconds != 0 || restartTask.Spec.Restart.Window.Nanos != 0) {
		if restartInfo.restartedInstances == nil {
			restartInfo.restartedInstances = list.New()
		}

		restartedInstance := restartedInstance{
			timestamp: time.Now(),
		}

		restartInfo.restartedInstances.PushBack(restartedInstance)
	}
}

// DelayStart starts a timer that moves the task from READY to RUNNING once:
// - The restart delay has elapsed (if applicable)
// - The old task that it's replacing has stopped running (or this times out)
// It must be called during an Update transaction to ensure that it does not
// miss events. The purpose of the store.Tx argument is to avoid accidental
// calls outside an Update transaction.
func (r *Supervisor) DelayStart(ctx context.Context, _ store.Tx, oldTask *api.Task, newTaskID string, delay time.Duration, waitStop bool) <-chan struct{} {
	ctx, cancel := context.WithCancel(context.Background())
	doneCh := make(chan struct{})

	r.mu.Lock()
	for {
		oldDelay, ok := r.delays[newTaskID]
		if !ok {
			break
		}
		oldDelay.cancel()
		r.mu.Unlock()
		// Note that this channel read should only block for a very
		// short time, because we cancelled the existing delay and
		// that should cause it to stop immediately.
		<-oldDelay.doneCh
		r.mu.Lock()
	}
	r.delays[newTaskID] = &delayedStart{cancel: cancel, doneCh: doneCh}
	r.mu.Unlock()

	var watch chan events.Event
	cancelWatch := func() {}

	if waitStop && oldTask != nil {
		// Wait for either the old task to complete, or the old task's
		// node to become unavailable.
		watch, cancelWatch = state.Watch(
			r.store.WatchQueue(),
			api.EventUpdateTask{
				Task:   &api.Task{ID: oldTask.ID, Status: api.TaskStatus{State: api.TaskStateRunning}},
				Checks: []api.TaskCheckFunc{api.TaskCheckID, state.TaskCheckStateGreaterThan},
			},
			api.EventUpdateNode{
				Node:   &api.Node{ID: oldTask.NodeID, Status: api.NodeStatus{State: api.NodeStatus_DOWN}},
				Checks: []api.NodeCheckFunc{api.NodeCheckID, state.NodeCheckState},
			},
			api.EventDeleteNode{
				Node:   &api.Node{ID: oldTask.NodeID},
				Checks: []api.NodeCheckFunc{api.NodeCheckID},
			},
		)
	}

	go func() {
		defer func() {
			cancelWatch()
			r.mu.Lock()
			delete(r.delays, newTaskID)
			r.mu.Unlock()
			close(doneCh)
		}()

		oldTaskTimer := time.NewTimer(r.TaskTimeout)
		defer oldTaskTimer.Stop()

		// Wait for the delay to elapse, if one is specified.
		if delay != 0 {
			select {
			case <-time.After(delay):
			case <-ctx.Done():
				return
			}
		}

		if waitStop && oldTask != nil {
			select {
			case <-watch:
			case <-oldTaskTimer.C:
			case <-ctx.Done():
				return
			}
		}

		err := r.store.Update(func(tx store.Tx) error {
			err := r.StartNow(tx, newTaskID)
			if err != nil {
				log.G(ctx).WithError(err).WithField("task.id", newTaskID).Error("moving task out of delayed state failed")
			}
			return nil
		})
		if err != nil {
			log.G(ctx).WithError(err).WithField("task.id", newTaskID).Error("task restart transaction failed")
		}
	}()

	return doneCh
}

// StartNow moves the task into the RUNNING state so it will proceed to start
// up.
func (r *Supervisor) StartNow(tx store.Tx, taskID string) error {
	t := store.GetTask(tx, taskID)
	if t == nil || t.DesiredState >= api.TaskStateRunning {
		return nil
	}
	t.DesiredState = api.TaskStateRunning
	return store.UpdateTask(tx, t)
}

// Cancel cancels a pending restart.
func (r *Supervisor) Cancel(taskID string) {
	r.mu.Lock()
	delay, ok := r.delays[taskID]
	r.mu.Unlock()

	if !ok {
		return
	}

	delay.cancel()
	<-delay.doneCh
}

// CancelAll aborts all pending restarts and waits for any instances of
// StartNow that have already triggered to complete.
func (r *Supervisor) CancelAll() {
	var cancelled []delayedStart

	r.mu.Lock()
	for _, delay := range r.delays {
		delay.cancel()
	}
	r.mu.Unlock()

	for _, delay := range cancelled {
		<-delay.doneCh
	}
}

// ClearServiceHistory forgets restart history related to a given service ID.
func (r *Supervisor) ClearServiceHistory(serviceID string) {
	r.mu.Lock()
	delete(r.historyByService, serviceID)
	r.mu.Unlock()
}
<commit_msg>restart: Track SpecVersion in instance restart history<commit_after>package restart

import (
	"container/list"
	"errors"
	"sync"
	"time"

	"github.com/docker/go-events"
	"github.com/docker/swarmkit/api"
	"github.com/docker/swarmkit/api/defaults"
	"github.com/docker/swarmkit/log"
	"github.com/docker/swarmkit/manager/orchestrator"
	"github.com/docker/swarmkit/manager/state"
	"github.com/docker/swarmkit/manager/state/store"
	gogotypes "github.com/gogo/protobuf/types"
	"golang.org/x/net/context"
)

const defaultOldTaskTimeout = time.Minute

type restartedInstance struct {
	timestamp time.Time
}

type instanceRestartInfo struct {
	// counter of restarts for this instance.
	totalRestarts uint64
	// Linked list of restartedInstance structs. Only used when
	// Restart.MaxAttempts and Restart.Window are both
	// nonzero.
	restartedInstances *list.List
}

type delayedStart struct {
	// cancel is called to cancel the delayed start.
	cancel func()
	doneCh chan struct{}

	// waiter is set to true if the next restart is waiting for this delay
	// to complete.
	waiter bool
}

type instanceTuple struct {
	instance    uint64 // unset for global tasks
	serviceID   string
	nodeID      string // unset for replicated tasks
	specVersion api.Version
}

// Supervisor initiates and manages restarts. It's responsible for
// delaying restarts when applicable.
type Supervisor struct {
	mu               sync.Mutex
	store            *store.MemoryStore
	delays           map[string]*delayedStart
	historyByService map[string]map[instanceTuple]*instanceRestartInfo
	TaskTimeout      time.Duration
}

// NewSupervisor creates a new RestartSupervisor.
func NewSupervisor(store *store.MemoryStore) *Supervisor {
	return &Supervisor{
		store:            store,
		delays:           make(map[string]*delayedStart),
		historyByService: make(map[string]map[instanceTuple]*instanceRestartInfo),
		TaskTimeout:      defaultOldTaskTimeout,
	}
}

func (r *Supervisor) waitRestart(ctx context.Context, oldDelay *delayedStart, cluster *api.Cluster, taskID string) {
	// Wait for the last restart delay to elapse.
	select {
	case <-oldDelay.doneCh:
	case <-ctx.Done():
		return
	}

	// Start the next restart
	err := r.store.Update(func(tx store.Tx) error {
		t := store.GetTask(tx, taskID)
		if t == nil {
			return nil
		}
		if t.DesiredState > api.TaskStateRunning {
			return nil
		}
		service := store.GetService(tx, t.ServiceID)
		if service == nil {
			return nil
		}
		return r.Restart(ctx, tx, cluster, service, *t)
	})

	if err != nil {
		log.G(ctx).WithError(err).Errorf("failed to restart task after waiting for previous restart")
	}
}

// Restart initiates a new task to replace t if appropriate under the service's
// restart policy.
func (r *Supervisor) Restart(ctx context.Context, tx store.Tx, cluster *api.Cluster, service *api.Service, t api.Task) error {
	// TODO(aluzzardi): This function should not depend on `service`.

	// Is the old task still in the process of restarting? If so, wait for
	// its restart delay to elapse, to avoid tight restart loops (for
	// example, when the image doesn't exist).
	r.mu.Lock()
	oldDelay, ok := r.delays[t.ID]
	if ok {
		if !oldDelay.waiter {
			oldDelay.waiter = true
			go r.waitRestart(ctx, oldDelay, cluster, t.ID)
		}
		r.mu.Unlock()
		return nil
	}
	r.mu.Unlock()

	// Sanity check: was the task shut down already by a separate call to
	// Restart? If so, we must avoid restarting it, because this will create
	// an extra task. This should never happen unless there is a bug.
	if t.DesiredState > api.TaskStateRunning {
		return errors.New("Restart called on task that was already shut down")
	}

	t.DesiredState = api.TaskStateShutdown
	err := store.UpdateTask(tx, &t)
	if err != nil {
		log.G(ctx).WithError(err).Errorf("failed to set task desired state to dead")
		return err
	}

	if !r.shouldRestart(ctx, &t, service) {
		return nil
	}

	var restartTask *api.Task

	if orchestrator.IsReplicatedService(service) {
		restartTask = orchestrator.NewTask(cluster, service, t.Slot, "")
	} else if orchestrator.IsGlobalService(service) {
		restartTask = orchestrator.NewTask(cluster, service, 0, t.NodeID)
	} else {
		log.G(ctx).Error("service not supported by restart supervisor")
		return nil
	}

	n := store.GetNode(tx, t.NodeID)

	restartTask.DesiredState = api.TaskStateReady

	var restartDelay time.Duration
	// Restart delay is not applied to drained nodes
	if n == nil || n.Spec.Availability != api.NodeAvailabilityDrain {
		if t.Spec.Restart != nil && t.Spec.Restart.Delay != nil {
			var err error
			restartDelay, err = gogotypes.DurationFromProto(t.Spec.Restart.Delay)
			if err != nil {
				log.G(ctx).WithError(err).Error("invalid restart delay; using default")
				restartDelay, _ = gogotypes.DurationFromProto(defaults.Service.Task.Restart.Delay)
			}
		} else {
			restartDelay, _ = gogotypes.DurationFromProto(defaults.Service.Task.Restart.Delay)
		}
	}

	waitStop := true

	// Normally we wait for the old task to stop running, but we skip this
	// if the old task is already dead or the node it's assigned to is down.
	if (n != nil && n.Status.State == api.NodeStatus_DOWN) || t.Status.State > api.TaskStateRunning {
		waitStop = false
	}

	if err := store.CreateTask(tx, restartTask); err != nil {
		log.G(ctx).WithError(err).WithField("task.id", restartTask.ID).Error("task create failed")
		return err
	}

	r.recordRestartHistory(restartTask)

	r.DelayStart(ctx, tx, &t, restartTask.ID, restartDelay, waitStop)
	return nil
}

func (r *Supervisor) shouldRestart(ctx context.Context, t *api.Task, service *api.Service) bool {
	// TODO(aluzzardi): This function should not depend on `service`.

	condition := orchestrator.RestartCondition(t)

	if condition != api.RestartOnAny &&
		(condition != api.RestartOnFailure || t.Status.State == api.TaskStateCompleted) {
		return false
	}

	if t.Spec.Restart == nil || t.Spec.Restart.MaxAttempts == 0 {
		return true
	}

	instanceTuple := instanceTuple{
		instance:  t.Slot,
		serviceID: t.ServiceID,
	}

	// Instance is not meaningful for "global" tasks, so they need to be
	// indexed by NodeID.
	if orchestrator.IsGlobalService(service) {
		instanceTuple.nodeID = t.NodeID
	}

	r.mu.Lock()
	defer r.mu.Unlock()

	restartInfo := r.historyByService[t.ServiceID][instanceTuple]
	if restartInfo == nil {
		return true
	}

	if t.Spec.Restart.Window == nil || (t.Spec.Restart.Window.Seconds == 0 && t.Spec.Restart.Window.Nanos == 0) {
		return restartInfo.totalRestarts < t.Spec.Restart.MaxAttempts
	}

	if restartInfo.restartedInstances == nil {
		return true
	}

	window, err := gogotypes.DurationFromProto(t.Spec.Restart.Window)
	if err != nil {
		log.G(ctx).WithError(err).Error("invalid restart lookback window")
		return restartInfo.totalRestarts < t.Spec.Restart.MaxAttempts
	}
	lookback := time.Now().Add(-window)

	var next *list.Element
	for e := restartInfo.restartedInstances.Front(); e != nil; e = next {
		next = e.Next()

		if e.Value.(restartedInstance).timestamp.After(lookback) {
			break
		}
		restartInfo.restartedInstances.Remove(e)
	}

	numRestarts := uint64(restartInfo.restartedInstances.Len())

	if numRestarts == 0 {
		restartInfo.restartedInstances = nil
	}

	return numRestarts < t.Spec.Restart.MaxAttempts
}

func (r *Supervisor) recordRestartHistory(restartTask *api.Task) {
	if restartTask.Spec.Restart == nil || restartTask.Spec.Restart.MaxAttempts == 0 {
		// No limit on the number of restarts, so no need to record
		// history.
		return
	}
	tuple := instanceTuple{
		instance:  restartTask.Slot,
		serviceID: restartTask.ServiceID,
		nodeID:    restartTask.NodeID,
	}
	if restartTask.SpecVersion != nil {
		tuple.specVersion = *restartTask.SpecVersion
	}

	r.mu.Lock()
	defer r.mu.Unlock()

	if r.historyByService[restartTask.ServiceID] == nil {
		r.historyByService[restartTask.ServiceID] = make(map[instanceTuple]*instanceRestartInfo)
	}
	if r.historyByService[restartTask.ServiceID][tuple] == nil {
		r.historyByService[restartTask.ServiceID][tuple] = &instanceRestartInfo{}
	}

	restartInfo := r.historyByService[restartTask.ServiceID][tuple]
	restartInfo.totalRestarts++

	if restartTask.Spec.Restart.Window != nil && (restartTask.Spec.Restart.Window.Seconds != 0 || restartTask.Spec.Restart.Window.Nanos != 0) {
		if restartInfo.restartedInstances == nil {
			restartInfo.restartedInstances = list.New()
		}

		restartedInstance := restartedInstance{
			timestamp: time.Now(),
		}

		restartInfo.restartedInstances.PushBack(restartedInstance)
	}
}

// DelayStart starts a timer that moves the task from READY to RUNNING once:
// - The restart delay has elapsed (if applicable)
// - The old task that it's replacing has stopped running (or this times out)
// It must be called during an Update transaction to ensure that it does not
// miss events. The purpose of the store.Tx argument is to avoid accidental
// calls outside an Update transaction.
func (r *Supervisor) DelayStart(ctx context.Context, _ store.Tx, oldTask *api.Task, newTaskID string, delay time.Duration, waitStop bool) <-chan struct{} {
	ctx, cancel := context.WithCancel(context.Background())
	doneCh := make(chan struct{})

	r.mu.Lock()
	for {
		oldDelay, ok := r.delays[newTaskID]
		if !ok {
			break
		}
		oldDelay.cancel()
		r.mu.Unlock()
		// Note that this channel read should only block for a very
		// short time, because we cancelled the existing delay and
		// that should cause it to stop immediately.
		<-oldDelay.doneCh
		r.mu.Lock()
	}
	r.delays[newTaskID] = &delayedStart{cancel: cancel, doneCh: doneCh}
	r.mu.Unlock()

	var watch chan events.Event
	cancelWatch := func() {}

	if waitStop && oldTask != nil {
		// Wait for either the old task to complete, or the old task's
		// node to become unavailable.
		watch, cancelWatch = state.Watch(
			r.store.WatchQueue(),
			api.EventUpdateTask{
				Task:   &api.Task{ID: oldTask.ID, Status: api.TaskStatus{State: api.TaskStateRunning}},
				Checks: []api.TaskCheckFunc{api.TaskCheckID, state.TaskCheckStateGreaterThan},
			},
			api.EventUpdateNode{
				Node:   &api.Node{ID: oldTask.NodeID, Status: api.NodeStatus{State: api.NodeStatus_DOWN}},
				Checks: []api.NodeCheckFunc{api.NodeCheckID, state.NodeCheckState},
			},
			api.EventDeleteNode{
				Node:   &api.Node{ID: oldTask.NodeID},
				Checks: []api.NodeCheckFunc{api.NodeCheckID},
			},
		)
	}

	go func() {
		defer func() {
			cancelWatch()
			r.mu.Lock()
			delete(r.delays, newTaskID)
			r.mu.Unlock()
			close(doneCh)
		}()

		oldTaskTimer := time.NewTimer(r.TaskTimeout)
		defer oldTaskTimer.Stop()

		// Wait for the delay to elapse, if one is specified.
		if delay != 0 {
			select {
			case <-time.After(delay):
			case <-ctx.Done():
				return
			}
		}

		if waitStop && oldTask != nil {
			select {
			case <-watch:
			case <-oldTaskTimer.C:
			case <-ctx.Done():
				return
			}
		}

		err := r.store.Update(func(tx store.Tx) error {
			err := r.StartNow(tx, newTaskID)
			if err != nil {
				log.G(ctx).WithError(err).WithField("task.id", newTaskID).Error("moving task out of delayed state failed")
			}
			return nil
		})
		if err != nil {
			log.G(ctx).WithError(err).WithField("task.id", newTaskID).Error("task restart transaction failed")
		}
	}()

	return doneCh
}

// StartNow moves the task into the RUNNING state so it will proceed to start
// up.
func (r *Supervisor) StartNow(tx store.Tx, taskID string) error {
	t := store.GetTask(tx, taskID)
	if t == nil || t.DesiredState >= api.TaskStateRunning {
		return nil
	}
	t.DesiredState = api.TaskStateRunning
	return store.UpdateTask(tx, t)
}

// Cancel cancels a pending restart.
func (r *Supervisor) Cancel(taskID string) {
	r.mu.Lock()
	delay, ok := r.delays[taskID]
	r.mu.Unlock()

	if !ok {
		return
	}

	delay.cancel()
	<-delay.doneCh
}

// CancelAll aborts all pending restarts and waits for any instances of
// StartNow that have already triggered to complete.
func (r *Supervisor) CancelAll() {
	var cancelled []delayedStart

	r.mu.Lock()
	for _, delay := range r.delays {
		delay.cancel()
	}
	r.mu.Unlock()

	for _, delay := range cancelled {
		<-delay.doneCh
	}
}

// ClearServiceHistory forgets restart history related to a given service ID.
func (r *Supervisor) ClearServiceHistory(serviceID string) {
	r.mu.Lock()
	delete(r.historyByService, serviceID)
	r.mu.Unlock()
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"sync"

	log "github.com/Sirupsen/logrus"
	"github.com/davecgh/go-spew/spew"
)

var (
	logMutex = &sync.Mutex{}
)

func logBlockError(err error, args ...interface{}) {
	logMutex.Lock()
	defer logMutex.Unlock()

	var out string

	out += fmt.Sprintf("Failed to handle block with error %s\nContents:\n", err)

	for _, arg := range args {
		out += fmt.Sprintf("%s\n", spew.Sdump(arg))
	}

	log.Error(out)

	f, err := os.OpenFile("errors.txt", os.O_CREATE|os.O_WRONLY, 0600)
	if err != nil {
		log.Error("Unable to open error log file", err)
		return
	}
	defer f.Close()

	if _, err = f.WriteString(out); err != nil {
		log.Error("Unable to write in error log file", err)
	}
}
<commit_msg>Add append flag<commit_after>package main

import (
	"fmt"
	"os"
	"sync"

	log "github.com/Sirupsen/logrus"
	"github.com/davecgh/go-spew/spew"
)

var (
	logMutex = &sync.Mutex{}
)

func logBlockError(err error, args ...interface{}) {
	logMutex.Lock()
	defer logMutex.Unlock()

	var out string

	out += fmt.Sprintf("Failed to handle block with error %s\nContents:\n", err)

	for _, arg := range args {
		out += fmt.Sprintf("%s\n", spew.Sdump(arg))
	}

	log.Error(out)

	f, err := os.OpenFile("errors.txt", os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600)
	if err != nil {
		log.Error("Unable to open error log file", err)
		return
	}
	defer f.Close()

	if _, err = f.WriteString(out); err != nil {
		log.Error("Unable to write in error log file", err)
	}
}
<|endoftext|>


<commit_before>package kiwi

import (
	"sync"
	"time"
)

type (
	// Logger keeps context and log record. There are many loggers initialized
	// in different places of application. Loggers are safe for
	// concurrent usage.
	Logger struct {
		sync.RWMutex
		contextSrc map[interface{}]interface{}
		context    map[string]value
		pairs      map[string]value
	}
	// Record allows log data from any custom types in they conform this interface.
	// Also types that conform fmt.Stringer can be used. But as they not have IsQuoted() check
	// they always treated as strings and displayed in quotes.
	Record interface {
		String() string
		IsQuoted() bool
	}
	value struct {
		Val    string
		Func   interface{}
		Type   uint8
		Quoted bool
	}
)

// NewLogger creates logger instance.
func NewLogger() *Logger {
	return &Logger{
		contextSrc: make(map[interface{}]interface{}),
		context:    make(map[string]value),
		pairs:      make(map[string]value)}
}

// Log is the most common method for flushing previously added key-val pairs to an output.
// After current record is flushed all pairs removed from a record except contextSrc pairs.
func (l *Logger) Log(keyVals ...interface{}) {
	if len(keyVals) > 0 {
		l.Add(keyVals...)
	}
	l.Lock()
	record := l.pairs
	l.pairs = make(map[string]value)
	l.Unlock()
	var key string
	for i, val := range keyVals {
		if i%2 == 0 {
			key = toRecordKey(val)
			continue
		}
		record[key] = toRecordValue(val)
	}
	// for odd number of key-val pairs just add label without value
	if len(keyVals)%2 == 1 {
		record[key] = value{"", nil, voidVal, false}
	}
	l.RLock()
	for key, val := range l.context {
		// pairs override context
		if _, ok := record[key]; !ok {
			record[key] = val
		}
	}
	l.RUnlock()
	passRecordToOutput(record)
}

// Add a new key-value pairs to the log record. If a key already added then value will be
// updated. If a key already exists in a contextSrc then it will be overriden by a new
// value for a current record only. After flushing a record with Log() old context value
// will be restored.
func (l *Logger) Add(keyVals ...interface{}) *Logger {
	var key string
	l.Lock()
	for i, val := range keyVals {
		if i%2 == 0 {
			key = toRecordKey(val)
			continue
		}
		l.pairs[key] = toRecordValue(val)
	}
	// for odd number of key-val pairs just add label without value
	if len(keyVals)%2 == 1 {
		l.pairs[key] = value{"", nil, voidVal, false}
	}
	l.Unlock()
	return l
}

// With defines a context for the logger.
func (l *Logger) With(keyVals ...interface{}) *Logger {
	var (
		keySrc interface{}
		key    string
	)
	l.Lock()
	for i, val := range keyVals {
		if i%2 == 0 {
			keySrc = val
			key = toRecordKey(val)
			continue
		}
		l.contextSrc[keySrc] = val
		l.context[key] = toRecordValue(val)
	}
	// for odd number of key-val pairs just add label without value
	if len(keyVals)%2 == 1 {
		l.contextSrc[keySrc] = nil
		l.context[key] = value{"", nil, voidVal, false}
	}
	l.Unlock()
	return l
}

// Without drops some keys from a context for the logger.
func (l *Logger) Without(keys ...interface{}) *Logger {
	l.Lock()
	for _, key := range keys {
		if _, ok := l.contextSrc[key]; ok {
			delete(l.contextSrc, key)
			delete(l.context, toRecordKey(key))
		}
	}
	l.Unlock()
	return l
}

// WithTimestamp adds "timestamp" field to the context.
func (l *Logger) WithTimestamp(format string) *Logger {
	l.Lock()
	l.contextSrc["timestamp"] = func() string { return time.Now().Format(format) }
	l.context["timestamp"] = value{"", func() string { return time.Now().Format(format) }, stringVal, true}
	l.Unlock()
	return l
}

// Reset logger values added after last Log() call. It keeps contextSrc untouched.
func (l *Logger) Reset() *Logger {
	l.Lock()
	l.pairs = make(map[string]value)
	l.Unlock()
	return l
}

// ResetContext resets the context of the logger.
func (l *Logger) ResetContext() *Logger {
	l.Lock()
	l.contextSrc = make(map[interface{}]interface{})
	l.context = make(map[string]value)
	l.Unlock()
	return l
}

// GetContext returns copy of the context saved in the logger.
func (l *Logger) GetContext() map[interface{}]interface{} {
	var contextSrc = make(map[interface{}]interface{})
	l.RLock()
	for k, v := range l.contextSrc {
		contextSrc[k] = v
	}
	l.RUnlock()
	return contextSrc
}

// GetContextValue returns single context value for the key.
func (l *Logger) GetContextValue(key string) interface{} {
	l.RLock()
	value := l.contextSrc[key]
	l.RUnlock()
	return value
}

// GetRecord returns copy of current set of keys and values prepared for logging
// as strings. With context key-vals included.
// The most of Logger operations return *Logger itself but it made for operations
// chaining only. If you need get log pairs use GelRecord() for it.
func (l *Logger) GetRecord() map[string]string {
	var merged = make(map[string]string)
	l.RLock()
	for k, v := range l.context {
		merged[k] = v.Val
	}
	for k, v := range l.pairs {
		merged[k] = v.Val
	}
	l.RUnlock()
	return merged
}

// Flush confirms that all outputs got the last logged record.
func (l *Logger) Flush() {
	// XXX
	time.Sleep(100 * time.Millisecond)
}
<commit_msg>Create new logger from existing logger context.<commit_after>package kiwi

import (
	"sync"
	"time"
)

type (
	// Logger keeps context and log record. There are many loggers initialized
	// in different places of application. Loggers are safe for
	// concurrent usage.
	Logger struct {
		sync.RWMutex
		contextSrc map[interface{}]interface{}
		context    map[string]value
		pairs      map[string]value
	}
	// Record allows log data from any custom types in they conform this interface.
	// Also types that conform fmt.Stringer can be used. But as they not have IsQuoted() check
	// they always treated as strings and displayed in quotes.
	Record interface {
		String() string
		IsQuoted() bool
	}
	value struct {
		Val    string
		Func   interface{}
		Type   uint8
		Quoted bool
	}
)

// NewLogger creates logger instance.
func NewLogger() *Logger {
	return &Logger{
		contextSrc: make(map[interface{}]interface{}),
		context:    make(map[string]value),
		pairs:      make(map[string]value)}
}

// Copy creates copy of logger instance.
func (l *Logger) Copy() *Logger {
	var (
		newContextSrc = make(map[interface{}]interface{})
		newContext    = make(map[string]value)
	)
	l.RLock()
	for k, v := range l.contextSrc {
		newContextSrc[k] = v
	}
	for k, v := range l.context {
		newContext[k] = v
	}
	l.RUnlock()
	return &Logger{
		contextSrc: newContextSrc,
		context:    newContext,
		pairs:      make(map[string]value)}
}

// Log is the most common method for flushing previously added key-val pairs to an output.
// After current record is flushed all pairs removed from a record except contextSrc pairs.
func (l *Logger) Log(keyVals ...interface{}) {
	if len(keyVals) > 0 {
		l.Add(keyVals...)
	}
	l.Lock()
	record := l.pairs
	l.pairs = make(map[string]value)
	l.Unlock()
	var key string
	for i, val := range keyVals {
		if i%2 == 0 {
			key = toRecordKey(val)
			continue
		}
		record[key] = toRecordValue(val)
	}
	// for odd number of key-val pairs just add label without value
	if len(keyVals)%2 == 1 {
		record[key] = value{"", nil, voidVal, false}
	}
	l.RLock()
	for key, val := range l.context {
		// pairs override context
		if _, ok := record[key]; !ok {
			record[key] = val
		}
	}
	l.RUnlock()
	passRecordToOutput(record)
}

// Add a new key-value pairs to the log record. If a key already added then value will be
// updated. If a key already exists in a contextSrc then it will be overriden by a new
// value for a current record only. After flushing a record with Log() old context value
// will be restored.
func (l *Logger) Add(keyVals ...interface{}) *Logger {
	var key string
	l.Lock()
	for i, val := range keyVals {
		if i%2 == 0 {
			key = toRecordKey(val)
			continue
		}
		l.pairs[key] = toRecordValue(val)
	}
	// for odd number of key-val pairs just add label without value
	if len(keyVals)%2 == 1 {
		l.pairs[key] = value{"", nil, voidVal, false}
	}
	l.Unlock()
	return l
}

// With defines a context for the logger.
func (l *Logger) With(keyVals ...interface{}) *Logger {
	var (
		keySrc interface{}
		key    string
	)
	l.Lock()
	for i, val := range keyVals {
		if i%2 == 0 {
			keySrc = val
			key = toRecordKey(val)
			continue
		}
		l.contextSrc[keySrc] = val
		l.context[key] = toRecordValue(val)
	}
	// for odd number of key-val pairs just add label without value
	if len(keyVals)%2 == 1 {
		l.contextSrc[keySrc] = nil
		l.context[key] = value{"", nil, voidVal, false}
	}
	l.Unlock()
	return l
}

// Without drops some keys from a context for the logger.
func (l *Logger) Without(keys ...interface{}) *Logger {
	l.Lock()
	for _, key := range keys {
		if _, ok := l.contextSrc[key]; ok {
			delete(l.contextSrc, key)
			delete(l.context, toRecordKey(key))
		}
	}
	l.Unlock()
	return l
}

// WithTimestamp adds "timestamp" field to the context.
func (l *Logger) WithTimestamp(format string) *Logger {
	l.Lock()
	l.contextSrc["timestamp"] = func() string { return time.Now().Format(format) }
	l.context["timestamp"] = value{"", func() string { return time.Now().Format(format) }, stringVal, true}
	l.Unlock()
	return l
}

// Reset logger values added after last Log() call. It keeps contextSrc untouched.
func (l *Logger) Reset() *Logger {
	l.Lock()
	l.pairs = make(map[string]value)
	l.Unlock()
	return l
}

// ResetContext resets the context of the logger.
func (l *Logger) ResetContext() *Logger {
	l.Lock()
	l.contextSrc = make(map[interface{}]interface{})
	l.context = make(map[string]value)
	l.Unlock()
	return l
}

// GetContext returns copy of the context saved in the logger.
func (l *Logger) GetContext() map[interface{}]interface{} {
	var contextSrc = make(map[interface{}]interface{})
	l.RLock()
	for k, v := range l.contextSrc {
		contextSrc[k] = v
	}
	l.RUnlock()
	return contextSrc
}

// GetContextValue returns single context value for the key.
func (l *Logger) GetContextValue(key string) interface{} {
	l.RLock()
	value := l.contextSrc[key]
	l.RUnlock()
	return value
}

// GetRecord returns copy of current set of keys and values prepared for logging
// as strings. With context key-vals included.
// The most of Logger operations return *Logger itself but it made for operations
// chaining only. If you need get log pairs use GelRecord() for it.
func (l *Logger) GetRecord() map[string]string {
	var merged = make(map[string]string)
	l.RLock()
	for k, v := range l.context {
		merged[k] = v.Val
	}
	for k, v := range l.pairs {
		merged[k] = v.Val
	}
	l.RUnlock()
	return merged
}

// Flush confirms that all outputs got the last logged record.
func (l *Logger) Flush() {
	// XXX
	time.Sleep(100 * time.Millisecond)
}
<|endoftext|>


<commit_before>// Package logger is a simple wrapper around log.Logger with usual logging levels "error", "warning", "notice", "info"
// and "debug".
package logger

import (
	"fmt"
	"log"
	"sync"
)

const defaultLevel = "info"

const (
	_ = iota
	// LevelError represents the error logging level.
	LevelError
	// LevelWarning represents the warning logging level.
	LevelWarning
	// LevelNotice represents the notice logging level.
	LevelNotice
	// LevelInfo represents the info logging level.
	LevelInfo
	// LevelDebug represents the debug logging level.
	LevelDebug
)

var levelMap = map[string]int{
	"error":   LevelError,
	"warning": LevelWarning,
	"notice":  LevelNotice,
	"info":    LevelInfo,
	"debug":   LevelDebug,
}

// Logger represents a logger instance.
type Logger struct {
	backends []backend
	context  string

	wg sync.WaitGroup

	sync.Mutex
}

// NewLogger returns a new Logger instance initialized with the given configuration.
// If no configs are passed are parameter, log messages will effectively be discarded.
func NewLogger(configs ...interface{}) (*Logger, error) {
	// Initialize logger backends
	logger := &Logger{
		backends: []backend{},
		wg:       sync.WaitGroup{},
	}

	for _, config := range configs {
		var (
			backend backend
			err     error
		)

		switch config.(type) {
		case FileConfig:
			backend, err = newFileBackend(config.(FileConfig), logger)

		case SyslogConfig:
			backend, err = newSyslogBackend(config.(SyslogConfig), logger)

		default:
			err = ErrUnsupportedBackend
		}

		if err != nil {
			return nil, err
		}

		logger.backends = append(logger.backends, backend)
	}

	return logger, nil
}

// Logger returns a log.Logger instance for a given logging level.
func (l *Logger) Logger(level int) *log.Logger {
	return log.New(newWriter(l, level), "", 0)
}

// Context clones the Logger instance and sets the context to the provided string.
func (l *Logger) Context(context string) *Logger {
	return &Logger{
		backends: l.backends,
		context:  context,
		wg:       sync.WaitGroup{},
	}
}

// Error prints an error message in the logging system.
func (l *Logger) Error(format string, v ...interface{}) *Logger {
	l.write(LevelError, format, v...)
	return l
}

// Warning prints a warning message in the logging system.
func (l *Logger) Warning(format string, v ...interface{}) *Logger {
	l.write(LevelWarning, format, v...)
	return l
}

// Notice prints a notice message in the logging system.
func (l *Logger) Notice(format string, v ...interface{}) *Logger {
	l.write(LevelNotice, format, v...)
	return l
}

// Info prints an information message in the logging system.
func (l *Logger) Info(format string, v ...interface{}) *Logger {
	l.write(LevelInfo, format, v...)
	return l
}

// Debug prints a debug message in the logging system.
func (l *Logger) Debug(format string, v ...interface{}) *Logger {
	l.write(LevelDebug, format, v...)
	return l
}

// Close closes the logger output file.
func (l *Logger) Close() {
	for _, b := range l.backends {
		b.Close()
	}
}

func (l *Logger) write(level int, format string, v ...interface{}) {
	var mesg string

	l.Lock()
	defer l.Unlock()

	// Set message
	if l.context != "" {
		mesg = l.context + ": "
	}

	if len(v) > 0 {
		mesg += fmt.Sprintf(format, v...)
	} else {
		mesg += format
	}

	// Write messages to backends
	l.wg.Add(len(l.backends))

	for _, b := range l.backends {
		go func(b backend) {
			b.Write(level, mesg)
			l.wg.Done()
		}(b)
	}

	l.wg.Wait()
}
<commit_msg>Add method to get current context<commit_after>// Package logger is a simple wrapper around log.Logger with usual logging levels "error", "warning", "notice", "info"
// and "debug".
package logger

import (
	"fmt"
	"log"
	"sync"
)

const defaultLevel = "info"

const (
	_ = iota
	// LevelError represents the error logging level.
	LevelError
	// LevelWarning represents the warning logging level.
	LevelWarning
	// LevelNotice represents the notice logging level.
	LevelNotice
	// LevelInfo represents the info logging level.
	LevelInfo
	// LevelDebug represents the debug logging level.
	LevelDebug
)

var levelMap = map[string]int{
	"error":   LevelError,
	"warning": LevelWarning,
	"notice":  LevelNotice,
	"info":    LevelInfo,
	"debug":   LevelDebug,
}

// Logger represents a logger instance.
type Logger struct {
	backends []backend
	context  string

	wg sync.WaitGroup

	sync.Mutex
}

// NewLogger returns a new Logger instance initialized with the given configuration.
// If no configs are passed are parameter, log messages will effectively be discarded.
func NewLogger(configs ...interface{}) (*Logger, error) {
	// Initialize logger backends
	logger := &Logger{
		backends: []backend{},
		wg:       sync.WaitGroup{},
	}

	for _, config := range configs {
		var (
			backend backend
			err     error
		)

		switch config.(type) {
		case FileConfig:
			backend, err = newFileBackend(config.(FileConfig), logger)

		case SyslogConfig:
			backend, err = newSyslogBackend(config.(SyslogConfig), logger)

		default:
			err = ErrUnsupportedBackend
		}

		if err != nil {
			return nil, err
		}

		logger.backends = append(logger.backends, backend)
	}

	return logger, nil
}

// Logger returns a log.Logger instance for a given logging level.
func (l *Logger) Logger(level int) *log.Logger {
	return log.New(newWriter(l, level), "", 0)
}

// Context clones the Logger instance and sets the context to the provided string.
func (l *Logger) Context(context string) *Logger {
	return &Logger{
		backends: l.backends,
		context:  context,
		wg:       sync.WaitGroup{},
	}
}

// CurrentContext returns the current logger context.
func (l *Logger) CurrentContext() string {
	return l.context
}

// Error prints an error message in the logging system.
func (l *Logger) Error(format string, v ...interface{}) *Logger {
	l.write(LevelError, format, v...)
	return l
}

// Warning prints a warning message in the logging system.
func (l *Logger) Warning(format string, v ...interface{}) *Logger {
	l.write(LevelWarning, format, v...)
	return l
}

// Notice prints a notice message in the logging system.
func (l *Logger) Notice(format string, v ...interface{}) *Logger {
	l.write(LevelNotice, format, v...)
	return l
}

// Info prints an information message in the logging system.
func (l *Logger) Info(format string, v ...interface{}) *Logger {
	l.write(LevelInfo, format, v...)
	return l
}

// Debug prints a debug message in the logging system.
func (l *Logger) Debug(format string, v ...interface{}) *Logger {
	l.write(LevelDebug, format, v...)
	return l
}

// Close closes the logger output file.
func (l *Logger) Close() {
	for _, b := range l.backends {
		b.Close()
	}
}

func (l *Logger) write(level int, format string, v ...interface{}) {
	var mesg string

	l.Lock()
	defer l.Unlock()

	// Set message
	if l.context != "" {
		mesg = l.context + ": "
	}

	if len(v) > 0 {
		mesg += fmt.Sprintf(format, v...)
	} else {
		mesg += format
	}

	// Write messages to backends
	l.wg.Add(len(l.backends))

	for _, b := range l.backends {
		go func(b backend) {
			b.Write(level, mesg)
			l.wg.Done()
		}(b)
	}

	l.wg.Wait()
}
<|endoftext|>


<commit_before>package logrus

import (
	"io"
	"os"
	"sync"
)

type Logger struct {
	// The logs are `io.Copy`'d to this in a mutex. It's common to set this to a
	// file, or leave it default which is `os.Stdout`. You can also set this to
	// something more adventorous, such as logging to Kafka.
	Out io.Writer
	// Hooks for the logger instance. These allow firing events based on logging
	// levels and log entries. For example, to send errors to an error tracking
	// service, log to StatsD or dump the core on fatal errors.
	Hooks levelHooks
	// All log entries pass through the formatter before logged to Out. The
	// included formatters are `TextFormatter` and `JSONFormatter` for which
	// TextFormatter is the default. In development (when a TTY is attached) it
	// logs with colors, but to a file it wouldn't. You can easily implement your
	// own that implements the `Formatter` interface, see the `README` or included
	// formatters for examples.
	Formatter Formatter
	// The logging level the logger should log at. This is typically (and defaults
	// to) `logrus.Info`, which allows Info(), Warn(), Error() and Fatal() to be
	// logged. `logrus.Debug` is useful in
	Level Level
	// Used to sync writing to the log.
	mu sync.Mutex
}

// Creates a new logger. Configuration should be set by changing `Formatter`,
// `Out` and `Hooks` directly on the default logger instance. You can also just
// instantiate your own:
//
//    var log = &Logger{
//      Out: os.Stderr,
//      Formatter: new(JSONFormatter),
//      Hooks: make(levelHooks),
//      Level: logrus.DebugLevel,
//    }
//
// It's recommended to make this a global instance called `log`.
func New() *Logger {
	return &Logger{
		Out:       os.Stderr,
		Formatter: new(TextFormatter),
		Hooks:     make(levelHooks),
		Level:     InfoLevel,
	}
}

// Adds a field to the log entry, note that you it doesn't log until you call
// Debug, Print, Info, Warn, Fatal or Panic. It only creates a log entry.
// Ff you want multiple fields, use `WithFields`.
func (logger *Logger) WithField(key string, value interface{}) *Entry {
	return NewEntry(logger).WithField(key, value)
}

// Adds a struct of fields to the log entry. All it does is call `WithField` for
// each `Field`.
func (logger *Logger) WithFields(fields Fields) *Entry {
	return NewEntry(logger).WithFields(fields)
}

func (logger *Logger) Debugf(format string, args ...interface{}) {
	if logger.Level >= DebugLevel {
		NewEntry(logger).Debugf(format, args...)
	}
}

func (logger *Logger) Infof(format string, args ...interface{}) {
	if logger.Level >= InfoLevel {
		NewEntry(logger).Infof(format, args...)
	}
}

func (logger *Logger) Printf(format string, args ...interface{}) {
	NewEntry(logger).Printf(format, args...)
}

func (logger *Logger) Warnf(format string, args ...interface{}) {
	if logger.Level >= WarnLevel {
		NewEntry(logger).Warnf(format, args...)
	}
}

func (logger *Logger) Warningf(format string, args ...interface{}) {
	if logger.Level >= WarnLevel {
		NewEntry(logger).Warnf(format, args...)
	}
}

func (logger *Logger) Errorf(format string, args ...interface{}) {
	if logger.Level >= ErrorLevel {
		NewEntry(logger).Errorf(format, args...)
	}
}

func (logger *Logger) Fatalf(format string, args ...interface{}) {
	if logger.Level >= FatalLevel {
		NewEntry(logger).Fatalf(format, args...)
	}
}

func (logger *Logger) Panicf(format string, args ...interface{}) {
	if logger.Level >= PanicLevel {
		NewEntry(logger).Panicf(format, args...)
	}
}

func (logger *Logger) Debug(args ...interface{}) {
	if logger.Level >= DebugLevel {
		NewEntry(logger).Debug(args...)
	}
}

func (logger *Logger) Info(args ...interface{}) {
	if logger.Level >= InfoLevel {
		NewEntry(logger).Info(args...)
	}
}

func (logger *Logger) Print(args ...interface{}) {
	NewEntry(logger).Info(args...)
}

func (logger *Logger) Warn(args ...interface{}) {
	if logger.Level >= WarnLevel {
		NewEntry(logger).Warn(args...)
	}
}

func (logger *Logger) Warning(args ...interface{}) {
	if logger.Level >= WarnLevel {
		NewEntry(logger).Warn(args...)
	}
}

func (logger *Logger) Error(args ...interface{}) {
	if logger.Level >= ErrorLevel {
		NewEntry(logger).Error(args...)
	}
}

func (logger *Logger) Fatal(args ...interface{}) {
	if logger.Level >= FatalLevel {
		NewEntry(logger).Fatal(args...)
	}
}

func (logger *Logger) Panic(args ...interface{}) {
	if logger.Level >= PanicLevel {
		NewEntry(logger).Panic(args...)
	}
}

func (logger *Logger) Debugln(args ...interface{}) {
	if logger.Level >= DebugLevel {
		NewEntry(logger).Debugln(args...)
	}
}

func (logger *Logger) Infoln(args ...interface{}) {
	if logger.Level >= InfoLevel {
		NewEntry(logger).Infoln(args...)
	}
}

func (logger *Logger) Println(args ...interface{}) {
	NewEntry(logger).Println(args...)
}

func (logger *Logger) Warnln(args ...interface{}) {
	if logger.Level >= WarnLevel {
		NewEntry(logger).Warnln(args...)
	}
}

func (logger *Logger) Warningln(args ...interface{}) {
	if logger.Level >= WarnLevel {
		NewEntry(logger).Warnln(args...)
	}
}

func (logger *Logger) Errorln(args ...interface{}) {
	if logger.Level >= ErrorLevel {
		NewEntry(logger).Errorln(args...)
	}
}

func (logger *Logger) Fatalln(args ...interface{}) {
	if logger.Level >= FatalLevel {
		NewEntry(logger).Fatalln(args...)
	}
}

func (logger *Logger) Panicln(args ...interface{}) {
	if logger.Level >= PanicLevel {
		NewEntry(logger).Panicln(args...)
	}
}
<commit_msg>Fix Fatal*() function of logger to match the behavior of Fatal*() functions of entry<commit_after>package logrus

import (
	"io"
	"os"
	"sync"
)

type Logger struct {
	// The logs are `io.Copy`'d to this in a mutex. It's common to set this to a
	// file, or leave it default which is `os.Stdout`. You can also set this to
	// something more adventorous, such as logging to Kafka.
	Out io.Writer
	// Hooks for the logger instance. These allow firing events based on logging
	// levels and log entries. For example, to send errors to an error tracking
	// service, log to StatsD or dump the core on fatal errors.
	Hooks levelHooks
	// All log entries pass through the formatter before logged to Out. The
	// included formatters are `TextFormatter` and `JSONFormatter` for which
	// TextFormatter is the default. In development (when a TTY is attached) it
	// logs with colors, but to a file it wouldn't. You can easily implement your
	// own that implements the `Formatter` interface, see the `README` or included
	// formatters for examples.
	Formatter Formatter
	// The logging level the logger should log at. This is typically (and defaults
	// to) `logrus.Info`, which allows Info(), Warn(), Error() and Fatal() to be
	// logged. `logrus.Debug` is useful in
	Level Level
	// Used to sync writing to the log.
	mu sync.Mutex
}

// Creates a new logger. Configuration should be set by changing `Formatter`,
// `Out` and `Hooks` directly on the default logger instance. You can also just
// instantiate your own:
//
//    var log = &Logger{
//      Out: os.Stderr,
//      Formatter: new(JSONFormatter),
//      Hooks: make(levelHooks),
//      Level: logrus.DebugLevel,
//    }
//
// It's recommended to make this a global instance called `log`.
func New() *Logger {
	return &Logger{
		Out:       os.Stderr,
		Formatter: new(TextFormatter),
		Hooks:     make(levelHooks),
		Level:     InfoLevel,
	}
}

// Adds a field to the log entry, note that you it doesn't log until you call
// Debug, Print, Info, Warn, Fatal or Panic. It only creates a log entry.
// Ff you want multiple fields, use `WithFields`.
func (logger *Logger) WithField(key string, value interface{}) *Entry {
	return NewEntry(logger).WithField(key, value)
}

// Adds a struct of fields to the log entry. All it does is call `WithField` for
// each `Field`.
func (logger *Logger) WithFields(fields Fields) *Entry {
	return NewEntry(logger).WithFields(fields)
}

func (logger *Logger) Debugf(format string, args ...interface{}) {
	if logger.Level >= DebugLevel {
		NewEntry(logger).Debugf(format, args...)
	}
}

func (logger *Logger) Infof(format string, args ...interface{}) {
	if logger.Level >= InfoLevel {
		NewEntry(logger).Infof(format, args...)
	}
}

func (logger *Logger) Printf(format string, args ...interface{}) {
	NewEntry(logger).Printf(format, args...)
}

func (logger *Logger) Warnf(format string, args ...interface{}) {
	if logger.Level >= WarnLevel {
		NewEntry(logger).Warnf(format, args...)
	}
}

func (logger *Logger) Warningf(format string, args ...interface{}) {
	if logger.Level >= WarnLevel {
		NewEntry(logger).Warnf(format, args...)
	}
}

func (logger *Logger) Errorf(format string, args ...interface{}) {
	if logger.Level >= ErrorLevel {
		NewEntry(logger).Errorf(format, args...)
	}
}

func (logger *Logger) Fatalf(format string, args ...interface{}) {
	if logger.Level >= FatalLevel {
		NewEntry(logger).Fatalf(format, args...)
	}
	os.Exit(1)
}

func (logger *Logger) Panicf(format string, args ...interface{}) {
	if logger.Level >= PanicLevel {
		NewEntry(logger).Panicf(format, args...)
	}
}

func (logger *Logger) Debug(args ...interface{}) {
	if logger.Level >= DebugLevel {
		NewEntry(logger).Debug(args...)
	}
}

func (logger *Logger) Info(args ...interface{}) {
	if logger.Level >= InfoLevel {
		NewEntry(logger).Info(args...)
	}
}

func (logger *Logger) Print(args ...interface{}) {
	NewEntry(logger).Info(args...)
}

func (logger *Logger) Warn(args ...interface{}) {
	if logger.Level >= WarnLevel {
		NewEntry(logger).Warn(args...)
	}
}

func (logger *Logger) Warning(args ...interface{}) {
	if logger.Level >= WarnLevel {
		NewEntry(logger).Warn(args...)
	}
}

func (logger *Logger) Error(args ...interface{}) {
	if logger.Level >= ErrorLevel {
		NewEntry(logger).Error(args...)
	}
}

func (logger *Logger) Fatal(args ...interface{}) {
	if logger.Level >= FatalLevel {
		NewEntry(logger).Fatal(args...)
	}
	os.Exit(1)
}

func (logger *Logger) Panic(args ...interface{}) {
	if logger.Level >= PanicLevel {
		NewEntry(logger).Panic(args...)
	}
}

func (logger *Logger) Debugln(args ...interface{}) {
	if logger.Level >= DebugLevel {
		NewEntry(logger).Debugln(args...)
	}
}

func (logger *Logger) Infoln(args ...interface{}) {
	if logger.Level >= InfoLevel {
		NewEntry(logger).Infoln(args...)
	}
}

func (logger *Logger) Println(args ...interface{}) {
	NewEntry(logger).Println(args...)
}

func (logger *Logger) Warnln(args ...interface{}) {
	if logger.Level >= WarnLevel {
		NewEntry(logger).Warnln(args...)
	}
}

func (logger *Logger) Warningln(args ...interface{}) {
	if logger.Level >= WarnLevel {
		NewEntry(logger).Warnln(args...)
	}
}

func (logger *Logger) Errorln(args ...interface{}) {
	if logger.Level >= ErrorLevel {
		NewEntry(logger).Errorln(args...)
	}
}

func (logger *Logger) Fatalln(args ...interface{}) {
	if logger.Level >= FatalLevel {
		NewEntry(logger).Fatalln(args...)
	}
	os.Exit(1)
}

func (logger *Logger) Panicln(args ...interface{}) {
	if logger.Level >= PanicLevel {
		NewEntry(logger).Panicln(args...)
	}
}
<|endoftext|>


<commit_before>package mantle

import (
	"github.com/goibibo/mantle/backends"
)

//only strings are supported
type Mantle interface {
	Get(key string) string
	Set(key string, value interface{}) bool
	Delete(keys ...interface{}) int
	Setex(key string, duration int, value interface{}) bool
	MGet(keys ...interface{}) []string
	MSet(keyValMap map[string]interface{}) bool
	Expire(key string, duration int) bool
	Execute(cmd string, args ...interface{}) (interface{}, error)
}

//helper func
func redisConns(settings mantle.PoolSettings) *mantle.Redis {
	redis := &mantle.Redis{}
	redis.Configure(settings)
	return redis
}

//generic pool settings
func getSettings(o *Orm) mantle.PoolSettings {
	return mantle.PoolSettings{
		HostAndPorts: o.HostAndPorts,
		Capacity:     o.Capacity,
		MaxCapacity:  o.Capacity,
		Options:      o.Options}

}

//this struct is exported
type Orm struct {
	//redis|memcache|cassandra
	Driver string
	//arrays of ip:port,ip:port
	HostAndPorts []string
	//pool size
	Capacity int
	//any other options thats needed for creating a connection
	Options map[string]string
}

//mantle is a wrapper for many nosql dbs
func (o *Orm) New() Mantle {
	settings := getSettings(o)
	if o.Driver == "memcache" {
		return redisConns(settings)
	} else {
		return redisConns(settings)
	}
}

//override mantle and get a redis client
func (o *Orm) GetRedisConn() (*mantle.RedisConn, error) {
	settings := getSettings(o)
	redisPool := redisConns(settings)
	return redisPool.GetClient()
}
<commit_msg>adding memcache<commit_after>package mantle

import (
	"github.com/goibibo/mantle/backends"
)

//only strings are supported
type Mantle interface {
	Get(key string) string
	Set(key string, value interface{}) bool
	Delete(keys ...interface{}) int
	Setex(key string, duration int, value interface{}) bool
	MGet(keys ...interface{}) []string
	MSet(keyValMap map[string]interface{}) bool
	Expire(key string, duration int) bool
	Execute(cmd string, args ...interface{}) (interface{}, error)
}

//helper func
func redisConns(settings mantle.PoolSettings) *mantle.Redis {
	redis := &mantle.Redis{}
	redis.Configure(settings)
	return redis
}

func memcacheConns(settings mantle.PoolSettings) *mantle.Memcache {
	redis := &mantle.Memcache{}
	redis.Configure(settings)
	return redis
}

//generic pool settings
func getSettings(o *Orm) mantle.PoolSettings {
	return mantle.PoolSettings{
		HostAndPorts: o.HostAndPorts,
		Capacity:     o.Capacity,
		MaxCapacity:  o.Capacity,
		Options:      o.Options}

}

//this struct is exported
type Orm struct {
	//redis|memcache|cassandra
	Driver string
	//arrays of ip:port,ip:port
	HostAndPorts []string
	//pool size
	Capacity int
	//any other options thats needed for creating a connection
	Options map[string]string
}

//mantle is a wrapper for many nosql dbs
func (o *Orm) New() Mantle {
	settings := getSettings(o)
	if o.Driver == "memcache" {
		return memcacheConns(settings)
	} else {
		return redisConns(settings)
	}
}

//override mantle and get a redis client
func (o *Orm) GetRedisConn() (*mantle.RedisConn, error) {
	settings := getSettings(o)
	redisPool := redisConns(settings)
	return redisPool.GetClient()
}
<|endoftext|>