<commit_before>// Copyright 2019 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package hmac

import (
	"context"
	"fmt"
	"io/ioutil"
	"os"
	"testing"
	"time"

	"github.com/GoogleCloudPlatform/golang-samples/internal/testutil"

	"cloud.google.com/go/storage"
)

var serviceAccountEmail = os.Getenv("GOLANG_SAMPLES_SERVICE_ACCOUNT_EMAIL")
var storageClient *storage.Client

func TestMain(m *testing.M) {
	if serviceAccountEmail == "" {
		fmt.Fprintln(os.Stderr, "GOLANG_SAMPLES_SERVICE_ACCOUNT_EMAIL not set. Skipping.")
		return
	}
	ctx := context.Background()
	storageClient, _ = storage.NewClient(ctx)
	defer storageClient.Close()

	os.Exit(m.Run())
}

func TestListKeys(t *testing.T) {
	tc := testutil.SystemTest(t)
	key, err := createTestKey(tc.ProjectID, t)
	defer deleteTestKey(key)
	if err != nil {
		t.Fatalf("Error in key creation: %s", err)
	}

	testutil.Retry(t, 10, 10*time.Second, func(r *testutil.R) {
		keys, err := listHMACKeys(ioutil.Discard, tc.ProjectID)
		if err != nil {
			r.Errorf("listHMACKeys raised error: %s", err)
		}
		if len(keys) < 1 {
			r.Errorf("Should have at least one key listed.")
		}
	})
}

func TestCreateKey(t *testing.T) {
	tc := testutil.SystemTest(t)
	key, err := createHMACKey(ioutil.Discard, tc.ProjectID, serviceAccountEmail)
	defer deleteTestKey(key)

	if err != nil {
		t.Errorf("createHMACKey raised error: %s", err)
	}
	if key == nil {
		t.Errorf("Returned nil key.")
	}
	if key.State != "ACTIVE" {
		t.Errorf("State of key is %s, should be ACTIVE", key.State)
	}
}

func TestActivateKey(t *testing.T) {
	tc := testutil.SystemTest(t)
	key, err := createTestKey(tc.ProjectID, t)
	defer deleteTestKey(key)
	if err != nil {
		t.Errorf("Error in key creation: %s", err)
	}

	// Key must first be deactivated in order to update to active state.
	ctx := context.Background()
	handle := storageClient.HMACKeyHandle(key.ProjectID, key.AccessID)
	handle.Update(ctx, storage.HMACKeyAttrsToUpdate{State: "INACTIVE"})

	key, err = activateHMACKey(ioutil.Discard, key.AccessID, key.ProjectID)

	if err != nil {
		t.Errorf("Error in activateHMACKey: %s", err)
	}
	if key.State != "ACTIVE" {
		t.Errorf("State of key is %s, should be ACTIVE", key.State)
	}
}

func TestDeactivateKey(t *testing.T) {
	tc := testutil.SystemTest(t)
	key, err := createTestKey(tc.ProjectID, t)
	defer deleteTestKey(key)
	if err != nil {
		t.Errorf("Error in key creation: %s", err)
	}

	key, err = deactivateHMACKey(ioutil.Discard, key.AccessID, key.ProjectID)
	if err != nil {
		t.Errorf("Error in deactivateHMACKey: %s", err)
	}
	if key.State != "INACTIVE" {
		t.Errorf("State of key is %s, should be INACTIVE", key.State)
	}
}

func TestGetKey(t *testing.T) {
	tc := testutil.SystemTest(t)
	key, err := createTestKey(tc.ProjectID, t)
	defer deleteTestKey(key)
	if err != nil {
		t.Errorf("Error in key creation: %s", err)
	}

	testutil.Retry(t, 10, 10*time.Second, func(r *testutil.R) {
		key, err = getHMACKey(ioutil.Discard, key.AccessID, key.ProjectID)
		if err != nil {
			r.Errorf("Error in getHMACKey: %s", err)
			return
		}
		if key == nil {
			r.Errorf("Returned nil key.")
		}
	})
}

func TestDeleteKey(t *testing.T) {
	tc := testutil.SystemTest(t)
	key, err := createTestKey(tc.ProjectID, t)
	defer deleteTestKey(key)
	if err != nil {
		t.Errorf("Error in key creation: %s", err)
	}

	// Keys must be in INACTIVE state before deletion.
	ctx := context.Background()
	handle := storageClient.HMACKeyHandle(key.ProjectID, key.AccessID)
	handle.Update(ctx, storage.HMACKeyAttrsToUpdate{State: "INACTIVE"})

	err = deleteHMACKey(ioutil.Discard, key.AccessID, key.ProjectID)
	if err != nil {
		t.Errorf("Error in deleteHMACKey: %s", err)
	}
	key, _ = handle.Get(ctx)
	if key.State != "DELETED" {
		t.Errorf("State of key is %s, should be DELETED", key.State)
	}

}

// Create a key for testing purposes.
func createTestKey(projectID string, t *testing.T) (*storage.HMACKey, error) {
	ctx := context.Background()
	var key *storage.HMACKey
	var err error

	testutil.Retry(t, 10, 10*time.Second, func(r *testutil.R) {
		key, err = storageClient.CreateHMACKey(ctx, projectID, serviceAccountEmail)
		if err != nil {
			r.Errorf("Error in CreateHMACKey: %v", err)
			return
		}

		// Nil key check should not happen but is added to handle flaky
		// "nil pointer dereference" error.
		if key == nil {
			r.Errorf("Returned nil key.")
			return
		}
	})

	return key, err
}

// Deactivate and delete the given key. Should operate as a teardown method.
func deleteTestKey(key *storage.HMACKey) {
	ctx := context.Background()
	handle := storageClient.HMACKeyHandle(key.ProjectID, key.AccessID)
	if key.State == "ACTIVE" {
		handle.Update(ctx, storage.HMACKeyAttrsToUpdate{State: "INACTIVE"})
	}
	if key.State != "DELETED" {
		handle.Delete(ctx)
	}
}
<commit_msg>test(storage): fix HMAC key creation (#2071)<commit_after>// Copyright 2019 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package hmac

import (
	"context"
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"testing"
	"time"

	"github.com/GoogleCloudPlatform/golang-samples/internal/testutil"

	"cloud.google.com/go/storage"
)

var serviceAccountEmail = os.Getenv("GOLANG_SAMPLES_SERVICE_ACCOUNT_EMAIL")
var storageClient *storage.Client

func TestMain(m *testing.M) {
	if serviceAccountEmail == "" {
		fmt.Fprintln(os.Stderr, "GOLANG_SAMPLES_SERVICE_ACCOUNT_EMAIL not set. Skipping.")
		return
	}
	ctx := context.Background()
	storageClient, _ = storage.NewClient(ctx)
	defer storageClient.Close()

	os.Exit(m.Run())
}

func TestListKeys(t *testing.T) {
	tc := testutil.SystemTest(t)
	key, err := createTestKey(tc.ProjectID, t)
	defer deleteTestKey(key)
	if err != nil {
		t.Fatalf("Error in key creation: %s", err)
	}

	testutil.Retry(t, 10, 10*time.Second, func(r *testutil.R) {
		keys, err := listHMACKeys(ioutil.Discard, tc.ProjectID)
		if err != nil {
			r.Errorf("listHMACKeys raised error: %s", err)
		}
		if len(keys) < 1 {
			r.Errorf("Should have at least one key listed.")
		}
	})
}

func TestCreateKey(t *testing.T) {
	tc := testutil.SystemTest(t)
	key, err := createHMACKey(ioutil.Discard, tc.ProjectID, serviceAccountEmail)
	defer deleteTestKey(key)

	if err != nil {
		t.Errorf("createHMACKey raised error: %s", err)
	}
	if key == nil {
		t.Errorf("Returned nil key.")
	}
	if key.State != "ACTIVE" {
		t.Errorf("State of key is %s, should be ACTIVE", key.State)
	}
}

func TestActivateKey(t *testing.T) {
	tc := testutil.SystemTest(t)
	key, err := createTestKey(tc.ProjectID, t)
	defer deleteTestKey(key)
	if err != nil {
		t.Fatalf("Error in key creation: %s", err)
	}

	// Key must first be deactivated in order to update to active state.
	ctx := context.Background()
	handle := storageClient.HMACKeyHandle(key.ProjectID, key.AccessID)
	handle.Update(ctx, storage.HMACKeyAttrsToUpdate{State: "INACTIVE"})

	key, err = activateHMACKey(ioutil.Discard, key.AccessID, key.ProjectID)

	if err != nil {
		t.Errorf("Error in activateHMACKey: %s", err)
	}
	if key.State != "ACTIVE" {
		t.Errorf("State of key is %s, should be ACTIVE", key.State)
	}
}

func TestDeactivateKey(t *testing.T) {
	tc := testutil.SystemTest(t)
	key, err := createTestKey(tc.ProjectID, t)
	defer deleteTestKey(key)
	if err != nil {
		t.Fatalf("Error in key creation: %s", err)
	}

	key, err = deactivateHMACKey(ioutil.Discard, key.AccessID, key.ProjectID)
	if err != nil {
		t.Errorf("Error in deactivateHMACKey: %s", err)
	}
	if key.State != "INACTIVE" {
		t.Errorf("State of key is %s, should be INACTIVE", key.State)
	}
}

func TestGetKey(t *testing.T) {
	tc := testutil.SystemTest(t)
	key, err := createTestKey(tc.ProjectID, t)
	defer deleteTestKey(key)
	if err != nil {
		t.Fatalf("Error in key creation: %s", err)
	}

	testutil.Retry(t, 10, 10*time.Second, func(r *testutil.R) {
		key, err = getHMACKey(ioutil.Discard, key.AccessID, key.ProjectID)
		if err != nil {
			r.Errorf("Error in getHMACKey: %s", err)
			return
		}
		if key == nil {
			r.Errorf("Returned nil key.")
		}
	})
}

func TestDeleteKey(t *testing.T) {
	tc := testutil.SystemTest(t)
	key, err := createTestKey(tc.ProjectID, t)
	defer deleteTestKey(key)
	if err != nil {
		t.Fatalf("Error in key creation: %s", err)
	}

	// Keys must be in INACTIVE state before deletion.
	ctx := context.Background()
	handle := storageClient.HMACKeyHandle(key.ProjectID, key.AccessID)
	handle.Update(ctx, storage.HMACKeyAttrsToUpdate{State: "INACTIVE"})

	err = deleteHMACKey(ioutil.Discard, key.AccessID, key.ProjectID)
	if err != nil {
		t.Errorf("Error in deleteHMACKey: %s", err)
	}
	key, _ = handle.Get(ctx)
	if key.State != "DELETED" {
		t.Errorf("State of key is %s, should be DELETED", key.State)
	}

}

// Create a key for testing purposes.
func createTestKey(projectID string, t *testing.T) (*storage.HMACKey, error) {
	ctx := context.Background()
	var key *storage.HMACKey
	var err error

	testutil.Retry(t, 10, 10*time.Second, func(r *testutil.R) {
		key, err = storageClient.CreateHMACKey(ctx, projectID, serviceAccountEmail)
		if err != nil {
			r.Errorf("Error in CreateHMACKey: %v", err)
			return
		}

		// Nil key check should not happen but is added to handle flaky
		// "nil pointer dereference" error.
		if key == nil {
			r.Errorf("CreateHMACKey returned nil key.")
			err = errors.New("CreateHMACKey returned nil key")
			return
		}
	})

	return key, err
}

// Deactivate and delete the given key. Should operate as a teardown method.
func deleteTestKey(key *storage.HMACKey) {
	if key == nil {
		return
	}
	ctx := context.Background()
	handle := storageClient.HMACKeyHandle(key.ProjectID, key.AccessID)
	if key.State == "ACTIVE" {
		handle.Update(ctx, storage.HMACKeyAttrsToUpdate{State: "INACTIVE"})
	}
	if key.State != "DELETED" {
		handle.Delete(ctx)
	}
}
<|endoftext|>


<commit_before>package yang

import (
	"github.com/openconfig/goyang/pkg/yang"
	"github.com/oshothebig/pbast"
)

// e must be YANG module
func Transform(e *yang.Entry) pbast.File {
	if _, ok := e.Node.(*yang.Module); !ok {
		return pbast.File{}
	}

	return transformModule(entry{e})
}

func transformModule(e entry) pbast.File {
	namespace := e.Namespace().Name
	elems := guessElements(namespace)
	f := pbast.NewFile(pbast.NewPackageWithElements(elems))

	f = f.AddService(transformRPCs(e))

	return f
}

func transformRPCs(e entry) pbast.Service {
	rpcs := e.rpcs()
	if len(rpcs) == 0 {
		return pbast.Service{}
	}

	s := pbast.NewService(CamelCase(e.Name))
	for _, rpc := range rpcs {
		s = s.AddRPC(transformRPC(rpc))
	}

	return s
}

func transformRPC(e entry) pbast.RPC {
	method := CamelCase(e.Name)

	return pbast.NewRPC(
		method,
		pbast.NewReturnType(method+"Request"),
		pbast.NewReturnType(method+"Response"),
	)
}
<commit_msg>Inline variable<commit_after>package yang

import (
	"github.com/openconfig/goyang/pkg/yang"
	"github.com/oshothebig/pbast"
)

// e must be YANG module
func Transform(e *yang.Entry) pbast.File {
	if _, ok := e.Node.(*yang.Module); !ok {
		return pbast.File{}
	}

	return transformModule(entry{e})
}

func transformModule(e entry) pbast.File {
	namespace := e.Namespace().Name
	f := pbast.NewFile(pbast.NewPackageWithElements(guessElements(namespace)))

	f = f.AddService(transformRPCs(e))

	return f
}

func transformRPCs(e entry) pbast.Service {
	rpcs := e.rpcs()
	if len(rpcs) == 0 {
		return pbast.Service{}
	}

	s := pbast.NewService(CamelCase(e.Name))
	for _, rpc := range rpcs {
		s = s.AddRPC(transformRPC(rpc))
	}

	return s
}

func transformRPC(e entry) pbast.RPC {
	method := CamelCase(e.Name)

	return pbast.NewRPC(
		method,
		pbast.NewReturnType(method+"Request"),
		pbast.NewReturnType(method+"Response"),
	)
}
<|endoftext|>


<commit_before>package cache

import (
	"errors"
	"log"
	"sync/atomic"
	"time"

	"github.com/go-redis/redis"

	"github.com/go-redis/cache/internal/lrucache"
	"github.com/go-redis/cache/internal/singleflight"
)

var ErrCacheMiss = errors.New("cache: key is missing")

type rediser interface {
	Set(key string, value interface{}, expiration time.Duration) *redis.StatusCmd
	Get(key string) *redis.StringCmd
	Del(keys ...string) *redis.IntCmd
}

type Item struct {
	Key    string
	Object interface{}

	// Func returns object to be cached.
	Func func() (interface{}, error)

	// Expiration is the cache expiration time.
	// Default expiration is 1 hour.
	Expiration time.Duration
}

func (item *Item) object() (interface{}, error) {
	if item.Object != nil {
		return item.Object, nil
	}
	if item.Func != nil {
		return item.Func()
	}
	return nil, nil
}

func (item *Item) exp() time.Duration {
	if item.Expiration < 0 {
		return 0
	}
	if item.Expiration < time.Second {
		return time.Hour
	}
	return item.Expiration
}

type Codec struct {
	Redis rediser

	localCache *lrucache.Cache

	Marshal   func(interface{}) ([]byte, error)
	Unmarshal func([]byte, interface{}) error

	group singleflight.Group

	hits        uint64
	misses      uint64
	localHits   uint64
	localMisses uint64
}

// UseLocalCache causes Codec to cache items in local LRU cache.
func (cd *Codec) UseLocalCache(maxLen int, expiration time.Duration) {
	cd.localCache = lrucache.New(maxLen, expiration)
}

// Set caches the item.
func (cd *Codec) Set(item *Item) error {
	_, err := cd.set(item)
	return err
}

func (cd *Codec) set(item *Item) ([]byte, error) {
	object, err := item.object()
	if err != nil {
		return nil, err
	}

	b, err := cd.Marshal(object)
	if err != nil {
		log.Printf("cache: Marshal key=%q failed: %s", item.Key, err)
		return nil, err
	}

	if cd.localCache != nil {
		cd.localCache.Set(item.Key, b)
	}

	if cd.Redis == nil {
		return b, nil
	}

	err = cd.Redis.Set(item.Key, b, item.exp()).Err()
	if err != nil {
		log.Printf("cache: Set key=%q failed: %s", item.Key, err)
		return nil, err
	}
	return b, nil
}

// Exists reports whether object for the given key exists.
func (cd *Codec) Exists(key string) bool {
	return cd.Get(key, nil) == nil
}

// Get gets the object for the given key.
func (cd *Codec) Get(key string, object interface{}) error {
	return cd.get(key, object, false)
}

func (cd *Codec) get(key string, object interface{}, onlyLocalCache bool) error {
	b, err := cd.getBytes(key, onlyLocalCache)
	if err != nil {
		return err
	}

	if object == nil || len(b) == 0 {
		return nil
	}

	if err := cd.Unmarshal(b, object); err != nil {
		log.Printf("cache: key=%q Unmarshal(%T) failed: %s", key, object, err)
		return err
	}

	return nil
}

func (cd *Codec) getBytes(key string, onlyLocalCache bool) ([]byte, error) {
	if cd.localCache != nil {
		b, ok := cd.localCache.Get(key)
		if ok {
			atomic.AddUint64(&cd.localHits, 1)
			return b, nil
		}
		atomic.AddUint64(&cd.localMisses, 1)
	}

	if onlyLocalCache || cd.Redis == nil {
		return nil, ErrCacheMiss
	}

	b, err := cd.Redis.Get(key).Bytes()
	if err != nil {
		atomic.AddUint64(&cd.misses, 1)
		if err == redis.Nil {
			return nil, ErrCacheMiss
		}
		log.Printf("cache: Get key=%q failed: %s", key, err)
		return nil, err
	}
	atomic.AddUint64(&cd.hits, 1)

	if cd.localCache != nil {
		cd.localCache.Set(key, b)
	}
	return b, nil
}

// Once gets the item.Object for the given item.Key from the cache or
// executes, caches, and returns the results of the given item.Func,
// making sure that only one execution is in-flight for a given item.Key
// at a time. If a duplicate comes in, the duplicate caller waits for the
// original to complete and receives the same results.
func (cd *Codec) Once(item *Item) error {
	b, err := cd.getItemBytesOnce(item)
	if err != nil {
		return err
	}

	if item.Object == nil || len(b) == 0 {
		return nil
	}

	err = cd.Unmarshal(b, item.Object)
	if err != nil {
		log.Printf("cache: key=%q Unmarshal(%T) failed: %s", item.Key, item.Object, err)
		return err
	}

	return nil
}

func (cd *Codec) getItemBytesOnce(item *Item) ([]byte, error) {
	if cd.localCache != nil {
		b, err := cd.getItemBytesFast(item)
		if err == nil {
			return b, nil
		}
	}

	obj, err := cd.group.Do(item.Key, func() (interface{}, error) {
		b, err := cd.getItemBytes(item)
		if err == nil {
			return b, nil
		}

		obj, err := item.Func()
		if err != nil {
			return nil, err
		}

		b, err = cd.set(&Item{
			Key:        item.Key,
			Object:     obj,
			Expiration: item.Expiration,
		})
		return b, err
	})
	if err != nil {
		return nil, err
	}
	return obj.([]byte), nil
}

func (cd *Codec) getItemBytes(item *Item) ([]byte, error) {
	return cd.getBytes(item.Key, false)
}

func (cd *Codec) getItemBytesFast(item *Item) ([]byte, error) {
	return cd.getBytes(item.Key, true)
}

func (cd *Codec) Delete(key string) error {
	if cd.localCache != nil {
		cd.localCache.Delete(key)
	}

	if cd.Redis == nil {
		return nil
	}

	deleted, err := cd.Redis.Del(key).Result()
	if err != nil {
		log.Printf("cache: Del key=%q failed: %s", key, err)
		return err
	}
	if deleted == 0 {
		return ErrCacheMiss
	}
	return nil
}

type Stats struct {
	Hits        uint64
	Misses      uint64
	LocalHits   uint64
	LocalMisses uint64
}

// Stats returns cache statistics.
func (cd *Codec) Stats() *Stats {
	stats := Stats{
		Hits:   atomic.LoadUint64(&cd.hits),
		Misses: atomic.LoadUint64(&cd.misses),
	}
	if cd.localCache != nil {
		stats.LocalHits = atomic.LoadUint64(&cd.localHits)
		stats.LocalMisses = atomic.LoadUint64(&cd.localMisses)
	}
	return &stats
}
<commit_msg>Check that either Redis or localCache is configured<commit_after>package cache

import (
	"errors"
	"log"
	"sync/atomic"
	"time"

	"github.com/go-redis/redis"

	"github.com/go-redis/cache/internal/lrucache"
	"github.com/go-redis/cache/internal/singleflight"
)

var ErrCacheMiss = errors.New("cache: key is missing")
var errRedisLocalCacheNil = errors.New("cache: both Redis and LocalCache are nil")

type rediser interface {
	Set(key string, value interface{}, expiration time.Duration) *redis.StatusCmd
	Get(key string) *redis.StringCmd
	Del(keys ...string) *redis.IntCmd
}

type Item struct {
	Key    string
	Object interface{}

	// Func returns object to be cached.
	Func func() (interface{}, error)

	// Expiration is the cache expiration time.
	// Default expiration is 1 hour.
	Expiration time.Duration
}

func (item *Item) object() (interface{}, error) {
	if item.Object != nil {
		return item.Object, nil
	}
	if item.Func != nil {
		return item.Func()
	}
	return nil, nil
}

func (item *Item) exp() time.Duration {
	if item.Expiration < 0 {
		return 0
	}
	if item.Expiration < time.Second {
		return time.Hour
	}
	return item.Expiration
}

type Codec struct {
	Redis rediser

	localCache *lrucache.Cache

	Marshal   func(interface{}) ([]byte, error)
	Unmarshal func([]byte, interface{}) error

	group singleflight.Group

	hits        uint64
	misses      uint64
	localHits   uint64
	localMisses uint64
}

// UseLocalCache causes Codec to cache items in local LRU cache.
func (cd *Codec) UseLocalCache(maxLen int, expiration time.Duration) {
	cd.localCache = lrucache.New(maxLen, expiration)
}

// Set caches the item.
func (cd *Codec) Set(item *Item) error {
	_, err := cd.set(item)
	return err
}

func (cd *Codec) set(item *Item) ([]byte, error) {
	object, err := item.object()
	if err != nil {
		return nil, err
	}

	b, err := cd.Marshal(object)
	if err != nil {
		log.Printf("cache: Marshal key=%q failed: %s", item.Key, err)
		return nil, err
	}

	if cd.localCache != nil {
		cd.localCache.Set(item.Key, b)
	}

	if cd.Redis == nil {
		if cd.localCache == nil {
			return nil, errRedisLocalCacheNil
		}
		return b, nil
	}

	err = cd.Redis.Set(item.Key, b, item.exp()).Err()
	if err != nil {
		log.Printf("cache: Set key=%q failed: %s", item.Key, err)
		return nil, err
	}
	return b, nil
}

// Exists reports whether object for the given key exists.
func (cd *Codec) Exists(key string) bool {
	return cd.Get(key, nil) == nil
}

// Get gets the object for the given key.
func (cd *Codec) Get(key string, object interface{}) error {
	return cd.get(key, object, false)
}

func (cd *Codec) get(key string, object interface{}, onlyLocalCache bool) error {
	b, err := cd.getBytes(key, onlyLocalCache)
	if err != nil {
		return err
	}

	if object == nil || len(b) == 0 {
		return nil
	}

	if err := cd.Unmarshal(b, object); err != nil {
		log.Printf("cache: key=%q Unmarshal(%T) failed: %s", key, object, err)
		return err
	}

	return nil
}

func (cd *Codec) getBytes(key string, onlyLocalCache bool) ([]byte, error) {
	if cd.localCache != nil {
		b, ok := cd.localCache.Get(key)
		if ok {
			atomic.AddUint64(&cd.localHits, 1)
			return b, nil
		}
		atomic.AddUint64(&cd.localMisses, 1)
	}

	if onlyLocalCache {
		return nil, ErrCacheMiss
	}
	if cd.Redis == nil {
		if cd.localCache == nil {
			return nil, errRedisLocalCacheNil
		}
		return nil, ErrCacheMiss
	}

	b, err := cd.Redis.Get(key).Bytes()
	if err != nil {
		atomic.AddUint64(&cd.misses, 1)
		if err == redis.Nil {
			return nil, ErrCacheMiss
		}
		log.Printf("cache: Get key=%q failed: %s", key, err)
		return nil, err
	}
	atomic.AddUint64(&cd.hits, 1)

	if cd.localCache != nil {
		cd.localCache.Set(key, b)
	}
	return b, nil
}

// Once gets the item.Object for the given item.Key from the cache or
// executes, caches, and returns the results of the given item.Func,
// making sure that only one execution is in-flight for a given item.Key
// at a time. If a duplicate comes in, the duplicate caller waits for the
// original to complete and receives the same results.
func (cd *Codec) Once(item *Item) error {
	b, err := cd.getItemBytesOnce(item)
	if err != nil {
		return err
	}

	if item.Object == nil || len(b) == 0 {
		return nil
	}

	err = cd.Unmarshal(b, item.Object)
	if err != nil {
		log.Printf("cache: key=%q Unmarshal(%T) failed: %s", item.Key, item.Object, err)
		return err
	}

	return nil
}

func (cd *Codec) getItemBytesOnce(item *Item) ([]byte, error) {
	if cd.localCache != nil {
		b, err := cd.getItemBytesFast(item)
		if err == nil {
			return b, nil
		}
	}

	obj, err := cd.group.Do(item.Key, func() (interface{}, error) {
		b, err := cd.getItemBytes(item)
		if err == nil {
			return b, nil
		}

		obj, err := item.Func()
		if err != nil {
			return nil, err
		}

		b, err = cd.set(&Item{
			Key:        item.Key,
			Object:     obj,
			Expiration: item.Expiration,
		})
		return b, err
	})
	if err != nil {
		return nil, err
	}
	return obj.([]byte), nil
}

func (cd *Codec) getItemBytes(item *Item) ([]byte, error) {
	return cd.getBytes(item.Key, false)
}

func (cd *Codec) getItemBytesFast(item *Item) ([]byte, error) {
	return cd.getBytes(item.Key, true)
}

func (cd *Codec) Delete(key string) error {
	if cd.localCache != nil {
		cd.localCache.Delete(key)
	}

	if cd.Redis == nil {
		if cd.localCache == nil {
			return errRedisLocalCacheNil
		}
		return nil
	}

	deleted, err := cd.Redis.Del(key).Result()
	if err != nil {
		log.Printf("cache: Del key=%q failed: %s", key, err)
		return err
	}
	if deleted == 0 {
		return ErrCacheMiss
	}
	return nil
}

type Stats struct {
	Hits        uint64
	Misses      uint64
	LocalHits   uint64
	LocalMisses uint64
}

// Stats returns cache statistics.
func (cd *Codec) Stats() *Stats {
	stats := Stats{
		Hits:   atomic.LoadUint64(&cd.hits),
		Misses: atomic.LoadUint64(&cd.misses),
	}
	if cd.localCache != nil {
		stats.LocalHits = atomic.LoadUint64(&cd.localHits)
		stats.LocalMisses = atomic.LoadUint64(&cd.localMisses)
	}
	return &stats
}
<|endoftext|>


<commit_before>package cache

import (
	"encoding/gob"
	"fmt"
	"io"
	"os"
	"reflect"
	"runtime"
	"sync"
	"time"
)

const Version = "1.0"

type Item struct {
	Object     interface{}
	Expiration *time.Time
}

// Returns true if the item has expired.
func (i *Item) Expired() bool {
	if i.Expiration == nil {
		return false
	}
	return i.Expiration.Before(time.Now())
}

type Cache struct {
	*cache
	// If this is confusing, see the comment at the bottom of the New() function
}

type cache struct {
	DefaultExpiration time.Duration
	Items             map[string]*Item
	// TODO: Calls to mu.Unlock are currently not deferred because defer
	// adds ~200 ns (as of 792c7561af4b+ tip.)
	mu                sync.Mutex
	janitor           *janitor
}

// Adds an item to the cache, replacing any existing item. If the duration is 0, the
// cache's default expiration time is used. If it is -1, the item never expires.
func (c *cache) Set(k string, x interface{}, d time.Duration) {
	c.mu.Lock()
	c.set(k, x, d)
	c.mu.Unlock()
}

func (c *cache) set(k string, x interface{}, d time.Duration) {
	var e *time.Time
	if d == 0 {
		d = c.DefaultExpiration
	}
	if d > 0 {
		t := time.Now().Add(d)
		e = &t
	}
	c.Items[k] = &Item{
		Object:     x,
		Expiration: e,
	}
}

// Adds an item to the cache only if an item doesn't already exist for the given key,
// or if the existing item has expired. Returns an error if not.
func (c *cache) Add(k string, x interface{}, d time.Duration) error {
	c.mu.Lock()
	_, found := c.get(k)
	if found {
		c.mu.Unlock()
		return fmt.Errorf("Item %s already exists", k)
	}
	c.set(k, x, d)
	c.mu.Unlock()
	return nil
}

// Sets a new value for the cache item only if it already exists. Returns an error if
// it does not.
func (c *cache) Replace(k string, x interface{}, d time.Duration) error {
	c.mu.Lock()
	_, found := c.get(k)
	if !found {
		c.mu.Unlock()
		return fmt.Errorf("Item %s doesn't exist", k)
	}
	c.set(k, x, d)
	c.mu.Unlock()
	return nil
}

// Gets an item from the cache. Returns the item or nil, and a bool indicating whether
// the given key was found in the cache.
func (c *cache) Get(k string) (interface{}, bool) {
	c.mu.Lock()
	x, found := c.get(k)
	c.mu.Unlock()
	return x, found
}

func (c *cache) get(k string) (interface{}, bool) {
	item, found := c.Items[k]
	if !found {
		return nil, false
	}
	if item.Expired() {
		c.delete(k)
		return nil, false
	}
	return item.Object, true
}

// Increment an item of type int, int8, int16, int32, int64, uintptr, uint, uint8,
// uint32, uint64, float32 or float64 by n. Returns an error if the item's value is
// not an integer, if it was not found, or if it is not possible to increment it by
// n. Passing a negative number will cause the item to be decremented.
func (c *cache) IncrementFloat(k string, n float64) error {
	c.mu.Lock()
	v, found := c.Items[k]
	if !found || v.Expired() {
		c.mu.Unlock()
		return fmt.Errorf("Item not found")
	}

	t := reflect.TypeOf(v.Object)
	switch t.Kind() {
	default:
		c.mu.Unlock()
		return fmt.Errorf("The value of %s is not an integer", k)
	case reflect.Uint:
		v.Object = v.Object.(uint) + uint(n)
	case reflect.Uintptr:
		v.Object = v.Object.(uintptr) + uintptr(n)
	case reflect.Uint8:
		v.Object = v.Object.(uint8) + uint8(n)
	case reflect.Uint16:
		v.Object = v.Object.(uint16) + uint16(n)
	case reflect.Uint32:
		v.Object = v.Object.(uint32) + uint32(n)
	case reflect.Uint64:
		v.Object = v.Object.(uint64) + uint64(n)
	case reflect.Int:
		v.Object = v.Object.(int) + int(n)
	case reflect.Int8:
		v.Object = v.Object.(int8) + int8(n)
	case reflect.Int16:
		v.Object = v.Object.(int16) + int16(n)
	case reflect.Int32:
		v.Object = v.Object.(int32) + int32(n)
	case reflect.Int64:
		v.Object = v.Object.(int64) + int64(n)
	case reflect.Float32:
		v.Object = v.Object.(float32) + float32(n)
	case reflect.Float64:
		v.Object = v.Object.(float64) + n
	}
	c.mu.Unlock()
	return nil
}

// Increment an item of type int, int8, int16, int32, int64, uintptr, uint, uint8,
// uint32, or uint64, float32 or float64 by n. Returns an error if the item's value
// is not an integer, if it was not found, or if it is not possible to increment it
// by n. Passing a negative number will cause the item to be decremented.
func (c *cache) Increment(k string, n int64) error {
	return c.IncrementFloat(k, float64(n))
}

// Decrement an item of type int, int8, int16, int32, int64, uintptr, uint, uint8,
// uint32, or uint64, float32 or float64 by n. Returns an error if the item's value
// is not an integer, if it was not found, or if it is not possible to decrement it
// by n.
func (c *cache) Decrement(k string, n int64) error {
	return c.Increment(k, n*-1)
}

// Deletes an item from the cache. Does nothing if the item does not exist in the cache.
func (c *cache) Delete(k string) {
	c.mu.Lock()
	c.delete(k)
	c.mu.Unlock()
}

func (c *cache) delete(k string) {
	delete(c.Items, k)
}

// Deletes all expired items from the cache.
func (c *cache) DeleteExpired() {
	c.mu.Lock()
	for k, v := range c.Items {
		if v.Expired() {
			c.delete(k)
		}
	}
	c.mu.Unlock()
}

// Writes the cache's items (using Gob) to an io.Writer.
func (c *cache) Save(w io.Writer) error {
	enc := gob.NewEncoder(w)

	var err error
	defer func() {
		if x := recover(); x != nil {
			fmt.Printf(`The Gob library paniced while registering the cache's item types!
Information: %v

The cache will not be saved.
Please report under what conditions this happened, and particularly what special type of objects
were stored in cache, at https://github.com/pmylund/go-cache/issues/new
`, x)
			err = fmt.Errorf("Error registering item types with Gob library")
		}
	}()
	for _, v := range c.Items {
		gob.Register(v.Object)
	}
	err = enc.Encode(&c.Items)
	return err
}

// Saves the cache's items to the given filename, creating the file if it
// doesn't exist, and overwriting it if it does.
func (c *cache) SaveFile(fname string) error {
	fp, err := os.Create(fname)
	if err != nil {
		return err
	}
	return c.Save(fp)
}

// Adds (Gob-serialized) cache items from an io.Reader, excluding any items that
// already exist in the current cache.
func (c *cache) Load(r io.Reader) error {
	dec := gob.NewDecoder(r)
	items := map[string]*Item{}
	err := dec.Decode(&items)
	if err == nil {
		for k, v := range items {
			_, found := c.Items[k]
			if !found {
				c.Items[k] = v
			}
		}
	}
	return err
}

// Loads and adds cache items from the given filename, excluding any items that
// already exist in the current cache.
func (c *cache) LoadFile(fname string) error {
	fp, err := os.Open(fname)
	if err != nil {
		return err
	}
	return c.Load(fp)
}

// Deletes all items from the cache.
func (c *cache) Flush() {
	c.mu.Lock()
	c.Items = map[string]*Item{}
	c.mu.Unlock()
}

type janitor struct {
	Interval time.Duration
	stop     chan bool
}

func (j *janitor) Run(c *cache) {
	j.stop = make(chan bool)
	tick := time.Tick(j.Interval)
	for {
		select {
		case <-tick:
			c.DeleteExpired()
		case <-j.stop:
			return
		}
	}
}

func (j *janitor) Stop() {
	j.stop <- true
}

func stopJanitor(c *Cache) {
	c.janitor.Stop()
}

// Returns a new cache with a given default expiration duration and default cleanup
// interval. If the expiration duration is less than 1, the items in the cache never
// expire and must be deleted manually. If the cleanup interval is less than one,
// expired items are not deleted from the cache before their next lookup or before
// calling DeleteExpired.
func New(de, ci time.Duration) *Cache {
	if de == 0 {
		de = -1
	}
	c := &cache{
		DefaultExpiration: de,
		Items:             map[string]*Item{},
		mu:                sync.Mutex{},
	}
	if ci > 0 {
		j := &janitor{
			Interval: ci,
		}
		c.janitor = j
		go j.Run(c)
	}
	// This trick ensures that the janitor goroutine (which--granted it was enabled--is
	// running DeleteExpired on c forever) does not keep the returned C object from being
	// garbage collected. When it is garbage collected, the finalizer stops the janitor
	// goroutine, after which c is collected.
	C := &Cache{c}
	if ci > 0 {
		runtime.SetFinalizer(C, stopJanitor)
	}
	return C
}
<commit_msg>Properly handle Gob panic<commit_after>package cache

import (
	"encoding/gob"
	"fmt"
	"io"
	"os"
	"reflect"
	"runtime"
	"sync"
	"time"
)

const Version = "1.0"

type Item struct {
	Object     interface{}
	Expiration *time.Time
}

// Returns true if the item has expired.
func (i *Item) Expired() bool {
	if i.Expiration == nil {
		return false
	}
	return i.Expiration.Before(time.Now())
}

type Cache struct {
	*cache
	// If this is confusing, see the comment at the bottom of the New() function
}

type cache struct {
	DefaultExpiration time.Duration
	Items             map[string]*Item
	// TODO: Calls to mu.Unlock are currently not deferred because defer
	// adds ~200 ns (as of 792c7561af4b+ tip.)
	mu                sync.Mutex
	janitor           *janitor
}

// Adds an item to the cache, replacing any existing item. If the duration is 0, the
// cache's default expiration time is used. If it is -1, the item never expires.
func (c *cache) Set(k string, x interface{}, d time.Duration) {
	c.mu.Lock()
	c.set(k, x, d)
	c.mu.Unlock()
}

func (c *cache) set(k string, x interface{}, d time.Duration) {
	var e *time.Time
	if d == 0 {
		d = c.DefaultExpiration
	}
	if d > 0 {
		t := time.Now().Add(d)
		e = &t
	}
	c.Items[k] = &Item{
		Object:     x,
		Expiration: e,
	}
}

// Adds an item to the cache only if an item doesn't already exist for the given key,
// or if the existing item has expired. Returns an error if not.
func (c *cache) Add(k string, x interface{}, d time.Duration) error {
	c.mu.Lock()
	_, found := c.get(k)
	if found {
		c.mu.Unlock()
		return fmt.Errorf("Item %s already exists", k)
	}
	c.set(k, x, d)
	c.mu.Unlock()
	return nil
}

// Sets a new value for the cache item only if it already exists. Returns an error if
// it does not.
func (c *cache) Replace(k string, x interface{}, d time.Duration) error {
	c.mu.Lock()
	_, found := c.get(k)
	if !found {
		c.mu.Unlock()
		return fmt.Errorf("Item %s doesn't exist", k)
	}
	c.set(k, x, d)
	c.mu.Unlock()
	return nil
}

// Gets an item from the cache. Returns the item or nil, and a bool indicating whether
// the given key was found in the cache.
func (c *cache) Get(k string) (interface{}, bool) {
	c.mu.Lock()
	x, found := c.get(k)
	c.mu.Unlock()
	return x, found
}

func (c *cache) get(k string) (interface{}, bool) {
	item, found := c.Items[k]
	if !found {
		return nil, false
	}
	if item.Expired() {
		c.delete(k)
		return nil, false
	}
	return item.Object, true
}

// Increment an item of type int, int8, int16, int32, int64, uintptr, uint, uint8,
// uint32, uint64, float32 or float64 by n. Returns an error if the item's value is
// not an integer, if it was not found, or if it is not possible to increment it by
// n. Passing a negative number will cause the item to be decremented.
func (c *cache) IncrementFloat(k string, n float64) error {
	c.mu.Lock()
	v, found := c.Items[k]
	if !found || v.Expired() {
		c.mu.Unlock()
		return fmt.Errorf("Item not found")
	}

	t := reflect.TypeOf(v.Object)
	switch t.Kind() {
	default:
		c.mu.Unlock()
		return fmt.Errorf("The value of %s is not an integer", k)
	case reflect.Uint:
		v.Object = v.Object.(uint) + uint(n)
	case reflect.Uintptr:
		v.Object = v.Object.(uintptr) + uintptr(n)
	case reflect.Uint8:
		v.Object = v.Object.(uint8) + uint8(n)
	case reflect.Uint16:
		v.Object = v.Object.(uint16) + uint16(n)
	case reflect.Uint32:
		v.Object = v.Object.(uint32) + uint32(n)
	case reflect.Uint64:
		v.Object = v.Object.(uint64) + uint64(n)
	case reflect.Int:
		v.Object = v.Object.(int) + int(n)
	case reflect.Int8:
		v.Object = v.Object.(int8) + int8(n)
	case reflect.Int16:
		v.Object = v.Object.(int16) + int16(n)
	case reflect.Int32:
		v.Object = v.Object.(int32) + int32(n)
	case reflect.Int64:
		v.Object = v.Object.(int64) + int64(n)
	case reflect.Float32:
		v.Object = v.Object.(float32) + float32(n)
	case reflect.Float64:
		v.Object = v.Object.(float64) + n
	}
	c.mu.Unlock()
	return nil
}

// Increment an item of type int, int8, int16, int32, int64, uintptr, uint, uint8,
// uint32, or uint64, float32 or float64 by n. Returns an error if the item's value
// is not an integer, if it was not found, or if it is not possible to increment it
// by n. Passing a negative number will cause the item to be decremented.
func (c *cache) Increment(k string, n int64) error {
	return c.IncrementFloat(k, float64(n))
}

// Decrement an item of type int, int8, int16, int32, int64, uintptr, uint, uint8,
// uint32, or uint64, float32 or float64 by n. Returns an error if the item's value
// is not an integer, if it was not found, or if it is not possible to decrement it
// by n.
func (c *cache) Decrement(k string, n int64) error {
	return c.Increment(k, n*-1)
}

// Deletes an item from the cache. Does nothing if the item does not exist in the cache.
func (c *cache) Delete(k string) {
	c.mu.Lock()
	c.delete(k)
	c.mu.Unlock()
}

func (c *cache) delete(k string) {
	delete(c.Items, k)
}

// Deletes all expired items from the cache.
func (c *cache) DeleteExpired() {
	c.mu.Lock()
	for k, v := range c.Items {
		if v.Expired() {
			c.delete(k)
		}
	}
	c.mu.Unlock()
}

// Writes the cache's items (using Gob) to an io.Writer.
func (c *cache) Save(w io.Writer) (err error) {
	enc := gob.NewEncoder(w)

	defer func() {
		if x := recover(); x != nil {
			fmt.Printf(`The Gob library paniced while registering the cache's item types!
Information: %v

The cache will not be saved.
Please report under what conditions this happened, and particularly what special type of objects
were stored in cache, at https://github.com/pmylund/go-cache/issues/new
`, x)
			err = fmt.Errorf("Error registering item types with Gob library")
		}
	}()
	for _, v := range c.Items {
		gob.Register(v.Object)
	}
	err = enc.Encode(&c.Items)
	return
}

// Saves the cache's items to the given filename, creating the file if it
// doesn't exist, and overwriting it if it does.
func (c *cache) SaveFile(fname string) error {
	fp, err := os.Create(fname)
	if err != nil {
		return err
	}
	return c.Save(fp)
}

// Adds (Gob-serialized) cache items from an io.Reader, excluding any items that
// already exist in the current cache.
func (c *cache) Load(r io.Reader) error {
	dec := gob.NewDecoder(r)
	items := map[string]*Item{}
	err := dec.Decode(&items)
	if err == nil {
		for k, v := range items {
			_, found := c.Items[k]
			if !found {
				c.Items[k] = v
			}
		}
	}
	return err
}

// Loads and adds cache items from the given filename, excluding any items that
// already exist in the current cache.
func (c *cache) LoadFile(fname string) error {
	fp, err := os.Open(fname)
	if err != nil {
		return err
	}
	return c.Load(fp)
}

// Deletes all items from the cache.
func (c *cache) Flush() {
	c.mu.Lock()
	c.Items = map[string]*Item{}
	c.mu.Unlock()
}

type janitor struct {
	Interval time.Duration
	stop     chan bool
}

func (j *janitor) Run(c *cache) {
	j.stop = make(chan bool)
	tick := time.Tick(j.Interval)
	for {
		select {
		case <-tick:
			c.DeleteExpired()
		case <-j.stop:
			return
		}
	}
}

func (j *janitor) Stop() {
	j.stop <- true
}

func stopJanitor(c *Cache) {
	c.janitor.Stop()
}

// Returns a new cache with a given default expiration duration and default cleanup
// interval. If the expiration duration is less than 1, the items in the cache never
// expire and must be deleted manually. If the cleanup interval is less than one,
// expired items are not deleted from the cache before their next lookup or before
// calling DeleteExpired.
func New(de, ci time.Duration) *Cache {
	if de == 0 {
		de = -1
	}
	c := &cache{
		DefaultExpiration: de,
		Items:             map[string]*Item{},
		mu:                sync.Mutex{},
	}
	if ci > 0 {
		j := &janitor{
			Interval: ci,
		}
		c.janitor = j
		go j.Run(c)
	}
	// This trick ensures that the janitor goroutine (which--granted it was enabled--is
	// running DeleteExpired on c forever) does not keep the returned C object from being
	// garbage collected. When it is garbage collected, the finalizer stops the janitor
	// goroutine, after which c is collected.
	C := &Cache{c}
	if ci > 0 {
		runtime.SetFinalizer(C, stopJanitor)
	}
	return C
}
<|endoftext|>


<commit_before>// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package runtime

import "unsafe"

// FIXME: Improve network poller for hurd.
// This is based on the former libgo/runtime/netpoll_select.c implementation
// except that it uses poll instead of select and is written in Go.
// It's also based on Solaris implementation for the arming mechanisms
// Inspiration was also taken from netpoll_aix.go and netpoll_solaris.go

//From /usr/include/x86_64-linux-gnu/sys/poll.h
//go:noescape
//extern poll
func libc_poll(pfds *pollfd, nfds int32, timeout int32) int32

//go:noescape
//extern pipe2
func libc_pipe2(fd *int32, flags int32) int32

//pollfd represents the poll structure for GNU/Hurd operating system.
type pollfd struct {
	fd      int32 // File descriptor to poll.
	events  int16 // Types of events poller cares about.
	revents int16 // Types of events that actually occurred.
}

//From /usr/include/i386-gnu/bits/poll.h
const _POLLIN = 01    // There is data to read.
const _POLLPRI = 02   // There is urgent data to read.
const _POLLOUT = 04   // Writing now will not block.
const _POLLERR = 010  // Error condition.
const _POLLHUP = 020  // Hung up.
const _POLLNVAL = 040 // Invalid polling request.

var (
	pfds           []pollfd
	pds            []*pollDesc
	mtxpoll        mutex
	mtxset         mutex
	rdwake         int32
	wrwake         int32
	pendingUpdates int32
)

const pollVerbose = false

func netpollinit() {
	var p [2]int32

	// Create the pipe we use to wakeup poll.
	if err := libc_pipe2(&p[0], _O_CLOEXEC|_O_NONBLOCK); err < 0 {
		throw("runtime:netpollinit(): failed to create pipe2")
	}
	rdwake = p[0]
	wrwake = p[1]

	// Pre-allocate array of pollfd structures for poll.
	if pollVerbose {
		println("*** allocating")
	}
	pfds = make([]pollfd, 1, 128)
	if pollVerbose {
		println("*** allocating done", &pfds[0])
	}

	// Poll the read side of the pipe.
	pfds[0].fd = int32(rdwake)
	pfds[0].events = int16(_POLLIN)
	pfds[0].revents = int16(0)

	pds = make([]*pollDesc, 1, 128)
	// Checks for pd != nil are made in netpoll.
	pds[0] = nil
}

func netpolldescriptor() uintptr {
	// Both fds must be returned.
	if rdwake > 0xFFFF || wrwake > 0xFFFF {
		throw("netpolldescriptor: invalid fd number")
	}
	return uintptr(rdwake<<16 | wrwake)
}

// netpollwakeup writes on wrwake to wakeup poll before any changes.
func netpollwakeup() {
	if pendingUpdates == 0 {
		pendingUpdates = 1
		if pollVerbose {
			println("*** writing 1 byte")
		}
		b := [1]byte{0}
		write(uintptr(wrwake), unsafe.Pointer(&b[0]), 1)
	}
}

func netpollopen(fd uintptr, pd *pollDesc) int32 {
	if pollVerbose {
		println("*** netpollopen", fd)
	}
	lock(&mtxpoll)
	netpollwakeup()

	lock(&mtxset)
	unlock(&mtxpoll)

	pd.user = uint32(len(pfds))
	pfds = append(pfds, pollfd{fd: int32(fd)})
	pds = append(pds, pd)
	unlock(&mtxset)
	return 0
}

func netpollclose(fd uintptr) int32 {
	if pollVerbose {
		println("*** netpollclose", fd)
	}
	lock(&mtxpoll)
	netpollwakeup()

	lock(&mtxset)
	unlock(&mtxpoll)

	for i := 0; i < len(pfds); i++ {
		if pfds[i].fd == int32(fd) {
			pfds[i] = pfds[len(pfds)-1]
			pfds = pfds[:len(pfds)-1]

			pds[i] = pds[len(pds)-1]
			pds[i].user = uint32(i)
			pds = pds[:len(pds)-1]
			break
		}
	}
	unlock(&mtxset)
	return 0
}

func netpollarm(pd *pollDesc, mode int) {
	if pollVerbose {
		println("*** netpollarm", pd.fd, mode)
	}
	lock(&mtxpoll)
	netpollwakeup()

	lock(&mtxset)
	unlock(&mtxpoll)

	switch mode {
	case 'r':
		pfds[pd.user].events |= _POLLIN
	case 'w':
		pfds[pd.user].events |= _POLLOUT
	}
	unlock(&mtxset)
}

// polls for ready network connections
// returns list of goroutines that become runnable
//go:nowritebarrierrec
func netpoll(block bool) gList {
	timeout := int32(0)
	if !block {
		timeout = 0
		return gList{}
	}
	if pollVerbose {
		println("*** netpoll", block)
	}
retry:
	lock(&mtxpoll)
	lock(&mtxset)
	pendingUpdates = 0
	unlock(&mtxpoll)

	if pollVerbose {
		println("*** netpoll before poll")
	}
	n := libc_poll(&pfds[0], int32(len(pfds)), timeout)
	if pollVerbose {
		println("*** netpoll after poll", n)
	}
	if n < 0 {
		e := errno()
		if e != _EINTR {
			println("errno=", e, " len(pfds)=", len(pfds))
			throw("poll failed")
		}
		if pollVerbose {
			println("*** poll failed")
		}
		unlock(&mtxset)
		goto retry
	}
	// Check if some descriptors need to be changed
	if n != 0 && pfds[0].revents&(_POLLIN|_POLLHUP|_POLLERR) != 0 {
		var b [1]byte
		for read(rdwake, unsafe.Pointer(&b[0]), 1) == 1 {
			if pollVerbose {
				println("*** read 1 byte from pipe")
			}
		}
		// Do not look at the other fds in this case as the mode may have changed
		// XXX only additions of flags are made, so maybe it is ok
		unlock(&mtxset)
		goto retry
	}
	var toRun gList
	for i := 0; i < len(pfds) && n > 0; i++ {
		pfd := &pfds[i]

		var mode int32
		if pfd.revents&(_POLLIN|_POLLHUP|_POLLERR) != 0 {
			mode += 'r'
			pfd.events &= ^_POLLIN
		}
		if pfd.revents&(_POLLOUT|_POLLHUP|_POLLERR) != 0 {
			mode += 'w'
			pfd.events &= ^_POLLOUT
		}
		if mode != 0 {
			if pollVerbose {
				println("*** netpollready i=", i, "revents=", pfd.revents, "events=", pfd.events, "pd=", pds[i])
			}
			netpollready(&toRun, pds[i], mode)
			n--
		}
	}
	unlock(&mtxset)
	if block && toRun.empty() {
		goto retry
	}
	if pollVerbose {
		println("*** netpoll returning end")
	}
	return toRun
}
<commit_msg>runtime: update netpoll_hurd.go for go1.14beta1 changes<commit_after>// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package runtime

import "unsafe"

// FIXME: Improve network poller for hurd.
// This is based on the former libgo/runtime/netpoll_select.c implementation
// except that it uses poll instead of select and is written in Go.
// It's also based on Solaris implementation for the arming mechanisms
// Inspiration was also taken from netpoll_aix.go and netpoll_solaris.go

//From /usr/include/x86_64-linux-gnu/sys/poll.h
//go:noescape
//extern poll
func libc_poll(pfds *pollfd, nfds int32, timeout int32) int32

//go:noescape
//extern pipe2
func libc_pipe2(fd *int32, flags int32) int32

//pollfd represents the poll structure for GNU/Hurd operating system.
type pollfd struct {
	fd      int32 // File descriptor to poll.
	events  int16 // Types of events poller cares about.
	revents int16 // Types of events that actually occurred.
}

//From /usr/include/i386-gnu/bits/poll.h
const _POLLIN = 01    // There is data to read.
const _POLLPRI = 02   // There is urgent data to read.
const _POLLOUT = 04   // Writing now will not block.
const _POLLERR = 010  // Error condition.
const _POLLHUP = 020  // Hung up.
const _POLLNVAL = 040 // Invalid polling request.

var (
	pfds           []pollfd
	pds            []*pollDesc
	mtxpoll        mutex
	mtxset         mutex
	rdwake         int32
	wrwake         int32
	pendingUpdates int32
)

const pollVerbose = false

func netpollinit() {
	var p [2]int32

	// Create the pipe we use to wakeup poll.
	if err := libc_pipe2(&p[0], _O_CLOEXEC|_O_NONBLOCK); err < 0 {
		throw("runtime:netpollinit(): failed to create pipe2")
	}
	rdwake = p[0]
	wrwake = p[1]

	// Pre-allocate array of pollfd structures for poll.
	if pollVerbose {
		println("*** allocating")
	}
	pfds = make([]pollfd, 1, 128)
	if pollVerbose {
		println("*** allocating done", &pfds[0])
	}

	// Poll the read side of the pipe.
	pfds[0].fd = int32(rdwake)
	pfds[0].events = int16(_POLLIN)
	pfds[0].revents = int16(0)

	pds = make([]*pollDesc, 1, 128)
	// Checks for pd != nil are made in netpoll.
	pds[0] = nil
}

func netpolldescriptor() uintptr {
	// Both fds must be returned.
	if rdwake > 0xFFFF || wrwake > 0xFFFF {
		throw("netpolldescriptor: invalid fd number")
	}
	return uintptr(rdwake<<16 | wrwake)
}

func netpollIsPollDescriptor(fd uintptr) bool {
	return fd == uintptr(rdwake) || fd == uintptr(wrwake)
}

// netpollwakeup writes on wrwake to wakeup poll before any changes.
func netpollwakeup() {
	if pendingUpdates == 0 {
		pendingUpdates = 1
		if pollVerbose {
			println("*** writing 1 byte")
		}
		b := [1]byte{0}
		write(uintptr(wrwake), unsafe.Pointer(&b[0]), 1)
	}
}

func netpollopen(fd uintptr, pd *pollDesc) int32 {
	if pollVerbose {
		println("*** netpollopen", fd)
	}
	lock(&mtxpoll)
	netpollwakeup()

	lock(&mtxset)
	unlock(&mtxpoll)

	pd.user = uint32(len(pfds))
	pfds = append(pfds, pollfd{fd: int32(fd)})
	pds = append(pds, pd)
	unlock(&mtxset)
	return 0
}

func netpollclose(fd uintptr) int32 {
	if pollVerbose {
		println("*** netpollclose", fd)
	}
	lock(&mtxpoll)
	netpollwakeup()

	lock(&mtxset)
	unlock(&mtxpoll)

	for i := 0; i < len(pfds); i++ {
		if pfds[i].fd == int32(fd) {
			pfds[i] = pfds[len(pfds)-1]
			pfds = pfds[:len(pfds)-1]

			pds[i] = pds[len(pds)-1]
			pds[i].user = uint32(i)
			pds = pds[:len(pds)-1]
			break
		}
	}
	unlock(&mtxset)
	return 0
}

func netpollarm(pd *pollDesc, mode int) {
	if pollVerbose {
		println("*** netpollarm", pd.fd, mode)
	}
	lock(&mtxpoll)
	netpollwakeup()

	lock(&mtxset)
	unlock(&mtxpoll)

	switch mode {
	case 'r':
		pfds[pd.user].events |= _POLLIN
	case 'w':
		pfds[pd.user].events |= _POLLOUT
	}
	unlock(&mtxset)
}

// netpollBreak interrupts an epollwait.
func netpollBreak() {
	netpollwakeup()
}

// netpoll checks for ready network connections.
// Returns list of goroutines that become runnable.
// delay < 0: blocks indefinitely
// delay == 0: does not block, just polls
// delay > 0: block for up to that many nanoseconds
//go:nowritebarrierrec
func netpoll(delay int64) gList {
	timeout := int32(0)
	if delay < 0 {
		timeout = 0
	} else if delay == 0 {
		// TODO: call poll with timeout == 0
		return gList{}
	} else if delay < 1e6 {
		timeout = 1
	} else if delay < 1e15 {
		timeout = int32(delay / 1e6)
	} else {
		// An arbitrary cap on how long to wait for a timer.
		// 1e9 ms == ~11.5 days.
		timeout = 1e9
	}
retry:
	lock(&mtxpoll)
	lock(&mtxset)
	pendingUpdates = 0
	unlock(&mtxpoll)

	n := libc_poll(&pfds[0], int32(len(pfds)), timeout)
	if n < 0 {
		e := errno()
		if e != _EINTR {
			println("errno=", e, " len(pfds)=", len(pfds))
			throw("poll failed")
		}
		unlock(&mtxset)
		// If a timed sleep was interrupted, just return to
		// recalculate how long we should sleep now.
		if timeout > 0 {
			return gList{}
		}
		goto retry
	}
	// Check if some descriptors need to be changed
	if n != 0 && pfds[0].revents&(_POLLIN|_POLLHUP|_POLLERR) != 0 {
		if delay != 0 {
			// A netpollwakeup could be picked up by a
			// non-blocking poll. Only clear the wakeup
			// if blocking.
			var b [1]byte
			for read(rdwake, unsafe.Pointer(&b[0]), 1) == 1 {
			}
		}
		// Still look at the other fds even if the mode may have
		// changed, as netpollBreak might have been called.
		n--
	}
	var toRun gList
	for i := 1; i < len(pfds) && n > 0; i++ {
		pfd := &pfds[i]

		var mode int32
		if pfd.revents&(_POLLIN|_POLLHUP|_POLLERR) != 0 {
			mode += 'r'
			pfd.events &= ^_POLLIN
		}
		if pfd.revents&(_POLLOUT|_POLLHUP|_POLLERR) != 0 {
			mode += 'w'
			pfd.events &= ^_POLLOUT
		}
		if mode != 0 {
			pds[i].everr = false
			if pfd.revents == _POLLERR {
				pds[i].everr = true
			}
			netpollready(&toRun, pds[i], mode)
			n--
		}
	}
	unlock(&mtxset)
	return toRun
}
<|endoftext|>


<commit_before>package mount

import (
	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/util"
	"github.com/hanwen/go-fuse/v2/fuse"
	"sync"
)

type InodeToPath struct {
	sync.RWMutex
	nextInodeId uint64
	inode2path  map[uint64]*InodeEntry
	path2inode  map[util.FullPath]uint64
}
type InodeEntry struct {
	util.FullPath
	nlookup          uint64
	isDirectory      bool
	isChildrenCached bool
}

func NewInodeToPath(root util.FullPath) *InodeToPath {
	t := &InodeToPath{
		inode2path: make(map[uint64]*InodeEntry),
		path2inode: make(map[util.FullPath]uint64),
	}
	t.inode2path[1] = &InodeEntry{root, 1, true, false}
	t.path2inode[root] = 1
	return t
}

func (i *InodeToPath) Lookup(path util.FullPath, unixTime int64, isDirectory bool, isHardlink bool, possibleInode uint64, isLookup bool) uint64 {
	i.Lock()
	defer i.Unlock()
	inode, found := i.path2inode[path]
	if !found {
		if possibleInode == 0 {
			inode = path.AsInode(unixTime)
		} else {
			inode = possibleInode
		}
		if !isHardlink {
			for _, found := i.inode2path[inode]; found; inode++ {
				_, found = i.inode2path[inode]
			}
		}
	}
	i.path2inode[path] = inode

	if _, found := i.inode2path[inode]; found {
		if isLookup {
			i.inode2path[inode].nlookup++
		}
	} else {
		if !isLookup {
			i.inode2path[inode] = &InodeEntry{path, 0, isDirectory, false}
		} else {
			i.inode2path[inode] = &InodeEntry{path, 1, isDirectory, false}
		}
	}

	return inode
}

func (i *InodeToPath) AllocateInode(path util.FullPath, unixTime int64) uint64 {
	if path == "/" {
		return 1
	}
	i.Lock()
	defer i.Unlock()
	inode := path.AsInode(unixTime)
	for _, found := i.inode2path[inode]; found; inode++ {
		_, found = i.inode2path[inode]
	}
	return inode
}

func (i *InodeToPath) GetInode(path util.FullPath) uint64 {
	if path == "/" {
		return 1
	}
	i.Lock()
	defer i.Unlock()
	inode, found := i.path2inode[path]
	if !found {
		// glog.Fatalf("GetInode unknown inode for %s", path)
		// this could be the parent for mount point
	}
	return inode
}

func (i *InodeToPath) GetPath(inode uint64) (util.FullPath, fuse.Status) {
	i.RLock()
	defer i.RUnlock()
	path, found := i.inode2path[inode]
	if !found {
		return "", fuse.ENOENT
	}
	return path.FullPath, fuse.OK
}

func (i *InodeToPath) HasPath(path util.FullPath) bool {
	i.RLock()
	defer i.RUnlock()
	_, found := i.path2inode[path]
	return found
}

func (i *InodeToPath) MarkChildrenCached(fullpath util.FullPath) {
	i.RLock()
	defer i.RUnlock()
	inode, found := i.path2inode[fullpath]
	if !found {
		glog.Fatalf("MarkChildrenCached not found inode %v", fullpath)
	}
	path, found := i.inode2path[inode]
	path.isChildrenCached = true
}

func (i *InodeToPath) IsChildrenCached(fullpath util.FullPath) bool {
	i.RLock()
	defer i.RUnlock()
	inode, found := i.path2inode[fullpath]
	if !found {
		return false
	}
	path, found := i.inode2path[inode]
	if found {
		return path.isChildrenCached
	}
	return false
}

func (i *InodeToPath) HasInode(inode uint64) bool {
	if inode == 1 {
		return true
	}
	i.RLock()
	defer i.RUnlock()
	_, found := i.inode2path[inode]
	return found
}

func (i *InodeToPath) RemovePath(path util.FullPath) {
	i.Lock()
	defer i.Unlock()
	inode, found := i.path2inode[path]
	if found {
		delete(i.path2inode, path)
		delete(i.inode2path, inode)
	}
}

func (i *InodeToPath) MovePath(sourcePath, targetPath util.FullPath) (replacedInode uint64) {
	i.Lock()
	defer i.Unlock()
	sourceInode, sourceFound := i.path2inode[sourcePath]
	targetInode, targetFound := i.path2inode[targetPath]
	if sourceFound {
		delete(i.path2inode, sourcePath)
		i.path2inode[targetPath] = sourceInode
	} else {
		// it is possible some source folder items has not been visited before
		// so no need to worry about their source inodes
		return
	}
	i.inode2path[sourceInode].FullPath = targetPath
	if targetFound {
		delete(i.inode2path, targetInode)
	} else {
		i.inode2path[sourceInode].nlookup++
	}
	return targetInode
}

func (i *InodeToPath) Forget(inode, nlookup uint64, onForgetDir func(dir util.FullPath)) {
	i.Lock()
	path, found := i.inode2path[inode]
	if found {
		path.nlookup -= nlookup
		if path.nlookup <= 0 {
			delete(i.path2inode, path.FullPath)
			delete(i.inode2path, inode)
		}
	}
	i.Unlock()
	if found {
		if path.isDirectory && path.nlookup <= 0 && onForgetDir != nil {
			path.isChildrenCached = false
			onForgetDir(path.FullPath)
		}
	}
}
<commit_msg>mount: delete target entry first during move<commit_after>package mount

import (
	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/util"
	"github.com/hanwen/go-fuse/v2/fuse"
	"sync"
)

type InodeToPath struct {
	sync.RWMutex
	nextInodeId uint64
	inode2path  map[uint64]*InodeEntry
	path2inode  map[util.FullPath]uint64
}
type InodeEntry struct {
	util.FullPath
	nlookup          uint64
	isDirectory      bool
	isChildrenCached bool
}

func NewInodeToPath(root util.FullPath) *InodeToPath {
	t := &InodeToPath{
		inode2path: make(map[uint64]*InodeEntry),
		path2inode: make(map[util.FullPath]uint64),
	}
	t.inode2path[1] = &InodeEntry{root, 1, true, false}
	t.path2inode[root] = 1
	return t
}

func (i *InodeToPath) Lookup(path util.FullPath, unixTime int64, isDirectory bool, isHardlink bool, possibleInode uint64, isLookup bool) uint64 {
	i.Lock()
	defer i.Unlock()
	inode, found := i.path2inode[path]
	if !found {
		if possibleInode == 0 {
			inode = path.AsInode(unixTime)
		} else {
			inode = possibleInode
		}
		if !isHardlink {
			for _, found := i.inode2path[inode]; found; inode++ {
				_, found = i.inode2path[inode]
			}
		}
	}
	i.path2inode[path] = inode

	if _, found := i.inode2path[inode]; found {
		if isLookup {
			i.inode2path[inode].nlookup++
		}
	} else {
		if !isLookup {
			i.inode2path[inode] = &InodeEntry{path, 0, isDirectory, false}
		} else {
			i.inode2path[inode] = &InodeEntry{path, 1, isDirectory, false}
		}
	}

	return inode
}

func (i *InodeToPath) AllocateInode(path util.FullPath, unixTime int64) uint64 {
	if path == "/" {
		return 1
	}
	i.Lock()
	defer i.Unlock()
	inode := path.AsInode(unixTime)
	for _, found := i.inode2path[inode]; found; inode++ {
		_, found = i.inode2path[inode]
	}
	return inode
}

func (i *InodeToPath) GetInode(path util.FullPath) uint64 {
	if path == "/" {
		return 1
	}
	i.Lock()
	defer i.Unlock()
	inode, found := i.path2inode[path]
	if !found {
		// glog.Fatalf("GetInode unknown inode for %s", path)
		// this could be the parent for mount point
	}
	return inode
}

func (i *InodeToPath) GetPath(inode uint64) (util.FullPath, fuse.Status) {
	i.RLock()
	defer i.RUnlock()
	path, found := i.inode2path[inode]
	if !found {
		return "", fuse.ENOENT
	}
	return path.FullPath, fuse.OK
}

func (i *InodeToPath) HasPath(path util.FullPath) bool {
	i.RLock()
	defer i.RUnlock()
	_, found := i.path2inode[path]
	return found
}

func (i *InodeToPath) MarkChildrenCached(fullpath util.FullPath) {
	i.RLock()
	defer i.RUnlock()
	inode, found := i.path2inode[fullpath]
	if !found {
		glog.Fatalf("MarkChildrenCached not found inode %v", fullpath)
	}
	path, found := i.inode2path[inode]
	path.isChildrenCached = true
}

func (i *InodeToPath) IsChildrenCached(fullpath util.FullPath) bool {
	i.RLock()
	defer i.RUnlock()
	inode, found := i.path2inode[fullpath]
	if !found {
		return false
	}
	path, found := i.inode2path[inode]
	if found {
		return path.isChildrenCached
	}
	return false
}

func (i *InodeToPath) HasInode(inode uint64) bool {
	if inode == 1 {
		return true
	}
	i.RLock()
	defer i.RUnlock()
	_, found := i.inode2path[inode]
	return found
}

func (i *InodeToPath) RemovePath(path util.FullPath) {
	i.Lock()
	defer i.Unlock()
	inode, found := i.path2inode[path]
	if found {
		delete(i.path2inode, path)
		delete(i.inode2path, inode)
	}
}

func (i *InodeToPath) MovePath(sourcePath, targetPath util.FullPath) (replacedInode uint64) {
	i.Lock()
	defer i.Unlock()
	sourceInode, sourceFound := i.path2inode[sourcePath]
	targetInode, targetFound := i.path2inode[targetPath]
	if targetFound {
		delete(i.inode2path, targetInode)
		delete(i.path2inode, targetPath)
	}
	if sourceFound {
		delete(i.path2inode, sourcePath)
		i.path2inode[targetPath] = sourceInode
	} else {
		// it is possible some source folder items has not been visited before
		// so no need to worry about their source inodes
		return
	}
	if entry, entryFound := i.inode2path[sourceInode]; entryFound {
		entry.FullPath = targetPath
		entry.isChildrenCached = false
		if !targetFound {
			entry.nlookup++
		}
	} else {
		glog.Errorf("MovePath %s to %s: sourceInode %s not found", sourcePath, targetPath, sourceInode)
	}
	return targetInode
}

func (i *InodeToPath) Forget(inode, nlookup uint64, onForgetDir func(dir util.FullPath)) {
	i.Lock()
	path, found := i.inode2path[inode]
	if found {
		path.nlookup -= nlookup
		if path.nlookup <= 0 {
			delete(i.path2inode, path.FullPath)
			delete(i.inode2path, inode)
		}
	}
	i.Unlock()
	if found {
		if path.isDirectory && path.nlookup <= 0 && onForgetDir != nil {
			path.isChildrenCached = false
			onForgetDir(path.FullPath)
		}
	}
}
<|endoftext|>


<commit_before>package awsprovider

import (
	"koding/kites/kloud/api/amazon"
	"koding/kites/kloud/klient"
	"koding/kites/kloud/machinestate"
	"time"

	"github.com/koding/kite"
	"golang.org/x/net/context"
	"labix.org/v2/mgo"
	"labix.org/v2/mgo/bson"
)

func (m *Machine) Info(ctx context.Context) (map[string]string, error) {
	dbState := m.State()
	resultState := dbState
	reason := "not known yet"

	// return lazily if it's a progress state, such as "Building, Stopping,
	// etc.."
	if dbState.InProgress() {
		return map[string]string{
			"State": dbState.String(),
		}, nil
	}

	defer func() {
		// Update db state if the up-to-date state is different than the db.
		if resultState != dbState {
			m.Log.Info("Info decision: Inconsistent state between the machine and db document. Updating state to '%s'. Reason: %s",
				resultState, reason)

			if err := m.checkAndUpdateState(resultState); err != nil {
				m.Log.Debug("Info decision: Error while updating the machine state. Err: %v", m.Id, err)
			}
		}
	}()

	instance, err := m.Session.AWSClient.Instance()
	if err == nil {
		resultState = amazon.StatusToState(instance.State.Name)
	} else if err == amazon.ErrNoInstances {
		resultState = machinestate.NotInitialized
	} else {
		// if it's something else, return it back
		return nil, err
	}

	// This happens when a machine was destroyed recently in one hour span.
	// The machine is still available in AWS but it's been marked as
	// Terminated. Because we still have the machine document, mark it as
	// NotInitialized so the user can build again.
	if resultState == machinestate.Terminated || resultState == machinestate.Terminating {
		resultState = machinestate.NotInitialized
		if err := m.markAsNotInitialized(); err != nil {
			return nil, err
		}
	}

	if resultState == machinestate.Running {
		// Check if klient is running first.
		klientRef, err := klient.ConnectTimeout(m.Session.Kite, m.QueryString, time.Second*10)
		if err == nil {
			// we could connect to it, which is more than enough
			klientRef.Close()

			reason = "Klient is active and healthy."
			resultState = machinestate.Running

			return map[string]string{
				"State": resultState.String(),
			}, nil
		}

		if err == klient.ErrDialingFailed || err == kite.ErrNoKitesAvailable {
			// klient state is still machinestate.Unknown.
			m.Log.Debug("Klient is not registered to Kontrol. Err: %s", err)

			// XXX: AWS call reduction workaround.
			if dbState == machinestate.Stopped {
				m.Log.Debug("Info result: Returning db state '%s' because the klient is not available. Username: %s",
					dbState, m.Username)
				return map[string]string{
					"State": machinestate.Stopped.String(),
				}, nil
			}
		}
	}

	m.Log.Debug("Info result: '%s'. Username: %s", resultState, m.Username)
	return map[string]string{
		"State": resultState.String(),
	}, nil
}

// CheckAndUpdate state updates only if the given machine id is not used by
// anyone else
func (m *Machine) checkAndUpdateState(state machinestate.State) error {
	m.Log.Info("storage state update request to state %v", state)
	err := m.Session.DB.Run("jMachines", func(c *mgo.Collection) error {
		return c.Update(
			bson.M{
				"_id": m.Id,
				"assignee.inProgress": false, // only update if it's not locked by someone else
			},
			bson.M{
				"$set": bson.M{
					"status.state":      state.String(),
					"status.modifiedAt": time.Now().UTC(),
				},
			},
		)
	})

	if err == mgo.ErrNotFound {
		m.Log.Warning("info can't update db state because lock is acquired by someone else")
	}

	return err
}
<commit_msg>kloud/aws/info: change log to info<commit_after>package awsprovider

import (
	"koding/kites/kloud/api/amazon"
	"koding/kites/kloud/klient"
	"koding/kites/kloud/machinestate"
	"time"

	"github.com/koding/kite"
	"golang.org/x/net/context"
	"labix.org/v2/mgo"
	"labix.org/v2/mgo/bson"
)

func (m *Machine) Info(ctx context.Context) (map[string]string, error) {
	dbState := m.State()
	resultState := dbState
	reason := "not known yet"

	// return lazily if it's a progress state, such as "Building, Stopping,
	// etc.."
	if dbState.InProgress() {
		return map[string]string{
			"State": dbState.String(),
		}, nil
	}

	defer func() {
		// Update db state if the up-to-date state is different than the db.
		if resultState != dbState {
			m.Log.Info("Info decision: Inconsistent state between the machine and db document. Updating state to '%s'. Reason: %s",
				resultState, reason)

			if err := m.checkAndUpdateState(resultState); err != nil {
				m.Log.Debug("Info decision: Error while updating the machine state. Err: %v", m.Id, err)
			}
		}
	}()

	instance, err := m.Session.AWSClient.Instance()
	if err == nil {
		resultState = amazon.StatusToState(instance.State.Name)
	} else if err == amazon.ErrNoInstances {
		resultState = machinestate.NotInitialized
	} else {
		// if it's something else, return it back
		return nil, err
	}

	// This happens when a machine was destroyed recently in one hour span.
	// The machine is still available in AWS but it's been marked as
	// Terminated. Because we still have the machine document, mark it as
	// NotInitialized so the user can build again.
	if resultState == machinestate.Terminated || resultState == machinestate.Terminating {
		resultState = machinestate.NotInitialized
		if err := m.markAsNotInitialized(); err != nil {
			return nil, err
		}
	}

	if resultState == machinestate.Running {
		// Check if klient is running first.
		klientRef, err := klient.ConnectTimeout(m.Session.Kite, m.QueryString, time.Second*10)
		if err == nil {
			// we could connect to it, which is more than enough
			klientRef.Close()

			reason = "Klient is active and healthy."
			resultState = machinestate.Running

			return map[string]string{
				"State": resultState.String(),
			}, nil
		}

		if err == klient.ErrDialingFailed || err == kite.ErrNoKitesAvailable {
			// klient state is still machinestate.Unknown.
			m.Log.Debug("Klient is not registered to Kontrol. Err: %s", err)

			// XXX: AWS call reduction workaround.
			if dbState == machinestate.Stopped {
				m.Log.Debug("Info result: Returning db state '%s' because the klient is not available. Username: %s",
					dbState, m.Username)
				return map[string]string{
					"State": machinestate.Stopped.String(),
				}, nil
			}
		}
	}

	m.Log.Debug("Info result: '%s'. Username: %s", resultState, m.Username)
	return map[string]string{
		"State": resultState.String(),
	}, nil
}

// CheckAndUpdate state updates only if the given machine id is not used by
// anyone else
func (m *Machine) checkAndUpdateState(state machinestate.State) error {
	m.Log.Info("storage state update request to state %v", state)
	err := m.Session.DB.Run("jMachines", func(c *mgo.Collection) error {
		return c.Update(
			bson.M{
				"_id": m.Id,
				"assignee.inProgress": false, // only update if it's not locked by someone else
			},
			bson.M{
				"$set": bson.M{
					"status.state":      state.String(),
					"status.modifiedAt": time.Now().UTC(),
				},
			},
		)
	})

	if err == mgo.ErrNotFound {
		m.Log.Info("info can't update db state because lock is acquired by someone else")
	}

	return err
}
<|endoftext|>


<commit_before>package keychain

import (
	"crypto/ecdsa"
	ethkeystore "github.com/ethereum/go-ethereum/accounts/keystore"
	"github.com/ethereum/go-ethereum/crypto"
	"fmt"
	"sync"
)

// EncryptToStored encrypts the contents of Keychain using the authentication passphrase and scrypt
// difficulty parameters.
func EncryptToStored(kc *Keychain, auth string, scryptN, scryptP int) (*StoredKeychain, error) {
	storedKeyEncKeys := make(map[string][]byte)
	var mu sync.Mutex
	var wg sync.WaitGroup
	errs, done := make(chan error, 1), make(chan struct{}, 1)

	// encrypt all keys in parallel b/c each can be intensive, thanks to scrypt
	for s1, key1 := range kc.keyEncKeys {
		wg.Add(1)
		go func(s2 string, key2 *ecdsa.PrivateKey) {
			var err error
			defer wg.Done()
			mu.Lock()
			storedKeyEncKeys[s2], err = encryptKey(key2, auth, scryptN, scryptP)
			mu.Unlock()
			if err != nil {
				errs <- err
			}
		}(s1, key1)
	}

	go func () {
		wg.Wait()
		done <- struct{}{}
	}()

	select {
	case <- done:
		return &StoredKeychain{
			KeyEncKeys: storedKeyEncKeys,
		}, nil
	case err := <- errs:
		return nil, err
	}
}

// DecryptFromStored decrypts the contents of a StoredKeychain using the authentication passphrase.
func DecryptFromStored(stored *StoredKeychain, auth string) (*Keychain, error) {
	keyEncKeys := make(map[string]*ecdsa.PrivateKey)
	var mu sync.Mutex
	var wg sync.WaitGroup
	errs, done := make(chan error, 1), make(chan struct{}, 1)

	// decrypt all keys in parallel b/c each can be intensive, thanks to scrypt
	for _, keyJSON1 := range stored.KeyEncKeys {
		wg.Add(1)
		go func(keyJSON2 []byte) {
			defer wg.Done()
			keyEncKey, err := decryptKey(keyJSON2, auth)
			if err != nil {
				errs <- err
				return
			}
			mu.Lock()
			keyEncKeys[pubKeyString(keyEncKey)] = keyEncKey
			mu.Unlock()
		}(keyJSON1)
	}

	go func () {
		wg.Wait()
		done <- struct{}{}
	}()

	select {
	case <- done:
		return &Keychain{
			keyEncKeys: keyEncKeys,
		}, nil
	case err := <- errs:
		return nil, err
	}
}

func encryptKey(key *ecdsa.PrivateKey, auth string, scryptN, scryptP int) ([]byte, error) {
	ethKey := &ethkeystore.Key{
		// Address is not not used by libri, but required for encryption & decryption
		Address: crypto.PubkeyToAddress(key.PublicKey),
		PrivateKey: key,
	}
	return ethkeystore.EncryptKey(ethKey, auth, scryptN, scryptP)
}

func decryptKey(keyJSON []byte, auth string) (*ecdsa.PrivateKey, error) {
	ethKey, err := ethkeystore.DecryptKey(keyJSON, auth)
	if err != nil {
		return nil, err
	}
	return ethKey.PrivateKey, nil
}

func pubKeyString(privKey *ecdsa.PrivateKey) string {
	return fmt.Sprintf("%064X", privKey.X.Bytes())
}
<commit_msg>use lowercase hex for pub key string (#39)<commit_after>package keychain

import (
	"crypto/ecdsa"
	ethkeystore "github.com/ethereum/go-ethereum/accounts/keystore"
	"github.com/ethereum/go-ethereum/crypto"
	"fmt"
	"sync"
)

// EncryptToStored encrypts the contents of Keychain using the authentication passphrase and scrypt
// difficulty parameters.
func EncryptToStored(kc *Keychain, auth string, scryptN, scryptP int) (*StoredKeychain, error) {
	storedKeyEncKeys := make(map[string][]byte)
	var mu sync.Mutex
	var wg sync.WaitGroup
	errs, done := make(chan error, 1), make(chan struct{}, 1)

	// encrypt all keys in parallel b/c each can be intensive, thanks to scrypt
	for s1, key1 := range kc.keyEncKeys {
		wg.Add(1)
		go func(s2 string, key2 *ecdsa.PrivateKey) {
			var err error
			defer wg.Done()
			mu.Lock()
			storedKeyEncKeys[s2], err = encryptKey(key2, auth, scryptN, scryptP)
			mu.Unlock()
			if err != nil {
				errs <- err
			}
		}(s1, key1)
	}

	go func () {
		wg.Wait()
		done <- struct{}{}
	}()

	select {
	case <- done:
		return &StoredKeychain{
			KeyEncKeys: storedKeyEncKeys,
		}, nil
	case err := <- errs:
		return nil, err
	}
}

// DecryptFromStored decrypts the contents of a StoredKeychain using the authentication passphrase.
func DecryptFromStored(stored *StoredKeychain, auth string) (*Keychain, error) {
	keyEncKeys := make(map[string]*ecdsa.PrivateKey)
	var mu sync.Mutex
	var wg sync.WaitGroup
	errs, done := make(chan error, 1), make(chan struct{}, 1)

	// decrypt all keys in parallel b/c each can be intensive, thanks to scrypt
	for _, keyJSON1 := range stored.KeyEncKeys {
		wg.Add(1)
		go func(keyJSON2 []byte) {
			defer wg.Done()
			keyEncKey, err := decryptKey(keyJSON2, auth)
			if err != nil {
				errs <- err
				return
			}
			mu.Lock()
			keyEncKeys[pubKeyString(keyEncKey)] = keyEncKey
			mu.Unlock()
		}(keyJSON1)
	}

	go func () {
		wg.Wait()
		done <- struct{}{}
	}()

	select {
	case <- done:
		return &Keychain{
			keyEncKeys: keyEncKeys,
		}, nil
	case err := <- errs:
		return nil, err
	}
}

func encryptKey(key *ecdsa.PrivateKey, auth string, scryptN, scryptP int) ([]byte, error) {
	ethKey := &ethkeystore.Key{
		// Address is not not used by libri, but required for encryption & decryption
		Address: crypto.PubkeyToAddress(key.PublicKey),
		PrivateKey: key,
	}
	return ethkeystore.EncryptKey(ethKey, auth, scryptN, scryptP)
}

func decryptKey(keyJSON []byte, auth string) (*ecdsa.PrivateKey, error) {
	ethKey, err := ethkeystore.DecryptKey(keyJSON, auth)
	if err != nil {
		return nil, err
	}
	return ethKey.PrivateKey, nil
}

func pubKeyString(privKey *ecdsa.PrivateKey) string {
	return fmt.Sprintf("%064x", privKey.X.Bytes())
}
<|endoftext|>


<commit_before>package mailck

import (
	"fmt"
	"net"
	"net/smtp"
	"regexp"
	"strings"
)

var emailRexp = regexp.MustCompile("^[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\\.[A-Za-z]{2,64}$")

// Check checks the syntax and if valid, it checks the mailbox by connecting to
// the target mailserver
// The fromEmail is used as from address in the communication to the foreign mailserver.
func Check(fromEmail, checkEmail string) (result Result, err error) {
	if !CheckSyntax(checkEmail) {
		return InvalidSyntax, nil
	}

	if CheckDisposable(checkEmail) {
		return Disposable, nil
	}
	return CheckMailbox(fromEmail, checkEmail)
}

// CheckSyntax returns true for a valid email, false otherwise
func CheckSyntax(checkEmail string) bool {
	return emailRexp.Match([]byte(checkEmail))
}

// CheckMailbox checks the checkEmail by connecting to the target mailbox and returns the result.
// The fromEmail is used as from address in the communication to the foreign mailserver.
func CheckMailbox(fromEmail, checkEmail string) (result Result, err error) {
	mxList, err := net.LookupMX(hostname(checkEmail))
	// TODO: Distinguish between usual network errors
	if err != nil || len(mxList) == 0 {
		return InvalidDomain, nil
	}
	return checkMailbox(fromEmail, checkEmail, mxList, 25)
}

func checkMailbox(fromEmail, checkEmail string, mxList []*net.MX, port int) (result Result, err error) {
	// try to connect to one mx
	var c *smtp.Client
	for _, mx := range mxList {
		c, err = smtp.Dial(fmt.Sprintf("%v:%v", mx.Host, port))
		if err == nil {
			break
		}
	}
	if err != nil {
		return MailserverError, err
	}
	defer c.Close()
	defer c.Quit() // defer ist LIFO

	// HELO
	err = c.Hello(hostname(fromEmail))
	if err != nil {
		return MailserverError, err
	}

	// MAIL FROM
	err = c.Mail(fromEmail)
	if err != nil {
		return MailserverError, err
	}

	// RCPT TO
	id, err := c.Text.Cmd("RCPT TO:<%s>", checkEmail)
	if err != nil {
		return MailserverError, err
	}
	c.Text.StartResponse(id)
	code, _, err := c.Text.ReadResponse(25)
	c.Text.EndResponse(id)
	if code == 550 {
		return MailboxUnavailable, nil
	}

	if err != nil {
		return MailserverError, err
	}

	return Valid, nil
}

func hostname(mail string) string {
	return mail[strings.Index(mail, "@")+1:]
}
<commit_msg>Add context-aware variants of Check() and CheckMailbox()<commit_after>package mailck

import (
	"fmt"
	"net"
	"net/smtp"
	"regexp"
	"strings"
	"context"
)

var emailRexp = regexp.MustCompile("^[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\\.[A-Za-z]{2,64}$")

// Check checks the syntax and if valid, it checks the mailbox by connecting to
// the target mailserver
// The fromEmail is used as from address in the communication to the foreign mailserver.
func Check(fromEmail, checkEmail string) (result Result, err error) {
	if !CheckSyntax(checkEmail) {
		return InvalidSyntax, nil
	}

	if CheckDisposable(checkEmail) {
		return Disposable, nil
	}
	return CheckMailbox(fromEmail, checkEmail)
}

func CheckContext(ctx context.Context,fromEmail, checkEmail string) (result Result, err error) {
	if !CheckSyntax(checkEmail) {
		return InvalidSyntax, nil
	}

	if CheckDisposable(checkEmail) {
		return Disposable, nil
	}
	return CheckMailboxContext(ctx,fromEmail, checkEmail)
}
// CheckSyntax returns true for a valid email, false otherwise
func CheckSyntax(checkEmail string) bool {
	return emailRexp.Match([]byte(checkEmail))
}

var noContext = context.Background()
var defaultResolver = net.DefaultResolver
var defaultDialer = net.Dialer{}

// CheckMailbox checks the checkEmail by connecting to the target mailbox and returns the result.
// The fromEmail is used as from address in the communication to the foreign mailserver.
func CheckMailbox(fromEmail, checkEmail string) (result Result, err error) {
	mxList, err := net.LookupMX(hostname(checkEmail))
	// TODO: Distinguish between usual network errors
	if err != nil || len(mxList) == 0 {
		return InvalidDomain, nil
	}
	return checkMailbox(noContext, fromEmail, checkEmail, mxList, 25)
}

func CheckMailboxContext(ctx context.Context,fromEmail, checkEmail string) (result Result, err error) {
	mxList, err := defaultResolver.LookupMX(ctx,hostname(checkEmail))
	// TODO: Distinguish between usual network errors
	if err != nil || len(mxList) == 0 {
		return InvalidDomain, nil
	}
	return checkMailbox(ctx, fromEmail, checkEmail, mxList, 25)
}

func checkMailbox(ctx context.Context, fromEmail, checkEmail string, mxList []*net.MX, port int) (result Result, err error) {
	// try to connect to one mx
	var c *smtp.Client
	for _, mx := range mxList {
		conn, err := defaultDialer.DialContext(ctx,"tcp", fmt.Sprintf("%v:%v", mx.Host, port))
		if err != nil {
			return MailserverError, err
		}
		c, err = smtp.NewClient(conn,mx.Host)
		if err == nil {
			break
		}
	}
	if err != nil {
		return MailserverError, err
	}
	defer c.Close()
	defer c.Quit() // defer ist LIFO

	// HELO
	err = c.Hello(hostname(fromEmail))
	if err != nil {
		return MailserverError, err
	}

	// MAIL FROM
	err = c.Mail(fromEmail)
	if err != nil {
		return MailserverError, err
	}

	// RCPT TO
	id, err := c.Text.Cmd("RCPT TO:<%s>", checkEmail)
	if err != nil {
		return MailserverError, err
	}
	c.Text.StartResponse(id)
	code, _, err := c.Text.ReadResponse(25)
	c.Text.EndResponse(id)
	if code == 550 {
		return MailboxUnavailable, nil
	}

	if err != nil {
		return MailserverError, err
	}

	return Valid, nil
}

func hostname(mail string) string {
	return mail[strings.Index(mail, "@")+1:]
}
<|endoftext|>