<commit_before>package kubeconfig

import (
	"crypto/rand"
	"crypto/sha256"
	"encoding/base32"
	"encoding/base64"
	"errors"
	"fmt"
	"strings"

	"github.com/moby/locker"
	v3 "github.com/rancher/rancher/pkg/apis/management.cattle.io/v3"
	v1 "github.com/rancher/rancher/pkg/apis/provisioning.cattle.io/v1"
	"github.com/rancher/rancher/pkg/features"
	mgmtcontrollers "github.com/rancher/rancher/pkg/generated/controllers/management.cattle.io/v3"
	"github.com/rancher/rancher/pkg/settings"
	"github.com/rancher/rancher/pkg/wrangler"
	appcontroller "github.com/rancher/wrangler/pkg/generated/controllers/apps/v1"
	corecontrollers "github.com/rancher/wrangler/pkg/generated/controllers/core/v1"
	"github.com/rancher/wrangler/pkg/name"
	"github.com/rancher/wrangler/pkg/randomtoken"
	corev1 "k8s.io/api/core/v1"
	apierror "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/rest"
	"k8s.io/client-go/tools/clientcmd"
	clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
)

const (
	userIDLabel     = "authn.management.cattle.io/token-userId"
	tokenKindLabel  = "authn.management.cattle.io/kind"
	tokenHashedAnno = "authn.management.cattle.io/token-hashed"

	hashFormat = "$%d:%s:%s" // $version:salt:hash -> $1:abc:def
	Version    = 2
)

type Manager struct {
	deploymentCache  appcontroller.DeploymentCache
	daemonsetCache   appcontroller.DaemonSetCache
	tokens           mgmtcontrollers.TokenClient
	userCache        mgmtcontrollers.UserCache
	users            mgmtcontrollers.UserClient
	secretCache      corecontrollers.SecretCache
	secrets          corecontrollers.SecretClient
	kubeConfigLocker locker.Locker
}

func New(clients *wrangler.Context) *Manager {
	return &Manager{
		deploymentCache: clients.Apps.Deployment().Cache(),
		daemonsetCache:  clients.Apps.DaemonSet().Cache(),
		tokens:          clients.Mgmt.Token(),
		userCache:       clients.Mgmt.User().Cache(),
		users:           clients.Mgmt.User(),
		secretCache:     clients.Core.Secret().Cache(),
		secrets:         clients.Core.Secret(),
	}
}

func getKubeConfigSecretName(clusterName string) string {
	return clusterName + "-kubeconfig"
}

func (m *Manager) getToken(clusterNamespace, clusterName string) (string, error) {
	kubeConfigSecretName := getKubeConfigSecretName(clusterName)
	if token, err := m.getSavedToken(clusterNamespace, kubeConfigSecretName); err != nil || token != "" {
		return token, err
	}

	// Need to be careful about caches being out of sync since we are dealing with multiple objects that
	// arent eventually consistent (because we delete and create the token for the user)
	if token, err := m.getSavedTokenNoCache(clusterNamespace, kubeConfigSecretName); err != nil || token != "" {
		return token, err
	}

	userName, err := m.EnsureUser(clusterNamespace, clusterName)
	if err != nil {
		return "", err
	}

	return m.createUserToken(userName)
}

func (m *Manager) EnsureUser(clusterNamespace, clusterName string) (string, error) {
	principalID := getPrincipalID(clusterNamespace, clusterName)
	userName := getUserNameForPrincipal(principalID)
	return userName, m.createUser(principalID, userName)
}

func getUserNameForPrincipal(principal string) string {
	hasher := sha256.New()
	hasher.Write([]byte(principal))
	sha := base32.StdEncoding.WithPadding(-1).EncodeToString(hasher.Sum(nil))[:10]
	return "u-" + strings.ToLower(sha)
}

func labelsForUser(principalID string) map[string]string {
	encodedPrincipalID := base32.HexEncoding.WithPadding(base32.NoPadding).EncodeToString([]byte(principalID))
	if len(encodedPrincipalID) > 63 {
		encodedPrincipalID = encodedPrincipalID[:63]
	}
	return map[string]string{
		encodedPrincipalID: "hashed-principal-name",
	}
}

func (m *Manager) getSavedToken(kubeConfigNamespace, kubeConfigName string) (string, error) {
	secret, err := m.secretCache.Get(kubeConfigNamespace, kubeConfigName)
	if apierror.IsNotFound(err) {
		return "", nil
	} else if err != nil {
		return "", err
	}
	return string(secret.Data["token"]), nil
}

func (m *Manager) getSavedTokenNoCache(kubeConfigNamespace, kubeConfigName string) (string, error) {
	secret, err := m.secrets.Get(kubeConfigNamespace, kubeConfigName, metav1.GetOptions{})
	if apierror.IsNotFound(err) {
		return "", nil
	} else if err != nil {
		return "", err
	}
	return string(secret.Data["token"]), nil
}

func getPrincipalID(clusterNamespace, clusterName string) string {
	return fmt.Sprintf("system://provisioning/%s/%s", clusterNamespace, clusterName)
}

func (m *Manager) createUser(principalID, userName string) error {
	_, err := m.userCache.Get(userName)
	if apierror.IsNotFound(err) {
		_, err = m.users.Create(&v3.User{
			ObjectMeta: metav1.ObjectMeta{
				Name:   userName,
				Labels: labelsForUser(principalID),
			},
			PrincipalIDs: []string{
				principalID,
			},
		})
	}
	return err
}

func (m *Manager) createUserToken(userName string) (string, error) {
	_, err := m.tokens.Get(userName, metav1.GetOptions{})
	if err == nil {
		err = m.tokens.Delete(userName, nil)
	}
	if err != nil && !apierror.IsNotFound(err) {
		return "", err
	}

	tokenValue, err := randomtoken.Generate()
	if err != nil {
		return "", fmt.Errorf("failed to generate token key: %w", err)
	}

	token := &v3.Token{
		ObjectMeta: metav1.ObjectMeta{
			Name: userName,
			Labels: map[string]string{
				userIDLabel:    userName,
				tokenKindLabel: "provisioning",
			},
			Annotations: map[string]string{},
		},
		UserID:       userName,
		AuthProvider: "local",
		IsDerived:    true,
		Token:        tokenValue,
	}

	if features.TokenHashing.Enabled() {
		tokenHash, err := createSHA256Hash(tokenValue)
		if err != nil {
			return "", err
		}
		token.Token = tokenHash
		token.Annotations[tokenHashedAnno] = "true"
	}

	_, err = m.tokens.Create(token)
	return fmt.Sprintf("%s:%s", userName, tokenValue), err
}

func createSHA256Hash(secretKey string) (string, error) {
	salt := make([]byte, 8)
	_, err := rand.Read(salt)
	if err != nil {
		return "", err
	}
	hash := sha256.Sum256([]byte(fmt.Sprintf("%s%s", salt, secretKey)))
	encSalt := base64.RawStdEncoding.EncodeToString(salt)
	encKey := base64.RawStdEncoding.EncodeToString(hash[:])
	return fmt.Sprintf(hashFormat, Version, encSalt, encKey), nil
}

func (m *Manager) GetCRTBForAdmin(cluster *v1.Cluster, status v1.ClusterStatus) (*v3.ClusterRoleTemplateBinding, error) {
	if status.ClusterName == "" {
		return nil, fmt.Errorf("management cluster is not assigned to v1.Cluster")
	}
	principalID := getPrincipalID(cluster.Namespace, cluster.Name)
	userName := getUserNameForPrincipal(principalID)
	return &v3.ClusterRoleTemplateBinding{
		ObjectMeta: metav1.ObjectMeta{
			Name:      name.SafeConcatName(status.ClusterName, "owner"),
			Namespace: status.ClusterName,
		},
		ClusterName:       status.ClusterName,
		UserName:          userName,
		UserPrincipalName: principalID,
		RoleTemplateName:  "cluster-owner",
	}, nil
}

func (m *Manager) getKubeConfigData(clusterNamespace, clusterName, secretName, managementClusterName string) (map[string][]byte, error) {
	secret, err := m.secretCache.Get(clusterNamespace, secretName)
	if err == nil {
		if secret.Data == nil || secret.Data["token"] == nil {
			// Check if we require a new secret based on the token value and annotation(s). We delete the old secret since it may contain
			// annotations, owner references, etc. that are out of date. We will then continue to create the new secret.
			if err := m.secrets.Delete(clusterNamespace, secretName, &metav1.DeleteOptions{}); err != nil && !apierror.IsNotFound(err) {
				return nil, err
			}
		} else {
			return secret.Data, nil
		}
	} else if !apierror.IsNotFound(err) {
		return nil, err
	}

	lockID := clusterNamespace + "/" + clusterName
	m.kubeConfigLocker.Lock(lockID)
	defer m.kubeConfigLocker.Unlock(lockID)

	secret, err = m.secrets.Get(clusterNamespace, secretName, metav1.GetOptions{})
	if err == nil {
		return secret.Data, nil
	}

	tokenValue, err := m.getToken(clusterNamespace, clusterName)
	if err != nil {
		return nil, err
	}

	serverURL, cacert := settings.InternalServerURL.Get(), settings.InternalCACerts.Get()
	if serverURL == "" {
		return nil, errors.New("server url is missing, can't generate kubeconfig for fleet import cluster")
	}

	data, err := clientcmd.Write(clientcmdapi.Config{
		Clusters: map[string]*clientcmdapi.Cluster{
			"cluster": {
				Server:                   fmt.Sprintf("%s/k8s/clusters/%s", serverURL, managementClusterName),
				CertificateAuthorityData: []byte(strings.TrimSpace(cacert)),
			},
		},
		AuthInfos: map[string]*clientcmdapi.AuthInfo{
			"user": {
				Token: tokenValue,
			},
		},
		Contexts: map[string]*clientcmdapi.Context{
			"default": {
				Cluster:  "cluster",
				AuthInfo: "user",
			},
		},
		CurrentContext: "default",
	})
	if err != nil {
		return nil, err
	}

	secret, err = m.secrets.Create(&corev1.Secret{
		ObjectMeta: metav1.ObjectMeta{
			Namespace: clusterNamespace,
			Name:      secretName,
		},
		Data: map[string][]byte{
			"value": data,
			"token": []byte(tokenValue),
		},
	})
	if err != nil {
		return nil, err
	}

	return secret.Data, nil
}

func (m *Manager) GetRESTConfig(cluster *v1.Cluster, status v1.ClusterStatus) (*rest.Config, error) {
	secret, err := m.GetKubeConfig(cluster, status)
	if err != nil {
		return nil, err
	}

	return clientcmd.RESTConfigFromKubeConfig(secret.Data["value"])
}

func (m *Manager) GetKubeConfig(cluster *v1.Cluster, status v1.ClusterStatus) (*corev1.Secret, error) {
	if cluster.Spec.ClusterAPIConfig != nil {
		return m.secretCache.Get(cluster.Namespace, getKubeConfigSecretName(cluster.Spec.ClusterAPIConfig.ClusterName))
	}

	var (
		secretName = getKubeConfigSecretName(cluster.Name)
	)

	data, err := m.getKubeConfigData(cluster.Namespace, cluster.Name, secretName, status.ClusterName)
	if err != nil {
		return nil, err
	}

	return &corev1.Secret{
		ObjectMeta: metav1.ObjectMeta{
			Namespace: cluster.Namespace,
			Name:      secretName,
		},
		Data: data,
	}, nil
}
<commit_msg>Make ${cluster}-kubeconfig secret be owned by the prov cluster<commit_after>package kubeconfig

import (
	"crypto/rand"
	"crypto/sha256"
	"encoding/base32"
	"encoding/base64"
	"errors"
	"fmt"
	"strings"

	"github.com/moby/locker"
	v3 "github.com/rancher/rancher/pkg/apis/management.cattle.io/v3"
	v1 "github.com/rancher/rancher/pkg/apis/provisioning.cattle.io/v1"
	"github.com/rancher/rancher/pkg/features"
	mgmtcontrollers "github.com/rancher/rancher/pkg/generated/controllers/management.cattle.io/v3"
	"github.com/rancher/rancher/pkg/settings"
	"github.com/rancher/rancher/pkg/wrangler"
	appcontroller "github.com/rancher/wrangler/pkg/generated/controllers/apps/v1"
	corecontrollers "github.com/rancher/wrangler/pkg/generated/controllers/core/v1"
	"github.com/rancher/wrangler/pkg/name"
	"github.com/rancher/wrangler/pkg/randomtoken"
	corev1 "k8s.io/api/core/v1"
	apierror "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/rest"
	"k8s.io/client-go/tools/clientcmd"
	clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
)

const (
	userIDLabel     = "authn.management.cattle.io/token-userId"
	tokenKindLabel  = "authn.management.cattle.io/kind"
	tokenHashedAnno = "authn.management.cattle.io/token-hashed"

	hashFormat = "$%d:%s:%s" // $version:salt:hash -> $1:abc:def
	Version    = 2
)

type Manager struct {
	deploymentCache  appcontroller.DeploymentCache
	daemonsetCache   appcontroller.DaemonSetCache
	tokens           mgmtcontrollers.TokenClient
	userCache        mgmtcontrollers.UserCache
	users            mgmtcontrollers.UserClient
	secretCache      corecontrollers.SecretCache
	secrets          corecontrollers.SecretClient
	kubeConfigLocker locker.Locker
}

func New(clients *wrangler.Context) *Manager {
	return &Manager{
		deploymentCache: clients.Apps.Deployment().Cache(),
		daemonsetCache:  clients.Apps.DaemonSet().Cache(),
		tokens:          clients.Mgmt.Token(),
		userCache:       clients.Mgmt.User().Cache(),
		users:           clients.Mgmt.User(),
		secretCache:     clients.Core.Secret().Cache(),
		secrets:         clients.Core.Secret(),
	}
}

func getKubeConfigSecretName(clusterName string) string {
	return clusterName + "-kubeconfig"
}

func (m *Manager) getToken(clusterNamespace, clusterName string) (string, error) {
	kubeConfigSecretName := getKubeConfigSecretName(clusterName)
	if token, err := m.getSavedToken(clusterNamespace, kubeConfigSecretName); err != nil || token != "" {
		return token, err
	}

	// Need to be careful about caches being out of sync since we are dealing with multiple objects that
	// arent eventually consistent (because we delete and create the token for the user)
	if token, err := m.getSavedTokenNoCache(clusterNamespace, kubeConfigSecretName); err != nil || token != "" {
		return token, err
	}

	userName, err := m.EnsureUser(clusterNamespace, clusterName)
	if err != nil {
		return "", err
	}

	return m.createUserToken(userName)
}

func (m *Manager) EnsureUser(clusterNamespace, clusterName string) (string, error) {
	principalID := getPrincipalID(clusterNamespace, clusterName)
	userName := getUserNameForPrincipal(principalID)
	return userName, m.createUser(principalID, userName)
}

func getUserNameForPrincipal(principal string) string {
	hasher := sha256.New()
	hasher.Write([]byte(principal))
	sha := base32.StdEncoding.WithPadding(-1).EncodeToString(hasher.Sum(nil))[:10]
	return "u-" + strings.ToLower(sha)
}

func labelsForUser(principalID string) map[string]string {
	encodedPrincipalID := base32.HexEncoding.WithPadding(base32.NoPadding).EncodeToString([]byte(principalID))
	if len(encodedPrincipalID) > 63 {
		encodedPrincipalID = encodedPrincipalID[:63]
	}
	return map[string]string{
		encodedPrincipalID: "hashed-principal-name",
	}
}

func (m *Manager) getSavedToken(kubeConfigNamespace, kubeConfigName string) (string, error) {
	secret, err := m.secretCache.Get(kubeConfigNamespace, kubeConfigName)
	if apierror.IsNotFound(err) {
		return "", nil
	} else if err != nil {
		return "", err
	}
	return string(secret.Data["token"]), nil
}

func (m *Manager) getSavedTokenNoCache(kubeConfigNamespace, kubeConfigName string) (string, error) {
	secret, err := m.secrets.Get(kubeConfigNamespace, kubeConfigName, metav1.GetOptions{})
	if apierror.IsNotFound(err) {
		return "", nil
	} else if err != nil {
		return "", err
	}
	return string(secret.Data["token"]), nil
}

func getPrincipalID(clusterNamespace, clusterName string) string {
	return fmt.Sprintf("system://provisioning/%s/%s", clusterNamespace, clusterName)
}

func (m *Manager) createUser(principalID, userName string) error {
	_, err := m.userCache.Get(userName)
	if apierror.IsNotFound(err) {
		_, err = m.users.Create(&v3.User{
			ObjectMeta: metav1.ObjectMeta{
				Name:   userName,
				Labels: labelsForUser(principalID),
			},
			PrincipalIDs: []string{
				principalID,
			},
		})
	}
	return err
}

func (m *Manager) createUserToken(userName string) (string, error) {
	_, err := m.tokens.Get(userName, metav1.GetOptions{})
	if err == nil {
		err = m.tokens.Delete(userName, nil)
	}
	if err != nil && !apierror.IsNotFound(err) {
		return "", err
	}

	tokenValue, err := randomtoken.Generate()
	if err != nil {
		return "", fmt.Errorf("failed to generate token key: %w", err)
	}

	token := &v3.Token{
		ObjectMeta: metav1.ObjectMeta{
			Name: userName,
			Labels: map[string]string{
				userIDLabel:    userName,
				tokenKindLabel: "provisioning",
			},
			Annotations: map[string]string{},
		},
		UserID:       userName,
		AuthProvider: "local",
		IsDerived:    true,
		Token:        tokenValue,
	}

	if features.TokenHashing.Enabled() {
		tokenHash, err := createSHA256Hash(tokenValue)
		if err != nil {
			return "", err
		}
		token.Token = tokenHash
		token.Annotations[tokenHashedAnno] = "true"
	}

	_, err = m.tokens.Create(token)
	return fmt.Sprintf("%s:%s", userName, tokenValue), err
}

func createSHA256Hash(secretKey string) (string, error) {
	salt := make([]byte, 8)
	_, err := rand.Read(salt)
	if err != nil {
		return "", err
	}
	hash := sha256.Sum256([]byte(fmt.Sprintf("%s%s", salt, secretKey)))
	encSalt := base64.RawStdEncoding.EncodeToString(salt)
	encKey := base64.RawStdEncoding.EncodeToString(hash[:])
	return fmt.Sprintf(hashFormat, Version, encSalt, encKey), nil
}

func (m *Manager) GetCRTBForAdmin(cluster *v1.Cluster, status v1.ClusterStatus) (*v3.ClusterRoleTemplateBinding, error) {
	if status.ClusterName == "" {
		return nil, fmt.Errorf("management cluster is not assigned to v1.Cluster")
	}
	principalID := getPrincipalID(cluster.Namespace, cluster.Name)
	userName := getUserNameForPrincipal(principalID)
	return &v3.ClusterRoleTemplateBinding{
		ObjectMeta: metav1.ObjectMeta{
			Name:      name.SafeConcatName(status.ClusterName, "owner"),
			Namespace: status.ClusterName,
		},
		ClusterName:       status.ClusterName,
		UserName:          userName,
		UserPrincipalName: principalID,
		RoleTemplateName:  "cluster-owner",
	}, nil
}

func (m *Manager) getKubeConfigData(cluster *v1.Cluster, secretName, managementClusterName string) (map[string][]byte, error) {
	secret, err := m.secretCache.Get(cluster.Namespace, secretName)
	if err == nil {
		if secret.Data == nil || secret.Data["token"] == nil || len(secret.OwnerReferences) == 0 {
			// Check if we require a new secret based on the token value and annotation(s). We delete the old secret since it may contain
			// annotations, owner references, etc. that are out of date. We will then continue to create the new secret.
			if err := m.secrets.Delete(cluster.Namespace, secretName, &metav1.DeleteOptions{}); err != nil && !apierror.IsNotFound(err) {
				return nil, err
			}
		} else {
			return secret.Data, nil
		}
	} else if !apierror.IsNotFound(err) {
		return nil, err
	}

	lockID := cluster.Namespace + "/" + cluster.Name
	m.kubeConfigLocker.Lock(lockID)
	defer m.kubeConfigLocker.Unlock(lockID)

	secret, err = m.secrets.Get(cluster.Namespace, secretName, metav1.GetOptions{})
	if err == nil {
		return secret.Data, nil
	}

	tokenValue, err := m.getToken(cluster.Namespace, cluster.Name)
	if err != nil {
		return nil, err
	}

	serverURL, cacert := settings.InternalServerURL.Get(), settings.InternalCACerts.Get()
	if serverURL == "" {
		return nil, errors.New("server url is missing, can't generate kubeconfig for fleet import cluster")
	}

	data, err := clientcmd.Write(clientcmdapi.Config{
		Clusters: map[string]*clientcmdapi.Cluster{
			"cluster": {
				Server:                   fmt.Sprintf("%s/k8s/clusters/%s", serverURL, managementClusterName),
				CertificateAuthorityData: []byte(strings.TrimSpace(cacert)),
			},
		},
		AuthInfos: map[string]*clientcmdapi.AuthInfo{
			"user": {
				Token: tokenValue,
			},
		},
		Contexts: map[string]*clientcmdapi.Context{
			"default": {
				Cluster:  "cluster",
				AuthInfo: "user",
			},
		},
		CurrentContext: "default",
	})
	if err != nil {
		return nil, err
	}

	secret, err = m.secrets.Create(&corev1.Secret{
		ObjectMeta: metav1.ObjectMeta{
			Namespace: cluster.Namespace,
			Name:      secretName,
			OwnerReferences: []metav1.OwnerReference{{
				APIVersion: "provisioning.cattle.io/v1",
				Kind:       "Cluster",
				Name:       cluster.Name,
				UID:        cluster.UID,
			}},
		},
		Data: map[string][]byte{
			"value": data,
			"token": []byte(tokenValue),
		},
	})
	if err != nil {
		return nil, err
	}

	return secret.Data, nil
}

func (m *Manager) GetRESTConfig(cluster *v1.Cluster, status v1.ClusterStatus) (*rest.Config, error) {
	secret, err := m.GetKubeConfig(cluster, status)
	if err != nil {
		return nil, err
	}

	return clientcmd.RESTConfigFromKubeConfig(secret.Data["value"])
}

func (m *Manager) GetKubeConfig(cluster *v1.Cluster, status v1.ClusterStatus) (*corev1.Secret, error) {
	if cluster.Spec.ClusterAPIConfig != nil {
		return m.secretCache.Get(cluster.Namespace, getKubeConfigSecretName(cluster.Spec.ClusterAPIConfig.ClusterName))
	}

	var (
		secretName = getKubeConfigSecretName(cluster.Name)
	)

	data, err := m.getKubeConfigData(cluster, secretName, status.ClusterName)
	if err != nil {
		return nil, err
	}

	return &corev1.Secret{
		ObjectMeta: metav1.ObjectMeta{
			Namespace: cluster.Namespace,
			Name:      secretName,
		},
		Data: data,
	}, nil
}
<|endoftext|>


<commit_before>// Package sync2 is the supplement of the standard library `sync`.
package sync2

import (
	"container/list"
	"fmt"
	"sync"
)

// ResourceLock is an interface about the resource lock.
type ResourceLock interface {
	// Lock locks the resource, which id is the resource id.
	//
	// When the resource named id has been locked, it should wait to be unlocked
	// by the locker.
	Lock(id string)

	// Unlock unlocks the resource, which id is the resource id.
	//
	// Notice: If the lock is not locked, it should panic.
	Unlock(id string)
}

type hashResourceLock map[uint8]resourceLock

func (h hashResourceLock) getResource(id string) resourceLock {
	return h[id[len(id)-1]]
}

func (h hashResourceLock) Lock(id string) {
	h.getResource(id).Lock(id)
}

func (h hashResourceLock) Unlock(id string) {
	h.getResource(id).Unlock(id)
}

// NewHashResourceLock returns a new ResourceLock based on hash.
//
// Notice: this implementation use the quadratic hash. The first hash does not
// use the lock, so it maybe have a better performance when the resources are
// not the same.
func NewHashResourceLock() ResourceLock {
	r := make(hashResourceLock, 256)
	for i := 0; i < 256; i++ {
		r[uint8(i)] = newResourceLock()
	}
	return r
}

// NewResourceLock is the alias of NewHashResourceLock.
//
// This is used to be compatible with the old.
func NewResourceLock() ResourceLock {
	return NewHashResourceLock()
}

// resourceLock is a lock to lock a certain resource by the resource id.
type resourceLock struct {
	locker    *sync.Mutex
	resources map[string]struct{}
	waiters   map[string]*list.List
}

// newResourceLock returns a new resourceLock.
func newResourceLock() resourceLock {
	return resourceLock{
		locker:    new(sync.Mutex),
		resources: make(map[string]struct{}),
		waiters:   make(map[string]*list.List),
	}
}

// Lock locks a certain resource by its id.
//
// Once Lock is called, Unlock must be called later, that's, Lock and Unlock
// must appear in pairs.
func (r resourceLock) Lock(id string) {
	for !r.lock(id) {
	}
}

func (r resourceLock) lock(id string) bool {
	r.locker.Lock()
	// Lock successfully
	if _, ok := r.resources[id]; !ok {
		r.resources[id] = struct{}{}
		r.locker.Unlock()
		return true
	}

	// Fail to lock, add the wait to wake up.
	waiter, ok := r.waiters[id]
	if !ok {
		waiter = list.New()
		r.waiters[id] = waiter
	}
	wake := make(chan struct{})
	waiter.PushBack(func() {
		wake <- struct{}{}
	})
	r.locker.Unlock()
	<-wake
	return false
}

// Unlock unlocks a certain resource by its id.
//
// If the lock is not locked, it will panic.
func (r resourceLock) Unlock(id string) {
	r.locker.Lock()
	if _, ok := r.resources[id]; ok {
		delete(r.resources, id)
		if waiter, ok := r.waiters[id]; ok {
			e := waiter.Front()
			e.Value.(func())()
			waiter.Remove(e)

			if waiter.Len() == 0 {
				delete(r.waiters, id)
			}
		}
		r.locker.Unlock()
		return
	}
	r.locker.Unlock()
	panic(fmt.Errorf("The lock[%s] is not locked", id))
}
<commit_msg>Update the doc<commit_after>// Package sync2 is the supplement of the standard library `sync`.
package sync2

import (
	"container/list"
	"fmt"
	"sync"
)

// ResourceLock is an interface about the resource lock.
type ResourceLock interface {
	// Lock locks the resource, which id is the resource id.
	//
	// When the resource named id has been locked, it should wait to be unlocked
	// by the locker.
	//
	// If id is empty, it should panic.
	Lock(id string)

	// Unlock unlocks the resource, which id is the resource id.
	//
	// If the lock is not locked or id is empty, it should panic.
	Unlock(id string)
}

type hashResourceLock map[uint8]resourceLock

func (h hashResourceLock) getResource(id string) resourceLock {
	return h[id[len(id)-1]]
}

func (h hashResourceLock) Lock(id string) {
	h.getResource(id).Lock(id)
}

func (h hashResourceLock) Unlock(id string) {
	h.getResource(id).Unlock(id)
}

// NewHashResourceLock returns a new ResourceLock based on hash.
//
// Notice: this implementation use the quadratic hash. The first hash does not
// use the lock, so it maybe have a better performance when the resources are
// not the same.
func NewHashResourceLock() ResourceLock {
	r := make(hashResourceLock, 256)
	for i := 0; i < 256; i++ {
		r[uint8(i)] = newResourceLock()
	}
	return r
}

// NewResourceLock is the alias of NewHashResourceLock.
//
// This is used to be compatible with the old.
func NewResourceLock() ResourceLock {
	return NewHashResourceLock()
}

// resourceLock is a lock to lock a certain resource by the resource id.
type resourceLock struct {
	locker    *sync.Mutex
	resources map[string]struct{}
	waiters   map[string]*list.List
}

// newResourceLock returns a new resourceLock.
func newResourceLock() resourceLock {
	return resourceLock{
		locker:    new(sync.Mutex),
		resources: make(map[string]struct{}),
		waiters:   make(map[string]*list.List),
	}
}

// Lock locks a certain resource by its id.
//
// Once Lock is called, Unlock must be called later, that's, Lock and Unlock
// must appear in pairs.
func (r resourceLock) Lock(id string) {
	for !r.lock(id) {
	}
}

func (r resourceLock) lock(id string) bool {
	r.locker.Lock()
	// Lock successfully
	if _, ok := r.resources[id]; !ok {
		r.resources[id] = struct{}{}
		r.locker.Unlock()
		return true
	}

	// Fail to lock, add the wait to wake up.
	waiter, ok := r.waiters[id]
	if !ok {
		waiter = list.New()
		r.waiters[id] = waiter
	}
	wake := make(chan struct{})
	waiter.PushBack(func() {
		wake <- struct{}{}
	})
	r.locker.Unlock()
	<-wake
	return false
}

// Unlock unlocks a certain resource by its id.
//
// If the lock is not locked, it will panic.
func (r resourceLock) Unlock(id string) {
	r.locker.Lock()
	if _, ok := r.resources[id]; ok {
		delete(r.resources, id)
		if waiter, ok := r.waiters[id]; ok {
			e := waiter.Front()
			e.Value.(func())()
			waiter.Remove(e)

			if waiter.Len() == 0 {
				delete(r.waiters, id)
			}
		}
		r.locker.Unlock()
		return
	}
	r.locker.Unlock()
	panic(fmt.Errorf("The lock[%s] is not locked", id))
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"image"
	_ "image/png"
	"io"
	"log"
	"net/http"
	"os"

	"github.com/generaltso/vibrant"
	"gopkg.in/gographics/imagick.v3/imagick"
)

func getImageColors(file string) (fgColor, bgColor string) {
	checkErr := func(err error) {
		if err != nil {
			panic(err)
		}
	}

	f, err := os.Open(file)
	checkErr(err)
	defer f.Close()

	img, _, err := image.Decode(f)
	checkErr(err)

	palette, err := vibrant.NewPaletteFromImage(img)
	checkErr(err)

	for name, swatch := range palette.ExtractAwesome() {
		if name == "Vibrant" {
			fgColor = swatch.Color.TitleTextColor().RGBHex()
			bgColor = swatch.Color.RGBHex()
		}
	}
	if fgColor == bgColor {
		fgColor = "#000"
	}
	return fgColor, bgColor
}

func createStoryText(fgColor, bgColor, text string) *imagick.MagickWand {
	mw := imagick.NewMagickWand()
	dw := imagick.NewDrawingWand()
	defer dw.Destroy()
	pw := imagick.NewPixelWand()
	defer pw.Destroy()

	pw.SetColor("none")
	mw.NewImage(1080, 103, pw)

	dw.SetFont("TTCommonsB")
	dw.SetFontSize(80)

	textMetrics := mw.QueryFontMetrics(dw, text)

	textWidth := textMetrics.TextWidth

	pw.SetColor(bgColor)
	dw.SetFillColor(pw)

	dw.RoundRectangle((1080-textWidth)/2-20, 0, (1080-textWidth)/2+textWidth+20, 103, 10, 10)

	pw.SetColor(fgColor)
	dw.SetFillColor(pw)
	dw.SetTextAlignment(imagick.ALIGN_CENTER)
	dw.Annotation(1080/2, 93-17, text)

	mw.DrawImage(dw)

	return mw
}

func downloadTeamLogo(filepath, url string) (err error) {
	// Create the file
	out, err := os.Create(filepath)
	if err != nil {
		return err
	}
	defer out.Close()

	// Get the data
	resp, err := http.Get(url)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	// Check server response
	if resp.StatusCode != http.StatusOK {
		return fmt.Errorf("bad status: %s", resp.Status)
	}

	// Writer the body to file
	_, err = io.Copy(out, resp.Body)
	if err != nil {
		return err
	}

	return nil
}

func copyFile(src, dst string) (int64, error) {
	sourceFileStat, err := os.Stat(src)
	if err != nil {
		return 0, err
	}

	if !sourceFileStat.Mode().IsRegular() {
		return 0, fmt.Errorf("%s is not a regular file", src)
	}

	source, err := os.Open(src)
	if err != nil {
		return 0, err
	}
	defer source.Close()

	destination, err := os.Create(dst)
	if err != nil {
		return 0, err
	}
	defer destination.Close()
	nBytes, err := io.Copy(destination, source)
	return nBytes, err
}

func getTeamsLogo(radiantTeamID, direTeamID int) {
	copyFile("assets/images/nologo.png", fmt.Sprintf("tmp/teams/%d.png", radiantTeamID))
	copyFile("assets/images/nologo.png", fmt.Sprintf("tmp/teams/%d.png", direTeamID))
	teamsData := getTeamsData()
	for _, team := range teamsData {
		teamID := team.TeamID
		if teamID == radiantTeamID || teamID == direTeamID {
			logo := team.LogoURL
			if logo == "" {
				logo = "https://github.com/dotaspirit/dotaspirit/raw/master/assets/images/nologo.png"
			}
			err := downloadTeamLogo(fmt.Sprintf("tmp/teams/%d.png", teamID), logo)
			if err != nil {
				log.Printf("Error downloading %d team logo", teamID)
			}
		}
	}
}

func guessResizeSize(width, height uint) (wOut, hOut uint) {
	if width > height {
		mult := 294 / width
		wOut = width * mult
		hOut = height * mult
	} else {
		mult := 294 / height
		wOut = width * mult
		hOut = height * mult
	}
	log.Printf("b4 %d %d, af %d %d", width, height, wOut, hOut)
	return uint(wOut), uint(hOut)
}

func makeStoryImage(matchData oDotaMatchData) {
	radiantTeamName := matchData.RadiantName
	direTeamName := matchData.DireName

	if radiantTeamName == "" {
		radiantTeamName = "Radiant"
	}

	if direTeamName == "" {
		direTeamName = "Dire"
	}

	radiantTeamID := matchData.RadiantTeamID
	direTeamID := matchData.DireTeamID
	matchID := matchData.MatchID

	getTeamsLogo(radiantTeamID, direTeamID)

	direTeamFile := fmt.Sprintf("tmp/teams/%d.png", direTeamID)
	radiantTeamFile := fmt.Sprintf("tmp/teams/%d.png", radiantTeamID)

	mw := imagick.NewMagickWand()
	defer mw.Destroy()
	dw := imagick.NewDrawingWand()
	defer dw.Destroy()
	pw := imagick.NewPixelWand()
	defer pw.Destroy()

	dw.SetFont("TTCommonsB")

	pw.SetColor("#242424")
	mw.NewImage(1080, 1920, pw)

	fgColor, bgColor := getImageColors(direTeamFile)

	direTeamText := createStoryText(fgColor, bgColor, direTeamName)

	mw.CompositeImage(direTeamText, imagick.COMPOSITE_OP_OVER, true, 0, ((1920/2)-294)/2+250)

	direTeamText.Destroy()

	fgColor, bgColor = getImageColors(radiantTeamFile)

	radiantTeamText := createStoryText(fgColor, bgColor, radiantTeamName)

	mw.CompositeImage(radiantTeamText, imagick.COMPOSITE_OP_OVER, true, 0, 1920/2+((1920/2)-294)/2+250)

	radiantTeamText.Destroy()

	direTeamLogo := imagick.NewMagickWand()
	direTeamLogo.ReadImage(direTeamFile)

	width := direTeamLogo.GetImageWidth()
	height := direTeamLogo.GetImageHeight()
	wGuessed, hGuessed := guessResizeSize(width, height)

	direTeamLogo.ResizeImage(wGuessed, hGuessed, imagick.FILTER_CUBIC)
	mw.CompositeImage(direTeamLogo, imagick.COMPOSITE_OP_OVER, true, (1080-294)/2, ((1920/2)-294)/2-100)
	direTeamLogo.Destroy()

	radiantTeamLogo := imagick.NewMagickWand()
	radiantTeamLogo.ReadImage(radiantTeamFile)

	width = radiantTeamLogo.GetImageWidth()
	height = radiantTeamLogo.GetImageHeight()
	wGuessed, hGuessed = guessResizeSize(width, height)

	radiantTeamLogo.ResizeImage(wGuessed, hGuessed, imagick.FILTER_CUBIC)
	mw.CompositeImage(radiantTeamLogo, imagick.COMPOSITE_OP_OVER, true, (1080-294)/2, 1920/2+((1920/2)-294)/2-100)
	radiantTeamLogo.Destroy()

	vs := createStoryText("#242424", "#ffffff", "VS")
	mw.CompositeImage(vs, imagick.COMPOSITE_OP_OVER, true, 0, 1920/2-51)
	vs.Destroy()

	mw.DrawImage(dw)
	storyFile := fmt.Sprintf("tmp/s%d.png", matchID)
	mw.WriteImage(storyFile)
}
<commit_msg>Fxi font name<commit_after>package main

import (
	"fmt"
	"image"
	_ "image/png"
	"io"
	"log"
	"net/http"
	"os"

	"github.com/generaltso/vibrant"
	"gopkg.in/gographics/imagick.v3/imagick"
)

func getImageColors(file string) (fgColor, bgColor string) {
	checkErr := func(err error) {
		if err != nil {
			panic(err)
		}
	}

	f, err := os.Open(file)
	checkErr(err)
	defer f.Close()

	img, _, err := image.Decode(f)
	checkErr(err)

	palette, err := vibrant.NewPaletteFromImage(img)
	checkErr(err)

	for name, swatch := range palette.ExtractAwesome() {
		if name == "Vibrant" {
			fgColor = swatch.Color.TitleTextColor().RGBHex()
			bgColor = swatch.Color.RGBHex()
		}
	}
	if fgColor == bgColor {
		fgColor = "#000"
	}
	return fgColor, bgColor
}

func createStoryText(fgColor, bgColor, text string) *imagick.MagickWand {
	mw := imagick.NewMagickWand()
	dw := imagick.NewDrawingWand()
	defer dw.Destroy()
	pw := imagick.NewPixelWand()
	defer pw.Destroy()

	pw.SetColor("none")
	mw.NewImage(1080, 103, pw)

	dw.SetFont("TT-Commons-Bold")
	dw.SetFontSize(80)

	textMetrics := mw.QueryFontMetrics(dw, text)

	textWidth := textMetrics.TextWidth

	pw.SetColor(bgColor)
	dw.SetFillColor(pw)

	dw.RoundRectangle((1080-textWidth)/2-20, 0, (1080-textWidth)/2+textWidth+20, 103, 10, 10)

	pw.SetColor(fgColor)
	dw.SetFillColor(pw)
	dw.SetTextAlignment(imagick.ALIGN_CENTER)
	dw.Annotation(1080/2, 93-17, text)

	mw.DrawImage(dw)

	return mw
}

func downloadTeamLogo(filepath, url string) (err error) {
	// Create the file
	out, err := os.Create(filepath)
	if err != nil {
		return err
	}
	defer out.Close()

	// Get the data
	resp, err := http.Get(url)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	// Check server response
	if resp.StatusCode != http.StatusOK {
		return fmt.Errorf("bad status: %s", resp.Status)
	}

	// Writer the body to file
	_, err = io.Copy(out, resp.Body)
	if err != nil {
		return err
	}

	return nil
}

func copyFile(src, dst string) (int64, error) {
	sourceFileStat, err := os.Stat(src)
	if err != nil {
		return 0, err
	}

	if !sourceFileStat.Mode().IsRegular() {
		return 0, fmt.Errorf("%s is not a regular file", src)
	}

	source, err := os.Open(src)
	if err != nil {
		return 0, err
	}
	defer source.Close()

	destination, err := os.Create(dst)
	if err != nil {
		return 0, err
	}
	defer destination.Close()
	nBytes, err := io.Copy(destination, source)
	return nBytes, err
}

func getTeamsLogo(radiantTeamID, direTeamID int) {
	copyFile("assets/images/nologo.png", fmt.Sprintf("tmp/teams/%d.png", radiantTeamID))
	copyFile("assets/images/nologo.png", fmt.Sprintf("tmp/teams/%d.png", direTeamID))
	teamsData := getTeamsData()
	for _, team := range teamsData {
		teamID := team.TeamID
		if teamID == radiantTeamID || teamID == direTeamID {
			logo := team.LogoURL
			if logo == "" {
				logo = "https://github.com/dotaspirit/dotaspirit/raw/master/assets/images/nologo.png"
			}
			err := downloadTeamLogo(fmt.Sprintf("tmp/teams/%d.png", teamID), logo)
			if err != nil {
				log.Printf("Error downloading %d team logo", teamID)
			}
		}
	}
}

func guessResizeSize(width, height uint) (wOut, hOut uint) {
	if width > height {
		mult := 294 / width
		wOut = width * mult
		hOut = height * mult
	} else {
		mult := 294 / height
		wOut = width * mult
		hOut = height * mult
	}
	log.Printf("b4 %d %d, af %d %d", width, height, wOut, hOut)
	return uint(wOut), uint(hOut)
}

func makeStoryImage(matchData oDotaMatchData) {
	radiantTeamName := matchData.RadiantName
	direTeamName := matchData.DireName

	if radiantTeamName == "" {
		radiantTeamName = "Radiant"
	}

	if direTeamName == "" {
		direTeamName = "Dire"
	}

	radiantTeamID := matchData.RadiantTeamID
	direTeamID := matchData.DireTeamID
	matchID := matchData.MatchID

	getTeamsLogo(radiantTeamID, direTeamID)

	direTeamFile := fmt.Sprintf("tmp/teams/%d.png", direTeamID)
	radiantTeamFile := fmt.Sprintf("tmp/teams/%d.png", radiantTeamID)

	mw := imagick.NewMagickWand()
	defer mw.Destroy()
	dw := imagick.NewDrawingWand()
	defer dw.Destroy()
	pw := imagick.NewPixelWand()
	defer pw.Destroy()

	dw.SetFont("TTCommonsB")

	pw.SetColor("#242424")
	mw.NewImage(1080, 1920, pw)

	fgColor, bgColor := getImageColors(direTeamFile)

	direTeamText := createStoryText(fgColor, bgColor, direTeamName)

	mw.CompositeImage(direTeamText, imagick.COMPOSITE_OP_OVER, true, 0, ((1920/2)-294)/2+250)

	direTeamText.Destroy()

	fgColor, bgColor = getImageColors(radiantTeamFile)

	radiantTeamText := createStoryText(fgColor, bgColor, radiantTeamName)

	mw.CompositeImage(radiantTeamText, imagick.COMPOSITE_OP_OVER, true, 0, 1920/2+((1920/2)-294)/2+250)

	radiantTeamText.Destroy()

	direTeamLogo := imagick.NewMagickWand()
	direTeamLogo.ReadImage(direTeamFile)

	width := direTeamLogo.GetImageWidth()
	height := direTeamLogo.GetImageHeight()
	wGuessed, hGuessed := guessResizeSize(width, height)

	direTeamLogo.ResizeImage(wGuessed, hGuessed, imagick.FILTER_CUBIC)
	mw.CompositeImage(direTeamLogo, imagick.COMPOSITE_OP_OVER, true, (1080-294)/2, ((1920/2)-294)/2-100)
	direTeamLogo.Destroy()

	radiantTeamLogo := imagick.NewMagickWand()
	radiantTeamLogo.ReadImage(radiantTeamFile)

	width = radiantTeamLogo.GetImageWidth()
	height = radiantTeamLogo.GetImageHeight()
	wGuessed, hGuessed = guessResizeSize(width, height)

	radiantTeamLogo.ResizeImage(wGuessed, hGuessed, imagick.FILTER_CUBIC)
	mw.CompositeImage(radiantTeamLogo, imagick.COMPOSITE_OP_OVER, true, (1080-294)/2, 1920/2+((1920/2)-294)/2-100)
	radiantTeamLogo.Destroy()

	vs := createStoryText("#242424", "#ffffff", "VS")
	mw.CompositeImage(vs, imagick.COMPOSITE_OP_OVER, true, 0, 1920/2-51)
	vs.Destroy()

	mw.DrawImage(dw)
	storyFile := fmt.Sprintf("tmp/s%d.png", matchID)
	mw.WriteImage(storyFile)
}
<|endoftext|>


<commit_before>package ticketmatic

import (
	"crypto/hmac"
	"crypto/sha256"
	"encoding/hex"
	"errors"
	"fmt"
	"net/url"
	"sort"
)

// Widget helper
type Widgets struct {
	AccountCode string
	AccessKey   string
	SecretKey   string
}

func NewWidgets(accountcode, accesskey, secretkey string) *Widgets {
	widgets := &Widgets{
		AccountCode: accountcode,
		AccessKey:   accesskey,
		SecretKey:   secretkey,
	}

	return widgets
}

type param struct {
	key   string
	value string
}

type paramSlice []param

func (a paramSlice) Len() int           { return len(a) }
func (a paramSlice) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a paramSlice) Less(i, j int) bool { return a[i].key < a[j].key }

func (w *Widgets) GenerateUrl(widget string, params map[string]string) string {
	v := url.Values{}
	v.Set("accesskey", w.AccessKey)

	for k, val := range params {
		v.Set(k, val)
	}

	sig := w.calculateSignature(params)

	v.Set("signature", sig)

	url := fmt.Sprintf("%s/widgets/%s/%s?%s", Server, w.AccountCode, widget, v.Encode())
	return url
}

func (w *Widgets) VerifyReturnUrl(params map[string]string) error {
	key, ok := params["accesskey"]
	if !ok || key != w.AccessKey {
		return errors.New("Failed to verify return URL: Bad access key")
	}
	delete(params, "accesskey")

	sig, ok := params["signature"]
	if !ok {
		return errors.New("Failed to verify return URL: Signature missing")
	}
	delete(params, "signature")

	expected := w.calculateSignature(params)
	if expected != sig {
		return errors.New("Failed to verify return URL: Signature mismatch")
	}

	return nil
}

// This variant of VerifyReturnUrl allows you to pass in req.Form directly
func (w *Widgets) VerifyFormReturnUrl(values url.Values) error {
	params := make(map[string]string)
	for k, v := range values {
		for i, s := range v {
			if i == 0 {
				params[k] = s
			} else {
				params[k] += s
			}
		}
	}

	return w.VerifyReturnUrl(params)
}

func (w *Widgets) calculateSignature(params map[string]string) string {
	delete(params, "l")

	paramObjs := make(paramSlice, 0)
	for k, v := range params {
		paramObjs = append(paramObjs, param{k, v})
	}

	sort.Sort(paramObjs)

	hash := ""
	for _, p := range paramObjs {
		hash += p.key + p.value
	}

	msg := fmt.Sprintf("%s%s%s", w.AccessKey, w.AccountCode, hash)

	mac := hmac.New(sha256.New, []byte(w.SecretKey))
	mac.Write([]byte(msg))
	return hex.EncodeToString(mac.Sum(nil))
}
<commit_msg>widget signature verification fix<commit_after>package ticketmatic

import (
	"crypto/hmac"
	"crypto/sha256"
	"encoding/hex"
	"errors"
	"fmt"
	"net/url"
	"sort"
)

// Widget helper
type Widgets struct {
	AccountCode string
	AccessKey   string
	SecretKey   string
}

func NewWidgets(accountcode, accesskey, secretkey string) *Widgets {
	widgets := &Widgets{
		AccountCode: accountcode,
		AccessKey:   accesskey,
		SecretKey:   secretkey,
	}

	return widgets
}

type param struct {
	key   string
	value string
}

type paramSlice []param

func (a paramSlice) Len() int           { return len(a) }
func (a paramSlice) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a paramSlice) Less(i, j int) bool { return a[i].key < a[j].key }

func (w *Widgets) GenerateUrl(widget string, params map[string]string) string {
	v := url.Values{}
	v.Set("accesskey", w.AccessKey)

	for k, val := range params {
		v.Set(k, val)
	}

	sig := w.calculateSignature(params)

	v.Set("signature", sig)

	url := fmt.Sprintf("%s/widgets/%s/%s?%s", Server, w.AccountCode, widget, v.Encode())
	return url
}

func (w *Widgets) VerifyReturnUrl(params map[string]string) error {
	key, ok := params["accesskey"]
	if !ok || key != w.AccessKey {
		return errors.New("Failed to verify return URL: Bad access key")
	}
	delete(params, "accesskey")

	sig, ok := params["signature"]
	if !ok {
		return errors.New("Failed to verify return URL: Signature missing")
	}
	delete(params, "signature")

	expected := w.calculateSignature(params)
	if expected != sig {
		return errors.New("Failed to verify return URL: Signature mismatch")
	}

	return nil
}

// This variant of VerifyReturnUrl allows you to pass in req.Form directly
func (w *Widgets) VerifyFormReturnUrl(values url.Values) error {
	params := make(map[string]string)
	for k, v := range values {
		for i, s := range v {
			if i == 0 {
				params[k] = s
			} else {
				params[k] += s
			}
		}
	}

	return w.VerifyReturnUrl(params)
}

func (w *Widgets) calculateSignature(params map[string]string) string {
	delete(params, "l")
	delete(params, "ordercode")

	paramObjs := make(paramSlice, 0)
	for k, v := range params {
		paramObjs = append(paramObjs, param{k, v})
	}

	sort.Sort(paramObjs)

	hash := ""
	for _, p := range paramObjs {
		hash += p.key + p.value
	}

	msg := fmt.Sprintf("%s%s%s", w.AccessKey, w.AccountCode, hash)

	mac := hmac.New(sha256.New, []byte(w.SecretKey))
	mac.Write([]byte(msg))
	return hex.EncodeToString(mac.Sum(nil))
}
<|endoftext|>


<commit_before>package gocbcore

import (
	"encoding/binary"
)

type memdClient struct {
	conn        memdConn
	opList      memdOpMap
	errorMap    *kvErrorMap
	closeNotify chan bool
	dcpAckSize  int
	dcpFlowRecv int
}

func newMemdClient(conn memdConn) *memdClient {
	client := memdClient{
		conn:        conn,
		closeNotify: make(chan bool),
	}
	client.run()
	return &client
}

func (client *memdClient) EnableDcpBufferAck(bufferAckSize int) {
	client.dcpAckSize = bufferAckSize
}

func (client *memdClient) maybeSendDcpBufferAck(packet *memdPacket) {
	packetLen := 24 + len(packet.Extras) + len(packet.Key) + len(packet.Value)

	client.dcpFlowRecv += packetLen
	if client.dcpFlowRecv < client.dcpAckSize {
		return
	}

	ackAmt := client.dcpFlowRecv

	extrasBuf := make([]byte, 4)
	binary.BigEndian.PutUint32(extrasBuf, uint32(ackAmt))

	err := client.conn.WritePacket(&memdPacket{
		Magic:  reqMagic,
		Opcode: cmdDcpBufferAck,
		Extras: extrasBuf,
	})
	if err != nil {
		logWarnf("Failed to dispatch DCP buffer ack: %s", err)
	}

	client.dcpFlowRecv -= ackAmt
}

func (client *memdClient) SetErrorMap(errorMap *kvErrorMap) {
	client.errorMap = errorMap
}

func (client *memdClient) Address() string {
	return client.conn.RemoteAddr()
}

func (client *memdClient) CloseNotify() chan bool {
	return client.closeNotify
}

func (client *memdClient) SendRequest(req *memdQRequest) error {
	client.opList.Add(req)

	err := client.conn.WritePacket(&req.memdPacket)
	if err != nil {
		logDebugf("memdClient write failure: %v", err)
		client.opList.Remove(req)
		return err
	}

	return nil
}

func (client *memdClient) resolveRequest(resp *memdQResponse) {
	opIndex := resp.Opaque

	// Find the request that goes with this response
	req := client.opList.FindAndMaybeRemove(opIndex)

	if req == nil {
		// There is no known request that goes with this response.  Ignore it.
		logDebugf("Received response with no corresponding request.")
		return
	}

	if req.RoutingCallback != nil {
		if req.RoutingCallback(resp, req) {
			logSchedf("Routing callback intercepted response")
			return
		}
	}

	// Grab an error object if one needs to exist.
	var err error
	if resp.Magic == resMagic {
		err = getMemdError(resp.Status, client.errorMap)
	}

	// Call the requests callback handler...
	logSchedf("Dispatching response callback. OP=0x%x. Opaque=%d", resp.Opcode, resp.Opaque)
	req.tryCallback(resp, err)
}

func (client *memdClient) run() {
	dcpBufferQ := make(chan *memdQResponse)
	killSwitch := make(chan bool, 1)
	go func() {
		for {
			select {
			case resp, more := <-dcpBufferQ:
				if !more {
					return
				}

				logSchedf("Resolving response OP=0x%x. Opaque=%d", resp.Opcode, resp.Opaque)
				client.resolveRequest(resp)

				if client.dcpAckSize > 0 {
					client.maybeSendDcpBufferAck(&resp.memdPacket)
				}
			case <-killSwitch:
				close(dcpBufferQ)
			}
		}
	}()

	go func() {
		for {
			resp := &memdQResponse{
				sourceAddr: client.conn.RemoteAddr(),
			}

			err := client.conn.ReadPacket(&resp.memdPacket)
			if err != nil {
				logErrorf("memdClient read failure: %v", err)
				break
			}

			// We handle DCP no-op's directly here so we can reply immediately.
			if resp.memdPacket.Opcode == cmdDcpNoop {
				err := client.conn.WritePacket(&memdPacket{
					Magic:  resMagic,
					Opcode: cmdDcpNoop,
				})
				if err != nil {
					logWarnf("Failed to dispatch DCP noop reply: %s", err)
				}
				continue
			}

			switch resp.memdPacket.Opcode {
			case cmdDcpDeletion:
				fallthrough
			case cmdDcpExpiration:
				fallthrough
			case cmdDcpMutation:
				fallthrough
			case cmdDcpSnapshotMarker:
				fallthrough
			case cmdDcpStreamEnd:
				fallthrough
			case cmdDcpStreamReq:
				dcpBufferQ <- resp
				continue
			default:
				logSchedf("Resolving response OP=0x%x. Opaque=%d", resp.Opcode, resp.Opaque)
				client.resolveRequest(resp)
			}
		}

		err := client.conn.Close()
		if err != nil {
			// Lets log an error, as this is non-fatal
			logErrorf("Failed to shut down client connection (%s)", err)
		}

		client.opList.Drain(func(req *memdQRequest) {
			req.tryCallback(nil, ErrNetwork)
		})

		killSwitch <- true
		client.closeNotify <- true
	}()
}

func (client *memdClient) Close() error {
	return client.conn.Close()
}
<commit_msg>Include the request opaque in the dcp noop response<commit_after>package gocbcore

import (
	"encoding/binary"
)

type memdClient struct {
	conn        memdConn
	opList      memdOpMap
	errorMap    *kvErrorMap
	closeNotify chan bool
	dcpAckSize  int
	dcpFlowRecv int
}

func newMemdClient(conn memdConn) *memdClient {
	client := memdClient{
		conn:        conn,
		closeNotify: make(chan bool),
	}
	client.run()
	return &client
}

func (client *memdClient) EnableDcpBufferAck(bufferAckSize int) {
	client.dcpAckSize = bufferAckSize
}

func (client *memdClient) maybeSendDcpBufferAck(packet *memdPacket) {
	packetLen := 24 + len(packet.Extras) + len(packet.Key) + len(packet.Value)

	client.dcpFlowRecv += packetLen
	if client.dcpFlowRecv < client.dcpAckSize {
		return
	}

	ackAmt := client.dcpFlowRecv

	extrasBuf := make([]byte, 4)
	binary.BigEndian.PutUint32(extrasBuf, uint32(ackAmt))

	err := client.conn.WritePacket(&memdPacket{
		Magic:  reqMagic,
		Opcode: cmdDcpBufferAck,
		Extras: extrasBuf,
	})
	if err != nil {
		logWarnf("Failed to dispatch DCP buffer ack: %s", err)
	}

	client.dcpFlowRecv -= ackAmt
}

func (client *memdClient) SetErrorMap(errorMap *kvErrorMap) {
	client.errorMap = errorMap
}

func (client *memdClient) Address() string {
	return client.conn.RemoteAddr()
}

func (client *memdClient) CloseNotify() chan bool {
	return client.closeNotify
}

func (client *memdClient) SendRequest(req *memdQRequest) error {
	client.opList.Add(req)

	err := client.conn.WritePacket(&req.memdPacket)
	if err != nil {
		logDebugf("memdClient write failure: %v", err)
		client.opList.Remove(req)
		return err
	}

	return nil
}

func (client *memdClient) resolveRequest(resp *memdQResponse) {
	opIndex := resp.Opaque

	// Find the request that goes with this response
	req := client.opList.FindAndMaybeRemove(opIndex)

	if req == nil {
		// There is no known request that goes with this response.  Ignore it.
		logDebugf("Received response with no corresponding request.")
		return
	}

	if req.RoutingCallback != nil {
		if req.RoutingCallback(resp, req) {
			logSchedf("Routing callback intercepted response")
			return
		}
	}

	// Grab an error object if one needs to exist.
	var err error
	if resp.Magic == resMagic {
		err = getMemdError(resp.Status, client.errorMap)
	}

	// Call the requests callback handler...
	logSchedf("Dispatching response callback. OP=0x%x. Opaque=%d", resp.Opcode, resp.Opaque)
	req.tryCallback(resp, err)
}

func (client *memdClient) run() {
	dcpBufferQ := make(chan *memdQResponse)
	killSwitch := make(chan bool, 1)
	go func() {
		for {
			select {
			case resp, more := <-dcpBufferQ:
				if !more {
					return
				}

				logSchedf("Resolving response OP=0x%x. Opaque=%d", resp.Opcode, resp.Opaque)
				client.resolveRequest(resp)

				if client.dcpAckSize > 0 {
					client.maybeSendDcpBufferAck(&resp.memdPacket)
				}
			case <-killSwitch:
				close(dcpBufferQ)
			}
		}
	}()

	go func() {
		for {
			resp := &memdQResponse{
				sourceAddr: client.conn.RemoteAddr(),
			}

			err := client.conn.ReadPacket(&resp.memdPacket)
			if err != nil {
				logErrorf("memdClient read failure: %v", err)
				break
			}

			// We handle DCP no-op's directly here so we can reply immediately.
			if resp.memdPacket.Opcode == cmdDcpNoop {
				err := client.conn.WritePacket(&memdPacket{
					Magic:  resMagic,
					Opcode: cmdDcpNoop,
					Opaque: resp.Opaque,
				})
				if err != nil {
					logWarnf("Failed to dispatch DCP noop reply: %s", err)
				}
				continue
			}

			switch resp.memdPacket.Opcode {
			case cmdDcpDeletion:
				fallthrough
			case cmdDcpExpiration:
				fallthrough
			case cmdDcpMutation:
				fallthrough
			case cmdDcpSnapshotMarker:
				fallthrough
			case cmdDcpStreamEnd:
				fallthrough
			case cmdDcpStreamReq:
				dcpBufferQ <- resp
				continue
			default:
				logSchedf("Resolving response OP=0x%x. Opaque=%d", resp.Opcode, resp.Opaque)
				client.resolveRequest(resp)
			}
		}

		err := client.conn.Close()
		if err != nil {
			// Lets log an error, as this is non-fatal
			logErrorf("Failed to shut down client connection (%s)", err)
		}

		client.opList.Drain(func(req *memdQRequest) {
			req.tryCallback(nil, ErrNetwork)
		})

		killSwitch <- true
		client.closeNotify <- true
	}()
}

func (client *memdClient) Close() error {
	return client.conn.Close()
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"sync"
)

func gen(num int) <-chan int {
	out := make(chan int)
	go func() {
		for i := 1; i <= num; i++ {
			out <- i
		}
		close(out)
	}()
	return out
}

func sq(in <-chan int) <-chan int {
	out := make(chan int)
	go func() {
		for n := range in {
			out <- n * n
		}
		close(out)
	}()
	return out
}

func merge(cs ...<-chan int) <-chan int {
	var wg sync.WaitGroup
	out := make(chan int)

	// Start an output goroutine for each input channel in cs.  output
	// copies values from c to out until c is closed, then calls wg.Done.
	output := func(c <-chan int) {
		for n := range c {
			out <- n
		}
		wg.Done()
	}
	wg.Add(len(cs))
	for _, c := range cs {
		go output(c)
	}

	// Start a goroutine to close out once all the output goroutines are
	// done.  This must start after the wg.Add call.
	go func() {
		wg.Wait()
		close(out)
	}()
	return out
}

func main() {
	// Set up the pipeline.
	in := gen(10000)
	c1 := sq(in)
	c2 := sq(in)

	// Consume the output.
	for n := range merge(c1, c2) {
		fmt.Println(n) // 1,4,9...
	}
}
<commit_msg>4 lines<commit_after>package main

import (
	"fmt"
	"sync"
)

func gen(num int) <-chan int {
	out := make(chan int)
	go func() {
		for i := 1; i <= num; i++ {
			out <- i
		}
		close(out)
	}()
	return out
}

func sq(in <-chan int) <-chan int {
	out := make(chan int)
	go func() {
		for n := range in {
			out <- n * n
		}
		close(out)
	}()
	return out
}

func merge(cs ...<-chan int) <-chan int {
	var wg sync.WaitGroup
	out := make(chan int)

	// Start an output goroutine for each input channel in cs.  output
	// copies values from c to out until c is closed, then calls wg.Done.
	output := func(c <-chan int) {
		for n := range c {
			out <- n
		}
		wg.Done()
	}
	wg.Add(len(cs))
	for _, c := range cs {
		go output(c)
	}

	// Start a goroutine to close out once all the output goroutines are
	// done.  This must start after the wg.Add call.
	go func() {
		wg.Wait()
		close(out)
	}()
	return out
}

func main() {
	// Set up the pipeline.
	in := gen(1000 * 1000 * 1000 * 1000 * 1000 * 1000)
	c1 := sq(in)
	c2 := sq(in)
	c3 := sq(in)
	c4 := sq(in)

	// Consume the output.
	for n := range merge(c1, c2, c3, c4) {
		fmt.Println(n) // 1,4,9...
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	smpp "github.com/CodeMonkeyKevin/smpp34"
)

func main() {
	// connect and bind
	trx, err := smpp.NewTransmitter(
		"localhost",
		9000,
		5,
		smpp.Params{
			"system_type": "CMT",
			"system_id":   "hugo",
			"password":    "ggoohu",
		},
	)
	if err != nil {
		fmt.Println("Connection Err:", err)
		return
	}

	for {
		pdu, err := trx.Read() // This is blocking
		if err != nil {
			fmt.Println("Read Err:", err)
			break
		}

		// EnquireLinks are auto handles
		switch pdu.GetHeader().Id {
		case smpp.DELIVER_SM:
			// received Deliver Sm
			for _, v := range pdu.MandatoryFieldsList() {
				f := pdu.GetField(v)
				fmt.Println(v, ":", f)
			}

			// Respond back to Deliver SM with Deliver SM Resp
			err := trx.DeliverSmResp(pdu.GetHeader().Sequence, smpp.ESME_ROK)

			if err != nil {
				fmt.Println("DeliverSmResp err:", err)
			}
		default:
			// ignore all other PDUs or do what you link with them
			fmt.Println("PDU ID:", pdu.GetHeader().Id)
		}
	}

	fmt.Println("ending...")
}
<commit_msg>Updated receiver rx example<commit_after>package main

import (
	"fmt"
	smpp "github.com/CodeMonkeyKevin/smpp34"
)

func main() {
	// connect and bind
	rx, err := smpp.NewReceiver(
		"localhost",
		9000,
		5,
		smpp.Params{
			"system_type": "CMT",
			"system_id":   "hugo",
			"password":    "ggoohu",
		},
	)
	if err != nil {
		fmt.Println("Connection Err:", err)
		return
	}

	for {
		pdu, err := rx.Read() // This is blocking
		if err != nil {
			fmt.Println("Read Err:", err)
			break
		}

		// EnquireLinks are auto handles
		switch pdu.GetHeader().Id {
		case smpp.DELIVER_SM:
			// received Deliver Sm
			for _, v := range pdu.MandatoryFieldsList() {
				f := pdu.GetField(v)
				fmt.Println(v, ":", f)
			}

			// Respond back to Deliver SM with Deliver SM Resp
			err := rx.DeliverSmResp(pdu.GetHeader().Sequence, smpp.ESME_ROK)

			if err != nil {
				fmt.Println("DeliverSmResp err:", err)
			}
		default:
			// ignore all other PDUs or do what you link with them
			fmt.Println("PDU ID:", pdu.GetHeader().Id)
		}
	}

	fmt.Println("ending...")
}
<|endoftext|>


<commit_before>package vagrantkite

import (
	"koding/kites/kloud/klient"
	"time"

	"github.com/hashicorp/terraform/helper/schema"
	"github.com/koding/kite"
)

func resourceVagrantKiteBuild() *schema.Resource {
	return &schema.Resource{
		Create: resourceMachineCreate,
		Read:   resourceMachineNoop,
		Update: resourceMachineNoop,
		Delete: resourceMachineNoop,

		Schema: map[string]*schema.Schema{
			"queryString": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
			},
			"vagrantFile": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
			},
			"filePath": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
			},
		},
	}
}

var klientFuncName = "vagrant.build"

type vagrantKiteReq struct {
	VagrantFile string
	FilePath    string
}

// resourceMachineCreate creates a new vagrant machine in remote klient host
func resourceMachineCreate(d *schema.ResourceData, meta interface{}) error {

	queryString := d.Get("queryString").(string)

	args := &vagrantKiteReq{
		VagrantFile: d.Get("vagrantFile").(string),
		FilePath:    d.Get("filePath").(string),
	}

	if err := sendCommand(klientFuncName, queryString, args); err != nil {
		return err
	}

	d.SetId(queryString)

	return nil
}

func sendCommand(command string, queryString string, args interface{}) error {
	c, err := NewClient()
	if err != nil {
		return err
	}
	defer c.Close()

	// Get the klient.
	klientRef, err := klient.ConnectTimeout(c.Kite, queryString, time.Second*10)
	if err != nil {
		if err == klient.ErrDialingFailed || err == kite.ErrNoKitesAvailable {
			c.Log.Error(
				"[%s] Klient is not registered to Kontrol. Err: %s",
				queryString,
				err.Error(),
			)

			return nil // if the machine is not open, we cant do anything
		}

		return err
	}
	defer klientRef.Close()

	_, err = klientRef.Client.Tell(command, args)
	return err
}

func resourceMachineNoop(d *schema.ResourceData, meta interface{}) error { return nil }
<commit_msg>Kite: func name can be constant<commit_after>package vagrantkite

import (
	"koding/kites/kloud/klient"
	"time"

	"github.com/hashicorp/terraform/helper/schema"
	"github.com/koding/kite"
)

func resourceVagrantKiteBuild() *schema.Resource {
	return &schema.Resource{
		Create: resourceMachineCreate,
		Read:   resourceMachineNoop,
		Update: resourceMachineNoop,
		Delete: resourceMachineNoop,

		Schema: map[string]*schema.Schema{
			"queryString": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
			},
			"vagrantFile": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
			},
			"filePath": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
			},
		},
	}
}

const klientFuncName = "vagrant.build"

type vagrantKiteReq struct {
	VagrantFile string
	FilePath    string
}

// resourceMachineCreate creates a new vagrant machine in remote klient host
func resourceMachineCreate(d *schema.ResourceData, meta interface{}) error {

	queryString := d.Get("queryString").(string)

	args := &vagrantKiteReq{
		VagrantFile: d.Get("vagrantFile").(string),
		FilePath:    d.Get("filePath").(string),
	}

	if err := sendCommand(klientFuncName, queryString, args); err != nil {
		return err
	}

	d.SetId(queryString)

	return nil
}

func sendCommand(command string, queryString string, args interface{}) error {
	c, err := NewClient()
	if err != nil {
		return err
	}
	defer c.Close()

	// Get the klient.
	klientRef, err := klient.ConnectTimeout(c.Kite, queryString, time.Second*10)
	if err != nil {
		if err == klient.ErrDialingFailed || err == kite.ErrNoKitesAvailable {
			c.Log.Error(
				"[%s] Klient is not registered to Kontrol. Err: %s",
				queryString,
				err.Error(),
			)

			return nil // if the machine is not open, we cant do anything
		}

		return err
	}
	defer klientRef.Close()

	_, err = klientRef.Client.Tell(command, args)
	return err
}

func resourceMachineNoop(d *schema.ResourceData, meta interface{}) error { return nil }
<|endoftext|>


<commit_before>// Copyright 2016 ego authors
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

package main

import (
	"fmt"
	"log"
	"os"

	"github.com/go-ego/riot"
	"github.com/go-ego/riot/types"
)

var (
	// searcher is coroutine safe
	searcher = riot.Engine{}
)

func initEngine() {
	var path = "./riot-index"

	searcher.Init(types.EngineOpts{
		Using: 1,
		IndexerOpts: &types.IndexerOpts{
			IndexType: types.DocIdsIndex,
		},
		UseStorage:    true,
		StorageFolder: path,
		StorageEngine: "bg", // bg: badger, lbd: leveldb, bolt: bolt
		// GseDict: "../../data/dict/dictionary.txt",
		GseDict:       "../../testdata/test_dict.txt",
		StopTokenFile: "../../data/dict/stop_tokens.txt",
	})
	defer searcher.Close()
	os.MkdirAll(path, 0777)

	text := "Google Is Experimenting With Virtual Reality Advertising"
	text1 := `Google accidentally pushed Bluetooth update for Home
	speaker early`
	text2 := `Google is testing another Search results layout with
	rounded cards, new colors, and the 4 mysterious colored dots again`

	// Add the document to the index, docId starts at 1
	searcher.Index(1, types.DocData{Content: text})
	searcher.Index(2, types.DocData{Content: text1})
	searcher.Index(3, types.DocData{Content: text2})
	searcher.Index(5, types.DocData{Content: text2})

	// Wait for the index to refresh
	// searcher.Flush()

	searcher.RemoveDoc(5)
	searcher.Flush()

	log.Println("recover index number: ", searcher.NumDocsIndexed())
}

func restoreIndex() {
	var path = "./riot-index"

	searcher.Init(types.EngineOpts{
		Using: 1,
		IndexerOpts: &types.IndexerOpts{
			IndexType: types.DocIdsIndex,
		},
		UseStorage:    true,
		StorageFolder: path,
		StorageEngine: "bg", // bg: badger, lbd: leveldb, bolt: bolt
		GseDict:       "../../data/dict/dictionary.txt",
		StopTokenFile: "../../data/dict/stop_tokens.txt",
	})
	defer searcher.Close()
	os.MkdirAll(path, 0777)

	// Wait for the index to refresh
	searcher.Flush()

	log.Println("recover index number: ", searcher.NumDocsIndexed())
}

func main() {
	initEngine()

	sea := searcher.Search(types.SearchReq{
		Text: "google testing",
		RankOpts: &types.RankOpts{
			OutputOffset: 0,
			MaxOutputs:   100,
		}})

	fmt.Println("search response: ", sea, "; docs = ", sea.Docs)

	// os.RemoveAll("riot-index")
}
<commit_msg>update store example<commit_after>// Copyright 2016 ego authors
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

package main

import (
	"fmt"
	"log"

	"github.com/go-ego/riot"
	"github.com/go-ego/riot/types"
)

var (
	// searcher is coroutine safe
	searcher = riot.Engine{}
)

func initEngine() {
	// var path = "./riot-index"
	searcher.Init(types.EngineOpts{
		Using: 1,
		IndexerOpts: &types.IndexerOpts{
			IndexType: types.DocIdsIndex,
		},
		UseStorage: true,
		// StorageFolder: path,
		StorageEngine: "bg", // bg: badger, lbd: leveldb, bolt: bolt
		// GseDict: "../../data/dict/dictionary.txt",
		GseDict:       "../../testdata/test_dict.txt",
		StopTokenFile: "../../data/dict/stop_tokens.txt",
	})
	defer searcher.Close()
	// os.MkdirAll(path, 0777)

	text := "Google Is Experimenting With Virtual Reality Advertising"
	text1 := `Google accidentally pushed Bluetooth update for Home
	speaker early`
	text2 := `Google is testing another Search results layout with
	rounded cards, new colors, and the 4 mysterious colored dots again`

	// Add the document to the index, docId starts at 1
	searcher.Index(1, types.DocData{Content: text})
	searcher.Index(2, types.DocData{Content: text1})
	searcher.Index(3, types.DocData{Content: text2})
	searcher.Index(5, types.DocData{Content: text2})

	// Wait for the index to refresh
	// searcher.Flush()

	searcher.RemoveDoc(5)
	searcher.Flush()

	log.Println("recover index number: ", searcher.NumDocsIndexed())
}

func restoreIndex() {
	// var path = "./riot-index"
	searcher.Init(types.EngineOpts{
		Using: 1,
		IndexerOpts: &types.IndexerOpts{
			IndexType: types.DocIdsIndex,
		},
		UseStorage: true,
		// StorageFolder: path,
		StorageEngine: "bg", // bg: badger, lbd: leveldb, bolt: bolt
		GseDict:       "../../data/dict/dictionary.txt",
		StopTokenFile: "../../data/dict/stop_tokens.txt",
	})
	defer searcher.Close()
	// os.MkdirAll(path, 0777)

	// Wait for the index to refresh
	searcher.Flush()

	log.Println("recover index number: ", searcher.NumDocsIndexed())
}

func main() {
	initEngine()

	sea := searcher.Search(types.SearchReq{
		Text: "google testing",
		RankOpts: &types.RankOpts{
			OutputOffset: 0,
			MaxOutputs:   100,
		}})

	fmt.Println("search response: ", sea, "; docs = ", sea.Docs)

	// os.RemoveAll("riot-index")
}
<|endoftext|>


<commit_before>package xeth

/*
 * eXtended ETHereum
 */

import (
	"github.com/ethereum/go-ethereum/core"
	"github.com/ethereum/go-ethereum/core/types"
	"github.com/ethereum/go-ethereum/crypto"
	"github.com/ethereum/go-ethereum/ethutil"
	"github.com/ethereum/go-ethereum/logger"
	"github.com/ethereum/go-ethereum/state"
)

var pipelogger = logger.NewLogger("XETH")

type VmVars struct {
	State *state.StateDB
}

type XEth struct {
	obj          core.EthManager
	blockManager *core.BlockManager
	blockChain   *core.ChainManager
	world        *World

	Vm VmVars
}

func New(obj core.EthManager) *XEth {
	pipe := &XEth{
		obj:          obj,
		blockManager: obj.BlockManager(),
		blockChain:   obj.ChainManager(),
	}
	pipe.world = NewWorld(pipe)

	return pipe
}

/*
 * State / Account accessors
 */
func (self *XEth) Balance(addr []byte) *ethutil.Value {
	return ethutil.NewValue(self.World().safeGet(addr).Balance)
}

func (self *XEth) Nonce(addr []byte) uint64 {
	return self.World().safeGet(addr).Nonce
}

func (self *XEth) Block(hash []byte) *types.Block {
	return self.blockChain.GetBlock(hash)
}

func (self *XEth) Storage(addr, storageAddr []byte) *ethutil.Value {
	return self.World().safeGet(addr).GetStorage(ethutil.BigD(storageAddr))
}

func (self *XEth) Exists(addr []byte) bool {
	return self.World().Get(addr) != nil
}

// Converts the given private key to an address
func (self *XEth) ToAddress(priv []byte) []byte {
	pair, err := crypto.NewKeyPairFromSec(priv)
	if err != nil {
		return nil
	}

	return pair.Address()
}

/*
 * Execution helpers
 */
func (self *XEth) Execute(addr []byte, data []byte, value, gas, price *ethutil.Value) ([]byte, error) {
	return self.ExecuteObject(&Object{self.World().safeGet(addr)}, data, value, gas, price)
}

func (self *XEth) ExecuteObject(object *Object, data []byte, value, gas, price *ethutil.Value) ([]byte, error) {
	var (
		initiator = state.NewStateObject(self.obj.KeyManager().KeyPair().Address())
		block     = self.blockChain.CurrentBlock
	)

	self.Vm.State = self.World().State().Copy()

	vmenv := NewEnv(self.Vm.State, block, value.BigInt(), initiator.Address())
	return vmenv.Call(initiator, object.Address(), data, gas.BigInt(), price.BigInt(), value.BigInt())
}

/*
 * Transactional methods
 */
func (self *XEth) TransactString(key *crypto.KeyPair, rec string, value, gas, price *ethutil.Value, data []byte) (*types.Transaction, error) {
	// Check if an address is stored by this address
	var hash []byte
	addr := self.World().Config().Get("NameReg").StorageString(rec).Bytes()
	if len(addr) > 0 {
		hash = addr
	} else if ethutil.IsHex(rec) {
		hash = ethutil.Hex2Bytes(rec[2:])
	} else {
		hash = ethutil.Hex2Bytes(rec)
	}

	return self.Transact(key, hash, value, gas, price, data)
}

func (self *XEth) Transact(key *crypto.KeyPair, to []byte, value, gas, price *ethutil.Value, data []byte) (*types.Transaction, error) {
	var hash []byte
	var contractCreation bool
	if types.IsContractAddr(to) {
		contractCreation = true
	} else {
		// Check if an address is stored by this address
		addr := self.World().Config().Get("NameReg").Storage(to).Bytes()
		if len(addr) > 0 {
			hash = addr
		} else {
			hash = to
		}
	}

	var tx *types.Transaction
	if contractCreation {
		tx = types.NewContractCreationTx(value.BigInt(), gas.BigInt(), price.BigInt(), data)
	} else {
		tx = types.NewTransactionMessage(hash, value.BigInt(), gas.BigInt(), price.BigInt(), data)
	}

	state := self.blockManager.TransState()
	nonce := state.GetNonce(key.Address())

	tx.Nonce = nonce
	tx.Sign(key.PrivateKey)
	err := self.obj.TxPool().Add(tx)
	if err != nil {
		return nil, err
	}

	state.SetNonce(key.Address(), nonce+1)

	if contractCreation {
		addr := tx.CreationAddress(self.World().State())
		pipelogger.Infof("Contract addr %x\n", addr)
	}

	return tx, nil
}

func (self *XEth) PushTx(tx *types.Transaction) ([]byte, error) {
	err := self.obj.TxPool().Add(tx)
	if err != nil {
		return nil, err
	}

	if tx.Recipient == nil {
		addr := tx.CreationAddress(self.World().State())
		pipelogger.Infof("Contract addr %x\n", addr)
		return addr, nil
	}
	return tx.Hash(), nil
}

func (self *XEth) CompileMutan(code string) ([]byte, error) {
	data, err := ethutil.Compile(code, false)
	if err != nil {
		return nil, err
	}

	return data, nil
}
<commit_msg>Added pre processing of transaction on the transient state<commit_after>package xeth

/*
 * eXtended ETHereum
 */

import (
	"github.com/ethereum/go-ethereum/core"
	"github.com/ethereum/go-ethereum/core/types"
	"github.com/ethereum/go-ethereum/crypto"
	"github.com/ethereum/go-ethereum/ethutil"
	"github.com/ethereum/go-ethereum/logger"
	"github.com/ethereum/go-ethereum/state"
)

var pipelogger = logger.NewLogger("XETH")

type VmVars struct {
	State *state.StateDB
}

type XEth struct {
	obj          core.EthManager
	blockManager *core.BlockManager
	blockChain   *core.ChainManager
	world        *World

	Vm VmVars
}

func New(obj core.EthManager) *XEth {
	pipe := &XEth{
		obj:          obj,
		blockManager: obj.BlockManager(),
		blockChain:   obj.ChainManager(),
	}
	pipe.world = NewWorld(pipe)

	return pipe
}

/*
 * State / Account accessors
 */
func (self *XEth) Balance(addr []byte) *ethutil.Value {
	return ethutil.NewValue(self.World().safeGet(addr).Balance)
}

func (self *XEth) Nonce(addr []byte) uint64 {
	return self.World().safeGet(addr).Nonce
}

func (self *XEth) Block(hash []byte) *types.Block {
	return self.blockChain.GetBlock(hash)
}

func (self *XEth) Storage(addr, storageAddr []byte) *ethutil.Value {
	return self.World().safeGet(addr).GetStorage(ethutil.BigD(storageAddr))
}

func (self *XEth) Exists(addr []byte) bool {
	return self.World().Get(addr) != nil
}

// Converts the given private key to an address
func (self *XEth) ToAddress(priv []byte) []byte {
	pair, err := crypto.NewKeyPairFromSec(priv)
	if err != nil {
		return nil
	}

	return pair.Address()
}

/*
 * Execution helpers
 */
func (self *XEth) Execute(addr []byte, data []byte, value, gas, price *ethutil.Value) ([]byte, error) {
	return self.ExecuteObject(&Object{self.World().safeGet(addr)}, data, value, gas, price)
}

func (self *XEth) ExecuteObject(object *Object, data []byte, value, gas, price *ethutil.Value) ([]byte, error) {
	var (
		initiator = state.NewStateObject(self.obj.KeyManager().KeyPair().Address())
		block     = self.blockChain.CurrentBlock
	)

	self.Vm.State = self.World().State().Copy()

	vmenv := NewEnv(self.Vm.State, block, value.BigInt(), initiator.Address())
	return vmenv.Call(initiator, object.Address(), data, gas.BigInt(), price.BigInt(), value.BigInt())
}

/*
 * Transactional methods
 */
func (self *XEth) TransactString(key *crypto.KeyPair, rec string, value, gas, price *ethutil.Value, data []byte) (*types.Transaction, error) {
	// Check if an address is stored by this address
	var hash []byte
	addr := self.World().Config().Get("NameReg").StorageString(rec).Bytes()
	if len(addr) > 0 {
		hash = addr
	} else if ethutil.IsHex(rec) {
		hash = ethutil.Hex2Bytes(rec[2:])
	} else {
		hash = ethutil.Hex2Bytes(rec)
	}

	return self.Transact(key, hash, value, gas, price, data)
}

func (self *XEth) Transact(key *crypto.KeyPair, to []byte, value, gas, price *ethutil.Value, data []byte) (*types.Transaction, error) {
	var hash []byte
	var contractCreation bool
	if types.IsContractAddr(to) {
		contractCreation = true
	} else {
		// Check if an address is stored by this address
		addr := self.World().Config().Get("NameReg").Storage(to).Bytes()
		if len(addr) > 0 {
			hash = addr
		} else {
			hash = to
		}
	}

	var tx *types.Transaction
	if contractCreation {
		tx = types.NewContractCreationTx(value.BigInt(), gas.BigInt(), price.BigInt(), data)
	} else {
		tx = types.NewTransactionMessage(hash, value.BigInt(), gas.BigInt(), price.BigInt(), data)
	}

	state := self.blockManager.TransState()
	nonce := state.GetNonce(key.Address())

	tx.Nonce = nonce
	tx.Sign(key.PrivateKey)

	// Do some pre processing for our "pre" events  and hooks
	block := self.blockChain.NewBlock(key.Address())
	coinbase := state.GetStateObject(key.Address())
	coinbase.SetGasPool(block.GasLimit)
	self.blockManager.ApplyTransactions(coinbase, state, block, types.Transactions{tx}, true)

	err := self.obj.TxPool().Add(tx)
	if err != nil {
		return nil, err
	}
	state.SetNonce(key.Address(), nonce+1)

	if contractCreation {
		addr := tx.CreationAddress(self.World().State())
		pipelogger.Infof("Contract addr %x\n", addr)
	}

	return tx, nil
}

func (self *XEth) PushTx(tx *types.Transaction) ([]byte, error) {
	err := self.obj.TxPool().Add(tx)
	if err != nil {
		return nil, err
	}

	if tx.Recipient == nil {
		addr := tx.CreationAddress(self.World().State())
		pipelogger.Infof("Contract addr %x\n", addr)
		return addr, nil
	}
	return tx.Hash(), nil
}

func (self *XEth) CompileMutan(code string) ([]byte, error) {
	data, err := ethutil.Compile(code, false)
	if err != nil {
		return nil, err
	}

	return data, nil
}
<|endoftext|>


<commit_before>package xkcd_test

import (
	"github.com/rkoesters/xkcd"
	"testing"
)

func TestGet(t *testing.T) {
	comic, err := xkcd.Get(221)
	if err != nil {
		t.Fatal(err)
	}

	expected := &xkcd.Comic{
		Num:       221,
		Title:     "Random Number",
		SafeTitle: "Random Number",
		Img:       "http://imgs.xkcd.com/comics/random_number.png",
		Alt:       "RFC 1149.5 specifies 4 as the standard IEEE-vetted random number.",
		Year:      "2007",
		Month:     "2",
		Day:       "9",
		News:      "",
		Link:      "",
		Transcript: `int getRandomNumber()
{
  return 4;
 chosen by fair dice roll.

 guarenteed to be random.
}
{{title text: RFC 1149.5 specifies 4 as the standard IEEE-vetted random number.}}`,
	}

	if *comic != *expected {
		t.Fail()
	}
}

func TestGetCurrent(t *testing.T) {
	comic1, err := xkcd.GetCurrent()
	if err != nil {
		t.Fatal(err)
	}

	comic2, err := xkcd.Get(comic1.Num)
	if err != nil {
		t.Fatal(err)
	}

	if *comic1 != *comic2 {
		t.Fail()
	}
}

func Test404(t *testing.T) {
	_, err := xkcd.Get(404)
	if err.Error() != "404 Not Found" {
		t.Fatal(err)
	}
}
<commit_msg>Added test for `New'.<commit_after>package xkcd_test

import (
	"encoding/json"
	"github.com/rkoesters/xkcd"
	"io"
	"testing"
)

func TestGet(t *testing.T) {
	comic, err := xkcd.Get(221)
	if err != nil {
		t.Fatal(err)
	}

	expected := &xkcd.Comic{
		Num:       221,
		Title:     "Random Number",
		SafeTitle: "Random Number",
		Img:       "http://imgs.xkcd.com/comics/random_number.png",
		Alt:       "RFC 1149.5 specifies 4 as the standard IEEE-vetted random number.",
		Year:      "2007",
		Month:     "2",
		Day:       "9",
		News:      "",
		Link:      "",
		Transcript: `int getRandomNumber()
{
  return 4;
 chosen by fair dice roll.

 guarenteed to be random.
}
{{title text: RFC 1149.5 specifies 4 as the standard IEEE-vetted random number.}}`,
	}

	if *comic != *expected {
		t.Fail()
	}
}

func TestGetCurrent(t *testing.T) {
	comic1, err := xkcd.GetCurrent()
	if err != nil {
		t.Fatal(err)
	}

	comic2, err := xkcd.Get(comic1.Num)
	if err != nil {
		t.Fatal(err)
	}

	if *comic1 != *comic2 {
		t.Fail()
	}
}

func Test404(t *testing.T) {
	_, err := xkcd.Get(404)
	if err.Error() != "404 Not Found" {
		t.Fatal(err)
	}
}

func TestNew(t *testing.T) {
	comic1, err := xkcd.GetCurrent()
	if err != nil {
		t.Fatal(err)
	}

	r, w := io.Pipe()

	go func() {
		e := json.NewEncoder(w)
		err = e.Encode(comic1)
		if err != nil {
			t.Fatal(err)
		}
	}()

	comic2, err := xkcd.New(r)
	if err != nil {
		t.Fatal(err)
	}

	t.Log("comic1: ", comic1)
	t.Log("comic2: ", comic2)

	if *comic1 != *comic2 {
		t.Fatal("comic1 and comic2 don't match")
	}
}
<|endoftext|>


<commit_before>package keylogger

import (
	"bytes"
	"encoding/binary"
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"strings"
	"syscall"

	"github.com/sirupsen/logrus"
)

// KeyLogger wrapper around file descriptior
type KeyLogger struct {
	fd *os.File
}

// New creates a new keylogger for a device path
func New(devPath string) (*KeyLogger, error) {
	k := &KeyLogger{}
	if !k.IsRoot() {
		return nil, errors.New("Must be run as root")
	}
	fd, err := os.Open(devPath)
	k.fd = fd
	return k, err
}

// FindKeyboardDevice by going through each device registered on OS
// Mostly it will contain keyword - keyboard
// Returns the file path which contains events
func FindKeyboardDevice() string {
	path := "/sys/class/input/event%d/device/name"
	resolved := "/dev/input/event%d"

	for i := 0; i < 255; i++ {
		buff, err := ioutil.ReadFile(fmt.Sprintf(path, i))
		if err != nil {
			logrus.Error(err)
		}

		// check if mouse is contained in the input event
		// if that is the case just skip.
		// We do this check as it seems that some mouses like the logitech MX mouse is also recognized as a mouse/keyboard
		if strings.Contains(strings.ToLower(string(buff)), "mouse") {
			continue
		}

		if strings.Contains(strings.ToLower(string(buff)), "keyboard") {
			return fmt.Sprintf(resolved, i)
		}
	}
	return ""
}

// IsRoot checks if the process is run with root permission
func (k *KeyLogger) IsRoot() bool {
	return syscall.Getuid() == 0 && syscall.Geteuid() == 0
}

// Read from file descriptor
// Blocking call, returns channel
// Make sure to close channel when finish
func (k *KeyLogger) Read() chan InputEvent {
	event := make(chan InputEvent)
	go func(event chan InputEvent) {
		for {
			e, err := k.read()
			if err != nil {
				logrus.Error(err)
				close(event)
				break
			}

			if e != nil {
				event <- *e
			}
		}
	}(event)
	return event
}

// read from file description and parse binary into go struct
func (k *KeyLogger) read() (*InputEvent, error) {
	buffer := make([]byte, eventsize)
	n, err := k.fd.Read(buffer)
	if err != nil {
		return nil, err
	}
	// no input, dont send error
	if n <= 0 {
		return nil, nil
	}
	return k.eventFromBuffer(buffer)
}

// eventFromBuffer parser bytes into InputEvent struct
func (k *KeyLogger) eventFromBuffer(buffer []byte) (*InputEvent, error) {
	event := &InputEvent{}
	err := binary.Read(bytes.NewBuffer(buffer), binary.LittleEndian, event)
	return event, err
}

// Close file descriptor
func (k *KeyLogger) Close() error {
	if k.fd == nil {
		return nil
	}
	return k.fd.Close()
}
<commit_msg>Add new API - FindAllKeyboardDevices<commit_after>package keylogger

import (
	"bytes"
	"encoding/binary"
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"strings"
	"syscall"

	"github.com/sirupsen/logrus"
)

// KeyLogger wrapper around file descriptior
type KeyLogger struct {
	fd *os.File
}

// New creates a new keylogger for a device path
func New(devPath string) (*KeyLogger, error) {
	k := &KeyLogger{}
	if !k.IsRoot() {
		return nil, errors.New("Must be run as root")
	}
	fd, err := os.Open(devPath)
	k.fd = fd
	return k, err
}

// FindKeyboardDevice by going through each device registered on OS
// Mostly it will contain keyword - keyboard
// Returns the file path which contains events
func FindKeyboardDevice() string {
	path := "/sys/class/input/event%d/device/name"
	resolved := "/dev/input/event%d"

	for i := 0; i < 255; i++ {
		buff, err := ioutil.ReadFile(fmt.Sprintf(path, i))
		if err != nil {
			logrus.Error(err)
		}

		// check if mouse is contained in the input event
		// if that is the case just skip.
		// We do this check as it seems that some mouses like the logitech MX mouse is also recognized as a mouse/keyboard
		if strings.Contains(strings.ToLower(string(buff)), "mouse") {
			continue
		}

		if strings.Contains(strings.ToLower(string(buff)), "keyboard") {
			return fmt.Sprintf(resolved, i)
		}
	}
	return ""
}

// Like FindKeyboardDevice, but finds all devices which contain keyword 'keyboard'
// Returns an array of file paths which contain keyboard events
func FindAllKeyboardDevices() []string {
	path := "/sys/class/input/event%d/device/name"
	resolved := "/dev/input/event%d"

	valid := make([]string, 0)

	for i := 0; i < 255; i++ {
		buff, err := ioutil.ReadFile(fmt.Sprintf(path, i))

		// prevent from checking non-existant files
		if os.IsNotExist(err) {
			break
		}
		if err != nil {
			logrus.Error(err)
		}

		// check if mouse is contained in the input event
		// if that is the case just skip.
		// We do this check as it seems that some mouses like the logitech MX mouse is also recognized as a mouse/keyboard
		if strings.Contains(strings.ToLower(string(buff)), "mouse") {
			continue
		}

		if strings.Contains(strings.ToLower(string(buff)), "keyboard") {
			valid = append(valid, fmt.Sprintf(resolved, i))
		}
	}
	return valid
}

// IsRoot checks if the process is run with root permission
func (k *KeyLogger) IsRoot() bool {
	return syscall.Getuid() == 0 && syscall.Geteuid() == 0
}

// Read from file descriptor
// Blocking call, returns channel
// Make sure to close channel when finish
func (k *KeyLogger) Read() chan InputEvent {
	event := make(chan InputEvent)
	go func(event chan InputEvent) {
		for {
			e, err := k.read()
			if err != nil {
				logrus.Error(err)
				close(event)
				break
			}

			if e != nil {
				event <- *e
			}
		}
	}(event)
	return event
}

// read from file description and parse binary into go struct
func (k *KeyLogger) read() (*InputEvent, error) {
	buffer := make([]byte, eventsize)
	n, err := k.fd.Read(buffer)
	if err != nil {
		return nil, err
	}
	// no input, dont send error
	if n <= 0 {
		return nil, nil
	}
	return k.eventFromBuffer(buffer)
}

// eventFromBuffer parser bytes into InputEvent struct
func (k *KeyLogger) eventFromBuffer(buffer []byte) (*InputEvent, error) {
	event := &InputEvent{}
	err := binary.Read(bytes.NewBuffer(buffer), binary.LittleEndian, event)
	return event, err
}

// Close file descriptor
func (k *KeyLogger) Close() error {
	if k.fd == nil {
		return nil
	}
	return k.fd.Close()
}
<|endoftext|>