<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package vsphere

import (
	"fmt"
	"sync"

	"github.com/onsi/ginkgo"
	"github.com/onsi/gomega"
	"k8s.io/api/core/v1"
	storagev1 "k8s.io/api/storage/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	clientset "k8s.io/client-go/kubernetes"
	"k8s.io/kubernetes/test/e2e/framework"
	e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
	e2epv "k8s.io/kubernetes/test/e2e/framework/pv"
	"k8s.io/kubernetes/test/e2e/storage/utils"
)

/*
	Induce stress to create volumes in parallel with multiple threads based on user configurable values for number of threads and iterations per thread.
	The following actions will be performed as part of this test.

	1. Create Storage Classes of 4 Categories (Default, SC with Non Default Datastore, SC with SPBM Policy, SC with VSAN Storage Capabilities.)
	2. READ VCP_STRESS_INSTANCES, VCP_STRESS_ITERATIONS, VSPHERE_SPBM_POLICY_NAME and VSPHERE_DATASTORE from System Environment.
	3. Launch goroutine for volume lifecycle operations.
	4. Each instance of routine iterates for n times, where n is read from system env - VCP_STRESS_ITERATIONS
	5. Each iteration creates 1 PVC, 1 POD using the provisioned PV, Verify disk is attached to the node, Verify pod can access the volume, delete the pod and finally delete the PVC.
*/
var _ = utils.SIGDescribe("vsphere cloud provider stress [Feature:vsphere]", func() {
	f := framework.NewDefaultFramework("vcp-stress")
	var (
		client        clientset.Interface
		namespace     string
		instances     int
		iterations    int
		policyName    string
		datastoreName string
		err           error
		scNames       = []string{storageclass1, storageclass2, storageclass3, storageclass4}
	)

	ginkgo.BeforeEach(func() {
		framework.SkipUnlessProviderIs("vsphere")
		client = f.ClientSet
		namespace = f.Namespace.Name

		nodeList := framework.GetReadySchedulableNodesOrDie(f.ClientSet)
		gomega.Expect(nodeList.Items).NotTo(gomega.BeEmpty(), "Unable to find ready and schedulable Node")

		// if VCP_STRESS_INSTANCES = 12 and VCP_STRESS_ITERATIONS is 10. 12 threads will run in parallel for 10 times.
		// Resulting 120 Volumes and POD Creation. Volumes will be provisioned with each different types of Storage Class,
		// Each iteration creates PVC, verify PV is provisioned, then creates a pod, verify volume is attached to the node, and then delete the pod and delete pvc.
		instances = GetAndExpectIntEnvVar(VCPStressInstances)
		gomega.Expect(instances <= volumesPerNode*len(nodeList.Items)).To(gomega.BeTrue(), fmt.Sprintf("Number of Instances should be less or equal: %v", volumesPerNode*len(nodeList.Items)))
		gomega.Expect(instances > len(scNames)).To(gomega.BeTrue(), "VCP_STRESS_INSTANCES should be greater than 3 to utilize all 4 types of storage classes")

		iterations = GetAndExpectIntEnvVar(VCPStressIterations)
		framework.ExpectNoError(err, "Error Parsing VCP_STRESS_ITERATIONS")
		gomega.Expect(iterations > 0).To(gomega.BeTrue(), "VCP_STRESS_ITERATIONS should be greater than 0")

		policyName = GetAndExpectStringEnvVar(SPBMPolicyName)
		datastoreName = GetAndExpectStringEnvVar(StorageClassDatastoreName)
	})

	ginkgo.It("vsphere stress tests", func() {
		scArrays := make([]*storagev1.StorageClass, len(scNames))
		for index, scname := range scNames {
			// Create vSphere Storage Class
			ginkgo.By(fmt.Sprintf("Creating Storage Class : %v", scname))
			var sc *storagev1.StorageClass
			var err error
			switch scname {
			case storageclass1:
				sc, err = client.StorageV1().StorageClasses().Create(getVSphereStorageClassSpec(storageclass1, nil, nil, ""))
			case storageclass2:
				var scVSanParameters map[string]string
				scVSanParameters = make(map[string]string)
				scVSanParameters[Policy_HostFailuresToTolerate] = "1"
				sc, err = client.StorageV1().StorageClasses().Create(getVSphereStorageClassSpec(storageclass2, scVSanParameters, nil, ""))
			case storageclass3:
				var scSPBMPolicyParameters map[string]string
				scSPBMPolicyParameters = make(map[string]string)
				scSPBMPolicyParameters[SpbmStoragePolicy] = policyName
				sc, err = client.StorageV1().StorageClasses().Create(getVSphereStorageClassSpec(storageclass3, scSPBMPolicyParameters, nil, ""))
			case storageclass4:
				var scWithDSParameters map[string]string
				scWithDSParameters = make(map[string]string)
				scWithDSParameters[Datastore] = datastoreName
				scWithDatastoreSpec := getVSphereStorageClassSpec(storageclass4, scWithDSParameters, nil, "")
				sc, err = client.StorageV1().StorageClasses().Create(scWithDatastoreSpec)
			}
			gomega.Expect(sc).NotTo(gomega.BeNil())
			framework.ExpectNoError(err)
			defer client.StorageV1().StorageClasses().Delete(scname, nil)
			scArrays[index] = sc
		}

		var wg sync.WaitGroup
		wg.Add(instances)
		for instanceCount := 0; instanceCount < instances; instanceCount++ {
			instanceId := fmt.Sprintf("Thread:%v", instanceCount+1)
			go PerformVolumeLifeCycleInParallel(f, client, namespace, instanceId, scArrays[instanceCount%len(scArrays)], iterations, &wg)
		}
		wg.Wait()
	})

})

// goroutine to perform volume lifecycle operations in parallel
func PerformVolumeLifeCycleInParallel(f *framework.Framework, client clientset.Interface, namespace string, instanceId string, sc *storagev1.StorageClass, iterations int, wg *sync.WaitGroup) {
	defer wg.Done()
	defer ginkgo.GinkgoRecover()

	for iterationCount := 0; iterationCount < iterations; iterationCount++ {
		logPrefix := fmt.Sprintf("Instance: [%v], Iteration: [%v] :", instanceId, iterationCount+1)
		ginkgo.By(fmt.Sprintf("%v Creating PVC using the Storage Class: %v", logPrefix, sc.Name))
		pvclaim, err := e2epv.CreatePVC(client, namespace, getVSphereClaimSpecWithStorageClass(namespace, "1Gi", sc))
		framework.ExpectNoError(err)
		defer e2epv.DeletePersistentVolumeClaim(client, pvclaim.Name, namespace)

		var pvclaims []*v1.PersistentVolumeClaim
		pvclaims = append(pvclaims, pvclaim)
		ginkgo.By(fmt.Sprintf("%v Waiting for claim: %v to be in bound phase", logPrefix, pvclaim.Name))
		persistentvolumes, err := e2epv.WaitForPVClaimBoundPhase(client, pvclaims, framework.ClaimProvisionTimeout)
		framework.ExpectNoError(err)

		ginkgo.By(fmt.Sprintf("%v Creating Pod using the claim: %v", logPrefix, pvclaim.Name))
		// Create pod to attach Volume to Node
		pod, err := e2epod.CreatePod(client, namespace, nil, pvclaims, false, "")
		framework.ExpectNoError(err)

		ginkgo.By(fmt.Sprintf("%v Waiting for the Pod: %v to be in the running state", logPrefix, pod.Name))
		err = f.WaitForPodRunningSlow(pod.Name)
		framework.ExpectNoError(err)

		// Get the copy of the Pod to know the assigned node name.
		pod, err = client.CoreV1().Pods(namespace).Get(pod.Name, metav1.GetOptions{})
		framework.ExpectNoError(err)

		ginkgo.By(fmt.Sprintf("%v Verifing the volume: %v is attached to the node VM: %v", logPrefix, persistentvolumes[0].Spec.VsphereVolume.VolumePath, pod.Spec.NodeName))
		isVolumeAttached, verifyDiskAttachedError := diskIsAttached(persistentvolumes[0].Spec.VsphereVolume.VolumePath, pod.Spec.NodeName)
		gomega.Expect(isVolumeAttached).To(gomega.BeTrue())
		framework.ExpectNoError(verifyDiskAttachedError)

		ginkgo.By(fmt.Sprintf("%v Verifing the volume: %v is accessible in the pod: %v", logPrefix, persistentvolumes[0].Spec.VsphereVolume.VolumePath, pod.Name))
		verifyVSphereVolumesAccessible(client, pod, persistentvolumes)

		ginkgo.By(fmt.Sprintf("%v Deleting pod: %v", logPrefix, pod.Name))
		err = e2epod.DeletePodWithWait(client, pod)
		framework.ExpectNoError(err)

		ginkgo.By(fmt.Sprintf("%v Waiting for volume: %v to be detached from the node: %v", logPrefix, persistentvolumes[0].Spec.VsphereVolume.VolumePath, pod.Spec.NodeName))
		err = waitForVSphereDiskToDetach(persistentvolumes[0].Spec.VsphereVolume.VolumePath, pod.Spec.NodeName)
		framework.ExpectNoError(err)

		ginkgo.By(fmt.Sprintf("%v Deleting the Claim: %v", logPrefix, pvclaim.Name))
		err = e2epv.DeletePersistentVolumeClaim(client, pvclaim.Name, namespace)
		framework.ExpectNoError(err)
	}
}
<commit_msg>E2E: Add missing Bootstrap() call<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package vsphere

import (
	"fmt"
	"sync"

	"github.com/onsi/ginkgo"
	"github.com/onsi/gomega"
	"k8s.io/api/core/v1"
	storagev1 "k8s.io/api/storage/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	clientset "k8s.io/client-go/kubernetes"
	"k8s.io/kubernetes/test/e2e/framework"
	e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
	e2epv "k8s.io/kubernetes/test/e2e/framework/pv"
	"k8s.io/kubernetes/test/e2e/storage/utils"
)

/*
	Induce stress to create volumes in parallel with multiple threads based on user configurable values for number of threads and iterations per thread.
	The following actions will be performed as part of this test.

	1. Create Storage Classes of 4 Categories (Default, SC with Non Default Datastore, SC with SPBM Policy, SC with VSAN Storage Capabilities.)
	2. READ VCP_STRESS_INSTANCES, VCP_STRESS_ITERATIONS, VSPHERE_SPBM_POLICY_NAME and VSPHERE_DATASTORE from System Environment.
	3. Launch goroutine for volume lifecycle operations.
	4. Each instance of routine iterates for n times, where n is read from system env - VCP_STRESS_ITERATIONS
	5. Each iteration creates 1 PVC, 1 POD using the provisioned PV, Verify disk is attached to the node, Verify pod can access the volume, delete the pod and finally delete the PVC.
*/
var _ = utils.SIGDescribe("vsphere cloud provider stress [Feature:vsphere]", func() {
	f := framework.NewDefaultFramework("vcp-stress")
	var (
		client        clientset.Interface
		namespace     string
		instances     int
		iterations    int
		policyName    string
		datastoreName string
		err           error
		scNames       = []string{storageclass1, storageclass2, storageclass3, storageclass4}
	)

	ginkgo.BeforeEach(func() {
		framework.SkipUnlessProviderIs("vsphere")
		Bootstrap(f)
		client = f.ClientSet
		namespace = f.Namespace.Name

		nodeList := framework.GetReadySchedulableNodesOrDie(f.ClientSet)
		gomega.Expect(nodeList.Items).NotTo(gomega.BeEmpty(), "Unable to find ready and schedulable Node")

		// if VCP_STRESS_INSTANCES = 12 and VCP_STRESS_ITERATIONS is 10. 12 threads will run in parallel for 10 times.
		// Resulting 120 Volumes and POD Creation. Volumes will be provisioned with each different types of Storage Class,
		// Each iteration creates PVC, verify PV is provisioned, then creates a pod, verify volume is attached to the node, and then delete the pod and delete pvc.
		instances = GetAndExpectIntEnvVar(VCPStressInstances)
		gomega.Expect(instances <= volumesPerNode*len(nodeList.Items)).To(gomega.BeTrue(), fmt.Sprintf("Number of Instances should be less or equal: %v", volumesPerNode*len(nodeList.Items)))
		gomega.Expect(instances > len(scNames)).To(gomega.BeTrue(), "VCP_STRESS_INSTANCES should be greater than 3 to utilize all 4 types of storage classes")

		iterations = GetAndExpectIntEnvVar(VCPStressIterations)
		framework.ExpectNoError(err, "Error Parsing VCP_STRESS_ITERATIONS")
		gomega.Expect(iterations > 0).To(gomega.BeTrue(), "VCP_STRESS_ITERATIONS should be greater than 0")

		policyName = GetAndExpectStringEnvVar(SPBMPolicyName)
		datastoreName = GetAndExpectStringEnvVar(StorageClassDatastoreName)
	})

	ginkgo.It("vsphere stress tests", func() {
		scArrays := make([]*storagev1.StorageClass, len(scNames))
		for index, scname := range scNames {
			// Create vSphere Storage Class
			ginkgo.By(fmt.Sprintf("Creating Storage Class : %v", scname))
			var sc *storagev1.StorageClass
			var err error
			switch scname {
			case storageclass1:
				sc, err = client.StorageV1().StorageClasses().Create(getVSphereStorageClassSpec(storageclass1, nil, nil, ""))
			case storageclass2:
				var scVSanParameters map[string]string
				scVSanParameters = make(map[string]string)
				scVSanParameters[Policy_HostFailuresToTolerate] = "1"
				sc, err = client.StorageV1().StorageClasses().Create(getVSphereStorageClassSpec(storageclass2, scVSanParameters, nil, ""))
			case storageclass3:
				var scSPBMPolicyParameters map[string]string
				scSPBMPolicyParameters = make(map[string]string)
				scSPBMPolicyParameters[SpbmStoragePolicy] = policyName
				sc, err = client.StorageV1().StorageClasses().Create(getVSphereStorageClassSpec(storageclass3, scSPBMPolicyParameters, nil, ""))
			case storageclass4:
				var scWithDSParameters map[string]string
				scWithDSParameters = make(map[string]string)
				scWithDSParameters[Datastore] = datastoreName
				scWithDatastoreSpec := getVSphereStorageClassSpec(storageclass4, scWithDSParameters, nil, "")
				sc, err = client.StorageV1().StorageClasses().Create(scWithDatastoreSpec)
			}
			gomega.Expect(sc).NotTo(gomega.BeNil())
			framework.ExpectNoError(err)
			defer client.StorageV1().StorageClasses().Delete(scname, nil)
			scArrays[index] = sc
		}

		var wg sync.WaitGroup
		wg.Add(instances)
		for instanceCount := 0; instanceCount < instances; instanceCount++ {
			instanceId := fmt.Sprintf("Thread:%v", instanceCount+1)
			go PerformVolumeLifeCycleInParallel(f, client, namespace, instanceId, scArrays[instanceCount%len(scArrays)], iterations, &wg)
		}
		wg.Wait()
	})

})

// goroutine to perform volume lifecycle operations in parallel
func PerformVolumeLifeCycleInParallel(f *framework.Framework, client clientset.Interface, namespace string, instanceId string, sc *storagev1.StorageClass, iterations int, wg *sync.WaitGroup) {
	defer wg.Done()
	defer ginkgo.GinkgoRecover()

	for iterationCount := 0; iterationCount < iterations; iterationCount++ {
		logPrefix := fmt.Sprintf("Instance: [%v], Iteration: [%v] :", instanceId, iterationCount+1)
		ginkgo.By(fmt.Sprintf("%v Creating PVC using the Storage Class: %v", logPrefix, sc.Name))
		pvclaim, err := e2epv.CreatePVC(client, namespace, getVSphereClaimSpecWithStorageClass(namespace, "1Gi", sc))
		framework.ExpectNoError(err)
		defer e2epv.DeletePersistentVolumeClaim(client, pvclaim.Name, namespace)

		var pvclaims []*v1.PersistentVolumeClaim
		pvclaims = append(pvclaims, pvclaim)
		ginkgo.By(fmt.Sprintf("%v Waiting for claim: %v to be in bound phase", logPrefix, pvclaim.Name))
		persistentvolumes, err := e2epv.WaitForPVClaimBoundPhase(client, pvclaims, framework.ClaimProvisionTimeout)
		framework.ExpectNoError(err)

		ginkgo.By(fmt.Sprintf("%v Creating Pod using the claim: %v", logPrefix, pvclaim.Name))
		// Create pod to attach Volume to Node
		pod, err := e2epod.CreatePod(client, namespace, nil, pvclaims, false, "")
		framework.ExpectNoError(err)

		ginkgo.By(fmt.Sprintf("%v Waiting for the Pod: %v to be in the running state", logPrefix, pod.Name))
		err = f.WaitForPodRunningSlow(pod.Name)
		framework.ExpectNoError(err)

		// Get the copy of the Pod to know the assigned node name.
		pod, err = client.CoreV1().Pods(namespace).Get(pod.Name, metav1.GetOptions{})
		framework.ExpectNoError(err)

		ginkgo.By(fmt.Sprintf("%v Verifing the volume: %v is attached to the node VM: %v", logPrefix, persistentvolumes[0].Spec.VsphereVolume.VolumePath, pod.Spec.NodeName))
		isVolumeAttached, verifyDiskAttachedError := diskIsAttached(persistentvolumes[0].Spec.VsphereVolume.VolumePath, pod.Spec.NodeName)
		gomega.Expect(isVolumeAttached).To(gomega.BeTrue())
		framework.ExpectNoError(verifyDiskAttachedError)

		ginkgo.By(fmt.Sprintf("%v Verifing the volume: %v is accessible in the pod: %v", logPrefix, persistentvolumes[0].Spec.VsphereVolume.VolumePath, pod.Name))
		verifyVSphereVolumesAccessible(client, pod, persistentvolumes)

		ginkgo.By(fmt.Sprintf("%v Deleting pod: %v", logPrefix, pod.Name))
		err = e2epod.DeletePodWithWait(client, pod)
		framework.ExpectNoError(err)

		ginkgo.By(fmt.Sprintf("%v Waiting for volume: %v to be detached from the node: %v", logPrefix, persistentvolumes[0].Spec.VsphereVolume.VolumePath, pod.Spec.NodeName))
		err = waitForVSphereDiskToDetach(persistentvolumes[0].Spec.VsphereVolume.VolumePath, pod.Spec.NodeName)
		framework.ExpectNoError(err)

		ginkgo.By(fmt.Sprintf("%v Deleting the Claim: %v", logPrefix, pvclaim.Name))
		err = e2epv.DeletePersistentVolumeClaim(client, pvclaim.Name, namespace)
		framework.ExpectNoError(err)
	}
}
<|endoftext|>


<commit_before>package models

import (
	"testing"
	"time"

	"appengine/aetest"
	"appengine/datastore"

	"github.com/taironas/gonawin/helpers"
	"github.com/taironas/gonawin/helpers/log"
)

func TestCreateTournament(t *testing.T) {
	c, err := aetest.NewContext(nil)
	if err != nil {
		t.Fatal(err)
	}
	defer c.Close()

	log.Infof(c, "Test Create Tournament")

	tests := []struct {
		name       string
		tournament Tournament
		want       *Tournament
	}{
		{
			name: "Simple create",
			tournament: Tournament{
				Name:        "Foo",
				Description: "Foo description",
				Start:       time.Now(),
				End:         time.Now(),
				AdminIds:    make([]int64, 1),
			},
			want: &Tournament{
				Name:            "Foo",
				Description:     "Foo description",
				Start:           time.Now(),
				End:             time.Now(),
				AdminIds:        make([]int64, 1),
				GroupIds:        make([]int64, 0),
				Matches1stStage: make([]int64, 0),
				Matches2ndStage: make([]int64, 0),
				UserIds:         make([]int64, 0),
				TeamIds:         make([]int64, 0),
			},
		},
	}
	for _, test := range tests {
		got, _ := CreateTournament(c, test.tournament.Name, test.tournament.Description, test.tournament.Start, test.tournament.End, test.tournament.AdminIds[0])
		if got == nil && test.want != nil {
			t.Errorf("TestCreateTournament(%q): got nil wanted %v", test.name, *test.want)
		} else if got != nil && test.want == nil {
			t.Errorf("TestCreateTournament(%q): got %v wanted nil", test.name, *got)
		} else if got == nil && test.want == nil {
			// This is OK
		} else if got.Name != test.want.Name ||
			got.Description != test.want.Description ||
			len(got.AdminIds) != len(test.want.AdminIds) ||
			len(got.GroupIds) != len(test.want.GroupIds) ||
			len(got.Matches1stStage) != len(test.want.Matches1stStage) ||
			len(got.Matches2ndStage) != len(test.want.Matches2ndStage) ||
			len(got.UserIds) != len(test.want.UserIds) ||
			len(got.TeamIds) != len(test.want.TeamIds) {
			t.Errorf("TestCreateTournament(%q): got %v wanted %v", test.name, *got, *test.want)
		}
	}
}

func TestDestroyTournament(t *testing.T) {
	c, err := aetest.NewContext(nil)
	if err != nil {
		t.Fatal(err)
	}
	defer c.Close()

	log.Infof(c, "Test Destory Tournament")

	test := struct {
		name       string
		tournament Tournament
		want       *Tournament
	}{
		name: "destroy tournament",
		tournament: Tournament{
			Name:        "Foo",
			Description: "Foo description",
			Start:       time.Now(),
			End:         time.Now(),
			AdminIds:    make([]int64, 1),
		},
		want: nil,
	}

	// create tournament
	tournament, _ := CreateTournament(c, test.tournament.Name, test.tournament.Description, test.tournament.Start, test.tournament.End, test.tournament.AdminIds[0])

	// perform a get query so that the results of the unapplied write are visible to subsequent global queries.
	dummy := Tournament{}
	key := TournamentKeyById(c, tournament.Id)
	if err := datastore.Get(c, key, &dummy); err != nil {
		t.Fatal(err)
	}

	// destory it
	if got := tournament.Destroy(c); got != nil {
		t.Errorf("TestDestroyTournament(%q): got %v wanted %v", test.name, got, test.want)
	}

	// make a query on datastore to be sure it is not there.
	if got, err1 := TournamentById(c, tournament.Id); err1 == nil {
		t.Errorf("TestDestroyTournament(%q): got %v wanted %v", test.name, got, test.want)
	}
}

func TestFindTournaments(t *testing.T) {

	c, err := aetest.NewContext(nil)
	if err != nil {
		t.Fatal(err)
	}
	defer c.Close()

	log.Infof(c, "Test Find Tournament")

	tournaments := []Tournament{
		Tournament{
			Name:        "bar",
			Description: "Foo description",
			Start:       time.Now(),
			End:         time.Now(),
			AdminIds:    make([]int64, 1),
		},
		Tournament{
			Name:        "foobar",
			Description: "foo description",
			Start:       time.Now(),
			End:         time.Now(),
			AdminIds:    make([]int64, 1),
		},
		Tournament{
			Name:        "foobarfoo",
			Description: "Foo description",
			Start:       time.Now(),
			End:         time.Now(),
			AdminIds:    make([]int64, 1),
		},
	}

	tests := []struct {
		name        string
		tournaments []Tournament
		queries     []string
		want        struct {
			Len   int
			Names []string
		}
	}{
		{
			name:        "find tournaments in empty datastore",
			tournaments: make([]Tournament, 0),
			queries:     []string{"foo"},
			want: struct {
				Len   int
				Names []string
			}{
				Len:   int(0),
				Names: make([]string, 0),
			},
		},
		{
			name:        "find tournaments in full datastore",
			tournaments: tournaments,
			queries:     []string{"bar", "foobar", "foobarfoo"},
			want: struct {
				Len   int
				Names []string
			}{
				Len:   int(1),
				Names: []string{"bar", "foobar", "foobarfoo"},
			},
		},
	}

	// create tournaments
	for _, test := range tests {
		for i, _ := range test.tournaments {
			if got, err1 := CreateTournament(
				c,
				test.tournaments[i].Name,
				test.tournaments[i].Description,
				test.tournaments[i].Start,
				test.tournaments[i].End,
				test.tournaments[i].AdminIds[0]); err1 != nil {
				t.Errorf("TestFindTournaments(%q): error creating tournaments: %v", test.name, err1)
			} else {
				// perform a get query so that the results of the unapplied write are visible to subsequent global queries.
				dummy := Tournament{}
				key := TournamentKeyById(c, got.Id)
				if err := datastore.Get(c, key, &dummy); err != nil {
					t.Fatal(err)
				}
			}
		}
	}

	// search tournaments
	log.Infof(c, "Test Find Tournament: start search ok tournaments")
	for _, test := range tests {
		for _, query := range test.queries {
			got := FindTournaments(c, "KeyName", query)
			if test.want.Len != len(got) {
				t.Errorf("TestFindTournaments(%q): got array of %v  wanted %v: query:%v t:%v", test.name, len(got), test.want.Len, query, got)
			}
			for _, tour := range got {
				if !helpers.SliceContains(test.want.Names, tour.Name) {
					t.Errorf("TestFindTournaments(%q): name not found. got %v  wanted among %v", test.name, tour.Name, test.want.Names)
				}
			}
		}
	}
}
<commit_msg>create checkTournament function<commit_after>package models

import (
	"errors"
	"fmt"
	"testing"
	"time"

	"appengine/aetest"
	"appengine/datastore"

	"github.com/taironas/gonawin/helpers"
	"github.com/taironas/gonawin/helpers/log"
)

func TestCreateTournament(t *testing.T) {
	c, err := aetest.NewContext(nil)
	if err != nil {
		t.Fatal(err)
	}
	defer c.Close()

	log.Infof(c, "Test Create Tournament")

	tests := []struct {
		name       string
		tournament Tournament
		want       *Tournament
	}{
		{
			name: "Simple create",
			tournament: Tournament{
				Name:        "Foo",
				Description: "Foo description",
				Start:       time.Now(),
				End:         time.Now(),
				AdminIds:    make([]int64, 1),
			},
			want: &Tournament{
				Name:            "Foo",
				Description:     "Foo description",
				Start:           time.Now(),
				End:             time.Now(),
				AdminIds:        make([]int64, 1),
				GroupIds:        make([]int64, 0),
				Matches1stStage: make([]int64, 0),
				Matches2ndStage: make([]int64, 0),
				UserIds:         make([]int64, 0),
				TeamIds:         make([]int64, 0),
			},
		},
	}
	for i, test := range tests {
		got, _ := CreateTournament(c, test.tournament.Name, test.tournament.Description, test.tournament.Start, test.tournament.End, test.tournament.AdminIds[0])
		if got == nil && test.want != nil {
			t.Errorf("TestCreateTournament(%q): got nil wanted %v", test.name, *test.want)
		} else if got != nil && test.want == nil {
			t.Errorf("TestCreateTournament(%q): got %v wanted nil", test.name, *got)
		} else if got == nil && test.want == nil {
			// This is OK
		} else if err = checkTournament(got, test.want); err != nil {
			t.Errorf("test %v - Error: %v", i, err)
		}
	}
}

func TestDestroyTournament(t *testing.T) {
	c, err := aetest.NewContext(nil)
	if err != nil {
		t.Fatal(err)
	}
	defer c.Close()

	log.Infof(c, "Test Destory Tournament")

	test := struct {
		name       string
		tournament Tournament
		want       *Tournament
	}{
		name: "destroy tournament",
		tournament: Tournament{
			Name:        "Foo",
			Description: "Foo description",
			Start:       time.Now(),
			End:         time.Now(),
			AdminIds:    make([]int64, 1),
		},
		want: nil,
	}

	// create tournament
	tournament, _ := CreateTournament(c, test.tournament.Name, test.tournament.Description, test.tournament.Start, test.tournament.End, test.tournament.AdminIds[0])

	// perform a get query so that the results of the unapplied write are visible to subsequent global queries.
	dummy := Tournament{}
	key := TournamentKeyById(c, tournament.Id)
	if err := datastore.Get(c, key, &dummy); err != nil {
		t.Fatal(err)
	}

	// destory it
	if got := tournament.Destroy(c); got != nil {
		t.Errorf("TestDestroyTournament(%q): got %v wanted %v", test.name, got, test.want)
	}

	// make a query on datastore to be sure it is not there.
	if got, err1 := TournamentById(c, tournament.Id); err1 == nil {
		t.Errorf("TestDestroyTournament(%q): got %v wanted %v", test.name, got, test.want)
	}
}

func TestFindTournaments(t *testing.T) {

	c, err := aetest.NewContext(nil)
	if err != nil {
		t.Fatal(err)
	}
	defer c.Close()

	log.Infof(c, "Test Find Tournament")

	tournaments := []Tournament{
		Tournament{
			Name:        "bar",
			Description: "Foo description",
			Start:       time.Now(),
			End:         time.Now(),
			AdminIds:    make([]int64, 1),
		},
		Tournament{
			Name:        "foobar",
			Description: "foo description",
			Start:       time.Now(),
			End:         time.Now(),
			AdminIds:    make([]int64, 1),
		},
		Tournament{
			Name:        "foobarfoo",
			Description: "Foo description",
			Start:       time.Now(),
			End:         time.Now(),
			AdminIds:    make([]int64, 1),
		},
	}

	tests := []struct {
		name        string
		tournaments []Tournament
		queries     []string
		want        struct {
			Len   int
			Names []string
		}
	}{
		{
			name:        "find tournaments in empty datastore",
			tournaments: make([]Tournament, 0),
			queries:     []string{"foo"},
			want: struct {
				Len   int
				Names []string
			}{
				Len:   int(0),
				Names: make([]string, 0),
			},
		},
		{
			name:        "find tournaments in full datastore",
			tournaments: tournaments,
			queries:     []string{"bar", "foobar", "foobarfoo"},
			want: struct {
				Len   int
				Names []string
			}{
				Len:   int(1),
				Names: []string{"bar", "foobar", "foobarfoo"},
			},
		},
	}

	// create tournaments
	for _, test := range tests {
		for i, _ := range test.tournaments {
			if got, err1 := CreateTournament(
				c,
				test.tournaments[i].Name,
				test.tournaments[i].Description,
				test.tournaments[i].Start,
				test.tournaments[i].End,
				test.tournaments[i].AdminIds[0]); err1 != nil {
				t.Errorf("TestFindTournaments(%q): error creating tournaments: %v", test.name, err1)
			} else {
				// perform a get query so that the results of the unapplied write are visible to subsequent global queries.
				dummy := Tournament{}
				key := TournamentKeyById(c, got.Id)
				if err := datastore.Get(c, key, &dummy); err != nil {
					t.Fatal(err)
				}
			}
		}
	}

	// search tournaments
	log.Infof(c, "Test Find Tournament: start search ok tournaments")
	for _, test := range tests {
		for _, query := range test.queries {
			got := FindTournaments(c, "KeyName", query)
			if test.want.Len != len(got) {
				t.Errorf("TestFindTournaments(%q): got array of %v  wanted %v: query:%v t:%v", test.name, len(got), test.want.Len, query, got)
			}
			for _, tour := range got {
				if !helpers.SliceContains(test.want.Names, tour.Name) {
					t.Errorf("TestFindTournaments(%q): name not found. got %v  wanted among %v", test.name, tour.Name, test.want.Names)
				}
			}
		}
	}
}

func checkTournament(got *Tournament, want *Tournament) error {
	var s string
	if got.Name != want.Name {
		s = fmt.Sprintf("want Name == %s, got %s", want.Name, got.Name)
	} else if got.Description != want.Description {
		s = fmt.Sprintf("want Description == %s, got %s", want.Description, got.Description)
	} else if len(got.AdminIds) != len(want.AdminIds) {
		s = fmt.Sprintf("want AdminIds count == %d, got %d", len(want.AdminIds), len(got.AdminIds))
	} else if len(got.GroupIds) != len(want.GroupIds) {
		s = fmt.Sprintf("want GroupIds count == %d, got %d", len(want.GroupIds), len(got.GroupIds))
	} else if len(got.Matches1stStage) != len(want.Matches1stStage) {
		s = fmt.Sprintf("want Matches1stStage count == %d, got %d", len(want.Matches1stStage), len(got.Matches1stStage))
	} else if len(got.Matches2ndStage) != len(want.Matches2ndStage) {
		s = fmt.Sprintf("want Matches2ndStage count == %d, got %d", len(want.Matches2ndStage), len(got.Matches2ndStage))
	} else if len(got.UserIds) != len(want.UserIds) {
		s = fmt.Sprintf("want UserIds count == %d, got %d", len(want.UserIds), len(got.UserIds))
	} else if len(got.TeamIds) != len(want.TeamIds) {
		s = fmt.Sprintf("want TeamIds count == %d, got %d", len(want.TeamIds), len(got.TeamIds))
	} else {
		return nil
	}

	return errors.New(s)
}
<|endoftext|>


<commit_before>package taskreaper

import (
	"sort"
	"time"

	"github.com/docker/swarmkit/api"
	"github.com/docker/swarmkit/log"
	"github.com/docker/swarmkit/manager/orchestrator"
	"github.com/docker/swarmkit/manager/state"
	"github.com/docker/swarmkit/manager/state/store"
	"golang.org/x/net/context"
)

const (
	// maxDirty is the size threshold for running a task pruning operation.
	maxDirty = 1000
	// reaperBatchingInterval is how often to prune old tasks.
	reaperBatchingInterval = 250 * time.Millisecond
)

// A TaskReaper deletes old tasks when more than TaskHistoryRetentionLimit tasks
// exist for the same service/instance or service/nodeid combination.
type TaskReaper struct {
	store *store.MemoryStore

	// taskHistory is the number of tasks to keep
	taskHistory int64

	// List of slot tubles to be inspected for task history cleanup.
	dirty map[orchestrator.SlotTuple]struct{}

	// List of tasks collected for cleanup, which includes two kinds of tasks
	// - serviceless orphaned tasks
	// - tasks with desired state REMOVE that have already been shut down
	cleanup  []string
	stopChan chan struct{}
	doneChan chan struct{}
}

// New creates a new TaskReaper.
func New(store *store.MemoryStore) *TaskReaper {
	return &TaskReaper{
		store:    store,
		dirty:    make(map[orchestrator.SlotTuple]struct{}),
		stopChan: make(chan struct{}),
		doneChan: make(chan struct{}),
	}
}

// Run is the TaskReaper's watch loop which collects candidates for cleanup.
// Task history is mainly used in task restarts but is also available for administrative purposes.
// Note that the task history is stored per-slot-per-service for replicated services
// and per-node-per-service for global services. History does not apply to serviceless
// since they are not attached to a service. In addition, the TaskReaper watch loop is also
// responsible for cleaning up tasks associated with slots that were removed as part of
// service scale down or service removal.
func (tr *TaskReaper) Run(ctx context.Context) {
	watcher, watchCancel := state.Watch(tr.store.WatchQueue(), api.EventCreateTask{}, api.EventUpdateTask{}, api.EventUpdateCluster{})

	defer func() {
		close(tr.doneChan)
		watchCancel()
	}()

	var orphanedTasks []*api.Task
	var removeTasks []*api.Task
	tr.store.View(func(readTx store.ReadTx) {
		var err error

		clusters, err := store.FindClusters(readTx, store.ByName(store.DefaultClusterName))
		if err == nil && len(clusters) == 1 {
			tr.taskHistory = clusters[0].Spec.Orchestration.TaskHistoryRetentionLimit
		}

		// On startup, scan the entire store and inspect orphaned tasks from previous life.
		orphanedTasks, err = store.FindTasks(readTx, store.ByTaskState(api.TaskStateOrphaned))
		if err != nil {
			log.G(ctx).WithError(err).Error("failed to find Orphaned tasks in task reaper init")
		}
		removeTasks, err = store.FindTasks(readTx, store.ByDesiredState(api.TaskStateRemove))
		if err != nil {
			log.G(ctx).WithError(err).Error("failed to find tasks with desired state REMOVE in task reaper init")
		}
	})

	if len(orphanedTasks)+len(removeTasks) > 0 {
		for _, t := range orphanedTasks {
			// Do not reap service tasks immediately.
			// Let them go through the regular history cleanup process
			// of checking TaskHistoryRetentionLimit.
			if t.ServiceID != "" {
				continue
			}

			// Serviceless tasks can be cleaned up right away since they are not attached to a service.
			tr.cleanup = append(tr.cleanup, t.ID)
		}
		// tasks with desired state REMOVE that have progressed beyond COMPLETE can be cleaned up
		// right away
		for _, t := range removeTasks {
			if t.Status.State >= api.TaskStateCompleted {
				tr.cleanup = append(tr.cleanup, t.ID)
			}
		}
		// Clean up tasks in 'cleanup' right away
		if len(tr.cleanup) > 0 {
			tr.tick()
		}
	}

	// Clean up when we hit TaskHistoryRetentionLimit or when the timer expires,
	// whichever happens first.
	timer := time.NewTimer(reaperBatchingInterval)

	// Watch for:
	// 1. EventCreateTask for cleaning slots, which is the best time to cleanup that node/slot.
	// 2. EventUpdateTask for cleaning
	//    - serviceless orphaned tasks (when orchestrator updates the task status to ORPHANED)
	//    - tasks which have desired state REMOVE and have been shut down by the agent
	//      (these are tasks which are associated with slots removed as part of service
	//       remove or scale down)
	// 3. EventUpdateCluster for TaskHistoryRetentionLimit update.
	for {
		select {
		case event := <-watcher:
			switch v := event.(type) {
			case api.EventCreateTask:
				t := v.Task
				tr.dirty[orchestrator.SlotTuple{
					Slot:      t.Slot,
					ServiceID: t.ServiceID,
					NodeID:    t.NodeID,
				}] = struct{}{}
			case api.EventUpdateTask:
				t := v.Task
				// add serviceless orphaned tasks
				if t.Status.State >= api.TaskStateOrphaned && t.ServiceID == "" {
					tr.cleanup = append(tr.cleanup, t.ID)
				}
				// add tasks that have progressed beyond COMPLETE and have desired state REMOVE. These
				// tasks are associated with slots that were removed as part of a service scale down
				// or service removal.
				if t.DesiredState == api.TaskStateRemove && t.Status.State >= api.TaskStateCompleted {
					tr.cleanup = append(tr.cleanup, t.ID)
				}
			case api.EventUpdateCluster:
				tr.taskHistory = v.Cluster.Spec.Orchestration.TaskHistoryRetentionLimit
			}

			if len(tr.dirty)+len(tr.cleanup) > maxDirty {
				timer.Stop()
				tr.tick()
			} else {
				timer.Reset(reaperBatchingInterval)
			}
		case <-timer.C:
			timer.Stop()
			tr.tick()
		case <-tr.stopChan:
			timer.Stop()
			return
		}
	}
}

// tick performs task history cleanup.
func (tr *TaskReaper) tick() {
	if len(tr.dirty) == 0 && len(tr.cleanup) == 0 {
		return
	}

	defer func() {
		tr.cleanup = nil
	}()

	deleteTasks := make(map[string]struct{})
	for _, tID := range tr.cleanup {
		deleteTasks[tID] = struct{}{}
	}

	// Check history of dirty tasks for cleanup.
	tr.store.View(func(tx store.ReadTx) {
		for dirty := range tr.dirty {
			service := store.GetService(tx, dirty.ServiceID)
			if service == nil {
				continue
			}

			taskHistory := tr.taskHistory

			// If MaxAttempts is set, keep at least one more than
			// that number of tasks (this overrides TaskHistoryRetentionLimit).
			// This is necessary to reconstruct restart history when the orchestrator starts up.
			// TODO(aaronl): Consider hiding tasks beyond the normal
			// retention limit in the UI.
			// TODO(aaronl): There are some ways to cut down the
			// number of retained tasks at the cost of more
			// complexity:
			//   - Don't force retention of tasks with an older spec
			//     version.
			//   - Don't force retention of tasks outside of the
			//     time window configured for restart lookback.
			if service.Spec.Task.Restart != nil && service.Spec.Task.Restart.MaxAttempts > 0 {
				taskHistory = int64(service.Spec.Task.Restart.MaxAttempts) + 1
			}

			// Negative value for TaskHistoryRetentionLimit is an indication to never clean up task history.
			if taskHistory < 0 {
				continue
			}

			var historicTasks []*api.Task

			switch service.Spec.GetMode().(type) {
			case *api.ServiceSpec_Replicated:
				// Clean out the slot for which we received EventCreateTask.
				var err error
				historicTasks, err = store.FindTasks(tx, store.BySlot(dirty.ServiceID, dirty.Slot))
				if err != nil {
					continue
				}

			case *api.ServiceSpec_Global:
				// Clean out the node history in case of global services.
				tasksByNode, err := store.FindTasks(tx, store.ByNodeID(dirty.NodeID))
				if err != nil {
					continue
				}

				for _, t := range tasksByNode {
					if t.ServiceID == dirty.ServiceID {
						historicTasks = append(historicTasks, t)
					}
				}
			}

			if int64(len(historicTasks)) <= taskHistory {
				continue
			}

			// TODO(aaronl): This could filter for non-running tasks and use quickselect
			// instead of sorting the whole slice.
			// TODO(aaronl): This sort should really use lamport time instead of wall
			// clock time. We should store a Version in the Status field.
			sort.Sort(orchestrator.TasksByTimestamp(historicTasks))

			runningTasks := 0
			for _, t := range historicTasks {
				if t.DesiredState <= api.TaskStateRunning || t.Status.State <= api.TaskStateRunning {
					// Don't delete running tasks
					runningTasks++
					continue
				}

				deleteTasks[t.ID] = struct{}{}

				taskHistory++
				if int64(len(historicTasks)) <= taskHistory {
					break
				}
			}

			if runningTasks <= 1 {
				delete(tr.dirty, dirty)
			}
		}
	})

	// Perform cleanup.
	if len(deleteTasks) > 0 {
		tr.store.Batch(func(batch *store.Batch) error {
			for taskID := range deleteTasks {
				batch.Update(func(tx store.Tx) error {
					return store.DeleteTask(tx, taskID)
				})
			}
			return nil
		})
	}
}

// Stop stops the TaskReaper and waits for the main loop to exit.
func (tr *TaskReaper) Stop() {
	// TODO(dperny) calling stop on the task reaper twice will cause a panic
	// because we try to close a channel that will already have been closed.
	close(tr.stopChan)
	<-tr.doneChan
}
<commit_msg>Make sure that task reaper stopChan is closed no more than one time<commit_after>package taskreaper

import (
	"sort"
	"sync"
	"time"

	"github.com/docker/swarmkit/api"
	"github.com/docker/swarmkit/log"
	"github.com/docker/swarmkit/manager/orchestrator"
	"github.com/docker/swarmkit/manager/state"
	"github.com/docker/swarmkit/manager/state/store"
	"golang.org/x/net/context"
)

const (
	// maxDirty is the size threshold for running a task pruning operation.
	maxDirty = 1000
	// reaperBatchingInterval is how often to prune old tasks.
	reaperBatchingInterval = 250 * time.Millisecond
)

// A TaskReaper deletes old tasks when more than TaskHistoryRetentionLimit tasks
// exist for the same service/instance or service/nodeid combination.
type TaskReaper struct {
	store *store.MemoryStore

	// closeOnce ensures that stopChan is closed only once
	closeOnce sync.Once

	// taskHistory is the number of tasks to keep
	taskHistory int64

	// List of slot tubles to be inspected for task history cleanup.
	dirty map[orchestrator.SlotTuple]struct{}

	// List of tasks collected for cleanup, which includes two kinds of tasks
	// - serviceless orphaned tasks
	// - tasks with desired state REMOVE that have already been shut down
	cleanup  []string
	stopChan chan struct{}
	doneChan chan struct{}
}

// New creates a new TaskReaper.
func New(store *store.MemoryStore) *TaskReaper {
	return &TaskReaper{
		store:    store,
		dirty:    make(map[orchestrator.SlotTuple]struct{}),
		stopChan: make(chan struct{}),
		doneChan: make(chan struct{}),
	}
}

// Run is the TaskReaper's watch loop which collects candidates for cleanup.
// Task history is mainly used in task restarts but is also available for administrative purposes.
// Note that the task history is stored per-slot-per-service for replicated services
// and per-node-per-service for global services. History does not apply to serviceless
// since they are not attached to a service. In addition, the TaskReaper watch loop is also
// responsible for cleaning up tasks associated with slots that were removed as part of
// service scale down or service removal.
func (tr *TaskReaper) Run(ctx context.Context) {
	watcher, watchCancel := state.Watch(tr.store.WatchQueue(), api.EventCreateTask{}, api.EventUpdateTask{}, api.EventUpdateCluster{})

	defer func() {
		close(tr.doneChan)
		watchCancel()
	}()

	var orphanedTasks []*api.Task
	var removeTasks []*api.Task
	tr.store.View(func(readTx store.ReadTx) {
		var err error

		clusters, err := store.FindClusters(readTx, store.ByName(store.DefaultClusterName))
		if err == nil && len(clusters) == 1 {
			tr.taskHistory = clusters[0].Spec.Orchestration.TaskHistoryRetentionLimit
		}

		// On startup, scan the entire store and inspect orphaned tasks from previous life.
		orphanedTasks, err = store.FindTasks(readTx, store.ByTaskState(api.TaskStateOrphaned))
		if err != nil {
			log.G(ctx).WithError(err).Error("failed to find Orphaned tasks in task reaper init")
		}
		removeTasks, err = store.FindTasks(readTx, store.ByDesiredState(api.TaskStateRemove))
		if err != nil {
			log.G(ctx).WithError(err).Error("failed to find tasks with desired state REMOVE in task reaper init")
		}
	})

	if len(orphanedTasks)+len(removeTasks) > 0 {
		for _, t := range orphanedTasks {
			// Do not reap service tasks immediately.
			// Let them go through the regular history cleanup process
			// of checking TaskHistoryRetentionLimit.
			if t.ServiceID != "" {
				continue
			}

			// Serviceless tasks can be cleaned up right away since they are not attached to a service.
			tr.cleanup = append(tr.cleanup, t.ID)
		}
		// tasks with desired state REMOVE that have progressed beyond COMPLETE can be cleaned up
		// right away
		for _, t := range removeTasks {
			if t.Status.State >= api.TaskStateCompleted {
				tr.cleanup = append(tr.cleanup, t.ID)
			}
		}
		// Clean up tasks in 'cleanup' right away
		if len(tr.cleanup) > 0 {
			tr.tick()
		}
	}

	// Clean up when we hit TaskHistoryRetentionLimit or when the timer expires,
	// whichever happens first.
	timer := time.NewTimer(reaperBatchingInterval)

	// Watch for:
	// 1. EventCreateTask for cleaning slots, which is the best time to cleanup that node/slot.
	// 2. EventUpdateTask for cleaning
	//    - serviceless orphaned tasks (when orchestrator updates the task status to ORPHANED)
	//    - tasks which have desired state REMOVE and have been shut down by the agent
	//      (these are tasks which are associated with slots removed as part of service
	//       remove or scale down)
	// 3. EventUpdateCluster for TaskHistoryRetentionLimit update.
	for {
		select {
		case event := <-watcher:
			switch v := event.(type) {
			case api.EventCreateTask:
				t := v.Task
				tr.dirty[orchestrator.SlotTuple{
					Slot:      t.Slot,
					ServiceID: t.ServiceID,
					NodeID:    t.NodeID,
				}] = struct{}{}
			case api.EventUpdateTask:
				t := v.Task
				// add serviceless orphaned tasks
				if t.Status.State >= api.TaskStateOrphaned && t.ServiceID == "" {
					tr.cleanup = append(tr.cleanup, t.ID)
				}
				// add tasks that have progressed beyond COMPLETE and have desired state REMOVE. These
				// tasks are associated with slots that were removed as part of a service scale down
				// or service removal.
				if t.DesiredState == api.TaskStateRemove && t.Status.State >= api.TaskStateCompleted {
					tr.cleanup = append(tr.cleanup, t.ID)
				}
			case api.EventUpdateCluster:
				tr.taskHistory = v.Cluster.Spec.Orchestration.TaskHistoryRetentionLimit
			}

			if len(tr.dirty)+len(tr.cleanup) > maxDirty {
				timer.Stop()
				tr.tick()
			} else {
				timer.Reset(reaperBatchingInterval)
			}
		case <-timer.C:
			timer.Stop()
			tr.tick()
		case <-tr.stopChan:
			timer.Stop()
			return
		}
	}
}

// tick performs task history cleanup.
func (tr *TaskReaper) tick() {
	if len(tr.dirty) == 0 && len(tr.cleanup) == 0 {
		return
	}

	defer func() {
		tr.cleanup = nil
	}()

	deleteTasks := make(map[string]struct{})
	for _, tID := range tr.cleanup {
		deleteTasks[tID] = struct{}{}
	}

	// Check history of dirty tasks for cleanup.
	tr.store.View(func(tx store.ReadTx) {
		for dirty := range tr.dirty {
			service := store.GetService(tx, dirty.ServiceID)
			if service == nil {
				continue
			}

			taskHistory := tr.taskHistory

			// If MaxAttempts is set, keep at least one more than
			// that number of tasks (this overrides TaskHistoryRetentionLimit).
			// This is necessary to reconstruct restart history when the orchestrator starts up.
			// TODO(aaronl): Consider hiding tasks beyond the normal
			// retention limit in the UI.
			// TODO(aaronl): There are some ways to cut down the
			// number of retained tasks at the cost of more
			// complexity:
			//   - Don't force retention of tasks with an older spec
			//     version.
			//   - Don't force retention of tasks outside of the
			//     time window configured for restart lookback.
			if service.Spec.Task.Restart != nil && service.Spec.Task.Restart.MaxAttempts > 0 {
				taskHistory = int64(service.Spec.Task.Restart.MaxAttempts) + 1
			}

			// Negative value for TaskHistoryRetentionLimit is an indication to never clean up task history.
			if taskHistory < 0 {
				continue
			}

			var historicTasks []*api.Task

			switch service.Spec.GetMode().(type) {
			case *api.ServiceSpec_Replicated:
				// Clean out the slot for which we received EventCreateTask.
				var err error
				historicTasks, err = store.FindTasks(tx, store.BySlot(dirty.ServiceID, dirty.Slot))
				if err != nil {
					continue
				}

			case *api.ServiceSpec_Global:
				// Clean out the node history in case of global services.
				tasksByNode, err := store.FindTasks(tx, store.ByNodeID(dirty.NodeID))
				if err != nil {
					continue
				}

				for _, t := range tasksByNode {
					if t.ServiceID == dirty.ServiceID {
						historicTasks = append(historicTasks, t)
					}
				}
			}

			if int64(len(historicTasks)) <= taskHistory {
				continue
			}

			// TODO(aaronl): This could filter for non-running tasks and use quickselect
			// instead of sorting the whole slice.
			// TODO(aaronl): This sort should really use lamport time instead of wall
			// clock time. We should store a Version in the Status field.
			sort.Sort(orchestrator.TasksByTimestamp(historicTasks))

			runningTasks := 0
			for _, t := range historicTasks {
				if t.DesiredState <= api.TaskStateRunning || t.Status.State <= api.TaskStateRunning {
					// Don't delete running tasks
					runningTasks++
					continue
				}

				deleteTasks[t.ID] = struct{}{}

				taskHistory++
				if int64(len(historicTasks)) <= taskHistory {
					break
				}
			}

			if runningTasks <= 1 {
				delete(tr.dirty, dirty)
			}
		}
	})

	// Perform cleanup.
	if len(deleteTasks) > 0 {
		tr.store.Batch(func(batch *store.Batch) error {
			for taskID := range deleteTasks {
				batch.Update(func(tx store.Tx) error {
					return store.DeleteTask(tx, taskID)
				})
			}
			return nil
		})
	}
}

// Stop stops the TaskReaper and waits for the main loop to exit.
// Stop can be called in two cases. One when the manager is
// shutting down, and the other when the manager (the leader) is
// becoming a follower. Since these two instances could race with
// each other, we use closeOnce here to ensure that TaskReaper.Stop()
// is called only once to avoid a panic.
func (tr *TaskReaper) Stop() {
	tr.closeOnce.Do(func() {
		close(tr.stopChan)
	})
	<-tr.doneChan
}
<|endoftext|>


<commit_before>// Copyright (c) 2015 Uber Technologies, Inc.

// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package thrift

import (
	"log"
	"strings"
	"sync"

	"github.com/apache/thrift/lib/go/thrift"
	tchannel "github.com/uber/tchannel-go"
	"golang.org/x/net/context"
)

type handler struct {
	server         TChanServer
	postResponseCB PostResponseCB
}

// Server handles incoming TChannel calls and forwards them to the matching TChanServer.
type Server struct {
	ch          tchannel.Registrar
	log         tchannel.Logger
	mut         sync.RWMutex
	handlers    map[string]handler
	metaHandler *metaHandler
}

// NewServer returns a server that can serve thrift services over TChannel.
func NewServer(registrar tchannel.Registrar) *Server {
	metaHandler := newMetaHandler()
	server := &Server{
		ch:          registrar,
		log:         registrar.Logger(),
		handlers:    make(map[string]handler),
		metaHandler: metaHandler,
	}
	server.Register(newTChanMetaServer(metaHandler))
	if ch, ok := registrar.(*tchannel.Channel); ok {
		// Register the meta endpoints on the "tchannel" service name.
		NewServer(ch.GetSubChannel("tchannel"))
	}
	return server
}

// Register registers the given TChanServer to be called on any incoming call for its' services.
// TODO(prashant): Replace Register call with this call.
func (s *Server) Register(svr TChanServer, opts ...RegisterOption) {
	service := svr.Service()
	handler := &handler{server: svr}
	for _, opt := range opts {
		opt.Apply(handler)
	}

	s.mut.Lock()
	s.handlers[service] = *handler
	s.mut.Unlock()

	for _, m := range svr.Methods() {
		s.ch.Register(s, service+"::"+m)
	}
}

// RegisterHealthHandler uses the user-specified function f for the Health endpoint.
func (s *Server) RegisterHealthHandler(f HealthFunc) {
	s.metaHandler.setHandler(f)
}

func (s *Server) onError(err error) {
	// TODO(prashant): Expose incoming call errors through options for NewServer.
	s.log.Errorf("thrift Server error: %v", err)
}

func (s *Server) handle(origCtx context.Context, handler handler, method string, call *tchannel.InboundCall) error {
	reader, err := call.Arg2Reader()
	if err != nil {
		return err
	}
	headers, err := readHeaders(reader)
	if err != nil {
		return err
	}
	if err := reader.Close(); err != nil {
		return err
	}

	reader, err = call.Arg3Reader()
	if err != nil {
		return err
	}

	ctx := WithHeaders(origCtx, headers)

	wp := getProtocolReader(reader)
	success, resp, err := handler.server.Handle(ctx, method, wp.protocol)
	thriftProtocolPool.Put(wp)

	if err != nil {
		if _, ok := err.(thrift.TProtocolException); ok {
			// We failed to parse the Thrift generated code, so convert the error to bad request.
			err = tchannel.NewSystemError(tchannel.ErrCodeBadRequest, err.Error())
		}

		reader.Close()
		call.Response().SendSystemError(err)
		return nil
	}
	if err := reader.Close(); err != nil {
		return err
	}

	if !success {
		call.Response().SetApplicationError()
	}

	writer, err := call.Response().Arg2Writer()
	if err != nil {
		return err
	}

	if err := writeHeaders(writer, ctx.ResponseHeaders()); err != nil {
		return err
	}
	if err := writer.Close(); err != nil {
		return err
	}

	writer, err = call.Response().Arg3Writer()
	wp = getProtocolWriter(writer)
	resp.Write(wp.protocol)
	thriftProtocolPool.Put(wp)
	err = writer.Close()

	if handler.postResponseCB != nil {
		handler.postResponseCB(method, resp)
	}

	return err
}

func getServiceMethod(operation string) (string, string, bool) {
	s := string(operation)
	sep := strings.Index(s, "::")
	if sep == -1 {
		return "", "", false
	}
	return s[:sep], s[sep+2:], true
}

// Handle handles an incoming TChannel call and forwards it to the correct handler.
func (s *Server) Handle(ctx context.Context, call *tchannel.InboundCall) {
	op := call.OperationString()
	service, method, ok := getServiceMethod(op)
	if !ok {
		log.Fatalf("Handle got call for %s which does not match the expected call format", op)
	}

	s.mut.RLock()
	handler, ok := s.handlers[service]
	s.mut.RUnlock()
	if !ok {
		log.Fatalf("Handle got call for service %v which is not registered", service)
	}

	if err := s.handle(ctx, handler, method, call); err != nil {
		s.onError(err)
	}
}
<commit_msg>thrift.Server shouldn't log an error on timeouts<commit_after>// Copyright (c) 2015 Uber Technologies, Inc.

// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package thrift

import (
	"log"
	"strings"
	"sync"

	"github.com/apache/thrift/lib/go/thrift"
	tchannel "github.com/uber/tchannel-go"
	"golang.org/x/net/context"
)

type handler struct {
	server         TChanServer
	postResponseCB PostResponseCB
}

// Server handles incoming TChannel calls and forwards them to the matching TChanServer.
type Server struct {
	ch          tchannel.Registrar
	log         tchannel.Logger
	mut         sync.RWMutex
	handlers    map[string]handler
	metaHandler *metaHandler
}

// NewServer returns a server that can serve thrift services over TChannel.
func NewServer(registrar tchannel.Registrar) *Server {
	metaHandler := newMetaHandler()
	server := &Server{
		ch:          registrar,
		log:         registrar.Logger(),
		handlers:    make(map[string]handler),
		metaHandler: metaHandler,
	}
	server.Register(newTChanMetaServer(metaHandler))
	if ch, ok := registrar.(*tchannel.Channel); ok {
		// Register the meta endpoints on the "tchannel" service name.
		NewServer(ch.GetSubChannel("tchannel"))
	}
	return server
}

// Register registers the given TChanServer to be called on any incoming call for its' services.
// TODO(prashant): Replace Register call with this call.
func (s *Server) Register(svr TChanServer, opts ...RegisterOption) {
	service := svr.Service()
	handler := &handler{server: svr}
	for _, opt := range opts {
		opt.Apply(handler)
	}

	s.mut.Lock()
	s.handlers[service] = *handler
	s.mut.Unlock()

	for _, m := range svr.Methods() {
		s.ch.Register(s, service+"::"+m)
	}
}

// RegisterHealthHandler uses the user-specified function f for the Health endpoint.
func (s *Server) RegisterHealthHandler(f HealthFunc) {
	s.metaHandler.setHandler(f)
}

func (s *Server) onError(err error) {
	// TODO(prashant): Expose incoming call errors through options for NewServer.
	// Timeouts should not be reported as errors.
	if se, ok := err.(tchannel.SystemError); ok && se.Code() == tchannel.ErrCodeTimeout {
		s.log.Debugf("thrift Server timeout: %v", err)
	} else {
		s.log.Errorf("thrift Server error: %v", err)
	}
}

func (s *Server) handle(origCtx context.Context, handler handler, method string, call *tchannel.InboundCall) error {
	reader, err := call.Arg2Reader()
	if err != nil {
		return err
	}
	headers, err := readHeaders(reader)
	if err != nil {
		return err
	}
	if err := reader.Close(); err != nil {
		return err
	}

	reader, err = call.Arg3Reader()
	if err != nil {
		return err
	}

	ctx := WithHeaders(origCtx, headers)

	wp := getProtocolReader(reader)
	success, resp, err := handler.server.Handle(ctx, method, wp.protocol)
	thriftProtocolPool.Put(wp)

	if err != nil {
		if _, ok := err.(thrift.TProtocolException); ok {
			// We failed to parse the Thrift generated code, so convert the error to bad request.
			err = tchannel.NewSystemError(tchannel.ErrCodeBadRequest, err.Error())
		}

		reader.Close()
		call.Response().SendSystemError(err)
		return nil
	}
	if err := reader.Close(); err != nil {
		return err
	}

	if !success {
		call.Response().SetApplicationError()
	}

	writer, err := call.Response().Arg2Writer()
	if err != nil {
		return err
	}

	if err := writeHeaders(writer, ctx.ResponseHeaders()); err != nil {
		return err
	}
	if err := writer.Close(); err != nil {
		return err
	}

	writer, err = call.Response().Arg3Writer()
	wp = getProtocolWriter(writer)
	resp.Write(wp.protocol)
	thriftProtocolPool.Put(wp)
	err = writer.Close()

	if handler.postResponseCB != nil {
		handler.postResponseCB(method, resp)
	}

	return err
}

func getServiceMethod(operation string) (string, string, bool) {
	s := string(operation)
	sep := strings.Index(s, "::")
	if sep == -1 {
		return "", "", false
	}
	return s[:sep], s[sep+2:], true
}

// Handle handles an incoming TChannel call and forwards it to the correct handler.
func (s *Server) Handle(ctx context.Context, call *tchannel.InboundCall) {
	op := call.OperationString()
	service, method, ok := getServiceMethod(op)
	if !ok {
		log.Fatalf("Handle got call for %s which does not match the expected call format", op)
	}

	s.mut.RLock()
	handler, ok := s.handlers[service]
	s.mut.RUnlock()
	if !ok {
		log.Fatalf("Handle got call for service %v which is not registered", service)
	}

	if err := s.handle(ctx, handler, method, call); err != nil {
		s.onError(err)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"koding/kite"
	"koding/kite/protocol"
)

type Sample struct{}

func main() {
	o := &protocol.Options{
		Username: "huseyin",
		Kitename: "fs-local",
		Version:  "1",
		Port:     "4005",
	}

	methods := map[string]string{
		"sample.hello": "Hello",
	}

	k := kite.New(o)
	k.AddMethods(new(Sample), methods)
	k.Start()
}

func (s Sample) Hello(r *protocol.KiteDnodeRequest, result *string) error {
	var params struct {
		Name string
	}
	if r.Args.Unmarshal(&params) != nil || params.Name == "" {
		return errors.New("{ name: [string] }")
	}

	*result = "Hello " + params.Name
	return nil
}
<commit_msg>remove unused file<commit_after><|endoftext|>


<commit_before>package dita

import (
	"encoding/json"
	"fmt"
	"log"
	"path/filepath"

	"github.com/bradfitz/slice"
	"github.com/raintreeinc/ditaconvert"
	"github.com/raintreeinc/knowledgebase/kb"
	"github.com/raintreeinc/knowledgebase/kb/items/index"
)

type Conversion struct {
	Group   kb.Slug
	Ditamap string

	Pages map[kb.Slug]*kb.Page
	Raw   map[kb.Slug][]byte
	Slugs []kb.Slug
	Nav   *index.Item

	LoadErrors    []error
	MappingErrors []error
	Errors        []ConversionError
}

func NewConversion(group kb.Slug, ditamap string) *Conversion {
	return &Conversion{
		Group:   group,
		Ditamap: ditamap,
		Pages:   make(map[kb.Slug]*kb.Page),
		Raw:     make(map[kb.Slug][]byte),
	}
}

type ConversionError struct {
	Path   string
	Slug   kb.Slug
	Fatal  error
	Errors []error
}

const maxPageSize = 1048575

func (context *Conversion) Run() {
	fs := ditaconvert.Dir(filepath.Dir(context.Ditamap))
	index := ditaconvert.NewIndex(fs)
	index.LoadMap(filepath.Base(context.Ditamap))

	context.LoadErrors = index.Errors

	mapping, mappingErrors := RemapTitles(context, index)
	context.MappingErrors = mappingErrors

	for slug, topic := range mapping.BySlug {
		page, errs, fatal := (&PageConversion{
			Conversion: context,
			Mapping:    mapping,
			Slug:       slug,
			Index:      index,
			Topic:      topic,
		}).Convert()

		if fatal != nil {
			context.Errors = append(context.Errors, ConversionError{
				Path:  topic.Path,
				Slug:  slug,
				Fatal: fatal,
			})
		} else if len(errs) > 0 {
			context.Errors = append(context.Errors, ConversionError{
				Path:   topic.Path,
				Slug:   slug,
				Errors: errs,
			})
		}

		data, err := json.Marshal(page)
		if err != nil {
			log.Println(err)
		}

		if len(data) > maxPageSize {
			context.Errors = append(context.Errors, ConversionError{
				Path:  topic.Path,
				Slug:  slug,
				Fatal: fmt.Errorf("Page is too large %vMB (%vB)", len(data)>>20, len(data)),
			})
		}

		context.Pages[slug] = page
		context.Raw[slug] = data
		context.Slugs = append(context.Slugs, slug)
	}

	slice.Sort(context.Slugs, func(i, j int) bool {
		return context.Slugs[i] < context.Slugs[j]
	})

	context.Nav = mapping.EntryToIndexItem(index.Nav)
}
<commit_msg>Show warning for pages larger than 0.25kb<commit_after>package dita

import (
	"encoding/json"
	"fmt"
	"path/filepath"

	"github.com/bradfitz/slice"
	"github.com/raintreeinc/ditaconvert"
	"github.com/raintreeinc/knowledgebase/kb"
	"github.com/raintreeinc/knowledgebase/kb/items/index"
)

type Conversion struct {
	Group   kb.Slug
	Ditamap string

	Pages map[kb.Slug]*kb.Page
	Raw   map[kb.Slug][]byte
	Slugs []kb.Slug
	Nav   *index.Item

	LoadErrors    []error
	MappingErrors []error
	Errors        []ConversionError
}

func NewConversion(group kb.Slug, ditamap string) *Conversion {
	return &Conversion{
		Group:   group,
		Ditamap: ditamap,
		Pages:   make(map[kb.Slug]*kb.Page),
		Raw:     make(map[kb.Slug][]byte),
	}
}

type ConversionError struct {
	Path   string
	Slug   kb.Slug
	Fatal  error
	Errors []error
}

const (
	maxPageSize         = 1 << 20 // 1MB
	recommendedPageSize = 1 << 18 // 0.25MB
)

func (context *Conversion) Run() {
	fs := ditaconvert.Dir(filepath.Dir(context.Ditamap))
	index := ditaconvert.NewIndex(fs)
	index.LoadMap(filepath.Base(context.Ditamap))

	context.LoadErrors = index.Errors

	mapping, mappingErrors := RemapTitles(context, index)
	context.MappingErrors = mappingErrors

	for slug, topic := range mapping.BySlug {
		page, errs, fatal := (&PageConversion{
			Conversion: context,
			Mapping:    mapping,
			Slug:       slug,
			Index:      index,
			Topic:      topic,
		}).Convert()

		if fatal != nil {
			context.Errors = append(context.Errors, ConversionError{
				Path:  topic.Path,
				Slug:  slug,
				Fatal: fatal,
			})
		} else if len(errs) > 0 {
			context.Errors = append(context.Errors, ConversionError{
				Path:   topic.Path,
				Slug:   slug,
				Errors: errs,
			})
		}

		data, err := json.Marshal(page)
		if err != nil {
			context.Errors = append(context.Errors, ConversionError{
				Path:  topic.Path,
				Slug:  slug,
				Fatal: fmt.Errorf("Marshaling page failed"),
			})
			continue
		}

		if len(data) > maxPageSize {
			context.Errors = append(context.Errors, ConversionError{
				Path:  topic.Path,
				Slug:  slug,
				Fatal: fmt.Errorf("Page is too large %.3fMB (%v bytes)", float64(len(data))/(1<<20), len(data)),
			})
			continue
		}

		if len(data) > recommendedPageSize {
			context.Errors = append(context.Errors, ConversionError{
				Path: topic.Path,
				Slug: slug,
				Errors: []error{
					fmt.Errorf("Page should be smaller %.3fMB (%v bytes)", float64(len(data))/(1<<20), len(data)),
				},
			})
		}

		context.Pages[slug] = page
		context.Raw[slug] = data
		context.Slugs = append(context.Slugs, slug)
	}

	slice.Sort(context.Slugs, func(i, j int) bool {
		return context.Slugs[i] < context.Slugs[j]
	})

	context.Nav = mapping.EntryToIndexItem(index.Nav)
}
<|endoftext|>


<commit_before>package nosurf

import (
	"crypto/sha256"
	"encoding/base64"
	"testing"
)

func TestGeneratesAValidToken(t *testing.T) {
	// We can't test much with any certainity here,
	// since we generate hashes from random numbers.

	// Basically we check that the length of the
	// encoded / decoded token is what it should be

	token := generateToken()
	l := len(token)

	if l != tokenLength {
		t.Errorf("Bad token length: expected %d, got %d", l, tokenLength)
	}

	hash, err := base64.StdEncoding.DecodeString(token)
	if err != nil {
		t.Fatal(err)
	}

	l = len(hash)
	if l != sha256.Size {
		t.Errorf("Bad decoded token length: expected %d, got %d", l, sha256.Size)
	}
}
<commit_msg>Oops, a mistake. Of course we *expected* sha256.Size<commit_after>package nosurf

import (
	"crypto/sha256"
	"encoding/base64"
	"testing"
)

func TestGeneratesAValidToken(t *testing.T) {
	// We can't test much with any certainity here,
	// since we generate hashes from random numbers.

	// Basically we check that the length of the
	// encoded / decoded token is what it should be

	token := generateToken()
	l := len(token)

	if l != tokenLength {
		t.Errorf("Bad token length: expected %d, got %d", l, tokenLength)
	}

	hash, err := base64.StdEncoding.DecodeString(token)
	if err != nil {
		t.Fatal(err)
	}

	l = len(hash)
	if l != sha256.Size {
		t.Errorf("Bad decoded token length: expected %d, got %d", sha256.Size, l)
	}
}
<|endoftext|>


<commit_before>package doc

import (
	"errors"
	"github.com/russross/blackfriday"
	"strings"
)

type Format string

const (
	Text             Format = "text"
	Markdown         Format = "markdown"
	ReStructuredText Format = "rst"
)

// ToHTML converts a source document in format to an HTML string. If conversion
// fails, it returns a failsafe plaintext-to-HTML conversion and a non-nil error.
func ToHTML(format Format, source string) (html string, err error) {
	switch format {
	case Markdown:
		var out []byte
		out = blackfriday.MarkdownCommon([]byte(source))
		html = string(out)
	case ReStructuredText:
		html, err = ReStructuredTextToHTML(source)
	case Text:
	default:
		err = ErrUnhandledFormat
	}
	if err != nil || html == "" {
		html = "
" + strings.TrimSpace(source) + "
"
	}
	return
}

var ErrUnhandledFormat = errors.New("unhandled doc format")
<commit_msg>Escape HTML in docstrings<commit_after>package doc

import (
	"errors"
	"github.com/russross/blackfriday"
	"html"
	"strings"
)

type Format string

const (
	Text             Format = "text"
	Markdown         Format = "markdown"
	ReStructuredText Format = "rst"
)

// ToHTML converts a source document in format to an HTML string. If conversion
// fails, it returns a failsafe plaintext-to-HTML conversion and a non-nil error.
func ToHTML(format Format, source string) (htmlSource string, err error) {
	switch format {
	case Markdown:
		var out []byte
		out = blackfriday.MarkdownCommon([]byte(source))
		htmlSource = string(out)
	case ReStructuredText:
		htmlSource, err = ReStructuredTextToHTML(source)
	case Text:
	default:
		err = ErrUnhandledFormat
	}
	if err != nil || htmlSource == "" {
		htmlSource = "
" + strings.TrimSpace(html.EscapeString(source)) + "
"
	}
	return
}

var ErrUnhandledFormat = errors.New("unhandled doc format")
<|endoftext|>


<commit_before>/*
   Nging is a toolbox for webmasters
   Copyright (C) 2018-present  Wenhui Shen <swh@admpub.com>

   This program is free software: you can redistribute it and/or modify
   it under the terms of the GNU Affero General Public License as published
   by the Free Software Foundation, either version 3 of the License, or
   (at your option) any later version.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU Affero General Public License for more details.

   You should have received a copy of the GNU Affero General Public License
   along with this program.  If not, see <https://www.gnu.org/licenses/>.
*/
package model

import (
	"strings"

	"github.com/webx-top/db"
	"github.com/webx-top/echo"
	"github.com/webx-top/echo/code"

	"github.com/admpub/nging/application/dbschema"
	"github.com/admpub/nging/application/library/perm"
	"github.com/admpub/nging/application/model/base"
	"github.com/admpub/nging/application/registry/navigate"
)

func NewUserRole(ctx echo.Context) *UserRole {
	return &UserRole{
		NgingUserRole: &dbschema.NgingUserRole{},
		base:          base.New(ctx),
	}
}

type UserRole struct {
	*dbschema.NgingUserRole
	base        *base.Base
	permActions *perm.Map
	permCmds    *perm.Map
}

func (u *UserRole) check() error {
	if len(u.Name) == 0 {
		return u.base.NewError(code.InvalidParameter, `角色名不能为空`)
	}
	var exists bool
	var err error
	if u.Id > 0 {
		exists, err = u.Exists2(u.Name, u.Id)
	} else {
		exists, err = u.Exists(u.Name)
	}
	if err != nil {
		return err
	}
	if exists {
		err = u.base.NewError(code.DataAlreadyExists, `角色名已经存在`)
	}
	return err
}

func (u *UserRole) Add() (interface{}, error) {
	if err := u.check(); err != nil {
		return nil, err
	}
	return u.NgingUserRole.Add()
}

func (u *UserRole) Edit(mw func(db.Result) db.Result, args ...interface{}) error {
	if err := u.check(); err != nil {
		return err
	}
	return u.NgingUserRole.Edit(mw, args...)
}

func (u *UserRole) Exists(name string) (bool, error) {
	return u.NgingUserRole.Exists(nil, db.Cond{`name`: name})
}

func (u *UserRole) ListByUser(user *dbschema.NgingUser) (roleList []*dbschema.NgingUserRole) {
	if len(user.RoleIds) > 0 {
		u.ListByOffset(nil, nil, 0, -1, db.And(
			db.Cond{`disabled`: `N`},
			db.Cond{`id`: db.In(strings.Split(user.RoleIds, `,`))},
		))
	}
	roleList = u.Objects()
	return
}

func (u *UserRole) CheckPerm2(roleList []*dbschema.NgingUserRole, perm string) (hasPerm bool) {
	perm = strings.TrimPrefix(perm, `/`)
	for _, role := range roleList {
		r := NewUserRole(nil)
		r.NgingUserRole = role
		if r.CheckPerm(perm) {
			hasPerm = true
			break
		}
	}
	return
}

func (u *UserRole) Exists2(name string, excludeID uint) (bool, error) {
	return u.NgingUserRole.Exists(nil, db.And(
		db.Cond{`name`: name},
		db.Cond{`id`: db.NotEq(excludeID)},
	))
}

func (u *UserRole) CleanPermAction(values []string) *UserRole {
	u.PermAction = perm.BuildPermActions(values)
	return u
}

func (u *UserRole) CheckPerm(permPath string) bool {
	if len(u.PermAction) == 0 {
		return false
	}
	if u.PermAction == `*` {
		return true
	}
	navTree := perm.NavTreeCached()
	if u.permActions == nil {
		u.permActions = perm.NewMap()
		u.permActions.Parse(u.PermAction, navTree)
	}
	return u.permActions.Check(permPath, navTree)
}

func (u *UserRole) CheckCmdPerm2(roleList []*dbschema.NgingUserRole, perm string) (hasPerm bool) {
	for _, role := range roleList {
		r := NewUserRole(nil)
		r.NgingUserRole = role
		if r.CheckCmdPerm(perm) {
			hasPerm = true
			break
		}
	}
	return
}

func (u *UserRole) CheckCmdPerm(permPath string) bool {
	if len(u.PermCmd) == 0 {
		return false
	}
	if u.PermCmd == `*` {
		return true
	}
	if u.permCmds == nil {
		u.permCmds = perm.NewMap()
		perms := strings.Split(u.PermCmd, `,`)
		for _, _perm := range perms {
			arr := strings.Split(_perm, `/`)
			result := u.permCmds.V
			for _, a := range arr {
				if _, y := result[a]; !y {
					result[a] = perm.NewMap()
				}
				result = result[a].V
			}
		}
	}

	arr := strings.Split(permPath, `/`)
	result := u.permCmds.V
	for _, a := range arr {
		v, y := result[a]
		if !y {
			return false
		}
		result = v.V
	}
	return true
}

//FilterNavigate 过滤导航菜单，只显示有权限的菜单
func (u *UserRole) FilterNavigate(roleList []*dbschema.NgingUserRole, navList *navigate.List) navigate.List {
	var result navigate.List
	if navList == nil {
		return result
	}
	for _, nav := range *navList {
		if !nav.Unlimited && !u.CheckPerm2(roleList, nav.Action) {
			continue
		}
		navCopy := *nav
		navCopy.Children = &navigate.List{}
		for _, child := range *nav.Children {
			var perm string
			if len(child.Action) > 0 {
				perm = nav.Action + `/` + child.Action
			} else {
				perm = nav.Action
			}
			if !nav.Unlimited && !u.CheckPerm2(roleList, perm) {
				continue
			}
			*navCopy.Children = append(*navCopy.Children, child)
		}
		result = append(result, &navCopy)
	}
	return result
}
<commit_msg>update<commit_after>/*
   Nging is a toolbox for webmasters
   Copyright (C) 2018-present  Wenhui Shen <swh@admpub.com>

   This program is free software: you can redistribute it and/or modify
   it under the terms of the GNU Affero General Public License as published
   by the Free Software Foundation, either version 3 of the License, or
   (at your option) any later version.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU Affero General Public License for more details.

   You should have received a copy of the GNU Affero General Public License
   along with this program.  If not, see <https://www.gnu.org/licenses/>.
*/
package model

import (
	"strings"

	"github.com/webx-top/db"
	"github.com/webx-top/echo"
	"github.com/webx-top/echo/code"

	"github.com/admpub/nging/application/dbschema"
	"github.com/admpub/nging/application/library/perm"
	"github.com/admpub/nging/application/model/base"
	"github.com/admpub/nging/application/registry/navigate"
)

func NewUserRole(ctx echo.Context) *UserRole {
	return &UserRole{
		NgingUserRole: &dbschema.NgingUserRole{},
		base:          base.New(ctx),
	}
}

type UserRole struct {
	*dbschema.NgingUserRole
	base        *base.Base
	permActions *perm.Map
	permCmds    *perm.Map
}

func (u *UserRole) check() error {
	if len(u.Name) == 0 {
		return u.base.NewError(code.InvalidParameter, `角色名不能为空`)
	}
	var exists bool
	var err error
	if u.Id > 0 {
		exists, err = u.Exists2(u.Name, u.Id)
	} else {
		exists, err = u.Exists(u.Name)
	}
	if err != nil {
		return err
	}
	if exists {
		err = u.base.NewError(code.DataAlreadyExists, `角色名已经存在`)
	}
	return err
}

func (u *UserRole) Add() (interface{}, error) {
	if err := u.check(); err != nil {
		return nil, err
	}
	return u.NgingUserRole.Add()
}

func (u *UserRole) Edit(mw func(db.Result) db.Result, args ...interface{}) error {
	if err := u.check(); err != nil {
		return err
	}
	return u.NgingUserRole.Edit(mw, args...)
}

func (u *UserRole) Exists(name string) (bool, error) {
	return u.NgingUserRole.Exists(nil, db.Cond{`name`: name})
}

func (u *UserRole) ListByUser(user *dbschema.NgingUser) (roleList []*dbschema.NgingUserRole) {
	if len(user.RoleIds) > 0 {
		u.ListByOffset(nil, nil, 0, -1, db.And(
			db.Cond{`disabled`: `N`},
			db.Cond{`id`: db.In(strings.Split(user.RoleIds, `,`))},
		))
	}
	roleList = u.Objects()
	return
}

func (u *UserRole) CheckPerm2(roleList []*dbschema.NgingUserRole, permPath string) bool {
	permPath = strings.TrimPrefix(permPath, `/`)
	if len(roleList) == 0 {
		return perm.NavTreeCached().Check(permPath, nil)
	}
	checked := map[string]struct{}{}
	var perms, sep string
	for _, role := range roleList {
		if len(role.PermAction) == 0 {
			continue
		}
		if role.PermAction == `*` {
			return true
		}
		for _, pa := range strings.Split(role.PermAction, `,`) {
			if _, ok := checked[pa]; !ok {
				checked[pa] = struct{}{}
				perms += sep + pa
				sep = `,`
			}
		}
	}
	navTree := perm.NavTreeCached()
	if u.permActions == nil {
		u.permActions = perm.NewMap()
		u.permActions.Parse(perms, navTree)
	}
	return u.permActions.Check(permPath, navTree)
}

func (u *UserRole) Exists2(name string, excludeID uint) (bool, error) {
	return u.NgingUserRole.Exists(nil, db.And(
		db.Cond{`name`: name},
		db.Cond{`id`: db.NotEq(excludeID)},
	))
}

func (u *UserRole) CleanPermAction(values []string) *UserRole {
	u.PermAction = perm.BuildPermActions(values)
	return u
}

func (u *UserRole) CheckPerm(permPath string) bool {
	if len(u.PermAction) == 0 {
		return false
	}
	if u.PermAction == `*` {
		return true
	}
	navTree := perm.NavTreeCached()
	if u.permActions == nil {
		u.permActions = perm.NewMap()
		u.permActions.Parse(u.PermAction, navTree)
	}
	return u.permActions.Check(permPath, navTree)
}

func (u *UserRole) CheckCmdPerm2(roleList []*dbschema.NgingUserRole, perm string) (hasPerm bool) {
	for _, role := range roleList {
		r := NewUserRole(nil)
		r.NgingUserRole = role
		if r.CheckCmdPerm(perm) {
			hasPerm = true
			break
		}
	}
	return
}

func (u *UserRole) CheckCmdPerm(permPath string) bool {
	if len(u.PermCmd) == 0 {
		return false
	}
	if u.PermCmd == `*` {
		return true
	}
	if u.permCmds == nil {
		u.permCmds = perm.NewMap()
		perms := strings.Split(u.PermCmd, `,`)
		for _, _perm := range perms {
			arr := strings.Split(_perm, `/`)
			result := u.permCmds.V
			for _, a := range arr {
				if _, y := result[a]; !y {
					result[a] = perm.NewMap()
				}
				result = result[a].V
			}
		}
	}

	arr := strings.Split(permPath, `/`)
	result := u.permCmds.V
	for _, a := range arr {
		v, y := result[a]
		if !y {
			return false
		}
		result = v.V
	}
	return true
}

//FilterNavigate 过滤导航菜单，只显示有权限的菜单
func (u *UserRole) FilterNavigate(roleList []*dbschema.NgingUserRole, navList *navigate.List) navigate.List {
	var result navigate.List
	if navList == nil {
		return result
	}
	for _, nav := range *navList {
		if !nav.Unlimited && !u.CheckPerm2(roleList, nav.Action) {
			continue
		}
		navCopy := *nav
		navCopy.Children = &navigate.List{}
		for _, child := range *nav.Children {
			var perm string
			if len(child.Action) > 0 {
				perm = nav.Action + `/` + child.Action
			} else {
				perm = nav.Action
			}
			if !nav.Unlimited && !u.CheckPerm2(roleList, perm) {
				continue
			}
			*navCopy.Children = append(*navCopy.Children, child)
		}
		result = append(result, &navCopy)
	}
	return result
}
<|endoftext|>


<commit_before>// Copyright 2016 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package oauth

import (
	"encoding/json"
	"io/ioutil"
	"net/http"
	"strconv"

	"github.com/pkg/errors"
	"github.com/tsuru/config"
	"github.com/tsuru/tsuru/auth"
	"github.com/tsuru/tsuru/auth/native"
	tsuruErrors "github.com/tsuru/tsuru/errors"
	"github.com/tsuru/tsuru/log"
	"golang.org/x/net/context"
	"golang.org/x/oauth2"
)

var (
	ErrMissingCodeError       = &tsuruErrors.ValidationError{Message: "You must provide code to login"}
	ErrMissingCodeRedirectUrl = &tsuruErrors.ValidationError{Message: "You must provide the used redirect url to login"}
	ErrEmptyAccessToken       = &tsuruErrors.NotAuthorizedError{Message: "Couldn't convert code to access token."}
	ErrEmptyUserEmail         = &tsuruErrors.NotAuthorizedError{Message: "Couldn't parse user email."}
)

type OAuthParser interface {
	Parse(infoResponse *http.Response) (string, error)
}

type OAuthScheme struct {
	BaseConfig   oauth2.Config
	InfoUrl      string
	CallbackPort int
	Parser       OAuthParser
}

func init() {
	auth.RegisterScheme("oauth", &OAuthScheme{})
}

// This method loads basic config and returns a copy of the
// config object.
func (s *OAuthScheme) loadConfig() (oauth2.Config, error) {
	if s.BaseConfig.ClientID != "" {
		return s.BaseConfig, nil
	}
	if s.Parser == nil {
		s.Parser = s
	}
	var emptyConfig oauth2.Config
	clientId, err := config.GetString("auth:oauth:client-id")
	if err != nil {
		return emptyConfig, err
	}
	clientSecret, err := config.GetString("auth:oauth:client-secret")
	if err != nil {
		return emptyConfig, err
	}
	scope, err := config.GetString("auth:oauth:scope")
	if err != nil {
		return emptyConfig, err
	}
	authURL, err := config.GetString("auth:oauth:auth-url")
	if err != nil {
		return emptyConfig, err
	}
	tokenURL, err := config.GetString("auth:oauth:token-url")
	if err != nil {
		return emptyConfig, err
	}
	infoURL, err := config.GetString("auth:oauth:info-url")
	if err != nil {
		return emptyConfig, err
	}
	callbackPort, err := config.GetInt("auth:oauth:callback-port")
	if err != nil {
		log.Debugf("auth:oauth:callback-port not found using random port: %s", err)
	}
	s.InfoUrl = infoURL
	s.CallbackPort = callbackPort
	s.BaseConfig = oauth2.Config{
		ClientID:     clientId,
		ClientSecret: clientSecret,
		Scopes:       []string{scope},
		Endpoint: oauth2.Endpoint{
			AuthURL:  authURL,
			TokenURL: tokenURL,
		},
	}
	return s.BaseConfig, nil
}

func (s *OAuthScheme) Login(params map[string]string) (auth.Token, error) {
	conf, err := s.loadConfig()
	if err != nil {
		return nil, err
	}
	code, ok := params["code"]
	if !ok {
		return nil, ErrMissingCodeError
	}
	redirectUrl, ok := params["redirectUrl"]
	if !ok {
		return nil, ErrMissingCodeRedirectUrl
	}
	conf.RedirectURL = redirectUrl
	oauthToken, err := conf.Exchange(context.Background(), code)
	if err != nil {
		return nil, err
	}
	return s.handleToken(oauthToken)
}

func (s *OAuthScheme) handleToken(t *oauth2.Token) (*Token, error) {
	if t.AccessToken == "" {
		return nil, ErrEmptyAccessToken
	}
	conf, err := s.loadConfig()
	if err != nil {
		return nil, err
	}
	client := conf.Client(context.Background(), t)
	response, err := client.Get(s.InfoUrl)
	if err != nil {
		return nil, err
	}
	defer response.Body.Close()
	email, err := s.Parser.Parse(response)
	if err != nil {
		return nil, err
	}
	if email == "" {
		return nil, ErrEmptyUserEmail
	}
	user, err := auth.GetUserByEmail(email)
	if err != nil {
		if err != auth.ErrUserNotFound {
			return nil, err
		}
		registrationEnabled, _ := config.GetBool("auth:user-registration")
		if !registrationEnabled {
			return nil, err
		}
		user = &auth.User{Email: email}
		err = user.Create()
		if err != nil {
			return nil, err
		}
	}
	token := Token{*t, email}
	err = token.save()
	if err != nil {
		return nil, err
	}
	return &token, nil
}

func (s *OAuthScheme) AppLogin(appName string) (auth.Token, error) {
	nativeScheme := native.NativeScheme{}
	return nativeScheme.AppLogin(appName)
}

func (s *OAuthScheme) AppLogout(token string) error {
	nativeScheme := native.NativeScheme{}
	return nativeScheme.AppLogout(token)
}

func (s *OAuthScheme) Logout(token string) error {
	return deleteToken(token)
}

func (s *OAuthScheme) Auth(header string) (auth.Token, error) {
	token, err := getToken(header)
	if err != nil {
		nativeScheme := native.NativeScheme{}
		token, nativeErr := nativeScheme.Auth(header)
		if nativeErr == nil && token.IsAppToken() {
			return token, nil
		}
		return nil, err
	}
	config, err := s.loadConfig()
	if err != nil {
		return nil, err
	}
	client := config.Client(context.Background(), &token.Token)
	rsp, err := client.Get(s.InfoUrl)
	if err != nil {
		return nil, err
	}
	defer rsp.Body.Close()
	return token, nil
}

func (s *OAuthScheme) Name() string {
	return "oauth"
}

func (s *OAuthScheme) Info() (auth.SchemeInfo, error) {
	config, err := s.loadConfig()
	if err != nil {
		return nil, err
	}
	config.RedirectURL = "__redirect_url__"
	return auth.SchemeInfo{"authorizeUrl": config.AuthCodeURL(""), "port": strconv.Itoa(s.CallbackPort)}, nil
}

func (s *OAuthScheme) Parse(infoResponse *http.Response) (string, error) {
	user := struct {
		Email string `json:"email"`
	}{}
	data, err := ioutil.ReadAll(infoResponse.Body)
	if err != nil {
		return "", errors.Wrap(err, "unable to read user data response")
	}
	if infoResponse.StatusCode != http.StatusOK {
		return "", errors.Errorf("unexpected user data response %d: %s", infoResponse.StatusCode, data)
	}
	err = json.Unmarshal(data, &user)
	if err != nil {
		return "", errors.Wrapf(err, "unable to parse user data: %s", data)
	}
	return user.Email, nil
}

func (s *OAuthScheme) Create(user *auth.User) (*auth.User, error) {
	user.Password = ""
	err := user.Create()
	if err != nil {
		return nil, err
	}
	return user, nil
}

func (s *OAuthScheme) Remove(u *auth.User) error {
	err := deleteAllTokens(u.Email)
	if err != nil {
		return err
	}
	return u.Delete()
}
<commit_msg>auth/oauth: add oath service metrics<commit_after>// Copyright 2017 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package oauth

import (
	"encoding/json"
	"io/ioutil"
	"net/http"
	"strconv"
	"time"

	"github.com/pkg/errors"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/tsuru/config"
	"github.com/tsuru/tsuru/auth"
	"github.com/tsuru/tsuru/auth/native"
	tsuruErrors "github.com/tsuru/tsuru/errors"
	"github.com/tsuru/tsuru/log"
	"golang.org/x/net/context"
	"golang.org/x/oauth2"
)

var (
	ErrMissingCodeError       = &tsuruErrors.ValidationError{Message: "You must provide code to login"}
	ErrMissingCodeRedirectUrl = &tsuruErrors.ValidationError{Message: "You must provide the used redirect url to login"}
	ErrEmptyAccessToken       = &tsuruErrors.NotAuthorizedError{Message: "Couldn't convert code to access token."}
	ErrEmptyUserEmail         = &tsuruErrors.NotAuthorizedError{Message: "Couldn't parse user email."}

	requestLatencies = prometheus.NewHistogram(prometheus.HistogramOpts{
		Name: "tsuru_oath_request_duration_seconds",
		Help: "The oath requests latency distributions.",
	})
	requestErrors = prometheus.NewCounter(prometheus.CounterOpts{
		Name: "tsuru_oath_request_errors_total",
		Help: "The total number of oath request errors.",
	})
)

type OAuthParser interface {
	Parse(infoResponse *http.Response) (string, error)
}

type OAuthScheme struct {
	BaseConfig   oauth2.Config
	InfoUrl      string
	CallbackPort int
	Parser       OAuthParser
}

func init() {
	auth.RegisterScheme("oauth", &OAuthScheme{})
	prometheus.MustRegister(requestLatencies)
	prometheus.MustRegister(requestErrors)
}

// This method loads basic config and returns a copy of the
// config object.
func (s *OAuthScheme) loadConfig() (oauth2.Config, error) {
	if s.BaseConfig.ClientID != "" {
		return s.BaseConfig, nil
	}
	if s.Parser == nil {
		s.Parser = s
	}
	var emptyConfig oauth2.Config
	clientId, err := config.GetString("auth:oauth:client-id")
	if err != nil {
		return emptyConfig, err
	}
	clientSecret, err := config.GetString("auth:oauth:client-secret")
	if err != nil {
		return emptyConfig, err
	}
	scope, err := config.GetString("auth:oauth:scope")
	if err != nil {
		return emptyConfig, err
	}
	authURL, err := config.GetString("auth:oauth:auth-url")
	if err != nil {
		return emptyConfig, err
	}
	tokenURL, err := config.GetString("auth:oauth:token-url")
	if err != nil {
		return emptyConfig, err
	}
	infoURL, err := config.GetString("auth:oauth:info-url")
	if err != nil {
		return emptyConfig, err
	}
	callbackPort, err := config.GetInt("auth:oauth:callback-port")
	if err != nil {
		log.Debugf("auth:oauth:callback-port not found using random port: %s", err)
	}
	s.InfoUrl = infoURL
	s.CallbackPort = callbackPort
	s.BaseConfig = oauth2.Config{
		ClientID:     clientId,
		ClientSecret: clientSecret,
		Scopes:       []string{scope},
		Endpoint: oauth2.Endpoint{
			AuthURL:  authURL,
			TokenURL: tokenURL,
		},
	}
	return s.BaseConfig, nil
}

func (s *OAuthScheme) Login(params map[string]string) (auth.Token, error) {
	conf, err := s.loadConfig()
	if err != nil {
		return nil, err
	}
	code, ok := params["code"]
	if !ok {
		return nil, ErrMissingCodeError
	}
	redirectUrl, ok := params["redirectUrl"]
	if !ok {
		return nil, ErrMissingCodeRedirectUrl
	}
	conf.RedirectURL = redirectUrl
	oauthToken, err := conf.Exchange(context.Background(), code)
	if err != nil {
		return nil, err
	}
	return s.handleToken(oauthToken)
}

func (s *OAuthScheme) handleToken(t *oauth2.Token) (*Token, error) {
	if t.AccessToken == "" {
		return nil, ErrEmptyAccessToken
	}
	conf, err := s.loadConfig()
	if err != nil {
		return nil, err
	}
	client := conf.Client(context.Background(), t)
	t0 := time.Now()
	response, err := client.Get(s.InfoUrl)
	requestLatencies.Observe(float64(time.Since(t0).Seconds()))
	if err != nil {
		requestErrors.Inc()
		return nil, err
	}
	defer response.Body.Close()
	email, err := s.Parser.Parse(response)
	if err != nil {
		return nil, err
	}
	if email == "" {
		return nil, ErrEmptyUserEmail
	}
	user, err := auth.GetUserByEmail(email)
	if err != nil {
		if err != auth.ErrUserNotFound {
			return nil, err
		}
		registrationEnabled, _ := config.GetBool("auth:user-registration")
		if !registrationEnabled {
			return nil, err
		}
		user = &auth.User{Email: email}
		err = user.Create()
		if err != nil {
			return nil, err
		}
	}
	token := Token{*t, email}
	err = token.save()
	if err != nil {
		return nil, err
	}
	return &token, nil
}

func (s *OAuthScheme) AppLogin(appName string) (auth.Token, error) {
	nativeScheme := native.NativeScheme{}
	return nativeScheme.AppLogin(appName)
}

func (s *OAuthScheme) AppLogout(token string) error {
	nativeScheme := native.NativeScheme{}
	return nativeScheme.AppLogout(token)
}

func (s *OAuthScheme) Logout(token string) error {
	return deleteToken(token)
}

func (s *OAuthScheme) Auth(header string) (auth.Token, error) {
	token, err := getToken(header)
	if err != nil {
		nativeScheme := native.NativeScheme{}
		token, nativeErr := nativeScheme.Auth(header)
		if nativeErr == nil && token.IsAppToken() {
			return token, nil
		}
		return nil, err
	}
	config, err := s.loadConfig()
	if err != nil {
		return nil, err
	}
	client := config.Client(context.Background(), &token.Token)
	t0 := time.Now()
	rsp, err := client.Get(s.InfoUrl)
	requestLatencies.Observe(float64(time.Since(t0).Seconds()))
	if err != nil {
		requestErrors.Inc()
		return nil, err
	}
	defer rsp.Body.Close()
	return token, nil
}

func (s *OAuthScheme) Name() string {
	return "oauth"
}

func (s *OAuthScheme) Info() (auth.SchemeInfo, error) {
	config, err := s.loadConfig()
	if err != nil {
		return nil, err
	}
	config.RedirectURL = "__redirect_url__"
	return auth.SchemeInfo{"authorizeUrl": config.AuthCodeURL(""), "port": strconv.Itoa(s.CallbackPort)}, nil
}

func (s *OAuthScheme) Parse(infoResponse *http.Response) (string, error) {
	user := struct {
		Email string `json:"email"`
	}{}
	data, err := ioutil.ReadAll(infoResponse.Body)
	if err != nil {
		return "", errors.Wrap(err, "unable to read user data response")
	}
	if infoResponse.StatusCode != http.StatusOK {
		return "", errors.Errorf("unexpected user data response %d: %s", infoResponse.StatusCode, data)
	}
	err = json.Unmarshal(data, &user)
	if err != nil {
		return "", errors.Wrapf(err, "unable to parse user data: %s", data)
	}
	return user.Email, nil
}

func (s *OAuthScheme) Create(user *auth.User) (*auth.User, error) {
	user.Password = ""
	err := user.Create()
	if err != nil {
		return nil, err
	}
	return user, nil
}

func (s *OAuthScheme) Remove(u *auth.User) error {
	err := deleteAllTokens(u.Email)
	if err != nil {
		return err
	}
	return u.Delete()
}
<|endoftext|>