<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package time_test

import (
	"errors"
	"fmt"
	"runtime"
	"sort"
	"sync"
	"sync/atomic"
	"testing"
	. "time"
)

func TestSleep(t *testing.T) {
	const delay = 100 * Millisecond
	go func() {
		Sleep(delay / 2)
		Interrupt()
	}()
	start := Now()
	Sleep(delay)
	duration := Now().Sub(start)
	if duration < delay {
		t.Fatalf("Sleep(%s) slept for only %s", delay, duration)
	}
}

// Test the basic function calling behavior. Correct queueing
// behavior is tested elsewhere, since After and AfterFunc share
// the same code.
func TestAfterFunc(t *testing.T) {
	i := 10
	c := make(chan bool)
	var f func()
	f = func() {
		i--
		if i >= 0 {
			AfterFunc(0, f)
			Sleep(1 * Second)
		} else {
			c <- true
		}
	}

	AfterFunc(0, f)
	<-c
}

func TestAfterStress(t *testing.T) {
	stop := uint32(0)
	go func() {
		for atomic.LoadUint32(&stop) == 0 {
			runtime.GC()
			// Yield so that the OS can wake up the timer thread,
			// so that it can generate channel sends for the main goroutine,
			// which will eventually set stop = 1 for us.
			Sleep(Nanosecond)
		}
	}()
	ticker := NewTicker(1)
	for i := 0; i < 100; i++ {
		<-ticker.C
	}
	ticker.Stop()
	atomic.StoreUint32(&stop, 1)
}

func benchmark(b *testing.B, bench func(n int)) {
	garbage := make([]*Timer, 1<<17)
	for i := 0; i < len(garbage); i++ {
		garbage[i] = AfterFunc(Hour, nil)
	}

	const batch = 1000
	P := runtime.GOMAXPROCS(-1)
	N := int32(b.N / batch)

	b.ResetTimer()

	var wg sync.WaitGroup
	wg.Add(P)

	for p := 0; p < P; p++ {
		go func() {
			for atomic.AddInt32(&N, -1) >= 0 {
				bench(batch)
			}
			wg.Done()
		}()
	}

	wg.Wait()

	b.StopTimer()
	for i := 0; i < len(garbage); i++ {
		garbage[i].Stop()
	}
}

func BenchmarkAfterFunc(b *testing.B) {
	benchmark(b, func(n int) {
		c := make(chan bool)
		var f func()
		f = func() {
			n--
			if n >= 0 {
				AfterFunc(0, f)
			} else {
				c <- true
			}
		}

		AfterFunc(0, f)
		<-c
	})
}

func BenchmarkAfter(b *testing.B) {
	benchmark(b, func(n int) {
		for i := 0; i < n; i++ {
			<-After(1)
		}
	})
}

func BenchmarkStop(b *testing.B) {
	benchmark(b, func(n int) {
		for i := 0; i < n; i++ {
			NewTimer(1 * Second).Stop()
		}
	})
}

func BenchmarkSimultaneousAfterFunc(b *testing.B) {
	benchmark(b, func(n int) {
		var wg sync.WaitGroup
		wg.Add(n)
		for i := 0; i < n; i++ {
			AfterFunc(0, wg.Done)
		}
		wg.Wait()
	})
}

func BenchmarkStartStop(b *testing.B) {
	benchmark(b, func(n int) {
		timers := make([]*Timer, n)
		for i := 0; i < n; i++ {
			timers[i] = AfterFunc(Hour, nil)
		}

		for i := 0; i < n; i++ {
			timers[i].Stop()
		}
	})
}

func TestAfter(t *testing.T) {
	const delay = 100 * Millisecond
	start := Now()
	end := <-After(delay)
	if duration := Now().Sub(start); duration < delay {
		t.Fatalf("After(%s) slept for only %d ns", delay, duration)
	}
	if min := start.Add(delay); end.Before(min) {
		t.Fatalf("After(%s) expect >= %s, got %s", delay, min, end)
	}
}

func TestAfterTick(t *testing.T) {
	const Count = 10
	Delta := 100 * Millisecond
	if testing.Short() {
		Delta = 10 * Millisecond
	}
	t0 := Now()
	for i := 0; i < Count; i++ {
		<-After(Delta)
	}
	t1 := Now()
	d := t1.Sub(t0)
	target := Delta * Count
	if d < target*9/10 {
		t.Fatalf("%d ticks of %s too fast: took %s, expected %s", Count, Delta, d, target)
	}
	if !testing.Short() && d > target*30/10 {
		t.Fatalf("%d ticks of %s too slow: took %s, expected %s", Count, Delta, d, target)
	}
}

func TestAfterStop(t *testing.T) {
	AfterFunc(100*Millisecond, func() {})
	t0 := NewTimer(50 * Millisecond)
	c1 := make(chan bool, 1)
	t1 := AfterFunc(150*Millisecond, func() { c1 <- true })
	c2 := After(200 * Millisecond)
	if !t0.Stop() {
		t.Fatalf("failed to stop event 0")
	}
	if !t1.Stop() {
		t.Fatalf("failed to stop event 1")
	}
	<-c2
	select {
	case <-t0.C:
		t.Fatalf("event 0 was not stopped")
	case <-c1:
		t.Fatalf("event 1 was not stopped")
	default:
	}
	if t1.Stop() {
		t.Fatalf("Stop returned true twice")
	}
}

func TestAfterQueuing(t *testing.T) {
	// This test flakes out on some systems,
	// so we'll try it a few times before declaring it a failure.
	const attempts = 3
	err := errors.New("!=nil")
	for i := 0; i < attempts && err != nil; i++ {
		if err = testAfterQueuing(t); err != nil {
			t.Logf("attempt %v failed: %v", i, err)
		}
	}
	if err != nil {
		t.Fatal(err)
	}
}

var slots = []int{5, 3, 6, 6, 6, 1, 1, 2, 7, 9, 4, 8, 0}

type afterResult struct {
	slot int
	t    Time
}

func await(slot int, result chan<- afterResult, ac <-chan Time) {
	result <- afterResult{slot, <-ac}
}

func testAfterQueuing(t *testing.T) error {
	Delta := 100 * Millisecond
	if testing.Short() {
		Delta = 20 * Millisecond
	}
	// make the result channel buffered because we don't want
	// to depend on channel queueing semantics that might
	// possibly change in the future.
	result := make(chan afterResult, len(slots))

	t0 := Now()
	for _, slot := range slots {
		go await(slot, result, After(Duration(slot)*Delta))
	}
	sort.Ints(slots)
	for _, slot := range slots {
		r := <-result
		if r.slot != slot {
			return fmt.Errorf("after slot %d, expected %d", r.slot, slot)
		}
		dt := r.t.Sub(t0)
		target := Duration(slot) * Delta
		if dt < target-Delta/2 || dt > target+Delta*10 {
			return fmt.Errorf("After(%s) arrived at %s, expected [%s,%s]", target, dt, target-Delta/2, target+Delta*10)
		}
	}
	return nil
}

func TestTimerStopStress(t *testing.T) {
	if testing.Short() {
		return
	}
	for i := 0; i < 100; i++ {
		go func(i int) {
			timer := AfterFunc(2*Second, func() {
				t.Fatalf("timer %d was not stopped", i)
			})
			Sleep(1 * Second)
			timer.Stop()
		}(i)
	}
	Sleep(3 * Second)
}

func TestSleepZeroDeadlock(t *testing.T) {
	// Sleep(0) used to hang, the sequence of events was as follows.
	// Sleep(0) sets G's status to Gwaiting, but then immediately returns leaving the status.
	// Then the goroutine calls e.g. new and falls down into the scheduler due to pending GC.
	// After the GC nobody wakes up the goroutine from Gwaiting status.
	defer runtime.GOMAXPROCS(runtime.GOMAXPROCS(4))
	c := make(chan bool)
	go func() {
		for i := 0; i < 100; i++ {
			runtime.GC()
		}
		c <- true
	}()
	for i := 0; i < 100; i++ {
		Sleep(0)
		tmp := make(chan bool, 1)
		tmp <- true
		<-tmp
	}
	<-c
}

func testReset(d Duration) error {
	t0 := NewTimer(2 * d)
	Sleep(d)
	if t0.Reset(3*d) != true {
		return errors.New("resetting unfired timer returned false")
	}
	Sleep(2 * d)
	select {
	case <-t0.C:
		return errors.New("timer fired early")
	default:
	}
	Sleep(2 * d)
	select {
	case <-t0.C:
	default:
		return errors.New("reset timer did not fire")
	}

	if t0.Reset(50*Millisecond) != false {
		return errors.New("resetting expired timer returned true")
	}
	return nil
}

func TestReset(t *testing.T) {
	// We try to run this test with increasingly larger multiples
	// until one works so slow, loaded hardware isn't as flaky,
	// but without slowing down fast machines unnecessarily.
	const unit = 25 * Millisecond
	tries := []Duration{
		1 * unit,
		3 * unit,
		7 * unit,
		15 * unit,
	}
	var err error
	for _, d := range tries {
		err = testReset(d)
		if err == nil {
			t.Logf("passed using duration %v", d)
			return
		}
	}
	t.Error(err)
}

// Test that sleeping for an interval so large it overflows does not
// result in a short sleep duration.
func TestOverflowSleep(t *testing.T) {
	const timeout = 25 * Millisecond
	const big = Duration(int64(1<<63 - 1))
	select {
	case <-After(big):
		t.Fatalf("big timeout fired")
	case <-After(timeout):
		// OK
	}
	const neg = Duration(-1 << 63)
	select {
	case <-After(neg):
		// OK
	case <-After(timeout):
		t.Fatalf("negative timeout didn't fire")
	}
}

// Test that a panic while deleting a timer does not leave
// the timers mutex held, deadlocking a ticker.Stop in a defer.
func TestIssue5745(t *testing.T) {
	ticker := NewTicker(Hour)
	defer func() {
		// would deadlock here before the fix due to
		// lock taken before the segfault.
		ticker.Stop()

		if r := recover(); r == nil {
			t.Error("Expected panic, but none happened.")
		}
	}()

	// cause a panic due to a segfault
	var timer *Timer
	timer.Stop()
	t.Error("Should be unreachable.")
}

func TestOverflowRuntimeTimer(t *testing.T) {
	if testing.Short() {
		t.Skip("skipping in short mode, see issue 6874")
	}
	if err := CheckRuntimeTimerOverflow(); err != nil {
		t.Fatalf(err.Error())
	}
}
<commit_msg>time: use RunParallel in benchmarks<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package time_test

import (
	"errors"
	"fmt"
	"runtime"
	"sort"
	"sync"
	"sync/atomic"
	"testing"
	. "time"
)

func TestSleep(t *testing.T) {
	const delay = 100 * Millisecond
	go func() {
		Sleep(delay / 2)
		Interrupt()
	}()
	start := Now()
	Sleep(delay)
	duration := Now().Sub(start)
	if duration < delay {
		t.Fatalf("Sleep(%s) slept for only %s", delay, duration)
	}
}

// Test the basic function calling behavior. Correct queueing
// behavior is tested elsewhere, since After and AfterFunc share
// the same code.
func TestAfterFunc(t *testing.T) {
	i := 10
	c := make(chan bool)
	var f func()
	f = func() {
		i--
		if i >= 0 {
			AfterFunc(0, f)
			Sleep(1 * Second)
		} else {
			c <- true
		}
	}

	AfterFunc(0, f)
	<-c
}

func TestAfterStress(t *testing.T) {
	stop := uint32(0)
	go func() {
		for atomic.LoadUint32(&stop) == 0 {
			runtime.GC()
			// Yield so that the OS can wake up the timer thread,
			// so that it can generate channel sends for the main goroutine,
			// which will eventually set stop = 1 for us.
			Sleep(Nanosecond)
		}
	}()
	ticker := NewTicker(1)
	for i := 0; i < 100; i++ {
		<-ticker.C
	}
	ticker.Stop()
	atomic.StoreUint32(&stop, 1)
}

func benchmark(b *testing.B, bench func(n int)) {
	garbage := make([]*Timer, 1<<17)
	for i := 0; i < len(garbage); i++ {
		garbage[i] = AfterFunc(Hour, nil)
	}
	b.ResetTimer()

	b.RunParallel(func(pb *testing.PB) {
		for pb.Next() {
			bench(1000)
		}
	})

	b.StopTimer()
	for i := 0; i < len(garbage); i++ {
		garbage[i].Stop()
	}
}

func BenchmarkAfterFunc(b *testing.B) {
	benchmark(b, func(n int) {
		c := make(chan bool)
		var f func()
		f = func() {
			n--
			if n >= 0 {
				AfterFunc(0, f)
			} else {
				c <- true
			}
		}

		AfterFunc(0, f)
		<-c
	})
}

func BenchmarkAfter(b *testing.B) {
	benchmark(b, func(n int) {
		for i := 0; i < n; i++ {
			<-After(1)
		}
	})
}

func BenchmarkStop(b *testing.B) {
	benchmark(b, func(n int) {
		for i := 0; i < n; i++ {
			NewTimer(1 * Second).Stop()
		}
	})
}

func BenchmarkSimultaneousAfterFunc(b *testing.B) {
	benchmark(b, func(n int) {
		var wg sync.WaitGroup
		wg.Add(n)
		for i := 0; i < n; i++ {
			AfterFunc(0, wg.Done)
		}
		wg.Wait()
	})
}

func BenchmarkStartStop(b *testing.B) {
	benchmark(b, func(n int) {
		timers := make([]*Timer, n)
		for i := 0; i < n; i++ {
			timers[i] = AfterFunc(Hour, nil)
		}

		for i := 0; i < n; i++ {
			timers[i].Stop()
		}
	})
}

func TestAfter(t *testing.T) {
	const delay = 100 * Millisecond
	start := Now()
	end := <-After(delay)
	if duration := Now().Sub(start); duration < delay {
		t.Fatalf("After(%s) slept for only %d ns", delay, duration)
	}
	if min := start.Add(delay); end.Before(min) {
		t.Fatalf("After(%s) expect >= %s, got %s", delay, min, end)
	}
}

func TestAfterTick(t *testing.T) {
	const Count = 10
	Delta := 100 * Millisecond
	if testing.Short() {
		Delta = 10 * Millisecond
	}
	t0 := Now()
	for i := 0; i < Count; i++ {
		<-After(Delta)
	}
	t1 := Now()
	d := t1.Sub(t0)
	target := Delta * Count
	if d < target*9/10 {
		t.Fatalf("%d ticks of %s too fast: took %s, expected %s", Count, Delta, d, target)
	}
	if !testing.Short() && d > target*30/10 {
		t.Fatalf("%d ticks of %s too slow: took %s, expected %s", Count, Delta, d, target)
	}
}

func TestAfterStop(t *testing.T) {
	AfterFunc(100*Millisecond, func() {})
	t0 := NewTimer(50 * Millisecond)
	c1 := make(chan bool, 1)
	t1 := AfterFunc(150*Millisecond, func() { c1 <- true })
	c2 := After(200 * Millisecond)
	if !t0.Stop() {
		t.Fatalf("failed to stop event 0")
	}
	if !t1.Stop() {
		t.Fatalf("failed to stop event 1")
	}
	<-c2
	select {
	case <-t0.C:
		t.Fatalf("event 0 was not stopped")
	case <-c1:
		t.Fatalf("event 1 was not stopped")
	default:
	}
	if t1.Stop() {
		t.Fatalf("Stop returned true twice")
	}
}

func TestAfterQueuing(t *testing.T) {
	// This test flakes out on some systems,
	// so we'll try it a few times before declaring it a failure.
	const attempts = 3
	err := errors.New("!=nil")
	for i := 0; i < attempts && err != nil; i++ {
		if err = testAfterQueuing(t); err != nil {
			t.Logf("attempt %v failed: %v", i, err)
		}
	}
	if err != nil {
		t.Fatal(err)
	}
}

var slots = []int{5, 3, 6, 6, 6, 1, 1, 2, 7, 9, 4, 8, 0}

type afterResult struct {
	slot int
	t    Time
}

func await(slot int, result chan<- afterResult, ac <-chan Time) {
	result <- afterResult{slot, <-ac}
}

func testAfterQueuing(t *testing.T) error {
	Delta := 100 * Millisecond
	if testing.Short() {
		Delta = 20 * Millisecond
	}
	// make the result channel buffered because we don't want
	// to depend on channel queueing semantics that might
	// possibly change in the future.
	result := make(chan afterResult, len(slots))

	t0 := Now()
	for _, slot := range slots {
		go await(slot, result, After(Duration(slot)*Delta))
	}
	sort.Ints(slots)
	for _, slot := range slots {
		r := <-result
		if r.slot != slot {
			return fmt.Errorf("after slot %d, expected %d", r.slot, slot)
		}
		dt := r.t.Sub(t0)
		target := Duration(slot) * Delta
		if dt < target-Delta/2 || dt > target+Delta*10 {
			return fmt.Errorf("After(%s) arrived at %s, expected [%s,%s]", target, dt, target-Delta/2, target+Delta*10)
		}
	}
	return nil
}

func TestTimerStopStress(t *testing.T) {
	if testing.Short() {
		return
	}
	for i := 0; i < 100; i++ {
		go func(i int) {
			timer := AfterFunc(2*Second, func() {
				t.Fatalf("timer %d was not stopped", i)
			})
			Sleep(1 * Second)
			timer.Stop()
		}(i)
	}
	Sleep(3 * Second)
}

func TestSleepZeroDeadlock(t *testing.T) {
	// Sleep(0) used to hang, the sequence of events was as follows.
	// Sleep(0) sets G's status to Gwaiting, but then immediately returns leaving the status.
	// Then the goroutine calls e.g. new and falls down into the scheduler due to pending GC.
	// After the GC nobody wakes up the goroutine from Gwaiting status.
	defer runtime.GOMAXPROCS(runtime.GOMAXPROCS(4))
	c := make(chan bool)
	go func() {
		for i := 0; i < 100; i++ {
			runtime.GC()
		}
		c <- true
	}()
	for i := 0; i < 100; i++ {
		Sleep(0)
		tmp := make(chan bool, 1)
		tmp <- true
		<-tmp
	}
	<-c
}

func testReset(d Duration) error {
	t0 := NewTimer(2 * d)
	Sleep(d)
	if t0.Reset(3*d) != true {
		return errors.New("resetting unfired timer returned false")
	}
	Sleep(2 * d)
	select {
	case <-t0.C:
		return errors.New("timer fired early")
	default:
	}
	Sleep(2 * d)
	select {
	case <-t0.C:
	default:
		return errors.New("reset timer did not fire")
	}

	if t0.Reset(50*Millisecond) != false {
		return errors.New("resetting expired timer returned true")
	}
	return nil
}

func TestReset(t *testing.T) {
	// We try to run this test with increasingly larger multiples
	// until one works so slow, loaded hardware isn't as flaky,
	// but without slowing down fast machines unnecessarily.
	const unit = 25 * Millisecond
	tries := []Duration{
		1 * unit,
		3 * unit,
		7 * unit,
		15 * unit,
	}
	var err error
	for _, d := range tries {
		err = testReset(d)
		if err == nil {
			t.Logf("passed using duration %v", d)
			return
		}
	}
	t.Error(err)
}

// Test that sleeping for an interval so large it overflows does not
// result in a short sleep duration.
func TestOverflowSleep(t *testing.T) {
	const timeout = 25 * Millisecond
	const big = Duration(int64(1<<63 - 1))
	select {
	case <-After(big):
		t.Fatalf("big timeout fired")
	case <-After(timeout):
		// OK
	}
	const neg = Duration(-1 << 63)
	select {
	case <-After(neg):
		// OK
	case <-After(timeout):
		t.Fatalf("negative timeout didn't fire")
	}
}

// Test that a panic while deleting a timer does not leave
// the timers mutex held, deadlocking a ticker.Stop in a defer.
func TestIssue5745(t *testing.T) {
	ticker := NewTicker(Hour)
	defer func() {
		// would deadlock here before the fix due to
		// lock taken before the segfault.
		ticker.Stop()

		if r := recover(); r == nil {
			t.Error("Expected panic, but none happened.")
		}
	}()

	// cause a panic due to a segfault
	var timer *Timer
	timer.Stop()
	t.Error("Should be unreachable.")
}

func TestOverflowRuntimeTimer(t *testing.T) {
	if testing.Short() {
		t.Skip("skipping in short mode, see issue 6874")
	}
	if err := CheckRuntimeTimerOverflow(); err != nil {
		t.Fatalf(err.Error())
	}
}
<|endoftext|>


<commit_before>// Copyright 2020 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package schedulers

import (
	"bytes"
	"encoding/json"
	"io/ioutil"
	"net/http"
	"reflect"
	"strings"
	"sync"
	"time"

	"github.com/gorilla/mux"
	"github.com/pingcap/pd/v4/server/core"
	"github.com/pingcap/pd/v4/server/schedule"
	"github.com/pingcap/pd/v4/server/statistics"
	"github.com/unrolled/render"
)

// params about hot region.
func initHotRegionScheduleConfig() *hotRegionSchedulerConfig {
	return &hotRegionSchedulerConfig{
		MinHotByteRate:        100,
		MinHotKeyRate:         10,
		MaxZombieRounds:       3,
		ByteRateRankStepRatio: 0.05,
		KeyRateRankStepRatio:  0.05,
		CountRankStepRatio:    0.01,
		GreatDecRatio:         0.95,
		MinorDecRatio:         0.99,
		MaxPeerNum:            1000,
		SrcToleranceRatio:     1.02, // Tolerate 2% difference
		DstToleranceRatio:     1.02, // Tolerate 2% difference
	}
}

type hotRegionSchedulerConfig struct {
	sync.RWMutex
	storage *core.Storage

	MinHotByteRate  float64 `json:"min-hot-byte-rate"`
	MinHotKeyRate   float64 `json:"min-hot-key-rate"`
	MaxZombieRounds int     `json:"max-zombie-rounds"`
	MaxPeerNum      int     `json:"max-peer-number"`

	// rank step ratio decide the step when calculate rank
	// step = max current * rank step ratio
	ByteRateRankStepRatio float64 `json:"byte-rate-rank-step-ratio"`
	KeyRateRankStepRatio  float64 `json:"key-rate-rank-step-ratio"`
	CountRankStepRatio    float64 `json:"count-rank-step-ratio"`
	GreatDecRatio         float64 `json:"great-dec-ratio"`
	MinorDecRatio         float64 `json:"minor-dec-ratio"`
	SrcToleranceRatio     float64 `json:"src-tolerance-ratio"`
	DstToleranceRatio     float64 `json:"dst-tolerance-ratio"`
}

func (conf *hotRegionSchedulerConfig) EncodeConfig() ([]byte, error) {
	conf.RLock()
	defer conf.RUnlock()
	return schedule.EncodeConfig(conf)
}

func (conf *hotRegionSchedulerConfig) GetMaxZombieDuration() time.Duration {
	conf.RLock()
	defer conf.RUnlock()
	return time.Duration(conf.MaxZombieRounds) * statistics.StoreHeartBeatReportInterval * time.Second
}

func (conf *hotRegionSchedulerConfig) GetMaxPeerNumber() int {
	conf.RLock()
	defer conf.RUnlock()
	return conf.MaxPeerNum
}

func (conf *hotRegionSchedulerConfig) GetSrcToleranceRatio() float64 {
	conf.RLock()
	defer conf.RUnlock()
	return conf.SrcToleranceRatio
}

func (conf *hotRegionSchedulerConfig) SetSrcToleranceRatio(tol float64) {
	conf.Lock()
	defer conf.Unlock()
	conf.SrcToleranceRatio = tol
}

func (conf *hotRegionSchedulerConfig) GetDstToleranceRatio() float64 {
	conf.RLock()
	defer conf.RUnlock()
	return conf.DstToleranceRatio
}

func (conf *hotRegionSchedulerConfig) SetDstToleranceRatio(tol float64) {
	conf.Lock()
	defer conf.Unlock()
	conf.DstToleranceRatio = tol
}

func (conf *hotRegionSchedulerConfig) GetByteRankStepRatio() float64 {
	conf.RLock()
	defer conf.RUnlock()
	return conf.ByteRateRankStepRatio
}

func (conf *hotRegionSchedulerConfig) GetKeyRankStepRatio() float64 {
	conf.RLock()
	defer conf.RUnlock()
	return conf.KeyRateRankStepRatio
}

func (conf *hotRegionSchedulerConfig) GetCountRankStepRatio() float64 {
	conf.RLock()
	defer conf.RUnlock()
	return conf.CountRankStepRatio
}

func (conf *hotRegionSchedulerConfig) GetGreatDecRatio() float64 {
	conf.RLock()
	defer conf.RUnlock()
	return conf.GreatDecRatio
}

func (conf *hotRegionSchedulerConfig) GetMinorGreatDecRatio() float64 {
	conf.RLock()
	defer conf.RUnlock()
	return conf.MinorDecRatio
}

func (conf *hotRegionSchedulerConfig) GetMinHotKeyRate() float64 {
	conf.RLock()
	defer conf.RUnlock()
	return conf.MinHotKeyRate
}

func (conf *hotRegionSchedulerConfig) GetMinHotByteRate() float64 {
	conf.RLock()
	defer conf.RUnlock()
	return conf.MinHotByteRate
}

func (conf *hotRegionSchedulerConfig) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	router := mux.NewRouter()
	router.HandleFunc("/list", conf.handleGetConfig).Methods("GET")
	router.HandleFunc("/config", conf.handleSetConfig).Methods("POST")
	router.ServeHTTP(w, r)
}

func (conf *hotRegionSchedulerConfig) handleGetConfig(w http.ResponseWriter, r *http.Request) {
	conf.RLock()
	defer conf.RUnlock()
	rd := render.New(render.Options{IndentJSON: true})
	rd.JSON(w, http.StatusOK, conf)
}

func (conf *hotRegionSchedulerConfig) handleSetConfig(w http.ResponseWriter, r *http.Request) {
	conf.Lock()
	defer conf.Unlock()
	rd := render.New(render.Options{IndentJSON: true})
	oldc, _ := json.Marshal(conf)
	data, err := ioutil.ReadAll(r.Body)
	r.Body.Close()
	if err != nil {
		rd.JSON(w, http.StatusInternalServerError, err.Error())
		return
	}

	if err := json.Unmarshal(data, conf); err != nil {
		rd.JSON(w, http.StatusInternalServerError, err.Error())
		return
	}
	newc, _ := json.Marshal(conf)
	if !bytes.Equal(oldc, newc) {
		conf.persist()
		rd.Text(w, http.StatusOK, "success")
	}

	m := make(map[string]interface{})
	if err := json.Unmarshal(data, &m); err != nil {
		rd.JSON(w, http.StatusInternalServerError, err.Error())
		return
	}
	t := reflect.TypeOf(conf).Elem()
	for i := 0; i < t.NumField(); i++ {
		jsonTag := t.Field(i).Tag.Get("json")
		if i := strings.Index(jsonTag, ","); i != -1 { // trim 'foobar,string' to 'foobar'
			jsonTag = jsonTag[:i]
		}
		if _, ok := m[jsonTag]; ok {
			rd.Text(w, http.StatusOK, "no changed")
			return
		}
	}

	rd.Text(w, http.StatusBadRequest, "config item not found")
}

func (conf *hotRegionSchedulerConfig) persist() error {
	data, err := schedule.EncodeConfig(conf)
	if err != nil {
		return err

	}
	return conf.storage.SaveScheduleConfig(HotRegionName, data)

}
<commit_msg>*: turn the default tolerance for hotregion (#2376)<commit_after>// Copyright 2020 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package schedulers

import (
	"bytes"
	"encoding/json"
	"io/ioutil"
	"net/http"
	"reflect"
	"strings"
	"sync"
	"time"

	"github.com/gorilla/mux"
	"github.com/pingcap/pd/v4/server/core"
	"github.com/pingcap/pd/v4/server/schedule"
	"github.com/pingcap/pd/v4/server/statistics"
	"github.com/unrolled/render"
)

// params about hot region.
func initHotRegionScheduleConfig() *hotRegionSchedulerConfig {
	return &hotRegionSchedulerConfig{
		MinHotByteRate:        100,
		MinHotKeyRate:         10,
		MaxZombieRounds:       3,
		ByteRateRankStepRatio: 0.05,
		KeyRateRankStepRatio:  0.05,
		CountRankStepRatio:    0.01,
		GreatDecRatio:         0.95,
		MinorDecRatio:         0.99,
		MaxPeerNum:            1000,
		SrcToleranceRatio:     1.05, // Tolerate 5% difference
		DstToleranceRatio:     1.05, // Tolerate 5% difference
	}
}

type hotRegionSchedulerConfig struct {
	sync.RWMutex
	storage *core.Storage

	MinHotByteRate  float64 `json:"min-hot-byte-rate"`
	MinHotKeyRate   float64 `json:"min-hot-key-rate"`
	MaxZombieRounds int     `json:"max-zombie-rounds"`
	MaxPeerNum      int     `json:"max-peer-number"`

	// rank step ratio decide the step when calculate rank
	// step = max current * rank step ratio
	ByteRateRankStepRatio float64 `json:"byte-rate-rank-step-ratio"`
	KeyRateRankStepRatio  float64 `json:"key-rate-rank-step-ratio"`
	CountRankStepRatio    float64 `json:"count-rank-step-ratio"`
	GreatDecRatio         float64 `json:"great-dec-ratio"`
	MinorDecRatio         float64 `json:"minor-dec-ratio"`
	SrcToleranceRatio     float64 `json:"src-tolerance-ratio"`
	DstToleranceRatio     float64 `json:"dst-tolerance-ratio"`
}

func (conf *hotRegionSchedulerConfig) EncodeConfig() ([]byte, error) {
	conf.RLock()
	defer conf.RUnlock()
	return schedule.EncodeConfig(conf)
}

func (conf *hotRegionSchedulerConfig) GetMaxZombieDuration() time.Duration {
	conf.RLock()
	defer conf.RUnlock()
	return time.Duration(conf.MaxZombieRounds) * statistics.StoreHeartBeatReportInterval * time.Second
}

func (conf *hotRegionSchedulerConfig) GetMaxPeerNumber() int {
	conf.RLock()
	defer conf.RUnlock()
	return conf.MaxPeerNum
}

func (conf *hotRegionSchedulerConfig) GetSrcToleranceRatio() float64 {
	conf.RLock()
	defer conf.RUnlock()
	return conf.SrcToleranceRatio
}

func (conf *hotRegionSchedulerConfig) SetSrcToleranceRatio(tol float64) {
	conf.Lock()
	defer conf.Unlock()
	conf.SrcToleranceRatio = tol
}

func (conf *hotRegionSchedulerConfig) GetDstToleranceRatio() float64 {
	conf.RLock()
	defer conf.RUnlock()
	return conf.DstToleranceRatio
}

func (conf *hotRegionSchedulerConfig) SetDstToleranceRatio(tol float64) {
	conf.Lock()
	defer conf.Unlock()
	conf.DstToleranceRatio = tol
}

func (conf *hotRegionSchedulerConfig) GetByteRankStepRatio() float64 {
	conf.RLock()
	defer conf.RUnlock()
	return conf.ByteRateRankStepRatio
}

func (conf *hotRegionSchedulerConfig) GetKeyRankStepRatio() float64 {
	conf.RLock()
	defer conf.RUnlock()
	return conf.KeyRateRankStepRatio
}

func (conf *hotRegionSchedulerConfig) GetCountRankStepRatio() float64 {
	conf.RLock()
	defer conf.RUnlock()
	return conf.CountRankStepRatio
}

func (conf *hotRegionSchedulerConfig) GetGreatDecRatio() float64 {
	conf.RLock()
	defer conf.RUnlock()
	return conf.GreatDecRatio
}

func (conf *hotRegionSchedulerConfig) GetMinorGreatDecRatio() float64 {
	conf.RLock()
	defer conf.RUnlock()
	return conf.MinorDecRatio
}

func (conf *hotRegionSchedulerConfig) GetMinHotKeyRate() float64 {
	conf.RLock()
	defer conf.RUnlock()
	return conf.MinHotKeyRate
}

func (conf *hotRegionSchedulerConfig) GetMinHotByteRate() float64 {
	conf.RLock()
	defer conf.RUnlock()
	return conf.MinHotByteRate
}

func (conf *hotRegionSchedulerConfig) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	router := mux.NewRouter()
	router.HandleFunc("/list", conf.handleGetConfig).Methods("GET")
	router.HandleFunc("/config", conf.handleSetConfig).Methods("POST")
	router.ServeHTTP(w, r)
}

func (conf *hotRegionSchedulerConfig) handleGetConfig(w http.ResponseWriter, r *http.Request) {
	conf.RLock()
	defer conf.RUnlock()
	rd := render.New(render.Options{IndentJSON: true})
	rd.JSON(w, http.StatusOK, conf)
}

func (conf *hotRegionSchedulerConfig) handleSetConfig(w http.ResponseWriter, r *http.Request) {
	conf.Lock()
	defer conf.Unlock()
	rd := render.New(render.Options{IndentJSON: true})
	oldc, _ := json.Marshal(conf)
	data, err := ioutil.ReadAll(r.Body)
	r.Body.Close()
	if err != nil {
		rd.JSON(w, http.StatusInternalServerError, err.Error())
		return
	}

	if err := json.Unmarshal(data, conf); err != nil {
		rd.JSON(w, http.StatusInternalServerError, err.Error())
		return
	}
	newc, _ := json.Marshal(conf)
	if !bytes.Equal(oldc, newc) {
		conf.persist()
		rd.Text(w, http.StatusOK, "success")
	}

	m := make(map[string]interface{})
	if err := json.Unmarshal(data, &m); err != nil {
		rd.JSON(w, http.StatusInternalServerError, err.Error())
		return
	}
	t := reflect.TypeOf(conf).Elem()
	for i := 0; i < t.NumField(); i++ {
		jsonTag := t.Field(i).Tag.Get("json")
		if i := strings.Index(jsonTag, ","); i != -1 { // trim 'foobar,string' to 'foobar'
			jsonTag = jsonTag[:i]
		}
		if _, ok := m[jsonTag]; ok {
			rd.Text(w, http.StatusOK, "no changed")
			return
		}
	}

	rd.Text(w, http.StatusBadRequest, "config item not found")
}

func (conf *hotRegionSchedulerConfig) persist() error {
	data, err := schedule.EncodeConfig(conf)
	if err != nil {
		return err

	}
	return conf.storage.SaveScheduleConfig(HotRegionName, data)

}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"log"
	"net/http"
	"time"
	"fmt"
	"math"
	"github.com/lox/httpcache"
	"github.com/elazarl/goproxy"
	"sync"
	"io"
	"net/textproto"
)

const (
	ProxyDateHeader = "Proxy-Date"
)

var Writes sync.WaitGroup

func TryCacheResponse(shared bool, cache httpcache.Cache, response *http.Response) (*http.Response) {
	if (response.StatusCode != http.StatusOK || response.StatusCode != http.StatusNotModified){
		return response
	}

	cacheRequest, err := NewCacheRequest(response.Request)
	if err != nil {
		return goproxy.NewResponse(response.Request, goproxy.ContentTypeText, http.StatusInternalServerError, err.Error())
	}

	if !cacheRequest.IsCacheable() {
		log.Printf("request not cacheable")
		return response
	}

	if (response.StatusCode == http.StatusNotModified){
		resource, err := lookup(cache, cacheRequest)
		if err != nil && err != httpcache.ErrNotFoundInCache {
			return goproxy.NewResponse(response.Request, goproxy.ContentTypeText, http.StatusInternalServerError, err.Error())
		} else if err == httpcache.ErrNotFoundInCache {
			return response
		}

		cache.Freshen(resource, cacheRequest.Key.ForMethod("GET").String())
		response := newCachedResponse(shared, response.Request, cacheRequest, resource)

		return response
	}

	if (response.StatusCode == http.StatusOK){
		if err != httpcache.ErrNotFoundInCache {
			return response
		}

		cachedResponse := newCacheAndForwardResponse(shared, cache, response.Request, cacheRequest, response)

		return cachedResponse
	}

	return response
}

func TryServeCachedResponse(shared bool, cache httpcache.Cache, request *http.Request) (*http.Request, *http.Response) {
	cacheRequest, err := NewCacheRequest(request)
	if err != nil {
		return request, goproxy.NewResponse(request, goproxy.ContentTypeText, http.StatusInternalServerError, err.Error())
	}

	if !cacheRequest.IsCacheable() {
		log.Printf("request not cacheable")
		return request, nil
	}

	cacheType := "private"
	if shared {
		cacheType = "shared"
	}

	resource, err := lookup(cache, cacheRequest)
	if err != nil && err != httpcache.ErrNotFoundInCache {
		return request, goproxy.NewResponse(request, goproxy.ContentTypeText, http.StatusInternalServerError, err.Error())
	}

	if err == httpcache.ErrNotFoundInCache {
		log.Printf("%s %s not in %s cache", request.Method, request.URL.String(), cacheType)
		if cacheRequest.CacheControl.Has("only-if-cached") {
			return request, goproxy.NewResponse(request, goproxy.ContentTypeText, http.StatusGatewayTimeout, "key not in cache")
		}

		return request, nil
	}

	log.Printf("%s %s found in %s cache", request.Method, request.URL.String(), cacheType)

	if needsValidation(shared, resource, cacheRequest) {
		if cacheRequest.CacheControl.Has("only-if-cached") {
			return request, goproxy.NewResponse(request, goproxy.ContentTypeText, http.StatusGatewayTimeout, "key not in cache")
		}

		log.Printf("validating cached response")
		return request, nil
	}

	log.Printf("serving from cache")

	response := newCachedResponse(shared, request, cacheRequest, resource)

	return request, response
}

func newCacheAndForwardResponse(shared bool, cache httpcache.Cache, request *http.Request, cacheRequest *cacheRequest, response * http.Response) *http.Response {
	resource := NewForwardOnlyResource(response.StatusCode, response.Body, response.Header)

	age, err := resource.Age()
	if err != nil {
		return response
	}

	contentLength := response.ContentLength

	now := httpcache.Clock()
	keys := []string{cacheRequest.Key.String()}

	if shared {
		resource.RemovePrivateHeaders()
	}

	headers := resource.Header()

	// store a secondary vary version
	if vary := headers.Get("Vary"); vary != "" {
		keys = append(keys, cacheRequest.Key.Vary(vary, cacheRequest.Request).String())
	}

	return response
}

func newCachedResponse(shared bool, request *http.Request, cacheRequest *cacheRequest, resource * httpcache.Resource) *http.Response {
	age, err := resource.Age()
	if err != nil {
		return goproxy.NewResponse(request, goproxy.ContentTypeText, http.StatusInternalServerError, "Error calculating age: "+err.Error())
	}

	contentLength, err := resource.ReadSeekCloser.Seek(0, 2)
	if err != nil {
		return goproxy.NewResponse(request, goproxy.ContentTypeText, http.StatusInternalServerError, "Error getting resource length: "+err.Error())
	}
	resource.ReadSeekCloser.Seek(0, 0)

	statusCode := resource.Status()

	headers := make(http.Header)

	for key, mimeheaders := range resource.Header() {
		for _, header := range mimeheaders {
			textproto.MIMEHeader(headers).Add(key, header)
		}
	}

	// http://httpwg.github.io/specs/rfc7234.html#warn.113
	if age > (time.Hour*24) && resource.HeuristicFreshness() > (time.Hour*24) {
		textproto.MIMEHeader(headers).Add("Warning", `113 - "Heuristic Expiration"`)
	}

	// http://httpwg.github.io/specs/rfc7234.html#warn.110
	freshness, err := freshness(shared, resource, cacheRequest)
	if err != nil || freshness <= 0 {
		textproto.MIMEHeader(headers).Add("Warning", `110 - "Response is Stale"`)
	}

	log.Printf("resource is %s old, updating age from %s", age.String(), headers.Get("Age"))

	textproto.MIMEHeader(headers).Set("Age", fmt.Sprintf("%.f", math.Floor(age.Seconds())))
	textproto.MIMEHeader(headers).Set("Via", resource.Via())

	body := resource.ReadSeekCloser

	response := newResponse(request, headers, body, contentLength, statusCode)

	return response
}

func newResponse(request *http.Request, headers http.Header, body io.ReadCloser, contentLength int64, statusCode int) *http.Response {
	response := &http.Response{}
	response.Request = request
	response.TransferEncoding = request.TransferEncoding
	response.Header = headers
	response.ContentLength = contentLength
	response.StatusCode = statusCode

	if request.Method == "HEAD" || response.StatusCode != http.StatusOK {
		response.Body = &BufferReadCloser{bytes.NewBufferString("")}
	} else {
		response.Body =	body
	}

	return response
}

func lookup(cache httpcache.Cache, cacheRequest *cacheRequest) (*httpcache.Resource, error) {
	cacheKey := cacheRequest.Key.String()
	resource, err := cache.Retrieve(cacheKey)

	// HEAD requests can possibly be served from GET
	if err == httpcache.ErrNotFoundInCache && cacheRequest.Method == "HEAD" {
		resource, err = cache.Retrieve(cacheRequest.Key.ForMethod("GET").String())
		if err != nil {
			return nil, err
		}

		if resource.HasExplicitExpiration() && cacheRequest.IsCacheable() {
			return resource, nil
		} else {
			return nil, httpcache.ErrNotFoundInCache
		}
	} else if err != nil {
		return resource, err
	}

	// Secondary lookup for Vary
	if vary := resource.Header().Get("Vary"); vary != "" {
		resource, err = cache.Retrieve(cacheRequest.Key.Vary(vary, cacheRequest.Request).String())
		if err != nil {
			return resource, err
		}
	}

	return resource, nil
}

func needsValidation(shared bool, resource * httpcache.Resource, cacheRequest *cacheRequest) bool {
	if resource.MustValidate(shared) {
		return true
	}

	freshness, err := freshness(shared, resource, cacheRequest)
	if err != nil {
		log.Printf("error calculating freshness: %s", err.Error())
		return true
	}

	if cacheRequest.CacheControl.Has("min-fresh") {
		reqMinFresh, err := cacheRequest.CacheControl.Duration("min-fresh")
		if err != nil {
			log.Printf("error parsing request min-fresh: %s", err.Error())
			return true
		}

		if freshness < reqMinFresh {
			log.Printf("resource is fresh, but won't satisfy min-fresh of %s", reqMinFresh)
			return true
		}
	}

	log.Printf("resource has a freshness of %s", freshness)

	if freshness <= 0 && cacheRequest.CacheControl.Has("max-stale") {
		if len(cacheRequest.CacheControl["max-stale"]) == 0 {
			log.Printf("resource is stale, but client sent max-stale")
			return false
		} else if maxStale, _ := cacheRequest.CacheControl.Duration("max-stale"); maxStale >= (freshness * -1) {
			log.Printf("resource is stale, but within allowed max-stale period of %s", maxStale)
			return false
		}
	}

	return freshness <= 0
}

// freshness returns the duration that a requested resource will be fresh for
func freshness(shared bool, resource * httpcache.Resource, cacheRequest *cacheRequest) (time.Duration, error) {
	maxAge, err := resource.MaxAge(shared)
	if err != nil {
		return time.Duration(0), err
	}

	if cacheRequest.CacheControl.Has("max-age") {
		reqMaxAge, err := cacheRequest.CacheControl.Duration("max-age")
		if err != nil {
			return time.Duration(0), err
		}

		if reqMaxAge < maxAge {
			log.Printf("using request max-age of %s", reqMaxAge.String())
			maxAge = reqMaxAge
		}
	}

	age, err := resource.Age()
	if err != nil {
		return time.Duration(0), err
	}

	if resource.IsStale() {
		return time.Duration(0), nil
	}

	if hFresh := resource.HeuristicFreshness(); hFresh > maxAge {
		log.Printf("using heuristic freshness of %q", hFresh)
		maxAge = hFresh
	}

	return maxAge - age, nil
}<commit_msg>Shelf check in<commit_after>package main

import (
	"bytes"
	"log"
	"net/http"
	"time"
	"fmt"
	"math"
	"github.com/lox/httpcache"
	"github.com/elazarl/goproxy"
	"sync"
	"io"
	"net/textproto"
)

const (
	ProxyDateHeader = "Proxy-Date"
)

var Writes sync.WaitGroup

func TryCacheResponse(shared bool, cache httpcache.Cache, response *http.Response) (*http.Response) {
	if (response.StatusCode != http.StatusOK || response.StatusCode != http.StatusNotModified){
		return response
	}

	cacheRequest, err := NewCacheRequest(response.Request)
	if err != nil {
		return goproxy.NewResponse(response.Request, goproxy.ContentTypeText, http.StatusInternalServerError, err.Error())
	}

	if !cacheRequest.IsCacheable() {
		log.Printf("request not cacheable")
		return response
	}

	if (response.StatusCode == http.StatusNotModified){
		resource, err := lookup(cache, cacheRequest)
		if err != nil && err != httpcache.ErrNotFoundInCache {
			return goproxy.NewResponse(response.Request, goproxy.ContentTypeText, http.StatusInternalServerError, err.Error())
		} else if err == httpcache.ErrNotFoundInCache {
			return response
		}

		cache.Freshen(resource, cacheRequest.Key.ForMethod("GET").String())
		response := newCachedResponse(shared, response.Request, cacheRequest, resource)

		return response
	}

	if (response.StatusCode == http.StatusOK){
		if err != httpcache.ErrNotFoundInCache {
			return response
		}

		cachedResponse := newCacheAndForwardResponse(shared, cache, cacheRequest, response)

		return cachedResponse
	}

	return response
}

func newCacheAndForwardResponse(shared bool, cache httpcache.Cache, cacheRequest *cacheRequest, response * http.Response) *http.Response {
	resource := NewForwardOnlyResource(response.StatusCode, response.Body, response.Header)

	age, err := resource.Age()
	if err != nil {
		return response
	}

	contentLength := response.ContentLength
	statusCode := resource.Status()

	if shared {
		resource.RemovePrivateHeaders()
	}

	headers := make(http.Header)

	for key, mimeheaders := range resource.Header() {
		for _, header := range mimeheaders {
			textproto.MIMEHeader(headers).Add(key, header)
		}
	}

	// http://httpwg.github.io/specs/rfc7234.html#warn.113
	if age > (time.Hour*24) && resource.HeuristicFreshness() > (time.Hour*24) {
		textproto.MIMEHeader(headers).Add("Warning", `113 - "Heuristic Expiration"`)
	}

	// http://httpwg.github.io/specs/rfc7234.html#warn.110
	freshness, err := freshness(shared, resource, cacheRequest)
	if err != nil || freshness <= 0 {
		textproto.MIMEHeader(headers).Add("Warning", `110 - "Response is Stale"`)
	}

	log.Printf("resource is %s old, updating age from %s", age.String(), headers.Get("Age"))

	textproto.MIMEHeader(headers).Set("Age", fmt.Sprintf("%.f", math.Floor(age.Seconds())))
	textproto.MIMEHeader(headers).Set("Via", resource.Via())

	body := resource.ReadCloser

	/////////////////////

	keys := []string{cacheRequest.Key.String()}

	// store a secondary vary version
	if vary := headers.Get("Vary"); vary != "" {
		keys = append(keys, cacheRequest.Key.Vary(vary, cacheRequest.Request).String())
	}

	if err := cache.Store(res, keys...); err != nil {
		log.Printf("storing resources %#v failed with error: %s", keys, err.Error())
	}

	/////////////////////

	forwardedResponse := newResponse(response.Request, headers, body, contentLength, statusCode)

	return forwardedResponse
}

func TryServeCachedResponse(shared bool, cache httpcache.Cache, request *http.Request) (*http.Request, *http.Response) {
	cacheRequest, err := NewCacheRequest(request)
	if err != nil {
		return request, goproxy.NewResponse(request, goproxy.ContentTypeText, http.StatusInternalServerError, err.Error())
	}

	if !cacheRequest.IsCacheable() {
		log.Printf("request not cacheable")
		return request, nil
	}

	cacheType := "private"
	if shared {
		cacheType = "shared"
	}

	resource, err := lookup(cache, cacheRequest)
	if err != nil && err != httpcache.ErrNotFoundInCache {
		return request, goproxy.NewResponse(request, goproxy.ContentTypeText, http.StatusInternalServerError, err.Error())
	}

	if err == httpcache.ErrNotFoundInCache {
		log.Printf("%s %s not in %s cache", request.Method, request.URL.String(), cacheType)
		if cacheRequest.CacheControl.Has("only-if-cached") {
			return request, goproxy.NewResponse(request, goproxy.ContentTypeText, http.StatusGatewayTimeout, "key not in cache")
		}

		return request, nil
	}

	log.Printf("%s %s found in %s cache", request.Method, request.URL.String(), cacheType)

	if needsValidation(shared, resource, cacheRequest) {
		if cacheRequest.CacheControl.Has("only-if-cached") {
			return request, goproxy.NewResponse(request, goproxy.ContentTypeText, http.StatusGatewayTimeout, "key not in cache")
		}

		log.Printf("validating cached response")
		return request, nil
	}

	log.Printf("serving from cache")

	response := newCachedResponse(shared, request, cacheRequest, resource)

	return request, response
}

func newCachedResponse(shared bool, request *http.Request, cacheRequest *cacheRequest, resource * httpcache.Resource) *http.Response {
	age, err := resource.Age()
	if err != nil {
		return goproxy.NewResponse(request, goproxy.ContentTypeText, http.StatusInternalServerError, "Error calculating age: "+err.Error())
	}

	contentLength, err := resource.ReadSeekCloser.Seek(0, 2)
	if err != nil {
		return goproxy.NewResponse(request, goproxy.ContentTypeText, http.StatusInternalServerError, "Error getting resource length: "+err.Error())
	}
	resource.ReadSeekCloser.Seek(0, 0)

	statusCode := resource.Status()

	headers := make(http.Header)

	for key, mimeheaders := range resource.Header() {
		for _, header := range mimeheaders {
			textproto.MIMEHeader(headers).Add(key, header)
		}
	}

	// http://httpwg.github.io/specs/rfc7234.html#warn.113
	if age > (time.Hour*24) && resource.HeuristicFreshness() > (time.Hour*24) {
		textproto.MIMEHeader(headers).Add("Warning", `113 - "Heuristic Expiration"`)
	}

	// http://httpwg.github.io/specs/rfc7234.html#warn.110
	freshness, err := freshness(shared, resource, cacheRequest)
	if err != nil || freshness <= 0 {
		textproto.MIMEHeader(headers).Add("Warning", `110 - "Response is Stale"`)
	}

	log.Printf("resource is %s old, updating age from %s", age.String(), headers.Get("Age"))

	textproto.MIMEHeader(headers).Set("Age", fmt.Sprintf("%.f", math.Floor(age.Seconds())))
	textproto.MIMEHeader(headers).Set("Via", resource.Via())

	body := resource.ReadSeekCloser

	response := newResponse(request, headers, body, contentLength, statusCode)

	return response
}

func newResponse(request *http.Request, headers http.Header, body io.ReadCloser, contentLength int64, statusCode int) *http.Response {
	response := &http.Response{}
	response.Request = request
	response.TransferEncoding = request.TransferEncoding
	response.Header = headers
	response.ContentLength = contentLength
	response.StatusCode = statusCode

	if request.Method == "HEAD" || response.StatusCode != http.StatusOK {
		body.Close()
		response.Body = &BufferReadCloser{bytes.NewBufferString("")}
	} else {
		response.Body =	body
	}

	return response
}

func lookup(cache httpcache.Cache, cacheRequest *cacheRequest) (*httpcache.Resource, error) {
	cacheKey := cacheRequest.Key.String()
	resource, err := cache.Retrieve(cacheKey)

	// HEAD requests can possibly be served from GET
	if err == httpcache.ErrNotFoundInCache && cacheRequest.Method == "HEAD" {
		resource, err = cache.Retrieve(cacheRequest.Key.ForMethod("GET").String())
		if err != nil {
			return nil, err
		}

		if resource.HasExplicitExpiration() && cacheRequest.IsCacheable() {
			return resource, nil
		} else {
			return nil, httpcache.ErrNotFoundInCache
		}
	} else if err != nil {
		return resource, err
	}

	// Secondary lookup for Vary
	if vary := resource.Header().Get("Vary"); vary != "" {
		resource, err = cache.Retrieve(cacheRequest.Key.Vary(vary, cacheRequest.Request).String())
		if err != nil {
			return resource, err
		}
	}

	return resource, nil
}

func needsValidation(shared bool, resource * httpcache.Resource, cacheRequest *cacheRequest) bool {
	if resource.MustValidate(shared) {
		return true
	}

	freshness, err := freshness(shared, resource, cacheRequest)
	if err != nil {
		log.Printf("error calculating freshness: %s", err.Error())
		return true
	}

	if cacheRequest.CacheControl.Has("min-fresh") {
		reqMinFresh, err := cacheRequest.CacheControl.Duration("min-fresh")
		if err != nil {
			log.Printf("error parsing request min-fresh: %s", err.Error())
			return true
		}

		if freshness < reqMinFresh {
			log.Printf("resource is fresh, but won't satisfy min-fresh of %s", reqMinFresh)
			return true
		}
	}

	log.Printf("resource has a freshness of %s", freshness)

	if freshness <= 0 && cacheRequest.CacheControl.Has("max-stale") {
		if len(cacheRequest.CacheControl["max-stale"]) == 0 {
			log.Printf("resource is stale, but client sent max-stale")
			return false
		} else if maxStale, _ := cacheRequest.CacheControl.Duration("max-stale"); maxStale >= (freshness * -1) {
			log.Printf("resource is stale, but within allowed max-stale period of %s", maxStale)
			return false
		}
	}

	return freshness <= 0
}

// freshness returns the duration that a requested resource will be fresh for
func freshness(shared bool, resource *ForwardOnlyResource, cacheRequest *cacheRequest) (time.Duration, error) {
	maxAge, err := resource.MaxAge(shared)
	if err != nil {
		return time.Duration(0), err
	}

	if cacheRequest.CacheControl.Has("max-age") {
		reqMaxAge, err := cacheRequest.CacheControl.Duration("max-age")
		if err != nil {
			return time.Duration(0), err
		}

		if reqMaxAge < maxAge {
			log.Printf("using request max-age of %s", reqMaxAge.String())
			maxAge = reqMaxAge
		}
	}

	age, err := resource.Age()
	if err != nil {
		return time.Duration(0), err
	}

	if resource.IsStale() {
		return time.Duration(0), nil
	}

	if hFresh := resource.HeuristicFreshness(); hFresh > maxAge {
		log.Printf("using heuristic freshness of %q", hFresh)
		maxAge = hFresh
	}

	return maxAge - age, nil
}

func freshness(shared bool, resource * httpcache.Resource, cacheRequest *cacheRequest) (time.Duration, error) {
	maxAge, err := resource.MaxAge(shared)
	if err != nil {
		return time.Duration(0), err
	}

	if cacheRequest.CacheControl.Has("max-age") {
		reqMaxAge, err := cacheRequest.CacheControl.Duration("max-age")
		if err != nil {
			return time.Duration(0), err
		}

		if reqMaxAge < maxAge {
			log.Printf("using request max-age of %s", reqMaxAge.String())
			maxAge = reqMaxAge
		}
	}

	age, err := resource.Age()
	if err != nil {
		return time.Duration(0), err
	}

	if resource.IsStale() {
		return time.Duration(0), nil
	}

	if hFresh := resource.HeuristicFreshness(); hFresh > maxAge {
		log.Printf("using heuristic freshness of %q", hFresh)
		maxAge = hFresh
	}

	return maxAge - age, nil
}<|endoftext|>


<commit_before>// Copyright 2017 The Upspin Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package cacheutil provides a mechanism to start the cacheserver
// if a config requires it and it is not already running.
// It is used by programs like upspin and upspinfs.
package cacheutil // import "upspin.io/cmd/cacheserver/cacheutil"

import (
	"flag"
	"fmt"
	"os"
	"os/exec"
	"time"

	"upspin.io/bind"
	"upspin.io/flags"
	"upspin.io/log"
	"upspin.io/rpc"
	"upspin.io/upspin"
)

var (
	writethrough = flag.Bool("writethrough", false, "make storage cache writethrough")
	cacheSize    = flag.Int64("cachesize", 5e9, "max disk `bytes` for cache")
)

// detach detaches a process from the parent process group,
// on platforms that support it.
var detach = func(*exec.Cmd) {}

// Start starts the cacheserver if the config requires it and it is not already running.
func Start(cfg upspin.Config) {
	if cfg == nil {
		return
	}
	ce, err := rpc.CacheEndpoint(cfg)
	if err != nil || ce == nil {
		// TODO(adg): log error message?
		return // not using a cache server
	}

	// Ping the cache server.
	if err := ping(cfg, ce); err == nil {
		return // cache server running
	}

	// Start a cache server.
	cacheErrorChan := make(chan bool)
	go func() {
		cmd := exec.Command(
			"cacheserver",
			"-cachedir="+flags.CacheDir,
			"-log="+log.GetLevel(),
			fmt.Sprintf("-writethrough=%v", *writethrough),
			fmt.Sprintf("-cachesize=%d", *cacheSize),
			"-config="+flags.Config,
			"-addr="+flags.NetAddr)
		detach(cmd)
		cmd.Stdout = os.Stdout
		cmd.Stderr = os.Stderr
		if err := cmd.Run(); err != nil {
			log.Info.Printf("Starting cacheserver: %s", err)
			fmt.Fprintf(os.Stderr, "Failed to start cacheserver; continuing without.\n")
			close(cacheErrorChan)
		}
	}()

	// Wait for it. Give up and continue without if it doesn't start in a timely fashion.
	for tries := 0; tries < 10; tries++ {
		time.Sleep(500 * time.Millisecond)
		select {
		case <-cacheErrorChan:
			return
		default:
		}
		if err := ping(cfg, ce); err == nil {
			return
		}
	}

	fmt.Fprintf(os.Stderr, "Timed out waiting for cacheserver to start.\n")
}

// ping determines if the cacheserver is functioning.
func ping(cfg upspin.Config, ce *upspin.Endpoint) error {
	store, err := bind.StoreServer(cfg, *ce)
	if err != nil {
		return err
	}
	msg, _, _, err := store.Get(upspin.HealthMetadata)
	if err == nil {
		log.Debug.Printf("Cacheserver said %q", string(msg))
	}
	return err
}
<commit_msg>cmd/cacheserver/cacheutil: Don't pass cacheserver flags that are at their default value<commit_after>// Copyright 2017 The Upspin Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package cacheutil provides a mechanism to start the cacheserver
// if a config requires it and it is not already running.
// It is used by programs like upspin and upspinfs.
package cacheutil // import "upspin.io/cmd/cacheserver/cacheutil"

import (
	"flag"
	"fmt"
	"os"
	"os/exec"
	"time"

	"upspin.io/bind"
	"upspin.io/log"
	"upspin.io/rpc"
	"upspin.io/upspin"
)

var (
	writethrough = flag.Bool("writethrough", false, "make storage cache writethrough")
	cacheSize    = flag.Int64("cachesize", 5e9, "max disk `bytes` for cache")
)

// detach detaches a process from the parent process group,
// on platforms that support it.
var detach = func(*exec.Cmd) {}

// Start starts the cacheserver if the config requires it and it is not already running.
func Start(cfg upspin.Config) {
	if cfg == nil {
		return
	}
	ce, err := rpc.CacheEndpoint(cfg)
	if err != nil || ce == nil {
		// TODO(adg): log error message?
		return // not using a cache server
	}

	// Ping the cache server.
	if err := ping(cfg, ce); err == nil {
		return // cache server running
	}

	// Start a cache server.
	cacheErrorChan := make(chan bool)
	go func() {
		args := []string{"-log=" + log.GetLevel()}
		args = addFlag(args, "config")
		args = addFlag(args, "addr")
		args = addFlag(args, "cachedir")
		args = addFlag(args, "cachesize")
		args = addFlag(args, "writethrough")
		cmd := exec.Command("cacheserver", args...)
		detach(cmd)
		cmd.Stdout = os.Stdout
		cmd.Stderr = os.Stderr
		if err := cmd.Run(); err != nil {
			log.Info.Printf("Starting cacheserver: %s", err)
			fmt.Fprintf(os.Stderr, "Failed to start cacheserver; continuing without.\n")
			close(cacheErrorChan)
		}
	}()

	// Wait for it. Give up and continue without if it doesn't start in a timely fashion.
	for tries := 0; tries < 10; tries++ {
		time.Sleep(500 * time.Millisecond)
		select {
		case <-cacheErrorChan:
			return
		default:
		}
		if err := ping(cfg, ce); err == nil {
			return
		}
	}

	fmt.Fprintf(os.Stderr, "Timed out waiting for cacheserver to start.\n")
}

// addFlag adds a flag to the command if it is at a non-default value.
func addFlag(args []string, name string) []string {
	f := flag.Lookup(name)
	if f == nil {
		return args
	}
	if f.Value.String() == f.DefValue {
		return args
	}
	return append(args, fmt.Sprintf("-%s=%s", name, f.Value.String()))
}

// ping determines if the cacheserver is functioning.
func ping(cfg upspin.Config, ce *upspin.Endpoint) error {
	store, err := bind.StoreServer(cfg, *ce)
	if err != nil {
		return err
	}
	msg, _, _, err := store.Get(upspin.HealthMetadata)
	if err == nil {
		log.Debug.Printf("Cacheserver said %q", string(msg))
	}
	return err
}
<|endoftext|>


<commit_before>/*-
 * Copyright (c) 2018, F5 Networks, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package main

import (
	"bufio"
	"fmt"
	"os/exec"
	"strings"
	"syscall"
	"time"

	"github.com/F5Networks/k8s-bigip-ctlr/pkg/writer"

	log "github.com/F5Networks/k8s-bigip-ctlr/pkg/vlogger"
)

func initializeDriverConfig(
	configWriter writer.Writer,
	global globalSection,
	bigIP bigIPSection,
) error {
	if nil == configWriter {
		return fmt.Errorf("config writer argument cannot be nil")
	}

	sectionNames := []string{"global", "bigip"}
	for i, v := range []interface{}{global, bigIP} {
		doneCh, errCh, err := configWriter.SendSection(sectionNames[i], v)
		if nil != err {
			return fmt.Errorf("failed writing global config section: %v", err)
		}
		select {
		case <-doneCh:
		case e := <-errCh:
			return fmt.Errorf("failed writing section %s - %v: %v",
				sectionNames[i], e, v)
		case <-time.After(1000 * time.Millisecond):
			log.Warning("Did not receive config write response in 1 second")
		}
	}

	return nil
}

func createDriverCmd(
	configFilename string,
	pyCmd string,
) *exec.Cmd {
	var cmd *exec.Cmd

	if pyCmd == "bigipconfigdriver.py" {
		cmdArgs := []string{
			"--config-file", configFilename,
			"--ctlr-prefix", "k8s"}
		cmd = exec.Command(pyCmd, cmdArgs...)
	} else {
		cmdName := "python"
		cmdArgs := []string{
			pyCmd,
			"--config-file", configFilename,
			"--ctlr-prefix", "k8s"}
		cmd = exec.Command(cmdName, cmdArgs...)
	}

	return cmd
}

func runBigIPDriver(pid chan<- int, cmd *exec.Cmd) {
	defer close(pid)

	// the config driver python logging goes to stderr by default
	cmdOut, err := cmd.StderrPipe()
	scanOut := bufio.NewScanner(cmdOut)
	go func() {
		for true {
			if scanOut.Scan() {
				if strings.Contains(scanOut.Text(), "DEBUG]") {
					log.Debug(scanOut.Text())
				} else if strings.Contains(scanOut.Text(), "WARNING]") {
					log.Warning(scanOut.Text())
				} else if strings.Contains(scanOut.Text(), "ERROR]") {
					log.Error(scanOut.Text())
				} else if strings.Contains(scanOut.Text(), "CRITICAL]") {
					log.Critical(scanOut.Text())
				} else {
					log.Info(scanOut.Text())
				}
			} else {
				break
			}
		}
	}()

	err = cmd.Start()
	if nil != err {
		log.Fatalf("Internal error: failed to start config driver: %v", err)
	}
	log.Infof("Started config driver sub-process at pid: %d", cmd.Process.Pid)

	pid <- cmd.Process.Pid

	err = cmd.Wait()
	var waitStatus syscall.WaitStatus
	if exitError, ok := err.(*exec.ExitError); ok {
		waitStatus = exitError.Sys().(syscall.WaitStatus)
		if waitStatus.Signaled() {
			log.Fatalf("Config driver signaled to stop: %d - %s",
				waitStatus.Signal(), waitStatus.Signal())
		} else {
			log.Fatalf("Config driver exited: %d", waitStatus.ExitStatus())
		}
	} else if nil != err {
		log.Fatalf("Config driver exited with error: %v", err)
	} else {
		waitStatus = cmd.ProcessState.Sys().(syscall.WaitStatus)
		log.Warningf("Config driver exited normally: %d", waitStatus.ExitStatus())
	}
}

// Start called to run the python driver
func startPythonDriver(
	configWriter writer.Writer,
	global globalSection,
	bigIP bigIPSection,
	pythonBaseDir string,
) (<-chan int, error) {
	var pyCmd string

	err := initializeDriverConfig(configWriter, global, bigIP)
	if nil != err {
		return nil, err
	}

	subPidCh := make(chan int)
	if len(pythonBaseDir) != 0 {
		log.Warning("DEPRECATED: python-basedir: option may no longer work as expected.")
		pyCmd = fmt.Sprintf("%s/bigipconfigdriver.py", pythonBaseDir)
	} else {
		pyCmd = "bigipconfigdriver.py"
	}
	cmd := createDriverCmd(
		configWriter.GetOutputFilename(),
		pyCmd,
	)
	go runBigIPDriver(subPidCh, cmd)

	return subPidCh, nil
}
<commit_msg>Allow default python dir for backwards compat<commit_after>/*-
 * Copyright (c) 2018, F5 Networks, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package main

import (
	"bufio"
	"fmt"
	"os/exec"
	"strings"
	"syscall"
	"time"

	"github.com/F5Networks/k8s-bigip-ctlr/pkg/writer"

	log "github.com/F5Networks/k8s-bigip-ctlr/pkg/vlogger"
)

func initializeDriverConfig(
	configWriter writer.Writer,
	global globalSection,
	bigIP bigIPSection,
) error {
	if nil == configWriter {
		return fmt.Errorf("config writer argument cannot be nil")
	}

	sectionNames := []string{"global", "bigip"}
	for i, v := range []interface{}{global, bigIP} {
		doneCh, errCh, err := configWriter.SendSection(sectionNames[i], v)
		if nil != err {
			return fmt.Errorf("failed writing global config section: %v", err)
		}
		select {
		case <-doneCh:
		case e := <-errCh:
			return fmt.Errorf("failed writing section %s - %v: %v",
				sectionNames[i], e, v)
		case <-time.After(1000 * time.Millisecond):
			log.Warning("Did not receive config write response in 1 second")
		}
	}

	return nil
}

func createDriverCmd(
	configFilename string,
	pyCmd string,
) *exec.Cmd {
	var cmd *exec.Cmd

	if pyCmd == "bigipconfigdriver.py" {
		cmdArgs := []string{
			"--config-file", configFilename,
			"--ctlr-prefix", "k8s"}
		cmd = exec.Command(pyCmd, cmdArgs...)
	} else {
		cmdName := "python"
		cmdArgs := []string{
			pyCmd,
			"--config-file", configFilename,
			"--ctlr-prefix", "k8s"}
		cmd = exec.Command(cmdName, cmdArgs...)
	}

	return cmd
}

func runBigIPDriver(pid chan<- int, cmd *exec.Cmd) {
	defer close(pid)

	// the config driver python logging goes to stderr by default
	cmdOut, err := cmd.StderrPipe()
	scanOut := bufio.NewScanner(cmdOut)
	go func() {
		for true {
			if scanOut.Scan() {
				if strings.Contains(scanOut.Text(), "DEBUG]") {
					log.Debug(scanOut.Text())
				} else if strings.Contains(scanOut.Text(), "WARNING]") {
					log.Warning(scanOut.Text())
				} else if strings.Contains(scanOut.Text(), "ERROR]") {
					log.Error(scanOut.Text())
				} else if strings.Contains(scanOut.Text(), "CRITICAL]") {
					log.Critical(scanOut.Text())
				} else {
					log.Info(scanOut.Text())
				}
			} else {
				break
			}
		}
	}()

	err = cmd.Start()
	if nil != err {
		log.Fatalf("Internal error: failed to start config driver: %v", err)
	}
	log.Infof("Started config driver sub-process at pid: %d", cmd.Process.Pid)

	pid <- cmd.Process.Pid

	err = cmd.Wait()
	var waitStatus syscall.WaitStatus
	if exitError, ok := err.(*exec.ExitError); ok {
		waitStatus = exitError.Sys().(syscall.WaitStatus)
		if waitStatus.Signaled() {
			log.Fatalf("Config driver signaled to stop: %d - %s",
				waitStatus.Signal(), waitStatus.Signal())
		} else {
			log.Fatalf("Config driver exited: %d", waitStatus.ExitStatus())
		}
	} else if nil != err {
		log.Fatalf("Config driver exited with error: %v", err)
	} else {
		waitStatus = cmd.ProcessState.Sys().(syscall.WaitStatus)
		log.Warningf("Config driver exited normally: %d", waitStatus.ExitStatus())
	}
}

// Start called to run the python driver
func startPythonDriver(
	configWriter writer.Writer,
	global globalSection,
	bigIP bigIPSection,
	pythonBaseDir string,
) (<-chan int, error) {
	var pyCmd string

	err := initializeDriverConfig(configWriter, global, bigIP)
	if nil != err {
		return nil, err
	}

	subPidCh := make(chan int)
	if len(pythonBaseDir) != 0 && pythonBaseDir != "/app/python" {
		log.Warning("DEPRECATED: python-basedir: option may no longer work as expected.")
		pyCmd = fmt.Sprintf("%s/bigipconfigdriver.py", pythonBaseDir)
	} else {
		pyCmd = "bigipconfigdriver.py"
	}
	cmd := createDriverCmd(
		configWriter.GetOutputFilename(),
		pyCmd,
	)
	go runBigIPDriver(subPidCh, cmd)

	return subPidCh, nil
}
<|endoftext|>


<commit_before>package commands

import (
	"bytes"
	"fmt"
	"io"
	"os"

	"github.com/github/git-lfs/git"
	"github.com/github/git-lfs/lfs"
	"github.com/spf13/cobra"
)

const (
	// cleanFilterBufferCapacity is the desired capacity of the
	// `*git.PacketWriter`'s internal buffer when the filter protocol
	// dictates the "clean" command. 512 bytes is (in most cases) enough to
	// hold an entire LFS pointer in memory.
	cleanFilterBufferCapacity = 512

	// smudgeFilterBufferCapacity is the desired capacity of the
	// `*git.PacketWriter`'s internal buffer when the filter protocol
	// dictates the "smudge" command.
	smudgeFilterBufferCapacity = git.MaxPacketLength
)

// filterSmudgeSkip is a command-line flag owned by the `filter-process` command
// dictating whether or not to skip the smudging process, leaving pointers as-is
// in the working tree.
var filterSmudgeSkip bool

// filterSmudge is a gateway to the `smudge()` function and serves to bail out
// immediately if the pointer decoded from "from" has no data (i.e., is empty).
// This function, unlike the implementation found in the legacy smudge command,
// only combines the `io.Reader`s when necessary, since the implementation
// found in `*git.PacketReader` blocks while waiting for the following packet.
func filterSmudge(from io.Reader, to io.Writer, filename string) error {
	var pbuf bytes.Buffer
	from = io.TeeReader(from, &pbuf)

	ptr, err := lfs.DecodePointer(from)
	if err != nil {
		// If we tried to decode a pointer out of the data given to us,
		// and the file was _empty_, write out an empty file in
		// response. This occurs because when the clean filter
		// encounters an empty file, and writes out an empty file,
		// instead of a pointer.
		//
		// TODO(taylor): figure out if there is more data on the reader,
		// and buffer that as well.
		if len(pbuf.Bytes()) == 0 {
			if _, cerr := io.Copy(to, &pbuf); cerr != nil {
				Panic(cerr, "Error writing data to stdout:")
			}
			return nil
		}

		return err
	}

	lfs.LinkOrCopyFromReference(ptr.Oid, ptr.Size)

	return smudge(to, ptr, filename, filterSmudgeSkip)
}

func filterCommand(cmd *cobra.Command, args []string) {
	requireStdin("This command should be run by the Git filter process")
	lfs.InstallHooks(false)

	s := git.NewFilterProcessScanner(os.Stdin, os.Stdout)

	if err := s.Init(); err != nil {
		ExitWithError(err)
	}
	if err := s.NegotiateCapabilities(); err != nil {
		ExitWithError(err)
	}

Scan:
	for s.Scan() {
		var err error
		var w *git.PktlineWriter

		req := s.Request()

		s.WriteStatus("success")

		switch req.Header["command"] {
		case "clean":
			w = git.NewPktlineWriter(os.Stdout, cleanFilterBufferCapacity)
			err = clean(req.Payload, w, req.Header["pathname"])
		case "smudge":
			w = git.NewPktlineWriter(os.Stdout, smudgeFilterBufferCapacity)
			err = filterSmudge(req.Payload, w, req.Header["pathname"])
		default:
			fmt.Errorf("Unknown command %s", cmd)
			break Scan
		}

		var status string
		if ferr := w.Flush(); ferr != nil {
			status = "error"
		} else {
			status = statusFromErr(err)
		}

		s.WriteStatus(status)
	}

	if err := s.Err(); err != nil && err != io.EOF {
		ExitWithError(err)
	}
}

// statusFromErr returns the status code that should be sent over the filter
// protocol based on a given error, "err".
func statusFromErr(err error) string {
	if err != nil && err != io.EOF {
		return "error"
	}
	return "success"
}

func init() {
	RegisterCommand("filter-process", filterCommand, func(cmd *cobra.Command) {
		cmd.Flags().BoolVarP(&filterSmudgeSkip, "skip", "s", false, "")
	})
}
<commit_msg>commands/fp: use statusFromErr<commit_after>package commands

import (
	"bytes"
	"fmt"
	"io"
	"os"

	"github.com/github/git-lfs/git"
	"github.com/github/git-lfs/lfs"
	"github.com/spf13/cobra"
)

const (
	// cleanFilterBufferCapacity is the desired capacity of the
	// `*git.PacketWriter`'s internal buffer when the filter protocol
	// dictates the "clean" command. 512 bytes is (in most cases) enough to
	// hold an entire LFS pointer in memory.
	cleanFilterBufferCapacity = 512

	// smudgeFilterBufferCapacity is the desired capacity of the
	// `*git.PacketWriter`'s internal buffer when the filter protocol
	// dictates the "smudge" command.
	smudgeFilterBufferCapacity = git.MaxPacketLength
)

// filterSmudgeSkip is a command-line flag owned by the `filter-process` command
// dictating whether or not to skip the smudging process, leaving pointers as-is
// in the working tree.
var filterSmudgeSkip bool

// filterSmudge is a gateway to the `smudge()` function and serves to bail out
// immediately if the pointer decoded from "from" has no data (i.e., is empty).
// This function, unlike the implementation found in the legacy smudge command,
// only combines the `io.Reader`s when necessary, since the implementation
// found in `*git.PacketReader` blocks while waiting for the following packet.
func filterSmudge(from io.Reader, to io.Writer, filename string) error {
	var pbuf bytes.Buffer
	from = io.TeeReader(from, &pbuf)

	ptr, err := lfs.DecodePointer(from)
	if err != nil {
		// If we tried to decode a pointer out of the data given to us,
		// and the file was _empty_, write out an empty file in
		// response. This occurs because when the clean filter
		// encounters an empty file, and writes out an empty file,
		// instead of a pointer.
		//
		// TODO(taylor): figure out if there is more data on the reader,
		// and buffer that as well.
		if len(pbuf.Bytes()) == 0 {
			if _, cerr := io.Copy(to, &pbuf); cerr != nil {
				Panic(cerr, "Error writing data to stdout:")
			}
			return nil
		}

		return err
	}

	lfs.LinkOrCopyFromReference(ptr.Oid, ptr.Size)

	return smudge(to, ptr, filename, filterSmudgeSkip)
}

func filterCommand(cmd *cobra.Command, args []string) {
	requireStdin("This command should be run by the Git filter process")
	lfs.InstallHooks(false)

	s := git.NewFilterProcessScanner(os.Stdin, os.Stdout)

	if err := s.Init(); err != nil {
		ExitWithError(err)
	}
	if err := s.NegotiateCapabilities(); err != nil {
		ExitWithError(err)
	}

Scan:
	for s.Scan() {
		var err error
		var w *git.PktlineWriter

		req := s.Request()

		s.WriteStatus(statusFromErr(nil))

		switch req.Header["command"] {
		case "clean":
			w = git.NewPktlineWriter(os.Stdout, cleanFilterBufferCapacity)
			err = clean(req.Payload, w, req.Header["pathname"])
		case "smudge":
			w = git.NewPktlineWriter(os.Stdout, smudgeFilterBufferCapacity)
			err = filterSmudge(req.Payload, w, req.Header["pathname"])
		default:
			fmt.Errorf("Unknown command %s", cmd)
			break Scan
		}

		var status string
		if ferr := w.Flush(); ferr != nil {
			status = statusFromErr(ferr)
		} else {
			status = statusFromErr(err)
		}

		s.WriteStatus(status)
	}

	if err := s.Err(); err != nil && err != io.EOF {
		ExitWithError(err)
	}
}

// statusFromErr returns the status code that should be sent over the filter
// protocol based on a given error, "err".
func statusFromErr(err error) string {
	if err != nil && err != io.EOF {
		return "error"
	}
	return "success"
}

func init() {
	RegisterCommand("filter-process", filterCommand, func(cmd *cobra.Command) {
		cmd.Flags().BoolVarP(&filterSmudgeSkip, "skip", "s", false, "")
	})
}
<|endoftext|>


<commit_before>// Copyright The OpenTelemetry Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//       http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package awsprometheusremotewriteexporter

import (
	"context"
	"os"
	"testing"

	"github.com/stretchr/testify/assert"
	"go.opentelemetry.io/collector/component"
	"go.opentelemetry.io/collector/config"
	"go.opentelemetry.io/collector/config/configcheck"
	"go.opentelemetry.io/collector/config/confighttp"
	"go.opentelemetry.io/collector/config/configtls"
	"go.uber.org/zap"
)

func TestType(t *testing.T) {
	af := NewFactory()
	assert.Equal(t, af.Type(), config.Type(typeStr))
}

//Tests whether or not the default Exporter factory can instantiate a properly interfaced Exporter with default conditions
func TestCreateDefaultConfig(t *testing.T) {
	af := NewFactory()
	cfg := af.CreateDefaultConfig()
	assert.NotNil(t, cfg, "failed to create default config")
	assert.NoError(t, configcheck.ValidateConfig(cfg))
}

//Tests whether or not a correct Metrics Exporter from the default Config parameters
func TestCreateMetricsExporter(t *testing.T) {
	af := NewFactory()
	validConfigWithAuth := af.CreateDefaultConfig().(*Config)
	validConfigWithAuth.AuthConfig = AuthConfig{Region: "region", Service: "service"}

	// Some form of AWS credentials chain required to test valid auth case
	// This is a set of mock credentials strictly for testing purposes. Users
	// should not set their credentials like this in production.
	os.Setenv("AWS_ACCESS_KEY", "mock_value")
	os.Setenv("AWS_SECRET_ACCESS_KEY", "mock_value2")

	invalidConfigWithAuth := af.CreateDefaultConfig().(*Config)
	invalidConfigWithAuth.AuthConfig = AuthConfig{Region: "", Service: "service"}

	invalidConfig := af.CreateDefaultConfig().(*Config)
	invalidConfig.HTTPClientSettings = confighttp.HTTPClientSettings{}

	invalidTLSConfig := af.CreateDefaultConfig().(*Config)
	invalidTLSConfig.HTTPClientSettings.TLSSetting = configtls.TLSClientSetting{
		TLSSetting: configtls.TLSSetting{
			CAFile:   "non-existent file",
			CertFile: "",
			KeyFile:  "",
		},
		Insecure:   false,
		ServerName: "",
	}

	tests := []struct {
		name        string
		cfg         config.Exporter
		params      component.ExporterCreateParams
		returnError bool
	}{
		{"success_case_with_auth",
			validConfigWithAuth,
			component.ExporterCreateParams{Logger: zap.NewNop()},
			false,
		},
		{"invalid_config_case",
			invalidConfig,
			component.ExporterCreateParams{Logger: zap.NewNop()},
			true,
		},
		{"invalid_tls_config_case",
			invalidTLSConfig,
			component.ExporterCreateParams{Logger: zap.NewNop()},
			true,
		},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			_, err := af.CreateMetricsExporter(context.Background(), tt.params, tt.cfg)
			if tt.returnError {
				assert.Error(t, err)
				return
			}
			assert.NoError(t, err)
		})
	}
}
<commit_msg>updated awsprometheusremotewriteeexporter for upcoming client auth extension changes (#3518)<commit_after>// Copyright The OpenTelemetry Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//       http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package awsprometheusremotewriteexporter

import (
	"context"
	"os"
	"testing"

	"github.com/stretchr/testify/assert"
	"go.opentelemetry.io/collector/component"
	"go.opentelemetry.io/collector/config"
	"go.opentelemetry.io/collector/config/configcheck"
	"go.opentelemetry.io/collector/config/confighttp"
	"go.opentelemetry.io/collector/config/configtls"
	"go.uber.org/zap"
)

func TestType(t *testing.T) {
	af := NewFactory()
	assert.Equal(t, af.Type(), config.Type(typeStr))
}

//Tests whether or not the default Exporter factory can instantiate a properly interfaced Exporter with default conditions
func TestCreateDefaultConfig(t *testing.T) {
	af := NewFactory()
	cfg := af.CreateDefaultConfig()
	assert.NotNil(t, cfg, "failed to create default config")
	assert.NoError(t, configcheck.ValidateConfig(cfg))
}

//Tests whether or not a correct Metrics Exporter from the default Config parameters
func TestCreateMetricsExporter(t *testing.T) {
	af := NewFactory()
	validConfigWithAuth := af.CreateDefaultConfig().(*Config)
	validConfigWithAuth.AuthConfig = AuthConfig{Region: "region", Service: "service"}

	// Some form of AWS credentials chain required to test valid auth case
	// This is a set of mock credentials strictly for testing purposes. Users
	// should not set their credentials like this in production.
	os.Setenv("AWS_ACCESS_KEY", "mock_value")
	os.Setenv("AWS_SECRET_ACCESS_KEY", "mock_value2")

	invalidConfigWithAuth := af.CreateDefaultConfig().(*Config)
	invalidConfigWithAuth.AuthConfig = AuthConfig{Region: "", Service: "service"}

	invalidConfig := af.CreateDefaultConfig().(*Config)
	invalidConfig.HTTPClientSettings = confighttp.HTTPClientSettings{}

	invalidTLSConfig := af.CreateDefaultConfig().(*Config)
	invalidTLSConfig.HTTPClientSettings.TLSSetting = configtls.TLSClientSetting{
		TLSSetting: configtls.TLSSetting{
			CAFile:   "non-existent file",
			CertFile: "",
			KeyFile:  "",
		},
		Insecure:   false,
		ServerName: "",
	}

	tests := []struct {
		name        string
		cfg         config.Exporter
		params      component.ExporterCreateParams
		returnError bool
	}{
		{
			name:        "success_case_with_auth",
			cfg:         validConfigWithAuth,
			params:      component.ExporterCreateParams{Logger: zap.NewNop()},
			returnError: false,
		},
		{
			name:        "invalid_config_case",
			cfg:         invalidConfig,
			params:      component.ExporterCreateParams{Logger: zap.NewNop()},
			returnError: true,
		},
		{
			name:        "invalid_tls_config_case",
			cfg:         invalidTLSConfig,
			params:      component.ExporterCreateParams{Logger: zap.NewNop()},
			returnError: true,
		},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			exp, err := af.CreateMetricsExporter(context.Background(), tt.params, tt.cfg)
			if tt.returnError {
				assert.Error(t, err)
				return
			}
			assert.NoError(t, err)
			assert.NotNil(t, exp)
		})
	}
}
<|endoftext|>


<commit_before>package collaboration

import (
	"fmt"
	mongomodels "koding/db/models"
	"koding/db/mongodb/modelhelper"
	"math/rand"
	socialapimodels "socialapi/models"
	"socialapi/workers/collaboration/models"
	"socialapi/workers/common/runner"
	"socialapi/workers/helper"
	"strconv"

	"github.com/koding/bongo"

	"labix.org/v2/mgo"
	"labix.org/v2/mgo/bson"

	"testing"
	"time"

	. "github.com/smartystreets/goconvey/convey"
)

func TestCollaborationOperationsDeleteDriveDoc(t *testing.T) {
	r := runner.New("collaboration-DeleteDriveDoc-tests")
	err := r.Init()
	if err != nil {
		panic(err)
	}

	defer r.Close()

	modelhelper.Initialize(r.Conf.Mongo)
	defer modelhelper.Close()

	redisConn := helper.MustInitRedisConn(r.Conf)
	defer redisConn.Close()

	handler := New(r.Log, redisConn, r.Conf, r.Kite)

	Convey("while testing DeleteDriveDoc", t, func() {
		req := &models.Ping{
			AccountId: 1,
			FileId:    fmt.Sprintf("%d", rand.Int63()),
		}
		Convey("should be able to create the file", func() {
			f, err := createTestFile(handler)
			So(err, ShouldBeNil)
			req.FileId = f.Id

			Convey("should be able to delete the created file", func() {
				err = handler.DeleteDriveDoc(req)
				So(err, ShouldBeNil)
			})

			Convey("if file id is nil response should be nil", func() {
				req := req
				req.FileId = ""
				err = handler.DeleteDriveDoc(req)
				So(err, ShouldBeNil)
			})
		})
	})
}

func TestCollaborationOperationsEndPrivateMessage(t *testing.T) {
	r := runner.New("collaboration-EndPrivateMessage-tests")
	err := r.Init()
	if err != nil {
		panic(err)
	}

	defer r.Close()

	modelhelper.Initialize(r.Conf.Mongo)
	defer modelhelper.Close()

	redisConn := helper.MustInitRedisConn(r.Conf)
	defer redisConn.Close()

	handler := New(r.Log, redisConn, r.Conf, r.Kite)

	Convey("while testing EndPrivateMessage", t, func() {
		req := &models.Ping{
			AccountId: 1,
			FileId:    fmt.Sprintf("%d", rand.Int63()),
		}
		Convey("should be able to create the channel first", func() {
			creator := socialapimodels.CreateAccountWithTest() // init account
			c := socialapimodels.NewChannel()                  // init channel
			c.CreatorId = creator.Id                           // set Creator id
			c.TypeConstant = socialapimodels.Channel_TYPE_COLLABORATION
			So(c.Create(), ShouldBeNil)
			cp, err := c.AddParticipant(creator.Id)
			So(err, ShouldBeNil)
			So(cp, ShouldNotBeNil)

			req.AccountId = c.CreatorId // set real account id
			req.ChannelId = c.Id        // set real channel id

			ws := &mongomodels.Workspace{
				ObjectId:     bson.NewObjectId(),
				Name:         "My Workspace",
				Slug:         "my-workspace",
				ChannelId:    strconv.FormatInt(req.ChannelId, 10),
				MachineUID:   bson.NewObjectId().Hex(),
				MachineLabel: "koding-vm-0",
				Owner:        "cihangir",
				RootPath:     "/home/cihangir",
				IsDefault:    true,
			}

			So(modelhelper.CreateWorkspace(ws), ShouldBeNil)

			Convey("should be able to delete channel", func() {
				err = handler.EndPrivateMessage(req)
				So(err, ShouldBeNil)
				Convey("deleted channel should not be exist", func() {
					channel := socialapimodels.NewChannel()
					err := channel.ById(req.ChannelId)
					So(err, ShouldEqual, bongo.RecordNotFound)
				})
				Convey("channel id in workspace should not be exist", func() {
					ws2, err := modelhelper.GetWorkspaceByChannelId(
						strconv.FormatInt(req.ChannelId, 10),
					)
					So(err, ShouldEqual, mgo.ErrNotFound)
					So(ws2, ShouldEqual, nil)
				})
			})

			Convey("if not a participant, should not be able to delete channel", func() {
				req.AccountId = 1
				err = handler.EndPrivateMessage(req)
				So(err, ShouldBeNil)
				Convey("channel should exist", func() {
					channel := socialapimodels.NewChannel()
					err := channel.ById(req.ChannelId)
					So(err, ShouldBeNil)
				})
			})

			Convey("if channel doesnt exists, should success", func() {
				req.ChannelId = 1
				err = handler.EndPrivateMessage(req)
				So(err, ShouldBeNil)
				Convey("channel should not exist", func() {
					channel := socialapimodels.NewChannel()
					err := channel.ById(req.ChannelId)
					So(err, ShouldEqual, bongo.RecordNotFound)
				})
			})
		})
	})
}

func TestCollaborationOperationsUnshareVM(t *testing.T) {
	r := runner.New("collaboration-UnshareVM-tests")
	err := r.Init()
	if err != nil {
		panic(err)
	}

	defer r.Close()

	modelhelper.Initialize(r.Conf.Mongo)
	defer modelhelper.Close()

	redisConn := helper.MustInitRedisConn(r.Conf)
	defer redisConn.Close()

	handler := New(r.Log, redisConn, r.Conf, r.Kite)

	Convey("while testing UnshareVM", t, func() {
		req := &models.Ping{
			AccountId: 1,
			FileId:    fmt.Sprintf("%d", rand.Int63()),
		}
		Convey("should be able to create the channel and workspace first", func() {
			creator := socialapimodels.CreateAccountWithTest() // init account
			c := socialapimodels.NewChannel()                  // init channel
			c.CreatorId = creator.Id                           // set Creator id
			c.TypeConstant = socialapimodels.Channel_TYPE_COLLABORATION
			So(c.Create(), ShouldBeNil)
			cp, err := c.AddParticipant(creator.Id)
			So(err, ShouldBeNil)
			So(cp, ShouldNotBeNil)

			req.AccountId = c.CreatorId // set real account id
			req.ChannelId = c.Id        // set real channel id

			// sample  machine struct
			m := &mongomodels.Machine{
				ObjectId: bson.NewObjectId(),
				Uid:      bson.NewObjectId().Hex(),
				Users: []mongomodels.MachineUser{

					mongomodels.MachineUser{ // real owner
						Id:    bson.ObjectIdHex(creator.OldId),
						Sudo:  true,
						Owner: true,
					},
					mongomodels.MachineUser{ // secondary owner
						Id:    bson.NewObjectId(),
						Sudo:  false,
						Owner: true,
					},
				},
				CreatedAt: time.Now().UTC(),
				Status: mongomodels.MachineStatus{
					State:      "running",
					ModifiedAt: time.Now().UTC(),
				},
				Assignee:    mongomodels.MachineAssignee{},
				UserDeleted: false,
			}

			So(modelhelper.CreateMachine(m), ShouldBeNil)

			ws := &mongomodels.Workspace{
				ObjectId:     bson.NewObjectId(),
				Name:         "My Workspace",
				Slug:         "my-workspace",
				ChannelId:    strconv.FormatInt(req.ChannelId, 10),
				MachineUID:   m.Uid,
				MachineLabel: "koding-vm-0",
				Owner:        "cihangir",
				RootPath:     "/home/cihangir",
				IsDefault:    true,
			}

			So(modelhelper.CreateWorkspace(ws), ShouldBeNil)

			Convey("should be able to UnshareVM", func() {
				err = handler.UnshareVM(req)
				So(err, ShouldBeNil)
				Convey("remove users should not be in the machine", func() {
					m2, err := modelhelper.GetMachineByUid(m.Uid)
					So(err, ShouldBeNil)
					So(m2, ShouldNotBeNil)
					So(len(m2.Users), ShouldEqual, 1)
					So(m2.Users[0].Id.Hex(), ShouldEqual, creator.OldId)
				})
			})
		})
	})
}
<commit_msg>Socialapi: implemented test cases for multiple workspaces<commit_after>package collaboration

import (
	"fmt"
	mongomodels "koding/db/models"
	"koding/db/mongodb/modelhelper"
	"math/rand"
	socialapimodels "socialapi/models"
	"socialapi/workers/collaboration/models"
	"socialapi/workers/common/runner"
	"socialapi/workers/helper"
	"strconv"

	"labix.org/v2/mgo"
	"labix.org/v2/mgo/bson"

	"testing"
	"time"

	"github.com/koding/bongo"
	. "github.com/smartystreets/goconvey/convey"
)

func TestCollaborationOperationsDeleteDriveDoc(t *testing.T) {
	r := runner.New("collaboration-DeleteDriveDoc-tests")
	err := r.Init()
	if err != nil {
		panic(err)
	}

	defer r.Close()

	modelhelper.Initialize(r.Conf.Mongo)
	defer modelhelper.Close()

	redisConn := helper.MustInitRedisConn(r.Conf)
	defer redisConn.Close()

	handler := New(r.Log, redisConn, r.Conf, r.Kite)

	Convey("while testing DeleteDriveDoc", t, func() {
		req := &models.Ping{
			AccountId: 1,
			FileId:    fmt.Sprintf("%d", rand.Int63()),
		}
		Convey("should be able to create the file", func() {
			f, err := createTestFile(handler)
			So(err, ShouldBeNil)
			req.FileId = f.Id

			Convey("should be able to delete the created file", func() {
				err = handler.DeleteDriveDoc(req)
				So(err, ShouldBeNil)
			})

			Convey("if file id is nil response should be nil", func() {
				req := req
				req.FileId = ""
				err = handler.DeleteDriveDoc(req)
				So(err, ShouldBeNil)
			})
		})
	})
}

func TestCollaborationOperationsEndPrivateMessage(t *testing.T) {
	r := runner.New("collaboration-EndPrivateMessage-tests")
	err := r.Init()
	if err != nil {
		panic(err)
	}

	defer r.Close()

	modelhelper.Initialize(r.Conf.Mongo)
	defer modelhelper.Close()

	redisConn := helper.MustInitRedisConn(r.Conf)
	defer redisConn.Close()

	handler := New(r.Log, redisConn, r.Conf, r.Kite)

	Convey("while testing EndPrivateMessage", t, func() {
		req := &models.Ping{
			AccountId: 1,
			FileId:    fmt.Sprintf("%d", rand.Int63()),
		}
		Convey("should be able to create the channel first", func() {
			creator := socialapimodels.CreateAccountWithTest() // init account
			c := socialapimodels.NewChannel()                  // init channel
			c.CreatorId = creator.Id                           // set Creator id
			c.TypeConstant = socialapimodels.Channel_TYPE_COLLABORATION
			So(c.Create(), ShouldBeNil)
			cp, err := c.AddParticipant(creator.Id)
			So(err, ShouldBeNil)
			So(cp, ShouldNotBeNil)

			req.AccountId = c.CreatorId // set real account id
			req.ChannelId = c.Id        // set real channel id

			ws := &mongomodels.Workspace{
				ObjectId:     bson.NewObjectId(),
				OriginId:     bson.NewObjectId(),
				Name:         "My Workspace",
				Slug:         "my-workspace",
				ChannelId:    strconv.FormatInt(req.ChannelId, 10),
				MachineUID:   bson.NewObjectId().Hex(),
				MachineLabel: "koding-vm-0",
				Owner:        "cihangir",
				RootPath:     "/home/cihangir",
				IsDefault:    true,
			}

			So(modelhelper.CreateWorkspace(ws), ShouldBeNil)

			Convey("should be able to delete channel", func() {
				err = handler.EndPrivateMessage(req)
				So(err, ShouldBeNil)
				Convey("deleted channel should not be exist", func() {
					channel := socialapimodels.NewChannel()
					err := channel.ById(req.ChannelId)
					So(err, ShouldEqual, bongo.RecordNotFound)
				})
				Convey("channel id in workspace should not be exist", func() {
					ws2, err := modelhelper.GetWorkspaceByChannelId(
						strconv.FormatInt(req.ChannelId, 10),
					)
					So(err, ShouldEqual, mgo.ErrNotFound)
					So(ws2, ShouldEqual, nil)
				})
			})

			Convey("if not a participant, should not be able to delete channel", func() {
				req.AccountId = 1
				err = handler.EndPrivateMessage(req)
				So(err, ShouldBeNil)
				Convey("channel should exist", func() {
					channel := socialapimodels.NewChannel()
					err := channel.ById(req.ChannelId)
					So(err, ShouldBeNil)
				})
			})

			Convey("if channel doesnt exists, should success", func() {
				req.ChannelId = 1
				err = handler.EndPrivateMessage(req)
				So(err, ShouldBeNil)
				Convey("channel should not exist", func() {
					channel := socialapimodels.NewChannel()
					err := channel.ById(req.ChannelId)
					So(err, ShouldEqual, bongo.RecordNotFound)
				})
			})
		})
	})
}

func TestCollaborationOperationsUnshareVM(t *testing.T) {
	r := runner.New("collaboration-UnshareVM-tests")
	err := r.Init()
	if err != nil {
		panic(err)
	}

	defer r.Close()

	modelhelper.Initialize(r.Conf.Mongo)
	defer modelhelper.Close()

	redisConn := helper.MustInitRedisConn(r.Conf)
	defer redisConn.Close()

	handler := New(r.Log, redisConn, r.Conf, r.Kite)

	Convey("while testing UnshareVM", t, func() {

		Convey("should be able to create the channel and workspace first", func() {

			Convey("should be able to UnshareVM", func() {

				creator, err := socialapimodels.CreateAccountInBothDbs() // init account
				So(err, ShouldBeNil)

				participant1, err := socialapimodels.CreateAccountInBothDbs()
				So(err, ShouldBeNil)

				participant2, err := socialapimodels.CreateAccountInBothDbs()
				So(err, ShouldBeNil)

				m1, m1ws1 := prepareSingleWorkspace(creator, participant1, participant2)

				channelId, err := strconv.ParseInt(m1ws1.ChannelId, 10, 64)
				So(err, ShouldBeNil)

				req1 := &models.Ping{
					AccountId: creator.Id,
					FileId:    fmt.Sprintf("%d", rand.Int63()),
					ChannelId: channelId,
				}

				err = handler.UnshareVM(req1)
				So(err, ShouldBeNil)
				Convey("remove users should not be in the machine", func() {
					mm1, err := modelhelper.GetMachineByUid(m1.Uid)
					So(err, ShouldBeNil)
					So(mm1, ShouldNotBeNil)
					So(len(mm1.Users), ShouldEqual, 1)
					ownerUser, err := modelhelper.GetUserByAccountId(creator.OldId)
					So(err, ShouldBeNil)
					So(mm1.Users[0].Id.Hex(), ShouldEqual, ownerUser.ObjectId.Hex())
				})
			})

			Convey("if participant and owner shares multiple workspaces", func() {

				creator, err := socialapimodels.CreateAccountInBothDbs() // init account
				So(err, ShouldBeNil)

				participant1, err := socialapimodels.CreateAccountInBothDbs()
				So(err, ShouldBeNil)

				participant2, err := socialapimodels.CreateAccountInBothDbs()
				So(err, ShouldBeNil)

				_, _, m2, m2ws1, _ := prepareWorkspace(creator, participant1, participant2)

				Convey("remove from first workspace", func() {
					channelId, err := strconv.ParseInt(m2ws1.ChannelId, 10, 64)
					So(err, ShouldBeNil)

					req := &models.Ping{
						AccountId: creator.Id,
						FileId:    fmt.Sprintf("%d", rand.Int63()),
						ChannelId: channelId,
					}

					err = handler.UnshareVM(req)
					So(err, ShouldBeNil)
					Convey("participant should still be in the second machine", func() {
						mm2, err := modelhelper.GetMachineByUid(m2.Uid)
						So(err, ShouldBeNil)
						So(mm2, ShouldNotBeNil)
						So(len(mm2.Users), ShouldEqual, 3)

						ownerUser, err := modelhelper.GetUserByAccountId(creator.OldId)
						So(err, ShouldBeNil)
						So(mm2.Users[0].Id.Hex(), ShouldEqual, ownerUser.ObjectId.Hex())

						participant1User, err := modelhelper.GetUserByAccountId(participant1.OldId)
						So(err, ShouldBeNil)
						So(mm2.Users[1].Id.Hex(), ShouldEqual, participant1User.ObjectId.Hex())

						participant2User, err := modelhelper.GetUserByAccountId(participant2.OldId)
						So(err, ShouldBeNil)
						So(mm2.Users[2].Id.Hex(), ShouldEqual, participant2User.ObjectId.Hex())
					})
				})
			})
		})
	})
}

func prepareWorkspace(creator, participant1, participant2 *socialapimodels.Account) (
	*mongomodels.Machine, // m1
	*mongomodels.Workspace, // m1 ws1
	*mongomodels.Machine, // m2
	*mongomodels.Workspace, // m2 ws1
	*mongomodels.Workspace, // m2 ws2
) {

	m1, m1ws1 := prepareSingleWorkspace(creator, participant1, participant2)

	ownerUser, err := modelhelper.GetUserByAccountId(creator.OldId)
	So(err, ShouldBeNil)

	participant1User, err := modelhelper.GetUserByAccountId(participant1.OldId)
	So(err, ShouldBeNil)

	participant2User, err := modelhelper.GetUserByAccountId(participant2.OldId)
	So(err, ShouldBeNil)

	// sample  machine struct
	m2 := &mongomodels.Machine{
		ObjectId: bson.NewObjectId(),
		Uid:      bson.NewObjectId().Hex(),
		Users: []mongomodels.MachineUser{
			mongomodels.MachineUser{ // real owner
				Id:    ownerUser.ObjectId,
				Sudo:  true,
				Owner: true,
			},
			mongomodels.MachineUser{ // secondary owner
				Id:    participant1User.ObjectId,
				Sudo:  false,
				Owner: true,
			},
			mongomodels.MachineUser{ // random
				Id:    participant2User.ObjectId,
				Sudo:  false,
				Owner: true,
			},
		},
		CreatedAt: time.Now().UTC(),
		Status: mongomodels.MachineStatus{
			State:      "running",
			ModifiedAt: time.Now().UTC(),
		},
		Assignee:    mongomodels.MachineAssignee{},
		UserDeleted: false,
	}

	So(modelhelper.CreateMachine(m2), ShouldBeNil)

	c3 := socialapimodels.NewChannel() // init channel
	c3.CreatorId = creator.Id          // set Creator id
	c3.TypeConstant = socialapimodels.Channel_TYPE_COLLABORATION
	So(c3.Create(), ShouldBeNil)

	c3p1, err := c3.AddParticipant(creator.Id)
	So(err, ShouldBeNil)
	So(c3p1, ShouldNotBeNil)

	c3p2, err := c3.AddParticipant(participant1.Id)
	So(err, ShouldBeNil)
	So(c3p2, ShouldNotBeNil)

	c3p3, err := c3.AddParticipant(participant2.Id)
	So(err, ShouldBeNil)
	So(c3p3, ShouldNotBeNil)

	c4 := socialapimodels.NewChannel() // init channel
	c4.CreatorId = creator.Id          // set Creator id
	c4.TypeConstant = socialapimodels.Channel_TYPE_COLLABORATION
	So(c4.Create(), ShouldBeNil)

	c4p1, err := c4.AddParticipant(creator.Id)
	So(err, ShouldBeNil)
	So(c4p1, ShouldNotBeNil)

	c4p2, err := c4.AddParticipant(participant1.Id)
	So(err, ShouldBeNil)
	So(c4p2, ShouldNotBeNil)

	c4p3, err := c4.AddParticipant(participant2.Id)
	So(err, ShouldBeNil)
	So(c4p3, ShouldNotBeNil)

	m2ws1 := &mongomodels.Workspace{
		ObjectId:     bson.NewObjectId(),
		OriginId:     ownerUser.ObjectId,
		Name:         "My Workspace",
		Slug:         "m2ws1",
		ChannelId:    strconv.FormatInt(c3.Id, 10),
		MachineUID:   m2.Uid,
		MachineLabel: "koding-vm-0",
		Owner:        "cihangir",
		RootPath:     "/home/cihangir",
		IsDefault:    true,
	}

	So(modelhelper.CreateWorkspace(m2ws1), ShouldBeNil)

	m2ws2 := &mongomodels.Workspace{
		ObjectId:     bson.NewObjectId(),
		OriginId:     ownerUser.ObjectId,
		Name:         "My Workspace",
		Slug:         "m2ws2",
		ChannelId:    strconv.FormatInt(c4.Id, 10),
		MachineUID:   m2.Uid,
		MachineLabel: "koding-vm-0",
		Owner:        "cihangir",
		RootPath:     "/home/cihangir",
		IsDefault:    true,
	}

	So(modelhelper.CreateWorkspace(m2ws2), ShouldBeNil)

	return m1, m1ws1, m2, m2ws1, m2ws2
}

func prepareSingleWorkspace(creator, participant1, participant2 *socialapimodels.Account) (
	*mongomodels.Machine, // m1
	*mongomodels.Workspace, // m1 ws1
) {

	ownerUser, err := modelhelper.GetUserByAccountId(creator.OldId)
	So(err, ShouldBeNil)

	participant1User, err := modelhelper.GetUserByAccountId(participant1.OldId)
	So(err, ShouldBeNil)

	participant2User, err := modelhelper.GetUserByAccountId(participant2.OldId)
	So(err, ShouldBeNil)

	//
	// create the first channel
	//
	c1 := socialapimodels.NewChannel() // init channel
	c1.CreatorId = creator.Id          // set Creator id
	c1.TypeConstant = socialapimodels.Channel_TYPE_COLLABORATION
	So(c1.Create(), ShouldBeNil)

	c1p1, err := c1.AddParticipant(creator.Id)
	So(err, ShouldBeNil)
	So(c1p1, ShouldNotBeNil)

	c1p2, err := c1.AddParticipant(participant1.Id)
	So(err, ShouldBeNil)
	So(c1p2, ShouldNotBeNil)

	c1p3, err := c1.AddParticipant(participant2.Id)
	So(err, ShouldBeNil)
	So(c1p3, ShouldNotBeNil)

	// sample  machine struct
	m1 := &mongomodels.Machine{
		ObjectId: bson.NewObjectId(),
		Uid:      bson.NewObjectId().Hex(),
		Users: []mongomodels.MachineUser{
			mongomodels.MachineUser{ // real owner
				Id:    ownerUser.ObjectId,
				Sudo:  true,
				Owner: true,
			},
			mongomodels.MachineUser{ // secondary owner
				Id:    participant1User.ObjectId,
				Sudo:  false,
				Owner: true,
			},
			mongomodels.MachineUser{ // random
				Id:    participant2User.ObjectId,
				Sudo:  false,
				Owner: true,
			},
		},
		CreatedAt: time.Now().UTC(),
		Status: mongomodels.MachineStatus{
			State:      "running",
			ModifiedAt: time.Now().UTC(),
		},
		Assignee:    mongomodels.MachineAssignee{},
		UserDeleted: false,
	}

	So(modelhelper.CreateMachine(m1), ShouldBeNil)

	m1ws1 := &mongomodels.Workspace{
		ObjectId:     bson.NewObjectId(),
		OriginId:     ownerUser.ObjectId,
		Name:         "My Workspace",
		Slug:         "m1ws1",
		ChannelId:    strconv.FormatInt(c1.Id, 10),
		MachineUID:   m1.Uid,
		MachineLabel: "koding-vm-0",
		Owner:        "cihangir",
		RootPath:     "/home/cihangir",
		IsDefault:    true,
	}

	So(modelhelper.CreateWorkspace(m1ws1), ShouldBeNil)

	return m1, m1ws1
}
<|endoftext|>


<commit_before>package main

import (
	"log"

	"github.com/mishudark/eventhus"
	"github.com/mishudark/eventhus/commandbus"
	"github.com/mishudark/eventhus/commandhandler/basic"
	"github.com/mishudark/eventhus/eventbus/nats"
	"github.com/mishudark/eventhus/eventstore/mongo"
	"github.com/mishudark/eventhus/examples/bank"
)

func config() (eventhus.CommandBus, error) {
	//register events
	reg := eventhus.NewEventRegister()
	reg.Set(bank.AccountCreated{})
	reg.Set(bank.DepositPerformed{})
	reg.Set(bank.WithdrawalPerformed{})

	//event store
	eventstore, err := mongo.NewClient("localhost", 27017, "bank")
	if err != nil {
		log.Println(err)
		return nil, err
	}

	//eventbus
	//rabbit, err := rabbitmq.NewClient("guest", "guest", "localhost", 5672)
	nat, err := nats.NewClient("nats://ruser:T0pS3cr3t@localhost:4222", false)
	if err != nil {
		log.Println(err)
		return nil, err
	}

	//repository
	repository := eventhus.NewRepository(eventstore, nat)

	//handlers
	commandRegister := eventhus.NewCommandRegister()
	commandHandler := basic.NewHandler(repository, &bank.Account{}, "bank", "account")

	//add commands to commandhandler
	commandRegister.Add(bank.CreateAccount{}, commandHandler)
	commandRegister.Add(bank.PerformDeposit{}, commandHandler)
	commandRegister.Add(bank.PerformWithdrawal{}, commandHandler)

	//commandbus
	commandBus := async.NewBus(commandRegister, 30)
	return commandBus, nil
}
<commit_msg>chore(command_handler): rename Handler -> CommandHandler<commit_after>package main

import (
	"log"

	"github.com/mishudark/eventhus"
	"github.com/mishudark/eventhus/commandbus"
	"github.com/mishudark/eventhus/commandhandler/basic"
	"github.com/mishudark/eventhus/eventbus/nats"
	"github.com/mishudark/eventhus/eventstore/mongo"
	"github.com/mishudark/eventhus/examples/bank"
)

func config() (eventhus.CommandBus, error) {
	//register events
	reg := eventhus.NewEventRegister()
	reg.Set(bank.AccountCreated{})
	reg.Set(bank.DepositPerformed{})
	reg.Set(bank.WithdrawalPerformed{})

	//event store
	eventstore, err := mongo.NewClient("localhost", 27017, "bank")
	if err != nil {
		log.Println(err)
		return nil, err
	}

	//eventbus
	//rabbit, err := rabbitmq.NewClient("guest", "guest", "localhost", 5672)
	nat, err := nats.NewClient("nats://ruser:T0pS3cr3t@localhost:4222", false)
	if err != nil {
		log.Println(err)
		return nil, err
	}

	//repository
	repository := eventhus.NewRepository(eventstore, nat)

	//handlers
	commandRegister := eventhus.NewCommandRegister()
	commandHandler := basic.NewCommandHandler(repository, &bank.Account{}, "bank", "account")

	//add commands to commandhandler
	commandRegister.Add(bank.CreateAccount{}, commandHandler)
	commandRegister.Add(bank.PerformDeposit{}, commandHandler)
	commandRegister.Add(bank.PerformWithdrawal{}, commandHandler)

	//commandbus
	commandBus := async.NewBus(commandRegister, 30)
	return commandBus, nil
}
<|endoftext|>