<commit_before>package dirtree

import (
	"testing"

	"github.com/ncw/rclone/fstest/mockdir"
	"github.com/ncw/rclone/fstest/mockobject"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

func TestNew(t *testing.T) {
	dt := New()
	assert.Equal(t, "", dt.String())
}

func TestParentDir(t *testing.T) {
	assert.Equal(t, "root/parent", parentDir("root/parent/file"))
	assert.Equal(t, "parent", parentDir("parent/file"))
	assert.Equal(t, "", parentDir("parent"))
	assert.Equal(t, "", parentDir(""))
}

func TestDirTreeAdd(t *testing.T) {
	dt := New()
	o := mockobject.New("potato")
	dt.Add(o)
	assert.Equal(t, `/
  potato
`, dt.String())
	o = mockobject.New("dir/subdir/sausage")
	dt.Add(o)
	assert.Equal(t, `/
  potato
dir/subdir/
  sausage
`, dt.String())
}

func TestDirTreeAddDir(t *testing.T) {
	dt := New()
	d := mockdir.New("potato")
	dt.Add(d)
	assert.Equal(t, `/
  potato/
`, dt.String())
	d = mockdir.New("dir/subdir/sausage")
	dt.AddDir(d)
	assert.Equal(t, `/
  potato/
dir/subdir/
  sausage/
dir/subdir/sausage/
`, dt.String())
}

func TestDirTreeAddEntry(t *testing.T) {
	dt := New()

	d := mockdir.New("dir/subdir/sausagedir")
	dt.AddEntry(d)
	o := mockobject.New("dir/subdir2/sausage2")
	dt.AddEntry(o)

	assert.Equal(t, `/
  dir/
dir/
  subdir/
  subdir2/
dir/subdir/
  sausagedir/
dir/subdir/sausagedir/
dir/subdir2/
  sausage2
`, dt.String())
}

func TestDirTreeFind(t *testing.T) {
	dt := New()

	parent, foundObj := dt.Find("dir/subdir/sausage")
	assert.Equal(t, "dir/subdir", parent)
	assert.Nil(t, foundObj)

	o := mockobject.New("dir/subdir/sausage")
	dt.Add(o)

	parent, foundObj = dt.Find("dir/subdir/sausage")
	assert.Equal(t, "dir/subdir", parent)
	assert.Equal(t, o, foundObj)
}

func TestDirTreeCheckParent(t *testing.T) {
	dt := New()

	o := mockobject.New("dir/subdir/sausage")
	dt.Add(o)

	assert.Equal(t, `dir/subdir/
  sausage
`, dt.String())

	dt.CheckParent("", "dir/subdir")

	assert.Equal(t, `/
  dir/
dir/
  subdir/
dir/subdir/
  sausage
`, dt.String())

}

func TestDirTreeCheckParents(t *testing.T) {
	dt := New()

	dt.Add(mockobject.New("dir/subdir/sausage"))
	dt.Add(mockobject.New("dir/subdir2/sausage2"))

	dt.CheckParents("")

	assert.Equal(t, `/
  dir/
dir/
  subdir/
  subdir2/
dir/subdir/
  sausage
dir/subdir2/
  sausage2
`, dt.String())
}

func TestDirTreeSort(t *testing.T) {
	dt := New()

	dt.Add(mockobject.New("dir/subdir/B"))
	dt.Add(mockobject.New("dir/subdir/A"))

	assert.Equal(t, `dir/subdir/
  B
  A
`, dt.String())

	dt.Sort()

	assert.Equal(t, `dir/subdir/
  A
  B
`, dt.String())
}

func TestDirTreeDirs(t *testing.T) {
	dt := New()

	dt.Add(mockobject.New("dir/subdir/sausage"))
	dt.Add(mockobject.New("dir/subdir2/sausage2"))

	dt.CheckParents("")

	assert.Equal(t, []string{
		"",
		"dir",
		"dir/subdir",
		"dir/subdir2",
	}, dt.Dirs())
}

func TestDirTreePrune(t *testing.T) {
	dt := New()

	dt.Add(mockobject.New("file"))
	dt.Add(mockobject.New("dir/subdir/sausage"))
	dt.Add(mockobject.New("dir/subdir2/sausage2"))
	dt.Add(mockobject.New("dir/file"))
	dt.Add(mockobject.New("dir2/file"))

	dt.CheckParents("")

	err := dt.Prune(map[string]bool{
		"dir": true,
	})
	require.NoError(t, err)

	assert.Equal(t, `/
  file
  dir2/
dir2/
  file
`, dt.String())

}
<commit_msg>dirtree: make tests more reliable<commit_after>package dirtree

import (
	"testing"

	"github.com/ncw/rclone/fstest/mockdir"
	"github.com/ncw/rclone/fstest/mockobject"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

func TestNew(t *testing.T) {
	dt := New()
	assert.Equal(t, "", dt.String())
}

func TestParentDir(t *testing.T) {
	assert.Equal(t, "root/parent", parentDir("root/parent/file"))
	assert.Equal(t, "parent", parentDir("parent/file"))
	assert.Equal(t, "", parentDir("parent"))
	assert.Equal(t, "", parentDir(""))
}

func TestDirTreeAdd(t *testing.T) {
	dt := New()
	o := mockobject.New("potato")
	dt.Add(o)
	assert.Equal(t, `/
  potato
`, dt.String())
	o = mockobject.New("dir/subdir/sausage")
	dt.Add(o)
	assert.Equal(t, `/
  potato
dir/subdir/
  sausage
`, dt.String())
}

func TestDirTreeAddDir(t *testing.T) {
	dt := New()
	d := mockdir.New("potato")
	dt.Add(d)
	assert.Equal(t, `/
  potato/
`, dt.String())
	d = mockdir.New("dir/subdir/sausage")
	dt.AddDir(d)
	assert.Equal(t, `/
  potato/
dir/subdir/
  sausage/
dir/subdir/sausage/
`, dt.String())
}

func TestDirTreeAddEntry(t *testing.T) {
	dt := New()

	d := mockdir.New("dir/subdir/sausagedir")
	dt.AddEntry(d)
	o := mockobject.New("dir/subdir2/sausage2")
	dt.AddEntry(o)

	assert.Equal(t, `/
  dir/
dir/
  subdir/
  subdir2/
dir/subdir/
  sausagedir/
dir/subdir/sausagedir/
dir/subdir2/
  sausage2
`, dt.String())
}

func TestDirTreeFind(t *testing.T) {
	dt := New()

	parent, foundObj := dt.Find("dir/subdir/sausage")
	assert.Equal(t, "dir/subdir", parent)
	assert.Nil(t, foundObj)

	o := mockobject.New("dir/subdir/sausage")
	dt.Add(o)

	parent, foundObj = dt.Find("dir/subdir/sausage")
	assert.Equal(t, "dir/subdir", parent)
	assert.Equal(t, o, foundObj)
}

func TestDirTreeCheckParent(t *testing.T) {
	dt := New()

	o := mockobject.New("dir/subdir/sausage")
	dt.Add(o)

	assert.Equal(t, `dir/subdir/
  sausage
`, dt.String())

	dt.CheckParent("", "dir/subdir")

	assert.Equal(t, `/
  dir/
dir/
  subdir/
dir/subdir/
  sausage
`, dt.String())

}

func TestDirTreeCheckParents(t *testing.T) {
	dt := New()

	dt.Add(mockobject.New("dir/subdir/sausage"))
	dt.Add(mockobject.New("dir/subdir2/sausage2"))

	dt.CheckParents("")
	dt.Sort() // sort since the exact order of adding parents is not defined

	assert.Equal(t, `/
  dir/
dir/
  subdir/
  subdir2/
dir/subdir/
  sausage
dir/subdir2/
  sausage2
`, dt.String())
}

func TestDirTreeSort(t *testing.T) {
	dt := New()

	dt.Add(mockobject.New("dir/subdir/B"))
	dt.Add(mockobject.New("dir/subdir/A"))

	assert.Equal(t, `dir/subdir/
  B
  A
`, dt.String())

	dt.Sort()

	assert.Equal(t, `dir/subdir/
  A
  B
`, dt.String())
}

func TestDirTreeDirs(t *testing.T) {
	dt := New()

	dt.Add(mockobject.New("dir/subdir/sausage"))
	dt.Add(mockobject.New("dir/subdir2/sausage2"))

	dt.CheckParents("")

	assert.Equal(t, []string{
		"",
		"dir",
		"dir/subdir",
		"dir/subdir2",
	}, dt.Dirs())
}

func TestDirTreePrune(t *testing.T) {
	dt := New()

	dt.Add(mockobject.New("file"))
	dt.Add(mockobject.New("dir/subdir/sausage"))
	dt.Add(mockobject.New("dir/subdir2/sausage2"))
	dt.Add(mockobject.New("dir/file"))
	dt.Add(mockobject.New("dir2/file"))

	dt.CheckParents("")

	err := dt.Prune(map[string]bool{
		"dir": true,
	})
	require.NoError(t, err)

	assert.Equal(t, `/
  file
  dir2/
dir2/
  file
`, dt.String())

}
<|endoftext|>


<commit_before>package morningStar

import (
	"bytes"
	"github.com/ViBiOh/funds/jsonHttp"
	"log"
	"net/http"
	"regexp"
	"sync"
	"time"
)

const refreshDelayInHours = 6
const maxConcurrentFetcher = 32

var requestList = regexp.MustCompile(`^/list$`)
var requestPerf = regexp.MustCompile(`^/(.+?)$`)

type performance struct {
	ID            string    `json:"id"`
	Isin          string    `json:"isin"`
	Label         string    `json:"label"`
	Category      string    `json:"category"`
	Rating        string    `json:"rating"`
	OneMonth      float64   `json:"1m"`
	ThreeMonths   float64   `json:"3m"`
	SixMonths     float64   `json:"6m"`
	OneYear       float64   `json:"1y"`
	VolThreeYears float64   `json:"v3y"`
	Score         float64   `json:"score"`
	Update        time.Time `json:"ts"`
}

func (perf *performance) computeScore() {
	score := (0.25 * perf.OneMonth) + (0.3 * perf.ThreeMonths) + (0.25 * perf.SixMonths) + (0.2 * perf.OneYear) - (0.1 * perf.VolThreeYears)
	perf.Score = float64(int(score*100)) / 100
}

type results struct {
	Results interface{} `json:"results"`
}

var cacheRequests = make(chan *cacheRequest, maxConcurrentFetcher)

func init() {
	go cacheServer(cacheRequests)
	go func() {
		refreshCache()
		c := time.Tick(refreshDelayInHours * time.Hour)
		for range c {
			refreshCache()
		}
	}()
}

func concurrentRetrievePerformances(ids [][]byte, wg *sync.WaitGroup, performances chan<- *performance, errors chan<- []byte) {
	tokens := make(chan int, maxConcurrentFetcher)

	clearSemaphores := func() {
		wg.Done()
		<-tokens
	}

	for _, id := range ids {
		tokens <- 1

		go func(morningStarID []byte) {
			defer clearSemaphores()
			perf, err := fetchPerformance(morningStarID)
			if err == nil {
				performances <- perf
			} else {
				errors <- morningStarID
			}
		}(id)
	}
}

func retrievePerformances(ids [][]byte) ([]*performance, [][]byte) {
	var wgFetch sync.WaitGroup
	wgFetch.Add(len(ids))

	var wgDrain sync.WaitGroup
	wgDrain.Add(2)

	performancesChan := make(chan *performance, 0)
	errorsChan := make(chan []byte, 0)

	performances := make([]*performance, 0, len(ids))
	errors := make([][]byte, 0)

	go concurrentRetrievePerformances(ids, &wgFetch, performancesChan, errorsChan)

	go func() {
		wgFetch.Wait()
		close(performancesChan)
		close(errorsChan)
	}()

	go func() {
		for perf := range performancesChan {
			performances = append(performances, perf)
		}
		wgDrain.Done()
	}()

	go func() {
		for error := range errorsChan {
			errors = append(errors, error)
		}
		wgDrain.Done()
	}()

	wgDrain.Wait()

	return performances, errors
}

func refreshCache() {
	log.Print(`Cache refresh - start`)
	defer log.Print(`Cache refresh - end`)

	performances, errors := retrievePerformances(morningStarIds)

	if len(errors) > 0 {
		log.Printf(`Errors while refreshing ids %s`, bytes.Join(errors, []byte(`, `)))
	}

	loadCache(cacheRequests, performances)
}

func retrievePerformance(morningStarID []byte) (*performance, error) {
	perf := getCache(cacheRequests, cleanID(morningStarID))
	if perf != nil {
		return perf, nil
	}

	perf, err := fetchPerformance(morningStarID)
	if err != nil {
		return nil, err
	}

	pushCache(cacheRequests, perf)
	morningStarIds = append(morningStarIds, morningStarID)

	return perf, nil
}

func performanceHandler(w http.ResponseWriter, morningStarID []byte) {
	perf, err := retrievePerformance(morningStarID)

	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	} else {
		jsonHttp.ResponseJSON(w, *perf)
	}
}

func listHandler(w http.ResponseWriter, r *http.Request) {
	perfs := make([]*performance, 0, len(morningStarIds))
	for perf := range listCache(cacheRequests) {
		perfs = append(perfs, perf)
	}

	jsonHttp.ResponseJSON(w, results{perfs})
}

// Handler for MorningStar request. Should be use with net/http
type Handler struct {
}

func (handler Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	w.Header().Add(`Access-Control-Allow-Origin`, `*`)
	w.Header().Add(`Access-Control-Allow-Headers`, `Content-Type`)
	w.Header().Add(`Access-Control-Allow-Methods`, `GET`)
	w.Header().Add(`X-Content-Type-Options`, `nosniff`)

	if r.Method == http.MethodOptions {
		w.Write(nil)
		return
	}

	urlPath := []byte(r.URL.Path)

	if requestList.Match(urlPath) {
		listHandler(w, r)
	} else if requestPerf.Match(urlPath) {
		performanceHandler(w, requestPerf.FindSubmatch(urlPath)[1])
	}
}
<commit_msg>Update morningStar.go<commit_after>package morningStar

import (
	"bytes"
	"github.com/ViBiOh/funds/jsonHttp"
	"log"
	"net/http"
	"regexp"
	"sync"
	"time"
)

const refreshDelayInHours = 6
const maxConcurrentFetcher = 32

var requestStatus = regexp.MustCompile(`^/status$`)
var requestList = regexp.MustCompile(`^/list$`)
var requestPerf = regexp.MustCompile(`^/(.+?)$`)

type performance struct {
	ID            string    `json:"id"`
	Isin          string    `json:"isin"`
	Label         string    `json:"label"`
	Category      string    `json:"category"`
	Rating        string    `json:"rating"`
	OneMonth      float64   `json:"1m"`
	ThreeMonths   float64   `json:"3m"`
	SixMonths     float64   `json:"6m"`
	OneYear       float64   `json:"1y"`
	VolThreeYears float64   `json:"v3y"`
	Score         float64   `json:"score"`
	Update        time.Time `json:"ts"`
}

func (perf *performance) computeScore() {
	score := (0.25 * perf.OneMonth) + (0.3 * perf.ThreeMonths) + (0.25 * perf.SixMonths) + (0.2 * perf.OneYear) - (0.1 * perf.VolThreeYears)
	perf.Score = float64(int(score*100)) / 100
}

type results struct {
	Results interface{} `json:"results"`
}

var cacheRequests = make(chan *cacheRequest, maxConcurrentFetcher)

func init() {
	go cacheServer(cacheRequests)
	go func() {
		refreshCache()
		c := time.Tick(refreshDelayInHours * time.Hour)
		for range c {
			refreshCache()
		}
	}()
}

func concurrentRetrievePerformances(ids [][]byte, wg *sync.WaitGroup, performances chan<- *performance, errors chan<- []byte) {
	tokens := make(chan int, maxConcurrentFetcher)

	clearSemaphores := func() {
		wg.Done()
		<-tokens
	}

	for _, id := range ids {
		tokens <- 1

		go func(morningStarID []byte) {
			defer clearSemaphores()
			perf, err := fetchPerformance(morningStarID)
			if err == nil {
				performances <- perf
			} else {
				errors <- morningStarID
			}
		}(id)
	}
}

func retrievePerformances(ids [][]byte) ([]*performance, [][]byte) {
	var wgFetch sync.WaitGroup
	wgFetch.Add(len(ids))

	var wgDrain sync.WaitGroup
	wgDrain.Add(2)

	performancesChan := make(chan *performance, 0)
	errorsChan := make(chan []byte, 0)

	performances := make([]*performance, 0, len(ids))
	errors := make([][]byte, 0)

	go concurrentRetrievePerformances(ids, &wgFetch, performancesChan, errorsChan)

	go func() {
		wgFetch.Wait()
		close(performancesChan)
		close(errorsChan)
	}()

	go func() {
		for perf := range performancesChan {
			performances = append(performances, perf)
		}
		wgDrain.Done()
	}()

	go func() {
		for error := range errorsChan {
			errors = append(errors, error)
		}
		wgDrain.Done()
	}()

	wgDrain.Wait()

	return performances, errors
}

func refreshCache() {
	log.Print(`Cache refresh - start`)
	defer log.Print(`Cache refresh - end`)

	performances, errors := retrievePerformances(morningStarIds)

	if len(errors) > 0 {
		log.Printf(`Errors while refreshing ids %s`, bytes.Join(errors, []byte(`, `)))
	}

	loadCache(cacheRequests, performances)
}

func retrievePerformance(morningStarID []byte) (*performance, error) {
	perf := getCache(cacheRequests, cleanID(morningStarID))
	if perf != nil {
		return perf, nil
	}

	perf, err := fetchPerformance(morningStarID)
	if err != nil {
		return nil, err
	}

	pushCache(cacheRequests, perf)
	morningStarIds = append(morningStarIds, morningStarID)

	return perf, nil
}

func performanceHandler(w http.ResponseWriter, morningStarID []byte) {
	perf, err := retrievePerformance(morningStarID)

	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	} else {
		jsonHttp.ResponseJSON(w, *perf)
	}
}

func listPerformances() []*performance {
	performances := make([]*performance, 0, len(morningStarIds))
	for perf := range listCache(cacheRequests) {
		performances = append(performances, perf)
	}

	return performances
}

func listHandler(w http.ResponseWriter, r *http.Request) {
	jsonHttp.ResponseJSON(w, results{listPerformances(})
}

func statusHandler(w http.ResponseWriter, r *http.Request) {
	if len(listPerformances()) > 0 {
		w.Write(`OK`)
	} else {
		w.Write('KO`)
	}
}

// Handler for MorningStar request. Should be use with net/http
type Handler struct {
}

func (handler Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	w.Header().Add(`Access-Control-Allow-Origin`, `*`)
	w.Header().Add(`Access-Control-Allow-Headers`, `Content-Type`)
	w.Header().Add(`Access-Control-Allow-Methods`, `GET`)
	w.Header().Add(`X-Content-Type-Options`, `nosniff`)

	if r.Method == http.MethodOptions {
		w.Write(nil)
		return
	}

	urlPath := []byte(r.URL.Path)

	if requestList.Match(urlPath) {
		listHandler(w, r)
	} else if requestPerf.Match(urlPath) {
		performanceHandler(w, requestPerf.FindSubmatch(urlPath)[1])
	} else if requestStatus.Match(urlPath) {
		statusHandler(w, r)
	}
}
<|endoftext|>


<commit_before>package models

import (
    "github.com/orc/db"
    "strconv"
)

type ParamValuesModel struct {
    Entity
}

type ParamValues struct {
    Id      int    `name:"id" type:"int" null:"NOT NULL" extra:"PRIMARY"`
    ParamId int    `name:"param_id" type:"int" null:"NOT NULL" extra:"REFERENCES" refTable:"params" refField:"id" refFieldShow:"name"`
    Value   string `name:"value" type:"text" null:"NULL" extra:""`
}

func (c *ModelManager) ParamValues() *ParamValuesModel {
    model := new(ParamValuesModel)

    model.TableName = "param_values"
    model.Caption = "Значение параметров"

    model.Columns = []string{"id", "param_id", "value"}
    model.ColNames = []string{"ID", "Параметр", "Значение"}

    model.Fields = new(ParamValues)
    model.WherePart = make(map[string]interface{}, 0)
    model.Condition = AND
    model.OrderBy = "id"
    model.Limit = "ALL"
    model.Offset = 0

    model.Sub = false
    model.SubTable = nil
    model.SubField = ""

    return model
}

func (this *ParamValuesModel) Select(fields []string, filters map[string]interface{}, limit, offset int, sord, sidx string) (result []interface{}) {
    if len(fields) == 0 {
        return nil
    }

    query := `SELECT `

    for _, field := range fields {
        switch field {
        case "id":
            query += "param_values.id, "
            break
        case "param_id":
            query += "params.name, "
            break
        case "value":
            query += "param_values.value, "
            break
        }
    }

    query = query[:len(query)-2]

    query += ` FROM param_values INNER JOIN params ON params.id = param_values.param_id`

    where, params := this.Where(filters)
    query += where

    if sidx != "" {
        query += ` ORDER BY param_values.`+sidx
    }

    query += ` `+ sord

    if limit != -1 {
        params = append(params, limit)
        query += ` LIMIT $`+strconv.Itoa(len(params))
    }

    if offset != -1 {
        params = append(params, offset)
        query += ` OFFSET $`+strconv.Itoa(len(params))
    }

    query += `;`

    return db.Query(query, params)
}
<commit_msg>add form name in param name of value<commit_after>package models

import (
    "github.com/orc/db"
    "strconv"
)

type ParamValuesModel struct {
    Entity
}

type ParamValues struct {
    Id      int    `name:"id" type:"int" null:"NOT NULL" extra:"PRIMARY"`
    ParamId int    `name:"param_id" type:"int" null:"NOT NULL" extra:"REFERENCES" refTable:"params" refField:"id" refFieldShow:"name"`
    Value   string `name:"value" type:"text" null:"NULL" extra:""`
}

func (c *ModelManager) ParamValues() *ParamValuesModel {
    model := new(ParamValuesModel)

    model.TableName = "param_values"
    model.Caption = "Значение параметров"

    model.Columns = []string{"id", "param_id", "value"}
    model.ColNames = []string{"ID", "Параметр", "Значение"}

    model.Fields = new(ParamValues)
    model.WherePart = make(map[string]interface{}, 0)
    model.Condition = AND
    model.OrderBy = "id"
    model.Limit = "ALL"
    model.Offset = 0

    model.Sub = false
    model.SubTable = nil
    model.SubField = ""

    return model
}

func (this *ParamValuesModel) GetModelRefDate() (fields []string, result map[string]interface{}) {
    fields = []string{"name"}

    query := `SELECT params.id, forms.name || ': ' || params.name as name
        FROM params
        INNER JOIN forms ON forms.id = params.form_id ORDER BY params.id`

    return fields, map[string]interface{}{"param_id": db.Query(query, nil)}
}

func (this *ParamValuesModel) Select(fields []string, filters map[string]interface{}, limit, offset int, sord, sidx string) (result []interface{}) {
    if len(fields) == 0 {
        return nil
    }

    query := `SELECT `

    for _, field := range fields {
        switch field {
        case "id":
            query += "param_values.id, "
            break
        case "param_id":
            query += "forms.name || ': ' || params.name as name, "
            break
        case "value":
            query += "param_values.value, "
            break
        }
    }

    query = query[:len(query)-2]

    query += ` FROM param_values
        INNER JOIN params ON params.id = param_values.param_id
        INNER JOIN forms ON forms.id = params.form_id`

    where, params := this.Where(filters)
    query += where

    if sidx != "" {
        query += ` ORDER BY param_values.`+sidx
    }

    query += ` `+ sord

    if limit != -1 {
        params = append(params, limit)
        query += ` LIMIT $`+strconv.Itoa(len(params))
    }

    if offset != -1 {
        params = append(params, offset)
        query += ` OFFSET $`+strconv.Itoa(len(params))
    }

    query += `;`

    return db.Query(query, params)
}
<|endoftext|>


<commit_before>package nodefflag

import (
	"flag"
	"testing"
	"time"
)

func TestEverything(t *testing.T) {
	fs := NewNDFlagSet("NDflag_test", flag.ExitOnError)
	fs.NDBool("test_bool", "bool value")
	fs.NDInt("test_int", "int value")
	fs.NDInt64("test_int64", "int64 value")
	fs.NDUint("test_uint", "uint value")
	fs.NDUint64("test_uint64", "uint64 value")
	fs.NDString("test_string", "string value")
	fs.NDFloat64("test_float64", "float64 value")
	fs.NDDuration("test_duration", "time.Duration value")

	visitor := func(f *flag.Flag) {
		g, ok := f.Value.(flag.Getter)
		if !ok {
			t.Errorf("Visit: value does not satisfy Getter: %T", f.Value)
		}
		switch f.Name {
		case "test_bool":
			ok = g.Get() == (*bool)(nil)
		case "test_int":
			ok = g.Get() == (*int)(nil)
		case "test_int64":
			ok = g.Get() == (*int64)(nil)
		case "test_uint":
			ok = g.Get() == (*uint)(nil)
		case "test_uint64":
			ok = g.Get() == (*uint64)(nil)
		case "test_string":
			ok = g.Get() == (*string)(nil)
		case "test_float64":
			ok = g.Get() == (*float64)(nil)
		case "test_duration":
			ok = g.Get() == (*time.Duration)(nil)
		}
		if !ok {
			t.Errorf("Visit: bad value %T(%v) for %s", g.Get(), g.Get(), f.Name)
		}
	}
	fs.VisitAll(visitor)

	_ = fs.Set("test_bool", "false")
	_ = fs.Set("test_int", "42")
	_ = fs.Set("test_int64", "-420")
	_ = fs.Set("test_uint", "80")
	_ = fs.Set("test_uint64", "800")
	_ = fs.Set("test_string", "your ad here")
	_ = fs.Set("test_float64", "123.45")
	_ = fs.Set("test_duration", "30s")

	visitor = func(f *flag.Flag) {
		var ok bool
		g := f.Value.(flag.Getter)
		switch f.Name {
		case "test_bool":
			ok = *(g.Get().(*bool)) == false
		case "test_int":
			ok = *(g.Get().(*int)) == 42
		case "test_int64":
			ok = *(g.Get().(*int64)) == int64(-420)
		case "test_uint":
			ok = *(g.Get().(*uint)) == uint(80)
		case "test_uint64":
			ok = *(g.Get().(*uint64)) == uint64(800)
		case "test_string":
			ok = *(g.Get().(*string)) == "your ad here"
		case "test_float64":
			ok = *(g.Get().(*float64)) == float64(123.45)
		case "test_duration":
			ok = g.Get().(*time.Duration).String() == "30s"
		}
		if !ok {
			t.Errorf("Visit: bad value %T(%v) for %s", g.Get(), g.Get(), f.Name)
		}
	}

	fs.Visit(visitor)
}
<commit_msg>better test<commit_after>package nodefflag

import (
	"flag"
	"testing"
	"time"
)

func TestNils(t *testing.T) {
	fs := nfs()
	visitor := func(f *flag.Flag) {
		g, ok := f.Value.(flag.Getter)
		if !ok {
			t.Errorf("Visit: value does not satisfy Getter: %T", f.Value)
		}
		switch f.Name {
		case "test_bool":
			ok = g.Get() == (*bool)(nil)
		case "test_int":
			ok = g.Get() == (*int)(nil)
		case "test_int64":
			ok = g.Get() == (*int64)(nil)
		case "test_uint":
			ok = g.Get() == (*uint)(nil)
		case "test_uint64":
			ok = g.Get() == (*uint64)(nil)
		case "test_string":
			ok = g.Get() == (*string)(nil)
		case "test_float64":
			ok = g.Get() == (*float64)(nil)
		case "test_duration":
			ok = g.Get() == (*time.Duration)(nil)
		}
		if !ok {
			t.Errorf("Visit: bad value %T(%v) for %s", g.Get(), g.Get(), f.Name)
		}
	}
	fs.VisitAll(visitor)
}
func TestSet(t *testing.T) {
	fs := nfs()

	_ = fs.Set("test_bool", "false")
	_ = fs.Set("test_int", "42")
	_ = fs.Set("test_int64", "-420")
	_ = fs.Set("test_uint", "80")
	_ = fs.Set("test_uint64", "800")
	_ = fs.Set("test_string", "your ad here")
	_ = fs.Set("test_float64", "123.45")
	_ = fs.Set("test_duration", "30s")

	visitor := func(f *flag.Flag) {
		var ok bool
		g := f.Value.(flag.Getter)
		switch f.Name {
		case "test_bool":
			ok = *(g.Get().(*bool)) == false
		case "test_int":
			ok = *(g.Get().(*int)) == 42
		case "test_int64":
			ok = *(g.Get().(*int64)) == int64(-420)
		case "test_uint":
			ok = *(g.Get().(*uint)) == uint(80)
		case "test_uint64":
			ok = *(g.Get().(*uint64)) == uint64(800)
		case "test_string":
			ok = *(g.Get().(*string)) == "your ad here"
		case "test_float64":
			ok = *(g.Get().(*float64)) == float64(123.45)
		case "test_duration":
			ok = g.Get().(*time.Duration).String() == "30s"
		}
		if !ok {
			t.Errorf("Visit: bad value %T(%v) for %s", g.Get(), g.Get(), f.Name)
		}
	}
	fs.Visit(visitor)

}

func TestPtrsMatch(t *testing.T) {
	fs := NewNDFlagSet("NDflag_test", flag.ExitOnError)
	var (
		bv    *bool
		iv    *int
		i64v  *int64
		uiv   *uint
		ui64v *uint64
		sv    *string
		fv    *float64
		dv    *time.Duration
	)
	fs.NDBoolVar(&bv, "test_bool", "bool value")
	fs.NDIntVar(&iv, "test_int", "int value")
	fs.NDInt64Var(&i64v, "test_int64", "int64 value")
	fs.NDUintVar(&uiv, "test_uint", "uint value")
	fs.NDUint64Var(&ui64v, "test_uint64", "uint64 value")
	fs.NDStringVar(&sv, "test_string", "string value")
	fs.NDFloat64Var(&fv, "test_float64", "float64 value")
	fs.NDDurationVar(&dv, "test_duration", "time.Duration value")

	_ = fs.Set("test_bool", "true")
	_ = fs.Set("test_int", "42")
	_ = fs.Set("test_int64", "-420")
	_ = fs.Set("test_uint", "80")
	_ = fs.Set("test_uint64", "800")
	_ = fs.Set("test_string", "your ad here")
	_ = fs.Set("test_float64", "123.45")
	_ = fs.Set("test_duration", "30s")
	visitor := func(f *flag.Flag) {
		var ok bool
		g := f.Value.(flag.Getter)
		switch f.Name {
		case "test_bool":
			ok = *bv == true
		case "test_int":
			ok = *iv == 42
		case "test_int64":
			ok = *i64v == int64(-420)
		case "test_uint":
			ok = *uiv == uint(80)
		case "test_uint64":
			ok = *ui64v == uint64(800)
		case "test_string":
			ok = *sv == "your ad here"

		case "test_float64":
			ok = *fv == float64(123.45)
		case "test_duration":
			ok = dv.String() == "30s"
		}
		if !ok {
			t.Errorf("Visit: bad value %T(%#v)%p for %s", g.Get(), g.Get(), g.Get(), f.Name)
		}
	}
	fs.Visit(visitor)
}

func nfs() *NDFlagSet {
	fs := NewNDFlagSet("NDflag_test", flag.ExitOnError)
	fs.NDBool("test_bool", "bool value")
	fs.NDInt("test_int", "int value")
	fs.NDInt64("test_int64", "int64 value")
	fs.NDUint("test_uint", "uint value")
	fs.NDUint64("test_uint64", "uint64 value")
	fs.NDString("test_string", "string value")
	fs.NDFloat64("test_float64", "float64 value")
	fs.NDDuration("test_duration", "time.Duration value")
	return fs
}
<|endoftext|>


<commit_before>// Copyright 2012 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package tar

import (
	"bytes"
	"io/ioutil"
	"os"
	"reflect"
	"testing"
	"time"
)

func TestFileInfoHeader(t *testing.T) {
	fi, err := os.Stat("testdata/small.txt")
	if err != nil {
		t.Fatal(err)
	}
	h, err := FileInfoHeader(fi, "")
	if err != nil {
		t.Fatalf("FileInfoHeader: %v", err)
	}
	if g, e := h.Name, "small.txt"; g != e {
		t.Errorf("Name = %q; want %q", g, e)
	}
	if g, e := h.Mode, int64(fi.Mode().Perm())|c_ISREG; g != e {
		t.Errorf("Mode = %#o; want %#o", g, e)
	}
	if g, e := h.Size, int64(5); g != e {
		t.Errorf("Size = %v; want %v", g, e)
	}
	if g, e := h.ModTime, fi.ModTime(); !g.Equal(e) {
		t.Errorf("ModTime = %v; want %v", g, e)
	}
}

func TestFileInfoHeaderDir(t *testing.T) {
	fi, err := os.Stat("testdata")
	if err != nil {
		t.Fatal(err)
	}
	h, err := FileInfoHeader(fi, "")
	if err != nil {
		t.Fatalf("FileInfoHeader: %v", err)
	}
	if g, e := h.Name, "testdata/"; g != e {
		t.Errorf("Name = %q; want %q", g, e)
	}
	const setsid = 02000 // see golang.org/issue/4867
	if g, e := h.Mode&^setsid, int64(fi.Mode().Perm())|c_ISDIR; g != e {
		t.Errorf("Mode = %#o; want %#o", g, e)
	}
	if g, e := h.Size, int64(0); g != e {
		t.Errorf("Size = %v; want %v", g, e)
	}
	if g, e := h.ModTime, fi.ModTime(); !g.Equal(e) {
		t.Errorf("ModTime = %v; want %v", g, e)
	}
}

func TestFileInfoHeaderSymlink(t *testing.T) {
	h, err := FileInfoHeader(symlink{}, "some-target")
	if err != nil {
		t.Fatal(err)
	}
	if g, e := h.Name, "some-symlink"; g != e {
		t.Errorf("Name = %q; want %q", g, e)
	}
	if g, e := h.Linkname, "some-target"; g != e {
		t.Errorf("Linkname = %q; want %q", g, e)
	}
}

type symlink struct{}

func (symlink) Name() string       { return "some-symlink" }
func (symlink) Size() int64        { return 0 }
func (symlink) Mode() os.FileMode  { return os.ModeSymlink }
func (symlink) ModTime() time.Time { return time.Time{} }
func (symlink) IsDir() bool        { return false }
func (symlink) Sys() interface{}   { return nil }

func TestRoundTrip(t *testing.T) {
	data := []byte("some file contents")

	var b bytes.Buffer
	tw := NewWriter(&b)
	hdr := &Header{
		Name:    "file.txt",
		Uid:     1 << 21, // too big for 8 octal digits
		Size:    int64(len(data)),
		ModTime: time.Now(),
	}
	// tar only supports second precision.
	hdr.ModTime = hdr.ModTime.Add(-time.Duration(hdr.ModTime.Nanosecond()) * time.Nanosecond)
	if err := tw.WriteHeader(hdr); err != nil {
		t.Fatalf("tw.WriteHeader: %v", err)
	}
	if _, err := tw.Write(data); err != nil {
		t.Fatalf("tw.Write: %v", err)
	}
	if err := tw.Close(); err != nil {
		t.Fatalf("tw.Close: %v", err)
	}

	// Read it back.
	tr := NewReader(&b)
	rHdr, err := tr.Next()
	if err != nil {
		t.Fatalf("tr.Next: %v", err)
	}
	if !reflect.DeepEqual(rHdr, hdr) {
		t.Errorf("Header mismatch.\n got %+v\nwant %+v", rHdr, hdr)
	}
	rData, err := ioutil.ReadAll(tr)
	if err != nil {
		t.Fatalf("Read: %v", err)
	}
	if !bytes.Equal(rData, data) {
		t.Errorf("Data mismatch.\n got %q\nwant %q", rData, data)
	}
}

type headerRoundTripTest struct {
	h  *Header
	fm os.FileMode
}

func TestHeaderRoundTrip(t *testing.T) {
	golden := []headerRoundTripTest{
		// regular file.
		{
			h: &Header{
				Name:     "test.txt",
				Mode:     0644 | c_ISREG,
				Size:     12,
				ModTime:  time.Unix(1360600916, 0),
				Typeflag: TypeReg,
			},
			fm: 0644,
		},
		// hard link.
		{
			h: &Header{
				Name:     "hard.txt",
				Mode:     0644 | c_ISLNK,
				Size:     0,
				ModTime:  time.Unix(1360600916, 0),
				Typeflag: TypeLink,
			},
			fm: 0644 | os.ModeSymlink,
		},
		// symbolic link.
		{
			h: &Header{
				Name:     "link.txt",
				Mode:     0777 | c_ISLNK,
				Size:     0,
				ModTime:  time.Unix(1360600852, 0),
				Typeflag: TypeSymlink,
			},
			fm: 0777 | os.ModeSymlink,
		},
		// character device node.
		{
			h: &Header{
				Name:     "dev/null",
				Mode:     0666 | c_ISCHR,
				Size:     0,
				ModTime:  time.Unix(1360578951, 0),
				Typeflag: TypeChar,
			},
			fm: 0666 | os.ModeDevice | os.ModeCharDevice,
		},
		// block device node.
		{
			h: &Header{
				Name:     "dev/sda",
				Mode:     0660 | c_ISBLK,
				Size:     0,
				ModTime:  time.Unix(1360578954, 0),
				Typeflag: TypeBlock,
			},
			fm: 0660 | os.ModeDevice,
		},
		// directory.
		{
			h: &Header{
				Name:     "dir/",
				Mode:     0755 | c_ISDIR,
				Size:     0,
				ModTime:  time.Unix(1360601116, 0),
				Typeflag: TypeDir,
			},
			fm: 0755 | os.ModeDir,
		},
		// fifo node.
		{
			h: &Header{
				Name:     "dev/initctl",
				Mode:     0600 | c_ISFIFO,
				Size:     0,
				ModTime:  time.Unix(1360578949, 0),
				Typeflag: TypeFifo,
			},
			fm: 0600 | os.ModeNamedPipe,
		},
		// setuid.
		{
			h: &Header{
				Name:     "bin/su",
				Mode:     0755 | c_ISREG | c_ISUID,
				Size:     23232,
				ModTime:  time.Unix(1355405093, 0),
				Typeflag: TypeReg,
			},
			fm: 0755 | os.ModeSetuid,
		},
		// setguid.
		{
			h: &Header{
				Name:     "group.txt",
				Mode:     0750 | c_ISREG | c_ISGID,
				Size:     0,
				ModTime:  time.Unix(1360602346, 0),
				Typeflag: TypeReg,
			},
			fm: 0750 | os.ModeSetgid,
		},
		// sticky.
		{
			h: &Header{
				Name:     "sticky.txt",
				Mode:     0600 | c_ISREG | c_ISVTX,
				Size:     7,
				ModTime:  time.Unix(1360602540, 0),
				Typeflag: TypeReg,
			},
			fm: 0600 | os.ModeSticky,
		},
	}

	for i, g := range golden {
		fi := g.h.FileInfo()
		h2, err := FileInfoHeader(fi, "")
		if err != nil {
			t.Error(err)
			continue
		}
		if got, want := h2.Name, g.h.Name; got != want {
			t.Errorf("i=%d: Name: got %v, want %v", i, got, want)
		}
		if got, want := h2.Size, g.h.Size; got != want {
			t.Errorf("i=%d: Size: got %v, want %v", i, got, want)
		}
		if got, want := h2.Mode, g.h.Mode; got != want {
			t.Errorf("i=%d: Mode: got %o, want %o", i, got, want)
		}
		if got, want := fi.Mode(), g.fm; got != want {
			t.Errorf("i=%d: fi.Mode: got %o, want %o", i, got, want)
		}
		if got, want := h2.ModTime, g.h.ModTime; got != want {
			t.Errorf("i=%d: ModTime: got %v, want %v", i, got, want)
		}
		if sysh, ok := fi.Sys().(*Header); !ok || sysh != g.h {
			t.Errorf("i=%d: Sys didn't return original *Header", i)
		}
	}
}
<commit_msg>archive/tar: simplify use of constants in test case.<commit_after>// Copyright 2012 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package tar

import (
	"bytes"
	"io/ioutil"
	"os"
	"reflect"
	"testing"
	"time"
)

func TestFileInfoHeader(t *testing.T) {
	fi, err := os.Stat("testdata/small.txt")
	if err != nil {
		t.Fatal(err)
	}
	h, err := FileInfoHeader(fi, "")
	if err != nil {
		t.Fatalf("FileInfoHeader: %v", err)
	}
	if g, e := h.Name, "small.txt"; g != e {
		t.Errorf("Name = %q; want %q", g, e)
	}
	if g, e := h.Mode, int64(fi.Mode().Perm())|c_ISREG; g != e {
		t.Errorf("Mode = %#o; want %#o", g, e)
	}
	if g, e := h.Size, int64(5); g != e {
		t.Errorf("Size = %v; want %v", g, e)
	}
	if g, e := h.ModTime, fi.ModTime(); !g.Equal(e) {
		t.Errorf("ModTime = %v; want %v", g, e)
	}
}

func TestFileInfoHeaderDir(t *testing.T) {
	fi, err := os.Stat("testdata")
	if err != nil {
		t.Fatal(err)
	}
	h, err := FileInfoHeader(fi, "")
	if err != nil {
		t.Fatalf("FileInfoHeader: %v", err)
	}
	if g, e := h.Name, "testdata/"; g != e {
		t.Errorf("Name = %q; want %q", g, e)
	}
	// Ignoring c_ISGID for golang.org/issue/4867
	if g, e := h.Mode&^c_ISGID, int64(fi.Mode().Perm())|c_ISDIR; g != e {
		t.Errorf("Mode = %#o; want %#o", g, e)
	}
	if g, e := h.Size, int64(0); g != e {
		t.Errorf("Size = %v; want %v", g, e)
	}
	if g, e := h.ModTime, fi.ModTime(); !g.Equal(e) {
		t.Errorf("ModTime = %v; want %v", g, e)
	}
}

func TestFileInfoHeaderSymlink(t *testing.T) {
	h, err := FileInfoHeader(symlink{}, "some-target")
	if err != nil {
		t.Fatal(err)
	}
	if g, e := h.Name, "some-symlink"; g != e {
		t.Errorf("Name = %q; want %q", g, e)
	}
	if g, e := h.Linkname, "some-target"; g != e {
		t.Errorf("Linkname = %q; want %q", g, e)
	}
}

type symlink struct{}

func (symlink) Name() string       { return "some-symlink" }
func (symlink) Size() int64        { return 0 }
func (symlink) Mode() os.FileMode  { return os.ModeSymlink }
func (symlink) ModTime() time.Time { return time.Time{} }
func (symlink) IsDir() bool        { return false }
func (symlink) Sys() interface{}   { return nil }

func TestRoundTrip(t *testing.T) {
	data := []byte("some file contents")

	var b bytes.Buffer
	tw := NewWriter(&b)
	hdr := &Header{
		Name:    "file.txt",
		Uid:     1 << 21, // too big for 8 octal digits
		Size:    int64(len(data)),
		ModTime: time.Now(),
	}
	// tar only supports second precision.
	hdr.ModTime = hdr.ModTime.Add(-time.Duration(hdr.ModTime.Nanosecond()) * time.Nanosecond)
	if err := tw.WriteHeader(hdr); err != nil {
		t.Fatalf("tw.WriteHeader: %v", err)
	}
	if _, err := tw.Write(data); err != nil {
		t.Fatalf("tw.Write: %v", err)
	}
	if err := tw.Close(); err != nil {
		t.Fatalf("tw.Close: %v", err)
	}

	// Read it back.
	tr := NewReader(&b)
	rHdr, err := tr.Next()
	if err != nil {
		t.Fatalf("tr.Next: %v", err)
	}
	if !reflect.DeepEqual(rHdr, hdr) {
		t.Errorf("Header mismatch.\n got %+v\nwant %+v", rHdr, hdr)
	}
	rData, err := ioutil.ReadAll(tr)
	if err != nil {
		t.Fatalf("Read: %v", err)
	}
	if !bytes.Equal(rData, data) {
		t.Errorf("Data mismatch.\n got %q\nwant %q", rData, data)
	}
}

type headerRoundTripTest struct {
	h  *Header
	fm os.FileMode
}

func TestHeaderRoundTrip(t *testing.T) {
	golden := []headerRoundTripTest{
		// regular file.
		{
			h: &Header{
				Name:     "test.txt",
				Mode:     0644 | c_ISREG,
				Size:     12,
				ModTime:  time.Unix(1360600916, 0),
				Typeflag: TypeReg,
			},
			fm: 0644,
		},
		// hard link.
		{
			h: &Header{
				Name:     "hard.txt",
				Mode:     0644 | c_ISLNK,
				Size:     0,
				ModTime:  time.Unix(1360600916, 0),
				Typeflag: TypeLink,
			},
			fm: 0644 | os.ModeSymlink,
		},
		// symbolic link.
		{
			h: &Header{
				Name:     "link.txt",
				Mode:     0777 | c_ISLNK,
				Size:     0,
				ModTime:  time.Unix(1360600852, 0),
				Typeflag: TypeSymlink,
			},
			fm: 0777 | os.ModeSymlink,
		},
		// character device node.
		{
			h: &Header{
				Name:     "dev/null",
				Mode:     0666 | c_ISCHR,
				Size:     0,
				ModTime:  time.Unix(1360578951, 0),
				Typeflag: TypeChar,
			},
			fm: 0666 | os.ModeDevice | os.ModeCharDevice,
		},
		// block device node.
		{
			h: &Header{
				Name:     "dev/sda",
				Mode:     0660 | c_ISBLK,
				Size:     0,
				ModTime:  time.Unix(1360578954, 0),
				Typeflag: TypeBlock,
			},
			fm: 0660 | os.ModeDevice,
		},
		// directory.
		{
			h: &Header{
				Name:     "dir/",
				Mode:     0755 | c_ISDIR,
				Size:     0,
				ModTime:  time.Unix(1360601116, 0),
				Typeflag: TypeDir,
			},
			fm: 0755 | os.ModeDir,
		},
		// fifo node.
		{
			h: &Header{
				Name:     "dev/initctl",
				Mode:     0600 | c_ISFIFO,
				Size:     0,
				ModTime:  time.Unix(1360578949, 0),
				Typeflag: TypeFifo,
			},
			fm: 0600 | os.ModeNamedPipe,
		},
		// setuid.
		{
			h: &Header{
				Name:     "bin/su",
				Mode:     0755 | c_ISREG | c_ISUID,
				Size:     23232,
				ModTime:  time.Unix(1355405093, 0),
				Typeflag: TypeReg,
			},
			fm: 0755 | os.ModeSetuid,
		},
		// setguid.
		{
			h: &Header{
				Name:     "group.txt",
				Mode:     0750 | c_ISREG | c_ISGID,
				Size:     0,
				ModTime:  time.Unix(1360602346, 0),
				Typeflag: TypeReg,
			},
			fm: 0750 | os.ModeSetgid,
		},
		// sticky.
		{
			h: &Header{
				Name:     "sticky.txt",
				Mode:     0600 | c_ISREG | c_ISVTX,
				Size:     7,
				ModTime:  time.Unix(1360602540, 0),
				Typeflag: TypeReg,
			},
			fm: 0600 | os.ModeSticky,
		},
	}

	for i, g := range golden {
		fi := g.h.FileInfo()
		h2, err := FileInfoHeader(fi, "")
		if err != nil {
			t.Error(err)
			continue
		}
		if got, want := h2.Name, g.h.Name; got != want {
			t.Errorf("i=%d: Name: got %v, want %v", i, got, want)
		}
		if got, want := h2.Size, g.h.Size; got != want {
			t.Errorf("i=%d: Size: got %v, want %v", i, got, want)
		}
		if got, want := h2.Mode, g.h.Mode; got != want {
			t.Errorf("i=%d: Mode: got %o, want %o", i, got, want)
		}
		if got, want := fi.Mode(), g.fm; got != want {
			t.Errorf("i=%d: fi.Mode: got %o, want %o", i, got, want)
		}
		if got, want := h2.ModTime, g.h.ModTime; got != want {
			t.Errorf("i=%d: ModTime: got %v, want %v", i, got, want)
		}
		if sysh, ok := fi.Sys().(*Header); !ok || sysh != g.h {
			t.Errorf("i=%d: Sys didn't return original *Header", i)
		}
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package model

import (
	"fmt"
	"path/filepath"

	"golang.org/x/sys/unix"
	"k8s.io/kops/upup/pkg/fi"
	"k8s.io/kops/upup/pkg/fi/nodeup/nodetasks"
)

// NetworkBuilder writes CNI assets
type NetworkBuilder struct {
	*NodeupModelContext
}

var _ fi.ModelBuilder = &NetworkBuilder{}

// Build is responsible for configuring the network cni
func (b *NetworkBuilder) Build(c *fi.ModelBuilderContext) error {
	var assetNames []string

	// @TODO need to clean up this code, it isn't the easiest to read
	networking := b.Cluster.Spec.Networking
	if networking == nil || networking.Classic != nil {
	} else if networking.Kubenet != nil || networking.GCE != nil {
		assetNames = append(assetNames, "bridge", "host-local", "loopback")
	} else if networking.External != nil {
		// external is based on kubenet
		assetNames = append(assetNames, "bridge", "host-local", "loopback")

	} else if networking.CNI != nil || networking.Weave != nil || networking.Flannel != nil || networking.Calico != nil || networking.Canal != nil || networking.Kuberouter != nil || networking.Romana != nil || networking.AmazonVPC != nil || networking.Cilium != nil {
		assetNames = append(assetNames, "bridge", "host-local", "loopback", "ptp", "portmap")
		// Do we need tuning?

		// TODO: Only when using flannel ?
		assetNames = append(assetNames, "flannel")
	} else if networking.Kopeio != nil {
		// TODO combine with External
		// Kopeio is based on kubenet / external
		assetNames = append(assetNames, "bridge", "host-local", "loopback")
	} else if networking.LyftVPC != nil {
		assetNames = append(assetNames, "cni-ipvlan-vpc-k8s-ipam", "cni-ipvlan-vpc-k8s-ipvlan", "cni-ipvlan-vpc-k8s-tool", "cni-ipvlan-vpc-k8s-unnumbered-ptp", "loopback")
	} else {
		return fmt.Errorf("no networking mode set")
	}

	for _, assetName := range assetNames {
		if err := b.addCNIBinAsset(c, assetName); err != nil {
			return err
		}
	}

	if networking.Cilium != nil {
		// systemd v238 includes the bpffs mount by default; and gives an error "has a bad unit file setting" if we try to mount it again (see mount_point_is_api)
		var alreadyMounted bool
		// bpffs magic number
		magic := uint32(0xCAFE4A11)
		var fsdata unix.Statfs_t
		err := unix.Statfs("/sys/fs/bpf", &fsdata)

		if err != nil {
			alreadyMounted = false
		} else {
			alreadyMounted = int32(magic) == int32(fsdata.Type)
		}

		if !alreadyMounted {
			unit := s(`
[Unit]
Description=Cilium BPF mounts
Documentation=http://docs.cilium.io/
DefaultDependencies=no
Before=local-fs.target umount.target kubelet.service

[Mount]
What=bpffs
Where=/sys/fs/bpf
Type=bpf

[Install]
WantedBy=multi-user.target
`)

			service := &nodetasks.Service{
				Name:       "sys-fs-bpf.mount",
				Definition: unit,
			}
			service.InitDefaults()
			c.AddTask(service)
		}
	}

	return nil
}

func (b *NetworkBuilder) addCNIBinAsset(c *fi.ModelBuilderContext, assetName string) error {
	assetPath := ""
	asset, err := b.Assets.Find(assetName, assetPath)
	if err != nil {
		return fmt.Errorf("error trying to locate asset %q: %v", assetName, err)
	}
	if asset == nil {
		return fmt.Errorf("unable to locate asset %q", assetName)
	}

	c.AddTask(&nodetasks.File{
		Path:     filepath.Join(b.CNIBinDir(), assetName),
		Contents: asset,
		Type:     nodetasks.FileType_File,
		Mode:     s("0755"),
	})

	return nil
}
<commit_msg>Return error if statfs fails<commit_after>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package model

import (
	"fmt"
	"path/filepath"

	"golang.org/x/sys/unix"
	"k8s.io/kops/upup/pkg/fi"
	"k8s.io/kops/upup/pkg/fi/nodeup/nodetasks"
)

// NetworkBuilder writes CNI assets
type NetworkBuilder struct {
	*NodeupModelContext
}

var _ fi.ModelBuilder = &NetworkBuilder{}

// Build is responsible for configuring the network cni
func (b *NetworkBuilder) Build(c *fi.ModelBuilderContext) error {
	var assetNames []string

	// @TODO need to clean up this code, it isn't the easiest to read
	networking := b.Cluster.Spec.Networking
	if networking == nil || networking.Classic != nil {
	} else if networking.Kubenet != nil || networking.GCE != nil {
		assetNames = append(assetNames, "bridge", "host-local", "loopback")
	} else if networking.External != nil {
		// external is based on kubenet
		assetNames = append(assetNames, "bridge", "host-local", "loopback")

	} else if networking.CNI != nil || networking.Weave != nil || networking.Flannel != nil || networking.Calico != nil || networking.Canal != nil || networking.Kuberouter != nil || networking.Romana != nil || networking.AmazonVPC != nil || networking.Cilium != nil {
		assetNames = append(assetNames, "bridge", "host-local", "loopback", "ptp", "portmap")
		// Do we need tuning?

		// TODO: Only when using flannel ?
		assetNames = append(assetNames, "flannel")
	} else if networking.Kopeio != nil {
		// TODO combine with External
		// Kopeio is based on kubenet / external
		assetNames = append(assetNames, "bridge", "host-local", "loopback")
	} else if networking.LyftVPC != nil {
		assetNames = append(assetNames, "cni-ipvlan-vpc-k8s-ipam", "cni-ipvlan-vpc-k8s-ipvlan", "cni-ipvlan-vpc-k8s-tool", "cni-ipvlan-vpc-k8s-unnumbered-ptp", "loopback")
	} else {
		return fmt.Errorf("no networking mode set")
	}

	for _, assetName := range assetNames {
		if err := b.addCNIBinAsset(c, assetName); err != nil {
			return err
		}
	}

	if networking.Cilium != nil {
		// systemd v238 includes the bpffs mount by default; and gives an error "has a bad unit file setting" if we try to mount it again (see mount_point_is_api)
		var alreadyMounted bool
		// bpffs magic number
		magic := uint32(0xCAFE4A11)
		var fsdata unix.Statfs_t
		err := unix.Statfs("/sys/fs/bpf", &fsdata)

		if err != nil {
			return fmt.Errorf("error checking for /sys/fs/bpf: %v", err)
		} else {
			alreadyMounted = int32(magic) == int32(fsdata.Type)
		}

		if !alreadyMounted {
			unit := s(`
[Unit]
Description=Cilium BPF mounts
Documentation=http://docs.cilium.io/
DefaultDependencies=no
Before=local-fs.target umount.target kubelet.service

[Mount]
What=bpffs
Where=/sys/fs/bpf
Type=bpf

[Install]
WantedBy=multi-user.target
`)

			service := &nodetasks.Service{
				Name:       "sys-fs-bpf.mount",
				Definition: unit,
			}
			service.InitDefaults()
			c.AddTask(service)
		}
	}

	return nil
}

func (b *NetworkBuilder) addCNIBinAsset(c *fi.ModelBuilderContext, assetName string) error {
	assetPath := ""
	asset, err := b.Assets.Find(assetName, assetPath)
	if err != nil {
		return fmt.Errorf("error trying to locate asset %q: %v", assetName, err)
	}
	if asset == nil {
		return fmt.Errorf("unable to locate asset %q", assetName)
	}

	c.AddTask(&nodetasks.File{
		Path:     filepath.Join(b.CNIBinDir(), assetName),
		Contents: asset,
		Type:     nodetasks.FileType_File,
		Mode:     s("0755"),
	})

	return nil
}
<|endoftext|>


<commit_before>// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package tls

import (
	"testing"
)

func TestRoundUp(t *testing.T) {
	if roundUp(0, 16) != 0 ||
		roundUp(1, 16) != 16 ||
		roundUp(15, 16) != 16 ||
		roundUp(16, 16) != 16 ||
		roundUp(17, 16) != 32 {
		t.Error("roundUp broken")
	}
}

var paddingTests = []struct {
	in          []byte
	good        bool
	expectedLen int
}{
	{[]byte{1, 2, 3, 4, 0}, true, 4},
	{[]byte{1, 2, 3, 4, 0, 1}, false, 0},
	{[]byte{1, 2, 3, 4, 99, 99}, false, 0},
	{[]byte{1, 2, 3, 4, 1, 1}, true, 4},
	{[]byte{1, 2, 3, 2, 2, 2}, true, 3},
	{[]byte{1, 2, 3, 3, 3, 3}, true, 2},
	{[]byte{1, 2, 3, 4, 3, 3}, false, 0},
	{[]byte{1, 4, 4, 4, 4, 4}, true, 1},
	{[]byte{5, 5, 5, 5, 5, 5}, true, 0},
	{[]byte{6, 6, 6, 6, 6, 6}, false, 0},
}

func TestRemovePadding(t *testing.T) {
	for i, test := range paddingTests {
		payload, good := removePadding(test.in)
		expectedGood := byte(255)
		if !test.good {
			expectedGood = 0
		}
		if good != expectedGood {
			t.Errorf("#%d: wrong validity, want:%d got:%d", expectedGood, good)
		}
		if good == 255 && len(payload) != test.expectedLen {
			t.Errorf("#%d: got %d, want %d", i, len(payload), test.expectedLen)
		}
	}
}
<commit_msg>tls: fix argument mistake in Error call.<commit_after>// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package tls

import (
	"testing"
)

func TestRoundUp(t *testing.T) {
	if roundUp(0, 16) != 0 ||
		roundUp(1, 16) != 16 ||
		roundUp(15, 16) != 16 ||
		roundUp(16, 16) != 16 ||
		roundUp(17, 16) != 32 {
		t.Error("roundUp broken")
	}
}

var paddingTests = []struct {
	in          []byte
	good        bool
	expectedLen int
}{
	{[]byte{1, 2, 3, 4, 0}, true, 4},
	{[]byte{1, 2, 3, 4, 0, 1}, false, 0},
	{[]byte{1, 2, 3, 4, 99, 99}, false, 0},
	{[]byte{1, 2, 3, 4, 1, 1}, true, 4},
	{[]byte{1, 2, 3, 2, 2, 2}, true, 3},
	{[]byte{1, 2, 3, 3, 3, 3}, true, 2},
	{[]byte{1, 2, 3, 4, 3, 3}, false, 0},
	{[]byte{1, 4, 4, 4, 4, 4}, true, 1},
	{[]byte{5, 5, 5, 5, 5, 5}, true, 0},
	{[]byte{6, 6, 6, 6, 6, 6}, false, 0},
}

func TestRemovePadding(t *testing.T) {
	for i, test := range paddingTests {
		payload, good := removePadding(test.in)
		expectedGood := byte(255)
		if !test.good {
			expectedGood = 0
		}
		if good != expectedGood {
			t.Errorf("#%d: wrong validity, want:%d got:%d", i, expectedGood, good)
		}
		if good == 255 && len(payload) != test.expectedLen {
			t.Errorf("#%d: got %d, want %d", i, len(payload), test.expectedLen)
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2010 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package pprof serves via its HTTP server runtime profiling data
// in the format expected by the pprof visualization tool.
// For more information about pprof, see
// http://code.google.com/p/google-perftools/.
//
// The package is typically only imported for the side effect of
// registering its HTTP handlers.
// The handled paths all begin with /debug/pprof/.
//
// To use pprof, link this package into your program:
//	import _ "net/http/pprof"
//
// Then use the pprof tool to look at the heap profile:
//
//	go tool pprof http://localhost:6060/debug/pprof/heap
//
// Or to look at a 30-second CPU profile:
//
//	go tool pprof http://localhost:6060/debug/pprof/profile
//
// Or to look at the thread creation profile:
//
//	go tool pprof http://localhost:6060/debug/pprof/thread
//
package pprof

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"net/http"
	"os"
	"runtime"
	"runtime/pprof"
	"strconv"
	"strings"
	"time"
)

func init() {
	http.Handle("/debug/pprof/cmdline", http.HandlerFunc(Cmdline))
	http.Handle("/debug/pprof/profile", http.HandlerFunc(Profile))
	http.Handle("/debug/pprof/heap", http.HandlerFunc(Heap))
	http.Handle("/debug/pprof/symbol", http.HandlerFunc(Symbol))
	http.Handle("/debug/pprof/thread", http.HandlerFunc(Thread))
}

// Cmdline responds with the running program's
// command line, with arguments separated by NUL bytes.
// The package initialization registers it as /debug/pprof/cmdline.
func Cmdline(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "text/plain; charset=utf-8")
	fmt.Fprintf(w, strings.Join(os.Args, "\x00"))
}

// Heap responds with the pprof-formatted heap profile.
// The package initialization registers it as /debug/pprof/heap.
func Heap(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "text/plain; charset=utf-8")
	pprof.WriteHeapProfile(w)
}

// Thread responds with the pprof-formatted thread creation profile.
// The package initialization registers it as /debug/pprof/thread.
func Thread(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "text/plain; charset=utf-8")
	pprof.WriteThreadProfile(w)
}

// Profile responds with the pprof-formatted cpu profile.
// The package initialization registers it as /debug/pprof/profile.
func Profile(w http.ResponseWriter, r *http.Request) {
	sec, _ := strconv.ParseInt(r.FormValue("seconds"), 10, 64)
	if sec == 0 {
		sec = 30
	}

	// Set Content Type assuming StartCPUProfile will work,
	// because if it does it starts writing.
	w.Header().Set("Content-Type", "application/octet-stream")
	if err := pprof.StartCPUProfile(w); err != nil {
		// StartCPUProfile failed, so no writes yet.
		// Can change header back to text content
		// and send error code.
		w.Header().Set("Content-Type", "text/plain; charset=utf-8")
		w.WriteHeader(http.StatusInternalServerError)
		fmt.Fprintf(w, "Could not enable CPU profiling: %s\n", err)
		return
	}
	time.Sleep(time.Duration(sec) * time.Second)
	pprof.StopCPUProfile()
}

// Symbol looks up the program counters listed in the request,
// responding with a table mapping program counters to function names.
// The package initialization registers it as /debug/pprof/symbol.
func Symbol(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "text/plain; charset=utf-8")

	// We have to read the whole POST body before
	// writing any output.  Buffer the output here.
	var buf bytes.Buffer

	// We don't know how many symbols we have, but we
	// do have symbol information.  Pprof only cares whether
	// this number is 0 (no symbols available) or > 0.
	fmt.Fprintf(&buf, "num_symbols: 1\n")

	var b *bufio.Reader
	if r.Method == "POST" {
		b = bufio.NewReader(r.Body)
	} else {
		b = bufio.NewReader(strings.NewReader(r.URL.RawQuery))
	}

	for {
		word, err := b.ReadSlice('+')
		if err == nil {
			word = word[0 : len(word)-1] // trim +
		}
		pc, _ := strconv.ParseUint(string(word), 0, 64)
		if pc != 0 {
			f := runtime.FuncForPC(uintptr(pc))
			if f != nil {
				fmt.Fprintf(&buf, "%#x %s\n", pc, f.Name())
			}
		}

		// Wait until here to check for err; the last
		// symbol will have an err because it doesn't end in +.
		if err != nil {
			if err != io.EOF {
				fmt.Fprintf(&buf, "reading request: %v\n", err)
			}
			break
		}
	}

	w.Write(buf.Bytes())
}
<commit_msg>net/http/pprof: link to blog post<commit_after>// Copyright 2010 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package pprof serves via its HTTP server runtime profiling data
// in the format expected by the pprof visualization tool.
// For more information about pprof, see
// http://code.google.com/p/google-perftools/.
//
// The package is typically only imported for the side effect of
// registering its HTTP handlers.
// The handled paths all begin with /debug/pprof/.
//
// To use pprof, link this package into your program:
//	import _ "net/http/pprof"
//
// Then use the pprof tool to look at the heap profile:
//
//	go tool pprof http://localhost:6060/debug/pprof/heap
//
// Or to look at a 30-second CPU profile:
//
//	go tool pprof http://localhost:6060/debug/pprof/profile
//
// Or to look at the thread creation profile:
//
//	go tool pprof http://localhost:6060/debug/pprof/thread
//
// For a study of the facility in action, visit
//
//	http://blog.golang.org/2011/06/profiling-go-programs.html
//
package pprof

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"net/http"
	"os"
	"runtime"
	"runtime/pprof"
	"strconv"
	"strings"
	"time"
)

func init() {
	http.Handle("/debug/pprof/cmdline", http.HandlerFunc(Cmdline))
	http.Handle("/debug/pprof/profile", http.HandlerFunc(Profile))
	http.Handle("/debug/pprof/heap", http.HandlerFunc(Heap))
	http.Handle("/debug/pprof/symbol", http.HandlerFunc(Symbol))
	http.Handle("/debug/pprof/thread", http.HandlerFunc(Thread))
}

// Cmdline responds with the running program's
// command line, with arguments separated by NUL bytes.
// The package initialization registers it as /debug/pprof/cmdline.
func Cmdline(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "text/plain; charset=utf-8")
	fmt.Fprintf(w, strings.Join(os.Args, "\x00"))
}

// Heap responds with the pprof-formatted heap profile.
// The package initialization registers it as /debug/pprof/heap.
func Heap(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "text/plain; charset=utf-8")
	pprof.WriteHeapProfile(w)
}

// Thread responds with the pprof-formatted thread creation profile.
// The package initialization registers it as /debug/pprof/thread.
func Thread(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "text/plain; charset=utf-8")
	pprof.WriteThreadProfile(w)
}

// Profile responds with the pprof-formatted cpu profile.
// The package initialization registers it as /debug/pprof/profile.
func Profile(w http.ResponseWriter, r *http.Request) {
	sec, _ := strconv.ParseInt(r.FormValue("seconds"), 10, 64)
	if sec == 0 {
		sec = 30
	}

	// Set Content Type assuming StartCPUProfile will work,
	// because if it does it starts writing.
	w.Header().Set("Content-Type", "application/octet-stream")
	if err := pprof.StartCPUProfile(w); err != nil {
		// StartCPUProfile failed, so no writes yet.
		// Can change header back to text content
		// and send error code.
		w.Header().Set("Content-Type", "text/plain; charset=utf-8")
		w.WriteHeader(http.StatusInternalServerError)
		fmt.Fprintf(w, "Could not enable CPU profiling: %s\n", err)
		return
	}
	time.Sleep(time.Duration(sec) * time.Second)
	pprof.StopCPUProfile()
}

// Symbol looks up the program counters listed in the request,
// responding with a table mapping program counters to function names.
// The package initialization registers it as /debug/pprof/symbol.
func Symbol(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "text/plain; charset=utf-8")

	// We have to read the whole POST body before
	// writing any output.  Buffer the output here.
	var buf bytes.Buffer

	// We don't know how many symbols we have, but we
	// do have symbol information.  Pprof only cares whether
	// this number is 0 (no symbols available) or > 0.
	fmt.Fprintf(&buf, "num_symbols: 1\n")

	var b *bufio.Reader
	if r.Method == "POST" {
		b = bufio.NewReader(r.Body)
	} else {
		b = bufio.NewReader(strings.NewReader(r.URL.RawQuery))
	}

	for {
		word, err := b.ReadSlice('+')
		if err == nil {
			word = word[0 : len(word)-1] // trim +
		}
		pc, _ := strconv.ParseUint(string(word), 0, 64)
		if pc != 0 {
			f := runtime.FuncForPC(uintptr(pc))
			if f != nil {
				fmt.Fprintf(&buf, "%#x %s\n", pc, f.Name())
			}
		}

		// Wait until here to check for err; the last
		// symbol will have an err because it doesn't end in +.
		if err != nil {
			if err != io.EOF {
				fmt.Fprintf(&buf, "reading request: %v\n", err)
			}
			break
		}
	}

	w.Write(buf.Bytes())
}
<|endoftext|>


<commit_before>package storage

import (
	"os"
	"sync"
	"syscall"
	"time"

	"github.com/containers/storage/pkg/stringid"
)

// A Locker represents a file lock where the file is used to cache an
// identifier of the last party that made changes to whatever's being protected
// by the lock.
type Locker interface {
	sync.Locker

	// Touch records, for others sharing the lock, that it was updated by the
	// caller.  It should only be called with the lock held.
	Touch() error

	// Modified() checks if the most recent writer was a party other than the
	// caller.  It should only be called with the lock held.
	Modified() (bool, error)
}

type lockfile struct {
	file string
	fd   uintptr
	me   string
}

// GetLockfile opens a lock file, creating it if necessary.  The Locker object
// return will be returned unlocked.
func GetLockfile(path string) (Locker, error) {
	fd, err := syscall.Open(path, os.O_RDWR|os.O_CREATE, syscall.S_IRUSR|syscall.S_IWUSR)
	if err != nil {
		return nil, err
	}
	return &lockfile{file: path, fd: uintptr(fd)}, nil
}

func (l *lockfile) Lock() {
	lk := syscall.Flock_t{
		Type:   syscall.F_WRLCK,
		Whence: int16(os.SEEK_SET),
		Start:  0,
		Len:    0,
		Pid:    int32(os.Getpid()),
	}
	for syscall.FcntlFlock(l.fd, syscall.F_SETLKW, &lk) != nil {
		time.Sleep(10 * time.Millisecond)
	}
}

func (l *lockfile) Unlock() {
	lk := syscall.Flock_t{
		Type:   syscall.F_UNLCK,
		Whence: int16(os.SEEK_SET),
		Start:  0,
		Len:    0,
		Pid:    int32(os.Getpid()),
	}
	for syscall.FcntlFlock(l.fd, syscall.F_SETLKW, &lk) != nil {
		time.Sleep(10 * time.Millisecond)
	}
}

func (l *lockfile) Touch() error {
	l.me = stringid.GenerateRandomID()
	id := []byte(l.me)
	_, err := syscall.Seek(int(l.fd), 0, os.SEEK_SET)
	if err != nil {
		return err
	}
	n, err := syscall.Write(int(l.fd), id)
	if err != nil {
		return err
	}
	if n != len(id) {
		return syscall.ENOSPC
	}
	return nil
}

func (l *lockfile) Modified() (bool, error) {
	id := []byte(l.me)
	_, err := syscall.Seek(int(l.fd), 0, os.SEEK_SET)
	if err != nil {
		return true, err
	}
	n, err := syscall.Read(int(l.fd), id)
	if err != nil {
		return true, err
	}
	if n != len(id) {
		return true, syscall.ENOSPC
	}
	return string(id) != l.me, nil
}
<commit_msg>Make use of lockfile goroutine-safe, too<commit_after>package storage

import (
	"os"
	"path/filepath"
	"sync"
	"syscall"
	"time"

	"github.com/containers/storage/pkg/stringid"
)

// A Locker represents a file lock where the file is used to cache an
// identifier of the last party that made changes to whatever's being protected
// by the lock.
type Locker interface {
	sync.Locker

	// Touch records, for others sharing the lock, that it was updated by the
	// caller.  It should only be called with the lock held.
	Touch() error

	// Modified() checks if the most recent writer was a party other than the
	// caller.  It should only be called with the lock held.
	Modified() (bool, error)
}

type lockfile struct {
	mu   sync.Mutex
	file string
	fd   uintptr
	me   string
}

var (
	lockfiles     map[string]*lockfile
	lockfilesLock sync.Mutex
)

// GetLockfile opens a lock file, creating it if necessary.  The Locker object
// return will be returned unlocked.
func GetLockfile(path string) (Locker, error) {
	lockfilesLock.Lock()
	defer lockfilesLock.Unlock()
	if locker, ok := lockfiles[filepath.Clean(path)]; ok {
		return locker, nil
	}
	fd, err := syscall.Open(filepath.Clean(path), os.O_RDWR|os.O_CREATE, syscall.S_IRUSR|syscall.S_IWUSR)
	if err != nil {
		return nil, err
	}
	locker := &lockfile{file: path, fd: uintptr(fd)}
	lockfiles[filepath.Clean(path)] = locker
	return locker, nil
}

func (l *lockfile) Lock() {
	lk := syscall.Flock_t{
		Type:   syscall.F_WRLCK,
		Whence: int16(os.SEEK_SET),
		Start:  0,
		Len:    0,
		Pid:    int32(os.Getpid()),
	}
	l.mu.Lock()
	for syscall.FcntlFlock(l.fd, syscall.F_SETLKW, &lk) != nil {
		time.Sleep(10 * time.Millisecond)
	}
}

func (l *lockfile) Unlock() {
	lk := syscall.Flock_t{
		Type:   syscall.F_UNLCK,
		Whence: int16(os.SEEK_SET),
		Start:  0,
		Len:    0,
		Pid:    int32(os.Getpid()),
	}
	for syscall.FcntlFlock(l.fd, syscall.F_SETLKW, &lk) != nil {
		time.Sleep(10 * time.Millisecond)
	}
	l.mu.Unlock()
}

func (l *lockfile) Touch() error {
	l.me = stringid.GenerateRandomID()
	id := []byte(l.me)
	_, err := syscall.Seek(int(l.fd), 0, os.SEEK_SET)
	if err != nil {
		return err
	}
	n, err := syscall.Write(int(l.fd), id)
	if err != nil {
		return err
	}
	if n != len(id) {
		return syscall.ENOSPC
	}
	return nil
}

func (l *lockfile) Modified() (bool, error) {
	id := []byte(l.me)
	_, err := syscall.Seek(int(l.fd), 0, os.SEEK_SET)
	if err != nil {
		return true, err
	}
	n, err := syscall.Read(int(l.fd), id)
	if err != nil {
		return true, err
	}
	if n != len(id) {
		return true, syscall.ENOSPC
	}
	return string(id) != l.me, nil
}
<|endoftext|>


<commit_before>package streamtols

import (
	"containers/heap"
	"github.com/bitly/go-simplejson"
	"log"
	"time"
)

func Join(inChan chan *simplejson.Json, outChan chan *simplejson.Json, RuleChan chan *simplejson.Json) {
	rules := <-RuleChan
	timeout, err := rules.Get("timeout").Float64()
	if err != nil {
		log.Fatal(err.Error())
	}
	key, err := rules.Get("key").Float64()
	if err != nil {
		log.Fatal(err.Error())
	}
	waitTimer := time.NewTimer(100 * time.Millisecond)
	cache = make(map[[]byte]*simplejson.Json)
	pq := &PriorityQueue{}
	heap.Init(pq)

	for {
		select {
		case msg := <-inChan:
			id, err := msg.Get(key).Bytes()

			cachedMsg = cache[id]

			if cachedMsg == nil {
				cache[id] = msg
				val, err := simplejson.NewJson([]byte("{}"))
				val.Set("id", id)
				valEncoded, err := val.Encode()
				if err != nil {
					log.Fatal(err.Error())
				}
				queueMessage := &PQMessage{
					val: &valEncoded,
					t:   time.Now(),
				}
				heap.Push(pq, queueMessage)
			} else {

                outChan <- join(cachedMsg, msg)

            }

		case <-waitTimer.C:
		}
		for {
			pqMsg, diff := pq.PeekAndShift(time.Now(), after)
			if pqMsg == nil {
				// either the queue is empty, or it's not time to emit
				waitTimer.Reset(diff)
				break
			}
			v := pqMsg.(*PQMessage).val
			queueMessage, err := simplejson.NewJson(*v)
			if err != nil {
				log.Fatal(err.Error())
			}

		}

	}

}
<commit_msg>moved join.go into drafts<commit_after><|endoftext|>


<commit_before>package service

import (
	"crypto/rsa"
	"fmt"
	"io/ioutil"
	"net/http"
	"strings"

	"github.com/dgrijalva/jwt-go"
	"github.com/gorilla/context"
	"github.com/microdevs/missy/config"
	"github.com/microdevs/missy/log"
)

var pubkey *rsa.PublicKey

func initPublicKey() {
	pubkeyLocation := config.GetInstance().Authorization.PublicKeyFile
	pubkeyPEM, err := ioutil.ReadFile(pubkeyLocation)
	if err != nil {
		log.Fatal("Unable to load public key file for token auth: ", err)
	}

	pkey, err := jwt.ParseRSAPublicKeyFromPEM(pubkeyPEM)
	if err != nil {
		log.Panic("Unable to parse public key for token auth: ", err)
	}
	pubkey = pkey
}

// StartTimerHandler is a middleware to start a timer for the request benchmark
func StartTimerHandler(h http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		context.Set(r, RequestTimer, NewTimer())
		h.ServeHTTP(w, r)
	})
}

// AuthHandler is a middleware to authenticating a user or machine by validating an JWT auth token passed in the header of the request
func AuthHandler(h http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		reqToken := r.Header.Get("Authorization")
		if reqToken == "" {
			http.Error(w, "Unauthorized", http.StatusUnauthorized)
			return
		}
		splitToken := strings.Split(reqToken, "Bearer ")
		reqToken = splitToken[1]
		claims := jwt.MapClaims{}
		token, err := jwt.ParseWithClaims(reqToken, &claims, func(t *jwt.Token) (interface{}, error) {
			return pubkey, nil
		})
		if err != nil {
			http.Error(w, "invalid token format", http.StatusBadRequest)
			return
		}

		if !token.Valid {
			log.Warn("Invalid token")
			http.Error(w, "Unauthorized", http.StatusUnauthorized)
			return
		}

		context.Set(r, "token", token)

		h.ServeHTTP(w, r)
	})
}

// FinalHandler measures the time of the request with the help of the timestamp taken in StartTimerHandler
// and writes it to a Prometheus metric. It will also write a log line of the request in the log file
func FinalHandler(h http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		mw := &ResponseWriter{w, http.StatusOK}
		h.ServeHTTP(mw, r)
		log.Infof("%s \"%s %s %s\" %d - %s", r.RemoteAddr, r.Method, r.URL, r.Proto, mw.Status, r.UserAgent())
		timer := context.Get(r, RequestTimer).(*Timer)
		prometheus := context.Get(r, PrometheusInstance).(*PrometheusHolder)
		prometheus.OnRequestFinished(r.Method, r.URL.Path, mw.Status, timer.durationMillis())
	})
}

// infoHandler
func (s *Service) infoHandler(w http.ResponseWriter, r *http.Request) {
	info := fmt.Sprintf("Name %s\nUptime %s", s.name, s.timer.Uptime())
	w.Write([]byte(info))
}

// healthHandler
func (s *Service) healthHandler(w http.ResponseWriter, r *http.Request) {
	w.Write([]byte("OK"))
}
<commit_msg>refactor(service): using log.Fatal instead of log.Panic<commit_after>package service

import (
	"crypto/rsa"
	"fmt"
	"io/ioutil"
	"net/http"
	"strings"

	"github.com/dgrijalva/jwt-go"
	"github.com/gorilla/context"
	"github.com/microdevs/missy/config"
	"github.com/microdevs/missy/log"
)

var pubkey *rsa.PublicKey

func initPublicKey() {
	pubkeyLocation := config.GetInstance().Authorization.PublicKeyFile
	pubkeyPEM, err := ioutil.ReadFile(pubkeyLocation)
	if err != nil {
		log.Fatal("Unable to load public key file for token auth: ", err)
	}

	pkey, err := jwt.ParseRSAPublicKeyFromPEM(pubkeyPEM)
	if err != nil {
		log.Fatal("Unable to parse public key for token auth: ", err)
	}
	pubkey = pkey
}

// StartTimerHandler is a middleware to start a timer for the request benchmark
func StartTimerHandler(h http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		context.Set(r, RequestTimer, NewTimer())
		h.ServeHTTP(w, r)
	})
}

// AuthHandler is a middleware to authenticating a user or machine by validating an JWT auth token passed in the header of the request
func AuthHandler(h http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		reqToken := r.Header.Get("Authorization")
		if reqToken == "" {
			http.Error(w, "Unauthorized", http.StatusUnauthorized)
			return
		}
		splitToken := strings.Split(reqToken, "Bearer ")
		reqToken = splitToken[1]
		claims := jwt.MapClaims{}
		token, err := jwt.ParseWithClaims(reqToken, &claims, func(t *jwt.Token) (interface{}, error) {
			return pubkey, nil
		})
		if err != nil {
			http.Error(w, "invalid token format", http.StatusBadRequest)
			return
		}

		if !token.Valid {
			log.Warn("Invalid token")
			http.Error(w, "Unauthorized", http.StatusUnauthorized)
			return
		}

		context.Set(r, "token", token)

		h.ServeHTTP(w, r)
	})
}

// FinalHandler measures the time of the request with the help of the timestamp taken in StartTimerHandler
// and writes it to a Prometheus metric. It will also write a log line of the request in the log file
func FinalHandler(h http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		mw := &ResponseWriter{w, http.StatusOK}
		h.ServeHTTP(mw, r)
		log.Infof("%s \"%s %s %s\" %d - %s", r.RemoteAddr, r.Method, r.URL, r.Proto, mw.Status, r.UserAgent())
		timer := context.Get(r, RequestTimer).(*Timer)
		prometheus := context.Get(r, PrometheusInstance).(*PrometheusHolder)
		prometheus.OnRequestFinished(r.Method, r.URL.Path, mw.Status, timer.durationMillis())
	})
}

// infoHandler
func (s *Service) infoHandler(w http.ResponseWriter, r *http.Request) {
	info := fmt.Sprintf("Name %s\nUptime %s", s.name, s.timer.Uptime())
	w.Write([]byte(info))
}

// healthHandler
func (s *Service) healthHandler(w http.ResponseWriter, r *http.Request) {
	w.Write([]byte("OK"))
}
<|endoftext|>


<commit_before>package gif

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"time"

	"github.com/ethereum/go-ethereum/log"
	"github.com/status-im/status-go/multiaccounts/accounts"
)

type Gif struct {
	ID         string `json:"id"`
	Title      string `json:"title"`
	URL        string `json:"url"`
	TinyURL    string `json:"tinyUrl"`
	Height     int    `json:"height"`
	IsFavorite bool   `json:"isFavorite"`
}

type Container struct {
	Items []Gif `json:"items"`
}

var tenorAPIKey = ""
var defaultParams = "&media_filter=minimal&limit=50&key="

const maxRetry = 3
const baseURL = "https://g.tenor.com/v1/"

func NewGifAPI(db *accounts.Database) *API {
	return &API{db, &http.Client{Timeout: time.Minute}}
}

// API is class with methods available over RPC.
type API struct {
	db         *accounts.Database
	httpClient *http.Client
}

func (api *API) SetTenorAPIKey(key string) (err error) {
	log.Info("[GifAPI::SetTenorAPIKey]")
	err = api.db.SaveSetting("gifs/api-key", key)
	if err != nil {
		return err
	}
	tenorAPIKey = key
	return nil
}

func (api *API) GetContentWithRetry(path string) (value string, err error) {
	var currentRetry = 0
	var response *http.Response

	for currentRetry < maxRetry {
		response, err = api.httpClient.Get(baseURL + path + defaultParams + tenorAPIKey)
		if err != nil {
			log.Error("can't get content from path %s", path)
			currentRetry++
			time.Sleep(100 * time.Millisecond)
		} else {
			break
		}
	}

	defer response.Body.Close()

	if response.StatusCode != http.StatusOK {
		return "", fmt.Errorf("Status error: %v", response.StatusCode)
	}

	data, err := ioutil.ReadAll(response.Body)
	if err != nil {
		return "", fmt.Errorf("Read body: %v", err)
	}

	return string(data), nil
}

func (api *API) FetchGifs(path string) (value string, err error) {
	log.Info("[GifAPI::fetchGifs]")
	return api.GetContentWithRetry(path)
}

func (api *API) UpdateRecentGifs(updatedGifs json.RawMessage) (err error) {
	log.Info("[GifAPI::updateRecentGifs]")
	recentGifsContainer := Container{}
	err = json.Unmarshal(updatedGifs, &recentGifsContainer)
	if err != nil {
		return err
	}
	err = api.db.SaveSetting("gifs/recent-gifs", recentGifsContainer.Items)
	if err != nil {
		return err
	}
	return nil
}

func (api *API) UpdateFavoriteGifs(updatedGifs json.RawMessage) (err error) {
	log.Info("[GifAPI::updateFavoriteGifs]", updatedGifs)
	favsGifsContainer := Container{}
	err = json.Unmarshal(updatedGifs, &favsGifsContainer)
	if err != nil {
		return err
	}
	err = api.db.SaveSetting("gifs/favorite-gifs", favsGifsContainer.Items)
	if err != nil {
		return err
	}
	return nil
}

func (api *API) GetRecentGifs() (recentGifs []Gif, err error) {
	log.Info("[GifAPI::getRecentGifs]")
	gifs, err := api.db.GifRecents()
	if err != nil {
		return nil, err
	}
	savedRecentGifs := []Gif{}
	err = json.Unmarshal(gifs, &savedRecentGifs)
	if err != nil {
		return nil, err
	}
	recentGifs = savedRecentGifs
	return recentGifs, nil
}

func (api *API) GetFavoriteGifs() (favoriteGifs []Gif, err error) {
	log.Info("[GifAPI::getFavoriteGifs]")
	gifs, err := api.db.GifFavorites()
	if err != nil {
		return nil, err
	}
	savedFavGifs := []Gif{}
	err = json.Unmarshal(gifs, &savedFavGifs)
	if err != nil {
		return nil, err
	}
	favoriteGifs = savedFavGifs
	return favoriteGifs, nil
}
<commit_msg>fix(@gif): fix retry when connection lost (#2520)<commit_after>package gif

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"time"

	"github.com/ethereum/go-ethereum/log"
	"github.com/status-im/status-go/multiaccounts/accounts"
)

type Gif struct {
	ID         string `json:"id"`
	Title      string `json:"title"`
	URL        string `json:"url"`
	TinyURL    string `json:"tinyUrl"`
	Height     int    `json:"height"`
	IsFavorite bool   `json:"isFavorite"`
}

type Container struct {
	Items []Gif `json:"items"`
}

var tenorAPIKey = ""
var defaultParams = "&media_filter=minimal&limit=50&key="

const maxRetry = 3
const baseURL = "https://g.tenor.com/v1/"

func NewGifAPI(db *accounts.Database) *API {
	return &API{db}
}

// API is class with methods available over RPC.
type API struct {
	db *accounts.Database
}

func (api *API) SetTenorAPIKey(key string) (err error) {
	log.Info("[GifAPI::SetTenorAPIKey]")
	err = api.db.SaveSetting("gifs/api-key", key)
	if err != nil {
		return err
	}
	tenorAPIKey = key
	return nil
}

func (api *API) GetContentWithRetry(path string) (value string, err error) {
	var currentRetry = 0
	var response *http.Response
	for currentRetry < maxRetry {
		transport := &http.Transport{
			Proxy:                 http.ProxyFromEnvironment,
			ResponseHeaderTimeout: time.Second * 1,
		}

		client := http.Client{
			Timeout:   1 * time.Second,
			Transport: transport,
		}

		response, err = client.Get(baseURL + path + defaultParams + tenorAPIKey)

		if err != nil {
			log.Error("can't get content from path %s with %s", path, err.Error())
			currentRetry++
			time.Sleep(100 * time.Millisecond)
		} else {
			break
		}
	}

	if response == nil {
		return "", fmt.Errorf("Could not reach Tenor API")
	}
	defer response.Body.Close()

	if response.StatusCode != http.StatusOK {
		return "", fmt.Errorf("Status error: %v", response.StatusCode)
	}

	data, err := ioutil.ReadAll(response.Body)
	if err != nil {
		return "", fmt.Errorf("Read body: %v", err)
	}

	return string(data), nil
}

func (api *API) FetchGifs(path string) (value string, err error) {
	log.Info("[GifAPI::fetchGifs]")
	return api.GetContentWithRetry(path)
}

func (api *API) UpdateRecentGifs(updatedGifs json.RawMessage) (err error) {
	log.Info("[GifAPI::updateRecentGifs]")
	recentGifsContainer := Container{}
	err = json.Unmarshal(updatedGifs, &recentGifsContainer)
	if err != nil {
		return err
	}
	err = api.db.SaveSetting("gifs/recent-gifs", recentGifsContainer.Items)
	if err != nil {
		return err
	}
	return nil
}

func (api *API) UpdateFavoriteGifs(updatedGifs json.RawMessage) (err error) {
	log.Info("[GifAPI::updateFavoriteGifs]", updatedGifs)
	favsGifsContainer := Container{}
	err = json.Unmarshal(updatedGifs, &favsGifsContainer)
	if err != nil {
		return err
	}
	err = api.db.SaveSetting("gifs/favorite-gifs", favsGifsContainer.Items)
	if err != nil {
		return err
	}
	return nil
}

func (api *API) GetRecentGifs() (recentGifs []Gif, err error) {
	log.Info("[GifAPI::getRecentGifs]")
	gifs, err := api.db.GifRecents()
	if err != nil {
		return nil, err
	}
	savedRecentGifs := []Gif{}
	err = json.Unmarshal(gifs, &savedRecentGifs)
	if err != nil {
		return nil, err
	}
	recentGifs = savedRecentGifs
	return recentGifs, nil
}

func (api *API) GetFavoriteGifs() (favoriteGifs []Gif, err error) {
	log.Info("[GifAPI::getFavoriteGifs]")
	gifs, err := api.db.GifFavorites()
	if err != nil {
		return nil, err
	}
	savedFavGifs := []Gif{}
	err = json.Unmarshal(gifs, &savedFavGifs)
	if err != nil {
		return nil, err
	}
	favoriteGifs = savedFavGifs
	return favoriteGifs, nil
}
<|endoftext|>


<commit_before>// Copyright 2018 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package submission

import (
	"context"
	"fmt"
	"math/rand"
	"strings"
	"sync"
	"time"

	ct "github.com/google/certificate-transparency-go"
	"github.com/google/certificate-transparency-go/ctpolicy"
)

const (
	// PostBatchInterval is duration between parallel batch call and subsequent
	// requests to Logs within group.
	// TODO(Mercurrent): optimize to avoid excessive requests.
	PostBatchInterval = time.Second
)

// Submitter is interface wrapping Log-request-response cycle and any processing.
type Submitter interface {
	SubmitToLog(ctx context.Context, logURL string, chain []ct.ASN1Cert) (*ct.SignedCertificateTimestamp, error)
}

// submissionResult holds outcome of a single-log submission.
type submissionResult struct {
	sct *ct.SignedCertificateTimestamp
	err error
}

type groupState struct {
	Name    string
	Success bool
}

// safeSubmissionState is a submission state-machine for set of Log-groups.
// When some group is complete cancels all requests that are not needed by any
// group.
type safeSubmissionState struct {
	mu          sync.Mutex
	logToGroups map[string]ctpolicy.GroupSet
	groupNeeds  map[string]int

	results map[string]*submissionResult
	cancels map[string]context.CancelFunc
}

func newSafeSubmissionState(groups ctpolicy.LogPolicyData) *safeSubmissionState {
	var s safeSubmissionState
	s.logToGroups = ctpolicy.GroupByLogs(groups)
	s.groupNeeds = make(map[string]int)
	for _, g := range groups {
		s.groupNeeds[g.Name] = g.MinInclusions
	}
	s.results = make(map[string]*submissionResult)
	s.cancels = make(map[string]context.CancelFunc)
	return &s
}

// request includes empty submissionResult in the set, returns whether
// the entry is requested for the first time.
func (sub *safeSubmissionState) request(logURL string, cancel context.CancelFunc) bool {
	sub.mu.Lock()
	defer sub.mu.Unlock()
	if sub.results[logURL] != nil {
		// Already requested.
		return false
	}
	sub.results[logURL] = &submissionResult{}
	isAwaited := false
	for g := range sub.logToGroups[logURL] {
		if sub.groupNeeds[g] > 0 {
			isAwaited = true
			break
		}
	}
	if !isAwaited {
		// No groups expecting result from this Log.
		return false
	}
	sub.cancels[logURL] = cancel
	return true
}

// setResult processes SCT-result. Writes it down if it is error or awaited-SCT.
// Re-calculates group-completion and cancels any running but
// not-awaited-anymore Log-requests.
func (sub *safeSubmissionState) setResult(logURL string, sct *ct.SignedCertificateTimestamp, err error) {
	sub.mu.Lock()
	defer sub.mu.Unlock()
	if sct == nil {
		sub.results[logURL] = &submissionResult{sct: sct, err: err}
		return
	}
	// If at least one group needs that SCT, result is set. Otherwise dumped.
	for groupName := range sub.logToGroups[logURL] {
		if sub.groupNeeds[groupName] > 0 {
			sub.results[logURL] = &submissionResult{sct: sct, err: err}
		}
		sub.groupNeeds[groupName]--
	}

	// Cancel any pending Log-requests for which there're no more awaiting
	// Log-groups.
	for logURL, groupSet := range sub.logToGroups {
		isAwaited := false
		for g := range groupSet {
			if sub.groupNeeds[g] > 0 {
				isAwaited = true
				break
			}
		}
		if !isAwaited && sub.cancels[logURL] != nil {
			sub.cancels[logURL]()
			sub.cancels[logURL] = nil
		}
	}
}

// groupComplete returns true iff the specified group has all the SCTs it needs.
func (sub *safeSubmissionState) groupComplete(groupName string) bool {
	sub.mu.Lock()
	defer sub.mu.Unlock()
	needs, ok := sub.groupNeeds[groupName]
	if !ok {
		return true
	}
	return needs <= 0
}

func (sub *safeSubmissionState) collectSCTs() []*AssignedSCT {
	sub.mu.Lock()
	defer sub.mu.Unlock()
	scts := []*AssignedSCT{}
	for logURL, r := range sub.results {
		if r != nil && r.sct != nil {
			scts = append(scts, &AssignedSCT{LogURL: logURL, SCT: r.sct})
		}
	}
	return scts
}

// postInterval calculates duration for consequent call.
// For first parallelStart calls duration is 0, while every next one gets
// additional dur interval.
func postInterval(idx int, parallelStart int, dur time.Duration) time.Duration {
	if idx < parallelStart {
		return time.Duration(0)
	}
	return time.Duration(idx+1-parallelStart) * dur
}

// groupRace shuffles logs within the group, submits avoiding
// duplicate-requests and collects responses.
func groupRace(ctx context.Context, chain []ct.ASN1Cert, group *ctpolicy.LogGroupInfo, parallelStart int,
	state *safeSubmissionState, submitter Submitter) groupState {
	groupURLs := make([]string, 0, len(group.LogURLs))
	for logURL := range group.LogURLs {
		groupURLs = append(groupURLs, logURL)
	}

	type count struct{}
	counter := make(chan count, len(groupURLs))

	countCall := func() {
		counter <- count{}
	}

	// Randomize the order in which we send requests to the logs in a group
	// so we maximize the distribution of logs we get SCTs from.
	for i, urlNum := range rand.Perm(len(groupURLs)) {
		subCtx, cancel := context.WithCancel(ctx)
		go func(i int, logURL string) {
			defer countCall()
			timeoutchan := time.After(postInterval(i, parallelStart, PostBatchInterval))

			select {
			case <-subCtx.Done():
				return
			case <-timeoutchan:
			}
			if state.groupComplete(group.Name) {
				cancel()
				return
			}
			if firstRequested := state.request(logURL, cancel); !firstRequested {
				return
			}
			sct, err := submitter.SubmitToLog(subCtx, logURL, chain)
			// TODO(Mercurrent): verify SCT
			state.setResult(logURL, sct, err)
		}(i, groupURLs[urlNum])
	}
	// Wait until either all logs within groups are processed or context is
	// cancelled.
	for i := 0; i < len(groupURLs); i++ {
		select {
		case <-ctx.Done():
			return groupState{Name: group.Name, Success: state.groupComplete(group.Name)}
		case <-counter:
			if state.groupComplete(group.Name) {
				return groupState{Name: group.Name, Success: true}
			}
		}
	}
	return groupState{Name: group.Name, Success: state.groupComplete(group.Name)}
}

func parallelNums(groups ctpolicy.LogPolicyData) map[string]int {
	nums := make(map[string]int)
	var subsetSum int
	for _, g := range groups {
		nums[g.Name] = g.MinInclusions
		if !g.IsBase {
			subsetSum += g.MinInclusions
		}
	}
	if _, hasBase := nums[ctpolicy.BaseName]; hasBase {
		if nums[ctpolicy.BaseName] >= subsetSum {
			nums[ctpolicy.BaseName] -= subsetSum
		} else {
			nums[ctpolicy.BaseName] = 0
		}
	}
	return nums
}

// AssignedSCT represents SCT with logURL of log-producer.
type AssignedSCT struct {
	LogURL string
	SCT    *ct.SignedCertificateTimestamp
}

func completenessError(groupComplete map[string]bool) error {
	failedGroups := []string{}
	for name, success := range groupComplete {
		if !success {
			failedGroups = append(failedGroups, name)
		}
	}
	if len(failedGroups) > 0 {
		return fmt.Errorf("log-group(s) %s didn't receive enough SCTs", strings.Join(failedGroups, ", "))
	}
	return nil
}

// GetSCTs picks required number of Logs according to policy-group logic and
// collects SCTs from them.
// Emits all collected SCTs even when any error produced.
func GetSCTs(ctx context.Context, submitter Submitter, chain []ct.ASN1Cert, groups ctpolicy.LogPolicyData) ([]*AssignedSCT, error) {
	parallelNums := parallelNums(groups)
	// channel listening to group-completion (failure) events from each single group-race.
	groupEvents := make(chan groupState, len(groups))
	submissions := newSafeSubmissionState(groups)
	for _, g := range groups {
		go func(g *ctpolicy.LogGroupInfo) {
			groupEvents <- groupRace(ctx, chain, g, parallelNums[g.Name], submissions, submitter)
		}(g)
	}

	// A group name being present in the map indicates that group has
	// completed processing, and the bool value indicates whether it
	// completed successfully.
	groupComplete := make(map[string]bool)
	// Terminates upon either all logs available are requested or required
	// number of SCTs is collected or upon context timeout.
	for i := 0; i < len(groups); i++ {
		select {
		case <-ctx.Done():
			return submissions.collectSCTs(), completenessError(groupComplete)
		case g := <-groupEvents:
			groupComplete[g.Name] = g.Success
		}
	}
	return submissions.collectSCTs(), completenessError(groupComplete)
}
<commit_msg>Init completeness map within before submission race. (#521)<commit_after>// Copyright 2018 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package submission

import (
	"context"
	"fmt"
	"math/rand"
	"strings"
	"sync"
	"time"

	ct "github.com/google/certificate-transparency-go"
	"github.com/google/certificate-transparency-go/ctpolicy"
)

const (
	// PostBatchInterval is duration between parallel batch call and subsequent
	// requests to Logs within group.
	// TODO(Mercurrent): optimize to avoid excessive requests.
	PostBatchInterval = time.Second
)

// Submitter is interface wrapping Log-request-response cycle and any processing.
type Submitter interface {
	SubmitToLog(ctx context.Context, logURL string, chain []ct.ASN1Cert) (*ct.SignedCertificateTimestamp, error)
}

// submissionResult holds outcome of a single-log submission.
type submissionResult struct {
	sct *ct.SignedCertificateTimestamp
	err error
}

type groupState struct {
	Name    string
	Success bool
}

// safeSubmissionState is a submission state-machine for set of Log-groups.
// When some group is complete cancels all requests that are not needed by any
// group.
type safeSubmissionState struct {
	mu          sync.Mutex
	logToGroups map[string]ctpolicy.GroupSet
	groupNeeds  map[string]int

	results map[string]*submissionResult
	cancels map[string]context.CancelFunc
}

func newSafeSubmissionState(groups ctpolicy.LogPolicyData) *safeSubmissionState {
	var s safeSubmissionState
	s.logToGroups = ctpolicy.GroupByLogs(groups)
	s.groupNeeds = make(map[string]int)
	for _, g := range groups {
		s.groupNeeds[g.Name] = g.MinInclusions
	}
	s.results = make(map[string]*submissionResult)
	s.cancels = make(map[string]context.CancelFunc)
	return &s
}

// request includes empty submissionResult in the set, returns whether
// the entry is requested for the first time.
func (sub *safeSubmissionState) request(logURL string, cancel context.CancelFunc) bool {
	sub.mu.Lock()
	defer sub.mu.Unlock()
	if sub.results[logURL] != nil {
		// Already requested.
		return false
	}
	sub.results[logURL] = &submissionResult{}
	isAwaited := false
	for g := range sub.logToGroups[logURL] {
		if sub.groupNeeds[g] > 0 {
			isAwaited = true
			break
		}
	}
	if !isAwaited {
		// No groups expecting result from this Log.
		return false
	}
	sub.cancels[logURL] = cancel
	return true
}

// setResult processes SCT-result. Writes it down if it is error or awaited-SCT.
// Re-calculates group-completion and cancels any running but
// not-awaited-anymore Log-requests.
func (sub *safeSubmissionState) setResult(logURL string, sct *ct.SignedCertificateTimestamp, err error) {
	sub.mu.Lock()
	defer sub.mu.Unlock()
	if sct == nil {
		sub.results[logURL] = &submissionResult{sct: sct, err: err}
		return
	}
	// If at least one group needs that SCT, result is set. Otherwise dumped.
	for groupName := range sub.logToGroups[logURL] {
		if sub.groupNeeds[groupName] > 0 {
			sub.results[logURL] = &submissionResult{sct: sct, err: err}
		}
		sub.groupNeeds[groupName]--
	}

	// Cancel any pending Log-requests for which there're no more awaiting
	// Log-groups.
	for logURL, groupSet := range sub.logToGroups {
		isAwaited := false
		for g := range groupSet {
			if sub.groupNeeds[g] > 0 {
				isAwaited = true
				break
			}
		}
		if !isAwaited && sub.cancels[logURL] != nil {
			sub.cancels[logURL]()
			sub.cancels[logURL] = nil
		}
	}
}

// groupComplete returns true iff the specified group has all the SCTs it needs.
func (sub *safeSubmissionState) groupComplete(groupName string) bool {
	sub.mu.Lock()
	defer sub.mu.Unlock()
	needs, ok := sub.groupNeeds[groupName]
	if !ok {
		return true
	}
	return needs <= 0
}

func (sub *safeSubmissionState) collectSCTs() []*AssignedSCT {
	sub.mu.Lock()
	defer sub.mu.Unlock()
	scts := []*AssignedSCT{}
	for logURL, r := range sub.results {
		if r != nil && r.sct != nil {
			scts = append(scts, &AssignedSCT{LogURL: logURL, SCT: r.sct})
		}
	}
	return scts
}

// postInterval calculates duration for consequent call.
// For first parallelStart calls duration is 0, while every next one gets
// additional dur interval.
func postInterval(idx int, parallelStart int, dur time.Duration) time.Duration {
	if idx < parallelStart {
		return time.Duration(0)
	}
	return time.Duration(idx+1-parallelStart) * dur
}

// groupRace shuffles logs within the group, submits avoiding
// duplicate-requests and collects responses.
func groupRace(ctx context.Context, chain []ct.ASN1Cert, group *ctpolicy.LogGroupInfo, parallelStart int,
	state *safeSubmissionState, submitter Submitter) groupState {
	groupURLs := make([]string, 0, len(group.LogURLs))
	for logURL := range group.LogURLs {
		groupURLs = append(groupURLs, logURL)
	}

	type count struct{}
	counter := make(chan count, len(groupURLs))

	countCall := func() {
		counter <- count{}
	}

	// Randomize the order in which we send requests to the logs in a group
	// so we maximize the distribution of logs we get SCTs from.
	for i, urlNum := range rand.Perm(len(groupURLs)) {
		subCtx, cancel := context.WithCancel(ctx)
		go func(i int, logURL string) {
			defer countCall()
			timeoutchan := time.After(postInterval(i, parallelStart, PostBatchInterval))

			select {
			case <-subCtx.Done():
				return
			case <-timeoutchan:
			}
			if state.groupComplete(group.Name) {
				cancel()
				return
			}
			if firstRequested := state.request(logURL, cancel); !firstRequested {
				return
			}
			sct, err := submitter.SubmitToLog(subCtx, logURL, chain)
			// TODO(Mercurrent): verify SCT
			state.setResult(logURL, sct, err)
		}(i, groupURLs[urlNum])
	}
	// Wait until either all logs within groups are processed or context is
	// cancelled.
	for i := 0; i < len(groupURLs); i++ {
		select {
		case <-ctx.Done():
			return groupState{Name: group.Name, Success: state.groupComplete(group.Name)}
		case <-counter:
			if state.groupComplete(group.Name) {
				return groupState{Name: group.Name, Success: true}
			}
		}
	}
	return groupState{Name: group.Name, Success: state.groupComplete(group.Name)}
}

func parallelNums(groups ctpolicy.LogPolicyData) map[string]int {
	nums := make(map[string]int)
	var subsetSum int
	for _, g := range groups {
		nums[g.Name] = g.MinInclusions
		if !g.IsBase {
			subsetSum += g.MinInclusions
		}
	}
	if _, hasBase := nums[ctpolicy.BaseName]; hasBase {
		if nums[ctpolicy.BaseName] >= subsetSum {
			nums[ctpolicy.BaseName] -= subsetSum
		} else {
			nums[ctpolicy.BaseName] = 0
		}
	}
	return nums
}

// AssignedSCT represents SCT with logURL of log-producer.
type AssignedSCT struct {
	LogURL string
	SCT    *ct.SignedCertificateTimestamp
}

func completenessError(groupComplete map[string]bool) error {
	failedGroups := []string{}
	for name, success := range groupComplete {
		if !success {
			failedGroups = append(failedGroups, name)
		}
	}
	if len(failedGroups) > 0 {
		return fmt.Errorf("log-group(s) %s didn't receive enough SCTs", strings.Join(failedGroups, ", "))
	}
	return nil
}

// GetSCTs picks required number of Logs according to policy-group logic and
// collects SCTs from them.
// Emits all collected SCTs even when any error produced.
func GetSCTs(ctx context.Context, submitter Submitter, chain []ct.ASN1Cert, groups ctpolicy.LogPolicyData) ([]*AssignedSCT, error) {
	groupComplete := make(map[string]bool)
	for _, g := range groups {
		groupComplete[g.Name] = false
	}

	parallelNums := parallelNums(groups)
	// channel listening to group-completion (failure) events from each single group-race.
	groupEvents := make(chan groupState, len(groups))
	submissions := newSafeSubmissionState(groups)
	for _, g := range groups {
		go func(g *ctpolicy.LogGroupInfo) {
			groupEvents <- groupRace(ctx, chain, g, parallelNums[g.Name], submissions, submitter)
		}(g)
	}

	// Terminates upon either all logs available are requested or required
	// number of SCTs is collected or upon context timeout.
	for i := 0; i < len(groups); i++ {
		select {
		case <-ctx.Done():
			return submissions.collectSCTs(), completenessError(groupComplete)
		case g := <-groupEvents:
			groupComplete[g.Name] = g.Success
		}
	}
	return submissions.collectSCTs(), completenessError(groupComplete)
}
<|endoftext|>