<commit_before>// SPDX-License-Identifier: ISC
// Copyright (c) 2014-2019 Bitmark Inc.
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package storage

import (
	"encoding/binary"
	"fmt"
	"os"
	"reflect"
	"sync"

	"github.com/syndtr/goleveldb/leveldb"
	ldb_opt "github.com/syndtr/goleveldb/leveldb/opt"

	"github.com/bitmark-inc/bitmarkd/fault"
	"github.com/bitmark-inc/logger"
)

// exported storage pools
//
// note all must be exported (i.e. initial capital) or initialisation will panic
type pools struct {
	Blocks            *PoolHandle `prefix:"B" database:"blocks"`
	BlockHeaderHash   *PoolHandle `prefix:"2" database:"blocks"`
	BlockOwnerPayment *PoolHandle `prefix:"H" database:"index"`
	BlockOwnerTxIndex *PoolHandle `prefix:"I" database:"index"`
	Assets            *PoolNB     `prefix:"A" database:"index"`
	Transactions      *PoolNB     `prefix:"T" database:"index"`
	OwnerNextCount    *PoolHandle `prefix:"N" database:"index"`
	OwnerList         *PoolHandle `prefix:"L" database:"index"`
	OwnerTxIndex      *PoolHandle `prefix:"D" database:"index"`
	OwnerData         *PoolHandle `prefix:"O" database:"index"`
	Shares            *PoolHandle `prefix:"F" database:"index"`
	ShareQuantity     *PoolHandle `prefix:"Q" database:"index"`
	TestData          *PoolHandle `prefix:"Z" database:"index"`
}

// Pool - the set of exported pools
var Pool pools

// for database version
var versionKey = []byte{0x00, 'V', 'E', 'R', 'S', 'I', 'O', 'N'}

const (
	currentBlockDBVersion = 0x301
	currentIndexDBVersion = 0x200
	ErrEmptyTransaction   = "Empty Transaction"
)

// holds the database handle
var poolData struct {
	sync.RWMutex
	dbBlocks  *leveldb.DB
	dbIndex   *leveldb.DB
	trx       Transaction
	blocksTrx *leveldb.Batch
	indexTrx  *leveldb.Batch
	cache     Cache
}

// pool access modes
const (
	ReadOnly  = true
	ReadWrite = false
)

// Initialise - open up the database connection
//
// this must be called before any pool is accessed
func Initialise(database string, readOnly bool) (bool, bool, error) {
	poolData.Lock()
	defer poolData.Unlock()

	ok := false
	mustMigrate := false
	mustReindex := false

	if nil != poolData.dbBlocks {
		return mustMigrate, mustReindex, fault.ErrAlreadyInitialised
	}

	defer func() {
		if !ok {
			dbClose()
		}
	}()

	blocksDatabase := database + "-blocks.leveldb"
	indexDatabase := database + "-index.leveldb"

	db, blocksVersion, err := getDB(blocksDatabase, readOnly)
	if nil != err {
		return mustMigrate, mustReindex, err
	}
	poolData.dbBlocks = db

	// ensure no database downgrade
	if blocksVersion > currentBlockDBVersion {
		logger.Criticalf("block database version: %d > current version: %d", blocksVersion, currentBlockDBVersion)
		return mustMigrate, mustReindex, fmt.Errorf("block database version: %d > current version: %d", blocksVersion, currentBlockDBVersion)
	}

	db, indexVersion, err := getDB(indexDatabase, readOnly)
	if nil != err {
		return mustMigrate, mustReindex, err
	}
	poolData.dbIndex = db

	// ensure no database downgrade
	if indexVersion > currentIndexDBVersion {
		logger.Criticalf("index database version: %d > current version: %d", indexVersion, currentIndexDBVersion)
		return mustMigrate, mustReindex, fmt.Errorf("index database version: %d > current version: %d", indexVersion, currentIndexDBVersion)
	}

	// prevent readOnly from modifying the database
	if readOnly && (blocksVersion != currentBlockDBVersion || indexVersion != currentIndexDBVersion) {
		logger.Criticalf("database is inconsistent: blocks: %d  index: %d  current: %d & %d", blocksVersion, indexVersion, currentBlockDBVersion, currentIndexDBVersion)
		return mustMigrate, mustReindex, fmt.Errorf("database is inconsistent: blocks: %d  index: %d  current: %d & %d", blocksVersion, indexVersion, currentBlockDBVersion, currentIndexDBVersion)
	}

	if 0 < blocksVersion && blocksVersion < currentBlockDBVersion {

		mustMigrate = true

		logger.Criticalf("block database version: %d < current version: %d", blocksVersion, currentBlockDBVersion)

	} else if 0 == blocksVersion {

		// database was empty so tag as current version
		err = putVersion(poolData.dbBlocks, currentBlockDBVersion)
		if err != nil {
			return mustMigrate, mustReindex, err
		}
	}

	// see if index need to be created or deleted and re-created
	if mustReindex || indexVersion < currentIndexDBVersion {

		mustReindex = true

		// close out current index
		poolData.dbIndex.Close()
		poolData.dbIndex = nil

		logger.Criticalf("drop index database: %s", indexDatabase)

		// erase the index completely
		err = os.RemoveAll(indexDatabase)
		if nil != err {
			return mustMigrate, mustReindex, err
		}

		// generate an empty index database
		poolData.dbIndex, _, err = getDB(indexDatabase, readOnly)
		if nil != err {
			return mustMigrate, mustReindex, err
		}

	}

	// this will be a struct type
	poolType := reflect.TypeOf(Pool)

	// get write access by using pointer + Elem()
	poolValue := reflect.ValueOf(&Pool).Elem()

	// databases
	poolData.blocksTrx = new(leveldb.Batch)
	poolData.indexTrx = new(leveldb.Batch)
	poolData.cache = newCache()
	blockDBAccess := newDA(poolData.dbBlocks, poolData.blocksTrx, poolData.cache)
	indexDBAccess := newDA(poolData.dbIndex, poolData.indexTrx, poolData.cache)
	access := []DataAccess{blockDBAccess, indexDBAccess}
	poolData.trx = newTransaction(access)

	// scan each field
	for i := 0; i < poolType.NumField(); i += 1 {

		fieldInfo := poolType.Field(i)

		prefixTag := fieldInfo.Tag.Get("prefix")
		if 1 != len(prefixTag) {
			return mustMigrate, mustReindex, fmt.Errorf("pool: %v has invalid prefix: %q", fieldInfo, prefixTag)
		}

		prefix := prefixTag[0]
		limit := []byte(nil)
		if prefix < 255 {
			limit = []byte{prefix + 1}
		}

		var dataAccess DataAccess
		switch dbName := fieldInfo.Tag.Get("database"); dbName {
		case "blocks":
			dataAccess = blockDBAccess
		case "index":
			dataAccess = indexDBAccess
		default:
			return mustMigrate, mustReindex, fmt.Errorf("pool: %v  has invalid database: %q", fieldInfo, dbName)
		}

		p := &PoolHandle{
			prefix:     prefix,
			limit:      limit,
			dataAccess: dataAccess,
		}

		if poolValue.Field(i).Type() == reflect.TypeOf((*PoolNB)(nil)) {
			pNB := &PoolNB{
				pool: p,
			}
			newNB := reflect.ValueOf(pNB)
			poolValue.Field(i).Set(newNB)
		} else {
			newPool := reflect.ValueOf(p)
			poolValue.Field(i).Set(newPool)
		}

	}

	ok = true // prevent db close
	return mustMigrate, mustReindex, nil
}

func dbClose() {
	if nil != poolData.dbIndex {
		poolData.dbIndex.Close()
		poolData.dbIndex = nil
	}
	if nil != poolData.dbBlocks {
		poolData.dbBlocks.Close()
		poolData.dbBlocks = nil
	}
}

// Finalise - close the database connection
func Finalise() {
	poolData.Lock()
	dbClose()
	poolData.Unlock()
}

// ReindexDone - called at the end of reindex
func ReindexDone() error {
	poolData.Lock()
	defer poolData.Unlock()
	return putVersion(poolData.dbIndex, currentIndexDBVersion)
}

// return:
//   databse handle
//   version number
func getDB(name string, readOnly bool) (*leveldb.DB, int, error) {
	opt := &ldb_opt.Options{
		ErrorIfExist:   false,
		ErrorIfMissing: readOnly,
		ReadOnly:       readOnly,
	}

	db, err := leveldb.OpenFile(name, opt)
	if nil != err {
		return nil, 0, err
	}

	versionValue, err := db.Get(versionKey, nil)
	if leveldb.ErrNotFound == err {
		return db, 0, nil
	} else if nil != err {
		db.Close()
		return nil, 0, err
	}

	if 4 != len(versionValue) {
		db.Close()
		return nil, 0, fmt.Errorf("incompatible database version length: expected: %d  actual: %d", 4, len(versionValue))
	}

	version := int(binary.BigEndian.Uint32(versionValue))
	return db, version, nil
}

func putVersion(db *leveldb.DB, version int) error {
	currentVersion := make([]byte, 4)
	binary.BigEndian.PutUint32(currentVersion, uint32(version))

	return db.Put(versionKey, currentVersion, nil)
}

func NewDBTransaction() (Transaction, error) {
	err := poolData.trx.Begin()
	if nil != err {
		return nil, err
	}
	return poolData.trx, nil
}
<commit_msg>[storage] remark some debug logs<commit_after>// SPDX-License-Identifier: ISC
// Copyright (c) 2014-2019 Bitmark Inc.
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package storage

import (
	"encoding/binary"
	"fmt"
	"os"
	"reflect"
	"sync"

	"github.com/syndtr/goleveldb/leveldb"
	ldb_opt "github.com/syndtr/goleveldb/leveldb/opt"

	"github.com/bitmark-inc/bitmarkd/fault"
	"github.com/bitmark-inc/logger"
)

// exported storage pools
//
// note all must be exported (i.e. initial capital) or initialisation will panic
type pools struct {
	Blocks            *PoolHandle `prefix:"B" database:"blocks"`
	BlockHeaderHash   *PoolHandle `prefix:"2" database:"blocks"`
	BlockOwnerPayment *PoolHandle `prefix:"H" database:"index"`
	BlockOwnerTxIndex *PoolHandle `prefix:"I" database:"index"`
	Assets            *PoolNB     `prefix:"A" database:"index"`
	Transactions      *PoolNB     `prefix:"T" database:"index"`
	OwnerNextCount    *PoolHandle `prefix:"N" database:"index"`
	OwnerList         *PoolHandle `prefix:"L" database:"index"`
	OwnerTxIndex      *PoolHandle `prefix:"D" database:"index"`
	OwnerData         *PoolHandle `prefix:"O" database:"index"`
	Shares            *PoolHandle `prefix:"F" database:"index"`
	ShareQuantity     *PoolHandle `prefix:"Q" database:"index"`
	TestData          *PoolHandle `prefix:"Z" database:"index"`
}

// Pool - the set of exported pools
var Pool pools

// for database version
var versionKey = []byte{0x00, 'V', 'E', 'R', 'S', 'I', 'O', 'N'}

const (
	currentBlockDBVersion = 0x301
	currentIndexDBVersion = 0x200
	ErrEmptyTransaction   = "Empty Transaction"
)

// holds the database handle
var poolData struct {
	sync.RWMutex
	dbBlocks  *leveldb.DB
	dbIndex   *leveldb.DB
	trx       Transaction
	blocksTrx *leveldb.Batch
	indexTrx  *leveldb.Batch
	cache     Cache
}

// pool access modes
const (
	ReadOnly  = true
	ReadWrite = false
)

// Initialise - open up the database connection
//
// this must be called before any pool is accessed
func Initialise(database string, readOnly bool) (bool, bool, error) {
	poolData.Lock()
	defer poolData.Unlock()

	ok := false
	mustMigrate := false
	mustReindex := false

	if nil != poolData.dbBlocks {
		return mustMigrate, mustReindex, fault.ErrAlreadyInitialised
	}

	defer func() {
		if !ok {
			dbClose()
		}
	}()

	blocksDatabase := database + "-blocks.leveldb"
	indexDatabase := database + "-index.leveldb"

	db, blocksVersion, err := getDB(blocksDatabase, readOnly)
	if nil != err {
		return mustMigrate, mustReindex, err
	}
	poolData.dbBlocks = db

	// ensure no database downgrade
	if blocksVersion > currentBlockDBVersion {
		logger.Criticalf("block database version: %d > current version: %d", blocksVersion, currentBlockDBVersion)
		return mustMigrate, mustReindex, fmt.Errorf("block database version: %d > current version: %d", blocksVersion, currentBlockDBVersion)
	}

	db, indexVersion, err := getDB(indexDatabase, readOnly)
	if nil != err {
		return mustMigrate, mustReindex, err
	}
	poolData.dbIndex = db

	// ensure no database downgrade
	if indexVersion > currentIndexDBVersion {
		logger.Criticalf("index database version: %d > current version: %d", indexVersion, currentIndexDBVersion)
		return mustMigrate, mustReindex, fmt.Errorf("index database version: %d > current version: %d", indexVersion, currentIndexDBVersion)
	}

	// prevent readOnly from modifying the database
	if readOnly && (blocksVersion != currentBlockDBVersion || indexVersion != currentIndexDBVersion) {
		logger.Criticalf("database is inconsistent: blocks: %d  index: %d  current: %d & %d", blocksVersion, indexVersion, currentBlockDBVersion, currentIndexDBVersion)
		return mustMigrate, mustReindex, fmt.Errorf("database is inconsistent: blocks: %d  index: %d  current: %d & %d", blocksVersion, indexVersion, currentBlockDBVersion, currentIndexDBVersion)
	}

	if 0 < blocksVersion && blocksVersion < currentBlockDBVersion {

		mustMigrate = true

		//logger.Criticalf("block database version: %d < current version: %d", blocksVersion, currentBlockDBVersion)

	} else if 0 == blocksVersion {

		// database was empty so tag as current version
		err = putVersion(poolData.dbBlocks, currentBlockDBVersion)
		if err != nil {
			return mustMigrate, mustReindex, err
		}
	}

	// see if index need to be created or deleted and re-created
	if mustReindex || indexVersion < currentIndexDBVersion {

		mustReindex = true

		// close out current index
		poolData.dbIndex.Close()
		poolData.dbIndex = nil

		//logger.Criticalf("drop index database: %s", indexDatabase)

		// erase the index completely
		err = os.RemoveAll(indexDatabase)
		if nil != err {
			return mustMigrate, mustReindex, err
		}

		// generate an empty index database
		poolData.dbIndex, _, err = getDB(indexDatabase, readOnly)
		if nil != err {
			return mustMigrate, mustReindex, err
		}

	}

	// this will be a struct type
	poolType := reflect.TypeOf(Pool)

	// get write access by using pointer + Elem()
	poolValue := reflect.ValueOf(&Pool).Elem()

	// databases
	poolData.blocksTrx = new(leveldb.Batch)
	poolData.indexTrx = new(leveldb.Batch)
	poolData.cache = newCache()
	blockDBAccess := newDA(poolData.dbBlocks, poolData.blocksTrx, poolData.cache)
	indexDBAccess := newDA(poolData.dbIndex, poolData.indexTrx, poolData.cache)
	access := []DataAccess{blockDBAccess, indexDBAccess}
	poolData.trx = newTransaction(access)

	// scan each field
	for i := 0; i < poolType.NumField(); i += 1 {

		fieldInfo := poolType.Field(i)

		prefixTag := fieldInfo.Tag.Get("prefix")
		if 1 != len(prefixTag) {
			return mustMigrate, mustReindex, fmt.Errorf("pool: %v has invalid prefix: %q", fieldInfo, prefixTag)
		}

		prefix := prefixTag[0]
		limit := []byte(nil)
		if prefix < 255 {
			limit = []byte{prefix + 1}
		}

		var dataAccess DataAccess
		switch dbName := fieldInfo.Tag.Get("database"); dbName {
		case "blocks":
			dataAccess = blockDBAccess
		case "index":
			dataAccess = indexDBAccess
		default:
			return mustMigrate, mustReindex, fmt.Errorf("pool: %v  has invalid database: %q", fieldInfo, dbName)
		}

		p := &PoolHandle{
			prefix:     prefix,
			limit:      limit,
			dataAccess: dataAccess,
		}

		if poolValue.Field(i).Type() == reflect.TypeOf((*PoolNB)(nil)) {
			pNB := &PoolNB{
				pool: p,
			}
			newNB := reflect.ValueOf(pNB)
			poolValue.Field(i).Set(newNB)
		} else {
			newPool := reflect.ValueOf(p)
			poolValue.Field(i).Set(newPool)
		}

	}

	ok = true // prevent db close
	return mustMigrate, mustReindex, nil
}

func dbClose() {
	if nil != poolData.dbIndex {
		poolData.dbIndex.Close()
		poolData.dbIndex = nil
	}
	if nil != poolData.dbBlocks {
		poolData.dbBlocks.Close()
		poolData.dbBlocks = nil
	}
}

// Finalise - close the database connection
func Finalise() {
	poolData.Lock()
	dbClose()
	poolData.Unlock()
}

// ReindexDone - called at the end of reindex
func ReindexDone() error {
	poolData.Lock()
	defer poolData.Unlock()
	return putVersion(poolData.dbIndex, currentIndexDBVersion)
}

// return:
//   databse handle
//   version number
func getDB(name string, readOnly bool) (*leveldb.DB, int, error) {
	opt := &ldb_opt.Options{
		ErrorIfExist:   false,
		ErrorIfMissing: readOnly,
		ReadOnly:       readOnly,
	}

	db, err := leveldb.OpenFile(name, opt)
	if nil != err {
		return nil, 0, err
	}

	versionValue, err := db.Get(versionKey, nil)
	if leveldb.ErrNotFound == err {
		return db, 0, nil
	} else if nil != err {
		db.Close()
		return nil, 0, err
	}

	if 4 != len(versionValue) {
		db.Close()
		return nil, 0, fmt.Errorf("incompatible database version length: expected: %d  actual: %d", 4, len(versionValue))
	}

	version := int(binary.BigEndian.Uint32(versionValue))
	return db, version, nil
}

func putVersion(db *leveldb.DB, version int) error {
	currentVersion := make([]byte, 4)
	binary.BigEndian.PutUint32(currentVersion, uint32(version))

	return db.Put(versionKey, currentVersion, nil)
}

func NewDBTransaction() (Transaction, error) {
	err := poolData.trx.Begin()
	if nil != err {
		return nil, err
	}
	return poolData.trx, nil
}
<|endoftext|>


<commit_before>package service

import (
    "time"
)

type GRPCServer interface {
    Serve(ctx context.Context, network, addr, pem, key string) error
}

type GRPCService struct {
    ID      string
    Service string
    Tags    []string
    Address string
    Port    int

    server GRPCServer

    TTL time.Duration

    c coordinator.Coordinator

    log *logger.Logger

    errorChan chan error
}

func NewGRPCService(id string, service string, tags []string, address string, port int, server GRPCServer, ttl time.Duration, c coordinator.Coordinator, log *logger.Logger) *GRPCService {
    return &GRPCService{
        ID:      id,
        Service: service,
        Tags:    tags,
        Address: address,
        Port:    port,

        server: server,

        TTL: ttl,
        c:   c,
        log: log,

        errorChan: make(chan error),
    }
}

func (g *GRPCService) Start(ctx context.Context, delayRegisterTime time.Duration) error {
    go func() {
        err := g.server.Serve(
            ctx,
            "tcp",
            fmt.Sprintf("%s:%d", g.Address, g.Port),
            g.pem,
            g.key,
        )
        g.errorChan <- err
    }()

    time.Sleep(delayRegisterTime)

    var err error

    select {
    case err = <-g.errorChan:
    default:
        err = c.Register(ctx, s)
    }

    return err
}
<commit_msg>update grpc<commit_after>package service

import (
    "fmt"
    "time"

    "golang.org/x/net/context"

    "github.com/servicekit/servicekit-go/coordinator"
    "github.com/servicekit/servicekit-go/logger"
)

type GRPCServer interface {
    Serve(ctx context.Context, network, addr, pem, key string) error
}

type GRPCService struct {
    ID      string
    Service string
    Tags    []string
    Address string
    Port    int

    server GRPCServer

    TTL time.Duration

    c coordinator.Coordinator

    log *logger.Logger

    errorChan chan error
}

func NewGRPCService(id string, service string, tags []string, address string, port int, server GRPCServer, ttl time.Duration, c coordinator.Coordinator, log *logger.Logger) *GRPCService {
    return &GRPCService{
        ID:      id,
        Service: service,
        Tags:    tags,
        Address: address,
        Port:    port,

        server: server,

        TTL: ttl,
        c:   c,
        log: log,

        errorChan: make(chan error),
    }
}

func (g *GRPCService) Start(ctx context.Context, delayRegisterTime time.Duration) error {
    go func() {
        err := g.server.Serve(
            ctx,
            "tcp",
            fmt.Sprintf("%s:%d", g.Address, g.Port),
            g.pem,
            g.key,
        )
        g.errorChan <- err
    }()

    time.Sleep(delayRegisterTime)

    var err error

    select {
    case err = <-g.errorChan:
    default:
        err = c.Register(ctx, s)
    }

    return err
}
<|endoftext|>


<commit_before>package stream

import (
	"fmt"
	"github.com/andygrunwald/gotrap/config"
	"github.com/andygrunwald/gotrap/gerrit"
	"github.com/andygrunwald/gotrap/github"
	"log"
	"regexp"
	"strings"
)

type Gotrap struct {
	githubClient github.GithubClient
	gerritClient gerrit.GerritInstance
	config       *config.Configuration
	message      gerrit.Message
}

func (trap *Gotrap) TakeAction() {
	// Stream events are documented
	// See https://git.eclipse.org/r/Documentation/cmd-stream-events.html
	switch trap.message.Type {
	case "patchset-created":
		log.Printf("> New patchset-created message incoming for ref \"%s\" in \"%s\" (%s)", trap.message.Patchset.Ref, trap.message.Change.Project, trap.message.Change.URL)

		// Check if Project is configured
		if _, err := trap.IsProjectConfigured(trap.message.Change.Project); err != nil {
			log.Printf("> %s", err)
			return
		}

		// Check if branch is configured
		if _, err := trap.IsBranchConfigured(trap.message.Change.Project, trap.message.Change.Branch); err != nil {
			log.Printf("> %s", err)
			return
		}

		// Check if change subject is excluded
		if res, matchedPattern := trap.IsSubjectExcludedByPattern(trap.message.Change.Subject); res == true {
			log.Printf("> Subject \"%s\" excluded by pattern \"%s\"", trap.message.Change.Subject, matchedPattern)
			return
		}

		// TODO: Get current revision number
		// If this revision / patchset number is not the current number
		// we will skip this patchset-created request, because
		// why should we create a pull request for an old patchset?
		// The current patchset will be delivered later as message.
		// So we won`t skip this changeset
		// https://review.typo3.org/a/changes/I640486e9f32da6ac1eba05e3c38d15a0aba41055/?o=CURRENT_REVISION
		if currentPatchset, _ := trap.gerritClient.IsPatchsetTheCurrentPatchset(trap.message.Change.ID, trap.message.Patchset.Number); currentPatchset == false {
			log.Printf("> Patchset skipped, because it is not the current one (Ref: %s of %s)", trap.message.Patchset.Ref, trap.message.Change.URL)
			return
		}

		// Create the pull request
		pullRequest, err := trap.githubClient.CreatePullRequestForPatchset(&trap.message)
		if err != nil {
			// TODO: I don`t have an idea what to do if i fail to create a PR
			log.Println("> Error during creating new pull request", err)
			return
		}

		log.Printf("> New pull request created: %s", *pullRequest.HTMLURL)

		// Poll travis ci and wait until the PR got a status
		s, _ := trap.githubClient.WaitUntilCommitStatusIsAvailable(*pullRequest)

		var vote int
		switch *s.State {
		// Success if the latest status for all contexts is success
		case "success":
			vote = 0

		// Failure if any of the contexts report as error or failure
		case "failure":
			vote = -1
		}

		// Generate detail information about status
		var statusDetails []string
		for _, repoStatus := range s.Statuses {
			if *repoStatus.State != "success" {
				continue
			}

			statusDetails = append(statusDetails, "Service: "+*repoStatus.Context)
			statusDetails = append(statusDetails, "Description: "+*repoStatus.Description)
			statusDetails = append(statusDetails, "URL: "+*repoStatus.TargetURL)
			statusDetails = append(statusDetails, "\n")
		}

		// Build template for Gerrit vote action
		msg := trap.gerritClient.Template
		msg = strings.Replace(msg, "%state%", *s.State, 1)
		msg = strings.Replace(msg, "%status%", strings.Join(statusDetails, "\n\n"), 1)
		msg = strings.Replace(msg, "%pr%", *pullRequest.HTMLURL, 1)

		// Post Command + Vote on Changeset
		trap.gerritClient.PostCommentOnChangeset(&trap.message, vote, msg)

		msg = "This PR will be closed, because the tests results were reported back to Gerrit. "
		msg += fmt.Sprintf("See [%s](%s) for details.", trap.message.Change.Subject, trap.message.Change.URL)

		_, err = trap.githubClient.AddCommentToPullRequest(pullRequest, msg)
		if err != nil {
			log.Printf("> Error during adding a comment to a pull request %s: %s", *pullRequest.HTMLURL, err)
		} else {
			log.Printf("> Comment added to pull request: %s", *pullRequest.HTMLURL)
		}

		_, err = trap.githubClient.ClosePullRequest(pullRequest)
		if err != nil {
			log.Printf("> Error during closing a pull request %s: %s", *pullRequest.HTMLURL, err)
		} else {
			log.Printf("> Pull request closed: %s", *pullRequest.HTMLURL)
		}

	case "change-abandoned":
		// We have to close all PR`s
		// change-restored
		// change-merged
		// ref-updated
		// ref-replicated
		// ref-replication-done
		// comment-added
		// topic-changed
		// ....
	default:
		log.Printf("> Skipped AMQP message (uncovered message type: %s)\n", trap.message.Type)
	}

	return
}

func (trap *Gotrap) IsProjectConfigured(project string) (bool, error) {
	if _, ok := trap.config.Gerrit.Projects[project]; !ok {
		return false, fmt.Errorf("Project \"%s\" is not configured", project)
	}

	return true, nil
}

func (trap *Gotrap) IsBranchConfigured(project, branch string) (bool, error) {
	// If no branch is configured, we assume that all branches should be covered
	if len(trap.config.Gerrit.Projects[project]) == 0 {
		return true, nil
	}

	// If the branch exists and is configured as true, then it is valid configured
	if val, ok := trap.config.Gerrit.Projects[project][branch]; ok && val == true {
		return true, nil
	}

	return false, fmt.Errorf("Branch \"%s\" not configured for project \"%s\"", branch, project)
}

func (trap *Gotrap) IsSubjectExcludedByPattern(subject string) (bool, string) {
	for _, pattern := range trap.config.Gerrit.ExcludePattern {
		if matched, err := regexp.MatchString(pattern, subject); err == nil && matched == true {
			return matched, pattern
		}
	}

	return false, ""
}
<commit_msg>Added some TODOs<commit_after>package stream

import (
	"fmt"
	"github.com/andygrunwald/gotrap/config"
	"github.com/andygrunwald/gotrap/gerrit"
	"github.com/andygrunwald/gotrap/github"
	"log"
	"regexp"
	"strings"
)

type Gotrap struct {
	githubClient github.GithubClient
	gerritClient gerrit.GerritInstance
	config       *config.Configuration
	message      gerrit.Message
}

func (trap *Gotrap) TakeAction() {
	// Stream events are documented
	// See https://git.eclipse.org/r/Documentation/cmd-stream-events.html
	switch trap.message.Type {
	case "patchset-created":
		log.Printf("> New patchset-created message incoming for ref \"%s\" in \"%s\" (%s)", trap.message.Patchset.Ref, trap.message.Change.Project, trap.message.Change.URL)

		// Check if Project is configured
		if _, err := trap.IsProjectConfigured(trap.message.Change.Project); err != nil {
			log.Printf("> %s", err)
			return
		}

		// Check if branch is configured
		if _, err := trap.IsBranchConfigured(trap.message.Change.Project, trap.message.Change.Branch); err != nil {
			log.Printf("> %s", err)
			return
		}

		// Check if change subject is excluded
		if res, matchedPattern := trap.IsSubjectExcludedByPattern(trap.message.Change.Subject); res == true {
			log.Printf("> Subject \"%s\" excluded by pattern \"%s\"", trap.message.Change.Subject, matchedPattern)
			return
		}

		// TODO: Get current revision number
		// If this revision / patchset number is not the current number
		// we will skip this patchset-created request, because
		// why should we create a pull request for an old patchset?
		// The current patchset will be delivered later as message.
		// So we won`t skip this changeset
		// https://review.typo3.org/a/changes/I640486e9f32da6ac1eba05e3c38d15a0aba41055/?o=CURRENT_REVISION
		if currentPatchset, _ := trap.gerritClient.IsPatchsetTheCurrentPatchset(trap.message.Change.ID, trap.message.Patchset.Number); currentPatchset == false {
			log.Printf("> Patchset skipped, because it is not the current one (Ref: %s of %s)", trap.message.Patchset.Ref, trap.message.Change.URL)
			return
		}

		// Create the pull request
		pullRequest, err := trap.githubClient.CreatePullRequestForPatchset(&trap.message)
		if err != nil {
			// TODO: I don`t have an idea what to do if i fail to create a PR
			log.Println("> Error during creating new pull request", err)
			return
		}

		log.Printf("> New pull request created: %s", *pullRequest.HTMLURL)

		// Poll travis ci and wait until the PR got a status
		s, _ := trap.githubClient.WaitUntilCommitStatusIsAvailable(*pullRequest)

		var vote int
		switch *s.State {
		// Success if the latest status for all contexts is success
		case "success":
			vote = 0

		// Failure if any of the contexts report as error or failure
		case "failure":
			vote = -1
		}

		// Generate detail information about status
		var statusDetails []string
		for _, repoStatus := range s.Statuses {
			if *repoStatus.State != "success" {
				continue
			}

			statusDetails = append(statusDetails, "Service: "+*repoStatus.Context)
			statusDetails = append(statusDetails, "Description: "+*repoStatus.Description)
			statusDetails = append(statusDetails, "URL: "+*repoStatus.TargetURL)
			statusDetails = append(statusDetails, "\n")
		}

		// Build template for Gerrit vote action
		// TODO: Replace message with text/template
		msg := trap.gerritClient.Template
		msg = strings.Replace(msg, "%state%", *s.State, 1)
		msg = strings.Replace(msg, "%status%", strings.Join(statusDetails, "\n\n"), 1)
		msg = strings.Replace(msg, "%pr%", *pullRequest.HTMLURL, 1)

		// Post Command + Vote on Changeset
		trap.gerritClient.PostCommentOnChangeset(&trap.message, vote, msg)

		// TODO: Replace message with text/template
		// TODO: Make text configurable
		msg = "This PR will be closed, because the tests results were reported back to Gerrit. "
		msg += fmt.Sprintf("See [%s](%s) for details.", trap.message.Change.Subject, trap.message.Change.URL)

		_, err = trap.githubClient.AddCommentToPullRequest(pullRequest, msg)
		if err != nil {
			log.Printf("> Error during adding a comment to a pull request %s: %s", *pullRequest.HTMLURL, err)
		} else {
			log.Printf("> Comment added to pull request: %s", *pullRequest.HTMLURL)
		}

		_, err = trap.githubClient.ClosePullRequest(pullRequest)
		if err != nil {
			log.Printf("> Error during closing a pull request %s: %s", *pullRequest.HTMLURL, err)
		} else {
			log.Printf("> Pull request closed: %s", *pullRequest.HTMLURL)
		}

	case "change-abandoned":
		// We have to close all PR`s
		// change-restored
		// change-merged
		// ref-updated
		// ref-replicated
		// ref-replication-done
		// comment-added
		// topic-changed
		// ....
	default:
		log.Printf("> Skipped AMQP message (uncovered message type: %s)\n", trap.message.Type)
	}

	return
}

func (trap *Gotrap) IsProjectConfigured(project string) (bool, error) {
	if _, ok := trap.config.Gerrit.Projects[project]; !ok {
		return false, fmt.Errorf("Project \"%s\" is not configured", project)
	}

	return true, nil
}

func (trap *Gotrap) IsBranchConfigured(project, branch string) (bool, error) {
	// If no branch is configured, we assume that all branches should be covered
	if len(trap.config.Gerrit.Projects[project]) == 0 {
		return true, nil
	}

	// If the branch exists and is configured as true, then it is valid configured
	if val, ok := trap.config.Gerrit.Projects[project][branch]; ok && val == true {
		return true, nil
	}

	return false, fmt.Errorf("Branch \"%s\" not configured for project \"%s\"", branch, project)
}

func (trap *Gotrap) IsSubjectExcludedByPattern(subject string) (bool, string) {
	for _, pattern := range trap.config.Gerrit.ExcludePattern {
		if matched, err := regexp.MatchString(pattern, subject); err == nil && matched == true {
			return matched, pattern
		}
	}

	return false, ""
}
<|endoftext|>


<commit_before>/*
 * Copyright 2019 Dgraph Labs, Inc. and Contributors
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package badger

import (
	"math"

	"github.com/dgraph-io/badger/pb"
	"github.com/dgraph-io/badger/table"
	"github.com/dgraph-io/badger/y"
	humanize "github.com/dustin/go-humanize"
	"github.com/pkg/errors"
)

const headStreamId uint32 = math.MaxUint32

// StreamWriter is used to write data coming from multiple streams. The streams must not have any
// overlapping key ranges. Within each stream, the keys must be sorted. Badger Stream framework is
// capable of generating such an output. So, this StreamWriter can be used at the other end to build
// BadgerDB at a much faster pace by writing SSTables (and value logs) directly to LSM tree levels
// without causing any compactions at all. This is way faster than using batched writer or using
// transactions, but only applicable in situations where the keys are pre-sorted and the DB is being
// bootstrapped. Existing data would get deleted when using this writer. So, this is only useful
// when restoring from backup or replicating DB across servers.
//
// StreamWriter should not be called on in-use DB instances. It is designed only to bootstrap new
// DBs.
type StreamWriter struct {
	db         *DB
	done       func()
	throttle   *y.Throttle
	maxVersion uint64
	writers    map[uint32]*sortedWriter
	closer     *y.Closer
}

// NewStreamWriter creates a StreamWriter. Right after creating StreamWriter, Prepare must be
// called. The memory usage of a StreamWriter is directly proportional to the number of streams
// possible. So, efforts must be made to keep the number of streams low. Stream framework would
// typically use 16 goroutines and hence create 16 streams.
func (db *DB) NewStreamWriter() *StreamWriter {
	return &StreamWriter{
		db: db,
		// throttle shouldn't make much difference. Memory consumption is based on the number of
		// concurrent streams being processed.
		throttle: y.NewThrottle(16),
		writers:  make(map[uint32]*sortedWriter),
		closer:   y.NewCloser(0),
	}
}

// Prepare should be called before writing any entry to StreamWriter. It deletes all data present in
// existing DB, stops compactions and any writes being done by other means. Be very careful when
// calling Prepare, because it could result in permanent data loss. Not calling Prepare would result
// in a corrupt Badger instance.
func (sw *StreamWriter) Prepare() error {
	var err error
	sw.done, err = sw.db.dropAll()
	return err
}

// Write writes KVList to DB. Each KV within the list contains the stream id which StreamWriter
// would use to demux the writes. Write is not thread safe and it should NOT be called concurrently.
func (sw *StreamWriter) Write(kvs *pb.KVList) error {
	if len(kvs.GetKv()) == 0 {
		return nil
	}
	streamReqs := make(map[uint32]*request)
	for _, kv := range kvs.Kv {
		var meta, userMeta byte
		if len(kv.Meta) > 0 {
			meta = kv.Meta[0]
		}
		if len(kv.UserMeta) > 0 {
			userMeta = kv.UserMeta[0]
		}
		if sw.maxVersion < kv.Version {
			sw.maxVersion = kv.Version
		}
		e := &Entry{
			Key:       y.KeyWithTs(kv.Key, kv.Version),
			Value:     kv.Value,
			UserMeta:  userMeta,
			ExpiresAt: kv.ExpiresAt,
			meta:      meta,
		}
		// If the value can be colocated with the key in LSM tree, we can skip
		// writing the value to value log.
		e.skipVlog = sw.db.shouldWriteValueToLSM(*e)
		req := streamReqs[kv.StreamId]
		if req == nil {
			req = &request{}
			streamReqs[kv.StreamId] = req
		}
		req.Entries = append(req.Entries, e)
	}
	var all []*request
	for _, req := range streamReqs {
		all = append(all, req)
	}
	if err := sw.db.vlog.write(all); err != nil {
		return err
	}

	for streamId, req := range streamReqs {
		writer, ok := sw.writers[streamId]
		if !ok {
			writer = sw.newWriter(streamId)
			sw.writers[streamId] = writer
		}
		writer.reqCh <- req
	}
	return nil
}

// Flush is called once we are done writing all the entries. It syncs DB directories. It also
// updates Oracle with maxVersion found in all entries (if DB is not managed).
func (sw *StreamWriter) Flush() error {
	defer sw.done()

	sw.closer.SignalAndWait()
	var maxHead valuePointer
	for _, writer := range sw.writers {
		if err := writer.Done(); err != nil {
			return err
		}
		if maxHead.Less(writer.head) {
			maxHead = writer.head
		}
	}

	// Encode and write the value log head into a new table.
	data := make([]byte, vptrSize)
	maxHead.Encode(data)
	headWriter := sw.newWriter(headStreamId)
	if err := headWriter.Add(
		y.KeyWithTs(head, sw.maxVersion),
		y.ValueStruct{Value: data}); err != nil {
		return err
	}
	if err := headWriter.Done(); err != nil {
		return err
	}

	if !sw.db.opt.managedTxns {
		if sw.db.orc != nil {
			sw.db.orc.Stop()
		}
		sw.db.orc = newOracle(sw.db.opt)
		sw.db.orc.nextTxnTs = sw.maxVersion
		sw.db.orc.txnMark.Done(sw.maxVersion)
		sw.db.orc.readMark.Done(sw.maxVersion)
		sw.db.orc.incrementNextTs()
	}

	// Wait for all files to be written.
	if err := sw.throttle.Finish(); err != nil {
		return err
	}

	// Now sync the directories, so all the files are registered.
	if sw.db.opt.ValueDir != sw.db.opt.Dir {
		if err := syncDir(sw.db.opt.ValueDir); err != nil {
			return err
		}
	}
	if err := syncDir(sw.db.opt.Dir); err != nil {
		return err
	}
	return sw.db.lc.validate()
}

type sortedWriter struct {
	db       *DB
	throttle *y.Throttle

	builder  *table.Builder
	lastKey  []byte
	streamId uint32
	reqCh    chan *request
	head     valuePointer
}

func (sw *StreamWriter) newWriter(streamId uint32) *sortedWriter {
	bopts := table.Options{
		BlockSize:         sw.db.opt.BlockSize,
		BloomFalsePostive: sw.db.opt.BloomFalsePositive,
	}
	w := &sortedWriter{
		db:       sw.db,
		streamId: streamId,
		throttle: sw.throttle,
		builder:  table.NewTableBuilder(bopts),
		reqCh:    make(chan *request, 3),
	}
	sw.closer.AddRunning(1)
	go w.handleRequests(sw.closer)
	return w
}

// ErrUnsortedKey is returned when any out of order key arrives at sortedWriter during call to Add.
var ErrUnsortedKey = errors.New("Keys not in sorted order")

func (w *sortedWriter) handleRequests(closer *y.Closer) {
	defer closer.Done()

	process := func(req *request) {
		for i, e := range req.Entries {
			vptr := req.Ptrs[i]
			if !vptr.IsZero() {
				y.AssertTrue(w.head.Less(vptr))
				w.head = vptr
			}

			var vs y.ValueStruct
			if e.skipVlog {
				vs = y.ValueStruct{
					Value:     e.Value,
					Meta:      e.meta,
					UserMeta:  e.UserMeta,
					ExpiresAt: e.ExpiresAt,
				}
			} else {
				vbuf := make([]byte, vptrSize)
				vs = y.ValueStruct{
					Value:     vptr.Encode(vbuf),
					Meta:      e.meta | bitValuePointer,
					UserMeta:  e.UserMeta,
					ExpiresAt: e.ExpiresAt,
				}
			}
			if err := w.Add(e.Key, vs); err != nil {
				panic(err)
			}
		}
	}

	for {
		select {
		case req := <-w.reqCh:
			process(req)
		case <-closer.HasBeenClosed():
			close(w.reqCh)
			for req := range w.reqCh {
				process(req)
			}
			return
		}
	}
}

// Add adds key and vs to sortedWriter.
func (w *sortedWriter) Add(key []byte, vs y.ValueStruct) error {
	if len(w.lastKey) > 0 && y.CompareKeys(key, w.lastKey) <= 0 {
		return ErrUnsortedKey
	}

	sameKey := y.SameKey(key, w.lastKey)
	// Same keys should go into the same SSTable.
	if !sameKey && w.builder.ReachedCapacity(w.db.opt.MaxTableSize) {
		if err := w.send(); err != nil {
			return err
		}
	}

	w.lastKey = y.SafeCopy(w.lastKey, key)
	return w.builder.Add(key, vs)
}

func (w *sortedWriter) send() error {
	if err := w.throttle.Do(); err != nil {
		return err
	}
	go func(builder *table.Builder) {
		data := builder.Finish()
		err := w.createTable(data)
		w.throttle.Done(err)
	}(w.builder)

	bopts := table.Options{
		BlockSize:         w.db.opt.BlockSize,
		BloomFalsePostive: w.db.opt.BloomFalsePositive,
	}
	w.builder = table.NewTableBuilder(bopts)
	return nil
}

// Done is called once we are done writing all keys and valueStructs
// to sortedWriter. It completes writing current SST to disk.
func (w *sortedWriter) Done() error {
	if w.builder.Empty() {
		return nil
	}
	return w.send()
}

func (w *sortedWriter) createTable(data []byte) error {
	if len(data) == 0 {
		return nil
	}
	fileID := w.db.lc.reserveFileID()
	fd, err := y.CreateSyncedFile(table.NewFilename(fileID, w.db.opt.Dir), true)
	if err != nil {
		return err
	}
	if _, err := fd.Write(data); err != nil {
		return err
	}
	opts := table.Options{
		LoadingMode: w.db.opt.TableLoadingMode,
		ChkMode:     w.db.opt.ChecksumVerificationMode,
	}
	tbl, err := table.OpenTable(fd, opts)
	if err != nil {
		return err
	}
	lc := w.db.lc

	var lhandler *levelHandler
	// We should start the levels from 1, because we need level 0 to set the !badger!head key. We
	// cannot mix up this key with other keys from the DB, otherwise we would introduce a range
	// overlap violation.
	y.AssertTrue(len(lc.levels) > 1)
	for _, l := range lc.levels[1:] {
		ratio := float64(l.getTotalSize()) / float64(l.maxTotalSize)
		if ratio < 1.0 {
			lhandler = l
			break
		}
	}
	if lhandler == nil {
		// If we're exceeding the size of the lowest level, shove it in the lowest level. Can't do
		// better than that.
		lhandler = lc.levels[len(lc.levels)-1]
	}
	if w.streamId == headStreamId {
		// This is a special !badger!head key. We should store it at level 0, separate from all the
		// other keys to avoid an overlap.
		lhandler = lc.levels[0]
	}
	// Now that table can be opened successfully, let's add this to the MANIFEST.
	change := &pb.ManifestChange{
		Id:    tbl.ID(),
		Op:    pb.ManifestChange_CREATE,
		Level: uint32(lhandler.level),
	}
	if err := w.db.manifest.addChanges([]*pb.ManifestChange{change}); err != nil {
		return err
	}
	if err := lhandler.replaceTables([]*table.Table{}, []*table.Table{tbl}); err != nil {
		return err
	}
	w.db.opt.Infof("Table created: %d at level: %d for stream: %d. Size: %s\n",
		fileID, lhandler.level, w.streamId, humanize.Bytes(uint64(tbl.Size())))
	return nil
}
<commit_msg>Making the stream writer APIs goroutine-safe (#959)<commit_after>/*
 * Copyright 2019 Dgraph Labs, Inc. and Contributors
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package badger

import (
	"math"
	"sync"

	"github.com/dgraph-io/badger/pb"
	"github.com/dgraph-io/badger/table"
	"github.com/dgraph-io/badger/y"
	humanize "github.com/dustin/go-humanize"
	"github.com/pkg/errors"
)

const headStreamId uint32 = math.MaxUint32

// StreamWriter is used to write data coming from multiple streams. The streams must not have any
// overlapping key ranges. Within each stream, the keys must be sorted. Badger Stream framework is
// capable of generating such an output. So, this StreamWriter can be used at the other end to build
// BadgerDB at a much faster pace by writing SSTables (and value logs) directly to LSM tree levels
// without causing any compactions at all. This is way faster than using batched writer or using
// transactions, but only applicable in situations where the keys are pre-sorted and the DB is being
// bootstrapped. Existing data would get deleted when using this writer. So, this is only useful
// when restoring from backup or replicating DB across servers.
//
// StreamWriter should not be called on in-use DB instances. It is designed only to bootstrap new
// DBs.
type StreamWriter struct {
	writeLock  sync.Mutex
	db         *DB
	done       func()
	throttle   *y.Throttle
	maxVersion uint64
	writers    map[uint32]*sortedWriter
	closer     *y.Closer
}

// NewStreamWriter creates a StreamWriter. Right after creating StreamWriter, Prepare must be
// called. The memory usage of a StreamWriter is directly proportional to the number of streams
// possible. So, efforts must be made to keep the number of streams low. Stream framework would
// typically use 16 goroutines and hence create 16 streams.
func (db *DB) NewStreamWriter() *StreamWriter {
	return &StreamWriter{
		db: db,
		// throttle shouldn't make much difference. Memory consumption is based on the number of
		// concurrent streams being processed.
		throttle: y.NewThrottle(16),
		writers:  make(map[uint32]*sortedWriter),
		closer:   y.NewCloser(0),
	}
}

// Prepare should be called before writing any entry to StreamWriter. It deletes all data present in
// existing DB, stops compactions and any writes being done by other means. Be very careful when
// calling Prepare, because it could result in permanent data loss. Not calling Prepare would result
// in a corrupt Badger instance.
func (sw *StreamWriter) Prepare() error {
	sw.writeLock.Lock()
	defer sw.writeLock.Unlock()

	var err error
	sw.done, err = sw.db.dropAll()
	return err
}

// Write writes KVList to DB. Each KV within the list contains the stream id which StreamWriter
// would use to demux the writes. Write is thread safe and can be called concurrently by mulitple
// goroutines.
func (sw *StreamWriter) Write(kvs *pb.KVList) error {
	if len(kvs.GetKv()) == 0 {
		return nil
	}
	streamReqs := make(map[uint32]*request)
	for _, kv := range kvs.Kv {
		var meta, userMeta byte
		if len(kv.Meta) > 0 {
			meta = kv.Meta[0]
		}
		if len(kv.UserMeta) > 0 {
			userMeta = kv.UserMeta[0]
		}
		if sw.maxVersion < kv.Version {
			sw.maxVersion = kv.Version
		}
		e := &Entry{
			Key:       y.KeyWithTs(kv.Key, kv.Version),
			Value:     kv.Value,
			UserMeta:  userMeta,
			ExpiresAt: kv.ExpiresAt,
			meta:      meta,
		}
		// If the value can be colocated with the key in LSM tree, we can skip
		// writing the value to value log.
		e.skipVlog = sw.db.shouldWriteValueToLSM(*e)
		req := streamReqs[kv.StreamId]
		if req == nil {
			req = &request{}
			streamReqs[kv.StreamId] = req
		}
		req.Entries = append(req.Entries, e)
	}
	var all []*request
	for _, req := range streamReqs {
		all = append(all, req)
	}

	sw.writeLock.Lock()
	defer sw.writeLock.Unlock()
	if err := sw.db.vlog.write(all); err != nil {
		return err
	}

	for streamId, req := range streamReqs {
		writer, ok := sw.writers[streamId]
		if !ok {
			writer = sw.newWriter(streamId)
			sw.writers[streamId] = writer
		}
		writer.reqCh <- req
	}
	return nil
}

// Flush is called once we are done writing all the entries. It syncs DB directories. It also
// updates Oracle with maxVersion found in all entries (if DB is not managed).
func (sw *StreamWriter) Flush() error {
	sw.writeLock.Lock()
	defer sw.writeLock.Unlock()

	defer sw.done()

	sw.closer.SignalAndWait()
	var maxHead valuePointer
	for _, writer := range sw.writers {
		if err := writer.Done(); err != nil {
			return err
		}
		if maxHead.Less(writer.head) {
			maxHead = writer.head
		}
	}

	// Encode and write the value log head into a new table.
	data := make([]byte, vptrSize)
	maxHead.Encode(data)
	headWriter := sw.newWriter(headStreamId)
	if err := headWriter.Add(
		y.KeyWithTs(head, sw.maxVersion),
		y.ValueStruct{Value: data}); err != nil {
		return err
	}
	if err := headWriter.Done(); err != nil {
		return err
	}

	if !sw.db.opt.managedTxns {
		if sw.db.orc != nil {
			sw.db.orc.Stop()
		}
		sw.db.orc = newOracle(sw.db.opt)
		sw.db.orc.nextTxnTs = sw.maxVersion
		sw.db.orc.txnMark.Done(sw.maxVersion)
		sw.db.orc.readMark.Done(sw.maxVersion)
		sw.db.orc.incrementNextTs()
	}

	// Wait for all files to be written.
	if err := sw.throttle.Finish(); err != nil {
		return err
	}

	// Now sync the directories, so all the files are registered.
	if sw.db.opt.ValueDir != sw.db.opt.Dir {
		if err := syncDir(sw.db.opt.ValueDir); err != nil {
			return err
		}
	}
	if err := syncDir(sw.db.opt.Dir); err != nil {
		return err
	}
	return sw.db.lc.validate()
}

type sortedWriter struct {
	db       *DB
	throttle *y.Throttle

	builder  *table.Builder
	lastKey  []byte
	streamId uint32
	reqCh    chan *request
	head     valuePointer
}

func (sw *StreamWriter) newWriter(streamId uint32) *sortedWriter {
	bopts := table.Options{
		BlockSize:         sw.db.opt.BlockSize,
		BloomFalsePostive: sw.db.opt.BloomFalsePositive,
	}
	w := &sortedWriter{
		db:       sw.db,
		streamId: streamId,
		throttle: sw.throttle,
		builder:  table.NewTableBuilder(bopts),
		reqCh:    make(chan *request, 3),
	}
	sw.closer.AddRunning(1)
	go w.handleRequests(sw.closer)
	return w
}

// ErrUnsortedKey is returned when any out of order key arrives at sortedWriter during call to Add.
var ErrUnsortedKey = errors.New("Keys not in sorted order")

func (w *sortedWriter) handleRequests(closer *y.Closer) {
	defer closer.Done()

	process := func(req *request) {
		for i, e := range req.Entries {
			vptr := req.Ptrs[i]
			if !vptr.IsZero() {
				y.AssertTrue(w.head.Less(vptr))
				w.head = vptr
			}

			var vs y.ValueStruct
			if e.skipVlog {
				vs = y.ValueStruct{
					Value:     e.Value,
					Meta:      e.meta,
					UserMeta:  e.UserMeta,
					ExpiresAt: e.ExpiresAt,
				}
			} else {
				vbuf := make([]byte, vptrSize)
				vs = y.ValueStruct{
					Value:     vptr.Encode(vbuf),
					Meta:      e.meta | bitValuePointer,
					UserMeta:  e.UserMeta,
					ExpiresAt: e.ExpiresAt,
				}
			}
			if err := w.Add(e.Key, vs); err != nil {
				panic(err)
			}
		}
	}

	for {
		select {
		case req := <-w.reqCh:
			process(req)
		case <-closer.HasBeenClosed():
			close(w.reqCh)
			for req := range w.reqCh {
				process(req)
			}
			return
		}
	}
}

// Add adds key and vs to sortedWriter.
func (w *sortedWriter) Add(key []byte, vs y.ValueStruct) error {
	if len(w.lastKey) > 0 && y.CompareKeys(key, w.lastKey) <= 0 {
		return ErrUnsortedKey
	}

	sameKey := y.SameKey(key, w.lastKey)
	// Same keys should go into the same SSTable.
	if !sameKey && w.builder.ReachedCapacity(w.db.opt.MaxTableSize) {
		if err := w.send(); err != nil {
			return err
		}
	}

	w.lastKey = y.SafeCopy(w.lastKey, key)
	return w.builder.Add(key, vs)
}

func (w *sortedWriter) send() error {
	if err := w.throttle.Do(); err != nil {
		return err
	}
	go func(builder *table.Builder) {
		data := builder.Finish()
		err := w.createTable(data)
		w.throttle.Done(err)
	}(w.builder)

	bopts := table.Options{
		BlockSize:         w.db.opt.BlockSize,
		BloomFalsePostive: w.db.opt.BloomFalsePositive,
	}
	w.builder = table.NewTableBuilder(bopts)
	return nil
}

// Done is called once we are done writing all keys and valueStructs
// to sortedWriter. It completes writing current SST to disk.
func (w *sortedWriter) Done() error {
	if w.builder.Empty() {
		return nil
	}
	return w.send()
}

func (w *sortedWriter) createTable(data []byte) error {
	if len(data) == 0 {
		return nil
	}
	fileID := w.db.lc.reserveFileID()
	fd, err := y.CreateSyncedFile(table.NewFilename(fileID, w.db.opt.Dir), true)
	if err != nil {
		return err
	}
	if _, err := fd.Write(data); err != nil {
		return err
	}
	opts := table.Options{
		LoadingMode: w.db.opt.TableLoadingMode,
		ChkMode:     w.db.opt.ChecksumVerificationMode,
	}
	tbl, err := table.OpenTable(fd, opts)
	if err != nil {
		return err
	}
	lc := w.db.lc

	var lhandler *levelHandler
	// We should start the levels from 1, because we need level 0 to set the !badger!head key. We
	// cannot mix up this key with other keys from the DB, otherwise we would introduce a range
	// overlap violation.
	y.AssertTrue(len(lc.levels) > 1)
	for _, l := range lc.levels[1:] {
		ratio := float64(l.getTotalSize()) / float64(l.maxTotalSize)
		if ratio < 1.0 {
			lhandler = l
			break
		}
	}
	if lhandler == nil {
		// If we're exceeding the size of the lowest level, shove it in the lowest level. Can't do
		// better than that.
		lhandler = lc.levels[len(lc.levels)-1]
	}
	if w.streamId == headStreamId {
		// This is a special !badger!head key. We should store it at level 0, separate from all the
		// other keys to avoid an overlap.
		lhandler = lc.levels[0]
	}
	// Now that table can be opened successfully, let's add this to the MANIFEST.
	change := &pb.ManifestChange{
		Id:    tbl.ID(),
		Op:    pb.ManifestChange_CREATE,
		Level: uint32(lhandler.level),
	}
	if err := w.db.manifest.addChanges([]*pb.ManifestChange{change}); err != nil {
		return err
	}
	if err := lhandler.replaceTables([]*table.Table{}, []*table.Table{tbl}); err != nil {
		return err
	}
	w.db.opt.Infof("Table created: %d at level: %d for stream: %d. Size: %s\n",
		fileID, lhandler.level, w.streamId, humanize.Bytes(uint64(tbl.Size())))
	return nil
}
<|endoftext|>


<commit_before>package main

type bash int

var BASH bash

func (b bash) Hook() string {
	return `PROMPT_COMMAND="eval \"\$(direnv export bash)\";$PROMPT_COMMAND"`
}

func (b bash) Escape(str string) string {
	return ShellEscape(str)
}

func (b bash) Export(key, value string) string {
	return "export " + b.Escape(key) + "=" + b.Escape(value) + ";"
}

func (b bash) Unset(key string) string {
	return "unset " + b.Escape(key) + ";"
}
<commit_msg>Makes the bash hook resistant to double-hooking.<commit_after>package main

type bash int

var BASH bash

const BASH_HOOK = `
_direnv_hook() {
  eval "$(direnv export bash)";
}
if ! [[ "$PROMPT_COMMAND" =~ _direnv_hook ]]; then
  PROMPT_COMMAND="_direnv_hook;$PROMPT_COMMAND";
fi
`

func (b bash) Hook() string {
	return BASH_HOOK
}

func (b bash) Escape(str string) string {
	return ShellEscape(str)
}

func (b bash) Export(key, value string) string {
	return "export " + b.Escape(key) + "=" + b.Escape(value) + ";"
}

func (b bash) Unset(key string) string {
	return "unset " + b.Escape(key) + ";"
}
<|endoftext|>


<commit_before>// send.go - mixnet client send
// Copyright (C) 2018  David Stainton.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

package session

import (
	"encoding/binary"
	"encoding/hex"
	"errors"
	"fmt"
	"io"
	"time"

	cConstants "github.com/katzenpost/client/constants"
	"github.com/katzenpost/core/constants"
	"github.com/katzenpost/core/crypto/rand"
	sConstants "github.com/katzenpost/core/sphinx/constants"
)

const roundTripTimeSlop = time.Duration(88 * time.Second)

var ReplyTimeoutError = errors.New("Failure waiting for reply, timeout reached")

func (s *Session) sendNext() error {
	msg, err := s.egressQueue.Peek()
	if err != nil {
		return err
	}
	if msg == nil {
		return errors.New("send next failure, message is nil")
	}
	m, ok := msg.(*Message)
	if !ok {
		return errors.New("send next failure, unknown message type")
	}
	err = s.doSend(m)
	if err != nil {
		return err
	}
	_, err = s.egressQueue.Pop()
	return err
}

func (s *Session) doSend(msg *Message) error {
	surbID := [sConstants.SURBIDLength]byte{}
	_, err := io.ReadFull(rand.Reader, surbID[:])
	if err != nil {
		return err
	}
	idStr := fmt.Sprintf("[%v]", hex.EncodeToString(surbID[:]))
	s.log.Debugf("doSend with SURB ID %x", idStr)
	key := []byte{}
	var eta time.Duration
	if msg.WithSURB {
		key, eta, err = s.minclient.SendCiphertext(msg.Recipient, msg.Provider, &surbID, msg.Payload)
	} else {
		err = s.minclient.SendUnreliableCiphertext(msg.Recipient, msg.Provider, msg.Payload)
	}
	if err != nil {
		return err
	}
	if msg.WithSURB {
		s.log.Debugf("doSend setting ReplyETA to %v", eta)
		msg.Key = key
		msg.SentAt = time.Now()
		msg.Sent = true
		msg.ReplyETA = eta
		s.surbIDMap.Store(surbID, msg)
	}
	if msg.IsBlocking {
		sentWaitChanRaw, ok := s.sentWaitChanMap.Load(*msg.ID)
		if !ok {
			err := fmt.Errorf("Impossible failure, sentWaitChan not found for message ID %x", *msg.ID)
			s.log.Error(err.Error())
			s.fatalErrCh <- err
		}
		sentWaitChan := sentWaitChanRaw.(chan *Message)
		sentWaitChan <- msg
	} else {
		s.eventCh.In() <- &MessageSentEvent{
			MessageID: msg.ID,
			Err:       nil, // XXX send error
		}
	}
	return nil
}

func (s *Session) sendLoopDecoy() error {
	s.log.Info("sending loop decoy")
	const loopService = "loop"
	serviceDesc, err := s.GetService(loopService)
	if err != nil {
		return err
	}
	payload := [constants.UserForwardPayloadLength]byte{}
	id := [cConstants.MessageIDLength]byte{}
	_, err = io.ReadFull(rand.Reader, id[:])
	if err != nil {
		return err
	}
	msg := &Message{
		ID:        &id,
		Recipient: serviceDesc.Name,
		Provider:  serviceDesc.Provider,
		Payload:   payload[:],
		WithSURB:  true,
		IsDecoy:   true,
	}
	defer s.incrementDecoyLoopTally()
	return s.doSend(msg)
}

func (s *Session) sendDropDecoy() error {
	s.log.Info("sending drop decoy")
	const loopService = "loop"
	serviceDesc, err := s.GetService(loopService)
	if err != nil {
		return err
	}
	payload := [constants.UserForwardPayloadLength]byte{}
	id := [cConstants.MessageIDLength]byte{}
	_, err = io.ReadFull(rand.Reader, id[:])
	if err != nil {
		return err
	}
	msg := &Message{
		ID:        &id,
		Recipient: serviceDesc.Name,
		Provider:  serviceDesc.Provider,
		Payload:   payload[:],
		WithSURB:  false,
		IsDecoy:   true,
	}
	return s.doSend(msg)
}

func (s *Session) composeMessage(recipient, provider string, message []byte, isBlocking bool) (*Message, error) {
	s.log.Debug("SendMessage")
	if len(message) > constants.UserForwardPayloadLength-4 {
		return nil, fmt.Errorf("invalid message size: %v", len(message))
	}
	payload := [constants.UserForwardPayloadLength]byte{}
	binary.BigEndian.PutUint32(payload[:4], uint32(len(message)))
	copy(payload[4:], message)
	id := [cConstants.MessageIDLength]byte{}
	_, err := io.ReadFull(rand.Reader, id[:])
	if err != nil {
		return nil, err
	}
	var msg = Message{
		ID:         &id,
		Recipient:  recipient,
		Provider:   provider,
		Payload:    payload[:],
		WithSURB:   true,
		SURBType:   cConstants.SurbTypeKaetzchen,
		IsBlocking: isBlocking,
	}
	return &msg, nil
}

// SendUnreliableMessage asynchronously sends message without any automatic retransmissions.
func (s *Session) SendUnreliableMessage(recipient, provider string, message []byte) (MessageID, error) {
	msg, err := s.composeMessage(recipient, provider, message, false)
	if err != nil {
		return nil, err
	}
	err = s.egressQueue.Push(msg)
	if err != nil {
		return nil, err
	}
	return msg.ID, nil
}

func (s *Session) BlockingSendUnreliableMessage(recipient, provider string, message []byte) ([]byte, error) {
	msg, err := s.composeMessage(recipient, provider, message, true)
	if err != nil {
		return nil, err
	}
	sentWaitChan := make(chan interface{})
	s.sentWaitChanMap.Store(*msg.ID, sentWaitChan)

	replyWaitChan := make(chan []byte)
	s.replyWaitChanMap.Store(*msg.ID, replyWaitChan)

	err = s.egressQueue.Push(msg)
	if err != nil {
		return nil, err
	}
	// wait until sent so that we know the ReplyETA for the waiting below
	sentMessageRaw := <-sentWaitChan
	sentMessage := sentMessageRaw.(*Message)
	s.sentWaitChanMap.Delete(*msg.ID)
	// wait for reply or round trip timeout
	select {
	case reply := <-replyWaitChan:
		s.replyWaitChanMap.Delete(*msg.ID)
		return reply, nil
	case <-time.After(sentMessage.ReplyETA + roundTripTimeSlop):
		return nil, ReplyTimeoutError
	}
	// unreachable
}
<commit_msg>Attempt to fix sync.Map usage bug of sentWaitChanMap<commit_after>// send.go - mixnet client send
// Copyright (C) 2018  David Stainton.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

package session

import (
	"encoding/binary"
	"encoding/hex"
	"errors"
	"fmt"
	"io"
	"time"

	cConstants "github.com/katzenpost/client/constants"
	"github.com/katzenpost/core/constants"
	"github.com/katzenpost/core/crypto/rand"
	sConstants "github.com/katzenpost/core/sphinx/constants"
)

const roundTripTimeSlop = time.Duration(88 * time.Second)

var ReplyTimeoutError = errors.New("Failure waiting for reply, timeout reached")

func (s *Session) sendNext() error {
	msg, err := s.egressQueue.Peek()
	if err != nil {
		return err
	}
	if msg == nil {
		return errors.New("send next failure, message is nil")
	}
	m, ok := msg.(*Message)
	if !ok {
		return errors.New("send next failure, unknown message type")
	}
	err = s.doSend(m)
	if err != nil {
		return err
	}
	_, err = s.egressQueue.Pop()
	return err
}

func (s *Session) doSend(msg *Message) error {
	surbID := [sConstants.SURBIDLength]byte{}
	_, err := io.ReadFull(rand.Reader, surbID[:])
	if err != nil {
		return err
	}
	idStr := fmt.Sprintf("[%v]", hex.EncodeToString(surbID[:]))
	s.log.Debugf("doSend with SURB ID %x", idStr)
	key := []byte{}
	var eta time.Duration
	if msg.WithSURB {
		key, eta, err = s.minclient.SendCiphertext(msg.Recipient, msg.Provider, &surbID, msg.Payload)
	} else {
		err = s.minclient.SendUnreliableCiphertext(msg.Recipient, msg.Provider, msg.Payload)
	}
	if err != nil {
		return err
	}
	if msg.WithSURB {
		s.log.Debugf("doSend setting ReplyETA to %v", eta)
		msg.Key = key
		msg.SentAt = time.Now()
		msg.Sent = true
		msg.ReplyETA = eta
		s.surbIDMap.Store(surbID, msg)
	}
	if msg.IsBlocking {
		sentWaitChanRaw, ok := s.sentWaitChanMap.Load(*msg.ID)
		if !ok {
			err := fmt.Errorf("Impossible failure, sentWaitChan not found for message ID %x", *msg.ID)
			s.log.Error(err.Error())
			s.fatalErrCh <- err
		}
		sentWaitChan := sentWaitChanRaw.(chan *Message)
		sentWaitChan <- msg
	} else {
		s.eventCh.In() <- &MessageSentEvent{
			MessageID: msg.ID,
			Err:       nil, // XXX send error
		}
	}
	return nil
}

func (s *Session) sendLoopDecoy() error {
	s.log.Info("sending loop decoy")
	const loopService = "loop"
	serviceDesc, err := s.GetService(loopService)
	if err != nil {
		return err
	}
	payload := [constants.UserForwardPayloadLength]byte{}
	id := [cConstants.MessageIDLength]byte{}
	_, err = io.ReadFull(rand.Reader, id[:])
	if err != nil {
		return err
	}
	msg := &Message{
		ID:        &id,
		Recipient: serviceDesc.Name,
		Provider:  serviceDesc.Provider,
		Payload:   payload[:],
		WithSURB:  true,
		IsDecoy:   true,
	}
	defer s.incrementDecoyLoopTally()
	return s.doSend(msg)
}

func (s *Session) sendDropDecoy() error {
	s.log.Info("sending drop decoy")
	const loopService = "loop"
	serviceDesc, err := s.GetService(loopService)
	if err != nil {
		return err
	}
	payload := [constants.UserForwardPayloadLength]byte{}
	id := [cConstants.MessageIDLength]byte{}
	_, err = io.ReadFull(rand.Reader, id[:])
	if err != nil {
		return err
	}
	msg := &Message{
		ID:        &id,
		Recipient: serviceDesc.Name,
		Provider:  serviceDesc.Provider,
		Payload:   payload[:],
		WithSURB:  false,
		IsDecoy:   true,
	}
	return s.doSend(msg)
}

func (s *Session) composeMessage(recipient, provider string, message []byte, isBlocking bool) (*Message, error) {
	s.log.Debug("SendMessage")
	if len(message) > constants.UserForwardPayloadLength-4 {
		return nil, fmt.Errorf("invalid message size: %v", len(message))
	}
	payload := [constants.UserForwardPayloadLength]byte{}
	binary.BigEndian.PutUint32(payload[:4], uint32(len(message)))
	copy(payload[4:], message)
	id := [cConstants.MessageIDLength]byte{}
	_, err := io.ReadFull(rand.Reader, id[:])
	if err != nil {
		return nil, err
	}
	var msg = Message{
		ID:         &id,
		Recipient:  recipient,
		Provider:   provider,
		Payload:    payload[:],
		WithSURB:   true,
		SURBType:   cConstants.SurbTypeKaetzchen,
		IsBlocking: isBlocking,
	}
	return &msg, nil
}

// SendUnreliableMessage asynchronously sends message without any automatic retransmissions.
func (s *Session) SendUnreliableMessage(recipient, provider string, message []byte) (MessageID, error) {
	msg, err := s.composeMessage(recipient, provider, message, false)
	if err != nil {
		return nil, err
	}
	err = s.egressQueue.Push(msg)
	if err != nil {
		return nil, err
	}
	return msg.ID, nil
}

func (s *Session) BlockingSendUnreliableMessage(recipient, provider string, message []byte) ([]byte, error) {
	msg, err := s.composeMessage(recipient, provider, message, true)
	if err != nil {
		return nil, err
	}
	sentWaitChan := make(chan *Message)
	s.sentWaitChanMap.Store(*msg.ID, sentWaitChan)

	replyWaitChan := make(chan []byte)
	s.replyWaitChanMap.Store(*msg.ID, replyWaitChan)

	err = s.egressQueue.Push(msg)
	if err != nil {
		return nil, err
	}
	// wait until sent so that we know the ReplyETA for the waiting below
	sentMessageRaw := <-sentWaitChan
	sentMessage := sentMessageRaw.(*Message)
	s.sentWaitChanMap.Delete(*msg.ID)
	// wait for reply or round trip timeout
	select {
	case reply := <-replyWaitChan:
		s.replyWaitChanMap.Delete(*msg.ID)
		return reply, nil
	case <-time.After(sentMessage.ReplyETA + roundTripTimeSlop):
		return nil, ReplyTimeoutError
	}
	// unreachable
}
<|endoftext|>


<commit_before>package sesstype

import (
	"fmt"
	"github.com/awalterschulze/gographviz"
	"os"
)

var (
	count                        = 0
	labelNodes map[string]string = make(map[string]string)
)

func GenDot(sess *Session) {
	graph := gographviz.NewEscape()
	graph.SetDir(true)
	graph.SetName("G")

	for role, root := range sess.types {
		sg := gographviz.NewSubGraph("cluster_" + role.Name())
		if root != nil {
			visitNode(root, graph, sg)
		}
		graph.AddSubGraph(graph.Name, sg.Name, map[string]string{"label": role.Name()})
	}

	f, err := os.OpenFile("output.dot", os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
	defer f.Close()
	if err != nil {
		panic(err)
	}

	_, err = f.WriteString(graph.String())
	if err != nil {
		panic(err)
	}
}

func CreateNode(node Node) *gographviz.Node {
	switch node := node.(type) {
	case *LabelNode:
		defer func() { count++ }()
		labelNodes[node.name] = fmt.Sprintf("label%d", count)
		dotNode := gographviz.Node{
			Name: labelNodes[node.name],
			Attrs: map[string]string{
				"label": fmt.Sprintf("\"%s\"", node.String()),
				"shape": "plaintext,",
			},
		}
		return &dotNode

	case *NewChanNode:
		defer func() { count++ }()
		return &gographviz.Node{
			Name: fmt.Sprintf("%s%d", node.Kind(), count),
			Attrs: map[string]string{
				"label": fmt.Sprintf("New channel %s", node.ch.Name()),
				"shape": "rect",
				"color": "red",
			},
		}

	case *SendNode:
		defer func() { count++ }()
		style := "solid"
		desc := ""
		if node.nondet {
			style = "dashed"
			desc = " nondet"
		}
		return &gographviz.Node{
			Name: fmt.Sprintf("%s%d", node.Kind(), count),
			Attrs: map[string]string{
				"label": fmt.Sprintf("Send %s%s", node.dest.Name(), desc),
				"shape": "rect",
				"style": style,
			},
		}

	case *RecvNode:
		defer func() { count++ }()
		style := "solid"
		desc := ""
		if node.nondet {
			style = "dashed"
			desc = " nondet"
		}
		return &gographviz.Node{
			Name: fmt.Sprintf("%s%d", node.Kind(), count),
			Attrs: map[string]string{
				"label": fmt.Sprintf("Receive %s%s", node.orig.Name(), desc),
				"shape": "rect",
				"style": style,
			},
		}

	case *GotoNode:
		return nil

	default:
		defer func() { count++ }()
		dotNode := gographviz.Node{
			Name: fmt.Sprintf("%s%d", node.Kind(), count),
			Attrs: map[string]string{
				"label": fmt.Sprintf("%s", node.String()),
				"shape": "rect",
			},
		}
		return &dotNode
	}
}

func visitNode(node Node, graph *gographviz.Escape, subgraph *gographviz.SubGraph) *gographviz.Node {
	gNode := CreateNode(node)
	if gNode != nil {
		if subgraph.Name == "cluster_main" && len(graph.Nodes.Nodes) == 0 {
			gNode.Attrs.Add("style", "bold")
		}
		graph.AddNode(subgraph.Name, gNode.Name, gNode.Attrs)
	}

	for _, child := range node.Children() {
		cNode := visitNode(child, graph, subgraph)
		if cNode != nil {
			graph.AddEdge(gNode.Name, cNode.Name, true, nil)
		} else {
			// Goto Node
			graph.AddEdge(gNode.Name, labelNodes[child.(*GotoNode).name], true, nil)
		}
	}

	return gNode
}
<commit_msg>Fix multi-subgraph generation<commit_after>package sesstype

import (
	"fmt"
	"github.com/awalterschulze/gographviz"
	"os"
)

var (
	count                        = 0
	labelNodes map[string]string = make(map[string]string)
)

func GenDot(sess *Session) {
	graph := gographviz.NewEscape()
	graph.SetDir(true)
	graph.SetName("G")

	for role, root := range sess.types {
		sg := gographviz.NewSubGraph("\"cluster_" + role.Name() + "\"")
		if root != nil {
			visitNode(root, graph, sg)
		}
		graph.AddSubGraph(graph.Name, sg.Name, map[string]string{"label": "\"" + role.Name() + "\""})
	}

	f, err := os.OpenFile("output.dot", os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
	defer f.Close()
	if err != nil {
		panic(err)
	}

	_, err = f.WriteString(graph.String())
	if err != nil {
		panic(err)
	}
}

func CreateNode(node Node) *gographviz.Node {
	switch node := node.(type) {
	case *LabelNode:
		defer func() { count++ }()
		labelNodes[node.name] = fmt.Sprintf("label%d", count)
		dotNode := gographviz.Node{
			Name: labelNodes[node.name],
			Attrs: map[string]string{
				"label": fmt.Sprintf("\"%s\"", node.String()),
				"shape": "plaintext,",
			},
		}
		return &dotNode

	case *NewChanNode:
		defer func() { count++ }()
		return &gographviz.Node{
			Name: fmt.Sprintf("%s%d", node.Kind(), count),
			Attrs: map[string]string{
				"label": fmt.Sprintf("New channel %s", node.ch.Name()),
				"shape": "rect",
				"color": "red",
			},
		}

	case *SendNode:
		defer func() { count++ }()
		style := "solid"
		desc := ""
		if node.nondet {
			style = "dashed"
			desc = " nondet"
		}
		return &gographviz.Node{
			Name: fmt.Sprintf("%s%d", node.Kind(), count),
			Attrs: map[string]string{
				"label": fmt.Sprintf("Send %s%s", node.dest.Name(), desc),
				"shape": "rect",
				"style": style,
			},
		}

	case *RecvNode:
		defer func() { count++ }()
		style := "solid"
		desc := ""
		if node.nondet {
			style = "dashed"
			desc = " nondet"
		}
		return &gographviz.Node{
			Name: fmt.Sprintf("%s%d", node.Kind(), count),
			Attrs: map[string]string{
				"label": fmt.Sprintf("Receive %s%s", node.orig.Name(), desc),
				"shape": "rect",
				"style": style,
			},
		}

	case *GotoNode:
		return nil

	default:
		defer func() { count++ }()
		dotNode := gographviz.Node{
			Name: fmt.Sprintf("%s%d", node.Kind(), count),
			Attrs: map[string]string{
				"label": fmt.Sprintf("\"%s\"", node.String()),
				"shape": "rect",
			},
		}
		return &dotNode
	}
}

func visitNode(node Node, graph *gographviz.Escape, subgraph *gographviz.SubGraph) *gographviz.Node {
	gNode := CreateNode(node)
	if gNode != nil {
		if subgraph.Name == "cluster_main" && len(graph.Nodes.Nodes) == 0 {
			gNode.Attrs.Add("style", "bold")
		}
		graph.AddNode(subgraph.Name, gNode.Name, gNode.Attrs)

		for _, child := range node.Children() {
			cNode := visitNode(child, graph, subgraph)
			if cNode != nil {
				graph.AddEdge(gNode.Name, cNode.Name, true, nil)
			} else {
				// Goto Node
				graph.AddEdge(gNode.Name, labelNodes[child.(*GotoNode).name], true, nil)
			}
		}
	}

	return gNode
}
<|endoftext|>


<commit_before>package shell

var Zsh = Shell(`# Put the line below in ~/.zshrc:
#
#   eval "$(jump shell zsh)"
#
# The following lines are autogenerated:

__jump_chpwd() {
  jump chdir
}

typeset -gaU chpwd_functions
chpwd_functions+=__jump_chpwd

j() {
  local dir="$(jump cd $@)"
  test -d "$dir" && cd "$dir"
}
`)
<commit_msg>Add sloppy zsh completion<commit_after>package shell

var Zsh = Shell(`# Put the line below in ~/.zshrc:
#
#   eval "$(jump shell zsh)"
#
# The following lines are autogenerated:

__jump_chpwd() {
  jump chdir
}

typeset -gaU chpwd_functions
chpwd_functions+=__jump_chpwd

j() {
  local dir="$(jump cd $@)"
  test -d "$dir" && cd "$dir"
}

jump_completion() {
  reply=($(jump hint $1 | head -3))
}

compctl -U -K jump_completion j
`)
<|endoftext|>


<commit_before><commit_msg>extend Nfa by functions to replace alphabet, states, initial states, final states<commit_after><|endoftext|>


<commit_before>package git

import (
	"bytes"
	"strconv"
	"time"
)

// Author and Committer information
type Signature struct {
	Email string
	Name  string
	When  time.Time
}

// Helper to get a signature from the commit line, which looks like this:
//     author Patrick Gundlach <gundlach@speedata.de> 1378823654 +0200
// but without the "author " at the beginning (this method should)
// be used for author and committer.
//
// FIXME: include timezone!
func newSignatureFromCommitline(line []byte) (*Signature, error) {
	sig := new(Signature)
	emailstart := bytes.IndexByte(line, '<')
	sig.Name = string(line[:emailstart-1])
	emailstop := bytes.IndexByte(line, '>')
	sig.Email = string(line[emailstart+1 : emailstop])
	timestop := bytes.IndexByte(line[emailstop+2:], ' ')
	timestring := string(line[emailstop+2 : emailstop+2+timestop])
	seconds, err := strconv.ParseInt(timestring, 10, 64)
	if err != nil {
		return nil, err
	}
	sig.When = time.Unix(seconds, 0)
	return sig, nil
}
<commit_msg>Implemented Stringer interface on Signature type (#23)<commit_after>package git

import (
	"bytes"
	"strconv"
	"time"
)

// Author and Committer information
type Signature struct {
	Email string
	Name  string
	When  time.Time
}

// Outputs the author signature in the same format as git log:
// "Author <email>". If needed, the "When" must be manually added to the
// string.
func (s Signature) String() string {
	return s.Name + " <" + s.Email + ">"
}

// Helper to get a signature from the commit line, which looks like this:
//     author Patrick Gundlach <gundlach@speedata.de> 1378823654 +0200
// but without the "author " at the beginning (this method should)
// be used for author and committer.
//
// FIXME: include timezone!
func newSignatureFromCommitline(line []byte) (*Signature, error) {
	sig := new(Signature)
	emailstart := bytes.IndexByte(line, '<')
	sig.Name = string(line[:emailstart-1])
	emailstop := bytes.IndexByte(line, '>')
	sig.Email = string(line[emailstart+1 : emailstop])
	timestop := bytes.IndexByte(line[emailstop+2:], ' ')
	timestring := string(line[emailstop+2 : emailstop+2+timestop])
	seconds, err := strconv.ParseInt(timestring, 10, 64)
	if err != nil {
		return nil, err
	}
	sig.When = time.Unix(seconds, 0)
	return sig, nil
}
<|endoftext|>


<commit_before>package sio

import (
	"encoding/binary"
	"io"
	"reflect"
)

// align4U32 returns sz adjusted to align at 4-byte boundaries
func align4U32(sz uint32) uint32 {
	return sz + (4-(sz&g_align))&g_align
}

// align4I32 returns sz adjusted to align at 4-byte boundaries
func align4I32(sz int32) int32 {
	return sz + (4-(sz&int32(g_align)))&int32(g_align)
}

// align4I64 returns sz adjusted to align at 4-byte boundaries
func align4I64(sz int64) int64 {
	return sz + (4-(sz&int64(g_align)))&int64(g_align)
}

func bread(r io.Reader, data interface{}) error {
	bo := binary.BigEndian
	rv := reflect.ValueOf(data)
	// fmt.Printf("::: [%v] :::...\n", rv.Type())
	// defer fmt.Printf("### [%v] [done]\n", rv.Type())

	switch rv.Type().Kind() {
	case reflect.Ptr:
		rv = rv.Elem()
	}
	switch rv.Type().Kind() {
	case reflect.Struct:
		// fmt.Printf(">>> struct: [%v]...\n", rv.Type())
		for i, n := 0, rv.NumField(); i < n; i++ {
			// fmt.Printf(">>> i=%d [%v] (%v)...\n", i, rv.Field(i).Type(), rv.Type().Name()+"."+rv.Type().Field(i).Name)
			err := bread(r, rv.Field(i).Addr().Interface())
			if err != nil {
				return err
			}
			// fmt.Printf(">>> i=%d [%v] (%v)...[done=>%v]\n", i, rv.Field(i).Type(), rv.Type().Name(), rv.Field(i).Interface())
		}
		// fmt.Printf(">>> struct: [%v]...[done]\n", rv.Type())
		return nil
	case reflect.String:
		// fmt.Printf("++++> string...\n")
		sz := uint32(0)
		err := bread(r, &sz)
		if err != nil {
			return err
		}
		strlen := align4U32(sz)
		// fmt.Printf("string [%d=>%d]...\n", sz, strlen)
		str := make([]byte, strlen)
		_, err = r.Read(str)
		if err != nil {
			return err
		}
		rv.SetString(string(str[:sz]))
		//fmt.Printf("<++++ [%s]\n", string(str))
		return nil

	case reflect.Slice:
		//fmt.Printf("<<< slice: [%v|%v]...\n", rv.Type(), rv.Type().Elem().Kind())
		sz := uint32(0)
		err := bread(r, &sz)
		if err != nil {
			return err
		}
		//fmt.Printf("<<< slice: %d [%v]\n", sz, rv.Type())
		slice := reflect.MakeSlice(rv.Type(), int(sz), int(sz))
		for i := 0; i < int(sz); i++ {
			err = bread(r, slice.Index(i).Addr().Interface())
			if err != nil {
				return err
			}
		}
		rv.Set(slice)
		//fmt.Printf("<<< slice: [%v]... [done] (%v)\n", rv.Type(), rv.Interface())
		return err

	case reflect.Map:
		//fmt.Printf(">>> map: [%v]...\n", rv.Type())
		sz := uint32(0)
		err := bread(r, &sz)
		if err != nil {
			return err
		}
		//fmt.Printf(">>> map: %d [%v]\n", sz, rv.Type())
		m := reflect.MakeMap(rv.Type())
		kt := rv.Type().Key()
		vt := rv.Type().Elem()
		for i, n := 0, int(sz); i < n; i++ {
			kv := reflect.New(kt)
			err = bread(r, kv.Interface())
			if err != nil {
				return err
			}
			vv := reflect.New(vt)
			err = bread(r, vv.Interface())
			if err != nil {
				return err
			}
			//fmt.Printf("m - %d: {%v} - {%v}\n", i, kv.Elem().Interface(), vv.Elem().Interface())
			m.SetMapIndex(kv.Elem(), vv.Elem())
		}
		rv.Set(m)
		return nil

	default:
		//fmt.Printf(">>> binary - [%v]...\n", rv.Type())
		err := binary.Read(r, bo, data)
		//fmt.Printf(">>> binary - [%v]... [done]\n", rv.Type())
		return err
	}
	panic("not reachable")
}

func bwrite(w io.Writer, data interface{}) error {

	bo := binary.BigEndian
	rrv := reflect.ValueOf(data)
	rv := reflect.Indirect(rrv)
	// fmt.Printf("::: [%v] :::...\n", rrv.Type())
	// defer fmt.Printf("### [%v] [done]\n", rrv.Type())

	switch rv.Type().Kind() {
	case reflect.Struct:
		//fmt.Printf(">>> struct: [%v]...\n", rv.Type())
		for i, n := 0, rv.NumField(); i < n; i++ {
			//fmt.Printf(">>> i=%d [%v] (%v)...\n", i, rv.Field(i).Type(), rv.Type().Name())
			err := bwrite(w, rv.Field(i).Addr().Interface())
			if err != nil {
				return err
			}
			//fmt.Printf(">>> i=%d [%v] (%v)...[done]\n", i, rv.Field(i).Type(), rv.Type().Name())
		}
		//fmt.Printf(">>> struct: [%v]...[done]\n", rv.Type())
		return nil
	case reflect.String:
		str := rv.String()
		sz := uint32(len(str))
		// fmt.Printf("++++> (%d) [%s]\n", sz, string(str))
		err := bwrite(w, &sz)
		if err != nil {
			return err
		}
		bstr := []byte(str)
		bstr = append(bstr, make([]byte, align4U32(sz)-sz)...)
		_, err = w.Write(bstr)
		if err != nil {
			return err
		}
		// fmt.Printf("<++++ (%d) [%s]\n", sz, string(str))
		return nil

	case reflect.Slice:
		// fmt.Printf(">>> slice: [%v|%v]...\n", rv.Type(), rv.Type().Elem().Kind())
		sz := uint32(rv.Len())
		// fmt.Printf(">>> slice: %d [%v]\n", sz, rv.Type())
		err := bwrite(w, &sz)
		if err != nil {
			return err
		}
		for i := 0; i < int(sz); i++ {
			err = bwrite(w, rv.Index(i).Addr().Interface())
			if err != nil {
				return err
			}
		}
		// fmt.Printf(">>> slice: [%v]... [done] (%v)\n", rv.Type(), rv.Interface())
		return err

	case reflect.Map:
		//fmt.Printf(">>> map: [%v]...\n", rv.Type())
		sz := uint32(rv.Len())
		err := bwrite(w, &sz)
		if err != nil {
			return err
		}
		//fmt.Printf(">>> map: %d [%v]\n", sz, rv.Type())
		for _, kv := range rv.MapKeys() {
			vv := rv.MapIndex(kv)
			err = bwrite(w, kv.Interface())
			if err != nil {
				return err
			}
			err = bwrite(w, vv.Interface())
			if err != nil {
				return err
			}
			//fmt.Printf("m - %d: {%v} - {%v}\n", i, kv.Elem().Interface(), vv.Elem().Interface())
		}
		return nil

	default:
		//fmt.Printf(">>> binary - [%v]...\n", rv.Type())
		err := binary.Write(w, bo, data)
		//fmt.Printf(">>> binary - [%v]... [done]\n", rv.Type())
		return err
	}
	panic("not reachable")

}

// EOF
<commit_msg>sio: expose bread as Unmarshal<commit_after>package sio

import (
	"bytes"
	"encoding/binary"
	"io"
	"reflect"
)

// align4U32 returns sz adjusted to align at 4-byte boundaries
func align4U32(sz uint32) uint32 {
	return sz + (4-(sz&g_align))&g_align
}

// align4I32 returns sz adjusted to align at 4-byte boundaries
func align4I32(sz int32) int32 {
	return sz + (4-(sz&int32(g_align)))&int32(g_align)
}

// align4I64 returns sz adjusted to align at 4-byte boundaries
func align4I64(sz int64) int64 {
	return sz + (4-(sz&int64(g_align)))&int64(g_align)
}

// Unmarshal unmarshals a slice of bytes into ptr.
// If ptr implements BinaryCodec, use it.
func Unmarshal(buf *bytes.Buffer, ptr interface{}) error {
	if ptr, ok := ptr.(BinaryCodec); ok {
		return ptr.UnmarshalBinary(buf)
	}
	return bread(buf, ptr)
}

func bread(r io.Reader, data interface{}) error {
	bo := binary.BigEndian
	rv := reflect.ValueOf(data)
	// fmt.Printf("::: [%v] :::...\n", rv.Type())
	// defer fmt.Printf("### [%v] [done]\n", rv.Type())

	switch rv.Type().Kind() {
	case reflect.Ptr:
		rv = rv.Elem()
	}
	switch rv.Type().Kind() {
	case reflect.Struct:
		// fmt.Printf(">>> struct: [%v]...\n", rv.Type())
		for i, n := 0, rv.NumField(); i < n; i++ {
			// fmt.Printf(">>> i=%d [%v] (%v)...\n", i, rv.Field(i).Type(), rv.Type().Name()+"."+rv.Type().Field(i).Name)
			err := bread(r, rv.Field(i).Addr().Interface())
			if err != nil {
				return err
			}
			// fmt.Printf(">>> i=%d [%v] (%v)...[done=>%v]\n", i, rv.Field(i).Type(), rv.Type().Name(), rv.Field(i).Interface())
		}
		// fmt.Printf(">>> struct: [%v]...[done]\n", rv.Type())
		return nil
	case reflect.String:
		// fmt.Printf("++++> string...\n")
		sz := uint32(0)
		err := bread(r, &sz)
		if err != nil {
			return err
		}
		strlen := align4U32(sz)
		// fmt.Printf("string [%d=>%d]...\n", sz, strlen)
		str := make([]byte, strlen)
		_, err = r.Read(str)
		if err != nil {
			return err
		}
		rv.SetString(string(str[:sz]))
		//fmt.Printf("<++++ [%s]\n", string(str))
		return nil

	case reflect.Slice:
		//fmt.Printf("<<< slice: [%v|%v]...\n", rv.Type(), rv.Type().Elem().Kind())
		sz := uint32(0)
		err := bread(r, &sz)
		if err != nil {
			return err
		}
		//fmt.Printf("<<< slice: %d [%v]\n", sz, rv.Type())
		slice := reflect.MakeSlice(rv.Type(), int(sz), int(sz))
		for i := 0; i < int(sz); i++ {
			err = bread(r, slice.Index(i).Addr().Interface())
			if err != nil {
				return err
			}
		}
		rv.Set(slice)
		//fmt.Printf("<<< slice: [%v]... [done] (%v)\n", rv.Type(), rv.Interface())
		return err

	case reflect.Map:
		//fmt.Printf(">>> map: [%v]...\n", rv.Type())
		sz := uint32(0)
		err := bread(r, &sz)
		if err != nil {
			return err
		}
		//fmt.Printf(">>> map: %d [%v]\n", sz, rv.Type())
		m := reflect.MakeMap(rv.Type())
		kt := rv.Type().Key()
		vt := rv.Type().Elem()
		for i, n := 0, int(sz); i < n; i++ {
			kv := reflect.New(kt)
			err = bread(r, kv.Interface())
			if err != nil {
				return err
			}
			vv := reflect.New(vt)
			err = bread(r, vv.Interface())
			if err != nil {
				return err
			}
			//fmt.Printf("m - %d: {%v} - {%v}\n", i, kv.Elem().Interface(), vv.Elem().Interface())
			m.SetMapIndex(kv.Elem(), vv.Elem())
		}
		rv.Set(m)
		return nil

	default:
		//fmt.Printf(">>> binary - [%v]...\n", rv.Type())
		err := binary.Read(r, bo, data)
		//fmt.Printf(">>> binary - [%v]... [done]\n", rv.Type())
		return err
	}
	panic("not reachable")
}

func bwrite(w io.Writer, data interface{}) error {

	bo := binary.BigEndian
	rrv := reflect.ValueOf(data)
	rv := reflect.Indirect(rrv)
	// fmt.Printf("::: [%v] :::...\n", rrv.Type())
	// defer fmt.Printf("### [%v] [done]\n", rrv.Type())

	switch rv.Type().Kind() {
	case reflect.Struct:
		//fmt.Printf(">>> struct: [%v]...\n", rv.Type())
		for i, n := 0, rv.NumField(); i < n; i++ {
			//fmt.Printf(">>> i=%d [%v] (%v)...\n", i, rv.Field(i).Type(), rv.Type().Name())
			err := bwrite(w, rv.Field(i).Addr().Interface())
			if err != nil {
				return err
			}
			//fmt.Printf(">>> i=%d [%v] (%v)...[done]\n", i, rv.Field(i).Type(), rv.Type().Name())
		}
		//fmt.Printf(">>> struct: [%v]...[done]\n", rv.Type())
		return nil
	case reflect.String:
		str := rv.String()
		sz := uint32(len(str))
		// fmt.Printf("++++> (%d) [%s]\n", sz, string(str))
		err := bwrite(w, &sz)
		if err != nil {
			return err
		}
		bstr := []byte(str)
		bstr = append(bstr, make([]byte, align4U32(sz)-sz)...)
		_, err = w.Write(bstr)
		if err != nil {
			return err
		}
		// fmt.Printf("<++++ (%d) [%s]\n", sz, string(str))
		return nil

	case reflect.Slice:
		// fmt.Printf(">>> slice: [%v|%v]...\n", rv.Type(), rv.Type().Elem().Kind())
		sz := uint32(rv.Len())
		// fmt.Printf(">>> slice: %d [%v]\n", sz, rv.Type())
		err := bwrite(w, &sz)
		if err != nil {
			return err
		}
		for i := 0; i < int(sz); i++ {
			err = bwrite(w, rv.Index(i).Addr().Interface())
			if err != nil {
				return err
			}
		}
		// fmt.Printf(">>> slice: [%v]... [done] (%v)\n", rv.Type(), rv.Interface())
		return err

	case reflect.Map:
		//fmt.Printf(">>> map: [%v]...\n", rv.Type())
		sz := uint32(rv.Len())
		err := bwrite(w, &sz)
		if err != nil {
			return err
		}
		//fmt.Printf(">>> map: %d [%v]\n", sz, rv.Type())
		for _, kv := range rv.MapKeys() {
			vv := rv.MapIndex(kv)
			err = bwrite(w, kv.Interface())
			if err != nil {
				return err
			}
			err = bwrite(w, vv.Interface())
			if err != nil {
				return err
			}
			//fmt.Printf("m - %d: {%v} - {%v}\n", i, kv.Elem().Interface(), vv.Elem().Interface())
		}
		return nil

	default:
		//fmt.Printf(">>> binary - [%v]...\n", rv.Type())
		err := binary.Write(w, bo, data)
		//fmt.Printf(">>> binary - [%v]... [done]\n", rv.Type())
		return err
	}
	panic("not reachable")

}

// EOF
<|endoftext|>


<commit_before>// Copyright (C) 2014 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.

// +build ignore

package main

import (
	"bytes"
	"compress/gzip"
	"flag"
	"fmt"
	"go/format"
	"io"
	"os"
	"path/filepath"
	"strings"
	"text/template"
	"time"
)

var tpl = template.Must(template.New("assets").Parse(`package auto

const Generated int64 = {{.Generated}}

func Assets() map[string][]byte {
	var assets = make(map[string][]byte, {{.Assets | len}})
{{range $asset := .Assets}}
	assets["{{$asset.Name}}"] = {{$asset.Data}}{{end}}
	return assets
}

`))

type asset struct {
	Name string
	Data string
}

var assets []asset

func walkerFor(basePath string) filepath.WalkFunc {
	return func(name string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}

		if strings.HasPrefix(filepath.Base(name), ".") {
			// Skip dotfiles
			return nil
		}

		if info.Mode().IsRegular() {
			fd, err := os.Open(name)
			if err != nil {
				return err
			}

			var buf bytes.Buffer
			gw := gzip.NewWriter(&buf)
			io.Copy(gw, fd)
			fd.Close()
			gw.Flush()
			gw.Close()

			name, _ = filepath.Rel(basePath, name)
			assets = append(assets, asset{
				Name: filepath.ToSlash(name),
				Data: fmt.Sprintf("%#v", buf.Bytes()), // "[]byte{0x00, 0x01, ...}"
			})
		}

		return nil
	}
}

type templateVars struct {
	Assets    []asset
	Generated int64
}

func main() {
	flag.Parse()

	filepath.Walk(flag.Arg(0), walkerFor(flag.Arg(0)))
	var buf bytes.Buffer
	tpl.Execute(&buf, templateVars{
		Assets:    assets,
		Generated: time.Now().Unix(),
	})
	bs, err := format.Source(buf.Bytes())
	if err != nil {
		panic(err)
	}
	os.Stdout.Write(bs)
}
<commit_msg>script: Use source data from environment when generating assets<commit_after>// Copyright (C) 2014 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.

// +build ignore

package main

import (
	"bytes"
	"compress/gzip"
	"flag"
	"fmt"
	"go/format"
	"io"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"text/template"
	"time"
)

var tpl = template.Must(template.New("assets").Parse(`package auto

const Generated int64 = {{.Generated}}

func Assets() map[string][]byte {
	var assets = make(map[string][]byte, {{.Assets | len}})
{{range $asset := .Assets}}
	assets["{{$asset.Name}}"] = {{$asset.Data}}{{end}}
	return assets
}

`))

type asset struct {
	Name string
	Data string
}

var assets []asset

func walkerFor(basePath string) filepath.WalkFunc {
	return func(name string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}

		if strings.HasPrefix(filepath.Base(name), ".") {
			// Skip dotfiles
			return nil
		}

		if info.Mode().IsRegular() {
			fd, err := os.Open(name)
			if err != nil {
				return err
			}

			var buf bytes.Buffer
			gw := gzip.NewWriter(&buf)
			io.Copy(gw, fd)
			fd.Close()
			gw.Flush()
			gw.Close()

			name, _ = filepath.Rel(basePath, name)
			assets = append(assets, asset{
				Name: filepath.ToSlash(name),
				Data: fmt.Sprintf("%#v", buf.Bytes()), // "[]byte{0x00, 0x01, ...}"
			})
		}

		return nil
	}
}

type templateVars struct {
	Assets    []asset
	Generated int64
}

func main() {
	flag.Parse()

	filepath.Walk(flag.Arg(0), walkerFor(flag.Arg(0)))
	var buf bytes.Buffer

	// Generated time is now, except if the SOURCE_DATE_EPOCH environment
	// variable is set (for reproducible builds).
	generated := time.Now().Unix()
	if s, _ := strconv.ParseInt(os.Getenv("SOURCE_DATE_EPOCH"), 10, 64); s > 0 {
		generated = s
	}

	tpl.Execute(&buf, templateVars{
		Assets:    assets,
		Generated: generated,
	})
	bs, err := format.Source(buf.Bytes())
	if err != nil {
		panic(err)
	}
	os.Stdout.Write(bs)
}
<|endoftext|>