<commit_before>package anomalyze

import (
	"fmt"
	"github.com/drewlanenga/govector"
	"math"
)

const (
	NA = math.SmallestNonzeroFloat64
)

type AnomalyzerConf struct {
	//Sensitivity   float64
	UpperBound    float64
	LowerBound    float64
	ActiveSize    int
	ReferenceSize int
	PermCount     int
	Methods       []string
}

type Anomalyzer struct {
	Conf *AnomalyzerConf
	Data govector.Vector
}

func validateConf(conf *AnomalyzerConf) error {
	// if supplied, make sure the detection methods are supported
	supportedMethods := []string{"magnitude", "diff", "rank", "fence", "ks"}
	minimumMethods := []string{"magnitude", "diff"}
	if conf.Methods == nil {
		conf.Methods = minimumMethods
	} else {
		for _, method := range conf.Methods {
			if !exists(method, supportedMethods) {
				return fmt.Errorf("Unsupported detection method '%s'", method)
			}
		}
	}

	// reference window must be at least of size 2 (for difference methods)
	if conf.ReferenceSize < 2 {
		return fmt.Errorf("Reference window must be at least of size 2, (%d) given", conf.ReferenceSize)
	}

	// window sizes must be positive ints
	if conf.ActiveSize < 1 || conf.ReferenceSize < 1 {
		return fmt.Errorf("Active and reference window sizes must be at least of size 1")
	}

	/*
		// validation for the magnitude test
		if conf.Sensitivity < 0 || conf.Sensitivity > 1 {
			return fmt.Errorf("Sensitivity must be between 0 and 1, %v given", conf.Sensitivity)
		}
	*/

	// validation for the fence test
	if exists("fence", conf.Methods) {
		if conf.UpperBound == conf.LowerBound {
			return fmt.Errorf("Fence test included with identical bounds on the fences")
		} else {
			if conf.UpperBound < conf.LowerBound {
				return fmt.Errorf("UpperBound (%v) was lower than the LowerBound (%v)", conf.UpperBound, conf.LowerBound)
			}
		}
	}

	// validation for the rank test
	if exists("rank", conf.Methods) || exists("ks", conf.Methods) {
		if conf.PermCount == 0 {
			conf.PermCount = 500
		}
	}

	// validation for diff test
	if exists("diff", conf.Methods) {
		if conf.ReferenceSize < 4 {
			return fmt.Errorf("The Difference method requires a minimum reference window of 4 points, (%v) given", conf.ReferenceSize)
		}
	}

	return nil
}

func index(needle string, haystack []string) int {
	for i, straw := range haystack {
		if straw == needle {
			return i
		}
	}
	return -1

}

func exists(needle string, haystack []string) bool {
	return index(needle, haystack) > -1
}

func NewAnomalyzer(conf *AnomalyzerConf, data []float64) (Anomalyzer, error) {
	err := validateConf(conf)
	if err != nil {
		return Anomalyzer{}, err
	}

	vector, err := govector.AsVector(data)
	if err != nil {
		return Anomalyzer{}, err
	}

	return Anomalyzer{conf, vector}, nil
}

func (a *Anomalyzer) Update(x []float64) {
	for _, val := range x {
		a.Data.Push(val)
	}
}

func (a Anomalyzer) Push(x float64) float64 {
	// add the new point to the data
	a.Data.Push(x)

	// evaluate the anomalous probability
	return a.Eval()
}

// Return the weighted average of four statistical tests
// for anomaly detection and return the probability that
// a behavior is anomalous.
func (a Anomalyzer) Eval() float64 {

	probs := make(govector.Vector, len(a.Conf.Methods))

	for i, method := range a.Conf.Methods {
		algorithm := Algorithms[method]
		probs[i] = cap(algorithm(a.Data, *a.Conf), 0, 1)
	}
	// ignore the error since the length of probs and
	// the weights will always be equal

	weights := a.getWeights(probs)
	weighted, _ := probs.WeightedMean(weights)

	// if all the weights are zero, then our weighted mean
	// function attempts to divide by zero which returns a
	// NaN. we'd like it to return 0.
	if math.IsNaN(weighted) {
		weighted = 0
	}
	return weighted
}

// Use essentially similar weights.  However, if either the magnitude
// or fence methods have high probabilities, upweight them significantly.
func (a Anomalyzer) getWeights(probs govector.Vector) govector.Vector {
	nmethods := len(a.Conf.Methods)
	weights := make(govector.Vector, nmethods)
	i := 0
	for i < nmethods {
		weights[i] = 0.5
		i++
	}

	// If either the magnitude and fence methods don't have any
	// probability to contribute, we don't want to hear about it.
	// If they do, we upweight them substantially.

	dynamicWeights := []string{"magnitude", "fence"}
	for _, dynamicWeight := range dynamicWeights {
		methodIndex := index(dynamicWeight, a.Conf.Methods)
		if methodIndex > -1 {
			if probs[methodIndex] > 0.8 {
				weights[methodIndex] = 1.0
			} else {
				weights[methodIndex] = 0.0
			}
		}
	}

	return weights
}
<commit_msg>Defaulted reference size to be a multiple of active size<commit_after>package anomalyze

import (
	"fmt"
	"github.com/drewlanenga/govector"
	"math"
)

const (
	NA = math.SmallestNonzeroFloat64
)

type AnomalyzerConf struct {
	//Sensitivity   float64
	UpperBound    float64
	LowerBound    float64
	ActiveSize    int
	ReferenceSize int
	PermCount     int
	Methods       []string
}

type Anomalyzer struct {
	Conf *AnomalyzerConf
	Data govector.Vector
}

func validateConf(conf *AnomalyzerConf) error {
	// if supplied, make sure the detection methods are supported
	supportedMethods := []string{"magnitude", "diff", "rank", "fence", "ks"}
	minimumMethods := []string{"magnitude", "diff"}
	if conf.Methods == nil {
		conf.Methods = minimumMethods
	} else {
		for _, method := range conf.Methods {
			if !exists(method, supportedMethods) {
				return fmt.Errorf("Unsupported detection method '%s'", method)
			}
		}
	}

	// if reference window is not specified, make it some multiple of the active window size
	if conf.ReferenceSize == 0 {
		conf.ReferenceSize = 3 * conf.ActiveSize
	}

	// reference window must be at least of size 2 (for difference methods)
	if conf.ReferenceSize < 2 {
		return fmt.Errorf("Reference window must be at least of size 2, (%d) given", conf.ReferenceSize)
	}

	// window sizes must be positive ints
	if conf.ActiveSize < 1 || conf.ReferenceSize < 1 {
		return fmt.Errorf("Active and reference window sizes must be at least of size 1")
	}

	/*
		// validation for the magnitude test
		if conf.Sensitivity < 0 || conf.Sensitivity > 1 {
			return fmt.Errorf("Sensitivity must be between 0 and 1, %v given", conf.Sensitivity)
		}
	*/

	// validation for the fence test
	if exists("fence", conf.Methods) {
		if conf.UpperBound == conf.LowerBound {
			return fmt.Errorf("Fence test included with identical bounds on the fences")
		} else {
			if conf.UpperBound < conf.LowerBound {
				return fmt.Errorf("UpperBound (%v) was lower than the LowerBound (%v)", conf.UpperBound, conf.LowerBound)
			}
		}
	}

	// validation for the rank test
	if exists("rank", conf.Methods) || exists("ks", conf.Methods) {
		if conf.PermCount == 0 {
			conf.PermCount = 500
		}
	}

	// validation for diff test
	if exists("diff", conf.Methods) {
		if conf.ReferenceSize < 4 {
			return fmt.Errorf("The Difference method requires a minimum reference window of 4 points, (%v) given", conf.ReferenceSize)
		}
	}

	return nil
}

func index(needle string, haystack []string) int {
	for i, straw := range haystack {
		if straw == needle {
			return i
		}
	}
	return -1

}

func exists(needle string, haystack []string) bool {
	return index(needle, haystack) > -1
}

func NewAnomalyzer(conf *AnomalyzerConf, data []float64) (Anomalyzer, error) {
	err := validateConf(conf)
	if err != nil {
		return Anomalyzer{}, err
	}

	vector, err := govector.AsVector(data)
	if err != nil {
		return Anomalyzer{}, err
	}

	return Anomalyzer{conf, vector}, nil
}

func (a *Anomalyzer) Update(x []float64) {
	for _, val := range x {
		a.Data.Push(val)
	}
}

func (a Anomalyzer) Push(x float64) float64 {
	// add the new point to the data
	a.Data.Push(x)

	// evaluate the anomalous probability
	return a.Eval()
}

// Return the weighted average of four statistical tests
// for anomaly detection and return the probability that
// a behavior is anomalous.
func (a Anomalyzer) Eval() float64 {

	probs := make(govector.Vector, len(a.Conf.Methods))

	for i, method := range a.Conf.Methods {
		algorithm := Algorithms[method]
		probs[i] = cap(algorithm(a.Data, *a.Conf), 0, 1)
	}
	// ignore the error since the length of probs and
	// the weights will always be equal

	weights := a.getWeights(probs)
	weighted, _ := probs.WeightedMean(weights)

	// if all the weights are zero, then our weighted mean
	// function attempts to divide by zero which returns a
	// NaN. we'd like it to return 0.
	if math.IsNaN(weighted) {
		weighted = 0
	}
	return weighted
}

// Use essentially similar weights.  However, if either the magnitude
// or fence methods have high probabilities, upweight them significantly.
func (a Anomalyzer) getWeights(probs govector.Vector) govector.Vector {
	nmethods := len(a.Conf.Methods)
	weights := make(govector.Vector, nmethods)
	i := 0
	for i < nmethods {
		weights[i] = 0.5
		i++
	}

	// If either the magnitude and fence methods don't have any
	// probability to contribute, we don't want to hear about it.
	// If they do, we upweight them substantially.

	dynamicWeights := []string{"magnitude", "fence"}
	for _, dynamicWeight := range dynamicWeights {
		methodIndex := index(dynamicWeight, a.Conf.Methods)
		if methodIndex > -1 {
			if probs[methodIndex] > 0.8 {
				weights[methodIndex] = 1.0
			} else {
				weights[methodIndex] = 0.0
			}
		}
	}

	return weights
}
<|endoftext|>


<commit_before>package bigip

import (
	"fmt"
	"github.com/scottdware/go-bigip"
)

func main() {
	// Connect to the BIG-IP system.
	f5 := bigip.NewSession("ltm.company.com", "admin", "secret")

	// Get a list of all VLAN's, and print their names to the console.
	vlans, err := b.Vlans()
	if err != nil {
		fmt.Println(err)
	}

	for _, vlan := range vlans.Vlans {
		fmt.Println(vlan.Name)
	}

	// Create a VLAN
	f5.CreateVlan("vlan1138", 1138)
	f5.CreateVlan("vlan421", 421)

	// Add an untagged interface to a VLAN.
	f5.AddInterfaceToVlan("vlan1138", "1.2", false)

	// Delete a VLAN.
	f5.DeleteVlan("vlan1138")

	// Create a couple of nodes.
	f5.CreateNode("web-server-1", "192.168.1.50")
	f5.CreateNode("web-server-2", "192.168.1.51")
	f5.CreateNode("ssl-web-server-1", "10.2.2.50")
	f5.CreateNode("ssl-web-server-2", "10.2.2.51")

	// Create a pool, and add members to it. When adding a member, you must
	// specify the port in the format of <node name>:<port>.
	f5.CreatePool("web_farm_80_pool")
	f5.AddPoolMember("web_farm_80_pool", "web-server-1:80")
	f5.AddPoolMember("web_farm_80_pool", "web-server-2:80")
	f5.CreatePool("ssl_443_pool")
	f5.AddPoolMember("ssl_443_pool", "ssl-web-server-1:443")
	f5.AddPoolMember("ssl_443_pool", "ssl-web-server-2:443")

	// Create a virtual server, with the above pool. The third field is the subnet
	// mask, and that can either be in CIDR notation or decimal. For any/all destinations
	// and ports, use '0' for the mask and/or port.
	f5.CreateVirtualServer("web_farm_VS", "0.0.0.0", "0.0.0.0", "web_farm_80_pool", 80)
	f5.CreateVirtualServer("ssl_web_farm_VS", "10.1.1.0", "24", "ssl_443_pool", 443)

	// Remove a pool member.
	f5.DeletePoolMember("web_farm_80_pool", "web-server-2:80")

	// Create a trunk, with LACP enabled.
	f5.CreateTrunk("Aggregated", "1.2, 1.4, 1.6", true)

	// Disable a virtual address.
	f5.VirtualAddressStatus("web_farm_VS", "disable")

	// Disable a pool member.
	f5.PoolMemberStatus("ssl_443_pool", "ssl-web-server-1:443", "disable")

	// Create a self IP.
	f5.CreateSelfIP("vlan1138", "10.10.10.1/24", "vlan1138")
	f5.CreateSelfIP("vlan421", "10.10.20.1/25", "vlan421")

	// Add a static route.
	f5.CreateRoute("servers", "10.1.1.0/24", "10.50.1.5")

	// Create a route domain.
	f5.CreateRouteDomain("vlans", 10, true, "vlan1138, vlan421")
}
<commit_msg>Fixed variable error<commit_after>package bigip

import (
	"fmt"
	"github.com/scottdware/go-bigip"
)

func main() {
	// Connect to the BIG-IP system.
	f5 := bigip.NewSession("ltm.company.com", "admin", "secret")

	// Get a list of all VLAN's, and print their names to the console.
	vlans, err := f5.Vlans()
	if err != nil {
		fmt.Println(err)
	}

	for _, vlan := range vlans.Vlans {
		fmt.Println(vlan.Name)
	}

	// Create a VLAN
	f5.CreateVlan("vlan1138", 1138)
	f5.CreateVlan("vlan421", 421)

	// Add an untagged interface to a VLAN.
	f5.AddInterfaceToVlan("vlan1138", "1.2", false)

	// Delete a VLAN.
	f5.DeleteVlan("vlan1138")

	// Create a couple of nodes.
	f5.CreateNode("web-server-1", "192.168.1.50")
	f5.CreateNode("web-server-2", "192.168.1.51")
	f5.CreateNode("ssl-web-server-1", "10.2.2.50")
	f5.CreateNode("ssl-web-server-2", "10.2.2.51")

	// Create a pool, and add members to it. When adding a member, you must
	// specify the port in the format of <node name>:<port>.
	f5.CreatePool("web_farm_80_pool")
	f5.AddPoolMember("web_farm_80_pool", "web-server-1:80")
	f5.AddPoolMember("web_farm_80_pool", "web-server-2:80")
	f5.CreatePool("ssl_443_pool")
	f5.AddPoolMember("ssl_443_pool", "ssl-web-server-1:443")
	f5.AddPoolMember("ssl_443_pool", "ssl-web-server-2:443")

	// Create a virtual server, with the above pool. The third field is the subnet
	// mask, and that can either be in CIDR notation or decimal. For any/all destinations
	// and ports, use '0' for the mask and/or port.
	f5.CreateVirtualServer("web_farm_VS", "0.0.0.0", "0.0.0.0", "web_farm_80_pool", 80)
	f5.CreateVirtualServer("ssl_web_farm_VS", "10.1.1.0", "24", "ssl_443_pool", 443)

	// Remove a pool member.
	f5.DeletePoolMember("web_farm_80_pool", "web-server-2:80")

	// Create a trunk, with LACP enabled.
	f5.CreateTrunk("Aggregated", "1.2, 1.4, 1.6", true)

	// Disable a virtual address.
	f5.VirtualAddressStatus("web_farm_VS", "disable")

	// Disable a pool member.
	f5.PoolMemberStatus("ssl_443_pool", "ssl-web-server-1:443", "disable")

	// Create a self IP.
	f5.CreateSelfIP("vlan1138", "10.10.10.1/24", "vlan1138")
	f5.CreateSelfIP("vlan421", "10.10.20.1/25", "vlan421")

	// Add a static route.
	f5.CreateRoute("servers", "10.1.1.0/24", "10.50.1.5")

	// Create a route domain.
	f5.CreateRouteDomain("vlans", 10, true, "vlan1138, vlan421")
}
<|endoftext|>


<commit_before><commit_msg>check os level err<commit_after><|endoftext|>


<commit_before><commit_msg>Add log level output functions<commit_after><|endoftext|>


<commit_before><commit_msg>added Levels()<commit_after><|endoftext|>


<commit_before>// Package account stores and tracks accounts within a Chain Core.
package account

import (
	"context"
//	stdsql "database/sql"
//	"encoding/json"
	"sync"
	"time"
    "fmt"

	"github.com/golang/groupcache/lru"
	//"github.com/lib/pq"

//	"chain/core/pin"
	"github.com/bytom/blockchain/signers"
	"github.com/bytom/blockchain/txbuilder"
	"github.com/bytom/crypto/ed25519/chainkd"
//	"chain/database/pg"
     dbm "github.com/tendermint/tmlibs/db"
	"github.com/bytom/errors"
	"github.com/bytom/log"
	"github.com/bytom/protocol"
	"github.com/bytom/protocol/vm/vmutil"
)

const maxAccountCache = 1000

var (
	ErrDuplicateAlias = errors.New("duplicate account alias")
	ErrBadIdentifier  = errors.New("either ID or alias must be specified, and not both")
)

func NewManager(db dbm.DB, chain *protocol.Chain/*, pinStore *pin.Store*/) *Manager {
	return &Manager{
		db:          db,
		chain:       chain,
		utxoDB:      newReserver(db, chain/*, pinStore*/),
//		pinStore:    pinStore,
		cache:       lru.New(maxAccountCache),
		aliasCache:  lru.New(maxAccountCache),
		delayedACPs: make(map[*txbuilder.TemplateBuilder][]*controlProgram),
	}
}

// Manager stores accounts and their associated control programs.
type Manager struct {
	db       dbm.DB
	chain    *protocol.Chain
	utxoDB   *reserver
	indexer  Saver
//	pinStore *pin.Store

	cacheMu    sync.Mutex
	cache      *lru.Cache
	aliasCache *lru.Cache

	delayedACPsMu sync.Mutex
	delayedACPs   map[*txbuilder.TemplateBuilder][]*controlProgram

	acpMu        sync.Mutex
	acpIndexNext uint64 // next acp index in our block
	acpIndexCap  uint64 // points to end of block
}

func (m *Manager) IndexAccounts(indexer Saver) {
	m.indexer = indexer
}

// ExpireReservations removes reservations that have expired periodically.
// It blocks until the context is canceled.
func (m *Manager) ExpireReservations(ctx context.Context, period time.Duration) {
	ticks := time.Tick(period)
	for {
		select {
		case <-ctx.Done():
			log.Printf(ctx, "Deposed, ExpireReservations exiting")
			return
		case <-ticks:
			err := m.utxoDB.ExpireReservations(ctx)
			if err != nil {
				log.Error(ctx, err)
			}
		}
	}
}

type Account struct {
	*signers.Signer
	Alias string
	Tags  map[string]interface{}
}

// Create creates a new Account.
func (m *Manager) Create(ctx context.Context, xpubs []chainkd.XPub, quorum int, alias string, tags map[string]interface{}, clientToken string) (*Account, error) {
	signer, err := signers.Create(ctx, m.db, "account", xpubs, quorum, clientToken)
	if err != nil {
		return nil, errors.Wrap(err)
	}

	/*tagsParam, err := tagsToNullString(tags)
	if err != nil {
		return nil, err
	}
    */
    var tagsParam []byte

/*	aliasSQL := stdsql.NullString{
		String: alias,
		Valid:  alias != "",
	}

	const q = `
		INSERT INTO accounts (account_id, alias, tags) VALUES ($1, $2, $3)
		ON CONFLICT (account_id) DO UPDATE SET alias = $2, tags = $3
	`
	_, err = m.db.ExecContext(ctx, q, signer.ID, aliasSQL, tagsParam)
	if pg.IsUniqueViolation(err) {
		return nil, errors.WithDetail(ErrDuplicateAlias, "an account with the provided alias already exists")
	} else if err != nil {
		return nil, errors.Wrap(err)
	}*/
    account_alias := []byte(fmt.Sprintf("account_alias:%v", signer.ID))
    account_tags := []byte(fmt.Sprintf("account_tags:%v", signer.ID))
    m.db.Set(account_alias, []byte(alias))
    m.db.Set(account_tags, []byte(tagsParam))
    alias_account := []byte(fmt.Sprintf("alias_account:%v", alias))
    m.db.Set(alias_account, []byte(signer.ID))

	account := &Account{
		Signer: signer,
		Alias:  alias,
		Tags:   tags,
	}

	err = m.indexAnnotatedAccount(ctx, account)
	if err != nil {
		return nil, errors.Wrap(err, "indexing annotated account")
	}

	return account, nil
}

// UpdateTags modifies the tags of the specified account. The account may be
// identified either by ID or Alias, but not both.
func (m *Manager) UpdateTags(ctx context.Context, id, alias *string, tags map[string]interface{}) error {
	if (id == nil) == (alias == nil) {
		return errors.Wrap(ErrBadIdentifier)
	}

/*	tagsParam, err := tagsToNullString(tags)
	if err != nil {
		return errors.Wrap(err, "convert tags")
	}
*/
	var (
		signer   *signers.Signer
		aliasStr string
	)

/*	if id != nil {
		signer, err = m.findByID(ctx, *id)
		if err != nil {
			return errors.Wrap(err, "get account by ID")
		}

		// An alias is required by indexAnnotatedAccount. The latter is a somewhat
		// complex function, so in the interest of not making a near-duplicate,
		// we'll satisfy its contract and provide an alias.
		const q = `SELECT alias FROM accounts WHERE account_id = $1`
		var a stdsql.NullString
		err := m.db.QueryRowContext(ctx, q, *id).Scan(&a)
		if err != nil {
			return errors.Wrap(err, "alias lookup")
		}
		if a.Valid {
			aliasStr = a.String
		}
	} else { // alias is guaranteed to be not nil due to bad identifier check
		aliasStr = *alias
		signer, err = m.FindByAlias(ctx, aliasStr)
		if err != nil {
			return errors.Wrap(err, "get account by alias")
		}
	}

	const q = `
		UPDATE accounts
		SET tags = $1
		WHERE account_id = $2
	`
	_, err = m.db.ExecContext(ctx, q, tagsParam, signer.ID)
	if err != nil {
		return errors.Wrap(err, "update entry in accounts table")
	}
*/
	return errors.Wrap(m.indexAnnotatedAccount(ctx, &Account{
		Signer: signer,
		Alias:  aliasStr,
		Tags:   tags,
	}), "update account index")
}

// FindByAlias retrieves an account's Signer record by its alias
func (m *Manager) FindByAlias(ctx context.Context, alias string) (*signers.Signer, error) {
	var accountID string

	m.cacheMu.Lock()
	cachedID, ok := m.aliasCache.Get(alias)
	m.cacheMu.Unlock()
	if ok {
		accountID = cachedID.(string)
	} else {
		/*const q = `SELECT account_id FROM accounts WHERE alias=$1`
		err := m.db.QueryRowContext(ctx, q, alias).Scan(&accountID)
		if err == stdsql.ErrNoRows {
			return nil, errors.WithDetailf(pg.ErrUserInputNotFound, "alias: %s", alias)
		}
		if err != nil {
			return nil, errors.Wrap(err)
		}*/
        bytez := m.db.Get([]byte(fmt.Sprintf("alias_account:%v", alias)))
        accountID = string(bytez[:])
		m.cacheMu.Lock()
		m.aliasCache.Add(alias, accountID)
		m.cacheMu.Unlock()
	}
	return m.findByID(ctx, accountID)
}

// findByID returns an account's Signer record by its ID.
func (m *Manager) findByID(ctx context.Context, id string) (*signers.Signer, error) {
	m.cacheMu.Lock()
	cached, ok := m.cache.Get(id)
	m.cacheMu.Unlock()
	if ok {
		return cached.(*signers.Signer), nil
	}
	account, err := signers.Find(ctx, m.db, "account", id)
	if err != nil {
		return nil, err
	}
	m.cacheMu.Lock()
	m.cache.Add(id, account)
	m.cacheMu.Unlock()
	return account, nil
}

type controlProgram struct {
	accountID      string
	keyIndex       uint64
	controlProgram []byte
	change         bool
	expiresAt      time.Time
}

func (m *Manager) createControlProgram(ctx context.Context, accountID string, change bool, expiresAt time.Time) (*controlProgram, error) {
	account, err := m.findByID(ctx, accountID)
	if err != nil {
		return nil, err
	}

	idx, err := m.nextIndex(ctx)
	if err != nil {
		return nil, err
	}

	path := signers.Path(account, signers.AccountKeySpace, idx)
	derivedXPubs := chainkd.DeriveXPubs(account.XPubs, path)
	derivedPKs := chainkd.XPubKeys(derivedXPubs)
	control, err := vmutil.P2SPMultiSigProgram(derivedPKs, account.Quorum)
	if err != nil {
		return nil, err
	}
	return &controlProgram{
		accountID:      account.ID,
		keyIndex:       idx,
		controlProgram: control,
		change:         change,
		expiresAt:      expiresAt,
	}, nil
}

// CreateControlProgram creates a control program
// that is tied to the Account and stores it in the database.
func (m *Manager) CreateControlProgram(ctx context.Context, accountID string, change bool, expiresAt time.Time) ([]byte, error) {
	cp, err := m.createControlProgram(ctx, accountID, change, expiresAt)
	if err != nil {
		return nil, err
	}
	/*err = m.insertAccountControlProgram(ctx, cp)
	if err != nil {
		return nil, err
	}*/
	return cp.controlProgram, nil
}

/*
func (m *Manager) insertAccountControlProgram(ctx context.Context, progs ...*controlProgram) error {
	const q = `
		INSERT INTO account_control_programs (signer_id, key_index, control_program, change, expires_at)
		SELECT unnest($1::text[]), unnest($2::bigint[]), unnest($3::bytea[]), unnest($4::boolean[]),
			unnest($5::timestamp with time zone[])
	`
	var (
		accountIDs   pq.StringArray
		keyIndexes   pq.Int64Array
		controlProgs pq.ByteaArray
		change       pq.BoolArray
		expirations  []stdsql.NullString
	)
	for _, p := range progs {
		accountIDs = append(accountIDs, p.accountID)
		keyIndexes = append(keyIndexes, int64(p.keyIndex))
		controlProgs = append(controlProgs, p.controlProgram)
		change = append(change, p.change)
		expirations = append(expirations, stdsql.NullString{
			String: p.expiresAt.Format(time.RFC3339),
			Valid:  !p.expiresAt.IsZero(),
		})
	}

	_, err := m.db.ExecContext(ctx, q, accountIDs, keyIndexes, controlProgs, change, pq.Array(expirations))
	return errors.Wrap(err)
}
*/

func (m *Manager) nextIndex(ctx context.Context) (uint64, error) {
	m.acpMu.Lock()
	defer m.acpMu.Unlock()

	if m.acpIndexNext >= m.acpIndexCap {
		/*var cap uint64
		const incrby = 10000 // account_control_program_seq increments by 10,000
		const q = `SELECT nextval('account_control_program_seq')`
		err := m.db.QueryRowContext(ctx, q).Scan(&cap)
		if err != nil {
			return 0, errors.Wrap(err, "scan")
		}
		m.acpIndexCap = cap
		m.acpIndexNext = cap - incrby*/
	}

	n := m.acpIndexNext
	m.acpIndexNext++
	return n, nil
}

/*
func tagsToNullString(tags map[string]interface{}) (*stdsql.NullString, error) {
	var tagsJSON []byte
	if len(tags) != 0 {
		var err error
		tagsJSON, err = json.Marshal(tags)
		if err != nil {
			return nil, errors.Wrap(err)
		}
	}
	return &stdsql.NullString{String: string(tagsJSON), Valid: len(tagsJSON) > 0}, nil
}
*/
<commit_msg>add insertAccountControlProgram<commit_after>// Package account stores and tracks accounts within a Chain Core.
package account

import (
	"context"
//	stdsql "database/sql"
//	"encoding/json"
	"sync"
	"time"
    "fmt"

	"github.com/golang/groupcache/lru"
	"github.com/lib/pq"

//	"chain/core/pin"
	"github.com/bytom/blockchain/signers"
	"github.com/bytom/blockchain/txbuilder"
	"github.com/bytom/crypto/ed25519/chainkd"
//	"chain/database/pg"
     dbm "github.com/tendermint/tmlibs/db"
	"github.com/bytom/errors"
	"github.com/bytom/log"
	"github.com/bytom/protocol"
	"github.com/bytom/protocol/vm/vmutil"
)

const maxAccountCache = 1000

var (
	ErrDuplicateAlias = errors.New("duplicate account alias")
	ErrBadIdentifier  = errors.New("either ID or alias must be specified, and not both")
)

func NewManager(db dbm.DB, chain *protocol.Chain/*, pinStore *pin.Store*/) *Manager {
	return &Manager{
		db:          db,
		chain:       chain,
		utxoDB:      newReserver(db, chain/*, pinStore*/),
//		pinStore:    pinStore,
		cache:       lru.New(maxAccountCache),
		aliasCache:  lru.New(maxAccountCache),
		delayedACPs: make(map[*txbuilder.TemplateBuilder][]*controlProgram),
	}
}

// Manager stores accounts and their associated control programs.
type Manager struct {
	db       dbm.DB
	chain    *protocol.Chain
	utxoDB   *reserver
	indexer  Saver
//	pinStore *pin.Store

	cacheMu    sync.Mutex
	cache      *lru.Cache
	aliasCache *lru.Cache

	delayedACPsMu sync.Mutex
	delayedACPs   map[*txbuilder.TemplateBuilder][]*controlProgram

	acpMu        sync.Mutex
	acpIndexNext uint64 // next acp index in our block
	acpIndexCap  uint64 // points to end of block
}

func (m *Manager) IndexAccounts(indexer Saver) {
	m.indexer = indexer
}

// ExpireReservations removes reservations that have expired periodically.
// It blocks until the context is canceled.
func (m *Manager) ExpireReservations(ctx context.Context, period time.Duration) {
	ticks := time.Tick(period)
	for {
		select {
		case <-ctx.Done():
			log.Printf(ctx, "Deposed, ExpireReservations exiting")
			return
		case <-ticks:
			err := m.utxoDB.ExpireReservations(ctx)
			if err != nil {
				log.Error(ctx, err)
			}
		}
	}
}

type Account struct {
	*signers.Signer
	Alias string
	Tags  map[string]interface{}
}

// Create creates a new Account.
func (m *Manager) Create(ctx context.Context, xpubs []chainkd.XPub, quorum int, alias string, tags map[string]interface{}, clientToken string) (*Account, error) {
	signer, err := signers.Create(ctx, m.db, "account", xpubs, quorum, clientToken)
	if err != nil {
		return nil, errors.Wrap(err)
	}
/*
	tagsParam , err := tagsToNullString(tags)
	if err != nil {
		return nil, err
	}
*/
    var tagsPar []byte

/*	aliasSQL := stdsql.NullString{
		String: alias,
		Valid:  alias != "",
	}

	const q = `
		INSERT INTO accounts (account_id, alias, tags) VALUES ($1, $2, $3)
		ON CONFLICT (account_id) DO UPDATE SET alias = $2, tags = $3
	`
	_, err = m.db.ExecContext(ctx, q, signer.ID, aliasSQL, tagsParam)
	if pg.IsUniqueViolation(err) {
		return nil, errors.WithDetail(ErrDuplicateAlias, "an account with the provided alias already exists")
	} else if err != nil {
		return nil, errors.Wrap(err)
	}*/
    account_alias := []byte(fmt.Sprintf("account_alias:%v", signer.ID))
    account_tags := []byte(fmt.Sprintf("account_tags:%v", signer.ID))
    m.db.Set(account_alias, []byte(alias))
    m.db.Set(account_tags, []byte(tagsPar))
    alias_account := []byte(fmt.Sprintf("alias_account:%v", alias))
    m.db.Set(alias_account, []byte(signer.ID))

	account := &Account{
		Signer: signer,
		Alias:  alias,
		Tags:   tags,
	}

	err = m.indexAnnotatedAccount(ctx, account)
	if err != nil {
		return nil, errors.Wrap(err, "indexing annotated account")
	}

	return account, nil
}

// UpdateTags modifies the tags of the specified account. The account may be
// identified either by ID or Alias, but not both.
func (m *Manager) UpdateTags(ctx context.Context, id, alias *string, tags map[string]interface{}) error {
	if (id == nil) == (alias == nil) {
		return errors.Wrap(ErrBadIdentifier)
	}
/*
	tagsParam , err := tagsToNullString(tags)
	if err != nil {
		return errors.Wrap(err, "convert tags")
	}
*/
	var (
		signer   *signers.Signer
		aliasStr string
	)

/*	if id != nil {
		signer, err = m.findByID(ctx, *id)
		if err != nil {
			return errors.Wrap(err, "get account by ID")
		}

		// An alias is required by indexAnnotatedAccount. The latter is a somewhat
		// complex function, so in the interest of not making a near-duplicate,
		// we'll satisfy its contract and provide an alias.
		const q = `SELECT alias FROM accounts WHERE account_id = $1`
		var a stdsql.NullString
		err := m.db.QueryRowContext(ctx, q, *id).Scan(&a)
		if err != nil {
			return errors.Wrap(err, "alias lookup")
		}
		if a.Valid {
			aliasStr = a.String
		}
	} else { // alias is guaranteed to be not nil due to bad identifier check
		aliasStr = *alias
		signer, err = m.FindByAlias(ctx, aliasStr)
		if err != nil {
			return errors.Wrap(err, "get account by alias")
		}
	}

	const q = `
		UPDATE accounts
		SET tags = $1
		WHERE account_id = $2
	`
	_, err = m.db.ExecContext(ctx, q, tagsParam, signer.ID)
	if err != nil {
		return errors.Wrap(err, "update entry in accounts table")
	}
*/
	return errors.Wrap(m.indexAnnotatedAccount(ctx, &Account{
		Signer: signer,
		Alias:  aliasStr,
		Tags:   tags,
	}), "update account index")
}

// FindByAlias retrieves an account's Signer record by its alias
func (m *Manager) FindByAlias(ctx context.Context, alias string) (*signers.Signer, error) {
	var accountID string

	m.cacheMu.Lock()
	cachedID, ok := m.aliasCache.Get(alias)
	m.cacheMu.Unlock()
	if ok {
		accountID = cachedID.(string)
	} else {
		/*const q = `SELECT account_id FROM accounts WHERE alias=$1`
		err := m.db.QueryRowContext(ctx, q, alias).Scan(&accountID)
		if err == stdsql.ErrNoRows {
			return nil, errors.WithDetailf(pg.ErrUserInputNotFound, "alias: %s", alias)
		}
		if err != nil {
			return nil, errors.Wrap(err)
		}*/
        bytez := m.db.Get([]byte(fmt.Sprintf("alias_account:%v", alias)))
        accountID = string(bytez[:])
		m.cacheMu.Lock()
		m.aliasCache.Add(alias, accountID)
		m.cacheMu.Unlock()
	}
	return m.findByID(ctx, accountID)
}

// findByID returns an account's Signer record by its ID.
func (m *Manager) findByID(ctx context.Context, id string) (*signers.Signer, error) {
	m.cacheMu.Lock()
	cached, ok := m.cache.Get(id)
	m.cacheMu.Unlock()
	if ok {
		return cached.(*signers.Signer), nil
	}
	account, err := signers.Find(ctx, m.db, "account", id)
	if err != nil {
		return nil, err
	}
	m.cacheMu.Lock()
	m.cache.Add(id, account)
	m.cacheMu.Unlock()
	return account, nil
}

type controlProgram struct {
	accountID      string
	keyIndex       uint64
	controlProgram []byte
	change         bool
	expiresAt      time.Time
}

func (m *Manager) createControlProgram(ctx context.Context, accountID string, change bool, expiresAt time.Time) (*controlProgram, error) {
	account, err := m.findByID(ctx, accountID)
	if err != nil {
		return nil, err
	}

	idx, err := m.nextIndex(ctx)
	if err != nil {
		return nil, err
	}

	path := signers.Path(account, signers.AccountKeySpace, idx)
	derivedXPubs := chainkd.DeriveXPubs(account.XPubs, path)
	derivedPKs := chainkd.XPubKeys(derivedXPubs)
	control, err := vmutil.P2SPMultiSigProgram(derivedPKs, account.Quorum)
	if err != nil {
		return nil, err
	}
	return &controlProgram{
		accountID:      account.ID,
		keyIndex:       idx,
		controlProgram: control,
		change:         change,
		expiresAt:      expiresAt,
	}, nil
}

// CreateControlProgram creates a control program
// that is tied to the Account and stores it in the database.
func (m *Manager) CreateControlProgram(ctx context.Context, accountID string, change bool, expiresAt time.Time) ([]byte, error) {
	cp, err := m.createControlProgram(ctx, accountID, change, expiresAt)
	if err != nil {
		return nil, err
	}
	err = m.insertAccountControlProgram(ctx, cp)
	if err != nil {
		return nil, err
	}
	return cp.controlProgram, nil
}

func (m *Manager) insertAccountControlProgram(ctx context.Context, progs ...*controlProgram) error {
	const q = `
		INSERT INTO account_control_programs (signer_id, key_index, control_program, change, expires_at)
		SELECT unnest($1::text[]), unnest($2::bigint[]), unnest($3::bytea[]), unnest($4::boolean[]),
			unnest($5::timestamp with time zone[])
	`
	var (
		accountIDs   pq.StringArray
		keyIndexes   pq.Int64Array
		controlProgs pq.ByteaArray
		change       pq.BoolArray
	//	expirations  []stdsql.NullString
	)
	for _, p := range progs {
		accountIDs = append(accountIDs, p.accountID)
		keyIndexes = append(keyIndexes, int64(p.keyIndex))
		controlProgs = append(controlProgs, p.controlProgram)
		change = append(change, p.change)
	/*	expirations = append(expirations, stdsql.NullString{
			String: p.expiresAt.Format(time.RFC3339),
			Valid:  !p.expiresAt.IsZero(),
		})*/
	}

//	_, err := m.dbm.ExecContext(ctx, q, accountIDs, keyIndexes, controlProgs, change, pq.Array(expirations))
	return errors.Wrap(nil)
}

func (m *Manager) nextIndex(ctx context.Context) (uint64, error) {
	m.acpMu.Lock()
	defer m.acpMu.Unlock()

	if m.acpIndexNext >= m.acpIndexCap {
		/*var cap uint64
		const incrby = 10000 // account_control_program_seq increments by 10,000
		const q = `SELECT nextval('account_control_program_seq')`
		err := m.db.QueryRowContext(ctx, q).Scan(&cap)
		if err != nil {
			return 0, errors.Wrap(err, "scan")
		}
		m.acpIndexCap = cap
		m.acpIndexNext = cap - incrby*/
	}

	n := m.acpIndexNext
	m.acpIndexNext++
	return n, nil
}

/*
func tagsToNullString(tags map[string]interface{}) (*stdsql.NullString, error) {
	var tagsJSON []byte
	if len(tags) != 0 {
		var err error
		tagsJSON, err = json.Marshal(tags)
		if err != nil {
			return nil, errors.Wrap(err)
		}
	}
	return &stdsql.NullString{String: string(tagsJSON), Valid: len(tagsJSON) > 0}, nil
}
*/
<|endoftext|>


<commit_before><commit_msg>Fill in ulIvBits when generating CK_GCM_PARAMS. (#83)<commit_after><|endoftext|>


<commit_before><commit_msg>Implement Parser.Escape()<commit_after><|endoftext|>


<commit_before>// Copyright 2013-2022 The btcsuite developers

package musig2

import (
	"bytes"
	"crypto/rand"
	"encoding/binary"
	"io"

	"github.com/btcsuite/btcd/btcec/v2"
	"github.com/btcsuite/btcd/btcec/v2/schnorr"
	"github.com/btcsuite/btcd/chaincfg/chainhash"
)

const (
	// PubNonceSize is the size of the public nonces. Each public nonce is
	// serialized the full compressed encoding, which uses 32 bytes for each
	// nonce.
	PubNonceSize = 66

	// SecNonceSize is the size of the secret nonces for musig2. The secret
	// nonces are the corresponding private keys to the public nonce points.
	SecNonceSize = 64
)

var (
	// NonceAuxTag is the tag used to optionally mix in the secret key with
	// the set of aux randomness.
	NonceAuxTag = []byte("MuSig/aux")

	// NonceGenTag is used to generate the value (from a set of required an
	// optional field) that will be used as the part of the secret nonce.
	NonceGenTag = []byte("Musig/nonce")

	byteOrder = binary.BigEndian
)

// zeroSecNonce is a secret nonce that's all zeroes. This is used to check that
// we're not attempting to re-use a nonce, and also protect callers from it.
var zeroSecNonce [SecNonceSize]byte

// Nonces holds the public and secret nonces required for musig2.
//
// TODO(roasbeef): methods on this to help w/ parsing, etc?
type Nonces struct {
	// PubNonce holds the two 33-byte compressed encoded points that serve
	// as the public set of nonces.
	PubNonce [PubNonceSize]byte

	// SecNonce holds the two 32-byte scalar values that are the private
	// keys to the two public nonces.
	SecNonce [SecNonceSize]byte
}

// secNonceToPubNonce takes our two secrete nonces, and produces their two
// corresponding EC points, serialized in compressed format.
func secNonceToPubNonce(secNonce [SecNonceSize]byte) [PubNonceSize]byte {
	var k1Mod, k2Mod btcec.ModNScalar
	k1Mod.SetByteSlice(secNonce[:btcec.PrivKeyBytesLen])
	k2Mod.SetByteSlice(secNonce[btcec.PrivKeyBytesLen:])

	var r1, r2 btcec.JacobianPoint
	btcec.ScalarBaseMultNonConst(&k1Mod, &r1)
	btcec.ScalarBaseMultNonConst(&k2Mod, &r2)

	// Next, we'll convert the key in jacobian format to a normal public
	// key expressed in affine coordinates.
	r1.ToAffine()
	r2.ToAffine()
	r1Pub := btcec.NewPublicKey(&r1.X, &r1.Y)
	r2Pub := btcec.NewPublicKey(&r2.X, &r2.Y)

	var pubNonce [PubNonceSize]byte

	// The public nonces are serialized as: R1 || R2, where both keys are
	// serialized in compressed format.
	copy(pubNonce[:], r1Pub.SerializeCompressed())
	copy(
		pubNonce[btcec.PubKeyBytesLenCompressed:],
		r2Pub.SerializeCompressed(),
	)

	return pubNonce
}

// NonceGenOption is a function option that allows callers to modify how nonce
// generation happens.
type NonceGenOption func(*nonceGenOpts)

// nonceGenOpts is the set of options that control how nonce generation
// happens.
type nonceGenOpts struct {
	// randReader is what we'll use to generate a set of random bytes. If
	// unspecified, then the normal crypto/rand rand.Read method will be
	// used in place.
	randReader io.Reader

	// secretKey is an optional argument that's used to further augment the
	// generated nonce by xor'ing it with this secret key.
	secretKey []byte

	// combinedKey is an optional argument that if specified, will be
	// combined along with the nonce generation.
	combinedKey []byte

	// msg is an optional argument that will be mixed into the nonce
	// derivation algorithm.
	msg []byte

	// auxInput is an optional argument that will be mixed into the nonce
	// derivation algorithm.
	auxInput []byte
}

// cryptoRandAdapter is an adapter struct that allows us to pass in the package
// level Read function from crypto/rand into a context that accepts an
// io.Reader.
type cryptoRandAdapter struct {
}

// Read implements the io.Reader interface for the crypto/rand package.  By
// default, we always use the crypto/rand reader, but the caller is able to
// specify their own generation, which can be useful for deterministic tests.
func (c *cryptoRandAdapter) Read(p []byte) (n int, err error) {
	return rand.Read(p)
}

// defaultNonceGenOpts returns the default set of nonce generation options.
func defaultNonceGenOpts() *nonceGenOpts {
	return &nonceGenOpts{
		randReader: &cryptoRandAdapter{},
	}
}

// WithCustomRand allows a caller to use a custom random number generator in
// place for crypto/rand. This should only really be used to generate
// determinstic tests.
func WithCustomRand(r io.Reader) NonceGenOption {
	return func(o *nonceGenOpts) {
		o.randReader = r
	}
}

// WithNonceSecretKeyAux allows a caller to optionally specify a secret key
// that should be used to augment the randomness used to generate the nonces.
func WithNonceSecretKeyAux(secKey *btcec.PrivateKey) NonceGenOption {
	return func(o *nonceGenOpts) {
		o.secretKey = secKey.Serialize()
	}
}

// WithNonceCombinedKeyAux allows a caller to optionally specify the combined
// key used in this signing session to further augment the randomness used to
// generate nonces.
func WithNonceCombinedKeyAux(combinedKey *btcec.PublicKey) NonceGenOption {
	return func(o *nonceGenOpts) {
		o.combinedKey = schnorr.SerializePubKey(combinedKey)
	}
}

// WithNonceMessageAux allows a caller to optionally specify a message to be
// mixed into the randomness generated to create the nonce.
func WithNonceMessageAux(msg [32]byte) NonceGenOption {
	return func(o *nonceGenOpts) {
		o.msg = msg[:]
	}
}

// WithNonceAuxInput is a set of auxiliary randomness, similar to BIP 340 that
// can be used to further augment the nonce generation process.
func WithNonceAuxInput(aux []byte) NonceGenOption {
	return func(o *nonceGenOpts) {
		o.auxInput = aux
	}
}

// lengthWriter is a function closure that allows a caller to control how the
// length prefix of a byte slice is written.
type lengthWriter func(w io.Writer, b []byte) error

// uint8Writer is an implementation of lengthWriter that writes the length of
// the byte slice using 1 byte.
func uint8Writer(w io.Writer, b []byte) error {
	return binary.Write(w, byteOrder, uint8(len(b)))
}

// uint32Writer is an implementation of lengthWriter that writes the length of
// the byte slice using 4 bytes.
func uint32Writer(w io.Writer, b []byte) error {
	return binary.Write(w, byteOrder, uint32(len(b)))
}

// writeBytesPrefix is used to write out: len(b) || b, to the passed io.Writer.
// The lengthWriter function closure is used to allow the caller to specify the
// precise byte packing of the length.
func writeBytesPrefix(w io.Writer, b []byte, lenWriter lengthWriter) error {
	// Write out the length of the byte first, followed by the set of bytes
	// itself.
	if err := lenWriter(w, b); err != nil {
		return err
	}

	if _, err := w.Write(b); err != nil {
		return err
	}

	return nil
}

// genNonceAuxBytes writes out the full byte string used to derive a secret
// nonce based on some initial randomness as well as the series of optional
// fields. The byte string used for derivation is:
//  * tagged_hash("MuSig/nonce", rand || len(aggpk) || aggpk || len(m)
//                              || m || len(in) || in || i).
//
// where i is the ith secret nonce being generated.
func genNonceAuxBytes(rand []byte, i int,
	opts *nonceGenOpts) (*chainhash.Hash, error) {

	var w bytes.Buffer

	// First, write out the randomness generated in the prior step.
	if _, err := w.Write(rand); err != nil {
		return nil, err
	}

	// Next, we'll write out: len(aggpk) || aggpk.
	err := writeBytesPrefix(&w, opts.combinedKey, uint8Writer)
	if err != nil {
		return nil, err
	}

	// Next, we'll write out the length prefixed message.
	err = writeBytesPrefix(&w, opts.msg, uint8Writer)
	if err != nil {
		return nil, err
	}

	// Finally we'll write out the auxiliary input.
	err = writeBytesPrefix(&w, opts.auxInput, uint32Writer)
	if err != nil {
		return nil, err
	}

	// Next we'll write out the interaction/index number which will
	// uniquely generate two nonces given the rest of the possibly static
	// parameters.
	if err := binary.Write(&w, byteOrder, uint8(i)); err != nil {
		return nil, err
	}

	// With the message buffer complete, we'll now derive the tagged hash
	// using our set of params.
	return chainhash.TaggedHash(NonceGenTag, w.Bytes()), nil
}

// GenNonces generates the secret nonces, as well as the public nonces which
// correspond to an EC point generated using the secret nonce as a private key.
func GenNonces(options ...NonceGenOption) (*Nonces, error) {
	opts := defaultNonceGenOpts()
	for _, opt := range options {
		opt(opts)
	}

	// First, we'll start out by generating 32 random bytes drawn from our
	// CSPRNG.
	var randBytes [32]byte
	if _, err := opts.randReader.Read(randBytes[:]); err != nil {
		return nil, err
	}

	// Using our randomness and the set of optional params, generate our
	// two secret nonces: k1 and k2.
	k1, err := genNonceAuxBytes(randBytes[:], 1, opts)
	if err != nil {
		return nil, err
	}
	k2, err := genNonceAuxBytes(randBytes[:], 2, opts)
	if err != nil {
		return nil, err
	}

	var k1Mod, k2Mod btcec.ModNScalar
	k1Mod.SetBytes((*[32]byte)(k1))
	k2Mod.SetBytes((*[32]byte)(k2))

	// The secret nonces are serialized as the concatenation of the two 32
	// byte secret nonce values.
	var nonces Nonces
	k1Mod.PutBytesUnchecked(nonces.SecNonce[:])
	k2Mod.PutBytesUnchecked(nonces.SecNonce[btcec.PrivKeyBytesLen:])

	// Next, we'll generate R_1 = k_1*G and R_2 = k_2*G. Along the way we
	// need to map our nonce values into mod n scalars so we can work with
	// the btcec API.
	nonces.PubNonce = secNonceToPubNonce(nonces.SecNonce)

	return &nonces, nil
}

// AggregateNonces aggregates the set of a pair of public nonces for each party
// into a single aggregated nonces to be used for multi-signing.
func AggregateNonces(pubNonces [][PubNonceSize]byte) ([PubNonceSize]byte, error) {
	// combineNonces is a helper function that aggregates (adds) up a
	// series of nonces encoded in compressed format. It uses a slicing
	// function to extra 33 bytes at a time from the packed 2x public
	// nonces.
	type nonceSlicer func([PubNonceSize]byte) []byte
	combineNonces := func(slicer nonceSlicer) (*btcec.PublicKey, error) {
		// Convert the set of nonces into jacobian coordinates we can
		// use to accumulate them all into each other.
		pubNonceJs := make([]*btcec.JacobianPoint, len(pubNonces))
		for i, pubNonceBytes := range pubNonces {
			// Using the slicer, extract just the bytes we need to
			// decode.
			var nonceJ btcec.JacobianPoint
			pubNonce, err := btcec.ParsePubKey(
				slicer(pubNonceBytes),
			)
			if err != nil {
				return nil, err
			}

			pubNonce.AsJacobian(&nonceJ)
			pubNonceJs[i] = &nonceJ
		}

		// Now that we have the set of complete nonces, we'll aggregate
		// them: R = R_i + R_i+1 + ... + R_i+n.
		var aggregateNonce btcec.JacobianPoint
		for _, pubNonceJ := range pubNonceJs {
			btcec.AddNonConst(
				&aggregateNonce, pubNonceJ, &aggregateNonce,
			)
		}

		// Now that we've aggregated all the points, we need to check
		// if this point is the point at infinity, if so, then we'll
		// just return the generator. At a later step, the malicious
		// party will be detected.
		if aggregateNonce == infinityPoint {
			// TODO(roasbeef): better way to get the generator w/
			// the new API? -- via old curve params instead?
			var generator btcec.JacobianPoint
			one := new(btcec.ModNScalar).SetInt(1)
			btcec.ScalarBaseMultNonConst(one, &generator)

			generator.ToAffine()
			return btcec.NewPublicKey(
				&generator.X, &generator.Y,
			), nil
		}

		aggregateNonce.ToAffine()
		return btcec.NewPublicKey(
			&aggregateNonce.X, &aggregateNonce.Y,
		), nil
	}

	// The final nonce public nonce is actually two nonces, one that
	// aggregate the first nonce of all the parties, and the other that
	// aggregates the second nonce of all the parties.
	var finalNonce [PubNonceSize]byte
	combinedNonce1, err := combineNonces(func(n [PubNonceSize]byte) []byte {
		return n[:btcec.PubKeyBytesLenCompressed]
	})
	if err != nil {
		return finalNonce, err
	}
	combinedNonce2, err := combineNonces(func(n [PubNonceSize]byte) []byte {
		return n[btcec.PubKeyBytesLenCompressed:]
	})
	if err != nil {
		return finalNonce, err
	}

	copy(finalNonce[:], combinedNonce1.SerializeCompressed())
	copy(
		finalNonce[btcec.PubKeyBytesLenCompressed:],
		combinedNonce2.SerializeCompressed(),
	)

	return finalNonce, nil
}
<commit_msg>btcec/schnorr/musig2: XOR rand with secret key<commit_after>// Copyright 2013-2022 The btcsuite developers

package musig2

import (
	"bytes"
	"crypto/rand"
	"encoding/binary"
	"io"

	"github.com/btcsuite/btcd/btcec/v2"
	"github.com/btcsuite/btcd/btcec/v2/schnorr"
	"github.com/btcsuite/btcd/chaincfg/chainhash"
)

const (
	// PubNonceSize is the size of the public nonces. Each public nonce is
	// serialized the full compressed encoding, which uses 32 bytes for each
	// nonce.
	PubNonceSize = 66

	// SecNonceSize is the size of the secret nonces for musig2. The secret
	// nonces are the corresponding private keys to the public nonce points.
	SecNonceSize = 64
)

var (
	// NonceAuxTag is the tag used to optionally mix in the secret key with
	// the set of aux randomness.
	NonceAuxTag = []byte("MuSig/aux")

	// NonceGenTag is used to generate the value (from a set of required an
	// optional field) that will be used as the part of the secret nonce.
	NonceGenTag = []byte("MuSig/nonce")

	byteOrder = binary.BigEndian
)

// zeroSecNonce is a secret nonce that's all zeroes. This is used to check that
// we're not attempting to re-use a nonce, and also protect callers from it.
var zeroSecNonce [SecNonceSize]byte

// Nonces holds the public and secret nonces required for musig2.
//
// TODO(roasbeef): methods on this to help w/ parsing, etc?
type Nonces struct {
	// PubNonce holds the two 33-byte compressed encoded points that serve
	// as the public set of nonces.
	PubNonce [PubNonceSize]byte

	// SecNonce holds the two 32-byte scalar values that are the private
	// keys to the two public nonces.
	SecNonce [SecNonceSize]byte
}

// secNonceToPubNonce takes our two secrete nonces, and produces their two
// corresponding EC points, serialized in compressed format.
func secNonceToPubNonce(secNonce [SecNonceSize]byte) [PubNonceSize]byte {
	var k1Mod, k2Mod btcec.ModNScalar
	k1Mod.SetByteSlice(secNonce[:btcec.PrivKeyBytesLen])
	k2Mod.SetByteSlice(secNonce[btcec.PrivKeyBytesLen:])

	var r1, r2 btcec.JacobianPoint
	btcec.ScalarBaseMultNonConst(&k1Mod, &r1)
	btcec.ScalarBaseMultNonConst(&k2Mod, &r2)

	// Next, we'll convert the key in jacobian format to a normal public
	// key expressed in affine coordinates.
	r1.ToAffine()
	r2.ToAffine()
	r1Pub := btcec.NewPublicKey(&r1.X, &r1.Y)
	r2Pub := btcec.NewPublicKey(&r2.X, &r2.Y)

	var pubNonce [PubNonceSize]byte

	// The public nonces are serialized as: R1 || R2, where both keys are
	// serialized in compressed format.
	copy(pubNonce[:], r1Pub.SerializeCompressed())
	copy(
		pubNonce[btcec.PubKeyBytesLenCompressed:],
		r2Pub.SerializeCompressed(),
	)

	return pubNonce
}

// NonceGenOption is a function option that allows callers to modify how nonce
// generation happens.
type NonceGenOption func(*nonceGenOpts)

// nonceGenOpts is the set of options that control how nonce generation
// happens.
type nonceGenOpts struct {
	// randReader is what we'll use to generate a set of random bytes. If
	// unspecified, then the normal crypto/rand rand.Read method will be
	// used in place.
	randReader io.Reader

	// secretKey is an optional argument that's used to further augment the
	// generated nonce by xor'ing it with this secret key.
	secretKey []byte

	// combinedKey is an optional argument that if specified, will be
	// combined along with the nonce generation.
	combinedKey []byte

	// msg is an optional argument that will be mixed into the nonce
	// derivation algorithm.
	msg []byte

	// auxInput is an optional argument that will be mixed into the nonce
	// derivation algorithm.
	auxInput []byte
}

// cryptoRandAdapter is an adapter struct that allows us to pass in the package
// level Read function from crypto/rand into a context that accepts an
// io.Reader.
type cryptoRandAdapter struct {
}

// Read implements the io.Reader interface for the crypto/rand package.  By
// default, we always use the crypto/rand reader, but the caller is able to
// specify their own generation, which can be useful for deterministic tests.
func (c *cryptoRandAdapter) Read(p []byte) (n int, err error) {
	return rand.Read(p)
}

// defaultNonceGenOpts returns the default set of nonce generation options.
func defaultNonceGenOpts() *nonceGenOpts {
	return &nonceGenOpts{
		randReader: &cryptoRandAdapter{},
	}
}

// WithCustomRand allows a caller to use a custom random number generator in
// place for crypto/rand. This should only really be used to generate
// determinstic tests.
func WithCustomRand(r io.Reader) NonceGenOption {
	return func(o *nonceGenOpts) {
		o.randReader = r
	}
}

// WithNonceSecretKeyAux allows a caller to optionally specify a secret key
// that should be used to augment the randomness used to generate the nonces.
func WithNonceSecretKeyAux(secKey *btcec.PrivateKey) NonceGenOption {
	return func(o *nonceGenOpts) {
		o.secretKey = secKey.Serialize()
	}
}

// WithNonceCombinedKeyAux allows a caller to optionally specify the combined
// key used in this signing session to further augment the randomness used to
// generate nonces.
func WithNonceCombinedKeyAux(combinedKey *btcec.PublicKey) NonceGenOption {
	return func(o *nonceGenOpts) {
		o.combinedKey = schnorr.SerializePubKey(combinedKey)
	}
}

// WithNonceMessageAux allows a caller to optionally specify a message to be
// mixed into the randomness generated to create the nonce.
func WithNonceMessageAux(msg [32]byte) NonceGenOption {
	return func(o *nonceGenOpts) {
		o.msg = msg[:]
	}
}

// WithNonceAuxInput is a set of auxiliary randomness, similar to BIP 340 that
// can be used to further augment the nonce generation process.
func WithNonceAuxInput(aux []byte) NonceGenOption {
	return func(o *nonceGenOpts) {
		o.auxInput = aux
	}
}

// lengthWriter is a function closure that allows a caller to control how the
// length prefix of a byte slice is written.
type lengthWriter func(w io.Writer, b []byte) error

// uint8Writer is an implementation of lengthWriter that writes the length of
// the byte slice using 1 byte.
func uint8Writer(w io.Writer, b []byte) error {
	return binary.Write(w, byteOrder, uint8(len(b)))
}

// uint32Writer is an implementation of lengthWriter that writes the length of
// the byte slice using 4 bytes.
func uint32Writer(w io.Writer, b []byte) error {
	return binary.Write(w, byteOrder, uint32(len(b)))
}

// writeBytesPrefix is used to write out: len(b) || b, to the passed io.Writer.
// The lengthWriter function closure is used to allow the caller to specify the
// precise byte packing of the length.
func writeBytesPrefix(w io.Writer, b []byte, lenWriter lengthWriter) error {
	// Write out the length of the byte first, followed by the set of bytes
	// itself.
	if err := lenWriter(w, b); err != nil {
		return err
	}

	if _, err := w.Write(b); err != nil {
		return err
	}

	return nil
}

// genNonceAuxBytes writes out the full byte string used to derive a secret
// nonce based on some initial randomness as well as the series of optional
// fields. The byte string used for derivation is:
//  * tagged_hash("MuSig/nonce", rand || len(aggpk) || aggpk || len(m)
//                              || m || len(in) || in || i).
//
// where i is the ith secret nonce being generated.
func genNonceAuxBytes(rand []byte, i int,
	opts *nonceGenOpts) (*chainhash.Hash, error) {

	var w bytes.Buffer

	// First, write out the randomness generated in the prior step.
	if _, err := w.Write(rand); err != nil {
		return nil, err
	}

	// Next, we'll write out: len(aggpk) || aggpk.
	err := writeBytesPrefix(&w, opts.combinedKey, uint8Writer)
	if err != nil {
		return nil, err
	}

	// Next, we'll write out the length prefixed message.
	err = writeBytesPrefix(&w, opts.msg, uint8Writer)
	if err != nil {
		return nil, err
	}

	// Finally we'll write out the auxiliary input.
	err = writeBytesPrefix(&w, opts.auxInput, uint32Writer)
	if err != nil {
		return nil, err
	}

	// Next we'll write out the interaction/index number which will
	// uniquely generate two nonces given the rest of the possibly static
	// parameters.
	if err := binary.Write(&w, byteOrder, uint8(i)); err != nil {
		return nil, err
	}

	// With the message buffer complete, we'll now derive the tagged hash
	// using our set of params.
	return chainhash.TaggedHash(NonceGenTag, w.Bytes()), nil
}

// GenNonces generates the secret nonces, as well as the public nonces which
// correspond to an EC point generated using the secret nonce as a private key.
func GenNonces(options ...NonceGenOption) (*Nonces, error) {
	opts := defaultNonceGenOpts()
	for _, opt := range options {
		opt(opts)
	}

	// First, we'll start out by generating 32 random bytes drawn from our
	// CSPRNG.
	var randBytes [32]byte
	if _, err := opts.randReader.Read(randBytes[:]); err != nil {
		return nil, err
	}

	// If the options contain a secret key, we XOR it with with the tagged
	// random bytes.
	if len(opts.secretKey) == 32 {
		taggedHash := chainhash.TaggedHash(NonceAuxTag, randBytes[:])

		for i := 0; i < chainhash.HashSize; i++ {
			randBytes[i] = opts.secretKey[i] ^ taggedHash[i]
		}
	}

	// Using our randomness and the set of optional params, generate our
	// two secret nonces: k1 and k2.
	k1, err := genNonceAuxBytes(randBytes[:], 1, opts)
	if err != nil {
		return nil, err
	}
	k2, err := genNonceAuxBytes(randBytes[:], 2, opts)
	if err != nil {
		return nil, err
	}

	var k1Mod, k2Mod btcec.ModNScalar
	k1Mod.SetBytes((*[32]byte)(k1))
	k2Mod.SetBytes((*[32]byte)(k2))

	// The secret nonces are serialized as the concatenation of the two 32
	// byte secret nonce values.
	var nonces Nonces
	k1Mod.PutBytesUnchecked(nonces.SecNonce[:])
	k2Mod.PutBytesUnchecked(nonces.SecNonce[btcec.PrivKeyBytesLen:])

	// Next, we'll generate R_1 = k_1*G and R_2 = k_2*G. Along the way we
	// need to map our nonce values into mod n scalars so we can work with
	// the btcec API.
	nonces.PubNonce = secNonceToPubNonce(nonces.SecNonce)

	return &nonces, nil
}

// AggregateNonces aggregates the set of a pair of public nonces for each party
// into a single aggregated nonces to be used for multi-signing.
func AggregateNonces(pubNonces [][PubNonceSize]byte) ([PubNonceSize]byte, error) {
	// combineNonces is a helper function that aggregates (adds) up a
	// series of nonces encoded in compressed format. It uses a slicing
	// function to extra 33 bytes at a time from the packed 2x public
	// nonces.
	type nonceSlicer func([PubNonceSize]byte) []byte
	combineNonces := func(slicer nonceSlicer) (*btcec.PublicKey, error) {
		// Convert the set of nonces into jacobian coordinates we can
		// use to accumulate them all into each other.
		pubNonceJs := make([]*btcec.JacobianPoint, len(pubNonces))
		for i, pubNonceBytes := range pubNonces {
			// Using the slicer, extract just the bytes we need to
			// decode.
			var nonceJ btcec.JacobianPoint
			pubNonce, err := btcec.ParsePubKey(
				slicer(pubNonceBytes),
			)
			if err != nil {
				return nil, err
			}

			pubNonce.AsJacobian(&nonceJ)
			pubNonceJs[i] = &nonceJ
		}

		// Now that we have the set of complete nonces, we'll aggregate
		// them: R = R_i + R_i+1 + ... + R_i+n.
		var aggregateNonce btcec.JacobianPoint
		for _, pubNonceJ := range pubNonceJs {
			btcec.AddNonConst(
				&aggregateNonce, pubNonceJ, &aggregateNonce,
			)
		}

		// Now that we've aggregated all the points, we need to check
		// if this point is the point at infinity, if so, then we'll
		// just return the generator. At a later step, the malicious
		// party will be detected.
		if aggregateNonce == infinityPoint {
			// TODO(roasbeef): better way to get the generator w/
			// the new API? -- via old curve params instead?
			var generator btcec.JacobianPoint
			one := new(btcec.ModNScalar).SetInt(1)
			btcec.ScalarBaseMultNonConst(one, &generator)

			generator.ToAffine()
			return btcec.NewPublicKey(
				&generator.X, &generator.Y,
			), nil
		}

		aggregateNonce.ToAffine()
		return btcec.NewPublicKey(
			&aggregateNonce.X, &aggregateNonce.Y,
		), nil
	}

	// The final nonce public nonce is actually two nonces, one that
	// aggregate the first nonce of all the parties, and the other that
	// aggregates the second nonce of all the parties.
	var finalNonce [PubNonceSize]byte
	combinedNonce1, err := combineNonces(func(n [PubNonceSize]byte) []byte {
		return n[:btcec.PubKeyBytesLenCompressed]
	})
	if err != nil {
		return finalNonce, err
	}
	combinedNonce2, err := combineNonces(func(n [PubNonceSize]byte) []byte {
		return n[btcec.PubKeyBytesLenCompressed:]
	})
	if err != nil {
		return finalNonce, err
	}

	copy(finalNonce[:], combinedNonce1.SerializeCompressed())
	copy(
		finalNonce[btcec.PubKeyBytesLenCompressed:],
		combinedNonce2.SerializeCompressed(),
	)

	return finalNonce, nil
}
<|endoftext|>


<commit_before>// Copyright 2014 Manu Martinez-Almeida.  All rights reserved.
// Use of this source code is governed by a MIT style
// license that can be found in the LICENSE file.

package ego

import (
	"bytes"
	"html/template"
	"log"
)

func init() {
	log.SetFlags(0)
}

// IsDebugging returns true if the framework is running in debug mode.
// Use SetMode(ego.Release) to disable debug mode.
func IsDebugging() bool {
	return egoMode == debugCode
}

func debugPrintRoute(httpMethod, absolutePath string, handlers HandlersChain) {
	if IsDebugging() {
		nuHandlers := len(handlers)
		handlerName := nameOfFunction(handlers.Last())
		debugPrint("%-6s %-25s --> %s (%d handlers)\n", httpMethod, absolutePath, handlerName, nuHandlers)
	}
}

func debugPrintLoadTemplate(tmpl *template.Template) {
	if IsDebugging() {
		var buf bytes.Buffer
		for _, tmpl := range tmpl.Templates() {
			buf.WriteString("\t- ")
			buf.WriteString(tmpl.Name())
			buf.WriteString("\n")
		}
		debugPrint("Loaded HTML Templates (%d): \n%s\n", len(tmpl.Templates()), buf.String())
	}
}

func debugPrint(format string, values ...interface{}) {
	if IsDebugging() {
		log.Printf("[EGO-debug] "+format, values...)
	}
}

func debugPrintWARNINGDefault() {
	debugPrint(`[WARNING] Creating an Engine instance with the Logger and Recovery middleware already attached.

	`)
}

func debugPrintWARNINGNew() {
	debugPrint(`[WARNING] Running in "debug" mode. Switch to "release" mode in production.
 - using env:	export EGO_MODE=release
 - using code:	ego.SetMode(ego.ReleaseMode)

`)
}

func debugPrintWARNINGSetHTMLTemplate() {
	debugPrint(`[WARNING] Since SetHTMLTemplate() is NOT thread-safe. It should only be called
at initialization. ie. before any route is registered or the router is listening in a socket:

	router := ego.Default()
	router.SetHTMLTemplate(template) // << good place

`)
}

func debugPrintError(err error) {
	if err != nil {
		debugPrint("[ERROR] %v\n", err)
	}
}
<commit_msg>Update comment debug<commit_after>// Copyright 2014 Manu Martinez-Almeida.  All rights reserved.
// Use of this source code is governed by a MIT style
// license that can be found in the LICENSE file.

package ego

import (
	"bytes"
	"html/template"
	"log"
)

func init() {
	log.SetFlags(0)
}

// IsDebugging returns true if the framework is running in debug mode.
// Use SetMode(ego.ReleaseMode) to disable debug mode.
func IsDebugging() bool {
	return egoMode == debugCode
}

func debugPrintRoute(httpMethod, absolutePath string, handlers HandlersChain) {
	if IsDebugging() {
		nuHandlers := len(handlers)
		handlerName := nameOfFunction(handlers.Last())
		debugPrint("%-6s %-25s --> %s (%d handlers)\n", httpMethod, absolutePath, handlerName, nuHandlers)
	}
}

func debugPrintLoadTemplate(tmpl *template.Template) {
	if IsDebugging() {
		var buf bytes.Buffer
		for _, tmpl := range tmpl.Templates() {
			buf.WriteString("\t- ")
			buf.WriteString(tmpl.Name())
			buf.WriteString("\n")
		}
		debugPrint("Loaded HTML Templates (%d): \n%s\n", len(tmpl.Templates()), buf.String())
	}
}

func debugPrint(format string, values ...interface{}) {
	if IsDebugging() {
		log.Printf("[EGO-debug] "+format, values...)
	}
}

func debugPrintWARNINGDefault() {
	debugPrint(`[WARNING] Creating an Engine instance with the Logger and Recovery middleware already attached.

	`)
}

func debugPrintWARNINGNew() {
	debugPrint(`[WARNING] Running in "debug" mode. Switch to "release" mode in production.
 - using env:	export EGO_MODE=release
 - using code:	ego.SetMode(ego.ReleaseMode)

`)
}

func debugPrintWARNINGSetHTMLTemplate() {
	debugPrint(`[WARNING] Since SetHTMLTemplate() is NOT thread-safe. It should only be called
at initialization. ie. before any route is registered or the router is listening in a socket:

	router := ego.Default()
	router.SetHTMLTemplate(template) // << good place

`)
}

func debugPrintError(err error) {
	if err != nil {
		debugPrint("[ERROR] %v\n", err)
	}
}
<|endoftext|>


<commit_before>// Copyright 2022 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"context"
	"fmt"
	"os"
	"path/filepath"

	bbpb "go.chromium.org/luci/buildbucket/proto"
	"go.chromium.org/luci/common/errors"
	"go.chromium.org/luci/common/logging"
	"go.chromium.org/luci/lucictx"
)

// Find CAS Client. It should have been downloaded when bbagent installs
// CIPD packages.
func findCasClient(b *bbpb.Build) (string, error) {
	var bbagentUtilityPath string
	for path, purpose := range b.Infra.Buildbucket.Agent.Purposes {
		if purpose == bbpb.BuildInfra_Buildbucket_Agent_PURPOSE_BBAGENT_UTILITY {
			bbagentUtilityPath = path
			break
		}
	}
	if bbagentUtilityPath == "" {
		return "", errors.Reason("Failed to find bbagent utility packages").Err()
	}

	casClient, err := processCmd(bbagentUtilityPath, "cas")
	if err != nil {
		return "", err
	}
	return casClient, nil
}

func generateCasCmd(casClient, outputDir string, casRef *bbpb.InputDataRef_CAS) []string {
	return []string{
		casClient,
		"download",
		"-cas-instance",
		casRef.CasInstance,
		"-digest",
		fmt.Sprintf("%s/%d", casRef.Digest.Hash, casRef.Digest.SizeBytes),
		"-dir",
		outputDir,
		"-log-level",
		"info",
	}
}

func execCasCmd(ctx context.Context, args []string) error {
	// Switch to swarming system account to download CAS inputs, it won't
	// work for non-swarming backends in the future.
	// TODO(crbug.com/1114804): Handle downloading CAS inputs within tasks
	// running on non-swarming backends.
	sysCtx, err := lucictx.SwitchLocalAccount(ctx, "system")
	if err != nil {
		return errors.Annotate(err, "could not switch to 'system' account in LUCI_CONTEXT").Err()
	}
	// TODO(crbug.com/1114804): remove log.
	logging.Infof(ctx, "Running command with system account")
	cmd := execCommandContext(sysCtx, args[0], args[1:]...)
	logging.Infof(ctx, "Running command: %s", cmd.String())
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	return errors.Annotate(cmd.Run(), "Failed to run cas download").Err()
}

func downloadCasFiles(ctx context.Context, b *bbpb.Build, workDir string) error {
	var casClient string
	var err error

	inputData := b.GetInfra().GetBuildbucket().GetAgent().GetInput().GetData()
	for dir, ref := range inputData {
		casRef := ref.GetCas()
		if casRef == nil {
			continue
		}

		if casClient == "" {
			if casClient, err = findCasClient(b); err != nil {
				return errors.Annotate(err, "download cas files").Err()
			}
		}

		outDir := filepath.Join(workDir, dir)
		cmdArgs := generateCasCmd(casClient, outDir, casRef)
		if err = execCasCmd(ctx, cmdArgs); err != nil {
			return errors.Annotate(err, "download cas files").Err()
		}
	}

	// TODO(chanli): populate ResolvedDataRef_CAS as more things use CAS in
	// buildbucket.
	return nil
}
<commit_msg>[bbagent] remove used log<commit_after>// Copyright 2022 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"context"
	"fmt"
	"os"
	"path/filepath"

	bbpb "go.chromium.org/luci/buildbucket/proto"
	"go.chromium.org/luci/common/errors"
	"go.chromium.org/luci/common/logging"
	"go.chromium.org/luci/lucictx"
)

// Find CAS Client. It should have been downloaded when bbagent installs
// CIPD packages.
func findCasClient(b *bbpb.Build) (string, error) {
	var bbagentUtilityPath string
	for path, purpose := range b.Infra.Buildbucket.Agent.Purposes {
		if purpose == bbpb.BuildInfra_Buildbucket_Agent_PURPOSE_BBAGENT_UTILITY {
			bbagentUtilityPath = path
			break
		}
	}
	if bbagentUtilityPath == "" {
		return "", errors.Reason("Failed to find bbagent utility packages").Err()
	}

	casClient, err := processCmd(bbagentUtilityPath, "cas")
	if err != nil {
		return "", err
	}
	return casClient, nil
}

func generateCasCmd(casClient, outputDir string, casRef *bbpb.InputDataRef_CAS) []string {
	return []string{
		casClient,
		"download",
		"-cas-instance",
		casRef.CasInstance,
		"-digest",
		fmt.Sprintf("%s/%d", casRef.Digest.Hash, casRef.Digest.SizeBytes),
		"-dir",
		outputDir,
		"-log-level",
		"info",
	}
}

func execCasCmd(ctx context.Context, args []string) error {
	// Switch to swarming system account to download CAS inputs, it won't
	// work for non-swarming backends in the future.
	// TODO(crbug.com/1114804): Handle downloading CAS inputs within tasks
	// running on non-swarming backends.
	sysCtx, err := lucictx.SwitchLocalAccount(ctx, "system")
	if err != nil {
		return errors.Annotate(err, "could not switch to 'system' account in LUCI_CONTEXT").Err()
	}
	cmd := execCommandContext(sysCtx, args[0], args[1:]...)
	logging.Infof(ctx, "Running command: %s", cmd.String())
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	return errors.Annotate(cmd.Run(), "Failed to run cas download").Err()
}

func downloadCasFiles(ctx context.Context, b *bbpb.Build, workDir string) error {
	var casClient string
	var err error

	inputData := b.GetInfra().GetBuildbucket().GetAgent().GetInput().GetData()
	for dir, ref := range inputData {
		casRef := ref.GetCas()
		if casRef == nil {
			continue
		}

		if casClient == "" {
			if casClient, err = findCasClient(b); err != nil {
				return errors.Annotate(err, "download cas files").Err()
			}
		}

		outDir := filepath.Join(workDir, dir)
		cmdArgs := generateCasCmd(casClient, outDir, casRef)
		if err = execCasCmd(ctx, cmdArgs); err != nil {
			return errors.Annotate(err, "download cas files").Err()
		}
	}

	// TODO(chanli): populate ResolvedDataRef_CAS as more things use CAS in
	// buildbucket.
	return nil
}
<|endoftext|>


<commit_before>package docker

import (
	"bytes"
	"fmt"
	"github.com/mitchellh/packer/packer"
	"io"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"syscall"
	"time"
)

type Communicator struct {
	ContainerId  string
	HostDir      string
	ContainerDir string
}

func (c *Communicator) Start(remote *packer.RemoteCmd) error {
	// Create a temporary file to store the output. Because of a bug in
	// Docker, sometimes all the output doesn't properly show up. This
	// file will capture ALL of the output, and we'll read that.
	//
	// https://github.com/dotcloud/docker/issues/2625
	outputFile, err := ioutil.TempFile(c.HostDir, "cmd")
	if err != nil {
		return err
	}
	outputFile.Close()
	defer os.Remove(outputFile.Name())

	// This file will store the exit code of the command once it is complete.
	exitCodePath := outputFile.Name() + "-exit"

	// Modify the remote command so that all the output of the commands
	// go to a single file and so that the exit code is redirected to
	// a single file. This lets us determine both when the command
	// is truly complete (because the file will have data), what the
	// exit status is (because Docker loses it because of the pty, not
	// Docker's fault), and get the output (Docker bug).
	remoteCmd := fmt.Sprintf("(%s) >%s 2>&1; echo $? >%s",
		remote.Command,
		filepath.Join(c.ContainerDir, filepath.Base(outputFile.Name())),
		filepath.Join(c.ContainerDir, filepath.Base(exitCodePath)))

	cmd := exec.Command("docker", "attach", c.ContainerId)
	stdin_w, err := cmd.StdinPipe()
	if err != nil {
		return err
	}

	log.Printf("Executing in container %s: %#v", c.ContainerId, remoteCmd)
	if err := cmd.Start(); err != nil {
		return err
	}

	go func() {
		defer stdin_w.Close()

		// This sleep needs to be here because of the issue linked to below.
		// Basically, without it, Docker will hang on reading stdin forever,
		// and won't see what we write, for some reason.
		//
		// https://github.com/dotcloud/docker/issues/2628
		time.Sleep(2 * time.Second)

		stdin_w.Write([]byte(remoteCmd + "\n"))
	}()

	err = cmd.Wait()
	if exitErr, ok := err.(*exec.ExitError); ok {
		exitStatus := 1

		// There is no process-independent way to get the REAL
		// exit status so we just try to go deeper.
		if status, ok := exitErr.Sys().(syscall.WaitStatus); ok {
			exitStatus = status.ExitStatus()
		}

		// Say that we ended, since if Docker itself failed, then
		// the command must've not run, or so we assume
		remote.SetExited(exitStatus)
		return nil
	}

	// Wait for the exit code to appear in our file...
	log.Println("Waiting for exit code to appear for remote command...")
	for {
		fi, err := os.Stat(exitCodePath)
		if err == nil && fi.Size() > 0 {
			break
		}

		time.Sleep(1 * time.Second)
	}

	// Read the exit code
	exitRaw, err := ioutil.ReadFile(exitCodePath)
	if err != nil {
		return err
	}

	exitStatus, err := strconv.ParseInt(string(bytes.TrimSpace(exitRaw)), 10, 0)
	if err != nil {
		return err
	}
	log.Printf("Executed command exit status: %d", exitStatus)

	// Read the output
	f, err := os.Open(outputFile.Name())
	if err != nil {
		return err
	}
	defer f.Close()

	if remote.Stdout != nil {
		io.Copy(remote.Stdout, f)
	} else {
		output, err := ioutil.ReadAll(f)
		if err != nil {
			return err
		}

		log.Printf("Command output: %s", string(output))
	}

	// Finally, we're done
	remote.SetExited(int(exitStatus))

	return nil
}

func (c *Communicator) Upload(dst string, src io.Reader) error {
	// Create a temporary file to store the upload
	tempfile, err := ioutil.TempFile(c.HostDir, "upload")
	if err != nil {
		return err
	}
	defer os.Remove(tempfile.Name())

	// Copy the contents to the temporary file
	_, err = io.Copy(tempfile, src)
	tempfile.Close()
	if err != nil {
		return err
	}

	// TODO(mitchellh): Copy the file into place
	cmd := &packer.RemoteCmd{
		Command: fmt.Sprintf("cp %s/%s %s", c.ContainerDir,
			filepath.Base(tempfile.Name()), dst),
	}

	if err := c.Start(cmd); err != nil {
		return err
	}

	// Wait for the copy to complete
	cmd.Wait()
	if cmd.ExitStatus != 0 {
		return fmt.Errorf("Upload failed with non-zero exit status: %d", cmd.ExitStatus)
	}

	return nil
}

func (c *Communicator) UploadDir(dst string, src string, exclude []string) error {
	return nil
}

func (c *Communicator) Download(src string, dst io.Writer) error {
	return nil
}
<commit_msg>builder/docker: remove the exit code file when we're done<commit_after>package docker

import (
	"bytes"
	"fmt"
	"github.com/mitchellh/packer/packer"
	"io"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"syscall"
	"time"
)

type Communicator struct {
	ContainerId  string
	HostDir      string
	ContainerDir string
}

func (c *Communicator) Start(remote *packer.RemoteCmd) error {
	// Create a temporary file to store the output. Because of a bug in
	// Docker, sometimes all the output doesn't properly show up. This
	// file will capture ALL of the output, and we'll read that.
	//
	// https://github.com/dotcloud/docker/issues/2625
	outputFile, err := ioutil.TempFile(c.HostDir, "cmd")
	if err != nil {
		return err
	}
	outputFile.Close()
	defer os.Remove(outputFile.Name())

	// This file will store the exit code of the command once it is complete.
	exitCodePath := outputFile.Name() + "-exit"
	defer os.Remove(exitCodePath)

	// Modify the remote command so that all the output of the commands
	// go to a single file and so that the exit code is redirected to
	// a single file. This lets us determine both when the command
	// is truly complete (because the file will have data), what the
	// exit status is (because Docker loses it because of the pty, not
	// Docker's fault), and get the output (Docker bug).
	remoteCmd := fmt.Sprintf("(%s) >%s 2>&1; echo $? >%s",
		remote.Command,
		filepath.Join(c.ContainerDir, filepath.Base(outputFile.Name())),
		filepath.Join(c.ContainerDir, filepath.Base(exitCodePath)))

	cmd := exec.Command("docker", "attach", c.ContainerId)
	stdin_w, err := cmd.StdinPipe()
	if err != nil {
		return err
	}

	log.Printf("Executing in container %s: %#v", c.ContainerId, remoteCmd)
	if err := cmd.Start(); err != nil {
		return err
	}

	go func() {
		defer stdin_w.Close()

		// This sleep needs to be here because of the issue linked to below.
		// Basically, without it, Docker will hang on reading stdin forever,
		// and won't see what we write, for some reason.
		//
		// https://github.com/dotcloud/docker/issues/2628
		time.Sleep(2 * time.Second)

		stdin_w.Write([]byte(remoteCmd + "\n"))
	}()

	err = cmd.Wait()
	if exitErr, ok := err.(*exec.ExitError); ok {
		exitStatus := 1

		// There is no process-independent way to get the REAL
		// exit status so we just try to go deeper.
		if status, ok := exitErr.Sys().(syscall.WaitStatus); ok {
			exitStatus = status.ExitStatus()
		}

		// Say that we ended, since if Docker itself failed, then
		// the command must've not run, or so we assume
		remote.SetExited(exitStatus)
		return nil
	}

	// Wait for the exit code to appear in our file...
	log.Println("Waiting for exit code to appear for remote command...")
	for {
		fi, err := os.Stat(exitCodePath)
		if err == nil && fi.Size() > 0 {
			break
		}

		time.Sleep(1 * time.Second)
	}

	// Read the exit code
	exitRaw, err := ioutil.ReadFile(exitCodePath)
	if err != nil {
		return err
	}

	exitStatus, err := strconv.ParseInt(string(bytes.TrimSpace(exitRaw)), 10, 0)
	if err != nil {
		return err
	}
	log.Printf("Executed command exit status: %d", exitStatus)

	// Read the output
	f, err := os.Open(outputFile.Name())
	if err != nil {
		return err
	}
	defer f.Close()

	if remote.Stdout != nil {
		io.Copy(remote.Stdout, f)
	} else {
		output, err := ioutil.ReadAll(f)
		if err != nil {
			return err
		}

		log.Printf("Command output: %s", string(output))
	}

	// Finally, we're done
	remote.SetExited(int(exitStatus))

	return nil
}

func (c *Communicator) Upload(dst string, src io.Reader) error {
	// Create a temporary file to store the upload
	tempfile, err := ioutil.TempFile(c.HostDir, "upload")
	if err != nil {
		return err
	}
	defer os.Remove(tempfile.Name())

	// Copy the contents to the temporary file
	_, err = io.Copy(tempfile, src)
	tempfile.Close()
	if err != nil {
		return err
	}

	// TODO(mitchellh): Copy the file into place
	cmd := &packer.RemoteCmd{
		Command: fmt.Sprintf("cp %s/%s %s", c.ContainerDir,
			filepath.Base(tempfile.Name()), dst),
	}

	if err := c.Start(cmd); err != nil {
		return err
	}

	// Wait for the copy to complete
	cmd.Wait()
	if cmd.ExitStatus != 0 {
		return fmt.Errorf("Upload failed with non-zero exit status: %d", cmd.ExitStatus)
	}

	return nil
}

func (c *Communicator) UploadDir(dst string, src string, exclude []string) error {
	return nil
}

func (c *Communicator) Download(src string, dst io.Writer) error {
	return nil
}
<|endoftext|>