<commit_before>package tenho

import (
	"fmt"
	"math/rand"
	"sort"
	"time"
)

func Start() {
	start := time.Now()
	// 処理
	var i float64
	i = 0
	for {
		i++
		end := time.Now()
		diff := float64(end.Sub(start) / 1000 / 1000 / 1000)
		m := int(i / diff)
		out := 0
		if m >= 0 {
			out = m
		}
		fmt.Printf("\r")
		fmt.Printf("\r%v回試行  %v秒経過 %v回/秒", i, diff, out)
		seed := time.Now().UnixNano()
		var hai string
		var ok bool
		hai, ok = tryOnce(seed)
		fmt.Printf("%v", hai)
		if ok {
			break
		}
	}
	fmt.Printf("\n")
}

type Hand []int

func tryOnce(seed int64) (string, bool) {
	hand := ShuffledHand(seed)
	hai := hand.HaiString()
	ok := Solve(hand)
	return hai, ok
}

// http://d.hatena.ne.jp/hake/20150930/p1
func shuffle(hand Hand) {
	for i := len(hand); i > 1; i-- {
		j := rand.Intn(i) // 0 .. i-1 の乱数発生
		hand[i-1], hand[j] = hand[j], hand[i-1]
	}
}

func ShuffledHand(seed int64) Hand {
	rand.Seed(seed)

	// データ要素数指定、および初期データ作成
	size := 136
	hand := make(Hand, size, size)
	for i := 0; i < size; i++ {
		hand[i] = i / 4
	}

	// シャッフル
	shuffle(hand)

	return hand[:14]
}

// 牌文字への変換(スペース区切り)
func (c Hand) HaiString() string {
	// http://qiita.com/ruiu/items/2bb83b29baeae2433a79
	// サイズ0、内部バッファの長さ69の[]byteの値を割り当てる
	b := make([]byte, 0, 70)

	// bに文字列を追加
	for j := 0; j < 14; j++ {
		// コードポイント上、普通の麻雀牌はU+1F000からの34個。
		// U+1F000 is 'MAHJONG TILE EAST WIND' ('東')
		// https://codepoints.net/U+1F000
		b = append(b, string(c[j]+0x1F000)...) // appendするには...が必要
		// 自分のMacではスペース区切りでないとうまく表示されないためスペースを挿入する
		// U+0020 is 'SPACE'
		// https://codepoints.net/U+0020
		b = append(b, string(0x20)...) // appendするには...が必要
	}
	return string(b)
}

// あがり判定する
func Solve(hand Hand) bool {
	if is_chitoitsu(hand) {
		return true
	}
	suit_grouped_hand := hand.GroupSuit()
	//fmt.Printf("%v", suit_grouped_hand)

	return suit_grouped_hand.Solve()
}

type SuitsGroupedHand [][]int

// スート分類
func (hand Hand) GroupSuit() SuitsGroupedHand {
	m := SuitsGroupedHand{{}, {}, {}, {}}
	for _, i := range hand {
		switch {
		case i < 7:
			m[0] = append(m[0], i)
		case i < 7+(9*1):
			m[1] = append(m[1], i-7)
		case i < 7+(9*2):
			m[2] = append(m[2], i-7-(9*1))
		case i < 7+(9*3):
			m[3] = append(m[3], i-7-(9*2))
		}
	}
	return m
}

func (m SuitsGroupedHand) Solve() bool {
	if !m.valid_mod3() {
		return false
	}
	if !m.valid_33332() {
		return false
	}
	return true
}

func (m SuitsGroupedHand) valid_mod3() bool {
	//スートのサイズを3で割った時
	//あまりが2であるすーとグループが1つであること
	c := 0
	for _, a := range m {
		switch len(a) % 3 {
		case 0:
			// noop
		case 1:
			return false
		case 2:
			c++
		}
	}
	return c == 1
}

func (m SuitsGroupedHand) valid_33332() bool {
	for i := 0; i < 4; i++ {
		if !valid_suit_group(m[i], i) {
			return false
		}
	}
	return true
}

func valid_suit_group(a Hand, i int) bool {
	// 第二引数は字牌のとき0

	//ソート
	sort.Ints(a)
	if len(a)%3 == 2 {
		//ペアを探す
		pair_numbers := pairable_numbers(a)
		//ペア候補がなかったらぬける
		if len(pair_numbers) == 0 {
			return false
		}
		//ペア候補毎に繰り返し処理
		for _, v := range pair_numbers {
			//ペアとなる２枚を除去
			rest := Hand{}
			c := 2
			for _, w := range a {
				// ペア候補以外は新スライスに入れる
				// ペア候補は３枚目以降は新スライスに入れる
				if w != v || c <= 0 {
					rest = append(rest, w)
				}
				if w == v {
					c--
				}
			}
			if valid_3cards(rest, i) {
				return true
			} else {
				continue
			}
		}
		return false
	} else if len(a)%3 == 0 {
		return valid_3cards(a, i)
	}
	// 到達しないはず
	panic("到達しないはず")
}

func valid_3cards(a Hand, i int) bool {
	// 刻子や順子のみで構成されている場合true
	// a is sorted
	// a.size % 3 is0
	// 第二引数は字牌のとき0
	ok := false
	for {
		a, ok = remove_kotsu(a)
		if ok {
			continue
		}
		if i > 0 {
			a, ok = remove_shuntsu(a)
			if ok {
				continue
			}
		}
		return len(a) == 0
	}
}

func remove_kotsu(a Hand) (Hand, bool) {
	// 刻子を除去できればtrue
	// a is sorted
	retval := a
	if len(a) < 3 {
		return retval, false
	}
	if a[0] == a[1] && a[0] == a[2] {
		retval = a[3:]
		return retval, true
	}
	return retval, false
}

func remove_shuntsu(a Hand) (Hand, bool) {
	// 順子を除去できればtrue
	// a is sorted
	rest := Hand{}
	first := -1
	second := -1
	found := false
	for _, v := range a {
		if found {
			rest = append(rest, v)
			continue
		}
		if first == -1 {
			first = v
		} else if second == -1 && first+1 == v {
			second = v
		} else if second != -1 && first+2 == v {
			//flush
			first = -1
			second = -1
			found = true
		} else {
			rest = append(rest, v)
		}
	}
	return rest, found
}

func pairable_numbers(sorted Hand) Hand {
	counter := []int{}
	a := 999 // 2つ前
	b := 999 // 1つ前
	for _, v := range sorted {
		if b == v && a != v {
			counter = append(counter, v)
		} else {
			b = v
		}
	}
	return counter
}

func is_chitoitsu(hand Hand) bool {
	//カウンタ
	c := map[int]int{}

	for _, v := range hand {
		count, ok := c[v]
		if ok {
			if count == 1 {
				c[v] = 2
			} else {
				// c[v] == 2
				return false
			}
		} else {
			c[v] = 1
		}
		//8個チェック
		if len(c) >= 8 {
			return false
		}
	}
	return true
}
<commit_msg>Refactor<commit_after>package tenho

import (
	"fmt"
	"math/rand"
	"sort"
	"time"
)

func Start() {
	start := time.Now()
	// 処理
	var i float64
	i = 0
	for {
		i++
		end := time.Now()
		diff := float64(end.Sub(start) / 1000 / 1000 / 1000)
		m := int(i / diff)
		out := 0
		if m >= 0 {
			out = m
		}
		fmt.Printf("\r")
		fmt.Printf("\r%v回試行  %v秒経過 %v回/秒", i, diff, out)
		seed := time.Now().UnixNano()
		var hai string
		var ok bool
		hai, ok = tryOnce(seed)
		fmt.Printf("%v", hai)
		if ok {
			break
		}
	}
	fmt.Printf("\n")
}

type Hand []int

func tryOnce(seed int64) (string, bool) {
	hand := ShuffledHand(seed)
	hai := hand.HaiString()
	ok := Solve(hand)
	return hai, ok
}

// http://d.hatena.ne.jp/hake/20150930/p1
func shuffle(hand Hand) {
	for i := len(hand); i > 1; i-- {
		j := rand.Intn(i) // 0 .. i-1 の乱数発生
		hand[i-1], hand[j] = hand[j], hand[i-1]
	}
}

func ShuffledHand(seed int64) Hand {
	rand.Seed(seed)

	// データ要素数指定、および初期データ作成
	size := 136
	hand := make(Hand, size, size)
	for i := 0; i < size; i++ {
		hand[i] = i / 4
	}

	// シャッフル
	shuffle(hand)

	return hand[:14]
}

// 牌文字への変換(スペース区切り)
func (c Hand) HaiString() string {
	// http://qiita.com/ruiu/items/2bb83b29baeae2433a79
	// サイズ0、内部バッファの長さ69の[]byteの値を割り当てる
	b := make([]byte, 0, 70)

	// bに文字列を追加
	for j := 0; j < 14; j++ {
		// コードポイント上、普通の麻雀牌はU+1F000からの34個。
		// U+1F000 is 'MAHJONG TILE EAST WIND' ('東')
		// https://codepoints.net/U+1F000
		b = append(b, string(c[j]+0x1F000)...) // appendするには...が必要
		// 自分のMacではスペース区切りでないとうまく表示されないためスペースを挿入する
		// U+0020 is 'SPACE'
		// https://codepoints.net/U+0020
		b = append(b, string(0x20)...) // appendするには...が必要
	}
	return string(b)
}

// あがり判定する
func Solve(hand Hand) bool {
	if is_chitoitsu(hand) {
		return true
	}
	suit_grouped_hand := hand.GroupSuit()
	//fmt.Printf("%v", suit_grouped_hand)

	return suit_grouped_hand.Solve()
}

type SuitsGroupedHand [][]int

// スート分類
func (hand Hand) GroupSuit() SuitsGroupedHand {
	m := SuitsGroupedHand{{}, {}, {}, {}}
	for _, i := range hand {
		switch {
		case i < 7:
			m[0] = append(m[0], i)
		case i < 7+(9*1):
			m[1] = append(m[1], i-7)
		case i < 7+(9*2):
			m[2] = append(m[2], i-7-(9*1))
		case i < 7+(9*3):
			m[3] = append(m[3], i-7-(9*2))
		}
	}
	return m
}

func (m SuitsGroupedHand) Solve() bool {
	return m.valid_mod3() && m.valid_33332()
}

func (m SuitsGroupedHand) valid_mod3() bool {
	//スートのサイズを3で割った時
	//あまりが2であるすーとグループが1つであること
	c := 0
	for _, a := range m {
		switch len(a) % 3 {
		case 0:
			// noop
		case 1:
			return false
		case 2:
			c++
		}
	}
	return c == 1
}

func (m SuitsGroupedHand) valid_33332() bool {
	for i := 0; i < 4; i++ {
		if !valid_suit_group(m[i], i) {
			return false
		}
	}
	return true
}

func valid_suit_group(a Hand, i int) bool {
	// 第二引数は字牌のとき0

	//ソート
	sort.Ints(a)
	if len(a)%3 == 2 {
		//ペアを探す
		pair_numbers := pairable_numbers(a)
		//ペア候補がなかったらぬける
		if len(pair_numbers) == 0 {
			return false
		}
		//ペア候補毎に繰り返し処理
		for _, v := range pair_numbers {
			//ペアとなる２枚を除去
			rest := Hand{}
			c := 2
			for _, w := range a {
				// ペア候補以外は新スライスに入れる
				// ペア候補は３枚目以降は新スライスに入れる
				if w != v || c <= 0 {
					rest = append(rest, w)
				}
				if w == v {
					c--
				}
			}
			if valid_3cards(rest, i) {
				return true
			} else {
				continue
			}
		}
		return false
	} else if len(a)%3 == 0 {
		return valid_3cards(a, i)
	}
	// 到達しないはず
	panic("到達しないはず")
}

func valid_3cards(a Hand, i int) bool {
	// 刻子や順子のみで構成されている場合true
	// a is sorted
	// a.size % 3 is0
	// 第二引数は字牌のとき0
	ok := false
	for {
		a, ok = remove_kotsu(a)
		if ok {
			continue
		}
		if i > 0 {
			a, ok = remove_shuntsu(a)
			if ok {
				continue
			}
		}
		return len(a) == 0
	}
}

func remove_kotsu(a Hand) (Hand, bool) {
	// 刻子を除去できればtrue
	// a is sorted
	retval := a
	if len(a) < 3 {
		return retval, false
	}
	if a[0] == a[1] && a[0] == a[2] {
		retval = a[3:]
		return retval, true
	}
	return retval, false
}

func remove_shuntsu(a Hand) (Hand, bool) {
	// 順子を除去できればtrue
	// a is sorted
	rest := Hand{}
	first := -1
	second := -1
	found := false
	for _, v := range a {
		if found {
			rest = append(rest, v)
			continue
		}
		if first == -1 {
			first = v
		} else if second == -1 && first+1 == v {
			second = v
		} else if second != -1 && first+2 == v {
			//flush
			first = -1
			second = -1
			found = true
		} else {
			rest = append(rest, v)
		}
	}
	return rest, found
}

func pairable_numbers(sorted Hand) Hand {
	counter := []int{}
	a := 999 // 2つ前
	b := 999 // 1つ前
	for _, v := range sorted {
		if b == v && a != v {
			counter = append(counter, v)
		} else {
			b = v
		}
	}
	return counter
}

func is_chitoitsu(hand Hand) bool {
	//カウンタ
	c := map[int]int{}

	for _, v := range hand {
		count, ok := c[v]
		if ok {
			if count == 1 {
				c[v] = 2
			} else {
				// c[v] == 2
				return false
			}
		} else {
			c[v] = 1
		}
		//8個チェック
		if len(c) >= 8 {
			return false
		}
	}
	return true
}
<|endoftext|>


<commit_before>package lnwallet

import (
	"github.com/roasbeef/btcd/btcec"
	"github.com/roasbeef/btcd/wire"
	"github.com/roasbeef/btcutil"
)

// WalletController defines an abstract interface for controlling a local Pure
// Go wallet, a local or remote wallet via an RPC mechanism, or possibly even
// a daemon assisted hardware wallet. This interface serves the purpose of
// allowing LightningWallet to be seamlessly compatible with several wallets
// such as: uspv, btcwallet, Bitcoin Core, Electrum, etc. This interface then
// serves as a "base wallet", with Lightning Network awareness taking place at
// a "higher" level of abstraction. Essentially, an overlay wallet.
// Implementors of this interface must closely adhere to the documented
// behavior of all interface methods in order to ensure identical behavior
// across all concrete implementations.
type WalletController interface {
	// ConfirmedBalance returns the sum of all the wallet's unspent outputs
	// that have at least confs confirmations. If confs is set to zero,
	// then all unspent outputs, including those currently in the mempool
	// will be included in the final sum.
	ConfirmedBalance(confs int32) btcutil.Amount

	// NewAddress returns the next external address for the wallet. The
	// type of address returned is dictated by the wallet's capabilities,
	// and may be of type: p2sh, p2pkh, p2wkh, p2wsh, etc.
	NewAddress(witness bool) (btcutil.Address, error)

	// NewChangeAddress returns a new change address for the wallet. If the
	// underlying wallet supports hd key chains, then this address should be
	// dervied from an internal branch.
	NewChangeAddress(witness bool) (btcutil.Address, error)

	// GetPrivKey retrives the underlying private key associated with the
	// passed address. If the wallet is unable to locate this private key
	// due to the address not being under control of the wallet, then an
	// error should be returned.
	GetPrivKey(a *btcutil.Address) (*btcec.PrivateKey, error)

	// NewRawKey returns a raw private key controlled by the wallet. These
	// keys are used for the 2-of-2 multi-sig outputs for funding
	// transactions, as well as the pub key used for commitment transactions.
	// TODO(roasbeef): key pool due to cancelled reservations??
	NewRawKey() (*btcec.PrivateKey, error)

	// FetchIdentityKey returns a private key which will be utilized as the
	// wallet's Lightning Network identity for authentication purposes.
	// TODO(roasbeef): rotate identity key?
	FetchIdentityKey() (*btcec.PrivateKey, error)

	// FundTransaction creates a new unsigned transactions paying to the
	// passed outputs, possibly using the specified change address. The
	// includeFee parameter dictates if the wallet should also provide
	// enough the funds necessary to create an adequate fee or not.
	FundTransaction(outputs []*wire.TxOut, changeAddr btcutil.Address,
		includeFee bool) (*wire.MsgTx, error)

	// SignTransaction performs potentially a sparse, or full signing of
	// all inputs within the passed transaction that are spendable by the
	// wallet.
	SignTransaction(tx *wire.MsgTx) error

	// BroadcastTransaction performs cursory validation (dust checks, etc),
	// then finally broadcasts the passed transaction to the Bitcoin network.
	BroadcastTransaction(tx *wire.MsgTx) error

	// SendMany funds, signs, and broadcasts a Bitcoin transaction paying
	// out to the specified outputs. In the case the wallet has insufficient
	// funds, or the outputs are non-standard, and error should be returned.
	SendMany(outputs []*wire.TxOut) (*wire.ShaHash, error)

	// ListUnspentWitness returns all unspent outputs which are version 0
	// witness programs. The 'confirms' parameter indicates the minimum
	// number of confirmations an output needs in order to be returned by
	// this method. Passing -1 as 'confirms' indicates that even unconfirmed
	// outputs should be returned.
	ListUnspentWitness(confirms int32) ([]*wire.OutPoint, error)

	// LockOutpoint marks an outpoint as locked meaning it will no longer
	// be deemed as eligble for coin selection. Locking outputs are utilized
	// in order to avoid race conditions when selecting inputs for usage when
	// funding a channel.
	LockOutpoint(o wire.OutPoint)

	// UnlockOutpoint unlocks an previously locked output, marking it
	// eligible for coin seleciton.
	UnlockOutpoint(o wire.OutPoint)

	// ImportScript imports the serialize public key script, or redeem
	// script into the wallet's database. Scripts to be imported include
	// the 2-of-2 script for funding transactions, commitment scripts,
	// HTLCs scripts, and so on.
	ImportScript(b []byte) error

	// Start initializes the wallet, making any neccessary connections,
	// starting up required goroutines etc.
	Start() error

	// Stop signals the wallet for shutdown. Shutdown may entail closing
	// any active sockets, database handles, stopping goroutines, etc.
	Stop() error

	// WaitForShutdown blocks until the wallet finishes the shutdown
	// procedure triggered by a prior call to Stop().
	WaitForShutdown() error

	// TODO(roasbeef): ImportPriv?
	//  * segwitty flag?
}
<commit_msg>lnwallet: revamp interfaces, add BlockChainIO and Singer<commit_after>package lnwallet

import (
	"errors"
	"fmt"
	"sync"

	"github.com/roasbeef/btcd/btcec"
	"github.com/roasbeef/btcd/txscript"
	"github.com/roasbeef/btcd/wire"
	"github.com/roasbeef/btcutil"
)

// ErrNotMine is an error denoting that a WalletController instance is unable
// to spend a specifid output.
var ErrNotMine = errors.New("the passed output doesn't belong to the wallet")

// AddressType is a enum-like type which denotes the possible address types
// WalletController supports.
type AddressType uint8

const (
	// WitnessPubKey represents a p2wkh address.
	WitnessPubKey AddressType = iota

	// NestedWitnessPubKey represents a p2sh output which is itself a
	// nested p2wkh output.
	NestedWitnessPubKey

	// PublicKey represents a regular p2pkh output.
	PubKeyHash
)

// Utxo is an unspent output denoted by its outpoint, and output value of the
// original output.
type Utxo struct {
	Value btcutil.Amount
	wire.OutPoint
}

// WalletController defines an abstract interface for controlling a local Pure
// Go wallet, a local or remote wallet via an RPC mechanism, or possibly even
// a daemon assisted hardware wallet. This interface serves the purpose of
// allowing LightningWallet to be seamlessly compatible with several wallets
// such as: uspv, btcwallet, Bitcoin Core, Electrum, etc. This interface then
// serves as a "base wallet", with Lightning Network awareness taking place at
// a "higher" level of abstraction. Essentially, an overlay wallet.
// Implementors of this interface must closely adhere to the documented
// behavior of all interface methods in order to ensure identical behavior
// across all concrete implementations.
type WalletController interface {
	// FetchInputInfo queries for the WalletController's knowledge of the
	// passed outpoint. If the base wallet determines this output is under
	// its control, then the original txout should be returned. Otherwise,
	// a non-nil error value of ErrNotMine should be returned instead.
	FetchInputInfo(prevOut *wire.OutPoint) (*wire.TxOut, error)

	// ConfirmedBalance returns the sum of all the wallet's unspent outputs
	// that have at least confs confirmations. If confs is set to zero,
	// then all unspent outputs, including those currently in the mempool
	// will be included in the final sum.
	ConfirmedBalance(confs int32, witness bool) (btcutil.Amount, error)

	// NewAddress returns the next external or internal address for the
	// wallet dicatated by the value of the `change` paramter. If change is
	// true, then an internal address should be used, otherwise an external
	// address should be returned. The type of address returned is dictated
	// by the wallet's capabilities, and may be of type: p2sh, p2pkh,
	// p2wkh, p2wsh, etc.
	NewAddress(addrType AddressType, change bool) (btcutil.Address, error)

	// GetPrivKey retrives the underlying private key associated with the
	// passed address. If the wallet is unable to locate this private key
	// due to the address not being under control of the wallet, then an
	// error should be returned.
	// TODO(roasbeef): should instead take tadge's derivation scheme in
	GetPrivKey(a btcutil.Address) (*btcec.PrivateKey, error)

	// NewRawKey returns a raw private key controlled by the wallet. These
	// keys are used for the 2-of-2 multi-sig outputs for funding
	// transactions, as well as the pub key used for commitment transactions.
	// TODO(roasbeef): may be scrapped, see above TODO
	NewRawKey() (*btcec.PublicKey, error)

	// FetchRootKey returns a root key which will be used by the
	// LightningWallet to deterministically generate secrets. The private
	// key returned by this method should remain constant in-between
	// WalletController restarts.
	FetchRootKey() (*btcec.PrivateKey, error)

	// SendOutputs funds, signs, and broadcasts a Bitcoin transaction
	// paying out to the specified outputs. In the case the wallet has
	// insufficient funds, or the outputs are non-standard, and error
	// should be returned.
	SendOutputs(outputs []*wire.TxOut) (*wire.ShaHash, error)

	// ListUnspentWitness returns all unspent outputs which are version 0
	// witness programs. The 'confirms' parameter indicates the minimum
	// number of confirmations an output needs in order to be returned by
	// this method. Passing -1 as 'confirms' indicates that even
	// unconfirmed outputs should be returned.
	ListUnspentWitness(confirms int32) ([]*Utxo, error)

	// LockOutpoint marks an outpoint as locked meaning it will no longer
	// be deemed as eligible for coin selection. Locking outputs are
	// utilized in order to avoid race conditions when selecting inputs for
	// usage when funding a channel.
	LockOutpoint(o wire.OutPoint)

	// UnlockOutpoint unlocks an previously locked output, marking it
	// eligible for coin seleciton.
	UnlockOutpoint(o wire.OutPoint)

	// PublishTransaction performs cursory validation (dust checks, etc),
	// then finally broadcasts the passed transaction to the Bitcoin network.
	PublishTransaction(tx *wire.MsgTx) error

	// Start initializes the wallet, making any neccessary connections,
	// starting up required goroutines etc.
	Start() error

	// Stop signals the wallet for shutdown. Shutdown may entail closing
	// any active sockets, database handles, stopping goroutines, etc.
	Stop() error
}

// BlockChainIO is a dedicated source which will be used to obtain queries
// related to the current state of the blockchain. The data returned by each of
// the defined methods within this interface should always return the most up
// to date data possible.
//
// TODO(roasbeef): move to diff package perhaps?
type BlockChainIO interface {
	// GetCurrentHeight returns the current height of the valid most-work
	// chain the implementation is aware of.
	GetCurrentHeight() (int32, error)

	// GetTxOut returns the original output referenced by the passed
	// outpoint.
	GetUtxo(txid *wire.ShaHash, index uint32) (*wire.TxOut, error)

	// GetTransaction returns the full transaction identified by the passed
	// transaction ID.
	GetTransaction(txid *wire.ShaHash) (*wire.MsgTx, error)
}

// SignDescriptor houses the necessary information required to succesfully sign
// a given output. This struct is used by the Signer interface in order to gain
// access to critial data needed to generate a valid signature.
type SignDescriptor struct {
	// Pubkey is the public key to which the signature should be generated
	// over. The Signer should then generate a signature with the private
	// key corresponding to this public key.
	PubKey *btcec.PublicKey

	// RedeemScript is the full script required to properly redeem the
	// output. This field will only be populated if a p2wsh or a p2sh
	// output is being signed.
	RedeemScript []byte

	// Output is the target output which should be signed. The PkScript and
	// Value fields within the output should be properly populated,
	// otherwise an invalid signature may be generated.
	Output *wire.TxOut

	// HashType is the target sighash type that should be used when
	// generating the final sighash, and signature.
	HashType txscript.SigHashType

	// SigHashes is the pre-computed sighash midstate to be used when
	// generating the final sighash for signing.
	SigHashes *txscript.TxSigHashes

	// InputIndex is the target input within the transaction that should be
	// signed.
	InputIndex int
}

// Signer represents an abstract object capable of generating raw signatures as
// well as full complete input scripts given a valid SignDescriptor and
// transaction. This interface fully abstracts away signing paving the way for
// Signer implementations such as hardware wallets, hardware tokens, HSM's, or
// simply a regular wallet.
type Signer interface {
	// SignOutputRaw generates a signature for the passed transaction
	// according to the data within the passed SignDescriptor.
	//
	// NOTE: The resulting signature should be void of a sighash byte.
	SignOutputRaw(tx *wire.MsgTx, signDesc *SignDescriptor) ([]byte, error)

	// ComputeInputScript generates a complete InputIndex for the passed
	// transaction with the signature as defined within the passed
	// SignDescriptor. This method should be capable of generating the
	// proper input script for both regular p2wkh output and p2wkh outputs
	// nested within a regualr p2sh output.
	ComputeInputScript(tx *wire.MsgTx, signDesc *SignDescriptor) (*InputScript, error)
}

// WalletDriver represents a "driver" for a particular concrete
// WalletController implementation. A driver is indentified by a globally
// unique string identifier along with a 'New()' method which is responsible
// for initializing a particular WalletController concrete implementation.
type WalletDriver struct {
	// WalletType is a string which uniquely identifes the WalletController
	// that this driver, drives.
	WalletType string

	// New creates a new instance of a concrete WalletController
	// implementation given a variadic set up arguments. The function takes
	// a varidaic number of interface paramters in order to provide
	// initialization flexibility, thereby accomodating several potential
	// WalletController implementations.
	New func(args ...interface{}) (WalletController, error)
}

var (
	wallets     = make(map[string]*WalletDriver)
	registerMtx sync.Mutex
)

// RegisteredWallets returns a slice of all currently registered notifiers.
//
// NOTE: This function is safe for concurrent access.
func RegisteredWallets() []*WalletDriver {
	registerMtx.Lock()
	defer registerMtx.Unlock()

	registeredWallets := make([]*WalletDriver, 0, len(wallets))
	for _, wallet := range wallets {
		registeredWallets = append(registeredWallets, wallet)
	}

	return registeredWallets
}

// RegisterWallet registers a WalletDriver which is capable of driving a
// concrete WalletController interface. In the case that this driver has
// already been registered, an error is returned.
//
// NOTE: This function is safe for concurrent access.
func RegisterWallet(driver *WalletDriver) error {
	registerMtx.Lock()
	defer registerMtx.Unlock()

	if _, ok := wallets[driver.WalletType]; ok {
		return fmt.Errorf("wallet already registered")
	}

	wallets[driver.WalletType] = driver

	return nil
}

// SupportedWallets returns a slice of strings that represents the walelt
// drivers that have been registered and are therefore supported.
//
// NOTE: This function is safe for concurrent access.
func SupportedWallets() []string {
	registerMtx.Lock()
	defer registerMtx.Unlock()

	supportedWallets := make([]string, 0, len(wallets))
	for walletName := range wallets {
		supportedWallets = append(supportedWallets, walletName)
	}

	return supportedWallets
}
<|endoftext|>


<commit_before>/*
 * This file is part of the KubeVirt project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * Copyright 2018 Red Hat, Inc.
 *
 */

package tests_test

import (
	"encoding/xml"

	"fmt"

	. "github.com/onsi/ginkgo"
	"github.com/onsi/ginkgo/extensions/table"
	. "github.com/onsi/gomega"

	k8sv1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/resource"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

	v1 "kubevirt.io/client-go/api/v1"
	"kubevirt.io/client-go/kubecli"
	"kubevirt.io/kubevirt/pkg/virt-launcher/virtwrap/api"
	"kubevirt.io/kubevirt/tests"
)

var _ = Describe("IOThreads", func() {
	tests.FlagParse()

	virtClient, err := kubecli.GetKubevirtClient()
	tests.PanicOnError(err)

	var vmi *v1.VirtualMachineInstance
	dedicated := true

	BeforeEach(func() {
		tests.BeforeTestCleanup()
		vmi = tests.NewRandomVMIWithEphemeralDisk(tests.ContainerDiskFor(tests.ContainerDiskAlpine))
	})

	Context("IOThreads Policies", func() {

		availableCPUs := tests.GetHighestCPUNumberAmongNodes(virtClient)

		It("Should honor shared ioThreadsPolicy for single disk", func() {
			policy := v1.IOThreadsPolicyShared
			vmi.Spec.Domain.IOThreadsPolicy = &policy

			vmi, err := virtClient.VirtualMachineInstance(tests.NamespaceTestDefault).Create(vmi)
			Expect(err).ToNot(HaveOccurred())

			tests.WaitForSuccessfulVMIStart(vmi)

			getOptions := metav1.GetOptions{}
			var newVMI *v1.VirtualMachineInstance

			newVMI, err = virtClient.VirtualMachineInstance(tests.NamespaceTestDefault).Get(vmi.Name, &getOptions)
			Expect(err).ToNot(HaveOccurred())

			domain, err := tests.GetRunningVirtualMachineInstanceDomainXML(virtClient, vmi)
			Expect(err).ToNot(HaveOccurred())
			domSpec := &api.DomainSpec{}
			Expect(xml.Unmarshal([]byte(domain), domSpec)).To(Succeed())

			expectedIOThreads := 1
			Expect(int(domSpec.IOThreads.IOThreads)).To(Equal(expectedIOThreads))

			Expect(len(newVMI.Spec.Domain.Devices.Disks)).To(Equal(1))
		})

		It("[test_id:864][ref_id:2065] Should honor a mix of shared and dedicated ioThreadsPolicy", func() {
			policy := v1.IOThreadsPolicyShared
			vmi.Spec.Domain.IOThreadsPolicy = &policy

			// The disk that came with the VMI
			vmi.Spec.Domain.Devices.Disks[0].DedicatedIOThread = &dedicated

			tests.AddEphemeralDisk(vmi, "shr1", "virtio", tests.ContainerDiskFor(tests.ContainerDiskCirros))
			tests.AddEphemeralDisk(vmi, "shr2", "virtio", tests.ContainerDiskFor(tests.ContainerDiskCirros))

			By("Creating VMI with 1 dedicated and 2 shared ioThreadPolicies")
			vmi, err := virtClient.VirtualMachineInstance(tests.NamespaceTestDefault).Create(vmi)
			Expect(err).ToNot(HaveOccurred())

			tests.WaitForSuccessfulVMIStart(vmi)

			getOptions := metav1.GetOptions{}
			var newVMI *v1.VirtualMachineInstance

			By("Fetching the VMI from the cluster")
			newVMI, err = virtClient.VirtualMachineInstance(tests.NamespaceTestDefault).Get(vmi.Name, &getOptions)
			Expect(err).ToNot(HaveOccurred())

			By("Fetching the domain XML from the running pod")
			domain, err := tests.GetRunningVirtualMachineInstanceDomainXML(virtClient, vmi)
			Expect(err).ToNot(HaveOccurred())
			domSpec := &api.DomainSpec{}
			Expect(xml.Unmarshal([]byte(domain), domSpec)).To(Succeed())

			By("Verifying the total number of ioThreads")
			expectedIOThreads := 2
			Expect(int(domSpec.IOThreads.IOThreads)).To(Equal(expectedIOThreads))

			By("Ensuring there are the expected number of disks")
			Expect(len(newVMI.Spec.Domain.Devices.Disks)).To(Equal(len(vmi.Spec.Domain.Devices.Disks)))

			By("Verifying the ioThread mapping for disks")
			disk0, err := getDiskByName(domSpec, "disk0")
			Expect(err).ToNot(HaveOccurred())
			disk1, err := getDiskByName(domSpec, "shr1")
			Expect(err).ToNot(HaveOccurred())
			disk2, err := getDiskByName(domSpec, "shr2")
			Expect(err).ToNot(HaveOccurred())

			By("Ensuring the ioThread ID for dedicated disk is unique")
			Expect(*disk1.Driver.IOThread).To(Equal(*disk2.Driver.IOThread))
			By("Ensuring that the ioThread ID's for shared disks are equal")
			Expect(*disk0.Driver.IOThread).ToNot(Equal(*disk1.Driver.IOThread))
		})

		table.DescribeTable("[ref_id:2065] should honor auto ioThreadPolicy", func(numCpus int, expectedIOThreads int) {
			if numCpus > availableCPUs {
				Skip(fmt.Sprintf("Testing environment does not contain a node with required %d CPUs number, the highest detected number is %d", numCpus, availableCPUs))
			}

			policy := v1.IOThreadsPolicyAuto
			vmi.Spec.Domain.IOThreadsPolicy = &policy

			vmi.Spec.Domain.Devices.Disks[0].DedicatedIOThread = &dedicated

			tests.AddEphemeralDisk(vmi, "ded2", "virtio", tests.ContainerDiskFor(tests.ContainerDiskCirros))
			vmi.Spec.Domain.Devices.Disks[1].DedicatedIOThread = &dedicated

			tests.AddEphemeralDisk(vmi, "shr1", "virtio", tests.ContainerDiskFor(tests.ContainerDiskCirros))
			tests.AddEphemeralDisk(vmi, "shr2", "virtio", tests.ContainerDiskFor(tests.ContainerDiskCirros))
			tests.AddEphemeralDisk(vmi, "shr3", "virtio", tests.ContainerDiskFor(tests.ContainerDiskCirros))
			tests.AddEphemeralDisk(vmi, "shr4", "virtio", tests.ContainerDiskFor(tests.ContainerDiskCirros))

			cpuReq := resource.MustParse(fmt.Sprintf("%d", numCpus))
			vmi.Spec.Domain.Resources.Requests[k8sv1.ResourceCPU] = cpuReq

			By("Creating VMI with 2 dedicated and 4 shared ioThreadPolicies")
			vmi, err := virtClient.VirtualMachineInstance(tests.NamespaceTestDefault).Create(vmi)
			Expect(err).ToNot(HaveOccurred())

			tests.WaitForSuccessfulVMIStart(vmi)

			getOptions := metav1.GetOptions{}
			var newVMI *v1.VirtualMachineInstance

			By("Fetching the VMI from the cluster")
			newVMI, err = virtClient.VirtualMachineInstance(tests.NamespaceTestDefault).Get(vmi.Name, &getOptions)
			Expect(err).ToNot(HaveOccurred())

			By("Fetching the domain XML from the running pod")
			domain, err := tests.GetRunningVirtualMachineInstanceDomainXML(virtClient, vmi)
			Expect(err).ToNot(HaveOccurred())
			domSpec := &api.DomainSpec{}
			Expect(xml.Unmarshal([]byte(domain), domSpec)).To(Succeed())

			By("Verifying the total number of ioThreads")
			Expect(int(domSpec.IOThreads.IOThreads)).To(Equal(expectedIOThreads))

			By("Ensuring there are the expected number of disks")
			Expect(len(newVMI.Spec.Domain.Devices.Disks)).To(Equal(len(vmi.Spec.Domain.Devices.Disks)))

			By("Verifying the ioThread mapping for disks")
			disk0, err := getDiskByName(domSpec, "disk0")
			Expect(err).ToNot(HaveOccurred())
			ded2, err := getDiskByName(domSpec, "ded2")
			Expect(err).ToNot(HaveOccurred())
			shr1, err := getDiskByName(domSpec, "shr1")
			Expect(err).ToNot(HaveOccurred())
			shr2, err := getDiskByName(domSpec, "shr2")
			Expect(err).ToNot(HaveOccurred())
			shr3, err := getDiskByName(domSpec, "shr2")
			Expect(err).ToNot(HaveOccurred())
			shr4, err := getDiskByName(domSpec, "shr2")
			Expect(err).ToNot(HaveOccurred())

			// the ioThreads for disks sh1 through shr4 will vary based on how many CPUs there are
			// but we already verified the total number of threads, so we know they're spread out
			// across the proper threadId pool.

			By("Ensuring disk0 has a unique threadId")
			Expect(*disk0.Driver.IOThread).ToNot(Equal(*ded2.Driver.IOThread), "disk0 should have a dedicated ioThread")
			Expect(*disk0.Driver.IOThread).ToNot(Equal(*shr1.Driver.IOThread), "disk0 should have a dedicated ioThread")
			Expect(*disk0.Driver.IOThread).ToNot(Equal(*shr2.Driver.IOThread), "disk0 should have a dedicated ioThread")
			Expect(*disk0.Driver.IOThread).ToNot(Equal(*shr3.Driver.IOThread), "disk0 should have a dedicated ioThread")
			Expect(*disk0.Driver.IOThread).ToNot(Equal(*shr4.Driver.IOThread), "disk0 should have a dedicated ioThread")

			By("Ensuring ded2 has a unique threadId")
			Expect(*ded2.Driver.IOThread).ToNot(Equal(*shr1.Driver.IOThread), "ded2 should have a dedicated ioThread")
			Expect(*ded2.Driver.IOThread).ToNot(Equal(*shr2.Driver.IOThread), "ded2 should have a dedicated ioThread")
			Expect(*ded2.Driver.IOThread).ToNot(Equal(*shr3.Driver.IOThread), "ded2 should have a dedicated ioThread")
			Expect(*ded2.Driver.IOThread).ToNot(Equal(*shr4.Driver.IOThread), "ded2 should have a dedicated ioThread")
		},
			// special case: there's always at least one thread for the shared pool:
			// two dedicated and one shared thread is 3 threads.
			table.Entry("for one CPU", 1, 3),
			table.Entry("[test_id:856] for two CPUs", 2, 4),
			table.Entry("[test_id:856] for three CPUs", 3, 6),
			// there's only 6 threads expected because there's 6 total disks, even
			// though the limit would have supported 8.
			table.Entry("for four CPUs", 4, 6),
		)
	})
})

func getDiskByName(domSpec *api.DomainSpec, diskName string) (*api.Disk, error) {
	for _, disk := range domSpec.Devices.Disks {
		if disk.Alias.Name == diskName {
			return &disk, nil
		}
	}
	return nil, fmt.Errorf("disk device '%s' not found", diskName)
}
<commit_msg>fail iothread tests when there is not enough cpu resources<commit_after>/*
 * This file is part of the KubeVirt project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * Copyright 2018 Red Hat, Inc.
 *
 */

package tests_test

import (
	"encoding/xml"

	"fmt"

	. "github.com/onsi/ginkgo"
	"github.com/onsi/ginkgo/extensions/table"
	. "github.com/onsi/gomega"

	k8sv1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/resource"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

	v1 "kubevirt.io/client-go/api/v1"
	"kubevirt.io/client-go/kubecli"
	"kubevirt.io/kubevirt/pkg/virt-launcher/virtwrap/api"
	"kubevirt.io/kubevirt/tests"
)

var _ = Describe("IOThreads", func() {
	tests.FlagParse()

	virtClient, err := kubecli.GetKubevirtClient()
	tests.PanicOnError(err)

	var vmi *v1.VirtualMachineInstance
	dedicated := true

	BeforeEach(func() {
		tests.BeforeTestCleanup()
		vmi = tests.NewRandomVMIWithEphemeralDisk(tests.ContainerDiskFor(tests.ContainerDiskAlpine))
	})

	Context("IOThreads Policies", func() {

		availableCPUs := tests.GetHighestCPUNumberAmongNodes(virtClient)

		It("Should honor shared ioThreadsPolicy for single disk", func() {
			policy := v1.IOThreadsPolicyShared
			vmi.Spec.Domain.IOThreadsPolicy = &policy

			vmi, err := virtClient.VirtualMachineInstance(tests.NamespaceTestDefault).Create(vmi)
			Expect(err).ToNot(HaveOccurred())

			tests.WaitForSuccessfulVMIStart(vmi)

			getOptions := metav1.GetOptions{}
			var newVMI *v1.VirtualMachineInstance

			newVMI, err = virtClient.VirtualMachineInstance(tests.NamespaceTestDefault).Get(vmi.Name, &getOptions)
			Expect(err).ToNot(HaveOccurred())

			domain, err := tests.GetRunningVirtualMachineInstanceDomainXML(virtClient, vmi)
			Expect(err).ToNot(HaveOccurred())
			domSpec := &api.DomainSpec{}
			Expect(xml.Unmarshal([]byte(domain), domSpec)).To(Succeed())

			expectedIOThreads := 1
			Expect(int(domSpec.IOThreads.IOThreads)).To(Equal(expectedIOThreads))

			Expect(len(newVMI.Spec.Domain.Devices.Disks)).To(Equal(1))
		})

		It("[test_id:864][ref_id:2065] Should honor a mix of shared and dedicated ioThreadsPolicy", func() {
			policy := v1.IOThreadsPolicyShared
			vmi.Spec.Domain.IOThreadsPolicy = &policy

			// The disk that came with the VMI
			vmi.Spec.Domain.Devices.Disks[0].DedicatedIOThread = &dedicated

			tests.AddEphemeralDisk(vmi, "shr1", "virtio", tests.ContainerDiskFor(tests.ContainerDiskCirros))
			tests.AddEphemeralDisk(vmi, "shr2", "virtio", tests.ContainerDiskFor(tests.ContainerDiskCirros))

			By("Creating VMI with 1 dedicated and 2 shared ioThreadPolicies")
			vmi, err := virtClient.VirtualMachineInstance(tests.NamespaceTestDefault).Create(vmi)
			Expect(err).ToNot(HaveOccurred())

			tests.WaitForSuccessfulVMIStart(vmi)

			getOptions := metav1.GetOptions{}
			var newVMI *v1.VirtualMachineInstance

			By("Fetching the VMI from the cluster")
			newVMI, err = virtClient.VirtualMachineInstance(tests.NamespaceTestDefault).Get(vmi.Name, &getOptions)
			Expect(err).ToNot(HaveOccurred())

			By("Fetching the domain XML from the running pod")
			domain, err := tests.GetRunningVirtualMachineInstanceDomainXML(virtClient, vmi)
			Expect(err).ToNot(HaveOccurred())
			domSpec := &api.DomainSpec{}
			Expect(xml.Unmarshal([]byte(domain), domSpec)).To(Succeed())

			By("Verifying the total number of ioThreads")
			expectedIOThreads := 2
			Expect(int(domSpec.IOThreads.IOThreads)).To(Equal(expectedIOThreads))

			By("Ensuring there are the expected number of disks")
			Expect(len(newVMI.Spec.Domain.Devices.Disks)).To(Equal(len(vmi.Spec.Domain.Devices.Disks)))

			By("Verifying the ioThread mapping for disks")
			disk0, err := getDiskByName(domSpec, "disk0")
			Expect(err).ToNot(HaveOccurred())
			disk1, err := getDiskByName(domSpec, "shr1")
			Expect(err).ToNot(HaveOccurred())
			disk2, err := getDiskByName(domSpec, "shr2")
			Expect(err).ToNot(HaveOccurred())

			By("Ensuring the ioThread ID for dedicated disk is unique")
			Expect(*disk1.Driver.IOThread).To(Equal(*disk2.Driver.IOThread))
			By("Ensuring that the ioThread ID's for shared disks are equal")
			Expect(*disk0.Driver.IOThread).ToNot(Equal(*disk1.Driver.IOThread))
		})

		table.DescribeTable("[ref_id:2065] should honor auto ioThreadPolicy", func(numCpus int, expectedIOThreads int) {
			Expect(
				numCpus <= availableCPUs).To(BeTrue(),
				fmt.Sprintf("Testing environment only has nodes with %d CPUs available, but required are %d CPUs", availableCPUs, numCpus),
			)

			policy := v1.IOThreadsPolicyAuto
			vmi.Spec.Domain.IOThreadsPolicy = &policy

			vmi.Spec.Domain.Devices.Disks[0].DedicatedIOThread = &dedicated

			tests.AddEphemeralDisk(vmi, "ded2", "virtio", tests.ContainerDiskFor(tests.ContainerDiskCirros))
			vmi.Spec.Domain.Devices.Disks[1].DedicatedIOThread = &dedicated

			tests.AddEphemeralDisk(vmi, "shr1", "virtio", tests.ContainerDiskFor(tests.ContainerDiskCirros))
			tests.AddEphemeralDisk(vmi, "shr2", "virtio", tests.ContainerDiskFor(tests.ContainerDiskCirros))
			tests.AddEphemeralDisk(vmi, "shr3", "virtio", tests.ContainerDiskFor(tests.ContainerDiskCirros))
			tests.AddEphemeralDisk(vmi, "shr4", "virtio", tests.ContainerDiskFor(tests.ContainerDiskCirros))

			cpuReq := resource.MustParse(fmt.Sprintf("%d", numCpus))
			vmi.Spec.Domain.Resources.Requests[k8sv1.ResourceCPU] = cpuReq

			By("Creating VMI with 2 dedicated and 4 shared ioThreadPolicies")
			vmi, err := virtClient.VirtualMachineInstance(tests.NamespaceTestDefault).Create(vmi)
			Expect(err).ToNot(HaveOccurred())

			tests.WaitForSuccessfulVMIStart(vmi)

			getOptions := metav1.GetOptions{}
			var newVMI *v1.VirtualMachineInstance

			By("Fetching the VMI from the cluster")
			newVMI, err = virtClient.VirtualMachineInstance(tests.NamespaceTestDefault).Get(vmi.Name, &getOptions)
			Expect(err).ToNot(HaveOccurred())

			By("Fetching the domain XML from the running pod")
			domain, err := tests.GetRunningVirtualMachineInstanceDomainXML(virtClient, vmi)
			Expect(err).ToNot(HaveOccurred())
			domSpec := &api.DomainSpec{}
			Expect(xml.Unmarshal([]byte(domain), domSpec)).To(Succeed())

			By("Verifying the total number of ioThreads")
			Expect(int(domSpec.IOThreads.IOThreads)).To(Equal(expectedIOThreads))

			By("Ensuring there are the expected number of disks")
			Expect(len(newVMI.Spec.Domain.Devices.Disks)).To(Equal(len(vmi.Spec.Domain.Devices.Disks)))

			By("Verifying the ioThread mapping for disks")
			disk0, err := getDiskByName(domSpec, "disk0")
			Expect(err).ToNot(HaveOccurred())
			ded2, err := getDiskByName(domSpec, "ded2")
			Expect(err).ToNot(HaveOccurred())
			shr1, err := getDiskByName(domSpec, "shr1")
			Expect(err).ToNot(HaveOccurred())
			shr2, err := getDiskByName(domSpec, "shr2")
			Expect(err).ToNot(HaveOccurred())
			shr3, err := getDiskByName(domSpec, "shr2")
			Expect(err).ToNot(HaveOccurred())
			shr4, err := getDiskByName(domSpec, "shr2")
			Expect(err).ToNot(HaveOccurred())

			// the ioThreads for disks sh1 through shr4 will vary based on how many CPUs there are
			// but we already verified the total number of threads, so we know they're spread out
			// across the proper threadId pool.

			By("Ensuring disk0 has a unique threadId")
			Expect(*disk0.Driver.IOThread).ToNot(Equal(*ded2.Driver.IOThread), "disk0 should have a dedicated ioThread")
			Expect(*disk0.Driver.IOThread).ToNot(Equal(*shr1.Driver.IOThread), "disk0 should have a dedicated ioThread")
			Expect(*disk0.Driver.IOThread).ToNot(Equal(*shr2.Driver.IOThread), "disk0 should have a dedicated ioThread")
			Expect(*disk0.Driver.IOThread).ToNot(Equal(*shr3.Driver.IOThread), "disk0 should have a dedicated ioThread")
			Expect(*disk0.Driver.IOThread).ToNot(Equal(*shr4.Driver.IOThread), "disk0 should have a dedicated ioThread")

			By("Ensuring ded2 has a unique threadId")
			Expect(*ded2.Driver.IOThread).ToNot(Equal(*shr1.Driver.IOThread), "ded2 should have a dedicated ioThread")
			Expect(*ded2.Driver.IOThread).ToNot(Equal(*shr2.Driver.IOThread), "ded2 should have a dedicated ioThread")
			Expect(*ded2.Driver.IOThread).ToNot(Equal(*shr3.Driver.IOThread), "ded2 should have a dedicated ioThread")
			Expect(*ded2.Driver.IOThread).ToNot(Equal(*shr4.Driver.IOThread), "ded2 should have a dedicated ioThread")
		},
			// special case: there's always at least one thread for the shared pool:
			// two dedicated and one shared thread is 3 threads.
			table.Entry("for one CPU", 1, 3),
			table.Entry("[test_id:856] for two CPUs", 2, 4),
			table.Entry("[test_id:856] for three CPUs", 3, 6),
			// there's only 6 threads expected because there's 6 total disks, even
			// though the limit would have supported 8.
			table.Entry("for four CPUs", 4, 6),
		)
	})
})

func getDiskByName(domSpec *api.DomainSpec, diskName string) (*api.Disk, error) {
	for _, disk := range domSpec.Devices.Disks {
		if disk.Alias.Name == diskName {
			return &disk, nil
		}
	}
	return nil, fmt.Errorf("disk device '%s' not found", diskName)
}
<|endoftext|>


<commit_before>package main

import (
	"net/http"
	"encoding/json"
)

func Index(w http.ResponseWriter, _ *http.Request) {
	spaceData, err := readSpaceData()

	if err == nil {
		ReturnJson(w, spaceData)
	} else {
		w.WriteHeader(http.StatusNotFound)
	}
}

func ReturnJson(w http.ResponseWriter, v interface{}) {
	w.Header().Set("Content-Type", "application/json; charset=UTF-8")
	w.WriteHeader(http.StatusOK)
	if err := json.NewEncoder(w).Encode(v); err != nil {
		w.WriteHeader(http.StatusInternalServerError)
		panic(err)
	}
}

func createEntry(i interface{}, w http.ResponseWriter, r *http.Request) {
	if err := json.NewDecoder(r.Body).Decode(i); err != nil {
		w.WriteHeader(422)
		if err := json.NewEncoder(w).Encode(err); err != nil {
			w.WriteHeader(http.StatusBadRequest)
			panic(err)
		}
	} else {
		w.WriteHeader(http.StatusCreated)
		if err := json.NewEncoder(w).Encode(i); err != nil {
			w.WriteHeader(http.StatusBadRequest)
			panic(err)
		}
	}
}<commit_msg>fixes return code<commit_after>package main

import (
	"net/http"
	"encoding/json"
)

func Index(w http.ResponseWriter, _ *http.Request) {
	spaceData, err := readSpaceData()

	if err == nil {
		ReturnJson(w, spaceData)
	} else {
		w.WriteHeader(http.StatusNotFound)
	}
}

func ReturnJson(w http.ResponseWriter, v interface{}) {
	w.Header().Set("Content-Type", "application/json; charset=UTF-8")
	w.WriteHeader(http.StatusOK)
	if err := json.NewEncoder(w).Encode(v); err != nil {
		w.WriteHeader(http.StatusInternalServerError)
		panic(err)
	}
}

func createEntry(i interface{}, w http.ResponseWriter, r *http.Request) {
	if err := json.NewDecoder(r.Body).Decode(i); err != nil {
		w.WriteHeader(422)
		if err := json.NewEncoder(w).Encode(err); err != nil {
			w.WriteHeader(http.StatusBadRequest)
			panic(err)
		}
	} else {
		w.WriteHeader(http.StatusOK)
		if err := json.NewEncoder(w).Encode(i); err != nil {
			w.WriteHeader(http.StatusBadRequest)
			panic(err)
		}
	}
}<|endoftext|>


<commit_before>package command

import (
	"encoding/json"
	"fmt"

	"github.com/codegangsta/cli"
	"github.com/dnaeon/gru/catalog"
)

// NewValidateCommand creates a new sub-command for
// validating module files
func NewValidateCommand() cli.Command {
	cmd := cli.Command{
		Name:   "validate",
		Usage:  "validate module file",
		Action: execValidateCommand,
		Flags: []cli.Flag{
			cli.BoolFlag{
				Name:  "tojson",
				Usage: "serialize the catalog to JSON",
			},
		},
	}

	return cmd
}

// Executes the "validate" command
func execValidateCommand(c *cli.Context) {
	if len(c.Args()) < 1 {
		displayError(errNoModuleName, 64)
	}

	main := c.Args()[0]
	katalog, err := catalog.Load(main, c.GlobalString("modulepath"))
	if err != nil {
		displayError(err, 1)
	}

	// Validate() returns a slice of errors
	resourceErrors := katalog.Validate()
	for _, err = range resourceErrors {
		fmt.Print(err)
	}

	if c.Bool("tojson") {
		data, err := json.MarshalIndent(katalog, "", "  ")
		if err != nil {
			displayError(err, 1)
		}
		fmt.Println(string(data))
	}

}
<commit_msg>gructl validate: do not print the json result if we've got errors<commit_after>package command

import (
	"encoding/json"
	"fmt"

	"github.com/codegangsta/cli"
	"github.com/dnaeon/gru/catalog"
)

// NewValidateCommand creates a new sub-command for
// validating module files
func NewValidateCommand() cli.Command {
	cmd := cli.Command{
		Name:   "validate",
		Usage:  "validate module file",
		Action: execValidateCommand,
		Flags: []cli.Flag{
			cli.BoolFlag{
				Name:  "tojson",
				Usage: "serialize the catalog to JSON",
			},
		},
	}

	return cmd
}

// Executes the "validate" command
func execValidateCommand(c *cli.Context) {
	if len(c.Args()) < 1 {
		displayError(errNoModuleName, 64)
	}

	main := c.Args()[0]
	katalog, err := catalog.Load(main, c.GlobalString("modulepath"))
	if err != nil {
		displayError(err, 1)
	}

	// Validate() returns a slice of errors
	foundErrors := katalog.Validate()
	for _, err = range foundErrors {
		fmt.Println(err)
	}

	if len(foundErrors) > 0 {
		return
	}

	if c.Bool("tojson") {
		data, err := json.MarshalIndent(katalog, "", "  ")
		if err != nil {
			displayError(err, 1)
		}
		fmt.Println(string(data))
	}

}
<|endoftext|>


<commit_before>package timetoken

import (
	"testing"
	"time"

	"github.com/stretchr/testify/assert"
)

func TokenManager(periodicFuncTimesCalled *int) *tokenManager {
	return NewTokenManager(1*time.Millisecond, 3*time.Millisecond, func(id string) {
		*periodicFuncTimesCalled += 1
	})
}

func TestPeriodicFunctionCalledUntilTTL(t *testing.T) {
	t.Parallel()
	periodicFuncTimesCalled := 0
	tm := TokenManager(&periodicFuncTimesCalled)
	tm.Add("id", 3*time.Millisecond)
	time.Sleep(2 * time.Millisecond)
	assert.Equal(t, 2, periodicFuncTimesCalled)
}

func TestPeriodicFunctionCalledEveryPeriodeUntilTTL(t *testing.T) {
	t.Parallel()
	periodicFuncTimesCalled := 0
	tm := TokenManager(&periodicFuncTimesCalled)
	tm.Add("id", 3*time.Millisecond)

	time.Sleep(1 * time.Millisecond)
	assert.Equal(t, 1, periodicFuncTimesCalled)

	time.Sleep(1 * time.Millisecond)
	assert.Equal(t, 2, periodicFuncTimesCalled)
}

func TestPeriodicFunctionNotCalledAfterTTL(t *testing.T) {
	t.Parallel()
	periodicFuncTimesCalled := 0
	tm := TokenManager(&periodicFuncTimesCalled)
	tm.Add("id", 2*time.Millisecond)

	time.Sleep(3 * time.Millisecond)
	assert.Equal(t, 2, periodicFuncTimesCalled)
}
<commit_msg>Add more waiting time to avoid test fails<commit_after>package timetoken

import (
	"testing"
	"time"

	"github.com/stretchr/testify/assert"
)

func TokenManager(periodicFuncTimesCalled *int) *tokenManager {
	return NewTokenManager(1*time.Millisecond, 3*time.Millisecond, func(id string) {
		*periodicFuncTimesCalled += 1
	})
}

func TestPeriodicFunctionCalledUntilTTL(t *testing.T) {
	t.Parallel()
	periodicFuncTimesCalled := 0
	tm := TokenManager(&periodicFuncTimesCalled)
	tm.Add("id", 3*time.Millisecond)
	time.Sleep(2 * time.Millisecond)
	assert.Equal(t, 2, periodicFuncTimesCalled)
}

func TestPeriodicFunctionCalledEveryPeriodeUntilTTL(t *testing.T) {
	t.Parallel()
	periodicFuncTimesCalled := 0
	tm := TokenManager(&periodicFuncTimesCalled)
	tm.Add("id", 3*time.Millisecond)

	time.Sleep(1 * time.Millisecond)
	assert.Equal(t, 1, periodicFuncTimesCalled)

	time.Sleep(1 * time.Millisecond)
	assert.Equal(t, 2, periodicFuncTimesCalled)
}

func TestPeriodicFunctionNotCalledAfterTTL(t *testing.T) {
	t.Parallel()
	periodicFuncTimesCalled := 0
	tm := TokenManager(&periodicFuncTimesCalled)
	tm.Add("id", 2*time.Millisecond)

	time.Sleep(4 * time.Millisecond)
	assert.Equal(t, 2, periodicFuncTimesCalled)
}
<|endoftext|>


<commit_before>package network

import (
	"fmt"
	"github.com/Sirupsen/logrus"
	"gitlab.com/gitlab-org/gitlab-ci-multi-runner/common"
	"gitlab.com/gitlab-org/gitlab-ci-multi-runner/helpers"
	"io"
	"mime/multipart"
	"net/http"
	"os"
	"path/filepath"
	"runtime"
)

const clientError = -100

type GitLabClient struct {
	clients map[string]*client
}

func (n *GitLabClient) getClient(runner common.RunnerCredentials) (c *client, err error) {
	if n.clients == nil {
		n.clients = make(map[string]*client)
	}
	key := fmt.Sprintf("%s_%s", runner.URL, runner.TLSCAFile)
	c = n.clients[key]
	if c == nil {
		c, err = newClient(runner)
		if err != nil {
			return
		}
		n.clients[key] = c
	}
	return
}

func (n *GitLabClient) getRunnerVersion(config common.RunnerConfig) common.VersionInfo {
	info := common.VersionInfo{
		Name:         common.NAME,
		Version:      common.VERSION,
		Revision:     common.REVISION,
		Platform:     runtime.GOOS,
		Architecture: runtime.GOARCH,
		Executor:     config.Executor,
	}

	if executor := common.GetExecutor(config.Executor); executor != nil {
		executor.GetFeatures(&info.Features)
	}

	if shell := common.GetShell(config.Shell); shell != nil {
		shell.GetFeatures(&info.Features)
	}

	return info
}

func (n *GitLabClient) doRaw(runner common.RunnerCredentials, method, uri string, request io.Reader, requestType string, headers http.Header) (res *http.Response, err error) {
	c, err := n.getClient(runner)
	if err != nil {
		return nil, err
	}

	return c.do(uri, method, request, requestType, headers)
}

func (n *GitLabClient) doJSON(runner common.RunnerCredentials, method, uri string, statusCode int, request interface{}, response interface{}) (int, string, string) {
	c, err := n.getClient(runner)
	if err != nil {
		return clientError, err.Error(), ""
	}

	return c.doJSON(uri, method, statusCode, request, response)
}

func (n *GitLabClient) GetBuild(config common.RunnerConfig) (*common.GetBuildResponse, bool) {
	request := common.GetBuildRequest{
		Info:  n.getRunnerVersion(config),
		Token: config.Token,
	}

	var response common.GetBuildResponse
	result, statusText, certificates := n.doJSON(config.RunnerCredentials, "POST", "builds/register.json", 201, &request, &response)

	switch result {
	case 201:
		config.Log().Println("Checking for builds...", "received")
		response.TLSCAChain = certificates
		return &response, true
	case 403:
		config.Log().Errorln("Checking for builds...", "forbidden")
		return nil, false
	case 204, 404:
		config.Log().Debugln("Checking for builds...", "nothing")
		return nil, true
	case clientError:
		config.Log().WithField("status", statusText).Errorln("Checking for builds...", "error")
		return nil, false
	default:
		config.Log().WithField("status", statusText).Warningln("Checking for builds...", "failed")
		return nil, true
	}
}

func (n *GitLabClient) RegisterRunner(runner common.RunnerCredentials, description, tags string) *common.RegisterRunnerResponse {
	// TODO: pass executor
	request := common.RegisterRunnerRequest{
		Info:        n.getRunnerVersion(common.RunnerConfig{}),
		Token:       runner.Token,
		Description: description,
		Tags:        tags,
	}

	var response common.RegisterRunnerResponse
	result, statusText, _ := n.doJSON(runner, "POST", "runners/register.json", 201, &request, &response)

	switch result {
	case 201:
		runner.Log().Println("Registering runner...", "succeeded")
		return &response
	case 403:
		runner.Log().Errorln("Registering runner...", "forbidden (check registration token)")
		return nil
	case clientError:
		runner.Log().WithField("status", statusText).Errorln("Registering runner...", "error")
		return nil
	default:
		runner.Log().WithField("status", statusText).Errorln("Registering runner...", "failed")
		return nil
	}
}

func (n *GitLabClient) DeleteRunner(runner common.RunnerCredentials) bool {
	request := common.DeleteRunnerRequest{
		Token: runner.Token,
	}

	result, statusText, _ := n.doJSON(runner, "DELETE", "runners/delete", 200, &request, nil)

	switch result {
	case 200:
		runner.Log().Println("Deleting runner...", "succeeded")
		return true
	case 403:
		runner.Log().Errorln("Deleting runner...", "forbidden")
		return false
	case clientError:
		runner.Log().WithField("status", statusText).Errorln("Deleting runner...", "error")
		return false
	default:
		runner.Log().WithField("status", statusText).Errorln("Deleting runner...", "failed")
		return false
	}
}

func (n *GitLabClient) VerifyRunner(runner common.RunnerCredentials) bool {
	request := common.VerifyRunnerRequest{
		Token: runner.Token,
	}

	// HACK: we use non-existing build id to check if receive forbidden or not found
	result, statusText, _ := n.doJSON(runner, "PUT", fmt.Sprintf("builds/%d", -1), 200, &request, nil)

	switch result {
	case 404:
		// this is expected due to fact that we ask for non-existing job
		runner.Log().Println("Verifying runner...", "is alive")
		return true
	case 403:
		runner.Log().Errorln("Verifying runner...", "is removed")
		return false
	case clientError:
		runner.Log().WithField("status", statusText).Errorln("Verifying runner...", "error")
		return false
	default:
		runner.Log().WithField("status", statusText).Errorln("Verifying runner...", "failed")
		return true
	}
}

func (n *GitLabClient) UpdateBuild(config common.RunnerConfig, id int, state common.BuildState, trace string) common.UpdateState {
	request := common.UpdateBuildRequest{
		Info:  n.getRunnerVersion(config),
		Token: config.Token,
		State: state,
		Trace: trace,
	}

	return n.updateBuild(config, id, request)
}

func (n *GitLabClient) UpdateBuildState(config common.RunnerConfig, id int, state common.BuildState) common.UpdateState {
	request := common.UpdateBuildRequest{
		Info:  n.getRunnerVersion(config),
		Token: config.Token,
		State: state,
	}

	return n.updateBuild(config, id, request)
}

func (n *GitLabClient) updateBuild(config common.RunnerConfig, id int, request common.UpdateBuildRequest) common.UpdateState {
	result, statusText, _ := n.doJSON(config.RunnerCredentials, "PUT", fmt.Sprintf("builds/%d.json", id), 200, &request, nil)
	switch result {
	case 200:
		config.Log().Println(id, "Submitting build to coordinator...", "ok")
		return common.UpdateSucceeded
	case 404:
		config.Log().Warningln(id, "Submitting build to coordinator...", "aborted")
		return common.UpdateAbort
	case 403:
		config.Log().Errorln(id, "Submitting build to coordinator...", "forbidden")
		return common.UpdateAbort
	case clientError:
		config.Log().WithField("status", statusText).Errorln(id, "Submitting build to coordinator...", "error")
		return common.UpdateAbort
	default:
		config.Log().WithField("status", statusText).Warningln(id, "Submitting build to coordinator...", "failed")
		return common.UpdateFailed
	}
}

func (n *GitLabClient) SendTracePart(config common.RunnerConfig, id int, tracePart string) common.UpdateState {
	request := common.SendTracePartRequest{
		Token:     config.Token,
		TracePart: tracePart,
	}

	result, statusText, _ := n.doJSON(config.RunnerCredentials, "PATCH", fmt.Sprintf("builds/%d/trace.txt", id), 200, &request, nil)

	switch result {
	case 200:
		config.Log().Println(id, "Appending trace to coordinator...", "ok")
		return common.UpdateSucceeded
	case 404:
		config.Log().Warningln(id, "Appending trace to coordinator...", "aborted")
		return common.UpdateAbort
	case 403:
		config.Log().Errorln(id, "Appending trace to coordinator...", "forbidden")
		return common.UpdateAbort
	case clientError:
		config.Log().WithField("status", statusText).Errorln(id, "Appending trace to coordinator...", "error")
		return common.UpdateAbort
	default:
		config.Log().WithField("status", statusText).Warningln(id, "Appending trace to coordinator...", "failed")
		return common.UpdateFailed
	}
}

func (n *GitLabClient) createArtifactsForm(mpw *multipart.Writer, reader io.Reader, baseName string) error {
	wr, err := mpw.CreateFormFile("file", baseName)
	if err != nil {
		return err
	}

	_, err = io.Copy(wr, reader)
	if err != nil {
		return err
	}
	return nil
}

func (n *GitLabClient) UploadRawArtifacts(config common.BuildCredentials, reader io.Reader, baseName string) common.UploadState {
	pr, pw := io.Pipe()
	defer pr.Close()

	mpw := multipart.NewWriter(pw)

	go func() {
		defer pw.Close()
		defer mpw.Close()
		err := n.createArtifactsForm(mpw, reader, baseName)
		if err != nil {
			pw.CloseWithError(err)
		}
	}()

	// TODO: Create proper interface for `doRaw` that can use other types than RunnerCredentials
	mappedConfig := common.RunnerCredentials{
		URL:       config.URL,
		Token:     config.Token,
		TLSCAFile: config.TLSCAFile,
	}

	headers := make(http.Header)
	headers.Set("BUILD-TOKEN", config.Token)
	res, err := n.doRaw(mappedConfig, "POST", fmt.Sprintf("builds/%d/artifacts", config.ID), pr, mpw.FormDataContentType(), headers)

	log := logrus.WithFields(logrus.Fields{
		"id":             config.ID,
		"token":          helpers.ShortenToken(config.Token),
		"responseStatus": res.Status,
	})

	if err != nil {
		log.Errorln("Uploading artifacts to coordinator...", "error", err.Error())
		return common.UploadFailed
	}
	defer res.Body.Close()

	switch res.StatusCode {
	case 201:
		log.Println("Uploading artifacts to coordinator...", "ok")
		return common.UploadSucceeded
	case 403:
		log.Errorln("Uploading artifacts to coordinator...", "forbidden")
		return common.UploadForbidden
	case 413:
		log.Errorln("Uploading artifacts to coordinator...", "too large archive")
		return common.UploadTooLarge
	default:
		log.Warningln("Uploading artifacts to coordinator...", "failed", res.Status)
		return common.UploadFailed
	}
}

func (n *GitLabClient) UploadArtifacts(config common.BuildCredentials, artifactsFile string) common.UploadState {
	log := logrus.WithFields(logrus.Fields{
		"id":    config.ID,
		"token": helpers.ShortenToken(config.Token),
	})

	file, err := os.Open(artifactsFile)
	if err != nil {
		log.Errorln("Uploading artifacts to coordinator...", "error", err.Error())
		return common.UploadFailed
	}
	defer file.Close()

	fi, err := file.Stat()
	if err != nil {
		log.Errorln("Uploading artifacts to coordinator...", "error", err.Error())
		return common.UploadFailed
	}
	if fi.IsDir() {
		log.Errorln("Uploading artifacts to coordinator...", "error", "cannot upload directories")
		return common.UploadFailed
	}

	baseName := filepath.Base(artifactsFile)
	return n.UploadRawArtifacts(config, file, baseName)
}

func (n *GitLabClient) DownloadArtifacts(config common.BuildCredentials, artifactsFile string) common.DownloadState {
	// TODO: Create proper interface for `doRaw` that can use other types than RunnerCredentials
	mappedConfig := common.RunnerCredentials{
		URL:       config.URL,
		Token:     config.Token,
		TLSCAFile: config.TLSCAFile,
	}

	headers := make(http.Header)
	headers.Set("BUILD-TOKEN", config.Token)
	res, err := n.doRaw(mappedConfig, "GET", fmt.Sprintf("builds/%d/artifacts", config.ID), nil, "", headers)

	log := logrus.WithFields(logrus.Fields{
		"id":             config.ID,
		"token":          helpers.ShortenToken(config.Token),
		"responseStatus": res.Status,
	})

	if err != nil {
		log.Errorln("Downloading artifacts from coordinator...", "error", err.Error())
		return common.DownloadFailed
	}
	defer res.Body.Close()

	switch res.StatusCode {
	case 200:
		file, err := os.Create(artifactsFile)
		if err == nil {
			defer file.Close()
			_, err = io.Copy(file, res.Body)
		}
		if err != nil {
			file.Close()
			os.Remove(file.Name())
			log.Errorln("Downloading artifacts from coordinator...", "error", err.Error())
			return common.DownloadFailed
		}
		log.Println("Downloading artifacts from coordinator...", "ok")
		return common.DownloadSucceeded
	case 403:
		log.Errorln("Downloading artifacts from coordinator...", "forbidden")
		return common.DownloadForbidden
	case 404:
		log.Errorln("Downloading artifacts from coordinator...", "not found")
		return common.DownloadNotFound
	default:
		log.Warningln("Downloading artifacts from coordinator...", "failed", res.Status)
		return common.DownloadFailed
	}
}

func (n *GitLabClient) ProcessBuild(config common.RunnerConfig, id int) common.BuildTrace {
	trace := newBuildTrace(n, config, id)
	trace.start()
	return trace
}
<commit_msg>Restore removed lines<commit_after>package network

import (
	"fmt"
	"github.com/Sirupsen/logrus"
	"gitlab.com/gitlab-org/gitlab-ci-multi-runner/common"
	"gitlab.com/gitlab-org/gitlab-ci-multi-runner/helpers"
	"io"
	"io/ioutil"
	"mime/multipart"
	"net/http"
	"os"
	"path/filepath"
	"runtime"
)

const clientError = -100

type GitLabClient struct {
	clients map[string]*client
}

func (n *GitLabClient) getClient(runner common.RunnerCredentials) (c *client, err error) {
	if n.clients == nil {
		n.clients = make(map[string]*client)
	}
	key := fmt.Sprintf("%s_%s", runner.URL, runner.TLSCAFile)
	c = n.clients[key]
	if c == nil {
		c, err = newClient(runner)
		if err != nil {
			return
		}
		n.clients[key] = c
	}
	return
}

func (n *GitLabClient) getRunnerVersion(config common.RunnerConfig) common.VersionInfo {
	info := common.VersionInfo{
		Name:         common.NAME,
		Version:      common.VERSION,
		Revision:     common.REVISION,
		Platform:     runtime.GOOS,
		Architecture: runtime.GOARCH,
		Executor:     config.Executor,
	}

	if executor := common.GetExecutor(config.Executor); executor != nil {
		executor.GetFeatures(&info.Features)
	}

	if shell := common.GetShell(config.Shell); shell != nil {
		shell.GetFeatures(&info.Features)
	}

	return info
}

func (n *GitLabClient) doRaw(runner common.RunnerCredentials, method, uri string, request io.Reader, requestType string, headers http.Header) (res *http.Response, err error) {
	c, err := n.getClient(runner)
	if err != nil {
		return nil, err
	}

	return c.do(uri, method, request, requestType, headers)
}

func (n *GitLabClient) doJSON(runner common.RunnerCredentials, method, uri string, statusCode int, request interface{}, response interface{}) (int, string, string) {
	c, err := n.getClient(runner)
	if err != nil {
		return clientError, err.Error(), ""
	}

	return c.doJSON(uri, method, statusCode, request, response)
}

func (n *GitLabClient) GetBuild(config common.RunnerConfig) (*common.GetBuildResponse, bool) {
	request := common.GetBuildRequest{
		Info:  n.getRunnerVersion(config),
		Token: config.Token,
	}

	var response common.GetBuildResponse
	result, statusText, certificates := n.doJSON(config.RunnerCredentials, "POST", "builds/register.json", 201, &request, &response)

	switch result {
	case 201:
		config.Log().Println("Checking for builds...", "received")
		response.TLSCAChain = certificates
		return &response, true
	case 403:
		config.Log().Errorln("Checking for builds...", "forbidden")
		return nil, false
	case 204, 404:
		config.Log().Debugln("Checking for builds...", "nothing")
		return nil, true
	case clientError:
		config.Log().WithField("status", statusText).Errorln("Checking for builds...", "error")
		return nil, false
	default:
		config.Log().WithField("status", statusText).Warningln("Checking for builds...", "failed")
		return nil, true
	}
}

func (n *GitLabClient) RegisterRunner(runner common.RunnerCredentials, description, tags string) *common.RegisterRunnerResponse {
	// TODO: pass executor
	request := common.RegisterRunnerRequest{
		Info:        n.getRunnerVersion(common.RunnerConfig{}),
		Token:       runner.Token,
		Description: description,
		Tags:        tags,
	}

	var response common.RegisterRunnerResponse
	result, statusText, _ := n.doJSON(runner, "POST", "runners/register.json", 201, &request, &response)

	switch result {
	case 201:
		runner.Log().Println("Registering runner...", "succeeded")
		return &response
	case 403:
		runner.Log().Errorln("Registering runner...", "forbidden (check registration token)")
		return nil
	case clientError:
		runner.Log().WithField("status", statusText).Errorln("Registering runner...", "error")
		return nil
	default:
		runner.Log().WithField("status", statusText).Errorln("Registering runner...", "failed")
		return nil
	}
}

func (n *GitLabClient) DeleteRunner(runner common.RunnerCredentials) bool {
	request := common.DeleteRunnerRequest{
		Token: runner.Token,
	}

	result, statusText, _ := n.doJSON(runner, "DELETE", "runners/delete", 200, &request, nil)

	switch result {
	case 200:
		runner.Log().Println("Deleting runner...", "succeeded")
		return true
	case 403:
		runner.Log().Errorln("Deleting runner...", "forbidden")
		return false
	case clientError:
		runner.Log().WithField("status", statusText).Errorln("Deleting runner...", "error")
		return false
	default:
		runner.Log().WithField("status", statusText).Errorln("Deleting runner...", "failed")
		return false
	}
}

func (n *GitLabClient) VerifyRunner(runner common.RunnerCredentials) bool {
	request := common.VerifyRunnerRequest{
		Token: runner.Token,
	}

	// HACK: we use non-existing build id to check if receive forbidden or not found
	result, statusText, _ := n.doJSON(runner, "PUT", fmt.Sprintf("builds/%d", -1), 200, &request, nil)

	switch result {
	case 404:
		// this is expected due to fact that we ask for non-existing job
		runner.Log().Println("Verifying runner...", "is alive")
		return true
	case 403:
		runner.Log().Errorln("Verifying runner...", "is removed")
		return false
	case clientError:
		runner.Log().WithField("status", statusText).Errorln("Verifying runner...", "error")
		return false
	default:
		runner.Log().WithField("status", statusText).Errorln("Verifying runner...", "failed")
		return true
	}
}

func (n *GitLabClient) UpdateBuild(config common.RunnerConfig, id int, state common.BuildState, trace string) common.UpdateState {
	request := common.UpdateBuildRequest{
		Info:  n.getRunnerVersion(config),
		Token: config.Token,
		State: state,
		Trace: trace,
	}

	return n.updateBuild(config, id, request)
}

func (n *GitLabClient) UpdateBuildState(config common.RunnerConfig, id int, state common.BuildState) common.UpdateState {
	request := common.UpdateBuildRequest{
		Info:  n.getRunnerVersion(config),
		Token: config.Token,
		State: state,
	}

	return n.updateBuild(config, id, request)
}

func (n *GitLabClient) updateBuild(config common.RunnerConfig, id int, request common.UpdateBuildRequest) common.UpdateState {
	result, statusText, _ := n.doJSON(config.RunnerCredentials, "PUT", fmt.Sprintf("builds/%d.json", id), 200, &request, nil)
	switch result {
	case 200:
		config.Log().Println(id, "Submitting build to coordinator...", "ok")
		return common.UpdateSucceeded
	case 404:
		config.Log().Warningln(id, "Submitting build to coordinator...", "aborted")
		return common.UpdateAbort
	case 403:
		config.Log().Errorln(id, "Submitting build to coordinator...", "forbidden")
		return common.UpdateAbort
	case clientError:
		config.Log().WithField("status", statusText).Errorln(id, "Submitting build to coordinator...", "error")
		return common.UpdateAbort
	default:
		config.Log().WithField("status", statusText).Warningln(id, "Submitting build to coordinator...", "failed")
		return common.UpdateFailed
	}
}

func (n *GitLabClient) SendTracePart(config common.RunnerConfig, id int, tracePart string) common.UpdateState {
	request := common.SendTracePartRequest{
		Token:     config.Token,
		TracePart: tracePart,
	}

	result, statusText, _ := n.doJSON(config.RunnerCredentials, "PATCH", fmt.Sprintf("builds/%d/trace.txt", id), 200, &request, nil)

	switch result {
	case 200:
		config.Log().Println(id, "Appending trace to coordinator...", "ok")
		return common.UpdateSucceeded
	case 404:
		config.Log().Warningln(id, "Appending trace to coordinator...", "aborted")
		return common.UpdateAbort
	case 403:
		config.Log().Errorln(id, "Appending trace to coordinator...", "forbidden")
		return common.UpdateAbort
	case clientError:
		config.Log().WithField("status", statusText).Errorln(id, "Appending trace to coordinator...", "error")
		return common.UpdateAbort
	default:
		config.Log().WithField("status", statusText).Warningln(id, "Appending trace to coordinator...", "failed")
		return common.UpdateFailed
	}
}

func (n *GitLabClient) createArtifactsForm(mpw *multipart.Writer, reader io.Reader, baseName string) error {
	wr, err := mpw.CreateFormFile("file", baseName)
	if err != nil {
		return err
	}

	_, err = io.Copy(wr, reader)
	if err != nil {
		return err
	}
	return nil
}

func (n *GitLabClient) UploadRawArtifacts(config common.BuildCredentials, reader io.Reader, baseName string) common.UploadState {
	pr, pw := io.Pipe()
	defer pr.Close()

	mpw := multipart.NewWriter(pw)

	go func() {
		defer pw.Close()
		defer mpw.Close()
		err := n.createArtifactsForm(mpw, reader, baseName)
		if err != nil {
			pw.CloseWithError(err)
		}
	}()

	// TODO: Create proper interface for `doRaw` that can use other types than RunnerCredentials
	mappedConfig := common.RunnerCredentials{
		URL:       config.URL,
		Token:     config.Token,
		TLSCAFile: config.TLSCAFile,
	}

	headers := make(http.Header)
	headers.Set("BUILD-TOKEN", config.Token)
	res, err := n.doRaw(mappedConfig, "POST", fmt.Sprintf("builds/%d/artifacts", config.ID), pr, mpw.FormDataContentType(), headers)

	log := logrus.WithFields(logrus.Fields{
		"id":             config.ID,
		"token":          helpers.ShortenToken(config.Token),
		"responseStatus": res.Status,
	})

	if err != nil {
		log.Errorln("Uploading artifacts to coordinator...", "error", err.Error())
		return common.UploadFailed
	}
	defer res.Body.Close()
	defer io.Copy(ioutil.Discard, res.Body)

	switch res.StatusCode {
	case 201:
		log.Println("Uploading artifacts to coordinator...", "ok")
		return common.UploadSucceeded
	case 403:
		log.Errorln("Uploading artifacts to coordinator...", "forbidden")
		return common.UploadForbidden
	case 413:
		log.Errorln("Uploading artifacts to coordinator...", "too large archive")
		return common.UploadTooLarge
	default:
		log.Warningln("Uploading artifacts to coordinator...", "failed", res.Status)
		return common.UploadFailed
	}
}

func (n *GitLabClient) UploadArtifacts(config common.BuildCredentials, artifactsFile string) common.UploadState {
	log := logrus.WithFields(logrus.Fields{
		"id":    config.ID,
		"token": helpers.ShortenToken(config.Token),
	})

	file, err := os.Open(artifactsFile)
	if err != nil {
		log.Errorln("Uploading artifacts to coordinator...", "error", err.Error())
		return common.UploadFailed
	}
	defer file.Close()

	fi, err := file.Stat()
	if err != nil {
		log.Errorln("Uploading artifacts to coordinator...", "error", err.Error())
		return common.UploadFailed
	}
	if fi.IsDir() {
		log.Errorln("Uploading artifacts to coordinator...", "error", "cannot upload directories")
		return common.UploadFailed
	}

	baseName := filepath.Base(artifactsFile)
	return n.UploadRawArtifacts(config, file, baseName)
}

func (n *GitLabClient) DownloadArtifacts(config common.BuildCredentials, artifactsFile string) common.DownloadState {
	// TODO: Create proper interface for `doRaw` that can use other types than RunnerCredentials
	mappedConfig := common.RunnerCredentials{
		URL:       config.URL,
		Token:     config.Token,
		TLSCAFile: config.TLSCAFile,
	}

	headers := make(http.Header)
	headers.Set("BUILD-TOKEN", config.Token)
	res, err := n.doRaw(mappedConfig, "GET", fmt.Sprintf("builds/%d/artifacts", config.ID), nil, "", headers)

	log := logrus.WithFields(logrus.Fields{
		"id":             config.ID,
		"token":          helpers.ShortenToken(config.Token),
		"responseStatus": res.Status,
	})

	if err != nil {
		log.Errorln("Downloading artifacts from coordinator...", "error", err.Error())
		return common.DownloadFailed
	}
	defer res.Body.Close()
	defer io.Copy(ioutil.Discard, res.Body)

	switch res.StatusCode {
	case 200:
		file, err := os.Create(artifactsFile)
		if err == nil {
			defer file.Close()
			_, err = io.Copy(file, res.Body)
		}
		if err != nil {
			file.Close()
			os.Remove(file.Name())
			log.Errorln("Downloading artifacts from coordinator...", "error", err.Error())
			return common.DownloadFailed
		}
		log.Println("Downloading artifacts from coordinator...", "ok")
		return common.DownloadSucceeded
	case 403:
		log.Errorln("Downloading artifacts from coordinator...", "forbidden")
		return common.DownloadForbidden
	case 404:
		log.Errorln("Downloading artifacts from coordinator...", "not found")
		return common.DownloadNotFound
	default:
		log.Warningln("Downloading artifacts from coordinator...", "failed", res.Status)
		return common.DownloadFailed
	}
}

func (n *GitLabClient) ProcessBuild(config common.RunnerConfig, id int) common.BuildTrace {
	trace := newBuildTrace(n, config, id)
	trace.start()
	return trace
}
<|endoftext|>


<commit_before>package main

import(
    "database/sql"
    _ "github.com/go-sql-driver/mysql"
    "encoding/json"
    "os"
    "bufio"
    "strings"
    "fmt"
)

func main() {
    host := os.Args[1]
    port := os.Args[2]
    user := os.Args[3]
    password := os.Args[4]
    database := os.Args[5]
    query := os.Args[6]

    connString := user + ":" + password + "@tcp(" + host + ":" + port + ")/" + database
    db, err := sql.Open("mysql", connString)
    if err != nil {
        panic(err.Error())
    }
    defer db.Close()

    rows, err := db.Query("select * from users")
    fmt.Fprintln(os.Stderr, "query end")
    defer rows.Close()
    if err != nil {
        panic(err.Error())
    }

    columns, err := rows.Columns()
    if err != nil {
        panic(err.Error())
    }
    values := make([]sql.RawBytes, len(columns))
    args := make([]interface{}, len(values))
    for i := range args {
        args[i] = &values[i]
    }

    f := bufio.NewWriter(os.Stdout)
    n := 0
    for rows.Next() {
        err := rows.Scan(args...)
        if err != nil {
            panic(err.Error())
        }

        sep := "{"
        for i, col := range values {
            fmt.Fprint(f, sep)
            sep = ","
            if col == nil {
                fmt.Fprint(f, columns[i])
                fmt.Fprint(f, ":null")
            } else {
                val := strings.Replace(string(col), "\"", "\\\"", -1)
                fmt.Fprint(f, columns[i])
                fmt.Fprint(f, ":\"")
                fmt.Fprint(f, val)
                fmt.Fprint(f, "\"")
            }
        }
        fmt.Fprintln(f, "}")
        n++
        if n % 100000 == 0 {
            fmt.Fprintln(os.Stderr, n, "lines")
        }
    }
}
<commit_msg>use JSON package to unmarshal<commit_after>package main

import(
    "database/sql"
    _ "github.com/go-sql-driver/mysql"
    "encoding/json"
    "os"
    "bufio"
    "fmt"
    "time"
)

func main() {
    if len(os.Args) != 7 {
        errorExit(fmt.Sprintf("%s: wrong number of arguments (%v for %v)", os.Args[0], len(os.Args), 6))
    }
    host := os.Args[1]
    port := os.Args[2]
    user := os.Args[3]
    password := os.Args[4]
    database := os.Args[5]
    query := os.Args[6]

    connString := user + ":" + password + "@tcp(" + host + ":" + port + ")/" + database
    db, err := sql.Open("mysql", connString)
    if err != nil {
        errorExit(err.Error())
    }
    defer db.Close()

    info("[SQL] %s", query)
    rows, err := db.Query(query)
    info("query returned")
    defer rows.Close()
    if err != nil {
        errorExit(err.Error())
    }

    columns, err := rows.Columns()
    if err != nil {
        errorExit(err.Error())
    }
    values := make([]sql.RawBytes, len(columns))
    args := make([]interface{}, len(columns))
    for i := range values {
        args[i] = &values[i]
    }

    f := bufio.NewWriter(os.Stdout)
    rec := make(map[string]interface{})
    n := 0
    for rows.Next() {
        err := rows.Scan(args...)
        if err != nil {
            errorExit(err.Error())
        }

        for i, val := range values {
            rec[columns[i]] = unmarshalValue(val)
        }
        data, err := json.Marshal(rec)
        if err != nil {
            errorExit(err.Error())
        }
        _, err = f.Write(data)
        if err != nil {
            errorExit(err.Error())
        }
        f.WriteString("\n")

        n++
        if n % 100000 == 0 {
            info("read %d records...", n)
        }
    }
    f.Flush()

    info("Total %d records", n)
}

func unmarshalValue(data sql.RawBytes) interface{} {
    if data == nil {
        return nil
    } else {
        // FIXME
        return string(data)
    }
}

func info(format string, values ...interface{}) {
    fmt.Fprintln(os.Stderr, time.Now().String() + ": " + fmt.Sprintf(format, values))
}

func errorExit(msg string) {
    // FIXME
    panic(msg)
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package alert

import (
	"context"
	"log"
	"strings"

	"github.com/knative/test-infra/shared/gcs"
	"github.com/knative/test-infra/tools/monitoring/config"
	"github.com/knative/test-infra/tools/monitoring/log_parser"
	"github.com/knative/test-infra/tools/monitoring/mail"
	"github.com/knative/test-infra/tools/monitoring/mysql"
	"github.com/knative/test-infra/tools/monitoring/prowapi"
	"github.com/knative/test-infra/tools/monitoring/subscriber"
)

// Client holds all the resources required to run alerting
type Client struct {
	*subscriber.Client
	*MailConfig
	db *mysql.DB
}

type MailConfig struct {
	*mail.Config
	Recipients []string
}

// Setup sets up the client required to run alerting workflow
func Setup(psClient *subscriber.Client, db *mysql.DB, mc *MailConfig) *Client {
	return &Client{psClient, mc, db}
}

// RunAlerting start the alerting workflow
func (c *Client) RunAlerting() {
	log.Println("Starting alerting workflow")
	go func() {
		err := c.ReceiveMessageAckAll(context.Background(), c.handleReportMessage)
		if err != nil {
			log.Printf("Failed to retrieve messages due to %v", err)
		}
	}()
}

func (c *Client) handleReportMessage(rmsg *prowapi.ReportMessage) {
	if rmsg.Status == prowapi.SuccessState || rmsg.Status == prowapi.FailureState || rmsg.Status == prowapi.AbortedState {
		config, err := config.ParseDefaultConfig()
		if err != nil {
			log.Printf("Failed to config yaml (%v): %v\n", config, err)
			return
		}

		rmsg.GCSPath = toGcsLink(rmsg.GCSPath)
		blPath, err := gcs.BuildLogPath(rmsg.GCSPath)
		if err != nil {
			log.Printf("Failed to construct build log url from gcs path %s. Error: %v\n", rmsg.GCSPath, err)
			return
		}
		buildLog, err := gcs.ReadURL(context.Background(), blPath)
		if err != nil {
			log.Printf("Failed to read from url %s. Error: %v\n", blPath, err)
			return
		}
		log.Printf("Build Log Content: %s\n", buildLog)

		errorLogs, err := log_parser.ParseLog(buildLog, config.CollectErrorPatterns())
		if err != nil {
			log.Printf("Failed to parse content %v. Error: %v\n", string(buildLog), err)
			return
		}

		log.Printf("Parsed errorLogs: %v\n", errorLogs)

		for _, el := range errorLogs {
			c.handleSingleError(config, rmsg, &el)
		}
	}
}

func (c *Client) handleSingleError(config *config.Config, rmsg *prowapi.ReportMessage, el *mysql.ErrorLog) {
	log.Println("Handling single error")
	var err error

	// Add the PR number if it is a pull request job
	log.Println("Adding Error Log to the table")
	if len(rmsg.Refs) <= 0 || len(rmsg.Refs[0].Pulls) <= 0 {
		err = c.db.AddErrorLog(el.Pattern, el.Msg, rmsg.JobName, 0, rmsg.GCSPath)
	} else {
		err = c.db.AddErrorLog(el.Pattern, el.Msg, rmsg.JobName, rmsg.Refs[0].Pulls[0].Number, rmsg.GCSPath)
	}
	if err != nil {
		log.Printf("Failed to insert error to db %+v\n", err)
		return
	}

	log.Println("Selecting the config")
	sc, noMatchErr := config.Select(el.Pattern, rmsg.JobName)
	if noMatchErr != nil {
		log.Printf("No matching config found for pattern (%v) job(%v): %v", el.Pattern, rmsg.JobName, noMatchErr)
		return
	}

	log.Println("Sending the alert")
	_, err = c.Alert(el.Pattern, sc, c.db)
	if err != nil {
		log.Printf("Failed to Alert %v", err)
	}
}

// TODO(yt3liu): Remove this hack after the gcs path does not contain extra link prefix
func toGcsLink(link string) string {
	return strings.Replace(strings.Replace(link, "https://gubernator.knative.dev/build/", "", 1),
		"https://prow.knative.dev/view/gcs/", "", 1)
}

func (m *MailConfig) sendAlert(c *mailContent) error {
	log.Printf("sending alert...")
	return m.Send(m.Recipients, c.subject(), c.body())
}

// Alert checks alert condition and alerts table and send alert mail conditionally
func (m *MailConfig) Alert(errorPattern string, s *config.SelectedConfig, db *mysql.DB) (bool, error) {
	log.Println("Fetcing error logs")
	errorLogs, err := db.ListErrorLogs(errorPattern, s.Duration())
	if err != nil {
		return false, err
	}

	log.Println("Building Report and checking alert conditions")
	report := newReport(errorLogs)
	if !report.CheckAlertCondition(s) {
		return false, nil
	}

	log.Println("checking if the alert is a fresh alert pattern")
	ok, err := db.IsFreshAlertPattern(errorPattern, s.Duration())
	if err != nil || !ok {
		return false, err
	}

	log.Println("Adding the new alert pattern to the database")
	if err := db.AddAlert(errorPattern); err != nil {
		return false, err
	}

	log.Println("Generating and sending the alert email")
	content := mailContent{*report, errorPattern, s.Hint, s.Duration()}
	err = m.sendAlert(&content)
	return err == nil, err
}
<commit_msg>clean up the log (#1145)<commit_after>/*
Copyright 2019 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package alert

import (
	"context"
	"log"
	"strings"

	"github.com/knative/test-infra/shared/gcs"
	"github.com/knative/test-infra/tools/monitoring/config"
	"github.com/knative/test-infra/tools/monitoring/log_parser"
	"github.com/knative/test-infra/tools/monitoring/mail"
	"github.com/knative/test-infra/tools/monitoring/mysql"
	"github.com/knative/test-infra/tools/monitoring/prowapi"
	"github.com/knative/test-infra/tools/monitoring/subscriber"
)

// Client holds all the resources required to run alerting
type Client struct {
	*subscriber.Client
	*MailConfig
	db *mysql.DB
}

type MailConfig struct {
	*mail.Config
	Recipients []string
}

// Setup sets up the client required to run alerting workflow
func Setup(psClient *subscriber.Client, db *mysql.DB, mc *MailConfig) *Client {
	return &Client{psClient, mc, db}
}

// RunAlerting start the alerting workflow
func (c *Client) RunAlerting() {
	log.Println("Starting alerting workflow")
	go func() {
		err := c.ReceiveMessageAckAll(context.Background(), c.handleReportMessage)
		if err != nil {
			log.Printf("Failed to retrieve messages due to %v", err)
		}
	}()
}

func (c *Client) handleReportMessage(rmsg *prowapi.ReportMessage) {
	if rmsg.Status == prowapi.SuccessState || rmsg.Status == prowapi.FailureState || rmsg.Status == prowapi.AbortedState {
		log.Printf("Received Pubsub message %v\n", rmsg)

		config, err := config.ParseDefaultConfig()
		if err != nil {
			log.Printf("Failed to config yaml (%v): %v\n", config, err)
			return
		}

		rmsg.GCSPath = toGcsLink(rmsg.GCSPath)
		blPath, err := gcs.BuildLogPath(rmsg.GCSPath)
		if err != nil {
			log.Printf("Failed to construct build log url from gcs path %s. Error: %v\n", rmsg.GCSPath, err)
			return
		}
		buildLog, err := gcs.ReadURL(context.Background(), blPath)
		if err != nil {
			log.Printf("Failed to read from url %s. Error: %v\n", blPath, err)
			return
		}

		errorLogs, err := log_parser.ParseLog(buildLog, config.CollectErrorPatterns())
		if err != nil {
			log.Printf("Failed to parse content %v. Error: %v\n", string(buildLog), err)
			return
		}
		log.Printf("Parsed errorLogs: %v\n", errorLogs)

		for _, el := range errorLogs {
			c.handleSingleError(config, rmsg, &el)
		}
	}
}

func (c *Client) handleSingleError(config *config.Config, rmsg *prowapi.ReportMessage, el *mysql.ErrorLog) {
	var err error

	// Add the PR number if it is a pull request job
	log.Println("Adding Error Log to the table")
	if len(rmsg.Refs) <= 0 || len(rmsg.Refs[0].Pulls) <= 0 {
		err = c.db.AddErrorLog(el.Pattern, el.Msg, rmsg.JobName, 0, rmsg.GCSPath)
	} else {
		err = c.db.AddErrorLog(el.Pattern, el.Msg, rmsg.JobName, rmsg.Refs[0].Pulls[0].Number, rmsg.GCSPath)
	}
	if err != nil {
		log.Printf("Failed to insert error to db %+v\n", err)
		return
	}

	log.Println("Selecting the config")
	sc, noMatchErr := config.Select(el.Pattern, rmsg.JobName)
	if noMatchErr != nil {
		log.Printf("No matching config found for pattern (%v) job(%v): %v", el.Pattern, rmsg.JobName, noMatchErr)
		return
	}

	log.Println("Sending the alert")
	_, err = c.Alert(el.Pattern, sc, c.db)
	if err != nil {
		log.Printf("Failed to Alert %v", err)
	}
}

// TODO(yt3liu): Remove this hack after the gcs path does not contain extra link prefix
func toGcsLink(link string) string {
	return strings.Replace(strings.Replace(link, "https://gubernator.knative.dev/build/", "", 1),
		"https://prow.knative.dev/view/gcs/", "", 1)
}

func (m *MailConfig) sendAlert(c *mailContent) error {
	log.Printf("sending alert...")
	return m.Send(m.Recipients, c.subject(), c.body())
}

// Alert checks alert condition and alerts table and send alert mail conditionally
func (m *MailConfig) Alert(errorPattern string, s *config.SelectedConfig, db *mysql.DB) (bool, error) {
	log.Println("Fetching error logs")
	errorLogs, err := db.ListErrorLogs(errorPattern, s.Duration())
	if err != nil {
		return false, err
	}

	log.Println("Building Report and checking alert conditions")
	report := newReport(errorLogs)
	if !report.CheckAlertCondition(s) {
		return false, nil
	}

	log.Println("checking if the alert is a fresh alert pattern")
	ok, err := db.IsFreshAlertPattern(errorPattern, s.Duration())
	if err != nil || !ok {
		return false, err
	}

	log.Println("Adding the new alert pattern to the database")
	if err := db.AddAlert(errorPattern); err != nil {
		return false, err
	}

	log.Println("Generating and sending the alert email")
	content := mailContent{*report, errorPattern, s.Hint, s.Duration()}
	err = m.sendAlert(&content)
	return err == nil, err
}
<|endoftext|>