<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package rsa

import (
	"big"
	"crypto/subtle"
	"io"
	"os"
)

// This file implements encryption and decryption using PKCS#1 v1.5 padding.

// EncryptPKCS1v15 encrypts the given message with RSA and the padding scheme from PKCS#1 v1.5.
// The message must be no longer than the length of the public modulus minus 11 bytes.
// WARNING: use of this function to encrypt plaintexts other than session keys
// is dangerous. Use RSA OAEP in new protocols.
func EncryptPKCS1v15(rand io.Reader, pub *PublicKey, msg []byte) (out []byte, err os.Error) {
	k := (pub.N.BitLen() + 7) / 8
	if len(msg) > k-11 {
		err = MessageTooLongError{}
		return
	}

	// EM = 0x02 || PS || 0x00 || M
	em := make([]byte, k-1)
	em[0] = 2
	ps, mm := em[1:len(em)-len(msg)-1], em[len(em)-len(msg):]
	err = nonZeroRandomBytes(ps, rand)
	if err != nil {
		return
	}
	em[len(em)-len(msg)-1] = 0
	copy(mm, msg)

	m := new(big.Int).SetBytes(em)
	c := encrypt(new(big.Int), pub, m)
	out = c.Bytes()
	return
}

// DecryptPKCS1v15 decrypts a plaintext using RSA and the padding scheme from PKCS#1 v1.5.
// If rand != nil, it uses RSA blinding to avoid timing side-channel attacks.
func DecryptPKCS1v15(rand io.Reader, priv *PrivateKey, ciphertext []byte) (out []byte, err os.Error) {
	valid, out, err := decryptPKCS1v15(rand, priv, ciphertext)
	if err == nil && valid == 0 {
		err = DecryptionError{}
	}

	return
}

// DecryptPKCS1v15SessionKey decrypts a session key using RSA and the padding scheme from PKCS#1 v1.5.
// If rand != nil, it uses RSA blinding to avoid timing side-channel attacks.
// It returns an error if the ciphertext is the wrong length or if the
// ciphertext is greater than the public modulus. Otherwise, no error is
// returned. If the padding is valid, the resulting plaintext message is copied
// into key. Otherwise, key is unchanged. These alternatives occur in constant
// time. It is intended that the user of this function generate a random
// session key beforehand and continue the protocol with the resulting value.
// This will remove any possibility that an attacker can learn any information
// about the plaintext.
// See ``Chosen Ciphertext Attacks Against Protocols Based on the RSA
// Encryption Standard PKCS #1'', Daniel Bleichenbacher, Advances in Cryptology
// (Crypto '98),
func DecryptPKCS1v15SessionKey(rand io.Reader, priv *PrivateKey, ciphertext []byte, key []byte) (err os.Error) {
	k := (priv.N.BitLen() + 7) / 8
	if k-(len(key)+3+8) < 0 {
		err = DecryptionError{}
		return
	}

	valid, msg, err := decryptPKCS1v15(rand, priv, ciphertext)
	if err != nil {
		return
	}

	valid &= subtle.ConstantTimeEq(int32(len(msg)), int32(len(key)))
	subtle.ConstantTimeCopy(valid, key, msg)
	return
}

func decryptPKCS1v15(rand io.Reader, priv *PrivateKey, ciphertext []byte) (valid int, msg []byte, err os.Error) {
	k := (priv.N.BitLen() + 7) / 8
	if k < 11 {
		err = DecryptionError{}
		return
	}

	c := new(big.Int).SetBytes(ciphertext)
	m, err := decrypt(rand, priv, c)
	if err != nil {
		return
	}

	em := leftPad(m.Bytes(), k)
	firstByteIsZero := subtle.ConstantTimeByteEq(em[0], 0)
	secondByteIsTwo := subtle.ConstantTimeByteEq(em[1], 2)

	// The remainder of the plaintext must be a string of non-zero random
	// octets, followed by a 0, followed by the message.
	//   lookingForIndex: 1 iff we are still looking for the zero.
	//   index: the offset of the first zero byte.
	var lookingForIndex, index int
	lookingForIndex = 1

	for i := 2; i < len(em); i++ {
		equals0 := subtle.ConstantTimeByteEq(em[i], 0)
		index = subtle.ConstantTimeSelect(lookingForIndex&equals0, i, index)
		lookingForIndex = subtle.ConstantTimeSelect(equals0, 0, lookingForIndex)
	}

	valid = firstByteIsZero & secondByteIsTwo & (^lookingForIndex & 1)
	msg = em[index+1:]
	return
}

// nonZeroRandomBytes fills the given slice with non-zero random octets.
func nonZeroRandomBytes(s []byte, rand io.Reader) (err os.Error) {
	_, err = io.ReadFull(rand, s)
	if err != nil {
		return
	}

	for i := 0; i < len(s); i++ {
		for s[i] == 0 {
			_, err = rand.Read(s[i : i+1])
			if err != nil {
				return
			}
			// In tests, the PRNG may return all zeros so we do
			// this to break the loop.
			s[i] ^= 0x42
		}
	}

	return
}

// Due to the design of PKCS#1 v1.5, we need to know the exact hash function in
// use. A generic hash.Hash will not do.
type PKCS1v15Hash int

const (
	HashMD5 PKCS1v15Hash = iota
	HashSHA1
	HashSHA256
	HashSHA384
	HashSHA512
	HashMD5SHA1 // combined MD5 and SHA1 hash used for RSA signing in TLS.
)

// These are ASN1 DER structures:
//   DigestInfo ::= SEQUENCE {
//     digestAlgorithm AlgorithmIdentifier,
//     digest OCTET STRING
//   }
// For performance, we don't use the generic ASN1 encoder. Rather, we
// precompute a prefix of the digest value that makes a valid ASN1 DER string
// with the correct contents.
var hashPrefixes = [][]byte{
	// HashMD5
	{0x30, 0x20, 0x30, 0x0c, 0x06, 0x08, 0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d, 0x02, 0x05, 0x05, 0x00, 0x04, 0x10},
	// HashSHA1
	{0x30, 0x21, 0x30, 0x09, 0x06, 0x05, 0x2b, 0x0e, 0x03, 0x02, 0x1a, 0x05, 0x00, 0x04, 0x14},
	// HashSHA256
	{0x30, 0x31, 0x30, 0x0d, 0x06, 0x09, 0x60, 0x86, 0x48, 0x01, 0x65, 0x03, 0x04, 0x02, 0x01, 0x05, 0x00, 0x04, 0x20},
	// HashSHA384
	{0x30, 0x41, 0x30, 0x0d, 0x06, 0x09, 0x60, 0x86, 0x48, 0x01, 0x65, 0x03, 0x04, 0x02, 0x02, 0x05, 0x00, 0x04, 0x30},
	// HashSHA512
	{0x30, 0x51, 0x30, 0x0d, 0x06, 0x09, 0x60, 0x86, 0x48, 0x01, 0x65, 0x03, 0x04, 0x02, 0x03, 0x05, 0x00, 0x04, 0x40},
	// HashMD5SHA1
	{}, // A special TLS case which doesn't use an ASN1 prefix.
}

// SignPKCS1v15 calcuates the signature of hashed using RSASSA-PSS-SIGN from RSA PKCS#1 v1.5.
// Note that hashed must be the result of hashing the input message using the
// given hash function.
func SignPKCS1v15(rand io.Reader, priv *PrivateKey, hash PKCS1v15Hash, hashed []byte) (s []byte, err os.Error) {
	hashLen, prefix, err := pkcs1v15HashInfo(hash, len(hashed))
	if err != nil {
		return
	}

	tLen := len(prefix) + hashLen
	k := (priv.N.BitLen() + 7) / 8
	if k < tLen+11 {
		return nil, MessageTooLongError{}
	}

	// EM = 0x00 || 0x01 || PS || 0x00 || T
	em := make([]byte, k)
	em[1] = 1
	for i := 2; i < k-tLen-1; i++ {
		em[i] = 0xff
	}
	copy(em[k-tLen:k-hashLen], prefix)
	copy(em[k-hashLen:k], hashed)

	m := new(big.Int).SetBytes(em)
	c, err := decrypt(rand, priv, m)
	if err == nil {
		s = c.Bytes()
	}
	return
}

// VerifyPKCS1v15 verifies an RSA PKCS#1 v1.5 signature.
// hashed is the result of hashing the input message using the given hash
// function and sig is the signature. A valid signature is indicated by
// returning a nil error.
func VerifyPKCS1v15(pub *PublicKey, hash PKCS1v15Hash, hashed []byte, sig []byte) (err os.Error) {
	hashLen, prefix, err := pkcs1v15HashInfo(hash, len(hashed))
	if err != nil {
		return
	}

	tLen := len(prefix) + hashLen
	k := (pub.N.BitLen() + 7) / 8
	if k < tLen+11 {
		err = VerificationError{}
		return
	}

	c := new(big.Int).SetBytes(sig)
	m := encrypt(new(big.Int), pub, c)
	em := leftPad(m.Bytes(), k)
	// EM = 0x00 || 0x01 || PS || 0x00 || T

	ok := subtle.ConstantTimeByteEq(em[0], 0)
	ok &= subtle.ConstantTimeByteEq(em[1], 1)
	ok &= subtle.ConstantTimeCompare(em[k-hashLen:k], hashed)
	ok &= subtle.ConstantTimeCompare(em[k-tLen:k-hashLen], prefix)
	ok &= subtle.ConstantTimeByteEq(em[k-tLen-1], 0)

	for i := 2; i < k-tLen-1; i++ {
		ok &= subtle.ConstantTimeByteEq(em[i], 0xff)
	}

	if ok != 1 {
		return VerificationError{}
	}

	return nil
}

func pkcs1v15HashInfo(hash PKCS1v15Hash, inLen int) (hashLen int, prefix []byte, err os.Error) {
	switch hash {
	case HashMD5:
		hashLen = 16
	case HashSHA1:
		hashLen = 20
	case HashSHA256:
		hashLen = 32
	case HashSHA384:
		hashLen = 48
	case HashSHA512:
		hashLen = 64
	case HashMD5SHA1:
		hashLen = 36
	default:
		return 0, nil, os.ErrorString("unknown hash function")
	}

	if inLen != hashLen {
		return 0, nil, os.ErrorString("input must be hashed message")
	}

	prefix = hashPrefixes[int(hash)]
	return
}
<commit_msg>crypto/rsa: correct docstring for SignPKCS1v15.<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package rsa

import (
	"big"
	"crypto/subtle"
	"io"
	"os"
)

// This file implements encryption and decryption using PKCS#1 v1.5 padding.

// EncryptPKCS1v15 encrypts the given message with RSA and the padding scheme from PKCS#1 v1.5.
// The message must be no longer than the length of the public modulus minus 11 bytes.
// WARNING: use of this function to encrypt plaintexts other than session keys
// is dangerous. Use RSA OAEP in new protocols.
func EncryptPKCS1v15(rand io.Reader, pub *PublicKey, msg []byte) (out []byte, err os.Error) {
	k := (pub.N.BitLen() + 7) / 8
	if len(msg) > k-11 {
		err = MessageTooLongError{}
		return
	}

	// EM = 0x02 || PS || 0x00 || M
	em := make([]byte, k-1)
	em[0] = 2
	ps, mm := em[1:len(em)-len(msg)-1], em[len(em)-len(msg):]
	err = nonZeroRandomBytes(ps, rand)
	if err != nil {
		return
	}
	em[len(em)-len(msg)-1] = 0
	copy(mm, msg)

	m := new(big.Int).SetBytes(em)
	c := encrypt(new(big.Int), pub, m)
	out = c.Bytes()
	return
}

// DecryptPKCS1v15 decrypts a plaintext using RSA and the padding scheme from PKCS#1 v1.5.
// If rand != nil, it uses RSA blinding to avoid timing side-channel attacks.
func DecryptPKCS1v15(rand io.Reader, priv *PrivateKey, ciphertext []byte) (out []byte, err os.Error) {
	valid, out, err := decryptPKCS1v15(rand, priv, ciphertext)
	if err == nil && valid == 0 {
		err = DecryptionError{}
	}

	return
}

// DecryptPKCS1v15SessionKey decrypts a session key using RSA and the padding scheme from PKCS#1 v1.5.
// If rand != nil, it uses RSA blinding to avoid timing side-channel attacks.
// It returns an error if the ciphertext is the wrong length or if the
// ciphertext is greater than the public modulus. Otherwise, no error is
// returned. If the padding is valid, the resulting plaintext message is copied
// into key. Otherwise, key is unchanged. These alternatives occur in constant
// time. It is intended that the user of this function generate a random
// session key beforehand and continue the protocol with the resulting value.
// This will remove any possibility that an attacker can learn any information
// about the plaintext.
// See ``Chosen Ciphertext Attacks Against Protocols Based on the RSA
// Encryption Standard PKCS #1'', Daniel Bleichenbacher, Advances in Cryptology
// (Crypto '98),
func DecryptPKCS1v15SessionKey(rand io.Reader, priv *PrivateKey, ciphertext []byte, key []byte) (err os.Error) {
	k := (priv.N.BitLen() + 7) / 8
	if k-(len(key)+3+8) < 0 {
		err = DecryptionError{}
		return
	}

	valid, msg, err := decryptPKCS1v15(rand, priv, ciphertext)
	if err != nil {
		return
	}

	valid &= subtle.ConstantTimeEq(int32(len(msg)), int32(len(key)))
	subtle.ConstantTimeCopy(valid, key, msg)
	return
}

func decryptPKCS1v15(rand io.Reader, priv *PrivateKey, ciphertext []byte) (valid int, msg []byte, err os.Error) {
	k := (priv.N.BitLen() + 7) / 8
	if k < 11 {
		err = DecryptionError{}
		return
	}

	c := new(big.Int).SetBytes(ciphertext)
	m, err := decrypt(rand, priv, c)
	if err != nil {
		return
	}

	em := leftPad(m.Bytes(), k)
	firstByteIsZero := subtle.ConstantTimeByteEq(em[0], 0)
	secondByteIsTwo := subtle.ConstantTimeByteEq(em[1], 2)

	// The remainder of the plaintext must be a string of non-zero random
	// octets, followed by a 0, followed by the message.
	//   lookingForIndex: 1 iff we are still looking for the zero.
	//   index: the offset of the first zero byte.
	var lookingForIndex, index int
	lookingForIndex = 1

	for i := 2; i < len(em); i++ {
		equals0 := subtle.ConstantTimeByteEq(em[i], 0)
		index = subtle.ConstantTimeSelect(lookingForIndex&equals0, i, index)
		lookingForIndex = subtle.ConstantTimeSelect(equals0, 0, lookingForIndex)
	}

	valid = firstByteIsZero & secondByteIsTwo & (^lookingForIndex & 1)
	msg = em[index+1:]
	return
}

// nonZeroRandomBytes fills the given slice with non-zero random octets.
func nonZeroRandomBytes(s []byte, rand io.Reader) (err os.Error) {
	_, err = io.ReadFull(rand, s)
	if err != nil {
		return
	}

	for i := 0; i < len(s); i++ {
		for s[i] == 0 {
			_, err = rand.Read(s[i : i+1])
			if err != nil {
				return
			}
			// In tests, the PRNG may return all zeros so we do
			// this to break the loop.
			s[i] ^= 0x42
		}
	}

	return
}

// Due to the design of PKCS#1 v1.5, we need to know the exact hash function in
// use. A generic hash.Hash will not do.
type PKCS1v15Hash int

const (
	HashMD5 PKCS1v15Hash = iota
	HashSHA1
	HashSHA256
	HashSHA384
	HashSHA512
	HashMD5SHA1 // combined MD5 and SHA1 hash used for RSA signing in TLS.
)

// These are ASN1 DER structures:
//   DigestInfo ::= SEQUENCE {
//     digestAlgorithm AlgorithmIdentifier,
//     digest OCTET STRING
//   }
// For performance, we don't use the generic ASN1 encoder. Rather, we
// precompute a prefix of the digest value that makes a valid ASN1 DER string
// with the correct contents.
var hashPrefixes = [][]byte{
	// HashMD5
	{0x30, 0x20, 0x30, 0x0c, 0x06, 0x08, 0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d, 0x02, 0x05, 0x05, 0x00, 0x04, 0x10},
	// HashSHA1
	{0x30, 0x21, 0x30, 0x09, 0x06, 0x05, 0x2b, 0x0e, 0x03, 0x02, 0x1a, 0x05, 0x00, 0x04, 0x14},
	// HashSHA256
	{0x30, 0x31, 0x30, 0x0d, 0x06, 0x09, 0x60, 0x86, 0x48, 0x01, 0x65, 0x03, 0x04, 0x02, 0x01, 0x05, 0x00, 0x04, 0x20},
	// HashSHA384
	{0x30, 0x41, 0x30, 0x0d, 0x06, 0x09, 0x60, 0x86, 0x48, 0x01, 0x65, 0x03, 0x04, 0x02, 0x02, 0x05, 0x00, 0x04, 0x30},
	// HashSHA512
	{0x30, 0x51, 0x30, 0x0d, 0x06, 0x09, 0x60, 0x86, 0x48, 0x01, 0x65, 0x03, 0x04, 0x02, 0x03, 0x05, 0x00, 0x04, 0x40},
	// HashMD5SHA1
	{}, // A special TLS case which doesn't use an ASN1 prefix.
}

// SignPKCS1v15 calculates the signature of hashed using RSASSA-PKCS1-V1_5-SIGN from RSA PKCS#1 v1.5.
// Note that hashed must be the result of hashing the input message using the
// given hash function.
func SignPKCS1v15(rand io.Reader, priv *PrivateKey, hash PKCS1v15Hash, hashed []byte) (s []byte, err os.Error) {
	hashLen, prefix, err := pkcs1v15HashInfo(hash, len(hashed))
	if err != nil {
		return
	}

	tLen := len(prefix) + hashLen
	k := (priv.N.BitLen() + 7) / 8
	if k < tLen+11 {
		return nil, MessageTooLongError{}
	}

	// EM = 0x00 || 0x01 || PS || 0x00 || T
	em := make([]byte, k)
	em[1] = 1
	for i := 2; i < k-tLen-1; i++ {
		em[i] = 0xff
	}
	copy(em[k-tLen:k-hashLen], prefix)
	copy(em[k-hashLen:k], hashed)

	m := new(big.Int).SetBytes(em)
	c, err := decrypt(rand, priv, m)
	if err == nil {
		s = c.Bytes()
	}
	return
}

// VerifyPKCS1v15 verifies an RSA PKCS#1 v1.5 signature.
// hashed is the result of hashing the input message using the given hash
// function and sig is the signature. A valid signature is indicated by
// returning a nil error.
func VerifyPKCS1v15(pub *PublicKey, hash PKCS1v15Hash, hashed []byte, sig []byte) (err os.Error) {
	hashLen, prefix, err := pkcs1v15HashInfo(hash, len(hashed))
	if err != nil {
		return
	}

	tLen := len(prefix) + hashLen
	k := (pub.N.BitLen() + 7) / 8
	if k < tLen+11 {
		err = VerificationError{}
		return
	}

	c := new(big.Int).SetBytes(sig)
	m := encrypt(new(big.Int), pub, c)
	em := leftPad(m.Bytes(), k)
	// EM = 0x00 || 0x01 || PS || 0x00 || T

	ok := subtle.ConstantTimeByteEq(em[0], 0)
	ok &= subtle.ConstantTimeByteEq(em[1], 1)
	ok &= subtle.ConstantTimeCompare(em[k-hashLen:k], hashed)
	ok &= subtle.ConstantTimeCompare(em[k-tLen:k-hashLen], prefix)
	ok &= subtle.ConstantTimeByteEq(em[k-tLen-1], 0)

	for i := 2; i < k-tLen-1; i++ {
		ok &= subtle.ConstantTimeByteEq(em[i], 0xff)
	}

	if ok != 1 {
		return VerificationError{}
	}

	return nil
}

func pkcs1v15HashInfo(hash PKCS1v15Hash, inLen int) (hashLen int, prefix []byte, err os.Error) {
	switch hash {
	case HashMD5:
		hashLen = 16
	case HashSHA1:
		hashLen = 20
	case HashSHA256:
		hashLen = 32
	case HashSHA384:
		hashLen = 48
	case HashSHA512:
		hashLen = 64
	case HashMD5SHA1:
		hashLen = 36
	default:
		return 0, nil, os.ErrorString("unknown hash function")
	}

	if inLen != hashLen {
		return 0, nil, os.ErrorString("input must be hashed message")
	}

	prefix = hashPrefixes[int(hash)]
	return
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// This file contains the exported entry points for invoking the parser.

package parser

import (
	"bytes"
	"go/ast"
	"go/scanner"
	"go/token"
	"io"
	"io/ioutil"
	"os"
	pathutil "path"
)

// If src != nil, readSource converts src to a []byte if possible;
// otherwise it returns an error. If src == nil, readSource returns
// the result of reading the file specified by filename.
//
func readSource(filename string, src interface{}) ([]byte, os.Error) {
	if src != nil {
		switch s := src.(type) {
		case string:
			return []byte(s), nil
		case []byte:
			return s, nil
		case *bytes.Buffer:
			// is io.Reader, but src is already available in []byte form
			if s != nil {
				return s.Bytes(), nil
			}
		case io.Reader:
			var buf bytes.Buffer
			_, err := io.Copy(&buf, s)
			if err != nil {
				return nil, err
			}
			return buf.Bytes(), nil
		default:
			return nil, os.ErrorString("invalid source")
		}
	}

	return ioutil.ReadFile(filename)
}

func (p *parser) parseEOF() os.Error {
	p.expect(token.EOF)
	return p.GetError(scanner.Sorted)
}

// ParseExpr parses a Go expression and returns the corresponding
// AST node. The filename, src, and scope arguments have the same interpretation
// as for ParseFile. If there is an error, the result expression
// may be nil or contain a partial AST.
//
func ParseExpr(filename string, src interface{}, scope *ast.Scope) (ast.Expr, os.Error) {
	data, err := readSource(filename, src)
	if err != nil {
		return nil, err
	}

	var p parser
	p.init(filename, data, scope, 0)
	return p.parseExpr(), p.parseEOF()
}

// ParseStmtList parses a list of Go statements and returns the list
// of corresponding AST nodes. The filename, src, and scope arguments have the same
// interpretation as for ParseFile. If there is an error, the node
// list may be nil or contain partial ASTs.
//
func ParseStmtList(filename string, src interface{}, scope *ast.Scope) ([]ast.Stmt, os.Error) {
	data, err := readSource(filename, src)
	if err != nil {
		return nil, err
	}

	var p parser
	p.init(filename, data, scope, 0)
	return p.parseStmtList(), p.parseEOF()
}

// ParseDeclList parses a list of Go declarations and returns the list
// of corresponding AST nodes.  The filename, src, and scope arguments have the same
// interpretation as for ParseFile. If there is an error, the node
// list may be nil or contain partial ASTs.
//
func ParseDeclList(filename string, src interface{}, scope *ast.Scope) ([]ast.Decl, os.Error) {
	data, err := readSource(filename, src)
	if err != nil {
		return nil, err
	}

	var p parser
	p.init(filename, data, scope, 0)
	return p.parseDeclList(), p.parseEOF()
}

// ParseFile parses a Go source file and returns a File node.
//
// If src != nil, ParseFile parses the file source from src. src may
// be provided in a variety of formats. At the moment the following types
// are supported: string, []byte, and io.Reader. In this case, filename is
// only used for source position information and error messages.
//
// If src == nil, ParseFile parses the file specified by filename.
//
// If scope != nil, it is the immediately surrounding scope for the file
// (the package scope) and it is used to lookup and declare identifiers.
// When parsing multiple files belonging to a package, the same scope should
// be provided to all files.
//
// The mode parameter controls the amount of source text parsed and other
// optional parser functionality.
//
// If the source couldn't be read, the returned AST is nil and the error
// indicates the specific failure. If the source was read but syntax
// errors were found, the result is a partial AST (with ast.BadX nodes
// representing the fragments of erroneous source code). Multiple errors
// are returned via a scanner.ErrorList which is sorted by file position.
//
func ParseFile(filename string, src interface{}, scope *ast.Scope, mode uint) (*ast.File, os.Error) {
	data, err := readSource(filename, src)
	if err != nil {
		return nil, err
	}

	var p parser
	p.init(filename, data, scope, mode)
	return p.parseFile(), p.GetError(scanner.NoMultiples) // parseFile() reads to EOF
}

// ParseFiles calls ParseFile for each file in the filenames list and returns
// a map of package name -> package AST with all the packages found. The mode
// bits are passed to ParseFile unchanged.
//
// Files with parse errors are ignored. In this case the map of packages may
// be incomplete (missing packages and/or incomplete packages) and the last
// error encountered is returned.
//
func ParseFiles(filenames []string, scope *ast.Scope, mode uint) (map[string]*ast.Package, os.Error) {
	pkgs := make(map[string]*ast.Package)
	var err os.Error
	for _, filename := range filenames {
		var src *ast.File
		src, err = ParseFile(filename, nil, scope, mode)
		if err == nil {
			name := src.Name.Name()
			pkg, found := pkgs[name]
			if !found {
				pkg = &ast.Package{name, scope, make(map[string]*ast.File)}
				pkgs[name] = pkg
			}
			pkg.Files[filename] = src
		}
	}

	return pkgs, err
}

// ParseDir calls ParseFile for the files in the directory specified by path and
// returns a map of package name -> package AST with all the packages found. If
// filter != nil, only the files with os.FileInfo entries passing through the filter
// are considered. The mode bits are passed to ParseFile unchanged.
//
// If the directory couldn't be read, a nil map and the respective error are
// returned. If a parse error occured, a non-nil but incomplete map and the
// error are returned.
//
func ParseDir(path string, filter func(*os.FileInfo) bool, mode uint) (map[string]*ast.Package, os.Error) {
	fd, err := os.Open(path, os.O_RDONLY, 0)
	if err != nil {
		return nil, err
	}
	defer fd.Close()

	list, err := fd.Readdir(-1)
	if err != nil {
		return nil, err
	}

	filenames := make([]string, len(list))
	n := 0
	for i := 0; i < len(list); i++ {
		d := &list[i]
		if filter == nil || filter(d) {
			filenames[n] = pathutil.Join(path, d.Name)
			n++
		}
	}
	filenames = filenames[0:n]

	var scope *ast.Scope = nil // for now tracking of declarations is disabled
	return ParseFiles(filenames, scope, mode)
}
<commit_msg>go/parser.ParseFiles: don't override error (to nil) if there is one<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// This file contains the exported entry points for invoking the parser.

package parser

import (
	"bytes"
	"go/ast"
	"go/scanner"
	"go/token"
	"io"
	"io/ioutil"
	"os"
	pathutil "path"
)

// If src != nil, readSource converts src to a []byte if possible;
// otherwise it returns an error. If src == nil, readSource returns
// the result of reading the file specified by filename.
//
func readSource(filename string, src interface{}) ([]byte, os.Error) {
	if src != nil {
		switch s := src.(type) {
		case string:
			return []byte(s), nil
		case []byte:
			return s, nil
		case *bytes.Buffer:
			// is io.Reader, but src is already available in []byte form
			if s != nil {
				return s.Bytes(), nil
			}
		case io.Reader:
			var buf bytes.Buffer
			_, err := io.Copy(&buf, s)
			if err != nil {
				return nil, err
			}
			return buf.Bytes(), nil
		default:
			return nil, os.ErrorString("invalid source")
		}
	}

	return ioutil.ReadFile(filename)
}

func (p *parser) parseEOF() os.Error {
	p.expect(token.EOF)
	return p.GetError(scanner.Sorted)
}

// ParseExpr parses a Go expression and returns the corresponding
// AST node. The filename, src, and scope arguments have the same interpretation
// as for ParseFile. If there is an error, the result expression
// may be nil or contain a partial AST.
//
func ParseExpr(filename string, src interface{}, scope *ast.Scope) (ast.Expr, os.Error) {
	data, err := readSource(filename, src)
	if err != nil {
		return nil, err
	}

	var p parser
	p.init(filename, data, scope, 0)
	return p.parseExpr(), p.parseEOF()
}

// ParseStmtList parses a list of Go statements and returns the list
// of corresponding AST nodes. The filename, src, and scope arguments have the same
// interpretation as for ParseFile. If there is an error, the node
// list may be nil or contain partial ASTs.
//
func ParseStmtList(filename string, src interface{}, scope *ast.Scope) ([]ast.Stmt, os.Error) {
	data, err := readSource(filename, src)
	if err != nil {
		return nil, err
	}

	var p parser
	p.init(filename, data, scope, 0)
	return p.parseStmtList(), p.parseEOF()
}

// ParseDeclList parses a list of Go declarations and returns the list
// of corresponding AST nodes.  The filename, src, and scope arguments have the same
// interpretation as for ParseFile. If there is an error, the node
// list may be nil or contain partial ASTs.
//
func ParseDeclList(filename string, src interface{}, scope *ast.Scope) ([]ast.Decl, os.Error) {
	data, err := readSource(filename, src)
	if err != nil {
		return nil, err
	}

	var p parser
	p.init(filename, data, scope, 0)
	return p.parseDeclList(), p.parseEOF()
}

// ParseFile parses a Go source file and returns a File node.
//
// If src != nil, ParseFile parses the file source from src. src may
// be provided in a variety of formats. At the moment the following types
// are supported: string, []byte, and io.Reader. In this case, filename is
// only used for source position information and error messages.
//
// If src == nil, ParseFile parses the file specified by filename.
//
// If scope != nil, it is the immediately surrounding scope for the file
// (the package scope) and it is used to lookup and declare identifiers.
// When parsing multiple files belonging to a package, the same scope should
// be provided to all files.
//
// The mode parameter controls the amount of source text parsed and other
// optional parser functionality.
//
// If the source couldn't be read, the returned AST is nil and the error
// indicates the specific failure. If the source was read but syntax
// errors were found, the result is a partial AST (with ast.BadX nodes
// representing the fragments of erroneous source code). Multiple errors
// are returned via a scanner.ErrorList which is sorted by file position.
//
func ParseFile(filename string, src interface{}, scope *ast.Scope, mode uint) (*ast.File, os.Error) {
	data, err := readSource(filename, src)
	if err != nil {
		return nil, err
	}

	var p parser
	p.init(filename, data, scope, mode)
	return p.parseFile(), p.GetError(scanner.NoMultiples) // parseFile() reads to EOF
}

// ParseFiles calls ParseFile for each file in the filenames list and returns
// a map of package name -> package AST with all the packages found. The mode
// bits are passed to ParseFile unchanged.
//
// Files with parse errors are ignored. In this case the map of packages may
// be incomplete (missing packages and/or incomplete packages) and the first
// error encountered is returned.
//
func ParseFiles(filenames []string, scope *ast.Scope, mode uint) (pkgs map[string]*ast.Package, first os.Error) {
	pkgs = make(map[string]*ast.Package)
	for _, filename := range filenames {
		if src, err := ParseFile(filename, nil, scope, mode); err == nil {
			name := src.Name.Name()
			pkg, found := pkgs[name]
			if !found {
				pkg = &ast.Package{name, scope, make(map[string]*ast.File)}
				pkgs[name] = pkg
			}
			pkg.Files[filename] = src
		} else if first == nil {
			first = err
		}
	}
	return
}

// ParseDir calls ParseFile for the files in the directory specified by path and
// returns a map of package name -> package AST with all the packages found. If
// filter != nil, only the files with os.FileInfo entries passing through the filter
// are considered. The mode bits are passed to ParseFile unchanged.
//
// If the directory couldn't be read, a nil map and the respective error are
// returned. If a parse error occured, a non-nil but incomplete map and the
// error are returned.
//
func ParseDir(path string, filter func(*os.FileInfo) bool, mode uint) (map[string]*ast.Package, os.Error) {
	fd, err := os.Open(path, os.O_RDONLY, 0)
	if err != nil {
		return nil, err
	}
	defer fd.Close()

	list, err := fd.Readdir(-1)
	if err != nil {
		return nil, err
	}

	filenames := make([]string, len(list))
	n := 0
	for i := 0; i < len(list); i++ {
		d := &list[i]
		if filter == nil || filter(d) {
			filenames[n] = pathutil.Join(path, d.Name)
			n++
		}
	}
	filenames = filenames[0:n]

	var scope *ast.Scope = nil // for now tracking of declarations is disabled
	return ParseFiles(filenames, scope, mode)
}
<|endoftext|>


<commit_before>// Copyright 2021 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package analytics

import (
	"context"
	"flag"
	"html/template"

	"go.chromium.org/luci/common/errors"
	"go.chromium.org/luci/server/module"
)

// ModuleName can be used to refer to this module when declaring dependencies.
var ModuleName = module.RegisterName("go.chromium.org/luci/server/analytics")

// ModuleOptions contain configuration of the analytics server module.
type ModuleOptions struct {
	// Google Analytics measurement ID to use like "G-XXXXXX".
	//
	// Default is empty, meaning Google analytics integration is disabled.
	AnalyticsID string
}

// Register registers the command line flags.
func (o *ModuleOptions) Register(f *flag.FlagSet) {
	f.StringVar(
		&o.AnalyticsID,
		"analytics-id",
		o.AnalyticsID,
		`Google analytics measurement ID in "G-XXXXXX" format.`,
	)
}

// NewModuleFromFlags initializes options through command line flags.
//
// Calling this function registers flags in flag.CommandLine. They are usually
// parsed in server.Main(...).
func NewModuleFromFlags() module.Module {
	opts := &ModuleOptions{}
	opts.Register(flag.CommandLine)
	return &analyticsModule{opts: opts}
}

// analyticsModule implements module.Module.
type analyticsModule struct {
	opts *ModuleOptions
}

// Name is part of module.Module interface.
func (*analyticsModule) Name() module.Name {
	return ModuleName
}

// Dependencies is part of module.Module interface.
func (*analyticsModule) Dependencies() []module.Dependency {
	return nil
}

var ctxKey = "go.chromium.org/luci/server/analytics/ctxKey"

// Initialize is part of module.Module interface.
func (m *analyticsModule) Initialize(ctx context.Context, host module.Host, opts module.HostOptions) (context.Context, error) {
	switch {
	case m.opts.AnalyticsID == "":
		return ctx, nil
	case rGA4Allowed.MatchString(m.opts.AnalyticsID):
		return context.WithValue(ctx, &ctxKey, makeGTagSnippet(m.opts.AnalyticsID)), nil
	case rAllowed.MatchString(m.opts.AnalyticsID):
		return context.WithValue(ctx, &ctxKey, makeUASnippet(m.opts.AnalyticsID)), nil
	default:
		return ctx, errors.Reason("given --analytics-id %q is not a measurement ID (like G-XXXXXX) or a tracking ID (like UA-XXXXX)", m.opts.AnalyticsID).Err()
	}
}

// Snippet returns the cached analytics snippet from ctx,
// which was generated by Initialize() to call into either Google's older
// library "analytics.js" or the newer "gtag.js", depending on the analytics ID
// passed in the command line. I.e. for IDs in the format "UA-XXXXXX" the
// snippet will be like the one documented in:
// https://developers.google.com/analytics/devguides/collection/analyticsjs#the_google_analytics_tag,
// and if in the format "G-XXXXX" the snippet will be of this other type:
// https://developers.google.com/analytics/devguides/collection/gtagjs#install_the_global_site_tag.
//
// If the analytics ID is missing or empty, this will return an empty template,
// so that it's safe to insert its return value into a page's source regardless.
func Snippet(ctx context.Context) template.HTML {
	v, _ := ctx.Value(&ctxKey).(template.HTML)
	return v
}
<commit_msg>Fix analytics module documentation.<commit_after>// Copyright 2021 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package analytics

import (
	"context"
	"flag"
	"html/template"

	"go.chromium.org/luci/common/errors"
	"go.chromium.org/luci/server/module"
)

// ModuleName can be used to refer to this module when declaring dependencies.
var ModuleName = module.RegisterName("go.chromium.org/luci/server/analytics")

// ModuleOptions contain configuration of the analytics server module.
type ModuleOptions struct {
	// Google Analytics measurement ID to use like "G-XXXXXX".
	//
	// Default is empty, meaning Google analytics integration is disabled.
	AnalyticsID string
}

// Register registers the command line flags.
func (o *ModuleOptions) Register(f *flag.FlagSet) {
	f.StringVar(
		&o.AnalyticsID,
		"analytics-id",
		o.AnalyticsID,
		`Google analytics measurement ID in "G-XXXXXX" format.`,
	)
}

// NewModuleFromFlags initializes options through command line flags.
//
// Calling this function registers flags in flag.CommandLine. They are usually
// parsed in server.Main(...).
func NewModuleFromFlags() module.Module {
	opts := &ModuleOptions{}
	opts.Register(flag.CommandLine)
	return &analyticsModule{opts: opts}
}

// analyticsModule implements module.Module.
type analyticsModule struct {
	opts *ModuleOptions
}

// Name is part of module.Module interface.
func (*analyticsModule) Name() module.Name {
	return ModuleName
}

// Dependencies is part of module.Module interface.
func (*analyticsModule) Dependencies() []module.Dependency {
	return nil
}

var ctxKey = "go.chromium.org/luci/server/analytics/ctxKey"

// Initialize is part of module.Module interface.
func (m *analyticsModule) Initialize(ctx context.Context, host module.Host, opts module.HostOptions) (context.Context, error) {
	switch {
	case m.opts.AnalyticsID == "":
		return ctx, nil
	case rGA4Allowed.MatchString(m.opts.AnalyticsID):
		return context.WithValue(ctx, &ctxKey, makeGTagSnippet(m.opts.AnalyticsID)), nil
	case rAllowed.MatchString(m.opts.AnalyticsID):
		return context.WithValue(ctx, &ctxKey, makeUASnippet(m.opts.AnalyticsID)), nil
	default:
		return ctx, errors.Reason("given --analytics-id %q is not a measurement ID (like G-XXXXXX) or a tracking ID (like UA-XXXXX)", m.opts.AnalyticsID).Err()
	}
}

// Snippet returns the analytics snippet to be inserted into the page's source as is.
//
// Depending on the kind of the AnalyticsID configured (see module's Options):
//
//  * format UA-XXXXX,
//    documented at https://developers.google.com/analytics/devguides/collection/analyticsjs#the_google_analytics_tag
//    will embed loading of "analytics.js"
//  * format G-XXXXXX-YY
//    documented at https://developers.google.com/analytics/devguides/collection/gtagjs#install_the_global_site_tag
//    will embed loading of "gtag.js"
//
// If the AnalyticsID isn't configured, this will return an empty template
// so that it's safe to insert its return value into a page's source regardless.
func Snippet(ctx context.Context) template.HTML {
	v, _ := ctx.Value(&ctxKey).(template.HTML)
	return v
}
<|endoftext|>


<commit_before>package consumer

import (
	"testing"
)

func TestNewTopologyEmptyConfig(t *testing.T) {
	ini := ""
	conf := newConfig(ini)
	_, err := NewTopology(conf)
	if err == nil {
		t.Error("should fail, config is empty")
	}
}

const singleQueue = `
[connection]
host = localhost

[queue]
name = db_events
routing_key = events

[exchange]
name = events
`

func TestNewTopologyWithSingleQueue(t *testing.T) {
	conf := newConfig(singleQueue)
	top, err := NewTopology(conf)
	if err != nil {
		t.Error("Should not make an error")
	}
	if top.Connection().host != "localhost" {
		t.Error("host on connection does not match")
	}
	bindings := top.Bindings()
	if len(bindings) != 1 {
		t.Error("incorrect bindings made")
	}
	if bindings[0].queue.name != "db_events" {
		t.Error("Incorrect name for first queue.")
	}
	if bindings[0].queue.routingKey != "events" {
		t.Error("Incorrect routingKey for first queue.")
	}
	if bindings[0].exchange.name != "events" {
		t.Error("Incorrect name for first exchange.")
	}
}

const multiQueue = `
[connection]
host = localhost

[queue-front]
name = front-events
routing_key = events

[queue-back]
name = back-events
routing_key = events

[exchange-back]
name = be-events
type = direct

[exchange-front]
name = fe-events
type = direct
`
func TestNewTopologyWithMultipleQueue(t *testing.T) {
	conf := newConfig(multiQueue)
	top, err := NewTopology(conf)
	if err != nil {
		t.Error("Should not make an error")
	}
	if top.Connection().host != "localhost" {
		t.Error("host on connection does not match")
	}
	bindings := top.Bindings()
	if len(bindings) != 2 {
		t.Error("incorrect bindings made")
	}
	if bindings[0].queue.name != "back-events" {
		t.Error("Incorrect name for first queue.")
	}
	if bindings[0].queue.routingKey != "events" {
		t.Error("Incorrect routingKey for first queue.")
	}
	if bindings[0].exchange.name != "be-events" {
		t.Error("Incorrect name for first exchange.")
	}

	if bindings[1].queue.name != "front-events" {
		t.Error("Incorrect name for second queue.")
	}
	if bindings[1].queue.routingKey != "events" {
		t.Error("Incorrect routingKey for second queue.")
	}
	if bindings[1].exchange.name != "fe-events" {
		t.Error("Incorrect name for second exchange.")
	}
}

const unmatchedQueue = `
[connection]
host = localhost

[queue-front]
name = front-events
routing_key = events

[queue-back]
name = back-events
routing_key = events

[exchange-back]
name = be-events
type = direct
`

func TestNewTopologyUnmatchedQueue(t *testing.T) {
	conf := newConfig(unmatchedQueue)
	_, err := NewTopology(conf)
	if err == nil {
		t.Error("Should make an error")
	}
}

const unmatchedExchange = `
[connection]
host = localhost

[exchange-front]
name = fe-events
type = direct

[queue-back]
name = back-events
routing_key = events

[exchange-back]
name = be-events
type = direct
`

func TestNewTopologyUnmatchedExchange(t *testing.T) {
	conf := newConfig(unmatchedExchange)
	_, err := NewTopology(conf)
	if err == nil {
		t.Error("Should make an error")
	}
}
<commit_msg>Add test for incomplete topology.<commit_after>package consumer

import (
	"testing"
)

func TestNewTopologyEmptyConfig(t *testing.T) {
	ini := ""
	conf := newConfig(ini)
	_, err := NewTopology(conf)
	if err == nil {
		t.Error("should fail, config is empty")
	}
}

const singleQueue = `
[connection]
host = localhost

[queue]
name = db_events
routing_key = events

[exchange]
name = events
`

func TestNewTopologyWithSingleQueue(t *testing.T) {
	conf := newConfig(singleQueue)
	top, err := NewTopology(conf)
	if err != nil {
		t.Error("Should not make an error")
	}
	if top.Connection().host != "localhost" {
		t.Error("host on connection does not match")
	}
	bindings := top.Bindings()
	if len(bindings) != 1 {
		t.Error("incorrect bindings made")
	}
	if bindings[0].queue.name != "db_events" {
		t.Error("Incorrect name for first queue.")
	}
	if bindings[0].queue.routingKey != "events" {
		t.Error("Incorrect routingKey for first queue.")
	}
	if bindings[0].exchange.name != "events" {
		t.Error("Incorrect name for first exchange.")
	}
}

const multiQueue = `
[connection]
host = localhost

[queue-front]
name = front-events
routing_key = events

[queue-back]
name = back-events
routing_key = events

[exchange-back]
name = be-events
type = direct

[exchange-front]
name = fe-events
type = direct
`
func TestNewTopologyWithMultipleQueue(t *testing.T) {
	conf := newConfig(multiQueue)
	top, err := NewTopology(conf)
	if err != nil {
		t.Error("Should not make an error")
	}
	if top.Connection().host != "localhost" {
		t.Error("host on connection does not match")
	}
	bindings := top.Bindings()
	if len(bindings) != 2 {
		t.Error("incorrect bindings made")
	}
	if bindings[0].queue.name != "back-events" {
		t.Error("Incorrect name for first queue.")
	}
	if bindings[0].queue.routingKey != "events" {
		t.Error("Incorrect routingKey for first queue.")
	}
	if bindings[0].exchange.name != "be-events" {
		t.Error("Incorrect name for first exchange.")
	}

	if bindings[1].queue.name != "front-events" {
		t.Error("Incorrect name for second queue.")
	}
	if bindings[1].queue.routingKey != "events" {
		t.Error("Incorrect routingKey for second queue.")
	}
	if bindings[1].exchange.name != "fe-events" {
		t.Error("Incorrect name for second exchange.")
	}
}

const unmatchedQueue = `
[connection]
host = localhost

[queue-front]
name = front-events
routing_key = events

[queue-back]
name = back-events
routing_key = events

[exchange-back]
name = be-events
type = direct
`

func TestNewTopologyUnmatchedQueue(t *testing.T) {
	conf := newConfig(unmatchedQueue)
	_, err := NewTopology(conf)
	if err == nil {
		t.Error("Should make an error")
	}
}

const unmatchedExchange = `
[connection]
host = localhost

[exchange-front]
name = fe-events
type = direct

[queue-back]
name = back-events
routing_key = events

[exchange-back]
name = be-events
type = direct
`

func TestNewTopologyUnmatchedExchange(t *testing.T) {
	conf := newConfig(unmatchedExchange)
	_, err := NewTopology(conf)
	if err == nil {
		t.Error("Should make an error")
	}
}

const missingBits = `
[connection]
host = localhost

[exchange-f]
name = fe-events
type = direct

[queue-b]
name = back-events
routing_key = events
`

func TestNewTopologyMisnamedBits(t *testing.T) {
	conf := newConfig(missingBits)
	_, err := NewTopology(conf)
	if err == nil {
		t.Error("Should make an error")
	}
}
<|endoftext|>


<commit_before>package rest

import (
	"fmt"
	"net/url"
	"path"

	"github.com/bitfinexcom/bitfinex-api-go/pkg/models/pulse"
	"github.com/bitfinexcom/bitfinex-api-go/pkg/models/pulseprofile"
)

type PulseService struct {
	requestFactory
	Synchronous
}

func (ps *PulseService) PublicPulseProfile(nickname string) (*pulseprofile.PulseProfile, error) {
	if (len(nickname)) == 0 {
		return nil, fmt.Errorf("nickname is required argument")
	}

	req := NewRequestWithMethod(path.Join("pulse", "profile", nickname), "GET")
	raw, err := ps.Request(req)
	if err != nil {
		return nil, err
	}

	pp, err := pulseprofile.NewFromRaw(raw)
	if err != nil {
		return nil, err
	}

	return pp, nil
}

func (ps *PulseService) PublicPulseHistory(limit, end string) ([]*pulse.Pulse, error) {
	req := NewRequestWithMethod(path.Join("pulse", "hist"), "GET")
	req.Params = make(url.Values)
	req.Params.Add("limit", limit)
	req.Params.Add("end", end)

	raw, err := ps.Request(req)
	if err != nil {
		return nil, err
	}

	pph, err := pulse.NewFromRaw(raw)
	if err != nil {
		return nil, err
	}

	return pph, nil
}
<commit_msg>RESTv2 new AddPulse function<commit_after>package rest

import (
	"encoding/json"
	"fmt"
	"net/url"
	"path"

	"github.com/bitfinexcom/bitfinex-api-go/pkg/models/pulse"
	"github.com/bitfinexcom/bitfinex-api-go/pkg/models/pulseprofile"
	"github.com/bitfinexcom/bitfinex-api-go/v2"
)

type PulseService struct {
	requestFactory
	Synchronous
}

// PublicPulseProfile get pulse profile by nickname
func (ps *PulseService) PublicPulseProfile(nickname string) (*pulseprofile.PulseProfile, error) {
	if (len(nickname)) == 0 {
		return nil, fmt.Errorf("nickname is required argument")
	}

	req := NewRequestWithMethod(path.Join("pulse", "profile", nickname), "GET")
	raw, err := ps.Request(req)
	if err != nil {
		return nil, err
	}

	pp, err := pulseprofile.NewFromRaw(raw)
	if err != nil {
		return nil, err
	}

	return pp, nil
}

// PublicPulseHistory returns public pulse messages
func (ps *PulseService) PublicPulseHistory(limit, end string) ([]*pulse.Pulse, error) {
	req := NewRequestWithMethod(path.Join("pulse", "hist"), "GET")
	req.Params = make(url.Values)
	req.Params.Add("limit", limit)
	req.Params.Add("end", end)

	raw, err := ps.Request(req)
	if err != nil {
		return nil, err
	}

	pph, err := pulse.NewFromRaw(raw)
	if err != nil {
		return nil, err
	}

	return pph, nil
}

// AddPulse submits pulse messages
func (ps *PulseService) AddPulse(p *pulse.Pulse) (*pulse.Pulse, error) {
	tl := len(p.Title)
	if tl < 16 || tl > 120 {
		return nil, fmt.Errorf("Title length min 16 and max 120 characters. Got:%d", tl)
	}

	payload, err := json.Marshal(p)
	if err != nil {
		return nil, err
	}

	req, err := ps.requestFactory.NewAuthenticatedRequestWithBytes(bitfinex.PermissionWrite, path.Join("pulse", "add"), payload)
	if err != nil {
		return nil, err
	}

	raw, err := ps.Request(req)
	if err != nil {
		return nil, err
	}

	pm, err := pulse.NewSingleFromRaw(raw)
	if err != nil {
		return nil, err
	}

	return pm, nil
}
<|endoftext|>


<commit_before>/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements.  See the NOTICE file
distributed with this work for additional information
regarding copyright ownership.  The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License.  You may obtain a copy of the License at

  http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied.  See the License for the
specific language governing permissions and limitations
under the License.
*/

package obcpbft

import (
	"encoding/base64"
	"fmt"
	"strconv"
	"sync"

	"github.com/openblockchain/obc-peer/openchain/consensus"
	"github.com/openblockchain/obc-peer/openchain/util"
	pb "github.com/openblockchain/obc-peer/protos"
)

type mockCPI struct {
	broadcasted [][]byte
	executed    [][]byte
	ledger      consensus.Ledger
}

func newMock() *mockCPI {
	mock := &mockCPI{
		make([][]byte, 0),
		make([][]byte, 0),
		NewMockLedger(nil, nil),
	}
	mock.ledger.PutBlock(0, SimpleGetBlock(0))
	return mock
}

func (mock *mockCPI) broadcast(msg []byte) {
	mock.broadcasted = append(mock.broadcasted, msg)
}

func (mock *mockCPI) unicast(msg []byte, receiverID uint64) (err error) {
	panic("not implemented")
}

func (mock *mockCPI) verify(tx []byte) error {
	return nil
}

func (mock *mockCPI) execute(tx []byte) {
	mock.executed = append(mock.executed, tx)
}

func (mock *mockCPI) viewChange(uint64) {
}

func (mock *mockCPI) BeginTxBatch(id interface{}) error {
	return mock.ledger.BeginTxBatch(id)
}

func (mock *mockCPI) ExecTXs(txs []*pb.Transaction) ([]byte, []error) {
	return mock.ledger.ExecTXs(txs)
}

func (mock *mockCPI) CommitTxBatch(id interface{}, txs []*pb.Transaction, proof []byte) error {
	return mock.ledger.CommitTxBatch(id, txs, proof)
}

func (mock *mockCPI) PreviewCommitTxBatchBlock(id interface{}, txs []*pb.Transaction, proof []byte) (*pb.Block, error) {
	return mock.ledger.PreviewCommitTxBatchBlock(id, txs, proof)
}

func (mock *mockCPI) RollbackTxBatch(id interface{}) error {
	return mock.ledger.RollbackTxBatch(id)
}

func (mock *mockCPI) GetBlock(id uint64) (block *pb.Block, err error) {
	return SimpleGetBlock(id), nil
}
func (mock *mockCPI) GetCurrentStateHash() (stateHash []byte, err error) {
	return mock.ledger.GetCurrentStateHash()
}
func (mock *mockCPI) GetBlockchainSize() (uint64, error) {
	return mock.ledger.GetBlockchainSize()
}
func (mock *mockCPI) HashBlock(block *pb.Block) ([]byte, error) {
	return mock.ledger.HashBlock(block)
}
func (mock *mockCPI) PutBlock(blockNumber uint64, block *pb.Block) error {
	return mock.ledger.PutBlock(blockNumber, block)
}
func (mock *mockCPI) ApplyStateDelta(delta []byte, unapply bool) error {
	return mock.ledger.ApplyStateDelta(delta, unapply)
}
func (mock *mockCPI) EmptyState() error {
	return mock.ledger.EmptyState()
}
func (mock *mockCPI) VerifyBlockchain(start, finish uint64) (uint64, error) {
	return mock.ledger.VerifyBlockchain(start, finish)
}
func (mock *mockCPI) GetRemoteBlocks(replicaId uint64, start, finish uint64) (<-chan *pb.SyncBlocks, error) {
	return mock.ledger.GetRemoteBlocks(replicaId, start, finish)
}
func (mock *mockCPI) GetRemoteStateSnapshot(replicaId uint64) (<-chan *pb.SyncStateSnapshot, error) {
	return mock.ledger.GetRemoteStateSnapshot(replicaId)
}
func (mock *mockCPI) GetRemoteStateDeltas(replicaId uint64, start, finish uint64) (<-chan *pb.SyncStateDeltas, error) {
	return mock.ledger.GetRemoteStateDeltas(replicaId, start, finish)
}

// =============================================================================
// Fake network structures
// =============================================================================

type closableConsenter interface {
	consensus.Consenter
	Close()
}

type taggedMsg struct {
	src int
	dst int
	msg []byte
}

type testnet struct {
	f        int
	cond     *sync.Cond
	closed   bool
	replicas []*instance
	msgs     []taggedMsg
	handles  []string
	filterFn func(int, int, []byte) []byte
}

type instance struct {
	id        int
	handle    string
	pbft      *pbftCore
	consenter closableConsenter
	net       *testnet
	ledger    consensus.Ledger

	deliver      func([]byte)
	execTxResult func([]*pb.Transaction) ([]byte, []error)
}

func (inst *instance) broadcast(payload []byte) {
	net := inst.net
	net.cond.L.Lock()
	/* msg := &Message{}
	_ = proto.Unmarshal(payload, msg)
	if fr := msg.GetFetchRequest(); fr != nil {
		fmt.Printf("Debug: replica %v broadcast fetch-request\n", inst.id)
	} */
	net.broadcastFilter(inst, payload)
	net.cond.Signal()
	net.cond.L.Unlock()
}

func (inst *instance) unicast(payload []byte, receiverID uint64) error {
	net := inst.net
	/* msg := &Message{}
	_ = proto.Unmarshal(payload, msg)
	if rr := msg.GetReturnRequest(); rr != nil {
		fmt.Printf("Debug: replica %v unicast return-request to %v\n", inst.id, receiverID)
		net.replicas[int(receiverID)].deliver(payload)
	} else {
		fmt.Printf("Debug: replica %v unicast (non return-request) to %v\n", inst.id, receiverID) */
	net.cond.L.Lock()
	net.msgs = append(net.msgs, taggedMsg{inst.id, int(receiverID), payload})
	net.cond.Signal()
	net.cond.L.Unlock()
	return nil
}

func (inst *instance) verify(payload []byte) error {
	return nil
}

func (inst *instance) execute(payload []byte) {

	tx := &pb.Transaction{
		Payload: payload,
	}

	txs := []*pb.Transaction{tx}
	txBatchID := base64.StdEncoding.EncodeToString(util.ComputeCryptoHash(payload))

	if err := inst.BeginTxBatch(txBatchID); err != nil {
		fmt.Printf("Failed to begin transaction %s: %v", txBatchID, err)
		return
	}

	_, errs := inst.ExecTXs(txs)

	if errs[len(txs)] != nil {
		fmt.Printf("Fail to execute transaction %s: %v", txBatchID, errs)
		if err := inst.RollbackTxBatch(txBatchID); err != nil {
			panic(fmt.Errorf("Unable to rollback transaction %s: %v", txBatchID, err))
		}
		return
	}

	if err := inst.CommitTxBatch(txBatchID, txs, nil); err != nil {
		fmt.Printf("Failed to commit transaction %s to the ledger: %v", txBatchID, err)
		if err = inst.RollbackTxBatch(txBatchID); err != nil {
			panic(fmt.Errorf("Unable to rollback transaction %s: %v", txBatchID, err))
		}
		return
	}

}

func (inst *instance) viewChange(uint64) {
}

func (inst *instance) GetNetworkHandles() (self string, network []string, err error) {
	return inst.handle, inst.net.handles, nil
}

func (inst *instance) GetReplicaHandle(id uint64) (handle string, err error) {
	_, network, _ := inst.GetNetworkHandles()
	if int(id) > (len(network) - 1) {
		return handle, fmt.Errorf("Replica ID is out of bounds")
	}
	return network[int(id)], nil
}

func (inst *instance) GetReplicaID(handle string) (id uint64, err error) {
	_, network, _ := inst.GetNetworkHandles()
	for i, v := range network {
		if v == handle {
			return uint64(i), nil
		}
	}
	err = fmt.Errorf("Couldn't find handle in list of handles in testnet")
	return
}

// Broadcast delivers to all replicas.  In contrast to the stack
// Broadcast, this will also deliver back to the replica.  We keep
// this behavior, because it exposes subtle bugs in the
// implementation.
func (inst *instance) Broadcast(msg *pb.OpenchainMessage) error {
	net := inst.net
	net.cond.L.Lock()
	net.broadcastFilter(inst, msg.Payload)
	net.cond.Signal()
	net.cond.L.Unlock()
	return nil
}

func (inst *instance) Unicast(msg *pb.OpenchainMessage, receiver string) error {
	net := inst.net
	net.cond.L.Lock()
	receiverID, err := inst.GetReplicaID(receiver)
	if err != nil {
		return fmt.Errorf("Couldn't unicast message to %s: %v", receiver, err)
	}
	net.msgs = append(net.msgs, taggedMsg{inst.id, int(receiverID), msg.Payload})
	net.cond.Signal()
	net.cond.L.Unlock()
	return nil
}

func (inst *instance) BeginTxBatch(id interface{}) error {
	return inst.ledger.BeginTxBatch(id)
}

func (inst *instance) ExecTXs(txs []*pb.Transaction) ([]byte, []error) {
	return inst.ledger.ExecTXs(txs)
}

func (inst *instance) CommitTxBatch(id interface{}, txs []*pb.Transaction, proof []byte) error {
	return inst.ledger.CommitTxBatch(id, txs, proof)
}

func (inst *instance) PreviewCommitTxBatchBlock(id interface{}, txs []*pb.Transaction, proof []byte) (*pb.Block, error) {
	return inst.ledger.PreviewCommitTxBatchBlock(id, txs, proof)
}

func (inst *instance) RollbackTxBatch(id interface{}) error {
	return inst.ledger.RollbackTxBatch(id)
}

func (inst *instance) GetBlock(id uint64) (block *pb.Block, err error) {
	return inst.ledger.GetBlock(id)
}
func (inst *instance) GetCurrentStateHash() (stateHash []byte, err error) {
	return inst.ledger.GetCurrentStateHash()
}
func (inst *instance) GetBlockchainSize() (uint64, error) {
	return inst.ledger.GetBlockchainSize()
}
func (inst *instance) HashBlock(block *pb.Block) ([]byte, error) {
	return inst.ledger.HashBlock(block)
}
func (inst *instance) PutBlock(blockNumber uint64, block *pb.Block) error {
	return inst.ledger.PutBlock(blockNumber, block)
}
func (inst *instance) ApplyStateDelta(delta []byte, unapply bool) error {
	return inst.ledger.ApplyStateDelta(delta, unapply)
}
func (inst *instance) EmptyState() error {
	return inst.ledger.EmptyState()
}
func (inst *instance) VerifyBlockchain(start, finish uint64) (uint64, error) {
	return inst.ledger.VerifyBlockchain(start, finish)
}
func (inst *instance) GetRemoteBlocks(replicaId uint64, start, finish uint64) (<-chan *pb.SyncBlocks, error) {
	return inst.ledger.GetRemoteBlocks(replicaId, start, finish)
}
func (inst *instance) GetRemoteStateSnapshot(replicaId uint64) (<-chan *pb.SyncStateSnapshot, error) {
	return inst.ledger.GetRemoteStateSnapshot(replicaId)
}
func (inst *instance) GetRemoteStateDeltas(replicaId uint64, start, finish uint64) (<-chan *pb.SyncStateDeltas, error) {
	return inst.ledger.GetRemoteStateDeltas(replicaId, start, finish)
}

func (net *testnet) broadcastFilter(inst *instance, payload []byte) {
	if net.filterFn != nil {
		payload = net.filterFn(inst.id, -1, payload)
	}
	if payload != nil {
		/* msg := &Message{}
		_ = proto.Unmarshal(payload, msg)
		if fr := msg.GetFetchRequest(); fr != nil {
			// treat fetch-request as a high-priority message that needs to be processed ASAP
			fmt.Printf("Debug: replica %v broadcastFilter for fetch-request\n", inst.id)
			net.deliverFilter(taggedMsg{inst.id, -1, payload})
		} else { */
		net.msgs = append(net.msgs, taggedMsg{inst.id, -1, payload})
	}
}

func (net *testnet) deliverFilter(msg taggedMsg) {
	if msg.dst == -1 {
		for id, inst := range net.replicas {
			payload := msg.msg
			if net.filterFn != nil {
				payload = net.filterFn(msg.src, id, payload)
			}
			if payload != nil {
				inst.deliver(msg.msg)
			}
		}
	} else {
		net.replicas[msg.dst].deliver(msg.msg)
	}
}

func (net *testnet) process() error {
	net.cond.L.Lock()
	defer net.cond.L.Unlock()

	for len(net.msgs) > 0 {
		msg := net.msgs[0]
		fmt.Printf("Debug: process iteration (%d messages to go, delivering now to destination %v)\n", len(net.msgs), msg.dst)
		net.msgs = net.msgs[1:]
		net.cond.L.Unlock()
		net.deliverFilter(msg)
		net.cond.L.Lock()
	}

	return nil
}

func (net *testnet) processContinually() {
	net.cond.L.Lock()
	defer net.cond.L.Unlock()
	for {
		if net.closed {
			break
		}
		if len(net.msgs) == 0 {
			net.cond.Wait()
		}
		net.cond.L.Unlock()
		net.process()
		net.cond.L.Lock()
	}
}

func makeTestnet(f int, initFn ...func(*instance)) *testnet {
	net := &testnet{f: f}
	net.cond = sync.NewCond(&sync.Mutex{})
	replicaCount := 3*f + 1

	for i := uint64(0); i < uint64(replicaCount); i++ {
	}

	ledgers := make(map[uint64]ReadOnlyLedger, replicaCount)
	for i := 0; i < replicaCount; i++ {
		inst := &instance{handle: "vp" + strconv.Itoa(i), id: i, net: net}
		ml := NewMockLedger(&ledgers, nil)
		ml.inst = inst
		ml.PutBlock(0, SimpleGetBlock(0))
		ledgers[uint64(i)] = ml
		inst.ledger = ml
		net.replicas = append(net.replicas, inst)
		net.handles = append(net.handles, inst.handle)
	}

	for _, inst := range net.replicas {
		for _, fn := range initFn {
			fn(inst)
		}
	}

	return net
}

func (net *testnet) close() {
	if net.closed {
		return
	}
	for _, inst := range net.replicas {
		if inst.pbft != nil {
			inst.pbft.close()
		}
		if inst.consenter != nil {
			inst.consenter.Close()
		}
	}
	net.cond.L.Lock()
	net.closed = true
	net.cond.Signal()
	net.cond.L.Unlock()
}
<commit_msg>Cleanup mock duplication<commit_after>/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements.  See the NOTICE file
distributed with this work for additional information
regarding copyright ownership.  The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License.  You may obtain a copy of the License at

  http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied.  See the License for the
specific language governing permissions and limitations
under the License.
*/

package obcpbft

import (
	"encoding/base64"
	"fmt"
	"strconv"
	"sync"

	"github.com/openblockchain/obc-peer/openchain/consensus"
	"github.com/openblockchain/obc-peer/openchain/util"
	pb "github.com/openblockchain/obc-peer/protos"
)

type mockCPI struct {
	broadcasted [][]byte
	*instance
}

func newMock() *mockCPI {
	mock := &mockCPI{
		make([][]byte, 0),
		&instance{},
	}
	mock.instance.ledger = NewMockLedger(nil, nil)
	mock.instance.ledger.PutBlock(0, SimpleGetBlock(0))
	return mock
}

func (mock *mockCPI) broadcast(msg []byte) {
	mock.broadcasted = append(mock.broadcasted, msg)
}

func (mock *mockCPI) unicast(msg []byte, receiverID uint64) (err error) {
	panic("not implemented")
}

// =============================================================================
// Fake network structures
// =============================================================================

type closableConsenter interface {
	consensus.Consenter
	Close()
}

type taggedMsg struct {
	src int
	dst int
	msg []byte
}

type testnet struct {
	f        int
	cond     *sync.Cond
	closed   bool
	replicas []*instance
	msgs     []taggedMsg
	handles  []string
	filterFn func(int, int, []byte) []byte
}

type instance struct {
	id        int
	handle    string
	pbft      *pbftCore
	consenter closableConsenter
	net       *testnet
	ledger    consensus.Ledger

	deliver      func([]byte)
	execTxResult func([]*pb.Transaction) ([]byte, []error)
}

func (inst *instance) broadcast(payload []byte) {
	net := inst.net
	net.cond.L.Lock()
	/* msg := &Message{}
	_ = proto.Unmarshal(payload, msg)
	if fr := msg.GetFetchRequest(); fr != nil {
		fmt.Printf("Debug: replica %v broadcast fetch-request\n", inst.id)
	} */
	net.broadcastFilter(inst, payload)
	net.cond.Signal()
	net.cond.L.Unlock()
}

func (inst *instance) unicast(payload []byte, receiverID uint64) error {
	net := inst.net
	/* msg := &Message{}
	_ = proto.Unmarshal(payload, msg)
	if rr := msg.GetReturnRequest(); rr != nil {
		fmt.Printf("Debug: replica %v unicast return-request to %v\n", inst.id, receiverID)
		net.replicas[int(receiverID)].deliver(payload)
	} else {
		fmt.Printf("Debug: replica %v unicast (non return-request) to %v\n", inst.id, receiverID) */
	net.cond.L.Lock()
	net.msgs = append(net.msgs, taggedMsg{inst.id, int(receiverID), payload})
	net.cond.Signal()
	net.cond.L.Unlock()
	return nil
}

func (inst *instance) verify(payload []byte) error {
	return nil
}

func (inst *instance) execute(payload []byte) {

	tx := &pb.Transaction{
		Payload: payload,
	}

	txs := []*pb.Transaction{tx}
	txBatchID := base64.StdEncoding.EncodeToString(util.ComputeCryptoHash(payload))

	if err := inst.BeginTxBatch(txBatchID); err != nil {
		fmt.Printf("Failed to begin transaction %s: %v", txBatchID, err)
		return
	}

	_, errs := inst.ExecTXs(txs)

	if errs[len(txs)] != nil {
		fmt.Printf("Fail to execute transaction %s: %v", txBatchID, errs)
		if err := inst.RollbackTxBatch(txBatchID); err != nil {
			panic(fmt.Errorf("Unable to rollback transaction %s: %v", txBatchID, err))
		}
		return
	}

	if err := inst.CommitTxBatch(txBatchID, txs, nil); err != nil {
		fmt.Printf("Failed to commit transaction %s to the ledger: %v", txBatchID, err)
		if err = inst.RollbackTxBatch(txBatchID); err != nil {
			panic(fmt.Errorf("Unable to rollback transaction %s: %v", txBatchID, err))
		}
		return
	}

}

func (inst *instance) viewChange(uint64) {
}

func (inst *instance) GetNetworkHandles() (self string, network []string, err error) {
	return inst.handle, inst.net.handles, nil
}

func (inst *instance) GetReplicaHandle(id uint64) (handle string, err error) {
	_, network, _ := inst.GetNetworkHandles()
	if int(id) > (len(network) - 1) {
		return handle, fmt.Errorf("Replica ID is out of bounds")
	}
	return network[int(id)], nil
}

func (inst *instance) GetReplicaID(handle string) (id uint64, err error) {
	_, network, _ := inst.GetNetworkHandles()
	for i, v := range network {
		if v == handle {
			return uint64(i), nil
		}
	}
	err = fmt.Errorf("Couldn't find handle in list of handles in testnet")
	return
}

// Broadcast delivers to all replicas.  In contrast to the stack
// Broadcast, this will also deliver back to the replica.  We keep
// this behavior, because it exposes subtle bugs in the
// implementation.
func (inst *instance) Broadcast(msg *pb.OpenchainMessage) error {
	net := inst.net
	net.cond.L.Lock()
	net.broadcastFilter(inst, msg.Payload)
	net.cond.Signal()
	net.cond.L.Unlock()
	return nil
}

func (inst *instance) Unicast(msg *pb.OpenchainMessage, receiver string) error {
	net := inst.net
	net.cond.L.Lock()
	receiverID, err := inst.GetReplicaID(receiver)
	if err != nil {
		return fmt.Errorf("Couldn't unicast message to %s: %v", receiver, err)
	}
	net.msgs = append(net.msgs, taggedMsg{inst.id, int(receiverID), msg.Payload})
	net.cond.Signal()
	net.cond.L.Unlock()
	return nil
}

func (inst *instance) BeginTxBatch(id interface{}) error {
	return inst.ledger.BeginTxBatch(id)
}

func (inst *instance) ExecTXs(txs []*pb.Transaction) ([]byte, []error) {
	return inst.ledger.ExecTXs(txs)
}

func (inst *instance) CommitTxBatch(id interface{}, txs []*pb.Transaction, proof []byte) error {
	return inst.ledger.CommitTxBatch(id, txs, proof)
}

func (inst *instance) PreviewCommitTxBatchBlock(id interface{}, txs []*pb.Transaction, proof []byte) (*pb.Block, error) {
	return inst.ledger.PreviewCommitTxBatchBlock(id, txs, proof)
}

func (inst *instance) RollbackTxBatch(id interface{}) error {
	return inst.ledger.RollbackTxBatch(id)
}

func (inst *instance) GetBlock(id uint64) (block *pb.Block, err error) {
	return inst.ledger.GetBlock(id)
}
func (inst *instance) GetCurrentStateHash() (stateHash []byte, err error) {
	return inst.ledger.GetCurrentStateHash()
}
func (inst *instance) GetBlockchainSize() (uint64, error) {
	return inst.ledger.GetBlockchainSize()
}
func (inst *instance) HashBlock(block *pb.Block) ([]byte, error) {
	return inst.ledger.HashBlock(block)
}
func (inst *instance) PutBlock(blockNumber uint64, block *pb.Block) error {
	return inst.ledger.PutBlock(blockNumber, block)
}
func (inst *instance) ApplyStateDelta(delta []byte, unapply bool) error {
	return inst.ledger.ApplyStateDelta(delta, unapply)
}
func (inst *instance) EmptyState() error {
	return inst.ledger.EmptyState()
}
func (inst *instance) VerifyBlockchain(start, finish uint64) (uint64, error) {
	return inst.ledger.VerifyBlockchain(start, finish)
}
func (inst *instance) GetRemoteBlocks(replicaId uint64, start, finish uint64) (<-chan *pb.SyncBlocks, error) {
	return inst.ledger.GetRemoteBlocks(replicaId, start, finish)
}
func (inst *instance) GetRemoteStateSnapshot(replicaId uint64) (<-chan *pb.SyncStateSnapshot, error) {
	return inst.ledger.GetRemoteStateSnapshot(replicaId)
}
func (inst *instance) GetRemoteStateDeltas(replicaId uint64, start, finish uint64) (<-chan *pb.SyncStateDeltas, error) {
	return inst.ledger.GetRemoteStateDeltas(replicaId, start, finish)
}

func (net *testnet) broadcastFilter(inst *instance, payload []byte) {
	if net.filterFn != nil {
		payload = net.filterFn(inst.id, -1, payload)
	}
	if payload != nil {
		/* msg := &Message{}
		_ = proto.Unmarshal(payload, msg)
		if fr := msg.GetFetchRequest(); fr != nil {
			// treat fetch-request as a high-priority message that needs to be processed ASAP
			fmt.Printf("Debug: replica %v broadcastFilter for fetch-request\n", inst.id)
			net.deliverFilter(taggedMsg{inst.id, -1, payload})
		} else { */
		net.msgs = append(net.msgs, taggedMsg{inst.id, -1, payload})
	}
}

func (net *testnet) deliverFilter(msg taggedMsg) {
	if msg.dst == -1 {
		for id, inst := range net.replicas {
			payload := msg.msg
			if net.filterFn != nil {
				payload = net.filterFn(msg.src, id, payload)
			}
			if payload != nil {
				inst.deliver(msg.msg)
			}
		}
	} else {
		net.replicas[msg.dst].deliver(msg.msg)
	}
}

func (net *testnet) process() error {
	net.cond.L.Lock()
	defer net.cond.L.Unlock()

	for len(net.msgs) > 0 {
		msg := net.msgs[0]
		fmt.Printf("Debug: process iteration (%d messages to go, delivering now to destination %v)\n", len(net.msgs), msg.dst)
		net.msgs = net.msgs[1:]
		net.cond.L.Unlock()
		net.deliverFilter(msg)
		net.cond.L.Lock()
	}

	return nil
}

func (net *testnet) processContinually() {
	net.cond.L.Lock()
	defer net.cond.L.Unlock()
	for {
		if net.closed {
			break
		}
		if len(net.msgs) == 0 {
			net.cond.Wait()
		}
		net.cond.L.Unlock()
		net.process()
		net.cond.L.Lock()
	}
}

func makeTestnet(f int, initFn ...func(*instance)) *testnet {
	net := &testnet{f: f}
	net.cond = sync.NewCond(&sync.Mutex{})
	replicaCount := 3*f + 1

	for i := uint64(0); i < uint64(replicaCount); i++ {
	}

	ledgers := make(map[uint64]ReadOnlyLedger, replicaCount)
	for i := 0; i < replicaCount; i++ {
		inst := &instance{handle: "vp" + strconv.Itoa(i), id: i, net: net}
		ml := NewMockLedger(&ledgers, nil)
		ml.inst = inst
		ml.PutBlock(0, SimpleGetBlock(0))
		ledgers[uint64(i)] = ml
		inst.ledger = ml
		net.replicas = append(net.replicas, inst)
		net.handles = append(net.handles, inst.handle)
	}

	for _, inst := range net.replicas {
		for _, fn := range initFn {
			fn(inst)
		}
	}

	return net
}

func (net *testnet) close() {
	if net.closed {
		return
	}
	for _, inst := range net.replicas {
		if inst.pbft != nil {
			inst.pbft.close()
		}
		if inst.consenter != nil {
			inst.consenter.Close()
		}
	}
	net.cond.L.Lock()
	net.closed = true
	net.cond.Signal()
	net.cond.L.Unlock()
}
<|endoftext|>


<commit_before>// Copyright 2013-2019 Aerospike, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package aerospike_test

import (
	"math"
	"math/rand"
	"time"

	. "github.com/aerospike/aerospike-client-go"
	// . "github.com/aerospike/aerospike-client-go/logger"
	// . "github.com/aerospike/aerospike-client-go/types"

	// . "github.com/aerospike/aerospike-client-go/utils/buffer"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

// ALL tests are isolated by SetName and Key, which are 50 random characters
var _ = Describe("Truncate operations test", func() {
	initTestVars()

	Context("Truncate", func() {
		var err error
		var ns = *namespace
		var set = randString(50)
		var key *Key
		var wpolicy = NewWritePolicy(0, 0)
		wpolicy.SendKey = true

		const keyCount = 1000
		bin1 := NewBin("Aerospike1", rand.Intn(math.MaxInt16))
		bin2 := NewBin("Aerospike2", randString(100))

		BeforeEach(func() {
			for i := 0; i < keyCount; i++ {
				key, err = NewKey(ns, set, i)
				Expect(err).ToNot(HaveOccurred())

				_, err = client.Operate(wpolicy, key, PutOp(bin1), PutOp(bin2), GetOp())
				Expect(err).ToNot(HaveOccurred())
			}
		})

		var countRecords = func(namespace, setName string) int {
			stmt := NewStatement(namespace, setName)
			res, err := client.Query(nil, stmt)
			Expect(err).ToNot(HaveOccurred())

			cnt := 0
			for rec := range res.Results() {
				Expect(rec.Err).ToNot(HaveOccurred())
				cnt++
			}

			return cnt
		}

		It("must truncate only the current set", func() {
			Expect(countRecords(ns, set)).To(Equal(keyCount))

			err := client.Truncate(nil, ns, set, nil)
			Expect(err).ToNot(HaveOccurred())

			time.Sleep(time.Second)
			Expect(countRecords(ns, set)).To(Equal(0))
		})

		It("must truncate the whole namespace", func() {
			Expect(countRecords(ns, "")).ToNot(Equal(0))

			err := client.Truncate(nil, ns, "", nil)
			Expect(err).ToNot(HaveOccurred())

			time.Sleep(time.Second)
			Expect(countRecords(ns, "")).To(Equal(0))
		})

		It("must truncate only older records", func() {
			t := time.Now()

			Expect(countRecords(ns, set)).To(Equal(keyCount))

			for i := keyCount; i < 2*keyCount; i++ {
				key, err = NewKey(ns, set, i)
				Expect(err).ToNot(HaveOccurred())

				_, err = client.Operate(wpolicy, key, PutOp(bin1), PutOp(bin2), GetOp())
				Expect(err).ToNot(HaveOccurred())
			}
			Expect(countRecords(ns, set)).To(Equal(2 * keyCount))

			err := client.Truncate(nil, ns, set, &t)
			Expect(err).ToNot(HaveOccurred())

			time.Sleep(1 * time.Second)
			Expect(countRecords(ns, set)).To(Equal(keyCount))
		})

	})
})
<commit_msg>Extend sleep time for truncate test<commit_after>// Copyright 2013-2019 Aerospike, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package aerospike_test

import (
	"math"
	"math/rand"
	"time"

	. "github.com/aerospike/aerospike-client-go"
	// . "github.com/aerospike/aerospike-client-go/logger"
	// . "github.com/aerospike/aerospike-client-go/types"

	// . "github.com/aerospike/aerospike-client-go/utils/buffer"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

// ALL tests are isolated by SetName and Key, which are 50 random characters
var _ = Describe("Truncate operations test", func() {
	initTestVars()

	Context("Truncate", func() {
		var err error
		var ns = *namespace
		var set = randString(50)
		var key *Key
		var wpolicy = NewWritePolicy(0, 0)
		wpolicy.SendKey = true

		const keyCount = 1000
		bin1 := NewBin("Aerospike1", rand.Intn(math.MaxInt16))
		bin2 := NewBin("Aerospike2", randString(100))

		BeforeEach(func() {
			for i := 0; i < keyCount; i++ {
				key, err = NewKey(ns, set, i)
				Expect(err).ToNot(HaveOccurred())

				_, err = client.Operate(wpolicy, key, PutOp(bin1), PutOp(bin2), GetOp())
				Expect(err).ToNot(HaveOccurred())
			}
		})

		var countRecords = func(namespace, setName string) int {
			stmt := NewStatement(namespace, setName)
			res, err := client.Query(nil, stmt)
			Expect(err).ToNot(HaveOccurred())

			cnt := 0
			for rec := range res.Results() {
				Expect(rec.Err).ToNot(HaveOccurred())
				cnt++
			}

			return cnt
		}

		It("must truncate only the current set", func() {
			Expect(countRecords(ns, set)).To(Equal(keyCount))

			err := client.Truncate(nil, ns, set, nil)
			Expect(err).ToNot(HaveOccurred())

			time.Sleep(time.Second)
			Expect(countRecords(ns, set)).To(Equal(0))
		})

		It("must truncate the whole namespace", func() {
			Expect(countRecords(ns, "")).ToNot(Equal(0))

			err := client.Truncate(nil, ns, "", nil)
			Expect(err).ToNot(HaveOccurred())

			time.Sleep(time.Second)
			Expect(countRecords(ns, "")).To(Equal(0))
		})

		It("must truncate only older records", func() {
			t := time.Now()

			Expect(countRecords(ns, set)).To(Equal(keyCount))

			for i := keyCount; i < 2*keyCount; i++ {
				key, err = NewKey(ns, set, i)
				Expect(err).ToNot(HaveOccurred())

				_, err = client.Operate(wpolicy, key, PutOp(bin1), PutOp(bin2), GetOp())
				Expect(err).ToNot(HaveOccurred())
			}
			Expect(countRecords(ns, set)).To(Equal(2 * keyCount))

			err := client.Truncate(nil, ns, set, &t)
			Expect(err).ToNot(HaveOccurred())

			time.Sleep(3 * time.Second)
			Expect(countRecords(ns, set)).To(Equal(keyCount))
		})

	})
})
<|endoftext|>


<commit_before>package ttlcache

import (
	"testing"
	"time"
)

func TestGetExisting(t *testing.T) {
	c := New(1 * time.Minute)
	key, value := "key", "value"
	c.Set(key, value)

	gotValue, ok := c.Get(key)

	if !ok {
		t.Error("ok = false, want true")
	}

	if got, want := gotValue, value; got != want {
		t.Errorf("c.Get(%q) = %v, want %q", key, got, want)
	}
}

func TestGetNonExistent(t *testing.T) {
	c := New(1 * time.Minute)
	key := "key"
	c.Set(key, "value")

	gotValue, ok := c.Get("no-key")

	if ok {
		t.Error("ok = true, want false")
	}

	if gotValue != nil {
		t.Errorf("c.Get(%q) = %v, want nil", key, gotValue)
	}
}

func TestGetTTL(t *testing.T) {
	c := New(10 * time.Millisecond)
	key, value := "key", "value"

	c.Set(key, value)
	time.Sleep(20 * time.Millisecond)

	gotValue, ok := c.Get(key)

	if gotValue != nil || ok {
		t.Errorf("c.Get(%q) = %v, %v; want <nil>, false", key, gotValue, ok)
	}
}

func TestExpire(t *testing.T) {
	c := New(1 * time.Minute)
	key, value := "key", "value"

	c.Set(key, value)
	c.Expire(key)

	gotValue, ok := c.Get(key)

	if gotValue != nil || ok {
		t.Errorf("c.Get(%q) = %v, %v; want <nil>, false", key, gotValue, ok)
	}
}

func TestExpireAll(t *testing.T) {
	c := New(1 * time.Minute)

	for i := 0; i < 10; i++ {
		c.Set(i, "value")
	}

	c.ExpireAll()

	key := 0
	gotValue, ok := c.Get(key)

	if gotValue != nil || ok {
		t.Errorf("c.Get(%d) = %q, %t; want <nil>, false", key, gotValue, ok)
	}

	if got, want := len(c.(*cache).items), 0; got != want {
		t.Errorf("cache has %d items, want %d", got, want)
	}
}

func TestSetTTLReset(t *testing.T) {
	c := New(20 * time.Millisecond)
	key, value := "key", "value"

	for i := 0; i < 10; i++ {
		c.Set(key, value)
		time.Sleep(10 * time.Millisecond)
	}

	gotValue, ok := c.Get(key)

	if gotValue != value || !ok {
		t.Errorf("c.Get(%q) = %v, %v; want %q, true", key, gotValue, ok, value)
	}
}
<commit_msg>Add benchmarks<commit_after>package ttlcache

import (
	"testing"
	"time"
)

func TestGetExisting(t *testing.T) {
	c := New(1 * time.Minute)
	key, value := "key", "value"
	c.Set(key, value)

	gotValue, ok := c.Get(key)

	if !ok {
		t.Error("ok = false, want true")
	}

	if got, want := gotValue, value; got != want {
		t.Errorf("c.Get(%q) = %v, want %q", key, got, want)
	}
}

func TestGetNonExistent(t *testing.T) {
	c := New(1 * time.Minute)
	key := "key"
	c.Set(key, "value")

	gotValue, ok := c.Get("no-key")

	if ok {
		t.Error("ok = true, want false")
	}

	if gotValue != nil {
		t.Errorf("c.Get(%q) = %v, want nil", key, gotValue)
	}
}

func TestGetTTL(t *testing.T) {
	c := New(10 * time.Millisecond)
	key, value := "key", "value"

	c.Set(key, value)
	time.Sleep(20 * time.Millisecond)

	gotValue, ok := c.Get(key)

	if gotValue != nil || ok {
		t.Errorf("c.Get(%q) = %v, %v; want <nil>, false", key, gotValue, ok)
	}
}

func TestExpire(t *testing.T) {
	c := New(1 * time.Minute)
	key, value := "key", "value"

	c.Set(key, value)
	c.Expire(key)

	gotValue, ok := c.Get(key)

	if gotValue != nil || ok {
		t.Errorf("c.Get(%q) = %v, %v; want <nil>, false", key, gotValue, ok)
	}
}

func TestExpireAll(t *testing.T) {
	c := New(1 * time.Minute)

	for i := 0; i < 10; i++ {
		c.Set(i, "value")
	}

	c.ExpireAll()

	key := 0
	gotValue, ok := c.Get(key)

	if gotValue != nil || ok {
		t.Errorf("c.Get(%d) = %q, %t; want <nil>, false", key, gotValue, ok)
	}

	if got, want := len(c.(*cache).items), 0; got != want {
		t.Errorf("cache has %d items, want %d", got, want)
	}
}

func TestSetTTLReset(t *testing.T) {
	c := New(20 * time.Millisecond)
	key, value := "key", "value"

	for i := 0; i < 10; i++ {
		c.Set(key, value)
		time.Sleep(10 * time.Millisecond)
	}

	gotValue, ok := c.Get(key)

	if gotValue != value || !ok {
		t.Errorf("c.Get(%q) = %v, %v; want %q, true", key, gotValue, ok, value)
	}
}

func BenchmarkGetExisting(b *testing.B) {
	c := New(5 * time.Minute)

	const numKeys = 100000

	for key := 0; key < numKeys; key++ {
		c.Set(key, "value")
	}

	b.ResetTimer()

	for i := 0; i < b.N; i++ {
		c.Get(i % numKeys)
	}

	b.StopTimer()

	c.ExpireAll()
}

func BenchmarkGetNonExistent(b *testing.B) {
	c := New(5 * time.Minute)

	b.ResetTimer()

	for i := 0; i < b.N; i++ {
		c.Get(i)
	}

	b.StopTimer()

	c.ExpireAll()
}

func BenchmarkSetExisting(b *testing.B) {
	c := New(5 * time.Minute)

	const numKeys = 100000

	for key := 0; key < numKeys; key++ {
		c.Set(key, "value")
	}

	b.ResetTimer()

	for i := 0; i < b.N; i++ {
		c.Set(i%numKeys, "value")
	}

	b.StopTimer()

	c.ExpireAll()
}

func BenchmarkSetNonExistent(b *testing.B) {
	c := New(5 * time.Minute)

	const numKeys = 100000

	b.ResetTimer()

	for i := 0; i < b.N; i++ {
		if i%numKeys == 0 {
			b.StopTimer()
			c.ExpireAll()
			b.StartTimer()
		}
		c.Set(i%numKeys, "value")
	}

	b.StopTimer()

	c.ExpireAll()
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"log"
	"net/http"
	"os"
	"os/signal"
	"time"

	"gopkg.in/jcelliott/turnpike.v2"
)

var (
	realm string
	port  int
	debug bool
)

func init() {
	flag.StringVar(&realm, "realm", "realm1", "realm name")
	flag.IntVar(&port, "port", 8000, "port to run on")
	flag.BoolVar(&debug, "debug", false, "enable debug logging")
}

func main() {
	flag.Parse()
	if debug {
		turnpike.Debug()
	}
	s := turnpike.NewBasicWebsocketServer(realm)
	shutdown := make(chan os.Signal, 1)
	signal.Notify(shutdown, os.Interrupt)
	go func() {
		<-shutdown
		s.Close()
		log.Println("shutting down server...")
		time.Sleep(time.Second)
		os.Exit(1)
	}()

	server := &http.Server{
		Handler: s,
		Addr:    ":8000",
	}
	log.Printf("turnpike server starting on port %d...", port)
	log.Fatal(server.ListenAndServe())
}
<commit_msg>Update import for govendor<commit_after>package main

import (
	"flag"
	"log"
	"net/http"
	"os"
	"os/signal"
	"time"

	"github.com/awakenetworks/turnpike"
)

var (
	realm string
	port  int
	debug bool
)

func init() {
	flag.StringVar(&realm, "realm", "realm1", "realm name")
	flag.IntVar(&port, "port", 8000, "port to run on")
	flag.BoolVar(&debug, "debug", false, "enable debug logging")
}

func main() {
	flag.Parse()
	if debug {
		turnpike.Debug()
	}
	s := turnpike.NewBasicWebsocketServer(realm)
	shutdown := make(chan os.Signal, 1)
	signal.Notify(shutdown, os.Interrupt)
	go func() {
		<-shutdown
		s.Close()
		log.Println("shutting down server...")
		time.Sleep(time.Second)
		os.Exit(1)
	}()

	server := &http.Server{
		Handler: s,
		Addr:    ":8000",
	}
	log.Printf("turnpike server starting on port %d...", port)
	log.Fatal(server.ListenAndServe())
}
<|endoftext|>


<commit_before>package twitch

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"strings"
	"time"
)

var favorites []string
var tournaments []string

func WatchFavorites(callback func(m string)) {
	favorites = FavoriteDota2Streams()
	for {
		time.Sleep(time.Second * 30)
		newFavorites := FavoriteDota2Streams()
		if len(newFavorites) == 0 {
			continue // sometimes the api delivers no results
		}

		for _, g := range newFavorites {
			if !inside(favorites, g) {
				callback(g + " started streaming.")
			}
		}
		favorites = newFavorites
	}
}

func WatchTournaments(callback func(m string)) {
	tournaments = TournamentStreams()
	for {
		time.Sleep(time.Second * 30)
		newTournaments := TournamentStreams()
		if len(newTournaments) == 0 {
			continue // sometimes the api delivers no results
		}

		for _, g := range newTournaments {
			if !inside(tournaments, g) {
				callback(g)
			}
		}
		tournaments = newTournaments
	}
}

func inside(haystack []string, needle string) bool {
	for _, g := range haystack {
		if g == needle {
			return true
		}
	}
	return false
}

func FavoriteDota2Streams() []string {
	f := favoriteList()
	concatenated := strings.Replace(f, "\n", ",", -1)
	requestURL := "https://api.twitch.tv/kraken/streams?game=Dota+2&channel=" + concatenated
	res, err := http.Get(requestURL)
	if err != nil {
		log.Fatal(err)
	}
	streams, err := ioutil.ReadAll(res.Body)
	res.Body.Close()
	if err != nil {
		log.Fatal(err)
	}

	var dat JSONResult
	if err := json.Unmarshal(streams, &dat); err != nil {
		panic(err)
	}

	sslice := make([]string, 0)
	for _, g := range dat.Streams {
		s := fmt.Sprintf("\u0002%s\u000F %s", g.Channel.DisplayName, g.Channel.URL)
		sslice = append(sslice, s)
	}

	return sslice
}

func TournamentStreams() []string {
	t := tournamentsList()
	concatenated := strings.Replace(t, "\n", ",", -1)
	requestURL := "https://api.twitch.tv/kraken/streams?game=Dota+2&channel=" + concatenated
	res, err := http.Get(requestURL)
	if err != nil {
		log.Fatal(err)
	}
	streams, err := ioutil.ReadAll(res.Body)
	res.Body.Close()
	if err != nil {
		log.Fatal(err)
	}

	var dat JSONResult
	if err := json.Unmarshal(streams, &dat); err != nil {
		panic(err)
	}

	sslice := make([]string, 0)
	for _, g := range dat.Streams {
		if isRebroadcast(g.Channel.Status) {
			continue
		}

		if containsVersus(g.Channel.Status) || containsLive(g.Channel.Status) {
			s := fmt.Sprintf("%s %s", g.Channel.Status, g.Channel.URL)
			sslice = append(sslice, s)
		}
	}

	return sslice
}

func TopDota2Streams() []string {
	requestURL := "https://api.twitch.tv/kraken/streams?game=Dota+2&limit=15"
	res, err := http.Get(requestURL)
	if err != nil {
		log.Fatal(err)
	}
	streams, err := ioutil.ReadAll(res.Body)
	res.Body.Close()
	if err != nil {
		log.Fatal(err)
	}

	var dat JSONResult
	if err := json.Unmarshal(streams, &dat); err != nil {
		panic(err)
	}

	limitOfStreams := 5
	c := 0

	sslice := make([]string, 0)
	for _, g := range dat.Streams {
		if c == limitOfStreams {
			break
		}
		if !isBlacklisted(g.Channel.Name) && g.Viewers > 100 && !isRebroadcast(g.Channel.Status) {
			s := fmt.Sprintf("\u0002%s\u000F (%d) %s", g.Channel.DisplayName, g.Viewers, g.Channel.URL)
			sslice = append(sslice, s)
			c++
		}
	}

	return sslice
}

func Dota2Streams() []string {
	requestURL := "https://api.twitch.tv/kraken/streams?game=Dota+2&limit=5"
	res, err := http.Get(requestURL)
	if err != nil {
		log.Fatal(err)
	}
	streams, err := ioutil.ReadAll(res.Body)
	res.Body.Close()
	if err != nil {
		log.Fatal(err)
	}

	var dat JSONResult
	if err := json.Unmarshal(streams, &dat); err != nil {
		panic(err)
	}

	sslice := make([]string, 0)
	for _, g := range dat.Streams {
		s := fmt.Sprintf("\u0002%s\u000F (%d) %s", g.Channel.DisplayName, g.Viewers, g.Channel.URL)
		sslice = append(sslice, s)
	}
	return sslice
}

func clientID() string {
	file, e := ioutil.ReadFile("./client.id")
	if e != nil {
		panic(e)
	}
	return string(file)
}

func favoriteList() string {
	file, e := ioutil.ReadFile("./favorites.txt")
	if e != nil {
		panic(e)
	}
	return string(file)
}

func tournamentsList() string {
	file, e := ioutil.ReadFile("./tournaments.txt")
	if e != nil {
		panic(e)
	}
	return string(file)
}

func isRebroadcast(stream string) bool {
	s := strings.ToLower(stream)
	return strings.Contains(s, "rebroadcast")
}

func containsVersus(stream string) bool {
	s := strings.ToLower(stream)
	return strings.Contains(s, " vs ")
}

func containsLive(stream string) bool {
	s := strings.ToLower(stream)
	return strings.Contains(s, "live")
}

func blacklistStreams() []string {
	file, e := ioutil.ReadFile("./blacklist.txt")
	if e != nil {
		panic(e)
	}
	return strings.Split(string(file), "\n")
}

func russianStreams() []string {
	file, e := ioutil.ReadFile("./russians.txt")
	if e != nil {
		panic(e)
	}
	return strings.Split(string(file), "\n")
}

func isBlacklisted(stream string) bool {
	blacklist := russianStreams()
	for _, b := range blacklist {
		if b == stream {
			return true
		}
	}
	blacklist = blacklistStreams()
	for _, b := range blacklist {
		if b == stream {
			return true
		}
	}
	return false
}

// JSON structs
type JSONResult struct {
	Streams []JSONStreams `json:"streams"`
}

type JSONStreams struct {
	Channel JSONChannel `json:"channel"`
	Viewers int         `json:"viewers"`
}

type JSONChannel struct {
	DisplayName string `json:"display_name"`
	Name        string `json:"name"`
	URL         string `json:"url"`
	Status      string `json:"status"`
}
<commit_msg>changed limit to 4<commit_after>package twitch

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"strings"
	"time"
)

var favorites []string
var tournaments []string

func WatchFavorites(callback func(m string)) {
	favorites = FavoriteDota2Streams()
	for {
		time.Sleep(time.Second * 30)
		newFavorites := FavoriteDota2Streams()
		if len(newFavorites) == 0 {
			continue // sometimes the api delivers no results
		}

		for _, g := range newFavorites {
			if !inside(favorites, g) {
				callback(g + " started streaming.")
			}
		}
		favorites = newFavorites
	}
}

func WatchTournaments(callback func(m string)) {
	tournaments = TournamentStreams()
	for {
		time.Sleep(time.Second * 30)
		newTournaments := TournamentStreams()
		if len(newTournaments) == 0 {
			continue // sometimes the api delivers no results
		}

		for _, g := range newTournaments {
			if !inside(tournaments, g) {
				callback(g)
			}
		}
		tournaments = newTournaments
	}
}

func inside(haystack []string, needle string) bool {
	for _, g := range haystack {
		if g == needle {
			return true
		}
	}
	return false
}

func FavoriteDota2Streams() []string {
	f := favoriteList()
	concatenated := strings.Replace(f, "\n", ",", -1)
	requestURL := "https://api.twitch.tv/kraken/streams?game=Dota+2&channel=" + concatenated
	res, err := http.Get(requestURL)
	if err != nil {
		log.Fatal(err)
	}
	streams, err := ioutil.ReadAll(res.Body)
	res.Body.Close()
	if err != nil {
		log.Fatal(err)
	}

	var dat JSONResult
	if err := json.Unmarshal(streams, &dat); err != nil {
		panic(err)
	}

	sslice := make([]string, 0)
	for _, g := range dat.Streams {
		s := fmt.Sprintf("\u0002%s\u000F %s", g.Channel.DisplayName, g.Channel.URL)
		sslice = append(sslice, s)
	}

	return sslice
}

func TournamentStreams() []string {
	t := tournamentsList()
	concatenated := strings.Replace(t, "\n", ",", -1)
	requestURL := "https://api.twitch.tv/kraken/streams?game=Dota+2&channel=" + concatenated
	res, err := http.Get(requestURL)
	if err != nil {
		log.Fatal(err)
	}
	streams, err := ioutil.ReadAll(res.Body)
	res.Body.Close()
	if err != nil {
		log.Fatal(err)
	}

	var dat JSONResult
	if err := json.Unmarshal(streams, &dat); err != nil {
		panic(err)
	}

	sslice := make([]string, 0)
	for _, g := range dat.Streams {
		if isRebroadcast(g.Channel.Status) {
			continue
		}

		if containsVersus(g.Channel.Status) || containsLive(g.Channel.Status) {
			s := fmt.Sprintf("%s %s", g.Channel.Status, g.Channel.URL)
			sslice = append(sslice, s)
		}
	}

	return sslice
}

func TopDota2Streams() []string {
	requestURL := "https://api.twitch.tv/kraken/streams?game=Dota+2&limit=15"
	res, err := http.Get(requestURL)
	if err != nil {
		log.Fatal(err)
	}
	streams, err := ioutil.ReadAll(res.Body)
	res.Body.Close()
	if err != nil {
		log.Fatal(err)
	}

	var dat JSONResult
	if err := json.Unmarshal(streams, &dat); err != nil {
		panic(err)
	}

	limitOfStreams := 4
	c := 0

	sslice := make([]string, 0)
	for _, g := range dat.Streams {
		if c == limitOfStreams {
			break
		}
		if !isBlacklisted(g.Channel.Name) && g.Viewers > 100 && !isRebroadcast(g.Channel.Status) {
			s := fmt.Sprintf("\u0002%s\u000F (%d) %s", g.Channel.DisplayName, g.Viewers, g.Channel.URL)
			sslice = append(sslice, s)
			c++
		}
	}

	return sslice
}

func Dota2Streams() []string {
	requestURL := "https://api.twitch.tv/kraken/streams?game=Dota+2&limit=4"
	res, err := http.Get(requestURL)
	if err != nil {
		log.Fatal(err)
	}
	streams, err := ioutil.ReadAll(res.Body)
	res.Body.Close()
	if err != nil {
		log.Fatal(err)
	}

	var dat JSONResult
	if err := json.Unmarshal(streams, &dat); err != nil {
		panic(err)
	}

	sslice := make([]string, 0)
	for _, g := range dat.Streams {
		s := fmt.Sprintf("\u0002%s\u000F (%d) %s", g.Channel.DisplayName, g.Viewers, g.Channel.URL)
		sslice = append(sslice, s)
	}
	return sslice
}

func clientID() string {
	file, e := ioutil.ReadFile("./client.id")
	if e != nil {
		panic(e)
	}
	return string(file)
}

func favoriteList() string {
	file, e := ioutil.ReadFile("./favorites.txt")
	if e != nil {
		panic(e)
	}
	return string(file)
}

func tournamentsList() string {
	file, e := ioutil.ReadFile("./tournaments.txt")
	if e != nil {
		panic(e)
	}
	return string(file)
}

func isRebroadcast(stream string) bool {
	s := strings.ToLower(stream)
	return strings.Contains(s, "rebroadcast")
}

func containsVersus(stream string) bool {
	s := strings.ToLower(stream)
	return strings.Contains(s, " vs ")
}

func containsLive(stream string) bool {
	s := strings.ToLower(stream)
	return strings.Contains(s, "live")
}

func blacklistStreams() []string {
	file, e := ioutil.ReadFile("./blacklist.txt")
	if e != nil {
		panic(e)
	}
	return strings.Split(string(file), "\n")
}

func russianStreams() []string {
	file, e := ioutil.ReadFile("./russians.txt")
	if e != nil {
		panic(e)
	}
	return strings.Split(string(file), "\n")
}

func isBlacklisted(stream string) bool {
	blacklist := russianStreams()
	for _, b := range blacklist {
		if b == stream {
			return true
		}
	}
	blacklist = blacklistStreams()
	for _, b := range blacklist {
		if b == stream {
			return true
		}
	}
	return false
}

// JSON structs
type JSONResult struct {
	Streams []JSONStreams `json:"streams"`
}

type JSONStreams struct {
	Channel JSONChannel `json:"channel"`
	Viewers int         `json:"viewers"`
}

type JSONChannel struct {
	DisplayName string `json:"display_name"`
	Name        string `json:"name"`
	URL         string `json:"url"`
	Status      string `json:"status"`
}
<|endoftext|>


<commit_before>// Usage: go test -run=TestCalibrate -calibrate

package bigfft

import (
	"flag"
	"fmt"
	"testing"
	"time"
)

var calibrate = flag.Bool("calibrate", false, "run calibration test")

// measureMul benchmarks math/big versus FFT for a given input size
// (in bits).
func measureMul(th int) (tBig, tFFT time.Duration) {
	bigLoad := func(b *testing.B) { benchmarkMulBig(b, th, th) }
	fftLoad := func(b *testing.B) { benchmarkMulFFT(b, th, th) }

	res1 := testing.Benchmark(bigLoad)
	res2 := testing.Benchmark(fftLoad)
	tBig = time.Duration(res1.NsPerOp())
	tFFT = time.Duration(res2.NsPerOp())
	return
}

func TestCalibrateThreshold(t *testing.T) {
	if !*calibrate {
		t.Log("not calibrating, use -calibrate to do so.")
		return
	}

	lower := int(1e3)   // math/big is faster at this size.
	upper := int(300e3) // FFT is faster at this size.

	big, fft := measureMul(lower)
	lowerX := float64(big) / float64(fft)
	fmt.Printf("speedup at size %d: %.2f\n", lower, lowerX)
	big, fft = measureMul(upper)
	upperX := float64(big) / float64(fft)
	fmt.Printf("speedup at size %d: %.2f\n", upper, upperX)
	for {
		mid := (lower + upper) / 2
		big, fft := measureMul(mid)
		X := float64(big) / float64(fft)
		fmt.Printf("speedup at size %d: %.2f\n", mid, X)
		switch {
		case X < 0.98:
			lower = mid
			lowerX = X
		case X > 1.02:
			upper = mid
			upperX = X
		default:
			fmt.Printf("speedup at size %d: %.2f\n", lower, lowerX)
			fmt.Printf("speedup at size %d: %.2f\n", upper, upperX)
			return
		}
	}
}

func measureFFTSize(w int, k uint) time.Duration {
	load := func(b *testing.B) {
		x := rndNat(w)
		y := rndNat(w)
		for i := 0; i < b.N; i++ {
			m := (w+w)>>k + 1
			xp := polyFromNat(x, k, m)
			yp := polyFromNat(y, k, m)
			rp := xp.Mul(&yp)
			_ = rp.Int()
		}
	}
	res := testing.Benchmark(load)
	return time.Duration(res.NsPerOp())
}

func TestCalibrateFFT(t *testing.T) {
	if !*calibrate {
		t.Log("not calibrating, use -calibrate to do so.")
		return
	}

K:
	for k := uint(3); k < 16; k++ {
		// Measure the speedup between k and k+1
		low := 10 << k
		hi := 500 << k
		low1, low2 := measureFFTSize(low, k), measureFFTSize(low, k+1)
		lowX := float64(low1) / float64(low2) // less than 1
		fmt.Printf("speedup of %d vs %d at size %d words: %.2f\n", k+1, k, low, lowX)
		hi1, hi2 := measureFFTSize(hi, k), measureFFTSize(hi, k+1)
		hiX := float64(hi1) / float64(hi2) // larger than 1
		fmt.Printf("speedup of %d vs %d at size %d words: %.2f\n", k+1, k, hi, hiX)
		for i := 0; i < 10; i++ {
			mid := (low + hi) / 2
			mid1, mid2 := measureFFTSize(mid, k), measureFFTSize(mid, k+1)
			midX := float64(mid1) / float64(mid2)
			fmt.Printf("speedup of %d vs %d at size %d words: %.2f\n", k+1, k, mid, midX)
			switch {
			case midX < 0.98:
				low = mid
				lowX = midX
			case midX > 1.02:
				hi = mid
				hiX = midX
			default:
				fmt.Printf("speedup of %d vs %d at size %d words: %.2f\n", k+1, k, low, lowX)
				fmt.Printf("speedup of %d vs %d at size %d words: %.2f\n", k+1, k, hi, hiX)
				continue K
			}
		}
	}
}
<commit_msg>Make FFT size calibration test finer.<commit_after>// Usage: go test -run=TestCalibrate -calibrate

package bigfft

import (
	"flag"
	"fmt"
	"testing"
	"time"
)

var calibrate = flag.Bool("calibrate", false, "run calibration test")

// measureMul benchmarks math/big versus FFT for a given input size
// (in bits).
func measureMul(th int) (tBig, tFFT time.Duration) {
	bigLoad := func(b *testing.B) { benchmarkMulBig(b, th, th) }
	fftLoad := func(b *testing.B) { benchmarkMulFFT(b, th, th) }

	res1 := testing.Benchmark(bigLoad)
	res2 := testing.Benchmark(fftLoad)
	tBig = time.Duration(res1.NsPerOp())
	tFFT = time.Duration(res2.NsPerOp())
	return
}

func TestCalibrateThreshold(t *testing.T) {
	if !*calibrate {
		t.Log("not calibrating, use -calibrate to do so.")
		return
	}

	lower := int(1e3)   // math/big is faster at this size.
	upper := int(300e3) // FFT is faster at this size.

	big, fft := measureMul(lower)
	lowerX := float64(big) / float64(fft)
	fmt.Printf("speedup at size %d: %.2f\n", lower, lowerX)
	big, fft = measureMul(upper)
	upperX := float64(big) / float64(fft)
	fmt.Printf("speedup at size %d: %.2f\n", upper, upperX)
	for {
		mid := (lower + upper) / 2
		big, fft := measureMul(mid)
		X := float64(big) / float64(fft)
		fmt.Printf("speedup at size %d: %.2f\n", mid, X)
		switch {
		case X < 0.98:
			lower = mid
			lowerX = X
		case X > 1.02:
			upper = mid
			upperX = X
		default:
			fmt.Printf("speedup at size %d: %.2f\n", lower, lowerX)
			fmt.Printf("speedup at size %d: %.2f\n", upper, upperX)
			return
		}
	}
}

func measureFFTSize(w int, k uint) time.Duration {
	load := func(b *testing.B) {
		x := rndNat(w)
		y := rndNat(w)
		for i := 0; i < b.N; i++ {
			m := (w+w)>>k + 1
			xp := polyFromNat(x, k, m)
			yp := polyFromNat(y, k, m)
			rp := xp.Mul(&yp)
			_ = rp.Int()
		}
	}
	res := testing.Benchmark(load)
	return time.Duration(res.NsPerOp())
}

func TestCalibrateFFT(t *testing.T) {
	if !*calibrate {
		t.Log("not calibrating, use -calibrate to do so.")
		return
	}

	lows := [...]int{10, 10, 10, 10,
		20, 50, 100, 200, 500, // 8
		1000, 2000, 5000, 10000, // 12
		20000, 50000, 100e3, 200e3, // 16
	}
	his := [...]int{100, 100, 100, 200,
		500, 1000, 2000, 5000, 10000, // 8
		50e3, 100e3, 200e3, 800e3, // 12
		2e6, 5e6, 10e6, 20e6, // 16
	}
	for k := uint(3); k < 16; k++ {
		// Measure the speedup between k and k+1
		low := lows[k] // FFT of size 1<<k known to be faster
		hi := his[k]   // FFT of size 2<<k known to be faster
		var sizes [9]int
		var speedups [9]float64
		for i := 0; i < 3; i++ {
			for idx := 1; idx <= 9; idx++ {
				sz := ((10-idx)*low + idx*hi) / 10
				t1, t2 := measureFFTSize(sz, k), measureFFTSize(sz, k+1)
				spd := float64(t1) / float64(t2)
				sizes[idx-1] = sz
				speedups[idx-1] = spd
				fmt.Printf("speedup of %d vs %d at size %d words: %.2f\n", k+1, k, sz, spd)
			}
			narrow := false
			for idx, s := range speedups {
				if s < .98 {
					low = sizes[idx]
					narrow = true
				} else {
					break
				}
			}
			for idx := range speedups {
				if speedups[8-idx] > 1.02 {
					hi = sizes[8-idx]
					narrow = true
				} else {
					break
				}
			}
			if low >= hi {
				panic("impossible")
			}
			if !narrow || (hi-low) <= 10 {
				break
			}
		}
		fmt.Printf("sizes: %d\n", sizes)
		fmt.Printf("speedups: %.2f\n", speedups)
	}
}
<|endoftext|>