<commit_before>package ledger

import (
	"errors"
	"fmt"
	"math"
	"sort"
	"strconv"
	"strings"
)

type Entry struct {
	Date        string // "Y-m-d"
	Description string
	Change      int // in cents
}

type Locale struct {
	Date              string
	Description       string
	Change            string
	DateSeperator     string
	DecimalPoint      string
	IntegralSeperator string
}

var locales = map[string]Locale{
	"en-US": {
		Date:              "Date",
		Description:       "Description",
		Change:            "Change",
		DateSeperator:     "/",
		DecimalPoint:      ".",
		IntegralSeperator: ",",
	},
	"nl-NL": {
		Date:              "Datum",
		Description:       "Omschrijving",
		Change:            "Verandering",
		DateSeperator:     "-",
		DecimalPoint:      ",",
		IntegralSeperator: ".",
	},
}

var currencyToSymbol = map[string]string{
	"USD": "$",
	"EUR": "€",
}

func FormatLedger(currency string, locale string, entries []Entry) (output string, e error) {
	if !isValidCurrency(currency) {
		return "", errors.New("invalid currency")
	}
	if !isValidLocale(locale) {
		return "", errors.New("invalid locale")
	}
	if !isValidDate(entries) {
		return "", errors.New("invalid date")
	}

	output += header(locale)
	for _, entry := range sortEntries(entries) {
		output += formatEntry(locale, currency, entry)
	}
	return output, nil
}

func sortEntries(entries []Entry) (sorted []Entry) {
	sorted = make([]Entry, len(entries))
	copy(sorted, entries)
	sort.Slice(sorted, func(i int, j int) bool {
		a := entries[i]
		b := entries[j]
		if a.Date != b.Date {
			return a.Date < b.Date
		} else if a.Description != b.Description {
			return a.Description < b.Description
		} else {
			return a.Change < b.Change
		}
	})
	return sorted
}

func formatEntry(locale string, currency string, entry Entry) (formatted string) {
	date := formatDate(locale, entry.Date)
	description := formatDescription(entry.Description)
	change := formatChange(locale, currency, entry.Change)

	// This conditional is necessary because expected output is aligned
	// differently for negative vs. non-negative values
	if isNegative(entry.Change) {
		return fmt.Sprintf("%-10s | %-25s | %13s\n", date, description, change)
	} else {
		return fmt.Sprintf("%-10s | %-25s | %12s\n", date, description, change)
	}
}

func header(locale string) (output string) {
	return fmt.Sprintf("%-10s | %-25s | %s\n", locales[locale].Date, locales[locale].Description, locales[locale].Change)
}

func formatDate(locale string, date string) string {
	year, month, day := date[0:4], date[5:7], date[8:10]

	if locale == "nl-NL" {
		return strings.Join([]string{day, month, year}, locales[locale].DateSeperator)
	} else if locale == "en-US" {
		return strings.Join([]string{month, day, year}, locales[locale].DateSeperator)
	}
	panic("invalid locale")
}

// formatDescription will ellipsize the description if it is longer than 25
// characters
func formatDescription(description string) string {
	if len(description) > 25 {
		return description[:22] + "..."
	}
	return description
}

func formatChange(locale string, currency string, cents int) (change string) {
	isNegative := isNegative(cents)
	paddedChange := fmt.Sprintf("%03s", strconv.Itoa(absoluteValue(cents)))
	change += formatCurrencySymbol(locale, currency)
	change += formatNumber(locale, paddedChange)
	if isNegative {
		change = formatNegative(locale, change)
	}
	return change
}

func formatNumber(locale string, paddedChange string) (formatted string) {
	integralPart := formatIntegralPart(paddedChange, locale)
	decimalPart := paddedChange[len(paddedChange)-2:]
	return strings.Join([]string{integralPart, decimalPart}, locales[locale].DecimalPoint)
}

func formatCurrencySymbol(locale string, currency string) (formatted string) {
	if locale == "en-US" {
		return currencyToSymbol[currency]
	} else if locale == "nl-NL" {
		return currencyToSymbol[currency] + " "
	}
	panic("invalid locale")
}

func formatNegative(locale string, change string) (formatted string) {
	if locale == "en-US" {
		return fmt.Sprintf("(%s)", change)
	} else if locale == "nl-NL" {
		return fmt.Sprintf("%s-", change)
	}
	panic("invalid locale")
}

func formatIntegralPart(centsStr string, locale string) (formatted string) {
	var parts []string
	remainder := centsStr[:len(centsStr)-2]
	for len(remainder) > 3 {
		parts = append([]string{remainder[len(remainder)-3:]}, parts...)
		remainder = remainder[:len(remainder)-3]
	}
	if len(remainder) > 0 {
		parts = append([]string{remainder}, parts...)
	}
	return strings.Join(parts, locales[locale].IntegralSeperator)
}

func isValidCurrency(currency string) bool {
	_, ok := currencyToSymbol[currency]
	return ok
}

func isValidLocale(locale string) bool {
	_, ok := locales[locale]
	return ok
}

func isValidDate(entries []Entry) bool {
	for _, entry := range entries {
		if len(entry.Date) != 10 {
			return false
		}
		_, d2, _, d4, _ := entry.Date[0:4], entry.Date[4], entry.Date[5:7], entry.Date[7], entry.Date[8:10]
		if d2 != '-' || d4 != '-' {
			return false
		}
	}
	return true
}

func isNegative(cents int) bool {
	return cents < 0
}

func absoluteValue(n int) int {
	return int(math.Abs(float64(n)))
}
<commit_msg>Misc cleanup<commit_after>package ledger

import (
	"errors"
	"fmt"
	"math"
	"sort"
	"strconv"
	"strings"
)

type Entry struct {
	Date        string // "Y-m-d"
	Description string
	Change      int // in cents
}

type Locale struct {
	Date              string
	Description       string
	Change            string
	DateSeperator     string
	DecimalPoint      string
	IntegralSeperator string
}

var locales = map[string]Locale{
	"en-US": {
		Date:              "Date",
		Description:       "Description",
		Change:            "Change",
		DateSeperator:     "/",
		DecimalPoint:      ".",
		IntegralSeperator: ",",
	},
	"nl-NL": {
		Date:              "Datum",
		Description:       "Omschrijving",
		Change:            "Verandering",
		DateSeperator:     "-",
		DecimalPoint:      ",",
		IntegralSeperator: ".",
	},
}

var currencyToSymbol = map[string]string{
	"USD": "$",
	"EUR": "€",
}

func FormatLedger(currency string, locale string, entries []Entry) (output string, e error) {
	if !isValidCurrency(currency) {
		return "", errors.New("invalid currency")
	}
	if !isValidLocale(locale) {
		return "", errors.New("invalid locale")
	}
	if !isValidDate(entries) {
		return "", errors.New("invalid date")
	}

	output += header(locale)
	for _, entry := range sortEntries(entries) {
		output += formatEntry(locale, currency, entry)
	}
	return output, nil
}

func sortEntries(entries []Entry) (sorted []Entry) {
	sorted = make([]Entry, len(entries))
	copy(sorted, entries)
	sort.Slice(sorted, func(i int, j int) bool {
		a := entries[i]
		b := entries[j]
		if a.Date != b.Date {
			return a.Date < b.Date
		} else if a.Description != b.Description {
			return a.Description < b.Description
		} else {
			return a.Change < b.Change
		}
	})
	return sorted
}

func formatEntry(locale string, currency string, entry Entry) (formatted string) {
	date := formatDate(locale, entry.Date)
	description := formatDescription(entry.Description)
	change := formatChange(locale, currency, entry.Change)

	// This conditional is necessary because expected output is aligned
	// differently for negative vs. non-negative values
	if isNegative(entry.Change) {
		return fmt.Sprintf("%-10s | %-25s | %13s\n", date, description, change)
	} else {
		return fmt.Sprintf("%-10s | %-25s | %12s\n", date, description, change)
	}
}

func header(locale string) (output string) {
	return fmt.Sprintf("%-10s | %-25s | %s\n", locales[locale].Date, locales[locale].Description, locales[locale].Change)
}

func formatDate(locale string, date string) string {
	year, month, day := date[0:4], date[5:7], date[8:10]

	if locale == "nl-NL" {
		return strings.Join([]string{day, month, year}, locales[locale].DateSeperator)
	} else if locale == "en-US" {
		return strings.Join([]string{month, day, year}, locales[locale].DateSeperator)
	}
	panic("invalid locale")
}

// formatDescription will ellipsize the description if it is longer than 25
// characters
func formatDescription(description string) string {
	if len(description) > 25 {
		return description[:22] + "..."
	}
	return description
}

func formatChange(locale string, currency string, cents int) (change string) {
	paddedChange := fmt.Sprintf("%03s", strconv.Itoa(absoluteValue(cents)))
	change += formatCurrencySymbol(locale, currency)
	change += formatNumber(locale, paddedChange)
	if isNegative(cents) {
		change = formatNegative(locale, change)
	}
	return change
}

func formatNumber(locale string, paddedChange string) (formatted string) {
	integralPart := formatIntegralPart(locale, paddedChange)
	decimalPart := paddedChange[len(paddedChange)-2:]
	return strings.Join([]string{integralPart, decimalPart}, locales[locale].DecimalPoint)
}

func formatCurrencySymbol(locale string, currency string) (formatted string) {
	if locale == "en-US" {
		return currencyToSymbol[currency]
	} else if locale == "nl-NL" {
		return currencyToSymbol[currency] + " "
	}
	panic("invalid locale")
}

func formatNegative(locale string, change string) (formatted string) {
	if locale == "en-US" {
		return fmt.Sprintf("(%s)", change)
	} else if locale == "nl-NL" {
		return fmt.Sprintf("%s-", change)
	}
	panic("invalid locale")
}

func formatIntegralPart(locale string, paddedChange string) (formatted string) {
	var parts []string
	remainder := paddedChange[:len(paddedChange)-2]
	for len(remainder) > 3 {
		parts = append([]string{remainder[len(remainder)-3:]}, parts...)
		remainder = remainder[:len(remainder)-3]
	}
	if len(remainder) > 0 {
		parts = append([]string{remainder}, parts...)
	}
	return strings.Join(parts, locales[locale].IntegralSeperator)
}

func isValidCurrency(currency string) bool {
	_, ok := currencyToSymbol[currency]
	return ok
}

func isValidLocale(locale string) bool {
	_, ok := locales[locale]
	return ok
}

func isValidDate(entries []Entry) bool {
	for _, entry := range entries {
		if len(entry.Date) != 10 {
			return false
		}
		seperatorA, seperatorB := entry.Date[4], entry.Date[7]
		if seperatorA != '-' || seperatorB != '-' {
			return false
		}
	}
	return true
}

func isNegative(cents int) bool {
	return cents < 0
}

func absoluteValue(n int) int {
	return int(math.Abs(float64(n)))
}
<|endoftext|>


<commit_before>// Output to librato - this is temporary

package main

import (
	"bytes"
	"encoding/json"
	"io/ioutil"
	"net/http"
)

type Librato struct {
	Source string
	Email  string
	Token  string
}

func NewLibrato(source, email, token string) *Librato {
	return &Librato{source, email, token}
}

func (l *Librato) Run(on_stats chan (*TimestampedStats)) {
	var stats *TimestampedStats
	for stats = range on_stats {
		// Don't bother posting if there are no metrics (it's an error anyway)
		if len(stats.Counters) == 0 && len(stats.Dists) == 0 {
			debug.Print("[librato] No stats to report")
			continue
		}

		l.Post(stats)
	}
}

func (l *Librato) Post(stats *TimestampedStats) {
	counters := make([]map[string]interface{}, 0)
	for key, value := range stats.Counters {
		counters = append(counters, map[string]interface{}{
			"name":  key,
			"value": value,
		})
	}

	gagues := make([]map[string]interface{}, 0)
	for key, value := range stats.Dists {
		gagues = append(gagues, map[string]interface{}{
			"name":        key,
			"count":       value.N,
			"sum":         value.Sum_x,
			"sum_squares": value.Sum_x2,
			"min":         value.Min,
			"max":         value.Max,
		})
	}

	data := map[string]interface{}{
		"source":       l.Source,
		"measure_time": stats.Timestamp,
		"counters":     counters,
		"gauges":       gagues,
	}

	json_data, err := json.Marshal(data)
	if nil != err {
		info.Printf("[librato] JSON error: %v", err)
		return
	}

	req, err := http.NewRequest(
		"POST",
		"https://metrics-api.librato.com/v1/metrics",
		bytes.NewBuffer(json_data),
	)
	if nil != err {
		info.Printf("[librato] Error creating request: %v", err)
		return
	}

	req.Header.Add("Content-Type", "application/json")
	req.SetBasicAuth(l.Email, l.Token)

	debug.Print("[librato] POSTING")

	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		debug.Printf("[librato] HTTP error: %v", err)
		return
	}

	if resp.StatusCode != 200 {
		debug.Printf("[librato] Invalid status: %v", resp.StatusCode)
		body, _ := ioutil.ReadAll(resp.Body)
		debug.Printf("[librato] HTTP body: %v", string(body))
		return
	}

	// Close the body as requested by the docs
	resp.Body.Close()

	debug.Print("[librato] DONE")
}
<commit_msg>Post counters to librato as gauges<commit_after>// Output to librato - this is temporary

package main

import (
	"bytes"
	"encoding/json"
	"io/ioutil"
	"net/http"
)

type Librato struct {
	Source string
	Email  string
	Token  string
}

func NewLibrato(source, email, token string) *Librato {
	return &Librato{source, email, token}
}

func (l *Librato) Run(on_stats chan (*TimestampedStats)) {
	var stats *TimestampedStats
	for stats = range on_stats {
		// Don't bother posting if there are no metrics (it's an error anyway)
		if len(stats.Counters) == 0 && len(stats.Dists) == 0 {
			debug.Print("[librato] No stats to report")
			continue
		}

		l.Post(stats)
	}
}

func (l *Librato) Post(stats *TimestampedStats) {
	gagues := make([]map[string]interface{}, 0)
	for key, value := range stats.Counters {
		gagues = append(gagues, map[string]interface{}{
			"name":  key,
			"value": value,
		})
	}

	for key, value := range stats.Dists {
		gagues = append(gagues, map[string]interface{}{
			"name":        key,
			"count":       value.N,
			"sum":         value.Sum_x,
			"sum_squares": value.Sum_x2,
			"min":         value.Min,
			"max":         value.Max,
		})
	}

	data := map[string]interface{}{
		"source":       l.Source,
		"measure_time": stats.Timestamp,
		"gauges":       gagues,
	}

	json_data, err := json.Marshal(data)
	if nil != err {
		info.Printf("[librato] JSON error: %v", err)
		return
	}

	req, err := http.NewRequest(
		"POST",
		"https://metrics-api.librato.com/v1/metrics",
		bytes.NewBuffer(json_data),
	)
	if nil != err {
		info.Printf("[librato] Error creating request: %v", err)
		return
	}

	req.Header.Add("Content-Type", "application/json")
	req.SetBasicAuth(l.Email, l.Token)

	debug.Print("[librato] POSTING")

	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		debug.Printf("[librato] HTTP error: %v", err)
		return
	}

	if resp.StatusCode != 200 {
		debug.Printf("[librato] Invalid status: %v", resp.StatusCode)
		body, _ := ioutil.ReadAll(resp.Body)
		debug.Printf("[librato] HTTP body: %v", string(body))
		return
	}

	// Close the body as requested by the docs
	resp.Body.Close()

	debug.Print("[librato] DONE")
}
<|endoftext|>


<commit_before>/*
Copyright IBM Corp. 2016 All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

		 http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package comm

import (
	"errors"
	"sync"
	"sync/atomic"

	"github.com/hyperledger/fabric/gossip/common"
	"github.com/hyperledger/fabric/gossip/util"
	proto "github.com/hyperledger/fabric/protos/gossip"
	"github.com/op/go-logging"
	"golang.org/x/net/context"
	"google.golang.org/grpc"
)

type handler func(message *proto.SignedGossipMessage)

type connFactory interface {
	createConnection(endpoint string, pkiID common.PKIidType) (*connection, error)
}

type connectionStore struct {
	logger           *logging.Logger          // logger
	isClosing        bool                     // whether this connection store is shutting down
	connFactory      connFactory              // creates a connection to remote peer
	sync.RWMutex                              // synchronize access to shared variables
	pki2Conn         map[string]*connection   // mapping between pkiID to connections
	destinationLocks map[string]*sync.RWMutex //mapping between pkiIDs and locks,
	// used to prevent concurrent connection establishment to the same remote endpoint
}

func newConnStore(connFactory connFactory, logger *logging.Logger) *connectionStore {
	return &connectionStore{
		connFactory:      connFactory,
		isClosing:        false,
		pki2Conn:         make(map[string]*connection),
		destinationLocks: make(map[string]*sync.RWMutex),
		logger:           logger,
	}
}

func (cs *connectionStore) getConnection(peer *RemotePeer) (*connection, error) {
	cs.RLock()
	isClosing := cs.isClosing
	cs.RUnlock()

	if isClosing {
		return nil, errors.New("Shutting down")
	}

	pkiID := peer.PKIID
	endpoint := peer.Endpoint

	cs.Lock()
	destinationLock, hasConnected := cs.destinationLocks[string(pkiID)]
	if !hasConnected {
		destinationLock = &sync.RWMutex{}
		cs.destinationLocks[string(pkiID)] = destinationLock
	}
	cs.Unlock()

	destinationLock.Lock()

	cs.RLock()
	conn, exists := cs.pki2Conn[string(pkiID)]
	if exists {
		cs.RUnlock()
		destinationLock.Unlock()
		return conn, nil
	}
	cs.RUnlock()

	createdConnection, err := cs.connFactory.createConnection(endpoint, pkiID)

	destinationLock.Unlock()

	cs.RLock()
	isClosing = cs.isClosing
	cs.RUnlock()
	if isClosing {
		return nil, errors.New("ConnStore is closing")
	}

	cs.Lock()
	delete(cs.destinationLocks, string(pkiID))
	defer cs.Unlock()

	// check again, maybe someone connected to us during the connection creation?
	conn, exists = cs.pki2Conn[string(pkiID)]

	if exists {
		if createdConnection != nil {
			createdConnection.close()
		}
		return conn, nil
	}

	// no one connected to us AND we failed connecting!
	if err != nil {
		return nil, err
	}

	// at this point in the code, we created a connection to a remote peer
	conn = createdConnection
	cs.pki2Conn[string(createdConnection.pkiID)] = conn

	go conn.serviceConnection()

	return conn, nil
}

func (cs *connectionStore) connNum() int {
	cs.RLock()
	defer cs.RUnlock()
	return len(cs.pki2Conn)
}

func (cs *connectionStore) closeConn(peer *RemotePeer) {
	cs.Lock()
	defer cs.Unlock()

	if conn, exists := cs.pki2Conn[string(peer.PKIID)]; exists {
		conn.close()
		delete(cs.pki2Conn, string(conn.pkiID))
	}
}

func (cs *connectionStore) shutdown() {
	cs.Lock()
	cs.isClosing = true
	pkiIds2conn := cs.pki2Conn

	var connections2Close []*connection
	for _, conn := range pkiIds2conn {
		connections2Close = append(connections2Close, conn)
	}
	cs.Unlock()

	wg := sync.WaitGroup{}
	for _, conn := range connections2Close {
		wg.Add(1)
		go func(conn *connection) {
			cs.closeByPKIid(conn.pkiID)
			wg.Done()
		}(conn)
	}
	wg.Wait()
}

func (cs *connectionStore) onConnected(serverStream proto.Gossip_GossipStreamServer, connInfo *proto.ConnectionInfo) *connection {
	cs.Lock()
	defer cs.Unlock()

	if c, exists := cs.pki2Conn[string(connInfo.Identity)]; exists {
		c.close()
	}

	return cs.registerConn(connInfo, serverStream)
}

func (cs *connectionStore) registerConn(connInfo *proto.ConnectionInfo, serverStream proto.Gossip_GossipStreamServer) *connection {
	conn := newConnection(nil, nil, nil, serverStream)
	conn.pkiID = connInfo.ID
	conn.info = connInfo
	conn.logger = cs.logger
	cs.pki2Conn[string(connInfo.ID)] = conn
	return conn
}

func (cs *connectionStore) closeByPKIid(pkiID common.PKIidType) {
	cs.Lock()
	defer cs.Unlock()
	if conn, exists := cs.pki2Conn[string(pkiID)]; exists {
		conn.close()
		delete(cs.pki2Conn, string(pkiID))
	}
}

func newConnection(cl proto.GossipClient, c *grpc.ClientConn, cs proto.Gossip_GossipStreamClient, ss proto.Gossip_GossipStreamServer) *connection {
	connection := &connection{
		outBuff:      make(chan *msgSending, util.GetIntOrDefault("peer.gossip.sendBuffSize", defSendBuffSize)),
		cl:           cl,
		conn:         c,
		clientStream: cs,
		serverStream: ss,
		stopFlag:     int32(0),
		stopChan:     make(chan struct{}, 1),
	}

	return connection
}

type connection struct {
	cancel       context.CancelFunc
	info         *proto.ConnectionInfo
	outBuff      chan *msgSending
	logger       *logging.Logger                 // logger
	pkiID        common.PKIidType                // pkiID of the remote endpoint
	handler      handler                         // function to invoke upon a message reception
	conn         *grpc.ClientConn                // gRPC connection to remote endpoint
	cl           proto.GossipClient              // gRPC stub of remote endpoint
	clientStream proto.Gossip_GossipStreamClient // client-side stream to remote endpoint
	serverStream proto.Gossip_GossipStreamServer // server-side stream to remote endpoint
	stopFlag     int32                           // indicates whether this connection is in process of stopping
	stopChan     chan struct{}                   // a method to stop the server-side gRPC call from a different go-routine
	sync.RWMutex                                 // synchronizes access to shared variables
}

func (conn *connection) close() {
	if conn.toDie() {
		return
	}

	amIFirst := atomic.CompareAndSwapInt32(&conn.stopFlag, int32(0), int32(1))
	if !amIFirst {
		return
	}

	conn.stopChan <- struct{}{}

	conn.Lock()

	if conn.clientStream != nil {
		conn.clientStream.CloseSend()
	}
	if conn.conn != nil {
		conn.conn.Close()
	}

	if conn.cancel != nil {
		conn.cancel()
	}

	conn.Unlock()

}

func (conn *connection) toDie() bool {
	return atomic.LoadInt32(&(conn.stopFlag)) == int32(1)
}

func (conn *connection) send(msg *proto.SignedGossipMessage, onErr func(error)) {
	conn.Lock()
	defer conn.Unlock()

	if len(conn.outBuff) == util.GetIntOrDefault("peer.gossip.sendBuffSize", defSendBuffSize) {
		return
	}

	m := &msgSending{
		envelope: msg.Envelope,
		onErr:    onErr,
	}

	conn.outBuff <- m
}

func (conn *connection) serviceConnection() error {
	errChan := make(chan error, 1)
	msgChan := make(chan *proto.SignedGossipMessage, util.GetIntOrDefault("peer.gossip.recvBuffSize", defRecvBuffSize))
	defer close(msgChan)

	// Call stream.Recv() asynchronously in readFromStream(),
	// and wait for either the Recv() call to end,
	// or a signal to close the connection, which exits
	// the method and makes the Recv() call to fail in the
	// readFromStream() method
	go conn.readFromStream(errChan, msgChan)

	go conn.writeToStream()

	for !conn.toDie() {
		select {
		case stop := <-conn.stopChan:
			conn.logger.Debug("Closing reading from stream")
			conn.stopChan <- stop
			return nil
		case err := <-errChan:
			return err
		case msg := <-msgChan:
			conn.handler(msg)
		}
	}
	return nil
}

func (conn *connection) writeToStream() {
	for !conn.toDie() {
		stream := conn.getStream()
		if stream == nil {
			conn.logger.Error(conn.pkiID, "Stream is nil, aborting!")
			return
		}
		select {
		case m := <-conn.outBuff:
			err := stream.Send(m.envelope)
			if err != nil {
				go m.onErr(err)
				return
			}
		case stop := <-conn.stopChan:
			conn.logger.Debug("Closing writing to stream")
			conn.stopChan <- stop
			return
		}
	}
}

func (conn *connection) readFromStream(errChan chan error, msgChan chan *proto.SignedGossipMessage) {
	defer func() {
		recover()
	}() // msgChan might be closed
	for !conn.toDie() {
		stream := conn.getStream()
		if stream == nil {
			conn.logger.Error(conn.pkiID, "Stream is nil, aborting!")
			errChan <- errors.New("Stream is nil")
			return
		}
		envelope, err := stream.Recv()
		if conn.toDie() {
			conn.logger.Debug(conn.pkiID, "canceling read because closing")
			return
		}
		if err != nil {
			errChan <- err
			conn.logger.Debug(conn.pkiID, "Got error, aborting:", err)
			return
		}
		msg, err := envelope.ToGossipMessage()
		if err != nil {
			errChan <- err
			conn.logger.Warning(conn.pkiID, "Got error, aborting:", err)
		}
		msgChan <- msg
	}
}

func (conn *connection) getStream() stream {
	conn.Lock()
	defer conn.Unlock()

	if conn.clientStream != nil && conn.serverStream != nil {
		e := "Both client and server stream are not nil, something went wrong"
		conn.logger.Error(e)
	}

	if conn.clientStream != nil {
		return conn.clientStream
	}

	if conn.serverStream != nil {
		return conn.serverStream
	}

	return nil
}

type msgSending struct {
	envelope *proto.Envelope
	onErr    func(error)
}
<commit_msg>[FAB-5293] Log buffer overflow events<commit_after>/*
Copyright IBM Corp. 2016 All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

		 http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package comm

import (
	"errors"
	"sync"
	"sync/atomic"

	"github.com/hyperledger/fabric/gossip/common"
	"github.com/hyperledger/fabric/gossip/util"
	proto "github.com/hyperledger/fabric/protos/gossip"
	"github.com/op/go-logging"
	"golang.org/x/net/context"
	"google.golang.org/grpc"
)

type handler func(message *proto.SignedGossipMessage)

type connFactory interface {
	createConnection(endpoint string, pkiID common.PKIidType) (*connection, error)
}

type connectionStore struct {
	logger           *logging.Logger          // logger
	isClosing        bool                     // whether this connection store is shutting down
	connFactory      connFactory              // creates a connection to remote peer
	sync.RWMutex                              // synchronize access to shared variables
	pki2Conn         map[string]*connection   // mapping between pkiID to connections
	destinationLocks map[string]*sync.RWMutex //mapping between pkiIDs and locks,
	// used to prevent concurrent connection establishment to the same remote endpoint
}

func newConnStore(connFactory connFactory, logger *logging.Logger) *connectionStore {
	return &connectionStore{
		connFactory:      connFactory,
		isClosing:        false,
		pki2Conn:         make(map[string]*connection),
		destinationLocks: make(map[string]*sync.RWMutex),
		logger:           logger,
	}
}

func (cs *connectionStore) getConnection(peer *RemotePeer) (*connection, error) {
	cs.RLock()
	isClosing := cs.isClosing
	cs.RUnlock()

	if isClosing {
		return nil, errors.New("Shutting down")
	}

	pkiID := peer.PKIID
	endpoint := peer.Endpoint

	cs.Lock()
	destinationLock, hasConnected := cs.destinationLocks[string(pkiID)]
	if !hasConnected {
		destinationLock = &sync.RWMutex{}
		cs.destinationLocks[string(pkiID)] = destinationLock
	}
	cs.Unlock()

	destinationLock.Lock()

	cs.RLock()
	conn, exists := cs.pki2Conn[string(pkiID)]
	if exists {
		cs.RUnlock()
		destinationLock.Unlock()
		return conn, nil
	}
	cs.RUnlock()

	createdConnection, err := cs.connFactory.createConnection(endpoint, pkiID)

	destinationLock.Unlock()

	cs.RLock()
	isClosing = cs.isClosing
	cs.RUnlock()
	if isClosing {
		return nil, errors.New("ConnStore is closing")
	}

	cs.Lock()
	delete(cs.destinationLocks, string(pkiID))
	defer cs.Unlock()

	// check again, maybe someone connected to us during the connection creation?
	conn, exists = cs.pki2Conn[string(pkiID)]

	if exists {
		if createdConnection != nil {
			createdConnection.close()
		}
		return conn, nil
	}

	// no one connected to us AND we failed connecting!
	if err != nil {
		return nil, err
	}

	// at this point in the code, we created a connection to a remote peer
	conn = createdConnection
	cs.pki2Conn[string(createdConnection.pkiID)] = conn

	go conn.serviceConnection()

	return conn, nil
}

func (cs *connectionStore) connNum() int {
	cs.RLock()
	defer cs.RUnlock()
	return len(cs.pki2Conn)
}

func (cs *connectionStore) closeConn(peer *RemotePeer) {
	cs.Lock()
	defer cs.Unlock()

	if conn, exists := cs.pki2Conn[string(peer.PKIID)]; exists {
		conn.close()
		delete(cs.pki2Conn, string(conn.pkiID))
	}
}

func (cs *connectionStore) shutdown() {
	cs.Lock()
	cs.isClosing = true
	pkiIds2conn := cs.pki2Conn

	var connections2Close []*connection
	for _, conn := range pkiIds2conn {
		connections2Close = append(connections2Close, conn)
	}
	cs.Unlock()

	wg := sync.WaitGroup{}
	for _, conn := range connections2Close {
		wg.Add(1)
		go func(conn *connection) {
			cs.closeByPKIid(conn.pkiID)
			wg.Done()
		}(conn)
	}
	wg.Wait()
}

func (cs *connectionStore) onConnected(serverStream proto.Gossip_GossipStreamServer, connInfo *proto.ConnectionInfo) *connection {
	cs.Lock()
	defer cs.Unlock()

	if c, exists := cs.pki2Conn[string(connInfo.Identity)]; exists {
		c.close()
	}

	return cs.registerConn(connInfo, serverStream)
}

func (cs *connectionStore) registerConn(connInfo *proto.ConnectionInfo, serverStream proto.Gossip_GossipStreamServer) *connection {
	conn := newConnection(nil, nil, nil, serverStream)
	conn.pkiID = connInfo.ID
	conn.info = connInfo
	conn.logger = cs.logger
	cs.pki2Conn[string(connInfo.ID)] = conn
	return conn
}

func (cs *connectionStore) closeByPKIid(pkiID common.PKIidType) {
	cs.Lock()
	defer cs.Unlock()
	if conn, exists := cs.pki2Conn[string(pkiID)]; exists {
		conn.close()
		delete(cs.pki2Conn, string(pkiID))
	}
}

func newConnection(cl proto.GossipClient, c *grpc.ClientConn, cs proto.Gossip_GossipStreamClient, ss proto.Gossip_GossipStreamServer) *connection {
	connection := &connection{
		outBuff:      make(chan *msgSending, util.GetIntOrDefault("peer.gossip.sendBuffSize", defSendBuffSize)),
		cl:           cl,
		conn:         c,
		clientStream: cs,
		serverStream: ss,
		stopFlag:     int32(0),
		stopChan:     make(chan struct{}, 1),
	}

	return connection
}

type connection struct {
	cancel       context.CancelFunc
	info         *proto.ConnectionInfo
	outBuff      chan *msgSending
	logger       *logging.Logger                 // logger
	pkiID        common.PKIidType                // pkiID of the remote endpoint
	handler      handler                         // function to invoke upon a message reception
	conn         *grpc.ClientConn                // gRPC connection to remote endpoint
	cl           proto.GossipClient              // gRPC stub of remote endpoint
	clientStream proto.Gossip_GossipStreamClient // client-side stream to remote endpoint
	serverStream proto.Gossip_GossipStreamServer // server-side stream to remote endpoint
	stopFlag     int32                           // indicates whether this connection is in process of stopping
	stopChan     chan struct{}                   // a method to stop the server-side gRPC call from a different go-routine
	sync.RWMutex                                 // synchronizes access to shared variables
}

func (conn *connection) close() {
	if conn.toDie() {
		return
	}

	amIFirst := atomic.CompareAndSwapInt32(&conn.stopFlag, int32(0), int32(1))
	if !amIFirst {
		return
	}

	conn.stopChan <- struct{}{}

	conn.Lock()

	if conn.clientStream != nil {
		conn.clientStream.CloseSend()
	}
	if conn.conn != nil {
		conn.conn.Close()
	}

	if conn.cancel != nil {
		conn.cancel()
	}

	conn.Unlock()

}

func (conn *connection) toDie() bool {
	return atomic.LoadInt32(&(conn.stopFlag)) == int32(1)
}

func (conn *connection) send(msg *proto.SignedGossipMessage, onErr func(error)) {
	conn.Lock()
	defer conn.Unlock()

	if len(conn.outBuff) == util.GetIntOrDefault("peer.gossip.sendBuffSize", defSendBuffSize) {
		if conn.logger.IsEnabledFor(logging.DEBUG) {
			conn.logger.Debug("Buffer to", conn.info.Endpoint, "overflown, dropping message", msg.String())
		}
		return
	}

	m := &msgSending{
		envelope: msg.Envelope,
		onErr:    onErr,
	}

	conn.outBuff <- m
}

func (conn *connection) serviceConnection() error {
	errChan := make(chan error, 1)
	msgChan := make(chan *proto.SignedGossipMessage, util.GetIntOrDefault("peer.gossip.recvBuffSize", defRecvBuffSize))
	defer close(msgChan)

	// Call stream.Recv() asynchronously in readFromStream(),
	// and wait for either the Recv() call to end,
	// or a signal to close the connection, which exits
	// the method and makes the Recv() call to fail in the
	// readFromStream() method
	go conn.readFromStream(errChan, msgChan)

	go conn.writeToStream()

	for !conn.toDie() {
		select {
		case stop := <-conn.stopChan:
			conn.logger.Debug("Closing reading from stream")
			conn.stopChan <- stop
			return nil
		case err := <-errChan:
			return err
		case msg := <-msgChan:
			conn.handler(msg)
		}
	}
	return nil
}

func (conn *connection) writeToStream() {
	for !conn.toDie() {
		stream := conn.getStream()
		if stream == nil {
			conn.logger.Error(conn.pkiID, "Stream is nil, aborting!")
			return
		}
		select {
		case m := <-conn.outBuff:
			err := stream.Send(m.envelope)
			if err != nil {
				go m.onErr(err)
				return
			}
		case stop := <-conn.stopChan:
			conn.logger.Debug("Closing writing to stream")
			conn.stopChan <- stop
			return
		}
	}
}

func (conn *connection) readFromStream(errChan chan error, msgChan chan *proto.SignedGossipMessage) {
	defer func() {
		recover()
	}() // msgChan might be closed
	for !conn.toDie() {
		stream := conn.getStream()
		if stream == nil {
			conn.logger.Error(conn.pkiID, "Stream is nil, aborting!")
			errChan <- errors.New("Stream is nil")
			return
		}
		envelope, err := stream.Recv()
		if conn.toDie() {
			conn.logger.Debug(conn.pkiID, "canceling read because closing")
			return
		}
		if err != nil {
			errChan <- err
			conn.logger.Debug(conn.pkiID, "Got error, aborting:", err)
			return
		}
		msg, err := envelope.ToGossipMessage()
		if err != nil {
			errChan <- err
			conn.logger.Warning(conn.pkiID, "Got error, aborting:", err)
		}
		msgChan <- msg
	}
}

func (conn *connection) getStream() stream {
	conn.Lock()
	defer conn.Unlock()

	if conn.clientStream != nil && conn.serverStream != nil {
		e := "Both client and server stream are not nil, something went wrong"
		conn.logger.Error(e)
	}

	if conn.clientStream != nil {
		return conn.clientStream
	}

	if conn.serverStream != nil {
		return conn.serverStream
	}

	return nil
}

type msgSending struct {
	envelope *proto.Envelope
	onErr    func(error)
}
<|endoftext|>


<commit_before>package gqt_test

import (
	"fmt"
	"os/exec"
	"path/filepath"
	"time"

	"github.com/cloudfoundry-incubator/garden"
	"github.com/cloudfoundry-incubator/guardian/gqt/runner"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gbytes"
	"github.com/onsi/gomega/gexec"
)

var _ = Describe("Destroying a Container", func() {
	var (
		client    *runner.RunningGarden
		container garden.Container
	)

	BeforeEach(func() {
		client = startGarden()
	})

	AfterEach(func() {
		Expect(client.DestroyAndStop()).To(Succeed())
	})

	JustBeforeEach(func() {
		Expect(client.Destroy(container.Handle())).To(Succeed())
	})

	Context("when running a process", func() {
		var (
			process         garden.Process
			initProcPid     int
			containerRootfs string
		)

		BeforeEach(func() {
			var err error

			container, err = client.Create(garden.ContainerSpec{})
			Expect(err).NotTo(HaveOccurred())

			initProcPid = initProcessPID(container.Handle())

			process, err = container.Run(garden.ProcessSpec{
				Path: "/bin/sh",
				Args: []string{
					"-c", "read x",
				},
			}, ginkgoIO)
			Expect(err).NotTo(HaveOccurred())

			info, err := container.Info()
			Expect(err).NotTo(HaveOccurred())
			containerRootfs = info.ContainerPath
		})

		It("should kill the containers init process", func() {
			var killExitCode = func() int {
				sess, err := gexec.Start(exec.Command("kill", "-0", fmt.Sprintf("%d", initProcPid)), GinkgoWriter, GinkgoWriter)
				Expect(err).NotTo(HaveOccurred())
				sess.Wait(1 * time.Second)
				return sess.ExitCode()
			}

			Eventually(killExitCode).Should(Equal(1))
		})

		It("should destroy the container's depot directory", func() {
			Expect(filepath.Join(client.DepotDir, container.Handle())).NotTo(BeAnExistingFile())
		})

		It("should destroy the container rootfs", func() {
			Expect(containerRootfs).NotTo(BeAnExistingFile())
		})
	})

	Context("when using a static subnet", func() {
		var (
			contIfaceName     string
			contHandle        string
			existingContainer garden.Container
		)

		BeforeEach(func() {
			var err error

			container, err = client.Create(garden.ContainerSpec{
				Network: "177.100.10.30/24",
			})
			Expect(err).NotTo(HaveOccurred())
			contIfaceName = ethInterfaceName(container)
			contHandle = container.Handle()

			existingContainer, err = client.Create(garden.ContainerSpec{
				Network: "168.100.20.10/24",
			})
			Expect(err).NotTo(HaveOccurred())
		})

		AfterEach(func() {
			err := client.Destroy(existingContainer.Handle())
			if err != nil {
				Expect(err).To(MatchError(garden.ContainerNotFoundError{Handle: existingContainer.Handle()}))
			}
		})

		It("should remove iptable entries", func() {
			out, err := exec.Command("iptables", "-w", "-S", "-t", "filter").CombinedOutput()
			Expect(err).NotTo(HaveOccurred())
			Expect(string(out)).NotTo(MatchRegexp("w-%d-instance.* 177.100.10.0/24", GinkgoParallelNode()))
			Expect(string(out)).To(ContainSubstring("168.100.20.0/24"))
		})

		It("should remove virtual ethernet cards", func() {
			ifconfigExits := func() int {
				session, err := gexec.Start(exec.Command("ifconfig", contIfaceName), GinkgoWriter, GinkgoWriter)
				Expect(err).NotTo(HaveOccurred())

				return session.Wait().ExitCode()
			}
			Eventually(ifconfigExits).ShouldNot(Equal(0))

			ifaceName := ethInterfaceName(existingContainer)
			session, err := gexec.Start(exec.Command("ifconfig", ifaceName), GinkgoWriter, GinkgoWriter)
			Expect(err).NotTo(HaveOccurred())
			Eventually(session).Should(gexec.Exit(0))
		})

		It("should remove the network bridge", func() {
			Expect(client.Destroy(existingContainer.Handle())).To(Succeed())

			session, err := gexec.Start(
				exec.Command("ifconfig"),
				GinkgoWriter, GinkgoWriter,
			)

			Expect(err).NotTo(HaveOccurred())
			Eventually(session).Should(gexec.Exit(0))
			Expect(session).NotTo(gbytes.Say("w%dbrdg-", GinkgoParallelNode()))
		})
	})
})

func ethInterfaceName(container garden.Container) string {
	buffer := gbytes.NewBuffer()
	proc, err := container.Run(
		garden.ProcessSpec{
			Path: "sh",
			Args: []string{"-c", "ifconfig | grep 'Ethernet' | cut -f 1 -d ' '"},
			User: "root",
		},
		garden.ProcessIO{
			Stdout: buffer,
			Stderr: GinkgoWriter,
		},
	)
	Expect(err).NotTo(HaveOccurred())
	Expect(proc.Wait()).To(Equal(0))

	contIfaceName := string(buffer.Contents()) // g3-abc-1

	return contIfaceName[:len(contIfaceName)-2] + "0" // g3-abc-0
}
<commit_msg>Retry ifconfig to avoid flakes<commit_after>package gqt_test

import (
	"fmt"
	"os/exec"
	"path/filepath"
	"time"

	"github.com/cloudfoundry-incubator/garden"
	"github.com/cloudfoundry-incubator/guardian/gqt/runner"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gbytes"
	"github.com/onsi/gomega/gexec"
)

var _ = Describe("Destroying a Container", func() {
	var (
		client    *runner.RunningGarden
		container garden.Container
	)

	BeforeEach(func() {
		client = startGarden()
	})

	AfterEach(func() {
		Expect(client.DestroyAndStop()).To(Succeed())
	})

	JustBeforeEach(func() {
		Expect(client.Destroy(container.Handle())).To(Succeed())
	})

	Context("when running a process", func() {
		var (
			process         garden.Process
			initProcPid     int
			containerRootfs string
		)

		BeforeEach(func() {
			var err error

			container, err = client.Create(garden.ContainerSpec{})
			Expect(err).NotTo(HaveOccurred())

			initProcPid = initProcessPID(container.Handle())

			process, err = container.Run(garden.ProcessSpec{
				Path: "/bin/sh",
				Args: []string{
					"-c", "read x",
				},
			}, ginkgoIO)
			Expect(err).NotTo(HaveOccurred())

			info, err := container.Info()
			Expect(err).NotTo(HaveOccurred())
			containerRootfs = info.ContainerPath
		})

		It("should kill the containers init process", func() {
			var killExitCode = func() int {
				sess, err := gexec.Start(exec.Command("kill", "-0", fmt.Sprintf("%d", initProcPid)), GinkgoWriter, GinkgoWriter)
				Expect(err).NotTo(HaveOccurred())
				sess.Wait(1 * time.Second)
				return sess.ExitCode()
			}

			Eventually(killExitCode).Should(Equal(1))
		})

		It("should destroy the container's depot directory", func() {
			Expect(filepath.Join(client.DepotDir, container.Handle())).NotTo(BeAnExistingFile())
		})

		It("should destroy the container rootfs", func() {
			Expect(containerRootfs).NotTo(BeAnExistingFile())
		})
	})

	Context("when using a static subnet", func() {
		var (
			contIfaceName     string
			contHandle        string
			existingContainer garden.Container
		)

		BeforeEach(func() {
			var err error

			container, err = client.Create(garden.ContainerSpec{
				Network: "177.100.10.30/24",
			})
			Expect(err).NotTo(HaveOccurred())
			contIfaceName = ethInterfaceName(container)
			contHandle = container.Handle()

			existingContainer, err = client.Create(garden.ContainerSpec{
				Network: "168.100.20.10/24",
			})
			Expect(err).NotTo(HaveOccurred())
		})

		AfterEach(func() {
			err := client.Destroy(existingContainer.Handle())
			if err != nil {
				Expect(err).To(MatchError(garden.ContainerNotFoundError{Handle: existingContainer.Handle()}))
			}
		})

		It("should remove iptable entries", func() {
			out, err := exec.Command("iptables", "-w", "-S", "-t", "filter").CombinedOutput()
			Expect(err).NotTo(HaveOccurred())
			Expect(string(out)).NotTo(MatchRegexp("w-%d-instance.* 177.100.10.0/24", GinkgoParallelNode()))
			Expect(string(out)).To(ContainSubstring("168.100.20.0/24"))
		})

		It("should remove virtual ethernet cards", func() {
			ifconfigExits := func() int {
				session, err := gexec.Start(exec.Command("ifconfig", contIfaceName), GinkgoWriter, GinkgoWriter)
				Expect(err).NotTo(HaveOccurred())

				return session.Wait().ExitCode()
			}
			Eventually(ifconfigExits).ShouldNot(Equal(0))

			ifaceName := ethInterfaceName(existingContainer)
			session, err := gexec.Start(exec.Command("ifconfig", ifaceName), GinkgoWriter, GinkgoWriter)
			Expect(err).NotTo(HaveOccurred())
			Eventually(session).Should(gexec.Exit(0))
		})

		It("should remove the network bridge", func() {
			Expect(client.Destroy(existingContainer.Handle())).To(Succeed())

			// ifconfig can be flakey when it runs while parallel tests are destroying
			// interfaces, so run it a few times to minimize chance of problems
			var session *gexec.Session
			for i := 0; i < 30; i++ {
				var err error
				session, err = gexec.Start(
					exec.Command("ifconfig"),
					GinkgoWriter, GinkgoWriter,
				)
				Expect(err).NotTo(HaveOccurred())

				session.Wait()
				if session.ExitCode() == 0 {
					break
				}

				time.Sleep(1 * time.Second)
			}

			Expect(session.ExitCode()).To(Equal(0))
			Expect(session).NotTo(gbytes.Say("w%dbrdg-", GinkgoParallelNode()))
		})
	})
})

func ethInterfaceName(container garden.Container) string {
	buffer := gbytes.NewBuffer()
	proc, err := container.Run(
		garden.ProcessSpec{
			Path: "sh",
			Args: []string{"-c", "ifconfig | grep 'Ethernet' | cut -f 1 -d ' '"},
			User: "root",
		},
		garden.ProcessIO{
			Stdout: buffer,
			Stderr: GinkgoWriter,
		},
	)
	Expect(err).NotTo(HaveOccurred())
	Expect(proc.Wait()).To(Equal(0))

	contIfaceName := string(buffer.Contents()) // g3-abc-1

	return contIfaceName[:len(contIfaceName)-2] + "0" // g3-abc-0
}
<|endoftext|>


<commit_before>package grifts

import (
	"html/template"
	"os"
	"os/exec"
	"path"
	"strings"

	mbgrift "github.com/markbates/grift/grift"
)

var _ = mbgrift.Desc("shoulders", "Generates a file listing all of the 3rd party packages used by buffalo.")
var _ = mbgrift.Add("shoulders", func(c *mbgrift.Context) error {
	giants := map[string]string{
		"github.com/markbates/refresh": "github.com/markbates/refresh",
		"github.com/markbates/grift":   "github.com/markbates/grift",
		"github.com/markbates/pop":     "github.com/markbates/pop",
		"github.com/spf13/cobra":       "github.com/spf13/cobra",
		"github.com/motemen/gore":      "github.com/motemen/gore",
	}

	for _, p := range []string{".", "./render"} {
		cmd := exec.Command("go", "list", "-f", `'* {{ join .Deps  "\n"}}'`, p)
		b, err := cmd.Output()
		if err != nil {
			return err
		}

		list := strings.Split(string(b), "\n")

		for _, g := range list {
			if strings.Contains(g, "github.com") {
				giants[g] = g
			}
		}
	}

	f, err := os.Create(path.Join(os.Getenv("GOPATH"), "src", "github.com", "gobuffalo", "buffalo", "SHOULDERS.md"))
	if err != nil {
		return err
	}
	t, err := template.New("").Parse(shouldersTemplate)
	if err != nil {
		return err
	}
	err = t.Execute(f, giants)
	if err != nil {
		return err
	}

	return commitAndPushShoulders()
})

func commitAndPushShoulders() error {
	cmd := exec.Command("git", "commit", "SHOULDERS.md", "-m", "Updated SHOULDERS.md")
	cmd.Stdin = os.Stdin
	cmd.Stderr = os.Stderr
	cmd.Stdout = os.Stdout
	err := cmd.Run()
	if err != nil {
		return err
	}

	cmd = exec.Command("git", "push", "origin")
	cmd.Stdin = os.Stdin
	cmd.Stderr = os.Stderr
	cmd.Stdout = os.Stdout
	return cmd.Run()
}

var shouldersTemplate = `
# Buffalo Stands on the Shoulders of Giants

Buffalo does not try to reinvent the wheel! Instead, it uses the already great wheels developed by the Go community and puts them altogether in the best way possible. Without these giants this project would not be possible. Please make sure to check them out and thank them for all of their hard work.

Thank you to the following **GIANTS**:

{{ range $k, $v := .}}
* [{{$k}}](https://{{$v}})
{{ end }}
`
<commit_msg>fix misnamed import<commit_after>package grifts

import (
	"html/template"
	"os"
	"os/exec"
	"path"
	"strings"

	"github.com/markbates/grift/grift"
)

var _ = grift.Desc("shoulders", "Generates a file listing all of the 3rd party packages used by buffalo.")
var _ = grift.Add("shoulders", func(c *grift.Context) error {
	giants := map[string]string{
		"github.com/markbates/refresh": "github.com/markbates/refresh",
		"github.com/markbates/grift":   "github.com/markbates/grift",
		"github.com/markbates/pop":     "github.com/markbates/pop",
		"github.com/spf13/cobra":       "github.com/spf13/cobra",
		"github.com/motemen/gore":      "github.com/motemen/gore",
	}

	for _, p := range []string{".", "./render"} {
		cmd := exec.Command("go", "list", "-f", `'* {{ join .Deps  "\n"}}'`, p)
		b, err := cmd.Output()
		if err != nil {
			return err
		}

		list := strings.Split(string(b), "\n")

		for _, g := range list {
			if strings.Contains(g, "github.com") {
				giants[g] = g
			}
		}
	}

	f, err := os.Create(path.Join(os.Getenv("GOPATH"), "src", "github.com", "gobuffalo", "buffalo", "SHOULDERS.md"))
	if err != nil {
		return err
	}
	t, err := template.New("").Parse(shouldersTemplate)
	if err != nil {
		return err
	}
	err = t.Execute(f, giants)
	if err != nil {
		return err
	}

	return commitAndPushShoulders()
})

func commitAndPushShoulders() error {
	cmd := exec.Command("git", "commit", "SHOULDERS.md", "-m", "Updated SHOULDERS.md")
	cmd.Stdin = os.Stdin
	cmd.Stderr = os.Stderr
	cmd.Stdout = os.Stdout
	err := cmd.Run()
	if err != nil {
		return err
	}

	cmd = exec.Command("git", "push", "origin")
	cmd.Stdin = os.Stdin
	cmd.Stderr = os.Stderr
	cmd.Stdout = os.Stdout
	return cmd.Run()
}

var shouldersTemplate = `
# Buffalo Stands on the Shoulders of Giants

Buffalo does not try to reinvent the wheel! Instead, it uses the already great wheels developed by the Go community and puts them altogether in the best way possible. Without these giants this project would not be possible. Please make sure to check them out and thank them for all of their hard work.

Thank you to the following **GIANTS**:

{{ range $k, $v := .}}
* [{{$k}}](https://{{$v}})
{{ end }}
`
<|endoftext|>


<commit_before>package hamt32_test

import (
	"flag"
	"fmt"
	"log"
	"math/rand"
	"os"
	"testing"
	"time"

	"github.com/lleo/go-hamt-functional/hamt32"
	"github.com/lleo/go-hamt/key"
	"github.com/lleo/go-hamt/stringkey"
	"github.com/lleo/stringutil"
	"github.com/pkg/errors"
)

//var numHugeKvs int = 1024
var numHugeKvs int = 2 * 1024 * 1024
var hugeKvs []key.KeyVal

var LookupHamt32 hamt32.Hamt
var DeleteHamt32 hamt32.Hamt

var StartTime = make(map[string]time.Time)
var RunTime = make(map[string]time.Duration)

var Inc = stringutil.Lower.Inc

var TableOption int

func TestMain(m *testing.M) {
	// flags
	var fullonly, componly, hybrid, all bool
	flag.BoolVar(&fullonly, "F", false, "Use full tables only and exclude C and H TableOption.")
	flag.BoolVar(&componly, "C", false, "Use compressed tables only and exclude F and H TableOption.")
	flag.BoolVar(&hybrid, "H", false, "Use compressed tables initially and exclude F and C TableOption.")
	flag.BoolVar(&all, "A", false, "Run all Tests w/ TableOption set to hamt32.FullTablesOnly, hamt32.CompTablesOnly, and hamt32.HybridTables; in that order.")

	flag.Parse()

	// If all flag not set, only one of -F, -C, or -H can be set.
	if !all {
		// only one flag may be set between fullonly, componly, and hybrid
		if (fullonly && (componly || hybrid)) ||
			(componly && (fullonly || hybrid)) ||
			(hybrid && (componly || fullonly)) {
			flag.PrintDefaults()
			os.Exit(1)
		}
	}

	// If no flags given, run all tests.
	if !(all || fullonly || componly || hybrid) {
		all = true
	}

	// log Config
	log.SetFlags(log.Lshortfile)

	var logfile, err = os.Create("test.log")
	if err != nil {
		log.Fatal(errors.Wrap(err, "failed to os.Create(\"test.log\")"))
	}
	defer logfile.Close()

	log.SetOutput(logfile)

	log.Println("TestMain: and so it begins...")

	hugeKvs = buildKeyVals(numHugeKvs)

	// execute
	var xit int

	if all {
		//Full Tables Only
		hamt32.GradeTables = false
		hamt32.FullTableInit = true
		log.Println("TestMain: Full Tables Only")
		log.Printf("TestMain: GradeTables=%t; FullTableInit=%t\n", hamt32.GradeTables, hamt32.FullTableInit)
		initialize()
		xit = m.Run()
		if xit != 0 {
			os.Exit(1)
		}

		//Compressed Tables Only
		hamt32.GradeTables = false
		hamt32.FullTableInit = false
		log.Println("TestMain: Compressed Tables Only")
		log.Printf("TestMain: GradeTables=%t; FullTableInit=%t\n", hamt32.GradeTables, hamt32.FullTableInit)
		initialize()
		xit = m.Run()
		if xit != 0 {
			os.Exit(1)
		}

		//Hybrid Tables
		hamt32.GradeTables = true
		hamt32.FullTableInit = false
		log.Println("TestMain: Hybrid Tables")
		log.Printf("TestMain: GradeTables=%t; FullTableInit=%t\n", hamt32.GradeTables, hamt32.FullTableInit)
		initialize()
		xit = m.Run()
	} else {
		if hybrid {
			hamt32.GradeTables = true
			hamt32.FullTableInit = false
			log.Println("TestMain: Hybrid Tables")
		} else if fullonly {
			hamt32.GradeTables = false
			hamt32.FullTableInit = true
			log.Println("TestMain: Full Tables Only")
		} else /* if componly */ {
			hamt32.GradeTables = false
			hamt32.FullTableInit = false
			log.Println("TestMain: Compressed Tables Only")
		}

		log.Printf("TestMain: GradeTables=%t; FullTableInit=%t\n", hamt32.GradeTables, hamt32.FullTableInit)
		initialize()
		xit = m.Run()
	}

	log.Println("\n", RunTimes())
	log.Println("TestMain: the end.")
	os.Exit(xit)
}

func RunTimes() string {
	var s = ""

	s += "Key                                                               Val\n"
	s += "=================================================================+==========\n"

	for key, val := range RunTime {
		s += fmt.Sprintf("%-65s %s\n", key, val)
	}
	return s
}

func initialize() {
	var funcName = "hamt32: initialize()"

	var metricName = funcName + ": build Lookup/Delete Hamt32"
	log.Println(metricName, "called.")
	log.Printf("initialize: GradeTables=%t; FullTableInit=%t\n", hamt32.GradeTables, hamt32.FullTableInit)
	StartTime[metricName] = time.Now()

	LookupHamt32 = hamt32.Hamt{}
	DeleteHamt32 = hamt32.Hamt{}

	for _, kv := range genRandomizedKvs(hugeKvs) {
		var inserted bool
		LookupHamt32, inserted = LookupHamt32.Put(kv.Key, kv.Val)
		if !inserted {
			log.Fatalf("failed to LookupHamt32.Put(%s, %v)", kv.Key, kv.Val)
		}

		DeleteHamt32, inserted = DeleteHamt32.Put(kv.Key, kv.Val)
		if !inserted {
			log.Fatalf("failed to DeleteHamt32.Put(%s, %v)", kv.Key, kv.Val)
		}
	}

	RunTime[metricName] = time.Since(StartTime[metricName])
}

func buildKeyVals(num int) []key.KeyVal {
	var kvs = make([]key.KeyVal, num, num)

	s := "aaa"
	for i := 0; i < num; i++ {
		kvs[i].Key = stringkey.New(s)
		kvs[i].Val = i

		s = Inc(s)
	}

	return kvs
}

func genRandomizedKvs(kvs []key.KeyVal) []key.KeyVal {
	randKvs := make([]key.KeyVal, len(kvs))
	copy(randKvs, kvs)

	//From: https://en.wikipedia.org/wiki/Fisher%E2%80%93Yates_shuffle#The_modern_algorithm
	for i := len(randKvs) - 1; i > 0; i-- {
		j := rand.Intn(i + 1)
		randKvs[i], randKvs[j] = randKvs[j], randKvs[i]
	}

	return randKvs
}
<commit_msg>* Renamed numHugeKvs to numKvs * Renamed hugeKvs to KVS * Renamed LookupHamt32 to TestHamt32 * Deleted DeleteHamt32<commit_after>package hamt32_test

import (
	"flag"
	"fmt"
	"log"
	"math/rand"
	"os"
	"testing"
	"time"

	"github.com/lleo/go-hamt-functional/hamt32"
	"github.com/lleo/go-hamt/key"
	"github.com/lleo/go-hamt/stringkey"
	"github.com/lleo/stringutil"
	"github.com/pkg/errors"
)

//var numKvs = 32
//var numKvs int = 1024
//var numKvs int = 2 * 1024 * 1024
//var numKvs int = (3 * 1024 * 1024) + (4 * 1024) // between 3m+2k & 3m+4k
var numKvs int = 512 * 1024

var KVS []key.KeyVal

var TestHamt32 hamt32.Hamt

var StartTime = make(map[string]time.Time)
var RunTime = make(map[string]time.Duration)

var Inc = stringutil.Lower.Inc

var TableOption int

func TestMain(m *testing.M) {
	// flags
	var fullonly, componly, hybrid, all bool
	flag.BoolVar(&fullonly, "F", false, "Use full tables only and exclude C and H TableOption.")
	flag.BoolVar(&componly, "C", false, "Use compressed tables only and exclude F and H TableOption.")
	flag.BoolVar(&hybrid, "H", false, "Use compressed tables initially and exclude F and C TableOption.")
	flag.BoolVar(&all, "A", false, "Run all Tests w/ TableOption set to hamt32.FullTablesOnly, hamt32.CompTablesOnly, and hamt32.HybridTables; in that order.")

	flag.Parse()

	// If all flag not set, only one of -F, -C, or -H can be set.
	if !all {
		// only one flag may be set between fullonly, componly, and hybrid
		if (fullonly && (componly || hybrid)) ||
			(componly && (fullonly || hybrid)) ||
			(hybrid && (componly || fullonly)) {
			flag.PrintDefaults()
			os.Exit(1)
		}
	}

	// If no flags given, run all tests.
	if !(all || fullonly || componly || hybrid) {
		all = true
	}

	// log Config
	log.SetFlags(log.Lshortfile)

	var logfile, err = os.Create("test.log")
	if err != nil {
		log.Fatal(errors.Wrap(err, "failed to os.Create(\"test.log\")"))
	}
	defer logfile.Close()

	log.SetOutput(logfile)

	log.Println("TestMain: and so it begins...")

	KVS = buildKeyVals(numKvs)

	// execute
	var xit int

	if all {
		//Full Tables Only
		hamt32.GradeTables = false
		hamt32.FullTableInit = true
		log.Println("TestMain: Full Tables Only")
		log.Printf("TestMain: GradeTables=%t; FullTableInit=%t\n", hamt32.GradeTables, hamt32.FullTableInit)
		initialize()
		xit = m.Run()
		if xit != 0 {
			os.Exit(1)
		}

		//Compressed Tables Only
		hamt32.GradeTables = false
		hamt32.FullTableInit = false
		log.Println("TestMain: Compressed Tables Only")
		log.Printf("TestMain: GradeTables=%t; FullTableInit=%t\n", hamt32.GradeTables, hamt32.FullTableInit)
		initialize()
		xit = m.Run()
		if xit != 0 {
			os.Exit(1)
		}

		//Hybrid Tables
		hamt32.GradeTables = true
		hamt32.FullTableInit = false
		log.Println("TestMain: Hybrid Tables")
		log.Printf("TestMain: GradeTables=%t; FullTableInit=%t\n", hamt32.GradeTables, hamt32.FullTableInit)
		initialize()
		xit = m.Run()
	} else {
		if hybrid {
			hamt32.GradeTables = true
			hamt32.FullTableInit = false
			log.Println("TestMain: Hybrid Tables")
		} else if fullonly {
			hamt32.GradeTables = false
			hamt32.FullTableInit = true
			log.Println("TestMain: Full Tables Only")
		} else /* if componly */ {
			hamt32.GradeTables = false
			hamt32.FullTableInit = false
			log.Println("TestMain: Compressed Tables Only")
		}

		log.Printf("TestMain: GradeTables=%t; FullTableInit=%t\n", hamt32.GradeTables, hamt32.FullTableInit)
		initialize()
		xit = m.Run()
	}

	log.Println("\n", RunTimes())
	log.Println("TestMain: the end.")
	os.Exit(xit)
}

func RunTimes() string {
	var s = ""

	s += "Key                                                               Val\n"
	s += "=================================================================+==========\n"

	for key, val := range RunTime {
		s += fmt.Sprintf("%-65s %s\n", key, val)
	}
	return s
}

func initialize() {
	var funcName = "hamt32: initialize()"

	var metricName = funcName + ": build TestHamt32"
	log.Println(metricName, "called.")
	log.Printf("initialize: GradeTables=%t; FullTableInit=%t\n", hamt32.GradeTables, hamt32.FullTableInit)
	StartTime[metricName] = time.Now()

	TestHamt32 = hamt32.Hamt{}

	for _, kv := range genRandomizedKvs(KVS) {
		var inserted bool
		TestHamt32, inserted = TestHamt32.Put(kv.Key, kv.Val)
		if !inserted {
			log.Fatalf("failed to TestHamt32.Put(%s, %v)", kv.Key, kv.Val)
		}
	}

	RunTime[metricName] = time.Since(StartTime[metricName])
}

func buildKeyVals(num int) []key.KeyVal {
	var kvs = make([]key.KeyVal, num, num)

	s := "aaa"
	for i := 0; i < num; i++ {
		kvs[i].Key = stringkey.New(s)
		kvs[i].Val = i

		s = Inc(s)
	}

	return kvs
}

func genRandomizedKvs(kvs []key.KeyVal) []key.KeyVal {
	randKvs := make([]key.KeyVal, len(kvs))
	copy(randKvs, kvs)

	//From: https://en.wikipedia.org/wiki/Fisher%E2%80%93Yates_shuffle#The_modern_algorithm
	for i := len(randKvs) - 1; i > 0; i-- {
		j := rand.Intn(i + 1)
		randKvs[i], randKvs[j] = randKvs[j], randKvs[i]
	}

	return randKvs
}
<|endoftext|>


<commit_before>package hclext

import (
	"fmt"
	"reflect"

	"github.com/hashicorp/hcl/v2"
)

// BodyContent is the result of applying a hclext.BodySchema to a hcl.Body.
// Unlike hcl.BodyContent, this does not have MissingItemRange.
// This difference is because hcl.BodyContent is the result for a single HCL file,
// while hclext.BodyContent is the result for a Terraform module.
type BodyContent struct {
	Attributes Attributes
	Blocks     Blocks
}

// Blocks is a sequence of Block.
type Blocks []*Block

// Block represents a nested block within a hcl.Body.
// Unlike hcl.Block, this has Body as hclext.BodyContent (struct), not hcl.Body (interface).
// Since interface is hard to send over a wire protocol, it is designed to always return only the attributes based on the schema.
// Instead, the hclext.BlockSchema can now be nested to extract the attributes within the nested block.
type Block struct {
	Type   string
	Labels []string
	Body   *BodyContent

	DefRange    hcl.Range
	TypeRange   hcl.Range
	LabelRanges []hcl.Range
}

// Attributes is a set of attributes keyed by their names.
// Please note that this is not strictly. Since hclext.BodyContent is the body from multiple files,
// top-level attributes can have the same name (it is not possible to specify the same name within a block).
// This exception is not considered here, as Terraform syntax does not allow top-level attributes.
type Attributes map[string]*Attribute

// Attribute represents an attribute from within a body.
type Attribute struct {
	Name string
	Expr hcl.Expression

	Range     hcl.Range
	NameRange hcl.Range
}

// Content is a wrapper for hcl.Content for working with nested schemas.
// Convert hclext.BodySchema to hcl.BodySchema, and convert hcl.BodyContent
// to hclext.BodyContent. It processes the nested body recursively.
func Content(body hcl.Body, schema *BodySchema) (*BodyContent, hcl.Diagnostics) {
	if reflect.ValueOf(body).IsNil() {
		return &BodyContent{}, hcl.Diagnostics{}
	}
	if schema == nil {
		schema = &BodySchema{}
	}

	hclS := &hcl.BodySchema{
		Attributes: make([]hcl.AttributeSchema, len(schema.Attributes)),
		Blocks:     make([]hcl.BlockHeaderSchema, len(schema.Blocks)),
	}
	for idx, attrS := range schema.Attributes {
		hclS.Attributes[idx] = hcl.AttributeSchema{Name: attrS.Name, Required: attrS.Required}
	}
	childS := map[string]*BodySchema{}
	for idx, blockS := range schema.Blocks {
		hclS.Blocks[idx] = hcl.BlockHeaderSchema{Type: blockS.Type, LabelNames: blockS.LabelNames}
		childS[blockS.Type] = blockS.Body
	}

	content := &hcl.BodyContent{}
	var diags hcl.Diagnostics
	switch schema.Mode {
	case SchemaDefaultMode:
		content, diags = body.Content(hclS)
	case SchemaJustAttributesMode:
		content.Attributes, diags = body.JustAttributes()
	default:
		panic(fmt.Sprintf("invalid SchemaMode: %s", schema.Mode))
	}

	ret := &BodyContent{
		Attributes: Attributes{},
		Blocks:     make(Blocks, len(content.Blocks)),
	}
	for name, attr := range content.Attributes {
		ret.Attributes[name] = &Attribute{
			Name:      attr.Name,
			Expr:      attr.Expr,
			Range:     attr.Range,
			NameRange: attr.NameRange,
		}
	}
	for idx, block := range content.Blocks {
		child, childDiags := Content(block.Body, childS[block.Type])
		diags = diags.Extend(childDiags)

		ret.Blocks[idx] = &Block{
			Type:        block.Type,
			Labels:      block.Labels,
			Body:        child,
			DefRange:    block.DefRange,
			TypeRange:   block.TypeRange,
			LabelRanges: block.LabelRanges,
		}
	}

	return ret, diags
}

// PartialContent is a wrapper for hcl.PartialContent for working with nested schemas.
// Convert hclext.BodySchema to hcl.BodySchema, and convert hcl.BodyContent
// to hclext.BodyContent. It processes the nested body recursively.
// Unlike hcl.PartialContent, it does not return the rest of the body.
func PartialContent(body hcl.Body, schema *BodySchema) (*BodyContent, hcl.Diagnostics) {
	if reflect.ValueOf(body).IsNil() {
		return &BodyContent{}, hcl.Diagnostics{}
	}
	if schema == nil {
		schema = &BodySchema{}
	}

	hclS := &hcl.BodySchema{
		Attributes: make([]hcl.AttributeSchema, len(schema.Attributes)),
		Blocks:     make([]hcl.BlockHeaderSchema, len(schema.Blocks)),
	}
	for idx, attrS := range schema.Attributes {
		hclS.Attributes[idx] = hcl.AttributeSchema{Name: attrS.Name, Required: attrS.Required}
	}
	childS := map[string]*BodySchema{}
	for idx, blockS := range schema.Blocks {
		hclS.Blocks[idx] = hcl.BlockHeaderSchema{Type: blockS.Type, LabelNames: blockS.LabelNames}
		childS[blockS.Type] = blockS.Body
	}

	content := &hcl.BodyContent{}
	var diags hcl.Diagnostics
	switch schema.Mode {
	case SchemaDefaultMode:
		content, _, diags = body.PartialContent(hclS)
	case SchemaJustAttributesMode:
		content.Attributes, diags = body.JustAttributes()
	default:
		panic(fmt.Sprintf("invalid SchemaMode: %s", schema.Mode))
	}

	ret := &BodyContent{
		Attributes: Attributes{},
		Blocks:     make(Blocks, len(content.Blocks)),
	}
	for name, attr := range content.Attributes {
		ret.Attributes[name] = &Attribute{
			Name:      attr.Name,
			Expr:      attr.Expr,
			Range:     attr.Range,
			NameRange: attr.NameRange,
		}
	}
	for idx, block := range content.Blocks {
		child, childDiags := PartialContent(block.Body, childS[block.Type])
		diags = diags.Extend(childDiags)

		ret.Blocks[idx] = &Block{
			Type:        block.Type,
			Labels:      block.Labels,
			Body:        child,
			DefRange:    block.DefRange,
			TypeRange:   block.TypeRange,
			LabelRanges: block.LabelRanges,
		}
	}

	return ret, diags
}

// IsEmpty returns whether the body content is empty
func (b *BodyContent) IsEmpty() bool {
	if b == nil {
		return true
	}
	return len(b.Attributes) == 0 && len(b.Blocks) == 0
}

// AsNative returns self as hcl.Attributes
func (as Attributes) AsNative() hcl.Attributes {
	ret := hcl.Attributes{}
	for name, attr := range as {
		ret[name] = attr.AsNative()
	}
	return ret
}

// AsNative returns self as hcl.Attribute
func (a *Attribute) AsNative() *hcl.Attribute {
	return &hcl.Attribute{
		Name:      a.Name,
		Expr:      a.Expr,
		Range:     a.Range,
		NameRange: a.NameRange,
	}
}

// OfType filters the receiving block sequence by block type name,
// returning a new block sequence including only the blocks of the
// requested type.
func (els Blocks) OfType(typeName string) Blocks {
	ret := make(Blocks, 0)
	for _, el := range els {
		if el.Type == typeName {
			ret = append(ret, el)
		}
	}
	return ret
}

// ByType transforms the receiving block sequence into a map from type
// name to block sequences of only that type.
func (els Blocks) ByType() map[string]Blocks {
	ret := make(map[string]Blocks)
	for _, el := range els {
		ty := el.Type
		if ret[ty] == nil {
			ret[ty] = make(Blocks, 0, 1)
		}
		ret[ty] = append(ret[ty], el)
	}
	return ret
}
<commit_msg>Add Copy() to structures (#206)<commit_after>package hclext

import (
	"fmt"
	"reflect"

	"github.com/hashicorp/hcl/v2"
)

// BodyContent is the result of applying a hclext.BodySchema to a hcl.Body.
// Unlike hcl.BodyContent, this does not have MissingItemRange.
// This difference is because hcl.BodyContent is the result for a single HCL file,
// while hclext.BodyContent is the result for a Terraform module.
type BodyContent struct {
	Attributes Attributes
	Blocks     Blocks
}

// Blocks is a sequence of Block.
type Blocks []*Block

// Block represents a nested block within a hcl.Body.
// Unlike hcl.Block, this has Body as hclext.BodyContent (struct), not hcl.Body (interface).
// Since interface is hard to send over a wire protocol, it is designed to always return only the attributes based on the schema.
// Instead, the hclext.BlockSchema can now be nested to extract the attributes within the nested block.
type Block struct {
	Type   string
	Labels []string
	Body   *BodyContent

	DefRange    hcl.Range
	TypeRange   hcl.Range
	LabelRanges []hcl.Range
}

// Attributes is a set of attributes keyed by their names.
// Please note that this is not strictly. Since hclext.BodyContent is the body from multiple files,
// top-level attributes can have the same name (it is not possible to specify the same name within a block).
// This exception is not considered here, as Terraform syntax does not allow top-level attributes.
type Attributes map[string]*Attribute

// Attribute represents an attribute from within a body.
type Attribute struct {
	Name string
	Expr hcl.Expression

	Range     hcl.Range
	NameRange hcl.Range
}

// Content is a wrapper for hcl.Content for working with nested schemas.
// Convert hclext.BodySchema to hcl.BodySchema, and convert hcl.BodyContent
// to hclext.BodyContent. It processes the nested body recursively.
func Content(body hcl.Body, schema *BodySchema) (*BodyContent, hcl.Diagnostics) {
	if reflect.ValueOf(body).IsNil() {
		return &BodyContent{}, hcl.Diagnostics{}
	}
	if schema == nil {
		schema = &BodySchema{}
	}

	hclS := &hcl.BodySchema{
		Attributes: make([]hcl.AttributeSchema, len(schema.Attributes)),
		Blocks:     make([]hcl.BlockHeaderSchema, len(schema.Blocks)),
	}
	for idx, attrS := range schema.Attributes {
		hclS.Attributes[idx] = hcl.AttributeSchema{Name: attrS.Name, Required: attrS.Required}
	}
	childS := map[string]*BodySchema{}
	for idx, blockS := range schema.Blocks {
		hclS.Blocks[idx] = hcl.BlockHeaderSchema{Type: blockS.Type, LabelNames: blockS.LabelNames}
		childS[blockS.Type] = blockS.Body
	}

	content := &hcl.BodyContent{}
	var diags hcl.Diagnostics
	switch schema.Mode {
	case SchemaDefaultMode:
		content, diags = body.Content(hclS)
	case SchemaJustAttributesMode:
		content.Attributes, diags = body.JustAttributes()
	default:
		panic(fmt.Sprintf("invalid SchemaMode: %s", schema.Mode))
	}

	ret := &BodyContent{
		Attributes: Attributes{},
		Blocks:     make(Blocks, len(content.Blocks)),
	}
	for name, attr := range content.Attributes {
		ret.Attributes[name] = &Attribute{
			Name:      attr.Name,
			Expr:      attr.Expr,
			Range:     attr.Range,
			NameRange: attr.NameRange,
		}
	}
	for idx, block := range content.Blocks {
		child, childDiags := Content(block.Body, childS[block.Type])
		diags = diags.Extend(childDiags)

		ret.Blocks[idx] = &Block{
			Type:        block.Type,
			Labels:      block.Labels,
			Body:        child,
			DefRange:    block.DefRange,
			TypeRange:   block.TypeRange,
			LabelRanges: block.LabelRanges,
		}
	}

	return ret, diags
}

// PartialContent is a wrapper for hcl.PartialContent for working with nested schemas.
// Convert hclext.BodySchema to hcl.BodySchema, and convert hcl.BodyContent
// to hclext.BodyContent. It processes the nested body recursively.
// Unlike hcl.PartialContent, it does not return the rest of the body.
func PartialContent(body hcl.Body, schema *BodySchema) (*BodyContent, hcl.Diagnostics) {
	if reflect.ValueOf(body).IsNil() {
		return &BodyContent{}, hcl.Diagnostics{}
	}
	if schema == nil {
		schema = &BodySchema{}
	}

	hclS := &hcl.BodySchema{
		Attributes: make([]hcl.AttributeSchema, len(schema.Attributes)),
		Blocks:     make([]hcl.BlockHeaderSchema, len(schema.Blocks)),
	}
	for idx, attrS := range schema.Attributes {
		hclS.Attributes[idx] = hcl.AttributeSchema{Name: attrS.Name, Required: attrS.Required}
	}
	childS := map[string]*BodySchema{}
	for idx, blockS := range schema.Blocks {
		hclS.Blocks[idx] = hcl.BlockHeaderSchema{Type: blockS.Type, LabelNames: blockS.LabelNames}
		childS[blockS.Type] = blockS.Body
	}

	content := &hcl.BodyContent{}
	var diags hcl.Diagnostics
	switch schema.Mode {
	case SchemaDefaultMode:
		content, _, diags = body.PartialContent(hclS)
	case SchemaJustAttributesMode:
		content.Attributes, diags = body.JustAttributes()
	default:
		panic(fmt.Sprintf("invalid SchemaMode: %s", schema.Mode))
	}

	ret := &BodyContent{
		Attributes: Attributes{},
		Blocks:     make(Blocks, len(content.Blocks)),
	}
	for name, attr := range content.Attributes {
		ret.Attributes[name] = &Attribute{
			Name:      attr.Name,
			Expr:      attr.Expr,
			Range:     attr.Range,
			NameRange: attr.NameRange,
		}
	}
	for idx, block := range content.Blocks {
		child, childDiags := PartialContent(block.Body, childS[block.Type])
		diags = diags.Extend(childDiags)

		ret.Blocks[idx] = &Block{
			Type:        block.Type,
			Labels:      block.Labels,
			Body:        child,
			DefRange:    block.DefRange,
			TypeRange:   block.TypeRange,
			LabelRanges: block.LabelRanges,
		}
	}

	return ret, diags
}

// IsEmpty returns whether the body content is empty
func (b *BodyContent) IsEmpty() bool {
	if b == nil {
		return true
	}
	return len(b.Attributes) == 0 && len(b.Blocks) == 0
}

// Copy returns a new BodyContent based on the original.
func (b *BodyContent) Copy() *BodyContent {
	out := &BodyContent{
		Attributes: Attributes{},
		Blocks:     make(Blocks, len(b.Blocks)),
	}

	for k, v := range b.Attributes {
		out.Attributes[k] = v.Copy()
	}
	copy(out.Blocks, b.Blocks)

	return out
}

// AsNative returns self as hcl.Attributes
func (as Attributes) AsNative() hcl.Attributes {
	ret := hcl.Attributes{}
	for name, attr := range as {
		ret[name] = attr.AsNative()
	}
	return ret
}

// AsNative returns self as hcl.Attribute
func (a *Attribute) AsNative() *hcl.Attribute {
	return &hcl.Attribute{
		Name:      a.Name,
		Expr:      a.Expr,
		Range:     a.Range,
		NameRange: a.NameRange,
	}
}

// Copy returns a new Attribute based on the original.
// Note that expr can be a shallow copy. So strictly speaking
// Copy is not a deep copy.
func (a *Attribute) Copy() *Attribute {
	return &Attribute{
		Name:      a.Name,
		Expr:      a.Expr,
		Range:     a.Range,
		NameRange: a.NameRange,
	}
}

// OfType filters the receiving block sequence by block type name,
// returning a new block sequence including only the blocks of the
// requested type.
func (els Blocks) OfType(typeName string) Blocks {
	ret := make(Blocks, 0)
	for _, el := range els {
		if el.Type == typeName {
			ret = append(ret, el)
		}
	}
	return ret
}

// ByType transforms the receiving block sequence into a map from type
// name to block sequences of only that type.
func (els Blocks) ByType() map[string]Blocks {
	ret := make(map[string]Blocks)
	for _, el := range els {
		ty := el.Type
		if ret[ty] == nil {
			ret[ty] = make(Blocks, 0, 1)
		}
		ret[ty] = append(ret[ty], el)
	}
	return ret
}

// Copy returns a new Block based on the original.
func (b *Block) Copy() *Block {
	out := &Block{
		Type:        b.Type,
		Labels:      make([]string, len(b.Labels)),
		Body:        b.Body.Copy(),
		DefRange:    b.DefRange,
		TypeRange:   b.TypeRange,
		LabelRanges: make([]hcl.Range, len(b.LabelRanges)),
	}

	copy(out.Labels, b.Labels)
	copy(out.LabelRanges, b.LabelRanges)

	return out
}
<|endoftext|>


<commit_before>package form

import (
	"encoding/json"
	"github.com/gogo/protobuf/parser"
	descriptor "github.com/gogo/protobuf/protoc-gen-gogo/descriptor"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"strings"
	"testing"
)

type Msg struct {
	Name      string
	WeirdName string
}

func TestCreateCustom(t *testing.T) {
	desc, err := parser.ParseFile("form.proto", ".")
	if err != nil {
		t.Fatal(err)
	}
	formStr = CreateCustom("WeirdMethod", "form", "Weird", desc, CustomBuildField)
	server := http.Server{
		Addr:    "localhost:8080",
		Handler: http.HandlerFunc(handle),
	}
	lis, err := net.Listen("tcp", "localhost:8080")
	if err != nil {
		t.Fatalf("Failed to listen: %v", err)
	}
	defer lis.Close()
	go server.Serve(lis)
	resp, err := http.Get("http://localhost:8080/WeirdMethod?json={%22Name%22:%22%22,%22WeirdName%22:%22another%20string%22,%22Number%22:null}")
	if err != nil {
		t.Fatal(err)
	}
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		t.Fatal(err)
	}
	t.Logf("%s", string(body))
	if strings.Contains(string(body), "404") {
		t.Fatal("404")
	}
	if !strings.Contains(string(body), `"WeirdName":"another string"`) {
		t.Fatal("could not find json value")
	}
}

func CustomBuildField(fileDescriptorSet *descriptor.FileDescriptorSet, msg *descriptor.DescriptorProto, f *descriptor.FieldDescriptorProto) string {
	fieldname := f.GetName()
	if fieldname != "WeirdName" {
		return BuildField(fileDescriptorSet, msg, f)
	}
	s := `s += '<div class="field form-group"><label class="col-sm-2 control-label">` + fieldname + `:
';
	s += '<select class="form-control" name="` + fieldname + `">';
	`
	stringOptions := []string{"option1", "another string", "192.168.1.1"}
	for _, stringOption := range stringOptions {
		s += `s += 	'<option value="` + stringOption + `" ' + selected("` + stringOption + `", json["` + fieldname + `"]) + '>` + stringOption + `</option>';
		`
	}
	s += `s += '</select>

';
	`
	return s
}

var formStr = ""

var header = `

	<title>WeirdTest</title>
	<link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.4/css/bootstrap.min.css">

WeirdTest

	`

var footer string = `

	`

func handle(w http.ResponseWriter, req *http.Request) {
	w.Write([]byte(header))
	jsonString := req.FormValue("json")
	someValue := false
	msg := &Msg{}
	if len(jsonString) > 0 {
		err := json.Unmarshal([]byte(jsonString), msg)
		if err != nil {
			if err != io.EOF {
				w.Write([]byte(err.Error()))
				return
			}
			w.Write([]byte(err.Error()))
		}
		someValue = true
	}
	w.Write([]byte(formStr))
	if someValue {
		w.Write([]byte("
" + jsonString + "
"))
	}
	w.Write([]byte(footer))
}
<commit_msg>copyright<commit_after>// Copyright (c) 2015, Vastech SA (PTY) LTD. All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
//     * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
//     * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

package form

import (
	"encoding/json"
	"github.com/gogo/protobuf/parser"
	descriptor "github.com/gogo/protobuf/protoc-gen-gogo/descriptor"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"strings"
	"testing"
)

type Msg struct {
	Name      string
	WeirdName string
}

func TestCreateCustom(t *testing.T) {
	desc, err := parser.ParseFile("form.proto", ".")
	if err != nil {
		t.Fatal(err)
	}
	formStr = CreateCustom("WeirdMethod", "form", "Weird", desc, CustomBuildField)
	server := http.Server{
		Addr:    "localhost:8080",
		Handler: http.HandlerFunc(handle),
	}
	lis, err := net.Listen("tcp", "localhost:8080")
	if err != nil {
		t.Fatalf("Failed to listen: %v", err)
	}
	defer lis.Close()
	go server.Serve(lis)
	resp, err := http.Get("http://localhost:8080/WeirdMethod?json={%22Name%22:%22%22,%22WeirdName%22:%22another%20string%22,%22Number%22:null}")
	if err != nil {
		t.Fatal(err)
	}
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		t.Fatal(err)
	}
	t.Logf("%s", string(body))
	if strings.Contains(string(body), "404") {
		t.Fatal("404")
	}
	if !strings.Contains(string(body), `"WeirdName":"another string"`) {
		t.Fatal("could not find json value")
	}
}

func CustomBuildField(fileDescriptorSet *descriptor.FileDescriptorSet, msg *descriptor.DescriptorProto, f *descriptor.FieldDescriptorProto) string {
	fieldname := f.GetName()
	if fieldname != "WeirdName" {
		return BuildField(fileDescriptorSet, msg, f)
	}
	s := `s += '<div class="field form-group"><label class="col-sm-2 control-label">` + fieldname + `:
';
	s += '<select class="form-control" name="` + fieldname + `">';
	`
	stringOptions := []string{"option1", "another string", "192.168.1.1"}
	for _, stringOption := range stringOptions {
		s += `s += 	'<option value="` + stringOption + `" ' + selected("` + stringOption + `", json["` + fieldname + `"]) + '>` + stringOption + `</option>';
		`
	}
	s += `s += '</select>

';
	`
	return s
}

var formStr = ""

var header = `

	<title>WeirdTest</title>
	<link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.4/css/bootstrap.min.css">

WeirdTest

	`

var footer string = `

	`

func handle(w http.ResponseWriter, req *http.Request) {
	w.Write([]byte(header))
	jsonString := req.FormValue("json")
	someValue := false
	msg := &Msg{}
	if len(jsonString) > 0 {
		err := json.Unmarshal([]byte(jsonString), msg)
		if err != nil {
			if err != io.EOF {
				w.Write([]byte(err.Error()))
				return
			}
			w.Write([]byte(err.Error()))
		}
		someValue = true
	}
	w.Write([]byte(formStr))
	if someValue {
		w.Write([]byte("
" + jsonString + "
"))
	}
	w.Write([]byte(footer))
}
<|endoftext|>


<commit_before>package data_test

import (
	"fmt"

	"github.com/DailyBurn/ratchet/data"
)

type testStruct struct {
	A int
	B int
}

func ExampleNewJSON() {
	t := testStruct{A: 1, B: 2}

	data, _ := data.NewJSON(t)

	fmt.Println(string(data))
	// Output: {"A":1,"B":2}
}

func ExampleParseJSON() {
	d := []byte(`{"A":1,"B":2}`)
	t := testStruct{}
	data.ParseJSON(data, &t)

	fmt.Println(fmt.Sprintf("%+v", t))
	// Output: {A:1 B:2}
}

func ExampleObjectsFromJSON() {
	d := []byte(`[{"One":1, "Two":2},
		      {"Three":3, "Four":4}]`)

	objects, _ := data.ObjectsFromJSON(d)

	fmt.Println(fmt.Sprintf("%+v", objects))
	// Output: [map[One:1 Two:2] map[Three:3 Four:4]]
}
<commit_msg>Fix data tests<commit_after>package data_test

import (
	"fmt"

	"github.com/DailyBurn/ratchet/data"
)

type testStruct struct {
	A int
	B int
}

func ExampleNewJSON() {
	t := testStruct{A: 1, B: 2}

	d, _ := data.NewJSON(t)

	fmt.Println(string(d))
	// Output: {"A":1,"B":2}
}

func ExampleParseJSON() {
	d := []byte(`{"A":1,"B":2}`)
	t := testStruct{}
	data.ParseJSON(d, &t)

	fmt.Println(fmt.Sprintf("%+v", t))
	// Output: {A:1 B:2}
}

func ExampleObjectsFromJSON() {
	d := []byte(`[{"One":1, "Two":2},
		      {"Three":3, "Four":4}]`)

	objects, _ := data.ObjectsFromJSON(d)

	fmt.Println(fmt.Sprintf("%+v", objects))
	// Output: [map[One:1 Two:2] map[Three:3 Four:4]]
}
<|endoftext|>


<commit_before>// Copyright 2017 The WPT Dashboard Project. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

package compute

import (
	"testing"
	"time"

	"github.com/stretchr/testify/assert"

	"github.com/w3c/wptdashboard/metrics"
	"github.com/w3c/wptdashboard/shared"
)

var timeA = time.Unix(0, 0)
var timeB = time.Unix(0, 1)

func TestGatherResultsById_TwoRuns_SameTest(t *testing.T) {
	runA := shared.TestRun{
		"ABrowser",
		"1.0",
		"MyOS",
		"1.0",
		"abcd",
		"http://example.com/a_run.json",
		timeA,
	}
	runB := shared.TestRun{
		"BBrowser",
		"1.0",
		"MyOS",
		"1.0",
		"dcba",
		"http://example.com/b_run.json",
		timeB,
	}
	testName := "Do a thing"
	results := &[]metrics.TestRunResults{
		{
			&runA,
			&metrics.TestResults{
				"A test",
				"OK",
				&testName,
				[]metrics.SubTest{},
			},
		},
		{
			&runB,
			&metrics.TestResults{
				"A test",
				"ERROR",
				&testName,
				[]metrics.SubTest{},
			},
		},
	}
	gathered := GatherResultsById(results)
	assert.Equal(t, len(gathered), 1) // Merged to single TestId: {"A test",""}.
	for testId, runStatusMap := range gathered {
		assert.Equal(t, testId, metrics.TestId{"A test", ""})
		for testRun, testStatus := range runStatusMap {
			if testRun == runA {
				assert.Equal(t, testStatus, metrics.CompleteTestStatus{
					metrics.TestStatus_fromString("OK"),
					metrics.SubTestStatus_fromString("STATUS_UNKNOWN"),
				})
			} else if testRun == runB {
				assert.Equal(t, testStatus, metrics.CompleteTestStatus{
					metrics.TestStatus_fromString("ERROR"),
					metrics.SubTestStatus_fromString("STATUS_UNKNOWN"),
				})
			} else {
				assert.Fail(t, "Invalid test run in GatherResultsById output")
			}
		}
	}
}
<commit_msg>Clean up tests in light of review comments from #357<commit_after>// Copyright 2017 The WPT Dashboard Project. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

package compute

import (
	"testing"
	"time"

	"github.com/stretchr/testify/assert"

	"github.com/w3c/wptdashboard/metrics"
	"github.com/w3c/wptdashboard/shared"
)

var timeA = time.Unix(0, 0)
var timeB = time.Unix(0, 1)

func TestGatherResultsById_TwoRuns_SameTest(t *testing.T) {
	runA := shared.TestRun{
		"ABrowser",
		"1.0",
		"MyOS",
		"1.0",
		"abcd",
		"http://example.com/a_run.json",
		timeA,
	}
	runB := shared.TestRun{
		"BBrowser",
		"1.0",
		"MyOS",
		"1.0",
		"dcba",
		"http://example.com/b_run.json",
		timeB,
	}
	testName := "Do a thing"
	results := &[]metrics.TestRunResults{
		{
			&runA,
			&metrics.TestResults{
				"A test",
				"OK",
				&testName,
				[]metrics.SubTest{},
			},
		},
		{
			&runB,
			&metrics.TestResults{
				"A test",
				"ERROR",
				&testName,
				[]metrics.SubTest{},
			},
		},
	}
	gathered := GatherResultsById(results)
	assert.Equal(t, 1, len(gathered)) // Merged to single TestId: {"A test",""}.
	for testId, runStatusMap := range gathered {
		assert.Equal(t, testId, metrics.TestId{"A test", ""})
		assert.Equal(t, 2, len(runStatusMap))
		assert.Equal(t, runStatusMap[runA], metrics.CompleteTestStatus{
			metrics.TestStatus_fromString("OK"),
			metrics.SubTestStatus_fromString("STATUS_UNKNOWN"),
		})
		assert.Equal(t, runStatusMap[runB], metrics.CompleteTestStatus{
			metrics.TestStatus_fromString("ERROR"),
			metrics.SubTestStatus_fromString("STATUS_UNKNOWN"),
		})
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	neturl "net/url"
	"strconv"
	"strings"
	"sync"
)

// Connection to a specific index in ES
type EsConn struct {
	hostPrefix string
	path       string
	types      []string
	client     *http.Client
}

var _ DataFlow = &EsConn{}

// http helper - check status and read the whole response body
func readRequest(client *http.Client, req *http.Request) (body []byte, err error) {
	resp, err := client.Do(req)
	if err != nil {
		return
	}
	defer resp.Body.Close()
	body, err = ioutil.ReadAll(resp.Body)
	if err != nil {
		return
	}
	if resp.StatusCode >= 300 {
		err = errors.New(fmt.Sprintf("Http error[%s]: %s", resp.Status, string(body)))
	}
	return
}

// set of _id that failed with anything but document exists
func parseBulkFailures(resp []byte) (fails map[string]bool, err error) {
	var respObj struct {
		Errors bool `json:"errors"`
		Items  []struct {
			Create struct {
				Id     string `json:"_id"`
				Status int    `json:"status"`
			} `json:"create"`
		} `json:"items"`
	}
	err = json.Unmarshal(resp, &respObj)
	if err != nil {
		return
	}
	if respObj.Errors {
		for _, item := range respObj.Items {
			if item.Create.Status != 201 {
				if item.Create.Status == 429 || item.Create.Status == 503 {
					fails[item.Create.Id] = true
				} else if item.Create.Status == 409 {
					// ignore already exists error
				} else {
					log.Printf("Failed %s with %d", item.Create.Id, item.Create.Status)
				}
			}
		}
	}
	return fails, nil
}

func ConnectES(url string) (*EsConn, error) {
	if !strings.HasPrefix(url, "http://") && !strings.HasPrefix(url, "https://") {
		url = "http://" + url
	}
	u, err := neturl.ParseRequestURI(url)
	if err != nil {
		return nil, err
	}
	hostPrefix := fmt.Sprintf("http://%s", u.Host)
	return &EsConn{hostPrefix: hostPrefix, path: u.Path, client: http.DefaultClient}, nil
}

func (this *EsConn) NewScroll(typeName string, size int) (scroll Scroll, err error) {
	query := fmt.Sprintf(`
        {
            "query": {
            	"match_all": {}
            },
            "size": %d
        }
    `, size)
	url := fmt.Sprintf("%s%s/%s/_search?search_type=scan&scroll=5m", this.hostPrefix, this.path, typeName)
	request, err := http.NewRequest("GET", url, bytes.NewBufferString(query))
	if err != nil {
		return
	}
	resp, err := readRequest(this.client, request)
	if err != nil {
		return
	}
	//
	getScrollId := func(data []byte) (string, error) {
		v := struct {
			ScrollID string `json:"_scroll_id"`
		}{}
		err := json.Unmarshal(data, &v)
		return v.ScrollID, err
	}

	scrollID, err := getScrollId(resp)
	if err != nil {
		return
	}
	scroll = Scroll{id: scrollID, es: this}
	return
}

func (this *Scroll) Next() ([]map[string]Any, error) {
	resp, err := this.es.client.Get(this.es.hostPrefix + "/_search/scroll?scroll=5m&scroll_id=" + this.id)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	data, errRead := ioutil.ReadAll(resp.Body)
	if errRead != nil {
		return nil, errRead
	}

	hits := struct {
		Hits struct {
			Hits []map[string]Any `json:"hits"`
		} `json:"hits"`
	}{}
	json.Unmarshal(data, &hits)
	return hits.Hits.Hits, errRead
}

// returns failed bulk ops if any single one failed; error on major filure
func (this *EsConn) Bulk(ops []Bulk) (ret []Bulk, err error) {
	bodyBuf := bytes.Buffer{}
	for _, b := range ops {
		b.Store(&bodyBuf)
	}
	url := fmt.Sprintf("%s%s/_bulk", this.hostPrefix, this.path)
	request, err := http.NewRequest("PUT", url, &bodyBuf)
	if err != nil {
		return nil, err
	}
	resp, err := readRequest(this.client, request)
	if err != nil {
		return
	}
	fails, err := parseBulkFailures(resp)
	if err != nil {
		return
	}
	if len(fails) > 0 {
		for _, op := range ops {
			if _, ok := fails[op.Id]; ok {
				ret = append(ret, op)
			}
		}
	}
	return
}

// delete index of EsConn
func (this *EsConn) DeleteIndex() (err error) {
	req, err := http.NewRequest("DELETE", this.hostPrefix+this.path, bytes.NewReader([]byte{}))
	if err != nil {
		return
	}
	_, err = readRequest(this.client, req)
	return
}

func (this *EsConn) PutIndex(metaString string, repls int, shards int) (err error) {
	// meta - all metadata including the old index name (top-level)
	meta := make(map[string]Index)
	err = json.Unmarshal([]byte(metaString), &meta)
	if err != nil {
		return
	}
	var oldIndexName string
	for k := range meta { // FIXME: iterating over map of 1 item to get the only key ?
		oldIndexName = k
	}
	this.types = nil
	for k := range meta[oldIndexName].Mappings {
		this.types = append(this.types, k)
	}
	log.Printf("Types %s", this.types)
	metaVal := meta[oldIndexName]
	metaVal.Aliases = map[string]Any{} // clear all aliases
	// metaBlob - all index meta data
	if repls >= 0 {
		metaVal.Settings.Index.NumRepls = strconv.Itoa(repls)
	}
	if repls >= 0 {
		metaVal.Settings.Index.NumShards = strconv.Itoa(shards)
	}

	metaBlob, err := json.Marshal(metaVal)
	if err != nil {
		return
	}

	reqMapping, err := http.NewRequest("PUT", this.hostPrefix+this.path, bytes.NewBuffer(metaBlob))
	if err != nil {
		return
	}
	_, err = readRequest(this.client, reqMapping)
	return
}

func (this *EsConn) GetIndex() (string, error) {
	resp, err := this.client.Get(this.hostPrefix + this.path)
	if err != nil {
		return "", err
	}
	defer resp.Body.Close()
	data, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return "", err
	}
	meta := make(map[string]Index)
	err = json.Unmarshal(data, &meta)
	if err != nil {
		return "", err
	}
	indexName := ""
	for k := range meta { // FIXME: iterating over map of 1 item to get the only key ?
		indexName = k
	}
	if err != nil {
		return "", err
	}
	this.types = nil
	log.Print(indexName)
	for k := range meta[indexName].Mappings {
		this.types = append(this.types, k)
	}
	metaVal := meta[indexName]
	// clear aliases
	metaVal.Aliases = map[string]Any{} // clear all aliases
	meta[indexName] = metaVal
	log.Printf("%v", this.types)
	if resp.StatusCode >= 400 {
		log.Fatalf("Elastic error[%s]: %s", resp.Status, string(data))
	}
	data, err = json.Marshal(meta)
	if err != nil {
		return "", err
	}
	return string(data), nil
}

func (this *EsConn) readBulk(typeName string, window, bulkSize int, dest chan<- []Bulk) {
	log.Printf("Exporting type: `%s`", typeName)
	scroll, err := this.NewScroll(typeName, window)
	if err != nil {
		panic(err)
	}
	batcher := Batcher{size: bulkSize, dest: dest}
	defer batcher.Flush()
	for hits, err := scroll.Next(); len(hits) != 0 && err == nil; hits, err = scroll.Next() {
		log.Printf("Fetched %d\n", len(hits))
		for _, h := range hits {
			bytes, err := json.Marshal(h["_source"])
			if err != nil {
				panic(err)
			}
			batcher.Put(Bulk{Id: h["_id"].(string), Type: typeName, Doc: bytes})
		}
	}
}

func (this *EsConn) StreamTo(window, bulkSize int, dest chan []Bulk) {
	defer close(dest)
	for _, t := range this.types {
		this.readBulk(t, window, bulkSize, dest)
	}
	return
}

func (this *EsConn) AcceptFrom(parallel int, src chan []Bulk) (err error) {
	var group sync.WaitGroup
	group.Add(parallel)
	for i := 0; i < parallel; i++ {
		go func() {
			var leftover []Bulk
			for batch := range src {
				batch = append(batch, leftover...)
				leftover, err = this.Bulk(batch)
				if err != nil {
					panic(err)
				}
				log.Printf("Imported %d/%d", len(batch)-len(leftover), len(batch))
			}
			for len(leftover) > 0 {
				log.Printf("Pushing in last %d", len(leftover))
				leftover, err = this.Bulk(leftover)
				if err != nil {
					panic(err)
				}
			}
			group.Done()
		}()
	}
	group.Wait()
	return
}
<commit_msg>Initialize map first.<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	neturl "net/url"
	"strconv"
	"strings"
	"sync"
)

// Connection to a specific index in ES
type EsConn struct {
	hostPrefix string
	path       string
	types      []string
	client     *http.Client
}

var _ DataFlow = &EsConn{}

// http helper - check status and read the whole response body
func readRequest(client *http.Client, req *http.Request) (body []byte, err error) {
	resp, err := client.Do(req)
	if err != nil {
		return
	}
	defer resp.Body.Close()
	body, err = ioutil.ReadAll(resp.Body)
	if err != nil {
		return
	}
	if resp.StatusCode >= 300 {
		err = errors.New(fmt.Sprintf("Http error[%s]: %s", resp.Status, string(body)))
	}
	return
}

// set of _id that failed with anything but document exists
func parseBulkFailures(resp []byte) (fails map[string]bool, err error) {
	fails = map[string]bool{}
	var respObj struct {
		Errors bool `json:"errors"`
		Items  []struct {
			Create struct {
				Id     string `json:"_id"`
				Status int    `json:"status"`
			} `json:"create"`
		} `json:"items"`
	}
	err = json.Unmarshal(resp, &respObj)
	if err != nil {
		return
	}
	if respObj.Errors {
		for _, item := range respObj.Items {
			if item.Create.Status != 201 {
				if item.Create.Status == 429 || item.Create.Status == 503 {
					fails[item.Create.Id] = true
				} else if item.Create.Status == 409 {
					// ignore already exists error
				} else {
					log.Printf("Failed %s with %d", item.Create.Id, item.Create.Status)
				}
			}
		}
	}
	return fails, nil
}

func ConnectES(url string) (*EsConn, error) {
	if !strings.HasPrefix(url, "http://") && !strings.HasPrefix(url, "https://") {
		url = "http://" + url
	}
	u, err := neturl.ParseRequestURI(url)
	if err != nil {
		return nil, err
	}
	hostPrefix := fmt.Sprintf("http://%s", u.Host)
	return &EsConn{hostPrefix: hostPrefix, path: u.Path, client: http.DefaultClient}, nil
}

func (this *EsConn) NewScroll(typeName string, size int) (scroll Scroll, err error) {
	query := fmt.Sprintf(`
        {
            "query": {
            	"match_all": {}
            },
            "size": %d
        }
    `, size)
	url := fmt.Sprintf("%s%s/%s/_search?search_type=scan&scroll=5m", this.hostPrefix, this.path, typeName)
	request, err := http.NewRequest("GET", url, bytes.NewBufferString(query))
	if err != nil {
		return
	}
	resp, err := readRequest(this.client, request)
	if err != nil {
		return
	}
	//
	getScrollId := func(data []byte) (string, error) {
		v := struct {
			ScrollID string `json:"_scroll_id"`
		}{}
		err := json.Unmarshal(data, &v)
		return v.ScrollID, err
	}

	scrollID, err := getScrollId(resp)
	if err != nil {
		return
	}
	scroll = Scroll{id: scrollID, es: this}
	return
}

func (this *Scroll) Next() ([]map[string]Any, error) {
	resp, err := this.es.client.Get(this.es.hostPrefix + "/_search/scroll?scroll=5m&scroll_id=" + this.id)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	data, errRead := ioutil.ReadAll(resp.Body)
	if errRead != nil {
		return nil, errRead
	}

	hits := struct {
		Hits struct {
			Hits []map[string]Any `json:"hits"`
		} `json:"hits"`
	}{}
	json.Unmarshal(data, &hits)
	return hits.Hits.Hits, errRead
}

// returns failed bulk ops if any single one failed; error on major filure
func (this *EsConn) Bulk(ops []Bulk) (ret []Bulk, err error) {
	bodyBuf := bytes.Buffer{}
	for _, b := range ops {
		b.Store(&bodyBuf)
	}
	url := fmt.Sprintf("%s%s/_bulk", this.hostPrefix, this.path)
	request, err := http.NewRequest("PUT", url, &bodyBuf)
	if err != nil {
		return nil, err
	}
	resp, err := readRequest(this.client, request)
	if err != nil {
		return
	}
	fails, err := parseBulkFailures(resp)
	if err != nil {
		return
	}
	if len(fails) > 0 {
		for _, op := range ops {
			if _, ok := fails[op.Id]; ok {
				ret = append(ret, op)
			}
		}
	}
	return
}

// delete index of EsConn
func (this *EsConn) DeleteIndex() (err error) {
	req, err := http.NewRequest("DELETE", this.hostPrefix+this.path, bytes.NewReader([]byte{}))
	if err != nil {
		return
	}
	_, err = readRequest(this.client, req)
	return
}

func (this *EsConn) PutIndex(metaString string, repls int, shards int) (err error) {
	// meta - all metadata including the old index name (top-level)
	meta := make(map[string]Index)
	err = json.Unmarshal([]byte(metaString), &meta)
	if err != nil {
		return
	}
	var oldIndexName string
	for k := range meta { // FIXME: iterating over map of 1 item to get the only key ?
		oldIndexName = k
	}
	this.types = nil
	for k := range meta[oldIndexName].Mappings {
		this.types = append(this.types, k)
	}
	log.Printf("Types %s", this.types)
	metaVal := meta[oldIndexName]
	metaVal.Aliases = map[string]Any{} // clear all aliases
	// metaBlob - all index meta data
	if repls >= 0 {
		metaVal.Settings.Index.NumRepls = strconv.Itoa(repls)
	}
	if repls >= 0 {
		metaVal.Settings.Index.NumShards = strconv.Itoa(shards)
	}

	metaBlob, err := json.Marshal(metaVal)
	if err != nil {
		return
	}

	reqMapping, err := http.NewRequest("PUT", this.hostPrefix+this.path, bytes.NewBuffer(metaBlob))
	if err != nil {
		return
	}
	_, err = readRequest(this.client, reqMapping)
	return
}

func (this *EsConn) GetIndex() (string, error) {
	resp, err := this.client.Get(this.hostPrefix + this.path)
	if err != nil {
		return "", err
	}
	defer resp.Body.Close()
	data, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return "", err
	}
	meta := make(map[string]Index)
	err = json.Unmarshal(data, &meta)
	if err != nil {
		return "", err
	}
	indexName := ""
	for k := range meta { // FIXME: iterating over map of 1 item to get the only key ?
		indexName = k
	}
	if err != nil {
		return "", err
	}
	this.types = nil
	log.Print(indexName)
	for k := range meta[indexName].Mappings {
		this.types = append(this.types, k)
	}
	metaVal := meta[indexName]
	// clear aliases
	metaVal.Aliases = map[string]Any{} // clear all aliases
	meta[indexName] = metaVal
	log.Printf("%v", this.types)
	if resp.StatusCode >= 400 {
		log.Fatalf("Elastic error[%s]: %s", resp.Status, string(data))
	}
	data, err = json.Marshal(meta)
	if err != nil {
		return "", err
	}
	return string(data), nil
}

func (this *EsConn) readBulk(typeName string, window, bulkSize int, dest chan<- []Bulk) {
	log.Printf("Exporting type: `%s`", typeName)
	scroll, err := this.NewScroll(typeName, window)
	if err != nil {
		panic(err)
	}
	batcher := Batcher{size: bulkSize, dest: dest}
	defer batcher.Flush()
	for hits, err := scroll.Next(); len(hits) != 0 && err == nil; hits, err = scroll.Next() {
		log.Printf("Fetched %d\n", len(hits))
		for _, h := range hits {
			bytes, err := json.Marshal(h["_source"])
			if err != nil {
				panic(err)
			}
			batcher.Put(Bulk{Id: h["_id"].(string), Type: typeName, Doc: bytes})
		}
	}
}

func (this *EsConn) StreamTo(window, bulkSize int, dest chan []Bulk) {
	defer close(dest)
	for _, t := range this.types {
		this.readBulk(t, window, bulkSize, dest)
	}
	return
}

func (this *EsConn) AcceptFrom(parallel int, src chan []Bulk) (err error) {
	var group sync.WaitGroup
	group.Add(parallel)
	for i := 0; i < parallel; i++ {
		go func() {
			var leftover []Bulk
			for batch := range src {
				batch = append(batch, leftover...)
				leftover, err = this.Bulk(batch)
				if err != nil {
					panic(err)
				}
				log.Printf("Imported %d/%d", len(batch)-len(leftover), len(batch))
			}
			for len(leftover) > 0 {
				log.Printf("Pushing in last %d", len(leftover))
				leftover, err = this.Bulk(leftover)
				if err != nil {
					panic(err)
				}
			}
			group.Done()
		}()
	}
	group.Wait()
	return
}
<|endoftext|>