<commit_before>package main

import (
	"crypto/tls"
	"crypto/x509"
	"fmt"
	"io/ioutil"
	"net"
	"os"
	"os/signal"
	"syscall"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/jessevdk/go-flags"
	"github.com/soheilhy/cmux"
	"google.golang.org/grpc"
	"google.golang.org/grpc/reflection"

	em "github.com/ContextLogic/eventmaster"
	emproto "github.com/ContextLogic/eventmaster/proto"
)

func main() {
	var config em.Config
	parser := flags.NewParser(&config, flags.Default)
	if _, err := parser.Parse(); err != nil {
		log.Fatalf("Error parsing flags: %v", err)
	}

	err := em.RegisterPromMetrics()
	if err != nil {
		log.Fatalf("Unable to register prometheus metrics: %v", err)
	}

	// Set up event store
	emConf, err := ParseEMConfig(config.ConfigFile)
	if err != nil {
		log.Fatalf("problem parsing config file: %v", err)
	}

	var ds em.DataStore
	if emConf.DataStore == "cassandra" {
		ds, err = em.NewCassandraStore(emConf.CassConfig)
		if err != nil {
			log.Fatalf("failed to create cassandra data store: %v", err)
		}
	} else {
		log.Fatalf("Unrecognized data store option")
	}
	store, err := em.NewEventStore(ds)
	if err != nil {
		log.Fatalf("Unable to create event store: %v", err)
	}
	if err := store.Update(); err != nil {
		fmt.Println("Error loading dcs and topics from Cassandra", err)
	}

	// Create listening socket for grpc server
	lis, err := net.Listen("tcp", fmt.Sprintf(":%d", config.Port))
	if err != nil {
		log.Fatalf("failed to listen: %v", err)
	}

	mux := cmux.New(lis)
	httpL := mux.Match(cmux.HTTP1Fast())
	grpcL := mux.Match(cmux.HTTP2HeaderField("content-type", "application/grpc"))

	var tlsConfig *tls.Config
	if config.CAFile != "" {
		cert, err := tls.LoadX509KeyPair(config.CertFile, config.KeyFile)
		if err != nil {
			log.Fatalf("Failed to load X509 key pair %v", err)
		}

		caCert, err := ioutil.ReadFile(config.CAFile)
		if err != nil {
			log.Fatalf("Failed to load CA cert file %v", err)
		}

		caCertPool := x509.NewCertPool()
		caCertPool.AppendCertsFromPEM(caCert)
		tlsConfig = &tls.Config{
			ClientAuth:   tls.RequireAndVerifyClientCert,
			ClientCAs:    caCertPool,
			Certificates: []tls.Certificate{cert},
			RootCAs:      caCertPool,
		}
	}

	httpS := em.NewHTTPServer(tlsConfig, store, config.Templates, config.StaticFiles)

	// Create the EventMaster grpc server
	grpcServer, err := em.NewGRPCServer(&config, store)
	if err != nil {
		log.Fatalf("Unable to start server: %v", err)
	}

	maxMsgSizeOpt := grpc.MaxMsgSize(1024 * 1024 * 100)
	// Create the gRPC server and register our service
	grpcS := grpc.NewServer(maxMsgSizeOpt)
	emproto.RegisterEventMasterServer(grpcS, grpcServer)
	reflection.Register(grpcS)

	go httpS.Serve(httpL)
	go grpcS.Serve(grpcL)

	go func() {
		fmt.Println("Starting server on port", config.Port)
		if err := mux.Serve(); err != nil {
			log.Fatalf("Error starting server: %v", err)
		}
	}()

	updateTicker := time.NewTicker(time.Second * time.Duration(emConf.UpdateInterval))
	go func() {
		for range updateTicker.C {
			if err := store.Update(); err != nil {
				fmt.Println("Error updating dcs and topics from cassandra:", err)
			}
		}
	}()
	rsyslogServer := &em.RsyslogServer{}

	if config.RsyslogServer {
		rsyslogServer, err = em.NewRsyslogServer(store, tlsConfig, config.RsyslogPort)
		if err != nil {
			log.Fatalf("Unable to start server: %v", err)
		}
		rsyslogServer.AcceptLogs()
	}

	stopChan := make(chan os.Signal)
	signal.Notify(stopChan, syscall.SIGTERM, syscall.SIGINT)

	<-stopChan
	fmt.Println("Got shutdown signal, gracefully shutting down...")
	updateTicker.Stop()
	store.CloseSession()
	grpcS.GracefulStop()
	lis.Close()
	if config.RsyslogServer {
		rsyslogServer.Stop()
	}
}
<commit_msg>Don't double output help message<commit_after>package main

import (
	"crypto/tls"
	"crypto/x509"
	"fmt"
	"io/ioutil"
	"net"
	"os"
	"os/signal"
	"syscall"
	"time"

	log "github.com/Sirupsen/logrus"
	flags "github.com/jessevdk/go-flags"
	"github.com/soheilhy/cmux"
	"google.golang.org/grpc"
	"google.golang.org/grpc/reflection"

	em "github.com/ContextLogic/eventmaster"
	emproto "github.com/ContextLogic/eventmaster/proto"
)

func main() {
	var config em.Config
	parser := flags.NewParser(&config, flags.Default)
	if _, err := parser.Parse(); err != nil {
		os.Exit(1)
	}

	err := em.RegisterPromMetrics()
	if err != nil {
		log.Fatalf("Unable to register prometheus metrics: %v", err)
	}

	// Set up event store
	emConf, err := ParseEMConfig(config.ConfigFile)
	if err != nil {
		log.Fatalf("problem parsing config file: %v", err)
	}

	var ds em.DataStore
	if emConf.DataStore == "cassandra" {
		ds, err = em.NewCassandraStore(emConf.CassConfig)
		if err != nil {
			log.Fatalf("failed to create cassandra data store: %v", err)
		}
	} else {
		log.Fatalf("Unrecognized data store option")
	}
	store, err := em.NewEventStore(ds)
	if err != nil {
		log.Fatalf("Unable to create event store: %v", err)
	}
	if err := store.Update(); err != nil {
		fmt.Println("Error loading dcs and topics from Cassandra", err)
	}

	// Create listening socket for grpc server
	lis, err := net.Listen("tcp", fmt.Sprintf(":%d", config.Port))
	if err != nil {
		log.Fatalf("failed to listen: %v", err)
	}

	mux := cmux.New(lis)
	httpL := mux.Match(cmux.HTTP1Fast())
	grpcL := mux.Match(cmux.HTTP2HeaderField("content-type", "application/grpc"))

	var tlsConfig *tls.Config
	if config.CAFile != "" {
		cert, err := tls.LoadX509KeyPair(config.CertFile, config.KeyFile)
		if err != nil {
			log.Fatalf("Failed to load X509 key pair %v", err)
		}

		caCert, err := ioutil.ReadFile(config.CAFile)
		if err != nil {
			log.Fatalf("Failed to load CA cert file %v", err)
		}

		caCertPool := x509.NewCertPool()
		caCertPool.AppendCertsFromPEM(caCert)
		tlsConfig = &tls.Config{
			ClientAuth:   tls.RequireAndVerifyClientCert,
			ClientCAs:    caCertPool,
			Certificates: []tls.Certificate{cert},
			RootCAs:      caCertPool,
		}
	}

	httpS := em.NewHTTPServer(tlsConfig, store, config.Templates, config.StaticFiles)

	// Create the EventMaster grpc server
	grpcServer, err := em.NewGRPCServer(&config, store)
	if err != nil {
		log.Fatalf("Unable to start server: %v", err)
	}

	maxMsgSizeOpt := grpc.MaxMsgSize(1024 * 1024 * 100)
	// Create the gRPC server and register our service
	grpcS := grpc.NewServer(maxMsgSizeOpt)
	emproto.RegisterEventMasterServer(grpcS, grpcServer)
	reflection.Register(grpcS)

	go httpS.Serve(httpL)
	go grpcS.Serve(grpcL)

	go func() {
		fmt.Println("Starting server on port", config.Port)
		if err := mux.Serve(); err != nil {
			log.Fatalf("Error starting server: %v", err)
		}
	}()

	updateTicker := time.NewTicker(time.Second * time.Duration(emConf.UpdateInterval))
	go func() {
		for range updateTicker.C {
			if err := store.Update(); err != nil {
				fmt.Println("Error updating dcs and topics from cassandra:", err)
			}
		}
	}()
	rsyslogServer := &em.RsyslogServer{}

	if config.RsyslogServer {
		rsyslogServer, err = em.NewRsyslogServer(store, tlsConfig, config.RsyslogPort)
		if err != nil {
			log.Fatalf("Unable to start server: %v", err)
		}
		rsyslogServer.AcceptLogs()
	}

	stopChan := make(chan os.Signal)
	signal.Notify(stopChan, syscall.SIGTERM, syscall.SIGINT)

	<-stopChan
	fmt.Println("Got shutdown signal, gracefully shutting down...")
	updateTicker.Stop()
	store.CloseSession()
	grpcS.GracefulStop()
	lis.Close()
	if config.RsyslogServer {
		rsyslogServer.Stop()
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"math/rand"
	"os"
	"os/signal"
	"strconv"
	"sync"
	"syscall"
	"text/template"

	inKafkaMdm "github.com/raintank/metrictank/input/kafkamdm"
	"github.com/raintank/metrictank/stats"
	"github.com/raintank/worldping-api/pkg/log"
	"github.com/rakyll/globalconf"
	"gopkg.in/raintank/schema.v1"
)

var (
	confFile = flag.String("config", "/etc/raintank/metrictank.ini", "configuration file path")
	format   = flag.String("format", "{{.Part}} {{.OrgId}} {{.Id}} {{.Name}} {{.Metric}} {{.Interval}} {{.Value}} {{.Time}} {{.Unit}} {{.Mtype}} {{.Tags}}", "template to render the data with")

	stdoutLock = sync.Mutex{}
)

type Data struct {
	Part int32
	schema.MetricData
}

type inputPrinter struct {
	template.Template
	data Data
}

func newInputPrinter(format string) inputPrinter {
	tpl := template.Must(template.New("format").Parse(format + "\n"))
	return inputPrinter{
		*tpl,
		Data{},
	}
}

func (ip inputPrinter) Process(metric *schema.MetricData, partition int32) {
	ip.data.MetricData = *metric
	ip.data.Part = partition
	stdoutLock.Lock()
	err := ip.Execute(os.Stdout, ip.data)
	stdoutLock.Unlock()
	if err != nil {
		log.Error(0, "executing template: %s", err)
	}
}

func main() {
	flag.Usage = func() {
		fmt.Fprintln(os.Stderr, "mt-kafka-mdm-sniff")
		fmt.Fprintln(os.Stderr)
		fmt.Fprintln(os.Stderr, "Inspects what's flowing through kafka (in mdm format) and reports it to you")
		fmt.Fprintf(os.Stderr, "\nFlags:\n\n")
		flag.PrintDefaults()
	}
	flag.Parse()
	log.NewLogger(0, "console", fmt.Sprintf(`{"level": %d, "formatting":false}`, 2))
	instance := "mt-kafka-mdm-sniff" + strconv.Itoa(rand.Int())

	// Only try and parse the conf file if it exists
	path := ""
	if _, err := os.Stat(*confFile); err == nil {
		path = *confFile
	}
	conf, err := globalconf.NewWithOptions(&globalconf.Options{
		Filename:  path,
		EnvPrefix: "MT_",
	})
	if err != nil {
		log.Fatal(4, "error with configuration file: %s", err)
		os.Exit(1)
	}
	inKafkaMdm.ConfigSetup()
	conf.ParseAll()

	// config may have had it disabled
	inKafkaMdm.Enabled = true
	// important: we don't want to share the same offset tracker as the mdm input of MT itself
	inKafkaMdm.DataDir = "/tmp/" + instance

	inKafkaMdm.ConfigProcess(instance)

	stats.NewDevnull() // make sure metrics don't pile up without getting discarded

	mdm := inKafkaMdm.New()
	mdm.Start(newInputPrinter(*format))
	sigChan := make(chan os.Signal, 1)
	signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)
	<-sigChan
	log.Info("stopping")
	mdm.Stop()
}
<commit_msg>add filter options to mt-kafka-mdm-sniff<commit_after>package main

import (
	"flag"
	"fmt"
	"math/rand"
	"os"
	"os/signal"
	"strconv"
	"strings"
	"sync"
	"syscall"
	"text/template"

	inKafkaMdm "github.com/raintank/metrictank/input/kafkamdm"
	"github.com/raintank/metrictank/stats"
	"github.com/raintank/worldping-api/pkg/log"
	"github.com/rakyll/globalconf"
	"gopkg.in/raintank/schema.v1"
)

var (
	confFile = flag.String("config", "/etc/raintank/metrictank.ini", "configuration file path")
	format   = flag.String("format", "{{.Part}} {{.OrgId}} {{.Id}} {{.Name}} {{.Metric}} {{.Interval}} {{.Value}} {{.Time}} {{.Unit}} {{.Mtype}} {{.Tags}}", "template to render the data with")
	prefix   = flag.String("prefix", "", "only show metrics that have this prefix")
	substr   = flag.String("substr", "", "only show metrics that have this substring")

	stdoutLock = sync.Mutex{}
)

type Data struct {
	Part int32
	schema.MetricData
}

type inputPrinter struct {
	template.Template
	data Data
}

func newInputPrinter(format string) inputPrinter {
	tpl := template.Must(template.New("format").Parse(format + "\n"))
	return inputPrinter{
		*tpl,
		Data{},
	}
}

func (ip inputPrinter) Process(metric *schema.MetricData, partition int32) {
	if *prefix != "" && !strings.HasPrefix(metric.Metric, *prefix) {
		return
	}
	if *substr != "" && !strings.Contains(metric.Metric, *substr) {
		return
	}
	ip.data.MetricData = *metric
	ip.data.Part = partition
	stdoutLock.Lock()
	err := ip.Execute(os.Stdout, ip.data)
	stdoutLock.Unlock()
	if err != nil {
		log.Error(0, "executing template: %s", err)
	}
}

func main() {
	flag.Usage = func() {
		fmt.Fprintln(os.Stderr, "mt-kafka-mdm-sniff")
		fmt.Fprintln(os.Stderr)
		fmt.Fprintln(os.Stderr, "Inspects what's flowing through kafka (in mdm format) and reports it to you")
		fmt.Fprintf(os.Stderr, "\nFlags:\n\n")
		flag.PrintDefaults()
	}
	flag.Parse()
	log.NewLogger(0, "console", fmt.Sprintf(`{"level": %d, "formatting":false}`, 2))
	instance := "mt-kafka-mdm-sniff" + strconv.Itoa(rand.Int())

	// Only try and parse the conf file if it exists
	path := ""
	if _, err := os.Stat(*confFile); err == nil {
		path = *confFile
	}
	conf, err := globalconf.NewWithOptions(&globalconf.Options{
		Filename:  path,
		EnvPrefix: "MT_",
	})
	if err != nil {
		log.Fatal(4, "error with configuration file: %s", err)
		os.Exit(1)
	}
	inKafkaMdm.ConfigSetup()
	conf.ParseAll()

	// config may have had it disabled
	inKafkaMdm.Enabled = true
	// important: we don't want to share the same offset tracker as the mdm input of MT itself
	inKafkaMdm.DataDir = "/tmp/" + instance

	inKafkaMdm.ConfigProcess(instance)

	stats.NewDevnull() // make sure metrics don't pile up without getting discarded

	mdm := inKafkaMdm.New()
	mdm.Start(newInputPrinter(*format))
	sigChan := make(chan os.Signal, 1)
	signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)
	<-sigChan
	log.Info("stopping")
	mdm.Stop()
}
<|endoftext|>


<commit_before>package procfinder

import (
	"fmt"
	"io/ioutil"
	"os"
	"os/user"
	"path/filepath"
	"strconv"
	"strings"
)

func readNetTCP() ([]string, error) {
	procTCP, err := ioutil.ReadFile("/proc/net/tcp")

	if err != nil {
		return nil, err
	}

	lines := strings.Split(string(procTCP), "\n")

	return lines[1 : len(lines)-1], nil
}

// Process represents a unix based process. This provides the direct path to the exe that
// it was originally spawned with, along with the nicename and process ID.
type Process struct {
	PID         string
	Name        string
	Exe         string
	User        string
	IP          string
	Port        int64
	ForeignIP   string
	ForeignPort int64
}

func removeEmpty(array []string) []string {
	// remove empty data from line
	var newArray []string
	for _, i := range array {
		if i != "" {
			newArray = append(newArray, i)
		}
	}

	return newArray
}

// convert hexadecimal to decimal
func hexToDec(h string) int64 {
	d, err := strconv.ParseInt(h, 16, 32)

	if err != nil {
		return 0
	}

	return d
}

// convert the ipv4 to decimal. would need to rearrange the ip because the default value
// is in little Endian order.
func ip(ip string) string {
	var out string

	// check ip size. if greater than 8, it is ipv6
	if len(ip) > 8 {
		i := []string{ip[30:32], ip[28:30], ip[26:28], ip[24:26], ip[22:24], ip[20:22], ip[18:20], ip[16:18], ip[14:16], ip[12:14], ip[10:12], ip[8:10], ip[6:8], ip[4:6], ip[2:4], ip[0:2]}
		out = fmt.Sprintf("%v%v:%v%v:%v%v:%v%v:%v%v:%v%v:%v%v:%v%v", i[14], i[15], i[13], i[12], i[10], i[11], i[8], i[9], i[6], i[7], i[4], i[5], i[2], i[3], i[0], i[1])
	} else {
		// ipv4
		i := []int64{hexToDec(ip[6:8]), hexToDec(ip[4:6]), hexToDec(ip[2:4]), hexToDec(ip[0:2])}

		out = fmt.Sprintf("%v.%v.%v.%v", i[0], i[1], i[2], i[3])
	}

	return out
}

// loop through all fd dirs of process on /proc to compare the inode and get the pid
func getPid(inode string) (pid string) {
	d, err := filepath.Glob("/proc/[0-9]*/fd/[0-9]*")
	if err != nil {
		return pid
	}

	for _, item := range d {
		path, _ := os.Readlink(item)
		if strings.Contains(path, inode) {
			pid = strings.Split(item, "/")[2]
		}
	}

	return pid
}

func getProcessExe(pid string) string {
	exe := fmt.Sprintf("/proc/%s/exe", pid)
	path, _ := os.Readlink(exe)
	return path
}

func getProcessName(pid string) (name string) {
	tmp, err := ioutil.ReadFile(fmt.Sprintf("/proc/%s/comm", pid))

	if err != nil {
		return ""
	}

	return strings.Split(string(tmp), "\n")[0]
}

func getUser(uid string) string {
	u, _ := user.LookupId(uid)
	return u.Username
}

// GetProcs crawls /proc/ for all pids that have bound ports
func GetProcs() (pl []*Process, err error) {
	tcp, err := readNetTCP()

	if err != nil {
		return nil, err
	}

	for _, line := range tcp {
		lineArray := removeEmpty(strings.Split(strings.TrimSpace(line), " "))
		ipPort := strings.Split(lineArray[1], ":")
		fipPort := strings.Split(lineArray[2], ":")

		proc := &Process{
			IP:          ip(ipPort[0]),
			Port:        hexToDec(ipPort[1]),
			ForeignIP:   ip(fipPort[0]),
			ForeignPort: hexToDec(fipPort[1]),
			User:        getUser(lineArray[7]),
			PID:         getPid(lineArray[9]),
		}

		proc.Exe = getProcessExe(proc.PID)
		proc.Name = getProcessName(proc.PID)

		pl = append(pl, proc)
	}

	return pl, nil
}
<commit_msg>add ip conversion failover<commit_after>package procfinder

import (
	"fmt"
	"io/ioutil"
	"os"
	"os/user"
	"path/filepath"
	"strconv"
	"strings"
)

func readNetTCP() ([]string, error) {
	procTCP, err := ioutil.ReadFile("/proc/net/tcp")

	if err != nil {
		return nil, err
	}

	lines := strings.Split(string(procTCP), "\n")

	return lines[1 : len(lines)-1], nil
}

// Process represents a unix based process. This provides the direct path to the exe that
// it was originally spawned with, along with the nicename and process ID.
type Process struct {
	PID         string
	Name        string
	Exe         string
	User        string
	IP          string
	Port        int64
	ForeignIP   string
	ForeignPort int64
}

func removeEmpty(array []string) []string {
	// remove empty data from line
	var newArray []string
	for _, i := range array {
		if i != "" {
			newArray = append(newArray, i)
		}
	}

	return newArray
}

// convert hexadecimal to decimal
func hexToDec(h string) int64 {
	d, err := strconv.ParseInt(h, 16, 32)

	if err != nil {
		return 0
	}

	return d
}

// convert the ipv4 to decimal. would need to rearrange the ip because the default value
// is in little Endian order.
func ip(ip string) string {
	var out string

	// check ip size. if greater than 8, it is ipv6
	if len(ip) > 8 && len(ip) <= 32 {
		i := []string{ip[30:32], ip[28:30], ip[26:28], ip[24:26], ip[22:24], ip[20:22], ip[18:20], ip[16:18], ip[14:16], ip[12:14], ip[10:12], ip[8:10], ip[6:8], ip[4:6], ip[2:4], ip[0:2]}
		out = fmt.Sprintf("%v%v:%v%v:%v%v:%v%v:%v%v:%v%v:%v%v:%v%v", i[14], i[15], i[13], i[12], i[10], i[11], i[8], i[9], i[6], i[7], i[4], i[5], i[2], i[3], i[0], i[1])
	} else if len(ip) <= 8 && len(ip) > 0 {
		// ipv4
		i := []int64{hexToDec(ip[6:8]), hexToDec(ip[4:6]), hexToDec(ip[2:4]), hexToDec(ip[0:2])}

		out = fmt.Sprintf("%v.%v.%v.%v", i[0], i[1], i[2], i[3])
	} else {
		return "0.0.0.0"
	}

	return out
}

// loop through all fd dirs of process on /proc to compare the inode and get the pid
func getPid(inode string) (pid string) {
	d, err := filepath.Glob("/proc/[0-9]*/fd/[0-9]*")
	if err != nil {
		return pid
	}

	for _, item := range d {
		path, _ := os.Readlink(item)
		if strings.Contains(path, inode) {
			pid = strings.Split(item, "/")[2]
		}
	}

	return pid
}

func getProcessExe(pid string) string {
	exe := fmt.Sprintf("/proc/%s/exe", pid)
	path, _ := os.Readlink(exe)
	return path
}

func getProcessName(pid string) (name string) {
	tmp, err := ioutil.ReadFile(fmt.Sprintf("/proc/%s/comm", pid))

	if err != nil {
		return ""
	}

	return strings.Split(string(tmp), "\n")[0]
}

func getUser(uid string) string {
	u, _ := user.LookupId(uid)
	return u.Username
}

// GetProcs crawls /proc/ for all pids that have bound ports
func GetProcs() (pl []*Process, err error) {
	tcp, err := readNetTCP()

	if err != nil {
		return nil, err
	}

	for _, line := range tcp {
		lineArray := removeEmpty(strings.Split(strings.TrimSpace(line), " "))
		ipPort := strings.Split(lineArray[1], ":")
		fipPort := strings.Split(lineArray[2], ":")

		proc := &Process{
			IP:          ip(ipPort[0]),
			Port:        hexToDec(ipPort[1]),
			ForeignIP:   ip(fipPort[0]),
			ForeignPort: hexToDec(fipPort[1]),
			User:        getUser(lineArray[7]),
			PID:         getPid(lineArray[9]),
		}

		proc.Exe = getProcessExe(proc.PID)
		proc.Name = getProcessName(proc.PID)

		pl = append(pl, proc)
	}

	return pl, nil
}
<|endoftext|>


<commit_before>package raft

import (
	"bytes"
	"encoding/json"
	"fmt"
	"github.com/arpith/mmapd/db"
	"net/http"
)

type appendEntryRequest struct {
	Term         int
	LeaderID     string
	PrevLogIndex int
	PrevLogTerm  int
	Entry        db.Entry
	LeaderCommit int
}

type appendRequest struct {
	Req        appendEntryRequest
	ReturnChan chan appendEntryResponse
}

type appendEntryResponse struct {
	Term    int
	Success bool
}

type followerResponse struct {
	ServerIndex int
	Resp        appendEntryResponse
}

func (s *server) appendEntry(command string, key string, value string, isCommitted chan bool) {
	entry := &db.Entry{command, key, value, s.term}
	index := -1
	if command != "" {
		s.db.Log.AppendEntry(*entry)
		index = len(s.db.Log.Entries) - 1
	}
	respChan := make(chan followerResponse)
	for i := 0; i < len(s.config); i++ {
		if s.config[i] != s.id {
			go s.sendAppendEntryRequest(i, index, respChan)
		}
	}
	responseCount := 0
	if len(s.config) > 1 {
		for {
			_ = <-respChan
			responseCount++
			for N := s.commitIndex + 1; N < len(s.db.Log.Entries); N++ {
				//Check if there exists an N > commitIndex
				count := 0
				for i := 0; i < len(s.matchIndex); i++ {
					if s.matchIndex[i] >= N {
						count++
					}
				}
				// Check if a majority of matchIndex[i] >= N
				cond1 := count > len(s.matchIndex)/2
				// Check if log[N].term == currentTerm
				cond2 := s.db.Log.Entries[N].Term == s.term
				if cond1 && cond2 {
					// Set commitIndex to N
					s.commitIndex = N
				} else {
					break
				}
			}
			if s.commitIndex == index {
				isCommitted <- true
				return
			}
		}
	} else {
		if command != "" {
			s.commitIndex++
			isCommitted <- true
		}
	}
}

func (s *server) sendAppendEntryRequest(followerIndex int, entryIndex int, respChan chan followerResponse) {
	entryP := &db.Entry{"", "", "", s.term}
	entry := *entryP
	if entryIndex != -1 {
		entry = s.db.Log.Entries[entryIndex]
	}
	follower := s.config[followerIndex]
	prevLogIndex := entryIndex - 1
	prevLogTerm := 0
	if prevLogIndex > 0 {
		prevLogTerm = s.db.Log.Entries[prevLogIndex].Term
	}
	a := &appendEntryRequest{s.term, s.id, prevLogIndex, prevLogTerm, entry, s.commitIndex}
	b := new(bytes.Buffer)
	json.NewEncoder(b).Encode(a)
	resp, err := http.Post("http://"+follower+"/append", "application/json", b)
	if err != nil {
		fmt.Println("Couldn't send append entry request to " + follower)
		fmt.Println(err)
		go s.sendAppendEntryRequest(followerIndex, entryIndex, respChan)
	} else {
		r := &appendEntryResponse{}
		err := json.NewDecoder(resp.Body).Decode(r)
		if err != nil {
			fmt.Println("Couldn't decode append entries response from " + s.config[followerIndex])
			return
		}
		defer resp.Body.Close()
		if r.Term > s.term {
			s.term = r.Term
			s.stepDown("Got append entry RPC response with term > current term")
		}
		if r.Success {
			s.term = r.Term
			s.nextIndex[followerIndex]++
			s.matchIndex[followerIndex]++
			followerResp := &followerResponse{followerIndex, *r}
			respChan <- *followerResp
		} else {
			s.nextIndex[followerIndex]--
			go s.sendAppendEntryRequest(followerIndex, s.nextIndex[followerIndex], respChan)
		}
	}
}

func (s *server) handleAppendEntryRequest(a appendRequest) {
	fmt.Println("Got append entry request: ", a.Req)
	returnChan := a.ReturnChan
	req := a.Req
	if req.Term < s.term {
		resp := &appendEntryResponse{s.term, false}
		returnChan <- *resp
	} else if req.PrevLogIndex > -1 &&
		len(s.db.Log.Entries) > req.PrevLogIndex &&
		s.db.Log.Entries[req.PrevLogIndex].Term != req.PrevLogTerm {
		resp := &appendEntryResponse{s.term, false}
		returnChan <- *resp
	} else {
		if req.Term > s.term {
			s.term = req.Term
			s.stepDown("append entries RPC has term greater than current term")
		}
		if req.Entry.Command == "" {
			s.term = req.Term
			s.stepDown("append entries RPC has term >= current term AND is a heartbeat")
			fmt.Println("got a heartbeat, responding true: term >= current term")
			resp := &appendEntryResponse{s.term, true}
			returnChan <- *resp
		} else {
			if len(s.db.Log.Entries) > req.PrevLogIndex+2 {
				if s.db.Log.Entries[req.PrevLogIndex+1].Term != req.Term {
					// If existing entry conflicts with new entry
					// Delete entry and all that follow it
					s.db.Log.SetEntries(s.db.Log.Entries[:req.PrevLogIndex])
				}
			}
			s.db.Log.AppendEntry(req.Entry)
			if req.LeaderCommit > s.commitIndex {
				// Set commit index to the min of the leader's commit index and index of last new entry
				if req.LeaderCommit < req.PrevLogIndex+1 {
					s.commitEntries(req.LeaderCommit)
				} else {
					s.commitEntries(req.PrevLogIndex + 1)
				}
			}
			resp := &appendEntryResponse{s.term, true}
			returnChan <- *resp
		}
	}
}
<commit_msg>attempt to commit entries on heartbeat as well<commit_after>package raft

import (
	"bytes"
	"encoding/json"
	"fmt"
	"github.com/arpith/mmapd/db"
	"net/http"
)

type appendEntryRequest struct {
	Term         int
	LeaderID     string
	PrevLogIndex int
	PrevLogTerm  int
	Entry        db.Entry
	LeaderCommit int
}

type appendRequest struct {
	Req        appendEntryRequest
	ReturnChan chan appendEntryResponse
}

type appendEntryResponse struct {
	Term    int
	Success bool
}

type followerResponse struct {
	ServerIndex int
	Resp        appendEntryResponse
}

func (s *server) appendEntry(command string, key string, value string, isCommitted chan bool) {
	entry := &db.Entry{command, key, value, s.term}
	index := -1
	if command != "" {
		s.db.Log.AppendEntry(*entry)
		index = len(s.db.Log.Entries) - 1
	}
	respChan := make(chan followerResponse)
	for i := 0; i < len(s.config); i++ {
		if s.config[i] != s.id {
			go s.sendAppendEntryRequest(i, index, respChan)
		}
	}
	responseCount := 0
	if len(s.config) > 1 {
		for {
			_ = <-respChan
			responseCount++
			for N := s.commitIndex + 1; N < len(s.db.Log.Entries); N++ {
				//Check if there exists an N > commitIndex
				count := 0
				for i := 0; i < len(s.matchIndex); i++ {
					if s.matchIndex[i] >= N {
						count++
					}
				}
				// Check if a majority of matchIndex[i] >= N
				cond1 := count > len(s.matchIndex)/2
				// Check if log[N].term == currentTerm
				cond2 := s.db.Log.Entries[N].Term == s.term
				if cond1 && cond2 {
					// Set commitIndex to N
					s.commitIndex = N
				} else {
					break
				}
			}
			if s.commitIndex == index {
				isCommitted <- true
				return
			}
		}
	} else {
		if command != "" {
			s.commitIndex++
			isCommitted <- true
		}
	}
}

func (s *server) sendAppendEntryRequest(followerIndex int, entryIndex int, respChan chan followerResponse) {
	entryP := &db.Entry{"", "", "", s.term}
	entry := *entryP
	if entryIndex != -1 {
		entry = s.db.Log.Entries[entryIndex]
	}
	follower := s.config[followerIndex]
	prevLogIndex := entryIndex - 1
	prevLogTerm := 0
	if prevLogIndex > 0 {
		prevLogTerm = s.db.Log.Entries[prevLogIndex].Term
	}
	a := &appendEntryRequest{s.term, s.id, prevLogIndex, prevLogTerm, entry, s.commitIndex}
	b := new(bytes.Buffer)
	json.NewEncoder(b).Encode(a)
	resp, err := http.Post("http://"+follower+"/append", "application/json", b)
	if err != nil {
		fmt.Println("Couldn't send append entry request to " + follower)
		fmt.Println(err)
		go s.sendAppendEntryRequest(followerIndex, entryIndex, respChan)
	} else {
		r := &appendEntryResponse{}
		err := json.NewDecoder(resp.Body).Decode(r)
		if err != nil {
			fmt.Println("Couldn't decode append entries response from " + s.config[followerIndex])
			return
		}
		defer resp.Body.Close()
		if r.Term > s.term {
			s.term = r.Term
			s.stepDown("Got append entry RPC response with term > current term")
		}
		if r.Success {
			s.term = r.Term
			s.nextIndex[followerIndex]++
			s.matchIndex[followerIndex]++
			followerResp := &followerResponse{followerIndex, *r}
			respChan <- *followerResp
		} else {
			s.nextIndex[followerIndex]--
			fmt.Println(s.nextIndex[followerIndex], len(s.db.Log.Entries))
			go s.sendAppendEntryRequest(followerIndex, s.nextIndex[followerIndex], respChan)
		}
	}
}

func (s *server) handleAppendEntryRequest(a appendRequest) {
	fmt.Println("Got append entry request: ", a.Req)
	returnChan := a.ReturnChan
	req := a.Req
	if req.Term < s.term {
		resp := &appendEntryResponse{s.term, false}
		returnChan <- *resp
	} else if req.PrevLogIndex > -1 &&
		len(s.db.Log.Entries) > req.PrevLogIndex &&
		s.db.Log.Entries[req.PrevLogIndex].Term != req.PrevLogTerm {
		resp := &appendEntryResponse{s.term, false}
		returnChan <- *resp
	} else {
		if req.Term > s.term {
			s.term = req.Term
			s.stepDown("append entries RPC has term greater than current term")
		}
		if req.Entry.Command == "" {
			s.term = req.Term
			s.stepDown("append entries RPC has term >= current term AND is a heartbeat")
			fmt.Println("got a heartbeat, responding true: term >= current term")
		} else {
			if len(s.db.Log.Entries) > req.PrevLogIndex+2 {
				if s.db.Log.Entries[req.PrevLogIndex+1].Term != req.Term {
					// If existing entry conflicts with new entry
					// Delete entry and all that follow it
					s.db.Log.SetEntries(s.db.Log.Entries[:req.PrevLogIndex])
				}
			}
			s.db.Log.AppendEntry(req.Entry)
		}
		if req.LeaderCommit > s.commitIndex {
			// Set commit index to the min of the leader's commit index and index of last new entry
			if req.LeaderCommit < req.PrevLogIndex+1 {
				s.commitEntries(req.LeaderCommit)
			} else {
				s.commitEntries(req.PrevLogIndex + 1)
			}
		}
		resp := &appendEntryResponse{s.term, true}
		returnChan <- *resp
	}
}
<|endoftext|>


<commit_before>package raft

import (
	"bytes"
	"encoding/json"
	"fmt"
	"github.com/arpith/mmapd/db"
	"net/http"
)

type appendEntryRequest struct {
	Term         int
	LeaderID     string
	PrevLogIndex int
	PrevLogTerm  int
	Entry        db.Entry
	LeaderCommit int
}

type appendRequest struct {
	Req        appendEntryRequest
	ReturnChan chan appendEntryResponse
}

type appendEntryResponse struct {
	Term    int
	Success bool
}

type followerResponse struct {
	ServerIndex int
	Resp        appendEntryResponse
}

func (s *server) appendEntry(command string, isCommitted chan bool) {
	entry := &db.Entry{command, s.term}
	index := -1
	if command != "" {
		s.db.Log.AppendEntry(*entry)
		index = len(s.db.Log.Entries)
	}
	respChan := make(chan followerResponse)
	for i := 0; i < len(s.config); i++ {
		if s.config[i] != s.id {
			go s.sendAppendEntryRequest(i, *entry, respChan)
		}
	}
	responseCount := 0
	if len(s.config) > 1 {
		for {
			_ = <-respChan
			responseCount++
			for N := s.commitIndex + 1; N < len(s.db.Log.Entries); N++ {
				//Check if there exists an N > commitIndex
				count := 0
				for i := 0; i < len(s.matchIndex); i++ {
					if s.matchIndex[i] >= N {
						count++
					}
				}
				// Check if a majority of matchIndex[i] >= N
				cond1 := count > len(s.matchIndex)/2
				// Check if log[N].term == currentTerm
				cond2 := s.db.Log.Entries[N].Term == s.term
				if cond1 && cond2 {
					// Set commitIndex to N
					s.commitIndex = N
				} else {
					break
				}
			}
			if s.commitIndex == index {
				isCommitted <- true
			}
		}
	} else {
		if command != "" {
			s.commitIndex++
			isCommitted <- true
		}
	}
}

func (s *server) sendAppendEntryRequest(followerIndex int, entry db.Entry, respChan chan followerResponse) {
	follower := s.config[followerIndex]
	prevLogIndex := len(s.db.Log.Entries) - 1
	prevLogTerm := 0
	if prevLogIndex > 0 {
		prevLogTerm = s.db.Log.Entries[prevLogIndex].Term
	}
	a := &appendEntryRequest{s.term, s.id, prevLogIndex, prevLogTerm, entry, s.commitIndex}
	b := new(bytes.Buffer)
	json.NewEncoder(b).Encode(a)
	resp, err := http.Post("http://"+follower+"/append", "application/json", b)
	if err != nil {
		fmt.Println("Couldn't send append entry request to " + follower)
		fmt.Println(err)
		return
		//		go s.sendAppendEntryRequest(followerIndex, entry, respChan)
	} else {
		r := &appendEntryResponse{}
		err := json.NewDecoder(resp.Body).Decode(r)
		if err != nil {
			fmt.Println("Couldn't decode append entries response from " + s.config[followerIndex])
			return
		}
		defer resp.Body.Close()
		if r.Term > s.term {
			s.term = r.Term
			s.state = "follower"
			s.electionTimeout.reset()
		}
		if r.Success {
			s.term = r.Term
			s.nextIndex[followerIndex]++
			s.matchIndex[followerIndex]++
			followerResp := &followerResponse{followerIndex, *r}
			respChan <- *followerResp
		} else {
			s.nextIndex[followerIndex]--
			//		go s.sendAppendEntryRequest(followerIndex, entry, respChan)
		}
	}
}

func (s *server) handleAppendEntryRequest(a appendRequest) {
	returnChan := a.ReturnChan
	req := a.Req
	if req.Term < s.term {
		resp := &appendEntryResponse{s.term, false}
		returnChan <- *resp
	} else if len(s.db.Log.Entries) > req.PrevLogIndex && s.db.Log.Entries[req.PrevLogIndex].Term != req.PrevLogTerm {
		resp := &appendEntryResponse{s.term, false}
		returnChan <- *resp
	} else {
		if s.db.Log.Entries[req.PrevLogIndex+1].Term != req.Term {
			// If existing entry conflicts with new entry
			// Delete entry and all that follow it
			s.db.Log.SetEntries(s.db.Log.Entries[:req.PrevLogIndex])
		}
		s.db.Log.AppendEntry(req.Entry)
		if req.LeaderCommit < s.commitIndex {
			// Set commit index to the min of the leader's commit index and index of last new entry
			if req.LeaderCommit < req.PrevLogIndex+1 {
				s.commitIndex = req.LeaderCommit
			} else {
				s.commitIndex = req.PrevLogIndex + 1
			}
		}
		resp := &appendEntryResponse{s.term, true}
		returnChan <- *resp
	}
}
<commit_msg>handle heartbeats when leader log is empty (is this really necessary??)<commit_after>package raft

import (
	"bytes"
	"encoding/json"
	"fmt"
	"github.com/arpith/mmapd/db"
	"net/http"
)

type appendEntryRequest struct {
	Term         int
	LeaderID     string
	PrevLogIndex int
	PrevLogTerm  int
	Entry        db.Entry
	LeaderCommit int
}

type appendRequest struct {
	Req        appendEntryRequest
	ReturnChan chan appendEntryResponse
}

type appendEntryResponse struct {
	Term    int
	Success bool
}

type followerResponse struct {
	ServerIndex int
	Resp        appendEntryResponse
}

func (s *server) appendEntry(command string, isCommitted chan bool) {
	entry := &db.Entry{command, s.term}
	index := -1
	if command != "" {
		s.db.Log.AppendEntry(*entry)
		index = len(s.db.Log.Entries)
	}
	respChan := make(chan followerResponse)
	for i := 0; i < len(s.config); i++ {
		if s.config[i] != s.id {
			go s.sendAppendEntryRequest(i, *entry, respChan)
		}
	}
	responseCount := 0
	if len(s.config) > 1 {
		for {
			_ = <-respChan
			responseCount++
			for N := s.commitIndex + 1; N < len(s.db.Log.Entries); N++ {
				//Check if there exists an N > commitIndex
				count := 0
				for i := 0; i < len(s.matchIndex); i++ {
					if s.matchIndex[i] >= N {
						count++
					}
				}
				// Check if a majority of matchIndex[i] >= N
				cond1 := count > len(s.matchIndex)/2
				// Check if log[N].term == currentTerm
				cond2 := s.db.Log.Entries[N].Term == s.term
				if cond1 && cond2 {
					// Set commitIndex to N
					s.commitIndex = N
				} else {
					break
				}
			}
			if s.commitIndex == index {
				isCommitted <- true
			}
		}
	} else {
		if command != "" {
			s.commitIndex++
			isCommitted <- true
		}
	}
}

func (s *server) sendAppendEntryRequest(followerIndex int, entry db.Entry, respChan chan followerResponse) {
	follower := s.config[followerIndex]
	prevLogIndex := len(s.db.Log.Entries) - 1
	prevLogTerm := 0
	if prevLogIndex > 0 {
		prevLogTerm = s.db.Log.Entries[prevLogIndex].Term
	}
	a := &appendEntryRequest{s.term, s.id, prevLogIndex, prevLogTerm, entry, s.commitIndex}
	b := new(bytes.Buffer)
	json.NewEncoder(b).Encode(a)
	resp, err := http.Post("http://"+follower+"/append", "application/json", b)
	if err != nil {
		fmt.Println("Couldn't send append entry request to " + follower)
		fmt.Println(err)
		return
		//		go s.sendAppendEntryRequest(followerIndex, entry, respChan)
	} else {
		r := &appendEntryResponse{}
		err := json.NewDecoder(resp.Body).Decode(r)
		if err != nil {
			fmt.Println("Couldn't decode append entries response from " + s.config[followerIndex])
			return
		}
		defer resp.Body.Close()
		if r.Term > s.term {
			s.term = r.Term
			s.state = "follower"
			s.electionTimeout.reset()
		}
		if r.Success {
			s.term = r.Term
			s.nextIndex[followerIndex]++
			s.matchIndex[followerIndex]++
			followerResp := &followerResponse{followerIndex, *r}
			respChan <- *followerResp
		} else {
			s.nextIndex[followerIndex]--
			//		go s.sendAppendEntryRequest(followerIndex, entry, respChan)
		}
	}
}

func (s *server) handleAppendEntryRequest(a appendRequest) {
	returnChan := a.ReturnChan
	req := a.Req
	if req.Term < s.term {
		resp := &appendEntryResponse{s.term, false}
		returnChan <- *resp
	} else if req.PrevLogIndex == -1 {
		resp := &appendEntryResponse{s.term, true}
		returnChan <- *resp
	} else if len(s.db.Log.Entries) > req.PrevLogIndex &&
		s.db.Log.Entries[req.PrevLogIndex].Term != req.PrevLogTerm {
		resp := &appendEntryResponse{s.term, false}
		returnChan <- *resp
	} else {
		if s.db.Log.Entries[req.PrevLogIndex+1].Term != req.Term {
			// If existing entry conflicts with new entry
			// Delete entry and all that follow it
			s.db.Log.SetEntries(s.db.Log.Entries[:req.PrevLogIndex])
		}
		s.db.Log.AppendEntry(req.Entry)
		if req.LeaderCommit < s.commitIndex {
			// Set commit index to the min of the leader's commit index and index of last new entry
			if req.LeaderCommit < req.PrevLogIndex+1 {
				s.commitIndex = req.LeaderCommit
			} else {
				s.commitIndex = req.PrevLogIndex + 1
			}
		}
		resp := &appendEntryResponse{s.term, true}
		returnChan <- *resp
	}
}
<|endoftext|>


<commit_before>package std

import (
	"regexp"
	"strings"
)

var re = regexp.MustCompile("^([^/]+)[.](com|org|net)/.*")

// Provider detects GH url
type Provider struct {
	URL         string
	providerURL string
	providerID  string
}

// New provider to work with url
func New() *Provider {
	return &Provider{}
}

// Is the current url about gh?
func (p *Provider) Is(url string) bool {
	x := cleanURL(url)
	return re.MatchString(x)
}

// SetURL of the provider
func (p *Provider) SetURL(url string) {
	p.URL = cleanURL(url)
	parts := strings.Split(p.URL, "/")
	p.providerURL = parts[0]
	parts2 := re.FindAllStringSubmatch(p.URL, -1)
	if len(parts2) > 0 {
		p.providerID = parts2[0][1]
	}
}

func cleanURL(url string) string {
	if url[:4] == "http" {
		url = url[4:]
	} else if url[:5] == "https" {
		url = url[5:]
	}
	if url[:3] == "://" {
		url = url[3:]
	}
	if url[:1] == "/" {
		url = url[1:]
	}
	return url
}

// GetUserName of the the current url.
func (p *Provider) GetUserName() string {
	ss := strings.Split(p.URL, "/")
	if len(ss) > 0 {
		return ss[1]
	}
	return ""
}

// GetProjectName of the the current url.
func (p *Provider) GetProjectName() string {
	ss := strings.Split(p.URL, "/")
	if len(ss) > 2 {
		return ss[2]
	}
	return ""
}

// GetProjectPath of the the current url.
func (p *Provider) GetProjectPath() string {
	ss := strings.Split(p.URL, "/")
	if len(ss) > 3 {
		return "/" + strings.Join(ss[3:], "/")
	}
	return ""
}

// GetProjectURL of the the current url.
func (p *Provider) GetProjectURL() string {
	ret := []string{}
	if x := p.GetProviderURL(); x != "" {
		if y := p.GetUserName(); y != "" {
			if z := p.GetProjectName(); z != "" {
				ret = append(ret, x)
				ret = append(ret, y)
				ret = append(ret, z)
			}
		}
	}
	return strings.Join(ret, "/")
}

// GetURL of the the current url.
func (p *Provider) GetURL() string {
	ret := []string{}
	if x := p.GetProjectURL(); x != "" {
		ret = append(ret, x)
		if y := p.GetProjectPath(); y != "" {
			ret = append(ret, y[1:]) //rm front /
		}
	}
	return strings.Join(ret, "/")
}

// GetProviderURL of the the current url.
func (p *Provider) GetProviderURL() string {
	return p.providerURL
}

// GetProviderID of the the current url.
func (p *Provider) GetProviderID() string {
	return p.providerID
}
<commit_msg>fix out of range index<commit_after>package std

import (
	"regexp"
	"strings"
)

var re = regexp.MustCompile("^([^/]+)[.](com|org|net)/.*")

// Provider detects GH url
type Provider struct {
	URL         string
	providerURL string
	providerID  string
}

// New provider to work with url
func New() *Provider {
	return &Provider{}
}

// Is the current url about gh?
func (p *Provider) Is(url string) bool {
	x := cleanURL(url)
	return re.MatchString(x)
}

// SetURL of the provider
func (p *Provider) SetURL(url string) {
	p.URL = cleanURL(url)
	parts := strings.Split(p.URL, "/")
	p.providerURL = parts[0]
	parts2 := re.FindAllStringSubmatch(p.URL, -1)
	if len(parts2) > 0 {
		p.providerID = parts2[0][1]
	}
}

func cleanURL(url string) string {
	if url[:4] == "http" {
		url = url[4:]
	} else if url[:5] == "https" {
		url = url[5:]
	}
	if url[:3] == "://" {
		url = url[3:]
	}
	if url[:1] == "/" {
		url = url[1:]
	}
	return url
}

// GetUserName of the the current url.
func (p *Provider) GetUserName() string {
	ss := strings.Split(p.URL, "/")
	if len(ss) > 1 {
		return ss[1]
	}
	return ""
}

// GetProjectName of the the current url.
func (p *Provider) GetProjectName() string {
	ss := strings.Split(p.URL, "/")
	if len(ss) > 2 {
		return ss[2]
	}
	return ""
}

// GetProjectPath of the the current url.
func (p *Provider) GetProjectPath() string {
	ss := strings.Split(p.URL, "/")
	if len(ss) > 3 {
		return "/" + strings.Join(ss[3:], "/")
	}
	return ""
}

// GetProjectURL of the the current url.
func (p *Provider) GetProjectURL() string {
	ret := []string{}
	if x := p.GetProviderURL(); x != "" {
		if y := p.GetUserName(); y != "" {
			if z := p.GetProjectName(); z != "" {
				ret = append(ret, x)
				ret = append(ret, y)
				ret = append(ret, z)
			}
		}
	}
	return strings.Join(ret, "/")
}

// GetURL of the the current url.
func (p *Provider) GetURL() string {
	ret := []string{}
	if x := p.GetProjectURL(); x != "" {
		ret = append(ret, x)
		if y := p.GetProjectPath(); y != "" {
			ret = append(ret, y[1:]) //rm front /
		}
	}
	return strings.Join(ret, "/")
}

// GetProviderURL of the the current url.
func (p *Provider) GetProviderURL() string {
	return p.providerURL
}

// GetProviderID of the the current url.
func (p *Provider) GetProviderID() string {
	return p.providerID
}
<|endoftext|>


<commit_before>// Copyright 2018 The Terraformer Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package aws

import (
	"fmt"
	"log"
	"strings"

	"github.com/GoogleCloudPlatform/terraformer/terraform_utils"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/route53"
)

var route53AllowEmptyValues = []string{}

var route53AdditionalFields = map[string]string{}

type Route53Generator struct {
	AWSService
}

func (g Route53Generator) createZonesResources(svc *route53.Route53) []terraform_utils.Resource {
	resources := []terraform_utils.Resource{}
	err := svc.ListHostedZonesPages(
		&route53.ListHostedZonesInput{},
		func(zones *route53.ListHostedZonesOutput, lastPage bool) bool {
			for _, zone := range zones.HostedZones {
				zoneID := cleanZoneID(aws.StringValue(zone.Id))
				resources = append(resources, terraform_utils.NewResource(
					zoneID,
					zoneID+"_"+strings.TrimSuffix(aws.StringValue(zone.Name), "."),
					"aws_route53_zone",
					"aws",
					map[string]string{
						"name":          aws.StringValue(zone.Name),
						"force_destroy": "false",
					},
					route53AllowEmptyValues,
					route53AdditionalFields,
				))
				records := g.createRecordsResources(svc, zoneID)
				resources = append(resources, records...)
			}
			return true
		},
	)

	if err != nil {
		log.Println(err)
		return resources
	}

	return resources
}

func (Route53Generator) createRecordsResources(svc *route53.Route53, zoneID string) []terraform_utils.Resource {
	resources := []terraform_utils.Resource{}
	listParams := &route53.ListResourceRecordSetsInput{
		HostedZoneId: aws.String(zoneID),
	}

	err := svc.ListResourceRecordSetsPages(
		listParams,
		func(recordSet *route53.ListResourceRecordSetsOutput, lastPage bool) bool {
			for _, record := range recordSet.ResourceRecordSets {
				resources = append(resources, terraform_utils.NewResource(
					fmt.Sprintf("%s_%s_%s", zoneID, aws.StringValue(record.Name), aws.StringValue(record.Type)),
					fmt.Sprintf("%s_%s_%s", zoneID, aws.StringValue(record.Name), aws.StringValue(record.Type)),
					"aws_route53_record",
					"aws",
					map[string]string{
						"name":    aws.StringValue(record.Name),
						"zone_id": zoneID,
						"type":    aws.StringValue(record.Type),
					},
					route53AllowEmptyValues,
					route53AdditionalFields,
				))
			}
			return true
		},
	)

	if err != nil {
		log.Println(err)
		return []terraform_utils.Resource{}
	}
	return resources
}

// Generate TerraformResources from AWS API,
// create terraform resource for each zone + each record
func (g *Route53Generator) InitResources() error {
	sess, _ := session.NewSession(&aws.Config{Region: aws.String(g.GetArgs()["region"])})
	svc := route53.New(sess)

	g.Resources = g.createZonesResources(svc)
	g.PopulateIgnoreKeys()
	return nil
}

func (g *Route53Generator) PostConvertHook() error {
	for i, resourceRecord := range g.Resources {
		if resourceRecord.InstanceInfo.Type == "aws_route53_zone" {
			continue
		}
		item := resourceRecord.Item
		zoneID := item["zone_id"].(string)
		for _, resourceZone := range g.Resources {
			if resourceZone.InstanceInfo.Type != "aws_route53_zone" {
				continue
			}
			if zoneID == resourceZone.InstanceState.ID {
				g.Resources[i].Item["zone_id"] = "${aws_route53_zone." + resourceZone.ResourceName + ".zone_id}"
			}
		}
		if _, aliasExist := resourceRecord.Item["alias"]; aliasExist {
			if _, ttlExist := resourceRecord.Item["ttl"]; ttlExist {
				delete(g.Resources[i].Item, "ttl")
			}
		}
	}
	return nil
}

// cleanZoneID is used to remove the leading /hostedzone/
func cleanZoneID(ID string) string {
	return cleanPrefix(ID, "/hostedzone/")
}

// cleanPrefix removes a string prefix from an ID
func cleanPrefix(ID, prefix string) string {
	return strings.TrimPrefix(ID, prefix)
}
<commit_msg>Include SetIdentifier into AWS Route 53 Record query to properly identify resources<commit_after>// Copyright 2018 The Terraformer Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package aws

import (
	"fmt"
	"log"
	"strings"

	"github.com/GoogleCloudPlatform/terraformer/terraform_utils"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/route53"
)

var route53AllowEmptyValues = []string{}

var route53AdditionalFields = map[string]string{}

type Route53Generator struct {
	AWSService
}

func (g Route53Generator) createZonesResources(svc *route53.Route53) []terraform_utils.Resource {
	resources := []terraform_utils.Resource{}
	err := svc.ListHostedZonesPages(
		&route53.ListHostedZonesInput{},
		func(zones *route53.ListHostedZonesOutput, lastPage bool) bool {
			for _, zone := range zones.HostedZones {
				zoneID := cleanZoneID(aws.StringValue(zone.Id))
				resources = append(resources, terraform_utils.NewResource(
					zoneID,
					zoneID+"_"+strings.TrimSuffix(aws.StringValue(zone.Name), "."),
					"aws_route53_zone",
					"aws",
					map[string]string{
						"name":          aws.StringValue(zone.Name),
						"force_destroy": "false",
					},
					route53AllowEmptyValues,
					route53AdditionalFields,
				))
				records := g.createRecordsResources(svc, zoneID)
				resources = append(resources, records...)
			}
			return true
		},
	)

	if err != nil {
		log.Println(err)
		return resources
	}

	return resources
}

func (Route53Generator) createRecordsResources(svc *route53.Route53, zoneID string) []terraform_utils.Resource {
	resources := []terraform_utils.Resource{}
	listParams := &route53.ListResourceRecordSetsInput{
		HostedZoneId: aws.String(zoneID),
	}

	err := svc.ListResourceRecordSetsPages(
		listParams,
		func(recordSet *route53.ListResourceRecordSetsOutput, lastPage bool) bool {
			for _, record := range recordSet.ResourceRecordSets {
				resources = append(resources, terraform_utils.NewResource(
					fmt.Sprintf("%s_%s_%s_%s", zoneID, aws.StringValue(record.Name), aws.StringValue(record.Type), aws.StringValue(record.SetIdentifier)),
					fmt.Sprintf("%s_%s_%s_%s", zoneID, aws.StringValue(record.Name), aws.StringValue(record.Type), aws.StringValue(record.SetIdentifier)),
					"aws_route53_record",
					"aws",
					map[string]string{
						"name":    aws.StringValue(record.Name),
						"zone_id": zoneID,
						"type":    aws.StringValue(record.Type),
						"set_identifier": aws.StringValue(record.SetIdentifier),
					},
					route53AllowEmptyValues,
					route53AdditionalFields,
				))
			}
			return true
		},
	)

	if err != nil {
		log.Println(err)
		return []terraform_utils.Resource{}
	}
	return resources
}

// Generate TerraformResources from AWS API,
// create terraform resource for each zone + each record
func (g *Route53Generator) InitResources() error {
	sess, _ := session.NewSession(&aws.Config{Region: aws.String(g.GetArgs()["region"])})
	svc := route53.New(sess)

	g.Resources = g.createZonesResources(svc)
	g.PopulateIgnoreKeys()
	return nil
}

func (g *Route53Generator) PostConvertHook() error {
	for i, resourceRecord := range g.Resources {
		if resourceRecord.InstanceInfo.Type == "aws_route53_zone" {
			continue
		}
		item := resourceRecord.Item
		zoneID := item["zone_id"].(string)
		for _, resourceZone := range g.Resources {
			if resourceZone.InstanceInfo.Type != "aws_route53_zone" {
				continue
			}
			if zoneID == resourceZone.InstanceState.ID {
				g.Resources[i].Item["zone_id"] = "${aws_route53_zone." + resourceZone.ResourceName + ".zone_id}"
			}
		}
		if _, aliasExist := resourceRecord.Item["alias"]; aliasExist {
			if _, ttlExist := resourceRecord.Item["ttl"]; ttlExist {
				delete(g.Resources[i].Item, "ttl")
			}
		}
	}
	return nil
}

// cleanZoneID is used to remove the leading /hostedzone/
func cleanZoneID(ID string) string {
	return cleanPrefix(ID, "/hostedzone/")
}

// cleanPrefix removes a string prefix from an ID
func cleanPrefix(ID, prefix string) string {
	return strings.TrimPrefix(ID, prefix)
}
<|endoftext|>


<commit_before>package proxy

import (
	"bufio"
	"errors"
	"fmt"
	"io"
	"net"
	"net/http"
	"net/http/httputil"
	"sync"

	"github.com/fsouza/go-dockerclient"
)

func (proxy *Proxy) Intercept(i interceptor, w http.ResponseWriter, r *http.Request) {
	if err := i.InterceptRequest(r); err != nil {
		switch err.(type) {
		case *docker.NoSuchContainer:
			http.Error(w, err.Error(), http.StatusNotFound)
		case *ErrNoSuchImage:
			http.Error(w, err.Error(), http.StatusNotFound)
		default:
			http.Error(w, err.Error(), http.StatusInternalServerError)
			Log.Warning("Error intercepting request: ", err)
		}
		return
	}

	conn, err := proxy.Dial()
	if err != nil {
		http.Error(w, "Could not connect to target", http.StatusInternalServerError)
		Log.Warning(err)
		return
	}
	client := httputil.NewClientConn(conn, nil)
	defer client.Close()

	resp, err := client.Do(r)
	if err != nil && err != httputil.ErrPersistEOF {
		http.Error(w, fmt.Sprintf("Could not make request to target: %v", err), http.StatusInternalServerError)
		Log.Warning("Error forwarding request: ", err)
		return
	}
	err = i.InterceptResponse(resp)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		Log.Warning("Error intercepting response: ", err)
		return
	}

	hdr := w.Header()
	for k, vs := range resp.Header {
		for _, v := range vs {
			hdr.Add(k, v)
		}
	}
	Log.Debugf("Response from target: %s %v", resp.Status, w.Header())

	if resp.Header.Get("Content-Type") == "application/vnd.docker.raw-stream" {
		doRawStream(w, resp, client)
	} else if resp.TransferEncoding != nil && resp.TransferEncoding[0] == "chunked" {
		doChunkedResponse(w, resp, client)
	} else {
		w.WriteHeader(resp.StatusCode)
		if _, err := io.Copy(w, resp.Body); err != nil {
			Log.Warning(err)
		}
	}
}

func doRawStream(w http.ResponseWriter, resp *http.Response, client *httputil.ClientConn) {
	down, downBuf, up, remaining, err := hijack(w, client)
	if err != nil {
		http.Error(w, "Unable to hijack connection for raw stream mode", http.StatusInternalServerError)
		return
	}
	defer down.Close()
	defer up.Close()

	if _, err := down.Write([]byte("HTTP/1.1 " + resp.Status + "\n")); err != nil {
		Log.Warning(err)
		return
	}

	if err := resp.Header.Write(down); err != nil {
		Log.Warning(err)
		return
	}

	if _, err := down.Write([]byte("\n")); err != nil {
		Log.Warning(err)
		return
	}

	var wg sync.WaitGroup
	wg.Add(2)
	go copyStream(down, io.MultiReader(remaining, up), &wg)
	go copyStream(up, downBuf, &wg)
	wg.Wait()
}

type closeWriter interface {
	CloseWrite() error
}

func copyStream(dst io.WriteCloser, src io.Reader, wg *sync.WaitGroup) {
	defer wg.Done()
	if _, err := io.Copy(dst, src); err != nil {
		Log.Warning(err)
	}
	var err error
	if c, ok := dst.(closeWriter); ok {
		err = c.CloseWrite()
	} else {
		err = dst.Close()
	}
	if err != nil {
		Log.Warningf("Error closing connection: %s", err)
	}
}

type writeFlusher interface {
	io.Writer
	http.Flusher
}

func doChunkedResponse(w http.ResponseWriter, resp *http.Response, client *httputil.ClientConn) {
	wf, ok := w.(writeFlusher)
	if !ok {
		http.Error(w, "Error forwarding chunked response body: flush not available", http.StatusInternalServerError)
		return
	}

	w.WriteHeader(resp.StatusCode)
	wf.Flush()

	up, remaining := client.Hijack()
	defer up.Close()

	var err error
	chunks := NewChunkedReader(io.MultiReader(remaining, up))
	for chunks.Next() && err == nil {
		_, err = io.Copy(wf, chunks.Chunk())
		wf.Flush()
	}
	if err == nil {
		err = chunks.Err()
	}
	if err != nil {
		Log.Errorf("Error forwarding chunked response body: %s", err)
	}
}

func hijack(w http.ResponseWriter, client *httputil.ClientConn) (down net.Conn, downBuf *bufio.ReadWriter, up net.Conn, remaining io.Reader, err error) {
	hj, ok := w.(http.Hijacker)
	if !ok {
		err = errors.New("Unable to cast to Hijack")
		return
	}
	down, downBuf, err = hj.Hijack()
	if err != nil {
		return
	}
	up, remaining = client.Hijack()
	return
}
<commit_msg>Use buffered writer for http headers<commit_after>package proxy

import (
	"bufio"
	"errors"
	"fmt"
	"io"
	"net"
	"net/http"
	"net/http/httputil"
	"sync"

	"github.com/fsouza/go-dockerclient"
)

func (proxy *Proxy) Intercept(i interceptor, w http.ResponseWriter, r *http.Request) {
	if err := i.InterceptRequest(r); err != nil {
		switch err.(type) {
		case *docker.NoSuchContainer:
			http.Error(w, err.Error(), http.StatusNotFound)
		case *ErrNoSuchImage:
			http.Error(w, err.Error(), http.StatusNotFound)
		default:
			http.Error(w, err.Error(), http.StatusInternalServerError)
			Log.Warning("Error intercepting request: ", err)
		}
		return
	}

	conn, err := proxy.Dial()
	if err != nil {
		http.Error(w, "Could not connect to target", http.StatusInternalServerError)
		Log.Warning(err)
		return
	}
	client := httputil.NewClientConn(conn, nil)
	defer client.Close()

	resp, err := client.Do(r)
	if err != nil && err != httputil.ErrPersistEOF {
		http.Error(w, fmt.Sprintf("Could not make request to target: %v", err), http.StatusInternalServerError)
		Log.Warning("Error forwarding request: ", err)
		return
	}
	err = i.InterceptResponse(resp)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		Log.Warning("Error intercepting response: ", err)
		return
	}

	hdr := w.Header()
	for k, vs := range resp.Header {
		for _, v := range vs {
			hdr.Add(k, v)
		}
	}
	Log.Debugf("Response from target: %s %v", resp.Status, w.Header())

	if resp.Header.Get("Content-Type") == "application/vnd.docker.raw-stream" {
		doRawStream(w, resp, client)
	} else if resp.TransferEncoding != nil && resp.TransferEncoding[0] == "chunked" {
		doChunkedResponse(w, resp, client)
	} else {
		w.WriteHeader(resp.StatusCode)
		if _, err := io.Copy(w, resp.Body); err != nil {
			Log.Warning(err)
		}
	}
}

func doRawStream(w http.ResponseWriter, resp *http.Response, client *httputil.ClientConn) {
	down, downBuf, up, remaining, err := hijack(w, client)
	if err != nil {
		http.Error(w, "Unable to hijack connection for raw stream mode", http.StatusInternalServerError)
		return
	}
	defer down.Close()
	defer up.Close()

	if _, err := downBuf.Write([]byte("HTTP/1.1 " + resp.Status + "\n")); err != nil {
		Log.Warning(err)
		return
	}

	if err := resp.Header.Write(downBuf); err != nil {
		Log.Warning(err)
		return
	}

	if _, err := downBuf.Write([]byte("\n")); err != nil {
		Log.Warning(err)
		return
	}

	if err = downBuf.Flush(); err != nil {
		Log.Warning(err)
		return
	}

	var wg sync.WaitGroup
	wg.Add(2)
	go copyStream(down, io.MultiReader(remaining, up), &wg)
	go copyStream(up, downBuf, &wg)
	wg.Wait()
}

type closeWriter interface {
	CloseWrite() error
}

func copyStream(dst io.WriteCloser, src io.Reader, wg *sync.WaitGroup) {
	defer wg.Done()
	if _, err := io.Copy(dst, src); err != nil {
		Log.Warning(err)
	}
	var err error
	if c, ok := dst.(closeWriter); ok {
		err = c.CloseWrite()
	} else {
		err = dst.Close()
	}
	if err != nil {
		Log.Warningf("Error closing connection: %s", err)
	}
}

type writeFlusher interface {
	io.Writer
	http.Flusher
}

func doChunkedResponse(w http.ResponseWriter, resp *http.Response, client *httputil.ClientConn) {
	wf, ok := w.(writeFlusher)
	if !ok {
		http.Error(w, "Error forwarding chunked response body: flush not available", http.StatusInternalServerError)
		return
	}

	w.WriteHeader(resp.StatusCode)
	wf.Flush()

	up, remaining := client.Hijack()
	defer up.Close()

	var err error
	chunks := NewChunkedReader(io.MultiReader(remaining, up))
	for chunks.Next() && err == nil {
		_, err = io.Copy(wf, chunks.Chunk())
		wf.Flush()
	}
	if err == nil {
		err = chunks.Err()
	}
	if err != nil {
		Log.Errorf("Error forwarding chunked response body: %s", err)
	}
}

func hijack(w http.ResponseWriter, client *httputil.ClientConn) (down net.Conn, downBuf *bufio.ReadWriter, up net.Conn, remaining io.Reader, err error) {
	hj, ok := w.(http.Hijacker)
	if !ok {
		err = errors.New("Unable to cast to Hijack")
		return
	}
	down, downBuf, err = hj.Hijack()
	if err != nil {
		return
	}
	up, remaining = client.Hijack()
	return
}
<|endoftext|>


<commit_before>package data

import (
	"encoding/csv"
	"time"
	"fmt"
	"strconv"
	"io"
)

func CSVParse(file io.Reader) (labels []string, data []Record) {
	labels, data = csvParse(file)
	return
}

func csvParse(file io.Reader) (labels []string, data []Record) {
	reader := csv.NewReader (file)
	tmpdata, err := reader.ReadAll()
	if  err != nil {
		fmt.Println(err)
	}
	labels = tmpdata[0]
	data = make([]Record, len(tmpdata)-1)
	for i := 1; i<len(tmpdata); i++ {
		data[i-1].Time, _ = time.Parse(ISO, tmpdata[i][0])
		data[i-1].Radiation, err = strconv.ParseFloat(tmpdata[i][1], 64)
		if err != nil {
			data[i-1].empty = true
		}
		data[i-1].Humidity, err = strconv.ParseFloat(tmpdata[i][2], 64)
		if err != nil {
			data[i-1].empty = true
		}
		data[i-1].Temperature, err = strconv.ParseFloat(tmpdata[i][2], 64)
		if err != nil {
			data[i-1].empty = true
		}
		data[i-1].Wind, err = strconv.ParseFloat(tmpdata[i][2], 64)
		if err != nil {
			data[i-1].empty = true
		}
		data[i-1].Power, err = strconv.ParseFloat(tmpdata[i][2], 64)
		if err != nil {
			data[i-1].Null = true
		}
	}
	data = fillRecords (data)
	return
}

func fillRecords (emptyData []Record) (data []Record){
	gradRad, gradHumidity, gradTemp, gradWind := 0.0, 0.0, 0.0, 0.0
	for i := 0; i<len(emptyData); i++ {
		if emptyData[i].empty && i > 0 {
			emptyData[i].Radiation = emptyData[i-1].Radiation + gradRad
			emptyData[i].Humidity = emptyData[i-1].Humidity + gradHumidity
			emptyData[i].Temperature = emptyData[i-1].Temperature + gradTemp
			emptyData[i].Wind = emptyData[i-1].Wind + gradWind
			emptyData[i].empty = false
		} else {
			if i + 4 < len (emptyData) {
				gradRad = (emptyData[i+4].Radiation - emptyData[i].Radiation)/4
				gradHumidity = (emptyData[i+4].Humidity - emptyData[i].Humidity)/4
				gradTemp = (emptyData[i+4].Temperature - emptyData[i].Temperature)/4
				gradWind = (emptyData[i+4].Wind - emptyData[i].Wind)/4
			} else {
				gradRad = 0
				gradHumidity = 0
				gradTemp = 0
				gradWind = 0
			}
		}
	}
	return
}
<commit_msg>That should be it<commit_after>package data

import (
	"encoding/csv"
	"time"
	"fmt"
	"strconv"
	"io"
)

func CSVParse(file io.Reader) (labels []string, data []Record) {
	labels, data = csvParse(file)
	return
}

func csvParse(file io.Reader) (labels []string, data []Record) {
	reader := csv.NewReader (file)
	tmpdata, err := reader.ReadAll()
	if  err != nil {
		fmt.Println(err)
	}
	labels = tmpdata[0]
	data = make([]Record, len(tmpdata)-1)
	for i := 1; i<len(tmpdata)-1; i++ {
		data[i-1].Time, _ = time.Parse(ISO, tmpdata[i][0])
		data[i-1].Radiation, err = strconv.ParseFloat(tmpdata[i][1], 64)
		if err != nil {
			data[i-1].empty = true
		}
		data[i-1].Humidity, err = strconv.ParseFloat(tmpdata[i][2], 64)
		if err != nil {
			data[i-1].empty = true
		}
		data[i-1].Temperature, err = strconv.ParseFloat(tmpdata[i][2], 64)
		if err != nil {
			data[i-1].empty = true
		}
		data[i-1].Wind, err = strconv.ParseFloat(tmpdata[i][2], 64)
		if err != nil {
			data[i-1].empty = true
		}
		data[i-1].Power, err = strconv.ParseFloat(tmpdata[i][2], 64)
		if err != nil {
			data[i-1].Null = true
		}
	}
	data = fillRecords (data)
	return
}

func fillRecords (emptyData []Record) (data []Record){
	gradRad, gradHumidity, gradTemp, gradWind := 0.0, 0.0, 0.0, 0.0
	for i := 0; i<len(emptyData); i++ {
		if emptyData[i].empty && i > 0 {
			emptyData[i].Radiation = emptyData[i-1].Radiation + gradRad
			emptyData[i].Humidity = emptyData[i-1].Humidity + gradHumidity
			emptyData[i].Temperature = emptyData[i-1].Temperature + gradTemp
			emptyData[i].Wind = emptyData[i-1].Wind + gradWind
			emptyData[i].empty = false
		} else {
			if i + 4 < len (emptyData) {
				gradRad = (emptyData[i+4].Radiation - emptyData[i].Radiation)/4
				gradHumidity = (emptyData[i+4].Humidity - emptyData[i].Humidity)/4
				gradTemp = (emptyData[i+4].Temperature - emptyData[i].Temperature)/4
				gradWind = (emptyData[i+4].Wind - emptyData[i].Wind)/4
			} else {
				gradRad = 0
				gradHumidity = 0
				gradTemp = 0
				gradWind = 0
			}
		}
	}
	return
}
<|endoftext|>


<commit_before>// Copyright 2013 Matthew Baird
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//     http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package elastigo

import (
	"bytes"
	"crypto/rand"
	"encoding/json"
	"flag"
	"fmt"
	"log"
	"net/url"
	"strconv"
	"testing"
	"time"

	"github.com/araddon/gou"
	"github.com/bmizerany/assert"
)

//  go test -bench=".*"
//  go test -bench="Bulk"

func init() {
	flag.Parse()
	if testing.Verbose() {
		gou.SetupLogging("debug")
	}
}

// take two ints, compare, need to be within 5%
func closeInt(a, b int) bool {
	c := float64(a) / float64(b)
	if c >= .95 && c <= 1.05 {
		return true
	}
	return false
}

func TestBulkIndexerBasic(t *testing.T) {
	testIndex := "users"
	var (
		buffers        = make([]*bytes.Buffer, 0)
		totalBytesSent int
		messageSets    int
	)

	InitTests(true)
	c := NewTestConn()

	c.DeleteIndex(testIndex)

	indexer := c.NewBulkIndexer(3)
	indexer.Sender = func(buf *bytes.Buffer) error {
		messageSets += 1
		totalBytesSent += buf.Len()
		buffers = append(buffers, buf)
		//log.Printf("buffer:%s", string(buf.Bytes()))
		return indexer.Send(buf)
	}
	indexer.Start()

	date := time.Unix(1257894000, 0)
	data := map[string]interface{}{
		"name": "smurfs",
		"age":  22,
		"date": "yesterday",
	}

	err := indexer.Index(testIndex, "user", "1", "", "", &date, data)
	waitFor(func() bool {
		return len(buffers) > 0
	}, 5)

	// part of request is url, so lets factor that in
	//totalBytesSent = totalBytesSent - len(*eshost)
	assert.T(t, len(buffers) == 1, fmt.Sprintf("Should have sent one operation but was %d", len(buffers)))
	assert.T(t, indexer.NumErrors() == 0 && err == nil, fmt.Sprintf("Should not have any errors. NumErrors: %v, err: %v", indexer.NumErrors(), err))
	expectedBytes := 129
	assert.T(t, totalBytesSent == expectedBytes, fmt.Sprintf("Should have sent %v bytes but was %v", expectedBytes, totalBytesSent))

	err = indexer.Index(testIndex, "user", "2", "", "", nil, data)
	waitFor(func() bool {
		return len(buffers) > 1
	}, 5)

	// this will test to ensure that Flush actually catches a doc
	indexer.Flush()
	totalBytesSent = totalBytesSent - len(*eshost)
	assert.T(t, err == nil, fmt.Sprintf("Should have nil error  =%v", err))
	assert.T(t, len(buffers) == 2, fmt.Sprintf("Should have another buffer ct=%d", len(buffers)))

	assert.T(t, indexer.NumErrors() == 0, fmt.Sprintf("Should not have any errors %d", indexer.NumErrors()))
	expectedBytes = 220
	assert.T(t, closeInt(totalBytesSent, expectedBytes), fmt.Sprintf("Should have sent %v bytes but was %v", expectedBytes, totalBytesSent))

	indexer.Stop()
}

func TestRefreshParam(t *testing.T) {
	var requrl *url.URL
	InitTests(true)
	c := NewTestConn()
	c.RequestTracer = func(method, urlStr, body string) {
		requrl, _ = url.Parse(urlStr)
	}
	date := time.Unix(1257894000, 0)
	data := map[string]interface{}{"name": "smurfs", "age": 22, "date": date}

	// Now tests small batches
	indexer := c.NewBulkIndexer(1)
	indexer.Refresh = true

	indexer.Start()
	<-time.After(time.Millisecond * 20)

	indexer.Index("users", "user", "2", "", "", &date, data)

	<-time.After(time.Millisecond * 200)
	//	indexer.Flush()
	indexer.Stop()

	assert.T(t, requrl.Query().Get("refresh") == "true", "Should have set refresh query param to true")
}

func TestWithoutRefreshParam(t *testing.T) {
	var requrl *url.URL
	InitTests(true)
	c := NewTestConn()
	c.RequestTracer = func(method, urlStr, body string) {
		requrl, _ = url.Parse(urlStr)
	}
	date := time.Unix(1257894000, 0)
	data := map[string]interface{}{"name": "smurfs", "age": 22, "date": date}

	// Now tests small batches
	indexer := c.NewBulkIndexer(1)

	indexer.Start()
	<-time.After(time.Millisecond * 20)

	indexer.Index("users", "user", "2", "", "", &date, data)

	<-time.After(time.Millisecond * 200)
	//	indexer.Flush()
	indexer.Stop()

	assert.T(t, requrl.Query().Get("refresh") == "false", "Should have set refresh query param to false")
}

// currently broken in drone.io
func XXXTestBulkUpdate(t *testing.T) {
	var (
		buffers        = make([]*bytes.Buffer, 0)
		totalBytesSent int
		messageSets    int
	)

	InitTests(true)
	c := NewTestConn()
	c.Port = "9200"
	indexer := c.NewBulkIndexer(3)
	indexer.Sender = func(buf *bytes.Buffer) error {
		messageSets += 1
		totalBytesSent += buf.Len()
		buffers = append(buffers, buf)
		return indexer.Send(buf)
	}
	indexer.Start()

	date := time.Unix(1257894000, 0)
	user := map[string]interface{}{
		"name": "smurfs", "age": 22, "date": date, "count": 1,
	}

	// Lets make sure the data is in the index ...
	_, err := c.Index("users", "user", "5", nil, user)

	// script and params
	data := map[string]interface{}{
		"script": "ctx._source.count += 2",
	}
	err = indexer.Update("users", "user", "5", "", "", &date, data)
	// So here's the deal. Flushing does seem to work, you just have to give the
	// channel a moment to recieve the message ...
	//	<- time.After(time.Millisecond * 20)
	//	indexer.Flush()

	waitFor(func() bool {
		return len(buffers) > 0
	}, 5)

	indexer.Stop()

	assert.T(t, indexer.NumErrors() == 0 && err == nil, fmt.Sprintf("Should not have any errors, bulkErrorCt:%v, err:%v", indexer.NumErrors(), err))

	response, err := c.Get("users", "user", "5", nil)
	assert.T(t, err == nil, fmt.Sprintf("Should not have any errors  %v", err))
	m := make(map[string]interface{})
	json.Unmarshal([]byte(*response.Source), &m)
	newCount := m["count"]
	assert.T(t, newCount.(float64) == 3,
		fmt.Sprintf("Should have update count: %#v ... %#v", m["count"], response))
}

func TestBulkSmallBatch(t *testing.T) {
	var (
		messageSets int
	)

	InitTests(true)
	c := NewTestConn()

	date := time.Unix(1257894000, 0)
	data := map[string]interface{}{"name": "smurfs", "age": 22, "date": date}

	// Now tests small batches
	indexer := c.NewBulkIndexer(1)
	indexer.BufferDelayMax = 100 * time.Millisecond
	indexer.BulkMaxDocs = 2
	messageSets = 0
	indexer.Sender = func(buf *bytes.Buffer) error {
		messageSets += 1
		return indexer.Send(buf)
	}
	indexer.Start()
	<-time.After(time.Millisecond * 20)

	indexer.Index("users", "user", "2", "", "", &date, data)
	indexer.Index("users", "user", "3", "", "", &date, data)
	indexer.Index("users", "user", "4", "", "", &date, data)
	<-time.After(time.Millisecond * 200)
	//	indexer.Flush()
	indexer.Stop()
	assert.T(t, messageSets == 2, fmt.Sprintf("Should have sent 2 message sets %d", messageSets))

}

func TestBulkDelete(t *testing.T) {
	InitTests(true)

	c := NewTestConn()
	indexer := c.NewBulkIndexer(1)
	sentBytes := []byte{}

	indexer.Sender = func(buf *bytes.Buffer) error {
		sentBytes = append(sentBytes, buf.Bytes()...)
		return nil
	}

	indexer.Start()

	indexer.Delete("fake", "fake_type", "1")

	indexer.Flush()
	indexer.Stop()

	sent := string(sentBytes)

	expected := `{"delete":{"_index":"fake","_type":"fake_type","_id":"1"}}
`
	asExpected := sent == expected
	assert.T(t, asExpected, fmt.Sprintf("Should have sent '%s' but actually sent '%s'", expected, sent))
}

func XXXTestBulkErrors(t *testing.T) {
	// lets set a bad port, and hope we get a conn refused error?
	c := NewTestConn()
	c.Port = "27845"
	defer func() {
		c.Port = "9200"
	}()
	indexer := c.NewBulkIndexerErrors(10, 1)
	indexer.Start()
	errorCt := 0
	go func() {
		for i := 0; i < 20; i++ {
			date := time.Unix(1257894000, 0)
			data := map[string]interface{}{"name": "smurfs", "age": 22, "date": date}
			indexer.Index("users", "user", strconv.Itoa(i), "", "", &date, data)
		}
	}()
	var errBuf *ErrorBuffer
	for errBuf = range indexer.ErrorChannel {
		errorCt++
		break
	}
	if errBuf.Buf.Len() > 0 {
		gou.Debug(errBuf.Err)
	}
	assert.T(t, errorCt > 0, fmt.Sprintf("ErrorCt should be > 0 %d", errorCt))
	indexer.Stop()
}

/*
BenchmarkSend	18:33:00 bulk_test.go:131: Sent 1 messages in 0 sets totaling 0 bytes
18:33:00 bulk_test.go:131: Sent 100 messages in 1 sets totaling 145889 bytes
18:33:01 bulk_test.go:131: Sent 10000 messages in 100 sets totaling 14608888 bytes
18:33:05 bulk_test.go:131: Sent 20000 messages in 99 sets totaling 14462790 bytes
   20000	    234526 ns/op

*/
func BenchmarkSend(b *testing.B) {
	InitTests(true)
	c := NewTestConn()
	b.StartTimer()
	totalBytes := 0
	sets := 0
	indexer := c.NewBulkIndexer(1)
	indexer.Sender = func(buf *bytes.Buffer) error {
		totalBytes += buf.Len()
		sets += 1
		//log.Println("got bulk")
		return indexer.Send(buf)
	}
	for i := 0; i < b.N; i++ {
		about := make([]byte, 1000)
		rand.Read(about)
		data := map[string]interface{}{"name": "smurfs", "age": 22, "date": time.Unix(1257894000, 0), "about": about}
		indexer.Index("users", "user", strconv.Itoa(i), "", "", nil, data)
	}
	log.Printf("Sent %d messages in %d sets totaling %d bytes \n", b.N, sets, totalBytes)
	if indexer.NumErrors() != 0 {
		b.Fail()
	}
}

/*
TODO:  this should be faster than above

BenchmarkSendBytes	18:33:05 bulk_test.go:169: Sent 1 messages in 0 sets totaling 0 bytes
18:33:05 bulk_test.go:169: Sent 100 messages in 2 sets totaling 292299 bytes
18:33:09 bulk_test.go:169: Sent 10000 messages in 99 sets totaling 14473800 bytes
   10000	    373529 ns/op

*/
func BenchmarkSendBytes(b *testing.B) {
	InitTests(true)
	c := NewTestConn()
	about := make([]byte, 1000)
	rand.Read(about)
	data := map[string]interface{}{"name": "smurfs", "age": 22, "date": time.Unix(1257894000, 0), "about": about}
	body, _ := json.Marshal(data)
	b.StartTimer()
	totalBytes := 0
	sets := 0
	indexer := c.NewBulkIndexer(1)
	indexer.Sender = func(buf *bytes.Buffer) error {
		totalBytes += buf.Len()
		sets += 1
		return indexer.Send(buf)
	}
	for i := 0; i < b.N; i++ {
		indexer.Index("users", "user", strconv.Itoa(i), "", "", nil, body)
	}
	log.Printf("Sent %d messages in %d sets totaling %d bytes \n", b.N, sets, totalBytes)
	if indexer.NumErrors() != 0 {
		b.Fail()
	}
}
<commit_msg>sync access to shared buffer int corebulk_test.<commit_after>// Copyright 2013 Matthew Baird
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//     http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package elastigo

import (
	"bytes"
	"crypto/rand"
	"encoding/json"
	"flag"
	"fmt"
	"github.com/araddon/gou"
	"github.com/bmizerany/assert"
	"log"
	"net/url"
	"strconv"
	"sync"
	"testing"
	"time"
)

//  go test -bench=".*"
//  go test -bench="Bulk"

type sharedBuffer struct {
	mu     sync.Mutex
	Buffer []*bytes.Buffer
}

func NewSharedBuffer() *sharedBuffer {
	return &sharedBuffer{
		Buffer: make([]*bytes.Buffer, 0),
	}
}

func (b *sharedBuffer) Append(buf *bytes.Buffer) {
	b.mu.Lock()
	defer b.mu.Unlock()
	b.Buffer = append(b.Buffer, buf)
}

func (b *sharedBuffer) Length() int {
	b.mu.Lock()
	defer b.mu.Unlock()
	return len(b.Buffer)
}

func init() {
	flag.Parse()
	if testing.Verbose() {
		gou.SetupLogging("debug")
	}
}

// take two ints, compare, need to be within 5%
func closeInt(a, b int) bool {
	c := float64(a) / float64(b)
	if c >= .95 && c <= 1.05 {
		return true
	}
	return false
}

func TestBulkIndexerBasic(t *testing.T) {
	testIndex := "users"
	var (
		buffers        = NewSharedBuffer()
		totalBytesSent int
		messageSets    int
	)

	InitTests(true)
	c := NewTestConn()

	c.DeleteIndex(testIndex)

	indexer := c.NewBulkIndexer(3)
	indexer.Sender = func(buf *bytes.Buffer) error {
		messageSets += 1
		totalBytesSent += buf.Len()
		buffers.Append(buf)
		//log.Printf("buffer:%s", string(buf.Bytes()))
		return indexer.Send(buf)
	}
	indexer.Start()

	date := time.Unix(1257894000, 0)
	data := map[string]interface{}{
		"name": "smurfs",
		"age":  22,
		"date": "yesterday",
	}

	err := indexer.Index(testIndex, "user", "1", "", "", &date, data)
	waitFor(func() bool {
		return buffers.Length() > 0
	}, 5)

	// part of request is url, so lets factor that in
	//totalBytesSent = totalBytesSent - len(*eshost)
	assert.T(t, buffers.Length() == 1, fmt.Sprintf("Should have sent one operation but was %d", buffers.Length()))
	assert.T(t, indexer.NumErrors() == 0 && err == nil, fmt.Sprintf("Should not have any errors. NumErrors: %v, err: %v", indexer.NumErrors(), err))
	expectedBytes := 129
	assert.T(t, totalBytesSent == expectedBytes, fmt.Sprintf("Should have sent %v bytes but was %v", expectedBytes, totalBytesSent))

	err = indexer.Index(testIndex, "user", "2", "", "", nil, data)
	waitFor(func() bool {
		return buffers.Length() > 1
	}, 5)

	// this will test to ensure that Flush actually catches a doc
	indexer.Flush()
	totalBytesSent = totalBytesSent - len(*eshost)
	assert.T(t, err == nil, fmt.Sprintf("Should have nil error  =%v", err))
	assert.T(t, buffers.Length() == 2, fmt.Sprintf("Should have another buffer ct=%d", buffers.Length()))

	assert.T(t, indexer.NumErrors() == 0, fmt.Sprintf("Should not have any errors %d", indexer.NumErrors()))
	expectedBytes = 220
	assert.T(t, closeInt(totalBytesSent, expectedBytes), fmt.Sprintf("Should have sent %v bytes but was %v", expectedBytes, totalBytesSent))

	indexer.Stop()
}

func TestRefreshParam(t *testing.T) {
	var requrl *url.URL
	InitTests(true)
	c := NewTestConn()
	c.RequestTracer = func(method, urlStr, body string) {
		requrl, _ = url.Parse(urlStr)
	}
	date := time.Unix(1257894000, 0)
	data := map[string]interface{}{"name": "smurfs", "age": 22, "date": date}

	// Now tests small batches
	indexer := c.NewBulkIndexer(1)
	indexer.Refresh = true

	indexer.Start()
	<-time.After(time.Millisecond * 20)

	indexer.Index("users", "user", "2", "", "", &date, data)

	<-time.After(time.Millisecond * 200)
	//	indexer.Flush()
	indexer.Stop()

	assert.T(t, requrl.Query().Get("refresh") == "true", "Should have set refresh query param to true")
}

func TestWithoutRefreshParam(t *testing.T) {
	var requrl *url.URL
	InitTests(true)
	c := NewTestConn()
	c.RequestTracer = func(method, urlStr, body string) {
		requrl, _ = url.Parse(urlStr)
	}
	date := time.Unix(1257894000, 0)
	data := map[string]interface{}{"name": "smurfs", "age": 22, "date": date}

	// Now tests small batches
	indexer := c.NewBulkIndexer(1)

	indexer.Start()
	<-time.After(time.Millisecond * 20)

	indexer.Index("users", "user", "2", "", "", &date, data)

	<-time.After(time.Millisecond * 200)
	//	indexer.Flush()
	indexer.Stop()

	assert.T(t, requrl.Query().Get("refresh") == "false", "Should have set refresh query param to false")
}

// currently broken in drone.io
func XXXTestBulkUpdate(t *testing.T) {
	var (
		buffers        = NewSharedBuffer()
		totalBytesSent int
		messageSets    int
	)

	InitTests(true)
	c := NewTestConn()
	c.Port = "9200"
	indexer := c.NewBulkIndexer(3)
	indexer.Sender = func(buf *bytes.Buffer) error {
		messageSets += 1
		totalBytesSent += buf.Len()
		buffers.Append(buf)
		return indexer.Send(buf)
	}
	indexer.Start()

	date := time.Unix(1257894000, 0)
	user := map[string]interface{}{
		"name": "smurfs", "age": 22, "date": date, "count": 1,
	}

	// Lets make sure the data is in the index ...
	_, err := c.Index("users", "user", "5", nil, user)

	// script and params
	data := map[string]interface{}{
		"script": "ctx._source.count += 2",
	}
	err = indexer.Update("users", "user", "5", "", "", &date, data)
	// So here's the deal. Flushing does seem to work, you just have to give the
	// channel a moment to recieve the message ...
	//	<- time.After(time.Millisecond * 20)
	//	indexer.Flush()

	waitFor(func() bool {
		return buffers.Length() > 0
	}, 5)

	indexer.Stop()

	assert.T(t, indexer.NumErrors() == 0 && err == nil, fmt.Sprintf("Should not have any errors, bulkErrorCt:%v, err:%v", indexer.NumErrors(), err))

	response, err := c.Get("users", "user", "5", nil)
	assert.T(t, err == nil, fmt.Sprintf("Should not have any errors  %v", err))
	m := make(map[string]interface{})
	json.Unmarshal([]byte(*response.Source), &m)
	newCount := m["count"]
	assert.T(t, newCount.(float64) == 3,
		fmt.Sprintf("Should have update count: %#v ... %#v", m["count"], response))
}

func TestBulkSmallBatch(t *testing.T) {
	var (
		messageSets int
	)

	InitTests(true)
	c := NewTestConn()

	date := time.Unix(1257894000, 0)
	data := map[string]interface{}{"name": "smurfs", "age": 22, "date": date}

	// Now tests small batches
	indexer := c.NewBulkIndexer(1)
	indexer.BufferDelayMax = 100 * time.Millisecond
	indexer.BulkMaxDocs = 2
	messageSets = 0
	indexer.Sender = func(buf *bytes.Buffer) error {
		messageSets += 1
		return indexer.Send(buf)
	}
	indexer.Start()
	<-time.After(time.Millisecond * 20)

	indexer.Index("users", "user", "2", "", "", &date, data)
	indexer.Index("users", "user", "3", "", "", &date, data)
	indexer.Index("users", "user", "4", "", "", &date, data)
	<-time.After(time.Millisecond * 200)
	//	indexer.Flush()
	indexer.Stop()
	assert.T(t, messageSets == 2, fmt.Sprintf("Should have sent 2 message sets %d", messageSets))

}

func TestBulkDelete(t *testing.T) {
	InitTests(true)

	c := NewTestConn()
	indexer := c.NewBulkIndexer(1)
	sentBytes := []byte{}

	indexer.Sender = func(buf *bytes.Buffer) error {
		sentBytes = append(sentBytes, buf.Bytes()...)
		return nil
	}

	indexer.Start()

	indexer.Delete("fake", "fake_type", "1")

	indexer.Flush()
	indexer.Stop()

	sent := string(sentBytes)

	expected := `{"delete":{"_index":"fake","_type":"fake_type","_id":"1"}}
`
	asExpected := sent == expected
	assert.T(t, asExpected, fmt.Sprintf("Should have sent '%s' but actually sent '%s'", expected, sent))
}

func XXXTestBulkErrors(t *testing.T) {
	// lets set a bad port, and hope we get a conn refused error?
	c := NewTestConn()
	c.Port = "27845"
	defer func() {
		c.Port = "9200"
	}()
	indexer := c.NewBulkIndexerErrors(10, 1)
	indexer.Start()
	errorCt := 0
	go func() {
		for i := 0; i < 20; i++ {
			date := time.Unix(1257894000, 0)
			data := map[string]interface{}{"name": "smurfs", "age": 22, "date": date}
			indexer.Index("users", "user", strconv.Itoa(i), "", "", &date, data)
		}
	}()
	var errBuf *ErrorBuffer
	for errBuf = range indexer.ErrorChannel {
		errorCt++
		break
	}
	if errBuf.Buf.Len() > 0 {
		gou.Debug(errBuf.Err)
	}
	assert.T(t, errorCt > 0, fmt.Sprintf("ErrorCt should be > 0 %d", errorCt))
	indexer.Stop()
}

/*
BenchmarkSend	18:33:00 bulk_test.go:131: Sent 1 messages in 0 sets totaling 0 bytes
18:33:00 bulk_test.go:131: Sent 100 messages in 1 sets totaling 145889 bytes
18:33:01 bulk_test.go:131: Sent 10000 messages in 100 sets totaling 14608888 bytes
18:33:05 bulk_test.go:131: Sent 20000 messages in 99 sets totaling 14462790 bytes
   20000	    234526 ns/op

*/
func BenchmarkSend(b *testing.B) {
	InitTests(true)
	c := NewTestConn()
	b.StartTimer()
	totalBytes := 0
	sets := 0
	indexer := c.NewBulkIndexer(1)
	indexer.Sender = func(buf *bytes.Buffer) error {
		totalBytes += buf.Len()
		sets += 1
		//log.Println("got bulk")
		return indexer.Send(buf)
	}
	for i := 0; i < b.N; i++ {
		about := make([]byte, 1000)
		rand.Read(about)
		data := map[string]interface{}{"name": "smurfs", "age": 22, "date": time.Unix(1257894000, 0), "about": about}
		indexer.Index("users", "user", strconv.Itoa(i), "", "", nil, data)
	}
	log.Printf("Sent %d messages in %d sets totaling %d bytes \n", b.N, sets, totalBytes)
	if indexer.NumErrors() != 0 {
		b.Fail()
	}
}

/*
TODO:  this should be faster than above

BenchmarkSendBytes	18:33:05 bulk_test.go:169: Sent 1 messages in 0 sets totaling 0 bytes
18:33:05 bulk_test.go:169: Sent 100 messages in 2 sets totaling 292299 bytes
18:33:09 bulk_test.go:169: Sent 10000 messages in 99 sets totaling 14473800 bytes
   10000	    373529 ns/op

*/
func BenchmarkSendBytes(b *testing.B) {
	InitTests(true)
	c := NewTestConn()
	about := make([]byte, 1000)
	rand.Read(about)
	data := map[string]interface{}{"name": "smurfs", "age": 22, "date": time.Unix(1257894000, 0), "about": about}
	body, _ := json.Marshal(data)
	b.StartTimer()
	totalBytes := 0
	sets := 0
	indexer := c.NewBulkIndexer(1)
	indexer.Sender = func(buf *bytes.Buffer) error {
		totalBytes += buf.Len()
		sets += 1
		return indexer.Send(buf)
	}
	for i := 0; i < b.N; i++ {
		indexer.Index("users", "user", strconv.Itoa(i), "", "", nil, body)
	}
	log.Printf("Sent %d messages in %d sets totaling %d bytes \n", b.N, sets, totalBytes)
	if indexer.NumErrors() != 0 {
		b.Fail()
	}
}
<|endoftext|>