<commit_before>// Copyright 2016, RadiantBlue Technologies, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"log"

	"github.com/venicegeo/pz-gocommon/elasticsearch"
	piazza "github.com/venicegeo/pz-gocommon/gocommon"
	pzsyslog "github.com/venicegeo/pz-gocommon/syslog"
	pzuuidgen "github.com/venicegeo/pz-uuidgen/uuidgen"
	pzworkflow "github.com/venicegeo/pz-workflow/workflow"
)

func main() {
	sys, logger, uuidgen := makeSystem()

	log.Printf("pz-workflow starting...")

	indices := makeIndexes(sys)

	workflowServer := makeWorkflow(sys, indices, logger, uuidgen)

	serverLoop(sys, workflowServer)
}

func makeWorkflow(sys *piazza.SystemConfig,
	indices []*elasticsearch.Index,
	logger *pzsyslog.Logger,
	uuidgen *pzuuidgen.Client) *pzworkflow.Server {
	workflowService := &pzworkflow.Service{}
	err := workflowService.Init(
		sys,
		logger,
		uuidgen,
		indices[0],
		indices[1],
		indices[2],
		indices[3],
		indices[4],
		indices[5])
	if err != nil {
		log.Fatal(err)
	}

	err = workflowService.InitCron()
	if err != nil {
		log.Fatal(err)
	}

	workflowServer := &pzworkflow.Server{}
	err = workflowServer.Init(workflowService)
	if err != nil {
		log.Fatal(err)
	}

	return workflowServer
}

func makeSystem() (
	*piazza.SystemConfig,
	*pzsyslog.Logger,
	*pzuuidgen.Client) {

	required := []piazza.ServiceName{
		piazza.PzElasticSearch,
		piazza.PzLogger,
		piazza.PzUuidgen,
		piazza.PzKafka,
		piazza.PzServiceController,
		piazza.PzIdam,
	}

	sys, err := piazza.NewSystemConfig(piazza.PzWorkflow, required)
	if err != nil {
		log.Fatal(err)
	}

	logWriter, err := pzsyslog.NewHttpWriter(sys)
	if err != nil {
		log.Fatal(err)
	}
	logger := pzsyslog.NewLogger(logWriter, "pz-uuidgen")

	uuidgen, err := pzuuidgen.NewClient(sys)
	if err != nil {
		log.Fatal(err)
	}

	return sys, logger, uuidgen
}

func serverLoop(sys *piazza.SystemConfig,
	workflowServer *pzworkflow.Server) {
	genericServer := piazza.GenericServer{Sys: sys}

	err := genericServer.Configure(workflowServer.Routes)
	if err != nil {
		log.Fatal(err)
	}

	done, err := genericServer.Start()
	if err != nil {
		log.Fatal(err)
	}

	err = <-done
	if err != nil {
		log.Fatal(err)
	}
}

func makeIndexes(sys *piazza.SystemConfig) []*elasticsearch.Index {
	eventtypesIndex, err := elasticsearch.NewIndex(sys, "eventtypes003", pzworkflow.EventTypeIndexSettings)
	if err != nil {
		log.Fatal(err)
	}
	eventsIndex, err := elasticsearch.NewIndex(sys, "events004", pzworkflow.EventIndexSettings)
	if err != nil {
		log.Fatal(err)
	}
	triggersIndex, err := elasticsearch.NewIndex(sys, "triggers003", pzworkflow.TriggerIndexSettings)
	if err != nil {
		log.Fatal(err)
	}
	alertsIndex, err := elasticsearch.NewIndex(sys, "alerts003", pzworkflow.AlertIndexSettings)
	if err != nil {
		log.Fatal(err)
	}
	cronIndex, err := elasticsearch.NewIndex(sys, "crons003", pzworkflow.CronIndexSettings)
	if err != nil {
		log.Fatal(err)
	}
	testElasticsearchIndex, err := elasticsearch.NewIndex(sys, "testelasticsearch003", pzworkflow.TestElasticsearchSettings)
	if err != nil {
		log.Fatal(err)
	}

	ret := []*elasticsearch.Index{
		eventtypesIndex, eventsIndex, triggersIndex,
		alertsIndex, cronIndex, testElasticsearchIndex,
	}
	return ret
}
<commit_msg>use proper new logger connection<commit_after>// Copyright 2016, RadiantBlue Technologies, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"log"

	"github.com/venicegeo/pz-gocommon/elasticsearch"
	piazza "github.com/venicegeo/pz-gocommon/gocommon"
	pzsyslog "github.com/venicegeo/pz-gocommon/syslog"
	pzuuidgen "github.com/venicegeo/pz-uuidgen/uuidgen"
	pzworkflow "github.com/venicegeo/pz-workflow/workflow"
)

func main() {
	sys, logger, uuidgen := makeSystem()

	log.Printf("pz-workflow starting...")

	indices := makeIndexes(sys)

	workflowServer := makeWorkflow(sys, indices, logger, uuidgen)

	serverLoop(sys, workflowServer)
}

func makeWorkflow(sys *piazza.SystemConfig,
	indices []*elasticsearch.Index,
	logger *pzsyslog.Logger,
	uuidgen *pzuuidgen.Client) *pzworkflow.Server {
	workflowService := &pzworkflow.Service{}
	err := workflowService.Init(
		sys,
		logger,
		uuidgen,
		indices[0],
		indices[1],
		indices[2],
		indices[3],
		indices[4],
		indices[5])
	if err != nil {
		log.Fatal(err)
	}

	err = workflowService.InitCron()
	if err != nil {
		log.Fatal(err)
	}

	workflowServer := &pzworkflow.Server{}
	err = workflowServer.Init(workflowService)
	if err != nil {
		log.Fatal(err)
	}

	return workflowServer
}

func makeSystem() (
	*piazza.SystemConfig,
	*pzsyslog.Logger,
	*pzuuidgen.Client) {

	required := []piazza.ServiceName{
		piazza.PzElasticSearch,
		piazza.PzLogger,
		piazza.PzUuidgen,
		piazza.PzKafka,
		piazza.PzServiceController,
		piazza.PzIdam,
	}

	sys, err := piazza.NewSystemConfig(piazza.PzWorkflow, required)
	if err != nil {
		log.Fatal(err)
	}

	logUrl, err := sys.GetURL(piazza.PzLogger)
	if err != nil {
		log.Fatal(err)
	}
	logWriter, err := pzsyslog.NewHttpWriter(logUrl)
	if err != nil {
		log.Fatal(err)
	}
	logger := pzsyslog.NewLogger(logWriter, "pz-uuidgen")

	uuidgen, err := pzuuidgen.NewClient(sys)
	if err != nil {
		log.Fatal(err)
	}

	return sys, logger, uuidgen
}

func serverLoop(sys *piazza.SystemConfig,
	workflowServer *pzworkflow.Server) {
	genericServer := piazza.GenericServer{Sys: sys}

	err := genericServer.Configure(workflowServer.Routes)
	if err != nil {
		log.Fatal(err)
	}

	done, err := genericServer.Start()
	if err != nil {
		log.Fatal(err)
	}

	err = <-done
	if err != nil {
		log.Fatal(err)
	}
}

func makeIndexes(sys *piazza.SystemConfig) []*elasticsearch.Index {
	eventtypesIndex, err := elasticsearch.NewIndex(sys, "eventtypes003", pzworkflow.EventTypeIndexSettings)
	if err != nil {
		log.Fatal(err)
	}
	eventsIndex, err := elasticsearch.NewIndex(sys, "events004", pzworkflow.EventIndexSettings)
	if err != nil {
		log.Fatal(err)
	}
	triggersIndex, err := elasticsearch.NewIndex(sys, "triggers003", pzworkflow.TriggerIndexSettings)
	if err != nil {
		log.Fatal(err)
	}
	alertsIndex, err := elasticsearch.NewIndex(sys, "alerts003", pzworkflow.AlertIndexSettings)
	if err != nil {
		log.Fatal(err)
	}
	cronIndex, err := elasticsearch.NewIndex(sys, "crons003", pzworkflow.CronIndexSettings)
	if err != nil {
		log.Fatal(err)
	}
	testElasticsearchIndex, err := elasticsearch.NewIndex(sys, "testelasticsearch003", pzworkflow.TestElasticsearchSettings)
	if err != nil {
		log.Fatal(err)
	}

	ret := []*elasticsearch.Index{
		eventtypesIndex, eventsIndex, triggersIndex,
		alertsIndex, cronIndex, testElasticsearchIndex,
	}
	return ret
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"log"
	"net/http"

	"github.com/zhulik/rutracker-proxy/selector"
)

var proxyTypes = map[string]selector.ProxyType{"http": selector.HTTP, "socks": selector.SOCKS}

func main() {
	port := flag.Int("p", 8080, "Proxy port")
	rotationTimeout := flag.Int("r", 5, "Proxy rotation timeout in minutes, 0 - disabled")
	proxyType := flag.String("t", "http", "Proxy type http|socks")

	flag.Parse()

	if p, ok := proxyTypes[*proxyType]; ok {
		log.Printf("Starting proxy with port=%d type=%s rotation timeout=%d", *port, *proxyType, *rotationTimeout)
		proxy := newProxy(p, *rotationTimeout, *port)
		log.Fatal(http.ListenAndServe(fmt.Sprintf(":%d", port), proxy))

	} else {
		log.Fatal("Unknown proxy type ", *proxyType)
	}
}
<commit_msg>Invalid port error fix (#7)<commit_after>package main

import (
	"flag"
	"fmt"
	"log"
	"net/http"

	"github.com/zhulik/rutracker-proxy/selector"
)

var proxyTypes = map[string]selector.ProxyType{"http": selector.HTTP, "socks": selector.SOCKS}

func main() {
	port := flag.Int("p", 8080, "Proxy port")
	rotationTimeout := flag.Int("r", 5, "Proxy rotation timeout in minutes, 0 - disabled")
	proxyType := flag.String("t", "http", "Proxy type http|socks")

	flag.Parse()

	if p, ok := proxyTypes[*proxyType]; ok {
		log.Printf("Starting proxy with port=%d type=%s rotation timeout=%d", *port, *proxyType, *rotationTimeout)
		proxy := newProxy(p, *rotationTimeout, *port)
		log.Fatal(http.ListenAndServe(fmt.Sprintf(":%d", *port), proxy))

	} else {
		log.Fatal("Unknown proxy type ", *proxyType)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"math/rand"
	"net/http"
	"time"

	"github.com/gorilla/mux"
	"github.com/sirupsen/logrus"
	"github.com/urfave/negroni"

	"github.com/ivan1993spb/snake-server/connections"
	"github.com/ivan1993spb/snake-server/handlers"
	"github.com/ivan1993spb/snake-server/middlewares"
)

const (
	defaultAddress     = ":8080"
	defaultGroupsLimit = 100
	defaultConnsLimit  = 1000
)

var (
	address     string
	groupsLimit int
	connsLimit  int
	seed        int64
	flagJSONLog bool
	logLevel    string
)

func init() {
	flag.StringVar(&address, "address", defaultAddress, "address to serve")
	flag.IntVar(&groupsLimit, "groups-limit", defaultGroupsLimit, "groups limit")
	flag.IntVar(&connsLimit, "conns-limit", defaultConnsLimit, "web-socket connections limit")
	flag.Int64Var(&seed, "seed", time.Now().UnixNano(), "random seed")
	flag.BoolVar(&flagJSONLog, "log-json", false, "use json format for logger")
	flag.StringVar(&logLevel, "log-level", "info", "set log level: panic, fatal, error, warning (warn), info or debug")
	flag.Parse()
}

func logger() logrus.FieldLogger {
	logger := logrus.New()
	if flagJSONLog {
		logger.Formatter = &logrus.JSONFormatter{}
	}
	if level, err := logrus.ParseLevel(logLevel); err != nil {
		logger.SetLevel(logrus.InfoLevel)
	} else {
		logger.SetLevel(level)
	}
	return logger
}

func main() {
	logger := logger()

	logger.WithFields(logrus.Fields{
		"conns_limit":  connsLimit,
		"groups_limit": groupsLimit,
		"seed":         seed,
		"log_level":    logLevel,
	}).Info("preparing to start server")

	rand.Seed(seed)

	groupManager, err := connections.NewConnectionGroupManager(logger, groupsLimit, connsLimit)
	if err != nil {
		logger.Fatalln("cannot create connections group manager:", err)
	}

	r := mux.NewRouter().StrictSlash(false)
	r.Path(handlers.URLRouteCreateGame).Methods(handlers.MethodCreateGame).Handler(handlers.NewCreateGameHandler(logger, groupManager))
	r.Path(handlers.URLRouteGetGameByID).Methods(handlers.MethodGetGame).Handler(handlers.NewGetGameHandler(logger, groupManager))
	r.Path(handlers.URLRouteDeleteGameByID).Methods(handlers.MethodDeleteGame).Handler(handlers.NewDeleteGameHandler(logger, groupManager))
	r.Path(handlers.URLRouteGetGames).Methods(handlers.MethodGetGames).Handler(handlers.NewGetGamesHandler(logger, groupManager))
	r.Path(handlers.URLRouteGameWebSocketByID).Methods(handlers.MethodGame).Handler(handlers.NewGameWebSocketHandler(logger, groupManager))

	// TODO: Use middlewares only for HTTP, not for WS connections!
	n := negroni.New(middlewares.NewRecovery(logger), middlewares.NewLogger(logger), middlewares.NewCORS())
	n.UseHandler(r)

	logger.WithField("address", address).Info("starting server")

	if err := http.ListenAndServe(address, n); err != nil {
		logger.Fatalf("server error: %s", err)
	}
}
<commit_msg>Refactor routes to use middlewares only for Rest API<commit_after>package main

import (
	"flag"
	"math/rand"
	"net/http"
	"time"

	"github.com/gorilla/mux"
	"github.com/sirupsen/logrus"
	"github.com/urfave/negroni"

	"github.com/ivan1993spb/snake-server/connections"
	"github.com/ivan1993spb/snake-server/handlers"
	"github.com/ivan1993spb/snake-server/middlewares"
)

const (
	defaultAddress     = ":8080"
	defaultGroupsLimit = 100
	defaultConnsLimit  = 1000
)

var (
	address     string
	groupsLimit int
	connsLimit  int
	seed        int64
	flagJSONLog bool
	logLevel    string
)

func init() {
	flag.StringVar(&address, "address", defaultAddress, "address to serve")
	flag.IntVar(&groupsLimit, "groups-limit", defaultGroupsLimit, "groups limit")
	flag.IntVar(&connsLimit, "conns-limit", defaultConnsLimit, "web-socket connections limit")
	flag.Int64Var(&seed, "seed", time.Now().UnixNano(), "random seed")
	flag.BoolVar(&flagJSONLog, "log-json", false, "use json format for logger")
	flag.StringVar(&logLevel, "log-level", "info", "set log level: panic, fatal, error, warning (warn), info or debug")
	flag.Parse()
}

func logger() logrus.FieldLogger {
	logger := logrus.New()
	if flagJSONLog {
		logger.Formatter = &logrus.JSONFormatter{}
	}
	if level, err := logrus.ParseLevel(logLevel); err != nil {
		logger.SetLevel(logrus.InfoLevel)
	} else {
		logger.SetLevel(level)
	}
	return logger
}

func main() {
	logger := logger()

	logger.WithFields(logrus.Fields{
		"conns_limit":  connsLimit,
		"groups_limit": groupsLimit,
		"seed":         seed,
		"log_level":    logLevel,
	}).Info("preparing to start server")

	rand.Seed(seed)

	groupManager, err := connections.NewConnectionGroupManager(logger, groupsLimit, connsLimit)
	if err != nil {
		logger.Fatalln("cannot create connections group manager:", err)
	}

	rootRouter := mux.NewRouter()

	// Web-Socket route
	rootRouter.Path(handlers.URLRouteGameWebSocketByID).Methods(handlers.MethodGame).Handler(handlers.NewGameWebSocketHandler(logger, groupManager))

	// API routes
	apiRouter := mux.NewRouter().StrictSlash(true)
	apiRouter.Path(handlers.URLRouteCreateGame).Methods(handlers.MethodCreateGame).Handler(handlers.NewCreateGameHandler(logger, groupManager))
	apiRouter.Path(handlers.URLRouteGetGameByID).Methods(handlers.MethodGetGame).Handler(handlers.NewGetGameHandler(logger, groupManager))
	apiRouter.Path(handlers.URLRouteDeleteGameByID).Methods(handlers.MethodDeleteGame).Handler(handlers.NewDeleteGameHandler(logger, groupManager))
	apiRouter.Path(handlers.URLRouteGetGames).Methods(handlers.MethodGetGames).Handler(handlers.NewGetGamesHandler(logger, groupManager))
	// Use middlewares for API routes
	rootRouter.NewRoute().Handler(negroni.New(middlewares.NewRecovery(logger), middlewares.NewLogger(logger), middlewares.NewCORS(), negroni.Wrap(apiRouter)))

	n := negroni.New()
	n.UseHandler(rootRouter)

	logger.WithField("address", address).Info("starting server")

	if err := http.ListenAndServe(address, n); err != nil {
		logger.Fatalf("server error: %s", err)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"math/rand"
	"net/http"
	"strconv"
	"strings"
	"time"

	"github.com/mediocregopher/lever"
	"github.com/mediocregopher/radix/v3"
)

// Prefix is a Markov chain prefix of one or more words.
type Prefix []string

// String returns the Prefix as a string (for use as a map key).
func (p Prefix) String() string {
	return strings.Join(p, " ")
}

// Shift removes the first word from the Prefix and appends the given word.
func (p Prefix) Shift(word string) {
	copy(p, p[1:])
	p[len(p)-1] = word
}

var p *radix.Pool

func prefixKey(chain string, prefix Prefix) string {
	return fmt.Sprintf("markov:%s:%s", chain, prefix.String())
}

func main() {
	rand.Seed(time.Now().UTC().UnixNano())
	log.SetFlags(log.Llongfile)

	l := lever.New("markov", nil)
	l.Add(lever.Param{
		Name:        "-prefixLen",
		Default:     "2",
		Description: "Prefix length for the markov chain algorithm",
	})
	l.Add(lever.Param{
		Name:        "-listenAddr",
		Default:     ":8080",
		Description: "Address to listen for calls to the http interface on",
	})
	l.Add(lever.Param{
		Name:        "-redisAddr",
		Default:     "127.0.0.1:6379",
		Description: "Address for an instance of redis",
	})
	l.Add(lever.Param{
		Name:        "-timeout",
		Default:     "720",
		Description: "Hours a suffix is allowed to stay untouched before it is cleaned up",
	})
	l.Parse()

	redisAddr, _ := l.ParamStr("-redisAddr")
	var err error

	p, err = radix.NewPool("tcp", redisAddr, 10)
	if err != nil {
		log.Fatal(err)
	}

	prefixLen, _ := l.ParamInt("-prefixLen")
	timeout, _ := l.ParamInt("-timeout")
	go bobTheBuilder(prefixLen)
	go clydeTheCleaner(int64(timeout))

	http.HandleFunc("/build", func(w http.ResponseWriter, r *http.Request) {
		var suffixes []string
		for {
			var s string
			if _, err := fmt.Fscan(r.Body, &s); err != nil {
				break
			}
			suffixes = append(suffixes, strings.TrimSpace(s))
		}
		buildCh <- toBuild{suffixes, r.FormValue("chainName")}
	})

	http.HandleFunc("/generate", func(w http.ResponseWriter, r *http.Request) {
		numPartsStr := r.FormValue("numParts")
		if numPartsStr == "" {
			http.Error(w, "numParts argument must be specified", 400)
			return
		}

		numParts, err := strconv.Atoi(numPartsStr)
		if err != nil {
			http.Error(w, fmt.Sprintf("invalid value of numParts: %s", err), 400)
			return
		}

		prefix := make(Prefix, prefixLen)
		var words []string
		for {
			key := prefixKey(r.FormValue("chainName"), prefix)
			var suffixes []string
			if err := p.Do(radix.Cmd(&suffixes, "ZRANGE", key, "0", "-1")); err != nil {
				log.Fatal(err)
			} else if len(suffixes) == 0 {
				break
			}

			i := rand.Intn(len(suffixes))
			next := suffixes[i]
			words = append(words, next)
			prefix.Shift(next)

			if len(next) == 0 {
				continue
			}

			lastChar := next[len(next)-1]

			if lastChar == '!' ||
				lastChar == '?' ||
				lastChar == '.' ||
				(numParts == 1 &&
					(lastChar == ',' ||
						lastChar == ':' ||
						lastChar == ';')) {
				numParts--
			}

			switch lastChar {
			case '!', '?', '.', ',', ':', ';':
				numParts--
			}

			if numParts <= 0 {
				break
			}
		}

		fmt.Fprintln(w, strings.Join(words, " "))
	})

	listenAddr, _ := l.ParamStr("-listenAddr")
	log.Fatal(http.ListenAndServe(listenAddr, nil))
}

type toBuild struct {
	suffixes  []string
	chainName string
}

var buildCh = make(chan toBuild)

func bobTheBuilder(prefixLen int) {
	for toB := range buildCh {
		suffixes := toB.suffixes
		prefix := make(Prefix, prefixLen)
		ts := time.Now().Unix()
		for _, suffix := range suffixes {
			key := prefixKey(toB.chainName, prefix)
			if err := p.Do(radix.FlatCmd(nil, "ZADD", key, ts, suffix)); err != nil {
				log.Fatal(err)
			}
			prefix.Shift(suffix)
		}
	}
}

func clydeTheCleaner(timeout int64) {
	tick := time.Tick(30 * time.Second)
	for {
		expire := time.Now().Unix() - (timeout * 3600)
		scanner := radix.NewScanner(p, radix.ScanOpts{
			Command: "SCAN",
			Pattern: "markov:*",
		})
		var key string
		for scanner.Next(&key) {
			if err := p.Do(radix.FlatCmd(nil, "ZREMRANGEBYSCORE", key, "0", expire)); err != nil {
				log.Fatal(err)
			}
		}
		if err := scanner.Close(); err != nil {
			log.Fatal(err)
		}

		<-tick
	}
}
<commit_msg>get rid of bobTheBuilder, dunno why that was ever put in<commit_after>package main

import (
	"fmt"
	"log"
	"math/rand"
	"net/http"
	"strconv"
	"strings"
	"time"

	"github.com/mediocregopher/lever"
	"github.com/mediocregopher/radix/v3"
)

// Prefix is a Markov chain prefix of one or more words.
type Prefix []string

// String returns the Prefix as a string (for use as a map key).
func (p Prefix) String() string {
	return strings.Join(p, " ")
}

// Shift removes the first word from the Prefix and appends the given word.
func (p Prefix) Shift(word string) {
	copy(p, p[1:])
	p[len(p)-1] = word
}

var p *radix.Pool

func prefixKey(chain string, prefix Prefix) string {
	return fmt.Sprintf("markov:%s:%s", chain, prefix.String())
}

func main() {
	rand.Seed(time.Now().UTC().UnixNano())
	log.SetFlags(log.Llongfile)

	l := lever.New("markov", nil)
	l.Add(lever.Param{
		Name:        "-prefixLen",
		Default:     "2",
		Description: "Prefix length for the markov chain algorithm",
	})
	l.Add(lever.Param{
		Name:        "-listenAddr",
		Default:     ":8080",
		Description: "Address to listen for calls to the http interface on",
	})
	l.Add(lever.Param{
		Name:        "-redisAddr",
		Default:     "127.0.0.1:6379",
		Description: "Address for an instance of redis",
	})
	l.Add(lever.Param{
		Name:        "-timeout",
		Default:     "720",
		Description: "Hours a suffix is allowed to stay untouched before it is cleaned up",
	})
	l.Parse()

	redisAddr, _ := l.ParamStr("-redisAddr")
	var err error

	p, err = radix.NewPool("tcp", redisAddr, 10)
	if err != nil {
		log.Fatal(err)
	}

	prefixLen, _ := l.ParamInt("-prefixLen")
	timeout, _ := l.ParamInt("-timeout")
	go clydeTheCleaner(int64(timeout))

	http.HandleFunc("/build", func(w http.ResponseWriter, r *http.Request) {
		var suffixes []string
		for {
			var s string
			if _, err := fmt.Fscan(r.Body, &s); err != nil {
				break
			}
			suffixes = append(suffixes, strings.TrimSpace(s))
		}

		prefix := make(Prefix, prefixLen)
		ts := time.Now().Unix()
		for _, suffix := range suffixes {
			key := prefixKey(r.FormValue("chainName"), prefix)
			if err := p.Do(radix.FlatCmd(nil, "ZADD", key, ts, suffix)); err != nil {
				log.Fatal(err)
			}
			prefix.Shift(suffix)
		}
	})

	http.HandleFunc("/generate", func(w http.ResponseWriter, r *http.Request) {
		numPartsStr := r.FormValue("numParts")
		if numPartsStr == "" {
			http.Error(w, "numParts argument must be specified", 400)
			return
		}

		numParts, err := strconv.Atoi(numPartsStr)
		if err != nil {
			http.Error(w, fmt.Sprintf("invalid value of numParts: %s", err), 400)
			return
		}

		prefix := make(Prefix, prefixLen)
		var words []string
		for {
			key := prefixKey(r.FormValue("chainName"), prefix)
			var suffixes []string
			if err := p.Do(radix.Cmd(&suffixes, "ZRANGE", key, "0", "-1")); err != nil {
				log.Fatal(err)
			} else if len(suffixes) == 0 {
				break
			}

			i := rand.Intn(len(suffixes))
			next := suffixes[i]
			words = append(words, next)
			prefix.Shift(next)

			if len(next) == 0 {
				continue
			}

			lastChar := next[len(next)-1]

			if lastChar == '!' ||
				lastChar == '?' ||
				lastChar == '.' ||
				(numParts == 1 &&
					(lastChar == ',' ||
						lastChar == ':' ||
						lastChar == ';')) {
				numParts--
			}

			switch lastChar {
			case '!', '?', '.', ',', ':', ';':
				numParts--
			}

			if numParts <= 0 {
				break
			}
		}

		fmt.Fprintln(w, strings.Join(words, " "))
	})

	listenAddr, _ := l.ParamStr("-listenAddr")
	log.Fatal(http.ListenAndServe(listenAddr, nil))
}

func clydeTheCleaner(timeout int64) {
	tick := time.Tick(30 * time.Second)
	for {
		expire := time.Now().Unix() - (timeout * 3600)
		scanner := radix.NewScanner(p, radix.ScanOpts{
			Command: "SCAN",
			Pattern: "markov:*",
		})
		var key string
		for scanner.Next(&key) {
			if err := p.Do(radix.FlatCmd(nil, "ZREMRANGEBYSCORE", key, "0", expire)); err != nil {
				log.Fatal(err)
			}
		}
		if err := scanner.Close(); err != nil {
			log.Fatal(err)
		}

		<-tick
	}
}
<|endoftext|>


<commit_before>package main

import (
	"os/exec"
	"strings"

	"github.com/lxn/walk"
	. "github.com/lxn/walk/declarative"
)

func main() {
	var inTE, outTE *walk.TextEdit
	//out, _ := exec.Command("wmic","logicaldisk", "get", "name").Output()

	out, err := exec.Command("wmic", "desktopmonitor", "get", "screenheight").Output()
	if err != nil {
		panic(err)
	}

	//screenwidth

	MainWindow{
		Title:   "SCREAMO",
		MinSize: Size{600, 400},
		Layout:  VBox{},
		Children: []Widget{
			HSplitter{
				Children: []Widget{
					TextEdit{AssignTo: &inTE, Text: string(out)},
					TextEdit{AssignTo: &outTE, ReadOnly: true},
				},
			},
			PushButton{
				Text: "SCREAM",
				OnClicked: func() {
					outTE.SetText(strings.ToUpper(inTE.Text()))
				},
			},
		},
	}.Run()
}
<commit_msg>prueba para separar del texto las dimensiones de ancho y alto<commit_after>package main

import (
	"log"
	"os/exec"
	"strings"

	"github.com/lxn/walk"
	. "github.com/lxn/walk/declarative"
)

func main() {
	var inTE, outTE *walk.TextEdit
	//out, _ := exec.Command("wmic","logicaldisk", "get", "name").Output()

	out, err := exec.Command("wmic", "desktopmonitor", "get", "screenheight", ",", "screenwidth").Output()
	if err != nil {
		panic(err)
	}

	height, err := exec.Command("wmic", "desktopmonitor", "get", "screenwidth").Output()
	if err != nil {
		panic(err)
	}

	width, err := exec.Command("wmic", "desktopmonitor", "get", "screenheight").Output()
	if err != nil {
		panic(err)
	}

	// log.Println(strings.Replace(string(out), "ScreenHeight  ScreenWidth \n", "", 1))

	log.Println(strings.Split(string(height), ` `)[2])
	log.Println(strings.Split(string(width), ` `)[2])
	//screenwidth

	MainWindow{
		Title:   "SCREAMO",
		MinSize: Size{600, 400},
		Layout:  VBox{},
		Children: []Widget{
			HSplitter{
				Children: []Widget{
					TextEdit{AssignTo: &inTE, Text: string(out)},
					TextEdit{AssignTo: &outTE, ReadOnly: true},
				},
			},
			PushButton{
				Text: "SCREAM",
				OnClicked: func() {
					outTE.SetText(strings.ToUpper(inTE.Text()))
				},
			},
		},
	}.Run()
}
<|endoftext|>


<commit_before>// Go equivalent of the "DNS & BIND" book check-soa program.
// Created by Stephane Bortzmeyer.
package main

import (
	"errors"
	"fmt"
	"log"
	"math/rand"
	"os"
	"sort"
	"time"

	"github.com/miekg/dns"
)

const (
	// DefaultTimeout is default timeout many operation in this program will
	// use.
	DefaultTimeout time.Duration = 5 * time.Second
)

var (
	localm           *dns.Msg
	localc           *dns.Client
	conf             *dns.ClientConfig
	domain           string
	assembledDomain string
	nextNs          string
)

func localQuery(qname string, qtype uint16, server string) (*dns.Msg, error) {
	localm.SetQuestion(qname, qtype)
	r, _, err := localc.Exchange(localm, server+":53")
	if err != nil {
		return nil, err
	}
	if r == nil || r.Rcode == dns.RcodeNameError || r.Rcode == dns.RcodeSuccess {
		return r, err
	}

	return nil, errors.New("No name server to answer the question")
}

func getNsRecords(zone string, server string) ([]string, string, error) {
	zone = dns.Fqdn(zone)
	r, err := localQuery(zone, dns.TypeNS, server)
	if err != nil || r == nil {
		log.Fatal("Cannot retrieve the list of name servers for %s: %s\n", zone, err)
	}

	var success bool
	var nameservers []string
	var random_ns string

	for _, ans := range r.Answer {
		switch t := ans.(type) {
		case *dns.NS:
			nameserver := t.Ns
			nameservers = append(nameservers, nameserver)
			success = true
		}
	}

	if !success {
		// No "Answer" given by the server, check the Authority section if
		// additional nameservers were provided.
		for _, ans := range r.Ns {
			switch t := ans.(type) {
			case *dns.NS:
				nameserver := t.Ns
				nameservers = append(nameservers, nameserver)
				success = true
			}
		}
	}

	if !success {
		return nil, "", errors.New("No nameservers found for " + zone)
	}

	// Pick a random NS record for the next queries
	random_ns = nameservers[rand.Intn(len(nameservers))]

	sort.Strings(nameservers)

	return nameservers, random_ns, nil

}

func main() {
	if len(os.Args) != 2 {
		log.Fatal("%s ZONE\n", os.Args[0])
	}
	domain = os.Args[1]

	rand.Seed(time.Now().Unix())
	var err error
	conf, err = dns.ClientConfigFromFile("/etc/resolv.conf")
	if err != nil || conf == nil {
		log.Fatal("Cannot initialize the local resolver: %s\n", err)
	}
	localm = &dns.Msg{
		MsgHdr: dns.MsgHdr{
			RecursionDesired: true,
		},
		Question: make([]dns.Question, 1),
	}
	localc = &dns.Client{
		ReadTimeout: DefaultTimeout,
	}

	// Walk the root until you find the authoritative nameservers
	fmt.Printf("Retrieving list of root nameservers:\n")
	root_nameservers, nextNs, err := getNsRecords(".", conf.Servers[0])
	if err != nil {
		log.Fatal("Query failed: ", err)
	}
	for _, nameserver := range root_nameservers {
		if nameserver == nextNs {
			// We'll use this one for queries
			fmt.Println(" ➡ " + nameserver)
		} else {
			fmt.Println(" - " + nameserver)
		}
	}

	// We have list of root nameservers: split domain, query each part for NS records
	domain_pieces := dns.SplitDomainName(domain)
	assembledDomain = "."
	var ns []string

	for i := len(domain_pieces) - 1; i >= 0; i-- {
		fmt.Println("\n")
		element := domain_pieces[i]
		if assembledDomain == "." {
			assembledDomain = element + "."
		} else {
			assembledDomain = element + "." + assembledDomain
		}

		fmt.Println("Finding nameservers for zone '" + assembledDomain + "' using parent nameserver '" + nextNs + "'")
		ns, nextNs, err = getNsRecords(assembledDomain, nextNs)
		if err != nil {
			fmt.Println("Query failed: ", err)
		}

		// Print the nameservers for this zone, highlight the one we used to query
		for _, nameserver := range ns {
			if nameserver == nextNs && dns.Fqdn(domain) != assembledDomain {
				// We'll use this one for queries
				fmt.Println(" ➡ " + nameserver)
			} else {
				fmt.Println(" - " + nameserver)
			}
		}
	}

}
<commit_msg>Remove obsolete success variable<commit_after>// Go equivalent of the "DNS & BIND" book check-soa program.
// Created by Stephane Bortzmeyer.
package main

import (
	"errors"
	"fmt"
	"log"
	"math/rand"
	"os"
	"sort"
	"time"

	"github.com/miekg/dns"
)

const (
	// DefaultTimeout is default timeout many operation in this program will
	// use.
	DefaultTimeout time.Duration = 5 * time.Second
)

var (
	localm           *dns.Msg
	localc           *dns.Client
	conf             *dns.ClientConfig
	domain           string
	assembledDomain string
	nextNs          string
)

func localQuery(qname string, qtype uint16, server string) (*dns.Msg, error) {
	localm.SetQuestion(qname, qtype)
	r, _, err := localc.Exchange(localm, server+":53")
	if err != nil {
		return nil, err
	}
	if r == nil || r.Rcode == dns.RcodeNameError || r.Rcode == dns.RcodeSuccess {
		return r, err
	}

	return nil, errors.New("No name server to answer the question")
}

func getNsRecords(zone string, server string) ([]string, string, error) {
	zone = dns.Fqdn(zone)
	r, err := localQuery(zone, dns.TypeNS, server)
	if err != nil || r == nil {
		log.Fatal("Cannot retrieve the list of name servers for %s: %s\n", zone, err)
	}

	var nameservers []string
	var random_ns string

	for _, ans := range r.Answer {
		switch t := ans.(type) {
		case *dns.NS:
			nameserver := t.Ns
			nameservers = append(nameservers, nameserver)
		}
	}

	if len(nameservers) == 0 {
		// No "Answer" given by the server, check the Authority section if
		// additional nameservers were provided.
		for _, ans := range r.Ns {
			switch t := ans.(type) {
			case *dns.NS:
				nameserver := t.Ns
				nameservers = append(nameservers, nameserver)
			}
		}
	}

	if len(nameservers) == 0 {
		return nil, "", errors.New("No nameservers found for " + zone)
	}

	// Pick a random NS record for the next queries
	random_ns = nameservers[rand.Intn(len(nameservers))]

	sort.Strings(nameservers)

	return nameservers, random_ns, nil

}

func main() {
	if len(os.Args) != 2 {
		log.Fatal("%s ZONE\n", os.Args[0])
	}
	domain = os.Args[1]

	rand.Seed(time.Now().Unix())
	var err error
	conf, err = dns.ClientConfigFromFile("/etc/resolv.conf")
	if err != nil || conf == nil {
		log.Fatal("Cannot initialize the local resolver: %s\n", err)
	}
	localm = &dns.Msg{
		MsgHdr: dns.MsgHdr{
			RecursionDesired: true,
		},
		Question: make([]dns.Question, 1),
	}
	localc = &dns.Client{
		ReadTimeout: DefaultTimeout,
	}

	// Walk the root until you find the authoritative nameservers
	fmt.Printf("Retrieving list of root nameservers:\n")
	root_nameservers, nextNs, err := getNsRecords(".", conf.Servers[0])
	if err != nil {
		log.Fatal("Query failed: ", err)
	}
	for _, nameserver := range root_nameservers {
		if nameserver == nextNs {
			// We'll use this one for queries
			fmt.Println(" ➡ " + nameserver)
		} else {
			fmt.Println(" - " + nameserver)
		}
	}

	// We have list of root nameservers: split domain, query each part for NS records
	domain_pieces := dns.SplitDomainName(domain)
	assembledDomain = "."
	var ns []string

	for i := len(domain_pieces) - 1; i >= 0; i-- {
		fmt.Println("\n")
		element := domain_pieces[i]
		if assembledDomain == "." {
			assembledDomain = element + "."
		} else {
			assembledDomain = element + "." + assembledDomain
		}

		fmt.Println("Finding nameservers for zone '" + assembledDomain + "' using parent nameserver '" + nextNs + "'")
		ns, nextNs, err = getNsRecords(assembledDomain, nextNs)
		if err != nil {
			fmt.Println("Query failed: ", err)
		}

		// Print the nameservers for this zone, highlight the one we used to query
		for _, nameserver := range ns {
			if nameserver == nextNs && dns.Fqdn(domain) != assembledDomain {
				// We'll use this one for queries
				fmt.Println(" ➡ " + nameserver)
			} else {
				fmt.Println(" - " + nameserver)
			}
		}
	}

}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path"

	"github.com/BurntSushi/toml"
	"github.com/k0pernicus/goyave/configurationFile"
	"github.com/k0pernicus/goyave/consts"
	"github.com/k0pernicus/goyave/traces"
	"github.com/k0pernicus/goyave/utils"
	"github.com/k0pernicus/goyave/walk"
	"github.com/spf13/cobra"
)

var configurationFileStructure configurationFile.ConfigurationFile
var configurationFilePath string
var userHomeDir string

/*initialize get the configuration file existing in the system (or create it), and return
 *a pointer to his content.
 */
func initialize(configurationFileStructure *configurationFile.ConfigurationFile) {
	// Initialize all different traces structures
	traces.InitTraces(os.Stdout, os.Stderr, os.Stdout, os.Stdout)

	// Get the user home directory
	userHomeDir = utils.GetUserHomeDir()
	if len(userHomeDir) == 0 {
		log.Fatalf("Cannot get the user home dir.\n")
	}
	// Set the configuration path file
	configurationFilePath = path.Join(userHomeDir, consts.ConfigurationFileName)
	filePointer, err := os.OpenFile(configurationFilePath, os.O_RDWR|os.O_CREATE, 0755)
	if err != nil {
		log.Fatalf("Cannot open the file %s, due to error '%s'.\n", configurationFilePath, err)
	}
	defer filePointer.Close()
	var bytesArray []byte
	// Get the content of the goyave configuration file
	configurationFile.GetConfigurationFileContent(filePointer, &bytesArray)
	if _, err = toml.Decode(string(bytesArray[:]), configurationFileStructure); err != nil {
		log.Fatalln(err)
	}
}

/*kill saves the current state of the configuration structure in the configuration file
 */
func kill() {
	var outputBuffer bytes.Buffer
	if err := configurationFileStructure.Encode(&outputBuffer); err != nil {
		log.Fatalln("Cannot save the current configurationFile structure!")
	}
	if err := ioutil.WriteFile(configurationFilePath, outputBuffer.Bytes(), 0777); err != nil {
		log.Fatalln("Cannot access to your file to save the configurationFile structure!")
	}
}

func main() {

	/*rootCmd defines the global app, and some actions to run before and after the command running
	 */
	var rootCmd = &cobra.Command{
		Use:   "goyave",
		Short: "Goyave is a tool to take a look at your local git repositories",
		// Initialize the structure
		PersistentPreRun: func(cmd *cobra.Command, args []string) {
			initialize(&configurationFileStructure)
		},
		// Save the current configuration file structure, in the configuration file
		PersistentPostRun: func(cmd *cobra.Command, args []string) {
			kill()
		},
	}

	/*addCmd is a subcommand to add the current working directory as a VISIBLE one
	 */
	var addCmd = &cobra.Command{
		Use:   "add",
		Short: "Add the current path as a VISIBLE repository",
		Run: func(cmd *cobra.Command, args []string) {
			// Get the path where the command has been executed
			currentDir, err := os.Getwd()
			if err != nil {
				log.Fatalln("There was a problem retrieving the current directory")
			}
			if !utils.IsGitRepository(currentDir) {
				log.Fatalf("%s is not a git repository!\n", currentDir)
			}
			// If the path is/contains a .git directory, add this one as a VISIBLE repository
			if err := configurationFileStructure.AddRepository(currentDir, consts.VisibleFlag); err != nil {
				traces.WarningTracer.Printf("[%s] %s\n", currentDir, err)
			}
		},
	}

	/*crawlCmd is a subcommand to crawl your hard drive in order to get and save new git repositories
	 */
	var crawlCmd = &cobra.Command{
		Use:   "crawl",
		Short: "Crawl the hard drive in order to find git repositories",
		Run: func(cmd *cobra.Command, args []string) {
			// Get all git paths, and display them
			gitPaths, err := walk.RetrieveGitRepositories(userHomeDir)
			if err != nil {
				log.Fatalf("There was an error retrieving your git repositories: '%s'\n", err)
			}
			// For each git repository, check if it exists, and if not add it to the default target visibility
			for _, gitPath := range gitPaths {
				if err := configurationFileStructure.AddRepository(gitPath, configurationFileStructure.Local.DefaultTarget); err != nil {
					traces.WarningTracer.Printf("[%s] %s\n", gitPath, err)
				}
			}
			// For each VISIBLE repository, get some informations about his state and display it
		},
	}

	/*stateCmd is a subcommand to list the state of each local git repository
	 */
	var stateCmd = &cobra.Command{
		Use:   "state",
		Short: "Get the state of each local git repository",
		Run: func(cmd *cobra.Command, args []string) {
			for _, gitStruct := range configurationFileStructure.VisibleRepositories {
				gitStruct.Init()
				gitStruct.GitObject.Status()
			}
		},
	}

	rootCmd.AddCommand(addCmd, crawlCmd, stateCmd)

	if err := rootCmd.Execute(); err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
}
<commit_msg>Update the developer documentation<commit_after>package main

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path"

	"github.com/BurntSushi/toml"
	"github.com/k0pernicus/goyave/configurationFile"
	"github.com/k0pernicus/goyave/consts"
	"github.com/k0pernicus/goyave/traces"
	"github.com/k0pernicus/goyave/utils"
	"github.com/k0pernicus/goyave/walk"
	"github.com/spf13/cobra"
)

var configurationFileStructure configurationFile.ConfigurationFile
var configurationFilePath string
var userHomeDir string

/*initialize get the configuration file existing in the system (or create it), and return
 *a pointer to his content.
 */
func initialize(configurationFileStructure *configurationFile.ConfigurationFile) {
	// Initialize all different traces structures
	traces.InitTraces(os.Stdout, os.Stderr, os.Stdout, os.Stdout)

	// Get the user home directory
	userHomeDir = utils.GetUserHomeDir()
	if len(userHomeDir) == 0 {
		log.Fatalf("Cannot get the user home dir.\n")
	}
	// Set the configuration path file
	configurationFilePath = path.Join(userHomeDir, consts.ConfigurationFileName)
	filePointer, err := os.OpenFile(configurationFilePath, os.O_RDWR|os.O_CREATE, 0755)
	if err != nil {
		log.Fatalf("Cannot open the file %s, due to error '%s'.\n", configurationFilePath, err)
	}
	defer filePointer.Close()
	var bytesArray []byte
	// Get the content of the goyave configuration file
	configurationFile.GetConfigurationFileContent(filePointer, &bytesArray)
	if _, err = toml.Decode(string(bytesArray[:]), configurationFileStructure); err != nil {
		log.Fatalln(err)
	}
}

/*kill saves the current state of the configuration structure in the configuration file
 */
func kill() {
	var outputBuffer bytes.Buffer
	if err := configurationFileStructure.Encode(&outputBuffer); err != nil {
		log.Fatalln("Cannot save the current configurationFile structure!")
	}
	if err := ioutil.WriteFile(configurationFilePath, outputBuffer.Bytes(), 0777); err != nil {
		log.Fatalln("Cannot access to your file to save the configurationFile structure!")
	}
}

func main() {

	/*rootCmd defines the global app, and some actions to run before and after the command running
	 */
	var rootCmd = &cobra.Command{
		Use:   "goyave",
		Short: "Goyave is a tool to take a look at your local git repositories",
		// Initialize the structure
		PersistentPreRun: func(cmd *cobra.Command, args []string) {
			initialize(&configurationFileStructure)
		},
		// Save the current configuration file structure, in the configuration file
		PersistentPostRun: func(cmd *cobra.Command, args []string) {
			kill()
		},
	}

	/*addCmd is a subcommand to add the current working directory as a VISIBLE one
	 */
	var addCmd = &cobra.Command{
		Use:   "add",
		Short: "Add the current path as a VISIBLE repository",
		Run: func(cmd *cobra.Command, args []string) {
			// Get the path where the command has been executed
			currentDir, err := os.Getwd()
			if err != nil {
				log.Fatalln("There was a problem retrieving the current directory")
			}
			if !utils.IsGitRepository(currentDir) {
				log.Fatalf("%s is not a git repository!\n", currentDir)
			}
			// If the path is/contains a .git directory, add this one as a VISIBLE repository
			if err := configurationFileStructure.AddRepository(currentDir, consts.VisibleFlag); err != nil {
				traces.WarningTracer.Printf("[%s] %s\n", currentDir, err)
			}
		},
	}

	/*crawlCmd is a subcommand to crawl your hard drive in order to get and save new git repositories
	 */
	var crawlCmd = &cobra.Command{
		Use:   "crawl",
		Short: "Crawl the hard drive in order to find git repositories",
		Run: func(cmd *cobra.Command, args []string) {
			// Get all git paths, and display them
			gitPaths, err := walk.RetrieveGitRepositories(userHomeDir)
			if err != nil {
				log.Fatalf("There was an error retrieving your git repositories: '%s'\n", err)
			}
			// For each git repository, check if it exists, and if not add it to the default target visibility
			for _, gitPath := range gitPaths {
				if err := configurationFileStructure.AddRepository(gitPath, configurationFileStructure.Local.DefaultTarget); err != nil {
					traces.WarningTracer.Printf("[%s] %s\n", gitPath, err)
				}
			}
			// For each VISIBLE repository, get some informations about his state and display it
		},
	}

	/*stateCmd is a subcommand to list the state of each local git repository.
	 */
	var stateCmd = &cobra.Command{
		Use:   "state",
		Short: "Get the state of each local git repository",
		Run: func(cmd *cobra.Command, args []string) {
			for _, gitStruct := range configurationFileStructure.VisibleRepositories {
				gitStruct.Init()
				gitStruct.GitObject.Status()
			}
		},
	}

	rootCmd.AddCommand(addCmd, crawlCmd, stateCmd)

	if err := rootCmd.Execute(); err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>package main

import "flag"
import "fmt"
import log "github.com/Sirupsen/logrus"
import "strconv"
import "time"
import "net"
import "os"
import "encoding/json"

//import "github.com/Shopify/sarama"
import "github.com/google/gopacket"
import "github.com/google/gopacket/pcap"
import "github.com/google/gopacket/layers"

/*
Plans:

    code cleanup (e.g. break up handlePacket, switch everything to camelCase)
    stats output
    perf testing
    TCP flow support
release v2

    syslog logging
    logging to kafka
    add PF_RING support
release v3

    maybe use something with a larger keyspace than the query ID for the conntable map
    maybe not so many string conversions?
    add more Types to gopacket
*/

/*

DNS log entry struct and helper functions

*/
type dnsLogEntry struct {
	Query_ID      uint16 `json:"query_id"`
	Response_Code int    `json:"response_code"`
	Question      string `json:"question"`
	Question_Type string `json:"question_type"`
	Answer        string `json:"answer"`
	Answer_Type   string `json:"answer_type"`
	TTL           uint32 `json:"ttl"`
	Server        net.IP `json:"server"`
	Client        net.IP `json:"client"`
	Timestamp     string `json:"timestamp"`

	encoded []byte //to hold the marshaled data structure
	err     error  //encoding errors
}

func (dle *dnsLogEntry) ensureEncoded() {
	if dle.encoded == nil && dle.err == nil {
		dle.encoded, dle.err = json.Marshal(dle)
	}
}

func (dle *dnsLogEntry) Length() int {
	dle.ensureEncoded()
	return len(dle.encoded)
}

func (dle *dnsLogEntry) Encode() ([]byte, error) {
	dle.ensureEncoded()
	return dle.encoded, dle.err
}

type dnsMapEntry struct {
	entry    *layers.DNS
	inserted time.Time
}

//background task to clear out stale entries in the conntable
//one of these gets spun up for every packet handling thread
func cleanDnsCache(conntable *map[uint16]dnsMapEntry, max_age time.Duration, interval time.Duration) {

	for {
		time.Sleep(interval)

		//max_age should be negative, e.g. -1m
		cleanup_cutoff := time.Now().Add(max_age)
		for key, item := range *conntable {
			if item.inserted.Before(cleanup_cutoff) {
				log.Debug("conntable GC: cleanup query ID " + strconv.Itoa(int(key)))
				delete(*conntable, key)
			}
		}
	}
}

/*
    The gopacket DNS layer doesn't have a lot of good String()
    conversion methods, so we have to do a lot of that ourselves
    here.  Much of this should move back into gopacket.  Also a
    little worried about the perf impact of doing string conversions
    in this thread...
*/
func TypeString(dnsType layers.DNSType) string {
    switch dnsType {
        default:
        	//take a blind stab...at least this shouldn't *lose* data
        	return strconv.Itoa(int(dnsType))
        case layers.DNSTypeA:
        	return "A"
        case layers.DNSTypeAAAA:
        	return "AAAA"
        case layers.DNSTypeCNAME:
        	return "CNAME"
        case layers.DNSTypeMX:
        	return "MX"
        case layers.DNSTypeNS:
        	return "NS"
        case layers.DNSTypePTR:
        	return "PTR"
        case layers.DNSTypeTXT:
        	return "TXT"
        case layers.DNSTypeSOA:
        	return "SOA"
        case layers.DNSTypeSRV:
        	return "SRV"
        case 255: //ANY query per http://tools.ietf.org/html/rfc1035#page-12
		    return "ANY"
        }
}

/*
    The gopacket DNS layer doesn't have a lot of good String()
    conversion methods, so we have to do a lot of that ourselves
    here.  Much of this should move back into gopacket.  Also a
    little worried about the perf impact of doing string conversions
    in this thread...
*/
func RrString(rr layers.DNSResourceRecord) string {
    switch rr.Type {
        default:
        	//take a blind stab...at least this shouldn't *lose* data
        	return string(rr.Data)
        case layers.DNSTypeA:
        	return rr.IP.String()
        case layers.DNSTypeAAAA:
        	return rr.IP.String()
        case layers.DNSTypeCNAME:
        	return string(rr.CNAME)
        case layers.DNSTypeMX:
        	//TODO: add the priority
        	return string(rr.MX.Name)
        case layers.DNSTypeNS:
        	return string(rr.NS)
        case layers.DNSTypePTR:
        	return string(rr.PTR)
        case layers.DNSTypeTXT:
        	return string(rr.TXT)
        case layers.DNSTypeSOA:
        	//TODO: rebuild the full SOA string
        	return string(rr.SOA.RName)
        case layers.DNSTypeSRV:
        	//TODO: rebuild the full SRV string
        	return string(rr.SRV.Name)
        }
}

/* validate if DNS, make conntable entry and output
   to log channel if there is a match
*/
func handlePacket(packets chan gopacket.Packet, logC chan dnsLogEntry,
	gc_interval time.Duration, gc_age time.Duration) {

	//DNS IDs are stored as uint16s by the gopacket DNS layer
	//TODO: fix the memory leak of failed lookups by making this a ttlcache
	var conntable = make(map[uint16]dnsMapEntry)

	//setup garbage collection for this map
	go cleanDnsCache(&conntable, gc_age, gc_interval)

	for packet := range packets {
		//TODO: there must be a better way of doing this with gopacket
		var srcIP net.IP
		var dstIP net.IP

		if ipLayer := packet.Layer(layers.LayerTypeIPv4); ipLayer != nil {
			ipData, _ := ipLayer.(*layers.IPv4)
			srcIP = ipData.SrcIP
			dstIP = ipData.DstIP
		} else if ipLayer := packet.Layer(layers.LayerTypeIPv6); ipLayer != nil {
			ipData, _ := ipLayer.(*layers.IPv6)
			srcIP = ipData.SrcIP
			dstIP = ipData.DstIP
		} else {
			//non-IP transport?  Ignore this packet
			log.Debug("Got non-IP packet: " + packet.String())
			continue
		}

		if dnsLayer := packet.Layer(layers.LayerTypeDNS); dnsLayer != nil {
			// Get actual DNS data from this layer
			dns, _ := dnsLayer.(*layers.DNS)

			//skip non-query stuff (Updates, AXFRs, etc)
			if dns.OpCode != layers.DNSOpCodeQuery {
				log.Debug("Saw non-update DNS packet: " + packet.String())
				continue
			}

            item, found_item := conntable[dns.ID];

			//this is a Query Response packet
			if dns.QR && found_item {
				question := item.entry
				//We have both legs of the connection, so drop the connection from the table
				log.Debug("Got 'answer' leg of query ID: " + strconv.Itoa(int(question.ID)))
				delete(conntable, question.ID)

				/*
				   http://forums.devshed.com/dns-36/dns-packet-question-section-1-a-183026.html
				   multiple questions isn't really a thing, so we'll loop over the answers and
				   insert the question section from the original query.  This means a successful
				   ANY query may result in a lot of seperate log entries.  The query ID will be
				   the same on all of those entries, however, so you can rebuild the query that
				   way.

				   TODO: Also loop through Additional records in addition to Answers
				*/

                questionType := TypeString(question.Questions[0].Type)

				//a response code other than 0 means failure of some kind
				if dns.ResponseCode != 0 {

					logEntry := dnsLogEntry{
						Query_ID:      dns.ID,
						Question:      string(question.Questions[0].Name),
						Response_Code: int(dns.ResponseCode),
						Question_Type: questionType,
						Answer:        dns.ResponseCode.String(),
						Answer_Type:   "",
						TTL:           0,
						//this is the answer packet, which comes from the server...
						Server: srcIP,
						//...and goes to the client
						Client:    dstIP,
						Timestamp: time.Now().UTC().Format(time.RFC3339),
					}

					logC <- logEntry

					continue
				}

				for _, answer := range dns.Answers {

                    answerString := RrString(answer)
                    typeString := TypeString(answer.Type)

					logEntry := dnsLogEntry{
						Query_ID:      dns.ID,
						Question:      string(question.Questions[0].Name),
						Response_Code: int(dns.ResponseCode),
						Question_Type: questionType,
						Answer:        answerString,
						Answer_Type:   typeString,
						TTL:           answer.TTL,
						//this is the answer packet, which comes from the server...
						Server: srcIP,
						//...and goes to the client
						Client:    dstIP,
						Timestamp: time.Now().UTC().Format(time.RFC3339),
					}

					logC <- logEntry
				}

			} else if dns.QR && !found_item {
				//This might happen if we get a query ID collision
				log.Debug("Got a Query Response and can't find a query for ID " + strconv.Itoa(int(dns.ID)))
				continue
			} else {
				//This is the initial query.  save it for later.
				log.Debug("Got the 'question' leg of query ID " + strconv.Itoa(int(dns.ID)))
				mapEntry := dnsMapEntry{
					entry:    dns,
					inserted: time.Now(),
				}
				conntable[dns.ID] = mapEntry
			}
		}
	}
}

//Round-robin log messages to log sinks
func logConn(logC chan dnsLogEntry, stdout bool, file bool, kafka bool,
	filename string, kafka_brokers string, kafka_topic string) {

	var logs []chan dnsLogEntry

	if stdout {
		log.Debug("STDOUT logging enabled")
		stdoutChan := make(chan dnsLogEntry)
		logs = append(logs, stdoutChan)
		go logConnStdout(stdoutChan)
	}

	if file {
		log.Debug("file logging enabled to " + filename)
		fileChan := make(chan dnsLogEntry)
		logs = append(logs, fileChan)
		go logConnFile(fileChan, filename)
	}

	if kafka && false {
		log.Debug("kafka logging enabled")
		kafkaChan := make(chan dnsLogEntry)
		logs = append(logs, kafkaChan)
		go logConnKafka(kafkaChan, kafka_brokers, kafka_topic)
	}

	//setup is done, now we sit here and dispatch messages to the configured sinks
	for message := range logC {
		for _, logChan := range logs {
			logChan <- message
		}
	}
}

func logConnStdout(logC chan dnsLogEntry) {
	for message := range logC {
		encoded, _ := message.Encode()
		fmt.Println(string(encoded))
	}
}

func logConnFile(logC chan dnsLogEntry, filename string) {

	f, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0666)
	if err != nil {
		log.Debug("could not open logfile for writing!")
		panic(err)
	}

	defer f.Close()

	for message := range logC {
		encoded, _ := message.Encode()
		f.WriteString(string(encoded) + "\n")
	}
}

func logConnKafka(logC chan dnsLogEntry, kafka_brokers string, kafka_topic string) {
	for message := range logC {
		//marshal to JSON.  Maybe we should do this in the log thread?
		encoded, _ := message.Encode()
		fmt.Println("Kafka: " + string(encoded))
	}
}

func main() {

	var dev = flag.String("dev", "", "Capture Device")
	var kafka_brokers = flag.String("kafka_brokers", os.Getenv("KAFKA_PEERS"), "The Kafka brokers to connect to, as a comma separated list")
	var kafka_topic = flag.String("kafka_topic", "", "Kafka topic for output")
	var bpf = flag.String("bpf", "port 53", "BPF Filter")
	var pcapFile = flag.String("pcap", "", "pcap file")
	var logfile = flag.String("logfile", "", "log file (recommended for debug only")
	var quiet = flag.Bool("quiet", false, "do not log to stdout")
	var gc_age = flag.String("gc_age", "-1m", "How old a connection table entry should be before it is garbage collected.")
	var gc_interval = flag.String("gc_interval", "3m", "How often to run garbage collection.")
	var debug = flag.Bool("debug", false, "Enable debug logging")

	flag.Parse()

	var handle *pcap.Handle
	var err error

	if *dev != "" {
		handle, err = pcap.OpenLive(*dev, 65536, true, pcap.BlockForever)
		if err != nil {
			log.Fatal(err)
		}
	} else if *pcapFile != "" {
		handle, err = pcap.OpenOffline(*pcapFile)
		if err != nil {
			log.Fatal(err)
		}
	} else {
		log.Fatal("You must specify either a capture device or a pcap file")
	}

	defer handle.Close()

	err = handle.SetBPFFilter(*bpf)
	if err != nil {
		log.Fatal(err)
	}

	if *debug {
		log.SetLevel(log.DebugLevel)
	}

	/* spin up logging thread */
	var logChan = make(chan dnsLogEntry)
	var log_file bool = false
	var log_kafka bool = false

	if *logfile != "" {
		log_file = true
	}

	if *kafka_brokers != "" && *kafka_topic != "" {
		log_kafka = true
	}

	var gc_age_dur time.Duration
	var gc_interval_dur time.Duration

	if gc_age_tmp, err := time.ParseDuration(*gc_age); err != nil {
		log.Fatal("Your gc_age parameter was not parseable.  Use a string like '-1m'")
	} else {
		gc_age_dur = gc_age_tmp
	}

	if gc_interval_tmp, err := time.ParseDuration(*gc_interval); err != nil {
		log.Fatal("Your gc_age parameter was not parseable.  Use a string like '3m'")
	} else {
		gc_interval_dur = gc_interval_tmp
	}

	go logConn(logChan, !*quiet, log_file, log_kafka, *logfile, *kafka_brokers, *kafka_topic)

	/* init channels for the packet handlers and kick off handler threads */
	var channels [8]chan gopacket.Packet
	for i := 0; i < 8; i++ {
		channels[i] = make(chan gopacket.Packet)
		go handlePacket(channels[i], logChan, gc_interval_dur, gc_age_dur)
	}

	// Use the handle as a packet source to process all packets
	packetSource := gopacket.NewPacketSource(handle, handle.LinkType())
	for packet := range packetSource.Packets() {
		// Dispatch packets here
		if net := packet.NetworkLayer(); net != nil {
			/*  load balance the processiing over 8 threads
			    FashHash is consistant for A->B and B->A hashes, which simplifies
			    our connection tracking problem a bit by letting us keep
			    per-worker connection pools instead of a global pool.
			*/
			channels[int(net.NetworkFlow().FastHash())&0x7] <- packet
		}
	}
}
<commit_msg>refactor of some of the logging code<commit_after>package main

import "flag"
import "fmt"
import log "github.com/Sirupsen/logrus"
import "strconv"
import "time"
import "net"
import "os"
import "encoding/json"

//import "github.com/Shopify/sarama"
import "github.com/google/gopacket"
import "github.com/google/gopacket/pcap"
import "github.com/google/gopacket/layers"

/*
Plans:

    code cleanup (e.g. break up handlePacket, switch everything to camelCase)
    stats output
    perf testing
    TCP flow support
release v2

    syslog logging
    logging to kafka
    add PF_RING support
release v3

    maybe use something with a larger keyspace than the query ID for the conntable map
    maybe not so many string conversions?
    add more Types to gopacket
*/

/*

DNS log entry struct and helper functions

*/
type dnsLogEntry struct {
	Query_ID      uint16 `json:"query_id"`
	Response_Code int    `json:"response_code"`
	Question      string `json:"question"`
	Question_Type string `json:"question_type"`
	Answer        string `json:"answer"`
	Answer_Type   string `json:"answer_type"`
	TTL           uint32 `json:"ttl"`
	Server        net.IP `json:"server"`
	Client        net.IP `json:"client"`
	Timestamp     string `json:"timestamp"`

	encoded []byte //to hold the marshaled data structure
	err     error  //encoding errors
}

func (dle *dnsLogEntry) ensureEncoded() {
	if dle.encoded == nil && dle.err == nil {
		dle.encoded, dle.err = json.Marshal(dle)
	}
}

func (dle *dnsLogEntry) Length() int {
	dle.ensureEncoded()
	return len(dle.encoded)
}

func (dle *dnsLogEntry) Encode() ([]byte, error) {
	dle.ensureEncoded()
	return dle.encoded, dle.err
}

type dnsMapEntry struct {
	entry    *layers.DNS
	inserted time.Time
}

//background task to clear out stale entries in the conntable
//one of these gets spun up for every packet handling thread
func cleanDnsCache(conntable *map[uint16]dnsMapEntry, max_age time.Duration, interval time.Duration) {

	for {
		time.Sleep(interval)

		//max_age should be negative, e.g. -1m
		cleanup_cutoff := time.Now().Add(max_age)
		for key, item := range *conntable {
			if item.inserted.Before(cleanup_cutoff) {
				log.Debug("conntable GC: cleanup query ID " + strconv.Itoa(int(key)))
				delete(*conntable, key)
			}
		}
	}
}

/*
    The gopacket DNS layer doesn't have a lot of good String()
    conversion methods, so we have to do a lot of that ourselves
    here.  Much of this should move back into gopacket.  Also a
    little worried about the perf impact of doing string conversions
    in this thread...
*/
func TypeString(dnsType layers.DNSType) string {
    switch dnsType {
        default:
        	//take a blind stab...at least this shouldn't *lose* data
        	return strconv.Itoa(int(dnsType))
        case layers.DNSTypeA:
        	return "A"
        case layers.DNSTypeAAAA:
        	return "AAAA"
        case layers.DNSTypeCNAME:
        	return "CNAME"
        case layers.DNSTypeMX:
        	return "MX"
        case layers.DNSTypeNS:
        	return "NS"
        case layers.DNSTypePTR:
        	return "PTR"
        case layers.DNSTypeTXT:
        	return "TXT"
        case layers.DNSTypeSOA:
        	return "SOA"
        case layers.DNSTypeSRV:
        	return "SRV"
        case 255: //ANY query per http://tools.ietf.org/html/rfc1035#page-12
		    return "ANY"
        }
}

/*
    The gopacket DNS layer doesn't have a lot of good String()
    conversion methods, so we have to do a lot of that ourselves
    here.  Much of this should move back into gopacket.  Also a
    little worried about the perf impact of doing string conversions
    in this thread...
*/
func RrString(rr layers.DNSResourceRecord) string {
    switch rr.Type {
        default:
        	//take a blind stab...at least this shouldn't *lose* data
        	return string(rr.Data)
        case layers.DNSTypeA:
        	return rr.IP.String()
        case layers.DNSTypeAAAA:
        	return rr.IP.String()
        case layers.DNSTypeCNAME:
        	return string(rr.CNAME)
        case layers.DNSTypeMX:
        	//TODO: add the priority
        	return string(rr.MX.Name)
        case layers.DNSTypeNS:
        	return string(rr.NS)
        case layers.DNSTypePTR:
        	return string(rr.PTR)
        case layers.DNSTypeTXT:
        	return string(rr.TXT)
        case layers.DNSTypeSOA:
        	//TODO: rebuild the full SOA string
        	return string(rr.SOA.RName)
        case layers.DNSTypeSRV:
        	//TODO: rebuild the full SRV string
        	return string(rr.SRV.Name)
        }
}

/* validate if DNS, make conntable entry and output
   to log channel if there is a match
*/
func handlePacket(packets chan gopacket.Packet, logC chan dnsLogEntry,
	gc_interval time.Duration, gc_age time.Duration) {

	//DNS IDs are stored as uint16s by the gopacket DNS layer
	//TODO: fix the memory leak of failed lookups by making this a ttlcache
	var conntable = make(map[uint16]dnsMapEntry)

	//setup garbage collection for this map
	go cleanDnsCache(&conntable, gc_age, gc_interval)

	for packet := range packets {
		//TODO: there must be a better way of doing this with gopacket
		var srcIP net.IP
		var dstIP net.IP

		if ipLayer := packet.Layer(layers.LayerTypeIPv4); ipLayer != nil {
			ipData, _ := ipLayer.(*layers.IPv4)
			srcIP = ipData.SrcIP
			dstIP = ipData.DstIP
		} else if ipLayer := packet.Layer(layers.LayerTypeIPv6); ipLayer != nil {
			ipData, _ := ipLayer.(*layers.IPv6)
			srcIP = ipData.SrcIP
			dstIP = ipData.DstIP
		} else {
			//non-IP transport?  Ignore this packet
			log.Debug("Got non-IP packet: " + packet.String())
			continue
		}

		if dnsLayer := packet.Layer(layers.LayerTypeDNS); dnsLayer != nil {
			// Get actual DNS data from this layer
			dns, _ := dnsLayer.(*layers.DNS)

			//skip non-query stuff (Updates, AXFRs, etc)
			if dns.OpCode != layers.DNSOpCodeQuery {
				log.Debug("Saw non-update DNS packet: " + packet.String())
				continue
			}

            item, found_item := conntable[dns.ID];

			//this is a Query Response packet
			if dns.QR && found_item {
				question := item.entry
				//We have both legs of the connection, so drop the connection from the table
				log.Debug("Got 'answer' leg of query ID: " + strconv.Itoa(int(question.ID)))
				delete(conntable, question.ID)

				/*
				   http://forums.devshed.com/dns-36/dns-packet-question-section-1-a-183026.html
				   multiple questions isn't really a thing, so we'll loop over the answers and
				   insert the question section from the original query.  This means a successful
				   ANY query may result in a lot of seperate log entries.  The query ID will be
				   the same on all of those entries, however, so you can rebuild the query that
				   way.

				   TODO: Also loop through Additional records in addition to Answers
				*/

				//a response code other than 0 means failure of some kind
				if dns.ResponseCode != 0 {

					logEntry := dnsLogEntry{
						Query_ID:      dns.ID,
						Question:      string(question.Questions[0].Name),
						Response_Code: int(dns.ResponseCode),
						Question_Type: TypeString(question.Questions[0].Type),
						Answer:        dns.ResponseCode.String(),
						Answer_Type:   "",
						TTL:           0,
						//this is the answer packet, which comes from the server...
						Server: srcIP,
						//...and goes to the client
						Client:    dstIP,
						Timestamp: time.Now().UTC().Format(time.RFC3339),
					}

					logC <- logEntry

					continue
				}

				for _, answer := range dns.Answers {

					logEntry := dnsLogEntry{
						Query_ID:      dns.ID,
						Question:      string(question.Questions[0].Name),
						Response_Code: int(dns.ResponseCode),
						Question_Type: TypeString(question.Questions[0].Type),
						Answer:        RrString(answer),
						Answer_Type:   TypeString(answer.Type),
						TTL:           answer.TTL,
						//this is the answer packet, which comes from the server...
						Server: srcIP,
						//...and goes to the client
						Client:    dstIP,
						Timestamp: time.Now().UTC().Format(time.RFC3339),
					}

					logC <- logEntry
				}

			} else if dns.QR && !found_item {
				//This might happen if we get a query ID collision
				log.Debug("Got a Query Response and can't find a query for ID " + strconv.Itoa(int(dns.ID)))
				continue
			} else {
				//This is the initial query.  save it for later.
				log.Debug("Got the 'question' leg of query ID " + strconv.Itoa(int(dns.ID)))
				mapEntry := dnsMapEntry{
					entry:    dns,
					inserted: time.Now(),
				}
				conntable[dns.ID] = mapEntry
			}
		}
	}
}

//Round-robin log messages to log sinks
func logConn(logC chan dnsLogEntry, stdout bool,
	filename string, kafka_brokers string, kafka_topic string) {

	var logs []chan dnsLogEntry

	if stdout {
		log.Debug("STDOUT logging enabled")
		stdoutChan := make(chan dnsLogEntry)
		logs = append(logs, stdoutChan)
		go logConnStdout(stdoutChan)
	}

	if filename != "" {
		log.Debug("file logging enabled to " + filename)
		fileChan := make(chan dnsLogEntry)
		logs = append(logs, fileChan)
		go logConnFile(fileChan, filename)
	}

	if kafka_brokers != "" && kafka_topic != "" && false {
		log.Debug("kafka logging enabled")
		kafkaChan := make(chan dnsLogEntry)
		logs = append(logs, kafkaChan)
		go logConnKafka(kafkaChan, kafka_brokers, kafka_topic)
	}

	//setup is done, now we sit here and dispatch messages to the configured sinks
	for message := range logC {
		for _, logChan := range logs {
			logChan <- message
		}
	}
}

func logConnStdout(logC chan dnsLogEntry) {
	for message := range logC {
		encoded, _ := message.Encode()
		fmt.Println(string(encoded))
	}
}

func logConnFile(logC chan dnsLogEntry, filename string) {

	f, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0666)
	if err != nil {
		log.Debug("could not open logfile for writing!")
		panic(err)
	}

	defer f.Close()

	for message := range logC {
		encoded, _ := message.Encode()
		f.WriteString(string(encoded) + "\n")
	}
}

func logConnKafka(logC chan dnsLogEntry, kafka_brokers string, kafka_topic string) {
	for message := range logC {
		//marshal to JSON.  Maybe we should do this in the log thread?
		encoded, _ := message.Encode()
		fmt.Println("Kafka: " + string(encoded))
	}
}

func main() {

	var dev = flag.String("dev", "", "Capture Device")
	var kafka_brokers = flag.String("kafka_brokers", os.Getenv("KAFKA_PEERS"), "The Kafka brokers to connect to, as a comma separated list")
	var kafka_topic = flag.String("kafka_topic", "", "Kafka topic for output")
	var bpf = flag.String("bpf", "port 53", "BPF Filter")
	var pcapFile = flag.String("pcap", "", "pcap file")
	var logfile = flag.String("logfile", "", "log file (recommended for debug only")
	var quiet = flag.Bool("quiet", false, "do not log to stdout")
	var gc_age = flag.String("gc_age", "-1m", "How old a connection table entry should be before it is garbage collected.")
	var gc_interval = flag.String("gc_interval", "3m", "How often to run garbage collection.")
	var debug = flag.Bool("debug", false, "Enable debug logging")

	flag.Parse()

	var handle *pcap.Handle
	var err error

	if *dev != "" {
		handle, err = pcap.OpenLive(*dev, 65536, true, pcap.BlockForever)
		if err != nil {
			log.Fatal(err)
		}
	} else if *pcapFile != "" {
		handle, err = pcap.OpenOffline(*pcapFile)
		if err != nil {
			log.Fatal(err)
		}
	} else {
		log.Fatal("You must specify either a capture device or a pcap file")
	}

	defer handle.Close()

	err = handle.SetBPFFilter(*bpf)
	if err != nil {
		log.Fatal(err)
	}

	if *debug {
		log.SetLevel(log.DebugLevel)
	}

	/* spin up logging thread */
	var logChan = make(chan dnsLogEntry)

	gc_age_dur, err := time.ParseDuration(*gc_age)

	if err != nil {
		log.Fatal("Your gc_age parameter was not parseable.  Use a string like '-1m'")
	}

	gc_interval_dur, err := time.ParseDuration(*gc_interval)

	if err != nil {
		log.Fatal("Your gc_age parameter was not parseable.  Use a string like '3m'")
	}

	go logConn(logChan, !*quiet, *logfile, *kafka_brokers, *kafka_topic)

	/* init channels for the packet handlers and kick off handler threads */
	var channels [8]chan gopacket.Packet
	for i := 0; i < 8; i++ {
		channels[i] = make(chan gopacket.Packet)
		go handlePacket(channels[i], logChan, gc_interval_dur, gc_age_dur)
	}

	// Use the handle as a packet source to process all packets
	packetSource := gopacket.NewPacketSource(handle, handle.LinkType())
	for packet := range packetSource.Packets() {
		// Dispatch packets here
		if net := packet.NetworkLayer(); net != nil {
			/*  load balance the processiing over 8 threads
			    FashHash is consistant for A->B and B->A hashes, which simplifies
			    our connection tracking problem a bit by letting us keep
			    per-worker connection pools instead of a global pool.
			*/
			channels[int(net.NetworkFlow().FastHash())&0x7] <- packet
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"strings"
	"time"

	"github.com/PonyvilleFM/aura/pvfm"
	"github.com/PonyvilleFM/aura/pvfm/pvl"
	pvfmschedule "github.com/PonyvilleFM/aura/pvfm/schedule"
	"github.com/PonyvilleFM/aura/pvfm/station"
	"github.com/bwmarrin/discordgo"
	"github.com/tebeka/strftime"
)

func pesterLink(s *discordgo.Session, m *discordgo.MessageCreate) {
	if musicLinkRegex.Match([]byte(m.Content)) {
		i, err := pvfm.GetStats()
		if err != nil {
			log.Println(err)
			return
		}

		if i.IsDJLive() && m.ChannelID == youtubeSpamRoomID {
			s.ChannelMessageSend(m.ChannelID, "Please be mindful sharing links to music when a DJ is performing. Thanks!")
		}
	}
}

func np(s *discordgo.Session, m *discordgo.Message, parv []string) error {
	i, err := pvfm.GetStats()
	if err != nil {
		log.Printf("Can't get info: %v, failing over to plan b", err)
		return doStationRequest(s, m, parv)
	}

	result := []string{}

	if i.Main.Nowplaying == "Fetching info..." {
		log.Println("Main information was bad, fetching from station directly...")

		err := doStationRequest(s, m, parv)
		if err != nil {
			return err
		}

		return nil
	} else {
		result = append(result, fmt.Sprintf(
			"Now playing: %s on Ponyville FM's main stream!\n",
			i.Main.Nowplaying,
		))
		result = append(result, fmt.Sprintf(
			"Now playing: %s on PVFM Chill!\n",
			i.Secondary.Nowplaying,
		))
		result = append(result, fmt.Sprintf(
			"Now playing: %s on PVFM Free!",
			i.MusicOnly.Nowplaying,
		))
	}

	s.ChannelMessageSend(m.ChannelID, strings.Join(result, "\n"))
	return nil
}

func dj(s *discordgo.Session, m *discordgo.Message, parv []string) error {
	cal, err := pvl.Get()
	if err != nil {
		return err
	}

	now := cal.Result[0]
	result := []string{}

	localTime := time.Now()
	thentime := time.Unix(now.StartTime, 0)
	if thentime.Unix() < localTime.Unix() {
		result = append(result, fmt.Sprintf("Currently live: %s\n", now.Title))
		now = cal.Result[1]
	}

	nowTime := time.Unix(now.StartTime, 0)
	zone, offset := nowTime.Zone()
	fmttime, _ := strftime.Format("%Y-%m-%d %H:%M:%S", nowTime)
	offset = offset / 60 / 60

	result = append(result, fmt.Sprintf("Next event: %s at %s \x02%s\x02 (%d)",
		now.Title,
		fmttime,
		zone, offset,
	))

	s.ChannelMessageSend(m.ChannelID, strings.Join(result, "\n"))
	return nil
}

func stats(s *discordgo.Session, m *discordgo.Message, parv []string) error {
	i, err := pvfm.GetStats()
	if err != nil {
		log.Printf("Error getting the station info: %v, falling back to plan b", err)
		return doStatsFromStation(s, m, parv)
	}

	st, err := station.GetStats()
	if err != nil {
		return err
	}

	var l int
	var peak int

	for _, source := range st.Icestats.Source {
		l = l + source.Listeners
		peak = peak + source.ListenerPeak
	}

	result := []string{
		fmt.Sprintf(
			"Current listeners across all streams: %d with a maximum of %d!",
			i.Listeners.Listeners, peak,
		),
		fmt.Sprintf(
			"Detailed: Main: %d listeners, Two: %d listeners, Free: %d listeners",
			i.Main.Listeners, i.Secondary.Listeners, i.MusicOnly.Listeners,
		),
	}

	s.ChannelMessageSend(m.ChannelID, strings.Join(result, "\n"))

	return nil
}

func schedule(s *discordgo.Session, m *discordgo.Message, parv []string) error {
	result := []string{}
	schEntries, err := pvfmschedule.Get()
	if err != nil {
		return err
	}

	for _, entry := range schEntries {
		result = append(result, entry.String())
	}

	s.ChannelMessageSend(m.ChannelID, strings.Join(result, "\n"))
	return nil
}

func doStationRequest(s *discordgo.Session, m *discordgo.Message, parv []string) error {
	stats, err := station.GetStats()
	if err != nil {
		return err
	}

	result := fmt.Sprintf(
		"Now playing: %s - %s on Ponyville FM!",
		stats.Icestats.Source[0].Title,
		stats.Icestats.Source[0].Artist,
	)

	s.ChannelMessageSend(m.ChannelID, result)
	return nil
}

func doStatsFromStation(s *discordgo.Session, m *discordgo.Message, parv []string) error {
	st, err := station.GetStats()
	if err != nil {
		return err
	}

	var l int
	var peak int

	for _, source := range st.Icestats.Source {
		l = l + source.Listeners
		peak = peak + source.ListenerPeak
	}

	result := []string{
		fmt.Sprintf("Current listeners: %d with a maximum of %d!", l, peak),
	}

	s.ChannelMessageSend(m.ChannelID, strings.Join(result, "\n"))
	return nil
}

func curTime(s *discordgo.Session, m *discordgo.Message, parv []string) error {
	s.ChannelMessageSend(m.ChannelID, fmt.Sprintf("The time currently is %s", time.Now().String()))

	return nil
}
<commit_msg>time command: Change to friendlier format, force UTC<commit_after>package main

import (
	"fmt"
	"log"
	"strings"
	"time"

	"github.com/PonyvilleFM/aura/pvfm"
	"github.com/PonyvilleFM/aura/pvfm/pvl"
	pvfmschedule "github.com/PonyvilleFM/aura/pvfm/schedule"
	"github.com/PonyvilleFM/aura/pvfm/station"
	"github.com/bwmarrin/discordgo"
	"github.com/tebeka/strftime"
)

func pesterLink(s *discordgo.Session, m *discordgo.MessageCreate) {
	if musicLinkRegex.Match([]byte(m.Content)) {
		i, err := pvfm.GetStats()
		if err != nil {
			log.Println(err)
			return
		}

		if i.IsDJLive() && m.ChannelID == youtubeSpamRoomID {
			s.ChannelMessageSend(m.ChannelID, "Please be mindful sharing links to music when a DJ is performing. Thanks!")
		}
	}
}

func np(s *discordgo.Session, m *discordgo.Message, parv []string) error {
	i, err := pvfm.GetStats()
	if err != nil {
		log.Printf("Can't get info: %v, failing over to plan b", err)
		return doStationRequest(s, m, parv)
	}

	result := []string{}

	if i.Main.Nowplaying == "Fetching info..." {
		log.Println("Main information was bad, fetching from station directly...")

		err := doStationRequest(s, m, parv)
		if err != nil {
			return err
		}

		return nil
	} else {
		result = append(result, fmt.Sprintf(
			"Now playing: %s on Ponyville FM's main stream!\n",
			i.Main.Nowplaying,
		))
		result = append(result, fmt.Sprintf(
			"Now playing: %s on PVFM Chill!\n",
			i.Secondary.Nowplaying,
		))
		result = append(result, fmt.Sprintf(
			"Now playing: %s on PVFM Free!",
			i.MusicOnly.Nowplaying,
		))
	}

	s.ChannelMessageSend(m.ChannelID, strings.Join(result, "\n"))
	return nil
}

func dj(s *discordgo.Session, m *discordgo.Message, parv []string) error {
	cal, err := pvl.Get()
	if err != nil {
		return err
	}

	now := cal.Result[0]
	result := []string{}

	localTime := time.Now()
	thentime := time.Unix(now.StartTime, 0)
	if thentime.Unix() < localTime.Unix() {
		result = append(result, fmt.Sprintf("Currently live: %s\n", now.Title))
		now = cal.Result[1]
	}

	nowTime := time.Unix(now.StartTime, 0)
	zone, offset := nowTime.Zone()
	fmttime, _ := strftime.Format("%Y-%m-%d %H:%M:%S", nowTime)
	offset = offset / 60 / 60

	result = append(result, fmt.Sprintf("Next event: %s at %s \x02%s\x02 (%d)",
		now.Title,
		fmttime,
		zone, offset,
	))

	s.ChannelMessageSend(m.ChannelID, strings.Join(result, "\n"))
	return nil
}

func stats(s *discordgo.Session, m *discordgo.Message, parv []string) error {
	i, err := pvfm.GetStats()
	if err != nil {
		log.Printf("Error getting the station info: %v, falling back to plan b", err)
		return doStatsFromStation(s, m, parv)
	}

	st, err := station.GetStats()
	if err != nil {
		return err
	}

	var l int
	var peak int

	for _, source := range st.Icestats.Source {
		l = l + source.Listeners
		peak = peak + source.ListenerPeak
	}

	result := []string{
		fmt.Sprintf(
			"Current listeners across all streams: %d with a maximum of %d!",
			i.Listeners.Listeners, peak,
		),
		fmt.Sprintf(
			"Detailed: Main: %d listeners, Two: %d listeners, Free: %d listeners",
			i.Main.Listeners, i.Secondary.Listeners, i.MusicOnly.Listeners,
		),
	}

	s.ChannelMessageSend(m.ChannelID, strings.Join(result, "\n"))

	return nil
}

func schedule(s *discordgo.Session, m *discordgo.Message, parv []string) error {
	result := []string{}
	schEntries, err := pvfmschedule.Get()
	if err != nil {
		return err
	}

	for _, entry := range schEntries {
		result = append(result, entry.String())
	}

	s.ChannelMessageSend(m.ChannelID, strings.Join(result, "\n"))
	return nil
}

func doStationRequest(s *discordgo.Session, m *discordgo.Message, parv []string) error {
	stats, err := station.GetStats()
	if err != nil {
		return err
	}

	result := fmt.Sprintf(
		"Now playing: %s - %s on Ponyville FM!",
		stats.Icestats.Source[0].Title,
		stats.Icestats.Source[0].Artist,
	)

	s.ChannelMessageSend(m.ChannelID, result)
	return nil
}

func doStatsFromStation(s *discordgo.Session, m *discordgo.Message, parv []string) error {
	st, err := station.GetStats()
	if err != nil {
		return err
	}

	var l int
	var peak int

	for _, source := range st.Icestats.Source {
		l = l + source.Listeners
		peak = peak + source.ListenerPeak
	}

	result := []string{
		fmt.Sprintf("Current listeners: %d with a maximum of %d!", l, peak),
	}

	s.ChannelMessageSend(m.ChannelID, strings.Join(result, "\n"))
	return nil
}

func curTime(s *discordgo.Session, m *discordgo.Message, parv []string) error {
	s.ChannelMessageSend(m.ChannelID, fmt.Sprintf("The time currently is %s", time.Now().UTC().Format("2006-01-02 15:04:05 UTC")))

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/signal"
	"runtime/pprof"
	"strings"

	"gopkg.in/alecthomas/kingpin.v3-unstable"

	"github.com/alecthomas/chroma"
	"github.com/alecthomas/chroma/formatters"
	"github.com/alecthomas/chroma/formatters/html"
	"github.com/alecthomas/chroma/lexers"
	"github.com/alecthomas/chroma/styles"
)

var (
	profileFlag = kingpin.Flag("profile", "Enable profiling to file.").Hidden().String()
	listFlag    = kingpin.Flag("list", "List lexers, styles and formatters.").Bool()

	lexerFlag     = kingpin.Flag("lexer", "Lexer to use when formatting.").PlaceHolder("autodetect").Short('l').Enum(lexers.Names(true)...)
	styleFlag     = kingpin.Flag("style", "Style to use for formatting.").Short('s').Default("swapoff").Enum(styles.Names()...)
	formatterFlag = kingpin.Flag("formatter", "Formatter to use.").Default("terminal").Short('f').Enum(formatters.Names()...)

	htmlFlag            = kingpin.Flag("html", "Enable HTML mode (equivalent to '--formatter html').").Bool()
	htmlPrefixFlag      = kingpin.Flag("html-prefix", "HTML CSS class prefix.").PlaceHolder("PREFIX").String()
	htmlStylesFlag      = kingpin.Flag("html-styles", "Output HTML CSS styles.").Bool()
	htmlOnlyFlag        = kingpin.Flag("html-only", "Output HTML fragment.").Bool()
	htmlInlineStyleFlag = kingpin.Flag("html-inline-styles", "Output HTML with inline styles (no classes).").Bool()

	filesArgs = kingpin.Arg("files", "Files to highlight.").ExistingFiles()
)

func main() {
	kingpin.CommandLine.Help = `
Chroma is a general purpose syntax highlighting library and corresponding
command, for Go.
`
	kingpin.Parse()
	if *listFlag {
		listAll()
		return
	}
	if *profileFlag != "" {
		f, err := os.Create(*profileFlag)
		kingpin.FatalIfError(err, "")
		pprof.StartCPUProfile(f)
		signals := make(chan os.Signal, 1)
		signal.Notify(signals, os.Interrupt)
		go func() {
			<-signals
			pprof.StopCPUProfile()
			os.Exit(128 + 3)
		}()
		defer pprof.StopCPUProfile()
	}
	w := bufio.NewWriterSize(os.Stdout, 16384)
	defer w.Flush()
	if *htmlFlag {
		*formatterFlag = "html"
	}
	if *formatterFlag == "html" {
		options := []html.Option{}
		if *htmlPrefixFlag != "" {
			options = append(options, html.ClassPrefix(*htmlPrefixFlag))
		}
		if !*htmlOnlyFlag {
			options = append(options, html.Standalone())
		}
		if !*htmlInlineStyleFlag {
			options = append(options, html.WithClasses())
		}
		if *htmlStylesFlag {
			formatter := html.New(html.WithClasses())
			formatter.WriteCSS(w, styles.Get(*styleFlag))
			return
		}
		formatters.Register("html", html.New(options...))
	}
	writer := getWriter(w)
	if len(*filesArgs) == 0 {
		contents, err := ioutil.ReadAll(os.Stdin)
		kingpin.FatalIfError(err, "")
		lex("", string(contents), writer)
	} else {
		for _, filename := range *filesArgs {
			contents, err := ioutil.ReadFile(filename)
			kingpin.FatalIfError(err, "")
			lex(filename, string(contents), writer)
		}
	}
}

func listAll() {
	fmt.Println("lexers:")
	for _, l := range lexers.Registry.Lexers {
		config := l.Config()
		fmt.Printf("  %s\n", config.Name)
		filenames := []string{}
		filenames = append(filenames, config.Filenames...)
		filenames = append(filenames, config.AliasFilenames...)
		fmt.Printf("    aliases: %s\n", strings.Join(config.Aliases, " "))
		fmt.Printf("    filenames: %s\n", strings.Join(filenames, " "))
		fmt.Printf("    mimetypes: %s\n", strings.Join(config.MimeTypes, " "))
	}
	fmt.Println()
	fmt.Printf("styles:")
	for name := range styles.Registry {
		fmt.Printf(" %s", name)
	}
	fmt.Println()
	fmt.Printf("formatters:")
	for name := range formatters.Registry {
		fmt.Printf(" %s", name)
	}
	fmt.Println()
}

func lex(path string, contents string, writer func(*chroma.Token)) {
	lexer := selexer(path, contents)
	if lexer == nil {
		lexer = lexers.Fallback
	}
	lexer = chroma.Coalesce(lexer)
	err := lexer.Tokenise(nil, string(contents), writer)
	kingpin.FatalIfError(err, "")
}

func selexer(path, contents string) (lexer chroma.Lexer) {
	if *lexerFlag != "" {
		return lexers.Get(*lexerFlag)
	}
	if path != "" {
		lexer := lexers.Match(path)
		if lexer != nil {
			return lexer
		}
	}
	return lexers.Analyse(contents)
}

func getWriter(w io.Writer) func(*chroma.Token) {
	style := styles.Get(*styleFlag)
	formatter := formatters.Get(*formatterFlag)
	// formatter := formatters.TTY8
	writer, err := formatter.Format(w, style)
	kingpin.FatalIfError(err, "")
	return writer
}
<commit_msg>support windows colors<commit_after>package main

import (
	"bufio"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/signal"
	"runtime"
	"runtime/pprof"
	"strings"

	"gopkg.in/alecthomas/kingpin.v3-unstable"

	"github.com/alecthomas/chroma"
	"github.com/alecthomas/chroma/formatters"
	"github.com/alecthomas/chroma/formatters/html"
	"github.com/alecthomas/chroma/lexers"
	"github.com/alecthomas/chroma/styles"
	"github.com/mattn/go-colorable"
	"github.com/mattn/go-isatty"
)

var (
	profileFlag = kingpin.Flag("profile", "Enable profiling to file.").Hidden().String()
	listFlag    = kingpin.Flag("list", "List lexers, styles and formatters.").Bool()

	lexerFlag     = kingpin.Flag("lexer", "Lexer to use when formatting.").PlaceHolder("autodetect").Short('l').Enum(lexers.Names(true)...)
	styleFlag     = kingpin.Flag("style", "Style to use for formatting.").Short('s').Default("swapoff").Enum(styles.Names()...)
	formatterFlag = kingpin.Flag("formatter", "Formatter to use.").Default("terminal").Short('f').Enum(formatters.Names()...)

	htmlFlag            = kingpin.Flag("html", "Enable HTML mode (equivalent to '--formatter html').").Bool()
	htmlPrefixFlag      = kingpin.Flag("html-prefix", "HTML CSS class prefix.").PlaceHolder("PREFIX").String()
	htmlStylesFlag      = kingpin.Flag("html-styles", "Output HTML CSS styles.").Bool()
	htmlOnlyFlag        = kingpin.Flag("html-only", "Output HTML fragment.").Bool()
	htmlInlineStyleFlag = kingpin.Flag("html-inline-styles", "Output HTML with inline styles (no classes).").Bool()

	filesArgs = kingpin.Arg("files", "Files to highlight.").ExistingFiles()
)

func main() {
	kingpin.CommandLine.Help = `
Chroma is a general purpose syntax highlighting library and corresponding
command, for Go.
`
	kingpin.Parse()
	if *listFlag {
		listAll()
		return
	}
	if *profileFlag != "" {
		f, err := os.Create(*profileFlag)
		kingpin.FatalIfError(err, "")
		pprof.StartCPUProfile(f)
		signals := make(chan os.Signal, 1)
		signal.Notify(signals, os.Interrupt)
		go func() {
			<-signals
			pprof.StopCPUProfile()
			os.Exit(128 + 3)
		}()
		defer pprof.StopCPUProfile()
	}
	var out io.Writer = os.Stdout
	if runtime.GOOS == "windows" && isatty.IsTerminal(os.Stdout.Fd()) {
		out = colorable.NewColorableStdout()
	}
	w := bufio.NewWriterSize(out, 16384)
	defer w.Flush()
	if *htmlFlag {
		*formatterFlag = "html"
	}
	if *formatterFlag == "html" {
		options := []html.Option{}
		if *htmlPrefixFlag != "" {
			options = append(options, html.ClassPrefix(*htmlPrefixFlag))
		}
		if !*htmlOnlyFlag {
			options = append(options, html.Standalone())
		}
		if !*htmlInlineStyleFlag {
			options = append(options, html.WithClasses())
		}
		if *htmlStylesFlag {
			formatter := html.New(html.WithClasses())
			formatter.WriteCSS(w, styles.Get(*styleFlag))
			return
		}
		formatters.Register("html", html.New(options...))
	}
	writer := getWriter(w)
	if len(*filesArgs) == 0 {
		contents, err := ioutil.ReadAll(os.Stdin)
		kingpin.FatalIfError(err, "")
		lex("", string(contents), writer)
	} else {
		for _, filename := range *filesArgs {
			contents, err := ioutil.ReadFile(filename)
			kingpin.FatalIfError(err, "")
			lex(filename, string(contents), writer)
		}
	}
}

func listAll() {
	fmt.Println("lexers:")
	for _, l := range lexers.Registry.Lexers {
		config := l.Config()
		fmt.Printf("  %s\n", config.Name)
		filenames := []string{}
		filenames = append(filenames, config.Filenames...)
		filenames = append(filenames, config.AliasFilenames...)
		fmt.Printf("    aliases: %s\n", strings.Join(config.Aliases, " "))
		fmt.Printf("    filenames: %s\n", strings.Join(filenames, " "))
		fmt.Printf("    mimetypes: %s\n", strings.Join(config.MimeTypes, " "))
	}
	fmt.Println()
	fmt.Printf("styles:")
	for name := range styles.Registry {
		fmt.Printf(" %s", name)
	}
	fmt.Println()
	fmt.Printf("formatters:")
	for name := range formatters.Registry {
		fmt.Printf(" %s", name)
	}
	fmt.Println()
}

func lex(path string, contents string, writer func(*chroma.Token)) {
	lexer := selexer(path, contents)
	if lexer == nil {
		lexer = lexers.Fallback
	}
	lexer = chroma.Coalesce(lexer)
	err := lexer.Tokenise(nil, string(contents), writer)
	kingpin.FatalIfError(err, "")
}

func selexer(path, contents string) (lexer chroma.Lexer) {
	if *lexerFlag != "" {
		return lexers.Get(*lexerFlag)
	}
	if path != "" {
		lexer := lexers.Match(path)
		if lexer != nil {
			return lexer
		}
	}
	return lexers.Analyse(contents)
}

func getWriter(w io.Writer) func(*chroma.Token) {
	style := styles.Get(*styleFlag)
	formatter := formatters.Get(*formatterFlag)
	// formatter := formatters.TTY8
	writer, err := formatter.Format(w, style)
	kingpin.FatalIfError(err, "")
	return writer
}
<|endoftext|>


<commit_before>// estab exports elasticsearch fields as tab separated values
package main

import (
	"bufio"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"runtime/pprof"
	"strconv"
	"strings"

	"github.com/belogik/goes"
	"github.com/miku/estab"
)

func main() {

	host := flag.String("host", "localhost", "elasticsearch host")
	port := flag.String("port", "9200", "elasticsearch port")
	indicesString := flag.String("indices", "", "indices to search (or all)")
	fieldsString := flag.String("f", "_id _index", "field or fields space separated")
	timeout := flag.String("timeout", "10m", "scroll timeout")
	size := flag.Int("size", 10000, "scroll batch size")
	nullValue := flag.String("null", "NOT_AVAILABLE", "value for empty fields")
	separator := flag.String("separator", "|", "separator to use for multiple field values")
	delimiter := flag.String("delimiter", "\t", "column delimiter")
	limit := flag.Int("limit", -1, "maximum number of docs to return (return all by default)")
	version := flag.Bool("v", false, "prints current program version")
	cpuprofile := flag.String("cpuprofile", "", "write cpu profile to file")
	queryString := flag.String("query", "", "custom query to run")
	raw := flag.Bool("raw", false, "stream out the raw json records")
	header := flag.Bool("header", false, "output header row with field names")
	singleValue := flag.Bool("1", false, "one value per line (works only with a single column in -f)")
	zeroAsNull := flag.Bool("zero-as-null", false, "treat zero length strings as null values")
	precision := flag.Int("precision", 0, "precision for numeric output")

	flag.Parse()

	if *cpuprofile != "" {
		f, err := os.Create(*cpuprofile)
		if err != nil {
			log.Fatal(err)
		}
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}

	if *version {
		fmt.Println(estab.Version)
		os.Exit(0)
	}

	var query map[string]interface{}
	if *queryString == "" {
		query = map[string]interface{}{
			"query": map[string]interface{}{
				"match_all": map[string]interface{}{},
			},
		}
	} else {
		err := json.Unmarshal([]byte(*queryString), &query)
		if err != nil {
			log.Fatal(err)
		}
	}

	indices := strings.Fields(*indicesString)
	fields := strings.Fields(*fieldsString)

	if *raw && *singleValue {
		log.Fatal("-1 xor -raw ")
	}

	if *singleValue && len(fields) > 1 {
		log.Fatalf("-1 works only with a single column, %d given: %s\n", len(fields), strings.Join(fields, " "))
	}

	if !*raw {
		query["fields"] = fields
	}

	conn := goes.NewConnection(*host, *port)
	scanResponse, err := conn.Scan(query, indices, []string{""}, *timeout, *size)
	if err != nil {
		log.Fatal(err)
	}

	w := bufio.NewWriter(os.Stdout)
	defer w.Flush()
	i := 0

	if *header {
		fmt.Fprintln(w, strings.Join(fields, *delimiter))
	}

	for {
		scrollResponse, err := conn.Scroll(scanResponse.ScrollId, *timeout)
		if err == io.EOF {
			break
		}
		if err != nil {
			log.Fatal(err)
		}
		if len(scrollResponse.Hits.Hits) == 0 {
			break
		}
		for _, hit := range scrollResponse.Hits.Hits {
			if i == *limit {
				return
			}
			if *raw {
				b, err := json.Marshal(hit)
				if err != nil {
					log.Fatal(err)
				}
				fmt.Fprintln(w, string(b))
				continue
			}

			var columns []string
			for _, f := range fields {
				var c []string
				switch f {
				case "_id":
					c = append(c, hit.Id)
				case "_index":
					c = append(c, hit.Index)
				case "_type":
					c = append(c, hit.Type)
				case "_score":
					c = append(c, strconv.FormatFloat(hit.Score, 'f', 6, 64))
				default:
					switch value := hit.Fields[f].(type) {
					case nil:
						c = []string{*nullValue}
					case []interface{}:
						for _, e := range value {
							switch e.(type) {
							case string:
								s := e.(string)
								if s == "" && *zeroAsNull {
									c = append(c, *nullValue)
								} else {
									c = append(c, e.(string))
								}
							case float64:
								c = append(c, strconv.FormatFloat(e.(float64), 'f', *precision, 64))
							}
						}
					default:
						log.Fatalf("unknown field type in response: %+v\n", hit.Fields[f])
					}
				}
				if *singleValue {
					for _, value := range c {
						fmt.Fprintln(w, value)
					}
				} else {
					columns = append(columns, strings.Join(c, *separator))
				}
			}
			if !*singleValue {
				fmt.Fprintln(w, strings.Join(columns, *delimiter))
			}
			i++
		}
	}
}
<commit_msg>Added boolean support<commit_after>// estab exports elasticsearch fields as tab separated values
package main

import (
	"bufio"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"runtime/pprof"
	"strconv"
	"strings"

	"github.com/belogik/goes"
	"github.com/miku/estab"
)

func main() {

	host := flag.String("host", "localhost", "elasticsearch host")
	port := flag.String("port", "9200", "elasticsearch port")
	indicesString := flag.String("indices", "", "indices to search (or all)")
	fieldsString := flag.String("f", "_id _index", "field or fields space separated")
	timeout := flag.String("timeout", "10m", "scroll timeout")
	size := flag.Int("size", 10000, "scroll batch size")
	nullValue := flag.String("null", "NOT_AVAILABLE", "value for empty fields")
	separator := flag.String("separator", "|", "separator to use for multiple field values")
	delimiter := flag.String("delimiter", "\t", "column delimiter")
	limit := flag.Int("limit", -1, "maximum number of docs to return (return all by default)")
	version := flag.Bool("v", false, "prints current program version")
	cpuprofile := flag.String("cpuprofile", "", "write cpu profile to file")
	queryString := flag.String("query", "", "custom query to run")
	raw := flag.Bool("raw", false, "stream out the raw json records")
	header := flag.Bool("header", false, "output header row with field names")
	singleValue := flag.Bool("1", false, "one value per line (works only with a single column in -f)")
	zeroAsNull := flag.Bool("zero-as-null", false, "treat zero length strings as null values")
	precision := flag.Int("precision", 0, "precision for numeric output")

	flag.Parse()

	if *cpuprofile != "" {
		f, err := os.Create(*cpuprofile)
		if err != nil {
			log.Fatal(err)
		}
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}

	if *version {
		fmt.Println(estab.Version)
		os.Exit(0)
	}

	var query map[string]interface{}
	if *queryString == "" {
		query = map[string]interface{}{
			"query": map[string]interface{}{
				"match_all": map[string]interface{}{},
			},
		}
	} else {
		err := json.Unmarshal([]byte(*queryString), &query)
		if err != nil {
			log.Fatal(err)
		}
	}

	indices := strings.Fields(*indicesString)
	fields := strings.Fields(*fieldsString)

	if *raw && *singleValue {
		log.Fatal("-1 xor -raw ")
	}

	if *singleValue && len(fields) > 1 {
		log.Fatalf("-1 works only with a single column, %d given: %s\n", len(fields), strings.Join(fields, " "))
	}

	if !*raw {
		query["fields"] = fields
	}

	conn := goes.NewConnection(*host, *port)
	scanResponse, err := conn.Scan(query, indices, []string{""}, *timeout, *size)
	if err != nil {
		log.Fatal(err)
	}

	w := bufio.NewWriter(os.Stdout)
	defer w.Flush()
	i := 0

	if *header {
		fmt.Fprintln(w, strings.Join(fields, *delimiter))
	}

	for {
		scrollResponse, err := conn.Scroll(scanResponse.ScrollId, *timeout)
		if err == io.EOF {
			break
		}
		if err != nil {
			log.Fatal(err)
		}
		if len(scrollResponse.Hits.Hits) == 0 {
			break
		}
		for _, hit := range scrollResponse.Hits.Hits {
			if i == *limit {
				return
			}
			if *raw {
				b, err := json.Marshal(hit)
				if err != nil {
					log.Fatal(err)
				}
				fmt.Fprintln(w, string(b))
				continue
			}

			var columns []string
			for _, f := range fields {
				var c []string
				switch f {
				case "_id":
					c = append(c, hit.Id)
				case "_index":
					c = append(c, hit.Index)
				case "_type":
					c = append(c, hit.Type)
				case "_score":
					c = append(c, strconv.FormatFloat(hit.Score, 'f', 6, 64))
				default:
					switch value := hit.Fields[f].(type) {
					case nil:
						c = []string{*nullValue}
					case []interface{}:
						for _, e := range value {
							switch e.(type) {
							case string:
								s := e.(string)
								if s == "" && *zeroAsNull {
									c = append(c, *nullValue)
								} else {
									c = append(c, e.(string))
								}
							case float64:
								c = append(c, strconv.FormatFloat(e.(float64), 'f', *precision, 64))
							case bool:
								c = append(c, strconv.FormatBool(e.(bool)))
							}
						}
					default:
						log.Fatalf("unknown field type in response: %+v\n", hit.Fields[f])
					}
				}
				if *singleValue {
					for _, value := range c {
						fmt.Fprintln(w, value)
					}
				} else {
					columns = append(columns, strings.Join(c, *separator))
				}
			}
			if !*singleValue {
				fmt.Fprintln(w, strings.Join(columns, *delimiter))
			}
			i++
		}
	}
}
<|endoftext|>