<commit_before>package main

import (
	"crypto/tls"
	"net/http"
	"net/http/httputil"
	"net/url"

	"github.com/golang/glog"
	"github.com/unrolled/secure"
	"golang.org/x/crypto/acme/autocert"
)

func run(backends map[string]*url.URL, hosts map[string]string, isDev bool, certMgr *autocert.Manager) {
	go httpServer(isDev)
	httpsServer(backends, hosts, isDev, certMgr)
}

type proxy struct {
	handlers map[string]http.Handler
}

func newProxy(backends map[string]*url.URL, hosts map[string]string) *proxy {
	handlers := make(map[string]http.Handler)

	for host, backendName := range hosts {
		backendURL := backends[backendName]
		handlers[host] = httputil.NewSingleHostReverseProxy(backendURL)
	}

	return &proxy{
		handlers: handlers,
	}
}

func (p *proxy) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
	h, ok := p.handlers[req.Host]
	if !ok {
		glog.Infof("Got request for non-existent hostname %q", req.Host)
		http.NotFound(rw, req)
		return
	}

	h.ServeHTTP(rw, req)
}

func httpsServer(backends map[string]*url.URL, hosts map[string]string, isDev bool, certMgr *autocert.Manager) {
	handler := newProxy(backends, hosts)
	server := &http.Server{
		Addr:    ":443",
		Handler: securify(isDev, handler),
		TLSConfig: &tls.Config{
			GetCertificate: certMgr.GetCertificate,
			MinVersion:     tls.VersionTLS12,
		},
	}
	glog.Fatal(server.ListenAndServeTLS("", ""))
}

func httpServer(isDev bool) {
	redirectHandler := http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
		u := &url.URL{
			Scheme:   "https",
			Host:     req.Host,
			Path:     req.URL.Path,
			RawQuery: req.URL.RawQuery,
		}
		http.Redirect(rw, req, u.String(), http.StatusMovedPermanently)
	})

	mux := http.NewServeMux()
	mux.Handle("/", securify(isDev, redirectHandler))

	glog.Fatal(http.ListenAndServe(":80", mux))
}

func securify(isDev bool, handler http.Handler) http.Handler {
	secureMiddleware := secure.New(secure.Options{
		STSSeconds:            60 * 60 * 24 * 365, // One year.
		STSIncludeSubdomains:  true,
		STSPreload:            true,
		FrameDeny:             true,
		ContentTypeNosniff:    true,
		BrowserXssFilter:      true,
		ContentSecurityPolicy: "object-src 'none'; script-src $NONCE 'unsafe-inline' 'strict-dynamic' https:; base-uri 'none';",
		IsDevelopment:         isDev,
	})
	return secureMiddleware.Handler(handler)
}
<commit_msg>Support the HTTP-01 challenge.<commit_after>package main

import (
	"crypto/tls"
	"net/http"
	"net/http/httputil"
	"net/url"

	"github.com/golang/glog"
	"github.com/unrolled/secure"
	"golang.org/x/crypto/acme/autocert"
)

func run(backends map[string]*url.URL, hosts map[string]string, isDev bool, certMgr *autocert.Manager) {
	go httpServer(isDev, certMgr)
	httpsServer(backends, hosts, isDev, certMgr)
}

type proxy struct {
	handlers map[string]http.Handler
}

func newProxy(backends map[string]*url.URL, hosts map[string]string) *proxy {
	handlers := make(map[string]http.Handler)

	for host, backendName := range hosts {
		backendURL := backends[backendName]
		handlers[host] = httputil.NewSingleHostReverseProxy(backendURL)
	}

	return &proxy{
		handlers: handlers,
	}
}

func (p *proxy) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
	h, ok := p.handlers[req.Host]
	if !ok {
		glog.Infof("Got request for non-existent hostname %q", req.Host)
		http.NotFound(rw, req)
		return
	}

	h.ServeHTTP(rw, req)
}

func httpsServer(backends map[string]*url.URL, hosts map[string]string, isDev bool, certMgr *autocert.Manager) {
	handler := newProxy(backends, hosts)
	server := &http.Server{
		Addr:    ":443",
		Handler: securify(isDev, handler),
		TLSConfig: &tls.Config{
			GetCertificate: certMgr.GetCertificate,
			MinVersion:     tls.VersionTLS12,
		},
	}
	glog.Fatal(server.ListenAndServeTLS("", ""))
}

func httpServer(isDev bool, certMgr *autocert.Manager) {
	redirectHandler := http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
		u := &url.URL{
			Scheme:   "https",
			Host:     req.Host,
			Path:     req.URL.Path,
			RawQuery: req.URL.RawQuery,
		}
		http.Redirect(rw, req, u.String(), http.StatusMovedPermanently)
	})

	mux := http.NewServeMux()
	mux.Handle("/", securify(isDev, redirectHandler))

	glog.Fatal(http.ListenAndServe(":80", certMgr.HTTPHandler(mux)))
}

func securify(isDev bool, handler http.Handler) http.Handler {
	secureMiddleware := secure.New(secure.Options{
		STSSeconds:            60 * 60 * 24 * 365, // One year.
		STSIncludeSubdomains:  true,
		STSPreload:            true,
		FrameDeny:             true,
		ContentTypeNosniff:    true,
		BrowserXssFilter:      true,
		ContentSecurityPolicy: "object-src 'none'; script-src $NONCE 'unsafe-inline' 'strict-dynamic' https:; base-uri 'none';",
		IsDevelopment:         isDev,
	})
	return secureMiddleware.Handler(handler)
}
<|endoftext|>


<commit_before>//   Copyright 2017 MSolution.IO
//
//   Licensed under the Apache License, Version 2.0 (the "License");
//   you may not use this file except in compliance with the License.
//   You may obtain a copy of the License at
//
//       http://www.apache.org/licenses/LICENSE-2.0
//
//   Unless required by applicable law or agreed to in writing, software
//   distributed under the License is distributed on an "AS IS" BASIS,
//   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//   See the License for the specific language governing permissions and
//   limitations under the License.

package main

import (
	"context"
	"fmt"
	"net/http"
	"os"
	"regexp"
	"time"

	uuid "github.com/satori/go.uuid"
	"github.com/trackit/jsonlog"

	_ "github.com/trackit/trackit/aws"
	_ "github.com/trackit/trackit/aws/routes"
	_ "github.com/trackit/trackit/aws/s3"
	"github.com/trackit/trackit/config"
	_ "github.com/trackit/trackit/costs"
	_ "github.com/trackit/trackit/costs/anomalies"
	_ "github.com/trackit/trackit/costs/diff"
	_ "github.com/trackit/trackit/costs/tags"
	"github.com/trackit/trackit/periodic"
	_ "github.com/trackit/trackit/plugins"
	_ "github.com/trackit/trackit/reports"
	"github.com/trackit/trackit/routes"
	_ "github.com/trackit/trackit/s3/costs"
	_ "github.com/trackit/trackit/tagging/routes"
	_ "github.com/trackit/trackit/usageReports/ec2"
	_ "github.com/trackit/trackit/usageReports/ec2Coverage"
	_ "github.com/trackit/trackit/usageReports/elasticache"
	_ "github.com/trackit/trackit/usageReports/es"
	_ "github.com/trackit/trackit/usageReports/lambda"
	_ "github.com/trackit/trackit/usageReports/rds"
	_ "github.com/trackit/trackit/usageReports/riEc2"
	_ "github.com/trackit/trackit/usageReports/riRds"
	_ "github.com/trackit/trackit/users"
	_ "github.com/trackit/trackit/users/shared_account"
)

var buildNumber string = "unknown-build"
var backendId = getBackendId()

func init() {
	jsonlog.DefaultLogger = jsonlog.DefaultLogger.WithLogLevel(jsonlog.LogLevelDebug)
}

var tasks = map[string]func(context.Context) error{
	"server":                      taskServer,
	"worker":                      taskWorker,
	"ingest":                      taskIngest,
	"ingest-due":                  taskIngestDue,
	"process-account":             taskProcessAccount,
	"process-account-plugins":     taskProcessAccountPlugins,
	"anomalies-detection":         taskAnomaliesDetection,
	"check-user-entitlement":      taskCheckEntitlement,
	"generate-spreadsheet":        taskSpreadsheet,
	"generate-tags-spreadsheet":   taskTagsSpreadsheet,
	"generate-master-spreadsheet": taskMasterSpreadsheet,
	"update-aws-identity":         taskUpdateAwsIdentity,
	"check-cost":                  taskCheckCost,
	"fetch-pricings":              taskFetchPricings,
	"ingest-limit":                taskIngestLimit,
	"update-tags":                 taskUpdateTags,
	"onboard-tagbot":              taskOnboardTagbot,
	"check-unused-accounts":       taskCheckUnusedAccounts,
	"worker":					   taskWorker,
}

// dockerHostnameRe matches the value of the HOSTNAME environment variable when
// generated by Docker from the container ID.
var dockerHostnameRe = regexp.MustCompile(`[0-9a-z]{12}`)

func main() {
	ctx := context.Background()
	logger := jsonlog.DefaultLogger
	logger.Info("Started.", struct {
		BackendId string `json:"backendId"`
	}{backendId})
	if task, ok := tasks[config.Task]; ok {
		task(ctx)
	} else {
		knownTasks := make([]string, 0, len(tasks))
		for k := range tasks {
			knownTasks = append(knownTasks, k)
		}
		logger.Error("Unknown task.", map[string]interface{}{
			"knownTasks": knownTasks,
			"chosen":     config.Task,
		})
	}
}

var sched periodic.Scheduler

func schedulePeriodicTasks() {
	sched.Register(taskIngestDue, 10*time.Minute, "ingest-due-updates")
	sched.Start()
}

func taskServer(ctx context.Context) error {
	logger := jsonlog.LoggerFromContextOrDefault(ctx)
	initializeHandlers()
	if config.Periodics {
		schedulePeriodicTasks()
		logger.Info("Scheduled periodic tasks.", nil)
	}
	logger.Info(fmt.Sprintf("Listening on %s.", config.HttpAddress), nil)
	err := http.ListenAndServe(config.HttpAddress, nil)
	logger.Error("Server stopped.", err.Error())
	return err
}

// initializeHandlers sets the HTTP server up with handler functions.
func initializeHandlers() {
	globalDecorators := []routes.Decorator{
		routes.RequestId{},
		routes.RouteLog{},
		routes.BackendId{backendId},
		routes.ErrorBody{},
		//routes.PanicAsError{},
		routes.Cors{
			AllowCredentials: true,
			AllowHeaders:     []string{"Content-Type", "Accept", "Authorization", "Cache-Status", "Cache-Error"},
			AllowOrigin:      []string{"*"},
		},
	}
	logger := jsonlog.DefaultLogger
	routes.DocumentationHandler().Register("/docs")
	for _, rh := range routes.RegisteredHandlers {
		applyDecoratorsAndHandle(rh.Pattern, rh.Handler, globalDecorators)
		logger.Info(fmt.Sprintf("Registered route %s.", rh.Pattern), nil)
	}
}

// applyDecoratorsAndHandle applies a list of decorators to a handler and
// registers it.
func applyDecoratorsAndHandle(p string, h routes.Handler, ds []routes.Decorator) {
	h = h.With(ds...)
	http.Handle(p, h)
}

// getBackendId returns an ID unique to the current process. It can also be set
// in the config to a determined string. It contains the build number.
func getBackendId() string {
	if config.BackendId != "" {
		return config.BackendId
	} else if hostname := os.Getenv("HOSTNAME"); dockerHostnameRe.Match([]byte(hostname)) {
		return fmt.Sprintf("%s-%s", hostname, buildNumber)
	} else {
		return fmt.Sprintf("%s-%s", uuid.NewV1().String(), buildNumber)
	}
}
<commit_msg>Fixed merge<commit_after>//   Copyright 2017 MSolution.IO
//
//   Licensed under the Apache License, Version 2.0 (the "License");
//   you may not use this file except in compliance with the License.
//   You may obtain a copy of the License at
//
//       http://www.apache.org/licenses/LICENSE-2.0
//
//   Unless required by applicable law or agreed to in writing, software
//   distributed under the License is distributed on an "AS IS" BASIS,
//   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//   See the License for the specific language governing permissions and
//   limitations under the License.

package main

import (
	"context"
	"fmt"
	"net/http"
	"os"
	"regexp"
	"time"

	uuid "github.com/satori/go.uuid"
	"github.com/trackit/jsonlog"

	_ "github.com/trackit/trackit/aws"
	_ "github.com/trackit/trackit/aws/routes"
	_ "github.com/trackit/trackit/aws/s3"
	"github.com/trackit/trackit/config"
	_ "github.com/trackit/trackit/costs"
	_ "github.com/trackit/trackit/costs/anomalies"
	_ "github.com/trackit/trackit/costs/diff"
	_ "github.com/trackit/trackit/costs/tags"
	"github.com/trackit/trackit/periodic"
	_ "github.com/trackit/trackit/plugins"
	_ "github.com/trackit/trackit/reports"
	"github.com/trackit/trackit/routes"
	_ "github.com/trackit/trackit/s3/costs"
	_ "github.com/trackit/trackit/tagging/routes"
	_ "github.com/trackit/trackit/usageReports/ec2"
	_ "github.com/trackit/trackit/usageReports/ec2Coverage"
	_ "github.com/trackit/trackit/usageReports/elasticache"
	_ "github.com/trackit/trackit/usageReports/es"
	_ "github.com/trackit/trackit/usageReports/lambda"
	_ "github.com/trackit/trackit/usageReports/rds"
	_ "github.com/trackit/trackit/usageReports/riEc2"
	_ "github.com/trackit/trackit/usageReports/riRds"
	_ "github.com/trackit/trackit/users"
	_ "github.com/trackit/trackit/users/shared_account"
)

var buildNumber string = "unknown-build"
var backendId = getBackendId()

func init() {
	jsonlog.DefaultLogger = jsonlog.DefaultLogger.WithLogLevel(jsonlog.LogLevelDebug)
}

var tasks = map[string]func(context.Context) error{
	"server":                      taskServer,
	"worker":                      taskWorker,
	"ingest":                      taskIngest,
	"ingest-due":                  taskIngestDue,
	"process-account":             taskProcessAccount,
	"process-account-plugins":     taskProcessAccountPlugins,
	"anomalies-detection":         taskAnomaliesDetection,
	"check-user-entitlement":      taskCheckEntitlement,
	"generate-spreadsheet":        taskSpreadsheet,
	"generate-tags-spreadsheet":   taskTagsSpreadsheet,
	"generate-master-spreadsheet": taskMasterSpreadsheet,
	"update-aws-identity":         taskUpdateAwsIdentity,
	"check-cost":                  taskCheckCost,
	"fetch-pricings":              taskFetchPricings,
	"ingest-limit":                taskIngestLimit,
	"update-tags":                 taskUpdateTags,
	"onboard-tagbot":              taskOnboardTagbot,
	"check-unused-accounts":       taskCheckUnusedAccounts,
}

// dockerHostnameRe matches the value of the HOSTNAME environment variable when
// generated by Docker from the container ID.
var dockerHostnameRe = regexp.MustCompile(`[0-9a-z]{12}`)

func main() {
	ctx := context.Background()
	logger := jsonlog.DefaultLogger
	logger.Info("Started.", struct {
		BackendId string `json:"backendId"`
	}{backendId})
	if task, ok := tasks[config.Task]; ok {
		task(ctx)
	} else {
		knownTasks := make([]string, 0, len(tasks))
		for k := range tasks {
			knownTasks = append(knownTasks, k)
		}
		logger.Error("Unknown task.", map[string]interface{}{
			"knownTasks": knownTasks,
			"chosen":     config.Task,
		})
	}
}

var sched periodic.Scheduler

func schedulePeriodicTasks() {
	sched.Register(taskIngestDue, 10*time.Minute, "ingest-due-updates")
	sched.Start()
}

func taskServer(ctx context.Context) error {
	logger := jsonlog.LoggerFromContextOrDefault(ctx)
	initializeHandlers()
	if config.Periodics {
		schedulePeriodicTasks()
		logger.Info("Scheduled periodic tasks.", nil)
	}
	logger.Info(fmt.Sprintf("Listening on %s.", config.HttpAddress), nil)
	err := http.ListenAndServe(config.HttpAddress, nil)
	logger.Error("Server stopped.", err.Error())
	return err
}

// initializeHandlers sets the HTTP server up with handler functions.
func initializeHandlers() {
	globalDecorators := []routes.Decorator{
		routes.RequestId{},
		routes.RouteLog{},
		routes.BackendId{backendId},
		routes.ErrorBody{},
		//routes.PanicAsError{},
		routes.Cors{
			AllowCredentials: true,
			AllowHeaders:     []string{"Content-Type", "Accept", "Authorization", "Cache-Status", "Cache-Error"},
			AllowOrigin:      []string{"*"},
		},
	}
	logger := jsonlog.DefaultLogger
	routes.DocumentationHandler().Register("/docs")
	for _, rh := range routes.RegisteredHandlers {
		applyDecoratorsAndHandle(rh.Pattern, rh.Handler, globalDecorators)
		logger.Info(fmt.Sprintf("Registered route %s.", rh.Pattern), nil)
	}
}

// applyDecoratorsAndHandle applies a list of decorators to a handler and
// registers it.
func applyDecoratorsAndHandle(p string, h routes.Handler, ds []routes.Decorator) {
	h = h.With(ds...)
	http.Handle(p, h)
}

// getBackendId returns an ID unique to the current process. It can also be set
// in the config to a determined string. It contains the build number.
func getBackendId() string {
	if config.BackendId != "" {
		return config.BackendId
	} else if hostname := os.Getenv("HOSTNAME"); dockerHostnameRe.Match([]byte(hostname)) {
		return fmt.Sprintf("%s-%s", hostname, buildNumber)
	} else {
		return fmt.Sprintf("%s-%s", uuid.NewV1().String(), buildNumber)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"net"
	"net/http"

	"github.com/Sirupsen/logrus"

	"code.google.com/p/go.net/websocket"
	"golang.org/x/net/context"
)

var newConns = make(chan *websocket.Conn)

func New(ctx context.Context, total, diff chan []byte, port string) {

	var worldData []byte
	conns := make([]chan []byte, 0)

	go handleConnections(port)

	for {
		select {
		case data := <-total:
			worldData = data

		case data := <-diff:
			for _, c := range conns {
				c <- data
			}

		case ws := <-newConns:
			go sendWorld(ws, worldData)
			// c := make(chan []byte)
			// conns = append(conns, c)
			// go sendDiffs(ctx, conn, c)
		}
	}
}

func handleWebSocket(conn *websocket.Conn) {
	logrus.Infof("Accepted conn: %v", conn)
	newConns <- conn
}

func handleConnections(port string) {
	http.Handle("/", websocket.Handler(handleWebSocket))
	err := http.ListenAndServe(port, nil)
	if err != nil {
		logrus.Error(err)
	}
}

func sendWorld(ws *websocket.Conn, data []byte) {
	err := websocket.Message.Send(ws, data)
	if err != nil {
		logrus.Error(err)
	}
	logrus.Infof("Sent world to conn: %v", ws)
}

func sendDiffs(ctx context.Context, conn net.Conn, diff chan []byte) {
	for {
		select {
		case data := <-diff:
			_, err := conn.Write(data)
			if err != nil {
				logrus.Error(err)
			}
			logrus.Infof("Sent diff to conn: %v", conn)
		case <-ctx.Done():
			return
		}
	}
}
<commit_msg>better logging<commit_after>package main

import (
	"net"
	"net/http"

	"github.com/Sirupsen/logrus"

	"code.google.com/p/go.net/websocket"
	"golang.org/x/net/context"
)

var newConns = make(chan *websocket.Conn)

func New(ctx context.Context, total, diff chan []byte, port string) {

	var worldData []byte
	conns := make([]chan []byte, 0)

	go handleConnections(port)

	for {
		select {
		case data := <-total:
			worldData = data

		case data := <-diff:
			for _, c := range conns {
				c <- data
			}

		case ws := <-newConns:
			go sendWorld(ws, worldData)
			// c := make(chan []byte)
			// conns = append(conns, c)
			// go sendDiffs(ctx, conn, c)
		}
	}
}

func handleWebSocket(conn *websocket.Conn) {
	logrus.Infof("Accepted conn: %v", conn)
	newConns <- conn
}

func handleConnections(port string) {
	http.Handle("/", websocket.Handler(handleWebSocket))
	err := http.ListenAndServe(port, nil)
	if err != nil {
		logrus.Error(err)
	}
}

func sendWorld(ws *websocket.Conn, data []byte) {
	err := websocket.Message.Send(ws, data)
	logrus.Infof("Checking for error", ws)
	if err != nil {
		logrus.Error(err)
	}
	logrus.Infof("Sent world to conn: %v", ws)
}

func sendDiffs(ctx context.Context, conn net.Conn, diff chan []byte) {
	for {
		select {
		case data := <-diff:
			_, err := conn.Write(data)
			if err != nil {
				logrus.Error(err)
			}
			logrus.Infof("Sent diff to conn: %v", conn)
		case <-ctx.Done():
			return
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"database/sql"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"log"
	"net/http"
	"runtime"
	"strings"

	"github.com/bgentry/speakeasy"

	"github.com/julienschmidt/httprouter"

	_ "github.com/lib/pq"
)

func post(res resource) httprouter.Handle {
	return func(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
		dec := json.NewDecoder(r.Body.(io.Reader))
		var rep report
		err := dec.Decode(&rep)
		if err != nil {
			log.Printf("error decoding json from request: %v\n", err)
			http.Error(w, "Error. Sorry. Contact bcgraham@gmail.com. Error code A.", http.StatusBadRequest)
			return
		}

		rep.RemoteAddr = r.RemoteAddr
		if rep.Submitter != ps[0].Value {
			log.Printf("SearchReport and URL disagree about identity of submitter: SR says \"%v\", URL says \"%v\".\n", rep.Submitter, ps[0].Value)
		}
		go func(r report) {
			_, err := res.stmt.Exec(res.args(r)...)
			if err != nil {
				log.Printf("error writing to db: %v\n", err)
				return
			}
		}(rep)
	}
}

func get(stmt *sql.Stmt) httprouter.Handle {
	return func(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
		users, err := getUserIDs(stmt, ps[0].Value)
		if err != nil {
			http.Error(w, "Error. Sorry. Contact bcgraham@gmail.com. Error code A.", http.StatusInternalServerError)
			return
		}
		enc := json.NewEncoder(w)
		err = enc.Encode(users)
		if err != nil {
			log.Printf("error encoding json and sending response: %v\n", err)
		}
	}
}

func main() {
	runtime.GOMAXPROCS(runtime.NumCPU())
	port := flag.String("port", ":8080", "port on which to run server")
	dbuser := flag.String("dbuser", "", "user for db connection")
	dbhost := flag.String("dbhost", "localhost:5432", "host and port of database server, formatted as 'host:port'")
	flag.Parse()
	dbpass, err := speakeasy.Ask("Enter password for database user '" + *dbuser + "':")
	if err != nil {
		log.Fatal(err)
	}
	credentials := mustParse(*dbuser, dbpass)
	db, err := sql.Open("postgres", "postgres://"+credentials+"@"+*dbhost+"/tsum?sslmode=disable")
	//	db, err := sql.Open("sqlite3", "users.db")
	if err != nil {
		log.Fatalf("Could not open db: %v", err)
	}

	strangers := mustPrepare(db, strangersSQL)
	reports := mustPrepare(db, reportsSQL)

	searches := resource{
		stmt: mustPrepare(db, searchSQL),
		args: report.searchArgs,
	}

	invites := resource{
		stmt: mustPrepare(db, inviteSQL),
		args: report.inviteArgs,
	}

	router := httprouter.New()

	router.GET("/tsum/:user/strangers/", get(strangers))
	router.GET("/tsum/:user/reports", get(reports))
	router.POST("/tsum/:user/reports/searches", post(searches))
	router.POST("/tsum/:user/reports/invites", post(invites))
	router.ServeFiles("/compiled/*filepath", http.Dir("$GOPATH/src/github.com/bcgraham/tsumtsum/ignore/compiled"))
	fmt.Println("Listening...")
	err = http.ListenAndServe(*port, router)
	if err != nil {
		fmt.Println(err)
	}
}

type report struct {
	Submitter  string
	UserID     string
	MID        string
	RemoteAddr string
}

type resource struct {
	stmt *sql.Stmt
	args func(report) []interface{}
}

func (r report) searchArgs() []interface{} {
	return []interface{}{
		r.Submitter,
		r.UserID,
		r.MID,
		r.RemoteAddr,
	}
}

func (r report) inviteArgs() []interface{} {
	return []interface{}{
		r.Submitter,
		r.MID,
		r.RemoteAddr,
	}
}

func getUserIDs(stmt *sql.Stmt, submitter string) (ids map[string]string, err error) {
	rows, err := stmt.Query(submitter)
	if err != nil {
		log.Printf("query error with submitter %v: %v\n", submitter, err)
		return make(map[string]string), err
	}
	defer rows.Close()

	ids = make(map[string]string)
	for rows.Next() {
		var id string
		err = rows.Scan(&id)
		if err != nil {
			log.Printf("error scanning rows: %v\n", err)
		}
		ids[id] = ""
	}
	return ids, nil
}

const strangersSQL = `SELECT id FROM validIDs WHERE id NOT IN (SELECT id FROM validIDsSubmitters WHERE submitter = $1) LIMIT 5000;`
const reportsSQL = `SELECT id FROM allIDsSubmitters WHERE submitter = $1`
const searchSQL = `INSERT INTO reports (submitter, userid, mid, ip) VALUES ($1, $2, $3, $4)`
const inviteSQL = `INSERT INTO invites (submitter, mid, ip) VALUES ($1, $2, $3)`

func mustPrepare(db *sql.DB, query string) *sql.Stmt {
	stmt, err := db.Prepare(query)
	if err != nil {
		log.Fatal(err)
	}
	return stmt
}

func mustParse(user, pass string) string {
	if user == "" {
		log.Fatal("Username for DB access cannot be blank.")
	}
	if pass == "" {
		log.Fatal("Password for DB access cannot be blank.")
	}
	if strings.Contains(user, ":") {
		log.Fatal("Username cannot contain colon.")
	}
	repl := strings.NewReplacer("\\", "\\\\", "'", "\\'")
	escPass := repl.Replace(pass)
	return user + ":" + escPass
}
<commit_msg>well, that didn't work<commit_after>package main

import (
	"database/sql"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"log"
	"net/http"
	"runtime"
	"strings"

	"github.com/bgentry/speakeasy"

	"github.com/julienschmidt/httprouter"

	_ "github.com/lib/pq"
)

func post(res resource) httprouter.Handle {
	return func(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
		dec := json.NewDecoder(r.Body.(io.Reader))
		var rep report
		err := dec.Decode(&rep)
		if err != nil {
			log.Printf("error decoding json from request: %v\n", err)
			http.Error(w, "Error. Sorry. Contact bcgraham@gmail.com. Error code A.", http.StatusBadRequest)
			return
		}

		rep.RemoteAddr = r.RemoteAddr
		if rep.Submitter != ps[0].Value {
			log.Printf("SearchReport and URL disagree about identity of submitter: SR says \"%v\", URL says \"%v\".\n", rep.Submitter, ps[0].Value)
		}
		go func(r report) {
			_, err := res.stmt.Exec(res.args(r)...)
			if err != nil {
				log.Printf("error writing to db: %v\n", err)
				return
			}
		}(rep)
	}
}

func get(stmt *sql.Stmt) httprouter.Handle {
	return func(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
		users, err := getUserIDs(stmt, ps[0].Value)
		if err != nil {
			http.Error(w, "Error. Sorry. Contact bcgraham@gmail.com. Error code A.", http.StatusInternalServerError)
			return
		}
		enc := json.NewEncoder(w)
		err = enc.Encode(users)
		if err != nil {
			log.Printf("error encoding json and sending response: %v\n", err)
		}
	}
}

func main() {
	runtime.GOMAXPROCS(runtime.NumCPU())
	port := flag.String("port", ":8080", "port on which to run server")
	dbuser := flag.String("dbuser", "", "user for db connection")
	dbhost := flag.String("dbhost", "localhost:5432", "host and port of database server, formatted as 'host:port'")
	flag.Parse()
	dbpass, err := speakeasy.Ask("Enter password for database user '" + *dbuser + "':")
	if err != nil {
		log.Fatal(err)
	}
	credentials := mustParse(*dbuser, dbpass)
	db, err := sql.Open("postgres", "postgres://"+credentials+"@"+*dbhost+"/tsum?sslmode=disable")
	//	db, err := sql.Open("sqlite3", "users.db")
	if err != nil {
		log.Fatalf("Could not open db: %v", err)
	}

	strangers := mustPrepare(db, strangersSQL)
	reports := mustPrepare(db, reportsSQL)

	searches := resource{
		stmt: mustPrepare(db, searchSQL),
		args: report.searchArgs,
	}

	invites := resource{
		stmt: mustPrepare(db, inviteSQL),
		args: report.inviteArgs,
	}

	router := httprouter.New()

	router.GET("/tsum/:user/strangers/", get(strangers))
	router.GET("/tsum/:user/reports", get(reports))
	router.POST("/tsum/:user/reports/searches", post(searches))
	router.POST("/tsum/:user/reports/invites", post(invites))
	router.ServeFiles("/compiled/*filepath", http.Dir("$GOPATH/src/github.com/bcgraham/tsumtsum/ignore/compiled"))
	fmt.Println("Listening...")
	err = http.ListenAndServe(*port, router)
	if err != nil {
		fmt.Println(err)
	}
}

type report struct {
	Submitter  string
	UserID     string
	MID        string
	RemoteAddr string
}

type resource struct {
	stmt *sql.Stmt
	args func(report) []interface{}
}

func (r report) searchArgs() []interface{} {
	return []interface{}{
		r.Submitter,
		r.UserID,
		r.MID,
		r.RemoteAddr,
	}
}

func (r report) inviteArgs() []interface{} {
	return []interface{}{
		r.Submitter,
		r.MID,
		r.RemoteAddr,
	}
}

func getUserIDs(stmt *sql.Stmt, submitter string) (ids map[string]string, err error) {
	rows, err := stmt.Query(submitter)
	if err != nil {
		log.Printf("query error with submitter %v: %v\n", submitter, err)
		return make(map[string]string), err
	}
	defer rows.Close()

	ids = make(map[string]string)
	for rows.Next() {
		var id string
		err = rows.Scan(&id)
		if err != nil {
			log.Printf("error scanning rows: %v\n", err)
		}
		ids[id] = ""
	}
	return ids, nil
}

const strangersSQL = `SELECT id FROM validIDsCache WHERE id NOT IN (SELECT id FROM validIDsSubmittersCache WHERE submitter = $1) LIMIT 5000;`
const reportsSQL = `SELECT id FROM allIDsSubmittersCache WHERE submitter = $1`
const searchSQL = `INSERT INTO reports (submitter, userid, mid, ip) VALUES ($1, $2, $3, $4)`
const inviteSQL = `INSERT INTO invites (submitter, mid, ip) VALUES ($1, $2, $3)`

func mustPrepare(db *sql.DB, query string) *sql.Stmt {
	stmt, err := db.Prepare(query)
	if err != nil {
		log.Fatal(err)
	}
	return stmt
}

func mustParse(user, pass string) string {
	if user == "" {
		log.Fatal("Username for DB access cannot be blank.")
	}
	if pass == "" {
		log.Fatal("Password for DB access cannot be blank.")
	}
	if strings.Contains(user, ":") {
		log.Fatal("Username cannot contain colon.")
	}
	repl := strings.NewReplacer("\\", "\\\\", "'", "\\'")
	escPass := repl.Replace(pass)
	return user + ":" + escPass
}
<|endoftext|>


<commit_before>package server

import (
	"fmt"
	"net"
	"os"

	"github.com/diegoholiveira/bottle/command"
	"github.com/diegoholiveira/bottle/queue"
)

type queues map[string]*queue.Queue

type Server struct {
	address *net.TCPAddr
	queues  queues
}

func (server *Server) Init(address string, port int) {
	ip := net.ParseIP(address)
	if ip == nil {
		fmt.Println("You must define a valid IP for bottle")
		os.Exit(1)
	}

	server.address = &net.TCPAddr{
		IP:   ip,
		Port: port,
	}

	server.queues = make(queues)
}

func (server *Server) Start() {
	fmt.Printf("Starting bottle at %s\n", server.address.String())

	listener, err := net.ListenTCP("tcp", server.address)
	if err != nil {
		fmt.Println("Its not possible to start bottle, please verify", err.Error())
		os.Exit(1)
	}

	for {
		conn, err := listener.Accept()
		if err != nil {
			continue
		}

		server.handle(conn)
	}
}

func (server *Server) handle(conn net.Conn) {
	defer conn.Close()

	var q *queue.Queue

	for {
		comm, err := command.NewCommandFromConnection(conn)
		if err != nil {
			conn.Write([]byte(err.Error()))
			return
		}

		var msg []byte

		if comm.Command == command.Quit {
			return
		}

		if comm.Command != command.Use && q == nil {
			conn.Write([]byte("Select a queue first"))

			continue
		}

		msg = []byte("OK")

		switch comm.Command {
		case command.Put:
			q.Push(comm.Data)
		case command.Get:
			if q.Len() == 0 {
				msg = []byte("NULL")
			} else {
				item := q.Pop()
				msg = []byte(item)
			}
		case command.Use:
			if _, ok := server.queues[comm.Data]; !ok {
				fmt.Printf("Creating a queue named %s\n", comm.Data)

				server.queues[comm.Data] = queue.New()
			}

			q = server.queues[comm.Data]
		case command.Purge:
			q = queue.New()
		}
		conn.Write(msg)
	}
}
<commit_msg>Allow multiples clients using go coroutines<commit_after>package server

import (
	"fmt"
	"net"
	"os"
	"unicode/utf8"

	"github.com/diegoholiveira/bottle/command"
	"github.com/diegoholiveira/bottle/queue"
)

type queues map[string]*queue.Queue

type Server struct {
	address *net.TCPAddr
	queues  queues
}

func (server *Server) Init(address string, port int) {
	ip := net.ParseIP(address)
	if ip == nil {
		fmt.Println("You must define a valid IP for bottle")
		os.Exit(1)
	}

	server.address = &net.TCPAddr{
		IP:   ip,
		Port: port,
	}

	server.queues = make(queues)
}

func (server *Server) Start() {
	fmt.Printf("Starting bottle at %s\n", server.address.String())

	listener, err := net.ListenTCP("tcp", server.address)
	if err != nil {
		fmt.Println("Its not possible to start bottle, please verify", err.Error())
		os.Exit(1)
	}

	for {
		conn, err := listener.Accept()
		if err != nil {
			continue
		}

		go server.handle(conn)
	}
}

func (server *Server) handle(conn net.Conn) {
	var q *queue.Queue

	for {
		comm, err := command.NewCommandFromConnection(conn)
		if err != nil {
			conn.Write([]byte(err.Error()))
			break
		}

		var msg []byte

		if comm.Command == command.Quit {
			break
		}

		if comm.Command != command.Use && q == nil {
			conn.Write([]byte("Select a queue first"))
			break
		}

		msg = []byte("OK")

		switch comm.Command {
		case command.Put:
			q.Push(comm.Data)
		case command.Get:
			msg = []byte("NULL")
			if q.Len() > 0 {
				if item := q.Pop(); utf8.RuneCountInString(item) > 0 {
					msg = []byte(item)

				}
			}
		case command.Use:
			if _, ok := server.queues[comm.Data]; !ok {
				fmt.Printf("Creating a queue named %s\n", comm.Data)

				server.queues[comm.Data] = queue.New()
			}

			q = server.queues[comm.Data]
		case command.Purge:
			q = queue.New()
		}
		conn.Write(msg)
	}

	conn.Close()
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"html/template"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"time"
)

type file struct {
	name string
	hash string
	size int64
}

type fileInfo struct {
	Name     string
	Hash     string
	Size     int64
	Path     string
	Modified time.Time
}

var f1 = file{"file1", "sjdalkjsda", 1234}
var f2 = file{"file2", "lkjlkjlkl", 4321}
var allFiles = map[string]file{"file1": f1, "file2": f2}

//Compile templates on start
var templates = template.Must(template.ParseFiles("templates/upload.html"))

//Display the named template
func display(w http.ResponseWriter, tmpl string, data interface{}) {
	templates.ExecuteTemplate(w, tmpl+".html", data)
}

func fileListHandler(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "application/json; charset=utf-8")
	json.NewEncoder(w).Encode(allFiles)
}

//This is where the action happens.
func uploadHandler(w http.ResponseWriter, r *http.Request) {
	switch r.Method {
	//GET displays the upload form.
	case "GET":
		log.Println("Get...")
		display(w, "upload", nil)

	//POST takes the uploaded file(s) and saves it to disk.
	case "POST":
		//get the multipart reader for the request.
		reader, err := r.MultipartReader()

		log.Println("Receiving stuff...")
		//hash := r.FormValue("sha-1")

		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}

		//copy each part to destination.
		for {
			part, err := reader.NextPart()
			if err == io.EOF {
				break
			}

			//if part.FileName() is empty, skip this iteration.
			if part.FileName() == "" {
				continue
			}

			dst, err := os.Create("/Users/partec/hackerschool/go/gobox/tmp/" + part.FileName())
			defer dst.Close()

			if err != nil {
				http.Error(w, err.Error(), http.StatusInternalServerError)
				return
			}

			if _, err := io.Copy(dst, part); err != nil {
				http.Error(w, err.Error(), http.StatusInternalServerError)
				return
			}
		}
		//display success message.
		display(w, "upload", "Upload successful.")
	default:
		w.WriteHeader(http.StatusMethodNotAllowed)
	}
}

func writeLocalMeta(fileMeta []byte) {
	f, err := os.OpenFile("localMeta.gob", os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600)
	if err != nil {
		panic(err)
	}

	defer f.Close()

	if _, err = f.WriteString(string(fileMeta) + "\n"); err != nil {
		panic(err)
	}
}

func handleMetaConnection(w http.ResponseWriter, r *http.Request) {
	switch r.Method {

	case "GET":
		log.Println("[+] GET REQUEST RECEIVED")

	case "POST":
		log.Println("[+] POST REQUEST RECEIVED")
		contents, err := ioutil.ReadAll(r.Body)
		if err != nil {
			panic("Error")
		}
		writeLocalMeta(contents)

	default:
		w.WriteHeader(http.StatusMethodNotAllowed)

	}
}

// filename, size, sha-1

func main() {
	fmt.Println("Server up...")
	http.HandleFunc("/upload", uploadHandler)

	http.HandleFunc("/files", fileListHandler)

	http.HandleFunc("/meta", handleMetaConnection)

	//static file handler.
	http.Handle("/assets/", http.StripPrefix("/assets/", http.FileServer(http.Dir("assets"))))

	//Listen on port 8080
	http.ListenAndServe(":4243", nil)
}
<commit_msg>S3 auth<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"time"

	"github.com/mitchellh/goamz/aws"
	"github.com/mitchellh/goamz/s3"
)

type uploadInfo struct {
	Task string
	File fileInfo
}
type fileInfo struct {
	Name     string
	Hash     string
	Size     int64
	Path     string
	Modified time.Time
}

func writeLocalMeta(fileMeta []byte) {
	f, err := os.OpenFile("localMeta.gob",
		os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600)
	if err != nil {
		panic(err)
	}

	defer f.Close()

	if _, err = f.WriteString(string(fileMeta) + "\n"); err != nil {
		panic(err)
	}
}

func evalAction(contents []byte) {
	var uploadData uploadInfo

	err := json.Unmarshal(contents, &uploadData)
	fileMeta := uploadData.File

	if err != nil {
		fmt.Errorf("Error: %s", err)
	}

	switch uploadData.Task {
	case "upload":
		fmt.Printf("[*]Uploading %s to S3...", fileMeta.Name)
	}
}

func authOnS3() {
	key := os.Getenv("GOBOX_AWS_ACCESS_KEY_ID")
	secret := os.Getenv("GOBOX_AWS_SECRET_ACCESS_KEY")

	auth, err := aws.GetAuth(key, secret)
	if err != nil {
		log.Fatal(err)
	}
	client := s3.New(auth, aws.USEast)
	resp, err := client.ListBuckets()

	if err != nil {
		log.Fatal(err)
	}

	log.Print(fmt.Sprintf("%T %+v", resp.Buckets[0], resp.Buckets[0]))
}

func handleMetaConnection(w http.ResponseWriter, r *http.Request) {
	switch r.Method {

	case "GET":
		log.Println("[+] GET REQUEST RECEIVED")

	case "POST":
		log.Println("[+] POST REQUEST RECEIVED")
		contents, err := ioutil.ReadAll(r.Body)
		if err != nil {
			panic("Error")
		}
		writeLocalMeta(contents)
		evalAction(contents)

	default:
		w.WriteHeader(http.StatusMethodNotAllowed)

	}
}

func main() {
	fmt.Println("[+] Server Initialized on port: 4243")
	//// Upload endpoint
	//http.HandleFunc("/upload", uploadHandler)

	//// files endpoint
	//http.HandleFunc("/files", fileListHandler)

	// meta endpoint
	http.HandleFunc("/meta", handleMetaConnection)

	//static file handler.
	http.Handle("/assets/",
		http.StripPrefix("/assets/",
			http.FileServer(http.Dir("assets"))))
	authOnS3()
	//Listen
	http.ListenAndServe(":4243", nil)
}
<|endoftext|>


<commit_before>package server

import (
	"encoding/json"
	"fmt"
	"html/template"
	"net/http"
	"os"
	"path/filepath"
	"strconv"
	"strings"

	"github.com/albertyw/reaction-pics/tumblr"
	"github.com/ikeikeikeike/go-sitemap-generator/v2/stm"
	"github.com/newrelic/go-agent/v3/newrelic"
	"github.com/pkg/errors"
	"github.com/rollbar/rollbar-go"
	"go.uber.org/zap"
)

const (
	maxResults = 20
)

var serverDir = filepath.Join(os.Getenv("ROOT_DIR"), "server")
var staticPath = fmt.Sprintf("%s/static/", serverDir)

// indexHandler is an http handler that returns the index page HTML
func indexHandler(w http.ResponseWriter, r *http.Request, d handlerDeps) {
	if r.URL.Path != "/" && !strings.HasPrefix(r.URL.Path, "/post/") {
		err := fmt.Errorf("file not found: %s", r.URL.Path)
		d.logger.Warn(err)
		rollbar.RequestError(rollbar.WARN, r, err)
		http.NotFound(w, r)
		return
	}
	t, err := template.ParseFiles(staticPath + "index.htm")
	if err != nil {
		err = errors.New("Cannot read post template")
		d.logger.Error(err)
		rollbar.RequestError(rollbar.ERR, r, err)
		http.Error(w, err.Error(), 500)
		return
	}
	templateData := struct {
		CacheString string
	}{d.appCacheString}
	err = t.Execute(w, templateData)
	if err != nil {
		err = errors.Wrap(err, "Cannot execute template")
		d.logger.Error(err)
		rollbar.RequestError(rollbar.ERR, r, err)
		http.Error(w, err.Error(), 500)
		return
	}
}

// searchHandler is an http handler to search data for keywords in json format
// It matches the query against post titles and then ranks posts by number of likes
func searchHandler(w http.ResponseWriter, r *http.Request, d handlerDeps) {
	query := r.URL.Query().Get("query")
	query = strings.ToLower(query)
	queriedBoard := d.board.FilterBoard(query)
	if query == "" {
		queriedBoard.RandomizePosts()
	}
	offsetString := r.URL.Query().Get("offset")
	offset, err := strconv.Atoi(offsetString)
	if err != nil {
		offset = 0
	}
	data := map[string]interface{}{
		"offset":       offset,
		"totalResults": len(queriedBoard.Posts),
	}
	queriedBoard.LimitBoard(offset, maxResults)
	queriedBoard.SortPostsByLikes()
	data["data"] = queriedBoard.PostsToJSON()
	dataBytes, _ := json.Marshal(data)
	fmt.Fprint(w, string(dataBytes))
}

// postDataHandler is an http handler to return post data by ID in json format
func postDataHandler(w http.ResponseWriter, r *http.Request, d handlerDeps) {
	pathStrings := strings.Split(r.URL.Path, "/")
	postIDString := pathStrings[2]
	postID, err := strconv.ParseInt(postIDString, 10, 64)
	if err != nil {
		err = errors.Wrap(err, "Cannot parse post id")
		d.logger.Warn(err)
		rollbar.RequestError(rollbar.WARN, r, err)
		http.NotFound(w, r)
		return
	}
	post := d.board.GetPostByID(postID)
	if post == nil {
		err = errors.New("Cannot find post")
		d.logger.Warn(err)
		rollbar.RequestError(rollbar.WARN, r, err)
		http.NotFound(w, r)
		return
	}
	data := map[string]interface{}{
		"offset":       0,
		"totalResults": 1,
		"data":         []tumblr.PostJSON{post.ToJSONStruct()},
	}
	marshalledPost, _ := json.Marshal(data)
	fmt.Fprint(w, string(marshalledPost))
}

// postHandler is an http handler that validates the correctness of a post url
// and returns the index page html to render it correct
func postHandler(w http.ResponseWriter, r *http.Request, d handlerDeps) {
	pathStrings := strings.Split(r.URL.Path, "/")
	postIDString := pathStrings[2]
	postID, err := strconv.ParseInt(postIDString, 10, 64)
	if err != nil {
		err = errors.Wrap(err, "Cannot parse post id")
		d.logger.Warn(err)
		rollbar.RequestError(rollbar.WARN, r, err)
		http.NotFound(w, r)
		return
	}
	foundPost := false
	for _, p := range d.board.Posts {
		if p.ID == postID {
			foundPost = true
		}
	}
	if !foundPost {
		err = errors.New("Cannot find post")
		d.logger.Warn(err)
		rollbar.RequestError(rollbar.WARN, r, err)
		http.NotFound(w, r)
		return
	}
	indexHandler(w, r, d)
}

// statsHandler returns internal stats about the reaction.pics DB as json
func statsHandler(w http.ResponseWriter, r *http.Request, d handlerDeps) {
	postCount := strconv.Itoa(len(d.board.Posts))
	data := map[string]interface{}{
		"postCount": postCount,
		"keywords":  d.board.Keywords(),
	}
	stats, _ := json.Marshal(data)
	fmt.Fprint(w, string(stats))
}

// sitemapHandler returns a sitemap of reaction.pics as an xml file
func sitemapHandler(w http.ResponseWriter, r *http.Request, d handlerDeps) {
	sm := stm.NewSitemap(0)
	sm.SetDefaultHost(os.Getenv("HOST"))

	sm.Create()
	sm.Add(stm.URL{{"loc", "/"}})
	for _, url := range d.board.URLs() {
		sm.Add(stm.URL{{"loc", url}})
	}
	w.Write(sm.XMLContent())
}

// staticHandler returns static files
func staticHandler(w http.ResponseWriter, r *http.Request, _ handlerDeps) {
	staticFS := rewriteFS(http.FileServer(http.Dir(staticPath)).ServeHTTP)
	staticFS(w, r)
}

func faviconHandler(w http.ResponseWriter, r *http.Request, _ handlerDeps) {
	faviconPath := filepath.Join(staticPath, "favicon", "favicon.ico")
	http.ServeFile(w, r, faviconPath)
}

func robotsTxtHandler(w http.ResponseWriter, r *http.Request, _ handlerDeps) {
	fmt.Fprint(w, "")
}

// Run starts up the HTTP server
func Run(newrelicApp *newrelic.Application, logger *zap.SugaredLogger) {
	board := tumblr.InitializeBoard()
	address := fmt.Sprintf(":%s", os.Getenv("PORT"))
	logger.Infof("server listening on %s", address)
	generator := newHandlerGenerator(board, newrelicApp, logger)
	http.Handle(generator.newHandler("/", indexHandler))
	http.Handle(generator.newHandler("/favicon.ico", faviconHandler))
	http.Handle(generator.newHandler("/robots.txt", robotsTxtHandler))
	http.Handle(generator.newHandler("/search", searchHandler))
	http.Handle(generator.newHandler("/postdata/", postDataHandler))
	http.Handle(generator.newHandler("/post/", postHandler))
	http.Handle(generator.newHandler("/stats.json", statsHandler))
	http.Handle(generator.newHandler("/sitemap.xml", sitemapHandler))
	http.Handle(generator.newHandler("/static/", staticHandler))
	http.ListenAndServe(address, nil)
}
<commit_msg>Add timeHandler<commit_after>package server

import (
	"encoding/json"
	"fmt"
	"html/template"
	"net/http"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"github.com/albertyw/reaction-pics/tumblr"
	"github.com/ikeikeikeike/go-sitemap-generator/v2/stm"
	"github.com/newrelic/go-agent/v3/newrelic"
	"github.com/pkg/errors"
	"github.com/rollbar/rollbar-go"
	"go.uber.org/zap"
)

const (
	maxResults = 20
)

var serverDir = filepath.Join(os.Getenv("ROOT_DIR"), "server")
var staticPath = fmt.Sprintf("%s/static/", serverDir)

// indexHandler is an http handler that returns the index page HTML
func indexHandler(w http.ResponseWriter, r *http.Request, d handlerDeps) {
	if r.URL.Path != "/" && !strings.HasPrefix(r.URL.Path, "/post/") {
		err := fmt.Errorf("file not found: %s", r.URL.Path)
		d.logger.Warn(err)
		rollbar.RequestError(rollbar.WARN, r, err)
		http.NotFound(w, r)
		return
	}
	t, err := template.ParseFiles(staticPath + "index.htm")
	if err != nil {
		err = errors.New("Cannot read post template")
		d.logger.Error(err)
		rollbar.RequestError(rollbar.ERR, r, err)
		http.Error(w, err.Error(), 500)
		return
	}
	templateData := struct {
		CacheString string
	}{d.appCacheString}
	err = t.Execute(w, templateData)
	if err != nil {
		err = errors.Wrap(err, "Cannot execute template")
		d.logger.Error(err)
		rollbar.RequestError(rollbar.ERR, r, err)
		http.Error(w, err.Error(), 500)
		return
	}
}

// searchHandler is an http handler to search data for keywords in json format
// It matches the query against post titles and then ranks posts by number of likes
func searchHandler(w http.ResponseWriter, r *http.Request, d handlerDeps) {
	query := r.URL.Query().Get("query")
	query = strings.ToLower(query)
	queriedBoard := d.board.FilterBoard(query)
	if query == "" {
		queriedBoard.RandomizePosts()
	}
	offsetString := r.URL.Query().Get("offset")
	offset, err := strconv.Atoi(offsetString)
	if err != nil {
		offset = 0
	}
	data := map[string]interface{}{
		"offset":       offset,
		"totalResults": len(queriedBoard.Posts),
	}
	queriedBoard.LimitBoard(offset, maxResults)
	queriedBoard.SortPostsByLikes()
	data["data"] = queriedBoard.PostsToJSON()
	dataBytes, _ := json.Marshal(data)
	fmt.Fprint(w, string(dataBytes))
}

// postDataHandler is an http handler to return post data by ID in json format
func postDataHandler(w http.ResponseWriter, r *http.Request, d handlerDeps) {
	pathStrings := strings.Split(r.URL.Path, "/")
	postIDString := pathStrings[2]
	postID, err := strconv.ParseInt(postIDString, 10, 64)
	if err != nil {
		err = errors.Wrap(err, "Cannot parse post id")
		d.logger.Warn(err)
		rollbar.RequestError(rollbar.WARN, r, err)
		http.NotFound(w, r)
		return
	}
	post := d.board.GetPostByID(postID)
	if post == nil {
		err = errors.New("Cannot find post")
		d.logger.Warn(err)
		rollbar.RequestError(rollbar.WARN, r, err)
		http.NotFound(w, r)
		return
	}
	data := map[string]interface{}{
		"offset":       0,
		"totalResults": 1,
		"data":         []tumblr.PostJSON{post.ToJSONStruct()},
	}
	marshalledPost, _ := json.Marshal(data)
	fmt.Fprint(w, string(marshalledPost))
}

// postHandler is an http handler that validates the correctness of a post url
// and returns the index page html to render it correct
func postHandler(w http.ResponseWriter, r *http.Request, d handlerDeps) {
	pathStrings := strings.Split(r.URL.Path, "/")
	postIDString := pathStrings[2]
	postID, err := strconv.ParseInt(postIDString, 10, 64)
	if err != nil {
		err = errors.Wrap(err, "Cannot parse post id")
		d.logger.Warn(err)
		rollbar.RequestError(rollbar.WARN, r, err)
		http.NotFound(w, r)
		return
	}
	foundPost := false
	for _, p := range d.board.Posts {
		if p.ID == postID {
			foundPost = true
		}
	}
	if !foundPost {
		err = errors.New("Cannot find post")
		d.logger.Warn(err)
		rollbar.RequestError(rollbar.WARN, r, err)
		http.NotFound(w, r)
		return
	}
	indexHandler(w, r, d)
}

// statsHandler returns internal stats about the reaction.pics DB as json
func statsHandler(w http.ResponseWriter, r *http.Request, d handlerDeps) {
	postCount := strconv.Itoa(len(d.board.Posts))
	data := map[string]interface{}{
		"postCount": postCount,
		"keywords":  d.board.Keywords(),
	}
	stats, _ := json.Marshal(data)
	fmt.Fprint(w, string(stats))
}

// sitemapHandler returns a sitemap of reaction.pics as an xml file
func sitemapHandler(w http.ResponseWriter, r *http.Request, d handlerDeps) {
	sm := stm.NewSitemap(0)
	sm.SetDefaultHost(os.Getenv("HOST"))

	sm.Create()
	sm.Add(stm.URL{{"loc", "/"}})
	for _, url := range d.board.URLs() {
		sm.Add(stm.URL{{"loc", url}})
	}
	w.Write(sm.XMLContent())
}

// staticHandler returns static files
func staticHandler(w http.ResponseWriter, r *http.Request, _ handlerDeps) {
	staticFS := rewriteFS(http.FileServer(http.Dir(staticPath)).ServeHTTP)
	staticFS(w, r)
}

func timeHandler(w http.ResponseWriter, r *http.Request, _ handlerDeps) {
	unixTime := int32(time.Now().Unix())
	data := map[string]interface{}{
		"unixtime": unixTime,
	}
	timeData, _ := json.Marshal(data)
	fmt.Fprint(w, string(timeData))
}

func faviconHandler(w http.ResponseWriter, r *http.Request, _ handlerDeps) {
	faviconPath := filepath.Join(staticPath, "favicon", "favicon.ico")
	http.ServeFile(w, r, faviconPath)
}

func robotsTxtHandler(w http.ResponseWriter, r *http.Request, _ handlerDeps) {
	fmt.Fprint(w, "")
}

// Run starts up the HTTP server
func Run(newrelicApp *newrelic.Application, logger *zap.SugaredLogger) {
	board := tumblr.InitializeBoard()
	address := fmt.Sprintf(":%s", os.Getenv("PORT"))
	logger.Infof("server listening on %s", address)
	generator := newHandlerGenerator(board, newrelicApp, logger)
	http.Handle(generator.newHandler("/", indexHandler))
	http.Handle(generator.newHandler("/favicon.ico", faviconHandler))
	http.Handle(generator.newHandler("/robots.txt", robotsTxtHandler))
	http.Handle(generator.newHandler("/search", searchHandler))
	http.Handle(generator.newHandler("/postdata/", postDataHandler))
	http.Handle(generator.newHandler("/post/", postHandler))
	http.Handle(generator.newHandler("/stats.json", statsHandler))
	http.Handle(generator.newHandler("/sitemap.xml", sitemapHandler))
	http.Handle(generator.newHandler("/static/", staticHandler))
	http.Handle(generator.newHandler("/time/", timeHandler))
	http.ListenAndServe(address, nil)
}
<|endoftext|>


<commit_before>package event_trigger

import (
	"log"
	"os"
	"testing"
	"time"

	"github.com/google/go-cmp/cmp"
	"github.com/karimra/gnmic/formatters"
)

type item struct {
	input  []*formatters.EventMsg
	output []*formatters.EventMsg
}

var testset = map[string]struct {
	processorType string
	processor     map[string]interface{}
	tests         []item
}{
	"init": {
		processorType: processorType,
		processor: map[string]interface{}{
			"debug": true,
			"action": map[string]interface{}{
				"type": "http",
			},
		},
		tests: []item{
			{
				input:  nil,
				output: nil,
			},
			{
				input:  make([]*formatters.EventMsg, 0),
				output: make([]*formatters.EventMsg, 0),
			},
			{
				input: []*formatters.EventMsg{
					{
						Name: "sub1",
						Tags: map[string]string{
							"tag1": "1",
						},
					},
				},
				output: []*formatters.EventMsg{
					{
						Name: "sub1",
						Tags: map[string]string{
							"tag1": "1",
						},
					},
				},
			},
		},
	},
	"with_condition": {
		processorType: processorType,
		processor: map[string]interface{}{
			"condition": `.values["counter1"] > 90`,
			"debug":     true,
			"action": map[string]interface{}{
				"type": "http",
				"url":  "http://remote-alerting-system:9090/",
			},
		},
		tests: []item{
			{
				input:  nil,
				output: nil,
			},
			{
				input:  make([]*formatters.EventMsg, 0),
				output: make([]*formatters.EventMsg, 0),
			},
			{
				input: []*formatters.EventMsg{
					{
						Name: "sub1",
						Tags: map[string]string{
							"tag1": "1",
						},
						Values: map[string]interface{}{
							"counter1": 91,
						},
					},
				},
				output: []*formatters.EventMsg{
					{
						Name: "sub1",
						Tags: map[string]string{
							"tag1": "1",
						},
						Values: map[string]interface{}{
							"counter1": 91,
						},
					},
				},
			},
			{
				input: []*formatters.EventMsg{
					{
						Name: "sub1",
						Tags: map[string]string{
							"tag1": "1",
						},
						Values: map[string]interface{}{
							"counter1": 89,
						},
					},
				},
				output: []*formatters.EventMsg{
					{
						Name: "sub1",
						Tags: map[string]string{
							"tag1": "1",
						},
						Values: map[string]interface{}{
							"counter1": 89,
						},
					},
				},
			},
			{
				input: []*formatters.EventMsg{
					{
						Name: "sub1",
						Tags: map[string]string{
							"tag1": "1",
						},
						Values: map[string]interface{}{
							"counter2": 91,
						},
					},
				},
				output: []*formatters.EventMsg{
					{
						Name: "sub1",
						Tags: map[string]string{
							"tag1": "1",
						},
						Values: map[string]interface{}{
							"counter2": 91,
						},
					},
				},
			},
		},
	},
}

var triggerOccWindowTestSet = map[string]struct {
	t   *Trigger
	now time.Time
	out bool
}{
	"defaults_0_occurrences": {
		t: &Trigger{
			logger:           log.New(os.Stderr, loggingPrefix, log.LstdFlags|log.Lmicroseconds),
			Debug:            true,
			MinOccurrences:   1,
			MaxOccurrences:   1,
			Window:           time.Minute,
			occurrencesTimes: []time.Time{},
		},
		out: true,
		now: time.Now(),
	},
	"defaults_with_1_occurrence_in_window": {
		t: &Trigger{
			logger:         log.New(os.Stderr, loggingPrefix, log.LstdFlags|log.Lmicroseconds),
			Debug:          true,
			MinOccurrences: 1,
			MaxOccurrences: 1,
			Window:         time.Minute,
			occurrencesTimes: []time.Time{
				time.Now().Add(-time.Second),
			},
		},
		out: false,
		now: time.Now(),
	},
	"defaults_with_1_occurrence_out_of_window": {
		t: &Trigger{
			logger:         log.New(os.Stderr, loggingPrefix, log.LstdFlags|log.Lmicroseconds),
			Debug:          true,
			MinOccurrences: 1,
			MaxOccurrences: 1,
			Window:         time.Minute,
			occurrencesTimes: []time.Time{
				time.Now().Add(-time.Hour),
			},
		},
		out: true,
		now: time.Now(),
	},
}

func TestEventTrigger(t *testing.T) {
	for name, ts := range testset {
		if pi, ok := formatters.EventProcessors[ts.processorType]; ok {
			t.Log("found processor")
			p := pi()
			err := p.Init(ts.processor, formatters.WithLogger(log.New(os.Stderr, loggingPrefix, log.LstdFlags|log.Lmicroseconds)))
			if err != nil {
				t.Errorf("failed to initialize processors: %v", err)
				return
			}
			t.Logf("processor: %+v", p)
			for i, item := range ts.tests {
				t.Run(name, func(t *testing.T) {
					t.Logf("running test item %d", i)
					outs := p.Apply(item.input...)
					for j := range outs {
						if !cmp.Equal(outs[j], item.output[j]) {
							t.Errorf("failed at %s item %d, index %d, expected %+v, got: %+v", name, i, j, item.output[j], outs[j])
						}
					}
				})
			}
		} else {
			t.Errorf("event processor %s not found", ts.processorType)
		}
	}
}

func TestOccurrenceTrigger(t *testing.T) {
	for name, ts := range triggerOccWindowTestSet {
		t.Run(name, func(t *testing.T) {
			ok := ts.t.evalOccurrencesWithinWindow(ts.now)
			t.Logf("%q result: %v", name, ok)
			if ok != ts.out {
				t.Errorf("failed at %s , expected %+v, got: %+v", name, ts.out, ok)
			}
		})
	}
}
<commit_msg>add tests<commit_after>package event_trigger

import (
	"log"
	"os"
	"testing"
	"time"

	"github.com/google/go-cmp/cmp"
	"github.com/karimra/gnmic/formatters"
)

type item struct {
	input  []*formatters.EventMsg
	output []*formatters.EventMsg
}

var testset = map[string]struct {
	processorType string
	processor     map[string]interface{}
	tests         []item
}{
	"init": {
		processorType: processorType,
		processor: map[string]interface{}{
			"debug": true,
			"action": map[string]interface{}{
				"type": "http",
			},
		},
		tests: []item{
			{
				input:  nil,
				output: nil,
			},
			{
				input:  make([]*formatters.EventMsg, 0),
				output: make([]*formatters.EventMsg, 0),
			},
			{
				input: []*formatters.EventMsg{
					{
						Name: "sub1",
						Tags: map[string]string{
							"tag1": "1",
						},
					},
				},
				output: []*formatters.EventMsg{
					{
						Name: "sub1",
						Tags: map[string]string{
							"tag1": "1",
						},
					},
				},
			},
		},
	},
	"with_condition": {
		processorType: processorType,
		processor: map[string]interface{}{
			"condition": `.values["counter1"] > 90`,
			"debug":     true,
			"action": map[string]interface{}{
				"type": "http",
				"url":  "http://remote-alerting-system:9090/",
			},
		},
		tests: []item{
			{
				input:  nil,
				output: nil,
			},
			{
				input:  make([]*formatters.EventMsg, 0),
				output: make([]*formatters.EventMsg, 0),
			},
			{
				input: []*formatters.EventMsg{
					{
						Name: "sub1",
						Tags: map[string]string{
							"tag1": "1",
						},
						Values: map[string]interface{}{
							"counter1": 91,
						},
					},
				},
				output: []*formatters.EventMsg{
					{
						Name: "sub1",
						Tags: map[string]string{
							"tag1": "1",
						},
						Values: map[string]interface{}{
							"counter1": 91,
						},
					},
				},
			},
			{
				input: []*formatters.EventMsg{
					{
						Name: "sub1",
						Tags: map[string]string{
							"tag1": "1",
						},
						Values: map[string]interface{}{
							"counter1": 89,
						},
					},
				},
				output: []*formatters.EventMsg{
					{
						Name: "sub1",
						Tags: map[string]string{
							"tag1": "1",
						},
						Values: map[string]interface{}{
							"counter1": 89,
						},
					},
				},
			},
			{
				input: []*formatters.EventMsg{
					{
						Name: "sub1",
						Tags: map[string]string{
							"tag1": "1",
						},
						Values: map[string]interface{}{
							"counter2": 91,
						},
					},
				},
				output: []*formatters.EventMsg{
					{
						Name: "sub1",
						Tags: map[string]string{
							"tag1": "1",
						},
						Values: map[string]interface{}{
							"counter2": 91,
						},
					},
				},
			},
		},
	},
}

var triggerOccWindowTestSet = map[string]struct {
	t   *Trigger
	now time.Time
	out bool
}{
	"defaults_0_occurrences": {
		t: &Trigger{
			logger:           log.New(os.Stderr, loggingPrefix, log.LstdFlags|log.Lmicroseconds),
			Debug:            true,
			MinOccurrences:   1,
			MaxOccurrences:   1,
			Window:           time.Minute,
			occurrencesTimes: []time.Time{},
		},
		out: true,
		now: time.Now(),
	},
	"defaults_with_1_occurrence_in_window": {
		t: &Trigger{
			logger:         log.New(os.Stderr, loggingPrefix, log.LstdFlags|log.Lmicroseconds),
			Debug:          true,
			MinOccurrences: 1,
			MaxOccurrences: 1,
			Window:         time.Minute,
			occurrencesTimes: []time.Time{
				time.Now().Add(-time.Second),
			},
		},
		out: false,
		now: time.Now(),
	},
	"defaults_with_1_occurrence_out_of_window": {
		t: &Trigger{
			logger:         log.New(os.Stderr, loggingPrefix, log.LstdFlags|log.Lmicroseconds),
			Debug:          true,
			MinOccurrences: 1,
			MaxOccurrences: 1,
			Window:         time.Minute,
			occurrencesTimes: []time.Time{
				time.Now().Add(-time.Hour),
			},
		},
		out: true,
		now: time.Now(),
	},
	"2max_1min_without_occurrences": {
		t: &Trigger{
			logger:           log.New(os.Stderr, loggingPrefix, log.LstdFlags|log.Lmicroseconds),
			Debug:            true,
			MinOccurrences:   1,
			MaxOccurrences:   2,
			Window:           time.Minute,
			occurrencesTimes: []time.Time{},
		},
		out: true,
		now: time.Now(),
	},
	"2max_1min_with_1occurrence_in_window": {
		t: &Trigger{
			logger:         log.New(os.Stderr, loggingPrefix, log.LstdFlags|log.Lmicroseconds),
			Debug:          true,
			MinOccurrences: 1,
			MaxOccurrences: 2,
			Window:         time.Minute,
			occurrencesTimes: []time.Time{
				time.Now().Add(-30 * time.Second),
			},
		},
		out: true,
		now: time.Now(),
	},
	"2max_1min_with_2occurrences_in_window": {
		t: &Trigger{
			logger:         log.New(os.Stderr, loggingPrefix, log.LstdFlags|log.Lmicroseconds),
			Debug:          true,
			MinOccurrences: 1,
			MaxOccurrences: 2,
			Window:         time.Minute,
			occurrencesTimes: []time.Time{
				time.Now().Add(-10 * time.Second),
				time.Now().Add(-30 * time.Second),
			},
		},
		out: false,
		now: time.Now(),
	},
	"2max_2min_without_occurrences": {
		t: &Trigger{
			logger:           log.New(os.Stderr, loggingPrefix, log.LstdFlags|log.Lmicroseconds),
			Debug:            true,
			MinOccurrences:   2,
			MaxOccurrences:   2,
			Window:           time.Minute,
			occurrencesTimes: []time.Time{},
		},
		out: false,
		now: time.Now(),
	},
	"2max_2min_with_1occurrence_in_window": {
		t: &Trigger{
			logger:         log.New(os.Stderr, loggingPrefix, log.LstdFlags|log.Lmicroseconds),
			Debug:          true,
			MinOccurrences: 2,
			MaxOccurrences: 2,
			Window:         time.Minute,
			occurrencesTimes: []time.Time{
				time.Now().Add(-30 * time.Second),
			},
		},
		out: true,
		now: time.Now(),
	},
	"2max_2min_with_2occurrences_in_window": {
		t: &Trigger{
			logger:         log.New(os.Stderr, loggingPrefix, log.LstdFlags|log.Lmicroseconds),
			Debug:          true,
			MinOccurrences: 2,
			MaxOccurrences: 2,
			Window:         time.Minute,
			occurrencesTimes: []time.Time{
				time.Now().Add(-10 * time.Second),
				time.Now().Add(-30 * time.Second),
			},
		},
		out: false,
		now: time.Now(),
	},
}

func TestEventTrigger(t *testing.T) {
	for name, ts := range testset {
		if pi, ok := formatters.EventProcessors[ts.processorType]; ok {
			t.Log("found processor")
			p := pi()
			err := p.Init(ts.processor, formatters.WithLogger(log.New(os.Stderr, loggingPrefix, log.LstdFlags|log.Lmicroseconds)))
			if err != nil {
				t.Errorf("failed to initialize processors: %v", err)
				return
			}
			t.Logf("processor: %+v", p)
			for i, item := range ts.tests {
				t.Run(name, func(t *testing.T) {
					t.Logf("running test item %d", i)
					outs := p.Apply(item.input...)
					for j := range outs {
						if !cmp.Equal(outs[j], item.output[j]) {
							t.Errorf("failed at %s item %d, index %d, expected %+v, got: %+v", name, i, j, item.output[j], outs[j])
						}
					}
				})
			}
		} else {
			t.Errorf("event processor %s not found", ts.processorType)
		}
	}
}

func TestOccurrenceTrigger(t *testing.T) {
	for name, ts := range triggerOccWindowTestSet {
		t.Run(name, func(t *testing.T) {
			ok := ts.t.evalOccurrencesWithinWindow(ts.now)
			t.Logf("%q result: %v", name, ok)
			if ok != ts.out {
				t.Errorf("failed at %s , expected %+v, got: %+v", name, ts.out, ok)
			}
		})
	}
}
<|endoftext|>


<commit_before>package config

import (
	"encoding/json"
	"fmt"

	"github.com/proidiot/gone/errors"
)

var typeRegistry = make(map[string]func() json.Unmarshaler)

func RegisterResourceType(
	typeName string,
	newFunc func() json.Unmarshaler,
) error {
	_, present := typeRegistry[typeName]
	if present || typeName == ReferenceResourceTypeName {
		return errors.New(
			fmt.Sprintf(
				"More than one resource type has registerred"+
					" the same name: %s",
				typeName,
			),
		)
	}

	typeRegistry[typeName] = newFunc
	return nil
}

func MustRegisterResourceType(
	typeName string,
	newFunc func() json.Unmarshaler,
) {
	e := RegisterResourceType(typeName, newFunc)
	if e != nil {
		panic(e)
	}
}

<commit_msg>Gofmt<commit_after>package config

import (
	"encoding/json"
	"fmt"

	"github.com/proidiot/gone/errors"
)

var typeRegistry = make(map[string]func() json.Unmarshaler)

func RegisterResourceType(
	typeName string,
	newFunc func() json.Unmarshaler,
) error {
	_, present := typeRegistry[typeName]
	if present || typeName == ReferenceResourceTypeName {
		return errors.New(
			fmt.Sprintf(
				"More than one resource type has registerred"+
					" the same name: %s",
				typeName,
			),
		)
	}

	typeRegistry[typeName] = newFunc
	return nil
}

func MustRegisterResourceType(
	typeName string,
	newFunc func() json.Unmarshaler,
) {
	e := RegisterResourceType(typeName, newFunc)
	if e != nil {
		panic(e)
	}
}
<|endoftext|>


<commit_before>package consumer

import (
	"container/list"
	"encoding/json"
	"flume-log-sdk/config"
	"flume-log-sdk/consumer/client"
	"flume-log-sdk/consumer/pool"
	"flume-log-sdk/rpc/flume"
	"fmt"
	"log"
	"math/rand"
	"sync/atomic"
	"time"
)

type counter struct {
	lastSuccValue int64

	currSuccValue int64

	lastFailValue int64

	currFailValue int64
}

// 用于向flume中作为sink 通过thrift客户端写入日志

type SourceServer struct {
	flumeClientPool *list.List
	isStop          bool
	monitorCount    counter
	business        string
	batchSize       int
	buffChannel     chan *flume.ThriftFlumeEvent
}

func newSourceServer(business string, flumePool *list.List) (server *SourceServer) {
	batchSize := 300
	sendbuff := 500
	buffChannel := make(chan *flume.ThriftFlumeEvent, sendbuff)
	sourceServer := &SourceServer{
		business:        business,
		flumeClientPool: flumePool,
		batchSize:       batchSize,
		buffChannel:     buffChannel}
	return sourceServer
}

func (self *SourceServer) monitor() (succ, fail int64, bufferSize int) {
	currSucc := self.monitorCount.currSuccValue
	currFail := self.monitorCount.currFailValue
	succ = (currSucc - self.monitorCount.lastSuccValue)
	fail = (currFail - self.monitorCount.lastFailValue)
	self.monitorCount.lastSuccValue = currSucc
	self.monitorCount.lastFailValue = currFail

	//自己的Buffer大小
	bufferSize = len(self.buffChannel)
	return
}

//启动pop
func (self *SourceServer) start() {

	self.isStop = false

	//创建chan ,buffer 为10
	sendbuff := make(chan []*flume.ThriftFlumeEvent, self.batchSize)
	//启动20个go程从channel获取
	for i := 0; i < 10; i++ {
		go func(ch chan []*flume.ThriftFlumeEvent) {
			for !self.isStop {
				events := <-ch
				self.innerSend(events)
			}
		}(sendbuff)
	}

	go func() {
		//批量收集数据
		pack := make([]*flume.ThriftFlumeEvent, 0, self.batchSize)
		for !self.isStop {
			event := <-self.buffChannel
			//如果总数大于batchsize则提交
			if len(pack) < self.batchSize {
				//批量提交
				pack = append(pack, event)
				continue
			}
			sendbuff <- pack[:len(pack)]
			pack = make([]*flume.ThriftFlumeEvent, 0, self.batchSize)
		}

		close(sendbuff)
	}()

	log.Printf("LOG_SOURCE|SOURCE SERVER [%s]|STARTED\n", self.business)
}

func (self *SourceServer) innerSend(events []*flume.ThriftFlumeEvent) {

	for i := 0; i < 3; i++ {
		pool := self.getFlumeClientPool()
		flumeclient, err := pool.Get(5 * time.Second)
		if nil != err || nil == flumeclient {
			log.Printf("LOG_SOURCE|GET FLUMECLIENT|FAIL|%s|%s|TRY:%d\n", self.business, err, i)
			continue
		}

		err = flumeclient.AppendBatch(events)
		defer func() {
			if err := recover(); nil != err {
				//回收这个坏的连接
				pool.ReleaseBroken(flumeclient)
			} else {
				pool.Release(flumeclient)
			}
		}()

		if nil != err {
			atomic.AddInt64(&self.monitorCount.currFailValue, int64(1*self.batchSize))
			log.Printf("LOG_SOURCE|SEND FLUME|FAIL|%s|%s|TRY:%d\n", self.business, err.Error(), i)

		} else {
			atomic.AddInt64(&self.monitorCount.currSuccValue, int64(1*self.batchSize))
			if rand.Int()%10000 == 0 {
				log.Printf("trace|send 2 flume succ|%s|%d\n", flumeclient.HostPort(), len(events))
			}
			break
		}

	}
}

//解析出decodecommand
func decodeCommand(resp []byte) (string, *flume.ThriftFlumeEvent) {
	var cmd config.Command
	err := json.Unmarshal(resp, &cmd)
	if nil != err {
		log.Printf("command unmarshal fail ! %T | error:%s\n", resp, err.Error())
		return "", nil
	}
	//
	momoid := cmd.Params["momoid"].(string)

	businessName := cmd.Params["businessName"].(string)

	action := cmd.Params["type"].(string)

	bodyContent := cmd.Params["body"]

	//将businessName 加入到body中
	bodyMap := bodyContent.(map[string]interface{})
	bodyMap["business_type"] = businessName

	body, err := json.Marshal(bodyContent)
	if nil != err {
		log.Printf("marshal log body fail %s", err.Error())
		return businessName, nil
	}

	//拼Body
	flumeBody := fmt.Sprintf("%s\t%s\t%s", momoid, action, string(body))
	event := client.NewFlumeEvent(businessName, action, []byte(flumeBody))
	return businessName, event
}

func (self *SourceServer) stop() {
	self.isStop = true
	time.Sleep(5 * time.Second)

	//遍历所有的flumeclientlink，将当前Business从该链表中移除
	for v := self.flumeClientPool.Back(); nil != v; v = v.Prev() {
		v.Value.(*pool.FlumePoolLink).DetachBusiness(self.business)
	}
	close(self.buffChannel)
	log.Printf("LOG_SOURCE|SOURCE SERVER|[%s]|STOPPED\n", self.business)
}

func (self *SourceServer) getFlumeClientPool() *pool.FlumeClientPool {

	//采用轮训算法
	e := self.flumeClientPool.Back()
	self.flumeClientPool.MoveToFront(e)
	return e.Value.(*pool.FlumePoolLink).FlumePool

}
<commit_msg>	modified:   consumer/log_source.go<commit_after>package consumer

import (
	"container/list"
	"encoding/json"
	"flume-log-sdk/config"
	"flume-log-sdk/consumer/client"
	"flume-log-sdk/consumer/pool"
	"flume-log-sdk/rpc/flume"
	"fmt"
	"log"
	"math/rand"
	"sync/atomic"
	"time"
)

type counter struct {
	lastSuccValue int64

	currSuccValue int64

	lastFailValue int64

	currFailValue int64
}

// 用于向flume中作为sink 通过thrift客户端写入日志

type SourceServer struct {
	flumeClientPool *list.List
	isStop          bool
	monitorCount    counter
	business        string
	batchSize       int
	buffChannel     chan *flume.ThriftFlumeEvent
}

func newSourceServer(business string, flumePool *list.List) (server *SourceServer) {
	batchSize := 300
	sendbuff := 500
	buffChannel := make(chan *flume.ThriftFlumeEvent, sendbuff)
	sourceServer := &SourceServer{
		business:        business,
		flumeClientPool: flumePool,
		batchSize:       batchSize,
		buffChannel:     buffChannel}
	return sourceServer
}

func (self *SourceServer) monitor() (succ, fail int64, bufferSize int) {
	currSucc := self.monitorCount.currSuccValue
	currFail := self.monitorCount.currFailValue
	succ = (currSucc - self.monitorCount.lastSuccValue)
	fail = (currFail - self.monitorCount.lastFailValue)
	self.monitorCount.lastSuccValue = currSucc
	self.monitorCount.lastFailValue = currFail

	//自己的Buffer大小
	bufferSize = len(self.buffChannel)
	return
}

//启动pop
func (self *SourceServer) start() {

	self.isStop = false

	//创建chan ,buffer 为10
	sendbuff := make(chan []*flume.ThriftFlumeEvent, self.batchSize)
	//启动20个go程从channel获取
	for i := 0; i < 10; i++ {
		go func(ch chan []*flume.ThriftFlumeEvent) {
			for !self.isStop {
				events := <-ch
				self.innerSend(events)
			}
		}(sendbuff)
	}

	go func() {
		//批量收集数据
		pack := make([]*flume.ThriftFlumeEvent, 0, self.batchSize)
		for !self.isStop {
			event := <-self.buffChannel
			//如果总数大于batchsize则提交
			if len(pack) < self.batchSize {
				//批量提交
				pack = append(pack, event)
				continue
			}
			sendbuff <- pack[:len(pack)]
			pack = make([]*flume.ThriftFlumeEvent, 0, self.batchSize)
		}

		close(sendbuff)
	}()

	log.Printf("LOG_SOURCE|SOURCE SERVER [%s]|STARTED\n", self.business)
}

func (self *SourceServer) innerSend(events []*flume.ThriftFlumeEvent) {

	for i := 0; i < 3; i++ {
		pool := self.getFlumeClientPool()
		flumeclient, err := pool.Get(5 * time.Second)
		if nil != err || nil == flumeclient {
			log.Printf("LOG_SOURCE|GET FLUMECLIENT|FAIL|%s|%s|TRY:%d\n", self.business, err, i)
			continue
		}

		err = flumeclient.AppendBatch(events)
		defer func() {
			if err := recover(); nil != err {
				//回收这个坏的连接
				pool.ReleaseBroken(flumeclient)
			} else {
				pool.Release(flumeclient)
			}
		}()

		if nil != err {
			atomic.AddInt64(&self.monitorCount.currFailValue, int64(len(events)))
			log.Printf("LOG_SOURCE|SEND FLUME|FAIL|%s|%s|TRY:%d\n", self.business, err.Error(), i)

		} else {
			atomic.AddInt64(&self.monitorCount.currSuccValue, int64(1*self.batchSize))
			if rand.Int()%10000 == 0 {
				log.Printf("trace|send 2 flume succ|%s|%d\n", flumeclient.HostPort(), len(events))
			}
			break
		}

	}
}

//解析出decodecommand
func decodeCommand(resp []byte) (string, *flume.ThriftFlumeEvent) {
	var cmd config.Command
	err := json.Unmarshal(resp, &cmd)
	if nil != err {
		log.Printf("command unmarshal fail ! %T | error:%s\n", resp, err.Error())
		return "", nil
	}
	//
	momoid := cmd.Params["momoid"].(string)

	businessName := cmd.Params["businessName"].(string)

	action := cmd.Params["type"].(string)

	bodyContent := cmd.Params["body"]

	//将businessName 加入到body中
	bodyMap := bodyContent.(map[string]interface{})
	bodyMap["business_type"] = businessName

	body, err := json.Marshal(bodyContent)
	if nil != err {
		log.Printf("marshal log body fail %s", err.Error())
		return businessName, nil
	}

	//拼Body
	flumeBody := fmt.Sprintf("%s\t%s\t%s", momoid, action, string(body))
	event := client.NewFlumeEvent(businessName, action, []byte(flumeBody))
	return businessName, event
}

func (self *SourceServer) stop() {
	self.isStop = true
	time.Sleep(5 * time.Second)

	//遍历所有的flumeclientlink，将当前Business从该链表中移除
	for v := self.flumeClientPool.Back(); nil != v; v = v.Prev() {
		v.Value.(*pool.FlumePoolLink).DetachBusiness(self.business)
	}
	close(self.buffChannel)
	log.Printf("LOG_SOURCE|SOURCE SERVER|[%s]|STOPPED\n", self.business)
}

func (self *SourceServer) getFlumeClientPool() *pool.FlumeClientPool {

	//采用轮训算法
	e := self.flumeClientPool.Back()
	self.flumeClientPool.MoveToFront(e)
	return e.Value.(*pool.FlumePoolLink).FlumePool

}
<|endoftext|>


<commit_before>package digests

import (
	"fmt"

	"github.com/ion-channel/ionic/scanner"
	"github.com/ion-channel/ionic/scans"
)

func buildsystemsDigests(status *scanner.ScanStatus, eval *scans.Evaluation) ([]Digest, error) {
	digests := make([]Digest, 0)

	c, err := createCompilerDigests(status, eval)
	if err != nil {
		return nil, err
	}
	digests = append(digests, *c)

	i, err := createImagesDigests(status, eval)
	if err != nil {
		return nil, err
	}
	digests = append(digests, *i)

	return digests, nil
}

func createCompilerDigests(status *scanner.ScanStatus, eval *scans.Evaluation) (*Digest, error) {
	d := NewDigest(status, compilersIndex, "compiler", "compilers")

	if eval != nil && !status.Errored() {
		b, ok := eval.TranslatedResults.Data.(scans.BuildsystemResults)
		if !ok {
			return nil, fmt.Errorf("error coercing evaluation translated results into buildss bytes")
		}

		d.MarshalSourceData(b, "buildsystems")

		switch len(b.Compilers) {
		case 0:
			err := d.AppendEval(eval, "chars", "none detected")
			if err != nil {
				return nil, fmt.Errorf("failed to create builds digest: %v", err.Error())
			}
		case 1:
			n := ""
			for _, c := range b.Compilers {
				n = c.Name
			}

			err := d.AppendEval(eval, "chars", n)
			if err != nil {
				return nil, fmt.Errorf("failed to create builds digest: %v", err.Error())
			}

			d.UseSingularTitle()
		default:
			err := d.AppendEval(eval, "count", len(b.Compilers))
			if err != nil {
				return nil, fmt.Errorf("failed to create builds digest: %v", err.Error())
			}
		}

		d.Evaluated = false // As of now there's no rule to evaluate this against so it's set to not evaluated.
	}

	return d, nil
}

func createImagesDigests(status *scanner.ScanStatus, eval *scans.Evaluation) (*Digest, error) {
	d := NewDigest(status, containerImagesIndex, "container image", "container images")

	if eval != nil && !status.Errored() {
		b, ok := eval.TranslatedResults.Data.(scans.BuildsystemResults)
		if !ok {
			return nil, fmt.Errorf("error coercing evaluation translated results into buildss bytes")
		}

		d.MarshalSourceData(b, "buildsystems")

		switch len(b.Dockerfile.Images) {
		case 0:
			err := d.AppendEval(eval, "chars", "none detected")
			if err != nil {
				return nil, fmt.Errorf("failed to create builds digest: %v", err.Error())
			}
		case 1:
			n := ""
			for _, c := range b.Dockerfile.Images {
				n = c.Name
			}

			err := d.AppendEval(eval, "chars", n)
			if err != nil {
				return nil, fmt.Errorf("failed to create builds digest: %v", err.Error())
			}

			d.UseSingularTitle()
		default:
			err := d.AppendEval(eval, "count", len(b.Dockerfile.Images))
			if err != nil {
				return nil, fmt.Errorf("failed to create builds digest: %v", err.Error())
			}
		}

		d.Evaluated = false // As of now there's no rule to evaluate this against so it's set to not evaluated.
	}

	return d, nil
}
<commit_msg>use only the relevant source data for buildsystems<commit_after>package digests

import (
	"fmt"

	"github.com/ion-channel/ionic/scanner"
	"github.com/ion-channel/ionic/scans"
)

func buildsystemsDigests(status *scanner.ScanStatus, eval *scans.Evaluation) ([]Digest, error) {
	digests := make([]Digest, 0)

	c, err := createCompilerDigests(status, eval)
	if err != nil {
		return nil, err
	}
	digests = append(digests, *c)

	i, err := createImagesDigests(status, eval)
	if err != nil {
		return nil, err
	}
	digests = append(digests, *i)

	return digests, nil
}

func createCompilerDigests(status *scanner.ScanStatus, eval *scans.Evaluation) (*Digest, error) {
	d := NewDigest(status, compilersIndex, "compiler", "compilers")

	if eval != nil && !status.Errored() {
		b, ok := eval.TranslatedResults.Data.(scans.BuildsystemResults)
		if !ok {
			return nil, fmt.Errorf("error coercing evaluation translated results into buildss bytes")
		}

		d.MarshalSourceData(b.Compilers, "compilers")

		switch len(b.Compilers) {
		case 0:
			err := d.AppendEval(eval, "chars", "none detected")
			if err != nil {
				return nil, fmt.Errorf("failed to create builds digest: %v", err.Error())
			}
		case 1:
			n := ""
			for _, c := range b.Compilers {
				n = c.Name
			}

			err := d.AppendEval(eval, "chars", n)
			if err != nil {
				return nil, fmt.Errorf("failed to create builds digest: %v", err.Error())
			}

			d.UseSingularTitle()
		default:
			err := d.AppendEval(eval, "count", len(b.Compilers))
			if err != nil {
				return nil, fmt.Errorf("failed to create builds digest: %v", err.Error())
			}
		}

		d.Evaluated = false // As of now there's no rule to evaluate this against so it's set to not evaluated.
	}

	return d, nil
}

func createImagesDigests(status *scanner.ScanStatus, eval *scans.Evaluation) (*Digest, error) {
	d := NewDigest(status, containerImagesIndex, "container image", "container images")

	if eval != nil && !status.Errored() {
		b, ok := eval.TranslatedResults.Data.(scans.BuildsystemResults)
		if !ok {
			return nil, fmt.Errorf("error coercing evaluation translated results into buildss bytes")
		}

		d.MarshalSourceData(b.Dockerfile.Images, "container images")

		switch len(b.Dockerfile.Images) {
		case 0:
			err := d.AppendEval(eval, "chars", "none detected")
			if err != nil {
				return nil, fmt.Errorf("failed to create builds digest: %v", err.Error())
			}
		case 1:
			n := ""
			for _, c := range b.Dockerfile.Images {
				n = c.Name
			}

			err := d.AppendEval(eval, "chars", n)
			if err != nil {
				return nil, fmt.Errorf("failed to create builds digest: %v", err.Error())
			}

			d.UseSingularTitle()
		default:
			err := d.AppendEval(eval, "count", len(b.Dockerfile.Images))
			if err != nil {
				return nil, fmt.Errorf("failed to create builds digest: %v", err.Error())
			}
		}

		d.Evaluated = false // As of now there's no rule to evaluate this against so it's set to not evaluated.
	}

	return d, nil
}
<|endoftext|>


<commit_before>package digitaloceanexporter

import (
	"strconv"

	"github.com/digitalocean/godo"
)

// DigitalOceanService is a wrapper around godo.Client.
type DigitalOceanService struct {
	C *godo.Client
}

// DropletCounter is a struct holding information about a Droplet.
type DropletCounter struct {
	status string
	region string
	size   string
}

// FlipCounter is a struct holding information about a Floating IP.
type FlipCounter struct {
	status string
	region string
}

// VolumeCounter is a struct holding information about a Block Storage Volume.
type VolumeCounter struct {
	status string
	region string
	size   string
}

var pageOpt = &godo.ListOptions{
	Page:    1,
	PerPage: 200,
}

// Droplets retrieves a count of Droplets grouped by status, size, and region.
func (s *DigitalOceanService) Droplets() (map[DropletCounter]int, error) {
	droplets, err := listDroplets(s)

	counters := make(map[DropletCounter]int)

	for _, d := range droplets {
		c := DropletCounter{
			d.Status,
			d.Region.Slug,
			d.Size.Slug,
		}
		counters[c]++
	}

	return counters, err
}

func listDroplets(s *DigitalOceanService) ([]godo.Droplet, error) {
	dropletList := []godo.Droplet{}

	for {
		droplets, resp, err := s.C.Droplets.List(pageOpt)

		if err != nil {
			return nil, err
		}

		for _, d := range droplets {
			dropletList = append(dropletList, d)
		}

		if resp.Links == nil || resp.Links.IsLastPage() {
			break
		}

		page, err := resp.Links.CurrentPage()
		if err != nil {
			return nil, err
		}

		pageOpt.Page = page + 1
	}

	return dropletList, nil
}

// FloatingIPs retrieves a count of Floating IPs grouped by status and region.
func (s *DigitalOceanService) FloatingIPs() (map[FlipCounter]int, error) {
	fips, err := listFips(s)

	counters := make(map[FlipCounter]int)

	for _, fip := range fips {
		var status string

		switch {
		case fip.Droplet == nil:
			status = "unassigned"
		default:
			status = "assigned"
		}
		c := FlipCounter{
			status,
			fip.Region.Slug,
		}
		counters[c]++
	}

	return counters, err
}

func listFips(s *DigitalOceanService) ([]godo.FloatingIP, error) {
	fipList := []godo.FloatingIP{}

	for {
		fips, resp, err := s.C.FloatingIPs.List(pageOpt)

		if err != nil {
			return nil, err
		}

		for _, f := range fips {
			fipList = append(fipList, f)
		}

		if resp.Links == nil || resp.Links.IsLastPage() {
			break
		}

		page, err := resp.Links.CurrentPage()
		if err != nil {
			return nil, err
		}

		pageOpt.Page = page + 1
	}

	return fipList, nil
}

// Volumes retrieves a count of Volumes grouped by status, size, and region.
func (s *DigitalOceanService) Volumes() (map[VolumeCounter]int, error) {
	volumes, err := listVolumes(s)

	counters := make(map[VolumeCounter]int)

	for _, v := range volumes {
		var status string

		switch {
		case len(v.DropletIDs) > 0:
			status = "attached"
		default:
			status = "unattached"
		}
		c := VolumeCounter{
			status,
			v.Region.Slug,
			strconv.FormatInt(v.SizeGigaBytes, 10),
		}
		counters[c]++
	}

	return counters, err
}

func listVolumes(s *DigitalOceanService) ([]godo.Volume, error) {
	volumeList := []godo.Volume{}

	for {
		fips, resp, err := s.C.Storage.ListVolumes(pageOpt)

		if err != nil {
			return nil, err
		}

		for _, f := range fips {
			volumeList = append(volumeList, f)
		}

		if resp.Links == nil || resp.Links.IsLastPage() {
			break
		}

		page, err := resp.Links.CurrentPage()
		if err != nil {
			return nil, err
		}

		pageOpt.Page = page + 1
	}

	return volumeList, nil
}
<commit_msg>Pass context.Context to godo<commit_after>package digitaloceanexporter

import (
	"context"
	"strconv"

	"github.com/digitalocean/godo"
)

// DigitalOceanService is a wrapper around godo.Client.
type DigitalOceanService struct {
	C *godo.Client
}

// DropletCounter is a struct holding information about a Droplet.
type DropletCounter struct {
	status string
	region string
	size   string
}

// FlipCounter is a struct holding information about a Floating IP.
type FlipCounter struct {
	status string
	region string
}

// VolumeCounter is a struct holding information about a Block Storage Volume.
type VolumeCounter struct {
	status string
	region string
	size   string
}

var pageOpt = &godo.ListOptions{
	Page:    1,
	PerPage: 200,
}

// Droplets retrieves a count of Droplets grouped by status, size, and region.
func (s *DigitalOceanService) Droplets() (map[DropletCounter]int, error) {
	droplets, err := listDroplets(s)

	counters := make(map[DropletCounter]int)

	for _, d := range droplets {
		c := DropletCounter{
			d.Status,
			d.Region.Slug,
			d.Size.Slug,
		}
		counters[c]++
	}

	return counters, err
}

func listDroplets(s *DigitalOceanService) ([]godo.Droplet, error) {
	ctx := context.TODO()
	dropletList := []godo.Droplet{}

	for {
		droplets, resp, err := s.C.Droplets.List(ctx, pageOpt)

		if err != nil {
			return nil, err
		}

		for _, d := range droplets {
			dropletList = append(dropletList, d)
		}

		if resp.Links == nil || resp.Links.IsLastPage() {
			break
		}

		page, err := resp.Links.CurrentPage()
		if err != nil {
			return nil, err
		}

		pageOpt.Page = page + 1
	}

	return dropletList, nil
}

// FloatingIPs retrieves a count of Floating IPs grouped by status and region.
func (s *DigitalOceanService) FloatingIPs() (map[FlipCounter]int, error) {
	fips, err := listFips(s)

	counters := make(map[FlipCounter]int)

	for _, fip := range fips {
		var status string

		switch {
		case fip.Droplet == nil:
			status = "unassigned"
		default:
			status = "assigned"
		}
		c := FlipCounter{
			status,
			fip.Region.Slug,
		}
		counters[c]++
	}

	return counters, err
}

func listFips(s *DigitalOceanService) ([]godo.FloatingIP, error) {
	ctx := context.TODO()
	fipList := []godo.FloatingIP{}

	for {
		fips, resp, err := s.C.FloatingIPs.List(ctx, pageOpt)

		if err != nil {
			return nil, err
		}

		for _, f := range fips {
			fipList = append(fipList, f)
		}

		if resp.Links == nil || resp.Links.IsLastPage() {
			break
		}

		page, err := resp.Links.CurrentPage()
		if err != nil {
			return nil, err
		}

		pageOpt.Page = page + 1
	}

	return fipList, nil
}

// Volumes retrieves a count of Volumes grouped by status, size, and region.
func (s *DigitalOceanService) Volumes() (map[VolumeCounter]int, error) {
	volumes, err := listVolumes(s)

	counters := make(map[VolumeCounter]int)

	for _, v := range volumes {
		var status string

		switch {
		case len(v.DropletIDs) > 0:
			status = "attached"
		default:
			status = "unattached"
		}
		c := VolumeCounter{
			status,
			v.Region.Slug,
			strconv.FormatInt(v.SizeGigaBytes, 10),
		}
		counters[c]++
	}

	return counters, err
}

func listVolumes(s *DigitalOceanService) ([]godo.Volume, error) {
	ctx := context.TODO()
	volumeList := []godo.Volume{}
	volumeParams := &godo.ListVolumeParams{
		ListOptions: pageOpt,
	}

	for {
		volumes, resp, err := s.C.Storage.ListVolumes(ctx, volumeParams)

		if err != nil {
			return nil, err
		}

		for _, v := range volumes {
			volumeList = append(volumeList, v)
		}

		if resp.Links == nil || resp.Links.IsLastPage() {
			break
		}

		page, err := resp.Links.CurrentPage()
		if err != nil {
			return nil, err
		}

		volumeParams.ListOptions.Page = page + 1
	}

	return volumeList, nil
}
<|endoftext|>


<commit_before>// Copyright 2017 Intel Corporation.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"flag"
	"fmt"
	"github.com/intel-go/yanff/flow"
	"github.com/intel-go/yanff/packet"
	"os"
)

var (
	outport uint
	inport  uint

	cloneNumber uint
)

func main() {
	flag.UintVar(&outport, "outport", 1, "port for sender")
	flag.UintVar(&inport, "inport", 0, "port for receiver")
	flag.Parse()

	// Initialize YANFF library at 10 available cores
	config := flow.Config{
		CPUCoresNumber: 10,
	}
	flow.SystemInit(&config)

	// Receive packets from zero port. One queue will be added automatically.
	f1 := flow.SetReceiver(uint8(inport))

	var pdp pcapdumperParameters
	flow.SetHandler(f1, pcapdumper, &pdp)

	// Send packets to control speed. One queue will be added automatically.
	flow.SetSender(f1, uint8(outport))

	flow.SystemStart()
}

type pcapdumperParameters struct {
	f *os.File
}

func (pd pcapdumperParameters) Copy() interface{} {
	filename := fmt.Sprintf("dumped%d.pcap", cloneNumber)
	f, err := os.Create(filename)
	if err != nil {
		fmt.Println("Cannot create file: ", err)
		os.Exit(0)
	}
	cloneNumber += 1
	packet.WritePcapGlobalHdr(f)
	pdp := pcapdumperParameters{f: f}
	return pdp
}

func (pd pcapdumperParameters) Delete() {
}

func pcapdumper(currentPacket *packet.Packet, context flow.UserContext) {
	pd := context.(pcapdumperParameters)
	currentPacket.WritePcapOnePacket(pd.f)
}
<commit_msg>Implement Delete method (addition to PR #87)<commit_after>// Copyright 2017 Intel Corporation.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"flag"
	"fmt"
	"github.com/intel-go/yanff/flow"
	"github.com/intel-go/yanff/packet"
	"os"
)

var (
	outport uint
	inport  uint

	cloneNumber uint
)

func main() {
	flag.UintVar(&outport, "outport", 1, "port for sender")
	flag.UintVar(&inport, "inport", 0, "port for receiver")
	flag.Parse()

	// Initialize YANFF library at 10 available cores
	config := flow.Config{
		CPUCoresNumber: 10,
	}
	flow.SystemInit(&config)

	// Receive packets from zero port. One queue will be added automatically.
	f1 := flow.SetReceiver(uint8(inport))

	var pdp pcapdumperParameters
	flow.SetHandler(f1, pcapdumper, &pdp)

	// Send packets to control speed. One queue will be added automatically.
	flow.SetSender(f1, uint8(outport))

	flow.SystemStart()
}

type pcapdumperParameters struct {
	f *os.File
}

func (pd pcapdumperParameters) Copy() interface{} {
	filename := fmt.Sprintf("dumped%d.pcap", cloneNumber)
	f, err := os.Create(filename)
	if err != nil {
		fmt.Println("Cannot create file: ", err)
		os.Exit(0)
	}
	cloneNumber += 1
	packet.WritePcapGlobalHdr(f)
	pdp := pcapdumperParameters{f: f}
	return pdp
}

func (pd pcapdumperParameters) Delete() {
	pd.f.Close()
}

func pcapdumper(currentPacket *packet.Packet, context flow.UserContext) {
	pd := context.(pcapdumperParameters)
	currentPacket.WritePcapOnePacket(pd.f)
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"

	"github.com/ttacon/box"
	"github.com/ttacon/pretty"
	"golang.org/x/oauth2"
)

var (
	clientId     = flag.String("cid", "", "OAuth Client ID")
	clientSecret = flag.String("csec", "", "OAuth Client Secret")

	accessToken  = flag.String("atok", "", "Access Token")
	refreshToken = flag.String("rtok", "", "Refresh Token")
)

func main() {
	flag.Parse()

	if len(*clientId) == 0 || len(*clientSecret) == 0 ||
		len(*accessToken) == 0 || len(*refreshToken) == 0 {
		fmt.Println("unfortunately all flags must be provided")
		return
	}

	// Set our OAuth2 configuration up
	var (
		configSource = box.NewConfigSource(
			&oauth2.Config{
				ClientID:     *clientId,
				ClientSecret: *clientSecret,
				Scopes:       nil,
				Endpoint: oauth2.Endpoint{
					AuthURL:  "https://app.box.com/api/oauth2/authorize",
					TokenURL: "https://app.box.com/api/oauth2/token",
				},
				RedirectURL: "http://localhost:8080/handle",
			},
		)
		tok = &oauth2.Token{
			TokenType:    "Bearer",
			AccessToken:  *accessToken,
			RefreshToken: *refreshToken,
		}
		c = configSource.NewClient(tok)
	)

	resp, longPollInfo, err := c.EventService().LongPollURL()
	fmt.Println("resp: ", resp)
	fmt.Println("err: ", err)
	pretty.Print(longPollInfo)

	// Print out the new tokens for next time
	fmt.Printf("\n%#v\n", tok)

	connInfo := longPollInfo.Entries[0]
	resp, event, _, err := c.EventService().ListenForEvent(connInfo, "")
	fmt.Println("resp: ", resp)
	fmt.Println("err: ", err)
	pretty.Println(event)
}
<commit_msg>remove non-removed return value from example<commit_after>package main

import (
	"flag"
	"fmt"

	"github.com/ttacon/box"
	"github.com/ttacon/pretty"
	"golang.org/x/oauth2"
)

var (
	clientId     = flag.String("cid", "", "OAuth Client ID")
	clientSecret = flag.String("csec", "", "OAuth Client Secret")

	accessToken  = flag.String("atok", "", "Access Token")
	refreshToken = flag.String("rtok", "", "Refresh Token")
)

func main() {
	flag.Parse()

	if len(*clientId) == 0 || len(*clientSecret) == 0 ||
		len(*accessToken) == 0 || len(*refreshToken) == 0 {
		fmt.Println("unfortunately all flags must be provided")
		return
	}

	// Set our OAuth2 configuration up
	var (
		configSource = box.NewConfigSource(
			&oauth2.Config{
				ClientID:     *clientId,
				ClientSecret: *clientSecret,
				Scopes:       nil,
				Endpoint: oauth2.Endpoint{
					AuthURL:  "https://app.box.com/api/oauth2/authorize",
					TokenURL: "https://app.box.com/api/oauth2/token",
				},
				RedirectURL: "http://localhost:8080/handle",
			},
		)
		tok = &oauth2.Token{
			TokenType:    "Bearer",
			AccessToken:  *accessToken,
			RefreshToken: *refreshToken,
		}
		c = configSource.NewClient(tok)
	)

	resp, longPollInfo, err := c.EventService().LongPollURL()
	fmt.Println("resp: ", resp)
	fmt.Println("err: ", err)
	pretty.Print(longPollInfo)

	// Print out the new tokens for next time
	fmt.Printf("\n%#v\n", tok)

	connInfo := longPollInfo.Entries[0]
	resp, event, err := c.EventService().ListenForEvent(connInfo, "")
	fmt.Println("resp: ", resp)
	fmt.Println("err: ", err)
	pretty.Println(event)
}
<|endoftext|>


<commit_before>// FIXME there must be a better way to pad constants in templates
package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"sort"
	"strings"
	"text/template"

	"github.com/twpayne/go-doarama"
)

var tmpl = template.Must(template.New("gat").Parse("" +
	"package doarama\n" +
	"\n" +
	"// Activity types\n" +
	"const (\n" +
	"{{range $constName, $id := .ConstActivityIDs}}\t{{$constName}} = {{$id}}\n" +
	"{{end}})\n" +
	"\n" +
	"// DefaultActivityTypes contains the default activity types.\n" +
	"var DefaultActivityTypes = ActivityTypes{\n" +
	"{{range $at := .ActivityTypes}}\t{ID: {{$at.ID}}, Name: {{$at.Name | printf \"%#v\"}}},\n" +
	"{{end}}}\n" +
	"\n" +
	"//go:generate go run cmd/generate-activity-types/generate-activity-types.go -o {{.Filename}}\n",
))

func constantize(s string) ([]string, error) {
	var result []string
	ss := strings.SplitN(s, " - ", 2)
	switch len(ss) {
	case 1:
		result = append(result, ss[0])
	case 2:
		prefix := ss[0]
		for _, desc := range strings.Split(ss[1], "/") {
			suffix := desc
			suffix = strings.TrimSuffix(suffix, " etc")
			suffix = strings.Replace(suffix, " ", "", -1)
			suffix = strings.TrimSpace(suffix)
			suffix = strings.Replace(suffix, "+", "And", -1)
			result = append(result, prefix+suffix)
		}
	default:
		return nil, fmt.Errorf("unable to parse %v", s)
	}
	return result, nil
}

func max(x, y int) int {
	if x > y {
		return x
	} else {
		return y
	}
}

func pad(s string, n int) string {
	m := len(s)
	switch {
	case m < n:
		return s + strings.Repeat(" ", n-m)
	case m == n:
		return s
	default:
		return s[:m]
	}
}

type byID doarama.ActivityTypes

func (ats byID) Len() int           { return len(ats) }
func (ats byID) Less(i, j int) bool { return ats[i].ID < ats[j].ID }
func (ats byID) Swap(i, j int)      { ats[i], ats[j] = ats[j], ats[i] }

func generateActivityIDs(filename string) error {
	f, err := os.Create(filename)
	if err != nil {
		return err
	}
	defer f.Close()
	client := doarama.NewClient(doarama.APIURL, "", "")
	activityTypes, err := client.ActivityTypes()
	if err != nil {
		return err
	}
	sort.Sort(byID(activityTypes))
	maxConstLen := 0
	maxNameLen := 0
	constActivityIDs := make(map[string]int)
	for _, at := range activityTypes {
		maxNameLen = max(maxNameLen, len(at.Name))
		ss, err := constantize(at.Name)
		if err != nil {
			return err
		}
		for _, s := range ss {
			constActivityIDs[s] = at.ID
			maxConstLen = max(maxConstLen, len(s))
		}
	}
	paddedConstActivityIDs := make(map[string]int)
	for name, id := range constActivityIDs {
		paddedConstActivityIDs[pad(name, maxConstLen)] = id
	}
	if err := tmpl.Execute(f, struct {
		ActivityTypes    doarama.ActivityTypes
		ConstActivityIDs map[string]int
		Filename         string
	}{
		ActivityTypes:    activityTypes,
		ConstActivityIDs: paddedConstActivityIDs,
		Filename:         filename,
	}); err != nil {
		return err
	}
	return nil
}

func main() {
	o := flag.String("o", "", "")
	flag.Parse()
	if err := generateActivityIDs(*o); err != nil {
		log.Fatal(err)
	}
}
<commit_msg>Restructure function to satisfy golint<commit_after>// FIXME there must be a better way to pad constants in templates
package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"sort"
	"strings"
	"text/template"

	"github.com/twpayne/go-doarama"
)

var tmpl = template.Must(template.New("gat").Parse("" +
	"package doarama\n" +
	"\n" +
	"// Activity types\n" +
	"const (\n" +
	"{{range $constName, $id := .ConstActivityIDs}}\t{{$constName}} = {{$id}}\n" +
	"{{end}})\n" +
	"\n" +
	"// DefaultActivityTypes contains the default activity types.\n" +
	"var DefaultActivityTypes = ActivityTypes{\n" +
	"{{range $at := .ActivityTypes}}\t{ID: {{$at.ID}}, Name: {{$at.Name | printf \"%#v\"}}},\n" +
	"{{end}}}\n" +
	"\n" +
	"//go:generate go run cmd/generate-activity-types/generate-activity-types.go -o {{.Filename}}\n",
))

func constantize(s string) ([]string, error) {
	var result []string
	ss := strings.SplitN(s, " - ", 2)
	switch len(ss) {
	case 1:
		result = append(result, ss[0])
	case 2:
		prefix := ss[0]
		for _, desc := range strings.Split(ss[1], "/") {
			suffix := desc
			suffix = strings.TrimSuffix(suffix, " etc")
			suffix = strings.Replace(suffix, " ", "", -1)
			suffix = strings.TrimSpace(suffix)
			suffix = strings.Replace(suffix, "+", "And", -1)
			result = append(result, prefix+suffix)
		}
	default:
		return nil, fmt.Errorf("unable to parse %v", s)
	}
	return result, nil
}

func max(x, y int) int {
	if x > y {
		return x
	}
	return y
}

func pad(s string, n int) string {
	m := len(s)
	switch {
	case m < n:
		return s + strings.Repeat(" ", n-m)
	case m == n:
		return s
	default:
		return s[:m]
	}
}

type byID doarama.ActivityTypes

func (ats byID) Len() int           { return len(ats) }
func (ats byID) Less(i, j int) bool { return ats[i].ID < ats[j].ID }
func (ats byID) Swap(i, j int)      { ats[i], ats[j] = ats[j], ats[i] }

func generateActivityIDs(filename string) error {
	f, err := os.Create(filename)
	if err != nil {
		return err
	}
	defer f.Close()
	client := doarama.NewClient(doarama.APIURL, "", "")
	activityTypes, err := client.ActivityTypes()
	if err != nil {
		return err
	}
	sort.Sort(byID(activityTypes))
	maxConstLen := 0
	maxNameLen := 0
	constActivityIDs := make(map[string]int)
	for _, at := range activityTypes {
		maxNameLen = max(maxNameLen, len(at.Name))
		ss, err := constantize(at.Name)
		if err != nil {
			return err
		}
		for _, s := range ss {
			constActivityIDs[s] = at.ID
			maxConstLen = max(maxConstLen, len(s))
		}
	}
	paddedConstActivityIDs := make(map[string]int)
	for name, id := range constActivityIDs {
		paddedConstActivityIDs[pad(name, maxConstLen)] = id
	}
	if err := tmpl.Execute(f, struct {
		ActivityTypes    doarama.ActivityTypes
		ConstActivityIDs map[string]int
		Filename         string
	}{
		ActivityTypes:    activityTypes,
		ConstActivityIDs: paddedConstActivityIDs,
		Filename:         filename,
	}); err != nil {
		return err
	}
	return nil
}

func main() {
	o := flag.String("o", "", "")
	flag.Parse()
	if err := generateActivityIDs(*o); err != nil {
		log.Fatal(err)
	}
}
<|endoftext|>