<commit_before>/*
 * Copyright 2013-2016 Fabian Groffen
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package main

import (
	"bufio"
	"expvar"
	"flag"
	"fmt"
	"io"
	"log"
	"net"
	"net/http"
	_ "net/http/pprof"
	"os"
	"regexp"
	"runtime"
	"strconv"
	"strings"
	"time"

	cfg "github.com/alyu/configparser"
	"github.com/dgryski/carbonzipper/mlog"
	"github.com/dgryski/httputil"
	whisper "github.com/grobian/go-whisper"
	g2g "github.com/peterbourgon/g2g"
)

var config = struct {
	WhisperData  string
	GraphiteHost string
}{
	WhisperData: "/var/lib/carbon/whisper",
}

// grouped expvars for /debug/vars and graphite
var Metrics = struct {
	MetricsReceived *expvar.Int
}{
	MetricsReceived: expvar.NewInt("metrics_received"),
}

var BuildVersion = "(development build)"

var logger mlog.Level

func handleConnection(conn net.Conn, schemas []*StorageSchema, aggrs []*StorageAggregation) {
	bufconn := bufio.NewReader(conn)

	for {
		line, err := bufconn.ReadBytes('\n')
		if err != nil {
			conn.Close()
			if err != io.EOF {
				logger.Logf("read failed: %s", err.Error())
			}
			break
		}

		elems := strings.Split(string(line), " ")
		if len(elems) != 3 {
			logger.Logf("invalid line: %s", string(line))
			continue
		}

		metric := elems[0]

		value, err := strconv.ParseFloat(elems[1], 64)
		if err != nil {
			logger.Logf("invalue value '%s': %s", elems[1], err.Error())
			continue
		}

		elems[2] = strings.TrimRight(elems[2], "\n")
		tsf, err := strconv.ParseFloat(elems[2], 64)
		if err != nil {
			logger.Logf("invalid timestamp '%s': %s", elems[2], err.Error())
			continue
		}
		ts := int(tsf)

		if metric == "" {
			logger.Logf("invalid line: %s", string(line))
			continue
		}

		if ts == 0 {
			logger.Logf("invalid timestamp (0): %s", string(line))
			continue
		}

		logger.Debugf("metric: %s, value: %f, ts: %d", metric, value, ts)

		// catch panics from whisper-go library
		defer func() {
			if r := recover(); r != nil {
				logger.Logf("recovering from whisper panic (metric: %s): %v", metric, r)
				err := conn.Close()
				if err != nil {
					logger.Logf("error while closing connection after whisper panic: %v", err)
				}
			}
		}()

		// do what we want to do
		path := config.WhisperData + "/" + strings.Replace(metric, ".", "/", -1) + ".wsp"
		w, err := whisper.Open(path)
		if err != nil && os.IsNotExist(err) {
			w = createMetric(metric, path, schemas, aggrs)
			if w == nil {
				continue
			}
		} else if err != nil {
			// some other error
			logger.Logf("failed to open whisper file %s: %v", path, err)
			continue
		}

		err = w.Update(value, int(ts))
		if err != nil {
			logger.Logf("failed to update whisper file %s: %v", path, err)
		}
		w.Close()

		Metrics.MetricsReceived.Add(1)
	}
}

func createMetric(metric, path string, schemas []*StorageSchema, aggrs []*StorageAggregation) *whisper.Whisper {
	var schema *StorageSchema
	for _, s := range schemas {
		if s.pattern.MatchString(metric) {
			schema = s
			break
		}
	}
	if schema == nil {
		logger.Logf("no storage schema defined for %s", metric)
		return nil
	}
	logger.Debugf("%s: found schema: %s", metric, schema.name)

	var aggr *StorageAggregation
	for _, a := range aggrs {
		if a.pattern.MatchString(metric) {
			aggr = a
			break
		}
	}

	// http://graphite.readthedocs.org/en/latest/config-carbon.html#storage-aggregation-conf
	aggrName := "(default)"
	aggrStr := "average"
	aggrType := whisper.Average
	xfilesf := float32(0.5)
	if aggr != nil {
		aggrName = aggr.name
		aggrStr = aggr.aggregationMethodStr
		aggrType = aggr.aggregationMethod
		xfilesf = float32(aggr.xFilesFactor)
	}

	logger.Logf("creating %s: %s, retention: %s (section %s), aggregationMethod: %s, xFilesFactor: %f (section %s)",
		metric, path, schema.retentionStr, schema.name,
		aggrStr, xfilesf, aggrName)

	// whisper.Create doesn't mkdir, so let's do it ourself
	lastslash := strings.LastIndex(path, "/")
	if lastslash != -1 {
		dir := path[0:lastslash]
		err := os.MkdirAll(dir, os.ModeDir|os.ModePerm)
		if err != nil {
			logger.Logf("error during mkdir(%q): %v\n", dir, err)
			return nil
		}

	}
	w, err := whisper.Create(path, schema.retentions, aggrType, xfilesf)
	if err != nil {
		logger.Logf("failed to create new whisper file %s: %v", path, err)
		return nil
	}

	return w
}

func listenAndServe(listen string, schemas []*StorageSchema, aggrs []*StorageAggregation) {
	l, err := net.Listen("tcp", listen)
	if err != nil {
		logger.Logf("failed to listen on %s: %s", listen, err.Error())
		os.Exit(1)
	}
	defer l.Close()
	for {
		conn, err := l.Accept()
		if err != nil {
			logger.Logf("failed to accept connection: %s", err.Error())
			continue
		}
		go handleConnection(conn, schemas, aggrs)
	}
}

type StorageSchema struct {
	name         string
	pattern      *regexp.Regexp
	retentionStr string
	retentions   whisper.Retentions
}

func readStorageSchemas(file string) ([]*StorageSchema, error) {
	config, err := cfg.Read(file)
	if err != nil {
		return nil, err
	}

	sections, err := config.AllSections()
	if err != nil {
		return nil, err
	}

	var ret []*StorageSchema
	for _, s := range sections {
		var sschema StorageSchema
		// this is mildly stupid, but I don't feel like forking
		// configparser just for this
		sschema.name =
			strings.Trim(strings.SplitN(s.String(), "\n", 2)[0], " []")
		if sschema.name == "" {
			continue
		}
		sschema.pattern, err = regexp.Compile(s.ValueOf("pattern"))
		if err != nil {
			logger.Logf("failed to parse pattern '%s'for [%s]: %s",
				s.ValueOf("pattern"), sschema.name, err.Error())
			continue
		}
		sschema.retentionStr = s.ValueOf("retentions")
		sschema.retentions, err = whisper.ParseRetentionDefs(sschema.retentionStr)
		logger.Debugf("adding schema [%s] pattern = %s retentions = %s",
			sschema.name, s.ValueOf("pattern"), sschema.retentionStr)

		ret = append(ret, &sschema)
	}

	return ret, nil
}

type StorageAggregation struct {
	name                 string
	pattern              *regexp.Regexp
	xFilesFactor         float64
	aggregationMethodStr string
	aggregationMethod    whisper.AggregationMethod
}

func readStorageAggregations(file string) ([]*StorageAggregation, error) {
	config, err := cfg.Read(file)
	if err != nil {
		return nil, err
	}

	sections, err := config.AllSections()
	if err != nil {
		return nil, err
	}

	var ret []*StorageAggregation
	for _, s := range sections {
		var saggr StorageAggregation
		// this is mildly stupid, but I don't feel like forking
		// configparser just for this
		saggr.name =
			strings.Trim(strings.SplitN(s.String(), "\n", 2)[0], " []")
		if saggr.name == "" {
			continue
		}
		saggr.pattern, err = regexp.Compile(s.ValueOf("pattern"))
		if err != nil {
			logger.Logf("failed to parse pattern '%s'for [%s]: %s",
				s.ValueOf("pattern"), saggr.name, err.Error())
			continue
		}
		saggr.xFilesFactor, err = strconv.ParseFloat(s.ValueOf("xFilesFactor"), 64)
		if err != nil {
			logger.Logf("failed to parse xFilesFactor '%s' in %s: %s",
				s.ValueOf("xFilesFactor"), saggr.name, err.Error())
			continue
		}

		saggr.aggregationMethodStr = s.ValueOf("aggregationMethod")
		switch saggr.aggregationMethodStr {
		case "average", "avg":
			saggr.aggregationMethod = whisper.Average
		case "sum":
			saggr.aggregationMethod = whisper.Sum
		case "last":
			saggr.aggregationMethod = whisper.Last
		case "max":
			saggr.aggregationMethod = whisper.Max
		case "min":
			saggr.aggregationMethod = whisper.Min
		default:
			logger.Logf("unknown aggregation method '%s'",
				s.ValueOf("aggregationMethod"))
			continue
		}

		logger.Debugf("adding aggregation [%s] pattern = %s aggregationMethod = %s xFilesFactor = %f",
			saggr.name, s.ValueOf("pattern"),
			saggr.aggregationMethodStr, saggr.xFilesFactor)
		ret = append(ret, &saggr)
	}

	return ret, nil
}

func main() {
	port := flag.Int("p", 2003, "port to bind to")
	reportport := flag.Int("reportport", 8080, "port to bind http report interface to")
	verbose := flag.Bool("v", false, "enable verbose logging")
	debug := flag.Bool("vv", false, "enable more verbose (debug) logging")
	whisperdata := flag.String("w", config.WhisperData, "location where whisper files are stored")
	maxprocs := flag.Int("maxprocs", runtime.NumCPU()*80/100, "GOMAXPROCS")
	logdir := flag.String("logdir", "/var/log/carbonwriter/", "logging directory")
	schemafile := flag.String("schemafile", "/etc/carbon/storage-schemas.conf", "storage-schemas.conf location")
	aggrfile := flag.String("aggrfile", "/etc/carbon/storage-aggregation.conf", "storage-aggregation.conf location")
	logtostdout := flag.Bool("stdout", false, "log also to stdout")

	flag.Parse()

	mlog.SetOutput(*logdir, "carbonwriter", *logtostdout)

	expvar.NewString("BuildVersion").Set(BuildVersion)
	log.Println("starting carbonwriter", BuildVersion)

	loglevel := mlog.Normal
	if *verbose {
		loglevel = mlog.Debug
	}
	if *debug {
		loglevel = mlog.Trace
	}

	logger = mlog.Level(loglevel)

	schemas, err := readStorageSchemas(*schemafile)
	if err != nil {
		logger.Logf("failed to read %s: %s", *schemafile, err.Error())
		os.Exit(1)
	}

	aggrs, err := readStorageAggregations(*aggrfile)
	if err != nil {
		logger.Logf("failed to read %s: %s", *aggrfile, err.Error())
		os.Exit(1)
	}

	config.WhisperData = strings.TrimRight(*whisperdata, "/")
	logger.Logf("writing whisper files to: %s", config.WhisperData)
	logger.Logf("reading storage schemas from: %s", *schemafile)
	logger.Logf("reading aggregation rules from: %s", *aggrfile)

	runtime.GOMAXPROCS(*maxprocs)
	logger.Logf("set GOMAXPROCS=%d", *maxprocs)

	httputil.PublishTrackedConnections("httptrack")

	// nothing in the config? check the environment
	if config.GraphiteHost == "" {
		if host := os.Getenv("GRAPHITEHOST") + ":" + os.Getenv("GRAPHITEPORT"); host != ":" {
			config.GraphiteHost = host
		}
	}

	// only register g2g if we have a graphite host
	if config.GraphiteHost != "" {

		logger.Logf("Using graphite host %v", config.GraphiteHost)

		// register our metrics with graphite
		graphite, err := g2g.NewGraphite(config.GraphiteHost, 60*time.Second, 10*time.Second)
		if err != nil {
			log.Fatalf("unable to connect to to graphite: %v: %v", config.GraphiteHost, err)
		}

		hostname, _ := os.Hostname()
		hostname = strings.Replace(hostname, ".", "_", -1)

		graphite.Register(fmt.Sprintf("carbon.writer.%s.metricsReceived", hostname), Metrics.MetricsReceived)
	}

	listen := fmt.Sprintf(":%d", *port)
	httplisten := fmt.Sprintf(":%d", *reportport)
	logger.Logf("listening on %s, statistics via %s", listen, httplisten)
	go listenAndServe(listen, schemas, aggrs)
	err = http.ListenAndServe(httplisten, nil)
	if err != nil {
		log.Fatalf("%s", err)
	}
	logger.Logf("stopped")
}
<commit_msg>g2g upstream changes, removed err<commit_after>/*
 * Copyright 2013-2016 Fabian Groffen
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package main

import (
	"bufio"
	"expvar"
	"flag"
	"fmt"
	"io"
	"log"
	"net"
	"net/http"
	_ "net/http/pprof"
	"os"
	"regexp"
	"runtime"
	"strconv"
	"strings"
	"time"

	cfg "github.com/alyu/configparser"
	"github.com/dgryski/carbonzipper/mlog"
	"github.com/dgryski/httputil"
	whisper "github.com/grobian/go-whisper"
	g2g "github.com/peterbourgon/g2g"
)

var config = struct {
	WhisperData  string
	GraphiteHost string
}{
	WhisperData: "/var/lib/carbon/whisper",
}

// grouped expvars for /debug/vars and graphite
var Metrics = struct {
	MetricsReceived *expvar.Int
}{
	MetricsReceived: expvar.NewInt("metrics_received"),
}

var BuildVersion = "(development build)"

var logger mlog.Level

func handleConnection(conn net.Conn, schemas []*StorageSchema, aggrs []*StorageAggregation) {
	bufconn := bufio.NewReader(conn)

	for {
		line, err := bufconn.ReadBytes('\n')
		if err != nil {
			conn.Close()
			if err != io.EOF {
				logger.Logf("read failed: %s", err.Error())
			}
			break
		}

		elems := strings.Split(string(line), " ")
		if len(elems) != 3 {
			logger.Logf("invalid line: %s", string(line))
			continue
		}

		metric := elems[0]

		value, err := strconv.ParseFloat(elems[1], 64)
		if err != nil {
			logger.Logf("invalue value '%s': %s", elems[1], err.Error())
			continue
		}

		elems[2] = strings.TrimRight(elems[2], "\n")
		tsf, err := strconv.ParseFloat(elems[2], 64)
		if err != nil {
			logger.Logf("invalid timestamp '%s': %s", elems[2], err.Error())
			continue
		}
		ts := int(tsf)

		if metric == "" {
			logger.Logf("invalid line: %s", string(line))
			continue
		}

		if ts == 0 {
			logger.Logf("invalid timestamp (0): %s", string(line))
			continue
		}

		logger.Debugf("metric: %s, value: %f, ts: %d", metric, value, ts)

		// catch panics from whisper-go library
		defer func() {
			if r := recover(); r != nil {
				logger.Logf("recovering from whisper panic (metric: %s): %v", metric, r)
				err := conn.Close()
				if err != nil {
					logger.Logf("error while closing connection after whisper panic: %v", err)
				}
			}
		}()

		// do what we want to do
		path := config.WhisperData + "/" + strings.Replace(metric, ".", "/", -1) + ".wsp"
		w, err := whisper.Open(path)
		if err != nil && os.IsNotExist(err) {
			w = createMetric(metric, path, schemas, aggrs)
			if w == nil {
				continue
			}
		} else if err != nil {
			// some other error
			logger.Logf("failed to open whisper file %s: %v", path, err)
			continue
		}

		err = w.Update(value, int(ts))
		if err != nil {
			logger.Logf("failed to update whisper file %s: %v", path, err)
		}
		w.Close()

		Metrics.MetricsReceived.Add(1)
	}
}

func createMetric(metric, path string, schemas []*StorageSchema, aggrs []*StorageAggregation) *whisper.Whisper {
	var schema *StorageSchema
	for _, s := range schemas {
		if s.pattern.MatchString(metric) {
			schema = s
			break
		}
	}
	if schema == nil {
		logger.Logf("no storage schema defined for %s", metric)
		return nil
	}
	logger.Debugf("%s: found schema: %s", metric, schema.name)

	var aggr *StorageAggregation
	for _, a := range aggrs {
		if a.pattern.MatchString(metric) {
			aggr = a
			break
		}
	}

	// http://graphite.readthedocs.org/en/latest/config-carbon.html#storage-aggregation-conf
	aggrName := "(default)"
	aggrStr := "average"
	aggrType := whisper.Average
	xfilesf := float32(0.5)
	if aggr != nil {
		aggrName = aggr.name
		aggrStr = aggr.aggregationMethodStr
		aggrType = aggr.aggregationMethod
		xfilesf = float32(aggr.xFilesFactor)
	}

	logger.Logf("creating %s: %s, retention: %s (section %s), aggregationMethod: %s, xFilesFactor: %f (section %s)",
		metric, path, schema.retentionStr, schema.name,
		aggrStr, xfilesf, aggrName)

	// whisper.Create doesn't mkdir, so let's do it ourself
	lastslash := strings.LastIndex(path, "/")
	if lastslash != -1 {
		dir := path[0:lastslash]
		err := os.MkdirAll(dir, os.ModeDir|os.ModePerm)
		if err != nil {
			logger.Logf("error during mkdir(%q): %v\n", dir, err)
			return nil
		}

	}
	w, err := whisper.Create(path, schema.retentions, aggrType, xfilesf)
	if err != nil {
		logger.Logf("failed to create new whisper file %s: %v", path, err)
		return nil
	}

	return w
}

func listenAndServe(listen string, schemas []*StorageSchema, aggrs []*StorageAggregation) {
	l, err := net.Listen("tcp", listen)
	if err != nil {
		logger.Logf("failed to listen on %s: %s", listen, err.Error())
		os.Exit(1)
	}
	defer l.Close()
	for {
		conn, err := l.Accept()
		if err != nil {
			logger.Logf("failed to accept connection: %s", err.Error())
			continue
		}
		go handleConnection(conn, schemas, aggrs)
	}
}

type StorageSchema struct {
	name         string
	pattern      *regexp.Regexp
	retentionStr string
	retentions   whisper.Retentions
}

func readStorageSchemas(file string) ([]*StorageSchema, error) {
	config, err := cfg.Read(file)
	if err != nil {
		return nil, err
	}

	sections, err := config.AllSections()
	if err != nil {
		return nil, err
	}

	var ret []*StorageSchema
	for _, s := range sections {
		var sschema StorageSchema
		// this is mildly stupid, but I don't feel like forking
		// configparser just for this
		sschema.name =
			strings.Trim(strings.SplitN(s.String(), "\n", 2)[0], " []")
		if sschema.name == "" {
			continue
		}
		sschema.pattern, err = regexp.Compile(s.ValueOf("pattern"))
		if err != nil {
			logger.Logf("failed to parse pattern '%s'for [%s]: %s",
				s.ValueOf("pattern"), sschema.name, err.Error())
			continue
		}
		sschema.retentionStr = s.ValueOf("retentions")
		sschema.retentions, err = whisper.ParseRetentionDefs(sschema.retentionStr)
		logger.Debugf("adding schema [%s] pattern = %s retentions = %s",
			sschema.name, s.ValueOf("pattern"), sschema.retentionStr)

		ret = append(ret, &sschema)
	}

	return ret, nil
}

type StorageAggregation struct {
	name                 string
	pattern              *regexp.Regexp
	xFilesFactor         float64
	aggregationMethodStr string
	aggregationMethod    whisper.AggregationMethod
}

func readStorageAggregations(file string) ([]*StorageAggregation, error) {
	config, err := cfg.Read(file)
	if err != nil {
		return nil, err
	}

	sections, err := config.AllSections()
	if err != nil {
		return nil, err
	}

	var ret []*StorageAggregation
	for _, s := range sections {
		var saggr StorageAggregation
		// this is mildly stupid, but I don't feel like forking
		// configparser just for this
		saggr.name =
			strings.Trim(strings.SplitN(s.String(), "\n", 2)[0], " []")
		if saggr.name == "" {
			continue
		}
		saggr.pattern, err = regexp.Compile(s.ValueOf("pattern"))
		if err != nil {
			logger.Logf("failed to parse pattern '%s'for [%s]: %s",
				s.ValueOf("pattern"), saggr.name, err.Error())
			continue
		}
		saggr.xFilesFactor, err = strconv.ParseFloat(s.ValueOf("xFilesFactor"), 64)
		if err != nil {
			logger.Logf("failed to parse xFilesFactor '%s' in %s: %s",
				s.ValueOf("xFilesFactor"), saggr.name, err.Error())
			continue
		}

		saggr.aggregationMethodStr = s.ValueOf("aggregationMethod")
		switch saggr.aggregationMethodStr {
		case "average", "avg":
			saggr.aggregationMethod = whisper.Average
		case "sum":
			saggr.aggregationMethod = whisper.Sum
		case "last":
			saggr.aggregationMethod = whisper.Last
		case "max":
			saggr.aggregationMethod = whisper.Max
		case "min":
			saggr.aggregationMethod = whisper.Min
		default:
			logger.Logf("unknown aggregation method '%s'",
				s.ValueOf("aggregationMethod"))
			continue
		}

		logger.Debugf("adding aggregation [%s] pattern = %s aggregationMethod = %s xFilesFactor = %f",
			saggr.name, s.ValueOf("pattern"),
			saggr.aggregationMethodStr, saggr.xFilesFactor)
		ret = append(ret, &saggr)
	}

	return ret, nil
}

func main() {
	port := flag.Int("p", 2003, "port to bind to")
	reportport := flag.Int("reportport", 8080, "port to bind http report interface to")
	verbose := flag.Bool("v", false, "enable verbose logging")
	debug := flag.Bool("vv", false, "enable more verbose (debug) logging")
	whisperdata := flag.String("w", config.WhisperData, "location where whisper files are stored")
	maxprocs := flag.Int("maxprocs", runtime.NumCPU()*80/100, "GOMAXPROCS")
	logdir := flag.String("logdir", "/var/log/carbonwriter/", "logging directory")
	schemafile := flag.String("schemafile", "/etc/carbon/storage-schemas.conf", "storage-schemas.conf location")
	aggrfile := flag.String("aggrfile", "/etc/carbon/storage-aggregation.conf", "storage-aggregation.conf location")
	logtostdout := flag.Bool("stdout", false, "log also to stdout")

	flag.Parse()

	mlog.SetOutput(*logdir, "carbonwriter", *logtostdout)

	expvar.NewString("BuildVersion").Set(BuildVersion)
	log.Println("starting carbonwriter", BuildVersion)

	loglevel := mlog.Normal
	if *verbose {
		loglevel = mlog.Debug
	}
	if *debug {
		loglevel = mlog.Trace
	}

	logger = mlog.Level(loglevel)

	schemas, err := readStorageSchemas(*schemafile)
	if err != nil {
		logger.Logf("failed to read %s: %s", *schemafile, err.Error())
		os.Exit(1)
	}

	aggrs, err := readStorageAggregations(*aggrfile)
	if err != nil {
		logger.Logf("failed to read %s: %s", *aggrfile, err.Error())
		os.Exit(1)
	}

	config.WhisperData = strings.TrimRight(*whisperdata, "/")
	logger.Logf("writing whisper files to: %s", config.WhisperData)
	logger.Logf("reading storage schemas from: %s", *schemafile)
	logger.Logf("reading aggregation rules from: %s", *aggrfile)

	runtime.GOMAXPROCS(*maxprocs)
	logger.Logf("set GOMAXPROCS=%d", *maxprocs)

	httputil.PublishTrackedConnections("httptrack")

	// nothing in the config? check the environment
	if config.GraphiteHost == "" {
		if host := os.Getenv("GRAPHITEHOST") + ":" + os.Getenv("GRAPHITEPORT"); host != ":" {
			config.GraphiteHost = host
		}
	}

	// only register g2g if we have a graphite host
	if config.GraphiteHost != "" {

		logger.Logf("Using graphite host %v", config.GraphiteHost)

		// register our metrics with graphite
		graphite := g2g.NewGraphite(config.GraphiteHost, 60*time.Second, 10*time.Second)

		hostname, _ := os.Hostname()
		hostname = strings.Replace(hostname, ".", "_", -1)

		graphite.Register(fmt.Sprintf("carbon.writer.%s.metricsReceived", hostname), Metrics.MetricsReceived)
	}

	listen := fmt.Sprintf(":%d", *port)
	httplisten := fmt.Sprintf(":%d", *reportport)
	logger.Logf("listening on %s, statistics via %s", listen, httplisten)
	go listenAndServe(listen, schemas, aggrs)
	err = http.ListenAndServe(httplisten, nil)
	if err != nil {
		log.Fatalf("%s", err)
	}
	logger.Logf("stopped")
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"fmt"
	"log"
	"os"

	telegramBotApi "gopkg.in/telegram-bot-api.v4"
	"strings"
)

//UpdatesTimeout Telegram API poll timeout
const UpdatesTimeout = 60

const CommandAdd = "add"
const CommandList = "list"
const CommandClose = "close"
const CommandShutdown = "shutdown"

type Request struct {
	ID     int `storm:"id,increment"`
	Name   string
	Closed bool
}

func main() {
	bot := getPersistentBot()

	for update := range bot.getUpdatesChan() {
		if update.Message == nil {
			continue
		}

		handleUpdate(update, bot)
	}

	defer bot.FinishWork()
}

//Logic of messages handling
func handleUpdate(update telegramBotApi.Update, bot BotCommunicationInterface) {
	//log.Printf("[%s] %s", update.Message.From.UserName, update.Message.Text)
	message := update.Message

	if message.IsCommand() {
		commandArguments := message.CommandArguments()

		switch command := strings.ToLower(message.Command()); command {
		case CommandAdd:
			result, _ := bot.AddRequest(commandArguments)
			bot.SendReply(update, result)
		case CommandList:
			bot.SendReply(update, bot.GetRequestsText())
		case CommandClose:
			result, _ := bot.CloseRequest(commandArguments)
			bot.SendReply(update, result)
		case CommandShutdown:
			username := message.From.UserName
			if bot.IsAdmin(username) {
				log.Printf("Shutdown initiated by '%s' ", username)

				bot.SendReply(update, "Shutdown initiated")
				bot.Shutdown()
			} else {
				bot.SendReply(update, "You are not authorized to perform shutdown command")
			}
		default:
			bot.SendReply(update, fmt.Sprintf("I can't understand command '%s'", command))
		}
	}
}

func (request Request) String() string {
	return request.Name
}

func readTokenFile() string {
	file, err := os.Open("token")
	if err != nil {
		log.Panic("token file reading error:", err)
	}
	defer file.Close()

	scanner := bufio.NewScanner(file)
	scanner.Scan()

	return scanner.Text()
}
<commit_msg>easier to read/understand<commit_after>package main

import (
	"bufio"
	"fmt"
	"log"
	"os"

	telegramBotApi "gopkg.in/telegram-bot-api.v4"
	"strings"
)

//UpdatesTimeout Telegram API poll timeout
const UpdatesTimeout = 60

const CommandAdd = "add"
const CommandList = "list"
const CommandClose = "close"
const CommandShutdown = "shutdown"

type Request struct {
	ID     int `storm:"id,increment"`
	Name   string
	Closed bool
}

func main() {
	bot := getPersistentBot()

	for update := range bot.getUpdatesChan() {
		if update.Message == nil {
			continue
		}

		handleUpdate(update, bot)
	}

	defer bot.FinishWork()
}

//Logic of messages handling
func handleUpdate(update telegramBotApi.Update, bot BotCommunicationInterface) {
	//log.Printf("[%s] %s", update.Message.From.UserName, update.Message.Text)
	message := update.Message

	if message.IsCommand() {
		commandArguments := message.CommandArguments()

		switch command := message.Command(); strings.ToLower(command) {
		case CommandAdd:
			result, _ := bot.AddRequest(commandArguments)
			bot.SendReply(update, result)
		case CommandList:
			bot.SendReply(update, bot.GetRequestsText())
		case CommandClose:
			result, _ := bot.CloseRequest(commandArguments)
			bot.SendReply(update, result)
		case CommandShutdown:
			username := message.From.UserName
			if bot.IsAdmin(username) {
				log.Printf("Shutdown initiated by '%s' ", username)

				bot.SendReply(update, "Shutdown initiated")
				bot.Shutdown()
			} else {
				bot.SendReply(update, "You are not authorized to perform shutdown command")
			}
		default:
			bot.SendReply(update, fmt.Sprintf("I can't understand command '%s'", command))
		}
	}
}

func (request Request) String() string {
	return request.Name
}

func readTokenFile() string {
	file, err := os.Open("token")
	if err != nil {
		log.Panic("token file reading error:", err)
	}
	defer file.Close()

	scanner := bufio.NewScanner(file)
	scanner.Scan()

	return scanner.Text()
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"flag"
	"fmt"
	envoyv2 "github.com/envoyproxy/go-control-plane/envoy/api/v2"
	"github.com/envoyproxy/go-control-plane/envoy/api/v2/core"
	envoyEndpoint "github.com/envoyproxy/go-control-plane/envoy/api/v2/endpoint"
	"github.com/envoyproxy/go-control-plane/envoy/api/v2/listener"
	"github.com/envoyproxy/go-control-plane/envoy/api/v2/route"
	hcm "github.com/envoyproxy/go-control-plane/envoy/config/filter/network/http_connection_manager/v2"
	discovery "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v2"
	"github.com/envoyproxy/go-control-plane/pkg/cache"
	xds "github.com/envoyproxy/go-control-plane/pkg/server"
	"github.com/envoyproxy/go-control-plane/pkg/util"
	"github.com/gogo/protobuf/types"
	log "github.com/sirupsen/logrus"
	"google.golang.org/grpc"
	v12 "k8s.io/api/core/v1"
	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/rest"
	"k8s.io/client-go/tools/clientcmd"
	"knative.dev/serving/pkg/apis/serving/v1alpha1"
	servingv1alpha1 "knative.dev/serving/pkg/client/clientset/versioned/typed/serving/v1alpha1"
	"net"
	"net/http"
	"os"
	"time"

	"path/filepath"
)

var (
	config cache.SnapshotCache
)

const (
	grpcMaxConcurrentStreams = 1000000
	nodeID                   = "3scale-courier"
)

func homeDir() string {
	if h := os.Getenv("HOME"); h != "" {
		return h
	}
	return os.Getenv("USERPROFILE") // windows
}

func init() {
	// Log as JSON instead of the default ASCII formatter.
	log.SetFormatter(&log.JSONFormatter{})

	// Output to stdout instead of the default stderr
	// Can be any io.Writer, see below for File example
	log.SetOutput(os.Stdout)

	// Only log the warning severity or above.
	log.SetLevel(log.InfoLevel)
}

func main() {

	var clusterCache []cache.Resource
	var kubeconfig *string
	version := 1
	ctx := context.Background()
	xdsConf := cache.NewSnapshotCache(true, Hasher{}, nil)

	srv := xds.NewServer(xdsConf, nil)

	go RunManagementServer(ctx, srv, 18000)
	go RunManagementGateway(ctx, srv, 19001)

	// Get the config, $HOME/.kube/config
	// TODO: Read from env var
	if home := homeDir(); home != "" {
		kubeconfig = flag.String("kubeconfig", filepath.Join(home, ".kube", "config"), "(optional) absolute path to the kubeconfig file")
	} else {
		kubeconfig = flag.String("kubeconfig", "", "absolute path to the kubeconfig file")
	}
	flag.Parse()

	var config *rest.Config
	// use the current context in kubeconfig
	config, err := clientcmd.BuildConfigFromFlags("", *kubeconfig)
	if err != nil {
		config, _ = rest.InClusterConfig()
	}

	servingClient, err := servingv1alpha1.NewForConfig(config)
	if err != nil {
		panic(err)
	}
	k8sClient, err := kubernetes.NewForConfig(config)
	if err != nil {
		panic(err)
	}

	for {
		serviceList, err := servingClient.Services("serverless").List(v1.ListOptions{})
		if err != nil {
			panic(err)
		}

		virtualHosts := []*route.VirtualHost{}
		routeCache := []cache.Resource{}

		for _, service := range serviceList.Items {
			log.WithFields(log.Fields{"name": service.GetName(), "host": service.Status.URL.Host}, ).Info("Knative serving service found")

			listOptions := v1.ListOptions{LabelSelector: "serving.knative.dev/service=" + service.Name}
			endpointList, err := k8sClient.CoreV1().Endpoints("serverless").List(listOptions)
			if err != nil {
				panic(err)
			}

			r := route.Route{
				Match: &route.RouteMatch{
					PathSpecifier: &route.RouteMatch_Prefix{
						Prefix: "/",
					},
				},
				Action: &route.Route_Route{
					Route: &route.RouteAction{
						ClusterSpecifier: &route.RouteAction_Cluster{
							Cluster: "knative_" + service.GetName(),
						},
					},
				},
			}

			routeCache = append(routeCache, &r)

			revision, err := getRevision(service)

			if err != nil {
				log.Errorf("%s", err)
				break
			}

			virtualHost := route.VirtualHost{
				Name:    service.GetName(),
				Domains: []string{service.Status.URL.Host},
				Routes:  []*route.Route{&r},
				RequestHeadersToAdd: []*core.HeaderValueOption{{
					Header: &core.HeaderValue{
						Key:   "Knative-Serving-Namespace",
						Value: service.Namespace,
					},
					Append: &types.BoolValue{
						Value: true,
					},
				},
					{
						Header: &core.HeaderValue{
							Key:   "Knative-Serving-Revision",
							Value: revision,
						},
						Append: &types.BoolValue{
							Value: true,
						},
					},
				},
			}

			virtualHosts = append(virtualHosts, &virtualHost)

			lbEndpoints := []*envoyEndpoint.LbEndpoint{}

			for _, endpoint := range endpointList.Items {
				log.Info(endpoint.Name)

				for _, subset := range endpoint.Subsets {

					port, err := getHTTPPort(subset)

					if err != nil {
						break
					}

					for _, address := range subset.Addresses {

						serviceEndpoint := &core.Address{
							Address: &core.Address_SocketAddress{
								SocketAddress: &core.SocketAddress{
									Protocol: core.TCP,
									Address:  address.IP,
									PortSpecifier: &core.SocketAddress_PortValue{
										PortValue: port,
									},
									Ipv4Compat: true,
								},
							},
						}

						lbEndpoint := envoyEndpoint.LbEndpoint{
							HostIdentifier: &envoyEndpoint.LbEndpoint_Endpoint{
								Endpoint: &envoyEndpoint.Endpoint{
									Address: serviceEndpoint,
								},
							},
						}
						lbEndpoints = append(lbEndpoints, &lbEndpoint)

					}

				}

			}

			connectTimeout := 5 * time.Second

			cluster := envoyv2.Cluster{
				Name:        "knative_" + service.GetName(),
				AltStatName: "",
				ClusterDiscoveryType: &envoyv2.Cluster_Type{
					Type: envoyv2.Cluster_STATIC,
				},
				ConnectTimeout: &connectTimeout,
				LoadAssignment: &envoyv2.ClusterLoadAssignment{
					ClusterName: "knative_" + service.GetName(),
					Endpoints: []*envoyEndpoint.LocalityLbEndpoints{
						{
							LbEndpoints: lbEndpoints,
						},
					},
				},
			}
			log.Info(cluster)

			clusterCache = append(clusterCache, &cluster)
		}

		manager := &hcm.HttpConnectionManager{
			CodecType:  hcm.AUTO,
			StatPrefix: "ingress_http",
			RouteSpecifier: &hcm.HttpConnectionManager_RouteConfig{
				RouteConfig: &envoyv2.RouteConfiguration{
					Name:         "local_route",
					VirtualHosts: virtualHosts,
				},
			},
			HttpFilters: []*hcm.HttpFilter{
				{
					Name: util.Router,
				},
			},
		}

		//TODO: ?????
		pbst, err := util.MessageToStruct(manager)
		if err != nil {
			panic(err)
		}

		listener := &envoyv2.Listener{
			Name: "listener_0",
			Address: &core.Address{
				Address: &core.Address_SocketAddress{
					SocketAddress: &core.SocketAddress{
						Protocol: core.TCP,
						Address:  "0.0.0.0",
						PortSpecifier: &core.SocketAddress_PortValue{
							PortValue: uint32(80),
						},
					},
				},
			},
			FilterChains: []*listener.FilterChain{{
				Filters: []*listener.Filter{{
					Name:       util.HTTPConnectionManager,
					ConfigType: &listener.Filter_Config{Config: pbst},
				}},
			}},
		}
		version = version + 1

		listenerCache := []cache.Resource{listener}

		snapshot := cache.NewSnapshot(fmt.Sprintf("%d", version), nil, clusterCache, routeCache, listenerCache)

		err = xdsConf.SetSnapshot(nodeID, snapshot)
		if err != nil {
			log.Error(err)
		}

		time.Sleep(5 * time.Second)
	}
}

// RunManagementServer starts an xDS server at the given Port.
func RunManagementServer(ctx context.Context, server xds.Server, port uint) {
	var grpcOptions []grpc.ServerOption
	grpcOptions = append(grpcOptions, grpc.MaxConcurrentStreams(grpcMaxConcurrentStreams))
	grpcServer := grpc.NewServer(grpcOptions...)
	lis, err := net.Listen("tcp", fmt.Sprintf(":%d", port))
	if err != nil {
		log.Error("Failed to listen")
	}

	// register services
	discovery.RegisterAggregatedDiscoveryServiceServer(grpcServer, server)
	envoyv2.RegisterEndpointDiscoveryServiceServer(grpcServer, server)
	envoyv2.RegisterClusterDiscoveryServiceServer(grpcServer, server)
	envoyv2.RegisterRouteDiscoveryServiceServer(grpcServer, server)
	envoyv2.RegisterListenerDiscoveryServiceServer(grpcServer, server)

	log.Printf("Starting Management Server on Port %d\n", port)
	go func() {
		if err = grpcServer.Serve(lis); err != nil {
			fmt.Errorf("%s", err)
		}
	}()
	<-ctx.Done()
	grpcServer.GracefulStop()
}

// RunManagementGateway starts an HTTP gateway to an xDS server.
func RunManagementGateway(ctx context.Context, srv xds.Server, port uint) {
	log.Printf("Starting HTTP/1.1 gateway on Port %d\n", port)
	server := &http.Server{Addr: fmt.Sprintf(":%d", port), Handler: &xds.HTTPGateway{Server: srv}}
	go func() {
		if err := server.ListenAndServe(); err != nil {
			panic(err)
		}
	}()

	<-ctx.Done()
	if err := server.Shutdown(ctx); err != nil {
		panic(err)
	}
}

// Hasher returns node ID as an ID
type Hasher struct {
}

func (h Hasher) ID(node *core.Node) string {
	if node == nil {
		return "unknown"
	}
	return node.Id
}

func getHTTPPort(subset v12.EndpointSubset) (uint32, error) {

	for _, port := range subset.Ports {
		if port.Name == "http" {
			return uint32(port.Port), nil
		}
	}

	return 0, fmt.Errorf("http port not found")

}

func getRevision(service v1alpha1.Service) (string, error) {
	for _, traffic := range service.Status.Traffic {
		if traffic.Percent == 100 {
			return traffic.RevisionName, nil
		}
	}

	return "", fmt.Errorf("no revision with 100% of traffic")
}
<commit_msg>main: update config only when endpoints/servings change<commit_after>package main

import (
	"context"
	"flag"
	"fmt"
	envoyv2 "github.com/envoyproxy/go-control-plane/envoy/api/v2"
	"github.com/envoyproxy/go-control-plane/envoy/api/v2/core"
	envoyEndpoint "github.com/envoyproxy/go-control-plane/envoy/api/v2/endpoint"
	"github.com/envoyproxy/go-control-plane/envoy/api/v2/listener"
	"github.com/envoyproxy/go-control-plane/envoy/api/v2/route"
	hcm "github.com/envoyproxy/go-control-plane/envoy/config/filter/network/http_connection_manager/v2"
	discovery "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v2"
	"github.com/envoyproxy/go-control-plane/pkg/cache"
	xds "github.com/envoyproxy/go-control-plane/pkg/server"
	"github.com/envoyproxy/go-control-plane/pkg/util"
	"github.com/gogo/protobuf/types"
	log "github.com/sirupsen/logrus"
	"google.golang.org/grpc"
	v12 "k8s.io/api/core/v1"
	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/fields"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/rest"
	"k8s.io/client-go/tools/clientcmd"
	"knative.dev/serving/pkg/apis/serving/v1alpha1"
	servingv1alpha1 "knative.dev/serving/pkg/client/clientset/versioned/typed/serving/v1alpha1"
	"net"
	"net/http"
	"os"
	"time"

	"path/filepath"
	k8s_cache "k8s.io/client-go/tools/cache"
)

var (
	config cache.SnapshotCache
)

const (
	grpcMaxConcurrentStreams = 1000000
	nodeID                   = "3scale-courier"
)

func homeDir() string {
	if h := os.Getenv("HOME"); h != "" {
		return h
	}
	return os.Getenv("USERPROFILE") // windows
}

func init() {
	// Log as JSON instead of the default ASCII formatter.
	log.SetFormatter(&log.JSONFormatter{})

	// Output to stdout instead of the default stderr
	// Can be any io.Writer, see below for File example
	log.SetOutput(os.Stdout)

	// Only log the warning severity or above.
	log.SetLevel(log.InfoLevel)
}

// Pushes an event to the "events" channel received when an serving is added/deleted/updated.
func watchChangesInServings(servingClient *servingv1alpha1.ServingV1alpha1Client, namespace string, events chan<- string, stopChan <-chan struct{}) {
	restClient := servingClient.RESTClient()

	watchlist := k8s_cache.NewListWatchFromClient(restClient, "services", namespace,
		fields.Everything())

	_, controller := k8s_cache.NewInformer(
		watchlist,
		&v1alpha1.Service{},
		time.Second*1,
		k8s_cache.ResourceEventHandlerFuncs{
			AddFunc: func(obj interface{}) {
				events <- "change"
			},

			DeleteFunc: func(obj interface{}) {
				events <- "change"
			},

			UpdateFunc: func(oldObj, newObj interface{}) {
				if oldObj != newObj {
					events <- "change"
				}
			},
		},
	)

	controller.Run(stopChan)
}

// Pushes an event to the "events" channel received when an endpoint is added/deleted/updated.
func watchChangesInEndpoints(kubernetesClient *kubernetes.Clientset, namespace string, events chan<- string, stopChan <-chan struct{}) {
	restClient := kubernetesClient.CoreV1().RESTClient()

	watchlist := k8s_cache.NewListWatchFromClient(restClient, "endpoints", namespace,
		fields.Everything())

	_, controller := k8s_cache.NewInformer(
		watchlist,
		&v12.Endpoints{},
		time.Second*1,
		k8s_cache.ResourceEventHandlerFuncs{
			AddFunc: func(obj interface{}) {
				events <- "change"
			},

			DeleteFunc: func(obj interface{}) {
				events <- "change"
			},

			UpdateFunc: func(oldObj, newObj interface{}) {
				if oldObj != newObj {
					events <- "change"
				}
			},
		},
	)

	controller.Run(stopChan)
}

func main() {

	var clusterCache []cache.Resource
	var kubeconfig *string
	version := 1
	ctx := context.Background()
	xdsConf := cache.NewSnapshotCache(true, Hasher{}, nil)

	srv := xds.NewServer(xdsConf, nil)

	go RunManagementServer(ctx, srv, 18000)
	go RunManagementGateway(ctx, srv, 19001)

	// Get the config, $HOME/.kube/config
	// TODO: Read from env var
	if home := homeDir(); home != "" {
		kubeconfig = flag.String("kubeconfig", filepath.Join(home, ".kube", "config"), "(optional) absolute path to the kubeconfig file")
	} else {
		kubeconfig = flag.String("kubeconfig", "", "absolute path to the kubeconfig file")
	}
	flag.Parse()

	var config *rest.Config
	// use the current context in kubeconfig
	config, err := clientcmd.BuildConfigFromFlags("", *kubeconfig)
	if err != nil {
		config, _ = rest.InClusterConfig()
	}

	servingClient, err := servingv1alpha1.NewForConfig(config)
	if err != nil {
		panic(err)
	}
	k8sClient, err := kubernetes.NewForConfig(config)
	if err != nil {
		panic(err)
	}

	eventsChan := make(chan string)

	stopChanEndpoints := make(chan struct{})
	go watchChangesInEndpoints(k8sClient, "serverless", eventsChan, stopChanEndpoints)

	stopChanServings := make(chan struct{})
	go watchChangesInServings(servingClient, "serverless", eventsChan, stopChanServings)

	for {
		serviceList, err := servingClient.Services("serverless").List(v1.ListOptions{})
		if err != nil {
			panic(err)
		}

		virtualHosts := []*route.VirtualHost{}
		routeCache := []cache.Resource{}

		for _, service := range serviceList.Items {
			log.WithFields(log.Fields{"name": service.GetName(), "host": service.Status.URL.Host}, ).Info("Knative serving service found")

			listOptions := v1.ListOptions{LabelSelector: "serving.knative.dev/service=" + service.Name}
			endpointList, err := k8sClient.CoreV1().Endpoints("serverless").List(listOptions)
			if err != nil {
				panic(err)
			}

			r := route.Route{
				Match: &route.RouteMatch{
					PathSpecifier: &route.RouteMatch_Prefix{
						Prefix: "/",
					},
				},
				Action: &route.Route_Route{
					Route: &route.RouteAction{
						ClusterSpecifier: &route.RouteAction_Cluster{
							Cluster: "knative_" + service.GetName(),
						},
					},
				},
			}

			routeCache = append(routeCache, &r)

			revision, err := getRevision(service)

			if err != nil {
				log.Errorf("%s", err)
				break
			}

			virtualHost := route.VirtualHost{
				Name:    service.GetName(),
				Domains: []string{service.Status.URL.Host},
				Routes:  []*route.Route{&r},
				RequestHeadersToAdd: []*core.HeaderValueOption{{
					Header: &core.HeaderValue{
						Key:   "Knative-Serving-Namespace",
						Value: service.Namespace,
					},
					Append: &types.BoolValue{
						Value: true,
					},
				},
					{
						Header: &core.HeaderValue{
							Key:   "Knative-Serving-Revision",
							Value: revision,
						},
						Append: &types.BoolValue{
							Value: true,
						},
					},
				},
			}

			virtualHosts = append(virtualHosts, &virtualHost)

			lbEndpoints := []*envoyEndpoint.LbEndpoint{}

			for _, endpoint := range endpointList.Items {
				log.Info(endpoint.Name)

				for _, subset := range endpoint.Subsets {

					port, err := getHTTPPort(subset)

					if err != nil {
						break
					}

					for _, address := range subset.Addresses {

						serviceEndpoint := &core.Address{
							Address: &core.Address_SocketAddress{
								SocketAddress: &core.SocketAddress{
									Protocol: core.TCP,
									Address:  address.IP,
									PortSpecifier: &core.SocketAddress_PortValue{
										PortValue: port,
									},
									Ipv4Compat: true,
								},
							},
						}

						lbEndpoint := envoyEndpoint.LbEndpoint{
							HostIdentifier: &envoyEndpoint.LbEndpoint_Endpoint{
								Endpoint: &envoyEndpoint.Endpoint{
									Address: serviceEndpoint,
								},
							},
						}
						lbEndpoints = append(lbEndpoints, &lbEndpoint)

					}

				}

			}

			connectTimeout := 5 * time.Second

			cluster := envoyv2.Cluster{
				Name:        "knative_" + service.GetName(),
				AltStatName: "",
				ClusterDiscoveryType: &envoyv2.Cluster_Type{
					Type: envoyv2.Cluster_STATIC,
				},
				ConnectTimeout: &connectTimeout,
				LoadAssignment: &envoyv2.ClusterLoadAssignment{
					ClusterName: "knative_" + service.GetName(),
					Endpoints: []*envoyEndpoint.LocalityLbEndpoints{
						{
							LbEndpoints: lbEndpoints,
						},
					},
				},
			}
			log.Info(cluster)

			clusterCache = append(clusterCache, &cluster)
		}

		manager := &hcm.HttpConnectionManager{
			CodecType:  hcm.AUTO,
			StatPrefix: "ingress_http",
			RouteSpecifier: &hcm.HttpConnectionManager_RouteConfig{
				RouteConfig: &envoyv2.RouteConfiguration{
					Name:         "local_route",
					VirtualHosts: virtualHosts,
				},
			},
			HttpFilters: []*hcm.HttpFilter{
				{
					Name: util.Router,
				},
			},
		}

		//TODO: ?????
		pbst, err := util.MessageToStruct(manager)
		if err != nil {
			panic(err)
		}

		listener := &envoyv2.Listener{
			Name: "listener_0",
			Address: &core.Address{
				Address: &core.Address_SocketAddress{
					SocketAddress: &core.SocketAddress{
						Protocol: core.TCP,
						Address:  "0.0.0.0",
						PortSpecifier: &core.SocketAddress_PortValue{
							PortValue: uint32(80),
						},
					},
				},
			},
			FilterChains: []*listener.FilterChain{{
				Filters: []*listener.Filter{{
					Name:       util.HTTPConnectionManager,
					ConfigType: &listener.Filter_Config{Config: pbst},
				}},
			}},
		}
		version = version + 1

		listenerCache := []cache.Resource{listener}

		snapshot := cache.NewSnapshot(fmt.Sprintf("%d", version), nil, clusterCache, routeCache, listenerCache)

		err = xdsConf.SetSnapshot(nodeID, snapshot)
		if err != nil {
			log.Error(err)
		}

		<- eventsChan // Block until there's a change in the endpoints or servings
	}
}

// RunManagementServer starts an xDS server at the given Port.
func RunManagementServer(ctx context.Context, server xds.Server, port uint) {
	var grpcOptions []grpc.ServerOption
	grpcOptions = append(grpcOptions, grpc.MaxConcurrentStreams(grpcMaxConcurrentStreams))
	grpcServer := grpc.NewServer(grpcOptions...)
	lis, err := net.Listen("tcp", fmt.Sprintf(":%d", port))
	if err != nil {
		log.Error("Failed to listen")
	}

	// register services
	discovery.RegisterAggregatedDiscoveryServiceServer(grpcServer, server)
	envoyv2.RegisterEndpointDiscoveryServiceServer(grpcServer, server)
	envoyv2.RegisterClusterDiscoveryServiceServer(grpcServer, server)
	envoyv2.RegisterRouteDiscoveryServiceServer(grpcServer, server)
	envoyv2.RegisterListenerDiscoveryServiceServer(grpcServer, server)

	log.Printf("Starting Management Server on Port %d\n", port)
	go func() {
		if err = grpcServer.Serve(lis); err != nil {
			fmt.Errorf("%s", err)
		}
	}()
	<-ctx.Done()
	grpcServer.GracefulStop()
}

// RunManagementGateway starts an HTTP gateway to an xDS server.
func RunManagementGateway(ctx context.Context, srv xds.Server, port uint) {
	log.Printf("Starting HTTP/1.1 gateway on Port %d\n", port)
	server := &http.Server{Addr: fmt.Sprintf(":%d", port), Handler: &xds.HTTPGateway{Server: srv}}
	go func() {
		if err := server.ListenAndServe(); err != nil {
			panic(err)
		}
	}()

	<-ctx.Done()
	if err := server.Shutdown(ctx); err != nil {
		panic(err)
	}
}

// Hasher returns node ID as an ID
type Hasher struct {
}

func (h Hasher) ID(node *core.Node) string {
	if node == nil {
		return "unknown"
	}
	return node.Id
}

func getHTTPPort(subset v12.EndpointSubset) (uint32, error) {

	for _, port := range subset.Ports {
		if port.Name == "http" {
			return uint32(port.Port), nil
		}
	}

	return 0, fmt.Errorf("http port not found")

}

func getRevision(service v1alpha1.Service) (string, error) {
	for _, traffic := range service.Status.Traffic {
		if traffic.Percent == 100 {
			return traffic.RevisionName, nil
		}
	}

	return "", fmt.Errorf("no revision with 100% of traffic")
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	vegeta "github.com/tsenart/vegeta/lib"
	"io"
	"log"
	"os"
	"runtime"
	"time"
)

func init() {
	runtime.GOMAXPROCS(runtime.NumCPU())
}

func main() {
	var (
		rate     = flag.Uint64("rate", 50, "Requests per second")
		targetsf = flag.String("targets", "targets.txt", "Targets file")
		ordering = flag.String("ordering", "random", "Attack ordering [sequential, random]")
		duration = flag.Duration("duration", 10*time.Second, "Duration of the test")
		reporter = flag.String("reporter", "text", "Reporter to use [text, plot:timings]")
		output   = flag.String("output", "stdout", "Reporter output file")
	)
	flag.Parse()

	if flag.NFlag() == 0 {
		flag.Usage()
		return
	}

	if err := run(*rate, *duration, *targetsf, *ordering, *reporter, *output); err != nil {
		log.Fatal(err)
	}
}

const (
	errRatePrefix        = "Rate: "
	errDurationPrefix    = "Duration: "
	errOutputFilePrefix  = "Output file: "
	errTargetsFilePrefix = "Targets file: "
	errOrderingPrefix    = "Ordering: "
	errReportingPrefix   = "Reporting: "
)

// run is an utility function that validates the attack arguments, sets up the
// required resources, launches the attack and reports the results
func run(rate uint64, duration time.Duration, targetsf, ordering, reporter, output string) error {
	if rate == 0 {
		return fmt.Errorf(errRatePrefix + "can't be zero")
	}

	if duration == 0 {
		return fmt.Errorf(errDurationPrefix + "can't be zero")
	}

	var out io.Writer
	switch output {
	case "stdout":
		out = os.Stdout
	default:
		file, err := os.Create(output)
		if err != nil {
			return fmt.Errorf(errOutputFilePrefix+"(%s): %s", output, err)
		}
		defer file.Close()
		out = file
	}

	var rep vegeta.Reporter
	switch reporter {
	case "text":
		rep = vegeta.NewTextReporter()
	case "plot:timings":
		rep = vegeta.NewTimingsPlotReporter()
	default:
		log.Println("Reporter provided is not supported. Using text")
		rep = vegeta.NewTextReporter()
	}

	targets, err := vegeta.NewTargetsFromFile(targetsf)
	if err != nil {
		return fmt.Errorf(errTargetsFilePrefix+"(%s): %s", targetsf, err)
	}

	switch ordering {
	case "random":
		targets.Shuffle(time.Now().UnixNano())
	case "sequential":
		break
	default:
		return fmt.Errorf(errOrderingPrefix+"`%s` is invalid", ordering)
	}

	log.Printf("Vegeta is attacking %d targets in %s order for %s...\n", len(targets), ordering, duration)
	vegeta.Attack(targets, rate, duration, rep)
	log.Println("Done!")

	log.Printf("Writing report to '%s'...", output)
	if err = rep.Report(out); err != nil {
		return fmt.Errorf(errReportingPrefix+"%s", err)
	}
	return nil
}
<commit_msg>Parameterize number of CPUs to be used<commit_after>package main

import (
	"flag"
	"fmt"
	vegeta "github.com/tsenart/vegeta/lib"
	"io"
	"log"
	"os"
	"runtime"
	"time"
)

func main() {
	var (
		rate     = flag.Uint64("rate", 50, "Requests per second")
		targetsf = flag.String("targets", "targets.txt", "Targets file")
		ordering = flag.String("ordering", "random", "Attack ordering [sequential, random]")
		duration = flag.Duration("duration", 10*time.Second, "Duration of the test")
		reporter = flag.String("reporter", "text", "Reporter to use [text, plot:timings]")
		output   = flag.String("output", "stdout", "Reporter output file")
		cpus     = flag.Int("cpus", runtime.NumCPU(), "Number of CPUs to use")
	)
	flag.Parse()

	if flag.NFlag() == 0 {
		flag.Usage()
		return
	}

	runtime.GOMAXPROCS(*cpus)

	if err := run(*rate, *duration, *targetsf, *ordering, *reporter, *output); err != nil {
		log.Fatal(err)
	}
}

const (
	errRatePrefix        = "Rate: "
	errDurationPrefix    = "Duration: "
	errOutputFilePrefix  = "Output file: "
	errTargetsFilePrefix = "Targets file: "
	errOrderingPrefix    = "Ordering: "
	errReportingPrefix   = "Reporting: "
)

// run is an utility function that validates the attack arguments, sets up the
// required resources, launches the attack and reports the results
func run(rate uint64, duration time.Duration, targetsf, ordering, reporter, output string) error {
	if rate == 0 {
		return fmt.Errorf(errRatePrefix + "can't be zero")
	}

	if duration == 0 {
		return fmt.Errorf(errDurationPrefix + "can't be zero")
	}

	var out io.Writer
	switch output {
	case "stdout":
		out = os.Stdout
	default:
		file, err := os.Create(output)
		if err != nil {
			return fmt.Errorf(errOutputFilePrefix+"(%s): %s", output, err)
		}
		defer file.Close()
		out = file
	}

	var rep vegeta.Reporter
	switch reporter {
	case "text":
		rep = vegeta.NewTextReporter()
	case "plot:timings":
		rep = vegeta.NewTimingsPlotReporter()
	default:
		log.Println("Reporter provided is not supported. Using text")
		rep = vegeta.NewTextReporter()
	}

	targets, err := vegeta.NewTargetsFromFile(targetsf)
	if err != nil {
		return fmt.Errorf(errTargetsFilePrefix+"(%s): %s", targetsf, err)
	}

	switch ordering {
	case "random":
		targets.Shuffle(time.Now().UnixNano())
	case "sequential":
		break
	default:
		return fmt.Errorf(errOrderingPrefix+"`%s` is invalid", ordering)
	}

	log.Printf("Vegeta is attacking %d targets in %s order for %s...\n", len(targets), ordering, duration)
	vegeta.Attack(targets, rate, duration, rep)
	log.Println("Done!")

	log.Printf("Writing report to '%s'...", output)
	if err = rep.Report(out); err != nil {
		return fmt.Errorf(errReportingPrefix+"%s", err)
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"bytes"
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"reflect"
	"strings"
	"sync"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/defaults"
	"github.com/aws/aws-sdk-go/service/s3"
	"launchpad.net/gommap"
)

var expectedDelims = flag.Uint("numDelims", 0, "specify the number of times the delimiter is expected to appear in each line")
var bufferSize = flag.Uint("bufferSize", 8196, "specify the buffer size to use why scanning through files")
var srcBucket = flag.String("srcBucket", "", "specify the full path the bucket that contains the files needing fixing")
var destBucket = flag.String("destBucket", "", "specify a full path to the bucket where the results will be stored")
var badBucket = flag.String("badBucket", "", "specify a full path to the bucket where the unfixable bad results will be stored")
var objPrefix = flag.String("objPrefix", "", "specify the prefix of attached to the objects in question")

func checkFlags() {
	flag.Parse()

	if *expectedDelims == 0 || *srcBucket == "" || *destBucket == "" || *badBucket == "" || *objPrefix == "" {
		flag.PrintDefaults()
		return
	}
}

func main() {
	// validate & parse the flags sent into the command
	checkFlags()

	// check existence of s3 bucket as src
	greeting, _ := getObjectsInBucket()
	fmt.Println(greeting)
	// get array of all object names
	// loop a goroutine for each file to:
	//// -stream object to a mmapped file
	//// -fix the lines
	//// -upload bad rows to separate files
	//// -reupload to s3
	// when all are done
}

func getObjectsInBucket() (*s3.ListObjectsOutput, error) {
	defaults.DefaultConfig.Region = aws.String("us-west-2")

	svc := s3.New(nil)

	params := &s3.ListObjectsInput{
		Bucket: aws.String(*srcBucket), // Required
		Prefix: aws.String(*objPrefix),
	}
	resp, err := svc.ListObjects(params)

	if err != nil {
		// Print the error, cast err to awserr.Error to get the Code and
		// Message from an error.
		fmt.Println(err.Error())
		return resp, err
	}

	// Pretty-print the response data.
	fmt.Println(reflect.TypeOf(resp))
	return resp, nil
}

func fixFile() {
	file, err := os.Open("./data/0000.txt")
	check(err)

	mmap, err := gommap.Map(file.Fd(), gommap.PROT_READ, gommap.MAP_PRIVATE)
	check(err)

	numLines, err := countLines(bytes.NewReader(mmap))
	check(err)

	lines := bytes.SplitN(mmap, []byte{'\n'}, numLines)

	lines[numLines-1] = bytes.Trim(lines[numLines-1], "\n")

	sub := [][][]byte{
		lines[:(numLines / 4)],
		lines[(numLines / 4):(numLines / 2)],
		lines[(numLines / 2) : (numLines/2)+(numLines/4)],
		lines[(numLines/2)+(numLines/4) : numLines],
	}

	jobs := make(chan []byte)
	results := make(chan string)

	wg := new(sync.WaitGroup)
	for w := 0; w <= 3; w++ {
		wg.Add(1)
		go normalizeLines(jobs, results, wg)
	}

	go func() {
		for i := 0; i <= 3; i++ {
			jobs <- bytes.Join(sub[i], []byte{'\n'})
		}
		close(jobs)
	}()

	go func() {
		wg.Wait()
		close(results)
	}()

	for v := range results {
		if strings.HasPrefix(v, "trimmed:") {
			//fmt.Fprintf(os.Stdout, "%s\n", v[strings.IndexAny(v, ":")+1:])
		} else if strings.HasPrefix(v, "bad:") {
			fmt.Fprintf(os.Stdout, "%s\n", v[strings.IndexAny(v, ":")+1:])
		} else {
			//fmt.Fprintf(os.Stdout, "%s\n", v)
		}
	}

}

func normalizeLines(jobs <-chan []byte, results chan<- string, wg *sync.WaitGroup) {
	//defer wg.Done()

	j := <-jobs
	scanner := bufio.NewScanner(bytes.NewReader(j))
	for scanner.Scan() {
		line := scanner.Text()

		delimCount := uint(strings.Count(line, "|"))
		values := strings.Split(line, "|")

		if delimCount < *expectedDelims {
			var fixBuf bytes.Buffer

			fixBuf.WriteString("trimmed:")
			fixBuf.WriteString(strings.TrimSpace(line))

			scanner.Scan()
			line2 := scanner.Text()
			fixBuf.WriteString(strings.TrimSpace(line2))

			scanner.Scan()
			line3 := scanner.Text()
			fixBuf.WriteString(strings.TrimSpace(line3))

			results <- fixBuf.String()
		} else if delimCount > *expectedDelims {
			var badBuf bytes.Buffer

			badBuf.WriteString("bad:")
			badBuf.WriteString(line)

			results <- badBuf.String()
		} else {
			var goodBuf bytes.Buffer

			for _, value := range values {
				trimmedValue := strings.TrimSpace(value)

				goodBuf.WriteString(trimmedValue)

				if !strings.Contains(value, "\n") {
					goodBuf.WriteString("\\|")
				} else {
					goodBuf.WriteString("\n")
				}
			}

			results <- goodBuf.String()
		}
	}

	wg.Done()
}

func check(e error) {
	if e != nil {
		log.Fatal(e)
	}
}

func countLines(r io.Reader) (int, error) {
	// play with this buffer size to optimize for speed
	buf := make([]byte, *bufferSize)
	lineCount := 0
	lineSep := []byte{'\n'}

	for {
		c, err := r.Read(buf)
		if err != nil && err != io.EOF {
			return lineCount, err
		}

		lineCount += bytes.Count(buf[:c], lineSep)

		if err == io.EOF {
			break
		}
	}

	return lineCount, nil
}
<commit_msg>Use objects.Contents<commit_after>package main

import (
	"bufio"
	"bytes"
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"reflect"
	"strings"
	"sync"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/defaults"
	"github.com/aws/aws-sdk-go/service/s3"
	"launchpad.net/gommap"
)

var expectedDelims = flag.Uint("numDelims", 0, "specify the number of times the delimiter is expected to appear in each line")
var bufferSize = flag.Uint("bufferSize", 8196, "specify the buffer size to use why scanning through files")
var srcBucket = flag.String("srcBucket", "", "specify the full path the bucket that contains the files needing fixing")
var destBucket = flag.String("destBucket", "", "specify a full path to the bucket where the results will be stored")
var badBucket = flag.String("badBucket", "", "specify a full path to the bucket where the unfixable bad results will be stored")
var objPrefix = flag.String("objPrefix", "", "specify the prefix of attached to the objects in question")

func checkFlags() {
	flag.Parse()

	if *expectedDelims == 0 || *srcBucket == "" || *destBucket == "" || *badBucket == "" || *objPrefix == "" {
		flag.PrintDefaults()
		return
	}
}

func main() {
	// validate & parse the flags sent into the command
	checkFlags()

	// check existence of s3 bucket as src
	objects, _ := getObjectsInBucket()
	fmt.Println(*objects.Contents[0].Key)
	// get array of all object names - use objects.Contents
	// loop a goroutine for each file to:
	//// -stream object to a mmapped file
	//// -fix the lines
	//// -upload bad rows to separate files
	//// -reupload to s3
	// when all are done
}

func getObjectsInBucket() (*s3.ListObjectsOutput, error) {
	defaults.DefaultConfig.Region = aws.String("us-west-2")

	svc := s3.New(nil)

	params := &s3.ListObjectsInput{
		Bucket: aws.String(*srcBucket), // Required
		Prefix: aws.String(*objPrefix),
	}
	resp, err := svc.ListObjects(params)

	if err != nil {
		// Print the error, cast err to awserr.Error to get the Code and
		// Message from an error.
		fmt.Println(err.Error())
		return resp, err
	}

	// Pretty-print the response data.
	fmt.Println(reflect.TypeOf(resp))
	return resp, nil
}

func fixFile() {
	file, err := os.Open("./data/0000.txt")
	check(err)

	mmap, err := gommap.Map(file.Fd(), gommap.PROT_READ, gommap.MAP_PRIVATE)
	check(err)

	numLines, err := countLines(bytes.NewReader(mmap))
	check(err)

	lines := bytes.SplitN(mmap, []byte{'\n'}, numLines)

	lines[numLines-1] = bytes.Trim(lines[numLines-1], "\n")

	// dear lord, fix this
	sub := [][][]byte{
		lines[:(numLines / 4)],
		lines[(numLines / 4):(numLines / 2)],
		lines[(numLines / 2) : (numLines/2)+(numLines/4)],
		lines[(numLines/2)+(numLines/4) : numLines],
	}

	jobs := make(chan []byte)
	results := make(chan string)

	wg := new(sync.WaitGroup)
	for w := 0; w <= 3; w++ {
		wg.Add(1)
		go normalizeLines(jobs, results, wg)
	}

	go func() {
		for i := 0; i <= 3; i++ {
			jobs <- bytes.Join(sub[i], []byte{'\n'})
		}
		close(jobs)
	}()

	go func() {
		wg.Wait()
		close(results)
	}()

	for v := range results {
		if strings.HasPrefix(v, "trimmed:") {
			//fmt.Fprintf(os.Stdout, "%s\n", v[strings.IndexAny(v, ":")+1:])
		} else if strings.HasPrefix(v, "bad:") {
			fmt.Fprintf(os.Stdout, "%s\n", v[strings.IndexAny(v, ":")+1:])
		} else {
			//fmt.Fprintf(os.Stdout, "%s\n", v)
		}
	}

}

func normalizeLines(jobs <-chan []byte, results chan<- string, wg *sync.WaitGroup) {
	//defer wg.Done()

	j := <-jobs
	scanner := bufio.NewScanner(bytes.NewReader(j))
	for scanner.Scan() {
		line := scanner.Text()

		delimCount := uint(strings.Count(line, "|"))
		values := strings.Split(line, "|")

		if delimCount < *expectedDelims {
			var fixBuf bytes.Buffer

			fixBuf.WriteString("trimmed:")
			fixBuf.WriteString(strings.TrimSpace(line))

			scanner.Scan()
			line2 := scanner.Text()
			fixBuf.WriteString(strings.TrimSpace(line2))

			scanner.Scan()
			line3 := scanner.Text()
			fixBuf.WriteString(strings.TrimSpace(line3))

			results <- fixBuf.String()
		} else if delimCount > *expectedDelims {
			var badBuf bytes.Buffer

			badBuf.WriteString("bad:")
			badBuf.WriteString(line)

			results <- badBuf.String()
		} else {
			var goodBuf bytes.Buffer

			for _, value := range values {
				trimmedValue := strings.TrimSpace(value)

				goodBuf.WriteString(trimmedValue)

				if !strings.Contains(value, "\n") {
					goodBuf.WriteString("\\|")
				} else {
					goodBuf.WriteString("\n")
				}
			}

			results <- goodBuf.String()
		}
	}

	wg.Done()
}

func check(e error) {
	if e != nil {
		log.Fatal(e)
	}
}

func countLines(r io.Reader) (int, error) {
	// play with this buffer size to optimize for speed
	buf := make([]byte, *bufferSize)
	lineCount := 0
	lineSep := []byte{'\n'}

	for {
		c, err := r.Read(buf)
		if err != nil && err != io.EOF {
			return lineCount, err
		}

		lineCount += bytes.Count(buf[:c], lineSep)

		if err == io.EOF {
			break
		}
	}

	return lineCount, nil
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/boltdb/bolt"
    "github.com/golang/glog"
    "flag"
	"github.com/gorilla/mux"
    "github.com/gorilla/context"
    "github.com/gorilla/schema"
    "net/http"
	"html/template"
    texttemplate "text/template"
    "encoding/gob"
    "github.com/BurntSushi/toml"
    "bytes"
    "io"
    "errors"

//    "github.com/russross/blackfriday"
    "github.com/shurcooL/github_flavored_markdown"
	"fmt"
)

const (
    // CountWidgetPerPage count widgets per page
    CountWidgetPerPage = 250
)

var db *bolt.DB
var dbFile = flag.String("db", "/tmp/gong.db", "Path to the BoltDB file")
var addr = flag.String("bind", ":8080", "Listen addres")

func widgetName(name string) string {
    if len(name) == 0 {
        return "pages/__index"
    }
    return name
}

func setupStorage(file string) (*bolt.DB, error) {
    db, err := bolt.Open(file, 0600, nil)

    if err != nil {
        return nil, err
    }

    err = db.Update(func(tx *bolt.Tx) error {
        if _, err := tx.CreateBucketIfNotExists([]byte("widgets")); err != nil {
            return err
        }

		return nil
	})

	if err != nil {
		db.Close()
		return nil, err
	}

	return db, nil
}

func main()  {
    flag.Parse()
    gob.Register(WidgetData{})

    var err error
    if db, err = setupStorage(*dbFile); err != nil {
        glog.Fatal("setup db: ", err)
    }
    defer db.Close()

    r := mux.NewRouter()
    r.Path(`/@{space:(widgets)}:{action:(edit)}/{name:[A-Za-z0-9_.\-/]*}`).HandlerFunc(EditHandler).Methods("GET")
    r.Path(`/@{space:(widgets)}:{action:(edit)}/{name:[A-Za-z0-9_.\-/]*}`).HandlerFunc(OnSaveHandler).Methods("POST")
    r.Path(`/{name:[A-Za-z0-9_.\-/]*}`).HandlerFunc(ViewHandler).Methods("GET")

    http.ListenAndServe(*addr, r)
}

func ViewHandler(w http.ResponseWriter, r *http.Request) {
    ctx := NewContext(w, r);
    widgetName := widgetName(mux.Vars(r)["name"])
    ctx.V["Name"] = widgetName
//    spaceName := mux.Vars(r)["space"]

    ctx.RenderWidget(w, widgetName)
}

func EditHandler(w http.ResponseWriter, r *http.Request) {
    ctx := NewContext(w, r);
    spaceName := mux.Vars(r)["space"]
    widgetName := widgetName(mux.Vars(r)["name"])
    ctx.V["WidgetName"] = widgetName
    ctx.V["FormAction"] = "/@"+ spaceName+ ":"+mux.Vars(r)["action"]+"/"+widgetName
    ctx.V["Widget"], ctx.V["WidgetError"] = GetWidgetData(widgetName)

    ctx.V["CurrentWidgetViewLink"] = "/"+widgetName

    t := template.Must(template.New("page_edit").Funcs(ctx.Funcs).Parse(editTpl))

    if err := ctx.RenderWidget(bytes.NewBufferString(""), widgetName); err != nil {
        ctx.V["Error"] = err
    }

    if err := t.Execute(w, ctx); err != nil {
        glog.Warningln("")
    }
}

func OnSaveHandler(w http.ResponseWriter, r *http.Request) {
//    ctx := NewContext(w, r);
    widgetName := widgetName(mux.Vars(r)["name"])
//    spaceName := mux.Vars(r)["space"]

    err := r.ParseForm()

    if err != nil {
        http.Error(w, "error parse form", http.StatusBadRequest)
        return
    }

    decoder := schema.NewDecoder()
    dto := &WidgetData{}
    err = decoder.Decode(dto, r.PostForm)

    if err != nil {
        http.Error(w, "error decode form data", http.StatusBadRequest)
        return
    }

    dto.Name = widgetName

    if err := SaveWidgetData(dto); err != nil {
        http.Error(w, "error save data", http.StatusBadRequest)
        return
    }

    // TODO: Задать верный URL редиректа
    http.Redirect(w, r, r.URL.String(), http.StatusFound)
}

func NewContext(w http.ResponseWriter, r *http.Request) *Context {
    model := new(Context)
    model.V = make(map[string]interface{})
    model.Self = NewConfig()

    model.Funcs = template.FuncMap{
        "eq": func(a, b interface{}) bool {
            return a == b
        },
        "md": func(v interface{}) template.HTML {
            var str string
            switch v.(type) {
                case string, template.HTML:
                    str = v.(string)
                default:
                    str = fmt.Sprintf("%v", v)
            }
            return template.HTML(github_flavored_markdown.Markdown([]byte(str)))
        },
        "widget": func(ctx *Context, args ...interface{}) template.HTML {

            if len(args) == 0 {
                return ""
            }

            _widgetName, ok :=  args[0].(string)

            if !ok {
                glog.Warningf("template funcs: widget: not valid widget name %v\n", args)
                return ""
            }

            buff := bytes.NewBufferString("")
            ctx.V["Name"] = _widgetName
            ctx.RenderWidget(buff, _widgetName)

            // TODO: render widget
            return template.HTML(buff.String())
        },
    }
    model.ResponseWriter = w
    model.Request = r

    return model
}

// WidgetData настройки виджета
//  Config - toml настройки
//  Raw - html|css|js|etc
type WidgetData struct {
    Name string `schema:"-"`
    Config string `schema:"config"`
    Raw string `schema:"raw"`
}

func (w *WidgetData) gobEncode() ([]byte, error) {
    buf := new(bytes.Buffer)
    enc := gob.NewEncoder(buf)
    err := enc.Encode(w)
    if err != nil {
        return nil, err
    }
    return buf.Bytes(), nil
}

func gobDecode(data []byte) (*WidgetData, error) {
    var w *WidgetData
    buf := bytes.NewBuffer(data)
    dec := gob.NewDecoder(buf)
    err := dec.Decode(&w)
    if err != nil {
        return nil, err
    }
    return w, nil
}

func NewConfig() Config {
    return Config(map[string]interface{}{})
}

type Config map[string]interface{}

func (c Config) Get(name string) interface{} {
    return c[name]
}

func (c Config) GetAsBool(name string) bool {
    _value, ok := c[name].(bool)
    return ok && _value
}

func (c Config) GetAsString(name string) string {
    _value, ok := c[name].(string)
    if !ok {
        return ""
    }
    return _value
}

func (c *Config) SetNewConfig(v interface{}) {
    if _v, ok := v.(map[string]interface{}); ok {
        *c = _v
    } else {
        *c = NewConfig()
    }
}

type Context struct {
    V map[string]interface{}
    Self Config
    Funcs template.FuncMap
    Request *http.Request
    ResponseWriter http.ResponseWriter

    TraceWidgets []string
    depth int
}

// IsEditable Context mode editable
func (p *Context) IsEditable() bool {
    isEditable, ok := context.GetOk(p.Request, "flags.is_editable")
    if !ok {
        return false
    }
    return isEditable.(bool)
}

func (p *Context) RenderWidget(writer io.Writer, name string) error {
    p.TraceWidgets = append(p.TraceWidgets, name)

    if p.depth > CountWidgetPerPage {
        return errors.New("more widgets per page (for example 'layout' is looped, clear 'layout')")
    }
    p.depth++

    widget, err := GetWidgetData(name)

    if err != nil {
        return err
    }

    configBuff := bytes.NewBufferString("")

    ttmpl, err := texttemplate.New("compilation_config").Funcs(texttemplate.FuncMap(p.Funcs)).Parse(widget.Config)

    if err != nil {
        return err
    }

    tt := texttemplate.Must(ttmpl, err)
    if err := tt.Execute(configBuff, p); err != nil {
        glog.Warningf("render widget: toml template compile: %v", err)
    }

    if _, err := toml.Decode(configBuff.String(), &p.V); err != nil {
        glog.Warningf("render widget: toml decode: %v", err)
    }

    // settings current widget
    p.Self.SetNewConfig(p.V["self"])
    delete(p.V, "self")

    widgetBuff := bytes.NewBufferString("")

    switch p.Self.GetAsString("render") {
        case "markdown", "md":
            widgetBuff.Write(github_flavored_markdown.Markdown([]byte(widget.Raw)))
//            widgetBuff.Write(blackfriday.MarkdownCommon([]byte(widget.Raw)))
        default:
            widgetBuff.WriteString(widget.Raw)
    }

    if layout := p.Self.GetAsString("layout"); len(layout) > 0 {
        contentBuff := bytes.NewBufferString("")

        tmpl, err := template.New("compilation_page_layout").Funcs(p.Funcs).Parse(widgetBuff.String());

        if err != nil {
            glog.Warningf("render widget: compile template: %s, \n===\n%s\n===\n", name, []byte(widget.Raw))
            return err
        }

        template.Must(tmpl, err).Execute(contentBuff, p)

        p.V["Content"] = contentBuff.String()

        return p.RenderWidget(writer, layout)
    }

    tmpl, err := template.New("compilation_data").Funcs(p.Funcs).Parse(widgetBuff.String())

    if err != nil {
        return err
    }

    t := template.Must(tmpl, err)
    return t.Execute(writer, p)
}

// GetWidgetData get widget by name
func GetWidgetData(name string) (*WidgetData, error) {
    var _bytes []byte
    err := db.View(func(tx *bolt.Tx) error {
        _bytes = tx.Bucket([]byte("widgets")).Get([]byte(name))
        return nil
    })

    if err != nil {
        return &WidgetData{Config: defaultConfig, Raw: defaultRaw}, nil
    }

    widget, err := gobDecode(_bytes);

    if err != nil {
        return &WidgetData{Config: defaultConfig, Raw: defaultRaw}, nil
    }

    return widget, nil
}

// SaveWidgetData save widget
func SaveWidgetData(dto *WidgetData) error {
    return db.Update(func(tx *bolt.Tx) error{
        _bytes, err := dto.gobEncode()
        if err != nil {
            return err
        }
        return tx.Bucket([]byte("widgets")).Put([]byte(dto.Name), _bytes)
    })
}

const defaultConfig = `
title = "page title" # if used layout

[self]
render = "" # or markdown
# if the render=markdown in the widget data should not have dynamic parameters

# layout = "" # current widget will be in the .V.Content variable
link_edit = "/@widgets:edit/{{.V.Name}}" # example dynamic parameter
link_title = "edit"
`
const defaultRaw = `<a href="{{.Self.link_edit}}">{{.Self.link_title}}</a>`

const editTpl = `<!DOCTYPE html>
<title>{{.V.Title}}</title>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
<meta charset="utf-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge" />
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<link rel="stylesheet" href="//cdnjs.cloudflare.com/ajax/libs/uikit/2.25.0/css/uikit.almost-flat.min.css">

        /

{{.V.WidgetName}}

        {{ if .V.Error}}

{{.V.Error}}

        {{ end }}
        <form class="uk-form uk-form-stacked" action="{{.V.FormAction}}" method="POST" >

                Config (toml):

{{.V.Widget.Config}}

                    <textarea id="src_config" name="config" style="display: none;">{{.V.Widget.Config}}</textarea>

                Data (html or css or javascript or text or etc):

{{.V.Widget.Raw}}

                    <textarea id="src_raw" name="raw" style="display: none;">{{.V.Widget.Raw}}</textarea>

                View widget
                <button class="uk-button">Save</button>
                Cancel

        </form>

Linked widgets:

        {{range $widgetName := .TraceWidgets}}

{{$widgetName}}

        {{ end }}

`<commit_msg>added helper mode<commit_after>package main

import (
	"github.com/boltdb/bolt"
    "github.com/golang/glog"
    "flag"
	"github.com/gorilla/mux"
    "github.com/gorilla/context"
    "github.com/gorilla/schema"
    "net/http"
	"html/template"
    texttemplate "text/template"
    "encoding/gob"
    "github.com/BurntSushi/toml"
    "bytes"
    "io"
    "errors"

//    "github.com/russross/blackfriday"
    "github.com/shurcooL/github_flavored_markdown"
	"fmt"
)

const (
    // CountWidgetPerPage count widgets per page
    CountWidgetPerPage = 250
)

var db *bolt.DB
var dbFile = flag.String("db", "/tmp/gong.db", "Path to the BoltDB file")
var addr = flag.String("bind", ":8080", "Listen addres")

func widgetName(name string) string {
    if len(name) == 0 {
        return "pages/__index"
    }
    return name
}

func setupStorage(file string) (*bolt.DB, error) {
    db, err := bolt.Open(file, 0600, nil)

    if err != nil {
        return nil, err
    }

    err = db.Update(func(tx *bolt.Tx) error {
        if _, err := tx.CreateBucketIfNotExists([]byte("widgets")); err != nil {
            return err
        }

		return nil
	})

	if err != nil {
		db.Close()
		return nil, err
	}

	return db, nil
}

func main()  {
    flag.Parse()
    gob.Register(WidgetData{})

    var err error
    if db, err = setupStorage(*dbFile); err != nil {
        glog.Fatal("setup db: ", err)
    }
    defer db.Close()

    r := mux.NewRouter()
    r.Path(`/@{space:(widgets)}:{action:(edit)}/{name:[A-Za-z0-9_.\-/]*}`).HandlerFunc(EditHandler).Methods("GET")
    r.Path(`/@{space:(widgets)}:{action:(edit)}/{name:[A-Za-z0-9_.\-/]*}`).HandlerFunc(OnSaveHandler).Methods("POST")
    r.Path(`/{name:[A-Za-z0-9_.\-/]*}`).HandlerFunc(ViewHandler).Methods("GET")

    http.ListenAndServe(*addr, r)
}

func ViewHandler(w http.ResponseWriter, r *http.Request) {
    ctx := NewContext(w, r);
    widgetName := widgetName(mux.Vars(r)["name"])
    ctx.V["Name"] = widgetName
    ctx.V["ShowHelpLink"] = len(r.URL.Query().Get("editable")) > 0
//    spaceName := mux.Vars(r)["space"]

    ctx.RenderWidget(w, widgetName)
}

func EditHandler(w http.ResponseWriter, r *http.Request) {
    ctx := NewContext(w, r);
    spaceName := mux.Vars(r)["space"]
    widgetName := widgetName(mux.Vars(r)["name"])
    ctx.V["WidgetName"] = widgetName
    ctx.V["FormAction"] = "/@"+ spaceName+ ":"+mux.Vars(r)["action"]+"/"+widgetName
    ctx.V["Widget"], ctx.V["WidgetError"] = GetWidgetData(widgetName)

    ctx.V["CurrentWidgetViewLink"] = "/"+widgetName

    t := template.Must(template.New("page_edit").Funcs(ctx.Funcs).Parse(editTpl))

    if err := ctx.RenderWidget(bytes.NewBufferString(""), widgetName); err != nil {
        ctx.V["Error"] = err
    }

    if err := t.Execute(w, ctx); err != nil {
        glog.Warningln("")
    }
}

func OnSaveHandler(w http.ResponseWriter, r *http.Request) {
//    ctx := NewContext(w, r);
    widgetName := widgetName(mux.Vars(r)["name"])
//    spaceName := mux.Vars(r)["space"]

    err := r.ParseForm()

    if err != nil {
        http.Error(w, "error parse form", http.StatusBadRequest)
        return
    }

    decoder := schema.NewDecoder()
    dto := &WidgetData{}
    err = decoder.Decode(dto, r.PostForm)

    if err != nil {
        http.Error(w, "error decode form data", http.StatusBadRequest)
        return
    }

    dto.Name = widgetName

    if err := SaveWidgetData(dto); err != nil {
        http.Error(w, "error save data", http.StatusBadRequest)
        return
    }

    // TODO: Задать верный URL редиректа
    http.Redirect(w, r, r.URL.String(), http.StatusFound)
}

func NewContext(w http.ResponseWriter, r *http.Request) *Context {
    model := new(Context)
    model.V = make(map[string]interface{})
    model.Self = NewConfig()

    model.Funcs = template.FuncMap{
        "eq": func(a, b interface{}) bool {
            return a == b
        },
        "md": func(v interface{}) template.HTML {
            var str string
            switch v.(type) {
                case string, template.HTML:
                    str = v.(string)
                default:
                    str = fmt.Sprintf("%v", v)
            }
            return template.HTML(github_flavored_markdown.Markdown([]byte(str)))
        },
        "widget": func(ctx *Context, args ...interface{}) template.HTML {

            if len(args) == 0 {
                return ""
            }

            _widgetName, ok :=  args[0].(string)

            if !ok {
                glog.Warningf("template funcs: widget: not valid widget name %v\n", args)
                return ""
            }

            buff := bytes.NewBufferString("")
            ctx.V["Name"] = _widgetName
            ctx.RenderWidget(buff, _widgetName)

            // TODO: render widget
            return template.HTML(buff.String())
        },
    }
    model.ResponseWriter = w
    model.Request = r

    return model
}

// WidgetData настройки виджета
//  Config - toml настройки
//  Raw - html|css|js|etc
type WidgetData struct {
    Name string `schema:"-"`
    Config string `schema:"config"`
    Raw string `schema:"raw"`
}

func (w *WidgetData) gobEncode() ([]byte, error) {
    buf := new(bytes.Buffer)
    enc := gob.NewEncoder(buf)
    err := enc.Encode(w)
    if err != nil {
        return nil, err
    }
    return buf.Bytes(), nil
}

func gobDecode(data []byte) (*WidgetData, error) {
    var w *WidgetData
    buf := bytes.NewBuffer(data)
    dec := gob.NewDecoder(buf)
    err := dec.Decode(&w)
    if err != nil {
        return nil, err
    }
    return w, nil
}

func NewConfig() Config {
    return Config(map[string]interface{}{})
}

type Config map[string]interface{}

func (c Config) Get(name string) interface{} {
    return c[name]
}

func (c Config) GetAsBool(name string) bool {
    _value, ok := c[name].(bool)
    return ok && _value
}

func (c Config) GetAsString(name string) string {
    _value, ok := c[name].(string)
    if !ok {
        return ""
    }
    return _value
}

func (c *Config) SetNewConfig(v interface{}) {
    if _v, ok := v.(map[string]interface{}); ok {
        *c = _v
    } else {
        *c = NewConfig()
    }
}

type Context struct {
    V map[string]interface{}
    Self Config
    Funcs template.FuncMap
    Request *http.Request
    ResponseWriter http.ResponseWriter

    TraceWidgets []string
    depth int
}

// IsEditable Context mode editable
func (p *Context) IsEditable() bool {
    isEditable, ok := context.GetOk(p.Request, "flags.is_editable")
    if !ok {
        return false
    }
    return isEditable.(bool)
}

func (p *Context) RenderWidget(writer io.Writer, name string) error {
    p.TraceWidgets = append(p.TraceWidgets, name)

    if p.depth > CountWidgetPerPage {
        return errors.New("more widgets per page (for example 'layout' is looped, clear 'layout')")
    }
    p.depth++

    widget, err := GetWidgetData(name)

    if err != nil {
        return err
    }

    configBuff := bytes.NewBufferString("")

    ttmpl, err := texttemplate.New("compilation_config").Funcs(texttemplate.FuncMap(p.Funcs)).Parse(widget.Config)

    if err != nil {
        return err
    }

    tt := texttemplate.Must(ttmpl, err)
    if err := tt.Execute(configBuff, p); err != nil {
        glog.Warningf("render widget: toml template compile: %v", err)
    }

    if _, err := toml.Decode(configBuff.String(), &p.V); err != nil {
        glog.Warningf("render widget: toml decode: %v", err)
    }

    // settings current widget
    p.Self.SetNewConfig(p.V["self"])
    delete(p.V, "self")

    widgetBuff := bytes.NewBufferString("")

    switch p.Self.GetAsString("render") {
        case "markdown", "md":
            widgetBuff.Write(github_flavored_markdown.Markdown([]byte(widget.Raw)))
//            widgetBuff.Write(blackfriday.MarkdownCommon([]byte(widget.Raw)))
        default:
            widgetBuff.WriteString(widget.Raw)
    }

    if layout := p.Self.GetAsString("layout"); len(layout) > 0 {
        contentBuff := bytes.NewBufferString("")

        tmpl, err := template.New("compilation_page_layout").Funcs(p.Funcs).Parse(widgetBuff.String());

        if err != nil {
            glog.Warningf("render widget: compile template: %s, \n===\n%s\n===\n", name, []byte(widget.Raw))
            return err
        }

        template.Must(tmpl, err).Execute(contentBuff, p)

        p.V["Content"] = template.HTML(contentBuff.String())

        return p.RenderWidget(writer, layout)
    }

    tmpl, err := template.New("compilation_data").Funcs(p.Funcs).Parse(widgetBuff.String())

    if err != nil {
        return err
    }

    t := template.Must(tmpl, err)
    return t.Execute(writer, p)
}

// GetWidgetData get widget by name
func GetWidgetData(name string) (*WidgetData, error) {
    var _bytes []byte
    err := db.View(func(tx *bolt.Tx) error {
        _bytes = tx.Bucket([]byte("widgets")).Get([]byte(name))
        return nil
    })

    if err != nil {
        return &WidgetData{Config: defaultConfig, Raw: defaultRaw}, nil
    }

    widget, err := gobDecode(_bytes);

    if err != nil {
        return &WidgetData{Config: defaultConfig, Raw: defaultRaw}, nil
    }

    return widget, nil
}

// SaveWidgetData save widget
func SaveWidgetData(dto *WidgetData) error {
    return db.Update(func(tx *bolt.Tx) error{
        _bytes, err := dto.gobEncode()
        if err != nil {
            return err
        }
        return tx.Bucket([]byte("widgets")).Put([]byte(dto.Name), _bytes)
    })
}

const defaultConfig = `
title = "page title" # if used layout

[self]
render = "" # or markdown
# if the render=markdown in the widget data should not have dynamic parameters

# layout = "" # current widget will be in the .V.Content variable
link_edit = "/@widgets:edit/{{.V.Name}}" # example dynamic parameter
link_title = "edit"
`
const defaultRaw = `{{if .V.ShowHelpLink}}<a href="{{.Self.link_edit}}" title="{{.V.Name}}">edit</a>{{end}}`

const editTpl = `<!DOCTYPE html>
<title>{{.V.Title}}</title>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
<meta charset="utf-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge" />
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<link rel="stylesheet" href="//cdnjs.cloudflare.com/ajax/libs/uikit/2.25.0/css/uikit.almost-flat.min.css">

        /

{{.V.WidgetName}}

        {{ if .V.Error}}

{{.V.Error}}

        {{ end }}
        <form class="uk-form uk-form-stacked" action="{{.V.FormAction}}" method="POST" >

                Config (toml):

{{.V.Widget.Config}}

                    <textarea id="src_config" name="config" style="display: none;">{{.V.Widget.Config}}</textarea>

                Data (html or css or javascript or text or etc):

{{.V.Widget.Raw}}

                    <textarea id="src_raw" name="raw" style="display: none;">{{.V.Widget.Raw}}</textarea>

                View widget
                <button class="uk-button">Save</button>
                Cancel

        </form>

Linked widgets:

        {{range $widgetName := .TraceWidgets}}

{{$widgetName}}

        {{ end }}

`<|endoftext|>