<commit_before>package main

//go:generate esc -o templates.go ./templates

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"strings"
	"text/tabwriter"

	"github.com/bukalapak/snowboard/adapter/drafter"
	"github.com/bukalapak/snowboard/adapter/drafterc"
	snowboard "github.com/bukalapak/snowboard/parser"
	"github.com/fsnotify/fsnotify"
	xerrors "github.com/pkg/errors"
	"github.com/urfave/cli"
)

var versionStr string
var (
	engine  snowboard.Parser
	engineC snowboard.Parser
)

func main() {
	engine = drafter.Engine{}
	engineC = drafterc.Engine{}

	cli.VersionPrinter = func(c *cli.Context) {
		fmt.Fprintf(c.App.Writer, "Snowboard version: %s\n", c.App.Version)
		fmt.Fprintf(c.App.Writer, "Drafter version: %s\n", engine.Version())
	}

	if versionStr == "" {
		versionStr = "HEAD"
	}

	app := cli.NewApp()
	app.Name = "snowboard"
	app.Usage = "API blueprint toolkit"
	app.Version = versionStr
	app.Commands = []cli.Command{
		{
			Name:  "lint",
			Usage: "Validate API blueprint",
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "i",
					Usage: "API blueprint file",
				},
				cli.BoolFlag{
					Name:  "c",
					Usage: "Use character index instead of line and row number",
				},
			},
			Action: func(c *cli.Context) error {
				err := validate(c, c.String("i"), c.Bool("c"))

				if strings.Contains(err.Error(), "read failed") {
					return xerrors.Cause(err)
				}

				io.WriteString(c.App.Writer, err.Error())
				return nil
			},
		},
		{
			Name:  "html",
			Usage: "Render HTML documentation",
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "i",
					Usage: "API blueprint file",
				},
				cli.StringFlag{
					Name:  "o",
					Value: "index.html",
					Usage: "HTML file",
				},
				cli.StringFlag{
					Name:  "t",
					Value: "alpha",
					Usage: "Template for HTML documentation",
				},
				cli.BoolFlag{
					Name:  "s",
					Usage: "Serve HTML via HTTP server",
				},
				cli.StringFlag{
					Name:  "b",
					Value: "127.0.0.1:8088",
					Usage: "HTTP server listen address",
				},
			},
			Action: func(c *cli.Context) error {
				if c.Bool("s") {
					return watchHTML(c, c.String("i"), c.String("o"), c.String("t"), c.String("b"))
				}

				return renderHTML(c, c.String("i"), c.String("o"), c.String("t"))
			},
		},
		{
			Name:  "apib",
			Usage: "Render API blueprint",
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "i",
					Usage: "API blueprint file",
				},
				cli.StringFlag{
					Name:  "o",
					Usage: "API blueprint output file",
				},
			},
			Action: func(c *cli.Context) error {
				return renderAPIB(c, c.String("i"), c.String("o"))
			},
		},
		{
			Name:  "mock",
			Usage: "Run Mock server",
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "i",
					Usage: "API blueprint file",
				},
				cli.StringFlag{
					Name:  "b",
					Value: "127.0.0.1:8087",
					Usage: "HTTP server listen address",
				},
			},
			Action: func(c *cli.Context) error {
				return serveMock(c, c.String("b"), c.String("i"))
			},
		},
		{
			Name:  "adapter",
			Usage: "Snowboard adapter",
			Flags: []cli.Flag{
				cli.BoolFlag{
					Name:  "c",
					Usage: "Copy snowboard adapter to desired location",
				},
				cli.StringFlag{
					Name:  "p",
					Value: ".",
					Usage: "Adapter location dir",
				},
			},
			Action: func(c *cli.Context) error {
				if c.Bool("c") {
					return installAdapters(c, c.String("p"))
				}

				return nil
			},
		},
	}
	app.Run(os.Args)
}

func readFile(fn string) ([]byte, error) {
	info, err := os.Stat(fn)
	if err != nil {
		return nil, errors.New("File is not exist")
	}

	if info.IsDir() {
		return nil, errors.New("File is a directory")
	}

	return ioutil.ReadFile(fn)
}

func readTemplate(fn string) ([]byte, error) {
	tf, err := readFile(fn)
	if err == nil {
		return tf, nil
	}

	fs := FS(false)
	ff, err := fs.Open("/templates/" + fn + ".html")
	if err != nil {
		return nil, err
	}

	defer ff.Close()
	return ioutil.ReadAll(ff)
}

func renderHTML(c *cli.Context, input, output, tplFile string) error {
	bp, err := snowboard.Load(input, engine)
	if err != nil {
		return err
	}

	of, err := os.Create(output)
	if err != nil {
		return err
	}
	defer of.Close()

	tf, err := readTemplate(tplFile)
	if err != nil {
		return err
	}

	err = snowboard.HTML(string(tf), of, bp)
	if err != nil {
		return err
	}

	fmt.Fprintln(c.App.Writer, "HTML has been generated!")
	return nil
}

func renderAPIB(c *cli.Context, input, output string) error {
	b, err := snowboard.Read(input)
	if err != nil {
		return err
	}

	of, err := os.Create(output)
	if err != nil {
		return err
	}
	defer of.Close()

	_, err = io.Copy(of, bytes.NewReader(b))
	if err != nil {
		return err
	}

	fmt.Fprintln(c.App.Writer, "API blueprint has been generated!")
	return nil
}

func validate(c *cli.Context, input string, charIdx bool) error {
	b, err := readFile(input)
	if err != nil {
		return xerrors.Wrap(err, "read failed")
	}

	bf := bytes.NewReader(b)

	if charIdx {
		out, err := snowboard.Validate(bf, engine)
		if err != nil {
			return err
		}

		if out == nil {
			fmt.Fprintln(c.App.Writer, "OK")
			return nil
		}

		var buf bytes.Buffer

		s := "--------"
		w := tabwriter.NewWriter(&buf, 8, 0, 0, ' ', tabwriter.Debug)
		fmt.Fprintln(w, "Char Index\tDescription")
		fmt.Fprintf(w, "%s\t%s\n", s, strings.Repeat(s, 8))

		for _, n := range out.Annotations {
			for _, m := range n.SourceMaps {
				fmt.Fprintf(w, "%d:%d\t%s\n", m.Row, m.Col, n.Description)
			}
		}

		w.Flush()

		if len(out.Annotations) > 0 {
			return errors.New(buf.String())
		}
	} else {
		b, err := engineC.Validate(bf)
		if err != nil {
			return err
		}

		var buf bytes.Buffer

		w := tabwriter.NewWriter(&buf, 8, 0, 0, ' ', tabwriter.Debug)
		fmt.Fprintln(w, "Location\tSeverity\tDescription")

		fmt.Fprintf(w, "%s\t%s\t%s\n", dash(42), dash(16), dash(80))

		if string(b) == "" {
			fmt.Fprintln(c.App.Writer, "OK")
			return nil
		}

		ns := strings.Split(string(b), "\n")

		for _, n := range ns {
			nn := strings.SplitN(n, "; ", 2)
			mm := strings.SplitN(nn[0], "  ", 2)
			fmt.Fprintf(w, "%s\t%s\t%s\n", nn[1], mm[0], mm[1])
		}

		w.Flush()
		return errors.New(buf.String())
	}

	return nil
}

func dash(n int) string {
	return strings.Repeat("-", n)
}

func watchHTML(c *cli.Context, input, output, tplFile, bind string) error {
	watcher, err := fsnotify.NewWatcher()
	if err != nil {
		return err
	}
	defer watcher.Close()

	done := make(chan bool)
	go func() {
		for {
			select {
			case event := <-watcher.Events:
				if event.Op&fsnotify.Write == fsnotify.Write {
					renderHTML(c, input, output, tplFile)
				}
			case err := <-watcher.Errors:
				fmt.Fprintln(c.App.Writer, err)
			}
		}
	}()

	err = watcher.Add(input)
	if err != nil {
		return err
	}

	if _, err = os.Stat(tplFile); err == nil {
		err = watcher.Add(tplFile)
		if err != nil {
			return err
		}
	}

	renderHTML(c, input, output, tplFile)
	serveHTML(bind, output)

	<-done

	return nil
}

func serveHTML(bind, output string) error {
	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		http.ServeFile(w, r, output)
	})

	return http.ListenAndServe(bind, nil)
}

func serveMock(c *cli.Context, bind, input string) error {
	bp, err := snowboard.Load(input, engine)
	if err != nil {
		return err
	}

	fmt.Fprintf(c.App.Writer, "Mock server is ready. Use %s\n", bind)
	fmt.Fprintln(c.App.Writer, "Available Routes:")

	ms := snowboard.Mock(bp)
	for _, m := range ms {
		fmt.Fprintf(c.App.Writer, "%s\t%d\t%s\n", m.Method, m.StatusCode, m.Pattern)
	}

	h := snowboard.MockHandler(ms)
	return http.ListenAndServe(bind, h)
}

func installAdapters(c *cli.Context, dir string) error {
	n := drafterc.Engine{}
	name, err := n.CopyExec(dir)
	if err != nil {
		return err
	}

	fmt.Fprintln(c.App.Writer, "Snowboard adapter installed!\nLocation: "+name)
	return nil
}
<commit_msg>handle non-error<commit_after>package main

//go:generate esc -o templates.go ./templates

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"strings"
	"text/tabwriter"

	"github.com/bukalapak/snowboard/adapter/drafter"
	"github.com/bukalapak/snowboard/adapter/drafterc"
	snowboard "github.com/bukalapak/snowboard/parser"
	"github.com/fsnotify/fsnotify"
	xerrors "github.com/pkg/errors"
	"github.com/urfave/cli"
)

var versionStr string
var (
	engine  snowboard.Parser
	engineC snowboard.Parser
)

func main() {
	engine = drafter.Engine{}
	engineC = drafterc.Engine{}

	cli.VersionPrinter = func(c *cli.Context) {
		fmt.Fprintf(c.App.Writer, "Snowboard version: %s\n", c.App.Version)
		fmt.Fprintf(c.App.Writer, "Drafter version: %s\n", engine.Version())
	}

	if versionStr == "" {
		versionStr = "HEAD"
	}

	app := cli.NewApp()
	app.Name = "snowboard"
	app.Usage = "API blueprint toolkit"
	app.Version = versionStr
	app.Commands = []cli.Command{
		{
			Name:  "lint",
			Usage: "Validate API blueprint",
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "i",
					Usage: "API blueprint file",
				},
				cli.BoolFlag{
					Name:  "c",
					Usage: "Use character index instead of line and row number",
				},
			},
			Action: func(c *cli.Context) error {
				if err := validate(c, c.String("i"), c.Bool("c")); err != nil {
					if strings.Contains(err.Error(), "read failed") {
						return xerrors.Cause(err)
					}

					io.WriteString(c.App.Writer, err.Error())
				}

				return nil
			},
		},
		{
			Name:  "html",
			Usage: "Render HTML documentation",
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "i",
					Usage: "API blueprint file",
				},
				cli.StringFlag{
					Name:  "o",
					Value: "index.html",
					Usage: "HTML file",
				},
				cli.StringFlag{
					Name:  "t",
					Value: "alpha",
					Usage: "Template for HTML documentation",
				},
				cli.BoolFlag{
					Name:  "s",
					Usage: "Serve HTML via HTTP server",
				},
				cli.StringFlag{
					Name:  "b",
					Value: "127.0.0.1:8088",
					Usage: "HTTP server listen address",
				},
			},
			Action: func(c *cli.Context) error {
				if c.Bool("s") {
					return watchHTML(c, c.String("i"), c.String("o"), c.String("t"), c.String("b"))
				}

				return renderHTML(c, c.String("i"), c.String("o"), c.String("t"))
			},
		},
		{
			Name:  "apib",
			Usage: "Render API blueprint",
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "i",
					Usage: "API blueprint file",
				},
				cli.StringFlag{
					Name:  "o",
					Usage: "API blueprint output file",
				},
			},
			Action: func(c *cli.Context) error {
				return renderAPIB(c, c.String("i"), c.String("o"))
			},
		},
		{
			Name:  "mock",
			Usage: "Run Mock server",
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "i",
					Usage: "API blueprint file",
				},
				cli.StringFlag{
					Name:  "b",
					Value: "127.0.0.1:8087",
					Usage: "HTTP server listen address",
				},
			},
			Action: func(c *cli.Context) error {
				return serveMock(c, c.String("b"), c.String("i"))
			},
		},
		{
			Name:  "adapter",
			Usage: "Snowboard adapter",
			Flags: []cli.Flag{
				cli.BoolFlag{
					Name:  "c",
					Usage: "Copy snowboard adapter to desired location",
				},
				cli.StringFlag{
					Name:  "p",
					Value: ".",
					Usage: "Adapter location dir",
				},
			},
			Action: func(c *cli.Context) error {
				if c.Bool("c") {
					return installAdapters(c, c.String("p"))
				}

				return nil
			},
		},
	}
	app.Run(os.Args)
}

func readFile(fn string) ([]byte, error) {
	info, err := os.Stat(fn)
	if err != nil {
		return nil, errors.New("File is not exist")
	}

	if info.IsDir() {
		return nil, errors.New("File is a directory")
	}

	return ioutil.ReadFile(fn)
}

func readTemplate(fn string) ([]byte, error) {
	tf, err := readFile(fn)
	if err == nil {
		return tf, nil
	}

	fs := FS(false)
	ff, err := fs.Open("/templates/" + fn + ".html")
	if err != nil {
		return nil, err
	}

	defer ff.Close()
	return ioutil.ReadAll(ff)
}

func renderHTML(c *cli.Context, input, output, tplFile string) error {
	bp, err := snowboard.Load(input, engine)
	if err != nil {
		return err
	}

	of, err := os.Create(output)
	if err != nil {
		return err
	}
	defer of.Close()

	tf, err := readTemplate(tplFile)
	if err != nil {
		return err
	}

	err = snowboard.HTML(string(tf), of, bp)
	if err != nil {
		return err
	}

	fmt.Fprintln(c.App.Writer, "HTML has been generated!")
	return nil
}

func renderAPIB(c *cli.Context, input, output string) error {
	b, err := snowboard.Read(input)
	if err != nil {
		return err
	}

	of, err := os.Create(output)
	if err != nil {
		return err
	}
	defer of.Close()

	_, err = io.Copy(of, bytes.NewReader(b))
	if err != nil {
		return err
	}

	fmt.Fprintln(c.App.Writer, "API blueprint has been generated!")
	return nil
}

func validate(c *cli.Context, input string, charIdx bool) error {
	b, err := readFile(input)
	if err != nil {
		return xerrors.Wrap(err, "read failed")
	}

	bf := bytes.NewReader(b)

	if charIdx {
		out, err := snowboard.Validate(bf, engine)
		if err != nil {
			return err
		}

		if out == nil {
			fmt.Fprintln(c.App.Writer, "OK")
			return nil
		}

		var buf bytes.Buffer

		s := "--------"
		w := tabwriter.NewWriter(&buf, 8, 0, 0, ' ', tabwriter.Debug)
		fmt.Fprintln(w, "Char Index\tDescription")
		fmt.Fprintf(w, "%s\t%s\n", s, strings.Repeat(s, 8))

		for _, n := range out.Annotations {
			for _, m := range n.SourceMaps {
				fmt.Fprintf(w, "%d:%d\t%s\n", m.Row, m.Col, n.Description)
			}
		}

		w.Flush()

		if len(out.Annotations) > 0 {
			return errors.New(buf.String())
		}
	} else {
		b, err := engineC.Validate(bf)
		if err != nil {
			return err
		}

		var buf bytes.Buffer

		w := tabwriter.NewWriter(&buf, 8, 0, 0, ' ', tabwriter.Debug)
		fmt.Fprintln(w, "Location\tSeverity\tDescription")

		fmt.Fprintf(w, "%s\t%s\t%s\n", dash(42), dash(16), dash(80))

		if string(b) == "" {
			fmt.Fprintln(c.App.Writer, "OK")
			return nil
		}

		ns := strings.Split(string(b), "\n")

		for _, n := range ns {
			nn := strings.SplitN(n, "; ", 2)
			mm := strings.SplitN(nn[0], "  ", 2)
			fmt.Fprintf(w, "%s\t%s\t%s\n", nn[1], mm[0], mm[1])
		}

		w.Flush()
		return errors.New(buf.String())
	}

	return nil
}

func dash(n int) string {
	return strings.Repeat("-", n)
}

func watchHTML(c *cli.Context, input, output, tplFile, bind string) error {
	watcher, err := fsnotify.NewWatcher()
	if err != nil {
		return err
	}
	defer watcher.Close()

	done := make(chan bool)
	go func() {
		for {
			select {
			case event := <-watcher.Events:
				if event.Op&fsnotify.Write == fsnotify.Write {
					renderHTML(c, input, output, tplFile)
				}
			case err := <-watcher.Errors:
				fmt.Fprintln(c.App.Writer, err)
			}
		}
	}()

	err = watcher.Add(input)
	if err != nil {
		return err
	}

	if _, err = os.Stat(tplFile); err == nil {
		err = watcher.Add(tplFile)
		if err != nil {
			return err
		}
	}

	renderHTML(c, input, output, tplFile)
	serveHTML(bind, output)

	<-done

	return nil
}

func serveHTML(bind, output string) error {
	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		http.ServeFile(w, r, output)
	})

	return http.ListenAndServe(bind, nil)
}

func serveMock(c *cli.Context, bind, input string) error {
	bp, err := snowboard.Load(input, engine)
	if err != nil {
		return err
	}

	fmt.Fprintf(c.App.Writer, "Mock server is ready. Use %s\n", bind)
	fmt.Fprintln(c.App.Writer, "Available Routes:")

	ms := snowboard.Mock(bp)
	for _, m := range ms {
		fmt.Fprintf(c.App.Writer, "%s\t%d\t%s\n", m.Method, m.StatusCode, m.Pattern)
	}

	h := snowboard.MockHandler(ms)
	return http.ListenAndServe(bind, h)
}

func installAdapters(c *cli.Context, dir string) error {
	n := drafterc.Engine{}
	name, err := n.CopyExec(dir)
	if err != nil {
		return err
	}

	fmt.Fprintln(c.App.Writer, "Snowboard adapter installed!\nLocation: "+name)
	return nil
}
<|endoftext|>


<commit_before>package main // import nirenjan.org/overlord

import (
	"nirenjan.org/overlord/internal/cmds/cli"

	// Overlord modules
	_ "nirenjan.org/overlord/internal/cmds/backup"
	_ "nirenjan.org/overlord/internal/cmds/init"
	_ "nirenjan.org/overlord/internal/cmds/journal"
	_ "nirenjan.org/overlord/internal/cmds/version"
)

func main() {
	cli.Parse()
}
<commit_msg>Fix import comment<commit_after>package main // import "nirenjan.org/overlord"

import (
	"nirenjan.org/overlord/internal/cmds/cli"

	// Overlord modules
	_ "nirenjan.org/overlord/internal/cmds/backup"
	_ "nirenjan.org/overlord/internal/cmds/init"
	_ "nirenjan.org/overlord/internal/cmds/journal"
	_ "nirenjan.org/overlord/internal/cmds/version"
)

func main() {
	cli.Parse()
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"net/url"
	"strings"
	"sync"
	"time"

	"github.com/jpillora/backoff"
	"github.com/jpillora/opts"
	"github.com/miekg/dns"
	"github.com/shirou/gopsutil/load"
	"github.com/shirou/gopsutil/mem"
)

var VERSION = "0"

type config struct {
	URL      string        `help:"InfluxDB URL"`
	Database string        `help:"InfluxDB database"`
	Interval time.Duration `help:"Time between reports"`
	Tags     string        `help:"InfluxDB tags in the form \"<tag>=<value>,<tag>=<value>\""`
	DNS      string        `help:"DNS server (used to lookup URL)"`
}

func main() {
	c := config{
		URL:      "http://localhost:8086",
		Database: "test",
		Interval: 5 * time.Minute,
	}

	opts.New(&c).Name("sysflux").Version(VERSION).Parse()

	//validate config
	u, err := url.Parse(c.URL)
	if err != nil || u.Host == "" {
		log.Fatal("Invalid URL")
	}
	if u.Path == "" {
		u.Path = "/write"
	}
	v := url.Values{}
	v.Set("db", c.Database)
	u.RawQuery = v.Encode()

	tags := ""
	if c.Tags != "" {
		tags = "," + c.Tags
	}

	//good to go
	log.Printf("Using InfluxDB endpoint: %s", u)
	log.Printf("Current time is %s UTC", time.Now().Format(time.RFC3339))

	lock := sync.Mutex{}
	entries := []string{}

	send := func() error {
		body := strings.NewReader(strings.Join(entries, "\n"))
		if c.DNS != "" {
			h, p, _ := net.SplitHostPort(u.Host)
			ips, err := lookup(h, c.DNS)
			if err != nil {
				return fmt.Errorf("Lookup failed: %s", err)
			}
			u.Host = ips[0] + ":" + p
		}
		resp, err := http.Post(u.String(), "application/x-www-form-urlencoded", body)
		if err != nil {
			return fmt.Errorf("HTTP POST failed: %s", err)
		}
		defer resp.Body.Close()
		if resp.StatusCode != http.StatusNoContent {
			msg, err := ioutil.ReadAll(resp.Body)
			if err != nil {
				return fmt.Errorf("Response download failed: %s", err)
			}
			return fmt.Errorf("Response: %d => %s", resp.StatusCode, msg)
		}
		log.Printf("Success")
		//clear once recieved!
		entries = nil
		return nil
	}

	report := func() {
		t := time.Now().UnixNano()
		if l, err := load.LoadAvg(); err == nil {
			entries = append(entries, fmt.Sprintf("cpu_load_short%s value=%f %d", tags, l.Load1*100, t))
			entries = append(entries, fmt.Sprintf("cpu_load_medium%s value=%f %d", tags, l.Load5*100, t))
			entries = append(entries, fmt.Sprintf("cpu_load_long%s value=%f %d", tags, l.Load15*100, t))
		}
		if v, err := mem.VirtualMemory(); err == nil {
			entries = append(entries, fmt.Sprintf("mem_usage%s value=%f %d", tags, v.UsedPercent, t))
		}
	}

	//send loop
	go func() {
		b := backoff.Backoff{}
		for {
			wait := time.Second
			lock.Lock()
			if len(entries) > 0 {
				if err := send(); err == nil {
					b.Reset()
				} else {
					log.Println(err)
					wait = b.Duration()
				}
			}
			lock.Unlock()
			time.Sleep(wait)
		}
	}()

	//report loop
	for {
		lock.Lock()
		report()
		lock.Unlock()
		time.Sleep(c.Interval)
	}
}

var dnsClient = dns.Client{}

func lookup(domain, server string) ([]string, error) {
	msg := dns.Msg{}
	msg.SetQuestion(domain+".", dns.TypeA)
	r, _, err := dnsClient.Exchange(&msg, server+":53")
	if err != nil {
		return nil, err
	}
	l := len(r.Answer)
	if l == 0 {
		return nil, fmt.Errorf("No answers")
	}
	ips := make([]string)
	for i, answer := range r.Answer {
		ans := answer.(*dns.A)
		ips[i] = ans.A.String()
	}
	return ips, nil
}
<commit_msg>add max queued<commit_after>package main

import (
	"fmt"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"net/url"
	"strings"
	"sync"
	"time"

	"github.com/jpillora/backoff"
	"github.com/jpillora/opts"
	"github.com/miekg/dns"
	"github.com/shirou/gopsutil/load"
	"github.com/shirou/gopsutil/mem"
)

var VERSION = "0"

const MAX_QUEUED = 10e3

type config struct {
	URL      string        `help:"InfluxDB URL"`
	Database string        `help:"InfluxDB database"`
	Interval time.Duration `help:"Time between reports"`
	Tags     string        `help:"InfluxDB tags in the form \"<tag>=<value>,<tag>=<value>\""`
	DNS      string        `help:"DNS server (used to lookup URL)"`
}

func main() {
	c := config{
		URL:      "http://localhost:8086",
		Database: "test",
		Interval: 5 * time.Minute,
	}

	opts.New(&c).Name("sysflux").Version(VERSION).Parse()

	//validate config
	u, err := url.Parse(c.URL)
	if err != nil || u.Host == "" {
		log.Fatal("Invalid URL")
	}
	if u.Path == "" {
		u.Path = "/write"
	}
	v := url.Values{}
	v.Set("db", c.Database)
	u.RawQuery = v.Encode()

	tags := ""
	if c.Tags != "" {
		tags = "," + c.Tags
	}

	//good to go
	log.Printf("Using InfluxDB endpoint: %s", u)
	log.Printf("Current time is %s UTC", time.Now().Format(time.RFC3339))

	lock := sync.Mutex{}
	entries := []string{}

	send := func() error {
		body := strings.NewReader(strings.Join(entries, "\n"))
		if c.DNS != "" {
			h, p, _ := net.SplitHostPort(u.Host)
			ips, err := lookup(h, c.DNS)
			if err != nil {
				return fmt.Errorf("Lookup failed: %s", err)
			}
			u.Host = ips[0] + ":" + p
		}
		resp, err := http.Post(u.String(), "application/x-www-form-urlencoded", body)
		if err != nil {
			return fmt.Errorf("HTTP POST failed: %s", err)
		}
		defer resp.Body.Close()
		if resp.StatusCode != http.StatusNoContent {
			msg, err := ioutil.ReadAll(resp.Body)
			if err != nil {
				return fmt.Errorf("Response download failed: %s", err)
			}
			return fmt.Errorf("Response: %d => %s", resp.StatusCode, msg)
		}
		log.Printf("Success")
		//clear once recieved!
		entries = nil
		return nil
	}

	report := func() {
		t := time.Now().UnixNano()
		if l, err := load.LoadAvg(); err == nil {
			entries = append(entries, fmt.Sprintf("cpu_load_short%s value=%f %d", tags, l.Load1*100, t))
			entries = append(entries, fmt.Sprintf("cpu_load_medium%s value=%f %d", tags, l.Load5*100, t))
			entries = append(entries, fmt.Sprintf("cpu_load_long%s value=%f %d", tags, l.Load15*100, t))
		}
		if v, err := mem.VirtualMemory(); err == nil {
			entries = append(entries, fmt.Sprintf("mem_usage%s value=%f %d", tags, v.UsedPercent, t))
		}
		if len(entries) > MAX_QUEUED {
			entries = entries[len(entries)-MAX_QUEUED:]
		}
	}

	//send loop
	go func() {
		b := backoff.Backoff{}
		for {
			wait := time.Second
			lock.Lock()
			if len(entries) > 0 {
				if err := send(); err == nil {
					b.Reset()
				} else {
					log.Println(err)
					wait = b.Duration()
				}
			}
			lock.Unlock()
			time.Sleep(wait)
		}
	}()

	//report loop
	for {
		lock.Lock()
		report()
		lock.Unlock()
		time.Sleep(c.Interval)
	}
}

var dnsClient = dns.Client{}

func lookup(domain, server string) ([]string, error) {
	msg := dns.Msg{}
	msg.SetQuestion(domain+".", dns.TypeA)
	r, _, err := dnsClient.Exchange(&msg, server+":53")
	if err != nil {
		return nil, err
	}
	l := len(r.Answer)
	if l == 0 {
		return nil, fmt.Errorf("No answers")
	}
	ips := make([]string, l)
	for i, answer := range r.Answer {
		ans := answer.(*dns.A)
		ips[i] = ans.A.String()
	}
	return ips, nil
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"log"

	"github.com/StackExchange/tsaf/conf"
	"github.com/StackExchange/tsaf/relay"
	"github.com/StackExchange/tsaf/sched"
	"github.com/StackExchange/tsaf/web"
)

var (
	confFile = flag.String("c", "dev.conf", "config file location")
)

func main() {
	flag.Parse()
	c, err := conf.ParseFile(*confFile)
	if err != nil {
		log.Fatal(err)
	}
	webDir := c.Global.Get("webDir", "web")
	webListen := c.Global.Get("webListen", ":8070")
	relayListen := c.Global.Get("relayListen", ":4242")
	tsdbHost := c.Global["tsdbHost"]
	if tsdbHost == "" {
		log.Fatal("tsaf: no tsdbHost in config file")
	}
	tsdbHttp := "http://" + tsdbHost + "/"
	go func() { log.Fatal(relay.RelayHTTP(relayListen, tsdbHost)) }()
	go func() { log.Fatal(web.Listen(webListen, webDir, tsdbHttp)) }()
	go func() { log.Fatal(sched.Run(c)) }()
	select {}
}
<commit_msg>New conf loader; remove broken sched<commit_after>package main

import (
	"flag"
	"log"

	"github.com/StackExchange/tsaf/conf"
	"github.com/StackExchange/tsaf/relay"
	"github.com/StackExchange/tsaf/web"
)

var (
	confFile = flag.String("c", "dev.conf", "config file location")
)

func main() {
	flag.Parse()
	c, err := conf.ParseFile(*confFile)
	if err != nil {
		log.Fatal(err)
	}
	tsdbHttp := "http://" + c.TsdbHost + "/"
	go func() { log.Fatal(relay.RelayHTTP(c.RelayListen, c.TsdbHost)) }()
	go func() { log.Fatal(web.Listen(c.HttpListen, c.WebDir, tsdbHttp)) }()
	select {}
}
<|endoftext|>


<commit_before>// Download Montreal traffic camera images
// Uses open data from the city of Montreal at:
//   - http://donnees.ville.montreal.qc.ca/dataset/cameras-observation-routiere
// See in action via city map application at @ http://ville.montreal.qc.ca/circulation/
package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"log"
	"os"
	"path"
	"sync"
	"time"

	"github.com/fatih/color"
)

const (
	// See http://donnees.ville.montreal.qc.ca/dataset/cameras-observation-routiere
	URLGeoJSON string = "http://ville.montreal.qc.ca/circulation/sites/ville.montreal.qc.ca.circulation/files/cameras-de-circulation.json"
	URLAbout   string = "http://donnees.ville.montreal.qc.ca/dataset/cameras-observation-routiere"
	URLBase    string = "http://www1.ville.montreal.qc.ca/Circulation-Cameras/GEN%03d.jpeg"
)

var (
	// functions to colorize strings for use in sprintf-style functions
	bluef  = color.New(color.FgBlue, color.Bold).SprintFunc()
	greenf = color.New(color.FgGreen, color.Bold).SprintFunc()
)

// Create date/timestampped subdirectories for saving images
func MakeTimeStampDir(parentDir string) string {
	timeStampDir := time.Now().Format(path.Join(parentDir, "060102/150405"))
	err := os.MkdirAll(timeStampDir, 0755)
	if err != nil {
		log.Fatalf("Couldn't create image directory: %s\n", err)
	}

	return timeStampDir
}

func main() {
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "\n%s: Download Montreal traffic camera images\n", os.Args[0])
		fmt.Fprintf(os.Stderr, "Data source: %s\n\n", URLAbout)
		flag.PrintDefaults()
		fmt.Fprintf(os.Stderr, "\n")
	}

	concurrency := flag.Int("c", 90, "max concurrent downloads")
	parentDir := flag.String("d", "images", "parent directory for downloaded files")
	flag.Parse()

	// timestamped directory to store images in
	tsDir := MakeTimeStampDir(*parentDir)

	// max number of workers (counting semaphore)
	workers := make(chan int, *concurrency)

	var fc FeatureCollection
	geoJSON := download(URLGeoJSON)
	if err := json.Unmarshal(geoJSON, &fc); err != nil {
		log.Printf("Error unmarshalling GeoJSON data: %v\n", err)
	}

	// wg waits for all downloads to complete
	var wg sync.WaitGroup
	for _, f := range fc.Features {
		imgURL := f.Properties.URLImageEnDirect
		if imgURL == "" {
			continue
		}

		// use one counting semaphore slot (when full, will block until slot free)
		workers <- 1
		wg.Add(1)

		imgFile := path.Base(imgURL)
		if imgFile == "" {
			log.Fatalf("Couldn't derive filename: %s\n", imgURL)
		}

		go func(URL, file, dir string) {
			defer wg.Done()
			img := download(URL)
			saveFile(img, file, dir)

			// when download finished, free one slot in workers
			<-workers
		}(imgURL, imgFile, tsDir)
	}

	wg.Wait()
}
<commit_msg>Simplify local image filenames<commit_after>// Download Montreal traffic camera images
// Uses open data from the city of Montreal at:
//   - http://donnees.ville.montreal.qc.ca/dataset/cameras-observation-routiere
// See in action via city map application at @ http://ville.montreal.qc.ca/circulation/
package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"log"
	"os"
	"path"
	"regexp"
	"strconv"
	"sync"
	"time"

	"github.com/fatih/color"
)

const (
	// See http://donnees.ville.montreal.qc.ca/dataset/cameras-observation-routiere
	URLGeoJSON string = "http://ville.montreal.qc.ca/circulation/sites/ville.montreal.qc.ca.circulation/files/cameras-de-circulation.json"
	URLAbout   string = "http://donnees.ville.montreal.qc.ca/dataset/cameras-observation-routiere"
	URLBase    string = "http://www1.ville.montreal.qc.ca/Circulation-Cameras/GEN%03d.jpeg"
)

var (
	// functions to colorize strings for use in sprintf-style functions
	bluef  = color.New(color.FgBlue, color.Bold).SprintFunc()
	greenf = color.New(color.FgGreen, color.Bold).SprintFunc()

	reDigits = regexp.MustCompile(`\d+`)
)

// Create date/timestampped subdirectories for saving images
func MakeTimeStampDir(parentDir string) string {
	timeStampDir := time.Now().Format(path.Join(parentDir, "060102/150405"))
	err := os.MkdirAll(timeStampDir, 0755)
	if err != nil {
		log.Fatalf("Couldn't create image directory: %s\n", err)
	}

	return timeStampDir
}

func main() {
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "\n%s: Download Montreal traffic camera images\n", os.Args[0])
		fmt.Fprintf(os.Stderr, "Data source: %s\n\n", URLAbout)
		flag.PrintDefaults()
		fmt.Fprintf(os.Stderr, "\n")
	}

	concurrency := flag.Int("c", 90, "max concurrent downloads")
	parentDir := flag.String("d", "images", "parent directory for downloaded files")
	flag.Parse()

	// timestamped directory to store images in
	tsDir := MakeTimeStampDir(*parentDir)

	// max number of workers (counting semaphore)
	workers := make(chan int, *concurrency)

	var fc FeatureCollection
	geoJSON := download(URLGeoJSON)
	if err := json.Unmarshal(geoJSON, &fc); err != nil {
		log.Printf("Error unmarshalling GeoJSON data: %v\n", err)
	}

	// wg waits for all downloads to complete
	var wg sync.WaitGroup
	for _, f := range fc.Features {
		imgURL := f.Properties.URLImageEnDirect
		if imgURL == "" {
			continue
		}

		// use one counting semaphore slot (when full, will block until slot free)
		workers <- 1
		wg.Add(1)

		imgFileServer := path.Base(imgURL)
		if imgFileServer == "" {
			log.Fatalf("Couldn't derive filename: %s\n", imgURL)
		}
		imgNum, err := strconv.Atoi(reDigits.FindString(imgFileServer))
		if err != nil {
			log.Printf("Error deriving image number: %s\n", err)
		}
		imgFile := fmt.Sprintf("%03d.jpg", imgNum)

		go func(URL, file, dir string) {
			defer wg.Done()
			img := download(URL)
			saveFile(img, file, dir)

			// when download finished, free one slot in workers
			<-workers
		}(imgURL, imgFile, tsDir)
	}

	wg.Wait()
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"fmt"
	"log"
	"os"
	"regexp"
	"runtime"
	"sync"
	"time"

	"github.com/sirupsen/logrus"

	"github.com/c9s/gomon/logger"
	"github.com/c9s/gomon/notify"
	"github.com/fsnotify/fsnotify"
)

const (
	name     = "gomon"
	version  = "1.3.0"
	revision = "HEAD"
)

var notifier notify.Notifier = nil

func main() {
	dirArgs, cmdArgs := options.Parse(os.Args)
	dirArgs = FilterExistPaths(dirArgs)

	var matchAll = false
	var alwaysNotify = false

	if options.Bool("h") {
		fmt.Println("Usage: gomon [options] [dir] [-- command]")
		for _, option := range options {
			if _, ok := option.value.(string); ok {
				fmt.Printf("  -%s=%s: %s\n", option.flag, option.value, option.description)
			} else {
				fmt.Printf("  -%s: %s\n", option.flag, option.description)
			}
		}
		os.Exit(0)
	}
	if options.Bool("v") {
		fmt.Printf("gomon %s\n", version)
		os.Exit(0)
	}

	if options.Bool("install-growl-icons") {
		notify.InstallGrowlIcons()
		os.Exit(0)
		return
	}

	matchAll = options.Bool("matchall")
	alwaysNotify = options.Bool("alwaysnotify")

	// dynamically build the command list
	var cmds = CommandList{}
	if options.Bool("f") {
		cmds.Add(goCommands["fmt"])
	}
	if options.Bool("t") {
		cmds.Add(goCommands["test"])
	}
	if options.Bool("l") {
		cmds.Add(goCommands["lint"])
	}
	if options.Bool("b") {
		cmds.Add(goCommands["build"])
	}
	if options.Bool("r") {
		cmds.Add(goCommands["run"])
	}
	if options.Bool("i") {
		cmds.Add(goCommands["install"])
	}
	if options.Bool("x") {
		cmds.AppendOption("-x")
	}

	if options.Bool("d") {
		logger.Instance().SetLevel(logrus.DebugLevel)
	}

	if len(cmdArgs) > 0 {
		cmds.Add(Command(cmdArgs))
	} else if cmds.Len() == 0 {
		// default to go build
		cmds.Add(goCommands["build"])
	}

	if len(dirArgs) == 0 {
		var cwd, err = os.Getwd()
		if err != nil {
			log.Fatal(err)
		}
		dirArgs = []string{cwd}
	}

	if runtime.GOOS == "darwin" {
		logger.Infoln("Setting up Notification Center for OS X ...")
		notifier = notify.NewOSXNotifier()
	}
	if notifier == nil {
		if _, err := os.Stat("/Applications/Growl.app"); err == nil {
			logger.Infoln("Found Growl.app, setting up GNTP notifier...")
			notifier = notify.NewGNTPNotifier(options.String("gntp"), "gomon")
		}
	}
	if notifier == nil {
		notifier = notify.NewTextNotifier()
	}

	watcher, err := fsnotify.NewWatcher()
	if err != nil {
		logger.Error(err)
		return
	}
	defer watcher.Close()

	var numdir = 0
	for _, dir := range dirArgs {
		if options.Bool("R") {
			subfolders := Subfolders(dir)
			for _, f := range subfolders {
				err = watcher.Add(f)
				numdir++
				if err != nil {
					log.Fatal(err)
				}
			}
		} else {
			err = watcher.Add(dir)
			numdir++
			if err != nil {
				log.Fatal(err)
			}
		}
	}

	logger.Infof("Watching %v for commands %v.", dirArgs, cmds)
	logger.Infof("%d directories watched", numdir)

	var jobRunner = &JobRunner{
		builder: &JobBuilder{
			// Job template arguments
			Commands:        cmds.commands,
			Args:            []string{},
			AppendFilename:  options.Bool("F"),
			ChangeDirectory: options.Bool("chdir"),
		},
	}

	var patternStr string = options.String("m")
	if len(patternStr) == 0 {
		// the empty regexp matches everything anyway
		matchAll = true
	}

	var pattern = regexp.MustCompile(patternStr)
	var once sync.Once
	var ctx = context.Background()

	for {
		select {

		case <-ctx.Done():
			return

		case err := <-watcher.Errors:
			log.Println("Error:", err)

		case e := <-watcher.Events:
			var matched = matchAll
			if !matched {
				matched = pattern.MatchString(e.Name)
			}

			if !matched {
				if options.Bool("d") {
					logger.Debugf("Ignored file=%s", e)
				}
				continue
			}

			if options.Bool("d") {
				logger.Debugf("Event=%+v", e)
			} else {
				switch e.Op {
				case fsnotify.Create:
					logger.Infoln("Create", e.Name)
				case fsnotify.Write:
					logger.Infoln("Write", e.Name)
				case fsnotify.Remove:
					logger.Infoln("Remove", e.Name)
				case fsnotify.Rename:
					logger.Infoln("Rename", e.Name)
				}
			}

			// TODO: time.ParseDuration
			// go fmt vim plugin will rename the file and then create a new file
			// In order to handle the batch operation, a delay is needed.
			go func(filename string) {
				once.Do(func() {
					// duration to avoid to run commands frequency at once
					<-time.After(500 * time.Millisecond)

					var duration, err = jobRunner.RunAndNotify(context.Background(), filename, alwaysNotify)
					if err != nil {
						logger.Errorf("Build failed: %v", err.Error())
					} else {
						logger.Infoln("Successful build:", duration)
					}
				})
				once = sync.Once{}
			}(e.Name)

		}
	}

}
<commit_msg>Fix build<commit_after>package main

import (
	"context"
	"fmt"
	"log"
	"os"
	"regexp"
	"runtime"
	"sync"
	"time"

	"github.com/sirupsen/logrus"

	"github.com/c9s/gomon/logger"
	"github.com/c9s/gomon/notify"
	"github.com/fsnotify/fsnotify"
)

const (
	name     = "gomon"
	version  = "1.3.0"
	revision = "HEAD"
)

var notifier notify.Notifier = nil

func main() {
	dirArgs, cmdArgs := options.Parse(os.Args)
	dirArgs = filterExistPaths(dirArgs)

	var matchAll = false
	var alwaysNotify = false

	if options.Bool("h") {
		fmt.Println("Usage: gomon [options] [dir] [-- command]")
		for _, option := range options {
			if _, ok := option.value.(string); ok {
				fmt.Printf("  -%s=%s: %s\n", option.flag, option.value, option.description)
			} else {
				fmt.Printf("  -%s: %s\n", option.flag, option.description)
			}
		}
		os.Exit(0)
	}
	if options.Bool("v") {
		fmt.Printf("gomon %s\n", version)
		os.Exit(0)
	}

	if options.Bool("install-growl-icons") {
		notify.InstallGrowlIcons()
		os.Exit(0)
		return
	}

	matchAll = options.Bool("matchall")
	alwaysNotify = options.Bool("alwaysnotify")

	// dynamically build the command list
	var cmds = CommandList{}
	if options.Bool("f") {
		cmds.Add(goCommands["fmt"])
	}
	if options.Bool("t") {
		cmds.Add(goCommands["test"])
	}
	if options.Bool("l") {
		cmds.Add(goCommands["lint"])
	}
	if options.Bool("b") {
		cmds.Add(goCommands["build"])
	}
	if options.Bool("r") {
		cmds.Add(goCommands["run"])
	}
	if options.Bool("i") {
		cmds.Add(goCommands["install"])
	}
	if options.Bool("x") {
		cmds.AppendOption("-x")
	}

	if options.Bool("d") {
		logger.Instance().SetLevel(logrus.DebugLevel)
	}

	if len(cmdArgs) > 0 {
		cmds.Add(Command(cmdArgs))
	} else if cmds.Len() == 0 {
		// default to go build
		cmds.Add(goCommands["build"])
	}

	if len(dirArgs) == 0 {
		var cwd, err = os.Getwd()
		if err != nil {
			log.Fatal(err)
		}
		dirArgs = []string{cwd}
	}

	if runtime.GOOS == "darwin" {
		logger.Infoln("Setting up Notification Center for OS X ...")
		notifier = notify.NewOSXNotifier()
	}
	if notifier == nil {
		if _, err := os.Stat("/Applications/Growl.app"); err == nil {
			logger.Infoln("Found Growl.app, setting up GNTP notifier...")
			notifier = notify.NewGNTPNotifier(options.String("gntp"), "gomon")
		}
	}
	if notifier == nil {
		notifier = notify.NewTextNotifier()
	}

	watcher, err := fsnotify.NewWatcher()
	if err != nil {
		logger.Error(err)
		return
	}
	defer watcher.Close()

	var numdir = 0
	for _, dir := range dirArgs {
		if options.Bool("R") {
			subfolders := subfolders(dir)
			for _, f := range subfolders {
				err = watcher.Add(f)
				numdir++
				if err != nil {
					log.Fatal(err)
				}
			}
		} else {
			err = watcher.Add(dir)
			numdir++
			if err != nil {
				log.Fatal(err)
			}
		}
	}

	logger.Infof("Watching %v for commands %v.", dirArgs, cmds)
	logger.Infof("%d directories watched", numdir)

	var jobRunner = &JobRunner{
		builder: &JobBuilder{
			// Job template arguments
			Commands:        cmds.commands,
			Args:            []string{},
			AppendFilename:  options.Bool("F"),
			ChangeDirectory: options.Bool("chdir"),
		},
	}

	var patternStr string = options.String("m")
	if len(patternStr) == 0 {
		// the empty regexp matches everything anyway
		matchAll = true
	}

	var pattern = regexp.MustCompile(patternStr)
	var once sync.Once
	var ctx = context.Background()

	for {
		select {

		case <-ctx.Done():
			return

		case err := <-watcher.Errors:
			log.Println("Error:", err)

		case e := <-watcher.Events:
			var matched = matchAll
			if !matched {
				matched = pattern.MatchString(e.Name)
			}

			if !matched {
				if options.Bool("d") {
					logger.Debugf("Ignored file=%s", e)
				}
				continue
			}

			if options.Bool("d") {
				logger.Debugf("Event=%+v", e)
			} else {
				switch e.Op {
				case fsnotify.Create:
					logger.Infoln("Create", e.Name)
				case fsnotify.Write:
					logger.Infoln("Write", e.Name)
				case fsnotify.Remove:
					logger.Infoln("Remove", e.Name)
				case fsnotify.Rename:
					logger.Infoln("Rename", e.Name)
				}
			}

			// TODO: time.ParseDuration
			// go fmt vim plugin will rename the file and then create a new file
			// In order to handle the batch operation, a delay is needed.
			go func(filename string) {
				once.Do(func() {
					// duration to avoid to run commands frequency at once
					<-time.After(500 * time.Millisecond)

					var duration, err = jobRunner.RunAndNotify(context.Background(), filename, alwaysNotify)
					if err != nil {
						logger.Errorf("Build failed: %v", err.Error())
					} else {
						logger.Infoln("Successful build:", duration)
					}
				})
				once = sync.Once{}
			}(e.Name)

		}
	}

}
<|endoftext|>


<commit_before>// Copyright 2015 Gareth Watts
// Licensed under an MIT license
// See the LICENSE file for details

/*
Command dyndump dumps a single DynamoDB table to a JSON formatted output

It supports parallel connections for increased throughput, and rate
limiting to a specified read capacity.

By default the JSON output provides an array of objects, with keys for each
column in the row of the table and values mapping to the values in the row.

Numeric values are returned by DynamoDB as strings, but are converted to
floats unless the -string-nums option is specified.

Specifying -typed changes the values in each object to become an object
with keys of "type" and "value" where "type" may be one of DynamoDB's
supported types:
* binary
* binary-set
* bool
*list
* map
* number
* number-set
* null
* string
* string-set

AWS credentials required to connect to DynamoDB must be passed in using
environment variables:
* AWS_ACCESS_KEY_ID
* AWS_SECRET_KEY
*/
package main

import (
	"flag"
	"fmt"
	"io"
	"os"
	"os/signal"
	"syscall"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/service/dynamodb"
	"github.com/gwatts/dyndump/dyndump"
)

const (
	maxParallel    = 1000
	statsFrequency = 2 * time.Second
)

var (
	consistentRead   = flag.Bool("consistent-read", false, "Enable consistent reads (at 2x capacity use)")
	includeTypes     = flag.Bool("typed", true, "Include type names in JSON output")
	maxItems         = flag.Int64("maxitems", 0, "Maximum number of items to read.  Set to 0 to read all items")
	numbersAsStrings = flag.Bool("string-nums", false, "Output numbers as exact value strings instead of converting")
	outFilename      = flag.String("target", "-", "Filename to write data to.  Defaults to stdout")
	parallel         = flag.Int("parallel", 4, "Number of concurrent channels to open to DynamoDB")
	readCapacity     = flag.Int("read-capacity", 5, "Average aggregate read capacity to use for scan (set to 0 for unlimited)")
	region           = flag.String("region", "us-west-2", "AWS Region")
	silent           = flag.Bool("silent", false, "Don't print progress to stderr")
	tableName        = flag.String("tablename", "", "DynamoDB table name to dump")
)

func fail(format string, a ...interface{}) {
	fmt.Fprintf(os.Stderr, format+"\n", a...)
	os.Exit(100)
}

func usage(err string) {
	fmt.Fprintf(os.Stderr, "Error: "+err+"\n\nUsage:\n")
	flag.PrintDefaults()
	os.Exit(101)
}

func main() {
	flag.Parse()

	if *tableName == "" {
		usage("No tablename supplied")
	}
	if *parallel < 1 || *parallel > maxParallel {
		usage("Invalid value for -parallel")
	}
	if *readCapacity < 0 {
		usage("Invalid value for -read-capacity")
	}

	config := &aws.Config{
		Credentials: credentials.NewEnvCredentials(),
		Region:      aws.String(*region),
	}

	var out io.Writer = os.Stdout
	if *outFilename != "" && *outFilename != "-" {
		var err error
		out, err = os.Create(*outFilename)
		if err != nil {
			fail("Failed to open file for write: %s", err)
		}
	}

	sigchan := make(chan os.Signal, 1)
	signal.Notify(sigchan, syscall.SIGTERM, syscall.SIGKILL, syscall.SIGINT)

	writer := dyndump.NewJSONItemEncoder(out, *includeTypes, *numbersAsStrings)

	if !*silent {
		fmt.Fprintf(os.Stderr, "Beginning scan: table=%q readCapacity=%d parallel=%d\n",
			*tableName, *readCapacity, *parallel)
	}

	f := dyndump.Fetcher{
		Dyn:            dynamodb.New(config),
		TableName:      *tableName,
		ConsistentRead: *consistentRead,
		MaxParallel:    *parallel,
		MaxItems:       *maxItems,
		ReadCapacity:   float64(*readCapacity),
		Writer:         writer,
	}

	doneChan := make(chan error)

	go func() {
		doneChan <- f.Run()
	}()

	var ticker <-chan time.Time
	if !*silent {
		ticker = time.Tick(statsFrequency)
	}

	lastTime := time.Now()
	startTime := lastTime
	var lastStats dyndump.Stats

LOOP:
	for {
		select {
		case <-ticker:
			deltaTime := time.Since(lastTime)
			lastTime = time.Now()
			stats := f.Stats()
			itemsDelta := float64(stats.ItemsRead-lastStats.ItemsRead) / float64(deltaTime/time.Second)
			capacityDelta := (stats.CapacityUsed - lastStats.CapacityUsed) / float64(deltaTime/time.Second)
			lastStats = stats
			fmt.Fprintf(os.Stderr, "Total items: %-10d Items/sec: %-10.1f  Capacity: %-6.1f\n",
				stats.ItemsRead, itemsDelta, capacityDelta)

		case <-sigchan:
			fmt.Fprintf(os.Stderr, "\nAborting..")
			f.Stop()
			<-doneChan
			fmt.Fprintf(os.Stderr, "Aborted.\n")
			break LOOP

		case err := <-doneChan:
			if err != nil {
				fail("Processing failed: %s", err)
			}
			break LOOP
		}
	}

	finalStats := f.Stats()
	deltaSeconds := float64(time.Since(startTime) / time.Second)

	if !*silent {
		fmt.Fprintf(os.Stderr, "Avg items/sec: %.2f\n", float64(finalStats.ItemsRead)/deltaSeconds)
		fmt.Fprintf(os.Stderr, "Avg capacity/sec: %.2f\n", finalStats.CapacityUsed/deltaSeconds)
		fmt.Fprintln(os.Stderr, "Total items read: ", f.Stats().ItemsRead)
	}
}
<commit_msg>Make compatible with 1.0 of aws-sdk-go<commit_after>// Copyright 2015 Gareth Watts
// Licensed under an MIT license
// See the LICENSE file for details

/*
Command dyndump dumps a single DynamoDB table to a JSON formatted output

It supports parallel connections for increased throughput, and rate
limiting to a specified read capacity.

By default the JSON output provides an array of objects, with keys for each
column in the row of the table and values mapping to the values in the row.

Numeric values are returned by DynamoDB as strings, but are converted to
floats unless the -string-nums option is specified.

Specifying -typed changes the values in each object to become an object
with keys of "type" and "value" where "type" may be one of DynamoDB's
supported types:
* binary
* binary-set
* bool
*list
* map
* number
* number-set
* null
* string
* string-set

AWS credentials required to connect to DynamoDB must be passed in using
environment variables:
* AWS_ACCESS_KEY_ID
* AWS_SECRET_KEY
*/
package main

import (
	"flag"
	"fmt"
	"io"
	"os"
	"os/signal"
	"syscall"
	"time"

	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/dynamodb"
	"github.com/gwatts/dyndump/dyndump"
)

const (
	maxParallel    = 1000
	statsFrequency = 2 * time.Second
)

var (
	consistentRead   = flag.Bool("consistent-read", false, "Enable consistent reads (at 2x capacity use)")
	includeTypes     = flag.Bool("typed", true, "Include type names in JSON output")
	maxItems         = flag.Int64("maxitems", 0, "Maximum number of items to read.  Set to 0 to read all items")
	numbersAsStrings = flag.Bool("string-nums", false, "Output numbers as exact value strings instead of converting")
	outFilename      = flag.String("target", "-", "Filename to write data to.  Defaults to stdout")
	parallel         = flag.Int("parallel", 4, "Number of concurrent channels to open to DynamoDB")
	readCapacity     = flag.Int("read-capacity", 5, "Average aggregate read capacity to use for scan (set to 0 for unlimited)")
	silent           = flag.Bool("silent", false, "Don't print progress to stderr")
	tableName        = flag.String("tablename", "", "DynamoDB table name to dump")
)

func fail(format string, a ...interface{}) {
	fmt.Fprintf(os.Stderr, format+"\n", a...)
	os.Exit(100)
}

func usage(err string) {
	fmt.Fprintf(os.Stderr, "Error: "+err+"\n\nUsage:\n")
	flag.PrintDefaults()
	fmt.Fprintln(os.Stderr, "\nSpecify AWS_REGION, AWS_ACCESS_KEY and AWS_SECRET_KEY environment variables as required")
	os.Exit(101)
}

func main() {
	flag.Parse()

	if *tableName == "" {
		usage("No tablename supplied")
	}
	if *parallel < 1 || *parallel > maxParallel {
		usage("Invalid value for -parallel")
	}
	if *readCapacity < 0 {
		usage("Invalid value for -read-capacity")
	}

	var out io.Writer = os.Stdout
	if *outFilename != "" && *outFilename != "-" {
		var err error
		out, err = os.Create(*outFilename)
		if err != nil {
			fail("Failed to open file for write: %s", err)
		}
	}

	sigchan := make(chan os.Signal, 1)
	signal.Notify(sigchan, syscall.SIGTERM, syscall.SIGKILL, syscall.SIGINT)

	writer := dyndump.NewJSONItemEncoder(out, *includeTypes, *numbersAsStrings)

	if !*silent {
		fmt.Fprintf(os.Stderr, "Beginning scan: table=%q readCapacity=%d parallel=%d\n",
			*tableName, *readCapacity, *parallel)
	}

	f := dyndump.Fetcher{
		Dyn:            dynamodb.New(session.New()),
		TableName:      *tableName,
		ConsistentRead: *consistentRead,
		MaxParallel:    *parallel,
		MaxItems:       *maxItems,
		ReadCapacity:   float64(*readCapacity),
		Writer:         writer,
	}

	doneChan := make(chan error)

	go func() {
		doneChan <- f.Run()
	}()

	var ticker <-chan time.Time
	if !*silent {
		ticker = time.Tick(statsFrequency)
	}

	lastTime := time.Now()
	startTime := lastTime
	var lastStats dyndump.Stats

LOOP:
	for {
		select {
		case <-ticker:
			deltaTime := time.Since(lastTime)
			lastTime = time.Now()
			stats := f.Stats()
			itemsDelta := float64(stats.ItemsRead-lastStats.ItemsRead) / float64(deltaTime/time.Second)
			capacityDelta := (stats.CapacityUsed - lastStats.CapacityUsed) / float64(deltaTime/time.Second)
			lastStats = stats
			fmt.Fprintf(os.Stderr, "Total items: %-10d Items/sec: %-10.1f  Capacity: %-6.1f\n",
				stats.ItemsRead, itemsDelta, capacityDelta)

		case <-sigchan:
			fmt.Fprintf(os.Stderr, "\nAborting..")
			f.Stop()
			<-doneChan
			fmt.Fprintf(os.Stderr, "Aborted.\n")
			break LOOP

		case err := <-doneChan:
			if err != nil {
				fail("Processing failed: %s", err)
			}
			break LOOP
		}
	}

	finalStats := f.Stats()
	deltaSeconds := float64(time.Since(startTime) / time.Second)

	if !*silent {
		fmt.Fprintf(os.Stderr, "Avg items/sec: %.2f\n", float64(finalStats.ItemsRead)/deltaSeconds)
		fmt.Fprintf(os.Stderr, "Avg capacity/sec: %.2f\n", finalStats.CapacityUsed/deltaSeconds)
		fmt.Fprintln(os.Stderr, "Total items read: ", f.Stats().ItemsRead)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"net/http"
	"net/url"
	"os"
	"strings"

	"github.com/zenazn/goji"
	"github.com/zenazn/goji/web"

	"github.com/owm-inc/weatherchecker-go/db"
	"github.com/owm-inc/weatherchecker-go/structs"
)

type JsonResponse struct {
	Code    int         `json:"code"`
	Message string      `json:"message"`
	Content interface{} `json:"content"`
}

var sources = structs.CreateSources()

var mongoDsn string

var db_instance = db.Db()
var locations = structs.NewLocationTable(db_instance)
var history = structs.NewWeatherHistory(db_instance)

func MarshalPrintStuff(stuff interface{}, w http.ResponseWriter) {
	data, _ := json.Marshal(stuff)
	jsonString := string(data)
	fmt.Fprintf(w, jsonString)
}

func MarshalPrintResponse(code int, message string, content interface{}, w http.ResponseWriter) {
	MarshalPrintStuff(JsonResponse{Code: code, Message: message, Content: content}, w)
}

func PrintLocationEntry(locationEntry structs.LocationEntry, w http.ResponseWriter) {
	MarshalPrintResponse(200, "OK", map[string][]structs.LocationEntry{"location_entry": []structs.LocationEntry{locationEntry}}, w)
}

func PrintHistoryEntry(historyEntry []structs.HistoryDataEntry, w http.ResponseWriter) {
	MarshalPrintResponse(200, "OK", map[string][]structs.HistoryDataEntry{"history_entry": historyEntry}, w)
}

func PrintLocations(w http.ResponseWriter) {
	MarshalPrintResponse(200, "OK", map[string][]structs.LocationEntry{"locations": locations.ReadLocations()}, w)
}

func PrintSources(w http.ResponseWriter) {
	MarshalPrintResponse(200, "OK", map[string][]structs.SourceEntry{"sources": sources}, w)
}

func PrintStatus(err error, successMessage string, w http.ResponseWriter) {
	var status int
	var message string

	if err != nil {
		status = 500
		message = err.Error()
	} else {
		status = 200
		message = successMessage
	}

	MarshalPrintResponse(status, message, make(map[string]string), w)
}

func MakeMissingParamsList(query_holder url.Values, required_params []string) (missing []string) {
	for _, entry := range required_params {
		if query_holder.Get(entry) == "" {
			missing = append(missing, entry)
		}
	}

	return missing
}

func ReadSources(c web.C, w http.ResponseWriter, r *http.Request) {
	PrintSources(w)
}

func CreateLocation(c web.C, w http.ResponseWriter, r *http.Request) {
	query_holder := r.URL.Query()

	missing := MakeMissingParamsList(query_holder, []string{"city_name", "iso_country", "iso_country", "country_name", "latitude", "longitude"})

	if len(missing) > 0 {
		MarshalPrintResponse(500, "The following parameters are missing: "+strings.Join(missing, ", "), make(map[string]string), w)
	} else {
		locationEntry := locations.CreateLocation(
			query_holder.Get("city_name"),
			query_holder.Get("iso_country"),
			query_holder.Get("country_name"),
			query_holder.Get("latitude"),
			query_holder.Get("longitude"),
			query_holder.Get("accuweather_id"),
			query_holder.Get("accuweather_city_name"),
			query_holder.Get("gismeteo_id"),
			query_holder.Get("gismeteo_city_name"))
		PrintLocationEntry(locationEntry, w)
	}
}

func ReadLocations(c web.C, w http.ResponseWriter, r *http.Request) {
	PrintLocations(w)
}

func UpdateLocation(c web.C, w http.ResponseWriter, r *http.Request) {
	query_holder := r.URL.Query()

	missing := MakeMissingParamsList(query_holder, []string{"location_id", "city_name", "iso_country", "iso_country", "country_name", "latitude", "longitude"})
	if len(missing) > 0 {
		MarshalPrintResponse(500, "The following parameters are missing: "+strings.Join(missing, ", "), make(map[string]string), w)
	} else {
		locationEntry, _ := locations.UpdateLocation(
			query_holder.Get("location_id"),
			query_holder.Get("city_name"),
			query_holder.Get("iso_country"),
			query_holder.Get("country_name"),
			query_holder.Get("latitude"),
			query_holder.Get("longitude"),
			query_holder.Get("accuweather_id"),
			query_holder.Get("accuweather_city_name"),
			query_holder.Get("gismeteo_id"),
			query_holder.Get("gismeteo_city_name"))
		PrintLocationEntry(locationEntry, w)
	}
}

func DeleteLocation(c web.C, w http.ResponseWriter, r *http.Request) {
	missing := make([]string, 0)

	query_holder := r.URL.Query()

	MakeMissingParamsList(query_holder, []string{"location_id"})

	if len(missing) > 0 {
		MarshalPrintResponse(500, "The following parameters are missing: "+strings.Join(missing, ", "), make(map[string]string), w)
	} else {
		err := locations.DeleteLocation(query_holder.Get("location_id"))
		PrintStatus(err, "Location removed successfully.", w)
	}
}

func ClearLocations(c web.C, w http.ResponseWriter, r *http.Request) {
	err := locations.Clear()

	PrintStatus(err, "Locations cleared successfully.", w)
}

func ReadHistory(c web.C, w http.ResponseWriter, r *http.Request) {
	query_holder := r.URL.Query()
	entryid := query_holder.Get("entryid")
	source := query_holder.Get("source")
	wtype := query_holder.Get("wtype")
	country := query_holder.Get("country")
	locationid := query_holder.Get("locationid")
	requeststart := query_holder.Get("requeststart")
	requestend := query_holder.Get("requestend")

	history_data := history.ReadHistory(entryid, source, wtype, country, locationid, requeststart, requestend)
	history_filters := map[string]string{"entryid": entryid, "source": source, "wtype": wtype, "country": country, "locationid": locationid, "requeststart": requeststart, "requestend": requestend}

	MarshalPrintResponse(200, "OK", map[string]interface{}{"history": map[string]interface{}{"data": history_data, "filters": history_filters}}, w)
}

func RefreshHistory(c web.C, w http.ResponseWriter, r *http.Request) {
	query_holder := r.URL.Query()

	var wtypes []string
	wtype := query_holder.Get("wtype")

	if wtype == "" {
		wtypes = []string{"current", "forecast"}
	} else if wtype != "current" && wtype != "forecast" && wtype != "" {
		PrintStatus(errors.New(""), "Invalid wtype specified.", w)
		return
	} else {
		wtypes = []string{wtype}
	}

	locations_query := locations.ReadLocations()
	historyEntry := history.CreateHistoryEntry(locations_query, sources, wtypes)
	PrintHistoryEntry(historyEntry, w)
}

func ClearHistory(c web.C, w http.ResponseWriter, r *http.Request) {
	err := history.Clear()

	PrintStatus(err, "History cleared successfully.", w)
}

func Api(c *web.C, h http.Handler) http.Handler {
	fn := func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Access-Control-Allow-Origin", "*")
		h.ServeHTTP(w, r)
	}
	return http.HandlerFunc(fn)
}

func init() {
	flag.StringVar(&mongoDsn, "mongo", "mongodb://127.0.0.1:27017/weatherchecker", "MongoDB DSN")
}

func main() {
	flag.Parse()

	if os.Getenv("MONGO") != "" {
		mongoDsn = os.Getenv("MONGO")
	}

	fmt.Println("Connecting to MongoDB at", mongoDsn)
	err := db_instance.Connect(mongoDsn)
	if err != nil {
		fmt.Println(fmt.Sprintf("Database error: %s", err))
		return
	}
	defer db_instance.Disconnect()

	const ApiVer = "0.1"

	const ApiEntrypoint = "/api" + "/" + ApiVer

	const SourcesEntrypoint = ApiEntrypoint + "/sources"
	const LocationEntrypoint = ApiEntrypoint + "/locations"
	const HistoryEntrypoint = ApiEntrypoint + "/history"

	goji.Use(Api)
	goji.Get(SourcesEntrypoint, ReadSources)
	goji.Get(LocationEntrypoint, ReadLocations)
	goji.Get(LocationEntrypoint+"/add", CreateLocation)
	goji.Get(LocationEntrypoint+"/edit", UpdateLocation)
	goji.Get(LocationEntrypoint+"/remove", DeleteLocation)
	goji.Get(LocationEntrypoint+"/clear", ClearLocations)
	goji.Get(HistoryEntrypoint, ReadHistory)
	goji.Get(HistoryEntrypoint+"/refresh", RefreshHistory)
	goji.Get(HistoryEntrypoint+"/clear", ClearHistory)
	goji.Serve()
}
<commit_msg>"code" -> "status"<commit_after>package main

import (
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"net/http"
	"net/url"
	"os"
	"strings"

	"github.com/zenazn/goji"
	"github.com/zenazn/goji/web"

	"github.com/owm-inc/weatherchecker-go/db"
	"github.com/owm-inc/weatherchecker-go/structs"
)

type JsonResponse struct {
	Status  int         `json:"status"`
	Message string      `json:"message"`
	Content interface{} `json:"content"`
}

var sources = structs.CreateSources()

var mongoDsn string

var db_instance = db.Db()
var locations = structs.NewLocationTable(db_instance)
var history = structs.NewWeatherHistory(db_instance)

func MarshalPrintStuff(stuff interface{}, w http.ResponseWriter) {
	data, _ := json.Marshal(stuff)
	jsonString := string(data)
	fmt.Fprintf(w, jsonString)
}

func MarshalPrintResponse(status int, message string, content interface{}, w http.ResponseWriter) {
	MarshalPrintStuff(JsonResponse{Status: status, Message: message, Content: content}, w)
}

func PrintLocationEntry(locationEntry structs.LocationEntry, w http.ResponseWriter) {
	MarshalPrintResponse(200, "OK", map[string][]structs.LocationEntry{"location_entry": []structs.LocationEntry{locationEntry}}, w)
}

func PrintHistoryEntry(historyEntry []structs.HistoryDataEntry, w http.ResponseWriter) {
	MarshalPrintResponse(200, "OK", map[string][]structs.HistoryDataEntry{"history_entry": historyEntry}, w)
}

func PrintLocations(w http.ResponseWriter) {
	MarshalPrintResponse(200, "OK", map[string][]structs.LocationEntry{"locations": locations.ReadLocations()}, w)
}

func PrintSources(w http.ResponseWriter) {
	MarshalPrintResponse(200, "OK", map[string][]structs.SourceEntry{"sources": sources}, w)
}

func PrintStatus(err error, successMessage string, w http.ResponseWriter) {
	var status int
	var message string

	if err != nil {
		status = 500
		message = err.Error()
	} else {
		status = 200
		message = successMessage
	}

	MarshalPrintResponse(status, message, make(map[string]string), w)
}

func MakeMissingParamsList(query_holder url.Values, required_params []string) (missing []string) {
	for _, entry := range required_params {
		if query_holder.Get(entry) == "" {
			missing = append(missing, entry)
		}
	}

	return missing
}

func ReadSources(c web.C, w http.ResponseWriter, r *http.Request) {
	PrintSources(w)
}

func CreateLocation(c web.C, w http.ResponseWriter, r *http.Request) {
	query_holder := r.URL.Query()

	missing := MakeMissingParamsList(query_holder, []string{"city_name", "iso_country", "iso_country", "country_name", "latitude", "longitude"})

	if len(missing) > 0 {
		MarshalPrintResponse(500, "The following parameters are missing: "+strings.Join(missing, ", "), make(map[string]string), w)
	} else {
		locationEntry := locations.CreateLocation(
			query_holder.Get("city_name"),
			query_holder.Get("iso_country"),
			query_holder.Get("country_name"),
			query_holder.Get("latitude"),
			query_holder.Get("longitude"),
			query_holder.Get("accuweather_id"),
			query_holder.Get("accuweather_city_name"),
			query_holder.Get("gismeteo_id"),
			query_holder.Get("gismeteo_city_name"))
		PrintLocationEntry(locationEntry, w)
	}
}

func ReadLocations(c web.C, w http.ResponseWriter, r *http.Request) {
	PrintLocations(w)
}

func UpdateLocation(c web.C, w http.ResponseWriter, r *http.Request) {
	query_holder := r.URL.Query()

	missing := MakeMissingParamsList(query_holder, []string{"location_id", "city_name", "iso_country", "iso_country", "country_name", "latitude", "longitude"})
	if len(missing) > 0 {
		MarshalPrintResponse(500, "The following parameters are missing: "+strings.Join(missing, ", "), make(map[string]string), w)
	} else {
		locationEntry, _ := locations.UpdateLocation(
			query_holder.Get("location_id"),
			query_holder.Get("city_name"),
			query_holder.Get("iso_country"),
			query_holder.Get("country_name"),
			query_holder.Get("latitude"),
			query_holder.Get("longitude"),
			query_holder.Get("accuweather_id"),
			query_holder.Get("accuweather_city_name"),
			query_holder.Get("gismeteo_id"),
			query_holder.Get("gismeteo_city_name"))
		PrintLocationEntry(locationEntry, w)
	}
}

func DeleteLocation(c web.C, w http.ResponseWriter, r *http.Request) {
	missing := make([]string, 0)

	query_holder := r.URL.Query()

	MakeMissingParamsList(query_holder, []string{"location_id"})

	if len(missing) > 0 {
		MarshalPrintResponse(500, "The following parameters are missing: "+strings.Join(missing, ", "), make(map[string]string), w)
	} else {
		err := locations.DeleteLocation(query_holder.Get("location_id"))
		PrintStatus(err, "Location removed successfully.", w)
	}
}

func ClearLocations(c web.C, w http.ResponseWriter, r *http.Request) {
	err := locations.Clear()

	PrintStatus(err, "Locations cleared successfully.", w)
}

func ReadHistory(c web.C, w http.ResponseWriter, r *http.Request) {
	query_holder := r.URL.Query()
	entryid := query_holder.Get("entryid")
	source := query_holder.Get("source")
	wtype := query_holder.Get("wtype")
	country := query_holder.Get("country")
	locationid := query_holder.Get("locationid")
	requeststart := query_holder.Get("requeststart")
	requestend := query_holder.Get("requestend")

	history_data := history.ReadHistory(entryid, source, wtype, country, locationid, requeststart, requestend)
	history_filters := map[string]string{"entryid": entryid, "source": source, "wtype": wtype, "country": country, "locationid": locationid, "requeststart": requeststart, "requestend": requestend}

	MarshalPrintResponse(200, "OK", map[string]interface{}{"history": map[string]interface{}{"data": history_data, "filters": history_filters}}, w)
}

func RefreshHistory(c web.C, w http.ResponseWriter, r *http.Request) {
	query_holder := r.URL.Query()

	var wtypes []string
	wtype := query_holder.Get("wtype")

	if wtype == "" {
		wtypes = []string{"current", "forecast"}
	} else if wtype != "current" && wtype != "forecast" && wtype != "" {
		PrintStatus(errors.New(""), "Invalid wtype specified.", w)
		return
	} else {
		wtypes = []string{wtype}
	}

	locations_query := locations.ReadLocations()
	historyEntry := history.CreateHistoryEntry(locations_query, sources, wtypes)
	PrintHistoryEntry(historyEntry, w)
}

func ClearHistory(c web.C, w http.ResponseWriter, r *http.Request) {
	err := history.Clear()

	PrintStatus(err, "History cleared successfully.", w)
}

func Api(c *web.C, h http.Handler) http.Handler {
	fn := func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Access-Control-Allow-Origin", "*")
		h.ServeHTTP(w, r)
	}
	return http.HandlerFunc(fn)
}

func init() {
	flag.StringVar(&mongoDsn, "mongo", "mongodb://127.0.0.1:27017/weatherchecker", "MongoDB DSN")
}

func main() {
	flag.Parse()

	if os.Getenv("MONGO") != "" {
		mongoDsn = os.Getenv("MONGO")
	}

	fmt.Println("Connecting to MongoDB at", mongoDsn)
	err := db_instance.Connect(mongoDsn)
	if err != nil {
		fmt.Println(fmt.Sprintf("Database error: %s", err))
		return
	}
	defer db_instance.Disconnect()

	const ApiVer = "0.1"

	const ApiEntrypoint = "/api" + "/" + ApiVer

	const SourcesEntrypoint = ApiEntrypoint + "/sources"
	const LocationEntrypoint = ApiEntrypoint + "/locations"
	const HistoryEntrypoint = ApiEntrypoint + "/history"

	goji.Use(Api)
	goji.Get(SourcesEntrypoint, ReadSources)
	goji.Get(LocationEntrypoint, ReadLocations)
	goji.Get(LocationEntrypoint+"/add", CreateLocation)
	goji.Get(LocationEntrypoint+"/edit", UpdateLocation)
	goji.Get(LocationEntrypoint+"/remove", DeleteLocation)
	goji.Get(LocationEntrypoint+"/clear", ClearLocations)
	goji.Get(HistoryEntrypoint, ReadHistory)
	goji.Get(HistoryEntrypoint+"/refresh", RefreshHistory)
	goji.Get(HistoryEntrypoint+"/clear", ClearHistory)
	goji.Serve()
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net/url"
	"os"
	"strings"

	"github.com/lair-framework/api-server/client"
	"github.com/lair-framework/go-lair"
	"github.com/lair-framework/go-nikto"
)

const (
	version = "2.0.0"
	tool    = "nikto"
	usage   = `
Usage:
  drone-nikto <id> <filename>
  export LAIR_ID=<id>; drone-nikto <filename>
Options:
  -v              show version and exit
  -h              show usage and exit
  -k              allow insecure SSL connections
  -force-ports    disable data protection in the API server for excessive ports
  -tags           a comma separated list of tags to add to every host that is imported
`
)

func buildProject(nikto *nikto.NiktoData, exproject *lair.Project, tags []string) (map[string]bool, error) {
	var bNotFound map[string]bool
	exproject.Tool = tool

	command := make([]lair.Command, 0)
	for _, scan := range nikto.NiktoScan {
		for _, item := range scan.ScanDetails {
			// Update project with Nikto command statement
			command = append(command, lair.Command{
				Tool:    tool,
				Command: scan.Options,
			})

			// Confirm if Host and Port existing in current project
			found := false
			hname := false
			for x, h := range exproject.Hosts {
				if item.TargetIP == h.IPv4 {
					for y, p := range h.Services {
						if item.TargetPort == p.Port {
							found = true

							// Append host tags
							exproject.Hosts[x].Tags = append(h.Tags, tags...)

							// Update Modified By
							h.LastModifiedBy = tool
							p.LastModifiedBy = tool

							// Append Note information to Service
							note := &lair.Note{
								Title:          fmt.Sprintf("Nikto v%v (%v:%v)", scan.Version, item.HostHeader, item.TargetPort),
								LastModifiedBy: tool,
							}

							scheme, err := url.Parse(item.TargetHostname)
							if err != nil {
								return bNotFound, err
							}
							if scheme.Scheme == "https" {
								note.Content = fmt.Sprintf("SSL Information:\nSubject: %v\nCiphers: %v\nIssuer: %v\n\n", item.SSL.Info, item.SSL.Ciphers, item.SSL.Issuers)
							}

							for _, item := range item.Items {
								if item.OSVDBID > 0 {
									note.Content += fmt.Sprintf("%v URI: %v OSVDBID: %v\n", item.Description, item.URI, item.OSVDBID)
								} else {
									note.Content += fmt.Sprintf("%v URI: %v\n", item.Description, item.URI)
								}
							}
							note.Content += fmt.Sprintf("\nStart: %s End: %s", scan.ScanStart, scan.ScanEnd)
							exproject.Hosts[x].Services[y].Notes = append(p.Notes, *note)
						}
					}
					// Check for existing hostname and append
					for _, n := range h.Hostnames {
						if n == item.TargetHostname {
							hname = true
						}
					}
					if !hname && (item.TargetHostname != item.TargetIP) {
						exproject.Hosts[x].Hostnames = append(h.Hostnames, item.TargetHostname)
					}
				}
			}
			if !found {
				bNotFound[fmt.Sprintf("%s (%s:%v)", item.TargetHostname, item.TargetIP, item.TargetPort)] = true
				continue
			}
		}
	}
	exproject.Commands = command
	return bNotFound, nil
}

func main() {
	showVersion := flag.Bool("v", false, "")
	insecureSSL := flag.Bool("k", false, "")
	forcePorts := flag.Bool("force-ports", false, "")
	tags := flag.String("tags", "", "")
	flag.Usage = func() {
		fmt.Println(usage)
	}
	flag.Parse()
	if *showVersion {
		log.Println(version)
		os.Exit(0)
	}
	lairURL := os.Getenv("LAIR_API_SERVER")
	if lairURL == "" {
		log.Fatal("Fatal: Missing LAIR_API_SERVER environment variable")
	}
	lairPID := os.Getenv("LAIR_ID")
	var filename string
	switch len(flag.Args()) {
	case 2:
		lairPID = flag.Arg(0)
		filename = flag.Arg(1)
	case 1:
		filename = flag.Arg(0)
	default:
		log.Fatal("Fatal: Missing required argument")
	}

	u, err := url.Parse(lairURL)
	if err != nil {
		log.Fatalf("Fatal: Error parsing LAIR_API_SERVER URL. Error %s", err.Error())
	}
	if u.User == nil {
		log.Fatal("Fatal: Missing username and/or password")
	}
	user := u.User.Username()
	pass, _ := u.User.Password()
	if user == "" || pass == "" {
		log.Fatal("Fatal: Missing username and/or password")
	}
	c, err := client.New(&client.COptions{
		User:               user,
		Password:           pass,
		Host:               u.Host,
		Scheme:             u.Scheme,
		InsecureSkipVerify: *insecureSSL,
	})
	if err != nil {
		log.Fatalf("Fatal: Error setting up client. Error %s", err.Error())
	}

	buf, err := ioutil.ReadFile(filename)
	if err != nil {
		log.Fatalf("Fatal: Could not open file. Error %s", err.Error())
	}

	niktoData, err := nikto.Parse(buf)
	if err != nil {
		log.Fatalf("Fatal: Error parsing nikto data. Error %s", err.Error())
	}

	hostTags := []string{}
	if *tags != "" {
		hostTags = strings.Split(*tags, ",")
	}

	project, err := c.ExportProject(lairPID)
	if err != nil {
		log.Fatalf("Fatal: Unable to export project. Error %s", err.Error())
	}
	bNotFound, err := buildProject(niktoData, &project, hostTags)
	if err != nil {
		log.Fatal(err.Error())
	}

	res, err := c.ImportProject(&client.DOptions{ForcePorts: *forcePorts}, &project)
	if err != nil {
		log.Fatalf("Fatal: Unable to import project. Error %s", err)
	}

	defer res.Body.Close()
	droneRes := &client.Response{}
	body, err := ioutil.ReadAll(res.Body)
	if err != nil {
		log.Fatalf("Fatal: Error %s", err.Error())
	}
	if err := json.Unmarshal(body, droneRes); err != nil {
		log.Fatalf("Fatal: Could not unmarshal JSON. Error %s", err.Error())
	}
	if droneRes.Status == "Error" {
		log.Fatalf("Fatal: Import failed. Error %s", droneRes.Message)
	}

	if len(bNotFound) > 0 {
		log.Printf("Info: The following host ports contained Nikto results but did not exist in project %s", lairPID)
	}
	for k := range bNotFound {
		fmt.Println(k)
	}

	log.Println("Success: Operation completed successfully")
}
<commit_msg>Update Lair Project Nikto command information<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net/url"
	"os"
	"strings"

	"github.com/lair-framework/api-server/client"
	"github.com/lair-framework/go-lair"
	"github.com/lair-framework/go-nikto"
)

const (
	version = "2.0.0"
	tool    = "nikto"
	usage   = `
Usage:
  drone-nikto <id> <filename>
  export LAIR_ID=<id>; drone-nikto <filename>
Options:
  -v              show version and exit
  -h              show usage and exit
  -k              allow insecure SSL connections
  -force-ports    disable data protection in the API server for excessive ports
  -tags           a comma separated list of tags to add to every host that is imported
`
)

func buildProject(nikto *nikto.NiktoData, exproject *lair.Project, tags []string) (map[string]bool, error) {
	var bNotFound map[string]bool
	exproject.Tool = tool
	var string command

	for _, scan := range nikto.NiktoScan {
		for _, item := range scan.ScanDetails {
			command = scan.Options

			// Confirm if Host and Port existing in current project
			found := false
			hname := false
			for x, h := range exproject.Hosts {
				if item.TargetIP == h.IPv4 {
					for y, p := range h.Services {
						if item.TargetPort == p.Port {
							found = true

							// Append host tags
							exproject.Hosts[x].Tags = append(h.Tags, tags...)

							// Update Modified By
							h.LastModifiedBy = tool
							p.LastModifiedBy = tool

							// Append Note information to Service
							note := &lair.Note{
								Title:          fmt.Sprintf("Nikto v%v (%v:%v)", scan.Version, item.HostHeader, item.TargetPort),
								LastModifiedBy: tool,
							}

							scheme, err := url.Parse(item.TargetHostname)
							if err != nil {
								return bNotFound, err
							}
							if scheme.Scheme == "https" {
								note.Content = fmt.Sprintf("SSL Information:\nSubject: %v\nCiphers: %v\nIssuer: %v\n\n", item.SSL.Info, item.SSL.Ciphers, item.SSL.Issuers)
							}

							for _, item := range item.Items {
								if item.OSVDBID > 0 {
									note.Content += fmt.Sprintf("%v URI: %v OSVDBID: %v\n", item.Description, item.URI, item.OSVDBID)
								} else {
									note.Content += fmt.Sprintf("%v URI: %v\n", item.Description, item.URI)
								}
							}
							note.Content += fmt.Sprintf("\nStart: %s End: %s", scan.ScanStart, scan.ScanEnd)
							exproject.Hosts[x].Services[y].Notes = append(p.Notes, *note)
						}
					}
					// Check for existing hostname and append
					for _, n := range h.Hostnames {
						if n == item.TargetHostname {
							hname = true
						}
					}
					if !hname && (item.TargetHostname != item.TargetIP) {
						exproject.Hosts[x].Hostnames = append(h.Hostnames, item.TargetHostname)
					}
				}
			}
			if !found {
				bNotFound[fmt.Sprintf("%s (%s:%v)", item.TargetHostname, item.TargetIP, item.TargetPort)] = true
				continue
			}
		}
	}
	// Update project with Nikto command
	com := make([]lair.Command, 0)
	com = append(com, lair.Command{
		Tool:    tool,
		Command: command,
	})
	exproject.Commands = command

	return bNotFound, nil
}

func main() {
	showVersion := flag.Bool("v", false, "")
	insecureSSL := flag.Bool("k", false, "")
	forcePorts := flag.Bool("force-ports", false, "")
	tags := flag.String("tags", "", "")
	flag.Usage = func() {
		fmt.Println(usage)
	}
	flag.Parse()
	if *showVersion {
		log.Println(version)
		os.Exit(0)
	}
	lairURL := os.Getenv("LAIR_API_SERVER")
	if lairURL == "" {
		log.Fatal("Fatal: Missing LAIR_API_SERVER environment variable")
	}
	lairPID := os.Getenv("LAIR_ID")
	var filename string
	switch len(flag.Args()) {
	case 2:
		lairPID = flag.Arg(0)
		filename = flag.Arg(1)
	case 1:
		filename = flag.Arg(0)
	default:
		log.Fatal("Fatal: Missing required argument")
	}

	u, err := url.Parse(lairURL)
	if err != nil {
		log.Fatalf("Fatal: Error parsing LAIR_API_SERVER URL. Error %s", err.Error())
	}
	if u.User == nil {
		log.Fatal("Fatal: Missing username and/or password")
	}
	user := u.User.Username()
	pass, _ := u.User.Password()
	if user == "" || pass == "" {
		log.Fatal("Fatal: Missing username and/or password")
	}
	c, err := client.New(&client.COptions{
		User:               user,
		Password:           pass,
		Host:               u.Host,
		Scheme:             u.Scheme,
		InsecureSkipVerify: *insecureSSL,
	})
	if err != nil {
		log.Fatalf("Fatal: Error setting up client. Error %s", err.Error())
	}

	buf, err := ioutil.ReadFile(filename)
	if err != nil {
		log.Fatalf("Fatal: Could not open file. Error %s", err.Error())
	}

	niktoData, err := nikto.Parse(buf)
	if err != nil {
		log.Fatalf("Fatal: Error parsing nikto data. Error %s", err.Error())
	}

	hostTags := []string{}
	if *tags != "" {
		hostTags = strings.Split(*tags, ",")
	}

	project, err := c.ExportProject(lairPID)
	if err != nil {
		log.Fatalf("Fatal: Unable to export project. Error %s", err.Error())
	}
	bNotFound, err := buildProject(niktoData, &project, hostTags)
	if err != nil {
		log.Fatal(err.Error())
	}

	res, err := c.ImportProject(&client.DOptions{ForcePorts: *forcePorts}, &project)
	if err != nil {
		log.Fatalf("Fatal: Unable to import project. Error %s", err)
	}

	defer res.Body.Close()
	droneRes := &client.Response{}
	body, err := ioutil.ReadAll(res.Body)
	if err != nil {
		log.Fatalf("Fatal: Error %s", err.Error())
	}
	if err := json.Unmarshal(body, droneRes); err != nil {
		log.Fatalf("Fatal: Could not unmarshal JSON. Error %s", err.Error())
	}
	if droneRes.Status == "Error" {
		log.Fatalf("Fatal: Import failed. Error %s", droneRes.Message)
	}

	if len(bNotFound) > 0 {
		log.Printf("Info: The following host ports contained Nikto results but did not exist in project %s", lairPID)
	}
	for k := range bNotFound {
		fmt.Println(k)
	}

	log.Println("Success: Operation completed successfully")
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"net/http"
	"os"
	"time"

	"github.com/deoliveiraromain/task-rest-api/configuration"
	"github.com/deoliveiraromain/task-rest-api/db"
	"github.com/deoliveiraromain/task-rest-api/handlers"
	"github.com/deoliveiraromain/task-rest-api/routes"
	"gopkg.in/mgo.v2"
)

func main() {
	//get config
	conf, err := configuration.GetConfig()
	if err != nil {
		log.Printf("Error getting config [%s]", err)
		os.Exit(1)
	}
	// Connect to our local mongo
	log.Printf("Config port %d", conf.Port)
	log.Printf("Config MongoHost %s", conf.MongoHost)
	//s, err := mgo.Dial("mongodb://" + conf.MongoHost + ":27017")
	maxWait := time.Duration(5 * time.Second)
	s, err := mgo.DialWithTimeout("mongodb://"+conf.MongoHost+":27017", maxWait)
	defer s.Close()

	// Check if connection error, is mongo running?
	if err != nil {
		panic(err)
	}
	con := db.NewMongo(s, "todos")

	router := routes.NewRouter()
	router.HandleFunc("/", serveWelcome)

	tc := handlers.NewTaskController(con)
	tc.Register(router)

	portStr := fmt.Sprintf(":%s", conf.Port)
	log.Printf("Serving on %s", portStr)
	log.Fatal(http.ListenAndServe(portStr, router))
}

// ServeHTTP is the http.Handler interface implementation
func serveWelcome(w http.ResponseWriter, r *http.Request) {
	fmt.Fprint(w, "Welcome on TODO list API written in GO.!\n")
}
<commit_msg>fix again mongo<commit_after>package main

import (
	"fmt"
	"log"
	"net/http"
	"os"
	"time"

	"github.com/deoliveiraromain/task-rest-api/configuration"
	"github.com/deoliveiraromain/task-rest-api/db"
	"github.com/deoliveiraromain/task-rest-api/handlers"
	"github.com/deoliveiraromain/task-rest-api/routes"
	"gopkg.in/mgo.v2"
)

func main() {
	//get config
	conf, err := configuration.GetConfig()
	if err != nil {
		log.Printf("Error getting config [%s]", err)
		os.Exit(1)
	}
	// Connect to our local mongo
	log.Printf("Config port %s", conf.Port)
	log.Printf("Config MongoHost %s", conf.MongoHost)
	//s, err := mgo.Dial("mongodb://" + conf.MongoHost + ":27017")
	maxWait := time.Duration(5 * time.Second)
	s, err := mgo.DialWithTimeout(conf.MongoHost, maxWait)
	defer s.Close()

	// Check if connection error, is mongo running?
	if err != nil {
		panic(err)
	}
	con := db.NewMongo(s, "todos")

	router := routes.NewRouter()
	router.HandleFunc("/", serveWelcome)

	tc := handlers.NewTaskController(con)
	tc.Register(router)

	portStr := fmt.Sprintf(":%s", conf.Port)
	log.Printf("Serving on %s", portStr)
	log.Fatal(http.ListenAndServe(portStr, router))
}

// ServeHTTP is the http.Handler interface implementation
func serveWelcome(w http.ResponseWriter, r *http.Request) {
	fmt.Fprint(w, "Welcome on TODO list API written in GO.!\n")
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"os"
	"os/signal"
	"strconv"
	"strings"

	"encoding/json"

	"github.com/Shopify/sarama"
	"github.com/companieshouse/chs.go/avro"
	"github.com/companieshouse/chs.go/avro/schema"
	"github.com/companieshouse/chs.go/kafka/producer"
	"github.com/companieshouse/kafka-management-tool/schemas"
)

// Assigns all flags to variables
var (
	brokerPtr         = flag.String("broker", "", "Broker address")
	topicPtr          = flag.String("topic", "", "Topic name")
	schemaPtr         = flag.String("schema", "", "Schema name")
	schemaRegistryPtr = flag.String("schema-registry", "", "Schema Registry")
	partitionPtr      = flag.Int64("partition", 0, "Partition (default: 0)")
	offsetPtr         = flag.String("offset", "", "Offset number, can be single offset number i.e. 10 or a range separated by the - operator i.e. 10-20")
	jsonOutPtr        = flag.Int64("json-out", 0, "Print deserialized JSON message (default: 0)")
)

func main() {
	flag.Parse()

	// check if all mandatory flags have been provided
	validateFlags()

	// Print flags and parameters of the tool
	printFlags()

	// create offset array
	offsetArray := createOffsetArray(*offsetPtr)

	// create default config for sarama
	config := sarama.NewConfig()

	//create consumer
	consumer, err := sarama.NewConsumer([]string{*brokerPtr}, config)
	if err != nil {
		panic(err)
	}

	// create messages chan
	messages := make(chan *sarama.ConsumerMessage)
	go consumePartition(consumer, *topicPtr, int32(*partitionPtr), messages, offsetArray)

	// create signals chan
	signals := make(chan os.Signal, 1)
	signal.Notify(signals, os.Interrupt)

	consumed := 0
ConsumerLoop:
	for {
		select {
		case msg := <-messages:

			// make messageBytes channel
			messageBytes := make(chan []byte)

			// if jsonOutPtr is set to 1, then output JSON and republish
			// otherwise just republish message
			if *jsonOutPtr == 1 {
				go outputJSON(msg, messageBytes)
			} else {
				go func() {
					messageBytes <- msg.Value
				}()
			}

			// create producer message
			producerMessage := &producer.Message{
				Value: <-messageBytes,
				Topic: *topicPtr,
			}

			//create new producer
			p, err := producer.New(&producer.Config{Acks: &producer.WaitForAll, BrokerAddrs: []string{*brokerPtr}})
			if err != nil {
				fmt.Println("error creating producer")
				panic(err)
				os.Exit(1)
			}

			// republish message
			fmt.Printf("Republishing message for offset: %v \n", msg.Offset)
			partition, offset, err := p.Send(producerMessage)
			if err != nil {
				fmt.Println("error republishing message with offset: %v, \n", msg.Offset)
				panic(err)
				os.Exit(1)
			}
			fmt.Printf("Successfully republished message with offset %v to topic: %v using partition: %v new offset: %v \n", msg.Offset, *topicPtr, partition, offset)
			fmt.Println("---------------------")

			consumed++

			// check if we have hit the end of the array of messages
			// if so, exit the tool by breaking the ConsumerLoop
			if consumed == len(offsetArray) {
				break ConsumerLoop
			}
		case <-signals:
			fmt.Println("break")
			break ConsumerLoop
		}
	}
}

func outputJSON(msg *sarama.ConsumerMessage, messageBytes chan []byte) {
	schemaStruct := schemas.IdentifySchema(*topicPtr)
	var err error

	// Get schema from schema registry and use it to create avro consumer
	fmt.Printf("Getting schema for topic: %v \n", *topicPtr)
	schema, err := schema.Get(*schemaRegistryPtr, *schemaPtr)
	if err != nil {
		fmt.Println("error getting schema")
		panic(err)
		os.Exit(1)
	}
	fmt.Printf("Retrieved schema for topic: %v \n", *topicPtr)

	consumerAvro := &avro.Schema{
		Definition: schema,
	}

	// Unmarshal message value and assign it to schemaStruct
	fmt.Printf("Unmarshalling message for offset: %v \n", msg.Offset)
	if err = consumerAvro.Unmarshal(msg.Value, schemaStruct); err != nil {
		fmt.Errorf("error unmarshalling avro message for offset: %v \n", msg.Offset)
		panic(err)
		os.Exit(1)
	}
	fmt.Printf("Message successfully unmarshalled for offset: %v \n", msg.Offset)

	// Print the unmarshalled message out to the terminal
	fmt.Println("Unmarshalled message:")
	data, err := json.Marshal(schemaStruct)
	if err != nil {
		fmt.Errorf("error marshalling JSON message for offset: %v \n", msg.Offset)
		panic(err)
		os.Exit(1)
	}
	fmt.Println(string(data))

	// create avro producer
	producerAvro := &avro.Schema{
		Definition: schema,
	}

	// Marshall schemaStruct that contains message value ready for republishing
	fmt.Printf("Marshalling message for offset: %v \n", msg.Offset)
	message, err := producerAvro.Marshal(schemaStruct)
	if err != nil {
		fmt.Errorf("error marshalling avro message for offset: %v \n", msg.Offset)
		panic(err)
		os.Exit(1)
	}
	fmt.Printf("Message successfully marshalled for offset: %v \n", msg.Offset)

	// output the marshalled message into the messageBytes chan
	messageBytes <- message
}

// Validates the flags to make sure all mandatory flags have been supplied
// throw an error if not the case
func validateFlags() {
	flag.VisitAll(func(f *flag.Flag) {
		if string(f.Value.String()) == "" {
			fmt.Printf("Value not supplied for: %v \n", f.Name)
			os.Exit(1)
		}
	})
}

// Prints the flags and parameters of the tool
func printFlags() {
	fmt.Println("---------------------")
	fmt.Println("Parameters:")
	flag.VisitAll(func(f *flag.Flag) {
		fmt.Printf("%v: %v\n", f.Name, f.Value)
	})
	fmt.Println("---------------------")
}

// The offset slice is passed into method and is iterated over and each message
// related to the offset in the topic/partition is outputted into the 'out' chan
func consumePartition(consumer sarama.Consumer, topic string, partition int32, out chan *sarama.ConsumerMessage, offsetArray []int64) {
	for _, offset := range offsetArray {
		partitionConsumer, err := consumer.ConsumePartition(topic, partition, int64(offset))
		if err != nil {
			panic(err)
		}

		select {
		case msg := <-partitionConsumer.Messages():
			partitionConsumer.Close()
			out <- msg
		}
	}
}

// This creates the offset array depending on whether a range or a single value
// is entered as an argument to the tool
func createOffsetArray(offset string) []int64 {
	arraySize := make([]int64, 0)
	if strings.ContainsAny(offset, "-") {
		slice := strings.Split(offset, "-")
		minRange, err := strconv.ParseInt(slice[0], 10, 64)
		if err != nil {
			panic(err)
		}

		maxRange, err := strconv.ParseInt(slice[1], 10, 64)
		if err != nil {
			panic(err)
		}

		for value := minRange; value <= maxRange; value++ {
			arraySize = append(arraySize, value)
		}
	} else {
		value, err := strconv.ParseInt(offset, 10, 64)
		if err != nil {
			panic(err)
		}
		arraySize = append(arraySize, value)
	}
	return arraySize
}
<commit_msg>Add nill check for schema<commit_after>package main

import (
	"flag"
	"fmt"
	"os"
	"os/signal"
	"strconv"
	"strings"

	"encoding/json"

	"github.com/Shopify/sarama"
	"github.com/companieshouse/chs.go/avro"
	"github.com/companieshouse/chs.go/avro/schema"
	"github.com/companieshouse/chs.go/kafka/producer"
	"github.com/companieshouse/kafka-management-tool/schemas"
)

// Assigns all flags to variables
var (
	brokerPtr         = flag.String("broker", "", "Broker address")
	topicPtr          = flag.String("topic", "", "Topic name")
	schemaPtr         = flag.String("schema", "", "Schema name")
	schemaRegistryPtr = flag.String("schema-registry", "", "Schema Registry")
	partitionPtr      = flag.Int64("partition", 0, "Partition (default: 0)")
	offsetPtr         = flag.String("offset", "", "Offset number, can be single offset number i.e. 10 or a range separated by the - operator i.e. 10-20")
	jsonOutPtr        = flag.Int64("json-out", 0, "Print deserialized JSON message (default: 0)")
)

func main() {
	flag.Parse()

	// check if all mandatory flags have been provided
	validateFlags()

	// Print flags and parameters of the tool
	printFlags()

	// create offset array
	offsetArray := createOffsetArray(*offsetPtr)

	// create default config for sarama
	config := sarama.NewConfig()

	//create consumer
	consumer, err := sarama.NewConsumer([]string{*brokerPtr}, config)
	if err != nil {
		panic(err)
	}

	// create messages chan
	messages := make(chan *sarama.ConsumerMessage)
	go consumePartition(consumer, *topicPtr, int32(*partitionPtr), messages, offsetArray)

	// create signals chan
	signals := make(chan os.Signal, 1)
	signal.Notify(signals, os.Interrupt)

	consumed := 0
ConsumerLoop:
	for {
		select {
		case msg := <-messages:

			// make messageBytes channel
			messageBytes := make(chan []byte)

			// if jsonOutPtr is set to 1, then output JSON and republish
			// otherwise just republish message
			if *jsonOutPtr == 1 {
				go outputJSON(msg, messageBytes)
			} else {
				go func() {
					messageBytes <- msg.Value
				}()
			}

			// create producer message
			producerMessage := &producer.Message{
				Value: <-messageBytes,
				Topic: *topicPtr,
			}

			//create new producer
			p, err := producer.New(&producer.Config{Acks: &producer.WaitForAll, BrokerAddrs: []string{*brokerPtr}})
			if err != nil {
				fmt.Println("error creating producer")
				panic(err)
				os.Exit(1)
			}

			// republish message
			fmt.Printf("Republishing message for offset: %v \n", msg.Offset)
			partition, offset, err := p.Send(producerMessage)
			if err != nil {
				fmt.Println("error republishing message with offset: %v, \n", msg.Offset)
				panic(err)
				os.Exit(1)
			}
			fmt.Printf("Successfully republished message with offset %v to topic: %v using partition: %v new offset: %v \n", msg.Offset, *topicPtr, partition, offset)
			fmt.Println("---------------------")

			consumed++

			// check if we have hit the end of the array of messages
			// if so, exit the tool by breaking the ConsumerLoop
			if consumed == len(offsetArray) {
				break ConsumerLoop
			}
		case <-signals:
			fmt.Println("break")
			break ConsumerLoop
		}
	}
}

func outputJSON(msg *sarama.ConsumerMessage, messageBytes chan []byte) {
	schemaStruct := schemas.IdentifySchema(*topicPtr)

	if schemaStruct == nil {
        fmt.Println("no schema identified")
        os.Exit(1)
    }

	var err error

	// Get schema from schema registry and use it to create avro consumer
	fmt.Printf("Getting schema for topic: %v \n", *topicPtr)
	schema, err := schema.Get(*schemaRegistryPtr, *schemaPtr)
	if err != nil {
		fmt.Println("error getting schema")
		panic(err)
		os.Exit(1)
	}
	fmt.Printf("Retrieved schema for topic: %v \n", *topicPtr)

	consumerAvro := &avro.Schema{
		Definition: schema,
	}

	// Unmarshal message value and assign it to schemaStruct
	fmt.Printf("Unmarshalling message for offset: %v \n", msg.Offset)
	if err = consumerAvro.Unmarshal(msg.Value, schemaStruct); err != nil {
		fmt.Errorf("error unmarshalling avro message for offset: %v \n", msg.Offset)
		panic(err)
		os.Exit(1)
	}
	fmt.Printf("Message successfully unmarshalled for offset: %v \n", msg.Offset)

	// Print the unmarshalled message out to the terminal
	fmt.Println("Unmarshalled message:")
	data, err := json.Marshal(schemaStruct)
	if err != nil {
		fmt.Errorf("error marshalling JSON message for offset: %v \n", msg.Offset)
		panic(err)
		os.Exit(1)
	}
	fmt.Println(string(data))

	// create avro producer
	producerAvro := &avro.Schema{
		Definition: schema,
	}

	// Marshall schemaStruct that contains message value ready for republishing
	fmt.Printf("Marshalling message for offset: %v \n", msg.Offset)
	message, err := producerAvro.Marshal(schemaStruct)
	if err != nil {
		fmt.Errorf("error marshalling avro message for offset: %v \n", msg.Offset)
		panic(err)
		os.Exit(1)
	}
	fmt.Printf("Message successfully marshalled for offset: %v \n", msg.Offset)

	// output the marshalled message into the messageBytes chan
	messageBytes <- message
}

// Validates the flags to make sure all mandatory flags have been supplied
// throw an error if not the case
func validateFlags() {
	flag.VisitAll(func(f *flag.Flag) {
		if string(f.Value.String()) == "" {
			fmt.Printf("Value not supplied for: %v \n", f.Name)
			os.Exit(1)
		}
	})
}

// Prints the flags and parameters of the tool
func printFlags() {
	fmt.Println("---------------------")
	fmt.Println("Parameters:")
	flag.VisitAll(func(f *flag.Flag) {
		fmt.Printf("%v: %v\n", f.Name, f.Value)
	})
	fmt.Println("---------------------")
}

// The offset slice is passed into method and is iterated over and each message
// related to the offset in the topic/partition is outputted into the 'out' chan
func consumePartition(consumer sarama.Consumer, topic string, partition int32, out chan *sarama.ConsumerMessage, offsetArray []int64) {
	for _, offset := range offsetArray {
		partitionConsumer, err := consumer.ConsumePartition(topic, partition, int64(offset))
		if err != nil {
			panic(err)
		}

		select {
		case msg := <-partitionConsumer.Messages():
			partitionConsumer.Close()
			out <- msg
		}
	}
}

// This creates the offset array depending on whether a range or a single value
// is entered as an argument to the tool
func createOffsetArray(offset string) []int64 {
	arraySize := make([]int64, 0)
	if strings.ContainsAny(offset, "-") {
		slice := strings.Split(offset, "-")
		minRange, err := strconv.ParseInt(slice[0], 10, 64)
		if err != nil {
			panic(err)
		}

		maxRange, err := strconv.ParseInt(slice[1], 10, 64)
		if err != nil {
			panic(err)
		}

		for value := minRange; value <= maxRange; value++ {
			arraySize = append(arraySize, value)
		}
	} else {
		value, err := strconv.ParseInt(offset, 10, 64)
		if err != nil {
			panic(err)
		}
		arraySize = append(arraySize, value)
	}
	return arraySize
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"log"
	"net/http"
	"os"
	"strconv"
	"time"

	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3/s3manager"

	"gopkg.in/asaskevich/govalidator.v6"
)

type DashboardSearchResult struct {
	Id    int    `json:"id"`
	Title string `json:"title"`
	Uri   string `json:"uri"`
}

func httpGet(url string) (*http.Response, error) {
	resp, err := http.Get(url)
	if err != nil {
		return nil, fmt.Errorf("Error retrieving dashboards from URL %v: %v", url, err)
	}
	if resp.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("Unexpected status code returned from Grafana API (got: %d, expected: 200, msg:%s)", resp.StatusCode, resp.Status)
	}
	return resp, nil
}

func main() {
	grafanaURL := flag.String("grafanaURL", "", "The URL of the grafana server to be backed up")
	s3Bucket := flag.String("s3Bucket", "", "The name of the S3 bucket where the backup should be stored")
	useSharedConfig := flag.Bool("useSharedConfig", false, "Controls whether to use the ~/.aws shared config")
	flag.Parse()

	if *grafanaURL == "" || *s3Bucket == "" {
		flag.Usage()
		os.Exit(1)
	}

	if !govalidator.IsURL(*grafanaURL) {
		log.Fatalf("Invalid grafanaURL: %v", *grafanaURL)
	}

	// S3 client
	var sess *session.Session
	if *useSharedConfig {
		sess = session.Must(session.NewSessionWithOptions(session.Options{
			SharedConfigState: session.SharedConfigEnable,
		}))
	} else {
		sess = session.Must(session.NewSession())
	}
	uploader := s3manager.NewUploader(sess)

	getAllDashboardsURL := *grafanaURL + "/api/search"
	searchResults := make([]DashboardSearchResult, 0)
	resp, err := httpGet(getAllDashboardsURL)
	if err != nil {
		log.Fatalf("Error retrieving dashboard list from URL %v: %v", getAllDashboardsURL, err)
	}
	defer resp.Body.Close()
	err = json.NewDecoder(resp.Body).Decode(&searchResults)
	if err != nil {
		log.Fatalf("Error retrieving decoding response body: %v", err)
	}

	getDashBaseURL := *grafanaURL + "/api/dashboards/"

	timeStr := time.Now().UTC().Format("2006-01-02T15:04:05-0700")
	backupDir := "grafana-backup_" + timeStr + "/dashboards/"
	for _, searchResult := range searchResults {
		// retrieve dashboard
		getDashURL := getDashBaseURL + searchResult.Uri
		resp, err := httpGet(getDashURL)
		if err != nil {
			log.Fatalf("Error retrieving dashboard from URL %v: %v", getDashURL, err)
		}

		//upload to S3
		filename := backupDir + strconv.Itoa(searchResult.Id)
		ui := &s3manager.UploadInput{
			Bucket: s3Bucket,
			Key:    &filename,
			Body:   resp.Body,
		}

		_, err = uploader.Upload(ui)
		if err != nil {
			log.Fatalf("Error uploading dashboard json for dashboard from URL %v: %v", getDashURL, err)
		}
		resp.Body.Close()
	}

	log.Printf("Backup to directory %v in bucket %v completed at %v", backupDir, s3Bucket, time.Now())
}
<commit_msg>append dashboard title for cleaner restores<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"log"
	"net/http"
	"os"
	"strconv"
	"time"

	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3/s3manager"

	"gopkg.in/asaskevich/govalidator.v6"
)

type DashboardSearchResult struct {
	Id    int    `json:"id"`
	Title string `json:"title"`
	Uri   string `json:"uri"`
}

func httpGet(url string) (*http.Response, error) {
	resp, err := http.Get(url)
	if err != nil {
		return nil, fmt.Errorf("Error retrieving dashboards from URL %v: %v", url, err)
	}
	if resp.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("Unexpected status code returned from Grafana API (got: %d, expected: 200, msg:%s)", resp.StatusCode, resp.Status)
	}
	return resp, nil
}

func main() {
	grafanaURL := flag.String("grafanaURL", "", "The URL of the grafana server to be backed up")
	s3Bucket := flag.String("s3Bucket", "", "The name of the S3 bucket where the backup should be stored")
	useSharedConfig := flag.Bool("useSharedConfig", false, "Controls whether to use the ~/.aws shared config")
	flag.Parse()

	if *grafanaURL == "" || *s3Bucket == "" {
		flag.Usage()
		os.Exit(1)
	}

	if !govalidator.IsURL(*grafanaURL) {
		log.Fatalf("Invalid grafanaURL: %v", *grafanaURL)
	}

	// S3 client
	var sess *session.Session
	if *useSharedConfig {
		sess = session.Must(session.NewSessionWithOptions(session.Options{
			SharedConfigState: session.SharedConfigEnable,
		}))
	} else {
		sess = session.Must(session.NewSession())
	}
	uploader := s3manager.NewUploader(sess)

	getAllDashboardsURL := *grafanaURL + "/api/search"
	searchResults := make([]DashboardSearchResult, 0)
	resp, err := httpGet(getAllDashboardsURL)
	if err != nil {
		log.Fatalf("Error retrieving dashboard list from URL %v: %v", getAllDashboardsURL, err)
	}
	defer resp.Body.Close()
	err = json.NewDecoder(resp.Body).Decode(&searchResults)
	if err != nil {
		log.Fatalf("Error retrieving decoding response body: %v", err)
	}

	getDashBaseURL := *grafanaURL + "/api/dashboards/"

	timeStr := time.Now().UTC().Format("2006-01-02T15:04:05-0700")
	backupDir := "grafana-backup_" + timeStr + "/dashboards/"
	for _, searchResult := range searchResults {
		// retrieve dashboard
		getDashURL := getDashBaseURL + searchResult.Uri
		resp, err := httpGet(getDashURL)
		if err != nil {
			log.Fatalf("Error retrieving dashboard from URL %v: %v", getDashURL, err)
		}

		//upload to S3
		filename := backupDir + strconv.Itoa(searchResult.Id) + "_" + searchResult.Title
		ui := &s3manager.UploadInput{
			Bucket: s3Bucket,
			Key:    &filename,
			Body:   resp.Body,
		}

		_, err = uploader.Upload(ui)
		if err != nil {
			log.Fatalf("Error uploading dashboard json for dashboard from URL %v: %v", getDashURL, err)
		}
		resp.Body.Close()
	}

	log.Printf("Backup to directory %v in bucket %v completed at %v", backupDir, s3Bucket, time.Now())
}
<|endoftext|>