<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"github.com/fhs/gompd/mpd"
	"net/http"
	"time"
)

type m map[string]interface{}

func writeJSONAttrList(w http.ResponseWriter, d []mpd.Attrs, l time.Time, err error) {
	w.Header().Add("Last-Modified", l.Format(http.TimeFormat))
	v := m{"errors": err, "data": d}
	b, jsonerr := json.Marshal(v)
	if jsonerr != nil {
		return
	}
	fmt.Fprintf(w, string(b))
	return
}

func writeJSONAttr(w http.ResponseWriter, d mpd.Attrs, l time.Time, err error) {
	w.Header().Add("Last-Modified", l.Format(http.TimeFormat))
	v := m{"errors": err, "data": d}
	b, jsonerr := json.Marshal(v)
	if jsonerr != nil {
		return
	}
	fmt.Fprintf(w, string(b))
	return
}

func writeJSON(w http.ResponseWriter, err error) {
	v := m{"errors": err}
	b, jsonerr := json.Marshal(v)
	if jsonerr != nil {
		return
	}
	fmt.Fprintf(w, string(b))
	return
}

func notModified(w http.ResponseWriter, l time.Time) {
	w.Header().Add("Last-Modified", l.Format(http.TimeFormat))
	w.WriteHeader(304)
	return
}

func handler(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintf(w, "Hello, World")
}

type apiHandler struct {
	player *Player
}

func (h *apiHandler) playlist(w http.ResponseWriter, r *http.Request) {
	d, l := h.player.Playlist()
	if modified(r, l) {
		writeJSONAttrList(w, d, l, nil)
	} else {
		notModified(w, l)
	}
}

func (h *apiHandler) library(w http.ResponseWriter, r *http.Request) {
	d, l := h.player.Library()
	if modified(r, l) {
		writeJSONAttrList(w, d, l, nil)
	} else {
		notModified(w, l)
	}
}

func (h *apiHandler) current(w http.ResponseWriter, r *http.Request) {
	method := r.FormValue("action")
	if method == "prev" {
		writeJSON(w, h.player.Prev())
	} else if method == "play" {
		writeJSON(w, h.player.Play())
	} else if method == "pause" {
		writeJSON(w, h.player.Pause())
	} else if method == "next" {
		writeJSON(w, h.player.Next())
	} else if method == "detail" {
		d, l := h.player.Comments()
		if modified(r, l) {
			writeJSONAttr(w, d, l, nil)
		} else {
			notModified(w, l)
		}
	} else {
		d, l := h.player.Current()
		if modified(r, l) {
			writeJSONAttr(w, d, l, nil)
		} else {
			notModified(w, l)
		}
	}
}

func modified(r *http.Request, l time.Time) bool {
	return r.Header.Get("If-Modified-Since") != l.Format(http.TimeFormat)
}

// App serves http request.
func App(p *Player, config ServerConfig) {
	var api = new(apiHandler)
	api.player = p
	http.HandleFunc("/api/library", api.library)
	http.HandleFunc("/api/songs", api.playlist)
	http.HandleFunc("/api/songs/current", api.current)
	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		http.ServeFile(w, r, "app.html")
	})
	http.HandleFunc("/app.css", func(w http.ResponseWriter, r *http.Request) {
		http.ServeFile(w, r, "app.css")
	})
	http.HandleFunc("/app.js", func(w http.ResponseWriter, r *http.Request) {
		http.ServeFile(w, r, "app.js")
	})
	http.HandleFunc("/jquery-3.1.1.js", func(w http.ResponseWriter, r *http.Request) {
		http.ServeFile(w, r, "jquery-3.1.1.js")
	})
	http.ListenAndServe(fmt.Sprintf(":%s", config.Port), nil)
}
<commit_msg>add json content type<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"github.com/fhs/gompd/mpd"
	"net/http"
	"time"
)

type m map[string]interface{}

func writeJSONAttrList(w http.ResponseWriter, d []mpd.Attrs, l time.Time, err error) {
	w.Header().Add("Last-Modified", l.Format(http.TimeFormat))
	w.Header().Add("Content-Type", "application/json; charset=utf-8")
	v := m{"errors": err, "data": d}
	b, jsonerr := json.Marshal(v)
	if jsonerr != nil {
		return
	}
	fmt.Fprintf(w, string(b))
	return
}

func writeJSONAttr(w http.ResponseWriter, d mpd.Attrs, l time.Time, err error) {
	w.Header().Add("Last-Modified", l.Format(http.TimeFormat))
	w.Header().Add("Content-Type", "application/json")
	v := m{"errors": err, "data": d}
	b, jsonerr := json.Marshal(v)
	if jsonerr != nil {
		return
	}
	fmt.Fprintf(w, string(b))
	return
}

func writeJSON(w http.ResponseWriter, err error) {
	w.Header().Add("Content-Type", "application/json")
	v := m{"errors": err}
	b, jsonerr := json.Marshal(v)
	if jsonerr != nil {
		return
	}
	fmt.Fprintf(w, string(b))
	return
}

func notModified(w http.ResponseWriter, l time.Time) {
	w.Header().Add("Content-Type", "application/json")
	w.Header().Add("Last-Modified", l.Format(http.TimeFormat))
	w.WriteHeader(304)
	return
}

func handler(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintf(w, "Hello, World")
}

type apiHandler struct {
	player *Player
}

func (h *apiHandler) playlist(w http.ResponseWriter, r *http.Request) {
	d, l := h.player.Playlist()
	if modified(r, l) {
		writeJSONAttrList(w, d, l, nil)
	} else {
		notModified(w, l)
	}
}

func (h *apiHandler) library(w http.ResponseWriter, r *http.Request) {
	d, l := h.player.Library()
	if modified(r, l) {
		writeJSONAttrList(w, d, l, nil)
	} else {
		notModified(w, l)
	}
}

func (h *apiHandler) current(w http.ResponseWriter, r *http.Request) {
	method := r.FormValue("action")
	if method == "prev" {
		writeJSON(w, h.player.Prev())
	} else if method == "play" {
		writeJSON(w, h.player.Play())
	} else if method == "pause" {
		writeJSON(w, h.player.Pause())
	} else if method == "next" {
		writeJSON(w, h.player.Next())
	} else if method == "detail" {
		d, l := h.player.Comments()
		if modified(r, l) {
			writeJSONAttr(w, d, l, nil)
		} else {
			notModified(w, l)
		}
	} else {
		d, l := h.player.Current()
		if modified(r, l) {
			writeJSONAttr(w, d, l, nil)
		} else {
			notModified(w, l)
		}
	}
}

func modified(r *http.Request, l time.Time) bool {
	return r.Header.Get("If-Modified-Since") != l.Format(http.TimeFormat)
}

// App serves http request.
func App(p *Player, config ServerConfig) {
	var api = new(apiHandler)
	api.player = p
	http.HandleFunc("/api/library", api.library)
	http.HandleFunc("/api/songs", api.playlist)
	http.HandleFunc("/api/songs/current", api.current)
	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		http.ServeFile(w, r, "app.html")
	})
	http.HandleFunc("/app.css", func(w http.ResponseWriter, r *http.Request) {
		http.ServeFile(w, r, "app.css")
	})
	http.HandleFunc("/app.js", func(w http.ResponseWriter, r *http.Request) {
		http.ServeFile(w, r, "app.js")
	})
	http.HandleFunc("/jquery-3.1.1.js", func(w http.ResponseWriter, r *http.Request) {
		http.ServeFile(w, r, "jquery-3.1.1.js")
	})
	http.ListenAndServe(fmt.Sprintf(":%s", config.Port), nil)
}
<|endoftext|>


<commit_before>/*
Package manners provides a wrapper for a standard net/http server that
ensures all active HTTP client have completed their current request
before the server shuts down.

It can be used a drop-in replacement for the standard http package,
or can wrap a pre-configured Server.

eg.
	myHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
	  w.Write([]byte("Hello\n"))
	})

	http.Handle("/hello", myHandler)

	log.Fatal(manners.ListenAndServe(":8080", nil))

or for a customized server:
	s := manners.NewWithServer(&http.Server{
		Addr:           ":8080",
		Handler:        myHandler,
		ReadTimeout:    10 * time.Second,
		WriteTimeout:   10 * time.Second,
		MaxHeaderBytes: 1 << 20,
	})
	log.Fatal(s.ListenAndServe())

The server will shutdown cleanly when the Close() method is called:

	manners.CloseOnInterrupt()
	http.Handle("/hello", myHandler)
	log.Fatal(manners.ListenAndServe(":8080", nil))
*/
package manners

import (
	"crypto/tls"
	"net"
	"net/http"
	"net/http/fcgi"
	"os"
	"sync"
	"sync/atomic"
	"strings"
	"os/signal"
	"syscall"
)

// interface describing a waitgroup, so unit
// tests can mock out an instrumentable version
type waitgroup interface {
	Add(delta int)
	Done()
	Wait()
}

// StateHandler can be called by the server if the state of the connection changes.
// Notice that it passed previous state and the new state as parameters.
type StateHandler func(net.Conn, http.ConnState, http.ConnState)

type Options struct {
	Server       *http.Server
	StateHandler StateHandler
	Listener     net.Listener
}

// NewServer creates a new GracefulServer. The server will begin shutting down when
// a value is passed to the Shutdown channel.
func NewServer() *GracefulServer {
	return NewWithServer(new(http.Server))
}

// NewWithServer wraps an existing http.Server object and returns a GracefulServer
// that supports all of the original Server operations.
func NewWithServer(s *http.Server) *GracefulServer {
	return &GracefulServer{
		Server:   s,
		shutdown: make(chan struct{}),
		wg:       new(sync.WaitGroup),
	}
}

func NewWithOptions(o Options) *GracefulServer {
	// Set up listener
	var listener *GracefulListener
	if o.Listener != nil {
		g, ok := o.Listener.(*GracefulListener)
		if !ok {
			listener = NewListener(o.Listener)
		} else {
			listener = g
		}
	}

	return &GracefulServer{
		listener:     listener,
		Server:       o.Server,
		stateHandler: o.StateHandler,
		shutdown:     make(chan struct{}),
		wg:           new(sync.WaitGroup),
	}
}

// A GracefulServer maintains a WaitGroup that counts how many in-flight
// requests the server is handling. When it receives a shutdown signal,
// it stops accepting new requests but does not actually shut down until
// all in-flight requests terminate.
//
// GracefulServer embeds the underlying net/http.Server making its non-override
// methods and properties avaiable.
//
// It must be initialized by calling NewServer or NewWithServer
type GracefulServer struct {
	*http.Server
	shutdown chan struct{}
	wg       waitgroup
	listener *GracefulListener

	// used by test code
	up chan net.Listener

	stateHandler StateHandler
}

// Close stops the server from accepting new requets and beings shutting down.
func (s *GracefulServer) Close() {
	close(s.shutdown)
}

func isUnixNetwork(addr string) bool {
	return strings.HasPrefix(addr, "/")
}

func chooseNetwork(addr string) string {
	if isUnixNetwork(addr) {
		return "unix"
	}
	return "tcp"
}

// ListenAndServe provides a graceful equivalent of net/http.Serve.ListenAndServe, and
// also supports the FastCGI equivalent.
func (s *GracefulServer) ListenAndServe() error {
	if s.listener == nil {
		netwk := chooseNetwork(s.Addr)
		oldListener, err := net.Listen(netwk, s.Addr)
		if err != nil {
			return err
		}
		s.listener = NewListener(oldListener.(*net.TCPListener))
	}
	return s.Serve(s.listener)
}

// ListenAndServeTLS provides a graceful equivalent of net/http.Serve.ListenAndServeTLS.
func (s *GracefulServer) ListenAndServeTLS(certFile, keyFile string) error {
	addr := s.Addr
	if addr == "" {
		addr = ":https"
	}
	config := &tls.Config{}
	if s.TLSConfig != nil {
		*config = *s.TLSConfig
	}
	if config.NextProtos == nil {
		config.NextProtos = []string{"http/1.1"}
	}

	var err error
	config.Certificates = make([]tls.Certificate, 1)
	config.Certificates[0], err = tls.LoadX509KeyPair(certFile, keyFile)
	if err != nil {
		return err
	}

	return s.ListenAndServeTLSWithConfig(config)
}

// ListenAndServeTLS provides a graceful equivalent of net/http.Serve.ListenAndServeTLS.
func (s *GracefulServer) ListenAndServeTLSWithConfig(config *tls.Config) error {
	addr := s.Addr
	if addr == "" {
		addr = ":https"
	}

	if s.listener == nil {
		// only "tcp" is supported with TLS
		ln, err := net.Listen("tcp", addr)
		if err != nil {
			return err
		}

		tlsListener := NewTLSListener(TCPKeepAliveListener{ln.(*net.TCPListener)}, config)
		s.listener = NewListener(tlsListener)
	}
	return s.Serve(s.listener)
}

func (gs *GracefulServer) GetFile() (*os.File, error) {
	return gs.listener.GetFile()
}

func (gs *GracefulServer) HijackListener(s *http.Server, config *tls.Config) (*GracefulServer, error) {
	listener, err := gs.listener.Clone()
	if err != nil {
		return nil, err
	}

	if config != nil {
		listener = NewTLSListener(TCPKeepAliveListener{listener.(*net.TCPListener)}, config)
	}

	other := NewWithServer(s)
	other.listener = NewListener(listener)
	return other, nil
}

// Serve provides a graceful equivalent net/http.Server.Serve.
//
// If listener is not an instance of *GracefulListener is will be wrapped
// to become one.
func (s *GracefulServer) Serve(listener net.Listener) error {
	// accept a net.Listener to preserve the interface compatibility with the standard
	// http.Server, but we except a GracefluListener
	gracefulListener, ok := listener.(*GracefulListener)
	if !ok {
		gracefulListener = NewListener(listener)
		listener = gracefulListener
	}
	s.listener = gracefulListener

	var closing int32

	go func() {
		<-s.shutdown
		atomic.StoreInt32(&closing, 1)
		s.Server.SetKeepAlivesEnabled(false)
		listener.Close()
	}()

	orgConnState := s.Server.ConnState
	s.ConnState = func(conn net.Conn, newState http.ConnState) {
		// Ugly hack, but it works. We pass the information about the underlying state via the only available interface in net.Conn
		// we do this not to override the tls.Conn, as the internal logic of http.Server depends on the type assertion (unfortunately)
		gconn := conn.LocalAddr().(*gracefulAddr).gconn
		switch newState {
		case http.StateNew:
			// new_conn -> StateNew
			s.StartRoutine()

		case http.StateActive:
			// (StateNew, StateIdle) -> StateActive
			if gconn.lastHTTPState == http.StateIdle {
				// transitioned from idle back to active
				s.StartRoutine()
			}

		case http.StateIdle:
			// StateActive -> StateIdle
			if atomic.LoadInt32(&closing) == 1 {
				// rapidly close newly idle connections; if not they may make
				// one more request before SetKeepAliveEnabled(false)  takes effect.
				conn.Close()
			}
			s.FinishRoutine()

		case http.StateClosed, http.StateHijacked:
			// (StateNew, StateActive, StateIdle) -> (StateClosed, StateHiJacked)
			if gconn.lastHTTPState != http.StateIdle {
				// if it was idle it's already been decremented
				s.FinishRoutine()
			}
		}
		if s.stateHandler != nil {
			s.stateHandler(conn, gconn.lastHTTPState, newState)
		}
		gconn.lastHTTPState = newState
		if orgConnState != nil {
			orgConnState(conn, newState)
		}
	}

	// only used by unit tests
	if s.up != nil {
		// notify test that server is up; wait for signal to continue
		s.up <- listener
	}

	var err error
	if isUnixNetwork(s.Server.Addr) {
		os.Chmod(s.Server.Addr, os.ModePerm)
		err = fcgi.Serve(listener, s.Server.Handler)
	} else {
		err = s.Server.Serve(listener)
	}

	// This block is reached when the server has received a shut down command.
	if err == nil {
		s.wg.Wait()
		return nil
	} else if _, ok := err.(listenerAlreadyClosed); ok {
		s.wg.Wait()
		return nil
	}
	return err
}

// StartRoutine increments the server's WaitGroup. Use this if a web request starts more
// goroutines and these goroutines are not guaranteed to finish before the
// request.
func (s *GracefulServer) StartRoutine() {
	s.wg.Add(1)
}

// FinishRoutine decrements the server's WaitGroup. Used this to complement StartRoutine().
func (s *GracefulServer) FinishRoutine() {
	s.wg.Done()
}

var (
	servers []*GracefulServer
	m       sync.Mutex
)

// ListenAndServe provides a graceful version of function provided by the net/http package.
// This supports HTTP but not HTTPS or FCGI.
func ListenAndServe(addr string, handler http.Handler) error {
	server := NewWithServer(&http.Server{Addr: addr, Handler: handler})
	m.Lock()
	servers = append(servers, server)
	m.Unlock()
	return server.ListenAndServe()
}

// ListenAndServe provides a graceful version of function provided by the net/http/fcgi package.
// This supports FCGI but not HTTP/HTTPS.
func ListenAndServeFCGI(addr string, handler http.Handler) error {
	server := NewWithServer(&http.Server{Addr: addr, Handler: handler})
	m.Lock()
	servers = append(servers, server)
	m.Unlock()
	return server.ListenAndServe()
}

// ListenAndServeTLS provides a graceful version of function provided by the net/http package.
// This supports HTTPS but not HTTP or FCGI.
func ListenAndServeTLS(addr string, certFile string, keyFile string, handler http.Handler) error {
	server := NewWithServer(&http.Server{Addr: addr, Handler: handler})
	m.Lock()
	servers = append(servers, server)
	m.Unlock()
	return server.ListenAndServeTLS(certFile, keyFile)
}

// Serve provides a graceful version of function provided by the net/http package.
func Serve(l net.Listener, handler http.Handler) error {
	server := NewWithServer(&http.Server{Handler: handler})
	m.Lock()
	servers = append(servers, server)
	m.Unlock()
	return server.Serve(l)
}

// Close triggers a shutdown of all running Graceful servers.
func Close() {
	m.Lock()
	for _, s := range servers {
		s.Close()
	}
	servers = nil
	m.Unlock()
}

// CloseOnInterrupt creates a go-routine that will call the Close() function when certain OS
// signals are received. If no signals are specified,
// the following are used: SIGINT, SIGTERM, SIGKILL, SIGQUIT, SIGHUP, SIGUSR1.
// This function must be called before ListenAndServe.
func CloseOnInterrupt(signals ...os.Signal) {
	go func() {
		sigchan := make(chan os.Signal, 1)
		if len(signals) > 0 {
			signal.Notify(sigchan, signals)
		} else {
			signal.Notify(sigchan, syscall.SIGINT, syscall.SIGTERM, syscall.SIGKILL,
				syscall.SIGQUIT, syscall.SIGHUP, syscall.SIGUSR1)
		}
		<-sigchan
		Close()
	}()
}
<commit_msg>Correction.<commit_after>/*
Package manners provides a wrapper for a standard net/http server that
ensures all active HTTP client have completed their current request
before the server shuts down.

It can be used a drop-in replacement for the standard http package,
or can wrap a pre-configured Server.

eg.
	myHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
	  w.Write([]byte("Hello\n"))
	})

	http.Handle("/hello", myHandler)

	log.Fatal(manners.ListenAndServe(":8080", nil))

or for a customized server:
	s := manners.NewWithServer(&http.Server{
		Addr:           ":8080",
		Handler:        myHandler,
		ReadTimeout:    10 * time.Second,
		WriteTimeout:   10 * time.Second,
		MaxHeaderBytes: 1 << 20,
	})
	log.Fatal(s.ListenAndServe())

The server will shutdown cleanly when the Close() method is called:

	manners.CloseOnInterrupt()
	http.Handle("/hello", myHandler)
	log.Fatal(manners.ListenAndServe(":8080", nil))
*/
package manners

import (
	"crypto/tls"
	"net"
	"net/http"
	"net/http/fcgi"
	"os"
	"sync"
	"sync/atomic"
	"strings"
	"os/signal"
	"syscall"
)

// interface describing a waitgroup, so unit
// tests can mock out an instrumentable version
type waitgroup interface {
	Add(delta int)
	Done()
	Wait()
}

// StateHandler can be called by the server if the state of the connection changes.
// Notice that it passed previous state and the new state as parameters.
type StateHandler func(net.Conn, http.ConnState, http.ConnState)

type Options struct {
	Server       *http.Server
	StateHandler StateHandler
	Listener     net.Listener
}

// NewServer creates a new GracefulServer. The server will begin shutting down when
// a value is passed to the Shutdown channel.
func NewServer() *GracefulServer {
	return NewWithServer(new(http.Server))
}

// NewWithServer wraps an existing http.Server object and returns a GracefulServer
// that supports all of the original Server operations.
func NewWithServer(s *http.Server) *GracefulServer {
	return &GracefulServer{
		Server:   s,
		shutdown: make(chan struct{}),
		wg:       new(sync.WaitGroup),
	}
}

func NewWithOptions(o Options) *GracefulServer {
	// Set up listener
	var listener *GracefulListener
	if o.Listener != nil {
		g, ok := o.Listener.(*GracefulListener)
		if !ok {
			listener = NewListener(o.Listener)
		} else {
			listener = g
		}
	}

	return &GracefulServer{
		listener:     listener,
		Server:       o.Server,
		stateHandler: o.StateHandler,
		shutdown:     make(chan struct{}),
		wg:           new(sync.WaitGroup),
	}
}

// A GracefulServer maintains a WaitGroup that counts how many in-flight
// requests the server is handling. When it receives a shutdown signal,
// it stops accepting new requests but does not actually shut down until
// all in-flight requests terminate.
//
// GracefulServer embeds the underlying net/http.Server making its non-override
// methods and properties avaiable.
//
// It must be initialized by calling NewServer or NewWithServer
type GracefulServer struct {
	*http.Server
	shutdown chan struct{}
	wg       waitgroup
	listener *GracefulListener

	// used by test code
	up chan net.Listener

	stateHandler StateHandler
}

// Close stops the server from accepting new requets and beings shutting down.
func (s *GracefulServer) Close() {
	close(s.shutdown)
}

func isUnixNetwork(addr string) bool {
	return strings.HasPrefix(addr, "/")
}

func chooseNetwork(addr string) string {
	if isUnixNetwork(addr) {
		return "unix"
	}
	return "tcp"
}

// ListenAndServe provides a graceful equivalent of net/http.Serve.ListenAndServe, and
// also supports the FastCGI equivalent.
func (s *GracefulServer) ListenAndServe() error {
	if s.listener == nil {
		netwk := chooseNetwork(s.Addr)
		oldListener, err := net.Listen(netwk, s.Addr)
		if err != nil {
			return err
		}
		s.listener = NewListener(oldListener.(*net.TCPListener))
	}
	return s.Serve(s.listener)
}

// ListenAndServeTLS provides a graceful equivalent of net/http.Serve.ListenAndServeTLS.
func (s *GracefulServer) ListenAndServeTLS(certFile, keyFile string) error {
	addr := s.Addr
	if addr == "" {
		addr = ":https"
	}
	config := &tls.Config{}
	if s.TLSConfig != nil {
		*config = *s.TLSConfig
	}
	if config.NextProtos == nil {
		config.NextProtos = []string{"http/1.1"}
	}

	var err error
	config.Certificates = make([]tls.Certificate, 1)
	config.Certificates[0], err = tls.LoadX509KeyPair(certFile, keyFile)
	if err != nil {
		return err
	}

	return s.ListenAndServeTLSWithConfig(config)
}

// ListenAndServeTLS provides a graceful equivalent of net/http.Serve.ListenAndServeTLS.
func (s *GracefulServer) ListenAndServeTLSWithConfig(config *tls.Config) error {
	addr := s.Addr
	if addr == "" {
		addr = ":https"
	}

	if s.listener == nil {
		// only "tcp" is supported with TLS
		ln, err := net.Listen("tcp", addr)
		if err != nil {
			return err
		}

		tlsListener := NewTLSListener(TCPKeepAliveListener{ln.(*net.TCPListener)}, config)
		s.listener = NewListener(tlsListener)
	}
	return s.Serve(s.listener)
}

func (gs *GracefulServer) GetFile() (*os.File, error) {
	return gs.listener.GetFile()
}

func (gs *GracefulServer) HijackListener(s *http.Server, config *tls.Config) (*GracefulServer, error) {
	listener, err := gs.listener.Clone()
	if err != nil {
		return nil, err
	}

	if config != nil {
		listener = NewTLSListener(TCPKeepAliveListener{listener.(*net.TCPListener)}, config)
	}

	other := NewWithServer(s)
	other.listener = NewListener(listener)
	return other, nil
}

// Serve provides a graceful equivalent net/http.Server.Serve.
//
// If listener is not an instance of *GracefulListener is will be wrapped
// to become one.
func (s *GracefulServer) Serve(listener net.Listener) error {
	// accept a net.Listener to preserve the interface compatibility with the standard
	// http.Server, but we except a GracefluListener
	gracefulListener, ok := listener.(*GracefulListener)
	if !ok {
		gracefulListener = NewListener(listener)
		listener = gracefulListener
	}
	s.listener = gracefulListener

	var closing int32

	go func() {
		<-s.shutdown
		atomic.StoreInt32(&closing, 1)
		s.Server.SetKeepAlivesEnabled(false)
		listener.Close()
	}()

	orgConnState := s.Server.ConnState
	s.ConnState = func(conn net.Conn, newState http.ConnState) {
		// Ugly hack, but it works. We pass the information about the underlying state via the only available interface in net.Conn
		// we do this not to override the tls.Conn, as the internal logic of http.Server depends on the type assertion (unfortunately)
		gconn := conn.LocalAddr().(*gracefulAddr).gconn
		switch newState {
		case http.StateNew:
			// new_conn -> StateNew
			s.StartRoutine()

		case http.StateActive:
			// (StateNew, StateIdle) -> StateActive
			if gconn.lastHTTPState == http.StateIdle {
				// transitioned from idle back to active
				s.StartRoutine()
			}

		case http.StateIdle:
			// StateActive -> StateIdle
			if atomic.LoadInt32(&closing) == 1 {
				// rapidly close newly idle connections; if not they may make
				// one more request before SetKeepAliveEnabled(false)  takes effect.
				conn.Close()
			}
			s.FinishRoutine()

		case http.StateClosed, http.StateHijacked:
			// (StateNew, StateActive, StateIdle) -> (StateClosed, StateHiJacked)
			if gconn.lastHTTPState != http.StateIdle {
				// if it was idle it's already been decremented
				s.FinishRoutine()
			}
		}
		if s.stateHandler != nil {
			s.stateHandler(conn, gconn.lastHTTPState, newState)
		}
		gconn.lastHTTPState = newState
		if orgConnState != nil {
			orgConnState(conn, newState)
		}
	}

	// only used by unit tests
	if s.up != nil {
		// notify test that server is up; wait for signal to continue
		s.up <- listener
	}

	var err error
	if isUnixNetwork(s.Server.Addr) {
		os.Chmod(s.Server.Addr, os.ModePerm)
		err = fcgi.Serve(listener, s.Server.Handler)
	} else {
		err = s.Server.Serve(listener)
	}

	// This block is reached when the server has received a shut down command.
	if err == nil {
		s.wg.Wait()
		return nil
	} else if _, ok := err.(listenerAlreadyClosed); ok {
		s.wg.Wait()
		return nil
	}
	return err
}

// StartRoutine increments the server's WaitGroup. Use this if a web request starts more
// goroutines and these goroutines are not guaranteed to finish before the
// request.
func (s *GracefulServer) StartRoutine() {
	s.wg.Add(1)
}

// FinishRoutine decrements the server's WaitGroup. Used this to complement StartRoutine().
func (s *GracefulServer) FinishRoutine() {
	s.wg.Done()
}

var (
	servers []*GracefulServer
	m       sync.Mutex
)

// ListenAndServe provides a graceful version of function provided by the net/http package.
// This supports HTTP but not HTTPS or FCGI.
func ListenAndServe(addr string, handler http.Handler) error {
	server := NewWithServer(&http.Server{Addr: addr, Handler: handler})
	m.Lock()
	servers = append(servers, server)
	m.Unlock()
	return server.ListenAndServe()
}

// ListenAndServe provides a graceful version of function provided by the net/http/fcgi package.
// This supports FCGI but not HTTP/HTTPS.
func ListenAndServeFCGI(addr string, handler http.Handler) error {
	server := NewWithServer(&http.Server{Addr: addr, Handler: handler})
	m.Lock()
	servers = append(servers, server)
	m.Unlock()
	return server.ListenAndServe()
}

// ListenAndServeTLS provides a graceful version of function provided by the net/http package.
// This supports HTTPS but not HTTP or FCGI.
func ListenAndServeTLS(addr string, certFile string, keyFile string, handler http.Handler) error {
	server := NewWithServer(&http.Server{Addr: addr, Handler: handler})
	m.Lock()
	servers = append(servers, server)
	m.Unlock()
	return server.ListenAndServeTLS(certFile, keyFile)
}

// Serve provides a graceful version of function provided by the net/http package.
func Serve(l net.Listener, handler http.Handler) error {
	server := NewWithServer(&http.Server{Handler: handler})
	m.Lock()
	servers = append(servers, server)
	m.Unlock()
	return server.Serve(l)
}

// Close triggers a shutdown of all running Graceful servers.
func Close() {
	m.Lock()
	for _, s := range servers {
		s.Close()
	}
	servers = nil
	m.Unlock()
}

// CloseOnInterrupt creates a go-routine that will call the Close() function when certain OS
// signals are received. If no signals are specified,
// the following are used: SIGINT, SIGTERM, SIGKILL, SIGQUIT, SIGHUP, SIGUSR1.
// This function must be called before ListenAndServe.
func CloseOnInterrupt(signals ...os.Signal) {
	go func() {
		sigchan := make(chan os.Signal, 1)
		if len(signals) > 0 {
			signal.Notify(sigchan, signals...)
		} else {
			signal.Notify(sigchan, syscall.SIGINT, syscall.SIGTERM, syscall.SIGKILL,
				syscall.SIGQUIT, syscall.SIGHUP, syscall.SIGUSR1)
		}
		<-sigchan
		Close()
	}()
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"html/template"
	"math/rand"
	"net/http"
)

var (
	recipes  []Recipe
	database []Recipe
)

func contains(s []string, e string) bool {
	for _, a := range s {
		if a == e {
			return true
		}
	}
	return false
}

func getRecipe(filters []func(Recipe) bool) Recipe {

	var recipe Recipe

	// Apply filters
	var subDb []Recipe
	var doAdd bool
	for _, r := range database {
		doAdd = true
		for _, f := range filters {
			if !f(r) {
				doAdd = false
				break
			}
		}

		if doAdd == true {
			subDb = append(subDb, r)
		}
	}

	recipe = subDb[rand.Intn(len(subDb))]

	return recipe
}

func handler(w http.ResponseWriter, r *http.Request) {
	t, _ := template.ParseFiles("home.html")
	t.Execute(w, "foo")
}

func recipesHandler(w http.ResponseWriter, r *http.Request) {
	t, _ := template.ParseFiles("recipes.html")

	for i, r := range recipes {
		t.Execute(w, map[string]interface{}{"Recipe": r, "Index": i})
	}
}

func groceriesHandler(w http.ResponseWriter, r *http.Request) {
	t, _ := template.ParseFiles("groceries.html")

	ingredients := make(map[string]Ingredient)
	var names []string

	for _, r := range recipes {
		fmt.Println(r)
		for _, i := range r.Ingredients {
			fmt.Println(i)
			if !contains(names, i.Name) {
				names = append(names, i.Name)
				fmt.Println(names)
				ingredients[i.Name] = i
				fmt.Println(ingredients)
			} else {
				ingr := ingredients[i.Name]
				ingr.Quantity += i.Quantity
				ingredients[i.Name] = ingr
			}
		}
		fmt.Println("")
	}

	for _, i := range ingredients {
		t.Execute(w, i)
	}
}

func main() {
	database = readRecipes(recipeDirectory)

	var filters []func(Recipe) bool
	for i := 0; i < 5; i++ {
		recipe := getRecipe(filters)
		filters = append(filters, func(r Recipe) bool {
			return recipe.Title != r.Title
		})
		recipes = append(recipes, recipe)
	}

	http.HandleFunc("/", handler)
	http.HandleFunc("/recipes", recipesHandler)
	http.HandleFunc("/groceries", groceriesHandler)
	http.ListenAndServe(":8080", nil)
}
<commit_msg>Removed unnecessary prints<commit_after>package main

import (
	"html/template"
	"math/rand"
	"net/http"
)

var (
	recipes  []Recipe
	database []Recipe
)

func contains(s []string, e string) bool {
	for _, a := range s {
		if a == e {
			return true
		}
	}
	return false
}

func getRecipe(filters []func(Recipe) bool) Recipe {

	var recipe Recipe

	// Apply filters
	var subDb []Recipe
	var doAdd bool
	for _, r := range database {
		doAdd = true
		for _, f := range filters {
			if !f(r) {
				doAdd = false
				break
			}
		}

		if doAdd == true {
			subDb = append(subDb, r)
		}
	}

	recipe = subDb[rand.Intn(len(subDb))]

	return recipe
}

func handler(w http.ResponseWriter, r *http.Request) {
	t, _ := template.ParseFiles("home.html")
	t.Execute(w, "foo")
}

func recipesHandler(w http.ResponseWriter, r *http.Request) {
	t, _ := template.ParseFiles("recipes.html")

	for i, r := range recipes {
		t.Execute(w, map[string]interface{}{"Recipe": r, "Index": i})
	}
}

func groceriesHandler(w http.ResponseWriter, r *http.Request) {
	t, _ := template.ParseFiles("groceries.html")

	ingredients := make(map[string]Ingredient)
	var names []string

	for _, r := range recipes {
		for _, i := range r.Ingredients {
			if !contains(names, i.Name) {
				names = append(names, i.Name)
				ingredients[i.Name] = i
			} else {
				ingr := ingredients[i.Name]
				ingr.Quantity += i.Quantity
				ingredients[i.Name] = ingr
			}
		}
	}

	for _, i := range ingredients {
		t.Execute(w, i)
	}
}

func main() {
	database = readRecipes(recipeDirectory)

	var filters []func(Recipe) bool
	for i := 0; i < 5; i++ {
		recipe := getRecipe(filters)
		filters = append(filters, func(r Recipe) bool {
			return recipe.Title != r.Title
		})
		recipes = append(recipes, recipe)
	}

	http.HandleFunc("/", handler)
	http.HandleFunc("/recipes", recipesHandler)
	http.HandleFunc("/groceries", groceriesHandler)
	http.ListenAndServe(":8080", nil)
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/base64"
	"encoding/binary"
	"errors"
	"fmt"
	"log"
	"net"
	"time"

	"github.com/thethingsnetwork/croft/lora"
	"github.com/thethingsnetwork/server-shared"
)

func readUDPMessages(port int, messages chan interface{}) {
	addr, err := net.ResolveUDPAddr("udp", fmt.Sprintf("0.0.0.0:%d", port))
	if err != nil {
		log.Fatalf("Failed to resolve address: %s", err.Error())
	}

	listener, err := net.ListenUDP("udp", addr)
	if err != nil {
		log.Fatalf("Failed to start UDP server: %s", err.Error())
	}
	defer listener.Close()

	conn := lora.NewConn(listener)
	for {
		msg, err := conn.ReadMessage()
		if err != nil {
			log.Printf("Failed to read message: %s", err.Error())
			continue
		}
		go handleMessage(msg, messages)
	}
}

func handleMessage(msg *lora.Message, messages chan interface{}) {
	switch msg.Header.Identifier {
	case lora.PUSH_DATA:
		publishPushMessagePayloads(msg.GatewayEui, msg.Payload.(lora.PushMessagePayload), messages)
	}

	err := msg.Ack()
	if err != nil {
		log.Printf("Failed to send ACK: %s", err.Error())
	}
}

func publishPushMessagePayloads(gatewayEui []byte, payload lora.PushMessagePayload, messages chan interface{}) {
	if payload.Stat != nil {
		stat, err := convertStat(gatewayEui, payload.Stat)
		if err != nil {
			log.Printf("Failed to convert Stat: %s", err.Error())
		} else {
			messages <- stat
		}
	}

	if payload.RXPK != nil {
		for _, rxpk := range payload.RXPK {
			packet, err := convertRXPK(gatewayEui, rxpk)
			if err != nil {
				log.Printf("Failed to convert RXPK: %s", err.Error())
				continue
			}
			messages <- packet
		}
	}
}

func convertStat(gatewayEui []byte, stat *lora.Stat) (*shared.GatewayStatus, error) {
	t, err := time.Parse(time.RFC822, stat.Time)
	if err != nil {
		log.Printf("Failed to parse time %s: %s", stat.Time, err.Error())
		t = time.Now()
	}

	return &shared.GatewayStatus{
		Eui:               fmt.Sprintf("%X", gatewayEui),
		Time:              t,
		Latitude:          &stat.Lati,
		Longitude:         &stat.Long,
		Altitude:          &stat.Alti,
		RxCount:           &stat.Rxnb,
		RxOk:              &stat.Rxok,
		RxForwarded:       &stat.Rxfw,
		AckRatio:          &stat.Ackr,
		DatagramsReceived: &stat.Dwnb,
		DatagramsSent:     &stat.Txnb,
	}, nil
}

func convertRXPK(gatewayEui []byte, rxpk *lora.RXPK) (*shared.RxPacket, error) {
	data, err := rxpk.ParseData()
	if err != nil {
		log.Printf("Failed to parse RXPK: %s", err.Error())
		return nil, err
	}

	networkKey, err := getNetworkKey(gatewayEui, data.DevAddr)
	if err != nil {
		log.Printf("Failed to get key (gateway %X, node %X): %s", gatewayEui, data.DevAddr, err.Error())
		return nil, err
	}

	var key []byte
	if data.FPort == 0 {
		key = networkKey
	} else {
		key, err = getAppKey(gatewayEui, data.DevAddr)
		if err != nil {
			log.Printf("Failed to get key (gateway %X, node %X): %s", gatewayEui, data.DevAddr, err.Error())
			return nil, err
		}
	}

	ok, err := data.TestIntegrity(networkKey)
	if err != nil {
		log.Printf("Failed to test integrity: %s", err.Error())
		return nil, err
	}
	if !ok {
		return nil, errors.New("Integrity test failed")
	}

	payload, err := data.DecryptPayload(key)
	if err != nil {
		log.Printf("Error decrypting data: %s", err.Error())
		return nil, err
	}

	return &shared.RxPacket{
		GatewayEui: fmt.Sprintf("%X", gatewayEui),
		NodeEui:    fmtDevAddr(data.DevAddr),
		Time:       rxpk.Time,
		Frequency:  &rxpk.Freq,
		DataRate:   rxpk.Datr,
		Rssi:       &rxpk.Rssi,
		Snr:        &rxpk.Lsnr,
		RawData:    rxpk.Data,
		Data:       base64.StdEncoding.EncodeToString(payload),
	}, nil
}

func fmtDevAddr(devAddr uint32) string {
	buf := new(bytes.Buffer)
	binary.Write(buf, binary.BigEndian, devAddr)
	return fmt.Sprintf("%X", buf.Bytes())
}

func getNetworkKey(gatewayEui []byte, devAddr uint32) ([]byte, error) {
	// TODO: Implement fetching the right network key. Now returning Semtech's default key
	key := []byte{0x2B, 0x7E, 0x15, 0x16, 0x28, 0xAE, 0xD2, 0xA6, 0xAB, 0xF7, 0x15, 0x88, 0x09, 0xCF, 0x4F, 0x3C}
	return key, nil
}

func getAppKey(gatewayEui []byte, devAddr uint32) ([]byte, error) {
	// TODO: Implement fetching the right application key
	return getNetworkKey(gatewayEui, devAddr)
}
<commit_msg>Back to server packet time<commit_after>package main

import (
	"bytes"
	"encoding/base64"
	"encoding/binary"
	"errors"
	"fmt"
	"log"
	"net"
	"time"

	"github.com/thethingsnetwork/croft/lora"
	"github.com/thethingsnetwork/server-shared"
)

func readUDPMessages(port int, messages chan interface{}) {
	addr, err := net.ResolveUDPAddr("udp", fmt.Sprintf("0.0.0.0:%d", port))
	if err != nil {
		log.Fatalf("Failed to resolve address: %s", err.Error())
	}

	listener, err := net.ListenUDP("udp", addr)
	if err != nil {
		log.Fatalf("Failed to start UDP server: %s", err.Error())
	}
	defer listener.Close()

	conn := lora.NewConn(listener)
	for {
		msg, err := conn.ReadMessage()
		if err != nil {
			log.Printf("Failed to read message: %s", err.Error())
			continue
		}
		go handleMessage(msg, messages)
	}
}

func handleMessage(msg *lora.Message, messages chan interface{}) {
	switch msg.Header.Identifier {
	case lora.PUSH_DATA:
		publishPushMessagePayloads(msg.GatewayEui, msg.Payload.(lora.PushMessagePayload), messages)
	}

	err := msg.Ack()
	if err != nil {
		log.Printf("Failed to send ACK: %s", err.Error())
	}
}

func publishPushMessagePayloads(gatewayEui []byte, payload lora.PushMessagePayload, messages chan interface{}) {
	if payload.Stat != nil {
		stat, err := convertStat(gatewayEui, payload.Stat)
		if err != nil {
			log.Printf("Failed to convert Stat: %s", err.Error())
		} else {
			messages <- stat
		}
	}

	if payload.RXPK != nil {
		for _, rxpk := range payload.RXPK {
			packet, err := convertRXPK(gatewayEui, rxpk)
			if err != nil {
				log.Printf("Failed to convert RXPK: %s", err.Error())
				continue
			}
			messages <- packet
		}
	}
}

func convertStat(gatewayEui []byte, stat *lora.Stat) (*shared.GatewayStatus, error) {
	t, err := time.Parse(time.RFC822, stat.Time)
	if err != nil {
		log.Printf("Failed to parse time %s: %s", stat.Time, err.Error())
		t = time.Now()
	}

	return &shared.GatewayStatus{
		Eui:               fmt.Sprintf("%X", gatewayEui),
		Time:              t,
		Latitude:          &stat.Lati,
		Longitude:         &stat.Long,
		Altitude:          &stat.Alti,
		RxCount:           &stat.Rxnb,
		RxOk:              &stat.Rxok,
		RxForwarded:       &stat.Rxfw,
		AckRatio:          &stat.Ackr,
		DatagramsReceived: &stat.Dwnb,
		DatagramsSent:     &stat.Txnb,
	}, nil
}

func convertRXPK(gatewayEui []byte, rxpk *lora.RXPK) (*shared.RxPacket, error) {
	data, err := rxpk.ParseData()
	if err != nil {
		log.Printf("Failed to parse RXPK: %s", err.Error())
		return nil, err
	}

	networkKey, err := getNetworkKey(gatewayEui, data.DevAddr)
	if err != nil {
		log.Printf("Failed to get key (gateway %X, node %X): %s", gatewayEui, data.DevAddr, err.Error())
		return nil, err
	}

	var key []byte
	if data.FPort == 0 {
		key = networkKey
	} else {
		key, err = getAppKey(gatewayEui, data.DevAddr)
		if err != nil {
			log.Printf("Failed to get key (gateway %X, node %X): %s", gatewayEui, data.DevAddr, err.Error())
			return nil, err
		}
	}

	ok, err := data.TestIntegrity(networkKey)
	if err != nil {
		log.Printf("Failed to test integrity: %s", err.Error())
		return nil, err
	}
	if !ok {
		return nil, errors.New("Integrity test failed")
	}

	payload, err := data.DecryptPayload(key)
	if err != nil {
		log.Printf("Error decrypting data: %s", err.Error())
		return nil, err
	}

	return &shared.RxPacket{
		GatewayEui: fmt.Sprintf("%X", gatewayEui),
		NodeEui:    fmtDevAddr(data.DevAddr),
		Time:       time.Now(), // rxpk.Time,
		Frequency:  &rxpk.Freq,
		DataRate:   rxpk.Datr,
		Rssi:       &rxpk.Rssi,
		Snr:        &rxpk.Lsnr,
		RawData:    rxpk.Data,
		Data:       base64.StdEncoding.EncodeToString(payload),
	}, nil
}

func fmtDevAddr(devAddr uint32) string {
	buf := new(bytes.Buffer)
	binary.Write(buf, binary.BigEndian, devAddr)
	return fmt.Sprintf("%X", buf.Bytes())
}

func getNetworkKey(gatewayEui []byte, devAddr uint32) ([]byte, error) {
	// TODO: Implement fetching the right network key. Now returning Semtech's default key
	key := []byte{0x2B, 0x7E, 0x15, 0x16, 0x28, 0xAE, 0xD2, 0xA6, 0xAB, 0xF7, 0x15, 0x88, 0x09, 0xCF, 0x4F, 0x3C}
	return key, nil
}

func getAppKey(gatewayEui []byte, devAddr uint32) ([]byte, error) {
	// TODO: Implement fetching the right application key
	return getNetworkKey(gatewayEui, devAddr)
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/xml"
	"fmt"
	"io/ioutil"
	"math"
	"net/http"
	"sort"
	"strconv"
)

type Server struct {
	Url      string `xml:"url,attr"`
	Lat      string `xml:"lat,attr"`
	Lon      string `xml:"lon,attr"`
	Name     string `xml:"name,attr"`
	Country  string `xml:"country,attr"`
	Sponsor  string `xml:"sponsor,attr"`
	Id       string `xml:"id,attr"`
	Url2     string `xml:"url2,attr"`
	Host     string `xml:"host,attr"`
	Distance float64
	DLSpeed  float64
	ULSpeed  float64
}

type List struct {
	Servers []Server `xml:"servers>server"`
}

// For sort =start=
type Servers []Server

type ByDistance struct {
	Servers
}

func (s Servers) Len() int {
	return len(s)
}

func (s Servers) Swap(i, j int) {
	s[i], s[j] = s[j], s[i]
}

func (b ByDistance) Less(i, j int) bool {
	return b.Servers[i].Distance < b.Servers[j].Distance
}

// For sort =end=

func FetchServerList(user User) List {
	// Fetch xml server data
	resp, err := http.Get("http://www.speedtest.net/speedtest-servers-static.php")
	CheckError(err)
	body, err := ioutil.ReadAll(resp.Body)
	CheckError(err)
	defer resp.Body.Close()

	// Decode xml
	decoder := xml.NewDecoder(bytes.NewReader(body))
	list := List{}
	for {
		t, _ := decoder.Token()
		if t == nil {
			break
		}
		switch se := t.(type) {
		case xml.StartElement:
			decoder.DecodeElement(&list, &se)
		}
	}

	// Calculate distance
	for i := range list.Servers {
		server := &list.Servers[i]
		sLat, _ := strconv.ParseFloat(server.Lat, 64)
		sLon, _ := strconv.ParseFloat(server.Lon, 64)
		uLat, _ := strconv.ParseFloat(user.Lat, 64)
		uLon, _ := strconv.ParseFloat(user.Lon, 64)
		server.Distance = Distance(sLat, sLon, uLat, uLon)
	}

	// Sort by distance
	sort.Sort(ByDistance{list.Servers})

	return list
}

func Distance(lat1 float64, lon1 float64, lat2 float64, lon2 float64) float64 {
	radius := 6378.137

	a1 := lat1 * math.Pi / 180.0
	b1 := lon1 * math.Pi / 180.0
	a2 := lat2 * math.Pi / 180.0
	b2 := lon2 * math.Pi / 180.0

	x := math.Sin(a1)*math.Sin(a2) + math.Cos(a1)*math.Cos(a2)*math.Cos(b2-b1)
	return radius * math.Acos(x)
}

func (l *List) FindServer(serverId []int) Servers {
	servers := Servers{}

	for _, sid := range serverId {
		for _, s := range l.Servers {
			id, _ := strconv.Atoi(s.Id)
			if sid == id {
				servers = append(servers, s)
			}
		}
	}

	if len(servers) == 0 {
		servers = append(servers, l.Servers[0])
	}

	return servers
}

func (l List) Show() {
	for _, s := range l.Servers {
		fmt.Printf("[%4s] %8.2fkm ", s.Id, s.Distance)
		fmt.Printf(s.Name + " (" + s.Country + ") by " + s.Sponsor + "\n")
	}
}

func (s Server) Show() {
	fmt.Printf(" \n")
	fmt.Printf("Target Server: [%4s] %8.2fkm ", s.Id, s.Distance)
	fmt.Printf(s.Name + " (" + s.Country + ") by " + s.Sponsor + "\n")
}

func (svrs Servers) StartTest() {
	for i, s := range svrs {
		s.Show()
		latency := PingTest(s.Url)
		dlSpeed := DownloadTest(s.Url, latency)
		ulSpeed := UploadTest(s.Url, latency)
		svrs[i].DLSpeed = dlSpeed
		svrs[i].ULSpeed = ulSpeed
	}
}

func (svrs Servers) ShowResult() {
	fmt.Printf(" \n")
	if len(svrs) == 1 {
		fmt.Printf("Download: %5.2f Mbit/s\n", svrs[0].DLSpeed)
		fmt.Printf("Upload: %5.2f Mbit/s\n", svrs[0].ULSpeed)
	} else {
		for _, s := range svrs {
			fmt.Printf("[%4s] Download: %5.2f Mbit/s, Upload: %5.2f Mbit/s\n", s.Id, s.DLSpeed, s.ULSpeed)
		}
		avgDL := 0.0
		avgUL := 0.0
		for _, s := range svrs {
			avgDL = avgDL + s.DLSpeed
			avgUL = avgUL + s.ULSpeed
		}
		fmt.Printf("Download Avg: %5.2f Mbit/s\n", avgDL/float64(len(svrs)))
		fmt.Printf("Upload Avg: %5.2f Mbit/s\n", avgUL/float64(len(svrs)))
	}
}
<commit_msg>[fix] add fail-safe<commit_after>package main

import (
	"bytes"
	"encoding/xml"
	"fmt"
	"io/ioutil"
	"math"
	"net/http"
	"sort"
	"strconv"
)

type Server struct {
	Url      string `xml:"url,attr"`
	Lat      string `xml:"lat,attr"`
	Lon      string `xml:"lon,attr"`
	Name     string `xml:"name,attr"`
	Country  string `xml:"country,attr"`
	Sponsor  string `xml:"sponsor,attr"`
	Id       string `xml:"id,attr"`
	Url2     string `xml:"url2,attr"`
	Host     string `xml:"host,attr"`
	Distance float64
	DLSpeed  float64
	ULSpeed  float64
}

type List struct {
	Servers []Server `xml:"servers>server"`
}

// For sort =start=
type Servers []Server

type ByDistance struct {
	Servers
}

func (s Servers) Len() int {
	return len(s)
}

func (s Servers) Swap(i, j int) {
	s[i], s[j] = s[j], s[i]
}

func (b ByDistance) Less(i, j int) bool {
	return b.Servers[i].Distance < b.Servers[j].Distance
}

// For sort =end=

func FetchServerList(user User) List {
	// Fetch xml server data
	resp, err := http.Get("http://www.speedtest.net/speedtest-servers-static.php")
	CheckError(err)
	body, err := ioutil.ReadAll(resp.Body)
	CheckError(err)
	defer resp.Body.Close()

	if len(body) == 0 {
		resp, err = http.Get("http://c.speedtest.net/speedtest-servers-static.php")
		CheckError(err)
		body, err = ioutil.ReadAll(resp.Body)
		CheckError(err)
		defer resp.Body.Close()
	}

	// Decode xml
	decoder := xml.NewDecoder(bytes.NewReader(body))
	list := List{}
	for {
		t, _ := decoder.Token()
		if t == nil {
			break
		}
		switch se := t.(type) {
		case xml.StartElement:
			decoder.DecodeElement(&list, &se)
		}
	}

	// Calculate distance
	for i := range list.Servers {
		server := &list.Servers[i]
		sLat, _ := strconv.ParseFloat(server.Lat, 64)
		sLon, _ := strconv.ParseFloat(server.Lon, 64)
		uLat, _ := strconv.ParseFloat(user.Lat, 64)
		uLon, _ := strconv.ParseFloat(user.Lon, 64)
		server.Distance = Distance(sLat, sLon, uLat, uLon)
	}

	// Sort by distance
	sort.Sort(ByDistance{list.Servers})

	return list
}

func Distance(lat1 float64, lon1 float64, lat2 float64, lon2 float64) float64 {
	radius := 6378.137

	a1 := lat1 * math.Pi / 180.0
	b1 := lon1 * math.Pi / 180.0
	a2 := lat2 * math.Pi / 180.0
	b2 := lon2 * math.Pi / 180.0

	x := math.Sin(a1)*math.Sin(a2) + math.Cos(a1)*math.Cos(a2)*math.Cos(b2-b1)
	return radius * math.Acos(x)
}

func (l *List) FindServer(serverId []int) Servers {
	servers := Servers{}

	for _, sid := range serverId {
		for _, s := range l.Servers {
			id, _ := strconv.Atoi(s.Id)
			if sid == id {
				servers = append(servers, s)
			}
		}
	}

	if len(servers) == 0 {
		servers = append(servers, l.Servers[0])
	}

	return servers
}

func (l List) Show() {
	for _, s := range l.Servers {
		fmt.Printf("[%4s] %8.2fkm ", s.Id, s.Distance)
		fmt.Printf(s.Name + " (" + s.Country + ") by " + s.Sponsor + "\n")
	}
}

func (s Server) Show() {
	fmt.Printf(" \n")
	fmt.Printf("Target Server: [%4s] %8.2fkm ", s.Id, s.Distance)
	fmt.Printf(s.Name + " (" + s.Country + ") by " + s.Sponsor + "\n")
}

func (svrs Servers) StartTest() {
	for i, s := range svrs {
		s.Show()
		latency := PingTest(s.Url)
		dlSpeed := DownloadTest(s.Url, latency)
		ulSpeed := UploadTest(s.Url, latency)
		svrs[i].DLSpeed = dlSpeed
		svrs[i].ULSpeed = ulSpeed
	}
}

func (svrs Servers) ShowResult() {
	fmt.Printf(" \n")
	if len(svrs) == 1 {
		fmt.Printf("Download: %5.2f Mbit/s\n", svrs[0].DLSpeed)
		fmt.Printf("Upload: %5.2f Mbit/s\n", svrs[0].ULSpeed)
	} else {
		for _, s := range svrs {
			fmt.Printf("[%4s] Download: %5.2f Mbit/s, Upload: %5.2f Mbit/s\n", s.Id, s.DLSpeed, s.ULSpeed)
		}
		avgDL := 0.0
		avgUL := 0.0
		for _, s := range svrs {
			avgDL = avgDL + s.DLSpeed
			avgUL = avgUL + s.ULSpeed
		}
		fmt.Printf("Download Avg: %5.2f Mbit/s\n", avgDL/float64(len(svrs)))
		fmt.Printf("Upload Avg: %5.2f Mbit/s\n", avgUL/float64(len(svrs)))
	}
}
<|endoftext|>


<commit_before>package main

import (
	"log"
	"net/http"
)

func main() {
	fs := http.FileServer(http.Dir("public/"))
	http.Handle("/", fs)

	log.Println("Listening...")
	http.ListenAndServe(":4747", nil)
}
<commit_msg>add Getenv to work in heroku<commit_after>package main

import (
	"log"
	"net/http"
	"os"
)

func main() {
	var home = os.Getenv("HOME")

	fs := http.FileServer(http.Dir(home + "/public/"))
	http.Handle("/", fs)

	log.Println("Listening...")
	http.ListenAndServe(":4747", nil)
}
<|endoftext|>


<commit_before>package main

import (
	"database/sql"
	"encoding/json"
	"flag"
	"fmt"
	"github.com/lib/pq"
	"io/ioutil"
	"log"
	"net/http"
	"time"
)

const OPEN311_API_URI = "http://311api.cityofchicago.org/open311/v2/requests.json?extensions=true&page_size=500"

type Open311Request struct {
	Lat, Long                                                                                float64
	Ward, Police_district                                                                                int
	Service_request_id, Status, Service_name, Service_code, Agency_responsible, Address, Channel, Media_url string
	Requested_datetime, Updated_datetime                                                                    string // FIXME: should these be proper time objects?
	Extended_attributes                                                                                map[string]interface{}
	Notes                                                                                []map[string]interface{}
}

type Worker struct {
	Db        *sql.DB
	LastRunAt time.Time
}

var worker Worker
var sr_number string
var backfill bool
var backfill_date string

func init() {
	// open database
	db, err := sql.Open("postgres", "dbname=cwfy sslmode=disable")
	if err != nil {
		log.Fatal("Cannot open database connection", err)
	}
	worker.Db = db

	// fetch SR num from command line, if present
	flag.StringVar(&sr_number, "sr-number", "", "SR number to fetch")
	flag.BoolVar(&backfill, "backfill", false, "run in reverse and backfill data")
	flag.StringVar(&backfill_date, "backfill-from", "", "date to start backfilling data from. Use RFC3339 format. Default will be the time of the least recently updated SR in the database.")
}

func main() {
	defer worker.Db.Close()
	flag.Parse()

	if sr_number != "" {
		sr := fetchSingleRequest(sr_number)
		sr.Save()
		return
	}

	start_backfill_from := backfill_date
	for {
		switch {
		case backfill:
			requests := backFillRequests(start_backfill_from)
			for _, request := range requests {
				request.Save()
			}

			start_backfill_from = requests[len(requests)-1].Updated_datetime // FIXME: is it safe to assume the items are sorted?

		case time.Since(worker.LastRunAt) > (30 * time.Second):
			// load requests from open311
			for _, request := range fetchRequests() {
				request.Save()
			}
			worker.LastRunAt = time.Now()
		default:
			log.Print("sleeping for 10 seconds")
			time.Sleep(10 * time.Second)
		}
	}
}

func (req Open311Request) String() string {
	// pretty print SR information
	return fmt.Sprintf("%s: %s at %s %f,%f, last update %s", req.Service_request_id, req.Service_name, req.Address, req.Lat, req.Long, req.Updated_datetime)
}

func (req Open311Request) Save() (persisted bool) {
	// create or update a SR

	// open311 says we should always ignore a SR that does not have a SR# assigned
	if req.Service_request_id == "" {
		log.Printf("cowardly refusing to create a new SR record because of empty SR#. Request type is %s", req.Service_name)
		return false
	}

	persisted = false

	// find existing record if exists
	var existing_id int
	err := worker.Db.QueryRow("SELECT id FROM service_requests WHERE service_request_id = $1", req.Service_request_id).Scan(&existing_id)
	switch {
	case err == sql.ErrNoRows:
		// log.Printf("did not find existing record %s", req.Service_request_id)
	case err != nil:
		// log.Print("error searching for existing SR", err)
	default:
		persisted = true
		// log.Printf("found existing sr %s", req.Service_request_id)
	}

	var stmt *sql.Stmt

	if !persisted {
		// create new record
		stmt, err = worker.Db.Prepare("INSERT INTO service_requests(service_request_id," +
			"status, service_name, service_code, agency_responsible, " +
			"address, requested_datetime, updated_datetime, lat, long," +
			"ward, police_district, media_url, channel, duplicate, parent_service_request_id, closed_datetime, notes) " +
			"VALUES ($1::varchar, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18); ")

		// "WHERE NOT EXISTS (SELECT 1 FROM service_requests WHERE service_request_id = $1);")

		if err != nil {
			log.Fatal("error preparing database insert statement", err)
		}

	} else {
		// update existing record
		stmt, err = worker.Db.Prepare("UPDATE service_requests SET " +
			"status = $2, service_name = $3, service_code = $4, agency_responsible = $5, " +
			"address = $6, requested_datetime = $7, updated_datetime = $8, lat = $9, long = $10," +
			"ward = $11, police_district = $12, media_url = $13, channel = $14, duplicate = $15, " +
			"parent_service_request_id = $16, updated_at = NOW(), closed_datetime = $17, notes = $18 WHERE service_request_id = $1;")

		if err != nil {
			log.Fatal("error preparing database update statement", err)
		}
	}

	tx, err := worker.Db.Begin()

	if err != nil {
		log.Fatal("error beginning transaction", err)
	}

	t := req.ExtractClosedDatetime()
	closed_time := pq.NullTime{Time: t, Valid: !t.IsZero()}
	notes_as_json, err := json.Marshal(req.Notes)
	if err != nil {
		log.Print("error marshaling notes to JSON: ", err)
	}
	_, err = tx.Stmt(stmt).Exec(req.Service_request_id,
		req.Status,
		req.Service_name,
		req.Service_code,
		req.Agency_responsible,
		req.Address,
		req.Requested_datetime,
		req.Updated_datetime,
		req.Lat,
		req.Long,
		req.Extended_attributes["ward"],
		req.Extended_attributes["police_district"],
		req.Media_url,
		req.Extended_attributes["channel"],
		req.Extended_attributes["duplicate"],
		req.Extended_attributes["parent_service_request_id"],
		closed_time,
		notes_as_json,
	)

	if err != nil {
		log.Fatalf("could not update %s because %s", req.Service_request_id, err)
	} else {
		var verb string
		switch {
		case !persisted && closed_time.Time.IsZero():
			verb = "CREATED"
		case !persisted && !closed_time.Time.IsZero():
			verb = "CREATED/CLOSED"
		case persisted && closed_time.Time.IsZero():
			verb = "UPDATED"
		case persisted && !closed_time.Time.IsZero():
			verb = "UPDATED/CLOSED"
		}

		log.Printf("[%s] %s", verb, req)
		persisted = true
	}

	err = tx.Commit()
	if err != nil {
		log.Fatal("error closing transaction", err)
	}

	return persisted

	// calculate closed time if necessary

}

func (req Open311Request) ExtractClosedDatetime() time.Time {
	// given an extended_attributes JSON blob, pluck out the closed time, if present
	// req.PrintNotes()

	var closed_at time.Time
	for _, note := range req.Notes {
		if note["type"] == "closed" {
			parsed_date, err := time.Parse("2006-01-02T15:04:05-07:00", note["datetime"].(string))
			if err != nil {
				log.Print("error parsing date", err)
			}
			log.Printf("SR %s closed at: %s", req, parsed_date)
			closed_at = parsed_date
		}
	}

	return closed_at
}

func (req Open311Request) PrintNotes() {
	fmt.Printf("Notes for SR %s:\n", req.Service_request_id)

	for _, note := range req.Notes {
		fmt.Printf("%+v\n", note)
	}
}

func fetchSingleRequest(sr_number string) (request Open311Request) {
	// given an SR, fetch the record
	log.Printf("fetching single SR %s", sr_number)
	open311_api_endpoint := fmt.Sprintf("http://311api.cityofchicago.org/open311/v2/requests/%s.json?extensions=true", sr_number)

	log.Printf("fetching from %s", open311_api_endpoint)
	resp, err := http.Get(open311_api_endpoint)
	defer resp.Body.Close()

	if err != nil {
		log.Fatalln("error fetching from Open311 endpoint", err)
	}

	// load response body
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Fatal("error loading response body", err)
	}

	// parse JSON and load into an array of Open311Request objects
	var requests []Open311Request

	err = json.Unmarshal(body, &requests)
	if err != nil {
		log.Fatal("error parsing JSON:", err)
	}

	log.Printf("received %d requests from Open311", len(requests))

	return requests[0]
}

func fetchRequests() (requests []Open311Request) {
	last_updated_at := time.Now()
	if err := worker.Db.QueryRow("SELECT MAX(updated_datetime) FROM service_requests;").Scan(&last_updated_at); err != nil {
		log.Print("[fetchRequests] error loading most recent SR, will fallback to current time: ", err)
	}

	log.Print("[fetchRequests] most recent SR timestamp ", last_updated_at.Format(time.RFC3339))

	// construct the request URI using base params and the proper time
	open311_api_endpoint := OPEN311_API_URI + "&updated_after=" + last_updated_at.Format(time.RFC3339)

	log.Printf("[fetchRequests] fetching from %s", open311_api_endpoint)
	resp, err := http.Get(open311_api_endpoint)
	defer resp.Body.Close()

	if err != nil {
		log.Fatalln("[fetchRequests] error fetching from Open311 endpoint", err)
	}

	// load response body
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Fatal("[fetchRequests] error loading response body", err)
	}

	// parse JSON and load into an array of Open311Request objects
	err = json.Unmarshal(body, &requests)
	if err != nil {
		log.Fatal("[fetchRequests] error parsing JSON:", err)
	}

	log.Printf("[fetchRequests] received %d requests from Open311", len(requests))

	return requests
}

func backFillRequests(start_from string) (requests []Open311Request) {
	var fetch_from time.Time

	if start_from == "" {
		err := worker.Db.QueryRow("SELECT updated_datetime FROM service_requests ORDER BY updated_datetime ASC LIMIT 1").Scan(&fetch_from)
		if err != nil {
			log.Println("error fetching oldest SR:", err)
		}
		log.Printf("no start_from value provided, so falling back to oldest (by last update) SR in the database: %s", fetch_from)
	} else {
		t, err := time.Parse(time.RFC3339, start_from)
		if err != nil {
			log.Fatal("[backfill] error parsing date to start from", err)
		}
		fetch_from = t
	}

	formatted_date_string_with_tz := fetch_from.Format(time.RFC3339)

	// construct the request URI using base params and the proper time
	open311_api_endpoint := OPEN311_API_URI + "&updated_before=" + formatted_date_string_with_tz

	log.Printf("[backfill] fetching from %s", open311_api_endpoint)
	resp, err := http.Get(open311_api_endpoint)
	defer resp.Body.Close()

	if err != nil {
		log.Fatalln("[backfill] error fetching from Open311 endpoint", err)
	}

	// load response body
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Fatal("[backfill] error loading response body", err)
	}

	// parse JSON and load into an array of Open311Request objects
	err = json.Unmarshal(body, &requests)
	if err != nil {
		log.Fatal("[backfill] error parsing JSON:", err)
	}

	log.Printf("[backfill] received %d requests from Open311", len(requests))

	return requests
}
<commit_msg>comment out transaction code<commit_after>package main

import (
	"database/sql"
	"encoding/json"
	"flag"
	"fmt"
	"github.com/lib/pq"
	"io/ioutil"
	"log"
	"net/http"
	"time"
)

const OPEN311_API_URI = "http://311api.cityofchicago.org/open311/v2/requests.json?extensions=true&page_size=500"

type Open311Request struct {
	Lat, Long                                                                                float64
	Ward, Police_district                                                                                int
	Service_request_id, Status, Service_name, Service_code, Agency_responsible, Address, Channel, Media_url string
	Requested_datetime, Updated_datetime                                                                    string // FIXME: should these be proper time objects?
	Extended_attributes                                                                                map[string]interface{}
	Notes                                                                                []map[string]interface{}
}

type Worker struct {
	Db        *sql.DB
	LastRunAt time.Time
}

var worker Worker
var sr_number string
var backfill bool
var backfill_date string

func init() {
	// open database
	db, err := sql.Open("postgres", "dbname=cwfy sslmode=disable")
	if err != nil {
		log.Fatal("Cannot open database connection", err)
	}
	worker.Db = db

	// fetch SR num from command line, if present
	flag.StringVar(&sr_number, "sr-number", "", "SR number to fetch")
	flag.BoolVar(&backfill, "backfill", false, "run in reverse and backfill data")
	flag.StringVar(&backfill_date, "backfill-from", "", "date to start backfilling data from. Use RFC3339 format. Default will be the time of the least recently updated SR in the database.")
}

func main() {
	defer worker.Db.Close()
	flag.Parse()

	if sr_number != "" {
		sr := fetchSingleRequest(sr_number)
		sr.Save()
		return
	}

	start_backfill_from := backfill_date
	for {
		switch {
		case backfill:
			requests := backFillRequests(start_backfill_from)
			for _, request := range requests {
				request.Save()
			}

			start_backfill_from = requests[len(requests)-1].Updated_datetime // FIXME: is it safe to assume the items are sorted?

		case time.Since(worker.LastRunAt) > (30 * time.Second):
			// load requests from open311
			for _, request := range fetchRequests() {
				request.Save()
			}
			worker.LastRunAt = time.Now()
		default:
			log.Print("sleeping for 10 seconds")
			time.Sleep(10 * time.Second)
		}
	}
}

func (req Open311Request) String() string {
	// pretty print SR information
	return fmt.Sprintf("%s: %s at %s %f,%f, last update %s", req.Service_request_id, req.Service_name, req.Address, req.Lat, req.Long, req.Updated_datetime)
}

func (req Open311Request) Save() (persisted bool) {
	// create or update a SR

	// open311 says we should always ignore a SR that does not have a SR# assigned
	if req.Service_request_id == "" {
		log.Printf("cowardly refusing to create a new SR record because of empty SR#. Request type is %s", req.Service_name)
		return false
	}

	persisted = false

	// find existing record if exists
	var existing_id int
	err := worker.Db.QueryRow("SELECT id FROM service_requests WHERE service_request_id = $1", req.Service_request_id).Scan(&existing_id)
	switch {
	case err == sql.ErrNoRows:
		// log.Printf("did not find existing record %s", req.Service_request_id)
	case err != nil:
		// log.Print("error searching for existing SR", err)
	default:
		persisted = true
		// log.Printf("found existing sr %s", req.Service_request_id)
	}

	var stmt *sql.Stmt

	if !persisted {
		// create new record
		stmt, err = worker.Db.Prepare("INSERT INTO service_requests(service_request_id," +
			"status, service_name, service_code, agency_responsible, " +
			"address, requested_datetime, updated_datetime, lat, long," +
			"ward, police_district, media_url, channel, duplicate, parent_service_request_id, closed_datetime, notes) " +
			"VALUES ($1::varchar, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18); ")

		// "WHERE NOT EXISTS (SELECT 1 FROM service_requests WHERE service_request_id = $1);")

		if err != nil {
			log.Fatal("error preparing database insert statement", err)
		}

	} else {
		// update existing record
		stmt, err = worker.Db.Prepare("UPDATE service_requests SET " +
			"status = $2, service_name = $3, service_code = $4, agency_responsible = $5, " +
			"address = $6, requested_datetime = $7, updated_datetime = $8, lat = $9, long = $10," +
			"ward = $11, police_district = $12, media_url = $13, channel = $14, duplicate = $15, " +
			"parent_service_request_id = $16, updated_at = NOW(), closed_datetime = $17, notes = $18 WHERE service_request_id = $1;")

		if err != nil {
			log.Fatal("error preparing database update statement", err)
		}
	}

	// tx, err := worker.Db.Begin()
	//
	// if err != nil {
	// 	log.Fatal("error beginning transaction", err)
	// }

	t := req.ExtractClosedDatetime()
	closed_time := pq.NullTime{Time: t, Valid: !t.IsZero()}
	notes_as_json, err := json.Marshal(req.Notes)
	if err != nil {
		log.Print("error marshaling notes to JSON: ", err)
	}

	_, err = stmt.Exec(req.Service_request_id,
		req.Status,
		req.Service_name,
		req.Service_code,
		req.Agency_responsible,
		req.Address,
		req.Requested_datetime,
		req.Updated_datetime,
		req.Lat,
		req.Long,
		req.Extended_attributes["ward"],
		req.Extended_attributes["police_district"],
		req.Media_url,
		req.Extended_attributes["channel"],
		req.Extended_attributes["duplicate"],
		req.Extended_attributes["parent_service_request_id"],
		closed_time,
		notes_as_json,
	)

	if err != nil {
		log.Fatalf("could not update %s because %s", req.Service_request_id, err)
	} else {
		var verb string
		switch {
		case !persisted && closed_time.Time.IsZero():
			verb = "CREATED"
		case !persisted && !closed_time.Time.IsZero():
			verb = "CREATED/CLOSED"
		case persisted && closed_time.Time.IsZero():
			verb = "UPDATED"
		case persisted && !closed_time.Time.IsZero():
			verb = "UPDATED/CLOSED"
		}

		log.Printf("[%s] %s", verb, req)
		persisted = true
	}

	// err = tx.Commit()
	// if err != nil {
	// 	log.Fatal("error closing transaction", err)
	// }

	return persisted

	// calculate closed time if necessary

}

func (req Open311Request) ExtractClosedDatetime() time.Time {
	// given an extended_attributes JSON blob, pluck out the closed time, if present
	// req.PrintNotes()

	var closed_at time.Time
	for _, note := range req.Notes {
		if note["type"] == "closed" {
			parsed_date, err := time.Parse("2006-01-02T15:04:05-07:00", note["datetime"].(string))
			if err != nil {
				log.Print("error parsing date", err)
			}
			log.Printf("SR %s closed at: %s", req, parsed_date)
			closed_at = parsed_date
		}
	}

	return closed_at
}

func (req Open311Request) PrintNotes() {
	fmt.Printf("Notes for SR %s:\n", req.Service_request_id)

	for _, note := range req.Notes {
		fmt.Printf("%+v\n", note)
	}
}

func fetchSingleRequest(sr_number string) (request Open311Request) {
	// given an SR, fetch the record
	log.Printf("fetching single SR %s", sr_number)
	open311_api_endpoint := fmt.Sprintf("http://311api.cityofchicago.org/open311/v2/requests/%s.json?extensions=true", sr_number)

	log.Printf("fetching from %s", open311_api_endpoint)
	resp, err := http.Get(open311_api_endpoint)
	defer resp.Body.Close()

	if err != nil {
		log.Fatalln("error fetching from Open311 endpoint", err)
	}

	// load response body
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Fatal("error loading response body", err)
	}

	// parse JSON and load into an array of Open311Request objects
	var requests []Open311Request

	err = json.Unmarshal(body, &requests)
	if err != nil {
		log.Fatal("error parsing JSON:", err)
	}

	log.Printf("received %d requests from Open311", len(requests))

	return requests[0]
}

func fetchRequests() (requests []Open311Request) {
	last_updated_at := time.Now()
	if err := worker.Db.QueryRow("SELECT MAX(updated_datetime) FROM service_requests;").Scan(&last_updated_at); err != nil {
		log.Print("[fetchRequests] error loading most recent SR, will fallback to current time: ", err)
	}

	log.Print("[fetchRequests] most recent SR timestamp ", last_updated_at.Format(time.RFC3339))

	// construct the request URI using base params and the proper time
	open311_api_endpoint := OPEN311_API_URI + "&updated_after=" + last_updated_at.Format(time.RFC3339)

	log.Printf("[fetchRequests] fetching from %s", open311_api_endpoint)
	resp, err := http.Get(open311_api_endpoint)
	defer resp.Body.Close()

	if err != nil {
		log.Fatalln("[fetchRequests] error fetching from Open311 endpoint", err)
	}

	// load response body
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Fatal("[fetchRequests] error loading response body", err)
	}

	// parse JSON and load into an array of Open311Request objects
	err = json.Unmarshal(body, &requests)
	if err != nil {
		log.Fatal("[fetchRequests] error parsing JSON:", err)
	}

	log.Printf("[fetchRequests] received %d requests from Open311", len(requests))

	return requests
}

func backFillRequests(start_from string) (requests []Open311Request) {
	var fetch_from time.Time

	if start_from == "" {
		err := worker.Db.QueryRow("SELECT updated_datetime FROM service_requests ORDER BY updated_datetime ASC LIMIT 1").Scan(&fetch_from)
		if err != nil {
			log.Println("error fetching oldest SR:", err)
		}
		log.Printf("no start_from value provided, so falling back to oldest (by last update) SR in the database: %s", fetch_from)
	} else {
		t, err := time.Parse(time.RFC3339, start_from)
		if err != nil {
			log.Fatal("[backfill] error parsing date to start from", err)
		}
		fetch_from = t
	}

	formatted_date_string_with_tz := fetch_from.Format(time.RFC3339)

	// construct the request URI using base params and the proper time
	open311_api_endpoint := OPEN311_API_URI + "&updated_before=" + formatted_date_string_with_tz

	log.Printf("[backfill] fetching from %s", open311_api_endpoint)
	resp, err := http.Get(open311_api_endpoint)
	defer resp.Body.Close()

	if err != nil {
		log.Fatalln("[backfill] error fetching from Open311 endpoint", err)
	}

	// load response body
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Fatal("[backfill] error loading response body", err)
	}

	// parse JSON and load into an array of Open311Request objects
	err = json.Unmarshal(body, &requests)
	if err != nil {
		log.Fatal("[backfill] error parsing JSON:", err)
	}

	log.Printf("[backfill] received %d requests from Open311", len(requests))

	return requests
}
<|endoftext|>


<commit_before>// Copyright 2015 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package workload

import (
	"reflect"
	"strings"

	"github.com/juju/errors"
	"gopkg.in/juju/charm.v5"
)

// Info holds information about a workload that Juju needs. Iff the
// workload has not been registered with Juju then the Status and
// Details fields will be zero values.
//
// A registered workload is one which has been defined in Juju (e.g. in
// charm metadata) and subsequently was launched by Juju (e.g. in a
// unit hook context).
type Info struct {
	charm.Workload

	// Status is the Juju-level status of the workload.
	Status Status

	// Details is the information about the workload which the plugin provided.
	Details Details
}

// ID composes a unique ID for the workload (relative to the unit/charm).
func (info Info) ID() string {
	if info.Details.ID == "" {
		return info.Workload.Name
	}
	return info.Workload.Name + "/" + info.Details.ID
}

// ParseID extracts the workload name and details ID from the provided string.
// The format is expected to be name/pluginID. If no separator is found, the
// whole string is assumed to be the name.
func ParseID(id string) (name, pluginID string) {
	parts := strings.SplitN(id, "/", 2)
	if len(parts) == 2 {
		return parts[0], parts[1]
	}
	return id, ""
}

// Validate checks the workload info to ensure it is correct.
func (info Info) Validate() error {
	if err := info.Workload.Validate(); err != nil {
		return errors.NewNotValid(err, "")
	}

	if err := info.Status.Validate(); err != nil {
		return errors.Trace(err)
	}

	if err := info.Details.Validate(); err != nil {
		return errors.Trace(err)
	}

	return nil
}

// IsTracked indicates whether the represented workload has
// is already being tracked by Juju.
func (info Info) IsTracked() bool {
	// An untracked workload will not have the Status and Details
	// fields set (they will be zero values). Thus a trackeded
	// workload can be identified by non-zero values in those fields.
	// We use that fact here.
	return !reflect.DeepEqual(info, Info{Workload: info.Workload})
}
<commit_msg>one more commit<commit_after>// Copyright 2015 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package workload

import (
	"reflect"
	"strings"

	"github.com/juju/errors"
	"gopkg.in/juju/charm.v5"
)

// Info holds information about a workload that Juju needs. Iff the
// workload has not been registered with Juju then the Status and
// Details fields will be zero values.
//
// A registered workload is one which has been defined in Juju (e.g. in
// charm metadata) and subsequently was launched by Juju (e.g. in a
// unit hook context).
type Info struct {
	charm.Workload

	// Status is the Juju-level status of the workload.
	Status Status

	// Details is the information about the workload which the plugin provided.
	Details Details
}

// ID composes a unique ID for the workload (relative to the unit/charm).
func (info Info) ID() string {
	if info.Details.ID == "" {
		// TODO(natefinch) remove this special case when we can be sure the ID
		// is never empty (and fix the tests).
		return info.Workload.Name
	}
	return info.Workload.Name + "/" + info.Details.ID
}

// ParseID extracts the workload name and details ID from the provided string.
// The format is expected to be name/pluginID. If no separator is found, the
// whole string is assumed to be the name.
func ParseID(id string) (name, pluginID string) {
	parts := strings.SplitN(id, "/", 2)
	if len(parts) == 2 {
		return parts[0], parts[1]
	}
	return id, ""
}

// Validate checks the workload info to ensure it is correct.
func (info Info) Validate() error {
	if err := info.Workload.Validate(); err != nil {
		return errors.NewNotValid(err, "")
	}

	if err := info.Status.Validate(); err != nil {
		return errors.Trace(err)
	}

	if err := info.Details.Validate(); err != nil {
		return errors.Trace(err)
	}

	return nil
}

// IsTracked indicates whether the represented workload has
// is already being tracked by Juju.
func (info Info) IsTracked() bool {
	// An untracked workload will not have the Status and Details
	// fields set (they will be zero values). Thus a trackeded
	// workload can be identified by non-zero values in those fields.
	// We use that fact here.
	return !reflect.DeepEqual(info, Info{Workload: info.Workload})
}
<|endoftext|>


<commit_before>// Copyright 2012-2018 Oliver Eilhard. All rights reserved.
// Use of this source code is governed by a MIT-license.
// See http://olivere.mit-license.org/license.txt for details.

package elastic

import (
	"context"
	"encoding/json"
	"fmt"
	"net/http"
	"net/url"
	"strings"

	"github.com/olivere/elastic/v7/uritemplates"
)

// XPackSecurityGetRoleService retrieves a role by its name.
// See https://www.elastic.co/guide/en/elasticsearch/reference/7.0/security-api-get-role.html.
type XPackSecurityGetRoleService struct {
	client *Client

	pretty     *bool       // pretty format the returned JSON response
	human      *bool       // return human readable values for statistics
	errorTrace *bool       // include the stack trace of returned errors
	filterPath []string    // list of filters used to reduce the response
	headers    http.Header // custom request-level HTTP headers

	name string
}

// NewXPackSecurityGetRoleService creates a new XPackSecurityGetRoleService.
func NewXPackSecurityGetRoleService(client *Client) *XPackSecurityGetRoleService {
	return &XPackSecurityGetRoleService{
		client: client,
	}
}

// Pretty tells Elasticsearch whether to return a formatted JSON response.
func (s *XPackSecurityGetRoleService) Pretty(pretty bool) *XPackSecurityGetRoleService {
	s.pretty = &pretty
	return s
}

// Human specifies whether human readable values should be returned in
// the JSON response, e.g. "7.5mb".
func (s *XPackSecurityGetRoleService) Human(human bool) *XPackSecurityGetRoleService {
	s.human = &human
	return s
}

// ErrorTrace specifies whether to include the stack trace of returned errors.
func (s *XPackSecurityGetRoleService) ErrorTrace(errorTrace bool) *XPackSecurityGetRoleService {
	s.errorTrace = &errorTrace
	return s
}

// FilterPath specifies a list of filters used to reduce the response.
func (s *XPackSecurityGetRoleService) FilterPath(filterPath ...string) *XPackSecurityGetRoleService {
	s.filterPath = filterPath
	return s
}

// Header adds a header to the request.
func (s *XPackSecurityGetRoleService) Header(name string, value string) *XPackSecurityGetRoleService {
	if s.headers == nil {
		s.headers = http.Header{}
	}
	s.headers.Add(name, value)
	return s
}

// Headers specifies the headers of the request.
func (s *XPackSecurityGetRoleService) Headers(headers http.Header) *XPackSecurityGetRoleService {
	s.headers = headers
	return s
}

// Name is name of the role to retrieve.
func (s *XPackSecurityGetRoleService) Name(name string) *XPackSecurityGetRoleService {
	s.name = name
	return s
}

// buildURL builds the URL for the operation.
func (s *XPackSecurityGetRoleService) buildURL() (string, url.Values, error) {
	// Build URL
	path, err := uritemplates.Expand("/_security/role/{name}", map[string]string{
		"name": s.name,
	})
	if err != nil {
		return "", url.Values{}, err
	}

	// Add query string parameters
	params := url.Values{}
	if v := s.pretty; v != nil {
		params.Set("pretty", fmt.Sprint(*v))
	}
	if v := s.human; v != nil {
		params.Set("human", fmt.Sprint(*v))
	}
	if v := s.errorTrace; v != nil {
		params.Set("error_trace", fmt.Sprint(*v))
	}
	if len(s.filterPath) > 0 {
		params.Set("filter_path", strings.Join(s.filterPath, ","))
	}
	return path, params, nil
}

// Validate checks if the operation is valid.
func (s *XPackSecurityGetRoleService) Validate() error {
	var invalid []string
	if s.name == "" {
		invalid = append(invalid, "Name")
	}
	if len(invalid) > 0 {
		return fmt.Errorf("missing required fields: %v", invalid)
	}
	return nil
}

// Do executes the operation.
func (s *XPackSecurityGetRoleService) Do(ctx context.Context) (*XPackSecurityGetRoleResponse, error) {
	// Check pre-conditions
	if err := s.Validate(); err != nil {
		return nil, err
	}

	// Get URL for request
	path, params, err := s.buildURL()
	if err != nil {
		return nil, err
	}

	// Get HTTP response
	res, err := s.client.PerformRequest(ctx, PerformRequestOptions{
		Method:  "GET",
		Path:    path,
		Params:  params,
		Headers: s.headers,
	})
	if err != nil {
		return nil, err
	}

	// Return operation response
	ret := XPackSecurityGetRoleResponse{}
	if err := json.Unmarshal(res.Body, &ret); err != nil {
		return nil, err
	}
	return &ret, nil
}

// XPackSecurityGetRoleResponse is the response of XPackSecurityGetRoleService.Do.
type XPackSecurityGetRoleResponse map[string]XPackSecurityRole

// XPackSecurityRole is the role object.
//
// The Java source for this struct is defined here:
// https://github.com/elastic/elasticsearch/blob/6.7/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/RoleDescriptor.java
type XPackSecurityRole struct {
	Cluster           []string                             `json:"cluster"`
	Indices           []XPackSecurityIndicesPermissions    `json:"indices"`
	Applications      []XPackSecurityApplicationPrivileges `json:"applications"`
	RunAs             []string                             `json:"run_as"`
	Global            map[string]interface{}               `json:"global"`
	Metadata          map[string]interface{}               `json:"metadata"`
	TransientMetadata map[string]interface{}               `json:"transient_metadata"`
}

// XPackSecurityApplicationPrivileges is the application privileges object
type XPackSecurityApplicationPrivileges struct {
	Application string   `json:"application"`
	Privileges  []string `json:"privileges"`
	Ressources  []string `json:"resources"`
}

// XPackSecurityIndicesPermissions is the indices permission object
type XPackSecurityIndicesPermissions struct {
	Names         []string    `json:"names"`
	Privileges    []string    `json:"privileges"`
	FieldSecurity interface{} `json:"field_security"`
	Query         string      `json:"query"`
}
<commit_msg>Ignore FieldSecurity when it is null (#1237)<commit_after>// Copyright 2012-2018 Oliver Eilhard. All rights reserved.
// Use of this source code is governed by a MIT-license.
// See http://olivere.mit-license.org/license.txt for details.

package elastic

import (
	"context"
	"encoding/json"
	"fmt"
	"net/http"
	"net/url"
	"strings"

	"github.com/olivere/elastic/v7/uritemplates"
)

// XPackSecurityGetRoleService retrieves a role by its name.
// See https://www.elastic.co/guide/en/elasticsearch/reference/7.0/security-api-get-role.html.
type XPackSecurityGetRoleService struct {
	client *Client

	pretty     *bool       // pretty format the returned JSON response
	human      *bool       // return human readable values for statistics
	errorTrace *bool       // include the stack trace of returned errors
	filterPath []string    // list of filters used to reduce the response
	headers    http.Header // custom request-level HTTP headers

	name string
}

// NewXPackSecurityGetRoleService creates a new XPackSecurityGetRoleService.
func NewXPackSecurityGetRoleService(client *Client) *XPackSecurityGetRoleService {
	return &XPackSecurityGetRoleService{
		client: client,
	}
}

// Pretty tells Elasticsearch whether to return a formatted JSON response.
func (s *XPackSecurityGetRoleService) Pretty(pretty bool) *XPackSecurityGetRoleService {
	s.pretty = &pretty
	return s
}

// Human specifies whether human readable values should be returned in
// the JSON response, e.g. "7.5mb".
func (s *XPackSecurityGetRoleService) Human(human bool) *XPackSecurityGetRoleService {
	s.human = &human
	return s
}

// ErrorTrace specifies whether to include the stack trace of returned errors.
func (s *XPackSecurityGetRoleService) ErrorTrace(errorTrace bool) *XPackSecurityGetRoleService {
	s.errorTrace = &errorTrace
	return s
}

// FilterPath specifies a list of filters used to reduce the response.
func (s *XPackSecurityGetRoleService) FilterPath(filterPath ...string) *XPackSecurityGetRoleService {
	s.filterPath = filterPath
	return s
}

// Header adds a header to the request.
func (s *XPackSecurityGetRoleService) Header(name string, value string) *XPackSecurityGetRoleService {
	if s.headers == nil {
		s.headers = http.Header{}
	}
	s.headers.Add(name, value)
	return s
}

// Headers specifies the headers of the request.
func (s *XPackSecurityGetRoleService) Headers(headers http.Header) *XPackSecurityGetRoleService {
	s.headers = headers
	return s
}

// Name is name of the role to retrieve.
func (s *XPackSecurityGetRoleService) Name(name string) *XPackSecurityGetRoleService {
	s.name = name
	return s
}

// buildURL builds the URL for the operation.
func (s *XPackSecurityGetRoleService) buildURL() (string, url.Values, error) {
	// Build URL
	path, err := uritemplates.Expand("/_security/role/{name}", map[string]string{
		"name": s.name,
	})
	if err != nil {
		return "", url.Values{}, err
	}

	// Add query string parameters
	params := url.Values{}
	if v := s.pretty; v != nil {
		params.Set("pretty", fmt.Sprint(*v))
	}
	if v := s.human; v != nil {
		params.Set("human", fmt.Sprint(*v))
	}
	if v := s.errorTrace; v != nil {
		params.Set("error_trace", fmt.Sprint(*v))
	}
	if len(s.filterPath) > 0 {
		params.Set("filter_path", strings.Join(s.filterPath, ","))
	}
	return path, params, nil
}

// Validate checks if the operation is valid.
func (s *XPackSecurityGetRoleService) Validate() error {
	var invalid []string
	if s.name == "" {
		invalid = append(invalid, "Name")
	}
	if len(invalid) > 0 {
		return fmt.Errorf("missing required fields: %v", invalid)
	}
	return nil
}

// Do executes the operation.
func (s *XPackSecurityGetRoleService) Do(ctx context.Context) (*XPackSecurityGetRoleResponse, error) {
	// Check pre-conditions
	if err := s.Validate(); err != nil {
		return nil, err
	}

	// Get URL for request
	path, params, err := s.buildURL()
	if err != nil {
		return nil, err
	}

	// Get HTTP response
	res, err := s.client.PerformRequest(ctx, PerformRequestOptions{
		Method:  "GET",
		Path:    path,
		Params:  params,
		Headers: s.headers,
	})
	if err != nil {
		return nil, err
	}

	// Return operation response
	ret := XPackSecurityGetRoleResponse{}
	if err := json.Unmarshal(res.Body, &ret); err != nil {
		return nil, err
	}
	return &ret, nil
}

// XPackSecurityGetRoleResponse is the response of XPackSecurityGetRoleService.Do.
type XPackSecurityGetRoleResponse map[string]XPackSecurityRole

// XPackSecurityRole is the role object.
//
// The Java source for this struct is defined here:
// https://github.com/elastic/elasticsearch/blob/6.7/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/RoleDescriptor.java
type XPackSecurityRole struct {
	Cluster           []string                             `json:"cluster"`
	Indices           []XPackSecurityIndicesPermissions    `json:"indices"`
	Applications      []XPackSecurityApplicationPrivileges `json:"applications"`
	RunAs             []string                             `json:"run_as"`
	Global            map[string]interface{}               `json:"global"`
	Metadata          map[string]interface{}               `json:"metadata"`
	TransientMetadata map[string]interface{}               `json:"transient_metadata"`
}

// XPackSecurityApplicationPrivileges is the application privileges object
type XPackSecurityApplicationPrivileges struct {
	Application string   `json:"application"`
	Privileges  []string `json:"privileges"`
	Ressources  []string `json:"resources"`
}

// XPackSecurityIndicesPermissions is the indices permission object
type XPackSecurityIndicesPermissions struct {
	Names         []string    `json:"names"`
	Privileges    []string    `json:"privileges"`
	FieldSecurity interface{} `json:"field_security,omitempty"`
	Query         string      `json:"query"`
}
<|endoftext|>


<commit_before>package imageScraper
//scrapes card fulls and crops according to the mtgjson AllSets-x.json
//provided
//
//all references to cardNames are to their image name that mtgimage
//graciously offers

import(

	"os"
	"log"
	"fmt"

	"io/ioutil"
	"encoding/json"

	"strings"

	"io"
	"net/http"

)

//how many scrapers we can have running at once.
//
//networks blocks are our most likely issue
const maxGatherers int = 4

//the basic data so we know where to acquire, and how to save, our images
const imageBaseUrl string = "http://mtgimage.com/card/"
const cropExtension string = ".crop.jpg"
const fullExtension string = ".jpg"

//the basic data so we know where to acquire those beautiful svgs
//
//all symbols are found at symbol'MODIFIER'BaseUrl/'symbolNAME'.svg
const symbolExtension string = ".svg"
const symbolOtherBaseUrl string = "http://mtgimage.com/symbol/other/"
var symbolOtherList = [...]string{
	"c", "chaos", "chaosdice", "creature",
	"enchantment", "forwardslash", "instant", "land",
	"multiple", "planeswalk", "planeswalker", "power",
	"q", "sorcery", "t", "tap", "toughness", "untap",
}
const symbolManaBaseUrl string = "http://mtgimage.com/symbol/mana/"
//programmatically scraped, newlines added manually for readability
var symbolManaList = [...]string{
	"0","1","10","100","1000000","11","12","13","14","15","16","17","18","19",
	"2","20","2b","2black","2blue","2g","2green","2r","2red","2u","2w","2white",
	"3","4","5","6","7","8","9","b","b2","bg","bh","bhalf","black","black2",
	"blackblue","blackgreen","blackh","blackhalf","blackp","blackphyrexian",
	"blackred","blacktwo","blackwhite","blue","blue2","blueblack","bluegreen",
	"blueh","bluehalf","bluep","bluephyrexian","bluered","bluetwo","bluewhite",
	"bp","bphyrexian","br","bu","bw","colorlesshalf","eight","eighteen","eleven",
	"fifteen","five","four","fourteen","g","g2","gb","gh","ghalf","gp",
	"gphyrexian","gr","green","green2","greenblack","greenblue","greenh",
	"greenhalf","greenp","greenphyrexian","greenred","greentwo","greenwhite",
	"gu","gw","h","half","halfb","halfblack","halfblue","halfcolorless","halfg",
	"halfgreen","halfr","halfred","halfu","halfw","halfwhite","hb","hg","hr",
	"hu","hundred","hw","infinity","million","nine","nineteen","one",
	"onehundred","onemillion","p","pb","pblack","pblue","pg","pgreen",
	"phyrexian","phyrexianblack","phyrexianblue","phyrexiangreen",
	"phyrexianred","phyrexianwhite","pr","pred","pu","pw","pwhite","r","r2",
	"rb","red","red2","redblack","redblue","redgreen","redh","redhalf",
	"redp","redphyrexian","redtwo","redwhite","rg","rh","rhalf","rp",
	"rphyrexian","ru","rw","s","seven","seventeen","six","sixteen","snow",
	"ten","thirteen","three","twelve","twenty","two","twoblack","twoblue",
	"twogreen","twored","twowhite","u","u2","ub","ug","uh","uhalf","up",
	"uphyrexian","ur","uw","w","w2","wb","wg","wh","whalf","white","white2",
	"whiteblack","whiteblue","whitegreen","whiteh","whitehalf","whitep",
	"whitephyrexian","whitered","whitetwo","wp","wphyrexian","wr","wu","x",
	"y","z","zero","∞",
}

const symbolSetBaseUrl string = "http://mtgimage.com/symbol/set/"

func getLogger(fName, name string) (aLogger *log.Logger) {
	file, err:= os.OpenFile(fName, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666)
	if err!=nil {
		fmt.Println("Starting logger failed, cannot write to logger to say logger failed. Oh god.")
		fmt.Println(err)
		os.Exit(0)
	}

	multi:= io.MultiWriter(file, os.Stdout)

	aLogger = log.New(multi, name, log.Ldate|log.Ltime|log.Lshortfile)

	return
}

// exists returns whether the given file or directory exists or not
func exists(path string) (bool, error) {
    _, err := os.Stat(path)
    if err == nil { return true, nil }
    if os.IsNotExist(err) { return false, nil }
    return false, err
}

type cardMap map[string]card

//the only component of each card that we require is the image
type card struct{
	ImageName string
}

//scrapes the images for the cards and sticks them in the provided folders
//which are relative to our current working directory
func ScrapeImages(fullLoc, cropLoc, symbolLoc string) {

	aLogger:= getLogger("imageLog.txt", "imageLog")

	//grab the card data hosted on disk
	cardData, err:= ioutil.ReadFile("AllCards-x.json")
	if err!=nil {
		aLogger.Fatalf("Failed to read AllCards-x.json")
	}

	//unmarshal it into a map of string to card with image name
	var aCardList cardMap
	err = json.Unmarshal(cardData, &aCardList)
	if err!=nil {
		aLogger.Fatalf("Failed to unmarshal card list")
	}

	//divide up the work
	imagesPerGatherer:= len(aCardList) / maxGatherers
	gathererFeeders := make([][]string, maxGatherers)
	imagesAssignedThisGatherer:= 0
	currentGathererFeeder:= 0
	for _, aCard:= range aCardList{

		gathererFeeders[currentGathererFeeder] = append(
			gathererFeeders[currentGathererFeeder],
			aCard.ImageName )

		imagesAssignedThisGatherer++
		if imagesAssignedThisGatherer>imagesPerGatherer {
			//reset the count
			imagesAssignedThisGatherer = 0

			//increment the gatherer we feed
			currentGathererFeeder++
		}
	}

	//get the workers going
	completionChan:= make(chan bool)
	for i := 0; i < maxGatherers; i++ {
		go getCardList(gathererFeeders[i], fullLoc, cropLoc,
		 completionChan, aLogger)
	}

	//grab the symbols, the other workers should take a lot longer
	//than these svg symbols
	getSymbols(symbolLoc, aLogger)

	//now let the gatherers do their jobs!
	//
	//we attempt to pop a bool off the completion chan until we have
	//each gatherer accounted for
	for i := 0; i < maxGatherers; i++ {
		_ = <- completionChan
	}

}

//allows parallel usage of card image gathering
func getCardList(cardNames []string, fullLoc, cropLoc string,
	 completionChan chan bool, aLogger *log.Logger){
	for _, aCardName := range cardNames {
		getCardImage(aCardName, fullLoc, cropLoc, aLogger)
	}

	completionChan <- true
}

func getCardImage(cardName, fullLoc, cropLoc string, aLogger *log.Logger) {

	fullURL:= imageBaseUrl + cardName + fullExtension
	cropURL:= imageBaseUrl + cardName + cropExtension
	fullPath:= fullLoc + cardName + fullExtension
	cropPath:= cropLoc + cardName + fullExtension

	//we make a quick check to see if we have both the full and the crop
	//already are present
	fullExists, _:= exists(fullPath)
	cropExists, _:= exists(cropPath)
	if fullExists && cropExists {
		return
	}

	getDumbImage(fullURL, fullPath, aLogger)
	getDumbImage(cropURL, cropPath, aLogger)

}

//acquires all symbols declared statically at the top
func getSymbols(symbolLoc string, aLogger *log.Logger) {

	aLogger.Println("Acquiring Symbols")

	var url string
	var path string

	//acquire the mana symbols first. Many of these are synonymous
	for _, aSymbol:= range symbolManaList{
		url = symbolManaBaseUrl + aSymbol + symbolExtension
		path = symbolLoc + aSymbol + symbolExtension

		getDumbImage(url, path, aLogger)
	}

	//now the other symbols that are much, much less meaty
	for _, aSymbol:= range symbolOtherList{
		url = symbolOtherBaseUrl + aSymbol + symbolExtension
		path = symbolLoc + aSymbol + symbolExtension

		getDumbImage(url, path, aLogger)
	}

	err:= getSetSymbols(symbolLoc, aLogger)
	if err!=nil {
		aLogger.Fatalf("Failed to get set symbols, ", err)
	}

	aLogger.Println("All symbols acquired")

}

type setMap map[string]set
type set struct{
	Name string
	Type string
}

func getSetSymbols(symbolLoc string, aLogger *log.Logger) error {

	setData, err:= ioutil.ReadFile("AllSets-x.json")
	if err!=nil {
		return fmt.Errorf("Failed to read AllSets-x.json")
	}

	//unmarshal it into a map of string to card with image name
	var aSetMap setMap
	err = json.Unmarshal(setData, &aSetMap)
	if err!=nil {
		return fmt.Errorf("Failed to unmarshal set map")
	}

	var url string
	var path string
	for setCode, aSet := range aSetMap {

		// Promo sets may or may not have a specific symbol.
		// We normalize them to the general promo symbol
		if aSet.Type == "promo" {
			setCode = "pwcq"
		}

		url = symbolSetBaseUrl + setCode + "/" + "c" + symbolExtension

		cleanedSetName:= strings.Replace(aSet.Name, ":", "", -1)
		cleanedSetName = strings.Replace(cleanedSetName, "–", "", -1)
		cleanedSetName = strings.Replace(cleanedSetName, "\"", "", -1)

		path = symbolLoc + cleanedSetName + symbolExtension
		getDumbImage(url, path, aLogger)
	}

	return nil

}

// Acquires an image dumbly. If it fails then it is logged.
func getDumbImage(url, outPath string, aLogger *log.Logger) {

	resp, err := http.Get(url)
	if err != nil {
		aLogger.Println("Failed to acquire ", url)
		return
	}
	defer resp.Body.Close()
	out, err := os.Create(outPath)
	if err != nil {
		aLogger.Println("Failed to acquire ", url)
		return
	}
	defer out.Close()
	io.Copy(out, resp.Body)

}<commit_msg>removed imageDB from cardData<commit_after><|endoftext|>