<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"github.com/jawher/mow.cli"
	"io"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"net/url"
	"os"
	"strings"
	"sync"
	"time"
)

func main() {

	app := cli.App("up-restutil", "A RESTful resource utility")

	app.Command("put-resources", "read json resources from stdin and PUT them to an endpoint", func(cmd *cli.Cmd) {
		idProp := cmd.StringArg("IDPROP", "", "property name of identity property")
		baseUrl := cmd.StringArg("BASEURL", "", "base URL to PUT resources to")
		user := cmd.StringOpt("user", "", "user for basic auth")
		pass := cmd.StringOpt("pass", "", "password for basic auth")
		cmd.Action = func() {
			if err := putAllRest(*baseUrl, *idProp, *user, *pass, 128); err != nil {
				log.Fatal(err)
			}
		}

	})

	app.Command("dump-resources", "read json resources from stdin and PUT them to an endpoint", func(cmd *cli.Cmd) {
		baseUrl := cmd.StringArg("BASEURL", "", "base URL to GET resources from. Must contain a __ids resource")
		throttle := cmd.IntOpt("throttle", 10, "Limit request rate for resource GET requests (requests per second)")
		cmd.Action = func() {
			if err := getAllRest(*baseUrl, *throttle); err != nil {
				log.Fatal(err)
			}
		}

	})

	app.Run(os.Args)
}

func putAllRest(baseurl string, idProperty string, user string, pass string, conns int) error {

	dec := json.NewDecoder(os.Stdin)

	docs := make(chan resource)

	transport.MaxIdleConnsPerHost = conns

	rp := &resourcePutter{baseurl, idProperty, user, pass}

	errs := make(chan error, 1)

	wg := sync.WaitGroup{}
	for i := 0; i < conns; i++ {
		wg.Add(1)
		go func() {
			if err := rp.putAll(docs); err != nil {
				select {
				case errs <- err:
				default:
				}
			}
			wg.Done()
		}()
	}

	for {
		var doc map[string]interface{}
		if err := dec.Decode(&doc); err != nil {
			if err == io.EOF {
				break
			}
			return err
		}
		select {
		case docs <- doc:
		case err := <-errs:
			return err
		}
	}

	close(docs)

	wg.Wait()

	select {
	case err := <-errs:
		return err
	default:
		return nil
	}

}

func (rp *resourcePutter) putAll(resources <-chan resource) error {
	for r := range resources {
		id := r[rp.idProperty]
		idStr, ok := id.(string)
		if !ok {
			log.Println("unable to extract id property from resource, skipping")
		}

		msg, err := json.Marshal(r)
		if err != nil {
			return err
		}
		b := rp.baseUrl
		if !strings.HasSuffix(b, "/") {
			b = b + "/"
		}
		u, err := url.Parse(b)
		if err != nil {
			return err
		}
		u, err = u.Parse(idStr)
		if err != nil {
			return err
		}
		req, err := http.NewRequest("PUT", u.String(), bytes.NewReader(msg))
		if err != nil {
			return err
		}
		if rp.user != "" && rp.pass != "" {
			req.SetBasicAuth(rp.user, rp.pass)
		}
		resp, err := httpClient.Do(req)
		if err != nil {
			return err
		}
		contents, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			return err
		}
		resp.Body.Close()
		if resp.StatusCode > 299 {
			return fmt.Errorf("http fail: %v :\n%s\n", resp.Status, contents)
		}

	}
	return nil
}

func getAllRest(baseURL string, throttle int) error {
	if baseURL == "" {
		return errors.New("baseURL must be provided")
	}
	if !strings.HasSuffix(baseURL, "/") {
		baseURL = baseURL + "/"
	}
	if throttle < 1 {
		log.Fatalf("Invalid throttle %d", throttle)
	}
	ticker := time.NewTicker(time.Second / time.Duration(throttle))

	messages := make(chan string, 128)

	go func() {
		fetchAll(baseURL, messages, ticker)
		close(messages)
	}()

	for msg := range messages {
		fmt.Println(msg)
	}
	return nil
}

func fetchAll(baseURL string, messages chan<- string, ticker *time.Ticker) {
	ids := make(chan string, 128)
	go fetchIDList(baseURL, ids)

	readers := 32

	transport.MaxIdleConnsPerHost = readers

	readWg := sync.WaitGroup{}

	for i := 0; i < readers; i++ {
		readWg.Add(1)
		go func(i int) {
			fetchMessages(baseURL, messages, ids, ticker)
			readWg.Done()
		}(i)
	}

	readWg.Wait()
}

func fetchIDList(baseURL string, ids chan<- string) {

	u, err := url.Parse(baseURL)
	if err != nil {
		log.Fatal(err)
	}
	u, err = u.Parse("./__ids")
	if err != nil {
		log.Fatal(err)
	}

	resp, err := httpClient.Get(u.String())
	if err != nil {
		panic(err)
	}
	defer resp.Body.Close()

	type listEntry struct {
		ID string `json:id`
	}

	var le listEntry
	dec := json.NewDecoder(resp.Body)
	for {
		err = dec.Decode(&le)
		if err != nil {
			if err == io.EOF {
				break
			}
			log.Fatal(err)
		}
		ids <- le.ID
	}

	close(ids)
}

func fetchMessages(baseURL string, messages chan<- string, ids <-chan string, ticker *time.Ticker) {
	for id := range ids {
		<-ticker.C
		resp, err := httpClient.Get(strings.Join([]string{baseURL, id}, ""))
		if err != nil {
			panic(err)
		}
		data, err := ioutil.ReadAll(resp.Body)
		resp.Body.Close()
		if err != nil {
			panic(err)
		}
		messages <- string(data)
	}
}

var (
	transport = &http.Transport{
		MaxIdleConnsPerHost: 32,
		Dial: (&net.Dialer{
			Timeout:   30 * time.Second,
			KeepAlive: 30 * time.Second,
		}).Dial,
	}

	httpClient = &http.Client{
		Transport: transport,
	}
)

type resource map[string]interface{}

type resourcePutter struct {
	baseUrl    string
	idProperty string
	user       string
	pass       string
}
<commit_msg>Concurrency option<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"github.com/jawher/mow.cli"
	"io"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"net/url"
	"os"
	"strings"
	"sync"
	"time"
)

func main() {

	app := cli.App("up-restutil", "A RESTful resource utility")

	app.Command("put-resources", "read json resources from stdin and PUT them to an endpoint", func(cmd *cli.Cmd) {
		user := cmd.StringOpt("user", "", "user for basic auth")
		pass := cmd.StringOpt("pass", "", "password for basic auth")
		concurrency := cmd.IntOpt("concurrency", 16, "number of concurrent requests to use")
		idProp := cmd.StringArg("IDPROP", "", "property name of identity property")
		baseUrl := cmd.StringArg("BASEURL", "", "base URL to PUT resources to")
		cmd.Action = func() {
			if err := putAllRest(*baseUrl, *idProp, *user, *pass, *concurrency); err != nil {
				log.Fatal(err)
			}
		}

	})

	app.Command("dump-resources", "read json resources from stdin and PUT them to an endpoint", func(cmd *cli.Cmd) {
		baseUrl := cmd.StringArg("BASEURL", "", "base URL to GET resources from. Must contain a __ids resource")
		throttle := cmd.IntOpt("throttle", 10, "Limit request rate for resource GET requests (requests per second)")
		cmd.Action = func() {
			if err := getAllRest(*baseUrl, *throttle); err != nil {
				log.Fatal(err)
			}
		}

	})

	app.Run(os.Args)
}

func putAllRest(baseurl string, idProperty string, user string, pass string, conns int) error {

	dec := json.NewDecoder(os.Stdin)

	docs := make(chan resource)

	transport.MaxIdleConnsPerHost = conns

	rp := &resourcePutter{baseurl, idProperty, user, pass}

	errs := make(chan error, 1)

	wg := sync.WaitGroup{}
	for i := 0; i < conns; i++ {
		wg.Add(1)
		go func() {
			if err := rp.putAll(docs); err != nil {
				select {
				case errs <- err:
				default:
				}
			}
			wg.Done()
		}()
	}

	for {
		var doc map[string]interface{}
		if err := dec.Decode(&doc); err != nil {
			if err == io.EOF {
				break
			}
			return err
		}
		select {
		case docs <- doc:
		case err := <-errs:
			return err
		}
	}

	close(docs)

	wg.Wait()

	select {
	case err := <-errs:
		return err
	default:
		return nil
	}

}

func (rp *resourcePutter) putAll(resources <-chan resource) error {
	for r := range resources {
		id := r[rp.idProperty]
		idStr, ok := id.(string)
		if !ok {
			log.Println("unable to extract id property from resource, skipping")
		}

		msg, err := json.Marshal(r)
		if err != nil {
			return err
		}
		b := rp.baseUrl
		if !strings.HasSuffix(b, "/") {
			b = b + "/"
		}
		u, err := url.Parse(b)
		if err != nil {
			return err
		}
		u, err = u.Parse(idStr)
		if err != nil {
			return err
		}
		req, err := http.NewRequest("PUT", u.String(), bytes.NewReader(msg))
		if err != nil {
			return err
		}
		if rp.user != "" && rp.pass != "" {
			req.SetBasicAuth(rp.user, rp.pass)
		}
		resp, err := httpClient.Do(req)
		if err != nil {
			return err
		}
		contents, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			return err
		}
		resp.Body.Close()
		if resp.StatusCode > 299 {
			return fmt.Errorf("http fail: %v :\n%s\n", resp.Status, contents)
		}

	}
	return nil
}

func getAllRest(baseURL string, throttle int) error {
	if baseURL == "" {
		return errors.New("baseURL must be provided")
	}
	if !strings.HasSuffix(baseURL, "/") {
		baseURL = baseURL + "/"
	}
	if throttle < 1 {
		log.Fatalf("Invalid throttle %d", throttle)
	}
	ticker := time.NewTicker(time.Second / time.Duration(throttle))

	messages := make(chan string, 128)

	go func() {
		fetchAll(baseURL, messages, ticker)
		close(messages)
	}()

	for msg := range messages {
		fmt.Println(msg)
	}
	return nil
}

func fetchAll(baseURL string, messages chan<- string, ticker *time.Ticker) {
	ids := make(chan string, 128)
	go fetchIDList(baseURL, ids)

	readers := 32

	transport.MaxIdleConnsPerHost = readers

	readWg := sync.WaitGroup{}

	for i := 0; i < readers; i++ {
		readWg.Add(1)
		go func(i int) {
			fetchMessages(baseURL, messages, ids, ticker)
			readWg.Done()
		}(i)
	}

	readWg.Wait()
}

func fetchIDList(baseURL string, ids chan<- string) {

	u, err := url.Parse(baseURL)
	if err != nil {
		log.Fatal(err)
	}
	u, err = u.Parse("./__ids")
	if err != nil {
		log.Fatal(err)
	}

	resp, err := httpClient.Get(u.String())
	if err != nil {
		panic(err)
	}
	defer resp.Body.Close()

	type listEntry struct {
		ID string `json:id`
	}

	var le listEntry
	dec := json.NewDecoder(resp.Body)
	for {
		err = dec.Decode(&le)
		if err != nil {
			if err == io.EOF {
				break
			}
			log.Fatal(err)
		}
		ids <- le.ID
	}

	close(ids)
}

func fetchMessages(baseURL string, messages chan<- string, ids <-chan string, ticker *time.Ticker) {
	for id := range ids {
		<-ticker.C
		resp, err := httpClient.Get(strings.Join([]string{baseURL, id}, ""))
		if err != nil {
			panic(err)
		}
		data, err := ioutil.ReadAll(resp.Body)
		resp.Body.Close()
		if err != nil {
			panic(err)
		}
		messages <- string(data)
	}
}

var (
	transport = &http.Transport{
		MaxIdleConnsPerHost: 32,
		Dial: (&net.Dialer{
			Timeout:   30 * time.Second,
			KeepAlive: 30 * time.Second,
		}).Dial,
	}

	httpClient = &http.Client{
		Transport: transport,
	}
)

type resource map[string]interface{}

type resourcePutter struct {
	baseUrl    string
	idProperty string
	user       string
	pass       string
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"

	"github.com/iulianclita/logy/parser"
	"github.com/spf13/cobra"
)

func main() {
	// Flag placeholders
	var (
		text      string
		filter    string
		lines     int
		page      int
		ext       string
		withRegex bool
		noColor   bool
	)
	// Define command
	appCmd := &cobra.Command{
		Use:   "logy /path/to/file",
		Short: "Filter and handle log files of any size with ease",
		Long:  `Filter and handle log files of any size with ease`,
		Args:  cobra.RangeArgs(0, 1),
		Run: func(cmd *cobra.Command, args []string) {
			if len(args) != 1 {
				if err := cmd.Help(); err != nil {
					fmt.Println(err)
					os.Exit(1)
				}
				return
			}
			path := args[0]
			// Create parser object
			p := parser.New(path, text, filter, lines, page, noColor, withRegex, ext)
			// Start parsing the given file
			p.Parse()
		},
	}

	// Parse flags
	appCmd.PersistentFlags().StringVarP(&text, "text", "t", "plain", "Text type to parse (plain/json)")
	appCmd.PersistentFlags().StringVarP(&filter, "filter", "f", "", "Text to filter by")
	appCmd.PersistentFlags().IntVarP(&lines, "lines", "l", 50, "Number of lines per page")
	appCmd.PersistentFlags().IntVarP(&page, "page", "p", 1, "Current page number")
	appCmd.PersistentFlags().StringVarP(&ext, "ext", "e", "", "Accepted file extensions to search in folder")
	appCmd.PersistentFlags().BoolVar(&withRegex, "with-regex", false, "Enable regex support")
	appCmd.PersistentFlags().BoolVar(&noColor, "no-color", false, "Disable color output")

	// Run command
	if err := appCmd.Execute(); err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
}
<commit_msg>Removed some empty spaces<commit_after>package main

import (
	"fmt"
	"os"

	"github.com/iulianclita/logy/parser"
	"github.com/spf13/cobra"
)

func main() {
	// Flag placeholders
	var (
		text      string
		filter    string
		lines     int
		page      int
		ext       string
		withRegex bool
		noColor   bool
	)
	// Define command
	appCmd := &cobra.Command{
		Use:   "logy /path/to/file",
		Short: "Filter and handle log files of any size with ease",
		Long:  `Filter and handle log files of any size with ease`,
		Args:  cobra.RangeArgs(0, 1),
		Run: func(cmd *cobra.Command, args []string) {
			if len(args) != 1 {
				if err := cmd.Help(); err != nil {
					fmt.Println(err)
					os.Exit(1)
				}
				return
			}
			path := args[0]
			// Create parser object
			p := parser.New(path, text, filter, lines, page, noColor, withRegex, ext)
			// Start parsing the given file
			p.Parse()
		},
	}
	// Parse flags
	appCmd.PersistentFlags().StringVarP(&text, "text", "t", "plain", "Text type to parse (plain/json)")
	appCmd.PersistentFlags().StringVarP(&filter, "filter", "f", "", "Text to filter by")
	appCmd.PersistentFlags().IntVarP(&lines, "lines", "l", 50, "Number of lines per page")
	appCmd.PersistentFlags().IntVarP(&page, "page", "p", 1, "Current page number")
	appCmd.PersistentFlags().StringVarP(&ext, "ext", "e", "", "Accepted file extensions to search in folder")
	appCmd.PersistentFlags().BoolVar(&withRegex, "with-regex", false, "Enable regex support")
	appCmd.PersistentFlags().BoolVar(&noColor, "no-color", false, "Disable color output")
	// Run command
	if err := appCmd.Execute(); err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2015 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"fmt"
	"log"
	"net/http"
	"os"
	"time"

	"github.com/golang/glog"
	"github.com/openshift/origin/pkg/util/proc"
	flag "github.com/spf13/pflag"
	clientset "k8s.io/client-go/1.4/kubernetes"
	"k8s.io/client-go/1.4/pkg/api"
	"k8s.io/client-go/1.4/pkg/api/v1"
	"k8s.io/client-go/1.4/pkg/apis/extensions/v1beta1"
	"k8s.io/client-go/1.4/pkg/runtime"
	"k8s.io/client-go/1.4/pkg/watch"
	restclient "k8s.io/client-go/1.4/rest"
	"k8s.io/client-go/1.4/tools/cache"
	"k8s.io/client-go/1.4/tools/clientcmd"

	"github.com/prometheus/client_golang/prometheus"
)

const (
	resyncPeriod = 5 * time.Minute
	metricsPath  = "/metrics"
	healthzPath  = "/healthz"
)

var (
	flags = flag.NewFlagSet("", flag.ContinueOnError)

	inCluster = flags.Bool("in-cluster", true, `If true, use the built in kubernetes
		cluster for creating the client`)

	apiserver = flags.String("apiserver", "", `The URL of the apiserver to use as a master`)

	kubeconfig = flags.String("kubeconfig", "./config", "absolute path to the kubeconfig file")

	port = flags.Int("port", 80, `Port to expose metrics on.`)
)

func main() {
	flags.Parse(os.Args)

	if *apiserver == "" && !(*inCluster) {
		glog.Fatalf("--apiserver not set and --in-cluster is false; apiserver must be set to a valid URL")
	}
	glog.Infof("apiServer set to: %v", *apiserver)

	proc.StartReaper()

	kubeClient, err := createKubeClient()
	if err != nil {
		glog.Fatalf("Failed to create client: %v", err)
	}

	initializeMetrics(kubeClient)
	metricsServer()
}

func createKubeClient() (kubeClient clientset.Interface, err error) {
	glog.Infof("Creating client")
	if *inCluster {
		config, err := restclient.InClusterConfig()
		if err != nil {
			return nil, err
		}
		// Allow overriding of apiserver even if using inClusterConfig
		// (necessary if kube-proxy isn't properly set up).
		if *apiserver != "" {
			config.Host = *apiserver
		}
		tokenPresent := false
		if len(config.BearerToken) > 0 {
			tokenPresent = true
		}
		glog.Infof("service account token present: %v", tokenPresent)
		glog.Infof("service host: %s", config.Host)
		if kubeClient, err = clientset.NewForConfig(config); err != nil {
			return nil, err
		}
	} else {
		loadingRules := clientcmd.NewDefaultClientConfigLoadingRules()
		// if you want to change the loading rules (which files in which order), you can do so here
		configOverrides := &clientcmd.ConfigOverrides{}
		// if you want to change override values or bind them to flags, there are methods to help you
		kubeConfig := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(loadingRules, configOverrides)
		config, err := kubeConfig.ClientConfig()
		//config, err := clientcmd.BuildConfigFromFlags("", *kubeconfig)
		//config, err := clientcmd.DefaultClientConfig.ClientConfig()
		if err != nil {
			return nil, err
		}
		kubeClient, err = clientset.NewForConfig(config)
		if err != nil {
			return nil, err
		}
	}

	// Informers don't seem to do a good job logging error messages when it
	// can't reach the server, making debugging hard. This makes it easier to
	// figure out if apiserver is configured incorrectly.
	glog.Infof("testing communication with server")
	_, err = kubeClient.Discovery().ServerVersion()
	if err != nil {
		return nil, fmt.Errorf("ERROR communicating with apiserver: %v", err)
	}

	return kubeClient, nil
}

func metricsServer() {
	// Address to listen on for web interface and telemetry
	listenAddress := fmt.Sprintf(":%d", *port)

	glog.Infof("Starting metrics server: %s", listenAddress)
	// Add metricsPath
	http.Handle(metricsPath, prometheus.UninstrumentedHandler())
	// Add healthzPath
	http.HandleFunc(healthzPath, func(w http.ResponseWriter, r *http.Request) {
		w.WriteHeader(200)
		w.Write([]byte("ok"))
	})
	// Add index
	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		w.Write([]byte(`
             <title>Kube Metrics Server</title>

Kube Metrics

<a href='` + metricsPath + `'>metrics

<a href='` + healthzPath + `'>healthz

             `))
	})
	log.Fatal(http.ListenAndServe(listenAddress, nil))
}

type DeploymentLister func() ([]v1beta1.Deployment, error)

func (l DeploymentLister) List() ([]v1beta1.Deployment, error) {
	return l()
}

type PodLister func() ([]v1.Pod, error)

func (l PodLister) List() ([]v1.Pod, error) {
	return l()
}

type NodeLister func() (v1.NodeList, error)

func (l NodeLister) List() (v1.NodeList, error) {
	return l()
}

// initializeMetrics creates a new controller from the given config.
func initializeMetrics(kubeClient clientset.Interface) {
	dplStore, dplController := cache.NewNamespaceKeyedIndexerAndReflector(
		&cache.ListWatch{
			ListFunc: func(options api.ListOptions) (runtime.Object, error) {
				return kubeClient.Extensions().Deployments(api.NamespaceAll).List(options)
			},
			WatchFunc: func(options api.ListOptions) (watch.Interface, error) {
				return kubeClient.Extensions().Deployments(api.NamespaceAll).Watch(options)
			},
		}, &v1beta1.Deployment{}, resyncPeriod)

	podStore, podController := cache.NewNamespaceKeyedIndexerAndReflector(
		cache.NewListWatchFromClient(
			kubeClient.Core().GetRESTClient(),
			"pods",
			api.NamespaceAll,
			nil,
		), &v1.Pod{}, resyncPeriod)

	nodeStore, nodeController := cache.NewNamespaceKeyedIndexerAndReflector(
		cache.NewListWatchFromClient(
			kubeClient.Core().GetRESTClient(),
			"nodes",
			api.NamespaceAll,
			nil,
		), &v1.Node{}, resyncPeriod)

	go dplController.Run()
	go podController.Run()
	go nodeController.Run()

	dplLister := DeploymentLister(func() (deployments []v1beta1.Deployment, err error) {
		for _, c := range dplStore.List() {
			deployments = append(deployments, *(c.(*v1beta1.Deployment)))
		}
		return deployments, nil
	})

	podLister := PodLister(func() (pods []v1.Pod, err error) {
		for _, m := range podStore.List() {
			pods = append(pods, *m.(*v1.Pod))
		}
		return pods, nil
	})

	nodeLister := NodeLister(func() (machines v1.NodeList, err error) {
		for _, m := range nodeStore.List() {
			machines.Items = append(machines.Items, *(m.(*v1.Node)))
		}
		return machines, nil
	})

	prometheus.MustRegister(&deploymentCollector{store: dplLister})
	prometheus.MustRegister(&podCollector{store: podLister})
	prometheus.MustRegister(&nodeCollector{store: nodeLister})
}
<commit_msg>don't panic when requesting help (#42)<commit_after>/*
Copyright 2015 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"fmt"
	"log"
	"net/http"
	"os"
	"time"

	"github.com/golang/glog"
	"github.com/openshift/origin/pkg/util/proc"
	flag "github.com/spf13/pflag"
	clientset "k8s.io/client-go/1.4/kubernetes"
	"k8s.io/client-go/1.4/pkg/api"
	"k8s.io/client-go/1.4/pkg/api/v1"
	"k8s.io/client-go/1.4/pkg/apis/extensions/v1beta1"
	"k8s.io/client-go/1.4/pkg/runtime"
	"k8s.io/client-go/1.4/pkg/watch"
	restclient "k8s.io/client-go/1.4/rest"
	"k8s.io/client-go/1.4/tools/cache"
	"k8s.io/client-go/1.4/tools/clientcmd"

	"github.com/prometheus/client_golang/prometheus"
)

const (
	resyncPeriod = 5 * time.Minute
	metricsPath  = "/metrics"
	healthzPath  = "/healthz"
)

var (
	flags = flag.NewFlagSet("", flag.ExitOnError)

	inCluster = flags.Bool("in-cluster", true, `If true, use the built in kubernetes cluster for creating the client`)

	apiserver = flags.String("apiserver", "", `The URL of the apiserver to use as a master`)

	kubeconfig = flags.String("kubeconfig", "./config", "absolute path to the kubeconfig file")

	help = flags.BoolP("help", "h", false, "Print help text")

	port = flags.Int("port", 80, `Port to expose metrics on.`)
)

func main() {
	flags.Usage = func() {
		fmt.Fprintf(os.Stderr, "Usage of %s:\n", os.Args[0])
		flags.PrintDefaults()
	}

	err := flags.Parse(os.Args)
	if err != nil {
		glog.Fatalf("Error: %s", err)
	}

	if *help {
		flags.Usage()
		os.Exit(0)
	}

	if *apiserver == "" && !(*inCluster) {
		glog.Fatalf("--apiserver not set and --in-cluster is false; apiserver must be set to a valid URL")
	}
	glog.Infof("apiServer set to: %v", *apiserver)

	proc.StartReaper()

	kubeClient, err := createKubeClient()
	if err != nil {
		glog.Fatalf("Failed to create client: %v", err)
	}

	initializeMetrics(kubeClient)
	metricsServer()
}

func createKubeClient() (kubeClient clientset.Interface, err error) {
	glog.Infof("Creating client")
	if *inCluster {
		config, err := restclient.InClusterConfig()
		if err != nil {
			return nil, err
		}
		// Allow overriding of apiserver even if using inClusterConfig
		// (necessary if kube-proxy isn't properly set up).
		if *apiserver != "" {
			config.Host = *apiserver
		}
		tokenPresent := false
		if len(config.BearerToken) > 0 {
			tokenPresent = true
		}
		glog.Infof("service account token present: %v", tokenPresent)
		glog.Infof("service host: %s", config.Host)
		if kubeClient, err = clientset.NewForConfig(config); err != nil {
			return nil, err
		}
	} else {
		loadingRules := clientcmd.NewDefaultClientConfigLoadingRules()
		// if you want to change the loading rules (which files in which order), you can do so here
		configOverrides := &clientcmd.ConfigOverrides{}
		// if you want to change override values or bind them to flags, there are methods to help you
		kubeConfig := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(loadingRules, configOverrides)
		config, err := kubeConfig.ClientConfig()
		//config, err := clientcmd.BuildConfigFromFlags("", *kubeconfig)
		//config, err := clientcmd.DefaultClientConfig.ClientConfig()
		if err != nil {
			return nil, err
		}
		kubeClient, err = clientset.NewForConfig(config)
		if err != nil {
			return nil, err
		}
	}

	// Informers don't seem to do a good job logging error messages when it
	// can't reach the server, making debugging hard. This makes it easier to
	// figure out if apiserver is configured incorrectly.
	glog.Infof("testing communication with server")
	_, err = kubeClient.Discovery().ServerVersion()
	if err != nil {
		return nil, fmt.Errorf("ERROR communicating with apiserver: %v", err)
	}

	return kubeClient, nil
}

func metricsServer() {
	// Address to listen on for web interface and telemetry
	listenAddress := fmt.Sprintf(":%d", *port)

	glog.Infof("Starting metrics server: %s", listenAddress)
	// Add metricsPath
	http.Handle(metricsPath, prometheus.UninstrumentedHandler())
	// Add healthzPath
	http.HandleFunc(healthzPath, func(w http.ResponseWriter, r *http.Request) {
		w.WriteHeader(200)
		w.Write([]byte("ok"))
	})
	// Add index
	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		w.Write([]byte(`
             <title>Kube Metrics Server</title>

Kube Metrics

<a href='` + metricsPath + `'>metrics

<a href='` + healthzPath + `'>healthz

             `))
	})
	log.Fatal(http.ListenAndServe(listenAddress, nil))
}

type DeploymentLister func() ([]v1beta1.Deployment, error)

func (l DeploymentLister) List() ([]v1beta1.Deployment, error) {
	return l()
}

type PodLister func() ([]v1.Pod, error)

func (l PodLister) List() ([]v1.Pod, error) {
	return l()
}

type NodeLister func() (v1.NodeList, error)

func (l NodeLister) List() (v1.NodeList, error) {
	return l()
}

// initializeMetrics creates a new controller from the given config.
func initializeMetrics(kubeClient clientset.Interface) {
	dplStore, dplController := cache.NewNamespaceKeyedIndexerAndReflector(
		&cache.ListWatch{
			ListFunc: func(options api.ListOptions) (runtime.Object, error) {
				return kubeClient.Extensions().Deployments(api.NamespaceAll).List(options)
			},
			WatchFunc: func(options api.ListOptions) (watch.Interface, error) {
				return kubeClient.Extensions().Deployments(api.NamespaceAll).Watch(options)
			},
		}, &v1beta1.Deployment{}, resyncPeriod)

	podStore, podController := cache.NewNamespaceKeyedIndexerAndReflector(
		cache.NewListWatchFromClient(
			kubeClient.Core().GetRESTClient(),
			"pods",
			api.NamespaceAll,
			nil,
		), &v1.Pod{}, resyncPeriod)

	nodeStore, nodeController := cache.NewNamespaceKeyedIndexerAndReflector(
		cache.NewListWatchFromClient(
			kubeClient.Core().GetRESTClient(),
			"nodes",
			api.NamespaceAll,
			nil,
		), &v1.Node{}, resyncPeriod)

	go dplController.Run()
	go podController.Run()
	go nodeController.Run()

	dplLister := DeploymentLister(func() (deployments []v1beta1.Deployment, err error) {
		for _, c := range dplStore.List() {
			deployments = append(deployments, *(c.(*v1beta1.Deployment)))
		}
		return deployments, nil
	})

	podLister := PodLister(func() (pods []v1.Pod, err error) {
		for _, m := range podStore.List() {
			pods = append(pods, *m.(*v1.Pod))
		}
		return pods, nil
	})

	nodeLister := NodeLister(func() (machines v1.NodeList, err error) {
		for _, m := range nodeStore.List() {
			machines.Items = append(machines.Items, *(m.(*v1.Node)))
		}
		return machines, nil
	})

	prometheus.MustRegister(&deploymentCollector{store: dplLister})
	prometheus.MustRegister(&podCollector{store: podLister})
	prometheus.MustRegister(&nodeCollector{store: nodeLister})
}
<|endoftext|>


<commit_before>// Copyright 2015 mokey Authors. All rights reserved.
// Use of this source code is governed by a BSD style
// license that can be found in the LICENSE file.

package main

import (
	"fmt"

	log "github.com/Sirupsen/logrus"
	"github.com/codegangsta/cli"
	"github.com/spf13/viper"
)

func init() {
	viper.SetConfigName("mokey")
	viper.SetConfigType("yaml")
	viper.AddConfigPath("/etc/mokey/")
}

func main() {
	app := cli.NewApp()
	app.Name = "mokey"
	app.Authors = []cli.Author{cli.Author{Name: "Andrew E. Bruno", Email: "aebruno2@buffalo.edu"}}
	app.Usage = "mokey"
	app.Version = "0.0.5"
	app.Flags = []cli.Flag{
		&cli.StringFlag{Name: "conf,c", Usage: "Path to conf file"},
		&cli.BoolFlag{Name: "debug,d", Usage: "Print debug messages"},
	}
	app.Before = func(c *cli.Context) error {
		if c.GlobalBool("debug") {
			log.SetLevel(log.InfoLevel)
		} else {
			log.SetLevel(log.WarnLevel)
		}

		conf := c.GlobalString("conf")
		if len(conf) > 0 {
			viper.SetConfigFile(conf)
		}

		err := viper.ReadInConfig()
		if err != nil {
			return fmt.Errorf("Failed reading config file - %s", err)
		}

		return nil
	}
	app.Commands = []cli.Command{
		{
			Name:  "server",
			Usage: "Run http server",
			Action: func(c *cli.Context) {
				Server()
			},
		},
		{
			Name:  "resetpw",
			Usage: "Send reset password email",
			Flags: []cli.Flag{
				&cli.StringFlag{Name: "uid, u", Usage: "User id"},
			},
			Action: func(c *cli.Context) {
				uid := c.String("uid")
				if len(uid) == 0 {
					log.Fatal("Please provide a user uid")
				}

				ResetPasswordEmail(uid)
			},
		},
		{
			Name:  "newacct",
			Usage: "Send new account email",
			Flags: []cli.Flag{
				&cli.StringFlag{Name: "uid, u", Usage: "User id"},
			},
			Action: func(c *cli.Context) {
				uid := c.String("uid")
				if len(uid) == 0 {
					log.Fatal("Please provide a user uid")
				}

				NewAccountEmail(uid)
			},
		},
		{
			Name:  "removeotp",
			Usage: "Disable TOTP",
			Flags: []cli.Flag{
				&cli.StringFlag{Name: "uid, u", Usage: "User id"},
			},
			Action: func(c *cli.Context) {
				uid := c.String("uid")
				if len(uid) == 0 {
					log.Fatal("Please provide a user uid")
				}

				DisableTOTP(uid)
			},
		}}

	app.RunAndExitOnError()
}
<commit_msg>Disable TOTP when sending new account email<commit_after>// Copyright 2015 mokey Authors. All rights reserved.
// Use of this source code is governed by a BSD style
// license that can be found in the LICENSE file.

package main

import (
	"fmt"

	log "github.com/Sirupsen/logrus"
	"github.com/codegangsta/cli"
	"github.com/spf13/viper"
)

func init() {
	viper.SetConfigName("mokey")
	viper.SetConfigType("yaml")
	viper.AddConfigPath("/etc/mokey/")
}

func main() {
	app := cli.NewApp()
	app.Name = "mokey"
	app.Authors = []cli.Author{cli.Author{Name: "Andrew E. Bruno", Email: "aebruno2@buffalo.edu"}}
	app.Usage = "mokey"
	app.Version = "0.0.5"
	app.Flags = []cli.Flag{
		&cli.StringFlag{Name: "conf,c", Usage: "Path to conf file"},
		&cli.BoolFlag{Name: "debug,d", Usage: "Print debug messages"},
	}
	app.Before = func(c *cli.Context) error {
		if c.GlobalBool("debug") {
			log.SetLevel(log.InfoLevel)
		} else {
			log.SetLevel(log.WarnLevel)
		}

		conf := c.GlobalString("conf")
		if len(conf) > 0 {
			viper.SetConfigFile(conf)
		}

		err := viper.ReadInConfig()
		if err != nil {
			return fmt.Errorf("Failed reading config file - %s", err)
		}

		return nil
	}
	app.Commands = []cli.Command{
		{
			Name:  "server",
			Usage: "Run http server",
			Action: func(c *cli.Context) {
				Server()
			},
		},
		{
			Name:  "resetpw",
			Usage: "Send reset password email",
			Flags: []cli.Flag{
				&cli.StringFlag{Name: "uid, u", Usage: "User id"},
			},
			Action: func(c *cli.Context) {
				uid := c.String("uid")
				if len(uid) == 0 {
					log.Fatal("Please provide a user uid")
				}

				ResetPasswordEmail(uid)
			},
		},
		{
			Name:  "newacct",
			Usage: "Send new account email",
			Flags: []cli.Flag{
				&cli.StringFlag{Name: "uid, u", Usage: "User id"},
			},
			Action: func(c *cli.Context) {
				uid := c.String("uid")
				if len(uid) == 0 {
					log.Fatal("Please provide a user uid")
				}

				NewAccountEmail(uid)
				DisableTOTP(uid)
			},
		},
		{
			Name:  "removeotp",
			Usage: "Disable TOTP",
			Flags: []cli.Flag{
				&cli.StringFlag{Name: "uid, u", Usage: "User id"},
			},
			Action: func(c *cli.Context) {
				uid := c.String("uid")
				if len(uid) == 0 {
					log.Fatal("Please provide a user uid")
				}

				DisableTOTP(uid)
			},
		}}

	app.RunAndExitOnError()
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"encoding/json"
	"fmt"
	"html"
	"io"
	"io/ioutil"
	"net/http"
	"net/http/httputil"
	"net/url"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"strings"
	"sync"

	"github.com/PuerkitoBio/goquery"
	"github.com/vincent-petithory/dataurl"
)

const yuyuteiURL = "http://yuyu-tei.jp/"
const wsDeckURL = "https://wsdecks.com/"
const hoTcURL = "http://www.heartofthecards.com/code/cardlist.html?card=WS_"

// Prox struct
type Prox struct {
	// target url of reverse proxy
	target *url.URL
	// instance of Go ReverseProxy thatwill do the job for us
	proxy *httputil.ReverseProxy
}

type siteConfig struct {
	Name   string
	Filter string
}

type cardsConfig struct {
	Dir  string
	Site siteConfig
}

type cardStruc struct {
	ID          string
	Translation string
}

// New proxy
func New(target string) *Prox {
	url, _ := url.Parse(target)
	// you should handle error on parsing
	return &Prox{target: url, proxy: httputil.NewSingleHostReverseProxy(url)}
}

func (p *Prox) handle(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("X-GoProxy", "GoProxy")
	// call to magic method from ReverseProxy object
	p.proxy.ServeHTTP(w, r)
}
func lowCostSystemToURL(syspath string) string {
	return strings.Replace(syspath, "\\", "/", -1)
}

func convertToJpg(filePath string) {
	// convert -density 150 -trim to_love-ru_darkness_2nd_trial_deck.pdf -quality 100 -sharpen 0x1.0 love.jpg
	cmd := exec.Command("convert", "-density", "150", "-trim", filePath, "-quality", "100", "-sharpen", "0x1.0", filePath+".jpg")
	err := cmd.Start()
	if err != nil {
		fmt.Println(err)
	}
	err = cmd.Wait()
}

func createCardsCodeFile(dirPath string) (string, error) {
	//TODO Do nothing if file exists
	dirPath += "/"
	out, err := os.Create(dirPath + "codes.txt")
	defer out.Close()
	if err != nil {
		fmt.Println(err)
		return "", err
	}
	cardList, err := filepath.Glob(dirPath + "*.gif")
	if err != nil {
		fmt.Println(err)
		return "", err
	}
	// PI/S40-056
	for _, card := range cardList {
		card = strings.Replace(card, dirPath, "", 1)
		card = strings.Replace(card, "_", "-", 1)
		card = strings.Replace(card, "_", "/", 1)
		ex := strings.Split(card, ".")[0]
		out.WriteString(ex + "\n")
	}
	return out.Name(), nil
}

func getTranslationHotC(codesPath string) []cardStruc {
	translations := []cardStruc{}
	file, err := os.Open(codesPath + "/codes.txt")
	scanner := bufio.NewScanner(file)
	if err != nil {
		fmt.Println(err)
	}
	fmt.Println("getTranslationHotC")
	for scanner.Scan() {
		// fmt.Println(scanner.Text())
		url := hoTcURL + scanner.Text()
		fmt.Println(url)
		doc, err := goquery.NewDocument(url)
		if err != nil {
			fmt.Println(err)
		}
		textHTML, err := doc.Find(".cards3").Slice(2, 3).Html()
		if err != nil {
			fmt.Println(err)
		}
		textHTML = strings.Replace(textHTML, "
", "
", -1)
		// html.UnescapeString(textHTML)
		card := cardStruc{ID: scanner.Text(), Translation: html.UnescapeString(textHTML)}

		translations = append(translations, card)
		// json.Marshal(card)
		// doc.Find(".card3").Get(2)
	}
	return translations
}

func getCardsConfig(link string) (cardsConfig, error) {
	uid := ""
	site := siteConfig{}

	if strings.Contains(link, yuyuteiURL) {
		site.Name = "yuyutei"
		site.Filter = ".card_list_box" + " .image img"
		parsedURL, _ := url.Parse(link)
		values, _ := url.ParseQuery(parsedURL.RawQuery)
		uid = values.Get("ver")
	} else if strings.Contains(link, wsDeckURL) {
		site.Name = "wsdeck"
		site.Filter = ".wscard" + " img"
		uid = filepath.Base(link)
	}
	dir := filepath.Join("static", site.Name, uid)
	cardsConfig := cardsConfig{Dir: dir, Site: site}
	if site.Filter == "" {
		return cardsConfig, fmt.Errorf("Url is not supported %v", link)
	}

	return cardsConfig, nil
}

func main() {
	proxy := New("http://localhost:8080")
	os.MkdirAll(filepath.Join("static", "yuyutei"), 0744)
	os.MkdirAll(filepath.Join("static", "wsdeck"), 0744)
	// static := http.FileServer(http.Dir("./"))
	http.HandleFunc("/", proxy.handle)

	http.HandleFunc("/translationimages", func(w http.ResponseWriter, r *http.Request) {
		link := r.PostFormValue("url")
		cardsConfig, err := getCardsConfig(link)
		if err != nil {
			fmt.Println(err)
		}
		result := getTranslationHotC(cardsConfig.Dir)
		b, err := json.Marshal(result)
		if err != nil {
			fmt.Println(err)
		}
		w.Write(b)
	})

	http.HandleFunc("/translationimages_old", func(w http.ResponseWriter, r *http.Request) {
		file := r.PostFormValue("file")
		filename := r.PostFormValue("filename")
		uid := strings.Replace(filename, filepath.Ext(filename), "", 1)
		dir := filepath.Join("static", uid)
		filePath := filepath.Join(dir, filename)

		data, err := dataurl.DecodeString(file)
		if err != nil {
			fmt.Println(err)
			http.Error(w, err.Error(), http.StatusBadRequest)
			return
		}
		// fmt.Println(dataURL.Data)
		os.MkdirAll(dir, 0777)
		ioutil.WriteFile(filePath, data.Data, 0644)
		if _, err := os.Stat(dir); os.IsNotExist(err) {
			convertToJpg(filePath)
		}
		if err != nil {
			fmt.Println(err)
			http.Error(w, err.Error(), http.StatusBadRequest)
			return
		}
		listJpg, err := filepath.Glob(filePath + "*.jpg")
		if err != nil {
			fmt.Println(err)
			http.Error(w, err.Error(), http.StatusBadRequest)
			return
		}
		for i, jpgfile := range listJpg {
			listJpg[i] = lowCostSystemToURL(jpgfile)
		}

		b, err := json.Marshal(listJpg)

		if err != nil {
			fmt.Println(err)
			http.Error(w, err.Error(), 500)
			return
		}

		w.Write(b)
	})

	http.HandleFunc("/cardimages", func(w http.ResponseWriter, r *http.Request) {
		var wg sync.WaitGroup

		result := []string{}
		link := r.PostFormValue("url")

		if link != "" {
			doc, err := goquery.NewDocument(link)
			imageURL := ""
			cardsConfig, err := getCardsConfig(link)

			if err != nil {
				fmt.Println(err)
			}
			if _, err := os.Stat(cardsConfig.Dir); os.IsNotExist(err) {
				os.MkdirAll(cardsConfig.Dir, 0744)
				doc.Find(cardsConfig.Site.Filter).Each(func(i int, s *goquery.Selection) {
					wg.Add(1)
					val, _ := s.Attr("src")
					if cardsConfig.Site.Name == "yuyutei" {
						big := strings.Replace(val, "90_126", "front", 1)
						imageURL = yuyuteiURL + big
					} else if cardsConfig.Site.Name == "wsdeck" {
						imageURL = wsDeckURL + val
					}

					go func(url string) {
						defer wg.Done()
						// fmt.Println("dir : ", dir)
						fileName := filepath.Join(cardsConfig.Dir, path.Base(url))
						out, err := os.Create(fileName)
						if err != nil {
							fmt.Println(err)
						}
						defer out.Close()
						reps, err := http.Get(url)

						if err != nil {
							fmt.Println(err)
						}

						file, err := io.Copy(out, reps.Body)
						if err != nil {
							fmt.Println(err)
						}

						fmt.Println("File", file)
						// fmt.Printf("Link: n-%d __ %v%v\n", i, imageURL, uid)
						defer reps.Body.Close()
						// fmt.Println("image url: ", strings.Replace(fileName, "\\", "/", 1))
						result = append(result, lowCostSystemToURL(fileName))
					}(imageURL)
				})
			} else {
				files, err := ioutil.ReadDir(cardsConfig.Dir)
				if err != nil {
					fmt.Println(err)
				}
				for _, file := range files {
					absPath := filepath.Join(cardsConfig.Dir, file.Name())
					urlPath := lowCostSystemToURL(absPath)
					result = append(result, urlPath)
				}

			}
			wg.Wait()
			fmt.Printf("Finish")
			createCardsCodeFile(cardsConfig.Dir)
		}

		b, err := json.Marshal(result)

		if err != nil {
			http.Error(w, err.Error(), 500)
			return
		}
		w.Write(b)
	})

	http.HandleFunc("/static/", func(w http.ResponseWriter, r *http.Request) {
		// fmt.Println("static", r.URL.Path[1:])
		http.ServeFile(w, r, r.URL.Path[1:])
	})

	http.ListenAndServe(":8010", nil)
}
<commit_msg>#14 handle suffix<commit_after>package main

import (
	"bufio"
	"encoding/json"
	"fmt"
	"html"
	"io"
	"io/ioutil"
	"net/http"
	"net/http/httputil"
	"net/url"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"strings"
	"sync"

	"github.com/PuerkitoBio/goquery"
	"github.com/vincent-petithory/dataurl"
)

const yuyuteiURL = "http://yuyu-tei.jp/"
const wsDeckURL = "https://wsdecks.com/"
const hoTcURL = "http://www.heartofthecards.com/code/cardlist.html?card=WS_"

// Prox struct
type Prox struct {
	// target url of reverse proxy
	target *url.URL
	// instance of Go ReverseProxy thatwill do the job for us
	proxy *httputil.ReverseProxy
}

type siteConfig struct {
	Name   string
	Filter string
}

type cardsConfig struct {
	Dir  string
	Site siteConfig
}

type cardStruc struct {
	ID          string
	Translation string
}

// New proxy
func New(target string) *Prox {
	url, _ := url.Parse(target)
	// you should handle error on parsing
	return &Prox{target: url, proxy: httputil.NewSingleHostReverseProxy(url)}
}

func (p *Prox) handle(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("X-GoProxy", "GoProxy")
	// call to magic method from ReverseProxy object
	p.proxy.ServeHTTP(w, r)
}
func lowCostSystemToURL(syspath string) string {
	return strings.Replace(syspath, "\\", "/", -1)
}

func convertToJpg(filePath string) {
	// convert -density 150 -trim to_love-ru_darkness_2nd_trial_deck.pdf -quality 100 -sharpen 0x1.0 love.jpg
	cmd := exec.Command("convert", "-density", "150", "-trim", filePath, "-quality", "100", "-sharpen", "0x1.0", filePath+".jpg")
	err := cmd.Start()
	if err != nil {
		fmt.Println(err)
	}
	err = cmd.Wait()
}

func createCardsCodeFile(dirPath string) (string, error) {
	//TODO Do nothing if file exists
	dirPath += "/"
	out, err := os.Create(dirPath + "codes.txt")
	defer out.Close()
	if err != nil {
		fmt.Println(err)
		return "", err
	}
	cardList, err := filepath.Glob(dirPath + "*.gif")
	if err != nil {
		fmt.Println(err)
		return "", err
	}
	// PI/S40-056
	for _, card := range cardList {
		card = strings.Replace(card, dirPath, "", 1)
		card = strings.Replace(card, "_", "-", 1)
		card = strings.Replace(card, "_", "/", 1)
		ex := strings.Split(card, ".")[0]
		out.WriteString(ex + "\n")
	}
	return out.Name(), nil
}

func getTranslationHotC(codesPath string) []cardStruc {
	translations := []cardStruc{}
	file, err := os.Open(codesPath + "/codes.txt")
	scanner := bufio.NewScanner(file)
	if err != nil {
		fmt.Println(err)
	}
	fmt.Println("getTranslationHotC")
	for scanner.Scan() {
		// fmt.Println(scanner.Text())
		url := hoTcURL + scanner.Text()
		fmt.Println(url)
		doc, err := goquery.NewDocument(url)
		if err != nil {
			fmt.Println(err)
		}
		textHTML, err := doc.Find(".cards3").Slice(2, 3).Html()
		if err != nil {
			fmt.Println(err)
		}
		textHTML = strings.Replace(textHTML, "
", "
", -1)
		// html.UnescapeString(textHTML)
		card := cardStruc{ID: scanner.Text(), Translation: html.UnescapeString(textHTML)}

		translations = append(translations, card)
		// json.Marshal(card)
		// doc.Find(".card3").Get(2)
	}
	return translations
}

func getCardsConfig(link string) (cardsConfig, error) {
	uid := ""
	site := siteConfig{}

	if strings.Contains(link, yuyuteiURL) {
		site.Name = "yuyutei"
		site.Filter = ".card_list_box" + " .image img"
		parsedURL, _ := url.Parse(link)
		values, _ := url.ParseQuery(parsedURL.RawQuery)
		uid = values.Get("ver")
	} else if strings.Contains(link, wsDeckURL) {
		site.Name = "wsdeck"
		site.Filter = ".wscard" + " img"
		uid = filepath.Base(link)
	}
	dir := filepath.Join("static", site.Name, uid)
	cardsConfig := cardsConfig{Dir: dir, Site: site}
	if site.Filter == "" {
		return cardsConfig, fmt.Errorf("Url is not supported %v", link)
	}

	return cardsConfig, nil
}

func main() {
	proxy := New("http://localhost:8080")
	os.MkdirAll(filepath.Join("static", "yuyutei"), 0744)
	os.MkdirAll(filepath.Join("static", "wsdeck"), 0744)
	// static := http.FileServer(http.Dir("./"))
	http.HandleFunc("/", proxy.handle)

	http.HandleFunc("/translationimages", func(w http.ResponseWriter, r *http.Request) {
		link := r.PostFormValue("url")
		cardsConfig, err := getCardsConfig(link)
		if err != nil {
			fmt.Println(err)
		}
		result := getTranslationHotC(cardsConfig.Dir)
		b, err := json.Marshal(result)
		if err != nil {
			fmt.Println(err)
		}
		w.Write(b)
	})

	http.HandleFunc("/translationimages_old", func(w http.ResponseWriter, r *http.Request) {
		file := r.PostFormValue("file")
		filename := r.PostFormValue("filename")
		uid := strings.Replace(filename, filepath.Ext(filename), "", 1)
		dir := filepath.Join("static", uid)
		filePath := filepath.Join(dir, filename)

		data, err := dataurl.DecodeString(file)
		if err != nil {
			fmt.Println(err)
			http.Error(w, err.Error(), http.StatusBadRequest)
			return
		}
		// fmt.Println(dataURL.Data)
		os.MkdirAll(dir, 0777)
		ioutil.WriteFile(filePath, data.Data, 0644)
		if _, err := os.Stat(dir); os.IsNotExist(err) {
			convertToJpg(filePath)
		}
		if err != nil {
			fmt.Println(err)
			http.Error(w, err.Error(), http.StatusBadRequest)
			return
		}
		listJpg, err := filepath.Glob(filePath + "*.jpg")
		if err != nil {
			fmt.Println(err)
			http.Error(w, err.Error(), http.StatusBadRequest)
			return
		}
		for i, jpgfile := range listJpg {
			listJpg[i] = lowCostSystemToURL(jpgfile)
		}

		b, err := json.Marshal(listJpg)

		if err != nil {
			fmt.Println(err)
			http.Error(w, err.Error(), 500)
			return
		}

		w.Write(b)
	})

	http.HandleFunc("/cardimages", func(w http.ResponseWriter, r *http.Request) {
		var wg sync.WaitGroup

		result := []string{}
		link := r.PostFormValue("url")

		if link != "" {
			doc, err := goquery.NewDocument(link)
			imageURL := ""
			cardsConfig, err := getCardsConfig(link)

			if err != nil {
				fmt.Println(err)
			}
			if _, err := os.Stat(cardsConfig.Dir); os.IsNotExist(err) {
				os.MkdirAll(cardsConfig.Dir, 0744)
				doc.Find(cardsConfig.Site.Filter).Each(func(i int, s *goquery.Selection) {
					wg.Add(1)
					val, _ := s.Attr("src")
					if cardsConfig.Site.Name == "yuyutei" {
						big := strings.Replace(val, "90_126", "front", 1)
						imageURL = yuyuteiURL + big
					} else if cardsConfig.Site.Name == "wsdeck" {
						imageURL = wsDeckURL + val
					}

					go func(url string) {
						defer wg.Done()
						// fmt.Println("dir : ", dir)
						fileName := filepath.Join(cardsConfig.Dir, path.Base(url))
						fileName = strings.Split(fileName, "?")[0]
						out, err := os.Create(fileName)
						if err != nil {
							fmt.Println(err)
						}
						defer out.Close()
						reps, err := http.Get(url)

						if err != nil {
							fmt.Println(err)
						}

						file, err := io.Copy(out, reps.Body)
						if err != nil {
							fmt.Println(err)
						}

						fmt.Println("File", file)
						// fmt.Printf("Link: n-%d __ %v%v\n", i, imageURL, uid)
						defer reps.Body.Close()
						// fmt.Println("image url: ", strings.Replace(fileName, "\\", "/", 1))
						result = append(result, lowCostSystemToURL(fileName))
					}(imageURL)
				})
			} else {
				files, err := ioutil.ReadDir(cardsConfig.Dir)
				if err != nil {
					fmt.Println(err)
				}
				for _, file := range files {
					absPath := filepath.Join(cardsConfig.Dir, file.Name())
					urlPath := lowCostSystemToURL(absPath)
					result = append(result, urlPath)
				}

			}
			wg.Wait()
			fmt.Printf("Finish")
			createCardsCodeFile(cardsConfig.Dir)
		}

		b, err := json.Marshal(result)

		if err != nil {
			http.Error(w, err.Error(), 500)
			return
		}
		w.Write(b)
	})

	http.HandleFunc("/static/", func(w http.ResponseWriter, r *http.Request) {
		// fmt.Println("static", r.URL.Path[1:])
		http.ServeFile(w, r, r.URL.Path[1:])
	})

	http.ListenAndServe(":8010", nil)
}
<|endoftext|>


<commit_before>// Copyright 2014 James McGuire. All rights reserved.
// Use of this source code is governed by the MIT
// license that can be found in the LICENSE file.

package main

import (
	"database/sql"
	"encoding/json"
	"fmt"
	"html"
	"io"
	"io/ioutil"
	"log"
	"math/rand"
	"net/http"
	"net/url"
	"os"
	"os/signal"
	"regexp"
	"runtime"
	"strings"
	"syscall"
	"time"
	"unicode/utf8"

	irc "github.com/sadbox/sadbot/Godeps/_workspace/src/github.com/fluffle/goirc/client"
	_ "github.com/sadbox/sadbot/Godeps/_workspace/src/github.com/go-sql-driver/mysql"
)

var (
	config   Config
	urlRegex = regexp.MustCompile(`(?i)\b((?:https?://|www\d{0,3}[.]|[` +
		`a-z0-9.\-]+[.][a-z]{2,4}/)(?:[^\s()<>]+|\(([^\s()<>]+|(\([^\s()<>]+` +
		`\)))*\))+(?:\(([^\s()<>]+|(\([^\s()<>]+\)))*\)|[^\s` + "`" + `!()\[` +
		`\]{};:'".,<>?«»“”‘’]))`)
	httpRegex      = regexp.MustCompile(`https?://.*`)
	findWhiteSpace = regexp.MustCompile(`\s+`)
	db             *sql.DB
	badWords       = make(map[string]*regexp.Regexp)
)

type Config struct {
	Channel       string
	DBConn        string
	Nick          string
	Ident         string
	FullName      string
	FlickrAPIKey  string
	WolframAPIKey string
	IRCPass       string
	RebuildWords  bool
	Commands      []struct {
		Name string
		Text string
	}
	BadWords []struct {
		Word  string
		Query string
	}
}

// Try and grab the title for any URL's posted in the channel
func sendUrl(channel, unparsedURL string, conn *irc.Conn) {
	if !httpRegex.MatchString(unparsedURL) {
		unparsedURL = `http://` + unparsedURL
	}
	postedUrl, err := url.Parse(unparsedURL)
	if err != nil {
		log.Println(err)
		return
	}
	log.Println("Fetching title for " + postedUrl.String() + " In channel " + channel)

	resp, err := http.Get(postedUrl.String())
	if err != nil {
		log.Println(err)
		return
	}
	defer resp.Body.Close()
	if resp.StatusCode >= 400 {
		log.Println("http server return error.")
		return
	}
	respbody := []byte{}
	if resp.Header.Get("Content-Type") == "" {
		buf := make([]byte, 512)
		bufsize, err := resp.Body.Read(buf)
		if err != nil {
			log.Println("adding content type failed")
		}
		resp.Header.Set("Content-Type", http.DetectContentType(buf[:bufsize]))
		respbody = append(respbody, buf[:bufsize]...)
	}

	if !strings.Contains(resp.Header.Get("Content-Type"), "text/html") {
		log.Println("content-type is not text/html")
		return
	}

	restofbody, err := ioutil.ReadAll(io.LimitReader(resp.Body, 50000))
	if err != nil {
		log.Println("error reading posted link")
		return
	}
	respbody = append(respbody, restofbody...)
	stringbody := string(respbody)
	titlestart := strings.Index(stringbody, "<title>")
	titleend := strings.Index(stringbody, "</title>")
	if titlestart != -1 && titlestart != -1 {
		title := string(respbody[titlestart+7 : titleend])
		title = strings.TrimSpace(title)
		if title != "" && utf8.ValidString(title) {
			// Example:
			// Title: sadbox . org (at sadbox.org)
			title = html.UnescapeString(title)
			title = findWhiteSpace.ReplaceAllString(title, " ")
			title = fmt.Sprintf("Title: %s (at %s)", title, postedUrl.Host)
			log.Println(title)
			conn.Privmsg(channel, title)
		}
	}
}

func logMessage(line *irc.Line, channel, message string) {
	_, err := db.Exec("insert into messages (Nick, Ident, Host, Src, Cmd, Channel,"+
		" Message, Time) values (?, ?, ?, ?, ?, ?, ?, ?)", line.Nick, line.Ident,
		line.Host, line.Src, line.Cmd, channel, message, line.Time)
	if err != nil {
		log.Println(err)
	}
	err = updateWords(line.Nick, message)
	if err != nil {
		log.Println(err)
	}
}

func checkForUrl(channel string, splitmessage []string, conn *irc.Conn) {
	urllist := []string{}
	numlinks := 0
NextWord:
	for _, word := range splitmessage {
		word = strings.TrimSpace(word)
		if urlRegex.MatchString(word) {
			for _, subUrl := range urllist {
				if subUrl == word {
					continue NextWord
				}
			}
			numlinks++
			if numlinks > 3 {
				break
			}
			urllist = append(urllist, word)
			go sendUrl(channel, word, conn)
		}
	}
}

// This function does all the dispatching for various commands
// as well as logging each message to the database
func handleMessage(conn *irc.Conn, line *irc.Line) {
	// This is so that the bot can properly respond to pm's
	var channel string
	if conn.Me.Nick == line.Args[0] {
		channel = line.Nick
	} else {
		channel = line.Args[0]
	}
	message := line.Args[1]
	splitmessage := strings.Split(message, " ")

	// sadbox-only commands!
	if line.Nick == "sadbox" {
		switch strings.TrimSpace(splitmessage[0]) {
		case "!dance":
			go dance(channel, conn)
		case "!audio":
			go conn.Privmsg(channel, "https://sadbox.org/static/stuff/audiophile.html")
		case "!cst":
			go conn.Privmsg(channel, "\u00039,13#CSTMASTERRACE")
		case "!chatter":
			go markov(channel, conn)
		}
	}

	// Special commands
	switch strings.TrimSpace(splitmessage[0]) {
	case "!haata":
		go haata(channel, conn)
	case "!search":
		go googSearch(channel, message, conn)
	case "!ask":
		go wolfram(channel, message, line.Nick, conn)
	case "!meebcast":
		var command string
		if len(splitmessage) >= 2 {
			command = strings.TrimSpace(splitmessage[1])
		}
		go meeba(channel, line.Nick, command, conn)
	}

	// Commands that are read in from the config file
	for _, command := range config.Commands {
		if strings.TrimSpace(splitmessage[0]) == command.Name {
			var response string
			if len(splitmessage) >= 2 {
				response = fmt.Sprintf("%s: %s", splitmessage[1], command.Text)
			} else {
				response = command.Text
			}
			go conn.Privmsg(channel, response)
		}
	}

	// This is what looks at each word and tries to figure out if it's a URL
	go checkForUrl(channel, splitmessage, conn)

	// Shove that shit in the database!
	go logMessage(line, channel, message)
}

func init() {
	log.Println("Starting sadbot")

	runtime.GOMAXPROCS(runtime.NumCPU())

	rand.Seed(time.Now().UTC().UnixNano())

	configfile, err := os.Open("config.json")
	if err != nil {
		log.Fatal(err)
	}
	err = json.NewDecoder(configfile).Decode(&config)
	if err != nil {
		log.Fatal(err)
	}

	for _, word := range config.BadWords {
		badWords[word.Word] = regexp.MustCompile(word.Query)
	}

	log.Println("Loaded config file!")
	log.Printf("Joining channel %s", config.Channel)
	log.Printf("Nick: %s", config.Nick)
	log.Printf("Ident: %s", config.Ident)
	log.Printf("FullName: %s", config.FullName)

	log.Printf("Found %d commands", len(config.Commands))
	for index, command := range config.Commands {
		log.Printf("%d %s: %s", index+1, command.Name, command.Text)
	}
}

func main() {
	var err error
	db, err = sql.Open("mysql", config.DBConn)
	if err != nil {
		log.Fatal(err)
	}
	defer db.Close()
	db.SetMaxIdleConns(100)
	db.SetMaxOpenConns(200)

	go makeMarkov()

	buildchan := make(chan os.Signal, 1)
	signal.Notify(buildchan, syscall.SIGUSR1)
	go func() {
		for _ = range buildchan {
			genTables()
		}
	}()

	c := irc.SimpleClient(config.Nick, config.Ident, config.FullName)

	c.SSL = true

	c.AddHandler(irc.CONNECTED,
		func(conn *irc.Conn, line *irc.Line) {
			conn.Join(config.Channel)
			log.Println("Connected!")
		})

	quit := make(chan bool)

	c.AddHandler(irc.DISCONNECTED,
		func(conn *irc.Conn, line *irc.Line) { quit <- true })

	c.AddHandler("PRIVMSG", handleMessage)
	c.AddHandler("ACTION", handleMessage)

	if err := c.Connect("irc.freenode.net", config.Nick+":"+config.IRCPass); err != nil {
		log.Fatalln("Connection error: %s\n", err)
	}

	<-quit
}
<commit_msg>more cleanup<commit_after>// Copyright 2014 James McGuire. All rights reserved.
// Use of this source code is governed by the MIT
// license that can be found in the LICENSE file.

package main

import (
	"database/sql"
	"encoding/json"
	"fmt"
	"html"
	"io"
	"io/ioutil"
	"log"
	"math/rand"
	"net/http"
	"net/url"
	"os"
	"os/signal"
	"regexp"
	"runtime"
	"strings"
	"syscall"
	"time"
	"unicode/utf8"

	irc "github.com/sadbox/sadbot/Godeps/_workspace/src/github.com/fluffle/goirc/client"
	_ "github.com/sadbox/sadbot/Godeps/_workspace/src/github.com/go-sql-driver/mysql"
)

var (
	config   Config
	urlRegex = regexp.MustCompile(`(?i)\b((?:https?://|www\d{0,3}[.]|[` +
		`a-z0-9.\-]+[.][a-z]{2,4}/)(?:[^\s()<>]+|\(([^\s()<>]+|(\([^\s()<>]+` +
		`\)))*\))+(?:\(([^\s()<>]+|(\([^\s()<>]+\)))*\)|[^\s` + "`" + `!()\[` +
		`\]{};:'".,<>?«»“”‘’]))`)
	httpRegex      = regexp.MustCompile(`https?://.*`)
	findWhiteSpace = regexp.MustCompile(`\s+`)
	db             *sql.DB
	badWords       = make(map[string]*regexp.Regexp)
)

type Config struct {
	Channel       string
	DBConn        string
	Nick          string
	Ident         string
	FullName      string
	FlickrAPIKey  string
	WolframAPIKey string
	IRCPass       string
	RebuildWords  bool
	Commands      []struct {
		Name string
		Text string
	}
	BadWords []struct {
		Word  string
		Query string
	}
}

// Try and grab the title for any URL's posted in the channel
func sendUrl(channel, unparsedURL string, conn *irc.Conn) {
	if !httpRegex.MatchString(unparsedURL) {
		unparsedURL = `http://` + unparsedURL
	}
	postedUrl, err := url.Parse(unparsedURL)
	if err != nil {
		log.Println(err)
		return
	}
	log.Println("Fetching title for " + postedUrl.String() + " In channel " + channel)

	resp, err := http.Get(postedUrl.String())
	if err != nil {
		log.Println(err)
		return
	}
	defer resp.Body.Close()
	if resp.StatusCode >= 400 {
		log.Println("http server return error.")
		return
	}
	respbody := []byte{}
	if resp.Header.Get("Content-Type") == "" {
		buf := make([]byte, 512)
		bufsize, err := resp.Body.Read(buf)
		if err != nil {
			log.Println("adding content type failed")
		}
		resp.Header.Set("Content-Type", http.DetectContentType(buf[:bufsize]))
		respbody = append(respbody, buf[:bufsize]...)
	}

	if !strings.Contains(resp.Header.Get("Content-Type"), "text/html") {
		log.Println("content-type is not text/html")
		return
	}

	restofbody, err := ioutil.ReadAll(io.LimitReader(resp.Body, 50000))
	if err != nil {
		log.Println("error reading posted link")
		return
	}
	respbody = append(respbody, restofbody...)
	stringbody := string(respbody)
	titlestart := strings.Index(stringbody, "<title>")
	titleend := strings.Index(stringbody, "</title>")
	if titlestart != -1 && titlestart != -1 {
		title := string(respbody[titlestart+7 : titleend])
		title = strings.TrimSpace(title)
		if title != "" && utf8.ValidString(title) {
			// Example:
			// Title: sadbox . org (at sadbox.org)
			title = html.UnescapeString(title)
			title = findWhiteSpace.ReplaceAllString(title, " ")
			title = fmt.Sprintf("Title: %s (at %s)", title, postedUrl.Host)
			log.Println(title)
			conn.Privmsg(channel, title)
		}
	}
}

func logMessage(line *irc.Line, channel, message string) {
	_, err := db.Exec("insert into messages (Nick, Ident, Host, Src, Cmd, Channel,"+
		" Message, Time) values (?, ?, ?, ?, ?, ?, ?, ?)", line.Nick, line.Ident,
		line.Host, line.Src, line.Cmd, channel, message, line.Time)
	if err != nil {
		log.Println(err)
	}
	err = updateWords(line.Nick, message)
	if err != nil {
		log.Println(err)
	}
}

func checkForUrl(channel string, splitmessage []string, conn *irc.Conn) {
	urllist := []string{}
	numlinks := 0
NextWord:
	for _, word := range splitmessage {
		word = strings.TrimSpace(word)
		if urlRegex.MatchString(word) {
			for _, subUrl := range urllist {
				if subUrl == word {
					continue NextWord
				}
			}
			numlinks++
			if numlinks > 3 {
				break
			}
			urllist = append(urllist, word)
			go sendUrl(channel, word, conn)
		}
	}
}

// This function does all the dispatching for various commands
// as well as logging each message to the database
func handleMessage(conn *irc.Conn, line *irc.Line) {
	// This is so that the bot can properly respond to pm's
	var channel string
	if conn.Me.Nick == line.Args[0] {
		channel = line.Nick
	} else {
		channel = line.Args[0]
	}
	message := line.Args[1]
	splitmessage := strings.Split(message, " ")
	cmd := strings.TrimSpace(splitmessage[0])

	// sadbox-only commands!
	if line.Nick == "sadbox" {
		switch cmd {
		case "!dance":
			go dance(channel, conn)
		case "!audio":
			go conn.Privmsg(channel, "https://sadbox.org/static/stuff/audiophile.html")
		case "!cst":
			go conn.Privmsg(channel, "\u00039,13#CSTMASTERRACE")
		case "!chatter":
			go markov(channel, conn)
		}
	}

	// Special commands
	switch cmd {
	case "!haata":
		go haata(channel, conn)
	case "!search":
		go googSearch(channel, message, conn)
	case "!ask":
		go wolfram(channel, message, line.Nick, conn)
	case "!meebcast":
		var command string
		if len(splitmessage) >= 2 {
			command = strings.TrimSpace(splitmessage[1])
		}
		go meeba(channel, line.Nick, command, conn)
	}

	// Commands that are read in from the config file
	for _, command := range config.Commands {
		if cmd == command.Name {
			var response string
			if len(splitmessage) >= 2 {
				response = fmt.Sprintf("%s: %s", splitmessage[1], command.Text)
			} else {
				response = command.Text
			}
			go conn.Privmsg(channel, response)
		}
	}

	// This is what looks at each word and tries to figure out if it's a URL
	go checkForUrl(channel, splitmessage, conn)

	// Shove that shit in the database!
	go logMessage(line, channel, message)
}

func init() {
	log.Println("Starting sadbot")

	runtime.GOMAXPROCS(runtime.NumCPU())

	rand.Seed(time.Now().UTC().UnixNano())

	configfile, err := os.Open("config.json")
	if err != nil {
		log.Fatal(err)
	}
	err = json.NewDecoder(configfile).Decode(&config)
	if err != nil {
		log.Fatal(err)
	}

	for _, word := range config.BadWords {
		badWords[word.Word] = regexp.MustCompile(word.Query)
	}

	log.Println("Loaded config file!")
	log.Printf("Joining channel %s", config.Channel)
	log.Printf("Nick: %s", config.Nick)
	log.Printf("Ident: %s", config.Ident)
	log.Printf("FullName: %s", config.FullName)

	log.Printf("Found %d commands", len(config.Commands))
	for index, command := range config.Commands {
		log.Printf("%d %s: %s", index+1, command.Name, command.Text)
	}
}

func main() {
	var err error
	db, err = sql.Open("mysql", config.DBConn)
	if err != nil {
		log.Fatal(err)
	}
	defer db.Close()
	db.SetMaxIdleConns(100)
	db.SetMaxOpenConns(200)

	go makeMarkov()

	buildchan := make(chan os.Signal, 1)
	signal.Notify(buildchan, syscall.SIGUSR1)
	go func() {
		for _ = range buildchan {
			genTables()
		}
	}()

	c := irc.SimpleClient(config.Nick, config.Ident, config.FullName)

	c.SSL = true

	c.AddHandler(irc.CONNECTED,
		func(conn *irc.Conn, line *irc.Line) {
			conn.Join(config.Channel)
			log.Println("Connected!")
		})

	quit := make(chan bool)

	c.AddHandler(irc.DISCONNECTED,
		func(conn *irc.Conn, line *irc.Line) { quit <- true })

	c.AddHandler("PRIVMSG", handleMessage)
	c.AddHandler("ACTION", handleMessage)

	if err := c.Connect("irc.freenode.net", config.Nick+":"+config.IRCPass); err != nil {
		log.Fatalln("Connection error: %s\n", err)
	}

	<-quit
}
<|endoftext|>


<commit_before>package main

import (
	"database/sql"
	"flag"
	"log"
	"log/syslog"
	"net/smtp"
	"os"
	"os/signal"
	"strconv"
	"strings"
	"syscall"
	"time"

	_ "github.com/go-sql-driver/mysql"
	"github.com/johansundell/cocapi"
)

var db *sql.DB
var mysqlUser, mysqlPass, mysqlDb, mysqlHost string
var queryInsertUpdateMember = `INSERT INTO members (tag, name, created, last_updated, active) VALUES (?, ?, null, null, 1) ON DUPLICATE KEY UPDATE member_id=LAST_INSERT_ID(member_id), last_updated = NOW(), active = 1`
var isCocUnderUpdate bool
var failedTries int
var emailTo, emailFrom string
var myClanTag, myKey string
var cocClient cocapi.Client

func init() {

	mysqlDb = "cocsniffer"
	mysqlHost = os.Getenv("MYSQL_COC_HOST")
	mysqlUser = os.Getenv("MYSQL_USER")
	mysqlPass = os.Getenv("MYSQL_PASS")

	emailTo = os.Getenv("EMAIL_TO")
	emailFrom = os.Getenv("EMAIL_FROM")

	myClanTag = os.Getenv("COC_CLANTAG")
	myKey = os.Getenv("COC_KEY")
}

func main() {
	useSyslog := flag.Bool("syslog", false, "Use syslog")
	flag.Parse()
	if *useSyslog {
		logwriter, e := syslog.New(syslog.LOG_NOTICE, "cocsniffer")
		if e == nil {
			log.SetOutput(logwriter)
		}
	}
	db, _ = sql.Open("mysql", mysqlUser+":"+mysqlPass+"@tcp("+mysqlHost+":3306)/"+mysqlDb)
	defer db.Close()

	cocClient = cocapi.NewClient(myKey)

	isCocUnderUpdate = false
	failedTries = 0
	getMembersData(myClanTag)
	//return
	ticker := time.NewTicker(10 * time.Second)
	quit := make(chan struct{})
	go func() {
		for {
			select {
			case <-ticker.C:
				getMembersData(myClanTag)
			case <-quit:
				ticker.Stop()
				return
			}
		}
	}()

	// Wait for SIGINT and SIGTERM (HIT CTRL-C)
	ch := make(chan os.Signal)
	signal.Notify(ch, syscall.SIGINT, syscall.SIGTERM)
	log.Println(<-ch)

	close(quit)
	log.Println("Bye ;)")
}

func getPlayerInfo() error {
	rows, err := db.Query("SELECT tag FROM members WHERE active = 1")
	if err != nil {
		return err
	}

	for rows.Next() {
		var tag string
		err := rows.Scan(&tag)
		if err != nil {
			log.Println(err)
			continue
		}
		player, err := cocClient.GetPlayerInfo(tag)
		if err != nil {
			log.Println(err)
			continue
		}
		if _, err := db.Exec("UPDATE members SET war_stars = ? WHERE tag = ?", player.WarStars, tag); err != nil {
			log.Println(err)
		}
		time.Sleep(250 * time.Millisecond)
	}
	return nil
}

func getMembersData(clan string) error {
	members, err := cocClient.GetMembers(clan)
	if err != nil {
		//log.Println(err)
		reportError(err)
		return err
	}

	if isCocUnderUpdate {
		isCocUnderUpdate = false
		sendEmail("COC Alert", "Servers are up again")
	}
	failedTries = 0

	var ids = make([]string, 0)
	for _, m := range members.Items {
		if result, err := db.Exec(queryInsertUpdateMember, m.Tag, m.Name); err != nil {
			log.Println(err)
		} else {
			if id, err := result.LastInsertId(); err != nil {
				log.Println(err)
			} else {
				donations := 0
				if err := db.QueryRow("SELECT current_donations FROM members WHERE member_id = ?", id).Scan(&donations); err != nil {
					//log.Println(err, "no 1")
				}
				//log.Println(m.Donations, donations)
				if m.Donations != donations && m.Donations != 0 {
					/*if _, err := db.Exec("UPDATE members SET prev_donations = ?, current_donations = ?, last_donation_time = NOW() WHERE member_id = ?", donations, m.Donations, id); err != nil {
						log.Println(err, "no 2")
					}*/
					//if m.Donations > donations {
					if _, err := db.Exec("INSERT donations (member_id, ts, current_donations, prev_donations) VALUES (?, NOW(), ?, ?)", id, m.Donations, donations); err != nil {
						log.Println(err, "no 3")
					}
					//}
				}

				received := 0
				if err := db.QueryRow("SELECT current FROM receive WHERE member_id = ?", id).Scan(&received); err != nil {
					//log.Println(err, "no 4", m.Name, id, m.DonationsReceived)
				}
				if m.DonationsReceived != received && m.DonationsReceived != 0 {
					if _, err := db.Exec("INSERT receive (member_id, ts, current, prev) VALUES ( ?, NOW(), ?, ? )", id, m.DonationsReceived, donations); err != nil {
						log.Println(err)
					}
				}
				ids = append(ids, strconv.Itoa(int(id)))
			}
		}
		if m.Role == "member" && m.Donations >= 1000 {
			log.Println("Found member that should be upgraded", m.Name)
			var alerted int
			db.QueryRow("SELECT alert_sent_donations FROM members WHERE tag = ?", m.Tag).Scan(&alerted)
			if alerted == 0 {
				sendEmail("Member "+m.Name+" should be upgraded", "Member "+m.Name+" should be upgraded")
				db.Exec("UPDATE members SET alert_sent_donations = 1 WHERE tag = ?", m.Tag)
			}
		}
	}
	db.Exec("UPDATE members SET exited = NOW() WHERE member_id NOT IN (" + strings.Join(ids, ", ") + ") AND active = 1")
	db.Exec("UPDATE members SET active = 0 WHERE member_id NOT IN (" + strings.Join(ids, ", ") + ")")
	//log.Println("done members func")
	return nil
}

func reportError(err error) {
	switch t := err.(type) {
	case *cocapi.ServerError:
		if t.ErrorCode == 503 {
			failedTries++
			if failedTries > 3 {
				if !isCocUnderUpdate {
					isCocUnderUpdate = true
					sendEmail("COC Alert", "Servers under update")
				}
			}
		}
		break
	default:
		log.Println("Fatal error coc:", t)
		break
	}
}

func sendEmail(subject, message string) bool {
	body := "To: " + emailTo + "\r\nSubject: " + subject + "\r\n\r\n" + message
	if err := smtp.SendMail("127.0.0.1:25", nil, emailFrom, []string{emailTo}, []byte(body)); err != nil {
		log.Println(err)
		return false
	}
	return true
}
<commit_msg>bug fix<commit_after>package main

import (
	"database/sql"
	"flag"
	"log"
	"log/syslog"
	"net/smtp"
	"os"
	"os/signal"
	"strconv"
	"strings"
	"syscall"
	"time"

	_ "github.com/go-sql-driver/mysql"
	"github.com/johansundell/cocapi"
)

var db *sql.DB
var mysqlUser, mysqlPass, mysqlDb, mysqlHost string
var queryInsertUpdateMember = `INSERT INTO members (tag, name, created, last_updated, active) VALUES (?, ?, null, null, 1) ON DUPLICATE KEY UPDATE member_id=LAST_INSERT_ID(member_id), last_updated = NOW(), active = 1`
var isCocUnderUpdate bool
var failedTries int
var emailTo, emailFrom string
var myClanTag, myKey string
var cocClient cocapi.Client

func init() {

	mysqlDb = "cocsniffer"
	mysqlHost = os.Getenv("MYSQL_COC_HOST")
	mysqlUser = os.Getenv("MYSQL_USER")
	mysqlPass = os.Getenv("MYSQL_PASS")

	emailTo = os.Getenv("EMAIL_TO")
	emailFrom = os.Getenv("EMAIL_FROM")

	myClanTag = os.Getenv("COC_CLANTAG")
	myKey = os.Getenv("COC_KEY")
}

func main() {
	useSyslog := flag.Bool("syslog", false, "Use syslog")
	flag.Parse()
	if *useSyslog {
		logwriter, e := syslog.New(syslog.LOG_NOTICE, "cocsniffer")
		if e == nil {
			log.SetOutput(logwriter)
		}
	}
	db, _ = sql.Open("mysql", mysqlUser+":"+mysqlPass+"@tcp("+mysqlHost+":3306)/"+mysqlDb)
	defer db.Close()

	cocClient = cocapi.NewClient(myKey)

	isCocUnderUpdate = false
	failedTries = 0
	getMembersData(myClanTag)
	//return
	ticker := time.NewTicker(10 * time.Second)
	quit := make(chan struct{})
	go func() {
		for {
			select {
			case <-ticker.C:
				getMembersData(myClanTag)
			case <-quit:
				ticker.Stop()
				return
			}
		}
	}()

	// Wait for SIGINT and SIGTERM (HIT CTRL-C)
	ch := make(chan os.Signal)
	signal.Notify(ch, syscall.SIGINT, syscall.SIGTERM)
	log.Println(<-ch)

	close(quit)
	log.Println("Bye ;)")
}

func getPlayerInfo() error {
	rows, err := db.Query("SELECT tag FROM members WHERE active = 1")
	if err != nil {
		return err
	}

	for rows.Next() {
		var tag string
		err := rows.Scan(&tag)
		if err != nil {
			log.Println(err)
			continue
		}
		player, err := cocClient.GetPlayerInfo(tag)
		if err != nil {
			log.Println(err)
			continue
		}
		if _, err := db.Exec("UPDATE members SET war_stars = ? WHERE tag = ?", player.WarStars, tag); err != nil {
			log.Println(err)
		}
		time.Sleep(250 * time.Millisecond)
	}
	return nil
}

func getMembersData(clan string) error {
	members, err := cocClient.GetMembers(clan)
	if err != nil {
		//log.Println(err)
		reportError(err)
		return err
	}

	if isCocUnderUpdate {
		isCocUnderUpdate = false
		sendEmail("COC Alert", "Servers are up again")
	}
	failedTries = 0

	var ids = make([]string, 0)
	for _, m := range members.Items {
		if result, err := db.Exec(queryInsertUpdateMember, m.Tag, m.Name); err != nil {
			log.Println(err)
		} else {
			if id, err := result.LastInsertId(); err != nil {
				log.Println(err)
			} else {
				donations := 0
				if err := db.QueryRow("SELECT current_donations FROM members WHERE member_id = ?", id).Scan(&donations); err != nil {
					log.Println(err, "no 1")
				}
				//log.Println(m.Donations, donations)
				if m.Donations != donations {
					if _, err := db.Exec("UPDATE members SET prev_donations = ?, current_donations = ?, last_donation_time = NOW() WHERE member_id = ?", donations, m.Donations, id); err != nil {
						log.Println(err)
					}
					//if m.Donations > donations {
					if _, err := db.Exec("INSERT donations (member_id, ts, current_donations, prev_donations) VALUES (?, NOW(), ?, ?)", id, m.Donations, donations); err != nil {
						log.Println(err, "")
					}
					//}
				}

				received := 0
				if err := db.QueryRow("SELECT current_rec FROM members WHERE member_id = ?", id).Scan(&received); err != nil {
					log.Println(err)
				}
				if m.DonationsReceived != received {
					if _, err := db.Exec("UPDATE members SET current_rec = ? WHERE member_id = ?", m.DonationsReceived, id); err != nil {
						log.Println(err)
					}
					if _, err := db.Exec("INSERT receive (member_id, ts, current, prev) VALUES ( ?, NOW(), ?, ? )", id, m.DonationsReceived, donations); err != nil {
						log.Println(err)
					}
				}
				ids = append(ids, strconv.Itoa(int(id)))
			}
		}
		if m.Role == "member" && m.Donations >= 1000 {
			log.Println("Found member that should be upgraded", m.Name)
			var alerted int
			db.QueryRow("SELECT alert_sent_donations FROM members WHERE tag = ?", m.Tag).Scan(&alerted)
			if alerted == 0 {
				sendEmail("Member "+m.Name+" should be upgraded", "Member "+m.Name+" should be upgraded")
				db.Exec("UPDATE members SET alert_sent_donations = 1 WHERE tag = ?", m.Tag)
			}
		}
	}
	db.Exec("UPDATE members SET exited = NOW() WHERE member_id NOT IN (" + strings.Join(ids, ", ") + ") AND active = 1")
	db.Exec("UPDATE members SET active = 0 WHERE member_id NOT IN (" + strings.Join(ids, ", ") + ")")
	//log.Println("done members func")
	return nil
}

func reportError(err error) {
	switch t := err.(type) {
	case *cocapi.ServerError:
		if t.ErrorCode == 503 {
			failedTries++
			if failedTries > 3 {
				if !isCocUnderUpdate {
					isCocUnderUpdate = true
					sendEmail("COC Alert", "Servers under update")
				}
			}
		}
		break
	default:
		log.Println("Fatal error coc:", t)
		break
	}
}

func sendEmail(subject, message string) bool {
	body := "To: " + emailTo + "\r\nSubject: " + subject + "\r\n\r\n" + message
	if err := smtp.SendMail("127.0.0.1:25", nil, emailFrom, []string{emailTo}, []byte(body)); err != nil {
		log.Println(err)
		return false
	}
	return true
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"github.com/gorilla/mux"
	"github.com/justinas/alice"
	"github.com/justinas/nosurf"
	"github.com/unrolled/render"
	"net/http"
)

var (
	env  = flag.String("env", "prod", "environment ('prod' or 'dev')")
	port = flag.String("port", "5000", "server port")
)

// should be settings
const (
	staticURL    = "/static/"
	staticDir    = "./static/"
	devServerURL = "http://localhost:8080"
)

func main() {

	flag.Parse()

	router := mux.NewRouter()

	router.PathPrefix(staticURL).Handler(
		http.StripPrefix(staticURL, http.FileServer(http.Dir(staticDir))))

	render := render.New()

	router.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		ctx := map[string]string{
			"staticURL": staticURL,
		}
		render.HTML(w, http.StatusOK, "index", ctx)
	})

	chain := alice.New(nosurf.NewPure).Then(router)

	if err := http.ListenAndServe(":"+*port, chain); err != nil {
		panic(err)
	}

}
<commit_msg>Adding JWT support<commit_after>package main

import (
	"flag"
	"fmt"
	"github.com/auth0/go-jwt-middleware"
	"github.com/dgrijalva/jwt-go"
	"github.com/gorilla/context"
	"github.com/gorilla/mux"
	"github.com/justinas/alice"
	"github.com/justinas/nosurf"
	"github.com/unrolled/render"
	"net/http"
)

var (
	env  = flag.String("env", "prod", "environment ('prod' or 'dev')")
	port = flag.String("port", "5000", "server port")
)

// should be settings
const (
	staticURL    = "/static/"
	staticDir    = "./static/"
	devServerURL = "http://localhost:8080"
)

func main() {

	flag.Parse()

	router := mux.NewRouter()

	router.PathPrefix(staticURL).Handler(
		http.StripPrefix(staticURL, http.FileServer(http.Dir(staticDir))))

	render := render.New()

	router.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		ctx := map[string]string{
			"staticURL": staticURL,
		}
		render.HTML(w, http.StatusOK, "index", ctx)
	})

	router.HandleFunc("/auth/", func(w http.ResponseWriter, r *http.Request) {
		user := context.Get(r, "user")
		fmt.Fprintf(w, "This is an authenticated request:\n")
		if user == nil {
			fmt.Fprintf(w, "No token set")
		} else {
			fmt.Fprintf(w, "Claim content:\n")
			for k, v := range user.(*jwt.Token).Claims {
				fmt.Fprintf(w, "%s: \t%#v", k, v)
			}
		}
	})

	jwtMiddleware := jwtmiddleware.New(jwtmiddleware.Options{
		ValidationKeyGetter: func(token *jwt.Token) (interface{}, error) {
			return []byte("My Secret"), nil
		},
		CredentialsOptional: true,
		SigningMethod:       jwt.SigningMethodHS256,
	})

	chain := alice.New(nosurf.NewPure, jwtMiddleware.Handler).Then(router)

	if err := http.ListenAndServe(":"+*port, chain); err != nil {
		panic(err)
	}

}
<|endoftext|>


<commit_before>package main

import (
	"github.com/hashicorp/terraform/plugin"
	"github.ibm.com/emerging-tech/terraform-provider-softlayer/softlayer"
)

func main() {
	plugin.Serve(&plugin.ServeOpts{
		ProviderFunc: softlayer.Provider,
	})
}
<commit_msg>Try specifying vcs suffix to get 'go get' working<commit_after>package main

import (
	"github.com/hashicorp/terraform/plugin"
	"github.ibm.com/emerging-tech/terraform-provider-softlayer.git/softlayer"
)

func main() {
	plugin.Serve(&plugin.ServeOpts{
		ProviderFunc: softlayer.Provider,
	})
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"net/http"
	"os"
	"strconv"
	"sync"
	"time"

	"github.com/influxdb/influxdb/client/v2"
)

var (
	listURL               = flag.String("list-url", "", "URL to JSON-formatted list of URLs to check")
	pollInterval          = flag.Uint("poll-interval", 60, "Polling interval (in seconds)")
	requestTimeout        = flag.Uint("req-timeout", 20, "Timeout for each HTTP request, must be < than poll-interval")
	debug                 = flag.Bool("debug", false, "To enable a more verbose mode for debugging")
	influxHost            = flag.String("influx-host", "http://localhost:8086", "InfluxDB host")
	influxDBName          = flag.String("influx-db", "uptime", "InfluxDB database name")
	influxMeasurementName = flag.String("influx-measurement", "page_load_time", "InfluxDB measurement to write to")

	// Global list of urls to fetch
	list *urlList

	errTimeout = errors.New("Request timed out")
)

type fetchableURL struct {
	URL         string
	Description string
	// More fields to come
}

type urlList struct {
	urls []fetchableURL
	lock sync.Mutex
}

type request struct {
	URL        string
	Duration   time.Duration
	StatusCode int
	done       chan struct{} // closes when request is done
	err        error
}

func main() {
	// Validate command line arguments
	if len(*listURL) == 0 {
		fmt.Println("Must pass list-url")
		os.Exit(1)
	}

	if *debug {
		fmt.Printf("Connecting to DB '%s' at %s\n", *influxDBName, *influxHost)
	}
	influxClient, err := client.NewHTTPClient(client.HTTPConfig{
		Addr: *influxHost,
	})
	if err != nil {
		fmt.Println("Error creating InfluxDB client:", err)
	}

	// Initialize global list
	list = &urlList{
		lock: sync.Mutex{},
	}

	// Fetch the initial list of URLs
	if err := updateURLsToFecth(*listURL, list); err != nil {
		fmt.Println("Error getting list of URLs:", err)
		os.Exit(1)
	}

	// Update the list of URLs to fetch periodically in a separate goroutine
	go func() {
		c := time.Tick(1 * time.Minute)
		for range c {
			if err := updateURLsToFecth(*listURL, list); err != nil {
				fmt.Println("Error updating list of URLs:", err)
			}
		}
	}()

	// Channel to send concurrent responses on
	respCh := make(chan request, 10)

	// Read responses (concurrently) as they come in
	go handleReponses(influxClient, len(list.urls), respCh)

	// The actual fetching of URLs
	for {
		list.lock.Lock()
		for _, u := range list.urls {
			// Setup request object
			req := request{
				URL:  u.URL,
				done: make(chan struct{}),
			}
			// Send it in it's own goroutine
			go send(req, time.Second*time.Duration(int(*requestTimeout)), respCh)
		}
		list.lock.Unlock()
		time.Sleep(time.Second * time.Duration(int(*pollInterval)))
	}
}

func send(req request, timeout time.Duration, resp chan request) {
	if *debug {
		fmt.Println("Sending a request to", req.URL)
	}
	start := time.Now()
	// Send the HTTP request in it's own goroutine to be able to abort the request if reached timeout
	go func() {
		resp, err := http.Get(req.URL)
		if err != nil {
			req.err = err
		} else {
			req.StatusCode = resp.StatusCode
		}
		// Indicate the request is finished
		close(req.done)

	}()
	// Either the request finished, or the timeout triggers
	select {
	case <-req.done:
		// How long did the request take, only applies if not timed out
		req.Duration = time.Since(start)
	case <-time.After(timeout):
		req.Duration = timeout
		req.err = errTimeout
	}
	// Send back on the response channel
	resp <- req
}

// Handles responses sent on passed channel
func handleReponses(cl client.Client, batchSize int, ch chan request) {
	bpConfig := client.BatchPointsConfig{
		Database:  *influxDBName,
		Precision: "ms",
	}
	// Create initial batch
	bp, _ := client.NewBatchPoints(bpConfig)
	for {
		select {
		case req := <-ch:
			tags := map[string]string{"url": req.URL}
			fields := map[string]interface{}{
				"duration": req.Duration,
			}
			if req.err != nil {
				if req.err == errTimeout {
					tags["status_code"] = strconv.Itoa(408)
				} else {
					// If error from the HTTP request, print to stdout and dont add new point
					fmt.Printf("Error for %s: %s\n", req.URL, req.err)
					continue
				}
			} else {
				tags["status_code"] = strconv.Itoa(req.StatusCode)
			}
			pt, err := client.NewPoint(*influxMeasurementName, tags, fields)
			if err != nil {
				fmt.Println("Error creating InfluxDB point: ", err.Error())
			}
			// Add point to current batch
			bp.AddPoint(pt)
			if *debug {
				fmt.Printf("%s\t%d\t%s\n", req.URL, req.StatusCode, req.Duration)
			}
			// If should send current batch
			if len(bp.Points()) == batchSize {
				if *debug {
					fmt.Println("Writing points to InfluxDB")
				}
				cl.Write(bp)
				// Create a new batch
				bp, _ = client.NewBatchPoints(bpConfig)
			}
		}
	}
}

// Update the list of urls to fecth
func updateURLsToFecth(url string, ul *urlList) error {
	if *debug {
		fmt.Println("Checking for URLs to fetch...")
	}
	resp, err := http.Get(url)
	if err != nil {
		return err
	}
	// Decode the JSON response
	decoder := json.NewDecoder(resp.Body)
	fetchableURLs := []fetchableURL{}
	if err := decoder.Decode(&fetchableURLs); err != nil {
		return err
	}
	// If new URLs were fetched, clear out the old ones and add the new ones
	list.lock.Lock()
	list.urls = fetchableURLs
	list.lock.Unlock()
	if *debug {
		fmt.Println("Current list of URLs is:")
		for _, u := range list.urls {
			fmt.Println(u.URL)
		}
	}
	return nil
}

func init() {
	flag.Parse()
}
<commit_msg>Write actual ns as duration, not formatted string<commit_after>package main

import (
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"net/http"
	"os"
	"strconv"
	"sync"
	"time"

	"github.com/influxdb/influxdb/client/v2"
)

var (
	listURL               = flag.String("list-url", "", "URL to JSON-formatted list of URLs to check")
	pollInterval          = flag.Uint("poll-interval", 60, "Polling interval (in seconds)")
	requestTimeout        = flag.Uint("req-timeout", 20, "Timeout for each HTTP request, must be < than poll-interval")
	debug                 = flag.Bool("debug", false, "To enable a more verbose mode for debugging")
	influxHost            = flag.String("influx-host", "http://localhost:8086", "InfluxDB host")
	influxDBName          = flag.String("influx-db", "uptime", "InfluxDB database name")
	influxMeasurementName = flag.String("influx-measurement", "page_load_time", "InfluxDB measurement to write to")

	// Global list of urls to fetch
	list *urlList

	errTimeout = errors.New("Request timed out")
)

type fetchableURL struct {
	URL         string
	Description string
	// More fields to come
}

type urlList struct {
	urls []fetchableURL
	lock sync.Mutex
}

type request struct {
	URL        string
	Duration   time.Duration
	StatusCode int
	done       chan struct{} // closes when request is done
	err        error
}

func main() {
	// Validate command line arguments
	if len(*listURL) == 0 {
		fmt.Println("Must pass list-url")
		os.Exit(1)
	}

	if *debug {
		fmt.Printf("Connecting to DB '%s' at %s\n", *influxDBName, *influxHost)
	}
	influxClient, err := client.NewHTTPClient(client.HTTPConfig{
		Addr: *influxHost,
	})
	if err != nil {
		fmt.Println("Error creating InfluxDB client:", err)
	}

	// Initialize global list
	list = &urlList{
		lock: sync.Mutex{},
	}

	// Fetch the initial list of URLs
	if err := updateURLsToFecth(*listURL, list); err != nil {
		fmt.Println("Error getting list of URLs:", err)
		os.Exit(1)
	}

	// Update the list of URLs to fetch periodically in a separate goroutine
	go func() {
		c := time.Tick(1 * time.Minute)
		for range c {
			if err := updateURLsToFecth(*listURL, list); err != nil {
				fmt.Println("Error updating list of URLs:", err)
			}
		}
	}()

	// Channel to send concurrent responses on
	respCh := make(chan request, 10)

	// Read responses (concurrently) as they come in
	go handleReponses(influxClient, len(list.urls), respCh)

	// The actual fetching of URLs
	for {
		list.lock.Lock()
		for _, u := range list.urls {
			// Setup request object
			req := request{
				URL:  u.URL,
				done: make(chan struct{}),
			}
			// Send it in it's own goroutine
			go send(req, time.Second*time.Duration(int(*requestTimeout)), respCh)
		}
		list.lock.Unlock()
		time.Sleep(time.Second * time.Duration(int(*pollInterval)))
	}
}

func send(req request, timeout time.Duration, resp chan request) {
	if *debug {
		fmt.Println("Sending a request to", req.URL)
	}
	start := time.Now()
	// Send the HTTP request in it's own goroutine to be able to abort the request if reached timeout
	go func() {
		resp, err := http.Get(req.URL)
		if err != nil {
			req.err = err
		} else {
			req.StatusCode = resp.StatusCode
		}
		// Indicate the request is finished
		close(req.done)

	}()
	// Either the request finished, or the timeout triggers
	select {
	case <-req.done:
		// How long did the request take, only applies if not timed out
		req.Duration = time.Since(start)
	case <-time.After(timeout):
		req.Duration = timeout
		req.err = errTimeout
	}
	// Send back on the response channel
	resp <- req
}

// Handles responses sent on passed channel
func handleReponses(cl client.Client, batchSize int, ch chan request) {
	bpConfig := client.BatchPointsConfig{
		Database:  *influxDBName,
		Precision: "ns",
	}
	// Create initial batch
	bp, _ := client.NewBatchPoints(bpConfig)
	for {
		select {
		case req := <-ch:
			tags := map[string]string{"url": req.URL}
			fields := map[string]interface{}{
				"duration": strconv.FormatInt(req.Duration.Nanoseconds(), 10),
			}
			if req.err != nil {
				if req.err == errTimeout {
					tags["status_code"] = strconv.Itoa(408)
				} else {
					// If error from the HTTP request, print to stdout and dont add new point
					fmt.Printf("Error for %s: %s\n", req.URL, req.err)
					continue
				}
			} else {
				tags["status_code"] = strconv.Itoa(req.StatusCode)
			}
			pt, err := client.NewPoint(*influxMeasurementName, tags, fields)
			if err != nil {
				fmt.Println("Error creating InfluxDB point: ", err.Error())
			}
			// Add point to current batch
			bp.AddPoint(pt)
			if *debug {
				fmt.Printf("%s\t%d\t%s\n", req.URL, req.StatusCode, req.Duration)
			}
			// If should send current batch
			if len(bp.Points()) == batchSize {
				if *debug {
					fmt.Println("Writing points to InfluxDB")
				}
				cl.Write(bp)
				// Create a new batch
				bp, _ = client.NewBatchPoints(bpConfig)
			}
		}
	}
}

// Update the list of urls to fecth
func updateURLsToFecth(url string, ul *urlList) error {
	if *debug {
		fmt.Println("Checking for URLs to fetch...")
	}
	resp, err := http.Get(url)
	if err != nil {
		return err
	}
	// Decode the JSON response
	decoder := json.NewDecoder(resp.Body)
	fetchableURLs := []fetchableURL{}
	if err := decoder.Decode(&fetchableURLs); err != nil {
		return err
	}
	// If new URLs were fetched, clear out the old ones and add the new ones
	list.lock.Lock()
	list.urls = fetchableURLs
	list.lock.Unlock()
	if *debug {
		fmt.Println("Current list of URLs is:")
		for _, u := range list.urls {
			fmt.Println(u.URL)
		}
	}
	return nil
}

func init() {
	flag.Parse()
}
<|endoftext|>


<commit_before>// Package main is the main entrypoint to the worker cleaner
package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"strings"
	"time"

	"github.com/GoogleCloudPlatform/kubernetes/pkg/client"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/fields"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/labels"
	"github.com/golang/glog"
	"gopkg.in/redis.v3"
)

const (
	workerKey        = "resque:worker"
	workersKey       = "resque:workers"
	statProcessedKey = "resque:stat:processed"
	statFailedKey    = "resque:stat:failed"
)

func main() {
	glog.Info("Kubernetes-Resque Worker Cleanup Service")

	namespace := os.Getenv("NAMESPACE")
	kubeClient := getKubernetesClient()
	redisClient := getRedisClient(true)

	glog.Info("Polling worker list every 5 min")
	for {

		runningPods := getLivingWorkers(kubeClient, namespace)
		redisWorkers := getWorkersFromRedis(redisClient)

		deadWorkers := getDeadWorkers(runningPods, redisWorkers)

		for _, dead := range deadWorkers {
			removeDeadWorker(redisClient, dead)
		}

		time.Sleep(5 * time.Minute)
	}
}

type resqueJob struct {
	Queue   string `json:queue`
	Payload json.RawMessage
}

func newResqueJob(data []byte) (resqueJob, error) {
	var j resqueJob
	err := json.Unmarshal(data, &j)
	return j, err
}

func requeueStuckJob(c *redis.Client, job []byte) {
	_, err := newResqueJob(job)
	if err != nil {
		glog.Warning("Could not deserialize job")
	}
}

func removeDeadWorker(c *redis.Client, worker string) {
	job, err := c.Get(fmt.Sprintf("%s:%s", workerKey, worker)).Bytes()
	if err != nil {
		glog.Warningf("Could not fetch job for obj: %s", err)
		return
	}
	if len(job) != 0 {
		// requeueStuckJob(c, job)
		glog.Warning("Job not empty for worker, skipping removal (not implemented)")
		return
	}

	c.Pipelined(func(pipe *redis.Pipeline) error {
		pipe.SRem(workersKey, worker)
		pipe.Del(fmt.Sprintf("%s:%s", workerKey, worker))
		pipe.Del(fmt.Sprintf("%s:%s:started", workerKey, worker))
		pipe.Del(fmt.Sprintf("%s:%s:shutdown", workerKey, worker))

		// delete stats
		pipe.Del(fmt.Sprintf("%s:%s", statProcessedKey, worker))
		pipe.Del(fmt.Sprintf("%s:%s", statFailedKey, worker))

		return nil
	})
}

func getDeadWorkers(running []string, listedWorkers []string) []string {
	var diff []string

	for _, worker := range listedWorkers {
		workerName := strings.SplitN(worker, ":", 2)[0]
		found := false
		for _, pod := range running {
			if workerName == pod {
				found = true
				break
			}
		}
		if !found {
			diff = append(diff, worker)
		}
	}

	return diff
}

func getLivingWorkers(c *client.Client, namespace string) []string {
	selector := "role=worker,app=vapor"
	labels, err := labels.Parse(selector)
	if err != nil {
		glog.Fatalf("Failed to parse selector %q: %v", selector, err)
	}

	pods, err := c.Pods(namespace).List(labels, fields.Everything())
	if err != nil {
		glog.Fatal("Failed to get pods")
	}

	podItems := pods.Items
	podNames := make([]string, len(podItems))
	for i, pod := range podItems {
		podNames[i] = pod.Name
	}
	return podNames
}

func getWorkersFromRedis(c *redis.Client) []string {
	workers, err := c.SMembers("resque:workers").Result()
	if err != nil {
		glog.Fatal("Failed to connect to Redis")
	}

	return workers
}

func getRedisClient(useSentinel bool) *redis.Client {
	if useSentinel {
		addr := fmt.Sprintf("%s:%s", os.Getenv("REDIS_SENTINEL_SERVICE_HOST"), os.Getenv("REDIS_SENTINEL_SERVICE_PORT"))
		return redis.NewFailoverClient(&redis.FailoverOptions{
			MasterName:    "mymaster",
			SentinelAddrs: []string{addr},
		})
	} else {
		return redis.NewClient(&redis.Options{
			Addr: "localhost:6379",
		})
	}
}

func getKubernetesClient() *client.Client {
	kubernetesService := os.Getenv("KUBERNETES_SERVICE_HOST")
	if kubernetesService == "" {
		glog.Fatalf("Please specify the Kubernetes server")
	}
	apiServer := fmt.Sprintf("https://%s:%s", kubernetesService, os.Getenv("KUBERNETES_SERVICE_PORT"))

	token, err := ioutil.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/token")
	if err != nil {
		glog.Fatalf("No service account token found")
	}

	config := client.Config{
		Host:        apiServer,
		BearerToken: string(token),
		Insecure:    true,
	}

	c, err := client.New(&config)
	if err != nil {
		glog.Fatalf("Failed to make client: %v", err)
	}
	return c
}
<commit_msg>Better nil check<commit_after>// Package main is the main entrypoint to the worker cleaner
package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"strings"
	"time"

	"github.com/GoogleCloudPlatform/kubernetes/pkg/client"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/fields"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/labels"
	"github.com/golang/glog"
	"gopkg.in/redis.v3"
)

const (
	workerKey        = "resque:worker"
	workersKey       = "resque:workers"
	statProcessedKey = "resque:stat:processed"
	statFailedKey    = "resque:stat:failed"
)

func main() {
	glog.Info("Kubernetes-Resque Worker Cleanup Service")

	namespace := os.Getenv("NAMESPACE")
	kubeClient := getKubernetesClient()
	redisClient := getRedisClient(true)

	glog.Info("Polling worker list every 5 min")
	for {

		runningPods := getLivingWorkers(kubeClient, namespace)
		redisWorkers := getWorkersFromRedis(redisClient)

		deadWorkers := getDeadWorkers(runningPods, redisWorkers)

		for _, dead := range deadWorkers {
			removeDeadWorker(redisClient, dead)
		}

		time.Sleep(5 * time.Minute)
	}
}

type resqueJob struct {
	Queue   string `json:queue`
	Payload json.RawMessage
}

func newResqueJob(data []byte) (resqueJob, error) {
	var j resqueJob
	err := json.Unmarshal(data, &j)
	return j, err
}

func requeueStuckJob(c *redis.Client, job []byte) {
	_, err := newResqueJob(job)
	if err != nil {
		glog.Warning("Could not deserialize job")
	}
}

func removeDeadWorker(c *redis.Client, worker string) {
	_, err := c.Get(fmt.Sprintf("%s:%s", workerKey, worker)).Bytes()
	if err != nil {
		// if error is redis: nil we just ignore
		if err.Error() != "redis: nil" {
			glog.Warningf("Could not fetch job for obj: %s", err)
			return
		}
	} else {
		// requeueStuckJob(c, job)
		glog.Warning("Job not empty for worker, skipping removal (not implemented)")
		return
	}

	c.Pipelined(func(pipe *redis.Pipeline) error {
		pipe.SRem(workersKey, worker)
		pipe.Del(fmt.Sprintf("%s:%s", workerKey, worker))
		pipe.Del(fmt.Sprintf("%s:%s:started", workerKey, worker))
		pipe.Del(fmt.Sprintf("%s:%s:shutdown", workerKey, worker))

		// delete stats
		pipe.Del(fmt.Sprintf("%s:%s", statProcessedKey, worker))
		pipe.Del(fmt.Sprintf("%s:%s", statFailedKey, worker))

		return nil
	})
}

func getDeadWorkers(running []string, listedWorkers []string) []string {
	var diff []string

	for _, worker := range listedWorkers {
		workerName := strings.SplitN(worker, ":", 2)[0]
		found := false
		for _, pod := range running {
			if workerName == pod {
				found = true
				break
			}
		}
		if !found {
			diff = append(diff, worker)
		}
	}

	return diff
}

func getLivingWorkers(c *client.Client, namespace string) []string {
	selector := "role=worker,app=vapor"
	labels, err := labels.Parse(selector)
	if err != nil {
		glog.Fatalf("Failed to parse selector %q: %v", selector, err)
	}

	pods, err := c.Pods(namespace).List(labels, fields.Everything())
	if err != nil {
		glog.Fatal("Failed to get pods")
	}

	podItems := pods.Items
	podNames := make([]string, len(podItems))
	for i, pod := range podItems {
		podNames[i] = pod.Name
	}
	return podNames
}

func getWorkersFromRedis(c *redis.Client) []string {
	workers, err := c.SMembers("resque:workers").Result()
	if err != nil {
		glog.Fatal("Failed to connect to Redis")
	}

	return workers
}

func getRedisClient(useSentinel bool) *redis.Client {
	if useSentinel {
		addr := fmt.Sprintf("%s:%s", os.Getenv("REDIS_SENTINEL_SERVICE_HOST"), os.Getenv("REDIS_SENTINEL_SERVICE_PORT"))
		return redis.NewFailoverClient(&redis.FailoverOptions{
			MasterName:    "mymaster",
			SentinelAddrs: []string{addr},
		})
	} else {
		return redis.NewClient(&redis.Options{
			Addr: "localhost:6379",
		})
	}
}

func getKubernetesClient() *client.Client {
	kubernetesService := os.Getenv("KUBERNETES_SERVICE_HOST")
	if kubernetesService == "" {
		glog.Fatalf("Please specify the Kubernetes server")
	}
	apiServer := fmt.Sprintf("https://%s:%s", kubernetesService, os.Getenv("KUBERNETES_SERVICE_PORT"))

	token, err := ioutil.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/token")
	if err != nil {
		glog.Fatalf("No service account token found")
	}

	config := client.Config{
		Host:        apiServer,
		BearerToken: string(token),
		Insecure:    true,
	}

	c, err := client.New(&config)
	if err != nil {
		glog.Fatalf("Failed to make client: %v", err)
	}
	return c
}
<|endoftext|>