<commit_before>package config

import (
	"testing"
)

const configFile = "../../fixtures/docker/config.json"
const irrelevantConfigFile = "../../fixtures/docker/config.json.irrelevant"
const invalidConfigFile = "../../fixtures/docker/config.json.invalid"
const corruptConfigFile = "../../fixtures/docker/config.json.corrupt"

func TestLoad(t *testing.T) {
	examples := map[string]string{
		"registry.company.io":     "user1:pass1",
		"registry.hub.docker.com": "user2:pass2",
	}

	c, err := Load(configFile)

	if err != nil {
		t.Fatalf("Error while loading '%s': %s", configFile, err.Error())
	}

	for registry, expected := range examples {
		username, password, defined := c.GetCredentials(registry)

		if !defined {
			t.Fatalf("Unable to get credentials from registry: %s", registry)
		}

		value := username + ":" + password

		if value != expected {
			t.Fatalf(
				"Unexpected 'username:password' for registry '%s': '%s' (expected: '%s')",
				registry,
				value,
				expected,
			)
		}
	}

	if c.IsEmpty() {
		t.Fatalf("Expected to load data set from config file: %s", configFile)
	}
}

func TestLoadWithIrrelevantConfigFile(t *testing.T) {
	c, err := Load(irrelevantConfigFile)

	if err != nil {
		t.Fatalf("Expected to not fail while loading irrelevant config file '%s'\nLoaded: %#v", irrelevantConfigFile, c)
	}

	if !c.IsEmpty() {
		t.Fatalf("Expected to load empty data set from irrelevant config file '%s'\nLoaded: %#v", irrelevantConfigFile, c)
	}
}

func TestLoadWithInvalidConfigFile(t *testing.T) {
	c, err := Load(invalidConfigFile)

	if err == nil {
		t.Fatalf("Expected to fail while loading invalid config file '%s'\nLoaded: %#v", invalidConfigFile, c)
	}
}

func TestLoadWithAbsentConfigFile(t *testing.T) {
	var err error

	_, err = Load("i/exist/only/in/your/magination")
	if err == nil {
		t.Fatalf("Expected to fail while trying to load absent config file")
	}

	DefaultDockerJSON = "i/exist/only/in/your/magination"

	_, err = Load("i/exist/only/in/your/magination")
	if err != nil {
		t.Fatalf(
			"Expected NOT to fail while trying to load absent config file from a default path: %s",
			err.Error(),
		)
	}
}

// Corrupt file = valid JSON file with badly encoded auth
func TestLoadWithCorruptConfigFile(t *testing.T) {
	const corruptRegistry = "registry.valencia.io"

	_, err := Load(corruptConfigFile)
	if err == nil {
		t.Fatalf("Expected to fail while loading corrupt config file")
	}

	DefaultDockerJSON = corruptConfigFile

	c, err := Load(corruptConfigFile)
	if err != nil {
		t.Fatalf(
			"Expected NOT to fail while loading corrupt config file from a default path: %s",
			err.Error(),
		)
	}

	_, _, defined := c.GetCredentials(corruptRegistry)
	if defined {
		t.Fatalf("Should NOT get credentials from a corrupt registry record: %s", corruptRegistry)
	}
}
<commit_msg>NORELEASE: Bump coverage for `docker/config`<commit_after>package config

import (
	"testing"
)

const configFile = "../../fixtures/docker/config.json"
const irrelevantConfigFile = "../../fixtures/docker/config.json.irrelevant"
const invalidConfigFile = "../../fixtures/docker/config.json.invalid"
const corruptConfigFile = "../../fixtures/docker/config.json.corrupt"

func TestGetRegistryAuth(t *testing.T) {
	examples := map[string]string{
		"registry.company.io":     "eyAidXNlcm5hbWUiOiAidXNlcjEiLCAicGFzc3dvcmQiOiAicGFzczEiIH0=",
		"registry.hub.docker.com": "eyAidXNlcm5hbWUiOiAidXNlcjIiLCAicGFzc3dvcmQiOiAicGFzczIiIH0=",
		"registry.mindundi.org":   "",
	}

	c, err := Load(configFile)

	if err != nil {
		t.Fatalf("Error while loading '%s': %s", configFile, err.Error())
	}

	for registry, expectedAuth := range examples {
		auth := c.GetRegistryAuth(registry)

		if auth != expectedAuth {
			t.Fatalf(
				"Unexpected authentication string for registry '%s': %s (expected: %s)",
				registry,
				auth,
				expectedAuth,
			)
		}
	}
}

func TestLoad(t *testing.T) {
	examples := map[string]string{
		"registry.company.io":     "user1:pass1",
		"registry.hub.docker.com": "user2:pass2",
	}

	c, err := Load(configFile)

	if err != nil {
		t.Fatalf("Error while loading '%s': %s", configFile, err.Error())
	}

	for registry, expected := range examples {
		username, password, defined := c.GetCredentials(registry)

		if !defined {
			t.Fatalf("Unable to get credentials from registry: %s", registry)
		}

		value := username + ":" + password

		if value != expected {
			t.Fatalf(
				"Unexpected 'username:password' for registry '%s': '%s' (expected: '%s')",
				registry,
				value,
				expected,
			)
		}
	}

	if c.IsEmpty() {
		t.Fatalf("Expected to load data set from config file: %s", configFile)
	}
}

func TestLoadWithIrrelevantConfigFile(t *testing.T) {
	c, err := Load(irrelevantConfigFile)

	if err != nil {
		t.Fatalf("Expected to not fail while loading irrelevant config file '%s'\nLoaded: %#v", irrelevantConfigFile, c)
	}

	if !c.IsEmpty() {
		t.Fatalf("Expected to load empty data set from irrelevant config file '%s'\nLoaded: %#v", irrelevantConfigFile, c)
	}
}

func TestLoadWithInvalidConfigFile(t *testing.T) {
	c, err := Load(invalidConfigFile)

	if err == nil {
		t.Fatalf("Expected to fail while loading invalid config file '%s'\nLoaded: %#v", invalidConfigFile, c)
	}
}

func TestLoadWithAbsentConfigFile(t *testing.T) {
	var err error

	_, err = Load("i/exist/only/in/your/magination")
	if err == nil {
		t.Fatalf("Expected to fail while trying to load absent config file")
	}

	DefaultDockerJSON = "i/exist/only/in/your/magination"

	_, err = Load("i/exist/only/in/your/magination")
	if err != nil {
		t.Fatalf(
			"Expected NOT to fail while trying to load absent config file from a default path: %s",
			err.Error(),
		)
	}
}

// Corrupt file = valid JSON file with badly encoded auth
func TestLoadWithCorruptConfigFile(t *testing.T) {
	const corruptRegistry = "registry.valencia.io"

	_, err := Load(corruptConfigFile)
	if err == nil {
		t.Fatalf("Expected to fail while loading corrupt config file")
	}

	DefaultDockerJSON = corruptConfigFile

	c, err := Load(corruptConfigFile)
	if err != nil {
		t.Fatalf(
			"Expected NOT to fail while loading corrupt config file from a default path: %s",
			err.Error(),
		)
	}

	_, _, defined := c.GetCredentials(corruptRegistry)
	if defined {
		t.Fatalf("Should NOT get credentials from a corrupt registry record: %s", corruptRegistry)
	}
}
<|endoftext|>


<commit_before>package lib

import (
	"sync"

	"github.com/streadway/amqp"
	"github.com/travis-ci/worker/lib/backend"
	"golang.org/x/net/context"
)

// A ProcessorPool spins up multiple Processors handling build jobs from the
// same queue.
type ProcessorPool struct {
	Context   context.Context
	Conn      *amqp.Connection
	Provider  backend.Provider
	Generator BuildScriptGenerator
	Canceller Canceller

	processorsLock sync.Mutex
	processors     []*Processor
}

// Run starts up a number of processors and connects them to the given queue.
// This method stalls until all processors have finished.
func (p *ProcessorPool) Run(poolSize uint16, queueName string) error {
	queue, err := NewJobQueue(p.Conn, queueName)
	if err != nil {
		return err
	}

	var wg sync.WaitGroup

	var i uint16
	for i = 0; i < poolSize; i++ {
		wg.Add(1)
		go func() {
			p.processor(queue)
			wg.Done()
		}()
	}

	wg.Wait()

	return nil
}

// GracefulShutdown causes each processor in the pool to start its graceful
// shutdown.
func (p *ProcessorPool) GracefulShutdown() {
	p.processorsLock.Lock()
	defer p.processorsLock.Unlock()

	for _, processor := range p.processors {
		processor.GracefulShutdown()
	}
}

func (p *ProcessorPool) processor(queue *JobQueue) {
	buildJobChan, err := queue.Jobs()
	if err != nil {
		// TODO: log
		return
	}

	proc := NewProcessor(p.Context, buildJobChan, p.Provider, p.Generator, p.Canceller)

	p.processorsLock.Lock()
	p.processors = append(p.processors, proc)
	p.processorsLock.Unlock()

	proc.Run()
}
<commit_msg>processor-pool: log if we can't subscribe to build queue<commit_after>package lib

import (
	"sync"

	"github.com/streadway/amqp"
	"github.com/travis-ci/worker/lib/backend"
	"github.com/travis-ci/worker/lib/context"
	gocontext "golang.org/x/net/context"
)

// A ProcessorPool spins up multiple Processors handling build jobs from the
// same queue.
type ProcessorPool struct {
	Context   gocontext.Context
	Conn      *amqp.Connection
	Provider  backend.Provider
	Generator BuildScriptGenerator
	Canceller Canceller

	processorsLock sync.Mutex
	processors     []*Processor
}

// Run starts up a number of processors and connects them to the given queue.
// This method stalls until all processors have finished.
func (p *ProcessorPool) Run(poolSize uint16, queueName string) error {
	queue, err := NewJobQueue(p.Conn, queueName)
	if err != nil {
		return err
	}

	var wg sync.WaitGroup

	var i uint16
	for i = 0; i < poolSize; i++ {
		wg.Add(1)
		go func() {
			p.processor(queue)
			wg.Done()
		}()
	}

	wg.Wait()

	return nil
}

// GracefulShutdown causes each processor in the pool to start its graceful
// shutdown.
func (p *ProcessorPool) GracefulShutdown() {
	p.processorsLock.Lock()
	defer p.processorsLock.Unlock()

	for _, processor := range p.processors {
		processor.GracefulShutdown()
	}
}

func (p *ProcessorPool) processor(queue *JobQueue) {
	buildJobChan, err := queue.Jobs()
	if err != nil {
		context.LoggerFromContext(p.Context).WithField("err", err).Error("couldn't subscribe to queue")
		return
	}

	proc := NewProcessor(p.Context, buildJobChan, p.Provider, p.Generator, p.Canceller)

	p.processorsLock.Lock()
	p.processors = append(p.processors, proc)
	p.processorsLock.Unlock()

	proc.Run()
}
<|endoftext|>


<commit_before>// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.
//
// Contributor: Julien Vehent jvehent@mozilla.com [:ulfr]
package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"log"
	"mig"
	"mig/client"
	migdb "mig/database"
	"os"
	"strings"

	"github.com/bobappleyard/readline"
)

var useShortNames bool

func main() {
	var err error
	defer func() {
		if e := recover(); e != nil {
			fmt.Printf("FATAL: %v\n", e)
		}
	}()
	homedir := client.FindHomedir()
	// command line options
	var config = flag.String("c", homedir+"/.migrc", "Load configuration from file")
	var shortnames = flag.Bool("s", false, "Shorten all agent names to first and last 5 characters)")
	var quiet = flag.Bool("q", false, "don't display banners and prompts")
	flag.Parse()

	// silence extra output
	out := os.Stdout
	if *quiet {
		out.Close()
		out, err = os.Open(os.DevNull)
		if err != nil {
			panic(err)
		}
	}
	defer out.Close()

	fmt.Fprintf(out, "\x1b[32;1m"+banner+"\x1b[0m")

	// append a space after completion
	readline.CompletionAppendChar = 0x20

	// shorten names for obfuscation, useful during demoes
	if *shortnames {
		useShortNames = true
	}

	// instanciate an API client
	conf, err := client.ReadConfiguration(*config)
	if err != nil {
		panic(err)
	}
	cli := client.NewClient(conf)

	err = printStatus(cli)
	if err != nil {
		log.Fatal(err)
	}
	fmt.Fprintf(out, "\nConnected to %s. Exit with \x1b[32;1mctrl+d\x1b[0m. Type \x1b[32;1mhelp\x1b[0m for help.\n", cli.Conf.API.URL)
	for {
		// completion
		var symbols = []string{"action", "agent", "create", "command", "help", "exit", "showcfg",
			"status", "investigator", "search", "where", "and"}
		readline.Completer = func(query, ctx string) []string {
			var res []string
			for _, sym := range symbols {
				if strings.HasPrefix(sym, query) {
					res = append(res, sym)
				}
			}
			return res
		}

		input, err := readline.String("\x1b[32;1mmig>\x1b[0m ")
		if err == io.EOF {
			break
		}
		if err != nil {
			fmt.Println("error: ", err)
			break
		}
		orders := strings.Split(strings.TrimSpace(input), " ")
		switch orders[0] {
		case "action":
			if len(orders) == 2 {
				err = actionReader(input, cli)
				if err != nil {
					log.Println(err)
				}
			} else {
				fmt.Println("error: missing action id in 'action <id>'")
			}
		case "agent":
			err = agentReader(input, cli)
			if err != nil {
				log.Println(err)
			}
		case "create":
			if len(orders) == 2 {
				switch orders[1] {
				case "action":
					var a mig.Action
					err = actionLauncher(a, cli)
				case "investigator":
					err = investigatorCreator(cli)
				default:
					fmt.Printf("unknown order 'create %s'\n", orders[1])
				}
				if err != nil {
					log.Println(err)
				}
			} else {
				fmt.Println("error: missing order, must be 'create <action|investigator>'")
			}
		case "command":
			err = commandReader(input, cli)
			if err != nil {
				log.Println(err)
			}
		case "exit":
			fmt.Printf("exit\n")
			goto exit
		case "help":
			fmt.Printf(`The following orders are available:
action <id|new>		enter interactive action mode. if <id> is given, go to reader mode. if "new" is given, enter launcher mode.
agent <id>		enter interactive agent reader mode for agent <id>
create action		enter interactive action creation mode
create investigator	create a new investigator, will prompt for name and public key
command <id>		enter command reader mode for command <id>
exit			leave
help			show this help
investigator <id>	enter interactive investigator management mode for investigator <id>
search			perform a search. see "search help" for more information.
showcfg			display running configuration
status			display platform status: connected agents, latest actions, ...
`)
		case "investigator":
			err = investigatorReader(input, cli)
			if err != nil {
				log.Println(err)
			}
		case "search":
			err = search(input, cli)
			if err != nil {
				log.Println(err)
			}
		case "showcfg":
			fmt.Printf("homedir = %s\n[api]\n    url = %s\n[gpg]\n    home = %s\n    keyid = %s\n",
				cli.Conf.API.URL, cli.Conf.Homedir, cli.Conf.GPG.Home, cli.Conf.GPG.KeyID)
		case "status":
			err = printStatus(cli)
			if err != nil {
				log.Println(err)
			}
		case "":
			break
		default:
			fmt.Printf("Unknown order '%s'\n", orders[0])
		}
		readline.AddHistory(input)
	}
exit:
	fmt.Fprintf(out, footer)
}

func printStatus(cli client.Client) (err error) {
	defer func() {
		if e := recover(); e != nil {
			err = fmt.Errorf("printStatus() -> %v", e)
		}
	}()
	st, err := cli.GetAPIResource("dashboard")
	if err != nil {
		panic(err)
	}
	agtout := make([]string, 5)
	agtout[0] = "Agents Summary:"
	actout := make([]string, 2)
	actout[0] = "Latest Actions:"
	actout[1] = "----    ID      ---- + ----         Name         ---- + ----    Date    ---- + ---- Investigators ----"
	for _, item := range st.Collection.Items {
		for _, data := range item.Data {
			switch data.Name {
			case "action":
				idstr, name, datestr, invs, err := actionPrintShort(data.Value)
				if err != nil {
					panic(err)
				}
				str := fmt.Sprintf("%s   %s   %s   %s", idstr, name, datestr, invs)
				actout = append(actout, str)
			case "active agents":
				agtout[1] = fmt.Sprintf("* %.0f active agents", data.Value)
			case "agents versions count":
				bData, err := json.Marshal(data.Value)
				if err != nil {
					panic(err)
				}
				var sum []migdb.AgentsSum
				err = json.Unmarshal(bData, &sum)
				if err != nil {
					panic(err)
				}
				for _, asum := range sum {
					s := fmt.Sprintf("* version %s: %.0f agent", asum.Version, asum.Count)
					if asum.Count > 1 {
						s += "s"
					}
					agtout = append(agtout, s)
				}
			case "agents started in the last 24 hours":
				agtout[2] = fmt.Sprintf("* %.0f agents (re)started in the last 24 hours", data.Value)
			case "endpoints running 2 or more agents":
				agtout[3] = fmt.Sprintf("* %.0f endpoints are running 2 or more agents", data.Value)
			case "endpoints that have disappeared over last 7 days":
				agtout[4] = fmt.Sprintf("* %.0f endpoints have disappeared over the last 7 days", data.Value)
			}
		}
	}
	fmt.Println("\x1b[31;1m+------\x1b[0m")
	for _, s := range agtout {
		fmt.Println("\x1b[31;1m|\x1b[0m " + s)
	}
	fmt.Println("\x1b[31;1m|\x1b[0m")
	for _, s := range actout {
		fmt.Println("\x1b[31;1m|\x1b[0m " + s)
		if len(actout) < 2 {
			fmt.Println("\x1b[31;1m|\x1b[0m * None")
			break
		}
	}
	fmt.Println("\x1b[31;1m+------\x1b[0m")
	return
}

func shorten(p string) string {
	if len(p) < 8 {
		return p
	}
	out := p[0:7]
	out += "."
	out += "."
	out += "."
	if len(p) > 18 {
		out += p[len(p)-7 : len(p)]
	}
	return out
}

var banner string = `
## ##                                     _.---._     .---.
# # # /-\ ---||  |    /\         __...---' .---. '---'-.   '.
#   #|   | / ||  |   /--\    .-''__.--' _.'( | )'.  '.  '._ :
#   # \_/ ---| \_ \_/    \ .'__-'_ .--'' ._'---'_.-.  '.   '-'.
     ###                         ~ -._ -._''---. -.    '-._   '.
      # |\ |\    /---------|          ~ -.._ _ _ _ ..-_ '.  '-._''--.._
      # | \| \  / |- |__ | |                       -~ -._  '-.  -. '-._''--.._.--''.
     ###|  \  \/  ---__| | |                            ~ ~-.__     -._  '-.__   '. '.
          #####                                               ~~ ~---...__ _    ._ .' '.
          #      /\  --- /-\ |--|----                                    ~  ~--.....--~
          # ### /--\  | |   ||-\  //
          #####/    \ |  \_/ |  \//__
`

var footer string = `
            .-._   _ _ _ _ _ _ _ _
 .-''-.__.-'Oo  '-' ' ' ' ' ' ' ' '-.
'.___ '    .   .--_'-' '-' '-' _'-' '._
 V: V 'vv-'   '_   '.       .'  _..' '.'.
   '=.____.=_.--'   :_.__.__:_   '.   : :
           (((____.-'        '-.  /   : :
                             (((-'\ .' /
                           _____..'  .'
                          '-._____.-'
              Gators are going back underwater.
`
<commit_msg>[minor] save and load history in console<commit_after>// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.
//
// Contributor: Julien Vehent jvehent@mozilla.com [:ulfr]
package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"github.com/bobappleyard/readline"
	"io"
	"log"
	"mig"
	"mig/client"
	migdb "mig/database"
	"os"
	"strconv"
	"strings"
)

var useShortNames bool

func main() {
	var err error
	defer func() {
		if e := recover(); e != nil {
			fmt.Printf("FATAL: %v\n", e)
		}
	}()
	homedir := client.FindHomedir()
	// command line options
	var config = flag.String("c", homedir+"/.migrc", "Load configuration from file")
	var shortnames = flag.Bool("s", false, "Shorten all agent names to first and last 5 characters)")
	var quiet = flag.Bool("q", false, "don't display banners and prompts")
	flag.Parse()

	// silence extra output
	out := os.Stdout
	if *quiet {
		out.Close()
		out, err = os.Open(os.DevNull)
		if err != nil {
			panic(err)
		}
	}
	defer out.Close()

	fmt.Fprintf(out, "\x1b[32;1m"+banner+"\x1b[0m")

	// append a space after completion
	readline.CompletionAppendChar = 0x20

	// shorten names for obfuscation, useful during demoes
	if *shortnames {
		useShortNames = true
	// load history
	historyfile := homedir + "/.mig_history"
	fi, err := os.Stat(historyfile)
	if err == nil && fi.Size() > 0 {
		err = readline.LoadHistory(historyfile)
		if err != nil {
			fmt.Fprintf(os.Stderr, "failed to load history from %s\n", historyfile)
		}
	}

	// instanciate an API client
	conf, err := client.ReadConfiguration(*config)
	if err != nil {
		panic(err)
	}
	cli := client.NewClient(conf)

	err = printStatus(cli)
	if err != nil {
		log.Fatal(err)
	}
	fmt.Fprintf(out, "\nConnected to %s. Exit with \x1b[32;1mctrl+d\x1b[0m. Type \x1b[32;1mhelp\x1b[0m for help.\n", cli.Conf.API.URL)
	for {
		// completion
		var symbols = []string{"action", "agent", "create", "command", "help", "history",
			"exit", "showcfg", "status", "investigator", "search", "where", "and"}
		readline.Completer = func(query, ctx string) []string {
			var res []string
			for _, sym := range symbols {
				if strings.HasPrefix(sym, query) {
					res = append(res, sym)
				}
			}
			return res
		}

		input, err := readline.String("\x1b[32;1mmig>\x1b[0m ")
		if err == io.EOF {
			break
		}
		if err != nil {
			fmt.Println("error: ", err)
			break
		}
		orders := strings.Split(strings.TrimSpace(input), " ")
		switch orders[0] {
		case "action":
			if len(orders) == 2 {
				err = actionReader(input, cli)
				if err != nil {
					log.Println(err)
				}
			} else {
				fmt.Println("error: missing action id in 'action <id>'")
			}
		case "agent":
			err = agentReader(input, cli)
			if err != nil {
				log.Println(err)
			}
		case "create":
			if len(orders) == 2 {
				switch orders[1] {
				case "action":
					var a mig.Action
					err = actionLauncher(a, cli)
				case "investigator":
					err = investigatorCreator(cli)
				default:
					fmt.Printf("unknown order 'create %s'\n", orders[1])
				}
				if err != nil {
					log.Println(err)
				}
			} else {
				fmt.Println("error: missing order, must be 'create <action|investigator>'")
			}
		case "command":
			err = commandReader(input, cli)
			if err != nil {
				log.Println(err)
			}
		case "exit":
			fmt.Printf("exit\n")
			goto exit
		case "help":
			fmt.Printf(`The following orders are available:
action <id|new>		enter interactive action mode. if <id> is given, go to reader mode. if "new" is given, enter launcher mode.
agent <id>		enter interactive agent reader mode for agent <id>
create action		enter interactive action creation mode
create investigator	create a new investigator, will prompt for name and public key
command <id>		enter command reader mode for command <id>
exit			leave
help			show this help
history <count>		print last <count> entries in history. count=10 by default.
investigator <id>	enter interactive investigator management mode for investigator <id>
search			perform a search. see "search help" for more information.
showcfg			display running configuration
status			display platform status: connected agents, latest actions, ...
`)
		case "history":
			var count int64 = 10
			if len(orders) > 1 {
				count, err = strconv.ParseInt(orders[1], 10, 64)
				if err != nil {
					log.Println(err)
					break
				}
			}
			for i := readline.HistorySize(); i > 0 && count > 0; i, count = i-1, count-1 {
				fmt.Println(readline.GetHistory(i - 1))
			}
		case "investigator":
			err = investigatorReader(input, cli)
			if err != nil {
				log.Println(err)
			}
		case "search":
			err = search(input, cli)
			if err != nil {
				log.Println(err)
			}
		case "showcfg":
			fmt.Printf("homedir = %s\n[api]\n    url = %s\n[gpg]\n    home = %s\n    keyid = %s\n",
				cli.Conf.API.URL, cli.Conf.Homedir, cli.Conf.GPG.Home, cli.Conf.GPG.KeyID)
		case "status":
			err = printStatus(cli)
			if err != nil {
				log.Println(err)
			}
		case "":
			break
		default:
			fmt.Printf("Unknown order '%s'\n", orders[0])
		}
		readline.AddHistory(input)
	}
exit:
	fmt.Fprintf(out, footer)
	err = readline.SaveHistory(historyfile)
	if err != nil {
		fmt.Fprintf(os.Stderr, "failed to save history to %s\n", historyfile)
	}
}

func printStatus(cli client.Client) (err error) {
	defer func() {
		if e := recover(); e != nil {
			err = fmt.Errorf("printStatus() -> %v", e)
		}
	}()
	st, err := cli.GetAPIResource("dashboard")
	if err != nil {
		panic(err)
	}
	agtout := make([]string, 5)
	agtout[0] = "Agents Summary:"
	actout := make([]string, 2)
	actout[0] = "Latest Actions:"
	actout[1] = "----    ID      ---- + ----         Name         ---- + ----    Date    ---- + ---- Investigators ----"
	for _, item := range st.Collection.Items {
		for _, data := range item.Data {
			switch data.Name {
			case "action":
				idstr, name, datestr, invs, err := actionPrintShort(data.Value)
				if err != nil {
					panic(err)
				}
				str := fmt.Sprintf("%s   %s   %s   %s", idstr, name, datestr, invs)
				actout = append(actout, str)
			case "active agents":
				agtout[1] = fmt.Sprintf("* %.0f active agents", data.Value)
			case "agents versions count":
				bData, err := json.Marshal(data.Value)
				if err != nil {
					panic(err)
				}
				var sum []migdb.AgentsSum
				err = json.Unmarshal(bData, &sum)
				if err != nil {
					panic(err)
				}
				for _, asum := range sum {
					s := fmt.Sprintf("* version %s: %.0f agent", asum.Version, asum.Count)
					if asum.Count > 1 {
						s += "s"
					}
					agtout = append(agtout, s)
				}
			case "agents started in the last 24 hours":
				agtout[2] = fmt.Sprintf("* %.0f agents (re)started in the last 24 hours", data.Value)
			case "endpoints running 2 or more agents":
				agtout[3] = fmt.Sprintf("* %.0f endpoints are running 2 or more agents", data.Value)
			case "endpoints that have disappeared over last 7 days":
				agtout[4] = fmt.Sprintf("* %.0f endpoints have disappeared over the last 7 days", data.Value)
			}
		}
	}
	fmt.Println("\x1b[31;1m+------\x1b[0m")
	for _, s := range agtout {
		fmt.Println("\x1b[31;1m|\x1b[0m " + s)
	}
	fmt.Println("\x1b[31;1m|\x1b[0m")
	for _, s := range actout {
		fmt.Println("\x1b[31;1m|\x1b[0m " + s)
		if len(actout) < 2 {
			fmt.Println("\x1b[31;1m|\x1b[0m * None")
			break
		}
	}
	fmt.Println("\x1b[31;1m+------\x1b[0m")
	return
}

func shorten(p string) string {
	if len(p) < 8 {
		return p
	}
	out := p[0:7]
	out += "."
	out += "."
	out += "."
	if len(p) > 18 {
		out += p[len(p)-7 : len(p)]
	}
	return out
}

var banner string = `
## ##                                     _.---._     .---.
# # # /-\ ---||  |    /\         __...---' .---. '---'-.   '.
#   #|   | / ||  |   /--\    .-''__.--' _.'( | )'.  '.  '._ :
#   # \_/ ---| \_ \_/    \ .'__-'_ .--'' ._'---'_.-.  '.   '-'.
     ###                         ~ -._ -._''---. -.    '-._   '.
      # |\ |\    /---------|          ~ -.._ _ _ _ ..-_ '.  '-._''--.._
      # | \| \  / |- |__ | |                       -~ -._  '-.  -. '-._''--.._.--''.
     ###|  \  \/  ---__| | |                            ~ ~-.__     -._  '-.__   '. '.
          #####                                               ~~ ~---...__ _    ._ .' '.
          #      /\  --- /-\ |--|----                                    ~  ~--.....--~
          # ### /--\  | |   ||-\  //
          #####/    \ |  \_/ |  \//__
`

var footer string = `
            .-._   _ _ _ _ _ _ _ _
 .-''-.__.-'Oo  '-' ' ' ' ' ' ' ' '-.
'.___ '    .   .--_'-' '-' '-' _'-' '._
 V: V 'vv-'   '_   '.       .'  _..' '.'.
   '=.____.=_.--'   :_.__.__:_   '.   : :
           (((____.-'        '-.  /   : :
                             (((-'\ .' /
                           _____..'  .'
                          '-._____.-'
              Gators are going back underwater.
`
<|endoftext|>


<commit_before>// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package token

import (
	"fmt"
	"testing"
)

func checkPos(t *testing.T, msg string, p, q Position) {
	if p.Filename != q.Filename {
		t.Errorf("%s: expected filename = %q; got %q", msg, q.Filename, p.Filename)
	}
	if p.Offset != q.Offset {
		t.Errorf("%s: expected offset = %d; got %d", msg, q.Offset, p.Offset)
	}
	if p.Line != q.Line {
		t.Errorf("%s: expected line = %d; got %d", msg, q.Line, p.Line)
	}
	if p.Column != q.Column {
		t.Errorf("%s: expected column = %d; got %d", msg, q.Column, p.Column)
	}
}

func TestNoPos(t *testing.T) {
	if NoPos.IsValid() {
		t.Errorf("NoPos should not be valid")
	}
	var fset *FileSet
	checkPos(t, "nil NoPos", fset.Position(NoPos), Position{})
	fset = NewFileSet()
	checkPos(t, "fset NoPos", fset.Position(NoPos), Position{})
}

var tests = []struct {
	filename string
	source   []byte // may be nil
	size     int
	lines    []int
}{
	{"a", []byte{}, 0, []int{}},
	{"b", []byte("01234"), 5, []int{0}},
	{"c", []byte("\n\n\n\n\n\n\n\n\n"), 9, []int{0, 1, 2, 3, 4, 5, 6, 7, 8}},
	{"d", nil, 100, []int{0, 5, 10, 20, 30, 70, 71, 72, 80, 85, 90, 99}},
	{"e", nil, 777, []int{0, 80, 100, 120, 130, 180, 267, 455, 500, 567, 620}},
	{"f", []byte("package p\n\nimport \"fmt\""), 23, []int{0, 10, 11}},
	{"g", []byte("package p\n\nimport \"fmt\"\n"), 24, []int{0, 10, 11}},
	{"h", []byte("package p\n\nimport \"fmt\"\n "), 25, []int{0, 10, 11, 24}},
}

func linecol(lines []int, offs int) (int, int) {
	prevLineOffs := 0
	for line, lineOffs := range lines {
		if offs < lineOffs {
			return line, offs - prevLineOffs + 1
		}
		prevLineOffs = lineOffs
	}
	return len(lines), offs - prevLineOffs + 1
}

func verifyPositions(t *testing.T, fset *FileSet, f *File, lines []int) {
	for offs := 0; offs < f.Size(); offs++ {
		p := f.Pos(offs)
		offs2 := f.Offset(p)
		if offs2 != offs {
			t.Errorf("%s, Offset: expected offset %d; got %d", f.Name(), offs, offs2)
		}
		line, col := linecol(lines, offs)
		msg := fmt.Sprintf("%s (offs = %d, p = %d)", f.Name(), offs, p)
		checkPos(t, msg, f.Position(f.Pos(offs)), Position{f.Name(), offs, line, col})
		checkPos(t, msg, fset.Position(p), Position{f.Name(), offs, line, col})
	}
}

func makeTestSource(size int, lines []int) []byte {
	src := make([]byte, size)
	for _, offs := range lines {
		if offs > 0 {
			src[offs-1] = '\n'
		}
	}
	return src
}

func TestPositions(t *testing.T) {
	const delta = 7 // a non-zero base offset increment
	fset := NewFileSet()
	for _, test := range tests {
		// verify consistency of test case
		if test.source != nil && len(test.source) != test.size {
			t.Errorf("%s: inconsistent test case: expected file size %d; got %d", test.filename, test.size, len(test.source))
		}

		// add file and verify name and size
		f := fset.AddFile(test.filename, fset.Base()+delta, test.size)
		if f.Name() != test.filename {
			t.Errorf("expected filename %q; got %q", test.filename, f.Name())
		}
		if f.Size() != test.size {
			t.Errorf("%s: expected file size %d; got %d", f.Name(), test.size, f.Size())
		}
		if fset.File(f.Pos(0)) != f {
			t.Errorf("%s: f.Pos(0) was not found in f", f.Name())
		}

		// add lines individually and verify all positions
		for i, offset := range test.lines {
			f.AddLine(offset)
			if f.LineCount() != i+1 {
				t.Errorf("%s, AddLine: expected line count %d; got %d", f.Name(), i+1, f.LineCount())
			}
			// adding the same offset again should be ignored
			f.AddLine(offset)
			if f.LineCount() != i+1 {
				t.Errorf("%s, AddLine: expected unchanged line count %d; got %d", f.Name(), i+1, f.LineCount())
			}
			verifyPositions(t, fset, f, test.lines[0:i+1])
		}

		// add lines with SetLines and verify all positions
		if ok := f.SetLines(test.lines); !ok {
			t.Errorf("%s: SetLines failed", f.Name())
		}
		if f.LineCount() != len(test.lines) {
			t.Errorf("%s, SetLines: expected line count %d; got %d", f.Name(), len(test.lines), f.LineCount())
		}
		verifyPositions(t, fset, f, test.lines)

		// add lines with SetLinesForContent and verify all positions
		src := test.source
		if src == nil {
			// no test source available - create one from scratch
			src = makeTestSource(test.size, test.lines)
		}
		f.SetLinesForContent(src)
		if f.LineCount() != len(test.lines) {
			t.Errorf("%s, SetLinesForContent: expected line count %d; got %d", f.Name(), len(test.lines), f.LineCount())
		}
		verifyPositions(t, fset, f, test.lines)
	}
}

func TestLineInfo(t *testing.T) {
	fset := NewFileSet()
	f := fset.AddFile("foo", fset.Base(), 500)
	lines := []int{0, 42, 77, 100, 210, 220, 277, 300, 333, 401}
	// add lines individually and provide alternative line information
	for _, offs := range lines {
		f.AddLine(offs)
		f.AddLineInfo(offs, "bar", 42)
	}
	// verify positions for all offsets
	for offs := 0; offs <= f.Size(); offs++ {
		p := f.Pos(offs)
		_, col := linecol(lines, offs)
		msg := fmt.Sprintf("%s (offs = %d, p = %d)", f.Name(), offs, p)
		checkPos(t, msg, f.Position(f.Pos(offs)), Position{"bar", offs, 42, col})
		checkPos(t, msg, fset.Position(p), Position{"bar", offs, 42, col})
	}
}

func TestFiles(t *testing.T) {
	fset := NewFileSet()
	for i, test := range tests {
		fset.AddFile(test.filename, fset.Base(), test.size)
		j := 0
		fset.Iterate(func(f *File) bool {
			if f.Name() != tests[j].filename {
				t.Errorf("expected filename = %s; got %s", tests[j].filename, f.Name())
			}
			j++
			return true
		})
		if j != i+1 {
			t.Errorf("expected %d files; got %d", i+1, j)
		}
	}
}
<commit_msg>go/token: add test for concurrent use of FileSet.Pos<commit_after>// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package token

import (
	"fmt"
	"math/rand"
	"sync"
	"testing"
)

func checkPos(t *testing.T, msg string, p, q Position) {
	if p.Filename != q.Filename {
		t.Errorf("%s: expected filename = %q; got %q", msg, q.Filename, p.Filename)
	}
	if p.Offset != q.Offset {
		t.Errorf("%s: expected offset = %d; got %d", msg, q.Offset, p.Offset)
	}
	if p.Line != q.Line {
		t.Errorf("%s: expected line = %d; got %d", msg, q.Line, p.Line)
	}
	if p.Column != q.Column {
		t.Errorf("%s: expected column = %d; got %d", msg, q.Column, p.Column)
	}
}

func TestNoPos(t *testing.T) {
	if NoPos.IsValid() {
		t.Errorf("NoPos should not be valid")
	}
	var fset *FileSet
	checkPos(t, "nil NoPos", fset.Position(NoPos), Position{})
	fset = NewFileSet()
	checkPos(t, "fset NoPos", fset.Position(NoPos), Position{})
}

var tests = []struct {
	filename string
	source   []byte // may be nil
	size     int
	lines    []int
}{
	{"a", []byte{}, 0, []int{}},
	{"b", []byte("01234"), 5, []int{0}},
	{"c", []byte("\n\n\n\n\n\n\n\n\n"), 9, []int{0, 1, 2, 3, 4, 5, 6, 7, 8}},
	{"d", nil, 100, []int{0, 5, 10, 20, 30, 70, 71, 72, 80, 85, 90, 99}},
	{"e", nil, 777, []int{0, 80, 100, 120, 130, 180, 267, 455, 500, 567, 620}},
	{"f", []byte("package p\n\nimport \"fmt\""), 23, []int{0, 10, 11}},
	{"g", []byte("package p\n\nimport \"fmt\"\n"), 24, []int{0, 10, 11}},
	{"h", []byte("package p\n\nimport \"fmt\"\n "), 25, []int{0, 10, 11, 24}},
}

func linecol(lines []int, offs int) (int, int) {
	prevLineOffs := 0
	for line, lineOffs := range lines {
		if offs < lineOffs {
			return line, offs - prevLineOffs + 1
		}
		prevLineOffs = lineOffs
	}
	return len(lines), offs - prevLineOffs + 1
}

func verifyPositions(t *testing.T, fset *FileSet, f *File, lines []int) {
	for offs := 0; offs < f.Size(); offs++ {
		p := f.Pos(offs)
		offs2 := f.Offset(p)
		if offs2 != offs {
			t.Errorf("%s, Offset: expected offset %d; got %d", f.Name(), offs, offs2)
		}
		line, col := linecol(lines, offs)
		msg := fmt.Sprintf("%s (offs = %d, p = %d)", f.Name(), offs, p)
		checkPos(t, msg, f.Position(f.Pos(offs)), Position{f.Name(), offs, line, col})
		checkPos(t, msg, fset.Position(p), Position{f.Name(), offs, line, col})
	}
}

func makeTestSource(size int, lines []int) []byte {
	src := make([]byte, size)
	for _, offs := range lines {
		if offs > 0 {
			src[offs-1] = '\n'
		}
	}
	return src
}

func TestPositions(t *testing.T) {
	const delta = 7 // a non-zero base offset increment
	fset := NewFileSet()
	for _, test := range tests {
		// verify consistency of test case
		if test.source != nil && len(test.source) != test.size {
			t.Errorf("%s: inconsistent test case: expected file size %d; got %d", test.filename, test.size, len(test.source))
		}

		// add file and verify name and size
		f := fset.AddFile(test.filename, fset.Base()+delta, test.size)
		if f.Name() != test.filename {
			t.Errorf("expected filename %q; got %q", test.filename, f.Name())
		}
		if f.Size() != test.size {
			t.Errorf("%s: expected file size %d; got %d", f.Name(), test.size, f.Size())
		}
		if fset.File(f.Pos(0)) != f {
			t.Errorf("%s: f.Pos(0) was not found in f", f.Name())
		}

		// add lines individually and verify all positions
		for i, offset := range test.lines {
			f.AddLine(offset)
			if f.LineCount() != i+1 {
				t.Errorf("%s, AddLine: expected line count %d; got %d", f.Name(), i+1, f.LineCount())
			}
			// adding the same offset again should be ignored
			f.AddLine(offset)
			if f.LineCount() != i+1 {
				t.Errorf("%s, AddLine: expected unchanged line count %d; got %d", f.Name(), i+1, f.LineCount())
			}
			verifyPositions(t, fset, f, test.lines[0:i+1])
		}

		// add lines with SetLines and verify all positions
		if ok := f.SetLines(test.lines); !ok {
			t.Errorf("%s: SetLines failed", f.Name())
		}
		if f.LineCount() != len(test.lines) {
			t.Errorf("%s, SetLines: expected line count %d; got %d", f.Name(), len(test.lines), f.LineCount())
		}
		verifyPositions(t, fset, f, test.lines)

		// add lines with SetLinesForContent and verify all positions
		src := test.source
		if src == nil {
			// no test source available - create one from scratch
			src = makeTestSource(test.size, test.lines)
		}
		f.SetLinesForContent(src)
		if f.LineCount() != len(test.lines) {
			t.Errorf("%s, SetLinesForContent: expected line count %d; got %d", f.Name(), len(test.lines), f.LineCount())
		}
		verifyPositions(t, fset, f, test.lines)
	}
}

func TestLineInfo(t *testing.T) {
	fset := NewFileSet()
	f := fset.AddFile("foo", fset.Base(), 500)
	lines := []int{0, 42, 77, 100, 210, 220, 277, 300, 333, 401}
	// add lines individually and provide alternative line information
	for _, offs := range lines {
		f.AddLine(offs)
		f.AddLineInfo(offs, "bar", 42)
	}
	// verify positions for all offsets
	for offs := 0; offs <= f.Size(); offs++ {
		p := f.Pos(offs)
		_, col := linecol(lines, offs)
		msg := fmt.Sprintf("%s (offs = %d, p = %d)", f.Name(), offs, p)
		checkPos(t, msg, f.Position(f.Pos(offs)), Position{"bar", offs, 42, col})
		checkPos(t, msg, fset.Position(p), Position{"bar", offs, 42, col})
	}
}

func TestFiles(t *testing.T) {
	fset := NewFileSet()
	for i, test := range tests {
		fset.AddFile(test.filename, fset.Base(), test.size)
		j := 0
		fset.Iterate(func(f *File) bool {
			if f.Name() != tests[j].filename {
				t.Errorf("expected filename = %s; got %s", tests[j].filename, f.Name())
			}
			j++
			return true
		})
		if j != i+1 {
			t.Errorf("expected %d files; got %d", i+1, j)
		}
	}
}

// issue 4345. Test concurrent use of FileSet.Pos does not trigger a
// race in the FileSet position cache.
func TestFileSetRace(t *testing.T) {
	fset := NewFileSet()
	for i := 0; i < 100; i++ {
		fset.AddFile(fmt.Sprintf("file-%d", i), fset.Base(), 1031)
	}
	max := int32(fset.Base())
	var stop sync.WaitGroup
	r := rand.New(rand.NewSource(7))
	for i := 0; i < 2; i++ {
		r := rand.New(rand.NewSource(r.Int63()))
		stop.Add(1)
		go func() {
			for i := 0; i < 1000; i++ {
				fset.Position(Pos(r.Int31n(max)))
			}
			stop.Done()
		}()
	}
	stop.Wait()
}
<|endoftext|>


<commit_before>package services

import (
	"github.com/jbrodriguez/actor"
	"github.com/jbrodriguez/go-tmdb"
	"github.com/jbrodriguez/mlog"
	"github.com/jbrodriguez/pubsub"

	// "io/ioutil"
	"fmt"
	"regexp"
	"strconv"
	"strings"
	"time"

	"jbrodriguez/mediagui/server/src/dto"
	"jbrodriguez/mediagui/server/src/lib"
	"jbrodriguez/mediagui/server/src/model"
)

// Scraper -
type Scraper struct {
	bus      *pubsub.PubSub
	settings *lib.Settings
	pool     *lib.Pool
	tmdb     *tmdb.Tmdb

	actor *actor.Actor

	// reRating   *regexp.Regexp
	// reVotes    *regexp.Regexp
	// reDirector *regexp.Regexp
	// reWriter   *regexp.Regexp
	// reActor    *regexp.Regexp
}

// NewScraper -
func NewScraper(bus *pubsub.PubSub, settings *lib.Settings) *Scraper {
	scraper := &Scraper{
		bus:      bus,
		settings: settings,
		actor:    actor.NewActor(bus),
	}

	// reRating = `<span[^>]*itemprop="ratingValue">([^<]*)</span>`
	// reVotes = `<span[^>]*itemprop="ratingCount">([^<]*)</span>`
	// reDirector = `<a href="/name[^"]*dr"[^>]*><span[^>]*>([^<]*)?</span>`
	// reWriter = `<a href="/name[^"]*wr"[^>]*><span[^>]*>([^<]*)?</span>`
	// reActor = `<a href="/name[^"]*st_sm"[^>]*><span[^>]*>([^<]*)?</span>`

	return scraper
}

// Start -
func (s *Scraper) Start() {
	mlog.Info("Starting service Scraper ...")

	var err error
	s.tmdb, err = tmdb.NewClient("e610ded10c3f47d05fe797961d90fea6", false)
	if err != nil {
		mlog.Fatalf("Unable to create tmdb client: %s", err)
	}

	s.actor.Register("/command/movie/scrape", s.scrapeMovie)
	s.actor.Register("/command/movie/rescrape", s.reScrapeMovie)
	s.actor.Register("/event/config/changed", s.configChanged)

	s.pool = lib.NewPool(12, 4000)

	go s.actor.React()
}

// Stop -
func (s *Scraper) Stop() {
	mlog.Info("Stopped service Scraper ...")
}

func (s *Scraper) scrapeMovie(msg *pubsub.Message) {
	movie := msg.Payload.(*model.Movie)

	scrape := &Scrape{
		s.bus,
		s.tmdb,
		&dto.Scrape{
			// BasePath: s.settings.WebDir,
			Movie:  movie,
			Forced: false,
		},
	}

	s.pool.Exec(scrape)
}

// Scrape -
type Scrape struct {
	bus  *pubsub.PubSub
	tmdb *tmdb.Tmdb
	dto  *dto.Scrape
}

// Execute -
func (s *Scrape) Execute(wid int) {
	movie := s.dto.Movie.(*model.Movie)

	lib.Notify(s.bus, "import:progress", fmt.Sprintf("SCRAPE REQUESTED (%d) [%s]", wid, movie.Title))

	now := time.Now().UTC().Format(time.RFC3339)
	movie.Added = now
	movie.Modified = now

	movie.Score = 0
	movie.ShowIfDuplicate = 1

	lib.Notify(s.bus, "import:progress", fmt.Sprintf("STARTED TMDB (%d) [%s]", wid, movie.Title))
	movies, err := s.tmdb.SearchMovie(movie.Title)
	if err != nil {
		s.bus.Pub(nil, "/event/workunit/done")

		mlog.Error(err)
		return
	}

	if movies.Total_Results == 0 {
		lib.Notify(s.bus, "import:progress", fmt.Sprintf("TMDB: NO MATCH FOUND (%d) [%s]", wid, movie.Title))

		msg := &pubsub.Message{Payload: s.dto}
		s.bus.Pub(msg, "/event/movie/tmdbnotfound")

		return
	} else if movies.Total_Results > 1 {
		lib.Notify(s.bus, "import:progress", fmt.Sprintf("TMDB: MORE THAN ONE (%d) [%s]", wid, movie.Title))
	}

	id := movies.Results[0].Id

	_scrape(wid, s.tmdb, id, movie)

	s.dto.BaseUrl = s.tmdb.BaseUrl
	s.dto.SecureBaseUrl = s.tmdb.SecureBaseUrl

	lib.Notify(s.bus, "import:progress", fmt.Sprintf("SCRAPE COMPLETED (%d) [%s]", wid, movie.Title))

	msg := &pubsub.Message{Payload: s.dto}
	s.bus.Pub(msg, "/event/movie/scraped")
}

func (s *Scraper) reScrapeMovie(msg *pubsub.Message) {
	movie := msg.Payload.(*model.Movie)

	reScrape := &ReScrape{
		s.bus,
		s.tmdb,
		&dto.Scrape{
			// BasePath: s.settings.WebDir,
			Movie:  movie,
			Forced: true,
		},
	}

	s.pool.Exec(reScrape)
}

// ReScrape -
type ReScrape struct {
	bus  *pubsub.PubSub
	tmdb *tmdb.Tmdb
	dto  *dto.Scrape
}

// Execute -
func (s *ReScrape) Execute(wid int) {
	movie := s.dto.Movie.(*model.Movie)

	// lib.Notify(s.bus, "import:progress", fmt.Sprintf("RESCRAPE REQUESTED (%d) [%d] %s", wid, movie.Id, movie.Title))
	mlog.Info("RESCRAPE REQUESTED (%d) [%d] %s", wid, movie.Id, movie.Title)

	now := time.Now().UTC().Format(time.RFC3339)
	movie.Modified = now

	id := movie.Tmdb_Id

	err := _scrape(wid, s.tmdb, id, movie)
	if err != nil {
		// lib.Notify(s.bus, "import:progress", fmt.Sprintf("RESCRAPE FAILED (%d) [%d] %s: %s", wid, movie.Id, movie.Title, err))
		mlog.Warning("RESCRAPE FAILED (%d) [%d] %s: %s", wid, movie.Id, movie.Title, err)
		s.bus.Pub(nil, "/event/workunit/done")

		return
	}

	s.dto.BaseUrl = s.tmdb.BaseUrl
	s.dto.SecureBaseUrl = s.tmdb.SecureBaseUrl

	// lib.Notify(s.bus, "import:progress", fmt.Sprintf("RESCRAPE COMPLETED (%d) [%d] %s", wid, movie.Id, movie.Title))
	mlog.Info("RESCRAPE COMPLETED (%d) [%d] %s", wid, movie.Id, movie.Title)

	msg := &pubsub.Message{Payload: s.dto}
	s.bus.Pub(msg, "/event/movie/rescraped")
}

func _scrape(wid int, tmdb *tmdb.Tmdb, id uint64, movie *model.Movie) error {
	// log.Printf("before getmovie [%d] %s", id, media.Movie.Title)
	// mlog.Info("[%s] before getmovie [%s]", movie.Title)
	gmr, err := tmdb.GetMovie(id)
	if err != nil {
		return fmt.Errorf("FAILED GETTING MOVIE (%d) [%s]", wid, movie.Title)
	}

	movie.Title = gmr.Title
	movie.Original_Title = gmr.Original_Title
	movie.Runtime = gmr.Runtime
	movie.Tmdb_Id = gmr.Id
	movie.Imdb_Id = gmr.Imdb_Id
	movie.Overview = gmr.Overview
	movie.Tagline = gmr.Tagline
	movie.Cover = gmr.Poster_Path
	movie.Backdrop = gmr.Backdrop_Path

	movie.Year = gmr.Release_Date[0:4]

	movie.Genres = ""
	for i := 0; i < len(gmr.Genres); i++ {
		attr := &gmr.Genres[i]
		if movie.Genres == "" {
			movie.Genres = attr.Name
		} else {
			if strings.Contains(movie.Genres, attr.Name) {
				continue
			}

			movie.Genres += "|" + attr.Name
		}
	}

	movie.Vote_Average = gmr.Vote_Average
	movie.Vote_Count = gmr.Vote_Count

	movie.Production_Countries = ""
	for i := 0; i < len(gmr.Production_Countries); i++ {
		attr := &gmr.Production_Countries[i]
		if movie.Production_Countries == "" {
			movie.Production_Countries = attr.Name
		} else {
			if strings.Contains(movie.Production_Countries, attr.Name) {
				continue
			}

			movie.Production_Countries += "|" + attr.Name
		}
	}

	// var omdb model.Omdb

	// lib.Notify(s.bus, "import:progress", fmt.Sprintf("STARTED OMDB [%s]", movie.Title))
	data, err := lib.RestGet(fmt.Sprintf("http://www.im%s.com/title/%s", "db", movie.Imdb_Id))
	if err != nil {
		return fmt.Errorf("OMDB Error: %s", err)
	}

	omdb := getOmdb(data)

	// lib.Notify(s.bus, "import:progress", fmt.Sprintf("omdb: %+v", omdb))

	vote := strings.Replace(omdb.Imdb_Vote, ",", "", -1)
	imdbVote, _ := strconv.ParseUint(vote, 0, 64)
	imdbRating, _ := strconv.ParseFloat(omdb.Imdb_Rating, 64)

	movie.Director = omdb.Director
	movie.Writer = omdb.Writer
	movie.Actors = omdb.Actors
	movie.Awards = omdb.Awards
	movie.Imdb_Rating = imdbRating
	movie.Imdb_Votes = imdbVote

	return nil
}

func (s *Scraper) configChanged(msg *pubsub.Message) {
	s.settings = msg.Payload.(*lib.Settings)
}

var reRating = regexp.MustCompile(`<span[^>]*itemprop="ratingValue">([^<]*)</span>`)
var reVotes = regexp.MustCompile(`<span[^>]*itemprop="ratingCount">([^<]*)</span>`)
var reDirector = regexp.MustCompile(`<a href="/name[^"]*dr"[^>]*><span[^>]*>([^<]*)?</span>`)
var reWriter = regexp.MustCompile(`<a href="/name[^"]*wr"[^>]*><span[^>]*>([^<]*)?</span>`)
var reActor = regexp.MustCompile(`<a href="/name[^"]*st_sm"[^>]*><span[^>]*>([^<]*)?</span>`)

func getOmdb(data string) *model.Omdb {
	omdb := &model.Omdb{}

	rating := reRating.FindStringSubmatch(data)
	if len(rating) > 0 {
		omdb.Imdb_Rating = rating[1]
	}

	votes := reVotes.FindStringSubmatch(data)
	if len(votes) > 0 {
		omdb.Imdb_Vote = votes[1]
	}

	directors := reDirector.FindAllStringSubmatch(data, -1)
	if len(directors) > 0 {
		for _, director := range directors {
			if omdb.Director == "" {
				omdb.Director = director[1]
			} else {
				omdb.Director += ", " + director[1]
			}
		}
	}

	writers := reWriter.FindAllStringSubmatch(data, -1)
	if len(writers) > 0 {
		for _, writer := range writers {
			if omdb.Writer == "" {
				omdb.Writer = writer[1]
			} else {
				omdb.Writer += ", " + writer[1]
			}
		}
	}

	actors := reActor.FindAllStringSubmatch(data, -1)
	if len(actors) > 0 {
		for _, actor := range actors {
			if omdb.Actors == "" {
				omdb.Actors = actor[1]
			} else {
				omdb.Actors += ", " + actor[1]
			}
		}
	}

	return omdb
}
<commit_msg>Fix imdb rating issue<commit_after>package services

import (
	"github.com/jbrodriguez/actor"
	"github.com/jbrodriguez/go-tmdb"
	"github.com/jbrodriguez/mlog"
	"github.com/jbrodriguez/pubsub"

	// "io/ioutil"
	"fmt"
	"regexp"
	"strconv"
	"strings"
	"time"

	"jbrodriguez/mediagui/server/src/dto"
	"jbrodriguez/mediagui/server/src/lib"
	"jbrodriguez/mediagui/server/src/model"
)

// Scraper -
type Scraper struct {
	bus      *pubsub.PubSub
	settings *lib.Settings
	pool     *lib.Pool
	tmdb     *tmdb.Tmdb

	actor *actor.Actor

	// reRating   *regexp.Regexp
	// reVotes    *regexp.Regexp
	// reDirector *regexp.Regexp
	// reWriter   *regexp.Regexp
	// reActor    *regexp.Regexp
}

// NewScraper -
func NewScraper(bus *pubsub.PubSub, settings *lib.Settings) *Scraper {
	scraper := &Scraper{
		bus:      bus,
		settings: settings,
		actor:    actor.NewActor(bus),
	}

	// reRating = `<span[^>]*itemprop="ratingValue">([^<]*)</span>`
	// reVotes = `<span[^>]*itemprop="ratingCount">([^<]*)</span>`
	// reDirector = `<a href="/name[^"]*dr"[^>]*><span[^>]*>([^<]*)?</span>`
	// reWriter = `<a href="/name[^"]*wr"[^>]*><span[^>]*>([^<]*)?</span>`
	// reActor = `<a href="/name[^"]*st_sm"[^>]*><span[^>]*>([^<]*)?</span>`

	return scraper
}

// Start -
func (s *Scraper) Start() {
	mlog.Info("Starting service Scraper ...")

	var err error
	s.tmdb, err = tmdb.NewClient("e610ded10c3f47d05fe797961d90fea6", false)
	if err != nil {
		mlog.Fatalf("Unable to create tmdb client: %s", err)
	}

	s.actor.Register("/command/movie/scrape", s.scrapeMovie)
	s.actor.Register("/command/movie/rescrape", s.reScrapeMovie)
	s.actor.Register("/event/config/changed", s.configChanged)

	s.pool = lib.NewPool(12, 4000)

	go s.actor.React()
}

// Stop -
func (s *Scraper) Stop() {
	mlog.Info("Stopped service Scraper ...")
}

func (s *Scraper) scrapeMovie(msg *pubsub.Message) {
	movie := msg.Payload.(*model.Movie)

	scrape := &Scrape{
		s.bus,
		s.tmdb,
		&dto.Scrape{
			// BasePath: s.settings.WebDir,
			Movie:  movie,
			Forced: false,
		},
	}

	s.pool.Exec(scrape)
}

// Scrape -
type Scrape struct {
	bus  *pubsub.PubSub
	tmdb *tmdb.Tmdb
	dto  *dto.Scrape
}

// Execute -
func (s *Scrape) Execute(wid int) {
	movie := s.dto.Movie.(*model.Movie)

	lib.Notify(s.bus, "import:progress", fmt.Sprintf("SCRAPE REQUESTED (%d) [%s]", wid, movie.Title))

	now := time.Now().UTC().Format(time.RFC3339)
	movie.Added = now
	movie.Modified = now

	movie.Score = 0
	movie.ShowIfDuplicate = 1

	lib.Notify(s.bus, "import:progress", fmt.Sprintf("STARTED TMDB (%d) [%s]", wid, movie.Title))
	movies, err := s.tmdb.SearchMovie(movie.Title)
	if err != nil {
		s.bus.Pub(nil, "/event/workunit/done")

		mlog.Error(err)
		return
	}

	if movies.Total_Results == 0 {
		lib.Notify(s.bus, "import:progress", fmt.Sprintf("TMDB: NO MATCH FOUND (%d) [%s]", wid, movie.Title))

		msg := &pubsub.Message{Payload: s.dto}
		s.bus.Pub(msg, "/event/movie/tmdbnotfound")

		return
	} else if movies.Total_Results > 1 {
		lib.Notify(s.bus, "import:progress", fmt.Sprintf("TMDB: MORE THAN ONE (%d) [%s]", wid, movie.Title))
	}

	id := movies.Results[0].Id

	_scrape(wid, s.tmdb, id, movie)

	s.dto.BaseUrl = s.tmdb.BaseUrl
	s.dto.SecureBaseUrl = s.tmdb.SecureBaseUrl

	lib.Notify(s.bus, "import:progress", fmt.Sprintf("SCRAPE COMPLETED (%d) [%s]", wid, movie.Title))

	msg := &pubsub.Message{Payload: s.dto}
	s.bus.Pub(msg, "/event/movie/scraped")
}

func (s *Scraper) reScrapeMovie(msg *pubsub.Message) {
	movie := msg.Payload.(*model.Movie)

	reScrape := &ReScrape{
		s.bus,
		s.tmdb,
		&dto.Scrape{
			// BasePath: s.settings.WebDir,
			Movie:  movie,
			Forced: true,
		},
	}

	s.pool.Exec(reScrape)
}

// ReScrape -
type ReScrape struct {
	bus  *pubsub.PubSub
	tmdb *tmdb.Tmdb
	dto  *dto.Scrape
}

// Execute -
func (s *ReScrape) Execute(wid int) {
	movie := s.dto.Movie.(*model.Movie)

	// lib.Notify(s.bus, "import:progress", fmt.Sprintf("RESCRAPE REQUESTED (%d) [%d] %s", wid, movie.Id, movie.Title))
	mlog.Info("RESCRAPE REQUESTED (%d) [%d] %s", wid, movie.Id, movie.Title)

	now := time.Now().UTC().Format(time.RFC3339)
	movie.Modified = now

	id := movie.Tmdb_Id

	err := _scrape(wid, s.tmdb, id, movie)
	if err != nil {
		// lib.Notify(s.bus, "import:progress", fmt.Sprintf("RESCRAPE FAILED (%d) [%d] %s: %s", wid, movie.Id, movie.Title, err))
		mlog.Warning("RESCRAPE FAILED (%d) [%d] %s: %s", wid, movie.Id, movie.Title, err)
		s.bus.Pub(nil, "/event/workunit/done")

		return
	}

	s.dto.BaseUrl = s.tmdb.BaseUrl
	s.dto.SecureBaseUrl = s.tmdb.SecureBaseUrl

	// lib.Notify(s.bus, "import:progress", fmt.Sprintf("RESCRAPE COMPLETED (%d) [%d] %s", wid, movie.Id, movie.Title))
	mlog.Info("RESCRAPE COMPLETED (%d) [%d] %s", wid, movie.Id, movie.Title)

	msg := &pubsub.Message{Payload: s.dto}
	s.bus.Pub(msg, "/event/movie/rescraped")
}

func _scrape(wid int, tmdb *tmdb.Tmdb, id uint64, movie *model.Movie) error {
	// log.Printf("before getmovie [%d] %s", id, media.Movie.Title)
	// mlog.Info("[%s] before getmovie [%s]", movie.Title)
	gmr, err := tmdb.GetMovie(id)
	if err != nil {
		return fmt.Errorf("FAILED GETTING MOVIE (%d) [%s]", wid, movie.Title)
	}

	movie.Title = gmr.Title
	movie.Original_Title = gmr.Original_Title
	movie.Runtime = gmr.Runtime
	movie.Tmdb_Id = gmr.Id
	movie.Imdb_Id = gmr.Imdb_Id
	movie.Overview = gmr.Overview
	movie.Tagline = gmr.Tagline
	movie.Cover = gmr.Poster_Path
	movie.Backdrop = gmr.Backdrop_Path

	movie.Year = gmr.Release_Date[0:4]

	movie.Genres = ""
	for i := 0; i < len(gmr.Genres); i++ {
		attr := &gmr.Genres[i]
		if movie.Genres == "" {
			movie.Genres = attr.Name
		} else {
			if strings.Contains(movie.Genres, attr.Name) {
				continue
			}

			movie.Genres += "|" + attr.Name
		}
	}

	movie.Vote_Average = gmr.Vote_Average
	movie.Vote_Count = gmr.Vote_Count

	movie.Production_Countries = ""
	for i := 0; i < len(gmr.Production_Countries); i++ {
		attr := &gmr.Production_Countries[i]
		if movie.Production_Countries == "" {
			movie.Production_Countries = attr.Name
		} else {
			if strings.Contains(movie.Production_Countries, attr.Name) {
				continue
			}

			movie.Production_Countries += "|" + attr.Name
		}
	}

	// var omdb model.Omdb

	// lib.Notify(s.bus, "import:progress", fmt.Sprintf("STARTED OMDB [%s]", movie.Title))
	data, err := lib.RestGet(fmt.Sprintf("http://www.im%s.com/title/%s", "db", movie.Imdb_Id))
	if err != nil {
		return fmt.Errorf("OMDB Error: %s", err)
	}

	omdb := getOmdb(data)

	// lib.Notify(s.bus, "import:progress", fmt.Sprintf("omdb: %+v", omdb))

	vote := strings.Replace(omdb.Imdb_Vote, ",", "", -1)
	imdbVote, _ := strconv.ParseUint(vote, 0, 64)
	imdbRating, _ := strconv.ParseFloat(omdb.Imdb_Rating, 64)

	movie.Director = omdb.Director
	movie.Writer = omdb.Writer
	movie.Actors = omdb.Actors
	movie.Awards = omdb.Awards
	movie.Imdb_Rating = imdbRating
	movie.Imdb_Votes = imdbVote

	return nil
}

func (s *Scraper) configChanged(msg *pubsub.Message) {
	s.settings = msg.Payload.(*lib.Settings)
}

var reRating = regexp.MustCompile(`<strong title[^>]*.*<span>([^<]*)<\/span>`)
var reVotes = regexp.MustCompile(`<span[^>]*itemprop="ratingCount">([^<]*)</span>`)
var reDirector = regexp.MustCompile(`<a href="/name[^"]*dr"[^>]*><span[^>]*>([^<]*)?</span>`)
var reWriter = regexp.MustCompile(`<a href="/name[^"]*wr"[^>]*><span[^>]*>([^<]*)?</span>`)
var reActor = regexp.MustCompile(`<a href="/name[^"]*st_sm"[^>]*><span[^>]*>([^<]*)?</span>`)

func getOmdb(data string) *model.Omdb {
	omdb := &model.Omdb{}

	rating := reRating.FindStringSubmatch(data)
	if len(rating) > 0 {
		omdb.Imdb_Rating = rating[1]
	}

	votes := reVotes.FindStringSubmatch(data)
	if len(votes) > 0 {
		omdb.Imdb_Vote = votes[1]
	}

	directors := reDirector.FindAllStringSubmatch(data, -1)
	if len(directors) > 0 {
		for _, director := range directors {
			if omdb.Director == "" {
				omdb.Director = director[1]
			} else {
				omdb.Director += ", " + director[1]
			}
		}
	}

	writers := reWriter.FindAllStringSubmatch(data, -1)
	if len(writers) > 0 {
		for _, writer := range writers {
			if omdb.Writer == "" {
				omdb.Writer = writer[1]
			} else {
				omdb.Writer += ", " + writer[1]
			}
		}
	}

	actors := reActor.FindAllStringSubmatch(data, -1)
	if len(actors) > 0 {
		for _, actor := range actors {
			if omdb.Actors == "" {
				omdb.Actors = actor[1]
			} else {
				omdb.Actors += ", " + actor[1]
			}
		}
	}

	return omdb
}
<|endoftext|>


<commit_before>//   Copyright 2018 MSolution.IO
//
//   Licensed under the Apache License, Version 2.0 (the "License");
//   you may not use this file except in compliance with the License.
//   You may obtain a copy of the License at
//
//       http://www.apache.org/licenses/LICENSE-2.0
//
//   Unless required by applicable law or agreed to in writing, software
//   distributed under the License is distributed on an "AS IS" BASIS,
//   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//   See the License for the specific language governing permissions and
//   limitations under the License.

package main

import (
	"context"
	"database/sql"
	"errors"
	"flag"
	"strconv"
	"github.com/trackit/jsonlog"

	"github.com/trackit/trackit-server/db"
	//"github.com/aws/aws-sdk-go/service/marketplacemetering"
	"github.com/aws/aws-sdk-go/service/marketplaceentitlementservice"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/trackit/trackit-server/config"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"fmt"
)

// taskCheckEntitlement checks the user Entitlement for AWS Marketplace users
func taskCheckEntitlement(ctx context.Context) error {
	args := flag.Args()
	logger := jsonlog.LoggerFromContextOrDefault(ctx)
	logger.Debug("Running task 'checkuserentitlement'.", map[string]interface{}{
		"args": args,
	})
	if len(args) != 1 {
		return errors.New("taskCheckEntitlement requires one integer argument")
	} else if userId, err := strconv.Atoi(args[0]); err != nil {
		return err
	} else {
		cuId, error := getCustomerIdentifier(db.Db, ctx, userId)
		if error != nil {
			return error
		} else if cuId == "" {
			return nil
		} else {
			res, err := getUserEntitlement(ctx, cuId)
			fmt.Print("This is entitlement : ")
			fmt.Print(res)
			_ = err
			//if res["Entitlement"] == "Ok" {
			//	updateCustomerEntitlement(db.Db, ctx, userId, 0)
			//} else {
			//	updateCustomerEntitlement(db.Db, ctx, userId, 1)
			//}
		}
	}
	return nil
}

func getUserEntitlement(ctx context.Context, customerIdentifier string) ([]*marketplaceentitlementservice.Entitlement, error){
	mySession := session.Must(session.NewSession(&aws.Config{
		Region: aws.String(config.AwsRegion),
	}))
	svc := marketplaceentitlementservice.New(mySession)
	var awsInput marketplaceentitlementservice.GetEntitlementsInput
	var filter = make(map[string][]*string)
	filter["CUSTOMER_IDENTIFIER"] = make([]*string, 0)
	filter["CUSTOMER_IDENTIFIER"] = append(filter["CUSTOMER_IDENTIFIER"], &customerIdentifier)
	awsInput.SetProductCode("f2k4zzzgvihcmlgrf4qlyo4sr")
	awsInput.SetFilter(filter)
	result, err := svc.GetEntitlements(&awsInput)
	if err != nil {
		aerr, ok := err.(awserr.Error)
		if !ok {
			return nil, errors.New("AWS error cast failed")
		}
		logger := jsonlog.LoggerFromContextOrDefault(ctx)
		logger.Error("Error when checking the AWS token", aerr.Message())
		return nil, nil
	}
	return result.Entitlements, nil
}

func getCustomerIdentifier(db *sql.DB, ctx context.Context, userId int) (string, error) {
	const sqlstr = `SELECT aws_customer_identifier FROM user WHERE id = ?`
	res, err := db.Query(sqlstr, userId)
	defer res.Close()
	if err != nil {
		return "", err
	}
	var token string
	for res.Next() {
		err := res.Scan(&token)
		if err != nil {
			return "", err
		}
	}
	return token, nil
}

func updateCustomerEntitlement(db *sql.DB, ctx context.Context, userId int, entitlementValue int) (error) {
	const sqlstr = `UPDATE user SET
		aws_customer_entitlement=?,
	WHERE id=?`
		var err error = nil
		//if err != nil {
		//	errorValue = err.Error()
		//}
		_, err = db.Exec(sqlstr, entitlementValue, userId)
		return err
}

//func checkUserEntitlement(ctx context.Context, amId int, cuId string) {
//	var entitlementInput marketplaceentitlementservice.GetEntitlementsInput
//	mySession := session.Must(session.NewSession(&aws.Config{
//		Region: aws.String(config.AwsRegion),
//	}))
//	svc := marketplaceentitlementservice.New(mySession)
//	entitlementInput.SetProductCode("f2k4zzzgvihcmlgrf4qlyo4sr")
//	entitlementInput.SetFilter(map["CUSTOMER_IDENTIFIER"][cuId])
//}
//
//func EntitlementUpdate(db *sql.DB, br s3.BillRepository) (int64, error) {
//	const sqlstr = `INSERT INTO user(
//		aws_entitlement
//	) VALUES (?)`
//	res, err := db.Exec(sqlstr, br.Id, backendId)
//	if err != nil {
//		return 0, err
//	}
//	return res.LastInsertId()
//}
//
//func updateCompletion(ctx context.Context, aaId, brId int, db *sql.DB, updateId int64, err error) {
//	rErr := registerUpdateCompletion(db, updateId, err)
//	if rErr != nil {
//		logger := jsonlog.LoggerFromContextOrDefault(ctx)
//		logger.Error("Failed to register ingestion completion.", map[string]interface{}{
//			"awsAccountId":     aaId,
//			"billRepositoryId": brId,
//			"error":            rErr.Error(),
//			"updateId":         updateId,
//		})
//	}
//}
//
//func registerUpdateCompletion(db *sql.DB, updateId int64, err error) error {
//	const sqlstr = `UPDATE aws_bill_update_job SET
//		completed=?,
//		error=?
//	WHERE id=?`
//	var errorValue string
//	var now = time.Now()
//	if err != nil {
//		errorValue = err.Error()
//	}
//	_, err = db.Exec(sqlstr, now, errorValue, updateId)
//	return err
//}
//
//const (
//	UpdateIntervalMinutes = 6 * 60
//	UpdateIntervalWindow  = 2 * 60
//)
//
//// updateBillRepositoryForNextUpdate plans the next update for a
//// BillRepository.
//func updateBillRepositoryForNextUpdate(ctx context.Context, tx *sql.Tx, br s3.BillRepository, latestManifest time.Time) error {
//	if latestManifest.After(br.LastImportedManifest) {
//		br.LastImportedManifest = latestManifest
//	}
//	updateDeltaMinutes := time.Duration(UpdateIntervalMinutes-UpdateIntervalWindow/2+rand.Int63n(UpdateIntervalWindow)) * time.Minute
//	br.NextUpdate = time.Now().Add(updateDeltaMinutes)
//	return s3.UpdateBillRepository(br, tx)
//}
<commit_msg>Check user entitlement<commit_after>//   Copyright 2018 MSolution.IO
//
//   Licensed under the Apache License, Version 2.0 (the "License");
//   you may not use this file except in compliance with the License.
//   You may obtain a copy of the License at
//
//       http://www.apache.org/licenses/LICENSE-2.0
//
//   Unless required by applicable law or agreed to in writing, software
//   distributed under the License is distributed on an "AS IS" BASIS,
//   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//   See the License for the specific language governing permissions and
//   limitations under the License.

package main

import (
	"context"
	"database/sql"
	"errors"
	"flag"
	"strconv"
	"time"

	"github.com/trackit/jsonlog"
	"github.com/aws/aws-sdk-go/service/marketplaceentitlementservice"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/aws"

	"github.com/trackit/trackit-server/db"
	"github.com/trackit/trackit-server/config"
)

// taskCheckEntitlement checks the user Entitlement for AWS Marketplace users
func taskCheckEntitlement(ctx context.Context) error {
	args := flag.Args()
	logger := jsonlog.LoggerFromContextOrDefault(ctx)
	logger.Debug("Running task 'checkuserentitlement'.", map[string]interface{}{
		"args": args,
	})
	if len(args) != 1 {
		return errors.New("taskCheckEntitlement requires one integer argument")
	} else if userId, err := strconv.Atoi(args[0]); err != nil {
		return err
	} else {
		cuId, error := getCustomerIdentifier(db.Db, ctx, userId)
		if error != nil {
			return error
		} else if cuId == "" {
			return nil
		} else {
			err = checkUserEntitlement(ctx, cuId, userId)
			if err != nil {
				logger.Error("Error occured while checking user entitlement", err)
				return err
			}
		}
	}
	return nil
}

// getUserEntitlement calls getEntitlements function to retrieve specific user entitlement from AWS marketplace.
func getUserEntitlement(ctx context.Context, customerIdentifier string) ([]*marketplaceentitlementservice.Entitlement, error){
	mySession := session.Must(session.NewSession(&aws.Config{
		Region: aws.String(config.AwsRegion),
	}))
	svc := marketplaceentitlementservice.New(mySession)
	var awsInput marketplaceentitlementservice.GetEntitlementsInput
	var filter = make(map[string][]*string)
	filter["CUSTOMER_IDENTIFIER"] = make([]*string, 0)
	filter["CUSTOMER_IDENTIFIER"] = append(filter["CUSTOMER_IDENTIFIER"], &customerIdentifier)
	awsInput.SetProductCode("f2k4zzzgvihcmlgrf4qlyo4sr")
	awsInput.SetFilter(filter)
	result, err := svc.GetEntitlements(&awsInput)
	if err != nil {
		aerr, ok := err.(awserr.Error)
		if !ok {
			return nil, errors.New("AWS error cast failed")
		}
		logger := jsonlog.LoggerFromContextOrDefault(ctx)
		logger.Error("Error when checking the AWS token", aerr.Message())
		return nil, err
	}
	return result.Entitlements, nil
}

// checUserEntitlement enables entitlement to be checked.
func checkUserEntitlement(ctx context.Context, cuId string, userId int) (error) {
	var expirationDate *time.Time
	res, err := getUserEntitlement(ctx, cuId)
	if err != nil {
		return err
	}
	for _, key := range res {
		expirationDate = key.ExpirationDate
	}
	err = checkExpirationDate(expirationDate, ctx, db.Db, userId)
	return err
}

// checkExpirationDate compares expiration date given by AWS to current time.
// According to result, an update is pushed to db.
func checkExpirationDate(expirationDate *time.Time, ctx context.Context, db *sql.DB, userId int) (error) {
	var err error
	currentTime := time.Now().Local()
	if expirationDate.String() > currentTime.String() {
		err = updateCustomerEntitlement(db, ctx, userId, 0)
	} else {
		err = updateCustomerEntitlement(db, ctx, userId, 1)
	}
	return err
}

// getCustomerIdentifier retreive AWS customer identifier from user Id.
func getCustomerIdentifier(db *sql.DB, ctx context.Context, userId int) (string, error) {
	const sqlstr = `SELECT aws_customer_identifier FROM user WHERE id = ?`
	res, err := db.Query(sqlstr, userId)
	defer res.Close()
	if err != nil {
		return "", err
	}
	var token string
	for res.Next() {
		err := res.Scan(&token)
		if err != nil {
			return "", err
		}
	}
	return token, nil
}

// updateCustomerEntitlement updates aws customer entitlement according to entitlement value.
func updateCustomerEntitlement(db *sql.DB, ctx context.Context, userId int, entitlementValue int) (error) {
	const sqlstr = `UPDATE user SET aws_customer_entitlement=? WHERE id=?`
	_, err := db.Exec(sqlstr, entitlementValue, userId)
	return err
}
<|endoftext|>


<commit_before>package urlvalue

import (
	"net/url"
	"regexp"
	"strconv"
	"strings"

	querybuilder "github.com/ngurajeka/go-querybuilder"
)

type builder struct {
	values url.Values
}

func New(values url.Values) querybuilder.Builder {
	return &builder{values: values}
}

func (b *builder) Compile() querybuilder.QueryBuilder {
	var qb = querybuilder.NewQuerybuilder(0, 10)
	qb = b.filter(qb)
	qb = b.order(qb)
	qb = b.pagination(qb)
	return qb
}

func (b *builder) filter(qb querybuilder.QueryBuilder) querybuilder.QueryBuilder {
	rf := regexp.MustCompile(`filter`)
	rw := regexp.MustCompile(`\W`)
	for q, v := range b.values {
		if rf.MatchString(q) != true {
			continue
		}
		qSplit := rf.Split(q, -1)
		fSplit := rw.Split(qSplit[1], -1)

		opr := querybuilder.DefaultOperator
		conj := querybuilder.DefaultConjunction
		for i, val := range fSplit {
			switch {
			case i == 3 && val != "":
				opr = querybuilder.Operator(val)
			case i == 5 && val == "or":
				conj = querybuilder.OR
			}
		}
		if opr == querybuilder.IN || opr == querybuilder.NOTIN {
			filter := querybuilder.New(fSplit[1], opr, conj, strings.Split(v[0], ","))
			qb.AddCondition(filter)
			continue
		}
		filter := querybuilder.New(fSplit[1], opr, conj, v[0])
		qb.AddCondition(filter)
	}
	return qb
}

func (b *builder) pagination(qb querybuilder.QueryBuilder) querybuilder.QueryBuilder {
	var (
		limit  = 10
		offset = 0
	)
	pageNumber := b.values.Get("pageNumber")
	pageSize := b.values.Get("pageSize")
	if pageSize != "" {
		_limit, err := strconv.Atoi(pageSize)
		if err == nil {
			limit = _limit
		}
	}
	if pageNumber != "" {
		number, err := strconv.Atoi(pageNumber)
		if err == nil {
			offset = (number - 1) * limit
		}
	}

	qb.SetLimit(limit)
	qb.SetNumber(pageNumber)
	qb.SetOffset(offset)
	return qb
}

func (b *builder) order(qb querybuilder.QueryBuilder) querybuilder.QueryBuilder {
	var (
		orderType querybuilder.Order
		sort      string
	)
	sort = b.values.Get("sort")
	if sort == "" {
		return qb
	}

	switch sort[:1] {
	case "+":
		orderType = querybuilder.Ascending(sort[1:])
	case "-":
		orderType = querybuilder.Descending(sort[1:])
	default:
		orderType = querybuilder.Ascending(sort)
	}

	qb.AddOrder(orderType)
	return qb
}
<commit_msg>fixing page number<commit_after>package urlvalue

import (
	"net/url"
	"regexp"
	"strconv"
	"strings"

	querybuilder "github.com/ngurajeka/go-querybuilder"
)

type builder struct {
	values url.Values
}

func New(values url.Values) querybuilder.Builder {
	return &builder{values: values}
}

func (b *builder) Compile() querybuilder.QueryBuilder {
	var qb = querybuilder.NewQuerybuilder(0, 10)
	qb = b.filter(qb)
	qb = b.order(qb)
	qb = b.pagination(qb)
	return qb
}

func (b *builder) filter(qb querybuilder.QueryBuilder) querybuilder.QueryBuilder {
	rf := regexp.MustCompile(`filter`)
	rw := regexp.MustCompile(`\W`)
	for q, v := range b.values {
		if rf.MatchString(q) != true {
			continue
		}
		qSplit := rf.Split(q, -1)
		fSplit := rw.Split(qSplit[1], -1)

		opr := querybuilder.DefaultOperator
		conj := querybuilder.DefaultConjunction
		for i, val := range fSplit {
			switch {
			case i == 3 && val != "":
				opr = querybuilder.Operator(val)
			case i == 5 && val == "or":
				conj = querybuilder.OR
			}
		}
		if opr == querybuilder.IN || opr == querybuilder.NOTIN {
			filter := querybuilder.New(fSplit[1], opr, conj, strings.Split(v[0], ","))
			qb.AddCondition(filter)
			continue
		}
		filter := querybuilder.New(fSplit[1], opr, conj, v[0])
		qb.AddCondition(filter)
	}
	return qb
}

func (b *builder) pagination(qb querybuilder.QueryBuilder) querybuilder.QueryBuilder {
	var (
		limit  = 10
		number = 1
		offset = 0
	)
	pageNumber := b.values.Get("pageNumber")
	pageSize := b.values.Get("pageSize")
	if pageSize != "" {
		_limit, err := strconv.Atoi(pageSize)
		if err == nil {
			limit = _limit
		}
	}
	if pageNumber != "" {
		_number, err := strconv.Atoi(pageNumber)
		if err == nil {
			number = _number
			offset = (number - 1) * limit
		}
	}

	qb.SetLimit(limit)
	qb.SetNumber(number)
	qb.SetOffset(offset)
	return qb
}

func (b *builder) order(qb querybuilder.QueryBuilder) querybuilder.QueryBuilder {
	var (
		orderType querybuilder.Order
		sort      string
	)
	sort = b.values.Get("sort")
	if sort == "" {
		return qb
	}

	switch sort[:1] {
	case "+":
		orderType = querybuilder.Ascending(sort[1:])
	case "-":
		orderType = querybuilder.Descending(sort[1:])
	default:
		orderType = querybuilder.Ascending(sort)
	}

	qb.AddOrder(orderType)
	return qb
}
<|endoftext|>


<commit_before>package repowatch

import (
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"time"

	"github.com/Cloud-Foundations/Dominator/lib/fsutil"
	"github.com/Cloud-Foundations/Dominator/lib/log"
	"github.com/Cloud-Foundations/tricorder/go/tricorder"
	"github.com/Cloud-Foundations/tricorder/go/tricorder/units"
)

type gitMetricsType struct {
	lastAttemptedPullTime  time.Time
	lastSuccessfulPullTime time.Time
	lastNotificationTime   time.Time
	latencyDistribution    *tricorder.CumulativeDistribution
}

func checkDirectory(directory string) error {
	if fi, err := os.Stat(directory); err != nil {
		return err
	} else if !fi.IsDir() {
		return fmt.Errorf("not a directory: %s", directory)
	}
	return nil
}

func gitCommand(repositoryDirectory string, command ...string) error {
	cmd := exec.Command("git", command...)
	cmd.Dir = repositoryDirectory
	if output, err := cmd.CombinedOutput(); err != nil {
		return errors.New(err.Error() + ": " + string(output))
	}
	return nil
}

func gitPull(repositoryDirectory string,
	metrics *gitMetricsType) (string, error) {
	metrics.lastAttemptedPullTime = time.Now()
	if err := gitCommand(repositoryDirectory, "pull"); err != nil {
		return "", err
	}
	metrics.lastSuccessfulPullTime = time.Now()
	metrics.latencyDistribution.Add(
		metrics.lastSuccessfulPullTime.Sub(metrics.lastAttemptedPullTime))
	return readLatestCommitId(repositoryDirectory)
}

func readLatestCommitId(repositoryDirectory string) (string, error) {
	commitId, err := ioutil.ReadFile(
		filepath.Join(repositoryDirectory, ".git/refs/heads/master"))
	if err != nil {
		return "", err
	}
	return strings.TrimSpace(string(commitId)), nil
}

func setupGitRepository(remoteURL, localDirectory string,
	metrics *gitMetricsType) (string, error) {
	if err := os.MkdirAll(localDirectory, fsutil.DirPerms); err != nil {
		return "", err
	}
	gitSubdir := filepath.Join(localDirectory, ".git")
	if _, err := os.Stat(gitSubdir); err != nil {
		if !os.IsNotExist(err) {
			return "", err
		}
		metrics.lastAttemptedPullTime = time.Now()
		err := gitCommand(localDirectory, "clone", remoteURL, ".")
		if err != nil {
			return "", err
		}
		metrics.lastSuccessfulPullTime = time.Now()
		return readLatestCommitId(localDirectory)
	} else {
		return gitPull(localDirectory, metrics) // Ensure freshness.
	}
}

func watch(remoteURL, localDirectory string, checkInterval time.Duration,
	metricDirectory string, logger log.DebugLogger) (<-chan string, error) {
	if checkInterval < time.Second {
		checkInterval = time.Second
	}
	if remoteURL == "" {
		return watchLocal(localDirectory, checkInterval, metricDirectory,
			logger)
	}
	return watchGit(remoteURL, localDirectory, checkInterval, metricDirectory,
		logger)
}

func watchGit(remoteURL, localDirectory string, checkInterval time.Duration,
	metricDirectory string, logger log.DebugLogger) (<-chan string, error) {
	notificationChannel := make(chan string, 1)
	metrics := &gitMetricsType{
		latencyDistribution: tricorder.NewGeometricBucketer(1, 1e5).
			NewCumulativeDistribution(),
	}
	err := tricorder.RegisterMetric(filepath.Join(metricDirectory,
		"git-pull-latency"), metrics.latencyDistribution,
		units.Millisecond, "latency of git pull calls")
	if err != nil {
		return nil, err
	}
	commitId, err := setupGitRepository(remoteURL, localDirectory, metrics)
	if err != nil {
		return nil, err
	}
	err = tricorder.RegisterMetric(filepath.Join(metricDirectory,
		"last-attempted-git-pull-time"), &metrics.lastAttemptedPullTime,
		units.None, "time of last attempted git pull")
	if err != nil {
		return nil, err
	}
	err = tricorder.RegisterMetric(filepath.Join(metricDirectory,
		"last-successful-git-pull-time"), &metrics.lastSuccessfulPullTime,
		units.None, "time of last successful git pull")
	if err != nil {
		return nil, err
	}
	err = tricorder.RegisterMetric(filepath.Join(metricDirectory,
		"last-notification-time"), &metrics.lastNotificationTime, units.None,
		"time of last git change notification")
	if err != nil {
		return nil, err
	}
	metrics.lastNotificationTime = time.Now()
	notificationChannel <- localDirectory
	go watchGitLoop(localDirectory, commitId, checkInterval, metrics,
		notificationChannel, logger)
	return notificationChannel, nil
}

func watchGitLoop(directory, lastCommitId string, checkInterval time.Duration,
	metrics *gitMetricsType, notificationChannel chan<- string,
	logger log.DebugLogger) {
	for {
		time.Sleep(checkInterval)
		if commitId, err := gitPull(directory, metrics); err != nil {
			logger.Println(err)
		} else if commitId != lastCommitId {
			lastCommitId = commitId
			metrics.lastNotificationTime = time.Now()
			notificationChannel <- directory
		}
	}
}

func watchLocal(directory string, checkInterval time.Duration,
	metricDirectory string, logger log.DebugLogger) (<-chan string, error) {
	if err := checkDirectory(directory); err != nil {
		return nil, err
	}
	var lastNotificationTime time.Time
	err := tricorder.RegisterMetric(filepath.Join(metricDirectory,
		"last-notification-time"), &lastNotificationTime, units.None,
		"time of last notification")
	if err != nil {
		return nil, err
	}
	notificationChannel := make(chan string, 1)
	go watchLocalLoop(directory, checkInterval, &lastNotificationTime,
		notificationChannel, logger)
	return notificationChannel, nil
}

func watchLocalLoop(directory string, checkInterval time.Duration,
	lastNotificationTime *time.Time, notificationChannel chan<- string,
	logger log.DebugLogger) {
	for ; ; time.Sleep(checkInterval) {
		if err := checkDirectory(directory); err != nil {
			logger.Println(err)
		} else {
			*lastNotificationTime = time.Now()
			notificationChannel <- directory
		}
	}
}
<commit_msg>lib/repowatch: expose last-commit-id metric.<commit_after>package repowatch

import (
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"time"

	"github.com/Cloud-Foundations/Dominator/lib/fsutil"
	"github.com/Cloud-Foundations/Dominator/lib/log"
	"github.com/Cloud-Foundations/tricorder/go/tricorder"
	"github.com/Cloud-Foundations/tricorder/go/tricorder/units"
)

type gitMetricsType struct {
	lastAttemptedPullTime  time.Time
	lastCommitId           string
	lastSuccessfulPullTime time.Time
	lastNotificationTime   time.Time
	latencyDistribution    *tricorder.CumulativeDistribution
}

func checkDirectory(directory string) error {
	if fi, err := os.Stat(directory); err != nil {
		return err
	} else if !fi.IsDir() {
		return fmt.Errorf("not a directory: %s", directory)
	}
	return nil
}

func gitCommand(repositoryDirectory string, command ...string) error {
	cmd := exec.Command("git", command...)
	cmd.Dir = repositoryDirectory
	if output, err := cmd.CombinedOutput(); err != nil {
		return errors.New(err.Error() + ": " + string(output))
	}
	return nil
}

func gitPull(repositoryDirectory string,
	metrics *gitMetricsType) (string, error) {
	metrics.lastAttemptedPullTime = time.Now()
	if err := gitCommand(repositoryDirectory, "pull"); err != nil {
		return "", err
	}
	metrics.lastSuccessfulPullTime = time.Now()
	metrics.latencyDistribution.Add(
		metrics.lastSuccessfulPullTime.Sub(metrics.lastAttemptedPullTime))
	return readLatestCommitId(repositoryDirectory)
}

func readLatestCommitId(repositoryDirectory string) (string, error) {
	commitId, err := ioutil.ReadFile(
		filepath.Join(repositoryDirectory, ".git/refs/heads/master"))
	if err != nil {
		return "", err
	}
	return strings.TrimSpace(string(commitId)), nil
}

func setupGitRepository(remoteURL, localDirectory string,
	metrics *gitMetricsType) (string, error) {
	if err := os.MkdirAll(localDirectory, fsutil.DirPerms); err != nil {
		return "", err
	}
	gitSubdir := filepath.Join(localDirectory, ".git")
	if _, err := os.Stat(gitSubdir); err != nil {
		if !os.IsNotExist(err) {
			return "", err
		}
		metrics.lastAttemptedPullTime = time.Now()
		err := gitCommand(localDirectory, "clone", remoteURL, ".")
		if err != nil {
			return "", err
		}
		metrics.lastSuccessfulPullTime = time.Now()
		return readLatestCommitId(localDirectory)
	} else {
		return gitPull(localDirectory, metrics) // Ensure freshness.
	}
}

func watch(remoteURL, localDirectory string, checkInterval time.Duration,
	metricDirectory string, logger log.DebugLogger) (<-chan string, error) {
	if checkInterval < time.Second {
		checkInterval = time.Second
	}
	if remoteURL == "" {
		return watchLocal(localDirectory, checkInterval, metricDirectory,
			logger)
	}
	return watchGit(remoteURL, localDirectory, checkInterval, metricDirectory,
		logger)
}

func watchGit(remoteURL, localDirectory string, checkInterval time.Duration,
	metricDirectory string, logger log.DebugLogger) (<-chan string, error) {
	notificationChannel := make(chan string, 1)
	metrics := &gitMetricsType{
		latencyDistribution: tricorder.NewGeometricBucketer(1, 1e5).
			NewCumulativeDistribution(),
	}
	err := tricorder.RegisterMetric(filepath.Join(metricDirectory,
		"git-pull-latency"), metrics.latencyDistribution,
		units.Millisecond, "latency of git pull calls")
	if err != nil {
		return nil, err
	}
	metrics.lastCommitId, err = setupGitRepository(remoteURL, localDirectory,
		metrics)
	if err != nil {
		return nil, err
	}
	err = tricorder.RegisterMetric(filepath.Join(metricDirectory,
		"last-attempted-git-pull-time"), &metrics.lastAttemptedPullTime,
		units.None, "time of last attempted git pull")
	if err != nil {
		return nil, err
	}
	err = tricorder.RegisterMetric(filepath.Join(metricDirectory,
		"last-commit-id"), &metrics.lastCommitId,
		units.None, "commit ID in master branch in  last successful git pull")
	if err != nil {
		return nil, err
	}
	err = tricorder.RegisterMetric(filepath.Join(metricDirectory,
		"last-successful-git-pull-time"), &metrics.lastSuccessfulPullTime,
		units.None, "time of last successful git pull")
	if err != nil {
		return nil, err
	}
	err = tricorder.RegisterMetric(filepath.Join(metricDirectory,
		"last-notification-time"), &metrics.lastNotificationTime, units.None,
		"time of last git change notification")
	if err != nil {
		return nil, err
	}
	metrics.lastNotificationTime = time.Now()
	notificationChannel <- localDirectory
	go watchGitLoop(localDirectory, checkInterval, metrics, notificationChannel,
		logger)
	return notificationChannel, nil
}

func watchGitLoop(directory string, checkInterval time.Duration,
	metrics *gitMetricsType, notificationChannel chan<- string,
	logger log.DebugLogger) {
	for {
		time.Sleep(checkInterval)
		if commitId, err := gitPull(directory, metrics); err != nil {
			logger.Println(err)
		} else if commitId != metrics.lastCommitId {
			metrics.lastCommitId = commitId
			metrics.lastNotificationTime = time.Now()
			notificationChannel <- directory
		}
	}
}

func watchLocal(directory string, checkInterval time.Duration,
	metricDirectory string, logger log.DebugLogger) (<-chan string, error) {
	if err := checkDirectory(directory); err != nil {
		return nil, err
	}
	var lastNotificationTime time.Time
	err := tricorder.RegisterMetric(filepath.Join(metricDirectory,
		"last-notification-time"), &lastNotificationTime, units.None,
		"time of last notification")
	if err != nil {
		return nil, err
	}
	notificationChannel := make(chan string, 1)
	go watchLocalLoop(directory, checkInterval, &lastNotificationTime,
		notificationChannel, logger)
	return notificationChannel, nil
}

func watchLocalLoop(directory string, checkInterval time.Duration,
	lastNotificationTime *time.Time, notificationChannel chan<- string,
	logger log.DebugLogger) {
	for ; ; time.Sleep(checkInterval) {
		if err := checkDirectory(directory); err != nil {
			logger.Println(err)
		} else {
			*lastNotificationTime = time.Now()
			notificationChannel <- directory
		}
	}
}
<|endoftext|>


<commit_before>package sitejsoncache

import (
	"akamai/atlas/linker"
	"akamai/atlas/sitelistcache"
	"akamai/atlas/stat"
	"akamai/atlas/svgtext"
	"bytes"
	"encoding/json"
	"flag"
	"github.com/russross/blackfriday"
	"io/ioutil"
	"log"
	"os"
	"path"
	"strings"
	"time"
)

var logSiteJsonCache = flag.Bool("log.cache.sitejson", false, "log sitejson cache ops")

func L(s string, v ...interface{}) {
	if *logSiteJsonCache {
		log.Printf("sjc "+s, v...)
	}
}

type Dep struct {
	name string
	fi   os.FileInfo
}

type Ent struct {
	text string
	deps []Dep
}

func (self Ent) MarshalJSON() ([]byte, error) {
	return json.Marshal(self.text)
}

type SiteJsonCache struct {
	*sitelistcache.SiteListCache
	ModTime time.Time
	Entries map[string]Ent
	Json    []byte
}

func New(siteListCache *sitelistcache.SiteListCache) *SiteJsonCache {
	return &SiteJsonCache{
		SiteListCache: siteListCache,
		ModTime:       time.Time{},
		Json:          nil,
		Entries:       map[string]Ent{},
	}
}

func (self *SiteJsonCache) Make() (built bool, err error) {
	L("make starting")
	built = false
	err = nil

	fresh, err := self.allFresh()
	L("make allFresh returned fresh %t, err %v", fresh, err)
	if err != nil {
		L("make exiting; allFresh returned err %v", err)
		return
	}

	if !fresh {
		built = true

		L("make rebuilding")
		err = self.rebuild()
		if err != nil {
			L("make exiting; rebuild returned err %v", err)
			return
		}
	}
	L("make done")
	return
}

func (self *SiteJsonCache) isFresh(a, b os.FileInfo) bool {
	return stat.IsFresh(a, b)
}

func (self *SiteJsonCache) allFresh() (fresh bool, err error) {
	L("allFresh")
	fresh = false
	err = nil

	for _, ent := range self.Entries {
		for _, dep := range ent.deps {
			var fi os.FileInfo
			fi, err = os.Stat(dep.name)
			if err != nil {
				if os.IsNotExist(err) {
					err = nil
				}
				return
			}
			fresh = self.isFresh(fi, dep.fi)
			if !fresh {
				return
			}
		}
	}

	return
}

func (self *SiteJsonCache) updateModTime(depModTime time.Time) bool {
	if depModTime.After(self.ModTime) {
		self.ModTime = depModTime
		return true
	}
	return false
}

func (self *SiteJsonCache) rebuild() (err error) {
	L("rebuild starting")
	self.Entries = map[string]Ent{}

	_, err = self.SiteListCache.Make()
	if err != nil {
		return
	}

	for name, slEnt := range self.SiteListCache.Entries {
		chart := slEnt.Chart
		if chart == nil {
			L("rebuild skipping name %s", name)
			continue
		}

		key := chart.Slug()
		L("rebuild found key %s", key)

		err = chart.Read()
		if err != nil {
			L("rebuild warning after read: %s", err)
			continue
		}

		chartBytes := chart.Bytes()
		//log.Printf("HandleSiteJsonGet(): found body: %s", body)

		dep := Dep{
			name: chart.Src(),
			fi:   chart.FileInfo(),
		}

		ent := Ent{}
		ent.text = string(chartBytes)
		ent.deps = []Dep{dep}

		self.updateModTime(dep.fi.ModTime())

		linkRenderer := linker.NewLinkRenderer()
		extFlags := 0
		extFlags |= blackfriday.EXTENSION_NO_INTRA_EMPHASIS
		extFlags |= blackfriday.EXTENSION_TABLES
		extFlags |= blackfriday.EXTENSION_FENCED_CODE
		extFlags |= blackfriday.EXTENSION_AUTOLINK
		extFlags |= blackfriday.EXTENSION_STRIKETHROUGH
		extFlags |= blackfriday.EXTENSION_SPACE_HEADERS
		blackfriday.Markdown([]byte(chart.Body()), linkRenderer, extFlags)
		L("rebuild found links: %s", linkRenderer.Links)

		for _, link := range linkRenderer.Links {
			// BUG(mistone): directory traversal
			sfx := strings.HasSuffix(link.Href, "svg")
			if sfx {
				svgPath := path.Clean(path.Join(chart.Dir(), link.Href))
				L("rebuild found svg: %s", svgPath)

				svgFile, err := os.Open(svgPath)
				if err != nil {
					L("rebuild warning: unable to open svg: %q, err %v", svgPath, err)
					continue
				}
				defer svgFile.Close()

				svgFI, err := svgFile.Stat()
				if err != nil {
					L("rebuild warning: unable to stat svg: %q, err %v", svgPath, err)
					continue
				}

				svgBody, err := ioutil.ReadAll(svgFile)
				if err != nil {
					L("rebuild warning: unable to read svg: %s, error: %s", svgPath, err)
					continue
				}

				cdata, err := svgtext.GetCData(svgBody)
				if err != nil {
					L("rebuild warning: unable to parse svg: %s, error: %s", svgPath, err)
					continue
				}
				L("rebuild found svg cdata items: %d", len(cdata))
				//L("rebuild found svg cdata: %s", cdata)
				L("rebuild done with svg: %s", svgPath)

				var buf bytes.Buffer
				for _, datum := range cdata {
					buf.WriteString("svg: ")
					buf.WriteString(datum)
					buf.WriteRune('\n')
				}

				ent.text = ent.text + "\n" + buf.String()
				ent.deps = append(ent.deps, Dep{
					name: svgPath,
					fi:   svgFI,
				})
				self.updateModTime(svgFI.ModTime())
			}
		}

		self.Entries[key] = ent
	}

	L("rebuild produced cache: %q", self.Entries)

	self.Json, err = json.Marshal(self.Entries)
	L("rebuild produced json: %q", string(self.Json))
	if err != nil {
		return
	}

	return
}
<commit_msg>Make sure to rebuild sitelist while checking sitejson freshness.<commit_after>package sitejsoncache

import (
	"akamai/atlas/linker"
	"akamai/atlas/sitelistcache"
	"akamai/atlas/stat"
	"akamai/atlas/svgtext"
	"bytes"
	"encoding/json"
	"flag"
	"github.com/russross/blackfriday"
	"io/ioutil"
	"log"
	"os"
	"path"
	"strings"
	"time"
)

var logSiteJsonCache = flag.Bool("log.cache.sitejson", false, "log sitejson cache ops")

func L(s string, v ...interface{}) {
	if *logSiteJsonCache {
		log.Printf("sjc "+s, v...)
	}
}

type Dep struct {
	name string
	fi   os.FileInfo
}

type Ent struct {
	text string
	deps []Dep
}

func (self Ent) MarshalJSON() ([]byte, error) {
	return json.Marshal(self.text)
}

type SiteJsonCache struct {
	*sitelistcache.SiteListCache
	ModTime time.Time
	Entries map[string]Ent
	Json    []byte
}

func New(siteListCache *sitelistcache.SiteListCache) *SiteJsonCache {
	return &SiteJsonCache{
		SiteListCache: siteListCache,
		ModTime:       time.Time{},
		Json:          nil,
		Entries:       map[string]Ent{},
	}
}

func (self *SiteJsonCache) Make() (built bool, err error) {
	L("make starting")
	built = false
	err = nil

	fresh, err := self.allFresh()
	L("make allFresh returned fresh %t, err %v", fresh, err)
	if err != nil {
		L("make exiting; allFresh returned err %v", err)
		return
	}

	if !fresh {
		built = true

		L("make rebuilding")
		err = self.rebuild()
		if err != nil {
			L("make exiting; rebuild returned err %v", err)
			return
		}
	}
	L("make done")
	return
}

func (self *SiteJsonCache) isFresh(a, b os.FileInfo) bool {
	return stat.IsFresh(a, b)
}

func (self *SiteJsonCache) allFresh() (fresh bool, err error) {
	L("allFresh")
	fresh = false
	err = nil

	if built, err := self.SiteListCache.Make(); built || err != nil {
		return false, err
	}

	for _, ent := range self.Entries {
		for _, dep := range ent.deps {
			var fi os.FileInfo
			fi, err = os.Stat(dep.name)
			if err != nil {
				if os.IsNotExist(err) {
					err = nil
				}
				return
			}
			fresh = self.isFresh(fi, dep.fi)
			if !fresh {
				return
			}
		}
	}

	return
}

func (self *SiteJsonCache) updateModTime(depModTime time.Time) bool {
	if depModTime.After(self.ModTime) {
		self.ModTime = depModTime
		return true
	}
	return false
}

func (self *SiteJsonCache) rebuild() (err error) {
	L("rebuild starting")
	self.Entries = map[string]Ent{}

	for name, slEnt := range self.SiteListCache.Entries {
		chart := slEnt.Chart
		if chart == nil {
			L("rebuild skipping name %s", name)
			continue
		}

		key := chart.Slug()
		L("rebuild found key %s", key)

		err = chart.Read()
		if err != nil {
			L("rebuild warning after read: %s", err)
			continue
		}

		chartBytes := chart.Bytes()
		//log.Printf("HandleSiteJsonGet(): found body: %s", body)

		dep := Dep{
			name: chart.Src(),
			fi:   chart.FileInfo(),
		}

		ent := Ent{}
		ent.text = string(chartBytes)
		ent.deps = []Dep{dep}

		self.updateModTime(dep.fi.ModTime())

		linkRenderer := linker.NewLinkRenderer()
		extFlags := 0
		extFlags |= blackfriday.EXTENSION_NO_INTRA_EMPHASIS
		extFlags |= blackfriday.EXTENSION_TABLES
		extFlags |= blackfriday.EXTENSION_FENCED_CODE
		extFlags |= blackfriday.EXTENSION_AUTOLINK
		extFlags |= blackfriday.EXTENSION_STRIKETHROUGH
		extFlags |= blackfriday.EXTENSION_SPACE_HEADERS
		blackfriday.Markdown([]byte(chart.Body()), linkRenderer, extFlags)
		L("rebuild found links: %s", linkRenderer.Links)

		for _, link := range linkRenderer.Links {
			// BUG(mistone): directory traversal
			sfx := strings.HasSuffix(link.Href, "svg")
			if sfx {
				svgPath := path.Clean(path.Join(chart.Dir(), link.Href))
				L("rebuild found svg: %s", svgPath)

				svgFile, err := os.Open(svgPath)
				if err != nil {
					L("rebuild warning: unable to open svg: %q, err %v", svgPath, err)
					continue
				}
				defer svgFile.Close()

				svgFI, err := svgFile.Stat()
				if err != nil {
					L("rebuild warning: unable to stat svg: %q, err %v", svgPath, err)
					continue
				}

				svgBody, err := ioutil.ReadAll(svgFile)
				if err != nil {
					L("rebuild warning: unable to read svg: %s, error: %s", svgPath, err)
					continue
				}

				cdata, err := svgtext.GetCData(svgBody)
				if err != nil {
					L("rebuild warning: unable to parse svg: %s, error: %s", svgPath, err)
					continue
				}
				L("rebuild found svg cdata items: %d", len(cdata))
				//L("rebuild found svg cdata: %s", cdata)
				L("rebuild done with svg: %s", svgPath)

				var buf bytes.Buffer
				for _, datum := range cdata {
					buf.WriteString("svg: ")
					buf.WriteString(datum)
					buf.WriteRune('\n')
				}

				ent.text = ent.text + "\n" + buf.String()
				ent.deps = append(ent.deps, Dep{
					name: svgPath,
					fi:   svgFI,
				})
				self.updateModTime(svgFI.ModTime())
			}
		}

		self.Entries[key] = ent
	}

	L("rebuild produced cache: %q", self.Entries)

	self.Json, err = json.Marshal(self.Entries)
	L("rebuild produced json: %q", string(self.Json))
	if err != nil {
		return
	}

	return
}
<|endoftext|>


<commit_before>package fs

import (
	"encoding/json"
	"fmt"
	"testing"

	"github.com/koding/fuseklient/transport"
	. "github.com/smartystreets/goconvey/convey"
)

// fakeTransport implements Transport; is used in testing Transport requests
// and mocking responses.
type fakeTransport struct {
	TripResponses map[string]interface{}
}

func (f *fakeTransport) Trip(methodName string, req interface{}, res interface{}) error {
	r, ok := f.TripResponses[methodName]
	if !ok {
		return fmt.Errorf("Expected '%s' to be in list of mocked responses.", methodName)
	}

	bytes, err := json.Marshal(r)
	if err != nil {
		return err
	}

	return json.Unmarshal(bytes, &res)
}

func TestFakeTransport(t *testing.T) {
	Convey("Given fake transport", t, func() {
		ft := &fakeTransport{
			TripResponses: map[string]interface{}{"indiana": struct{ Name string }{"jones"}},
		}

		Convey("It implements Transport", func() {
			var _ transport.Transport = (*fakeTransport)(nil)
		})

		Convey("It should return error if method don't exist", func() {
			So(ft.Trip("random", "", ""), ShouldNotBeNil)
		})

		Convey("It should return unmarshal mock into response for method", func() {
			res := struct{ Name string }{}

			So(ft.Trip("indiana", "", &res), ShouldBeNil)
			So(res.Name, ShouldEqual, "jones")
		})
	})
}
<commit_msg>panic if fakeTransport#Trip is called with unknown Trip name in tests<commit_after>package fs

import (
	"encoding/json"
	"fmt"
	"testing"

	"github.com/koding/fuseklient/transport"
	. "github.com/smartystreets/goconvey/convey"
)

// fakeTransport implements Transport; is used in testing Transport requests
// and mocking responses.
type fakeTransport struct {
	TripResponses map[string]interface{}
}

func (f *fakeTransport) Trip(methodName string, req interface{}, res interface{}) error {
	r, ok := f.TripResponses[methodName]
	if !ok {
		panic(fmt.Sprintf("Expected '%s' to be in list of mocked responses.", methodName))
	}

	bytes, err := json.Marshal(r)
	if err != nil {
		panic(err.Error())
	}

	return json.Unmarshal(bytes, &res)
}

func TestFakeTransport(t *testing.T) {
	Convey("Given fake transport", t, func() {
		ft := &fakeTransport{
			TripResponses: map[string]interface{}{"indiana": struct{ Name string }{"jones"}},
		}

		Convey("It implements Transport", func() {
			var _ transport.Transport = (*fakeTransport)(nil)
		})

		Convey("It should return unmarshal mock into response for method", func() {
			res := struct{ Name string }{}

			So(ft.Trip("indiana", "", &res), ShouldBeNil)
			So(res.Name, ShouldEqual, "jones")
		})
	})
}
<|endoftext|>